v0.25.1

src/balance/health.rs

@@ -0,0 +1,353 @@
+impl BackendState {
+    fn new(mode: HealthMode) -> Self {
+        let now = Instant::now();
+        Self {
+            health: Mutex::new(Health {
+                healthy: mode == HealthMode::Off,
+                active_healthy: mode == HealthMode::Off,
+                passive_healthy: true,
+                checked: false,
+                failures: 0,
+                passive_failures: 0,
+                successes: 0,
+                passive_successes: 0,
+                next_probe: now,
+                probing: false,
+                healthy_since: now,
+                result: if mode == HealthMode::Off {
+                    "disabled"
+                } else {
+                    "checking"
+                },
+                passive_result: "none",
+                last_probe_ms: 0,
+                latency_us: 0,
+                ewma_us: 0,
+                transitions: 0,
+                unhealthy_source: if mode == HealthMode::Off { "none" } else { "active" },
+                unhealthy_reason: if mode == HealthMode::Off { "none" } else { "checking" },
+                active_unhealthy_source: if mode == HealthMode::Off { "none" } else { "active" },
+                active_unhealthy_reason: if mode == HealthMode::Off { "none" } else { "checking" },
+                passive_unhealthy_source: "none",
+                passive_unhealthy_reason: "none",
+            }),
+            traffic: Mutex::new(Traffic {
+                at: now,
+                total: 0,
+                rate: 0.0,
+            }),
+            last_failure: Mutex::new(FailureObservation::default()),
+            active: AtomicU64::new(0),
+            requests: AtomicU64::new(0),
+            errors: AtomicU64::new(0),
+            bytes_in: AtomicU64::new(0),
+            bytes_out: AtomicU64::new(0),
+            last_status: AtomicU16::new(0),
+            active_health_failures_total: AtomicU64::new(0),
+            passive_health_failures_total: AtomicU64::new(0),
+            http_5xx_total: AtomicU64::new(0),
+            timeouts_total: AtomicU64::new(0),
+            connect_errors_total: AtomicU64::new(0),
+            io_errors_total: AtomicU64::new(0),
+            slow_responses_total: AtomicU64::new(0),
+            retry_attempts_total: AtomicU64::new(0),
+            retry_successes_total: AtomicU64::new(0),
+            capacity_rejections_total: AtomicU64::new(0),
+            queue_timeouts_total: AtomicU64::new(0),
+            dns_errors_total: AtomicU64::new(0),
+            tls_errors_total: AtomicU64::new(0),
+            response_header_timeouts_total: AtomicU64::new(0),
+            read_timeouts_total: AtomicU64::new(0),
+            send_timeouts_total: AtomicU64::new(0),
+            last_ejected_ms: AtomicU64::new(0),
+            last_recovered_ms: AtomicU64::new(0),
+        }
+    }
+    fn has_capacity(&self, limit: u32) -> bool {
+        limit == 0 || self.active.load(Ordering::Relaxed) < u64::from(limit)
+    }
+    fn try_acquire_active(&self, limit: u32) -> bool {
+        if limit == 0 {
+            self.active.fetch_add(1, Ordering::Relaxed);
+            return true;
+        }
+        let limit = u64::from(limit);
+        let mut current = self.active.load(Ordering::Relaxed);
+        loop {
+            if current >= limit {
+                return false;
+            }
+            match self.active.compare_exchange_weak(
+                current,
+                current + 1,
+                Ordering::AcqRel,
+                Ordering::Relaxed,
+            ) {
+                Ok(_) => return true,
+                Err(next) => current = next,
+            }
+        }
+    }
+    pub fn healthy(&self) -> bool {
+        lock(&self.health).healthy
+    }
+    fn weight(&self, configured: u16, slow_start: u64) -> f64 {
+        let h = lock(&self.health);
+        if slow_start == 0 {
+            return configured as f64;
+        }
+        let ratio = (h.healthy_since.elapsed().as_secs_f64() / slow_start as f64).clamp(0.05, 1.0);
+        (configured as f64 * ratio).max(0.05)
+    }
+    fn latency(&self) -> f64 {
+        lock(&self.health).ewma_us.max(1000) as f64 / 1000.0
+    }
+    fn traffic_rate(&self) -> f64 {
+        lock(&self.traffic).rate
+    }
+    fn sample_traffic(&self) {
+        let mut sample = lock(&self.traffic);
+        let elapsed = sample.at.elapsed().as_secs_f64();
+        if elapsed < 1.0 {
+            return;
+        }
+        let total = self
+            .bytes_in
+            .load(Ordering::Relaxed)
+            .saturating_add(self.bytes_out.load(Ordering::Relaxed));
+        let delta = total.saturating_sub(sample.total);
+        if delta == 0 {
+            // No bytes moved in this sampling window: report an actual idle backend
+            // instead of keeping a decaying EWMA tail that looks like live traffic.
+            sample.rate = 0.0;
+        } else {
+            let rate = delta as f64 / elapsed;
+            sample.rate = sample.rate * 0.5 + rate * 0.5;
+        }
+        sample.total = total;
+        sample.at = Instant::now();
+    }
+    fn claim_probe(&self, h: &HealthCheck) -> bool {
+        if h.mode == HealthMode::Off {
+            return false;
+        }
+        let mut state = lock(&self.health);
+        if state.probing || Instant::now() < state.next_probe {
+            return false;
+        }
+        state.probing = true;
+        state.next_probe = Instant::now() + Duration::from_millis(h.interval_ms);
+        true
+    }
+    fn update_latency(h: &mut Health, elapsed: Duration) {
+        let latency = elapsed.as_micros().min(u64::MAX as u128) as u64;
+        h.latency_us = latency;
+        h.ewma_us = if h.ewma_us == 0 {
+            latency
+        } else {
+            h.ewma_us.saturating_mul(7).saturating_add(latency) / 8
+        };
+    }
+
+    fn finish_transition(&self, h: &mut Health, previous: bool) {
+        h.healthy = h.active_healthy && h.passive_healthy;
+        if previous != h.healthy {
+            h.transitions = h.transitions.saturating_add(1);
+            if h.healthy {
+                self.last_recovered_ms
+                    .store(crate::logging::now_ms(), Ordering::Relaxed);
+                h.healthy_since = Instant::now();
+                h.unhealthy_source = "none";
+                h.unhealthy_reason = "none";
+                return;
+            } else {
+                self.last_ejected_ms
+                    .store(crate::logging::now_ms(), Ordering::Relaxed);
+            }
+        }
+        if h.healthy {
+            return;
+        }
+
+        let current_source_is_still_unhealthy =
+            (h.unhealthy_source.starts_with("active") && !h.active_healthy)
+                || (h.unhealthy_source.starts_with("passive") && !h.passive_healthy);
+        if current_source_is_still_unhealthy {
+            return;
+        }
+        if !h.active_healthy {
+            h.unhealthy_source = h.active_unhealthy_source;
+            h.unhealthy_reason = h.active_unhealthy_reason;
+        } else if !h.passive_healthy {
+            h.unhealthy_source = h.passive_unhealthy_source;
+            h.unhealthy_reason = h.passive_unhealthy_reason;
+        }
+    }
+
+    fn observe_active(&self, settings: &HealthCheck, result: &'static str, elapsed: Duration) {
+        if settings.mode == HealthMode::Off {
+            return;
+        }
+        let mut h = lock(&self.health);
+        let previous = h.healthy;
+        Self::update_latency(&mut h, elapsed);
+        h.probing = false;
+        h.checked = true;
+        h.last_probe_ms = crate::logging::now_ms();
+        h.result = result;
+        if result == "ok" {
+            h.failures = 0;
+            h.successes = h.successes.saturating_add(1);
+            if h.successes >= settings.rise {
+                h.active_healthy = true;
+            }
+        } else {
+            self.active_health_failures_total.fetch_add(1, Ordering::Relaxed);
+            h.successes = 0;
+            h.failures = h.failures.saturating_add(1);
+            if h.failures >= settings.fall {
+                h.active_healthy = false;
+            }
+        }
+        let source = match settings.mode {
+            HealthMode::Tcp => "active_tcp",
+            HealthMode::Http => "active_http",
+            HealthMode::Off => "active",
+        };
+        if result != "ok" {
+            self.record_failure("active_health", &format!("{source}:{result}"));
+        }
+        if h.active_healthy {
+            h.active_unhealthy_source = "none";
+            h.active_unhealthy_reason = "none";
+        } else {
+            h.active_unhealthy_source = source;
+            h.active_unhealthy_reason = result;
+            if previous || h.unhealthy_source == "active" || h.unhealthy_source == "none" {
+                h.unhealthy_source = source;
+                h.unhealthy_reason = result;
+            }
+        }
+        self.finish_transition(&mut h, previous);
+    }
+
+    fn record_failure(&self, class: &'static str, detail: &str) {
+        match class {
+            "dns" => {
+                self.dns_errors_total.fetch_add(1, Ordering::Relaxed);
+            }
+            "tls" => {
+                self.tls_errors_total.fetch_add(1, Ordering::Relaxed);
+            }
+            "response_header_timeout" => {
+                self.response_header_timeouts_total
+                    .fetch_add(1, Ordering::Relaxed);
+            }
+            "read_timeout" => {
+                self.read_timeouts_total.fetch_add(1, Ordering::Relaxed);
+            }
+            "send_timeout" => {
+                self.send_timeouts_total.fetch_add(1, Ordering::Relaxed);
+            }
+            _ => {}
+        }
+        let mut last = lock(&self.last_failure);
+        last.at_ms = crate::logging::now_ms();
+        last.class = class;
+        last.detail.clear();
+        last.detail.extend(detail.chars().take(1024));
+    }
+
+    fn record_passive_observation(&self, reason: &'static str) {
+        match reason {
+            "http_5xx" => { self.http_5xx_total.fetch_add(1, Ordering::Relaxed); }
+            "timeout" => { self.timeouts_total.fetch_add(1, Ordering::Relaxed); }
+            "connect_error" => { self.connect_errors_total.fetch_add(1, Ordering::Relaxed); }
+            "io_error" => { self.io_errors_total.fetch_add(1, Ordering::Relaxed); }
+            "slow_response" => { self.slow_responses_total.fetch_add(1, Ordering::Relaxed); }
+            _ => {}
+        }
+    }
+
+    fn observe_passive(
+        &self,
+        active: &HealthCheck,
+        passive: &PassiveHealth,
+        reason: &'static str,
+        elapsed: Duration,
+    ) {
+        self.record_passive_observation(reason);
+        {
+            let mut h = lock(&self.health);
+            Self::update_latency(&mut h, elapsed);
+            h.passive_result = reason;
+        }
+        if !passive.enabled {
+            return;
+        }
+        let mut h = lock(&self.health);
+        let previous = h.healthy;
+        if reason == "ok" {
+            h.passive_failures = 0;
+            h.passive_successes = h.passive_successes.saturating_add(1);
+            if h.passive_successes >= passive.rise(active) {
+                h.passive_healthy = true;
+                h.passive_unhealthy_source = "none";
+                h.passive_unhealthy_reason = "none";
+            }
+            self.finish_transition(&mut h, previous);
+            return;
+        }
+        if !passive.counts(reason) {
+            return;
+        }
+        self.passive_health_failures_total.fetch_add(1, Ordering::Relaxed);
+        h.passive_successes = 0;
+        h.passive_failures = h.passive_failures.saturating_add(1);
+        if h.passive_failures >= passive.fall(active) {
+            h.passive_healthy = false;
+        }
+        let source = match reason {
+            "connect_error" => "passive_connect_error",
+            "timeout" => "passive_timeout",
+            "io_error" => "passive_io_error",
+            "http_5xx" => "passive_http_5xx",
+            "slow_response" => "passive_slow",
+            _ => "passive",
+        };
+        if !h.passive_healthy {
+            h.passive_unhealthy_source = source;
+            h.passive_unhealthy_reason = reason;
+            if previous {
+                h.unhealthy_source = source;
+                h.unhealthy_reason = reason;
+            }
+        }
+        self.finish_transition(&mut h, previous);
+    }
+
+}
+#[derive(Debug)]
+pub struct Member {
+    pub backend: Arc<CompiledBackend>,
+    pub definition: PoolMember,
+    pub state: Arc<BackendState>,
+    token: String,
+}
+#[derive(Debug)]
+pub struct Pool {
+    pub id: String,
+    pub settings: Settings,
+    pub members: Vec<Arc<Member>>,
+    pub fingerprint: Vec<u8>,
+    cookie_name: String,
+    cursor: AtomicU64,
+    scheduling: Mutex<std::collections::HashMap<String, f64>>,
+}
+#[derive(Debug)]
+pub struct Lease {
+    pub member: Arc<Member>,
+    health: HealthCheck,
+    passive_health: PassiveHealth,
+    error_recorded: AtomicBool,
+}

src/balance/lease_runtime.rs

@@ -0,0 +1,294 @@
+impl Lease {
+    pub fn started(&self) {
+        self.member.state.requests.fetch_add(1, Ordering::Relaxed);
+    }
+    pub fn response(&self, status: u16, elapsed: Duration, request_body_empty: bool) {
+        self.member
+            .state
+            .last_status
+            .store(status, Ordering::Relaxed);
+        let is_5xx = status >= 500;
+        let is_slow = request_body_empty
+            && self.health.slow_response_ms > 0
+            && elapsed > Duration::from_millis(self.health.slow_response_ms);
+        if (is_5xx || is_slow) && !self.error_recorded.swap(true, Ordering::Relaxed) {
+            self.member.state.errors.fetch_add(1, Ordering::Relaxed);
+        }
+        if is_5xx {
+            self.member
+                .state
+                .record_failure("http_5xx", &format!("status={status}"));
+        }
+        let reason = if is_5xx
+            && (!is_slow
+                || !self.passive_health.enabled
+                || self.passive_health.count_http_5xx
+                || !self.passive_health.count_slow_responses)
+        {
+            if is_slow {
+                self.member.state.record_passive_observation("slow_response");
+            }
+            "http_5xx"
+        } else if is_slow {
+            if is_5xx {
+                self.member.state.record_passive_observation("http_5xx");
+            }
+            "slow_response"
+        } else {
+            "ok"
+        };
+        let before = self.member.state.healthy();
+        self.member
+            .state
+            .observe_passive(&self.health, &self.passive_health, reason, elapsed);
+        let after = self.member.state.healthy();
+        if before != after {
+            tracing::info!(
+                backend = %self.member.definition.id,
+                healthy = after,
+                source = "passive",
+                reason,
+                "backend health changed"
+            );
+        }
+    }
+    pub fn error(&self, reason: &'static str, elapsed: Duration) {
+        self.error_detail(reason, reason, elapsed);
+    }
+    pub fn error_detail(&self, class: &'static str, detail: &str, elapsed: Duration) {
+        if !self.error_recorded.swap(true, Ordering::Relaxed) {
+            self.member.state.errors.fetch_add(1, Ordering::Relaxed);
+        }
+        self.member.state.record_failure(class, detail);
+        let passive_reason = match class {
+            "connect" | "dns" | "tls" => "connect_error",
+            "timeout" | "body_timeout" | "response_header_timeout" | "read_timeout"
+            | "send_timeout" => "timeout",
+            _ => "io_error",
+        };
+        let before = self.member.state.healthy();
+        self.member.state.observe_passive(
+            &self.health,
+            &self.passive_health,
+            passive_reason,
+            elapsed,
+        );
+        let after = self.member.state.healthy();
+        if before != after {
+            tracing::info!(
+                backend = %self.member.definition.id,
+                healthy = after,
+                source = "passive",
+                reason = passive_reason,
+                "backend health changed"
+            );
+        }
+    }
+    pub fn bytes(&self, count: u64, incoming: bool) {
+        if incoming {
+            self.member
+                .state
+                .bytes_in
+                .fetch_add(count, Ordering::Relaxed);
+        } else {
+            self.member
+                .state
+                .bytes_out
+                .fetch_add(count, Ordering::Relaxed);
+        }
+    }
+    pub fn retry_attempted(&self) {
+        self.member
+            .state
+            .retry_attempts_total
+            .fetch_add(1, Ordering::Relaxed);
+    }
+    pub fn retry_succeeded(&self) {
+        self.member
+            .state
+            .retry_successes_total
+            .fetch_add(1, Ordering::Relaxed);
+    }
+}
+impl Drop for Lease {
+    fn drop(&mut self) {
+        self.member.state.active.fetch_sub(1, Ordering::Relaxed);
+    }
+}
+pub struct Selection {
+    pub lease: Arc<Lease>,
+    pub cookie: Option<HeaderValue>,
+}
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum SelectionFailure {
+    Unavailable,
+    Capacity,
+}
+
+impl Pool {
+    pub fn prometheus(&self) -> String {
+        use std::fmt::Write;
+        let mut text = String::new();
+        for member in &self.members {
+            let state = &member.state;
+            let labels = format!("vhost=\"{}\",backend=\"{}\"", self.id, member.definition.id);
+            for (metric, value) in [
+                (
+                    "up",
+                    u64::from(state.healthy() && member.definition.enabled),
+                ),
+                ("draining", u64::from(member.definition.drain)),
+                ("active_exchanges", state.active.load(Ordering::Relaxed)),
+                ("requests_total", state.requests.load(Ordering::Relaxed)),
+                ("errors_total", state.errors.load(Ordering::Relaxed)),
+                ("bytes_in_total", state.bytes_in.load(Ordering::Relaxed)),
+                ("bytes_out_total", state.bytes_out.load(Ordering::Relaxed)),
+            ] {
+                let _ = writeln!(text, "rust_proxy_backend_{metric}{{{labels}}} {value}");
+            }
+            for (metric, value) in [
+                ("active_health_failures_total", state.active_health_failures_total.load(Ordering::Relaxed)),
+                ("passive_health_failures_total", state.passive_health_failures_total.load(Ordering::Relaxed)),
+                ("http_5xx_total", state.http_5xx_total.load(Ordering::Relaxed)),
+                ("passive_http_5xx_total", state.http_5xx_total.load(Ordering::Relaxed)),
+                ("timeouts_total", state.timeouts_total.load(Ordering::Relaxed)),
+                ("passive_timeouts_total", state.timeouts_total.load(Ordering::Relaxed)),
+                ("connect_errors_total", state.connect_errors_total.load(Ordering::Relaxed)),
+                ("passive_connect_errors_total", state.connect_errors_total.load(Ordering::Relaxed)),
+                ("io_errors_total", state.io_errors_total.load(Ordering::Relaxed)),
+                ("passive_io_errors_total", state.io_errors_total.load(Ordering::Relaxed)),
+                ("slow_responses_total", state.slow_responses_total.load(Ordering::Relaxed)),
+                ("passive_slow_total", state.slow_responses_total.load(Ordering::Relaxed)),
+                ("retry_attempts_total", state.retry_attempts_total.load(Ordering::Relaxed)),
+                ("retry_successes_total", state.retry_successes_total.load(Ordering::Relaxed)),
+                ("capacity_rejections_total", state.capacity_rejections_total.load(Ordering::Relaxed)),
+                ("queue_timeouts_total", state.queue_timeouts_total.load(Ordering::Relaxed)),
+                ("dns_errors_total", state.dns_errors_total.load(Ordering::Relaxed)),
+                ("tls_errors_total", state.tls_errors_total.load(Ordering::Relaxed)),
+                ("response_header_timeouts_total", state.response_header_timeouts_total.load(Ordering::Relaxed)),
+                ("read_timeouts_total", state.read_timeouts_total.load(Ordering::Relaxed)),
+                ("send_timeouts_total", state.send_timeouts_total.load(Ordering::Relaxed)),
+            ] {
+                let _ = writeln!(text, "rust_proxy_backend_{metric}{{{labels}}} {value}");
+            }
+            for (metric, value) in [
+                ("last_ejected_timestamp_milliseconds", state.last_ejected_ms.load(Ordering::Relaxed)),
+                ("last_recovered_timestamp_milliseconds", state.last_recovered_ms.load(Ordering::Relaxed)),
+            ] {
+                let _ = writeln!(text, "rust_proxy_backend_{metric}{{{labels}}} {value}");
+            }
+            let _ = writeln!(
+                text,
+                "rust_proxy_backend_traffic_bytes_per_second{{{labels}}} {:.3}",
+                state.traffic_rate()
+            );
+            let _ = writeln!(
+                text,
+                "rust_proxy_backend_latency_milliseconds{{{labels}}} {:.3}",
+                state.latency()
+            );
+        }
+        text
+    }
+}
+async fn probe(member: Arc<Member>, settings: HealthCheck) {
+    let start = Instant::now();
+    let result = tokio::time::timeout(Duration::from_millis(settings.timeout_ms), async {
+        match settings.mode {
+            HealthMode::Tcp => {
+                crate::upstream::tcp_probe(&member.backend.origin, &member.backend.address_policy)
+                    .await
+                    .map(|_| "ok")
+                    .unwrap_or("connect")
+            }
+            HealthMode::Http => {
+                let request = Request::builder()
+                    .uri(format!(
+                        "{}{}",
+                        member.backend.request_origin, settings.path
+                    ))
+                    .header(
+                        "host",
+                        settings
+                            .host
+                            .as_deref()
+                            .unwrap_or_else(|| member.backend.host_header.to_str().unwrap_or("")),
+                    )
+                    .header("connection", "close")
+                    .header(
+                        "user-agent",
+                        concat!("RustProxy-health/", env!("CARGO_PKG_VERSION")),
+                    )
+                    .body(Body::empty());
+                match request {
+                    Ok(request) => match member.backend.websocket_client.request(request).await {
+                        Ok(response) => {
+                            if (settings.expected_status_min..=settings.expected_status_max)
+                                .contains(&response.status().as_u16())
+                            {
+                                "ok"
+                            } else {
+                                "http_status"
+                            }
+                        }
+                        Err(_) => "connect",
+                    },
+                    Err(_) => "request",
+                }
+            }
+            HealthMode::Off => "ok",
+        }
+    })
+    .await
+    .unwrap_or("timeout");
+    let elapsed = start.elapsed();
+    let result = if result == "ok"
+        && settings.slow_response_ms > 0
+        && elapsed > Duration::from_millis(settings.slow_response_ms)
+    {
+        "slow"
+    } else {
+        result
+    };
+    let before = member.state.healthy();
+    member.state.observe_active(&settings, result, elapsed);
+    if before != member.state.healthy() {
+        tracing::info!(backend = %member.definition.id, healthy = member.state.healthy(), reason = result, "backend health changed");
+    }
+}
+/// One supervisor, at most 16 probes, at most one outstanding probe per member.
+/// DNS workers have their own shared bound and never resolve arbitrary URLs.
+pub async fn health_loop(app: Weak<crate::runtime::App>, shutdown: CancellationToken) {
+    let mut interval = tokio::time::interval(Duration::from_millis(250));
+    interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
+    let mut checks = JoinSet::new();
+    let mut cursor = 0usize;
+    loop {
+        tokio::select! {
+            biased;
+            _ = shutdown.cancelled() => break,
+            _ = checks.join_next(), if !checks.is_empty() => {},
+            _ = interval.tick() => {
+                let Some(app) = app.upgrade() else { break; };
+                let snapshot = app.config.load_full();
+                let members: Vec<_> = snapshot.pools.iter().flat_map(|pool|
+                    pool.members.iter().map(move |member| (member, &pool.settings.health_check))).collect();
+                for (member, _) in &members { member.state.sample_traffic(); }
+                // Rotate the bounded probe window: slow first members must not
+                // starve later pools when all intervals equal their timeouts.
+                let start = cursor;
+                for offset in 0..members.len() {
+                    if checks.len() >= 16 { break; }
+                    let index = (start + offset) % members.len();
+                    let (member, health) = members[index];
+                    if member.definition.enabled && member.state.claim_probe(health) {
+                        checks.spawn(probe(member.clone(), health.clone()));
+                        cursor = (index + 1) % members.len();
+                    }
+                }
+            },
+        }
+    }
+    checks.abort_all();
+    while checks.join_next().await.is_some() {}
+}

src/balance/model.rs

@@ -0,0 +1,360 @@
+// Core pool model and policies.
+use crate::{config::CompiledBackend, vhost::PoolMember};
+use anyhow::{ensure, Result};
+use axum::body::Body;
+use http::{HeaderMap, HeaderValue, Request};
+use rand::{distributions::Alphanumeric, Rng};
+use serde::{Deserialize, Serialize};
+use serde_json::{json, Value};
+use sha2::{Digest, Sha256};
+use std::{
+    net::IpAddr,
+    sync::{
+        atomic::{AtomicBool, AtomicU16, AtomicU64, Ordering},
+        Arc, Mutex, MutexGuard, Weak,
+    },
+    time::{Duration, Instant},
+};
+use tokio::task::JoinSet;
+use tokio_util::sync::CancellationToken;
+
+fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
+    mutex.lock().unwrap_or_else(|e| e.into_inner())
+}
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum Algorithm {
+    RoundRobin,
+    #[default]
+    LeastConnections,
+    LeastTraffic,
+    Adaptive,
+    IpHash,
+}
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum HealthMode {
+    Off,
+    #[default]
+    Tcp,
+    Http,
+}
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum StickyMode {
+    #[default]
+    Off,
+    Cookie,
+    IpHash,
+}
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(default, deny_unknown_fields)]
+pub struct Keepalive {
+    pub enabled: bool,
+    pub idle_timeout_seconds: u64,
+    pub max_idle_per_host: usize,
+    pub tcp_keepalive_seconds: u64,
+}
+impl Default for Keepalive {
+    fn default() -> Self {
+        Self {
+            enabled: true,
+            idle_timeout_seconds: 30,
+            max_idle_per_host: 4,
+            tcp_keepalive_seconds: 30,
+        }
+    }
+}
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(default, deny_unknown_fields)]
+pub struct RetryPolicy {
+    pub max_retries: u8,
+    pub retry_connect_errors: bool,
+    pub retry_timeouts: bool,
+    pub retry_statuses: Vec<u16>,
+}
+impl Default for RetryPolicy {
+    fn default() -> Self {
+        Self {
+            max_retries: 1,
+            retry_connect_errors: true,
+            retry_timeouts: true,
+            retry_statuses: vec![502, 503, 504],
+        }
+    }
+}
+impl RetryPolicy {
+    pub fn retries_status(&self, status: u16) -> bool {
+        self.max_retries > 0 && self.retry_statuses.contains(&status)
+    }
+}
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(default, deny_unknown_fields)]
+pub struct HealthCheck {
+    pub mode: HealthMode,
+    pub path: String,
+    pub host: Option<String>,
+    pub interval_ms: u64,
+    pub timeout_ms: u64,
+    pub slow_response_ms: u64,
+    pub fall: u16,
+    pub rise: u16,
+    pub expected_status_min: u16,
+    pub expected_status_max: u16,
+}
+impl Default for HealthCheck {
+    fn default() -> Self {
+        Self {
+            mode: HealthMode::Tcp,
+            path: "/".into(),
+            host: None,
+            interval_ms: 5000,
+            timeout_ms: 2000,
+            slow_response_ms: 1500,
+            fall: 3,
+            rise: 2,
+            expected_status_min: 200,
+            expected_status_max: 399,
+        }
+    }
+}
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(default, deny_unknown_fields)]
+pub struct PassiveHealth {
+    pub enabled: bool,
+    pub count_connect_errors: bool,
+    pub count_timeouts: bool,
+    pub count_io_errors: bool,
+    pub count_http_5xx: bool,
+    pub count_slow_responses: bool,
+    pub fall: Option<u16>,
+    pub rise: Option<u16>,
+}
+impl Default for PassiveHealth {
+    fn default() -> Self {
+        Self {
+            enabled: false,
+            count_connect_errors: true,
+            count_timeouts: true,
+            count_io_errors: true,
+            count_http_5xx: true,
+            count_slow_responses: false,
+            fall: None,
+            rise: None,
+        }
+    }
+}
+impl PassiveHealth {
+    fn fall(&self, active: &HealthCheck) -> u16 {
+        self.fall.unwrap_or(active.fall)
+    }
+    fn rise(&self, active: &HealthCheck) -> u16 {
+        self.rise.unwrap_or(active.rise)
+    }
+    fn counts(&self, reason: &str) -> bool {
+        match reason {
+            "connect_error" => self.count_connect_errors,
+            "timeout" => self.count_timeouts,
+            "io_error" => self.count_io_errors,
+            "http_5xx" => self.count_http_5xx,
+            "slow_response" => self.count_slow_responses,
+            _ => false,
+        }
+    }
+}
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(default, deny_unknown_fields)]
+pub struct Sticky {
+    pub mode: StickyMode,
+    pub cookie_name: Option<String>,
+    pub ttl_seconds: u64,
+}
+impl Default for Sticky {
+    fn default() -> Self {
+        Self {
+            mode: StickyMode::Off,
+            cookie_name: None,
+            ttl_seconds: 3600,
+        }
+    }
+}
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(default, deny_unknown_fields)]
+pub struct Settings {
+    pub algorithm: Algorithm,
+    pub health_check: HealthCheck,
+    pub passive_health: PassiveHealth,
+    pub keepalive: Keepalive,
+    pub retry: RetryPolicy,
+    pub sticky: Sticky,
+    pub slow_start_seconds: u64,
+}
+impl Default for Settings {
+    fn default() -> Self {
+        Self {
+            algorithm: Algorithm::default(),
+            health_check: HealthCheck::default(),
+            passive_health: PassiveHealth::default(),
+            keepalive: Keepalive::default(),
+            retry: RetryPolicy::default(),
+            sticky: Sticky::default(),
+            slow_start_seconds: 10,
+        }
+    }
+}
+impl Settings {
+    pub fn validate(&self) -> Result<()> {
+        let h = &self.health_check;
+        let k = &self.keepalive;
+        ensure!(
+            (250..=300_000).contains(&h.interval_ms),
+            "health interval_ms must be 250..300000"
+        );
+        ensure!(
+            (50..=60_000).contains(&h.timeout_ms) && h.timeout_ms <= h.interval_ms,
+            "health timeout_ms must be 50..60000 and <= interval_ms"
+        );
+        ensure!(
+            h.slow_response_ms <= h.timeout_ms,
+            "slow_response_ms must be 0 (off) or <= timeout_ms"
+        );
+        ensure!(
+            (1..=20).contains(&h.fall) && (1..=20).contains(&h.rise),
+            "health fall/rise must be 1..20"
+        );
+        ensure!(
+            h.expected_status_min >= 100
+                && h.expected_status_max <= 599
+                && h.expected_status_min <= h.expected_status_max,
+            "invalid health HTTP status range"
+        );
+        ensure!(
+            h.path.starts_with('/')
+                && !h.path.starts_with("//")
+                && h.path.len() <= 1024
+                && !h.path.contains('#')
+                && !h.path.bytes().any(|b| b <= 32 || b == 127),
+            "invalid health HTTP path"
+        );
+        if let Some(host) = &h.host {
+            crate::config::request_host(host)?;
+            let _: HeaderValue = host.parse()?;
+        }
+        if let Some(fall) = self.passive_health.fall {
+            ensure!((1..=20).contains(&fall), "passive health fall must be 1..20");
+        }
+        if let Some(rise) = self.passive_health.rise {
+            ensure!((1..=20).contains(&rise), "passive health rise must be 1..20");
+        }
+        ensure!(
+            (1..=3600).contains(&k.idle_timeout_seconds)
+                && (1..=3600).contains(&k.tcp_keepalive_seconds)
+                && (1..=64).contains(&k.max_idle_per_host),
+            "invalid keepalive limits"
+        );
+        ensure!(self.retry.max_retries <= 3, "retry max_retries must be 0..3");
+        ensure!(
+            self.retry.retry_statuses.len() <= 8
+                && self
+                    .retry
+                    .retry_statuses
+                    .iter()
+                    .all(|status| (500..=599).contains(status)),
+            "retry_statuses must contain at most 8 HTTP 5xx status codes"
+        );
+        ensure!(
+            self.slow_start_seconds <= 300,
+            "slow_start_seconds must be 0..300"
+        );
+        ensure!(
+            (60..=604_800).contains(&self.sticky.ttl_seconds),
+            "sticky ttl_seconds must be 60..604800"
+        );
+        if let Some(name) = &self.sticky.cookie_name {
+            ensure!(!name.is_empty() && name.len() <= 96 && name.bytes().all(|b| b.is_ascii_alphanumeric() || matches!(b,b'_'|b'-')),
+                "affinity cookie name must use 1..96 ASCII letters, numbers, underscores or hyphens");
+            ensure!(
+                !name.eq_ignore_ascii_case(crate::auth::COOKIE_NAME),
+                "affinity cannot use the administrator session cookie name"
+            );
+        }
+        Ok(())
+    }
+}
+#[derive(Debug)]
+struct Health {
+    healthy: bool,
+    active_healthy: bool,
+    passive_healthy: bool,
+    checked: bool,
+    failures: u16,
+    passive_failures: u16,
+    successes: u16,
+    passive_successes: u16,
+    next_probe: Instant,
+    probing: bool,
+    healthy_since: Instant,
+    result: &'static str,
+    passive_result: &'static str,
+    last_probe_ms: u64,
+    latency_us: u64,
+    ewma_us: u64,
+    transitions: u64,
+    unhealthy_source: &'static str,
+    unhealthy_reason: &'static str,
+    active_unhealthy_source: &'static str,
+    active_unhealthy_reason: &'static str,
+    passive_unhealthy_source: &'static str,
+    passive_unhealthy_reason: &'static str,
+}
+#[derive(Debug)]
+struct Traffic {
+    at: Instant,
+    total: u64,
+    rate: f64,
+}
+#[derive(Debug, Clone)]
+struct FailureObservation {
+    at_ms: u64,
+    class: &'static str,
+    detail: String,
+}
+impl Default for FailureObservation {
+    fn default() -> Self {
+        Self {
+            at_ms: 0,
+            class: "none",
+            detail: String::new(),
+        }
+    }
+}
+#[derive(Debug)]
+pub struct BackendState {
+    health: Mutex<Health>,
+    traffic: Mutex<Traffic>,
+    last_failure: Mutex<FailureObservation>,
+    pub active: AtomicU64,
+    pub requests: AtomicU64,
+    pub errors: AtomicU64,
+    pub bytes_in: AtomicU64,
+    pub bytes_out: AtomicU64,
+    pub last_status: AtomicU16,
+    pub active_health_failures_total: AtomicU64,
+    pub passive_health_failures_total: AtomicU64,
+    pub http_5xx_total: AtomicU64,
+    pub timeouts_total: AtomicU64,
+    pub connect_errors_total: AtomicU64,
+    pub io_errors_total: AtomicU64,
+    pub slow_responses_total: AtomicU64,
+    pub retry_attempts_total: AtomicU64,
+    pub retry_successes_total: AtomicU64,
+    pub capacity_rejections_total: AtomicU64,
+    pub queue_timeouts_total: AtomicU64,
+    pub dns_errors_total: AtomicU64,
+    pub tls_errors_total: AtomicU64,
+    pub response_header_timeouts_total: AtomicU64,
+    pub read_timeouts_total: AtomicU64,
+    pub send_timeouts_total: AtomicU64,
+    pub last_ejected_ms: AtomicU64,
+    pub last_recovered_ms: AtomicU64,
+}

src/balance/pool_core.rs

@@ -0,0 +1,334 @@
+impl Pool {
+    pub fn new(
+        id: String,
+        settings: Settings,
+        definitions: &[PoolMember],
+        backends: Vec<Arc<CompiledBackend>>,
+    ) -> Result<Self> {
+        ensure!(
+            definitions.len() == backends.len() && !backends.is_empty(),
+            "invalid compiled pool"
+        );
+        let fingerprint = Sha256::digest(serde_json::to_vec(&(definitions, &settings))?).to_vec();
+        let cookie_name = settings
+            .sticky
+            .cookie_name
+            .clone()
+            .unwrap_or_else(|| format!("RP_AFFINITY_{id}"));
+        let members = definitions
+            .iter()
+            .cloned()
+            .zip(backends)
+            .map(|(definition, backend)| {
+                Arc::new(Member {
+                    state: Arc::new(BackendState::new(settings.health_check.mode)),
+                    definition,
+                    backend,
+                    token: rand::thread_rng()
+                        .sample_iter(&Alphanumeric)
+                        .take(32)
+                        .map(char::from)
+                        .collect(),
+                })
+            })
+            .collect();
+        Ok(Self {
+            id,
+            settings,
+            members,
+            fingerprint,
+            cookie_name,
+            cursor: AtomicU64::new(0),
+            scheduling: Mutex::new(std::collections::HashMap::new()),
+        })
+    }
+    pub fn reuse_observations(&mut self, previous: &Pool) {
+        for member in &mut self.members {
+            let Some(old) = previous
+                .members
+                .iter()
+                .find(|old| old.definition.id == member.definition.id)
+            else {
+                continue;
+            };
+            let same_origin = serde_json::to_vec(&member.definition.backend()).ok()
+                == serde_json::to_vec(&old.definition.backend()).ok();
+            let same_health = serde_json::to_vec(&(
+                &self.settings.health_check,
+                &self.settings.passive_health,
+            ))
+            .ok()
+                == serde_json::to_vec(&(
+                    &previous.settings.health_check,
+                    &previous.settings.passive_health,
+                ))
+                .ok();
+            if same_origin && same_health && (old.definition.enabled || !member.definition.enabled)
+            {
+                // This pool has not been published or cloned into routes yet.
+                if let Some(member) = Arc::get_mut(member) {
+                    member.state = old.state.clone();
+                    member.token = old.token.clone();
+                }
+            }
+        }
+    }
+    fn candidates_excluding(&self, excluded: &[String]) -> Vec<&Arc<Member>> {
+        let all: Vec<_> = self
+            .members
+            .iter()
+            .filter(|m| {
+                m.definition.pool_eligible
+                    && m.definition.enabled
+                    && !m.definition.drain
+                    && m.state.healthy()
+                    && m.state.has_capacity(m.definition.max_concurrent_requests)
+                    && !excluded.iter().any(|id| id == &m.definition.id)
+            })
+            .collect();
+        let primary = all.iter().any(|m| !m.definition.backup);
+        all.into_iter()
+            .filter(|m| !primary || !m.definition.backup)
+            .collect()
+    }
+    pub fn ready(&self) -> bool {
+        !self.eligible_healthy_excluding(&[]).is_empty()
+    }
+    pub fn cookie_affinity(&self) -> bool {
+        self.settings.sticky.mode == StickyMode::Cookie
+    }
+    fn affinity<'a>(&self, headers: &'a HeaderMap) -> Option<&'a str> {
+        let mut values = headers
+            .get_all("cookie")
+            .iter()
+            .filter_map(|v| v.to_str().ok())
+            .flat_map(|v| v.split(';'))
+            .filter_map(|pair| pair.trim().split_once('='))
+            .filter(|(name, _)| *name == self.cookie_name)
+            .map(|(_, value)| value);
+        let first = values.next()?;
+        if values.next().is_some() {
+            None
+        } else {
+            Some(first)
+        }
+    }
+    pub fn strip_affinity_response(&self, headers: &mut HeaderMap) {
+        if !self.cookie_affinity() {
+            return;
+        }
+        let retained: Vec<_> = headers
+            .get_all("set-cookie")
+            .iter()
+            .filter(|value| {
+                value
+                    .to_str()
+                    .ok()
+                    .and_then(|v| v.split_once('='))
+                    .is_none_or(|(name, _)| name.trim() != self.cookie_name)
+            })
+            .cloned()
+            .collect();
+        headers.remove("set-cookie");
+        for value in retained {
+            headers.append("set-cookie", value);
+        }
+    }
+    pub fn strip_affinity(&self, headers: &mut HeaderMap) {
+        if !self.cookie_affinity() {
+            return;
+        }
+        let values: Vec<HeaderValue> = headers
+            .get_all("cookie")
+            .iter()
+            .filter_map(|value| {
+                let Ok(text) = value.to_str() else {
+                    return Some(value.clone());
+                };
+                let clean = text
+                    .split(';')
+                    .filter(|part| {
+                        part.trim()
+                            .split_once('=')
+                            .is_none_or(|(n, _)| n != self.cookie_name)
+                    })
+                    .map(str::trim)
+                    .collect::<Vec<_>>()
+                    .join("; ");
+                if clean.is_empty() {
+                    None
+                } else {
+                    clean.parse().ok()
+                }
+            })
+            .collect();
+        headers.remove("cookie");
+        for value in values {
+            headers.append("cookie", value);
+        }
+    }
+    pub fn select_member(&self, id: &str) -> Option<Selection> {
+        let member = self.members.iter().find(|m| {
+            m.definition.id == id
+                && m.definition.enabled
+                && !m.definition.drain
+                && m.state.healthy()
+        })?;
+        if !member
+            .state
+            .try_acquire_active(member.definition.max_concurrent_requests)
+        {
+            return None;
+        }
+        Some(Selection {
+            lease: Arc::new(Lease {
+                member: member.clone(),
+                health: self.settings.health_check.clone(),
+                passive_health: self.settings.passive_health.clone(),
+                error_recorded: AtomicBool::new(false),
+            }),
+            cookie: None,
+        })
+    }
+    fn select_excluding(
+        &self,
+        peer: IpAddr,
+        headers: &HeaderMap,
+        tls: bool,
+        excluded: &[String],
+    ) -> Option<Selection> {
+        let mut scheduling = lock(&self.scheduling);
+        let members = self.candidates_excluding(excluded);
+        let existing = if self.cookie_affinity() {
+            self.affinity(headers)
+        } else {
+            None
+        };
+        // Drain stops new assignments while existing cookie-affine clients may
+        // finish their session. Disabled/unhealthy members never receive traffic.
+        let matched = existing.and_then(|token| {
+            self.members.iter().find(|m| {
+                m.token == token
+                    && m.definition.pool_eligible
+                    && m.definition.enabled
+                    && m.state.healthy()
+                    && m.state.has_capacity(m.definition.max_concurrent_requests)
+                    && !excluded.iter().any(|id| id == &m.definition.id)
+            })
+        });
+        if members.is_empty() && matched.is_none() {
+            return None;
+        }
+        let cursor = self.cursor.fetch_add(1, Ordering::Relaxed);
+        let selected = if let Some(member) = matched {
+            member
+        } else {
+            let weights: Vec<f64> = members
+                .iter()
+                .map(|m| {
+                    m.state
+                        .weight(m.definition.weight, self.settings.slow_start_seconds)
+                })
+                .collect();
+            if self.settings.algorithm == Algorithm::RoundRobin
+                && self.settings.sticky.mode != StickyMode::IpHash
+            {
+                // Smooth weighted round-robin: distribute weighted requests,
+                // rather than sending long consecutive bursts to one member.
+                let mut chosen = members[0];
+                let mut best = f64::NEG_INFINITY;
+                let mut total = 0.0;
+                for (member, weight) in members.iter().zip(&weights) {
+                    let current = scheduling
+                        .entry(member.definition.id.clone())
+                        .or_insert(0.0);
+                    *current += *weight;
+                    total += *weight;
+                    if *current > best {
+                        best = *current;
+                        chosen = *member;
+                    }
+                }
+                if let Some(current) = scheduling.get_mut(&chosen.definition.id) {
+                    *current -= total;
+                }
+                chosen
+            } else {
+                let mut chosen = members[(cursor as usize) % members.len()];
+                let mut best = f64::INFINITY;
+                for offset in 0..members.len() {
+                    let index = ((cursor as usize) % members.len() + offset) % members.len();
+                    let member = members[index];
+                    let active = member.state.active.load(Ordering::Relaxed) as f64;
+                    let algorithm = if self.settings.sticky.mode == StickyMode::IpHash {
+                        Algorithm::IpHash
+                    } else {
+                        self.settings.algorithm
+                    };
+                    let score = match algorithm {
+                        Algorithm::IpHash => {
+                            let digest = Sha256::digest(
+                                format!(
+                                    "{}|{}|{}",
+                                    self.id,
+                                    crate::vhost::canonical_ip(peer),
+                                    member.definition.id
+                                )
+                                .as_bytes(),
+                            );
+                            let n = u64::from_be_bytes([
+                                digest[0], digest[1], digest[2], digest[3], digest[4], digest[5],
+                                digest[6], digest[7],
+                            ]);
+                            // Weighted rendezvous: stable across requests, no client-controlled forwarding headers.
+                            let unit = (n as f64 + 1.0) / (u64::MAX as f64 + 2.0);
+                            -unit.max(f64::MIN_POSITIVE).ln() / member.definition.weight as f64
+                        }
+                        Algorithm::LeastTraffic => {
+                            (member.state.traffic_rate() + (active + 1.0) * 1024.0) / weights[index]
+                        }
+                        Algorithm::Adaptive => {
+                            ((active + 1.0) * member.state.latency()
+                                + member.state.traffic_rate() / 16_384.0)
+                                / weights[index]
+                        }
+                        _ => (active + 1.0) / weights[index],
+                    };
+                    if score < best {
+                        best = score;
+                        chosen = member;
+                    }
+                }
+                chosen
+            }
+        };
+        if !selected
+            .state
+            .try_acquire_active(selected.definition.max_concurrent_requests)
+        {
+            return None;
+        }
+        let cookie = if self.cookie_affinity() && existing != Some(selected.token.as_str()) {
+            format!(
+                "{}={}; Path=/; Max-Age={}; HttpOnly; SameSite=Lax{}",
+                self.cookie_name,
+                selected.token,
+                self.settings.sticky.ttl_seconds,
+                if tls { "; Secure" } else { "" }
+            )
+            .parse()
+            .ok()
+        } else {
+            None
+        };
+        Some(Selection {
+            lease: Arc::new(Lease {
+                member: selected.clone(),
+                health: self.settings.health_check.clone(),
+                passive_health: self.settings.passive_health.clone(),
+                error_recorded: AtomicBool::new(false),
+            }),
+            cookie,
+        })
+    }
+}

src/balance/pool_selection.rs

@@ -0,0 +1,132 @@
+impl Pool {
+    pub fn select(&self, peer: IpAddr, headers: &HeaderMap, tls: bool) -> Option<Selection> {
+        self.select_excluding(peer, headers, tls, &[])
+    }
+    pub fn select_retry(
+        &self,
+        peer: IpAddr,
+        headers: &HeaderMap,
+        tls: bool,
+        excluded: &[String],
+    ) -> Option<Selection> {
+        self.select_excluding(peer, headers, tls, excluded)
+    }
+    fn eligible_healthy_excluding(&self, excluded: &[String]) -> Vec<&Arc<Member>> {
+        let all: Vec<_> = self
+            .members
+            .iter()
+            .filter(|m| {
+                m.definition.pool_eligible
+                    && m.definition.enabled
+                    && !m.definition.drain
+                    && m.state.healthy()
+                    && !excluded.iter().any(|id| id == &m.definition.id)
+            })
+            .collect();
+        let primary = all.iter().any(|m| !m.definition.backup);
+        all.into_iter()
+            .filter(|m| !primary || !m.definition.backup)
+            .collect()
+    }
+    fn capacity_wait_ms(&self, excluded: &[String]) -> Option<u64> {
+        let members = self.eligible_healthy_excluding(excluded);
+        if members.is_empty() {
+            return None;
+        }
+        Some(
+            members
+                .iter()
+                .map(|m| m.definition.queue_timeout_ms)
+                .max()
+                .unwrap_or(0),
+        )
+    }
+    fn note_capacity_rejection(&self, excluded: &[String], timed_out: bool) {
+        if let Some(member) = self
+            .eligible_healthy_excluding(excluded)
+            .into_iter()
+            .min_by_key(|m| m.state.active.load(Ordering::Relaxed))
+        {
+            member
+                .state
+                .capacity_rejections_total
+                .fetch_add(1, Ordering::Relaxed);
+            if timed_out {
+                member
+                    .state
+                    .queue_timeouts_total
+                    .fetch_add(1, Ordering::Relaxed);
+            }
+        }
+    }
+    pub async fn select_wait(
+        &self,
+        peer: IpAddr,
+        headers: &HeaderMap,
+        tls: bool,
+    ) -> Result<Selection, SelectionFailure> {
+        self.select_retry_wait(peer, headers, tls, &[]).await
+    }
+    pub async fn select_member_wait(&self, id: &str) -> Result<Selection, SelectionFailure> {
+        let Some(member) = self.members.iter().find(|m| {
+            m.definition.id == id
+                && m.definition.enabled
+                && !m.definition.drain
+                && m.state.healthy()
+        }) else {
+            return Err(SelectionFailure::Unavailable);
+        };
+        let wait_ms = member.definition.queue_timeout_ms;
+        let started = Instant::now();
+        loop {
+            if !member.definition.enabled || member.definition.drain || !member.state.healthy() {
+                return Err(SelectionFailure::Unavailable);
+            }
+            if let Some(selection) = self.select_member(id) {
+                return Ok(selection);
+            }
+            if wait_ms == 0 || started.elapsed() >= Duration::from_millis(wait_ms) {
+                member
+                    .state
+                    .capacity_rejections_total
+                    .fetch_add(1, Ordering::Relaxed);
+                if wait_ms > 0 {
+                    member
+                        .state
+                        .queue_timeouts_total
+                        .fetch_add(1, Ordering::Relaxed);
+                }
+                return Err(SelectionFailure::Capacity);
+            }
+            tokio::time::sleep(Duration::from_millis(5)).await;
+        }
+    }
+    pub async fn select_retry_wait(
+        &self,
+        peer: IpAddr,
+        headers: &HeaderMap,
+        tls: bool,
+        excluded: &[String],
+    ) -> Result<Selection, SelectionFailure> {
+        if let Some(selection) = self.select_retry(peer, headers, tls, excluded) {
+            return Ok(selection);
+        }
+        let Some(wait_ms) = self.capacity_wait_ms(excluded) else {
+            return Err(SelectionFailure::Unavailable);
+        };
+        let started = Instant::now();
+        loop {
+            if self.eligible_healthy_excluding(excluded).is_empty() {
+                return Err(SelectionFailure::Unavailable);
+            }
+            if let Some(selection) = self.select_retry(peer, headers, tls, excluded) {
+                return Ok(selection);
+            }
+            if wait_ms == 0 || started.elapsed() >= Duration::from_millis(wait_ms) {
+                self.note_capacity_rejection(excluded, wait_ms > 0);
+                return Err(SelectionFailure::Capacity);
+            }
+            tokio::time::sleep(Duration::from_millis(5)).await;
+        }
+    }
+}

src/balance/pool_view.rs

@@ -0,0 +1,159 @@
+impl Pool {
+    pub fn view(&self) -> Value {
+        let members = self
+            .members
+            .iter()
+            .map(|m| {
+                let h = lock(&m.state.health);
+                let failure = lock(&m.state.last_failure);
+                let status = if !m.definition.enabled {
+                    "disabled"
+                } else if m.definition.drain {
+                    "draining"
+                } else if h.healthy {
+                    "up"
+                } else if !h.checked && self.settings.health_check.mode != HealthMode::Off {
+                    "checking"
+                } else {
+                    "down"
+                };
+
+                // Keep this as explicit Map construction instead of one very large
+                // json! invocation. serde_json's json_internal! macro is recursive
+                // and the diagnostics payload is large enough to hit rustc's default
+                // macro recursion limit on supported stable toolchains.
+                let mut item = serde_json::Map::new();
+                item.insert("id".into(), json!(m.definition.id));
+                item.insert("url".into(), json!(m.definition.url));
+                item.insert("weight".into(), json!(m.definition.weight));
+                item.insert("backup".into(), json!(m.definition.backup));
+                item.insert(
+                    "max_concurrent_requests".into(),
+                    json!(m.definition.max_concurrent_requests),
+                );
+                item.insert("queue_timeout_ms".into(), json!(m.definition.queue_timeout_ms));
+                item.insert("status".into(), json!(status));
+                item.insert("healthy".into(), json!(h.healthy));
+                item.insert("active".into(), json!(m.state.active.load(Ordering::Relaxed)));
+                item.insert(
+                    "requests".into(),
+                    json!(m.state.requests.load(Ordering::Relaxed)),
+                );
+                item.insert("errors".into(), json!(m.state.errors.load(Ordering::Relaxed)));
+                item.insert(
+                    "bytes_in".into(),
+                    json!(m.state.bytes_in.load(Ordering::Relaxed)),
+                );
+                item.insert(
+                    "bytes_out".into(),
+                    json!(m.state.bytes_out.load(Ordering::Relaxed)),
+                );
+                item.insert("traffic_bytes_per_second".into(), json!(m.state.traffic_rate()));
+                item.insert("latency_ms".into(), json!(h.latency_us as f64 / 1000.0));
+                item.insert("ewma_latency_ms".into(), json!(h.ewma_us as f64 / 1000.0));
+                item.insert("last_probe_ms".into(), json!(h.last_probe_ms));
+                item.insert("last_result".into(), json!(h.result));
+                item.insert("last_passive_result".into(), json!(h.passive_result));
+                item.insert("failures".into(), json!(h.failures));
+                item.insert("passive_failures".into(), json!(h.passive_failures));
+                item.insert("successes".into(), json!(h.successes));
+                item.insert("passive_successes".into(), json!(h.passive_successes));
+                item.insert("transitions".into(), json!(h.transitions));
+                item.insert("unhealthy_source".into(), json!(h.unhealthy_source));
+                item.insert("unhealthy_reason".into(), json!(h.unhealthy_reason));
+                item.insert(
+                    "http_5xx_total".into(),
+                    json!(m.state.http_5xx_total.load(Ordering::Relaxed)),
+                );
+                item.insert(
+                    "timeouts_total".into(),
+                    json!(m.state.timeouts_total.load(Ordering::Relaxed)),
+                );
+                item.insert(
+                    "connect_errors_total".into(),
+                    json!(m.state.connect_errors_total.load(Ordering::Relaxed)),
+                );
+                item.insert(
+                    "io_errors_total".into(),
+                    json!(m.state.io_errors_total.load(Ordering::Relaxed)),
+                );
+                item.insert(
+                    "slow_responses_total".into(),
+                    json!(m.state.slow_responses_total.load(Ordering::Relaxed)),
+                );
+                item.insert(
+                    "retry_attempts_total".into(),
+                    json!(m.state.retry_attempts_total.load(Ordering::Relaxed)),
+                );
+                item.insert(
+                    "retry_successes_total".into(),
+                    json!(m.state.retry_successes_total.load(Ordering::Relaxed)),
+                );
+                item.insert(
+                    "capacity_rejections_total".into(),
+                    json!(m.state.capacity_rejections_total.load(Ordering::Relaxed)),
+                );
+                item.insert(
+                    "queue_timeouts_total".into(),
+                    json!(m.state.queue_timeouts_total.load(Ordering::Relaxed)),
+                );
+                item.insert(
+                    "dns_errors_total".into(),
+                    json!(m.state.dns_errors_total.load(Ordering::Relaxed)),
+                );
+                item.insert(
+                    "tls_errors_total".into(),
+                    json!(m.state.tls_errors_total.load(Ordering::Relaxed)),
+                );
+                item.insert(
+                    "response_header_timeouts_total".into(),
+                    json!(m.state.response_header_timeouts_total.load(Ordering::Relaxed)),
+                );
+                item.insert(
+                    "read_timeouts_total".into(),
+                    json!(m.state.read_timeouts_total.load(Ordering::Relaxed)),
+                );
+                item.insert(
+                    "send_timeouts_total".into(),
+                    json!(m.state.send_timeouts_total.load(Ordering::Relaxed)),
+                );
+                item.insert("last_failure_at_ms".into(), json!(failure.at_ms));
+                item.insert("last_failure_class".into(), json!(failure.class));
+                item.insert("last_failure_detail".into(), json!(failure.detail.as_str()));
+                item.insert(
+                    "last_ejected_ms".into(),
+                    json!(m.state.last_ejected_ms.load(Ordering::Relaxed)),
+                );
+                item.insert(
+                    "last_recovered_ms".into(),
+                    json!(m.state.last_recovered_ms.load(Ordering::Relaxed)),
+                );
+                item.insert(
+                    "active_health_failures_total".into(),
+                    json!(m.state.active_health_failures_total.load(Ordering::Relaxed)),
+                );
+                item.insert(
+                    "passive_health_failures_total".into(),
+                    json!(m.state.passive_health_failures_total.load(Ordering::Relaxed)),
+                );
+                item.insert(
+                    "last_status".into(),
+                    json!(m.state.last_status.load(Ordering::Relaxed)),
+                );
+                Value::Object(item)
+            })
+            .collect::<Vec<_>>();
+
+        json!({
+            "id": self.id,
+            "algorithm": self.settings.algorithm,
+            "ready": self.ready(),
+            "health_mode": self.settings.health_check.mode,
+            "passive_health": self.settings.passive_health,
+            "retry": self.settings.retry,
+            "sticky_mode": self.settings.sticky.mode,
+            "keepalive": self.settings.keepalive,
+            "members": members,
+        })
+    }
+}

src/balance/tests.rs

@@ -0,0 +1,4 @@
+mod tests {
+    include!("tests/part_01.rs");
+    include!("tests/part_02.rs");
+}

src/balance/tests/part_01.rs

@@ -0,0 +1,393 @@
+    use super::*;
+    fn settings(mode: HealthMode) -> HealthCheck {
+        HealthCheck {
+            mode,
+            fall: 2,
+            rise: 2,
+            ..Default::default()
+        }
+    }
+    fn pool(algorithm: Algorithm, sticky: StickyMode) -> Arc<Pool> {
+        let cfg = serde_json::from_value(serde_json::json!({
+            "listeners":{"endpoints":[{"id":"http","binding":{"address":"127.0.0.1:8080"}}]},
+            "vhosts":[{
+            "id":"app","frontend":{"hosts":["app.localhost"],"listener_ids":["http"]},
+            "backends":[{"id":"a","url":"http://127.0.0.1:9000","allow_private":true,"weight":3},
+                {"id":"b","url":"http://127.0.0.1:9001","allow_private":true,"weight":1}],
+            "load_balancing":{"algorithm":algorithm,"sticky":{"mode":sticky},"slow_start_seconds":0,"health_check":{"mode":"off"}}
+        }]})).unwrap();
+        let policy = crate::config::ValidationPolicy {
+            proxy_addr: "127.0.0.1:8080".parse().unwrap(),
+            admin_addr: "127.0.0.1:9090".parse().unwrap(),
+            tls_addr: None,
+            cert_dir: None,
+            data_dir: None,
+        };
+        crate::config::compile(cfg, 0, &policy).unwrap().pools[0].clone()
+    }
+    fn capacity_pool(limit: u32, queue_timeout_ms: u64) -> Arc<Pool> {
+        let cfg = serde_json::from_value(serde_json::json!({
+            "listeners":{"endpoints":[{"id":"http","binding":{"address":"127.0.0.1:8080"}}]},
+            "vhosts":[{
+                "id":"limited","frontend":{"hosts":["limited.localhost"],"listener_ids":["http"]},
+                "backends":[{
+                    "id":"only","url":"http://127.0.0.1:9000","allow_private":true,
+                    "max_concurrent_requests":limit,"queue_timeout_ms":queue_timeout_ms
+                }],
+                "load_balancing":{"health_check":{"mode":"off"}}
+            }]
+        })).unwrap();
+        let policy = crate::config::ValidationPolicy {
+            proxy_addr: "127.0.0.1:8080".parse().unwrap(),
+            admin_addr: "127.0.0.1:9090".parse().unwrap(),
+            tls_addr: None,
+            cert_dir: None,
+            data_dir: None,
+        };
+        crate::config::compile(cfg, 0, &policy).unwrap().pools[0].clone()
+    }
+    fn select(pool: &Pool) -> Selection {
+        pool.select("127.0.0.1".parse().unwrap(), &HeaderMap::new(), false)
+            .unwrap()
+    }
+    #[test]
+    fn default_health_check_uses_tcp() {
+        assert_eq!(HealthMode::default(), HealthMode::Tcp);
+        assert_eq!(HealthCheck::default().mode, HealthMode::Tcp);
+    }
+    #[test]
+    fn legacy_settings_without_passive_health_default_to_off() {
+        let parsed: Settings = serde_json::from_value(serde_json::json!({
+            "health_check": { "mode": "off" }
+        }))
+        .unwrap();
+        assert!(!parsed.passive_health.enabled);
+        assert!(parsed.passive_health.count_connect_errors);
+        assert!(parsed.passive_health.count_timeouts);
+        assert!(parsed.passive_health.count_io_errors);
+        assert!(parsed.passive_health.count_http_5xx);
+        assert!(!parsed.passive_health.count_slow_responses);
+        assert_eq!(parsed.retry.max_retries, 1);
+        assert!(parsed.retry.retry_connect_errors);
+        assert!(parsed.retry.retry_timeouts);
+        assert_eq!(parsed.retry.retry_statuses, vec![502, 503, 504]);
+    }
+
+    #[test]
+    fn retry_policy_rejects_unbounded_or_non_5xx_configuration() {
+        let mut settings = Settings::default();
+        settings.retry.max_retries = 4;
+        assert!(settings.validate().is_err());
+        settings.retry.max_retries = 1;
+        settings.retry.retry_statuses = vec![429];
+        assert!(settings.validate().is_err());
+    }
+
+    #[test]
+    fn traffic_rate_drops_to_zero_after_an_idle_sample_window() {
+        let state = BackendState::new(HealthMode::Off);
+        state.bytes_out.store(2048, std::sync::atomic::Ordering::Relaxed);
+        lock(&state.traffic).at = std::time::Instant::now() - std::time::Duration::from_secs(2);
+        state.sample_traffic();
+        assert!(state.traffic_rate() > 0.0);
+        lock(&state.traffic).at = std::time::Instant::now() - std::time::Duration::from_secs(2);
+        state.sample_traffic();
+        assert_eq!(state.traffic_rate(), 0.0);
+    }
+    #[test]
+    fn active_failures_are_independent_from_passive_user_traffic() {
+        let state = BackendState::new(HealthMode::Http);
+        let settings = settings(HealthMode::Http);
+        let passive = PassiveHealth::default();
+        let fast = Duration::from_millis(5);
+        assert!(!state.healthy());
+        state.observe_active(&settings, "ok", fast);
+        assert!(!state.healthy());
+        state.observe_active(&settings, "ok", fast);
+        assert!(state.healthy());
+        state.observe_active(&settings, "status", fast);
+        state.observe_passive(&settings, &passive, "ok", fast);
+        state.observe_active(&settings, "status", fast);
+        assert!(!state.healthy());
+        state.observe_passive(&settings, &passive, "http_5xx", fast);
+        assert!(!state.healthy());
+        state.observe_active(&settings, "ok", fast);
+        assert!(!state.healthy());
+        state.observe_active(&settings, "ok", fast);
+        assert!(state.healthy());
+    }
+
+    #[test]
+    fn passive_health_is_opt_in_and_uses_separate_thresholds() {
+        let state = BackendState::new(HealthMode::Off);
+        let active = settings(HealthMode::Off);
+        let fast = Duration::from_millis(5);
+        for _ in 0..10 {
+            state.observe_passive(&active, &PassiveHealth::default(), "http_5xx", fast);
+        }
+        assert!(state.healthy());
+        assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 10);
+        assert_eq!(state.passive_health_failures_total.load(Ordering::Relaxed), 0);
+
+        let passive = PassiveHealth {
+            enabled: true,
+            fall: Some(2),
+            rise: Some(2),
+            ..Default::default()
+        };
+        state.observe_passive(&active, &passive, "http_5xx", fast);
+        assert!(state.healthy());
+        state.observe_passive(&active, &passive, "http_5xx", fast);
+        assert!(!state.healthy());
+        state.observe_passive(&active, &passive, "ok", fast);
+        assert!(!state.healthy());
+        state.observe_passive(&active, &passive, "ok", fast);
+        assert!(state.healthy());
+    }
+    #[test]
+    fn passive_http_5xx_can_be_explicitly_ignored_when_passive_health_is_on() {
+        let state = BackendState::new(HealthMode::Off);
+        let active = settings(HealthMode::Off);
+        let passive = PassiveHealth {
+            enabled: true,
+            count_http_5xx: false,
+            fall: Some(1),
+            ..Default::default()
+        };
+        for _ in 0..100 {
+            state.observe_passive(
+                &active,
+                &passive,
+                "http_5xx",
+                Duration::from_millis(5),
+            );
+        }
+        assert!(state.healthy());
+        assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 100);
+        assert_eq!(state.passive_health_failures_total.load(Ordering::Relaxed), 0);
+    }
+
+    #[test]
+    fn passive_connect_errors_use_the_explicit_fall_threshold() {
+        let state = BackendState::new(HealthMode::Off);
+        let active = settings(HealthMode::Off);
+        let passive = PassiveHealth {
+            enabled: true,
+            count_connect_errors: true,
+            fall: Some(3),
+            ..Default::default()
+        };
+        for _ in 0..2 {
+            state.observe_passive(
+                &active,
+                &passive,
+                "connect_error",
+                Duration::from_millis(5),
+            );
+            assert!(state.healthy());
+        }
+        state.observe_passive(
+            &active,
+            &passive,
+            "connect_error",
+            Duration::from_millis(5),
+        );
+        assert!(!state.healthy());
+        assert_eq!(state.connect_errors_total.load(Ordering::Relaxed), 3);
+        assert_eq!(state.passive_health_failures_total.load(Ordering::Relaxed), 3);
+    }
+
+    #[test]
+    fn active_health_can_stay_healthy_while_passive_health_is_off() {
+        let state = BackendState::new(HealthMode::Tcp);
+        let active = settings(HealthMode::Tcp);
+        let passive = PassiveHealth::default();
+        let fast = Duration::from_millis(5);
+        state.observe_active(&active, "ok", fast);
+        state.observe_active(&active, "ok", fast);
+        assert!(state.healthy());
+        for _ in 0..20 {
+            state.observe_passive(&active, &passive, "http_5xx", fast);
+        }
+        assert!(state.healthy());
+        assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 20);
+        assert_eq!(state.passive_health_failures_total.load(Ordering::Relaxed), 0);
+    }
+
+    #[test]
+    fn unhealthy_source_hands_off_when_the_original_health_domain_recovers() {
+        let state = BackendState::new(HealthMode::Tcp);
+        let active = HealthCheck {
+            fall: 1,
+            rise: 1,
+            ..settings(HealthMode::Tcp)
+        };
+        let passive = PassiveHealth {
+            enabled: true,
+            count_http_5xx: true,
+            fall: Some(1),
+            rise: Some(1),
+            ..Default::default()
+        };
+        let fast = Duration::from_millis(5);
+
+        state.observe_active(&active, "ok", fast);
+        state.observe_active(&active, "connect_error", fast);
+        {
+            let health = lock(&state.health);
+            assert_eq!(health.unhealthy_source, "active_tcp");
+        }
+        state.observe_passive(&active, &passive, "http_5xx", fast);
+        state.observe_active(&active, "ok", fast);
+        {
+            let health = lock(&state.health);
+            assert!(!health.healthy);
+            assert_eq!(health.unhealthy_source, "passive_http_5xx");
+            assert_eq!(health.unhealthy_reason, "http_5xx");
+        }
+        state.observe_passive(&active, &passive, "ok", fast);
+        assert!(state.healthy());
+    }
+
+    #[test]
+    fn disabled_checks_never_quarantine_and_probes_are_single_flight() {
+        let off = BackendState::new(HealthMode::Off);
+        let settings = settings(HealthMode::Off);
+        for _ in 0..5 {
+            off.observe_passive(&settings, &PassiveHealth::default(), "connect_error", Duration::ZERO);
+        }
+        assert!(off.healthy());
+        assert!(!off.claim_probe(&settings));
+        let on = BackendState::new(HealthMode::Http);
+        let settings = super::tests::settings(HealthMode::Http);
+        assert!(on.claim_probe(&settings));
+        assert!(!on.claim_probe(&settings));
+        on.observe_active(&settings, "ok", Duration::ZERO);
+        assert!(!on.claim_probe(&settings));
+    }
+    #[tokio::test]
+    async fn weighted_round_robin_distributes_three_to_one_without_body_collection() {
+        let pool = pool(Algorithm::RoundRobin, StickyMode::Off);
+        let mut counts = [0, 0];
+        for _ in 0..40 {
+            let selection = select(&pool);
+            counts[usize::from(selection.lease.member.definition.id == "b")] += 1;
+        }
+        assert_eq!(counts, [30, 10]);
+        assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 0);
+        assert_eq!(pool.members[0].state.requests.load(Ordering::Relaxed), 0); // Mere cache selection isn't origin traffic.
+    }
+    #[tokio::test]
+    async fn least_connections_counts_a_lease_until_its_last_owner_drops() {
+        let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
+        let mut held = Vec::new();
+        for _ in 0..3 {
+            let selection = select(&pool);
+            assert_eq!(selection.lease.member.definition.id, "a");
+            held.push(selection.lease);
+        }
+        let other = select(&pool);
+        assert_eq!(other.lease.member.definition.id, "b");
+        let clone = held[0].clone();
+        held.clear();
+        assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 1);
+        drop(clone);
+        assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 0);
+    }
+    #[tokio::test]
+    async fn failover_selection_never_reuses_an_attempted_backend() {
+        let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
+        let first = select(&pool);
+        let first_id = first.lease.member.definition.id.clone();
+        drop(first);
+        let retry = pool
+            .select_retry(
+                "127.0.0.1".parse().unwrap(),
+                &HeaderMap::new(),
+                false,
+                std::slice::from_ref(&first_id),
+            )
+            .unwrap();
+        assert_ne!(retry.lease.member.definition.id, first_id);
+    }
+    #[tokio::test]
+    async fn backend_concurrency_limit_queues_then_rejects_and_recovers() {
+        let pool = capacity_pool(1, 10);
+        let first = select(&pool);
+        let headers = HeaderMap::new();
+        let saturated = pool
+            .select_wait("127.0.0.1".parse().unwrap(), &headers, false)
+            .await;
+        assert!(matches!(saturated, Err(SelectionFailure::Capacity)));
+        let view = pool.view();
+        let member = &view["members"][0];
+        assert_eq!(member["capacity_rejections_total"], 1);
+        assert_eq!(member["queue_timeouts_total"], 1);
+        assert_eq!(member["active"], 1);
+        drop(first);
+        assert!(pool
+            .select_wait("127.0.0.1".parse().unwrap(), &headers, false)
+            .await
+            .is_ok());
+    }
+
+    #[tokio::test]
+    async fn zero_capacity_queue_timeout_rejects_without_counting_a_queue_timeout() {
+        let pool = capacity_pool(1, 0);
+        let first = select(&pool);
+        let headers = HeaderMap::new();
+        let saturated = pool
+            .select_wait("127.0.0.1".parse().unwrap(), &headers, false)
+            .await;
+        assert!(matches!(saturated, Err(SelectionFailure::Capacity)));
+        let view = pool.view();
+        let member = &view["members"][0];
+        assert_eq!(member["capacity_rejections_total"], 1);
+        assert_eq!(member["queue_timeouts_total"], 0);
+        drop(first);
+    }
+
+    #[test]
+    fn detailed_backend_failures_are_exposed_without_losing_aggregate_counters() {
+        let pool = capacity_pool(0, 0);
+        let selection = select(&pool);
+        selection
+            .lease
+            .error_detail("dns", "lookup backend.example failed", Duration::from_millis(7));
+        let view = pool.view();
+        let member = &view["members"][0];
+        assert_eq!(member["dns_errors_total"], 1);
+        assert_eq!(member["connect_errors_total"], 1);
+        assert_eq!(member["last_failure_class"], "dns");
+        assert_eq!(member["last_failure_detail"], "lookup backend.example failed");
+        assert!(member["last_failure_at_ms"].as_u64().unwrap_or_default() > 0);
+    }
+    #[tokio::test]
+    async fn traffic_and_adaptive_selection_use_observed_cost_and_weight() {
+        for algorithm in [Algorithm::LeastTraffic, Algorithm::Adaptive] {
+            let pool = pool(algorithm, StickyMode::Off);
+            lock(&pool.members[0].state.traffic).rate = 50_000_000.0;
+            lock(&pool.members[0].state.health).ewma_us = 1_000_000;
+            assert_eq!(select(&pool).lease.member.definition.id, "b");
+        }
+    }
+    #[tokio::test]
+    async fn ip_hash_is_stable_and_cannot_be_spoofed_with_forwarded_headers() {
+        let pool = pool(Algorithm::IpHash, StickyMode::Off);
+        let original = select(&pool).lease.member.definition.id.clone();
+        let mut headers = HeaderMap::new();
+        headers.insert("x-forwarded-for", "10.0.0.99".parse().unwrap());
+        for ip in ["127.0.0.1", "::ffff:127.0.0.1"] {
+            for _ in 0..8 {
+                assert_eq!(
+                    pool.select(ip.parse().unwrap(), &headers, false)
+                        .unwrap()
+                        .lease
+                        .member
+                        .definition
+                        .id,
+                    original
+                );
+            }
+        }
+    }

src/balance/tests/part_02.rs

@@ -0,0 +1,170 @@
+    #[tokio::test]
+    async fn opaque_cookie_is_removed_from_origin_requests_and_protected_on_responses() {
+        let pool = pool(Algorithm::RoundRobin, StickyMode::Cookie);
+        let first = select(&pool);
+        let cookie = first.cookie.unwrap().to_str().unwrap().to_owned();
+        assert!(
+            cookie.contains("HttpOnly")
+                && cookie.contains("SameSite=Lax")
+                && !cookie.contains("Secure")
+        );
+        let cookie = cookie.split(';').next().unwrap();
+        let mut headers = HeaderMap::new();
+        headers.insert("cookie", format!("{cookie}; app=ok").parse().unwrap());
+        let next = pool
+            .select("10.0.0.1".parse().unwrap(), &headers, false)
+            .unwrap();
+        assert_eq!(
+            next.lease.member.definition.id,
+            first.lease.member.definition.id
+        );
+        assert!(next.cookie.is_none());
+        pool.strip_affinity(&mut headers);
+        assert_eq!(headers["cookie"], "app=ok");
+        headers.append(
+            "set-cookie",
+            format!("{}=origin-spoof", pool.cookie_name)
+                .parse()
+                .unwrap(),
+        );
+        headers.append("set-cookie", "app=ok".parse().unwrap());
+        pool.strip_affinity_response(&mut headers);
+        assert_eq!(headers.get_all("set-cookie").iter().count(), 1);
+        assert_eq!(headers["set-cookie"], "app=ok");
+        let secure = pool
+            .select("127.0.0.1".parse().unwrap(), &HeaderMap::new(), true)
+            .unwrap();
+        assert!(secure
+            .cookie
+            .unwrap()
+            .to_str()
+            .unwrap()
+            .contains("; Secure"));
+    }
+    #[tokio::test]
+    async fn failed_sticky_member_is_replaced_and_duplicate_cookies_are_ignored() {
+        let pool = pool(Algorithm::RoundRobin, StickyMode::Cookie);
+        let first = select(&pool);
+        let cookie = first
+            .cookie
+            .unwrap()
+            .to_str()
+            .unwrap()
+            .split(';')
+            .next()
+            .unwrap()
+            .to_owned();
+        let mut headers = HeaderMap::new();
+        headers.insert("cookie", format!("{cookie}; {cookie}").parse().unwrap());
+        assert!(pool.affinity(&headers).is_none());
+        headers.insert("cookie", cookie.parse().unwrap());
+        lock(&first.lease.member.state.health).healthy = false;
+        let next = pool
+            .select("127.0.0.1".parse().unwrap(), &headers, false)
+            .unwrap();
+        assert_ne!(
+            next.lease.member.definition.id,
+            first.lease.member.definition.id
+        );
+        assert!(next.cookie.is_some());
+        lock(&next.lease.member.state.health).healthy = false;
+        assert!(pool
+            .select("127.0.0.1".parse().unwrap(), &headers, false)
+            .is_none());
+    }
+    #[tokio::test]
+    async fn slow_start_ramps_weight_and_error_is_counted_once_per_exchange() {
+        let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
+        let selected = select(&pool);
+        let state = &selected.lease.member.state;
+        assert!(state.weight(100, 100) < 100.0);
+        assert_eq!(state.weight(100, 0), 100.0);
+        selected.lease.started();
+        selected.lease.bytes(50, true);
+        selected.lease.bytes(100, false);
+        selected.lease.error("timeout", Duration::from_secs(1));
+        selected.lease.error("body", Duration::from_secs(2));
+        selected.lease.retry_attempted();
+        selected.lease.retry_succeeded();
+        assert_eq!(state.requests.load(Ordering::Relaxed), 1);
+        assert_eq!(state.errors.load(Ordering::Relaxed), 1);
+        assert_eq!(state.timeouts_total.load(Ordering::Relaxed), 1);
+        assert_eq!(state.io_errors_total.load(Ordering::Relaxed), 1);
+        assert_eq!(state.bytes_in.load(Ordering::Relaxed), 50);
+        assert_eq!(state.bytes_out.load(Ordering::Relaxed), 100);
+        assert_eq!(state.retry_attempts_total.load(Ordering::Relaxed), 1);
+        assert_eq!(state.retry_successes_total.load(Ordering::Relaxed), 1);
+        assert!(pool
+            .prometheus()
+            .contains("rust_proxy_backend_requests_total{vhost=\"app\",backend=\"a\"} 1"));
+        assert!(pool
+            .prometheus()
+            .contains("rust_proxy_backend_retry_attempts_total{vhost=\"app\",backend=\"a\"} 1"));
+    }
+    #[tokio::test]
+    async fn slow_5xx_counts_both_observability_classes_when_passive_is_off() {
+        let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
+        let selected = select(&pool);
+        selected.lease.response(500, Duration::from_secs(20), true);
+        let state = &selected.lease.member.state;
+        assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 1);
+        assert_eq!(state.slow_responses_total.load(Ordering::Relaxed), 1);
+        assert_eq!(state.passive_health_failures_total.load(Ordering::Relaxed), 0);
+        assert!(state.healthy());
+    }
+
+    #[tokio::test]
+    async fn client_upload_duration_does_not_trigger_passive_slow_failure() {
+        let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
+        let upload = select(&pool);
+        upload.lease.response(200, Duration::from_secs(20), false);
+        assert_eq!(upload.lease.member.state.errors.load(Ordering::Relaxed), 0);
+        let read = select(&pool);
+        read.lease.response(200, Duration::from_secs(20), true);
+        assert_eq!(read.lease.member.state.errors.load(Ordering::Relaxed), 1);
+    }
+    #[tokio::test]
+    async fn backup_is_only_new_assignment_when_no_primary_is_eligible() {
+        let mut pool = pool(Algorithm::RoundRobin, StickyMode::Off);
+        Arc::get_mut(&mut Arc::get_mut(&mut pool).unwrap().members[1])
+            .unwrap()
+            .definition
+            .backup = true;
+        for _ in 0..8 {
+            assert_eq!(select(&pool).lease.member.definition.id, "a");
+        }
+        lock(&pool.members[0].state.health).healthy = false;
+        assert_eq!(select(&pool).lease.member.definition.id, "b");
+        lock(&pool.members[0].state.health).healthy = true;
+        assert_eq!(select(&pool).lease.member.definition.id, "a");
+    }
+    #[tokio::test]
+    async fn drain_rejects_new_sessions_but_preserves_valid_existing_affinity() {
+        let mut pool = pool(Algorithm::RoundRobin, StickyMode::Cookie);
+        Arc::get_mut(&mut Arc::get_mut(&mut pool).unwrap().members[0])
+            .unwrap()
+            .definition
+            .drain = true;
+        assert_eq!(select(&pool).lease.member.definition.id, "b");
+        let mut headers = HeaderMap::new();
+        headers.insert(
+            "cookie",
+            format!("{}={}", pool.cookie_name, pool.members[0].token)
+                .parse()
+                .unwrap(),
+        );
+        let existing = pool
+            .select("127.0.0.1".parse().unwrap(), &headers, false)
+            .unwrap();
+        assert_eq!(existing.lease.member.definition.id, "a");
+        lock(&pool.members[0].state.health).healthy = false;
+        assert_eq!(
+            pool.select("127.0.0.1".parse().unwrap(), &headers, false)
+                .unwrap()
+                .lease
+                .member
+                .definition
+                .id,
+            "b"
+        );
+    }