RustProxy/src/balance/tests.rs

mod tests {
    use super::*;
    fn settings(mode: HealthMode) -> HealthCheck {
        HealthCheck {
            mode,
            fall: 2,
            rise: 2,
            ..Default::default()
        }
    }
    fn pool(algorithm: Algorithm, sticky: StickyMode) -> Arc<Pool> {
        let cfg = serde_json::from_value(serde_json::json!({
            "listeners":{"endpoints":[{"id":"http","binding":{"address":"127.0.0.1:8080"}}]},
            "vhosts":[{
            "id":"app","frontend":{"hosts":["app.localhost"],"listener_ids":["http"]},
            "backends":[{"id":"a","url":"http://127.0.0.1:9000","allow_private":true,"weight":3},
                {"id":"b","url":"http://127.0.0.1:9001","allow_private":true,"weight":1}],
            "load_balancing":{"algorithm":algorithm,"sticky":{"mode":sticky},"slow_start_seconds":0,"health_check":{"mode":"off"}}
        }]})).unwrap();
        let policy = crate::config::ValidationPolicy {
            proxy_addr: "127.0.0.1:8080".parse().unwrap(),
            admin_addr: "127.0.0.1:9090".parse().unwrap(),
            tls_addr: None,
            cert_dir: None,
            data_dir: None,
        };
        crate::config::compile(cfg, 0, &policy).unwrap().pools[0].clone()
    }
    fn capacity_pool(limit: u32, queue_timeout_ms: u64) -> Arc<Pool> {
        let cfg = serde_json::from_value(serde_json::json!({
            "listeners":{"endpoints":[{"id":"http","binding":{"address":"127.0.0.1:8080"}}]},
            "vhosts":[{
                "id":"limited","frontend":{"hosts":["limited.localhost"],"listener_ids":["http"]},
                "backends":[{
                    "id":"only","url":"http://127.0.0.1:9000","allow_private":true,
                    "max_concurrent_requests":limit,"queue_timeout_ms":queue_timeout_ms
                }],
                "load_balancing":{"health_check":{"mode":"off"}}
            }]
        }))
        .unwrap();
        let policy = crate::config::ValidationPolicy {
            proxy_addr: "127.0.0.1:8080".parse().unwrap(),
            admin_addr: "127.0.0.1:9090".parse().unwrap(),
            tls_addr: None,
            cert_dir: None,
            data_dir: None,
        };
        crate::config::compile(cfg, 0, &policy).unwrap().pools[0].clone()
    }
    fn select(pool: &Pool) -> Selection {
        pool.select("127.0.0.1".parse().unwrap(), &HeaderMap::new(), false)
            .unwrap()
    }
    #[test]
    fn per_backend_monitoring_can_be_disabled_or_overridden() {
        let mut defaults: crate::vhost::PoolMember = serde_json::from_value(serde_json::json!({
            "id": "error_404", "url": "http://127.0.0.1:8282", "pool_eligible": false
        }))
        .unwrap();
        let mut settings = Settings::default();
        settings.passive_health.enabled = true;
        assert!(defaults.health_check_enabled);
        assert_eq!(settings.health_for(&defaults).mode, HealthMode::Tcp);
        assert!(settings.passive_for(&defaults).enabled);
        defaults.health_check_enabled = false;
        assert_eq!(settings.health_for(&defaults).mode, HealthMode::Off);
        assert!(!settings.passive_for(&defaults).enabled);
        assert!(!defaults.pool_eligible);
        defaults.health_check_enabled = true;
        defaults.health_check = Some(HealthCheck {
            mode: HealthMode::Http,
            path: "/healthz".into(),
            ..Default::default()
        });
        let effective = settings.health_for(&defaults);
        assert_eq!(effective.mode, HealthMode::Http);
        assert_eq!(effective.path, "/healthz");
        assert_eq!(effective.host, settings.health_check.host);
        let restored: crate::vhost::PoolMember =
            serde_json::from_str(&serde_json::to_string(&defaults).unwrap()).unwrap();
        assert_eq!(settings.health_for(&restored).path, "/healthz");
    }
    #[test]
    fn default_health_check_uses_tcp() {
        assert_eq!(HealthMode::default(), HealthMode::Tcp);
        assert_eq!(HealthCheck::default().mode, HealthMode::Tcp);
    }
    #[test]
    fn legacy_settings_without_passive_health_default_to_off() {
        let parsed: Settings = serde_json::from_value(serde_json::json!({
            "health_check": { "mode": "off" }
        }))
        .unwrap();
        assert!(!parsed.passive_health.enabled);
        assert!(parsed.passive_health.count_connect_errors);
        assert!(parsed.passive_health.count_timeouts);
        assert!(parsed.passive_health.count_io_errors);
        assert!(parsed.passive_health.count_http_5xx);
        assert!(!parsed.passive_health.count_slow_responses);
        assert_eq!(parsed.retry.max_retries, 1);
        assert!(parsed.retry.retry_connect_errors);
        assert!(parsed.retry.retry_timeouts);
        assert_eq!(parsed.retry.retry_statuses, vec![502, 503, 504]);
        assert_eq!(parsed.keepalive.max_idle_per_host, 64);
        let explicit: Settings = serde_json::from_value(serde_json::json!({
            "keepalive": { "max_idle_per_host": 4 }
        }))
        .unwrap();
        assert_eq!(explicit.keepalive.max_idle_per_host, 4);
        assert_eq!(parsed.keepalive.pool_recycle_requests, 25_000);
        assert_eq!(parsed.keepalive.pool_recycle_seconds, 3_600);
        assert_eq!(parsed.keepalive.http1_max_buffer_bytes, 131_072);
    }

    #[test]
    fn upstream_pool_recycle_limits_are_bounded_and_can_be_disabled() {
        let mut settings = Settings::default();
        assert!(settings.validate().is_ok());

        settings.keepalive.pool_recycle_requests = 0;
        settings.keepalive.pool_recycle_seconds = 0;
        assert!(settings.validate().is_ok());

        settings.keepalive.pool_recycle_requests = 999;
        assert!(settings.validate().is_err());
        settings.keepalive.pool_recycle_requests = 1_000;
        settings.keepalive.pool_recycle_seconds = 59;
        assert!(settings.validate().is_err());
        settings.keepalive.pool_recycle_seconds = 60;
        settings.keepalive.http1_max_buffer_bytes = 8_191;
        assert!(settings.validate().is_err());
        settings.keepalive.http1_max_buffer_bytes = 8_192;
        assert!(settings.validate().is_ok());
    }

    #[test]
    fn compatible_pool_rebuild_reuses_upstream_clients() {
        let previous = pool(Algorithm::LeastConnections, StickyMode::Off);
        let mut rebuilt = pool(Algorithm::LeastConnections, StickyMode::Off);
        let rebuilt_mut = Arc::get_mut(&mut rebuilt).expect("fresh pool must be uniquely owned");

        assert!(!Arc::ptr_eq(
            &rebuilt_mut.members[0].backend,
            &previous.members[0].backend
        ));
        rebuilt_mut.reuse_observations(&previous);
        assert!(Arc::ptr_eq(
            &rebuilt_mut.members[0].backend,
            &previous.members[0].backend
        ));
    }

    #[test]
    fn retry_policy_rejects_unbounded_or_non_5xx_configuration() {
        let mut settings = Settings::default();
        settings.retry.max_retries = 4;
        assert!(settings.validate().is_err());
        settings.retry.max_retries = 1;
        settings.retry.retry_statuses = vec![429];
        assert!(settings.validate().is_err());
    }

    #[test]
    fn traffic_rate_drops_to_zero_after_an_idle_sample_window() {
        let state = BackendState::new(HealthMode::Off);
        state
            .bytes_out
            .store(2048, std::sync::atomic::Ordering::Relaxed);
        lock(&state.traffic).at = std::time::Instant::now() - std::time::Duration::from_secs(2);
        state.sample_traffic();
        assert!(state.traffic_rate() > 0.0);
        lock(&state.traffic).at = std::time::Instant::now() - std::time::Duration::from_secs(2);
        state.sample_traffic();
        assert_eq!(state.traffic_rate(), 0.0);
    }
    #[test]
    fn active_failures_are_independent_from_passive_user_traffic() {
        let state = BackendState::new(HealthMode::Http);
        let settings = settings(HealthMode::Http);
        let passive = PassiveHealth::default();
        let fast = Duration::from_millis(5);
        assert!(!state.healthy());
        state.observe_active(&settings, "ok", fast);
        assert!(!state.healthy());
        state.observe_active(&settings, "ok", fast);
        assert!(state.healthy());
        state.observe_active(&settings, "status", fast);
        state.observe_passive(&settings, &passive, "ok", fast);
        state.observe_active(&settings, "status", fast);
        assert!(!state.healthy());
        state.observe_passive(&settings, &passive, "http_5xx", fast);
        assert!(!state.healthy());
        state.observe_active(&settings, "ok", fast);
        assert!(!state.healthy());
        state.observe_active(&settings, "ok", fast);
        assert!(state.healthy());
    }

    #[test]
    fn passive_health_is_opt_in_and_uses_separate_thresholds() {
        let state = BackendState::new(HealthMode::Off);
        let active = settings(HealthMode::Off);
        let fast = Duration::from_millis(5);
        for _ in 0..10 {
            state.observe_passive(&active, &PassiveHealth::default(), "http_5xx", fast);
        }
        assert!(state.healthy());
        assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 10);
        assert_eq!(
            state.passive_health_failures_total.load(Ordering::Relaxed),
            0
        );

        let passive = PassiveHealth {
            enabled: true,
            fall: Some(2),
            rise: Some(2),
            ..Default::default()
        };
        state.observe_passive(&active, &passive, "http_5xx", fast);
        assert!(state.healthy());
        state.observe_passive(&active, &passive, "http_5xx", fast);
        assert!(!state.healthy());
        state.observe_passive(&active, &passive, "ok", fast);
        assert!(!state.healthy());
        state.observe_passive(&active, &passive, "ok", fast);
        assert!(state.healthy());
    }
    #[test]
    fn passive_http_5xx_can_be_explicitly_ignored_when_passive_health_is_on() {
        let state = BackendState::new(HealthMode::Off);
        let active = settings(HealthMode::Off);
        let passive = PassiveHealth {
            enabled: true,
            count_http_5xx: false,
            fall: Some(1),
            ..Default::default()
        };
        for _ in 0..100 {
            state.observe_passive(&active, &passive, "http_5xx", Duration::from_millis(5));
        }
        assert!(state.healthy());
        assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 100);
        assert_eq!(
            state.passive_health_failures_total.load(Ordering::Relaxed),
            0
        );
    }

    #[test]
    fn passive_connect_errors_use_the_explicit_fall_threshold() {
        let state = BackendState::new(HealthMode::Off);
        let active = settings(HealthMode::Off);
        let passive = PassiveHealth {
            enabled: true,
            count_connect_errors: true,
            fall: Some(3),
            ..Default::default()
        };
        for _ in 0..2 {
            state.observe_passive(&active, &passive, "connect_error", Duration::from_millis(5));
            assert!(state.healthy());
        }
        state.observe_passive(&active, &passive, "connect_error", Duration::from_millis(5));
        assert!(!state.healthy());
        assert_eq!(state.connect_errors_total.load(Ordering::Relaxed), 3);
        assert_eq!(
            state.passive_health_failures_total.load(Ordering::Relaxed),
            3
        );
    }

    #[test]
    fn active_health_can_stay_healthy_while_passive_health_is_off() {
        let state = BackendState::new(HealthMode::Tcp);
        let active = settings(HealthMode::Tcp);
        let passive = PassiveHealth::default();
        let fast = Duration::from_millis(5);
        state.observe_active(&active, "ok", fast);
        state.observe_active(&active, "ok", fast);
        assert!(state.healthy());
        for _ in 0..20 {
            state.observe_passive(&active, &passive, "http_5xx", fast);
        }
        assert!(state.healthy());
        assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 20);
        assert_eq!(
            state.passive_health_failures_total.load(Ordering::Relaxed),
            0
        );
    }

    #[test]
    fn unhealthy_source_hands_off_when_the_original_health_domain_recovers() {
        let state = BackendState::new(HealthMode::Tcp);
        let active = HealthCheck {
            fall: 1,
            rise: 1,
            ..settings(HealthMode::Tcp)
        };
        let passive = PassiveHealth {
            enabled: true,
            count_http_5xx: true,
            fall: Some(1),
            rise: Some(1),
            ..Default::default()
        };
        let fast = Duration::from_millis(5);

        state.observe_active(&active, "ok", fast);
        state.observe_active(&active, "connect_error", fast);
        {
            let health = lock(&state.health);
            assert_eq!(health.unhealthy_source, "active_tcp");
        }
        state.observe_passive(&active, &passive, "http_5xx", fast);
        state.observe_active(&active, "ok", fast);
        {
            let health = lock(&state.health);
            assert!(!health.healthy);
            assert_eq!(health.unhealthy_source, "passive_http_5xx");
            assert_eq!(health.unhealthy_reason, "http_5xx");
        }
        state.observe_passive(&active, &passive, "ok", fast);
        assert!(state.healthy());
    }

    #[test]
    fn disabled_checks_never_quarantine_and_probes_are_single_flight() {
        let off = BackendState::new(HealthMode::Off);
        let settings = settings(HealthMode::Off);
        for _ in 0..5 {
            off.observe_passive(
                &settings,
                &PassiveHealth::default(),
                "connect_error",
                Duration::ZERO,
            );
        }
        assert!(off.healthy());
        assert!(!off.claim_probe(&settings));
        let on = BackendState::new(HealthMode::Http);
        let settings = super::tests::settings(HealthMode::Http);
        assert!(on.claim_probe(&settings));
        assert!(!on.claim_probe(&settings));
        on.observe_active(&settings, "ok", Duration::ZERO);
        assert!(!on.claim_probe(&settings));
    }
    #[tokio::test]
    async fn weighted_round_robin_distributes_three_to_one_without_body_collection() {
        let pool = pool(Algorithm::RoundRobin, StickyMode::Off);
        let mut counts = [0, 0];
        for _ in 0..40 {
            let selection = select(&pool);
            counts[usize::from(selection.lease.member.definition.id == "b")] += 1;
        }
        assert_eq!(counts, [30, 10]);
        assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 0);
        assert_eq!(pool.members[0].state.requests.load(Ordering::Relaxed), 0);
    }
    #[tokio::test]
    async fn least_connections_counts_a_lease_until_its_last_owner_drops() {
        let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
        let mut held = Vec::new();
        for _ in 0..3 {
            let selection = select(&pool);
            assert_eq!(selection.lease.member.definition.id, "a");
            held.push(selection.lease);
        }
        let other = select(&pool);
        assert_eq!(other.lease.member.definition.id, "b");
        let clone = held[0].clone();
        held.clear();
        assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 1);
        drop(clone);
        assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 0);
    }
    #[tokio::test]
    async fn failover_selection_never_reuses_an_attempted_backend() {
        let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
        let first = select(&pool);
        let first_id = first.lease.member.definition.id.clone();
        drop(first);
        let retry = pool
            .select_retry(
                "127.0.0.1".parse().unwrap(),
                &HeaderMap::new(),
                false,
                std::slice::from_ref(&first_id),
            )
            .unwrap();
        assert_ne!(retry.lease.member.definition.id, first_id);
    }
    #[tokio::test]
    async fn backend_concurrency_limit_queues_then_rejects_and_recovers() {
        let pool = capacity_pool(1, 10);
        let first = select(&pool);
        let headers = HeaderMap::new();
        let saturated = pool
            .select_wait("127.0.0.1".parse().unwrap(), &headers, false)
            .await;
        assert!(matches!(saturated, Err(SelectionFailure::Capacity)));
        let view = pool.view();
        let member = &view["members"][0];
        assert_eq!(member["capacity_rejections_total"], 1);
        assert_eq!(member["queue_timeouts_total"], 1);
        assert_eq!(member["active"], 1);
        drop(first);
        assert!(pool
            .select_wait("127.0.0.1".parse().unwrap(), &headers, false)
            .await
            .is_ok());
    }

    #[tokio::test]
    async fn zero_capacity_queue_timeout_rejects_without_counting_a_queue_timeout() {
        let pool = capacity_pool(1, 0);
        let first = select(&pool);
        let headers = HeaderMap::new();
        let saturated = pool
            .select_wait("127.0.0.1".parse().unwrap(), &headers, false)
            .await;
        assert!(matches!(saturated, Err(SelectionFailure::Capacity)));
        let view = pool.view();
        let member = &view["members"][0];
        assert_eq!(member["capacity_rejections_total"], 1);
        assert_eq!(member["queue_timeouts_total"], 0);
        drop(first);
    }

    #[test]
    fn detailed_backend_failures_are_exposed_without_losing_aggregate_counters() {
        let pool = capacity_pool(0, 0);
        let selection = select(&pool);
        selection.lease.error_detail(
            "dns",
            "lookup backend.example failed",
            Duration::from_millis(7),
        );
        let view = pool.view();
        let member = &view["members"][0];
        assert_eq!(member["dns_errors_total"], 1);
        assert_eq!(member["connect_errors_total"], 1);
        assert_eq!(member["last_failure_class"], "dns");
        assert_eq!(
            member["last_failure_detail"],
            "lookup backend.example failed"
        );
        assert!(member["last_failure_at_ms"].as_u64().unwrap_or_default() > 0);
    }
    #[tokio::test]
    async fn traffic_and_adaptive_selection_use_observed_cost_and_weight() {
        for algorithm in [Algorithm::LeastTraffic, Algorithm::Adaptive] {
            let pool = pool(algorithm, StickyMode::Off);
            lock(&pool.members[0].state.traffic).rate = 50_000_000.0;
            lock(&pool.members[0].state.health).ewma_us = 1_000_000;
            assert_eq!(select(&pool).lease.member.definition.id, "b");
        }
    }
    #[tokio::test]
    async fn ip_hash_is_stable_and_cannot_be_spoofed_with_forwarded_headers() {
        let pool = pool(Algorithm::IpHash, StickyMode::Off);
        let original = select(&pool).lease.member.definition.id.clone();
        let mut headers = HeaderMap::new();
        headers.insert("x-forwarded-for", "10.0.0.99".parse().unwrap());
        for ip in ["127.0.0.1", "::ffff:127.0.0.1"] {
            for _ in 0..8 {
                assert_eq!(
                    pool.select(ip.parse().unwrap(), &headers, false)
                        .unwrap()
                        .lease
                        .member
                        .definition
                        .id,
                    original
                );
            }
        }
    }

    #[tokio::test]
    async fn opaque_cookie_is_removed_from_origin_requests_and_protected_on_responses() {
        let pool = pool(Algorithm::RoundRobin, StickyMode::Cookie);
        let first = select(&pool);
        let cookie = first.cookie.unwrap().to_str().unwrap().to_owned();
        assert!(
            cookie.contains("HttpOnly")
                && cookie.contains("SameSite=Lax")
                && !cookie.contains("Secure")
        );
        let cookie = cookie.split(';').next().unwrap();
        let mut headers = HeaderMap::new();
        headers.insert("cookie", format!("{cookie}; app=ok").parse().unwrap());
        let next = pool
            .select("10.0.0.1".parse().unwrap(), &headers, false)
            .unwrap();
        assert_eq!(
            next.lease.member.definition.id,
            first.lease.member.definition.id
        );
        assert!(next.cookie.is_none());
        pool.strip_affinity(&mut headers);
        assert_eq!(headers["cookie"], "app=ok");
        headers.append(
            "set-cookie",
            format!("{}=origin-spoof", pool.cookie_name)
                .parse()
                .unwrap(),
        );
        headers.append("set-cookie", "app=ok".parse().unwrap());
        pool.strip_affinity_response(&mut headers);
        assert_eq!(headers.get_all("set-cookie").iter().count(), 1);
        assert_eq!(headers["set-cookie"], "app=ok");
        let secure = pool
            .select("127.0.0.1".parse().unwrap(), &HeaderMap::new(), true)
            .unwrap();
        assert!(secure
            .cookie
            .unwrap()
            .to_str()
            .unwrap()
            .contains("; Secure"));
    }
    #[tokio::test]
    async fn failed_sticky_member_is_replaced_and_duplicate_cookies_are_ignored() {
        let pool = pool(Algorithm::RoundRobin, StickyMode::Cookie);
        let first = select(&pool);
        let cookie = first
            .cookie
            .unwrap()
            .to_str()
            .unwrap()
            .split(';')
            .next()
            .unwrap()
            .to_owned();
        let mut headers = HeaderMap::new();
        headers.insert("cookie", format!("{cookie}; {cookie}").parse().unwrap());
        assert!(pool.affinity(&headers).is_none());
        headers.insert("cookie", cookie.parse().unwrap());
        first.lease.member.state.set_healthy_for_test(false);
        let next = pool
            .select("127.0.0.1".parse().unwrap(), &headers, false)
            .unwrap();
        assert_ne!(
            next.lease.member.definition.id,
            first.lease.member.definition.id
        );
        assert!(next.cookie.is_some());
        next.lease.member.state.set_healthy_for_test(false);
        assert!(pool
            .select("127.0.0.1".parse().unwrap(), &headers, false)
            .is_none());
    }
    #[tokio::test]
    async fn slow_start_ramps_weight_and_error_is_counted_once_per_exchange() {
        let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
        let selected = select(&pool);
        let state = &selected.lease.member.state;
        assert!(state.weight(100, 100) < 100.0);
        assert_eq!(state.weight(100, 0), 100.0);
        selected.lease.started();
        selected.lease.bytes(50, true);
        selected.lease.bytes(100, false);
        selected.lease.error("timeout", Duration::from_secs(1));
        selected.lease.error("body", Duration::from_secs(2));
        selected.lease.retry_attempted();
        selected.lease.retry_succeeded();
        assert_eq!(state.requests.load(Ordering::Relaxed), 1);
        assert_eq!(state.errors.load(Ordering::Relaxed), 1);
        assert_eq!(state.timeouts_total.load(Ordering::Relaxed), 1);
        assert_eq!(state.io_errors_total.load(Ordering::Relaxed), 1);
        assert_eq!(state.bytes_in.load(Ordering::Relaxed), 50);
        assert_eq!(state.bytes_out.load(Ordering::Relaxed), 100);
        assert_eq!(state.retry_attempts_total.load(Ordering::Relaxed), 1);
        assert_eq!(state.retry_successes_total.load(Ordering::Relaxed), 1);
        assert!(pool
            .prometheus()
            .contains("rust_proxy_backend_requests_total{vhost=\"app\",backend=\"a\"} 1"));
        assert!(pool
            .prometheus()
            .contains("rust_proxy_backend_retry_attempts_total{vhost=\"app\",backend=\"a\"} 1"));
    }
    #[tokio::test]
    async fn slow_5xx_counts_both_observability_classes_when_passive_is_off() {
        let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
        let selected = select(&pool);
        selected.lease.response(500, Duration::from_secs(20), true);
        let state = &selected.lease.member.state;
        assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 1);
        assert_eq!(state.slow_responses_total.load(Ordering::Relaxed), 1);
        assert_eq!(
            state.passive_health_failures_total.load(Ordering::Relaxed),
            0
        );
        assert!(state.healthy());
    }

    #[tokio::test]
    async fn client_upload_duration_does_not_trigger_passive_slow_failure() {
        let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
        let upload = select(&pool);
        upload.lease.response(200, Duration::from_secs(20), false);
        assert_eq!(upload.lease.member.state.errors.load(Ordering::Relaxed), 0);
        let read = select(&pool);
        read.lease.response(200, Duration::from_secs(20), true);
        assert_eq!(read.lease.member.state.errors.load(Ordering::Relaxed), 1);
    }
    #[tokio::test]
    async fn backup_is_only_new_assignment_when_no_primary_is_eligible() {
        let mut pool = pool(Algorithm::RoundRobin, StickyMode::Off);
        Arc::get_mut(&mut Arc::get_mut(&mut pool).unwrap().members[1])
            .unwrap()
            .definition
            .backup = true;
        for _ in 0..8 {
            assert_eq!(select(&pool).lease.member.definition.id, "a");
        }
        pool.members[0].state.set_healthy_for_test(false);
        assert_eq!(select(&pool).lease.member.definition.id, "b");
        pool.members[0].state.set_healthy_for_test(true);
        assert_eq!(select(&pool).lease.member.definition.id, "a");
    }
    #[tokio::test]
    async fn drain_rejects_new_sessions_but_preserves_valid_existing_affinity() {
        let mut pool = pool(Algorithm::RoundRobin, StickyMode::Cookie);
        Arc::get_mut(&mut Arc::get_mut(&mut pool).unwrap().members[0])
            .unwrap()
            .definition
            .drain = true;
        assert_eq!(select(&pool).lease.member.definition.id, "b");
        let mut headers = HeaderMap::new();
        headers.insert(
            "cookie",
            format!("{}={}", pool.cookie_name, pool.members[0].token)
                .parse()
                .unwrap(),
        );
        let existing = pool
            .select("127.0.0.1".parse().unwrap(), &headers, false)
            .unwrap();
        assert_eq!(existing.lease.member.definition.id, "a");
        pool.members[0].state.set_healthy_for_test(false);
        assert_eq!(
            pool.select("127.0.0.1".parse().unwrap(), &headers, false)
                .unwrap()
                .lease
                .member
                .definition
                .id,
            "b"
        );
    }

    #[test]
    fn passive_ejection_remembers_first_real_error_and_trigger() {
        let state = BackendState::new(HealthMode::Off);
        let active = settings(HealthMode::Off);
        let passive = PassiveHealth {
            enabled: true,
            fall: Some(2),
            ..Default::default()
        };
        let elapsed = Duration::from_millis(3);
        assert!(state
            .observe_passive_with_detail(
                &active,
                &passive,
                "connect_error",
                "dns",
                "first lookup failed",
                elapsed
            )
            .is_none());
        let (first, failures) = state
            .observe_passive_with_detail(
                &active,
                &passive,
                "connect_error",
                "connect",
                "second TCP failed",
                elapsed,
            )
            .expect("the threshold crossing must return the first failure");
        assert_eq!(failures, 2);
        assert_eq!(first.class, "dns");
        assert_eq!(first.detail, "first lookup failed");
        assert!(!state.healthy());
        assert!(
            state
                .observe_passive_with_detail(
                    &active,
                    &passive,
                    "connect_error",
                    "connect",
                    "third TCP failed",
                    elapsed
                )
                .is_none(),
            "the ejection must be logged only once"
        );
    }

    #[test]
    fn passive_io_burst_is_one_observation_not_many_failed_connections() {
        let state = BackendState::new(HealthMode::Off);
        let active = settings(HealthMode::Off);
        let passive = PassiveHealth {
            enabled: true,
            fall: Some(3),
            ..Default::default()
        };
        for _ in 0..20 {
            state.observe_passive_with_detail(
                &active,
                &passive,
                "io_error",
                "io_error",
                "connection closed",
                Duration::from_millis(1),
            );
        }
        assert!(state.healthy());
        assert_eq!(state.io_errors_total.load(Ordering::Relaxed), 20);
        assert_eq!(
            state.passive_health_failures_total.load(Ordering::Relaxed),
            1
        );
    }

    #[test]
    fn passive_only_quarantine_expires_without_vhost_reload() {
        let state = BackendState::new(HealthMode::Off);
        let active = settings(HealthMode::Off);
        let passive = PassiveHealth {
            enabled: true,
            fall: Some(1),
            ..Default::default()
        };
        state.observe_passive(&active, &passive, "connect_error", Duration::ZERO);
        assert!(!state.healthy());
        lock(&state.health).passive_ejected_at = Some(Instant::now() - Duration::from_secs(10));
        assert!(
            state.healthy(),
            "a half-open traffic attempt must become possible"
        );
    }

    #[test]
    fn active_probe_successes_recover_a_passively_ejected_backend() {
        let state = BackendState::new(HealthMode::Http);
        let active = HealthCheck {
            fall: 1,
            rise: 1,
            ..settings(HealthMode::Http)
        };
        let passive = PassiveHealth {
            enabled: true,
            fall: Some(1),
            ..Default::default()
        };
        state.observe_active(&active, "ok", Duration::ZERO);
        assert!(state.healthy());
        state.observe_passive(&active, &passive, "connect_error", Duration::ZERO);
        assert!(!state.healthy());
        state.observe_active(&active, "ok", Duration::ZERO);
        assert!(!state.healthy());
        state.observe_active(&active, "ok", Duration::ZERO);
        assert!(state.healthy());
    }

    #[test]
    fn single_backend_fast_path_observes_capacity_and_health_without_affinity_cookie() {
        let pool = capacity_pool(1, 0);
        assert_eq!(pool.members.len(), 1);
        let peer = "127.0.0.1".parse().unwrap();
        let headers = HeaderMap::new();
        let first = pool.select(peer, &headers, false).expect("healthy backend");
        assert!(first.cookie.is_none());
        assert!(pool.select(peer, &headers, false).is_none());
        drop(first);
        pool.members[0].state.set_healthy_for_test(false);
        assert!(pool.select(peer, &headers, false).is_none());
        pool.members[0].state.set_healthy_for_test(true);
        assert!(pool.select(peer, &headers, false).is_some());
    }

    #[test]
    fn leases_reuse_immutable_backend_health_policies() {
        let pool = capacity_pool(0, 0);
        let first = select(&pool);
        let second = select(&pool);
        assert!(Arc::ptr_eq(&first.lease.health, &second.lease.health));
        assert!(Arc::ptr_eq(
            &first.lease.passive_health,
            &second.lease.passive_health
        ));
    }

    #[test]
    fn healthy_fast_path_tracks_active_and_passive_ejections() {
        let state = BackendState::new(HealthMode::Off);
        assert!(state.healthy());
        let active = settings(HealthMode::Off);
        let passive = PassiveHealth {
            enabled: true,
            fall: Some(1),
            rise: Some(1),
            ..Default::default()
        };
        state.observe_passive(&active, &passive, "connect_error", Duration::from_millis(2));
        assert!(!state.healthy());
        state.observe_passive(&active, &passive, "ok", Duration::from_millis(2));
        assert!(state.healthy());
    }

    #[test]
    fn successful_lease_response_keeps_passive_observability_and_failure_detail() {
        let pool = capacity_pool(0, 0);
        let selection = select(&pool);
        selection
            .lease
            .response(200, Duration::from_millis(5), true);
        let state = &selection.lease.member.state;
        assert_eq!(state.last_status.load(Ordering::Relaxed), 200);
        assert_eq!(lock(&state.health).passive_result, "ok");
        assert!(lock(&state.health).latency_us > 0);
        selection
            .lease
            .response(503, Duration::from_millis(5), true);
        assert_eq!(state.last_status.load(Ordering::Relaxed), 503);
        assert_eq!(lock(&state.last_failure).detail, "status=503");
    }
}