v0.25.1

src/balance/tests/part_01.rs

@@ -0,0 +1,393 @@
+    use super::*;
+    fn settings(mode: HealthMode) -> HealthCheck {
+        HealthCheck {
+            mode,
+            fall: 2,
+            rise: 2,
+            ..Default::default()
+        }
+    }
+    fn pool(algorithm: Algorithm, sticky: StickyMode) -> Arc<Pool> {
+        let cfg = serde_json::from_value(serde_json::json!({
+            "listeners":{"endpoints":[{"id":"http","binding":{"address":"127.0.0.1:8080"}}]},
+            "vhosts":[{
+            "id":"app","frontend":{"hosts":["app.localhost"],"listener_ids":["http"]},
+            "backends":[{"id":"a","url":"http://127.0.0.1:9000","allow_private":true,"weight":3},
+                {"id":"b","url":"http://127.0.0.1:9001","allow_private":true,"weight":1}],
+            "load_balancing":{"algorithm":algorithm,"sticky":{"mode":sticky},"slow_start_seconds":0,"health_check":{"mode":"off"}}
+        }]})).unwrap();
+        let policy = crate::config::ValidationPolicy {
+            proxy_addr: "127.0.0.1:8080".parse().unwrap(),
+            admin_addr: "127.0.0.1:9090".parse().unwrap(),
+            tls_addr: None,
+            cert_dir: None,
+            data_dir: None,
+        };
+        crate::config::compile(cfg, 0, &policy).unwrap().pools[0].clone()
+    }
+    fn capacity_pool(limit: u32, queue_timeout_ms: u64) -> Arc<Pool> {
+        let cfg = serde_json::from_value(serde_json::json!({
+            "listeners":{"endpoints":[{"id":"http","binding":{"address":"127.0.0.1:8080"}}]},
+            "vhosts":[{
+                "id":"limited","frontend":{"hosts":["limited.localhost"],"listener_ids":["http"]},
+                "backends":[{
+                    "id":"only","url":"http://127.0.0.1:9000","allow_private":true,
+                    "max_concurrent_requests":limit,"queue_timeout_ms":queue_timeout_ms
+                }],
+                "load_balancing":{"health_check":{"mode":"off"}}
+            }]
+        })).unwrap();
+        let policy = crate::config::ValidationPolicy {
+            proxy_addr: "127.0.0.1:8080".parse().unwrap(),
+            admin_addr: "127.0.0.1:9090".parse().unwrap(),
+            tls_addr: None,
+            cert_dir: None,
+            data_dir: None,
+        };
+        crate::config::compile(cfg, 0, &policy).unwrap().pools[0].clone()
+    }
+    fn select(pool: &Pool) -> Selection {
+        pool.select("127.0.0.1".parse().unwrap(), &HeaderMap::new(), false)
+            .unwrap()
+    }
+    #[test]
+    fn default_health_check_uses_tcp() {
+        assert_eq!(HealthMode::default(), HealthMode::Tcp);
+        assert_eq!(HealthCheck::default().mode, HealthMode::Tcp);
+    }
+    #[test]
+    fn legacy_settings_without_passive_health_default_to_off() {
+        let parsed: Settings = serde_json::from_value(serde_json::json!({
+            "health_check": { "mode": "off" }
+        }))
+        .unwrap();
+        assert!(!parsed.passive_health.enabled);
+        assert!(parsed.passive_health.count_connect_errors);
+        assert!(parsed.passive_health.count_timeouts);
+        assert!(parsed.passive_health.count_io_errors);
+        assert!(parsed.passive_health.count_http_5xx);
+        assert!(!parsed.passive_health.count_slow_responses);
+        assert_eq!(parsed.retry.max_retries, 1);
+        assert!(parsed.retry.retry_connect_errors);
+        assert!(parsed.retry.retry_timeouts);
+        assert_eq!(parsed.retry.retry_statuses, vec![502, 503, 504]);
+    }
+
+    #[test]
+    fn retry_policy_rejects_unbounded_or_non_5xx_configuration() {
+        let mut settings = Settings::default();
+        settings.retry.max_retries = 4;
+        assert!(settings.validate().is_err());
+        settings.retry.max_retries = 1;
+        settings.retry.retry_statuses = vec![429];
+        assert!(settings.validate().is_err());
+    }
+
+    #[test]
+    fn traffic_rate_drops_to_zero_after_an_idle_sample_window() {
+        let state = BackendState::new(HealthMode::Off);
+        state.bytes_out.store(2048, std::sync::atomic::Ordering::Relaxed);
+        lock(&state.traffic).at = std::time::Instant::now() - std::time::Duration::from_secs(2);
+        state.sample_traffic();
+        assert!(state.traffic_rate() > 0.0);
+        lock(&state.traffic).at = std::time::Instant::now() - std::time::Duration::from_secs(2);
+        state.sample_traffic();
+        assert_eq!(state.traffic_rate(), 0.0);
+    }
+    #[test]
+    fn active_failures_are_independent_from_passive_user_traffic() {
+        let state = BackendState::new(HealthMode::Http);
+        let settings = settings(HealthMode::Http);
+        let passive = PassiveHealth::default();
+        let fast = Duration::from_millis(5);
+        assert!(!state.healthy());
+        state.observe_active(&settings, "ok", fast);
+        assert!(!state.healthy());
+        state.observe_active(&settings, "ok", fast);
+        assert!(state.healthy());
+        state.observe_active(&settings, "status", fast);
+        state.observe_passive(&settings, &passive, "ok", fast);
+        state.observe_active(&settings, "status", fast);
+        assert!(!state.healthy());
+        state.observe_passive(&settings, &passive, "http_5xx", fast);
+        assert!(!state.healthy());
+        state.observe_active(&settings, "ok", fast);
+        assert!(!state.healthy());
+        state.observe_active(&settings, "ok", fast);
+        assert!(state.healthy());
+    }
+
+    #[test]
+    fn passive_health_is_opt_in_and_uses_separate_thresholds() {
+        let state = BackendState::new(HealthMode::Off);
+        let active = settings(HealthMode::Off);
+        let fast = Duration::from_millis(5);
+        for _ in 0..10 {
+            state.observe_passive(&active, &PassiveHealth::default(), "http_5xx", fast);
+        }
+        assert!(state.healthy());
+        assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 10);
+        assert_eq!(state.passive_health_failures_total.load(Ordering::Relaxed), 0);
+
+        let passive = PassiveHealth {
+            enabled: true,
+            fall: Some(2),
+            rise: Some(2),
+            ..Default::default()
+        };
+        state.observe_passive(&active, &passive, "http_5xx", fast);
+        assert!(state.healthy());
+        state.observe_passive(&active, &passive, "http_5xx", fast);
+        assert!(!state.healthy());
+        state.observe_passive(&active, &passive, "ok", fast);
+        assert!(!state.healthy());
+        state.observe_passive(&active, &passive, "ok", fast);
+        assert!(state.healthy());
+    }
+    #[test]
+    fn passive_http_5xx_can_be_explicitly_ignored_when_passive_health_is_on() {
+        let state = BackendState::new(HealthMode::Off);
+        let active = settings(HealthMode::Off);
+        let passive = PassiveHealth {
+            enabled: true,
+            count_http_5xx: false,
+            fall: Some(1),
+            ..Default::default()
+        };
+        for _ in 0..100 {
+            state.observe_passive(
+                &active,
+                &passive,
+                "http_5xx",
+                Duration::from_millis(5),
+            );
+        }
+        assert!(state.healthy());
+        assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 100);
+        assert_eq!(state.passive_health_failures_total.load(Ordering::Relaxed), 0);
+    }
+
+    #[test]
+    fn passive_connect_errors_use_the_explicit_fall_threshold() {
+        let state = BackendState::new(HealthMode::Off);
+        let active = settings(HealthMode::Off);
+        let passive = PassiveHealth {
+            enabled: true,
+            count_connect_errors: true,
+            fall: Some(3),
+            ..Default::default()
+        };
+        for _ in 0..2 {
+            state.observe_passive(
+                &active,
+                &passive,
+                "connect_error",
+                Duration::from_millis(5),
+            );
+            assert!(state.healthy());
+        }
+        state.observe_passive(
+            &active,
+            &passive,
+            "connect_error",
+            Duration::from_millis(5),
+        );
+        assert!(!state.healthy());
+        assert_eq!(state.connect_errors_total.load(Ordering::Relaxed), 3);
+        assert_eq!(state.passive_health_failures_total.load(Ordering::Relaxed), 3);
+    }
+
+    #[test]
+    fn active_health_can_stay_healthy_while_passive_health_is_off() {
+        let state = BackendState::new(HealthMode::Tcp);
+        let active = settings(HealthMode::Tcp);
+        let passive = PassiveHealth::default();
+        let fast = Duration::from_millis(5);
+        state.observe_active(&active, "ok", fast);
+        state.observe_active(&active, "ok", fast);
+        assert!(state.healthy());
+        for _ in 0..20 {
+            state.observe_passive(&active, &passive, "http_5xx", fast);
+        }
+        assert!(state.healthy());
+        assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 20);
+        assert_eq!(state.passive_health_failures_total.load(Ordering::Relaxed), 0);
+    }
+
+    #[test]
+    fn unhealthy_source_hands_off_when_the_original_health_domain_recovers() {
+        let state = BackendState::new(HealthMode::Tcp);
+        let active = HealthCheck {
+            fall: 1,
+            rise: 1,
+            ..settings(HealthMode::Tcp)
+        };
+        let passive = PassiveHealth {
+            enabled: true,
+            count_http_5xx: true,
+            fall: Some(1),
+            rise: Some(1),
+            ..Default::default()
+        };
+        let fast = Duration::from_millis(5);
+
+        state.observe_active(&active, "ok", fast);
+        state.observe_active(&active, "connect_error", fast);
+        {
+            let health = lock(&state.health);
+            assert_eq!(health.unhealthy_source, "active_tcp");
+        }
+        state.observe_passive(&active, &passive, "http_5xx", fast);
+        state.observe_active(&active, "ok", fast);
+        {
+            let health = lock(&state.health);
+            assert!(!health.healthy);
+            assert_eq!(health.unhealthy_source, "passive_http_5xx");
+            assert_eq!(health.unhealthy_reason, "http_5xx");
+        }
+        state.observe_passive(&active, &passive, "ok", fast);
+        assert!(state.healthy());
+    }
+
+    #[test]
+    fn disabled_checks_never_quarantine_and_probes_are_single_flight() {
+        let off = BackendState::new(HealthMode::Off);
+        let settings = settings(HealthMode::Off);
+        for _ in 0..5 {
+            off.observe_passive(&settings, &PassiveHealth::default(), "connect_error", Duration::ZERO);
+        }
+        assert!(off.healthy());
+        assert!(!off.claim_probe(&settings));
+        let on = BackendState::new(HealthMode::Http);
+        let settings = super::tests::settings(HealthMode::Http);
+        assert!(on.claim_probe(&settings));
+        assert!(!on.claim_probe(&settings));
+        on.observe_active(&settings, "ok", Duration::ZERO);
+        assert!(!on.claim_probe(&settings));
+    }
+    #[tokio::test]
+    async fn weighted_round_robin_distributes_three_to_one_without_body_collection() {
+        let pool = pool(Algorithm::RoundRobin, StickyMode::Off);
+        let mut counts = [0, 0];
+        for _ in 0..40 {
+            let selection = select(&pool);
+            counts[usize::from(selection.lease.member.definition.id == "b")] += 1;
+        }
+        assert_eq!(counts, [30, 10]);
+        assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 0);
+        assert_eq!(pool.members[0].state.requests.load(Ordering::Relaxed), 0); // Mere cache selection isn't origin traffic.
+    }
+    #[tokio::test]
+    async fn least_connections_counts_a_lease_until_its_last_owner_drops() {
+        let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
+        let mut held = Vec::new();
+        for _ in 0..3 {
+            let selection = select(&pool);
+            assert_eq!(selection.lease.member.definition.id, "a");
+            held.push(selection.lease);
+        }
+        let other = select(&pool);
+        assert_eq!(other.lease.member.definition.id, "b");
+        let clone = held[0].clone();
+        held.clear();
+        assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 1);
+        drop(clone);
+        assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 0);
+    }
+    #[tokio::test]
+    async fn failover_selection_never_reuses_an_attempted_backend() {
+        let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
+        let first = select(&pool);
+        let first_id = first.lease.member.definition.id.clone();
+        drop(first);
+        let retry = pool
+            .select_retry(
+                "127.0.0.1".parse().unwrap(),
+                &HeaderMap::new(),
+                false,
+                std::slice::from_ref(&first_id),
+            )
+            .unwrap();
+        assert_ne!(retry.lease.member.definition.id, first_id);
+    }
+    #[tokio::test]
+    async fn backend_concurrency_limit_queues_then_rejects_and_recovers() {
+        let pool = capacity_pool(1, 10);
+        let first = select(&pool);
+        let headers = HeaderMap::new();
+        let saturated = pool
+            .select_wait("127.0.0.1".parse().unwrap(), &headers, false)
+            .await;
+        assert!(matches!(saturated, Err(SelectionFailure::Capacity)));
+        let view = pool.view();
+        let member = &view["members"][0];
+        assert_eq!(member["capacity_rejections_total"], 1);
+        assert_eq!(member["queue_timeouts_total"], 1);
+        assert_eq!(member["active"], 1);
+        drop(first);
+        assert!(pool
+            .select_wait("127.0.0.1".parse().unwrap(), &headers, false)
+            .await
+            .is_ok());
+    }
+
+    #[tokio::test]
+    async fn zero_capacity_queue_timeout_rejects_without_counting_a_queue_timeout() {
+        let pool = capacity_pool(1, 0);
+        let first = select(&pool);
+        let headers = HeaderMap::new();
+        let saturated = pool
+            .select_wait("127.0.0.1".parse().unwrap(), &headers, false)
+            .await;
+        assert!(matches!(saturated, Err(SelectionFailure::Capacity)));
+        let view = pool.view();
+        let member = &view["members"][0];
+        assert_eq!(member["capacity_rejections_total"], 1);
+        assert_eq!(member["queue_timeouts_total"], 0);
+        drop(first);
+    }
+
+    #[test]
+    fn detailed_backend_failures_are_exposed_without_losing_aggregate_counters() {
+        let pool = capacity_pool(0, 0);
+        let selection = select(&pool);
+        selection
+            .lease
+            .error_detail("dns", "lookup backend.example failed", Duration::from_millis(7));
+        let view = pool.view();
+        let member = &view["members"][0];
+        assert_eq!(member["dns_errors_total"], 1);
+        assert_eq!(member["connect_errors_total"], 1);
+        assert_eq!(member["last_failure_class"], "dns");
+        assert_eq!(member["last_failure_detail"], "lookup backend.example failed");
+        assert!(member["last_failure_at_ms"].as_u64().unwrap_or_default() > 0);
+    }
+    #[tokio::test]
+    async fn traffic_and_adaptive_selection_use_observed_cost_and_weight() {
+        for algorithm in [Algorithm::LeastTraffic, Algorithm::Adaptive] {
+            let pool = pool(algorithm, StickyMode::Off);
+            lock(&pool.members[0].state.traffic).rate = 50_000_000.0;
+            lock(&pool.members[0].state.health).ewma_us = 1_000_000;
+            assert_eq!(select(&pool).lease.member.definition.id, "b");
+        }
+    }
+    #[tokio::test]
+    async fn ip_hash_is_stable_and_cannot_be_spoofed_with_forwarded_headers() {
+        let pool = pool(Algorithm::IpHash, StickyMode::Off);
+        let original = select(&pool).lease.member.definition.id.clone();
+        let mut headers = HeaderMap::new();
+        headers.insert("x-forwarded-for", "10.0.0.99".parse().unwrap());
+        for ip in ["127.0.0.1", "::ffff:127.0.0.1"] {
+            for _ in 0..8 {
+                assert_eq!(
+                    pool.select(ip.parse().unwrap(), &headers, false)
+                        .unwrap()
+                        .lease
+                        .member
+                        .definition
+                        .id,
+                    original
+                );
+            }
+        }
+    }

src/balance/tests/part_02.rs

@@ -0,0 +1,170 @@
+    #[tokio::test]
+    async fn opaque_cookie_is_removed_from_origin_requests_and_protected_on_responses() {
+        let pool = pool(Algorithm::RoundRobin, StickyMode::Cookie);
+        let first = select(&pool);
+        let cookie = first.cookie.unwrap().to_str().unwrap().to_owned();
+        assert!(
+            cookie.contains("HttpOnly")
+                && cookie.contains("SameSite=Lax")
+                && !cookie.contains("Secure")
+        );
+        let cookie = cookie.split(';').next().unwrap();
+        let mut headers = HeaderMap::new();
+        headers.insert("cookie", format!("{cookie}; app=ok").parse().unwrap());
+        let next = pool
+            .select("10.0.0.1".parse().unwrap(), &headers, false)
+            .unwrap();
+        assert_eq!(
+            next.lease.member.definition.id,
+            first.lease.member.definition.id
+        );
+        assert!(next.cookie.is_none());
+        pool.strip_affinity(&mut headers);
+        assert_eq!(headers["cookie"], "app=ok");
+        headers.append(
+            "set-cookie",
+            format!("{}=origin-spoof", pool.cookie_name)
+                .parse()
+                .unwrap(),
+        );
+        headers.append("set-cookie", "app=ok".parse().unwrap());
+        pool.strip_affinity_response(&mut headers);
+        assert_eq!(headers.get_all("set-cookie").iter().count(), 1);
+        assert_eq!(headers["set-cookie"], "app=ok");
+        let secure = pool
+            .select("127.0.0.1".parse().unwrap(), &HeaderMap::new(), true)
+            .unwrap();
+        assert!(secure
+            .cookie
+            .unwrap()
+            .to_str()
+            .unwrap()
+            .contains("; Secure"));
+    }
+    #[tokio::test]
+    async fn failed_sticky_member_is_replaced_and_duplicate_cookies_are_ignored() {
+        let pool = pool(Algorithm::RoundRobin, StickyMode::Cookie);
+        let first = select(&pool);
+        let cookie = first
+            .cookie
+            .unwrap()
+            .to_str()
+            .unwrap()
+            .split(';')
+            .next()
+            .unwrap()
+            .to_owned();
+        let mut headers = HeaderMap::new();
+        headers.insert("cookie", format!("{cookie}; {cookie}").parse().unwrap());
+        assert!(pool.affinity(&headers).is_none());
+        headers.insert("cookie", cookie.parse().unwrap());
+        lock(&first.lease.member.state.health).healthy = false;
+        let next = pool
+            .select("127.0.0.1".parse().unwrap(), &headers, false)
+            .unwrap();
+        assert_ne!(
+            next.lease.member.definition.id,
+            first.lease.member.definition.id
+        );
+        assert!(next.cookie.is_some());
+        lock(&next.lease.member.state.health).healthy = false;
+        assert!(pool
+            .select("127.0.0.1".parse().unwrap(), &headers, false)
+            .is_none());
+    }
+    #[tokio::test]
+    async fn slow_start_ramps_weight_and_error_is_counted_once_per_exchange() {
+        let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
+        let selected = select(&pool);
+        let state = &selected.lease.member.state;
+        assert!(state.weight(100, 100) < 100.0);
+        assert_eq!(state.weight(100, 0), 100.0);
+        selected.lease.started();
+        selected.lease.bytes(50, true);
+        selected.lease.bytes(100, false);
+        selected.lease.error("timeout", Duration::from_secs(1));
+        selected.lease.error("body", Duration::from_secs(2));
+        selected.lease.retry_attempted();
+        selected.lease.retry_succeeded();
+        assert_eq!(state.requests.load(Ordering::Relaxed), 1);
+        assert_eq!(state.errors.load(Ordering::Relaxed), 1);
+        assert_eq!(state.timeouts_total.load(Ordering::Relaxed), 1);
+        assert_eq!(state.io_errors_total.load(Ordering::Relaxed), 1);
+        assert_eq!(state.bytes_in.load(Ordering::Relaxed), 50);
+        assert_eq!(state.bytes_out.load(Ordering::Relaxed), 100);
+        assert_eq!(state.retry_attempts_total.load(Ordering::Relaxed), 1);
+        assert_eq!(state.retry_successes_total.load(Ordering::Relaxed), 1);
+        assert!(pool
+            .prometheus()
+            .contains("rust_proxy_backend_requests_total{vhost=\"app\",backend=\"a\"} 1"));
+        assert!(pool
+            .prometheus()
+            .contains("rust_proxy_backend_retry_attempts_total{vhost=\"app\",backend=\"a\"} 1"));
+    }
+    #[tokio::test]
+    async fn slow_5xx_counts_both_observability_classes_when_passive_is_off() {
+        let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
+        let selected = select(&pool);
+        selected.lease.response(500, Duration::from_secs(20), true);
+        let state = &selected.lease.member.state;
+        assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 1);
+        assert_eq!(state.slow_responses_total.load(Ordering::Relaxed), 1);
+        assert_eq!(state.passive_health_failures_total.load(Ordering::Relaxed), 0);
+        assert!(state.healthy());
+    }
+
+    #[tokio::test]
+    async fn client_upload_duration_does_not_trigger_passive_slow_failure() {
+        let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
+        let upload = select(&pool);
+        upload.lease.response(200, Duration::from_secs(20), false);
+        assert_eq!(upload.lease.member.state.errors.load(Ordering::Relaxed), 0);
+        let read = select(&pool);
+        read.lease.response(200, Duration::from_secs(20), true);
+        assert_eq!(read.lease.member.state.errors.load(Ordering::Relaxed), 1);
+    }
+    #[tokio::test]
+    async fn backup_is_only_new_assignment_when_no_primary_is_eligible() {
+        let mut pool = pool(Algorithm::RoundRobin, StickyMode::Off);
+        Arc::get_mut(&mut Arc::get_mut(&mut pool).unwrap().members[1])
+            .unwrap()
+            .definition
+            .backup = true;
+        for _ in 0..8 {
+            assert_eq!(select(&pool).lease.member.definition.id, "a");
+        }
+        lock(&pool.members[0].state.health).healthy = false;
+        assert_eq!(select(&pool).lease.member.definition.id, "b");
+        lock(&pool.members[0].state.health).healthy = true;
+        assert_eq!(select(&pool).lease.member.definition.id, "a");
+    }
+    #[tokio::test]
+    async fn drain_rejects_new_sessions_but_preserves_valid_existing_affinity() {
+        let mut pool = pool(Algorithm::RoundRobin, StickyMode::Cookie);
+        Arc::get_mut(&mut Arc::get_mut(&mut pool).unwrap().members[0])
+            .unwrap()
+            .definition
+            .drain = true;
+        assert_eq!(select(&pool).lease.member.definition.id, "b");
+        let mut headers = HeaderMap::new();
+        headers.insert(
+            "cookie",
+            format!("{}={}", pool.cookie_name, pool.members[0].token)
+                .parse()
+                .unwrap(),
+        );
+        let existing = pool
+            .select("127.0.0.1".parse().unwrap(), &headers, false)
+            .unwrap();
+        assert_eq!(existing.lease.member.definition.id, "a");
+        lock(&pool.members[0].state.health).healthy = false;
+        assert_eq!(
+            pool.select("127.0.0.1".parse().unwrap(), &headers, false)
+                .unwrap()
+                .lease
+                .member
+                .definition
+                .id,
+            "b"
+        );
+    }