mod tests { use super::*; fn settings(mode: HealthMode) -> HealthCheck { HealthCheck { mode, fall: 2, rise: 2, ..Default::default() } } fn pool(algorithm: Algorithm, sticky: StickyMode) -> Arc { let cfg = serde_json::from_value(serde_json::json!({ "listeners":{"endpoints":[{"id":"http","binding":{"address":"127.0.0.1:8080"}}]}, "vhosts":[{ "id":"app","frontend":{"hosts":["app.localhost"],"listener_ids":["http"]}, "backends":[{"id":"a","url":"http://127.0.0.1:9000","allow_private":true,"weight":3}, {"id":"b","url":"http://127.0.0.1:9001","allow_private":true,"weight":1}], "load_balancing":{"algorithm":algorithm,"sticky":{"mode":sticky},"slow_start_seconds":0,"health_check":{"mode":"off"}} }]})).unwrap(); let policy = crate::config::ValidationPolicy { proxy_addr: "127.0.0.1:8080".parse().unwrap(), admin_addr: "127.0.0.1:9090".parse().unwrap(), tls_addr: None, cert_dir: None, data_dir: None, }; crate::config::compile(cfg, 0, &policy).unwrap().pools[0].clone() } fn capacity_pool(limit: u32, queue_timeout_ms: u64) -> Arc { let cfg = serde_json::from_value(serde_json::json!({ "listeners":{"endpoints":[{"id":"http","binding":{"address":"127.0.0.1:8080"}}]}, "vhosts":[{ "id":"limited","frontend":{"hosts":["limited.localhost"],"listener_ids":["http"]}, "backends":[{ "id":"only","url":"http://127.0.0.1:9000","allow_private":true, "max_concurrent_requests":limit,"queue_timeout_ms":queue_timeout_ms }], "load_balancing":{"health_check":{"mode":"off"}} }] })) .unwrap(); let policy = crate::config::ValidationPolicy { proxy_addr: "127.0.0.1:8080".parse().unwrap(), admin_addr: "127.0.0.1:9090".parse().unwrap(), tls_addr: None, cert_dir: None, data_dir: None, }; crate::config::compile(cfg, 0, &policy).unwrap().pools[0].clone() } fn select(pool: &Pool) -> Selection { pool.select("127.0.0.1".parse().unwrap(), &HeaderMap::new(), false) .unwrap() } #[test] fn per_backend_monitoring_can_be_disabled_or_overridden() { let mut defaults: crate::vhost::PoolMember = serde_json::from_value(serde_json::json!({ "id": "error_404", "url": "http://127.0.0.1:8282", "pool_eligible": false })) .unwrap(); let mut settings = Settings::default(); settings.passive_health.enabled = true; assert!(defaults.health_check_enabled); assert_eq!(settings.health_for(&defaults).mode, HealthMode::Tcp); assert!(settings.passive_for(&defaults).enabled); defaults.health_check_enabled = false; assert_eq!(settings.health_for(&defaults).mode, HealthMode::Off); assert!(!settings.passive_for(&defaults).enabled); assert!(!defaults.pool_eligible); defaults.health_check_enabled = true; defaults.health_check = Some(HealthCheck { mode: HealthMode::Http, path: "/healthz".into(), ..Default::default() }); let effective = settings.health_for(&defaults); assert_eq!(effective.mode, HealthMode::Http); assert_eq!(effective.path, "/healthz"); assert_eq!(effective.host, settings.health_check.host); let restored: crate::vhost::PoolMember = serde_json::from_str(&serde_json::to_string(&defaults).unwrap()).unwrap(); assert_eq!(settings.health_for(&restored).path, "/healthz"); } #[test] fn default_health_check_uses_tcp() { assert_eq!(HealthMode::default(), HealthMode::Tcp); assert_eq!(HealthCheck::default().mode, HealthMode::Tcp); } #[test] fn legacy_settings_without_passive_health_default_to_off() { let parsed: Settings = serde_json::from_value(serde_json::json!({ "health_check": { "mode": "off" } })) .unwrap(); assert!(!parsed.passive_health.enabled); assert!(parsed.passive_health.count_connect_errors); assert!(parsed.passive_health.count_timeouts); assert!(parsed.passive_health.count_io_errors); assert!(parsed.passive_health.count_http_5xx); assert!(!parsed.passive_health.count_slow_responses); assert_eq!(parsed.retry.max_retries, 1); assert!(parsed.retry.retry_connect_errors); assert!(parsed.retry.retry_timeouts); assert_eq!(parsed.retry.retry_statuses, vec![502, 503, 504]); assert_eq!(parsed.keepalive.max_idle_per_host, 64); let explicit: Settings = serde_json::from_value(serde_json::json!({ "keepalive": { "max_idle_per_host": 4 } })) .unwrap(); assert_eq!(explicit.keepalive.max_idle_per_host, 4); assert_eq!(parsed.keepalive.pool_recycle_requests, 25_000); assert_eq!(parsed.keepalive.pool_recycle_seconds, 3_600); assert_eq!(parsed.keepalive.http1_max_buffer_bytes, 131_072); } #[test] fn upstream_pool_recycle_limits_are_bounded_and_can_be_disabled() { let mut settings = Settings::default(); assert!(settings.validate().is_ok()); settings.keepalive.pool_recycle_requests = 0; settings.keepalive.pool_recycle_seconds = 0; assert!(settings.validate().is_ok()); settings.keepalive.pool_recycle_requests = 999; assert!(settings.validate().is_err()); settings.keepalive.pool_recycle_requests = 1_000; settings.keepalive.pool_recycle_seconds = 59; assert!(settings.validate().is_err()); settings.keepalive.pool_recycle_seconds = 60; settings.keepalive.http1_max_buffer_bytes = 8_191; assert!(settings.validate().is_err()); settings.keepalive.http1_max_buffer_bytes = 8_192; assert!(settings.validate().is_ok()); } #[test] fn compatible_pool_rebuild_reuses_upstream_clients() { let previous = pool(Algorithm::LeastConnections, StickyMode::Off); let mut rebuilt = pool(Algorithm::LeastConnections, StickyMode::Off); let rebuilt_mut = Arc::get_mut(&mut rebuilt).expect("fresh pool must be uniquely owned"); assert!(!Arc::ptr_eq( &rebuilt_mut.members[0].backend, &previous.members[0].backend )); rebuilt_mut.reuse_observations(&previous); assert!(Arc::ptr_eq( &rebuilt_mut.members[0].backend, &previous.members[0].backend )); } #[test] fn retry_policy_rejects_unbounded_or_non_5xx_configuration() { let mut settings = Settings::default(); settings.retry.max_retries = 4; assert!(settings.validate().is_err()); settings.retry.max_retries = 1; settings.retry.retry_statuses = vec![429]; assert!(settings.validate().is_err()); } #[test] fn traffic_rate_drops_to_zero_after_an_idle_sample_window() { let state = BackendState::new(HealthMode::Off); state .bytes_out .store(2048, std::sync::atomic::Ordering::Relaxed); lock(&state.traffic).at = std::time::Instant::now() - std::time::Duration::from_secs(2); state.sample_traffic(); assert!(state.traffic_rate() > 0.0); lock(&state.traffic).at = std::time::Instant::now() - std::time::Duration::from_secs(2); state.sample_traffic(); assert_eq!(state.traffic_rate(), 0.0); } #[test] fn active_failures_are_independent_from_passive_user_traffic() { let state = BackendState::new(HealthMode::Http); let settings = settings(HealthMode::Http); let passive = PassiveHealth::default(); let fast = Duration::from_millis(5); assert!(!state.healthy()); state.observe_active(&settings, "ok", fast); assert!(!state.healthy()); state.observe_active(&settings, "ok", fast); assert!(state.healthy()); state.observe_active(&settings, "status", fast); state.observe_passive(&settings, &passive, "ok", fast); state.observe_active(&settings, "status", fast); assert!(!state.healthy()); state.observe_passive(&settings, &passive, "http_5xx", fast); assert!(!state.healthy()); state.observe_active(&settings, "ok", fast); assert!(!state.healthy()); state.observe_active(&settings, "ok", fast); assert!(state.healthy()); } #[test] fn passive_health_is_opt_in_and_uses_separate_thresholds() { let state = BackendState::new(HealthMode::Off); let active = settings(HealthMode::Off); let fast = Duration::from_millis(5); for _ in 0..10 { state.observe_passive(&active, &PassiveHealth::default(), "http_5xx", fast); } assert!(state.healthy()); assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 10); assert_eq!( state.passive_health_failures_total.load(Ordering::Relaxed), 0 ); let passive = PassiveHealth { enabled: true, fall: Some(2), rise: Some(2), ..Default::default() }; state.observe_passive(&active, &passive, "http_5xx", fast); assert!(state.healthy()); state.observe_passive(&active, &passive, "http_5xx", fast); assert!(!state.healthy()); state.observe_passive(&active, &passive, "ok", fast); assert!(!state.healthy()); state.observe_passive(&active, &passive, "ok", fast); assert!(state.healthy()); } #[test] fn passive_http_5xx_can_be_explicitly_ignored_when_passive_health_is_on() { let state = BackendState::new(HealthMode::Off); let active = settings(HealthMode::Off); let passive = PassiveHealth { enabled: true, count_http_5xx: false, fall: Some(1), ..Default::default() }; for _ in 0..100 { state.observe_passive(&active, &passive, "http_5xx", Duration::from_millis(5)); } assert!(state.healthy()); assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 100); assert_eq!( state.passive_health_failures_total.load(Ordering::Relaxed), 0 ); } #[test] fn passive_connect_errors_use_the_explicit_fall_threshold() { let state = BackendState::new(HealthMode::Off); let active = settings(HealthMode::Off); let passive = PassiveHealth { enabled: true, count_connect_errors: true, fall: Some(3), ..Default::default() }; for _ in 0..2 { state.observe_passive(&active, &passive, "connect_error", Duration::from_millis(5)); assert!(state.healthy()); } state.observe_passive(&active, &passive, "connect_error", Duration::from_millis(5)); assert!(!state.healthy()); assert_eq!(state.connect_errors_total.load(Ordering::Relaxed), 3); assert_eq!( state.passive_health_failures_total.load(Ordering::Relaxed), 3 ); } #[test] fn active_health_can_stay_healthy_while_passive_health_is_off() { let state = BackendState::new(HealthMode::Tcp); let active = settings(HealthMode::Tcp); let passive = PassiveHealth::default(); let fast = Duration::from_millis(5); state.observe_active(&active, "ok", fast); state.observe_active(&active, "ok", fast); assert!(state.healthy()); for _ in 0..20 { state.observe_passive(&active, &passive, "http_5xx", fast); } assert!(state.healthy()); assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 20); assert_eq!( state.passive_health_failures_total.load(Ordering::Relaxed), 0 ); } #[test] fn unhealthy_source_hands_off_when_the_original_health_domain_recovers() { let state = BackendState::new(HealthMode::Tcp); let active = HealthCheck { fall: 1, rise: 1, ..settings(HealthMode::Tcp) }; let passive = PassiveHealth { enabled: true, count_http_5xx: true, fall: Some(1), rise: Some(1), ..Default::default() }; let fast = Duration::from_millis(5); state.observe_active(&active, "ok", fast); state.observe_active(&active, "connect_error", fast); { let health = lock(&state.health); assert_eq!(health.unhealthy_source, "active_tcp"); } state.observe_passive(&active, &passive, "http_5xx", fast); state.observe_active(&active, "ok", fast); { let health = lock(&state.health); assert!(!health.healthy); assert_eq!(health.unhealthy_source, "passive_http_5xx"); assert_eq!(health.unhealthy_reason, "http_5xx"); } state.observe_passive(&active, &passive, "ok", fast); assert!(state.healthy()); } #[test] fn disabled_checks_never_quarantine_and_probes_are_single_flight() { let off = BackendState::new(HealthMode::Off); let settings = settings(HealthMode::Off); for _ in 0..5 { off.observe_passive( &settings, &PassiveHealth::default(), "connect_error", Duration::ZERO, ); } assert!(off.healthy()); assert!(!off.claim_probe(&settings)); let on = BackendState::new(HealthMode::Http); let settings = super::tests::settings(HealthMode::Http); assert!(on.claim_probe(&settings)); assert!(!on.claim_probe(&settings)); on.observe_active(&settings, "ok", Duration::ZERO); assert!(!on.claim_probe(&settings)); } #[tokio::test] async fn weighted_round_robin_distributes_three_to_one_without_body_collection() { let pool = pool(Algorithm::RoundRobin, StickyMode::Off); let mut counts = [0, 0]; for _ in 0..40 { let selection = select(&pool); counts[usize::from(selection.lease.member.definition.id == "b")] += 1; } assert_eq!(counts, [30, 10]); assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 0); assert_eq!(pool.members[0].state.requests.load(Ordering::Relaxed), 0); } #[tokio::test] async fn least_connections_counts_a_lease_until_its_last_owner_drops() { let pool = pool(Algorithm::LeastConnections, StickyMode::Off); let mut held = Vec::new(); for _ in 0..3 { let selection = select(&pool); assert_eq!(selection.lease.member.definition.id, "a"); held.push(selection.lease); } let other = select(&pool); assert_eq!(other.lease.member.definition.id, "b"); let clone = held[0].clone(); held.clear(); assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 1); drop(clone); assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 0); } #[tokio::test] async fn failover_selection_never_reuses_an_attempted_backend() { let pool = pool(Algorithm::LeastConnections, StickyMode::Off); let first = select(&pool); let first_id = first.lease.member.definition.id.clone(); drop(first); let retry = pool .select_retry( "127.0.0.1".parse().unwrap(), &HeaderMap::new(), false, std::slice::from_ref(&first_id), ) .unwrap(); assert_ne!(retry.lease.member.definition.id, first_id); } #[tokio::test] async fn backend_concurrency_limit_queues_then_rejects_and_recovers() { let pool = capacity_pool(1, 10); let first = select(&pool); let headers = HeaderMap::new(); let saturated = pool .select_wait("127.0.0.1".parse().unwrap(), &headers, false) .await; assert!(matches!(saturated, Err(SelectionFailure::Capacity))); let view = pool.view(); let member = &view["members"][0]; assert_eq!(member["capacity_rejections_total"], 1); assert_eq!(member["queue_timeouts_total"], 1); assert_eq!(member["active"], 1); drop(first); assert!(pool .select_wait("127.0.0.1".parse().unwrap(), &headers, false) .await .is_ok()); } #[tokio::test] async fn zero_capacity_queue_timeout_rejects_without_counting_a_queue_timeout() { let pool = capacity_pool(1, 0); let first = select(&pool); let headers = HeaderMap::new(); let saturated = pool .select_wait("127.0.0.1".parse().unwrap(), &headers, false) .await; assert!(matches!(saturated, Err(SelectionFailure::Capacity))); let view = pool.view(); let member = &view["members"][0]; assert_eq!(member["capacity_rejections_total"], 1); assert_eq!(member["queue_timeouts_total"], 0); drop(first); } #[test] fn detailed_backend_failures_are_exposed_without_losing_aggregate_counters() { let pool = capacity_pool(0, 0); let selection = select(&pool); selection.lease.error_detail( "dns", "lookup backend.example failed", Duration::from_millis(7), ); let view = pool.view(); let member = &view["members"][0]; assert_eq!(member["dns_errors_total"], 1); assert_eq!(member["connect_errors_total"], 1); assert_eq!(member["last_failure_class"], "dns"); assert_eq!( member["last_failure_detail"], "lookup backend.example failed" ); assert!(member["last_failure_at_ms"].as_u64().unwrap_or_default() > 0); } #[tokio::test] async fn traffic_and_adaptive_selection_use_observed_cost_and_weight() { for algorithm in [Algorithm::LeastTraffic, Algorithm::Adaptive] { let pool = pool(algorithm, StickyMode::Off); lock(&pool.members[0].state.traffic).rate = 50_000_000.0; lock(&pool.members[0].state.health).ewma_us = 1_000_000; assert_eq!(select(&pool).lease.member.definition.id, "b"); } } #[tokio::test] async fn ip_hash_is_stable_and_cannot_be_spoofed_with_forwarded_headers() { let pool = pool(Algorithm::IpHash, StickyMode::Off); let original = select(&pool).lease.member.definition.id.clone(); let mut headers = HeaderMap::new(); headers.insert("x-forwarded-for", "10.0.0.99".parse().unwrap()); for ip in ["127.0.0.1", "::ffff:127.0.0.1"] { for _ in 0..8 { assert_eq!( pool.select(ip.parse().unwrap(), &headers, false) .unwrap() .lease .member .definition .id, original ); } } } #[tokio::test] async fn opaque_cookie_is_removed_from_origin_requests_and_protected_on_responses() { let pool = pool(Algorithm::RoundRobin, StickyMode::Cookie); let first = select(&pool); let cookie = first.cookie.unwrap().to_str().unwrap().to_owned(); assert!( cookie.contains("HttpOnly") && cookie.contains("SameSite=Lax") && !cookie.contains("Secure") ); let cookie = cookie.split(';').next().unwrap(); let mut headers = HeaderMap::new(); headers.insert("cookie", format!("{cookie}; app=ok").parse().unwrap()); let next = pool .select("10.0.0.1".parse().unwrap(), &headers, false) .unwrap(); assert_eq!( next.lease.member.definition.id, first.lease.member.definition.id ); assert!(next.cookie.is_none()); pool.strip_affinity(&mut headers); assert_eq!(headers["cookie"], "app=ok"); headers.append( "set-cookie", format!("{}=origin-spoof", pool.cookie_name) .parse() .unwrap(), ); headers.append("set-cookie", "app=ok".parse().unwrap()); pool.strip_affinity_response(&mut headers); assert_eq!(headers.get_all("set-cookie").iter().count(), 1); assert_eq!(headers["set-cookie"], "app=ok"); let secure = pool .select("127.0.0.1".parse().unwrap(), &HeaderMap::new(), true) .unwrap(); assert!(secure .cookie .unwrap() .to_str() .unwrap() .contains("; Secure")); } #[tokio::test] async fn failed_sticky_member_is_replaced_and_duplicate_cookies_are_ignored() { let pool = pool(Algorithm::RoundRobin, StickyMode::Cookie); let first = select(&pool); let cookie = first .cookie .unwrap() .to_str() .unwrap() .split(';') .next() .unwrap() .to_owned(); let mut headers = HeaderMap::new(); headers.insert("cookie", format!("{cookie}; {cookie}").parse().unwrap()); assert!(pool.affinity(&headers).is_none()); headers.insert("cookie", cookie.parse().unwrap()); first.lease.member.state.set_healthy_for_test(false); let next = pool .select("127.0.0.1".parse().unwrap(), &headers, false) .unwrap(); assert_ne!( next.lease.member.definition.id, first.lease.member.definition.id ); assert!(next.cookie.is_some()); next.lease.member.state.set_healthy_for_test(false); assert!(pool .select("127.0.0.1".parse().unwrap(), &headers, false) .is_none()); } #[tokio::test] async fn slow_start_ramps_weight_and_error_is_counted_once_per_exchange() { let pool = pool(Algorithm::LeastConnections, StickyMode::Off); let selected = select(&pool); let state = &selected.lease.member.state; assert!(state.weight(100, 100) < 100.0); assert_eq!(state.weight(100, 0), 100.0); selected.lease.started(); selected.lease.bytes(50, true); selected.lease.bytes(100, false); selected.lease.error("timeout", Duration::from_secs(1)); selected.lease.error("body", Duration::from_secs(2)); selected.lease.retry_attempted(); selected.lease.retry_succeeded(); assert_eq!(state.requests.load(Ordering::Relaxed), 1); assert_eq!(state.errors.load(Ordering::Relaxed), 1); assert_eq!(state.timeouts_total.load(Ordering::Relaxed), 1); assert_eq!(state.io_errors_total.load(Ordering::Relaxed), 1); assert_eq!(state.bytes_in.load(Ordering::Relaxed), 50); assert_eq!(state.bytes_out.load(Ordering::Relaxed), 100); assert_eq!(state.retry_attempts_total.load(Ordering::Relaxed), 1); assert_eq!(state.retry_successes_total.load(Ordering::Relaxed), 1); assert!(pool .prometheus() .contains("rust_proxy_backend_requests_total{vhost=\"app\",backend=\"a\"} 1")); assert!(pool .prometheus() .contains("rust_proxy_backend_retry_attempts_total{vhost=\"app\",backend=\"a\"} 1")); } #[tokio::test] async fn slow_5xx_counts_both_observability_classes_when_passive_is_off() { let pool = pool(Algorithm::LeastConnections, StickyMode::Off); let selected = select(&pool); selected.lease.response(500, Duration::from_secs(20), true); let state = &selected.lease.member.state; assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 1); assert_eq!(state.slow_responses_total.load(Ordering::Relaxed), 1); assert_eq!( state.passive_health_failures_total.load(Ordering::Relaxed), 0 ); assert!(state.healthy()); } #[tokio::test] async fn client_upload_duration_does_not_trigger_passive_slow_failure() { let pool = pool(Algorithm::LeastConnections, StickyMode::Off); let upload = select(&pool); upload.lease.response(200, Duration::from_secs(20), false); assert_eq!(upload.lease.member.state.errors.load(Ordering::Relaxed), 0); let read = select(&pool); read.lease.response(200, Duration::from_secs(20), true); assert_eq!(read.lease.member.state.errors.load(Ordering::Relaxed), 1); } #[tokio::test] async fn backup_is_only_new_assignment_when_no_primary_is_eligible() { let mut pool = pool(Algorithm::RoundRobin, StickyMode::Off); Arc::get_mut(&mut Arc::get_mut(&mut pool).unwrap().members[1]) .unwrap() .definition .backup = true; for _ in 0..8 { assert_eq!(select(&pool).lease.member.definition.id, "a"); } pool.members[0].state.set_healthy_for_test(false); assert_eq!(select(&pool).lease.member.definition.id, "b"); pool.members[0].state.set_healthy_for_test(true); assert_eq!(select(&pool).lease.member.definition.id, "a"); } #[tokio::test] async fn drain_rejects_new_sessions_but_preserves_valid_existing_affinity() { let mut pool = pool(Algorithm::RoundRobin, StickyMode::Cookie); Arc::get_mut(&mut Arc::get_mut(&mut pool).unwrap().members[0]) .unwrap() .definition .drain = true; assert_eq!(select(&pool).lease.member.definition.id, "b"); let mut headers = HeaderMap::new(); headers.insert( "cookie", format!("{}={}", pool.cookie_name, pool.members[0].token) .parse() .unwrap(), ); let existing = pool .select("127.0.0.1".parse().unwrap(), &headers, false) .unwrap(); assert_eq!(existing.lease.member.definition.id, "a"); pool.members[0].state.set_healthy_for_test(false); assert_eq!( pool.select("127.0.0.1".parse().unwrap(), &headers, false) .unwrap() .lease .member .definition .id, "b" ); } #[test] fn passive_ejection_remembers_first_real_error_and_trigger() { let state = BackendState::new(HealthMode::Off); let active = settings(HealthMode::Off); let passive = PassiveHealth { enabled: true, fall: Some(2), ..Default::default() }; let elapsed = Duration::from_millis(3); assert!(state .observe_passive_with_detail( &active, &passive, "connect_error", "dns", "first lookup failed", elapsed ) .is_none()); let (first, failures) = state .observe_passive_with_detail( &active, &passive, "connect_error", "connect", "second TCP failed", elapsed, ) .expect("the threshold crossing must return the first failure"); assert_eq!(failures, 2); assert_eq!(first.class, "dns"); assert_eq!(first.detail, "first lookup failed"); assert!(!state.healthy()); assert!( state .observe_passive_with_detail( &active, &passive, "connect_error", "connect", "third TCP failed", elapsed ) .is_none(), "the ejection must be logged only once" ); } #[test] fn passive_io_burst_is_one_observation_not_many_failed_connections() { let state = BackendState::new(HealthMode::Off); let active = settings(HealthMode::Off); let passive = PassiveHealth { enabled: true, fall: Some(3), ..Default::default() }; for _ in 0..20 { state.observe_passive_with_detail( &active, &passive, "io_error", "io_error", "connection closed", Duration::from_millis(1), ); } assert!(state.healthy()); assert_eq!(state.io_errors_total.load(Ordering::Relaxed), 20); assert_eq!( state.passive_health_failures_total.load(Ordering::Relaxed), 1 ); } #[test] fn passive_only_quarantine_expires_without_vhost_reload() { let state = BackendState::new(HealthMode::Off); let active = settings(HealthMode::Off); let passive = PassiveHealth { enabled: true, fall: Some(1), ..Default::default() }; state.observe_passive(&active, &passive, "connect_error", Duration::ZERO); assert!(!state.healthy()); lock(&state.health).passive_ejected_at = Some(Instant::now() - Duration::from_secs(10)); assert!( state.healthy(), "a half-open traffic attempt must become possible" ); } #[test] fn active_probe_successes_recover_a_passively_ejected_backend() { let state = BackendState::new(HealthMode::Http); let active = HealthCheck { fall: 1, rise: 1, ..settings(HealthMode::Http) }; let passive = PassiveHealth { enabled: true, fall: Some(1), ..Default::default() }; state.observe_active(&active, "ok", Duration::ZERO); assert!(state.healthy()); state.observe_passive(&active, &passive, "connect_error", Duration::ZERO); assert!(!state.healthy()); state.observe_active(&active, "ok", Duration::ZERO); assert!(!state.healthy()); state.observe_active(&active, "ok", Duration::ZERO); assert!(state.healthy()); } #[test] fn single_backend_fast_path_observes_capacity_and_health_without_affinity_cookie() { let pool = capacity_pool(1, 0); assert_eq!(pool.members.len(), 1); let peer = "127.0.0.1".parse().unwrap(); let headers = HeaderMap::new(); let first = pool.select(peer, &headers, false).expect("healthy backend"); assert!(first.cookie.is_none()); assert!(pool.select(peer, &headers, false).is_none()); drop(first); pool.members[0].state.set_healthy_for_test(false); assert!(pool.select(peer, &headers, false).is_none()); pool.members[0].state.set_healthy_for_test(true); assert!(pool.select(peer, &headers, false).is_some()); } #[test] fn leases_reuse_immutable_backend_health_policies() { let pool = capacity_pool(0, 0); let first = select(&pool); let second = select(&pool); assert!(Arc::ptr_eq(&first.lease.health, &second.lease.health)); assert!(Arc::ptr_eq( &first.lease.passive_health, &second.lease.passive_health )); } #[test] fn healthy_fast_path_tracks_active_and_passive_ejections() { let state = BackendState::new(HealthMode::Off); assert!(state.healthy()); let active = settings(HealthMode::Off); let passive = PassiveHealth { enabled: true, fall: Some(1), rise: Some(1), ..Default::default() }; state.observe_passive(&active, &passive, "connect_error", Duration::from_millis(2)); assert!(!state.healthy()); state.observe_passive(&active, &passive, "ok", Duration::from_millis(2)); assert!(state.healthy()); } #[test] fn successful_lease_response_keeps_passive_observability_and_failure_detail() { let pool = capacity_pool(0, 0); let selection = select(&pool); selection .lease .response(200, Duration::from_millis(5), true); let state = &selection.lease.member.state; assert_eq!(state.last_status.load(Ordering::Relaxed), 200); assert_eq!(lock(&state.health).passive_result, "ok"); assert!(lock(&state.health).latency_us > 0); selection .lease .response(503, Duration::from_millis(5), true); assert_eq!(state.last_status.load(Ordering::Relaxed), 503); assert_eq!(lock(&state.last_failure).detail, "status=503"); } }