mod tests { use super::*; fn settings(mode: HealthMode) -> HealthCheck { HealthCheck { mode, fall: 2, rise: 2, ..Default::default() } } fn pool(algorithm: Algorithm, sticky: StickyMode) -> Arc { let cfg = serde_json::from_value(serde_json::json!({ "listeners":{"endpoints":[{"id":"http","binding":{"address":"127.0.0.1:8080"}}]}, "vhosts":[{ "id":"app","frontend":{"hosts":["app.localhost"],"listener_ids":["http"]}, "backends":[{"id":"a","url":"http://127.0.0.1:9000","allow_private":true,"weight":3}, {"id":"b","url":"http://127.0.0.1:9001","allow_private":true,"weight":1}], "load_balancing":{"algorithm":algorithm,"sticky":{"mode":sticky},"slow_start_seconds":0,"health_check":{"mode":"off"}} }]})).unwrap(); let policy = crate::config::ValidationPolicy { proxy_addr: "127.0.0.1:8080".parse().unwrap(), admin_addr: "127.0.0.1:9090".parse().unwrap(), tls_addr: None, cert_dir: None, data_dir: None, }; crate::config::compile(cfg, 0, &policy).unwrap().pools[0].clone() } fn capacity_pool(limit: u32, queue_timeout_ms: u64) -> Arc { let cfg = serde_json::from_value(serde_json::json!({ "listeners":{"endpoints":[{"id":"http","binding":{"address":"127.0.0.1:8080"}}]}, "vhosts":[{ "id":"limited","frontend":{"hosts":["limited.localhost"],"listener_ids":["http"]}, "backends":[{ "id":"only","url":"http://127.0.0.1:9000","allow_private":true, "max_concurrent_requests":limit,"queue_timeout_ms":queue_timeout_ms }], "load_balancing":{"health_check":{"mode":"off"}} }] })) .unwrap(); let policy = crate::config::ValidationPolicy { proxy_addr: "127.0.0.1:8080".parse().unwrap(), admin_addr: "127.0.0.1:9090".parse().unwrap(), tls_addr: None, cert_dir: None, data_dir: None, }; crate::config::compile(cfg, 0, &policy).unwrap().pools[0].clone() } fn select(pool: &Pool) -> Selection { pool.select("127.0.0.1".parse().unwrap(), &HeaderMap::new(), false) .unwrap() } #[test] fn per_backend_monitoring_can_be_disabled_or_overridden() { let mut defaults: crate::vhost::PoolMember = serde_json::from_value(serde_json::json!({ "id": "error_404", "url": "http://127.0.0.1:8282", "pool_eligible": false })) .unwrap(); let mut settings = Settings::default(); settings.passive_health.enabled = true; assert!(defaults.health_check_enabled); assert_eq!(settings.health_for(&defaults).mode, HealthMode::Tcp); assert!(settings.passive_for(&defaults).enabled); defaults.health_check_enabled = false; assert_eq!(settings.health_for(&defaults).mode, HealthMode::Off); assert!(!settings.passive_for(&defaults).enabled); assert!(!defaults.pool_eligible); defaults.health_check_enabled = true; defaults.health_check = Some(HealthCheck { mode: HealthMode::Http, path: "/healthz".into(), ..Default::default() }); let effective = settings.health_for(&defaults); assert_eq!(effective.mode, HealthMode::Http); assert_eq!(effective.path, "/healthz"); assert_eq!(effective.host, settings.health_check.host); let restored: crate::vhost::PoolMember = serde_json::from_str(&serde_json::to_string(&defaults).unwrap()).unwrap(); assert_eq!(settings.health_for(&restored).path, "/healthz"); } #[test] fn default_health_check_uses_tcp() { assert_eq!(HealthMode::default(), HealthMode::Tcp); assert_eq!(HealthCheck::default().mode, HealthMode::Tcp); } #[test] fn legacy_settings_without_passive_health_default_to_off() { let parsed: Settings = serde_json::from_value(serde_json::json!({ "health_check": { "mode": "off" } })) .unwrap(); assert!(!parsed.passive_health.enabled); assert!(parsed.passive_health.count_connect_errors); assert!(parsed.passive_health.count_timeouts); assert!(parsed.passive_health.count_io_errors); assert!(parsed.passive_health.count_http_5xx); assert!(!parsed.passive_health.count_slow_responses); assert_eq!(parsed.retry.max_retries, 1); assert!(parsed.retry.retry_connect_errors); assert!(parsed.retry.retry_timeouts); assert_eq!(parsed.retry.retry_statuses, vec![502, 503, 504]); assert_eq!(parsed.keepalive.max_idle_per_host, 64); let explicit: Settings = serde_json::from_value(serde_json::json!({ "keepalive": { "max_idle_per_host": 4 } })) .unwrap(); assert_eq!(explicit.keepalive.max_idle_per_host, 4); assert_eq!(parsed.keepalive.pool_recycle_requests, 25_000); assert_eq!(parsed.keepalive.pool_recycle_seconds, 3_600); assert_eq!(parsed.keepalive.http1_max_buffer_bytes, 131_072); } #[test] fn upstream_pool_recycle_limits_are_bounded_and_can_be_disabled() { let mut settings = Settings::default(); assert!(settings.validate().is_ok()); settings.keepalive.pool_recycle_requests = 0; settings.keepalive.pool_recycle_seconds = 0; assert!(settings.validate().is_ok()); settings.keepalive.pool_recycle_requests = 999; assert!(settings.validate().is_err()); settings.keepalive.pool_recycle_requests = 1_000; settings.keepalive.pool_recycle_seconds = 59; assert!(settings.validate().is_err()); settings.keepalive.pool_recycle_seconds = 60; settings.keepalive.http1_max_buffer_bytes = 8_191; assert!(settings.validate().is_err()); settings.keepalive.http1_max_buffer_bytes = 8_192; assert!(settings.validate().is_ok()); } #[test] fn compatible_pool_rebuild_reuses_upstream_clients() { let previous = pool(Algorithm::LeastConnections, StickyMode::Off); let mut rebuilt = pool(Algorithm::LeastConnections, StickyMode::Off); let rebuilt_mut = Arc::get_mut(&mut rebuilt).expect("fresh pool must be uniquely owned"); assert!(!Arc::ptr_eq( &rebuilt_mut.members[0].backend, &previous.members[0].backend )); rebuilt_mut.reuse_observations(&previous); assert!(Arc::ptr_eq( &rebuilt_mut.members[0].backend, &previous.members[0].backend )); } #[test] fn retry_policy_rejects_unbounded_or_non_5xx_configuration() { let mut settings = Settings::default(); settings.retry.max_retries = 4; assert!(settings.validate().is_err()); settings.retry.max_retries = 1; settings.retry.retry_statuses = vec![429]; assert!(settings.validate().is_err()); } #[test] fn traffic_rate_drops_to_zero_after_an_idle_sample_window() { let state = BackendState::new(HealthMode::Off); state .bytes_out .store(2048, std::sync::atomic::Ordering::Relaxed); lock(&state.traffic).at = std::time::Instant::now() - std::time::Duration::from_secs(2); state.sample_traffic(); assert!(state.traffic_rate() > 0.0); lock(&state.traffic).at = std::time::Instant::now() - std::time::Duration::from_secs(2); state.sample_traffic(); assert_eq!(state.traffic_rate(), 0.0); } #[test] fn active_failures_are_independent_from_passive_user_traffic() { let state = BackendState::new(HealthMode::Http); let settings = settings(HealthMode::Http); let passive = PassiveHealth::default(); let fast = Duration::from_millis(5); assert!(!state.healthy()); state.observe_active(&settings, "ok", fast); assert!(!state.healthy()); state.observe_active(&settings, "ok", fast); assert!(state.healthy()); state.observe_active(&settings, "status", fast); state.observe_passive(&settings, &passive, "ok", fast); state.observe_active(&settings, "status", fast); assert!(!state.healthy()); state.observe_passive(&settings, &passive, "http_5xx", fast); assert!(!state.healthy()); state.observe_active(&settings, "ok", fast); assert!(!state.healthy()); state.observe_active(&settings, "ok", fast); assert!(state.healthy()); } #[test] fn passive_health_is_opt_in_and_uses_separate_thresholds() { let state = BackendState::new(HealthMode::Off); let active = settings(HealthMode::Off); let fast = Duration::from_millis(5); for _ in 0..10 { state.observe_passive(&active, &PassiveHealth::default(), "http_5xx", fast); } assert!(state.healthy()); assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 10); assert_eq!( state.passive_health_failures_total.load(Ordering::Relaxed), 0 ); let passive = PassiveHealth { enabled: true, fall: Some(2), rise: Some(2), ..Default::default() }; state.observe_passive(&active, &passive, "http_5xx", fast); assert!(state.healthy()); state.observe_passive(&active, &passive, "http_5xx", fast); assert!(!state.healthy()); state.observe_passive(&active, &passive, "ok", fast); assert!(!state.healthy()); state.observe_passive(&active, &passive, "ok", fast); assert!(state.healthy()); } #[test] fn passive_http_5xx_can_be_explicitly_ignored_when_passive_health_is_on() { let state = BackendState::new(HealthMode::Off); let active = settings(HealthMode::Off); let passive = PassiveHealth { enabled: true, count_http_5xx: false, fall: Some(1), ..Default::default() }; for _ in 0..100 { state.observe_passive(&active, &passive, "http_5xx", Duration::from_millis(5)); } assert!(state.healthy()); assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 100); assert_eq!( state.passive_health_failures_total.load(Ordering::Relaxed), 0 ); } #[test] fn passive_connect_errors_use_the_explicit_fall_threshold() { let state = BackendState::new(HealthMode::Off); let active = settings(HealthMode::Off); let passive = PassiveHealth { enabled: true, count_connect_errors: true, fall: Some(3), ..Default::default() }; for _ in 0..2 { state.observe_passive(&active, &passive, "connect_error", Duration::from_millis(5)); assert!(state.healthy()); } state.observe_passive(&active, &passive, "connect_error", Duration::from_millis(5)); assert!(!state.healthy()); assert_eq!(state.connect_errors_total.load(Ordering::Relaxed), 3); assert_eq!( state.passive_health_failures_total.load(Ordering::Relaxed), 3 ); } #[test] fn active_health_can_stay_healthy_while_passive_health_is_off() { let state = BackendState::new(HealthMode::Tcp); let active = settings(HealthMode::Tcp); let passive = PassiveHealth::default(); let fast = Duration::from_millis(5); state.observe_active(&active, "ok", fast); state.observe_active(&active, "ok", fast); assert!(state.healthy()); for _ in 0..20 { state.observe_passive(&active, &passive, "http_5xx", fast); } assert!(state.healthy()); assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 20); assert_eq!( state.passive_health_failures_total.load(Ordering::Relaxed), 0 ); } #[test] fn unhealthy_source_hands_off_when_the_original_health_domain_recovers() { let state = BackendState::new(HealthMode::Tcp); let active = HealthCheck { fall: 1, rise: 1, ..settings(HealthMode::Tcp) }; let passive = PassiveHealth { enabled: true, count_http_5xx: true, fall: Some(1), rise: Some(1), ..Default::default() }; let fast = Duration::from_millis(5); state.observe_active(&active, "ok", fast); state.observe_active(&active, "connect_error", fast); { let health = lock(&state.health); assert_eq!(health.unhealthy_source, "active_tcp"); } state.observe_passive(&active, &passive, "http_5xx", fast); state.observe_active(&active, "ok", fast); { let health = lock(&state.health); assert!(!health.healthy); assert_eq!(health.unhealthy_source, "passive_http_5xx"); assert_eq!(health.unhealthy_reason, "http_5xx"); } state.observe_passive(&active, &passive, "ok", fast); assert!(state.healthy()); } #[test] fn disabled_checks_never_quarantine_and_probes_are_single_flight() { let off = BackendState::new(HealthMode::Off); let settings = settings(HealthMode::Off); for _ in 0..5 { off.observe_passive( &settings, &PassiveHealth::default(), "connect_error", Duration::ZERO, ); } assert!(off.healthy()); assert!(!off.claim_probe(&settings)); let on = BackendState::new(HealthMode::Http); let settings = super::tests::settings(HealthMode::Http); assert!(on.claim_probe(&settings)); assert!(!on.claim_probe(&settings)); on.observe_active(&settings, "ok", Duration::ZERO); assert!(!on.claim_probe(&settings)); } #[tokio::test] async fn weighted_round_robin_distributes_three_to_one_without_body_collection() { let pool = pool(Algorithm::RoundRobin, StickyMode::Off); let mut counts = [0, 0]; for _ in 0..40 { let selection = select(&pool); counts[usize::from(selection.lease.member.definition.id == "b")] += 1; } assert_eq!(counts, [30, 10]); assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 0); assert_eq!(pool.members[0].state.requests.load(Ordering::Relaxed), 0); } #[tokio::test] async fn least_connections_counts_a_lease_until_its_last_owner_drops() { let pool = pool(Algorithm::LeastConnections, StickyMode::Off); let mut held = Vec::new(); for _ in 0..3 { let selection = select(&pool); assert_eq!(selection.lease.member.definition.id, "a"); held.push(selection.lease); } let other = select(&pool); assert_eq!(other.lease.member.definition.id, "b"); let clone = held[0].clone(); held.clear(); assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 1); drop(clone); assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 0); } #[tokio::test] async fn failover_selection_never_reuses_an_attempted_backend() { let pool = pool(Algorithm::LeastConnections, StickyMode::Off); let first = select(&pool); let first_id = first.lease.member.definition.id.clone(); drop(first); let retry = pool .select_retry( "127.0.0.1".parse().unwrap(), &HeaderMap::new(), false, std::slice::from_ref(&first_id), ) .unwrap(); assert_ne!(retry.lease.member.definition.id, first_id); } #[tokio::test] async fn backend_concurrency_limit_queues_then_rejects_and_recovers() { let pool = capacity_pool(1, 10); let first = select(&pool); let headers = HeaderMap::new(); let saturated = pool .select_wait("127.0.0.1".parse().unwrap(), &headers, false) .await; assert!(matches!(saturated, Err(SelectionFailure::Capacity))); let view = pool.view(); let member = &view["members"][0]; assert_eq!(member["capacity_rejections_total"], 1); assert_eq!(member["queue_timeouts_total"], 1); assert_eq!(member["active"], 1); drop(first); assert!(pool .select_wait("127.0.0.1".parse().unwrap(), &headers, false) .await .is_ok()); } #[tokio::test] async fn zero_capacity_queue_timeout_rejects_without_counting_a_queue_timeout() { let pool = capacity_pool(1, 0); let first = select(&pool); let headers = HeaderMap::new(); let saturated = pool .select_wait("127.0.0.1".parse().unwrap(), &headers, false) .await; assert!(matches!(saturated, Err(SelectionFailure::Capacity))); let view = pool.view(); let member = &view["members"][0]; assert_eq!(member["capacity_rejections_total"], 1); assert_eq!(member["queue_timeouts_total"], 0); drop(first); } #[test] fn detailed_backend_failures_are_exposed_without_losing_aggregate_counters() { let pool = capacity_pool(0, 0); let selection = select(&pool); selection.lease.error_detail( "dns", "lookup backend.example failed", Duration::from_millis(7), ); let view = pool.view(); let member = &view["members"][0]; assert_eq!(member["dns_errors_total"], 1); assert_eq!(member["connect_errors_total"], 1); assert_eq!(member["last_failure_class"], "dns"); assert_eq!( member["last_failure_detail"], "lookup backend.example failed" ); assert!(member["last_failure_at_ms"].as_u64().unwrap_or_default() > 0); } #[tokio::test] async fn traffic_and_adaptive_selection_use_observed_cost_and_weight() { for algorithm in [Algorithm::LeastTraffic, Algorithm::Adaptive] { let pool = pool(algorithm, StickyMode::Off); lock(&pool.members[0].state.traffic).rate = 50_000_000.0; lock(&pool.members[0].state.health).ewma_us = 1_000_000; assert_eq!(select(&pool).lease.member.definition.id, "b"); } } #[tokio::test] async fn ip_hash_is_stable_and_cannot_be_spoofed_with_forwarded_headers() { let pool = pool(Algorithm::IpHash, StickyMode::Off); let original = select(&pool).lease.member.definition.id.clone(); let mut headers = HeaderMap::new(); headers.insert("x-forwarded-for", "10.0.0.99".parse().unwrap()); for ip in ["127.0.0.1", "::ffff:127.0.0.1"] { for _ in 0..8 { assert_eq!( pool.select(ip.parse().unwrap(), &headers, false) .unwrap() .lease .member .definition .id, original ); } } } #[tokio::test] async fn opaque_cookie_is_removed_from_origin_requests_and_protected_on_responses() { let pool = pool(Algorithm::RoundRobin, StickyMode::Cookie); let first = select(&pool); let cookie = first.cookie.unwrap().to_str().unwrap().to_owned(); assert!( cookie.contains("HttpOnly") && cookie.contains("SameSite=Lax") && !cookie.contains("Secure") ); let cookie = cookie.split(';').next().unwrap(); let mut headers = HeaderMap::new(); headers.insert("cookie", format!("{cookie}; app=ok").parse().unwrap()); let next = pool .select("10.0.0.1".parse().unwrap(), &headers, false) .unwrap(); assert_eq!( next.lease.member.definition.id, first.lease.member.definition.id ); assert!(next.cookie.is_none()); pool.strip_affinity(&mut headers); assert_eq!(headers["cookie"], "app=ok"); headers.append( "set-cookie", format!("{}=origin-spoof", pool.cookie_name) .parse() .unwrap(), ); headers.append("set-cookie", "app=ok".parse().unwrap()); pool.strip_affinity_response(&mut headers); assert_eq!(headers.get_all("set-cookie").iter().count(), 1); assert_eq!(headers["set-cookie"], "app=ok"); let secure = pool .select("127.0.0.1".parse().unwrap(), &HeaderMap::new(), true) .unwrap(); assert!(secure .cookie .unwrap() .to_str() .unwrap() .contains("; Secure")); } #[tokio::test] async fn failed_sticky_member_is_replaced_and_duplicate_cookies_are_ignored() { let pool = pool(Algorithm::RoundRobin, StickyMode::Cookie); let first = select(&pool); let cookie = first .cookie .unwrap() .to_str() .unwrap() .split(';') .next() .unwrap() .to_owned(); let mut headers = HeaderMap::new(); headers.insert("cookie", format!("{cookie}; {cookie}").parse().unwrap()); assert!(pool.affinity(&headers).is_none()); headers.insert("cookie", cookie.parse().unwrap()); first.lease.member.state.set_healthy_for_test(false); let next = pool .select("127.0.0.1".parse().unwrap(), &headers, false) .unwrap(); assert_ne!( next.lease.member.definition.id, first.lease.member.definition.id ); assert!(next.cookie.is_some()); next.lease.member.state.set_healthy_for_test(false); assert!(pool .select("127.0.0.1".parse().unwrap(), &headers, false) .is_none()); } #[tokio::test] async fn slow_start_ramps_weight_and_error_is_counted_once_per_exchange() { let pool = pool(Algorithm::LeastConnections, StickyMode::Off); let selected = select(&pool); let state = &selected.lease.member.state; assert!(state.weight(100, 100) < 100.0); assert_eq!(state.weight(100, 0), 100.0); selected.lease.started(); selected.lease.bytes(50, true); selected.lease.bytes(100, false); selected.lease.error("timeout", Duration::from_secs(1)); selected.lease.error("body", Duration::from_secs(2)); selected.lease.retry_attempted(); selected.lease.retry_succeeded(); assert_eq!(state.requests.load(Ordering::Relaxed), 1); assert_eq!(state.errors.load(Ordering::Relaxed), 1); assert_eq!(state.timeouts_total.load(Ordering::Relaxed), 1); assert_eq!(state.io_errors_total.load(Ordering::Relaxed), 1); assert_eq!(state.bytes_in.load(Ordering::Relaxed), 50); assert_eq!(state.bytes_out.load(Ordering::Relaxed), 100); assert_eq!(state.retry_attempts_total.load(Ordering::Relaxed), 1); assert_eq!(state.retry_successes_total.load(Ordering::Relaxed), 1); assert!(pool .prometheus() .contains("rust_proxy_backend_requests_total{vhost=\"app\",backend=\"a\"} 1")); assert!(pool .prometheus() .contains("rust_proxy_backend_retry_attempts_total{vhost=\"app\",backend=\"a\"} 1")); } #[tokio::test] async fn slow_5xx_counts_both_observability_classes_when_passive_is_off() { let pool = pool(Algorithm::LeastConnections, StickyMode::Off); let selected = select(&pool); selected.lease.response(500, Duration::from_secs(20), true); let state = &selected.lease.member.state; assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 1); assert_eq!(state.slow_responses_total.load(Ordering::Relaxed), 1); assert_eq!( state.passive_health_failures_total.load(Ordering::Relaxed), 0 ); assert!(state.healthy()); } #[tokio::test] async fn client_upload_duration_does_not_trigger_passive_slow_failure() { let pool = pool(Algorithm::LeastConnections, StickyMode::Off); let upload = select(&pool); upload.lease.response(200, Duration::from_secs(20), false); assert_eq!(upload.lease.member.state.errors.load(Ordering::Relaxed), 0); let read = select(&pool); read.lease.response(200, Duration::from_secs(20), true); assert_eq!(read.lease.member.state.errors.load(Ordering::Relaxed), 1); } #[tokio::test] async fn backup_is_only_new_assignment_when_no_primary_is_eligible() { let mut pool = pool(Algorithm::RoundRobin, StickyMode::Off); Arc::get_mut(&mut Arc::get_mut(&mut pool).unwrap().members[1]) .unwrap() .definition .backup = true; for _ in 0..8 { assert_eq!(select(&pool).lease.member.definition.id, "a"); } pool.members[0].state.set_healthy_for_test(false); assert_eq!(select(&pool).lease.member.definition.id, "b"); pool.members[0].state.set_healthy_for_test(true); assert_eq!(select(&pool).lease.member.definition.id, "a"); } #[tokio::test] async fn drain_rejects_new_sessions_but_preserves_valid_existing_affinity() { let mut pool = pool(Algorithm::RoundRobin, StickyMode::Cookie); Arc::get_mut(&mut Arc::get_mut(&mut pool).unwrap().members[0]) .unwrap() .definition .drain = true; assert_eq!(select(&pool).lease.member.definition.id, "b"); let mut headers = HeaderMap::new(); headers.insert( "cookie", format!("{}={}", pool.cookie_name, pool.members[0].token) .parse() .unwrap(), ); let existing = pool .select("127.0.0.1".parse().unwrap(), &headers, false) .unwrap(); assert_eq!(existing.lease.member.definition.id, "a"); pool.members[0].state.set_healthy_for_test(false); assert_eq!( pool.select("127.0.0.1".parse().unwrap(), &headers, false) .unwrap() .lease .member .definition .id, "b" ); } #[test] fn passive_ejection_remembers_first_real_error_and_trigger() { let state = BackendState::new(HealthMode::Off); let active = settings(HealthMode::Off); let passive = PassiveHealth { enabled: true, fall: Some(2), ..Default::default() }; let elapsed = Duration::from_millis(3); assert!(state .observe_passive_with_detail( &active, &passive, "connect_error", "dns", "first lookup failed", elapsed ) .is_none()); let (first, failures) = state .observe_passive_with_detail( &active, &passive, "connect_error", "connect", "second TCP failed", elapsed, ) .expect("the threshold crossing must return the first failure"); assert_eq!(failures, 2); assert_eq!(first.class, "dns"); assert_eq!(first.detail, "first lookup failed"); assert!(!state.healthy()); assert!( state .observe_passive_with_detail( &active, &passive, "connect_error", "connect", "third TCP failed", elapsed ) .is_none(), "the ejection must be logged only once" ); } #[test] fn passive_io_burst_is_one_observation_not_many_failed_connections() { let state = BackendState::new(HealthMode::Off); let active = settings(HealthMode::Off); let passive = PassiveHealth { enabled: true, fall: Some(3), ..Default::default() }; for _ in 0..20 { state.observe_passive_with_detail( &active, &passive, "io_error", "io_error", "connection closed", Duration::from_millis(1), ); } assert!(state.healthy()); assert_eq!(state.io_errors_total.load(Ordering::Relaxed), 20); assert_eq!( state.passive_health_failures_total.load(Ordering::Relaxed), 1 ); } #[test] fn passive_only_quarantine_expires_without_vhost_reload() { let state = BackendState::new(HealthMode::Off); let active = settings(HealthMode::Off); let passive = PassiveHealth { enabled: true, fall: Some(1), ..Default::default() }; state.observe_passive(&active, &passive, "connect_error", Duration::ZERO); assert!(!state.healthy()); lock(&state.health).passive_ejected_at = Some(Instant::now() - Duration::from_secs(10)); assert!( state.healthy(), "a half-open traffic attempt must become possible" ); } #[test] fn active_probe_successes_recover_a_passively_ejected_backend() { let state = BackendState::new(HealthMode::Http); let active = HealthCheck { fall: 1, rise: 1, ..settings(HealthMode::Http) }; let passive = PassiveHealth { enabled: true, fall: Some(1), ..Default::default() }; state.observe_active(&active, "ok", Duration::ZERO); assert!(state.healthy()); state.observe_passive(&active, &passive, "connect_error", Duration::ZERO); assert!(!state.healthy()); state.observe_active(&active, "ok", Duration::ZERO); assert!(!state.healthy()); state.observe_active(&active, "ok", Duration::ZERO); assert!(state.healthy()); } #[test] fn single_backend_fast_path_observes_capacity_and_health_without_affinity_cookie() { let pool = capacity_pool(1, 0); assert_eq!(pool.members.len(), 1); let peer = "127.0.0.1".parse().unwrap(); let headers = HeaderMap::new(); let first = pool.select(peer, &headers, false).expect("healthy backend"); assert!(first.cookie.is_none()); assert!(pool.select(peer, &headers, false).is_none()); drop(first); pool.members[0].state.set_healthy_for_test(false); assert!(pool.select(peer, &headers, false).is_none()); pool.members[0].state.set_healthy_for_test(true); assert!(pool.select(peer, &headers, false).is_some()); } #[test] fn leases_reuse_immutable_backend_health_policies() { let pool = capacity_pool(0, 0); let first = select(&pool); let second = select(&pool); assert!(Arc::ptr_eq(&first.lease.health, &second.lease.health)); assert!(Arc::ptr_eq( &first.lease.passive_health, &second.lease.passive_health )); } #[test] fn healthy_fast_path_tracks_active_and_passive_ejections() { let state = BackendState::new(HealthMode::Off); assert!(state.healthy()); let active = settings(HealthMode::Off); let passive = PassiveHealth { enabled: true, fall: Some(1), rise: Some(1), ..Default::default() }; state.observe_passive(&active, &passive, "connect_error", Duration::from_millis(2)); assert!(!state.healthy()); state.observe_passive(&active, &passive, "ok", Duration::from_millis(2)); assert!(state.healthy()); } #[test] fn successful_lease_response_keeps_passive_observability_and_failure_detail() { let pool = capacity_pool(0, 0); let selection = select(&pool); selection .lease .response(200, Duration::from_millis(5), true); let state = &selection.lease.member.state; assert_eq!(state.last_status.load(Ordering::Relaxed), 200); assert_eq!(lock(&state.health).passive_result, "ok"); assert!(lock(&state.health).latency_us > 0); selection .lease .response(503, Duration::from_millis(5), true); assert_eq!(state.last_status.load(Ordering::Relaxed), 503); assert_eq!(lock(&state.last_failure).detail, "status=503"); } fn health_port_config(backend_port: u16, health: serde_json::Value) -> crate::config::Config { serde_json::from_value(serde_json::json!({ "listeners":{"endpoints":[{"id":"http","binding":{"address":"127.0.0.1:8080"}}]}, "vhosts":[{ "id":"app","frontend":{"hosts":["app.localhost"],"listener_ids":["http"]}, "backends":[{"id":"a","url":format!("http://127.0.0.1:{backend_port}"),"allow_private":true}], "load_balancing":{"health_check":health} }]})).unwrap() } fn health_port_policy() -> crate::config::ValidationPolicy { crate::config::ValidationPolicy { proxy_addr: "127.0.0.1:8080".parse().unwrap(), admin_addr: "127.0.0.1:9090".parse().unwrap(), tls_addr: None, cert_dir: None, data_dir: None, } } #[test] fn health_check_port_compiles_separate_probe_target() { let policy = health_port_policy(); let snapshot = crate::config::compile( health_port_config(9000, serde_json::json!({"mode":"tcp","port":8079})), 0, &policy).unwrap(); let backend = &snapshot.pools[0].members[0].backend; assert_eq!(backend.origin, "http://127.0.0.1:9000"); assert_eq!(backend.health_probe.as_ref().unwrap().origin, "http://127.0.0.1:8079"); for health in [ serde_json::json!({"mode":"tcp","port":9000}), serde_json::json!({"mode":"tcp"}), serde_json::json!({"mode":"off","port":8079}), ] { let snapshot = crate::config::compile(health_port_config(9000, health), 0, &policy).unwrap(); assert!(snapshot.pools[0].members[0].backend.health_probe.is_none()); } assert!(crate::config::compile( health_port_config(9000, serde_json::json!({"mode":"tcp","port":0})), 0, &policy).is_err()); assert!(crate::config::compile( health_port_config(9000, serde_json::json!({"mode":"http","port":9090})), 0, &policy).is_err()); } #[tokio::test] async fn health_check_probes_configured_port_instead_of_traffic_port() { let traffic = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let traffic_port = traffic.local_addr().unwrap().port(); let closed_port = { let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); listener.local_addr().unwrap().port() }; let policy = health_port_policy(); let config = health_port_config(traffic_port, serde_json::json!({ "mode":"tcp","port":closed_port,"fall":1,"rise":1,"interval_ms":250,"timeout_ms":200,"slow_response_ms":0})); let snapshot = crate::config::compile(config, 0, &policy).unwrap(); let member = snapshot.pools[0].members[0].clone(); let settings = member.health_settings.clone(); probe(member.clone(), settings.clone()).await; assert!(!member.state.healthy(), "probe must use the closed health port"); let health = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let health_port = health.local_addr().unwrap().port(); drop(traffic); let config = health_port_config(traffic_port, serde_json::json!({ "mode":"tcp","port":health_port,"fall":1,"rise":1,"interval_ms":250,"timeout_ms":200,"slow_response_ms":0})); let snapshot = crate::config::compile(config, 0, &policy).unwrap(); let member = snapshot.pools[0].members[0].clone(); let settings = member.health_settings.clone(); probe(member.clone(), settings).await; assert!(member.state.healthy(), "probe must succeed on the open health port while traffic port is closed"); tokio::time::timeout(Duration::from_secs(1), health.accept()) .await .expect("health port must receive the probe connection") .unwrap(); } async fn one_shot_http(body: &'static str) -> (u16, tokio::task::JoinHandle<()>) { use tokio::io::{AsyncReadExt, AsyncWriteExt}; let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let port = listener.local_addr().unwrap().port(); let task = tokio::spawn(async move { while let Ok((mut socket, _)) = listener.accept().await { let mut buffer = [0u8; 2048]; let _ = socket.read(&mut buffer).await; let response = format!( "HTTP/1.1 200 OK\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}", body.len(), body ); let _ = socket.write_all(response.as_bytes()).await; } }); (port, task) } #[tokio::test] async fn http_health_check_requires_expected_body_text() { let policy = health_port_policy(); for (body, healthy) in [("{\"status\":\"ok\"}", true), ("{\"status\":\"degraded\"}", false)] { let (port, server) = one_shot_http(body).await; let config = health_port_config(port, serde_json::json!({ "mode":"http","path":"/health","expect_body":"\"status\":\"ok\"", "fall":1,"rise":1,"interval_ms":1000,"timeout_ms":800,"slow_response_ms":0})); let snapshot = crate::config::compile(config, 0, &policy).unwrap(); let member = snapshot.pools[0].members[0].clone(); probe(member.clone(), member.health_settings.clone()).await; assert_eq!(member.state.healthy(), healthy, "body {body}"); if !healthy { assert_eq!(lock(&member.state.health).result, "http_body"); } server.abort(); } for invalid in ["", "line\nbreak"] { assert!(crate::config::compile( health_port_config(9000, serde_json::json!({"mode":"http","expect_body":invalid})), 0, &policy).is_err()); } } #[tokio::test] async fn health_check_port_survives_live_reload_with_reused_clients() { let policy = health_port_policy(); let defaults = (policy.proxy_addr, policy.tls_addr); let first = crate::config::compile_reusing( health_port_config(9000, serde_json::json!({"mode":"tcp"})), 0, &policy, defaults, None).unwrap(); assert!(first.pools[0].members[0].health_probe.is_none()); let second = crate::config::compile_reusing( health_port_config(9000, serde_json::json!({"mode":"tcp","port":8079})), 1, &policy, defaults, Some(&first)).unwrap(); let member = &second.pools[0].members[0]; assert!(Arc::ptr_eq(&member.backend, &first.pools[0].members[0].backend), "traffic clients are reused"); assert_eq!(member.health_probe.as_ref().unwrap().origin, "http://127.0.0.1:8079"); let third = crate::config::compile_reusing( health_port_config(9000, serde_json::json!({"mode":"tcp"})), 2, &policy, defaults, Some(&second)).unwrap(); assert!(third.pools[0].members[0].health_probe.is_none()); } }