965 lines
39 KiB
Rust
965 lines
39 KiB
Rust
mod tests {
|
|
use super::*;
|
|
fn settings(mode: HealthMode) -> HealthCheck {
|
|
HealthCheck {
|
|
mode,
|
|
fall: 2,
|
|
rise: 2,
|
|
..Default::default()
|
|
}
|
|
}
|
|
fn pool(algorithm: Algorithm, sticky: StickyMode) -> Arc<Pool> {
|
|
let cfg = serde_json::from_value(serde_json::json!({
|
|
"listeners":{"endpoints":[{"id":"http","binding":{"address":"127.0.0.1:8080"}}]},
|
|
"vhosts":[{
|
|
"id":"app","frontend":{"hosts":["app.localhost"],"listener_ids":["http"]},
|
|
"backends":[{"id":"a","url":"http://127.0.0.1:9000","allow_private":true,"weight":3},
|
|
{"id":"b","url":"http://127.0.0.1:9001","allow_private":true,"weight":1}],
|
|
"load_balancing":{"algorithm":algorithm,"sticky":{"mode":sticky},"slow_start_seconds":0,"health_check":{"mode":"off"}}
|
|
}]})).unwrap();
|
|
let policy = crate::config::ValidationPolicy {
|
|
proxy_addr: "127.0.0.1:8080".parse().unwrap(),
|
|
admin_addr: "127.0.0.1:9090".parse().unwrap(),
|
|
tls_addr: None,
|
|
cert_dir: None,
|
|
data_dir: None,
|
|
};
|
|
crate::config::compile(cfg, 0, &policy).unwrap().pools[0].clone()
|
|
}
|
|
fn capacity_pool(limit: u32, queue_timeout_ms: u64) -> Arc<Pool> {
|
|
let cfg = serde_json::from_value(serde_json::json!({
|
|
"listeners":{"endpoints":[{"id":"http","binding":{"address":"127.0.0.1:8080"}}]},
|
|
"vhosts":[{
|
|
"id":"limited","frontend":{"hosts":["limited.localhost"],"listener_ids":["http"]},
|
|
"backends":[{
|
|
"id":"only","url":"http://127.0.0.1:9000","allow_private":true,
|
|
"max_concurrent_requests":limit,"queue_timeout_ms":queue_timeout_ms
|
|
}],
|
|
"load_balancing":{"health_check":{"mode":"off"}}
|
|
}]
|
|
}))
|
|
.unwrap();
|
|
let policy = crate::config::ValidationPolicy {
|
|
proxy_addr: "127.0.0.1:8080".parse().unwrap(),
|
|
admin_addr: "127.0.0.1:9090".parse().unwrap(),
|
|
tls_addr: None,
|
|
cert_dir: None,
|
|
data_dir: None,
|
|
};
|
|
crate::config::compile(cfg, 0, &policy).unwrap().pools[0].clone()
|
|
}
|
|
fn select(pool: &Pool) -> Selection {
|
|
pool.select("127.0.0.1".parse().unwrap(), &HeaderMap::new(), false)
|
|
.unwrap()
|
|
}
|
|
#[test]
|
|
fn per_backend_monitoring_can_be_disabled_or_overridden() {
|
|
let mut defaults: crate::vhost::PoolMember = serde_json::from_value(serde_json::json!({
|
|
"id": "error_404", "url": "http://127.0.0.1:8282", "pool_eligible": false
|
|
}))
|
|
.unwrap();
|
|
let mut settings = Settings::default();
|
|
settings.passive_health.enabled = true;
|
|
assert!(defaults.health_check_enabled);
|
|
assert_eq!(settings.health_for(&defaults).mode, HealthMode::Tcp);
|
|
assert!(settings.passive_for(&defaults).enabled);
|
|
defaults.health_check_enabled = false;
|
|
assert_eq!(settings.health_for(&defaults).mode, HealthMode::Off);
|
|
assert!(!settings.passive_for(&defaults).enabled);
|
|
assert!(!defaults.pool_eligible);
|
|
defaults.health_check_enabled = true;
|
|
defaults.health_check = Some(HealthCheck {
|
|
mode: HealthMode::Http,
|
|
path: "/healthz".into(),
|
|
..Default::default()
|
|
});
|
|
let effective = settings.health_for(&defaults);
|
|
assert_eq!(effective.mode, HealthMode::Http);
|
|
assert_eq!(effective.path, "/healthz");
|
|
assert_eq!(effective.host, settings.health_check.host);
|
|
let restored: crate::vhost::PoolMember =
|
|
serde_json::from_str(&serde_json::to_string(&defaults).unwrap()).unwrap();
|
|
assert_eq!(settings.health_for(&restored).path, "/healthz");
|
|
}
|
|
#[test]
|
|
fn default_health_check_uses_tcp() {
|
|
assert_eq!(HealthMode::default(), HealthMode::Tcp);
|
|
assert_eq!(HealthCheck::default().mode, HealthMode::Tcp);
|
|
}
|
|
#[test]
|
|
fn legacy_settings_without_passive_health_default_to_off() {
|
|
let parsed: Settings = serde_json::from_value(serde_json::json!({
|
|
"health_check": { "mode": "off" }
|
|
}))
|
|
.unwrap();
|
|
assert!(!parsed.passive_health.enabled);
|
|
assert!(parsed.passive_health.count_connect_errors);
|
|
assert!(parsed.passive_health.count_timeouts);
|
|
assert!(parsed.passive_health.count_io_errors);
|
|
assert!(parsed.passive_health.count_http_5xx);
|
|
assert!(!parsed.passive_health.count_slow_responses);
|
|
assert_eq!(parsed.retry.max_retries, 1);
|
|
assert!(parsed.retry.retry_connect_errors);
|
|
assert!(parsed.retry.retry_timeouts);
|
|
assert_eq!(parsed.retry.retry_statuses, vec![502, 503, 504]);
|
|
assert_eq!(parsed.keepalive.max_idle_per_host, 64);
|
|
let explicit: Settings = serde_json::from_value(serde_json::json!({
|
|
"keepalive": { "max_idle_per_host": 4 }
|
|
}))
|
|
.unwrap();
|
|
assert_eq!(explicit.keepalive.max_idle_per_host, 4);
|
|
assert_eq!(parsed.keepalive.pool_recycle_requests, 25_000);
|
|
assert_eq!(parsed.keepalive.pool_recycle_seconds, 3_600);
|
|
assert_eq!(parsed.keepalive.http1_max_buffer_bytes, 131_072);
|
|
}
|
|
|
|
#[test]
|
|
fn upstream_pool_recycle_limits_are_bounded_and_can_be_disabled() {
|
|
let mut settings = Settings::default();
|
|
assert!(settings.validate().is_ok());
|
|
|
|
settings.keepalive.pool_recycle_requests = 0;
|
|
settings.keepalive.pool_recycle_seconds = 0;
|
|
assert!(settings.validate().is_ok());
|
|
|
|
settings.keepalive.pool_recycle_requests = 999;
|
|
assert!(settings.validate().is_err());
|
|
settings.keepalive.pool_recycle_requests = 1_000;
|
|
settings.keepalive.pool_recycle_seconds = 59;
|
|
assert!(settings.validate().is_err());
|
|
settings.keepalive.pool_recycle_seconds = 60;
|
|
settings.keepalive.http1_max_buffer_bytes = 8_191;
|
|
assert!(settings.validate().is_err());
|
|
settings.keepalive.http1_max_buffer_bytes = 8_192;
|
|
assert!(settings.validate().is_ok());
|
|
}
|
|
|
|
#[test]
|
|
fn compatible_pool_rebuild_reuses_upstream_clients() {
|
|
let previous = pool(Algorithm::LeastConnections, StickyMode::Off);
|
|
let mut rebuilt = pool(Algorithm::LeastConnections, StickyMode::Off);
|
|
let rebuilt_mut = Arc::get_mut(&mut rebuilt).expect("fresh pool must be uniquely owned");
|
|
|
|
assert!(!Arc::ptr_eq(
|
|
&rebuilt_mut.members[0].backend,
|
|
&previous.members[0].backend
|
|
));
|
|
rebuilt_mut.reuse_observations(&previous);
|
|
assert!(Arc::ptr_eq(
|
|
&rebuilt_mut.members[0].backend,
|
|
&previous.members[0].backend
|
|
));
|
|
}
|
|
|
|
#[test]
|
|
fn retry_policy_rejects_unbounded_or_non_5xx_configuration() {
|
|
let mut settings = Settings::default();
|
|
settings.retry.max_retries = 4;
|
|
assert!(settings.validate().is_err());
|
|
settings.retry.max_retries = 1;
|
|
settings.retry.retry_statuses = vec![429];
|
|
assert!(settings.validate().is_err());
|
|
}
|
|
|
|
#[test]
|
|
fn traffic_rate_drops_to_zero_after_an_idle_sample_window() {
|
|
let state = BackendState::new(HealthMode::Off);
|
|
state
|
|
.bytes_out
|
|
.store(2048, std::sync::atomic::Ordering::Relaxed);
|
|
lock(&state.traffic).at = std::time::Instant::now() - std::time::Duration::from_secs(2);
|
|
state.sample_traffic();
|
|
assert!(state.traffic_rate() > 0.0);
|
|
lock(&state.traffic).at = std::time::Instant::now() - std::time::Duration::from_secs(2);
|
|
state.sample_traffic();
|
|
assert_eq!(state.traffic_rate(), 0.0);
|
|
}
|
|
#[test]
|
|
fn active_failures_are_independent_from_passive_user_traffic() {
|
|
let state = BackendState::new(HealthMode::Http);
|
|
let settings = settings(HealthMode::Http);
|
|
let passive = PassiveHealth::default();
|
|
let fast = Duration::from_millis(5);
|
|
assert!(!state.healthy());
|
|
state.observe_active(&settings, "ok", fast);
|
|
assert!(!state.healthy());
|
|
state.observe_active(&settings, "ok", fast);
|
|
assert!(state.healthy());
|
|
state.observe_active(&settings, "status", fast);
|
|
state.observe_passive(&settings, &passive, "ok", fast);
|
|
state.observe_active(&settings, "status", fast);
|
|
assert!(!state.healthy());
|
|
state.observe_passive(&settings, &passive, "http_5xx", fast);
|
|
assert!(!state.healthy());
|
|
state.observe_active(&settings, "ok", fast);
|
|
assert!(!state.healthy());
|
|
state.observe_active(&settings, "ok", fast);
|
|
assert!(state.healthy());
|
|
}
|
|
|
|
#[test]
|
|
fn passive_health_is_opt_in_and_uses_separate_thresholds() {
|
|
let state = BackendState::new(HealthMode::Off);
|
|
let active = settings(HealthMode::Off);
|
|
let fast = Duration::from_millis(5);
|
|
for _ in 0..10 {
|
|
state.observe_passive(&active, &PassiveHealth::default(), "http_5xx", fast);
|
|
}
|
|
assert!(state.healthy());
|
|
assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 10);
|
|
assert_eq!(
|
|
state.passive_health_failures_total.load(Ordering::Relaxed),
|
|
0
|
|
);
|
|
|
|
let passive = PassiveHealth {
|
|
enabled: true,
|
|
fall: Some(2),
|
|
rise: Some(2),
|
|
..Default::default()
|
|
};
|
|
state.observe_passive(&active, &passive, "http_5xx", fast);
|
|
assert!(state.healthy());
|
|
state.observe_passive(&active, &passive, "http_5xx", fast);
|
|
assert!(!state.healthy());
|
|
state.observe_passive(&active, &passive, "ok", fast);
|
|
assert!(!state.healthy());
|
|
state.observe_passive(&active, &passive, "ok", fast);
|
|
assert!(state.healthy());
|
|
}
|
|
#[test]
|
|
fn passive_http_5xx_can_be_explicitly_ignored_when_passive_health_is_on() {
|
|
let state = BackendState::new(HealthMode::Off);
|
|
let active = settings(HealthMode::Off);
|
|
let passive = PassiveHealth {
|
|
enabled: true,
|
|
count_http_5xx: false,
|
|
fall: Some(1),
|
|
..Default::default()
|
|
};
|
|
for _ in 0..100 {
|
|
state.observe_passive(&active, &passive, "http_5xx", Duration::from_millis(5));
|
|
}
|
|
assert!(state.healthy());
|
|
assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 100);
|
|
assert_eq!(
|
|
state.passive_health_failures_total.load(Ordering::Relaxed),
|
|
0
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn passive_connect_errors_use_the_explicit_fall_threshold() {
|
|
let state = BackendState::new(HealthMode::Off);
|
|
let active = settings(HealthMode::Off);
|
|
let passive = PassiveHealth {
|
|
enabled: true,
|
|
count_connect_errors: true,
|
|
fall: Some(3),
|
|
..Default::default()
|
|
};
|
|
for _ in 0..2 {
|
|
state.observe_passive(&active, &passive, "connect_error", Duration::from_millis(5));
|
|
assert!(state.healthy());
|
|
}
|
|
state.observe_passive(&active, &passive, "connect_error", Duration::from_millis(5));
|
|
assert!(!state.healthy());
|
|
assert_eq!(state.connect_errors_total.load(Ordering::Relaxed), 3);
|
|
assert_eq!(
|
|
state.passive_health_failures_total.load(Ordering::Relaxed),
|
|
3
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn active_health_can_stay_healthy_while_passive_health_is_off() {
|
|
let state = BackendState::new(HealthMode::Tcp);
|
|
let active = settings(HealthMode::Tcp);
|
|
let passive = PassiveHealth::default();
|
|
let fast = Duration::from_millis(5);
|
|
state.observe_active(&active, "ok", fast);
|
|
state.observe_active(&active, "ok", fast);
|
|
assert!(state.healthy());
|
|
for _ in 0..20 {
|
|
state.observe_passive(&active, &passive, "http_5xx", fast);
|
|
}
|
|
assert!(state.healthy());
|
|
assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 20);
|
|
assert_eq!(
|
|
state.passive_health_failures_total.load(Ordering::Relaxed),
|
|
0
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn unhealthy_source_hands_off_when_the_original_health_domain_recovers() {
|
|
let state = BackendState::new(HealthMode::Tcp);
|
|
let active = HealthCheck {
|
|
fall: 1,
|
|
rise: 1,
|
|
..settings(HealthMode::Tcp)
|
|
};
|
|
let passive = PassiveHealth {
|
|
enabled: true,
|
|
count_http_5xx: true,
|
|
fall: Some(1),
|
|
rise: Some(1),
|
|
..Default::default()
|
|
};
|
|
let fast = Duration::from_millis(5);
|
|
|
|
state.observe_active(&active, "ok", fast);
|
|
state.observe_active(&active, "connect_error", fast);
|
|
{
|
|
let health = lock(&state.health);
|
|
assert_eq!(health.unhealthy_source, "active_tcp");
|
|
}
|
|
state.observe_passive(&active, &passive, "http_5xx", fast);
|
|
state.observe_active(&active, "ok", fast);
|
|
{
|
|
let health = lock(&state.health);
|
|
assert!(!health.healthy);
|
|
assert_eq!(health.unhealthy_source, "passive_http_5xx");
|
|
assert_eq!(health.unhealthy_reason, "http_5xx");
|
|
}
|
|
state.observe_passive(&active, &passive, "ok", fast);
|
|
assert!(state.healthy());
|
|
}
|
|
|
|
#[test]
|
|
fn disabled_checks_never_quarantine_and_probes_are_single_flight() {
|
|
let off = BackendState::new(HealthMode::Off);
|
|
let settings = settings(HealthMode::Off);
|
|
for _ in 0..5 {
|
|
off.observe_passive(
|
|
&settings,
|
|
&PassiveHealth::default(),
|
|
"connect_error",
|
|
Duration::ZERO,
|
|
);
|
|
}
|
|
assert!(off.healthy());
|
|
assert!(!off.claim_probe(&settings));
|
|
let on = BackendState::new(HealthMode::Http);
|
|
let settings = super::tests::settings(HealthMode::Http);
|
|
assert!(on.claim_probe(&settings));
|
|
assert!(!on.claim_probe(&settings));
|
|
on.observe_active(&settings, "ok", Duration::ZERO);
|
|
assert!(!on.claim_probe(&settings));
|
|
}
|
|
#[tokio::test]
|
|
async fn weighted_round_robin_distributes_three_to_one_without_body_collection() {
|
|
let pool = pool(Algorithm::RoundRobin, StickyMode::Off);
|
|
let mut counts = [0, 0];
|
|
for _ in 0..40 {
|
|
let selection = select(&pool);
|
|
counts[usize::from(selection.lease.member.definition.id == "b")] += 1;
|
|
}
|
|
assert_eq!(counts, [30, 10]);
|
|
assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 0);
|
|
assert_eq!(pool.members[0].state.requests.load(Ordering::Relaxed), 0);
|
|
}
|
|
#[tokio::test]
|
|
async fn least_connections_counts_a_lease_until_its_last_owner_drops() {
|
|
let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
|
|
let mut held = Vec::new();
|
|
for _ in 0..3 {
|
|
let selection = select(&pool);
|
|
assert_eq!(selection.lease.member.definition.id, "a");
|
|
held.push(selection.lease);
|
|
}
|
|
let other = select(&pool);
|
|
assert_eq!(other.lease.member.definition.id, "b");
|
|
let clone = held[0].clone();
|
|
held.clear();
|
|
assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 1);
|
|
drop(clone);
|
|
assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 0);
|
|
}
|
|
#[tokio::test]
|
|
async fn failover_selection_never_reuses_an_attempted_backend() {
|
|
let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
|
|
let first = select(&pool);
|
|
let first_id = first.lease.member.definition.id.clone();
|
|
drop(first);
|
|
let retry = pool
|
|
.select_retry(
|
|
"127.0.0.1".parse().unwrap(),
|
|
&HeaderMap::new(),
|
|
false,
|
|
std::slice::from_ref(&first_id),
|
|
)
|
|
.unwrap();
|
|
assert_ne!(retry.lease.member.definition.id, first_id);
|
|
}
|
|
#[tokio::test]
|
|
async fn backend_concurrency_limit_queues_then_rejects_and_recovers() {
|
|
let pool = capacity_pool(1, 10);
|
|
let first = select(&pool);
|
|
let headers = HeaderMap::new();
|
|
let saturated = pool
|
|
.select_wait("127.0.0.1".parse().unwrap(), &headers, false)
|
|
.await;
|
|
assert!(matches!(saturated, Err(SelectionFailure::Capacity)));
|
|
let view = pool.view();
|
|
let member = &view["members"][0];
|
|
assert_eq!(member["capacity_rejections_total"], 1);
|
|
assert_eq!(member["queue_timeouts_total"], 1);
|
|
assert_eq!(member["active"], 1);
|
|
drop(first);
|
|
assert!(pool
|
|
.select_wait("127.0.0.1".parse().unwrap(), &headers, false)
|
|
.await
|
|
.is_ok());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn zero_capacity_queue_timeout_rejects_without_counting_a_queue_timeout() {
|
|
let pool = capacity_pool(1, 0);
|
|
let first = select(&pool);
|
|
let headers = HeaderMap::new();
|
|
let saturated = pool
|
|
.select_wait("127.0.0.1".parse().unwrap(), &headers, false)
|
|
.await;
|
|
assert!(matches!(saturated, Err(SelectionFailure::Capacity)));
|
|
let view = pool.view();
|
|
let member = &view["members"][0];
|
|
assert_eq!(member["capacity_rejections_total"], 1);
|
|
assert_eq!(member["queue_timeouts_total"], 0);
|
|
drop(first);
|
|
}
|
|
|
|
#[test]
|
|
fn detailed_backend_failures_are_exposed_without_losing_aggregate_counters() {
|
|
let pool = capacity_pool(0, 0);
|
|
let selection = select(&pool);
|
|
selection.lease.error_detail(
|
|
"dns",
|
|
"lookup backend.example failed",
|
|
Duration::from_millis(7),
|
|
);
|
|
let view = pool.view();
|
|
let member = &view["members"][0];
|
|
assert_eq!(member["dns_errors_total"], 1);
|
|
assert_eq!(member["connect_errors_total"], 1);
|
|
assert_eq!(member["last_failure_class"], "dns");
|
|
assert_eq!(
|
|
member["last_failure_detail"],
|
|
"lookup backend.example failed"
|
|
);
|
|
assert!(member["last_failure_at_ms"].as_u64().unwrap_or_default() > 0);
|
|
}
|
|
#[tokio::test]
|
|
async fn traffic_and_adaptive_selection_use_observed_cost_and_weight() {
|
|
for algorithm in [Algorithm::LeastTraffic, Algorithm::Adaptive] {
|
|
let pool = pool(algorithm, StickyMode::Off);
|
|
lock(&pool.members[0].state.traffic).rate = 50_000_000.0;
|
|
lock(&pool.members[0].state.health).ewma_us = 1_000_000;
|
|
assert_eq!(select(&pool).lease.member.definition.id, "b");
|
|
}
|
|
}
|
|
#[tokio::test]
|
|
async fn ip_hash_is_stable_and_cannot_be_spoofed_with_forwarded_headers() {
|
|
let pool = pool(Algorithm::IpHash, StickyMode::Off);
|
|
let original = select(&pool).lease.member.definition.id.clone();
|
|
let mut headers = HeaderMap::new();
|
|
headers.insert("x-forwarded-for", "10.0.0.99".parse().unwrap());
|
|
for ip in ["127.0.0.1", "::ffff:127.0.0.1"] {
|
|
for _ in 0..8 {
|
|
assert_eq!(
|
|
pool.select(ip.parse().unwrap(), &headers, false)
|
|
.unwrap()
|
|
.lease
|
|
.member
|
|
.definition
|
|
.id,
|
|
original
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn opaque_cookie_is_removed_from_origin_requests_and_protected_on_responses() {
|
|
let pool = pool(Algorithm::RoundRobin, StickyMode::Cookie);
|
|
let first = select(&pool);
|
|
let cookie = first.cookie.unwrap().to_str().unwrap().to_owned();
|
|
assert!(
|
|
cookie.contains("HttpOnly")
|
|
&& cookie.contains("SameSite=Lax")
|
|
&& !cookie.contains("Secure")
|
|
);
|
|
let cookie = cookie.split(';').next().unwrap();
|
|
let mut headers = HeaderMap::new();
|
|
headers.insert("cookie", format!("{cookie}; app=ok").parse().unwrap());
|
|
let next = pool
|
|
.select("10.0.0.1".parse().unwrap(), &headers, false)
|
|
.unwrap();
|
|
assert_eq!(
|
|
next.lease.member.definition.id,
|
|
first.lease.member.definition.id
|
|
);
|
|
assert!(next.cookie.is_none());
|
|
pool.strip_affinity(&mut headers);
|
|
assert_eq!(headers["cookie"], "app=ok");
|
|
headers.append(
|
|
"set-cookie",
|
|
format!("{}=origin-spoof", pool.cookie_name)
|
|
.parse()
|
|
.unwrap(),
|
|
);
|
|
headers.append("set-cookie", "app=ok".parse().unwrap());
|
|
pool.strip_affinity_response(&mut headers);
|
|
assert_eq!(headers.get_all("set-cookie").iter().count(), 1);
|
|
assert_eq!(headers["set-cookie"], "app=ok");
|
|
let secure = pool
|
|
.select("127.0.0.1".parse().unwrap(), &HeaderMap::new(), true)
|
|
.unwrap();
|
|
assert!(secure
|
|
.cookie
|
|
.unwrap()
|
|
.to_str()
|
|
.unwrap()
|
|
.contains("; Secure"));
|
|
}
|
|
#[tokio::test]
|
|
async fn failed_sticky_member_is_replaced_and_duplicate_cookies_are_ignored() {
|
|
let pool = pool(Algorithm::RoundRobin, StickyMode::Cookie);
|
|
let first = select(&pool);
|
|
let cookie = first
|
|
.cookie
|
|
.unwrap()
|
|
.to_str()
|
|
.unwrap()
|
|
.split(';')
|
|
.next()
|
|
.unwrap()
|
|
.to_owned();
|
|
let mut headers = HeaderMap::new();
|
|
headers.insert("cookie", format!("{cookie}; {cookie}").parse().unwrap());
|
|
assert!(pool.affinity(&headers).is_none());
|
|
headers.insert("cookie", cookie.parse().unwrap());
|
|
first.lease.member.state.set_healthy_for_test(false);
|
|
let next = pool
|
|
.select("127.0.0.1".parse().unwrap(), &headers, false)
|
|
.unwrap();
|
|
assert_ne!(
|
|
next.lease.member.definition.id,
|
|
first.lease.member.definition.id
|
|
);
|
|
assert!(next.cookie.is_some());
|
|
next.lease.member.state.set_healthy_for_test(false);
|
|
assert!(pool
|
|
.select("127.0.0.1".parse().unwrap(), &headers, false)
|
|
.is_none());
|
|
}
|
|
#[tokio::test]
|
|
async fn slow_start_ramps_weight_and_error_is_counted_once_per_exchange() {
|
|
let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
|
|
let selected = select(&pool);
|
|
let state = &selected.lease.member.state;
|
|
assert!(state.weight(100, 100) < 100.0);
|
|
assert_eq!(state.weight(100, 0), 100.0);
|
|
selected.lease.started();
|
|
selected.lease.bytes(50, true);
|
|
selected.lease.bytes(100, false);
|
|
selected.lease.error("timeout", Duration::from_secs(1));
|
|
selected.lease.error("body", Duration::from_secs(2));
|
|
selected.lease.retry_attempted();
|
|
selected.lease.retry_succeeded();
|
|
assert_eq!(state.requests.load(Ordering::Relaxed), 1);
|
|
assert_eq!(state.errors.load(Ordering::Relaxed), 1);
|
|
assert_eq!(state.timeouts_total.load(Ordering::Relaxed), 1);
|
|
assert_eq!(state.io_errors_total.load(Ordering::Relaxed), 1);
|
|
assert_eq!(state.bytes_in.load(Ordering::Relaxed), 50);
|
|
assert_eq!(state.bytes_out.load(Ordering::Relaxed), 100);
|
|
assert_eq!(state.retry_attempts_total.load(Ordering::Relaxed), 1);
|
|
assert_eq!(state.retry_successes_total.load(Ordering::Relaxed), 1);
|
|
assert!(pool
|
|
.prometheus()
|
|
.contains("rust_proxy_backend_requests_total{vhost=\"app\",backend=\"a\"} 1"));
|
|
assert!(pool
|
|
.prometheus()
|
|
.contains("rust_proxy_backend_retry_attempts_total{vhost=\"app\",backend=\"a\"} 1"));
|
|
}
|
|
#[tokio::test]
|
|
async fn slow_5xx_counts_both_observability_classes_when_passive_is_off() {
|
|
let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
|
|
let selected = select(&pool);
|
|
selected.lease.response(500, Duration::from_secs(20), true);
|
|
let state = &selected.lease.member.state;
|
|
assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 1);
|
|
assert_eq!(state.slow_responses_total.load(Ordering::Relaxed), 1);
|
|
assert_eq!(
|
|
state.passive_health_failures_total.load(Ordering::Relaxed),
|
|
0
|
|
);
|
|
assert!(state.healthy());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn client_upload_duration_does_not_trigger_passive_slow_failure() {
|
|
let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
|
|
let upload = select(&pool);
|
|
upload.lease.response(200, Duration::from_secs(20), false);
|
|
assert_eq!(upload.lease.member.state.errors.load(Ordering::Relaxed), 0);
|
|
let read = select(&pool);
|
|
read.lease.response(200, Duration::from_secs(20), true);
|
|
assert_eq!(read.lease.member.state.errors.load(Ordering::Relaxed), 1);
|
|
}
|
|
#[tokio::test]
|
|
async fn backup_is_only_new_assignment_when_no_primary_is_eligible() {
|
|
let mut pool = pool(Algorithm::RoundRobin, StickyMode::Off);
|
|
Arc::get_mut(&mut Arc::get_mut(&mut pool).unwrap().members[1])
|
|
.unwrap()
|
|
.definition
|
|
.backup = true;
|
|
for _ in 0..8 {
|
|
assert_eq!(select(&pool).lease.member.definition.id, "a");
|
|
}
|
|
pool.members[0].state.set_healthy_for_test(false);
|
|
assert_eq!(select(&pool).lease.member.definition.id, "b");
|
|
pool.members[0].state.set_healthy_for_test(true);
|
|
assert_eq!(select(&pool).lease.member.definition.id, "a");
|
|
}
|
|
#[tokio::test]
|
|
async fn drain_rejects_new_sessions_but_preserves_valid_existing_affinity() {
|
|
let mut pool = pool(Algorithm::RoundRobin, StickyMode::Cookie);
|
|
Arc::get_mut(&mut Arc::get_mut(&mut pool).unwrap().members[0])
|
|
.unwrap()
|
|
.definition
|
|
.drain = true;
|
|
assert_eq!(select(&pool).lease.member.definition.id, "b");
|
|
let mut headers = HeaderMap::new();
|
|
headers.insert(
|
|
"cookie",
|
|
format!("{}={}", pool.cookie_name, pool.members[0].token)
|
|
.parse()
|
|
.unwrap(),
|
|
);
|
|
let existing = pool
|
|
.select("127.0.0.1".parse().unwrap(), &headers, false)
|
|
.unwrap();
|
|
assert_eq!(existing.lease.member.definition.id, "a");
|
|
pool.members[0].state.set_healthy_for_test(false);
|
|
assert_eq!(
|
|
pool.select("127.0.0.1".parse().unwrap(), &headers, false)
|
|
.unwrap()
|
|
.lease
|
|
.member
|
|
.definition
|
|
.id,
|
|
"b"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn passive_ejection_remembers_first_real_error_and_trigger() {
|
|
let state = BackendState::new(HealthMode::Off);
|
|
let active = settings(HealthMode::Off);
|
|
let passive = PassiveHealth {
|
|
enabled: true,
|
|
fall: Some(2),
|
|
..Default::default()
|
|
};
|
|
let elapsed = Duration::from_millis(3);
|
|
assert!(state
|
|
.observe_passive_with_detail(
|
|
&active,
|
|
&passive,
|
|
"connect_error",
|
|
"dns",
|
|
"first lookup failed",
|
|
elapsed
|
|
)
|
|
.is_none());
|
|
let (first, failures) = state
|
|
.observe_passive_with_detail(
|
|
&active,
|
|
&passive,
|
|
"connect_error",
|
|
"connect",
|
|
"second TCP failed",
|
|
elapsed,
|
|
)
|
|
.expect("the threshold crossing must return the first failure");
|
|
assert_eq!(failures, 2);
|
|
assert_eq!(first.class, "dns");
|
|
assert_eq!(first.detail, "first lookup failed");
|
|
assert!(!state.healthy());
|
|
assert!(
|
|
state
|
|
.observe_passive_with_detail(
|
|
&active,
|
|
&passive,
|
|
"connect_error",
|
|
"connect",
|
|
"third TCP failed",
|
|
elapsed
|
|
)
|
|
.is_none(),
|
|
"the ejection must be logged only once"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn passive_io_burst_is_one_observation_not_many_failed_connections() {
|
|
let state = BackendState::new(HealthMode::Off);
|
|
let active = settings(HealthMode::Off);
|
|
let passive = PassiveHealth {
|
|
enabled: true,
|
|
fall: Some(3),
|
|
..Default::default()
|
|
};
|
|
for _ in 0..20 {
|
|
state.observe_passive_with_detail(
|
|
&active,
|
|
&passive,
|
|
"io_error",
|
|
"io_error",
|
|
"connection closed",
|
|
Duration::from_millis(1),
|
|
);
|
|
}
|
|
assert!(state.healthy());
|
|
assert_eq!(state.io_errors_total.load(Ordering::Relaxed), 20);
|
|
assert_eq!(
|
|
state.passive_health_failures_total.load(Ordering::Relaxed),
|
|
1
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn passive_only_quarantine_expires_without_vhost_reload() {
|
|
let state = BackendState::new(HealthMode::Off);
|
|
let active = settings(HealthMode::Off);
|
|
let passive = PassiveHealth {
|
|
enabled: true,
|
|
fall: Some(1),
|
|
..Default::default()
|
|
};
|
|
state.observe_passive(&active, &passive, "connect_error", Duration::ZERO);
|
|
assert!(!state.healthy());
|
|
lock(&state.health).passive_ejected_at = Some(Instant::now() - Duration::from_secs(10));
|
|
assert!(
|
|
state.healthy(),
|
|
"a half-open traffic attempt must become possible"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn active_probe_successes_recover_a_passively_ejected_backend() {
|
|
let state = BackendState::new(HealthMode::Http);
|
|
let active = HealthCheck {
|
|
fall: 1,
|
|
rise: 1,
|
|
..settings(HealthMode::Http)
|
|
};
|
|
let passive = PassiveHealth {
|
|
enabled: true,
|
|
fall: Some(1),
|
|
..Default::default()
|
|
};
|
|
state.observe_active(&active, "ok", Duration::ZERO);
|
|
assert!(state.healthy());
|
|
state.observe_passive(&active, &passive, "connect_error", Duration::ZERO);
|
|
assert!(!state.healthy());
|
|
state.observe_active(&active, "ok", Duration::ZERO);
|
|
assert!(!state.healthy());
|
|
state.observe_active(&active, "ok", Duration::ZERO);
|
|
assert!(state.healthy());
|
|
}
|
|
|
|
#[test]
|
|
fn single_backend_fast_path_observes_capacity_and_health_without_affinity_cookie() {
|
|
let pool = capacity_pool(1, 0);
|
|
assert_eq!(pool.members.len(), 1);
|
|
let peer = "127.0.0.1".parse().unwrap();
|
|
let headers = HeaderMap::new();
|
|
let first = pool.select(peer, &headers, false).expect("healthy backend");
|
|
assert!(first.cookie.is_none());
|
|
assert!(pool.select(peer, &headers, false).is_none());
|
|
drop(first);
|
|
pool.members[0].state.set_healthy_for_test(false);
|
|
assert!(pool.select(peer, &headers, false).is_none());
|
|
pool.members[0].state.set_healthy_for_test(true);
|
|
assert!(pool.select(peer, &headers, false).is_some());
|
|
}
|
|
|
|
#[test]
|
|
fn leases_reuse_immutable_backend_health_policies() {
|
|
let pool = capacity_pool(0, 0);
|
|
let first = select(&pool);
|
|
let second = select(&pool);
|
|
assert!(Arc::ptr_eq(&first.lease.health, &second.lease.health));
|
|
assert!(Arc::ptr_eq(
|
|
&first.lease.passive_health,
|
|
&second.lease.passive_health
|
|
));
|
|
}
|
|
|
|
#[test]
|
|
fn healthy_fast_path_tracks_active_and_passive_ejections() {
|
|
let state = BackendState::new(HealthMode::Off);
|
|
assert!(state.healthy());
|
|
let active = settings(HealthMode::Off);
|
|
let passive = PassiveHealth {
|
|
enabled: true,
|
|
fall: Some(1),
|
|
rise: Some(1),
|
|
..Default::default()
|
|
};
|
|
state.observe_passive(&active, &passive, "connect_error", Duration::from_millis(2));
|
|
assert!(!state.healthy());
|
|
state.observe_passive(&active, &passive, "ok", Duration::from_millis(2));
|
|
assert!(state.healthy());
|
|
}
|
|
|
|
#[test]
|
|
fn successful_lease_response_keeps_passive_observability_and_failure_detail() {
|
|
let pool = capacity_pool(0, 0);
|
|
let selection = select(&pool);
|
|
selection
|
|
.lease
|
|
.response(200, Duration::from_millis(5), true);
|
|
let state = &selection.lease.member.state;
|
|
assert_eq!(state.last_status.load(Ordering::Relaxed), 200);
|
|
assert_eq!(lock(&state.health).passive_result, "ok");
|
|
assert!(lock(&state.health).latency_us > 0);
|
|
selection
|
|
.lease
|
|
.response(503, Duration::from_millis(5), true);
|
|
assert_eq!(state.last_status.load(Ordering::Relaxed), 503);
|
|
assert_eq!(lock(&state.last_failure).detail, "status=503");
|
|
}
|
|
|
|
fn health_port_config(backend_port: u16, health: serde_json::Value) -> crate::config::Config {
|
|
serde_json::from_value(serde_json::json!({
|
|
"listeners":{"endpoints":[{"id":"http","binding":{"address":"127.0.0.1:8080"}}]},
|
|
"vhosts":[{
|
|
"id":"app","frontend":{"hosts":["app.localhost"],"listener_ids":["http"]},
|
|
"backends":[{"id":"a","url":format!("http://127.0.0.1:{backend_port}"),"allow_private":true}],
|
|
"load_balancing":{"health_check":health}
|
|
}]})).unwrap()
|
|
}
|
|
fn health_port_policy() -> crate::config::ValidationPolicy {
|
|
crate::config::ValidationPolicy {
|
|
proxy_addr: "127.0.0.1:8080".parse().unwrap(),
|
|
admin_addr: "127.0.0.1:9090".parse().unwrap(),
|
|
tls_addr: None,
|
|
cert_dir: None,
|
|
data_dir: None,
|
|
}
|
|
}
|
|
#[test]
|
|
fn health_check_port_compiles_separate_probe_target() {
|
|
let policy = health_port_policy();
|
|
let snapshot = crate::config::compile(
|
|
health_port_config(9000, serde_json::json!({"mode":"tcp","port":8079})), 0, &policy).unwrap();
|
|
let backend = &snapshot.pools[0].members[0].backend;
|
|
assert_eq!(backend.origin, "http://127.0.0.1:9000");
|
|
assert_eq!(backend.health_probe.as_ref().unwrap().origin, "http://127.0.0.1:8079");
|
|
for health in [
|
|
serde_json::json!({"mode":"tcp","port":9000}),
|
|
serde_json::json!({"mode":"tcp"}),
|
|
serde_json::json!({"mode":"off","port":8079}),
|
|
] {
|
|
let snapshot = crate::config::compile(health_port_config(9000, health), 0, &policy).unwrap();
|
|
assert!(snapshot.pools[0].members[0].backend.health_probe.is_none());
|
|
}
|
|
assert!(crate::config::compile(
|
|
health_port_config(9000, serde_json::json!({"mode":"tcp","port":0})), 0, &policy).is_err());
|
|
assert!(crate::config::compile(
|
|
health_port_config(9000, serde_json::json!({"mode":"http","port":9090})), 0, &policy).is_err());
|
|
}
|
|
#[tokio::test]
|
|
async fn health_check_probes_configured_port_instead_of_traffic_port() {
|
|
let traffic = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let traffic_port = traffic.local_addr().unwrap().port();
|
|
let closed_port = {
|
|
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
|
listener.local_addr().unwrap().port()
|
|
};
|
|
let policy = health_port_policy();
|
|
let config = health_port_config(traffic_port, serde_json::json!({
|
|
"mode":"tcp","port":closed_port,"fall":1,"rise":1,"interval_ms":250,"timeout_ms":200,"slow_response_ms":0}));
|
|
let snapshot = crate::config::compile(config, 0, &policy).unwrap();
|
|
let member = snapshot.pools[0].members[0].clone();
|
|
let settings = member.health_settings.clone();
|
|
probe(member.clone(), settings.clone()).await;
|
|
assert!(!member.state.healthy(), "probe must use the closed health port");
|
|
|
|
let health = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let health_port = health.local_addr().unwrap().port();
|
|
drop(traffic);
|
|
let config = health_port_config(traffic_port, serde_json::json!({
|
|
"mode":"tcp","port":health_port,"fall":1,"rise":1,"interval_ms":250,"timeout_ms":200,"slow_response_ms":0}));
|
|
let snapshot = crate::config::compile(config, 0, &policy).unwrap();
|
|
let member = snapshot.pools[0].members[0].clone();
|
|
let settings = member.health_settings.clone();
|
|
probe(member.clone(), settings).await;
|
|
assert!(member.state.healthy(), "probe must succeed on the open health port while traffic port is closed");
|
|
tokio::time::timeout(Duration::from_secs(1), health.accept())
|
|
.await
|
|
.expect("health port must receive the probe connection")
|
|
.unwrap();
|
|
}
|
|
|
|
async fn one_shot_http(body: &'static str) -> (u16, tokio::task::JoinHandle<()>) {
|
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let port = listener.local_addr().unwrap().port();
|
|
let task = tokio::spawn(async move {
|
|
while let Ok((mut socket, _)) = listener.accept().await {
|
|
let mut buffer = [0u8; 2048];
|
|
let _ = socket.read(&mut buffer).await;
|
|
let response = format!(
|
|
"HTTP/1.1 200 OK\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
|
|
body.len(), body
|
|
);
|
|
let _ = socket.write_all(response.as_bytes()).await;
|
|
}
|
|
});
|
|
(port, task)
|
|
}
|
|
#[tokio::test]
|
|
async fn http_health_check_requires_expected_body_text() {
|
|
let policy = health_port_policy();
|
|
for (body, healthy) in [("{\"status\":\"ok\"}", true), ("{\"status\":\"degraded\"}", false)] {
|
|
let (port, server) = one_shot_http(body).await;
|
|
let config = health_port_config(port, serde_json::json!({
|
|
"mode":"http","path":"/health","expect_body":"\"status\":\"ok\"",
|
|
"fall":1,"rise":1,"interval_ms":1000,"timeout_ms":800,"slow_response_ms":0}));
|
|
let snapshot = crate::config::compile(config, 0, &policy).unwrap();
|
|
let member = snapshot.pools[0].members[0].clone();
|
|
probe(member.clone(), member.health_settings.clone()).await;
|
|
assert_eq!(member.state.healthy(), healthy, "body {body}");
|
|
if !healthy {
|
|
assert_eq!(lock(&member.state.health).result, "http_body");
|
|
}
|
|
server.abort();
|
|
}
|
|
for invalid in ["", "line\nbreak"] {
|
|
assert!(crate::config::compile(
|
|
health_port_config(9000, serde_json::json!({"mode":"http","expect_body":invalid})), 0, &policy).is_err());
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn health_check_port_survives_live_reload_with_reused_clients() {
|
|
let policy = health_port_policy();
|
|
let defaults = (policy.proxy_addr, policy.tls_addr);
|
|
let first = crate::config::compile_reusing(
|
|
health_port_config(9000, serde_json::json!({"mode":"tcp"})), 0, &policy, defaults, None).unwrap();
|
|
assert!(first.pools[0].members[0].health_probe.is_none());
|
|
let second = crate::config::compile_reusing(
|
|
health_port_config(9000, serde_json::json!({"mode":"tcp","port":8079})), 1, &policy, defaults, Some(&first)).unwrap();
|
|
let member = &second.pools[0].members[0];
|
|
assert!(Arc::ptr_eq(&member.backend, &first.pools[0].members[0].backend), "traffic clients are reused");
|
|
assert_eq!(member.health_probe.as_ref().unwrap().origin, "http://127.0.0.1:8079");
|
|
let third = crate::config::compile_reusing(
|
|
health_port_config(9000, serde_json::json!({"mode":"tcp"})), 2, &policy, defaults, Some(&second)).unwrap();
|
|
assert!(third.pools[0].members[0].health_probe.is_none());
|
|
}
|
|
}
|