gru/RustProxy
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.25.5

Browse Source
This commit is contained in:
Mateusz Gruszczyński committed 2026-10-05 09:30:21 +02:00
1 parent 1dea7f55c8
commit c819f0a335
151 files changed
+7610 -4264
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/balance/tests/part_01.rs
+379 -369
View File
@@ -1,14 +1,14 @@
use super::*;
fn settings(mode: HealthMode) -> HealthCheck {
HealthCheck {
mode,
fall: 2,
rise: 2,
..Default::default()
}
use super::*;
fn settings(mode: HealthMode) -> HealthCheck {
HealthCheck {
mode,
fall: 2,
rise: 2,
..Default::default()
}
fn pool(algorithm: Algorithm, sticky: StickyMode) -> Arc<Pool> {
let cfg = serde_json::from_value(serde_json::json!({
}
fn pool(algorithm: Algorithm, sticky: StickyMode) -> Arc<Pool> {
let cfg = serde_json::from_value(serde_json::json!({
"listeners":{"endpoints":[{"id":"http","binding":{"address":"127.0.0.1:8080"}}]},
"vhosts":[{
"id":"app","frontend":{"hosts":["app.localhost"],"listener_ids":["http"]},
@@ -16,378 +16,388 @@
{"id":"b","url":"http://127.0.0.1:9001","allow_private":true,"weight":1}],
"load_balancing":{"algorithm":algorithm,"sticky":{"mode":sticky},"slow_start_seconds":0,"health_check":{"mode":"off"}}
}]})).unwrap();
let policy = crate::config::ValidationPolicy {
proxy_addr: "127.0.0.1:8080".parse().unwrap(),
admin_addr: "127.0.0.1:9090".parse().unwrap(),
tls_addr: None,
cert_dir: None,
data_dir: None,
};
crate::config::compile(cfg, 0, &policy).unwrap().pools[0].clone()
}
fn capacity_pool(limit: u32, queue_timeout_ms: u64) -> Arc<Pool> {
let cfg = serde_json::from_value(serde_json::json!({
"listeners":{"endpoints":[{"id":"http","binding":{"address":"127.0.0.1:8080"}}]},
"vhosts":[{
"id":"limited","frontend":{"hosts":["limited.localhost"],"listener_ids":["http"]},
"backends":[{
"id":"only","url":"http://127.0.0.1:9000","allow_private":true,
"max_concurrent_requests":limit,"queue_timeout_ms":queue_timeout_ms
}],
"load_balancing":{"health_check":{"mode":"off"}}
}]
})).unwrap();
let policy = crate::config::ValidationPolicy {
proxy_addr: "127.0.0.1:8080".parse().unwrap(),
admin_addr: "127.0.0.1:9090".parse().unwrap(),
tls_addr: None,
cert_dir: None,
data_dir: None,
};
crate::config::compile(cfg, 0, &policy).unwrap().pools[0].clone()
}
fn select(pool: &Pool) -> Selection {
pool.select("127.0.0.1".parse().unwrap(), &HeaderMap::new(), false)
.unwrap()
}
#[test]
fn default_health_check_uses_tcp() {
assert_eq!(HealthMode::default(), HealthMode::Tcp);
assert_eq!(HealthCheck::default().mode, HealthMode::Tcp);
}
#[test]
fn legacy_settings_without_passive_health_default_to_off() {
let parsed: Settings = serde_json::from_value(serde_json::json!({
"health_check": { "mode": "off" }
}))
.unwrap();
assert!(!parsed.passive_health.enabled);
assert!(parsed.passive_health.count_connect_errors);
assert!(parsed.passive_health.count_timeouts);
assert!(parsed.passive_health.count_io_errors);
assert!(parsed.passive_health.count_http_5xx);
assert!(!parsed.passive_health.count_slow_responses);
assert_eq!(parsed.retry.max_retries, 1);
assert!(parsed.retry.retry_connect_errors);
assert!(parsed.retry.retry_timeouts);
assert_eq!(parsed.retry.retry_statuses, vec![502, 503, 504]);
}
let policy = crate::config::ValidationPolicy {
proxy_addr: "127.0.0.1:8080".parse().unwrap(),
admin_addr: "127.0.0.1:9090".parse().unwrap(),
tls_addr: None,
cert_dir: None,
data_dir: None,
};
crate::config::compile(cfg, 0, &policy).unwrap().pools[0].clone()
}
fn capacity_pool(limit: u32, queue_timeout_ms: u64) -> Arc<Pool> {
let cfg = serde_json::from_value(serde_json::json!({
"listeners":{"endpoints":[{"id":"http","binding":{"address":"127.0.0.1:8080"}}]},
"vhosts":[{
"id":"limited","frontend":{"hosts":["limited.localhost"],"listener_ids":["http"]},
"backends":[{
"id":"only","url":"http://127.0.0.1:9000","allow_private":true,
"max_concurrent_requests":limit,"queue_timeout_ms":queue_timeout_ms
}],
"load_balancing":{"health_check":{"mode":"off"}}
}]
}))
.unwrap();
let policy = crate::config::ValidationPolicy {
proxy_addr: "127.0.0.1:8080".parse().unwrap(),
admin_addr: "127.0.0.1:9090".parse().unwrap(),
tls_addr: None,
cert_dir: None,
data_dir: None,
};
crate::config::compile(cfg, 0, &policy).unwrap().pools[0].clone()
}
fn select(pool: &Pool) -> Selection {
pool.select("127.0.0.1".parse().unwrap(), &HeaderMap::new(), false)
.unwrap()
}
#[test]
fn default_health_check_uses_tcp() {
assert_eq!(HealthMode::default(), HealthMode::Tcp);
assert_eq!(HealthCheck::default().mode, HealthMode::Tcp);
}
#[test]
fn legacy_settings_without_passive_health_default_to_off() {
let parsed: Settings = serde_json::from_value(serde_json::json!({
"health_check": { "mode": "off" }
}))
.unwrap();
assert!(!parsed.passive_health.enabled);
assert!(parsed.passive_health.count_connect_errors);
assert!(parsed.passive_health.count_timeouts);
assert!(parsed.passive_health.count_io_errors);
assert!(parsed.passive_health.count_http_5xx);
assert!(!parsed.passive_health.count_slow_responses);
assert_eq!(parsed.retry.max_retries, 1);
assert!(parsed.retry.retry_connect_errors);
assert!(parsed.retry.retry_timeouts);
assert_eq!(parsed.retry.retry_statuses, vec![502, 503, 504]);
}
#[test]
fn retry_policy_rejects_unbounded_or_non_5xx_configuration() {
let mut settings = Settings::default();
settings.retry.max_retries = 4;
assert!(settings.validate().is_err());
settings.retry.max_retries = 1;
settings.retry.retry_statuses = vec![429];
assert!(settings.validate().is_err());
}
#[test]
fn retry_policy_rejects_unbounded_or_non_5xx_configuration() {
let mut settings = Settings::default();
settings.retry.max_retries = 4;
assert!(settings.validate().is_err());
settings.retry.max_retries = 1;
settings.retry.retry_statuses = vec![429];
assert!(settings.validate().is_err());
}
#[test]
fn traffic_rate_drops_to_zero_after_an_idle_sample_window() {
let state = BackendState::new(HealthMode::Off);
state.bytes_out.store(2048, std::sync::atomic::Ordering::Relaxed);
lock(&state.traffic).at = std::time::Instant::now() - std::time::Duration::from_secs(2);
state.sample_traffic();
assert!(state.traffic_rate() > 0.0);
lock(&state.traffic).at = std::time::Instant::now() - std::time::Duration::from_secs(2);
state.sample_traffic();
assert_eq!(state.traffic_rate(), 0.0);
#[test]
fn traffic_rate_drops_to_zero_after_an_idle_sample_window() {
let state = BackendState::new(HealthMode::Off);
state
.bytes_out
.store(2048, std::sync::atomic::Ordering::Relaxed);
lock(&state.traffic).at = std::time::Instant::now() - std::time::Duration::from_secs(2);
state.sample_traffic();
assert!(state.traffic_rate() > 0.0);
lock(&state.traffic).at = std::time::Instant::now() - std::time::Duration::from_secs(2);
state.sample_traffic();
assert_eq!(state.traffic_rate(), 0.0);
}
#[test]
fn active_failures_are_independent_from_passive_user_traffic() {
let state = BackendState::new(HealthMode::Http);
let settings = settings(HealthMode::Http);
let passive = PassiveHealth::default();
let fast = Duration::from_millis(5);
assert!(!state.healthy());
state.observe_active(&settings, "ok", fast);
assert!(!state.healthy());
state.observe_active(&settings, "ok", fast);
assert!(state.healthy());
state.observe_active(&settings, "status", fast);
state.observe_passive(&settings, &passive, "ok", fast);
state.observe_active(&settings, "status", fast);
assert!(!state.healthy());
state.observe_passive(&settings, &passive, "http_5xx", fast);
assert!(!state.healthy());
state.observe_active(&settings, "ok", fast);
assert!(!state.healthy());
state.observe_active(&settings, "ok", fast);
assert!(state.healthy());
}
#[test]
fn passive_health_is_opt_in_and_uses_separate_thresholds() {
let state = BackendState::new(HealthMode::Off);
let active = settings(HealthMode::Off);
let fast = Duration::from_millis(5);
for _ in 0..10 {
state.observe_passive(&active, &PassiveHealth::default(), "http_5xx", fast);
}
#[test]
fn active_failures_are_independent_from_passive_user_traffic() {
let state = BackendState::new(HealthMode::Http);
let settings = settings(HealthMode::Http);
let passive = PassiveHealth::default();
let fast = Duration::from_millis(5);
assert!(!state.healthy());
state.observe_active(&settings, "ok", fast);
assert!(!state.healthy());
state.observe_active(&settings, "ok", fast);
assert!(state.healthy());
state.observe_active(&settings, "status", fast);
state.observe_passive(&settings, &passive, "ok", fast);
state.observe_active(&settings, "status", fast);
assert!(!state.healthy());
state.observe_passive(&settings, &passive, "http_5xx", fast);
assert!(!state.healthy());
state.observe_active(&settings, "ok", fast);
assert!(!state.healthy());
state.observe_active(&settings, "ok", fast);
assert!(state.healthy());
assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 10);
assert_eq!(
state.passive_health_failures_total.load(Ordering::Relaxed),
0
);
let passive = PassiveHealth {
enabled: true,
fall: Some(2),
rise: Some(2),
..Default::default()
};
state.observe_passive(&active, &passive, "http_5xx", fast);
assert!(state.healthy());
state.observe_passive(&active, &passive, "http_5xx", fast);
assert!(!state.healthy());
state.observe_passive(&active, &passive, "ok", fast);
assert!(!state.healthy());
state.observe_passive(&active, &passive, "ok", fast);
assert!(state.healthy());
}
#[test]
fn passive_http_5xx_can_be_explicitly_ignored_when_passive_health_is_on() {
let state = BackendState::new(HealthMode::Off);
let active = settings(HealthMode::Off);
let passive = PassiveHealth {
enabled: true,
count_http_5xx: false,
fall: Some(1),
..Default::default()
};
for _ in 0..100 {
state.observe_passive(&active, &passive, "http_5xx", Duration::from_millis(5));
}
assert!(state.healthy());
assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 100);
assert_eq!(
state.passive_health_failures_total.load(Ordering::Relaxed),
0
);
}
#[test]
fn passive_connect_errors_use_the_explicit_fall_threshold() {
let state = BackendState::new(HealthMode::Off);
let active = settings(HealthMode::Off);
let passive = PassiveHealth {
enabled: true,
count_connect_errors: true,
fall: Some(3),
..Default::default()
};
for _ in 0..2 {
state.observe_passive(&active, &passive, "connect_error", Duration::from_millis(5));
assert!(state.healthy());
}
state.observe_passive(&active, &passive, "connect_error", Duration::from_millis(5));
assert!(!state.healthy());
assert_eq!(state.connect_errors_total.load(Ordering::Relaxed), 3);
assert_eq!(
state.passive_health_failures_total.load(Ordering::Relaxed),
3
);
}
#[test]
fn passive_health_is_opt_in_and_uses_separate_thresholds() {
let state = BackendState::new(HealthMode::Off);
let active = settings(HealthMode::Off);
let fast = Duration::from_millis(5);
for _ in 0..10 {
state.observe_passive(&active, &PassiveHealth::default(), "http_5xx", fast);
}
assert!(state.healthy());
assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 10);
assert_eq!(state.passive_health_failures_total.load(Ordering::Relaxed), 0);
let passive = PassiveHealth {
enabled: true,
fall: Some(2),
rise: Some(2),
..Default::default()
};
#[test]
fn active_health_can_stay_healthy_while_passive_health_is_off() {
let state = BackendState::new(HealthMode::Tcp);
let active = settings(HealthMode::Tcp);
let passive = PassiveHealth::default();
let fast = Duration::from_millis(5);
state.observe_active(&active, "ok", fast);
state.observe_active(&active, "ok", fast);
assert!(state.healthy());
for _ in 0..20 {
state.observe_passive(&active, &passive, "http_5xx", fast);
assert!(state.healthy());
state.observe_passive(&active, &passive, "http_5xx", fast);
assert!(!state.healthy());
state.observe_passive(&active, &passive, "ok", fast);
assert!(!state.healthy());
state.observe_passive(&active, &passive, "ok", fast);
assert!(state.healthy());
}
#[test]
fn passive_http_5xx_can_be_explicitly_ignored_when_passive_health_is_on() {
let state = BackendState::new(HealthMode::Off);
let active = settings(HealthMode::Off);
let passive = PassiveHealth {
enabled: true,
count_http_5xx: false,
fall: Some(1),
..Default::default()
};
for _ in 0..100 {
state.observe_passive(
&active,
&passive,
"http_5xx",
Duration::from_millis(5),
);
}
assert!(state.healthy());
assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 100);
assert_eq!(state.passive_health_failures_total.load(Ordering::Relaxed), 0);
}
assert!(state.healthy());
assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 20);
assert_eq!(
state.passive_health_failures_total.load(Ordering::Relaxed),
0
);
}
#[test]
fn passive_connect_errors_use_the_explicit_fall_threshold() {
let state = BackendState::new(HealthMode::Off);
let active = settings(HealthMode::Off);
let passive = PassiveHealth {
enabled: true,
count_connect_errors: true,
fall: Some(3),
..Default::default()
};
for _ in 0..2 {
state.observe_passive(
&active,
&passive,
"connect_error",
Duration::from_millis(5),
);
assert!(state.healthy());
}
state.observe_passive(
&active,
&passive,
#[test]
fn unhealthy_source_hands_off_when_the_original_health_domain_recovers() {
let state = BackendState::new(HealthMode::Tcp);
let active = HealthCheck {
fall: 1,
rise: 1,
..settings(HealthMode::Tcp)
};
let passive = PassiveHealth {
enabled: true,
count_http_5xx: true,
fall: Some(1),
rise: Some(1),
..Default::default()
};
let fast = Duration::from_millis(5);
state.observe_active(&active, "ok", fast);
state.observe_active(&active, "connect_error", fast);
{
let health = lock(&state.health);
assert_eq!(health.unhealthy_source, "active_tcp");
}
state.observe_passive(&active, &passive, "http_5xx", fast);
state.observe_active(&active, "ok", fast);
{
let health = lock(&state.health);
assert!(!health.healthy);
assert_eq!(health.unhealthy_source, "passive_http_5xx");
assert_eq!(health.unhealthy_reason, "http_5xx");
}
state.observe_passive(&active, &passive, "ok", fast);
assert!(state.healthy());
}
#[test]
fn disabled_checks_never_quarantine_and_probes_are_single_flight() {
let off = BackendState::new(HealthMode::Off);
let settings = settings(HealthMode::Off);
for _ in 0..5 {
off.observe_passive(
&settings,
&PassiveHealth::default(),
"connect_error",
Duration::from_millis(5),
Duration::ZERO,
);
assert!(!state.healthy());
assert_eq!(state.connect_errors_total.load(Ordering::Relaxed), 3);
assert_eq!(state.passive_health_failures_total.load(Ordering::Relaxed), 3);
}
#[test]
fn active_health_can_stay_healthy_while_passive_health_is_off() {
let state = BackendState::new(HealthMode::Tcp);
let active = settings(HealthMode::Tcp);
let passive = PassiveHealth::default();
let fast = Duration::from_millis(5);
state.observe_active(&active, "ok", fast);
state.observe_active(&active, "ok", fast);
assert!(state.healthy());
for _ in 0..20 {
state.observe_passive(&active, &passive, "http_5xx", fast);
}
assert!(state.healthy());
assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 20);
assert_eq!(state.passive_health_failures_total.load(Ordering::Relaxed), 0);
}
#[test]
fn unhealthy_source_hands_off_when_the_original_health_domain_recovers() {
let state = BackendState::new(HealthMode::Tcp);
let active = HealthCheck {
fall: 1,
rise: 1,
..settings(HealthMode::Tcp)
};
let passive = PassiveHealth {
enabled: true,
count_http_5xx: true,
fall: Some(1),
rise: Some(1),
..Default::default()
};
let fast = Duration::from_millis(5);
state.observe_active(&active, "ok", fast);
state.observe_active(&active, "connect_error", fast);
{
let health = lock(&state.health);
assert_eq!(health.unhealthy_source, "active_tcp");
}
state.observe_passive(&active, &passive, "http_5xx", fast);
state.observe_active(&active, "ok", fast);
{
let health = lock(&state.health);
assert!(!health.healthy);
assert_eq!(health.unhealthy_source, "passive_http_5xx");
assert_eq!(health.unhealthy_reason, "http_5xx");
}
state.observe_passive(&active, &passive, "ok", fast);
assert!(state.healthy());
}
#[test]
fn disabled_checks_never_quarantine_and_probes_are_single_flight() {
let off = BackendState::new(HealthMode::Off);
let settings = settings(HealthMode::Off);
for _ in 0..5 {
off.observe_passive(&settings, &PassiveHealth::default(), "connect_error", Duration::ZERO);
}
assert!(off.healthy());
assert!(!off.claim_probe(&settings));
let on = BackendState::new(HealthMode::Http);
let settings = super::tests::settings(HealthMode::Http);
assert!(on.claim_probe(&settings));
assert!(!on.claim_probe(&settings));
on.observe_active(&settings, "ok", Duration::ZERO);
assert!(!on.claim_probe(&settings));
}
#[tokio::test]
async fn weighted_round_robin_distributes_three_to_one_without_body_collection() {
let pool = pool(Algorithm::RoundRobin, StickyMode::Off);
let mut counts = [0, 0];
for _ in 0..40 {
let selection = select(&pool);
counts[usize::from(selection.lease.member.definition.id == "b")] += 1;
}
assert_eq!(counts, [30, 10]);
assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 0);
assert_eq!(pool.members[0].state.requests.load(Ordering::Relaxed), 0); // Mere cache selection isn't origin traffic.
}
#[tokio::test]
async fn least_connections_counts_a_lease_until_its_last_owner_drops() {
let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
let mut held = Vec::new();
for _ in 0..3 {
let selection = select(&pool);
assert_eq!(selection.lease.member.definition.id, "a");
held.push(selection.lease);
}
let other = select(&pool);
assert_eq!(other.lease.member.definition.id, "b");
let clone = held[0].clone();
held.clear();
assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 1);
drop(clone);
assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 0);
}
#[tokio::test]
async fn failover_selection_never_reuses_an_attempted_backend() {
let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
let first = select(&pool);
let first_id = first.lease.member.definition.id.clone();
drop(first);
let retry = pool
.select_retry(
"127.0.0.1".parse().unwrap(),
&HeaderMap::new(),
false,
std::slice::from_ref(&first_id),
)
.unwrap();
assert_ne!(retry.lease.member.definition.id, first_id);
}
#[tokio::test]
async fn backend_concurrency_limit_queues_then_rejects_and_recovers() {
let pool = capacity_pool(1, 10);
let first = select(&pool);
let headers = HeaderMap::new();
let saturated = pool
.select_wait("127.0.0.1".parse().unwrap(), &headers, false)
.await;
assert!(matches!(saturated, Err(SelectionFailure::Capacity)));
let view = pool.view();
let member = &view["members"][0];
assert_eq!(member["capacity_rejections_total"], 1);
assert_eq!(member["queue_timeouts_total"], 1);
assert_eq!(member["active"], 1);
drop(first);
assert!(pool
.select_wait("127.0.0.1".parse().unwrap(), &headers, false)
.await
.is_ok());
}
#[tokio::test]
async fn zero_capacity_queue_timeout_rejects_without_counting_a_queue_timeout() {
let pool = capacity_pool(1, 0);
let first = select(&pool);
let headers = HeaderMap::new();
let saturated = pool
.select_wait("127.0.0.1".parse().unwrap(), &headers, false)
.await;
assert!(matches!(saturated, Err(SelectionFailure::Capacity)));
let view = pool.view();
let member = &view["members"][0];
assert_eq!(member["capacity_rejections_total"], 1);
assert_eq!(member["queue_timeouts_total"], 0);
drop(first);
}
#[test]
fn detailed_backend_failures_are_exposed_without_losing_aggregate_counters() {
let pool = capacity_pool(0, 0);
assert!(off.healthy());
assert!(!off.claim_probe(&settings));
let on = BackendState::new(HealthMode::Http);
let settings = super::tests::settings(HealthMode::Http);
assert!(on.claim_probe(&settings));
assert!(!on.claim_probe(&settings));
on.observe_active(&settings, "ok", Duration::ZERO);
assert!(!on.claim_probe(&settings));
}
#[tokio::test]
async fn weighted_round_robin_distributes_three_to_one_without_body_collection() {
let pool = pool(Algorithm::RoundRobin, StickyMode::Off);
let mut counts = [0, 0];
for _ in 0..40 {
let selection = select(&pool);
selection
.lease
.error_detail("dns", "lookup backend.example failed", Duration::from_millis(7));
let view = pool.view();
let member = &view["members"][0];
assert_eq!(member["dns_errors_total"], 1);
assert_eq!(member["connect_errors_total"], 1);
assert_eq!(member["last_failure_class"], "dns");
assert_eq!(member["last_failure_detail"], "lookup backend.example failed");
assert!(member["last_failure_at_ms"].as_u64().unwrap_or_default() > 0);
counts[usize::from(selection.lease.member.definition.id == "b")] += 1;
}
#[tokio::test]
async fn traffic_and_adaptive_selection_use_observed_cost_and_weight() {
for algorithm in [Algorithm::LeastTraffic, Algorithm::Adaptive] {
let pool = pool(algorithm, StickyMode::Off);
lock(&pool.members[0].state.traffic).rate = 50_000_000.0;
lock(&pool.members[0].state.health).ewma_us = 1_000_000;
assert_eq!(select(&pool).lease.member.definition.id, "b");
}
}
#[tokio::test]
async fn ip_hash_is_stable_and_cannot_be_spoofed_with_forwarded_headers() {
let pool = pool(Algorithm::IpHash, StickyMode::Off);
let original = select(&pool).lease.member.definition.id.clone();
let mut headers = HeaderMap::new();
headers.insert("x-forwarded-for", "10.0.0.99".parse().unwrap());
for ip in ["127.0.0.1", "::ffff:127.0.0.1"] {
for _ in 0..8 {
assert_eq!(
pool.select(ip.parse().unwrap(), &headers, false)
.unwrap()
.lease
.member
.definition
.id,
original
);
}
assert_eq!(counts, [30, 10]);
assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 0);
assert_eq!(pool.members[0].state.requests.load(Ordering::Relaxed), 0); // Mere cache selection isn't origin traffic.
}
#[tokio::test]
async fn least_connections_counts_a_lease_until_its_last_owner_drops() {
let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
let mut held = Vec::new();
for _ in 0..3 {
let selection = select(&pool);
assert_eq!(selection.lease.member.definition.id, "a");
held.push(selection.lease);
}
let other = select(&pool);
assert_eq!(other.lease.member.definition.id, "b");
let clone = held[0].clone();
held.clear();
assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 1);
drop(clone);
assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 0);
}
#[tokio::test]
async fn failover_selection_never_reuses_an_attempted_backend() {
let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
let first = select(&pool);
let first_id = first.lease.member.definition.id.clone();
drop(first);
let retry = pool
.select_retry(
"127.0.0.1".parse().unwrap(),
&HeaderMap::new(),
false,
std::slice::from_ref(&first_id),
)
.unwrap();
assert_ne!(retry.lease.member.definition.id, first_id);
}
#[tokio::test]
async fn backend_concurrency_limit_queues_then_rejects_and_recovers() {
let pool = capacity_pool(1, 10);
let first = select(&pool);
let headers = HeaderMap::new();
let saturated = pool
.select_wait("127.0.0.1".parse().unwrap(), &headers, false)
.await;
assert!(matches!(saturated, Err(SelectionFailure::Capacity)));
let view = pool.view();
let member = &view["members"][0];
assert_eq!(member["capacity_rejections_total"], 1);
assert_eq!(member["queue_timeouts_total"], 1);
assert_eq!(member["active"], 1);
drop(first);
assert!(pool
.select_wait("127.0.0.1".parse().unwrap(), &headers, false)
.await
.is_ok());
}
#[tokio::test]
async fn zero_capacity_queue_timeout_rejects_without_counting_a_queue_timeout() {
let pool = capacity_pool(1, 0);
let first = select(&pool);
let headers = HeaderMap::new();
let saturated = pool
.select_wait("127.0.0.1".parse().unwrap(), &headers, false)
.await;
assert!(matches!(saturated, Err(SelectionFailure::Capacity)));
let view = pool.view();
let member = &view["members"][0];
assert_eq!(member["capacity_rejections_total"], 1);
assert_eq!(member["queue_timeouts_total"], 0);
drop(first);
}
#[test]
fn detailed_backend_failures_are_exposed_without_losing_aggregate_counters() {
let pool = capacity_pool(0, 0);
let selection = select(&pool);
selection.lease.error_detail(
"dns",
"lookup backend.example failed",
Duration::from_millis(7),
);
let view = pool.view();
let member = &view["members"][0];
assert_eq!(member["dns_errors_total"], 1);
assert_eq!(member["connect_errors_total"], 1);
assert_eq!(member["last_failure_class"], "dns");
assert_eq!(
member["last_failure_detail"],
"lookup backend.example failed"
);
assert!(member["last_failure_at_ms"].as_u64().unwrap_or_default() > 0);
}
#[tokio::test]
async fn traffic_and_adaptive_selection_use_observed_cost_and_weight() {
for algorithm in [Algorithm::LeastTraffic, Algorithm::Adaptive] {
let pool = pool(algorithm, StickyMode::Off);
lock(&pool.members[0].state.traffic).rate = 50_000_000.0;
lock(&pool.members[0].state.health).ewma_us = 1_000_000;
assert_eq!(select(&pool).lease.member.definition.id, "b");
}
}
#[tokio::test]
async fn ip_hash_is_stable_and_cannot_be_spoofed_with_forwarded_headers() {
let pool = pool(Algorithm::IpHash, StickyMode::Off);
let original = select(&pool).lease.member.definition.id.clone();
let mut headers = HeaderMap::new();
headers.insert("x-forwarded-for", "10.0.0.99".parse().unwrap());
for ip in ["127.0.0.1", "::ffff:127.0.0.1"] {
for _ in 0..8 {
assert_eq!(
pool.select(ip.parse().unwrap(), &headers, false)
.unwrap()
.lease
.member
.definition
.id,
original
);
}
}
}
src/balance/tests/part_02.rs
+168 -165
View File
@@ -1,170 +1,173 @@
#[tokio::test]
async fn opaque_cookie_is_removed_from_origin_requests_and_protected_on_responses() {
let pool = pool(Algorithm::RoundRobin, StickyMode::Cookie);
let first = select(&pool);
let cookie = first.cookie.unwrap().to_str().unwrap().to_owned();
assert!(
cookie.contains("HttpOnly")
&& cookie.contains("SameSite=Lax")
&& !cookie.contains("Secure")
);
let cookie = cookie.split(';').next().unwrap();
let mut headers = HeaderMap::new();
headers.insert("cookie", format!("{cookie}; app=ok").parse().unwrap());
let next = pool
.select("10.0.0.1".parse().unwrap(), &headers, false)
.unwrap();
assert_eq!(
next.lease.member.definition.id,
first.lease.member.definition.id
);
assert!(next.cookie.is_none());
pool.strip_affinity(&mut headers);
assert_eq!(headers["cookie"], "app=ok");
headers.append(
"set-cookie",
format!("{}=origin-spoof", pool.cookie_name)
.parse()
.unwrap(),
);
headers.append("set-cookie", "app=ok".parse().unwrap());
pool.strip_affinity_response(&mut headers);
assert_eq!(headers.get_all("set-cookie").iter().count(), 1);
assert_eq!(headers["set-cookie"], "app=ok");
let secure = pool
.select("127.0.0.1".parse().unwrap(), &HeaderMap::new(), true)
.unwrap();
assert!(secure
.cookie
.unwrap()
.to_str()
.unwrap()
.contains("; Secure"));
}
#[tokio::test]
async fn failed_sticky_member_is_replaced_and_duplicate_cookies_are_ignored() {
let pool = pool(Algorithm::RoundRobin, StickyMode::Cookie);
let first = select(&pool);
let cookie = first
.cookie
.unwrap()
.to_str()
.unwrap()
.split(';')
.next()
.unwrap()
.to_owned();
let mut headers = HeaderMap::new();
headers.insert("cookie", format!("{cookie}; {cookie}").parse().unwrap());
assert!(pool.affinity(&headers).is_none());
headers.insert("cookie", cookie.parse().unwrap());
lock(&first.lease.member.state.health).healthy = false;
let next = pool
.select("127.0.0.1".parse().unwrap(), &headers, false)
.unwrap();
assert_ne!(
next.lease.member.definition.id,
first.lease.member.definition.id
);
assert!(next.cookie.is_some());
lock(&next.lease.member.state.health).healthy = false;
assert!(pool
.select("127.0.0.1".parse().unwrap(), &headers, false)
.is_none());
}
#[tokio::test]
async fn slow_start_ramps_weight_and_error_is_counted_once_per_exchange() {
let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
let selected = select(&pool);
let state = &selected.lease.member.state;
assert!(state.weight(100, 100) < 100.0);
assert_eq!(state.weight(100, 0), 100.0);
selected.lease.started();
selected.lease.bytes(50, true);
selected.lease.bytes(100, false);
selected.lease.error("timeout", Duration::from_secs(1));
selected.lease.error("body", Duration::from_secs(2));
selected.lease.retry_attempted();
selected.lease.retry_succeeded();
assert_eq!(state.requests.load(Ordering::Relaxed), 1);
assert_eq!(state.errors.load(Ordering::Relaxed), 1);
assert_eq!(state.timeouts_total.load(Ordering::Relaxed), 1);
assert_eq!(state.io_errors_total.load(Ordering::Relaxed), 1);
assert_eq!(state.bytes_in.load(Ordering::Relaxed), 50);
assert_eq!(state.bytes_out.load(Ordering::Relaxed), 100);
assert_eq!(state.retry_attempts_total.load(Ordering::Relaxed), 1);
assert_eq!(state.retry_successes_total.load(Ordering::Relaxed), 1);
assert!(pool
.prometheus()
.contains("rust_proxy_backend_requests_total{vhost=\"app\",backend=\"a\"} 1"));
assert!(pool
.prometheus()
.contains("rust_proxy_backend_retry_attempts_total{vhost=\"app\",backend=\"a\"} 1"));
}
#[tokio::test]
async fn slow_5xx_counts_both_observability_classes_when_passive_is_off() {
let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
let selected = select(&pool);
selected.lease.response(500, Duration::from_secs(20), true);
let state = &selected.lease.member.state;
assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 1);
assert_eq!(state.slow_responses_total.load(Ordering::Relaxed), 1);
assert_eq!(state.passive_health_failures_total.load(Ordering::Relaxed), 0);
assert!(state.healthy());
}
#[tokio::test]
async fn opaque_cookie_is_removed_from_origin_requests_and_protected_on_responses() {
let pool = pool(Algorithm::RoundRobin, StickyMode::Cookie);
let first = select(&pool);
let cookie = first.cookie.unwrap().to_str().unwrap().to_owned();
assert!(
cookie.contains("HttpOnly")
&& cookie.contains("SameSite=Lax")
&& !cookie.contains("Secure")
);
let cookie = cookie.split(';').next().unwrap();
let mut headers = HeaderMap::new();
headers.insert("cookie", format!("{cookie}; app=ok").parse().unwrap());
let next = pool
.select("10.0.0.1".parse().unwrap(), &headers, false)
.unwrap();
assert_eq!(
next.lease.member.definition.id,
first.lease.member.definition.id
);
assert!(next.cookie.is_none());
pool.strip_affinity(&mut headers);
assert_eq!(headers["cookie"], "app=ok");
headers.append(
"set-cookie",
format!("{}=origin-spoof", pool.cookie_name)
.parse()
.unwrap(),
);
headers.append("set-cookie", "app=ok".parse().unwrap());
pool.strip_affinity_response(&mut headers);
assert_eq!(headers.get_all("set-cookie").iter().count(), 1);
assert_eq!(headers["set-cookie"], "app=ok");
let secure = pool
.select("127.0.0.1".parse().unwrap(), &HeaderMap::new(), true)
.unwrap();
assert!(secure
.cookie
.unwrap()
.to_str()
.unwrap()
.contains("; Secure"));
}
#[tokio::test]
async fn failed_sticky_member_is_replaced_and_duplicate_cookies_are_ignored() {
let pool = pool(Algorithm::RoundRobin, StickyMode::Cookie);
let first = select(&pool);
let cookie = first
.cookie
.unwrap()
.to_str()
.unwrap()
.split(';')
.next()
.unwrap()
.to_owned();
let mut headers = HeaderMap::new();
headers.insert("cookie", format!("{cookie}; {cookie}").parse().unwrap());
assert!(pool.affinity(&headers).is_none());
headers.insert("cookie", cookie.parse().unwrap());
lock(&first.lease.member.state.health).healthy = false;
let next = pool
.select("127.0.0.1".parse().unwrap(), &headers, false)
.unwrap();
assert_ne!(
next.lease.member.definition.id,
first.lease.member.definition.id
);
assert!(next.cookie.is_some());
lock(&next.lease.member.state.health).healthy = false;
assert!(pool
.select("127.0.0.1".parse().unwrap(), &headers, false)
.is_none());
}
#[tokio::test]
async fn slow_start_ramps_weight_and_error_is_counted_once_per_exchange() {
let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
let selected = select(&pool);
let state = &selected.lease.member.state;
assert!(state.weight(100, 100) < 100.0);
assert_eq!(state.weight(100, 0), 100.0);
selected.lease.started();
selected.lease.bytes(50, true);
selected.lease.bytes(100, false);
selected.lease.error("timeout", Duration::from_secs(1));
selected.lease.error("body", Duration::from_secs(2));
selected.lease.retry_attempted();
selected.lease.retry_succeeded();
assert_eq!(state.requests.load(Ordering::Relaxed), 1);
assert_eq!(state.errors.load(Ordering::Relaxed), 1);
assert_eq!(state.timeouts_total.load(Ordering::Relaxed), 1);
assert_eq!(state.io_errors_total.load(Ordering::Relaxed), 1);
assert_eq!(state.bytes_in.load(Ordering::Relaxed), 50);
assert_eq!(state.bytes_out.load(Ordering::Relaxed), 100);
assert_eq!(state.retry_attempts_total.load(Ordering::Relaxed), 1);
assert_eq!(state.retry_successes_total.load(Ordering::Relaxed), 1);
assert!(pool
.prometheus()
.contains("rust_proxy_backend_requests_total{vhost=\"app\",backend=\"a\"} 1"));
assert!(pool
.prometheus()
.contains("rust_proxy_backend_retry_attempts_total{vhost=\"app\",backend=\"a\"} 1"));
}
#[tokio::test]
async fn slow_5xx_counts_both_observability_classes_when_passive_is_off() {
let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
let selected = select(&pool);
selected.lease.response(500, Duration::from_secs(20), true);
let state = &selected.lease.member.state;
assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 1);
assert_eq!(state.slow_responses_total.load(Ordering::Relaxed), 1);
assert_eq!(
state.passive_health_failures_total.load(Ordering::Relaxed),
0
);
assert!(state.healthy());
}
#[tokio::test]
async fn client_upload_duration_does_not_trigger_passive_slow_failure() {
let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
let upload = select(&pool);
upload.lease.response(200, Duration::from_secs(20), false);
assert_eq!(upload.lease.member.state.errors.load(Ordering::Relaxed), 0);
let read = select(&pool);
read.lease.response(200, Duration::from_secs(20), true);
assert_eq!(read.lease.member.state.errors.load(Ordering::Relaxed), 1);
}
#[tokio::test]
async fn backup_is_only_new_assignment_when_no_primary_is_eligible() {
let mut pool = pool(Algorithm::RoundRobin, StickyMode::Off);
Arc::get_mut(&mut Arc::get_mut(&mut pool).unwrap().members[1])
.unwrap()
.definition
.backup = true;
for _ in 0..8 {
assert_eq!(select(&pool).lease.member.definition.id, "a");
}
lock(&pool.members[0].state.health).healthy = false;
assert_eq!(select(&pool).lease.member.definition.id, "b");
lock(&pool.members[0].state.health).healthy = true;
#[tokio::test]
async fn client_upload_duration_does_not_trigger_passive_slow_failure() {
let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
let upload = select(&pool);
upload.lease.response(200, Duration::from_secs(20), false);
assert_eq!(upload.lease.member.state.errors.load(Ordering::Relaxed), 0);
let read = select(&pool);
read.lease.response(200, Duration::from_secs(20), true);
assert_eq!(read.lease.member.state.errors.load(Ordering::Relaxed), 1);
}
#[tokio::test]
async fn backup_is_only_new_assignment_when_no_primary_is_eligible() {
let mut pool = pool(Algorithm::RoundRobin, StickyMode::Off);
Arc::get_mut(&mut Arc::get_mut(&mut pool).unwrap().members[1])
.unwrap()
.definition
.backup = true;
for _ in 0..8 {
assert_eq!(select(&pool).lease.member.definition.id, "a");
}
#[tokio::test]
async fn drain_rejects_new_sessions_but_preserves_valid_existing_affinity() {
let mut pool = pool(Algorithm::RoundRobin, StickyMode::Cookie);
Arc::get_mut(&mut Arc::get_mut(&mut pool).unwrap().members[0])
lock(&pool.members[0].state.health).healthy = false;
assert_eq!(select(&pool).lease.member.definition.id, "b");
lock(&pool.members[0].state.health).healthy = true;
assert_eq!(select(&pool).lease.member.definition.id, "a");
}
#[tokio::test]
async fn drain_rejects_new_sessions_but_preserves_valid_existing_affinity() {
let mut pool = pool(Algorithm::RoundRobin, StickyMode::Cookie);
Arc::get_mut(&mut Arc::get_mut(&mut pool).unwrap().members[0])
.unwrap()
.definition
.drain = true;
assert_eq!(select(&pool).lease.member.definition.id, "b");
let mut headers = HeaderMap::new();
headers.insert(
"cookie",
format!("{}={}", pool.cookie_name, pool.members[0].token)
.parse()
.unwrap(),
);
let existing = pool
.select("127.0.0.1".parse().unwrap(), &headers, false)
.unwrap();
assert_eq!(existing.lease.member.definition.id, "a");
lock(&pool.members[0].state.health).healthy = false;
assert_eq!(
pool.select("127.0.0.1".parse().unwrap(), &headers, false)
.unwrap()
.lease
.member
.definition
.drain = true;
assert_eq!(select(&pool).lease.member.definition.id, "b");
let mut headers = HeaderMap::new();
headers.insert(
"cookie",
format!("{}={}", pool.cookie_name, pool.members[0].token)
.parse()
.unwrap(),
);
let existing = pool
.select("127.0.0.1".parse().unwrap(), &headers, false)
.unwrap();
assert_eq!(existing.lease.member.definition.id, "a");
lock(&pool.members[0].state.health).healthy = false;
assert_eq!(
pool.select("127.0.0.1".parse().unwrap(), &headers, false)
.unwrap()
.lease
.member
.definition
.id,
"b"
);
}
.id,
"b"
);
}