gru/RustProxy
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.25.5

Browse Source
This commit is contained in:
Mateusz Gruszczyński committed 2026-10-05 09:30:21 +02:00
1 parent 1dea7f55c8
commit c819f0a335
151 files changed
+7610 -4264
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/balance/health.rs
+42 -15
View File
@@ -24,10 +24,26 @@ impl BackendState {
latency_us: 0,
ewma_us: 0,
transitions: 0,
unhealthy_source: if mode == HealthMode::Off { "none" } else { "active" },
unhealthy_reason: if mode == HealthMode::Off { "none" } else { "checking" },
active_unhealthy_source: if mode == HealthMode::Off { "none" } else { "active" },
active_unhealthy_reason: if mode == HealthMode::Off { "none" } else { "checking" },
unhealthy_source: if mode == HealthMode::Off {
"none"
} else {
"active"
},
unhealthy_reason: if mode == HealthMode::Off {
"none"
} else {
"checking"
},
active_unhealthy_source: if mode == HealthMode::Off {
"none"
} else {
"active"
},
active_unhealthy_reason: if mode == HealthMode::Off {
"none"
} else {
"checking"
},
passive_unhealthy_source: "none",
passive_unhealthy_reason: "none",
}),
@@ -169,9 +185,9 @@ impl BackendState {
return;
}
let current_source_is_still_unhealthy =
(h.unhealthy_source.starts_with("active") && !h.active_healthy)
|| (h.unhealthy_source.starts_with("passive") && !h.passive_healthy);
let current_source_is_still_unhealthy = (h.unhealthy_source.starts_with("active")
&& !h.active_healthy)
|| (h.unhealthy_source.starts_with("passive") && !h.passive_healthy);
if current_source_is_still_unhealthy {
return;
}
@@ -202,7 +218,8 @@ impl BackendState {
h.active_healthy = true;
}
} else {
self.active_health_failures_total.fetch_add(1, Ordering::Relaxed);
self.active_health_failures_total
.fetch_add(1, Ordering::Relaxed);
h.successes = 0;
h.failures = h.failures.saturating_add(1);
if h.failures >= settings.fall {
@@ -260,11 +277,21 @@ impl BackendState {
fn record_passive_observation(&self, reason: &'static str) {
match reason {
"http_5xx" => { self.http_5xx_total.fetch_add(1, Ordering::Relaxed); }
"timeout" => { self.timeouts_total.fetch_add(1, Ordering::Relaxed); }
"connect_error" => { self.connect_errors_total.fetch_add(1, Ordering::Relaxed); }
"io_error" => { self.io_errors_total.fetch_add(1, Ordering::Relaxed); }
"slow_response" => { self.slow_responses_total.fetch_add(1, Ordering::Relaxed); }
"http_5xx" => {
self.http_5xx_total.fetch_add(1, Ordering::Relaxed);
}
"timeout" => {
self.timeouts_total.fetch_add(1, Ordering::Relaxed);
}
"connect_error" => {
self.connect_errors_total.fetch_add(1, Ordering::Relaxed);
}
"io_error" => {
self.io_errors_total.fetch_add(1, Ordering::Relaxed);
}
"slow_response" => {
self.slow_responses_total.fetch_add(1, Ordering::Relaxed);
}
_ => {}
}
}
@@ -301,7 +328,8 @@ impl BackendState {
if !passive.counts(reason) {
return;
}
self.passive_health_failures_total.fetch_add(1, Ordering::Relaxed);
self.passive_health_failures_total
.fetch_add(1, Ordering::Relaxed);
h.passive_successes = 0;
h.passive_failures = h.passive_failures.saturating_add(1);
if h.passive_failures >= passive.fall(active) {
@@ -325,7 +353,6 @@ impl BackendState {
}
self.finish_transition(&mut h, previous);
}
}
#[derive(Debug)]
pub struct Member {
src/balance/lease_runtime.rs
+99 -25
View File
@@ -26,7 +26,9 @@ impl Lease {
|| !self.passive_health.count_slow_responses)
{
if is_slow {
self.member.state.record_passive_observation("slow_response");
self.member
.state
.record_passive_observation("slow_response");
}
"http_5xx"
} else if is_slow {
@@ -62,7 +64,10 @@ impl Lease {
self.member.state.record_failure(class, detail);
let passive_reason = match class {
"connect" | "dns" | "tls" => "connect_error",
"timeout" | "body_timeout" | "response_header_timeout" | "read_timeout"
"timeout"
| "body_timeout"
| "response_header_timeout"
| "read_timeout"
| "send_timeout" => "timeout",
_ => "io_error",
};
@@ -147,33 +152,102 @@ impl Pool {
let _ = writeln!(text, "rust_proxy_backend_{metric}{{{labels}}} {value}");
}
for (metric, value) in [
("active_health_failures_total", state.active_health_failures_total.load(Ordering::Relaxed)),
("passive_health_failures_total", state.passive_health_failures_total.load(Ordering::Relaxed)),
("http_5xx_total", state.http_5xx_total.load(Ordering::Relaxed)),
("passive_http_5xx_total", state.http_5xx_total.load(Ordering::Relaxed)),
("timeouts_total", state.timeouts_total.load(Ordering::Relaxed)),
("passive_timeouts_total", state.timeouts_total.load(Ordering::Relaxed)),
("connect_errors_total", state.connect_errors_total.load(Ordering::Relaxed)),
("passive_connect_errors_total", state.connect_errors_total.load(Ordering::Relaxed)),
("io_errors_total", state.io_errors_total.load(Ordering::Relaxed)),
("passive_io_errors_total", state.io_errors_total.load(Ordering::Relaxed)),
("slow_responses_total", state.slow_responses_total.load(Ordering::Relaxed)),
("passive_slow_total", state.slow_responses_total.load(Ordering::Relaxed)),
("retry_attempts_total", state.retry_attempts_total.load(Ordering::Relaxed)),
("retry_successes_total", state.retry_successes_total.load(Ordering::Relaxed)),
("capacity_rejections_total", state.capacity_rejections_total.load(Ordering::Relaxed)),
("queue_timeouts_total", state.queue_timeouts_total.load(Ordering::Relaxed)),
("dns_errors_total", state.dns_errors_total.load(Ordering::Relaxed)),
("tls_errors_total", state.tls_errors_total.load(Ordering::Relaxed)),
("response_header_timeouts_total", state.response_header_timeouts_total.load(Ordering::Relaxed)),
("read_timeouts_total", state.read_timeouts_total.load(Ordering::Relaxed)),
("send_timeouts_total", state.send_timeouts_total.load(Ordering::Relaxed)),
(
"active_health_failures_total",
state.active_health_failures_total.load(Ordering::Relaxed),
),
(
"passive_health_failures_total",
state.passive_health_failures_total.load(Ordering::Relaxed),
),
(
"http_5xx_total",
state.http_5xx_total.load(Ordering::Relaxed),
),
(
"passive_http_5xx_total",
state.http_5xx_total.load(Ordering::Relaxed),
),
(
"timeouts_total",
state.timeouts_total.load(Ordering::Relaxed),
),
(
"passive_timeouts_total",
state.timeouts_total.load(Ordering::Relaxed),
),
(
"connect_errors_total",
state.connect_errors_total.load(Ordering::Relaxed),
),
(
"passive_connect_errors_total",
state.connect_errors_total.load(Ordering::Relaxed),
),
(
"io_errors_total",
state.io_errors_total.load(Ordering::Relaxed),
),
(
"passive_io_errors_total",
state.io_errors_total.load(Ordering::Relaxed),
),
(
"slow_responses_total",
state.slow_responses_total.load(Ordering::Relaxed),
),
(
"passive_slow_total",
state.slow_responses_total.load(Ordering::Relaxed),
),
(
"retry_attempts_total",
state.retry_attempts_total.load(Ordering::Relaxed),
),
(
"retry_successes_total",
state.retry_successes_total.load(Ordering::Relaxed),
),
(
"capacity_rejections_total",
state.capacity_rejections_total.load(Ordering::Relaxed),
),
(
"queue_timeouts_total",
state.queue_timeouts_total.load(Ordering::Relaxed),
),
(
"dns_errors_total",
state.dns_errors_total.load(Ordering::Relaxed),
),
(
"tls_errors_total",
state.tls_errors_total.load(Ordering::Relaxed),
),
(
"response_header_timeouts_total",
state.response_header_timeouts_total.load(Ordering::Relaxed),
),
(
"read_timeouts_total",
state.read_timeouts_total.load(Ordering::Relaxed),
),
(
"send_timeouts_total",
state.send_timeouts_total.load(Ordering::Relaxed),
),
] {
let _ = writeln!(text, "rust_proxy_backend_{metric}{{{labels}}} {value}");
}
for (metric, value) in [
("last_ejected_timestamp_milliseconds", state.last_ejected_ms.load(Ordering::Relaxed)),
("last_recovered_timestamp_milliseconds", state.last_recovered_ms.load(Ordering::Relaxed)),
(
"last_ejected_timestamp_milliseconds",
state.last_ejected_ms.load(Ordering::Relaxed),
),
(
"last_recovered_timestamp_milliseconds",
state.last_recovered_ms.load(Ordering::Relaxed),
),
] {
let _ = writeln!(text, "rust_proxy_backend_{metric}{{{labels}}} {value}");
}
src/balance/model.rs
+12 -3
View File
@@ -241,10 +241,16 @@ impl Settings {
let _: HeaderValue = host.parse()?;
}
if let Some(fall) = self.passive_health.fall {
ensure!((1..=20).contains(&fall), "passive health fall must be 1..20");
ensure!(
(1..=20).contains(&fall),
"passive health fall must be 1..20"
);
}
if let Some(rise) = self.passive_health.rise {
ensure!((1..=20).contains(&rise), "passive health rise must be 1..20");
ensure!(
(1..=20).contains(&rise),
"passive health rise must be 1..20"
);
}
ensure!(
(1..=3600).contains(&k.idle_timeout_seconds)
@@ -252,7 +258,10 @@ impl Settings {
&& (1..=64).contains(&k.max_idle_per_host),
"invalid keepalive limits"
);
ensure!(self.retry.max_retries <= 3, "retry max_retries must be 0..3");
ensure!(
self.retry.max_retries <= 3,
"retry max_retries must be 0..3"
);
ensure!(
self.retry.retry_statuses.len() <= 8
&& self
src/balance/pool_core.rs
+8 -10
View File
@@ -53,16 +53,14 @@ impl Pool {
};
let same_origin = serde_json::to_vec(&member.definition.backend()).ok()
== serde_json::to_vec(&old.definition.backend()).ok();
let same_health = serde_json::to_vec(&(
&self.settings.health_check,
&self.settings.passive_health,
))
.ok()
== serde_json::to_vec(&(
&previous.settings.health_check,
&previous.settings.passive_health,
))
.ok();
let same_health =
serde_json::to_vec(&(&self.settings.health_check, &self.settings.passive_health))
.ok()
== serde_json::to_vec(&(
&previous.settings.health_check,
&previous.settings.passive_health,
))
.ok();
if same_origin && same_health && (old.definition.enabled || !member.definition.enabled)
{
// This pool has not been published or cloned into routes yet.
src/balance/pool_view.rs
+24 -6
View File
@@ -31,15 +31,24 @@ impl Pool {
"max_concurrent_requests".into(),
json!(m.definition.max_concurrent_requests),
);
item.insert("queue_timeout_ms".into(), json!(m.definition.queue_timeout_ms));
item.insert(
"queue_timeout_ms".into(),
json!(m.definition.queue_timeout_ms),
);
item.insert("status".into(), json!(status));
item.insert("healthy".into(), json!(h.healthy));
item.insert("active".into(), json!(m.state.active.load(Ordering::Relaxed)));
item.insert(
"active".into(),
json!(m.state.active.load(Ordering::Relaxed)),
);
item.insert(
"requests".into(),
json!(m.state.requests.load(Ordering::Relaxed)),
);
item.insert("errors".into(), json!(m.state.errors.load(Ordering::Relaxed)));
item.insert(
"errors".into(),
json!(m.state.errors.load(Ordering::Relaxed)),
);
item.insert(
"bytes_in".into(),
json!(m.state.bytes_in.load(Ordering::Relaxed)),
@@ -48,7 +57,10 @@ impl Pool {
"bytes_out".into(),
json!(m.state.bytes_out.load(Ordering::Relaxed)),
);
item.insert("traffic_bytes_per_second".into(), json!(m.state.traffic_rate()));
item.insert(
"traffic_bytes_per_second".into(),
json!(m.state.traffic_rate()),
);
item.insert("latency_ms".into(), json!(h.latency_us as f64 / 1000.0));
item.insert("ewma_latency_ms".into(), json!(h.ewma_us as f64 / 1000.0));
item.insert("last_probe_ms".into(), json!(h.last_probe_ms));
@@ -107,7 +119,10 @@ impl Pool {
);
item.insert(
"response_header_timeouts_total".into(),
json!(m.state.response_header_timeouts_total.load(Ordering::Relaxed)),
json!(m
.state
.response_header_timeouts_total
.load(Ordering::Relaxed)),
);
item.insert(
"read_timeouts_total".into(),
@@ -134,7 +149,10 @@ impl Pool {
);
item.insert(
"passive_health_failures_total".into(),
json!(m.state.passive_health_failures_total.load(Ordering::Relaxed)),
json!(m
.state
.passive_health_failures_total
.load(Ordering::Relaxed)),
);
item.insert(
"last_status".into(),
src/balance/tests/part_01.rs
+379 -369
View File
@@ -1,14 +1,14 @@
use super::*;
fn settings(mode: HealthMode) -> HealthCheck {
HealthCheck {
mode,
fall: 2,
rise: 2,
..Default::default()
}
use super::*;
fn settings(mode: HealthMode) -> HealthCheck {
HealthCheck {
mode,
fall: 2,
rise: 2,
..Default::default()
}
fn pool(algorithm: Algorithm, sticky: StickyMode) -> Arc<Pool> {
let cfg = serde_json::from_value(serde_json::json!({
}
fn pool(algorithm: Algorithm, sticky: StickyMode) -> Arc<Pool> {
let cfg = serde_json::from_value(serde_json::json!({
"listeners":{"endpoints":[{"id":"http","binding":{"address":"127.0.0.1:8080"}}]},
"vhosts":[{
"id":"app","frontend":{"hosts":["app.localhost"],"listener_ids":["http"]},
@@ -16,378 +16,388 @@
{"id":"b","url":"http://127.0.0.1:9001","allow_private":true,"weight":1}],
"load_balancing":{"algorithm":algorithm,"sticky":{"mode":sticky},"slow_start_seconds":0,"health_check":{"mode":"off"}}
}]})).unwrap();
let policy = crate::config::ValidationPolicy {
proxy_addr: "127.0.0.1:8080".parse().unwrap(),
admin_addr: "127.0.0.1:9090".parse().unwrap(),
tls_addr: None,
cert_dir: None,
data_dir: None,
};
crate::config::compile(cfg, 0, &policy).unwrap().pools[0].clone()
}
fn capacity_pool(limit: u32, queue_timeout_ms: u64) -> Arc<Pool> {
let cfg = serde_json::from_value(serde_json::json!({
"listeners":{"endpoints":[{"id":"http","binding":{"address":"127.0.0.1:8080"}}]},
"vhosts":[{
"id":"limited","frontend":{"hosts":["limited.localhost"],"listener_ids":["http"]},
"backends":[{
"id":"only","url":"http://127.0.0.1:9000","allow_private":true,
"max_concurrent_requests":limit,"queue_timeout_ms":queue_timeout_ms
}],
"load_balancing":{"health_check":{"mode":"off"}}
}]
})).unwrap();
let policy = crate::config::ValidationPolicy {
proxy_addr: "127.0.0.1:8080".parse().unwrap(),
admin_addr: "127.0.0.1:9090".parse().unwrap(),
tls_addr: None,
cert_dir: None,
data_dir: None,
};
crate::config::compile(cfg, 0, &policy).unwrap().pools[0].clone()
}
fn select(pool: &Pool) -> Selection {
pool.select("127.0.0.1".parse().unwrap(), &HeaderMap::new(), false)
.unwrap()
}
#[test]
fn default_health_check_uses_tcp() {
assert_eq!(HealthMode::default(), HealthMode::Tcp);
assert_eq!(HealthCheck::default().mode, HealthMode::Tcp);
}
#[test]
fn legacy_settings_without_passive_health_default_to_off() {
let parsed: Settings = serde_json::from_value(serde_json::json!({
"health_check": { "mode": "off" }
}))
.unwrap();
assert!(!parsed.passive_health.enabled);
assert!(parsed.passive_health.count_connect_errors);
assert!(parsed.passive_health.count_timeouts);
assert!(parsed.passive_health.count_io_errors);
assert!(parsed.passive_health.count_http_5xx);
assert!(!parsed.passive_health.count_slow_responses);
assert_eq!(parsed.retry.max_retries, 1);
assert!(parsed.retry.retry_connect_errors);
assert!(parsed.retry.retry_timeouts);
assert_eq!(parsed.retry.retry_statuses, vec![502, 503, 504]);
}
let policy = crate::config::ValidationPolicy {
proxy_addr: "127.0.0.1:8080".parse().unwrap(),
admin_addr: "127.0.0.1:9090".parse().unwrap(),
tls_addr: None,
cert_dir: None,
data_dir: None,
};
crate::config::compile(cfg, 0, &policy).unwrap().pools[0].clone()
}
fn capacity_pool(limit: u32, queue_timeout_ms: u64) -> Arc<Pool> {
let cfg = serde_json::from_value(serde_json::json!({
"listeners":{"endpoints":[{"id":"http","binding":{"address":"127.0.0.1:8080"}}]},
"vhosts":[{
"id":"limited","frontend":{"hosts":["limited.localhost"],"listener_ids":["http"]},
"backends":[{
"id":"only","url":"http://127.0.0.1:9000","allow_private":true,
"max_concurrent_requests":limit,"queue_timeout_ms":queue_timeout_ms
}],
"load_balancing":{"health_check":{"mode":"off"}}
}]
}))
.unwrap();
let policy = crate::config::ValidationPolicy {
proxy_addr: "127.0.0.1:8080".parse().unwrap(),
admin_addr: "127.0.0.1:9090".parse().unwrap(),
tls_addr: None,
cert_dir: None,
data_dir: None,
};
crate::config::compile(cfg, 0, &policy).unwrap().pools[0].clone()
}
fn select(pool: &Pool) -> Selection {
pool.select("127.0.0.1".parse().unwrap(), &HeaderMap::new(), false)
.unwrap()
}
#[test]
fn default_health_check_uses_tcp() {
assert_eq!(HealthMode::default(), HealthMode::Tcp);
assert_eq!(HealthCheck::default().mode, HealthMode::Tcp);
}
#[test]
fn legacy_settings_without_passive_health_default_to_off() {
let parsed: Settings = serde_json::from_value(serde_json::json!({
"health_check": { "mode": "off" }
}))
.unwrap();
assert!(!parsed.passive_health.enabled);
assert!(parsed.passive_health.count_connect_errors);
assert!(parsed.passive_health.count_timeouts);
assert!(parsed.passive_health.count_io_errors);
assert!(parsed.passive_health.count_http_5xx);
assert!(!parsed.passive_health.count_slow_responses);
assert_eq!(parsed.retry.max_retries, 1);
assert!(parsed.retry.retry_connect_errors);
assert!(parsed.retry.retry_timeouts);
assert_eq!(parsed.retry.retry_statuses, vec![502, 503, 504]);
}
#[test]
fn retry_policy_rejects_unbounded_or_non_5xx_configuration() {
let mut settings = Settings::default();
settings.retry.max_retries = 4;
assert!(settings.validate().is_err());
settings.retry.max_retries = 1;
settings.retry.retry_statuses = vec![429];
assert!(settings.validate().is_err());
}
#[test]
fn retry_policy_rejects_unbounded_or_non_5xx_configuration() {
let mut settings = Settings::default();
settings.retry.max_retries = 4;
assert!(settings.validate().is_err());
settings.retry.max_retries = 1;
settings.retry.retry_statuses = vec![429];
assert!(settings.validate().is_err());
}
#[test]
fn traffic_rate_drops_to_zero_after_an_idle_sample_window() {
let state = BackendState::new(HealthMode::Off);
state.bytes_out.store(2048, std::sync::atomic::Ordering::Relaxed);
lock(&state.traffic).at = std::time::Instant::now() - std::time::Duration::from_secs(2);
state.sample_traffic();
assert!(state.traffic_rate() > 0.0);
lock(&state.traffic).at = std::time::Instant::now() - std::time::Duration::from_secs(2);
state.sample_traffic();
assert_eq!(state.traffic_rate(), 0.0);
#[test]
fn traffic_rate_drops_to_zero_after_an_idle_sample_window() {
let state = BackendState::new(HealthMode::Off);
state
.bytes_out
.store(2048, std::sync::atomic::Ordering::Relaxed);
lock(&state.traffic).at = std::time::Instant::now() - std::time::Duration::from_secs(2);
state.sample_traffic();
assert!(state.traffic_rate() > 0.0);
lock(&state.traffic).at = std::time::Instant::now() - std::time::Duration::from_secs(2);
state.sample_traffic();
assert_eq!(state.traffic_rate(), 0.0);
}
#[test]
fn active_failures_are_independent_from_passive_user_traffic() {
let state = BackendState::new(HealthMode::Http);
let settings = settings(HealthMode::Http);
let passive = PassiveHealth::default();
let fast = Duration::from_millis(5);
assert!(!state.healthy());
state.observe_active(&settings, "ok", fast);
assert!(!state.healthy());
state.observe_active(&settings, "ok", fast);
assert!(state.healthy());
state.observe_active(&settings, "status", fast);
state.observe_passive(&settings, &passive, "ok", fast);
state.observe_active(&settings, "status", fast);
assert!(!state.healthy());
state.observe_passive(&settings, &passive, "http_5xx", fast);
assert!(!state.healthy());
state.observe_active(&settings, "ok", fast);
assert!(!state.healthy());
state.observe_active(&settings, "ok", fast);
assert!(state.healthy());
}
#[test]
fn passive_health_is_opt_in_and_uses_separate_thresholds() {
let state = BackendState::new(HealthMode::Off);
let active = settings(HealthMode::Off);
let fast = Duration::from_millis(5);
for _ in 0..10 {
state.observe_passive(&active, &PassiveHealth::default(), "http_5xx", fast);
}
#[test]
fn active_failures_are_independent_from_passive_user_traffic() {
let state = BackendState::new(HealthMode::Http);
let settings = settings(HealthMode::Http);
let passive = PassiveHealth::default();
let fast = Duration::from_millis(5);
assert!(!state.healthy());
state.observe_active(&settings, "ok", fast);
assert!(!state.healthy());
state.observe_active(&settings, "ok", fast);
assert!(state.healthy());
state.observe_active(&settings, "status", fast);
state.observe_passive(&settings, &passive, "ok", fast);
state.observe_active(&settings, "status", fast);
assert!(!state.healthy());
state.observe_passive(&settings, &passive, "http_5xx", fast);
assert!(!state.healthy());
state.observe_active(&settings, "ok", fast);
assert!(!state.healthy());
state.observe_active(&settings, "ok", fast);
assert!(state.healthy());
assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 10);
assert_eq!(
state.passive_health_failures_total.load(Ordering::Relaxed),
0
);
let passive = PassiveHealth {
enabled: true,
fall: Some(2),
rise: Some(2),
..Default::default()
};
state.observe_passive(&active, &passive, "http_5xx", fast);
assert!(state.healthy());
state.observe_passive(&active, &passive, "http_5xx", fast);
assert!(!state.healthy());
state.observe_passive(&active, &passive, "ok", fast);
assert!(!state.healthy());
state.observe_passive(&active, &passive, "ok", fast);
assert!(state.healthy());
}
#[test]
fn passive_http_5xx_can_be_explicitly_ignored_when_passive_health_is_on() {
let state = BackendState::new(HealthMode::Off);
let active = settings(HealthMode::Off);
let passive = PassiveHealth {
enabled: true,
count_http_5xx: false,
fall: Some(1),
..Default::default()
};
for _ in 0..100 {
state.observe_passive(&active, &passive, "http_5xx", Duration::from_millis(5));
}
assert!(state.healthy());
assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 100);
assert_eq!(
state.passive_health_failures_total.load(Ordering::Relaxed),
0
);
}
#[test]
fn passive_connect_errors_use_the_explicit_fall_threshold() {
let state = BackendState::new(HealthMode::Off);
let active = settings(HealthMode::Off);
let passive = PassiveHealth {
enabled: true,
count_connect_errors: true,
fall: Some(3),
..Default::default()
};
for _ in 0..2 {
state.observe_passive(&active, &passive, "connect_error", Duration::from_millis(5));
assert!(state.healthy());
}
state.observe_passive(&active, &passive, "connect_error", Duration::from_millis(5));
assert!(!state.healthy());
assert_eq!(state.connect_errors_total.load(Ordering::Relaxed), 3);
assert_eq!(
state.passive_health_failures_total.load(Ordering::Relaxed),
3
);
}
#[test]
fn passive_health_is_opt_in_and_uses_separate_thresholds() {
let state = BackendState::new(HealthMode::Off);
let active = settings(HealthMode::Off);
let fast = Duration::from_millis(5);
for _ in 0..10 {
state.observe_passive(&active, &PassiveHealth::default(), "http_5xx", fast);
}
assert!(state.healthy());
assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 10);
assert_eq!(state.passive_health_failures_total.load(Ordering::Relaxed), 0);
let passive = PassiveHealth {
enabled: true,
fall: Some(2),
rise: Some(2),
..Default::default()
};
#[test]
fn active_health_can_stay_healthy_while_passive_health_is_off() {
let state = BackendState::new(HealthMode::Tcp);
let active = settings(HealthMode::Tcp);
let passive = PassiveHealth::default();
let fast = Duration::from_millis(5);
state.observe_active(&active, "ok", fast);
state.observe_active(&active, "ok", fast);
assert!(state.healthy());
for _ in 0..20 {
state.observe_passive(&active, &passive, "http_5xx", fast);
assert!(state.healthy());
state.observe_passive(&active, &passive, "http_5xx", fast);
assert!(!state.healthy());
state.observe_passive(&active, &passive, "ok", fast);
assert!(!state.healthy());
state.observe_passive(&active, &passive, "ok", fast);
assert!(state.healthy());
}
#[test]
fn passive_http_5xx_can_be_explicitly_ignored_when_passive_health_is_on() {
let state = BackendState::new(HealthMode::Off);
let active = settings(HealthMode::Off);
let passive = PassiveHealth {
enabled: true,
count_http_5xx: false,
fall: Some(1),
..Default::default()
};
for _ in 0..100 {
state.observe_passive(
&active,
&passive,
"http_5xx",
Duration::from_millis(5),
);
}
assert!(state.healthy());
assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 100);
assert_eq!(state.passive_health_failures_total.load(Ordering::Relaxed), 0);
}
assert!(state.healthy());
assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 20);
assert_eq!(
state.passive_health_failures_total.load(Ordering::Relaxed),
0
);
}
#[test]
fn passive_connect_errors_use_the_explicit_fall_threshold() {
let state = BackendState::new(HealthMode::Off);
let active = settings(HealthMode::Off);
let passive = PassiveHealth {
enabled: true,
count_connect_errors: true,
fall: Some(3),
..Default::default()
};
for _ in 0..2 {
state.observe_passive(
&active,
&passive,
"connect_error",
Duration::from_millis(5),
);
assert!(state.healthy());
}
state.observe_passive(
&active,
&passive,
#[test]
fn unhealthy_source_hands_off_when_the_original_health_domain_recovers() {
let state = BackendState::new(HealthMode::Tcp);
let active = HealthCheck {
fall: 1,
rise: 1,
..settings(HealthMode::Tcp)
};
let passive = PassiveHealth {
enabled: true,
count_http_5xx: true,
fall: Some(1),
rise: Some(1),
..Default::default()
};
let fast = Duration::from_millis(5);
state.observe_active(&active, "ok", fast);
state.observe_active(&active, "connect_error", fast);
{
let health = lock(&state.health);
assert_eq!(health.unhealthy_source, "active_tcp");
}
state.observe_passive(&active, &passive, "http_5xx", fast);
state.observe_active(&active, "ok", fast);
{
let health = lock(&state.health);
assert!(!health.healthy);
assert_eq!(health.unhealthy_source, "passive_http_5xx");
assert_eq!(health.unhealthy_reason, "http_5xx");
}
state.observe_passive(&active, &passive, "ok", fast);
assert!(state.healthy());
}
#[test]
fn disabled_checks_never_quarantine_and_probes_are_single_flight() {
let off = BackendState::new(HealthMode::Off);
let settings = settings(HealthMode::Off);
for _ in 0..5 {
off.observe_passive(
&settings,
&PassiveHealth::default(),
"connect_error",
Duration::from_millis(5),
Duration::ZERO,
);
assert!(!state.healthy());
assert_eq!(state.connect_errors_total.load(Ordering::Relaxed), 3);
assert_eq!(state.passive_health_failures_total.load(Ordering::Relaxed), 3);
}
#[test]
fn active_health_can_stay_healthy_while_passive_health_is_off() {
let state = BackendState::new(HealthMode::Tcp);
let active = settings(HealthMode::Tcp);
let passive = PassiveHealth::default();
let fast = Duration::from_millis(5);
state.observe_active(&active, "ok", fast);
state.observe_active(&active, "ok", fast);
assert!(state.healthy());
for _ in 0..20 {
state.observe_passive(&active, &passive, "http_5xx", fast);
}
assert!(state.healthy());
assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 20);
assert_eq!(state.passive_health_failures_total.load(Ordering::Relaxed), 0);
}
#[test]
fn unhealthy_source_hands_off_when_the_original_health_domain_recovers() {
let state = BackendState::new(HealthMode::Tcp);
let active = HealthCheck {
fall: 1,
rise: 1,
..settings(HealthMode::Tcp)
};
let passive = PassiveHealth {
enabled: true,
count_http_5xx: true,
fall: Some(1),
rise: Some(1),
..Default::default()
};
let fast = Duration::from_millis(5);
state.observe_active(&active, "ok", fast);
state.observe_active(&active, "connect_error", fast);
{
let health = lock(&state.health);
assert_eq!(health.unhealthy_source, "active_tcp");
}
state.observe_passive(&active, &passive, "http_5xx", fast);
state.observe_active(&active, "ok", fast);
{
let health = lock(&state.health);
assert!(!health.healthy);
assert_eq!(health.unhealthy_source, "passive_http_5xx");
assert_eq!(health.unhealthy_reason, "http_5xx");
}
state.observe_passive(&active, &passive, "ok", fast);
assert!(state.healthy());
}
#[test]
fn disabled_checks_never_quarantine_and_probes_are_single_flight() {
let off = BackendState::new(HealthMode::Off);
let settings = settings(HealthMode::Off);
for _ in 0..5 {
off.observe_passive(&settings, &PassiveHealth::default(), "connect_error", Duration::ZERO);
}
assert!(off.healthy());
assert!(!off.claim_probe(&settings));
let on = BackendState::new(HealthMode::Http);
let settings = super::tests::settings(HealthMode::Http);
assert!(on.claim_probe(&settings));
assert!(!on.claim_probe(&settings));
on.observe_active(&settings, "ok", Duration::ZERO);
assert!(!on.claim_probe(&settings));
}
#[tokio::test]
async fn weighted_round_robin_distributes_three_to_one_without_body_collection() {
let pool = pool(Algorithm::RoundRobin, StickyMode::Off);
let mut counts = [0, 0];
for _ in 0..40 {
let selection = select(&pool);
counts[usize::from(selection.lease.member.definition.id == "b")] += 1;
}
assert_eq!(counts, [30, 10]);
assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 0);
assert_eq!(pool.members[0].state.requests.load(Ordering::Relaxed), 0); // Mere cache selection isn't origin traffic.
}
#[tokio::test]
async fn least_connections_counts_a_lease_until_its_last_owner_drops() {
let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
let mut held = Vec::new();
for _ in 0..3 {
let selection = select(&pool);
assert_eq!(selection.lease.member.definition.id, "a");
held.push(selection.lease);
}
let other = select(&pool);
assert_eq!(other.lease.member.definition.id, "b");
let clone = held[0].clone();
held.clear();
assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 1);
drop(clone);
assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 0);
}
#[tokio::test]
async fn failover_selection_never_reuses_an_attempted_backend() {
let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
let first = select(&pool);
let first_id = first.lease.member.definition.id.clone();
drop(first);
let retry = pool
.select_retry(
"127.0.0.1".parse().unwrap(),
&HeaderMap::new(),
false,
std::slice::from_ref(&first_id),
)
.unwrap();
assert_ne!(retry.lease.member.definition.id, first_id);
}
#[tokio::test]
async fn backend_concurrency_limit_queues_then_rejects_and_recovers() {
let pool = capacity_pool(1, 10);
let first = select(&pool);
let headers = HeaderMap::new();
let saturated = pool
.select_wait("127.0.0.1".parse().unwrap(), &headers, false)
.await;
assert!(matches!(saturated, Err(SelectionFailure::Capacity)));
let view = pool.view();
let member = &view["members"][0];
assert_eq!(member["capacity_rejections_total"], 1);
assert_eq!(member["queue_timeouts_total"], 1);
assert_eq!(member["active"], 1);
drop(first);
assert!(pool
.select_wait("127.0.0.1".parse().unwrap(), &headers, false)
.await
.is_ok());
}
#[tokio::test]
async fn zero_capacity_queue_timeout_rejects_without_counting_a_queue_timeout() {
let pool = capacity_pool(1, 0);
let first = select(&pool);
let headers = HeaderMap::new();
let saturated = pool
.select_wait("127.0.0.1".parse().unwrap(), &headers, false)
.await;
assert!(matches!(saturated, Err(SelectionFailure::Capacity)));
let view = pool.view();
let member = &view["members"][0];
assert_eq!(member["capacity_rejections_total"], 1);
assert_eq!(member["queue_timeouts_total"], 0);
drop(first);
}
#[test]
fn detailed_backend_failures_are_exposed_without_losing_aggregate_counters() {
let pool = capacity_pool(0, 0);
assert!(off.healthy());
assert!(!off.claim_probe(&settings));
let on = BackendState::new(HealthMode::Http);
let settings = super::tests::settings(HealthMode::Http);
assert!(on.claim_probe(&settings));
assert!(!on.claim_probe(&settings));
on.observe_active(&settings, "ok", Duration::ZERO);
assert!(!on.claim_probe(&settings));
}
#[tokio::test]
async fn weighted_round_robin_distributes_three_to_one_without_body_collection() {
let pool = pool(Algorithm::RoundRobin, StickyMode::Off);
let mut counts = [0, 0];
for _ in 0..40 {
let selection = select(&pool);
selection
.lease
.error_detail("dns", "lookup backend.example failed", Duration::from_millis(7));
let view = pool.view();
let member = &view["members"][0];
assert_eq!(member["dns_errors_total"], 1);
assert_eq!(member["connect_errors_total"], 1);
assert_eq!(member["last_failure_class"], "dns");
assert_eq!(member["last_failure_detail"], "lookup backend.example failed");
assert!(member["last_failure_at_ms"].as_u64().unwrap_or_default() > 0);
counts[usize::from(selection.lease.member.definition.id == "b")] += 1;
}
#[tokio::test]
async fn traffic_and_adaptive_selection_use_observed_cost_and_weight() {
for algorithm in [Algorithm::LeastTraffic, Algorithm::Adaptive] {
let pool = pool(algorithm, StickyMode::Off);
lock(&pool.members[0].state.traffic).rate = 50_000_000.0;
lock(&pool.members[0].state.health).ewma_us = 1_000_000;
assert_eq!(select(&pool).lease.member.definition.id, "b");
}
}
#[tokio::test]
async fn ip_hash_is_stable_and_cannot_be_spoofed_with_forwarded_headers() {
let pool = pool(Algorithm::IpHash, StickyMode::Off);
let original = select(&pool).lease.member.definition.id.clone();
let mut headers = HeaderMap::new();
headers.insert("x-forwarded-for", "10.0.0.99".parse().unwrap());
for ip in ["127.0.0.1", "::ffff:127.0.0.1"] {
for _ in 0..8 {
assert_eq!(
pool.select(ip.parse().unwrap(), &headers, false)
.unwrap()
.lease
.member
.definition
.id,
original
);
}
assert_eq!(counts, [30, 10]);
assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 0);
assert_eq!(pool.members[0].state.requests.load(Ordering::Relaxed), 0); // Mere cache selection isn't origin traffic.
}
#[tokio::test]
async fn least_connections_counts_a_lease_until_its_last_owner_drops() {
let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
let mut held = Vec::new();
for _ in 0..3 {
let selection = select(&pool);
assert_eq!(selection.lease.member.definition.id, "a");
held.push(selection.lease);
}
let other = select(&pool);
assert_eq!(other.lease.member.definition.id, "b");
let clone = held[0].clone();
held.clear();
assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 1);
drop(clone);
assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 0);
}
#[tokio::test]
async fn failover_selection_never_reuses_an_attempted_backend() {
let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
let first = select(&pool);
let first_id = first.lease.member.definition.id.clone();
drop(first);
let retry = pool
.select_retry(
"127.0.0.1".parse().unwrap(),
&HeaderMap::new(),
false,
std::slice::from_ref(&first_id),
)
.unwrap();
assert_ne!(retry.lease.member.definition.id, first_id);
}
#[tokio::test]
async fn backend_concurrency_limit_queues_then_rejects_and_recovers() {
let pool = capacity_pool(1, 10);
let first = select(&pool);
let headers = HeaderMap::new();
let saturated = pool
.select_wait("127.0.0.1".parse().unwrap(), &headers, false)
.await;
assert!(matches!(saturated, Err(SelectionFailure::Capacity)));
let view = pool.view();
let member = &view["members"][0];
assert_eq!(member["capacity_rejections_total"], 1);
assert_eq!(member["queue_timeouts_total"], 1);
assert_eq!(member["active"], 1);
drop(first);
assert!(pool
.select_wait("127.0.0.1".parse().unwrap(), &headers, false)
.await
.is_ok());
}
#[tokio::test]
async fn zero_capacity_queue_timeout_rejects_without_counting_a_queue_timeout() {
let pool = capacity_pool(1, 0);
let first = select(&pool);
let headers = HeaderMap::new();
let saturated = pool
.select_wait("127.0.0.1".parse().unwrap(), &headers, false)
.await;
assert!(matches!(saturated, Err(SelectionFailure::Capacity)));
let view = pool.view();
let member = &view["members"][0];
assert_eq!(member["capacity_rejections_total"], 1);
assert_eq!(member["queue_timeouts_total"], 0);
drop(first);
}
#[test]
fn detailed_backend_failures_are_exposed_without_losing_aggregate_counters() {
let pool = capacity_pool(0, 0);
let selection = select(&pool);
selection.lease.error_detail(
"dns",
"lookup backend.example failed",
Duration::from_millis(7),
);
let view = pool.view();
let member = &view["members"][0];
assert_eq!(member["dns_errors_total"], 1);
assert_eq!(member["connect_errors_total"], 1);
assert_eq!(member["last_failure_class"], "dns");
assert_eq!(
member["last_failure_detail"],
"lookup backend.example failed"
);
assert!(member["last_failure_at_ms"].as_u64().unwrap_or_default() > 0);
}
#[tokio::test]
async fn traffic_and_adaptive_selection_use_observed_cost_and_weight() {
for algorithm in [Algorithm::LeastTraffic, Algorithm::Adaptive] {
let pool = pool(algorithm, StickyMode::Off);
lock(&pool.members[0].state.traffic).rate = 50_000_000.0;
lock(&pool.members[0].state.health).ewma_us = 1_000_000;
assert_eq!(select(&pool).lease.member.definition.id, "b");
}
}
#[tokio::test]
async fn ip_hash_is_stable_and_cannot_be_spoofed_with_forwarded_headers() {
let pool = pool(Algorithm::IpHash, StickyMode::Off);
let original = select(&pool).lease.member.definition.id.clone();
let mut headers = HeaderMap::new();
headers.insert("x-forwarded-for", "10.0.0.99".parse().unwrap());
for ip in ["127.0.0.1", "::ffff:127.0.0.1"] {
for _ in 0..8 {
assert_eq!(
pool.select(ip.parse().unwrap(), &headers, false)
.unwrap()
.lease
.member
.definition
.id,
original
);
}
}
}
src/balance/tests/part_02.rs
+168 -165
View File
@@ -1,170 +1,173 @@
#[tokio::test]
async fn opaque_cookie_is_removed_from_origin_requests_and_protected_on_responses() {
let pool = pool(Algorithm::RoundRobin, StickyMode::Cookie);
let first = select(&pool);
let cookie = first.cookie.unwrap().to_str().unwrap().to_owned();
assert!(
cookie.contains("HttpOnly")
&& cookie.contains("SameSite=Lax")
&& !cookie.contains("Secure")
);
let cookie = cookie.split(';').next().unwrap();
let mut headers = HeaderMap::new();
headers.insert("cookie", format!("{cookie}; app=ok").parse().unwrap());
let next = pool
.select("10.0.0.1".parse().unwrap(), &headers, false)
.unwrap();
assert_eq!(
next.lease.member.definition.id,
first.lease.member.definition.id
);
assert!(next.cookie.is_none());
pool.strip_affinity(&mut headers);
assert_eq!(headers["cookie"], "app=ok");
headers.append(
"set-cookie",
format!("{}=origin-spoof", pool.cookie_name)
.parse()
.unwrap(),
);
headers.append("set-cookie", "app=ok".parse().unwrap());
pool.strip_affinity_response(&mut headers);
assert_eq!(headers.get_all("set-cookie").iter().count(), 1);
assert_eq!(headers["set-cookie"], "app=ok");
let secure = pool
.select("127.0.0.1".parse().unwrap(), &HeaderMap::new(), true)
.unwrap();
assert!(secure
.cookie
.unwrap()
.to_str()
.unwrap()
.contains("; Secure"));
}
#[tokio::test]
async fn failed_sticky_member_is_replaced_and_duplicate_cookies_are_ignored() {
let pool = pool(Algorithm::RoundRobin, StickyMode::Cookie);
let first = select(&pool);
let cookie = first
.cookie
.unwrap()
.to_str()
.unwrap()
.split(';')
.next()
.unwrap()
.to_owned();
let mut headers = HeaderMap::new();
headers.insert("cookie", format!("{cookie}; {cookie}").parse().unwrap());
assert!(pool.affinity(&headers).is_none());
headers.insert("cookie", cookie.parse().unwrap());
lock(&first.lease.member.state.health).healthy = false;
let next = pool
.select("127.0.0.1".parse().unwrap(), &headers, false)
.unwrap();
assert_ne!(
next.lease.member.definition.id,
first.lease.member.definition.id
);
assert!(next.cookie.is_some());
lock(&next.lease.member.state.health).healthy = false;
assert!(pool
.select("127.0.0.1".parse().unwrap(), &headers, false)
.is_none());
}
#[tokio::test]
async fn slow_start_ramps_weight_and_error_is_counted_once_per_exchange() {
let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
let selected = select(&pool);
let state = &selected.lease.member.state;
assert!(state.weight(100, 100) < 100.0);
assert_eq!(state.weight(100, 0), 100.0);
selected.lease.started();
selected.lease.bytes(50, true);
selected.lease.bytes(100, false);
selected.lease.error("timeout", Duration::from_secs(1));
selected.lease.error("body", Duration::from_secs(2));
selected.lease.retry_attempted();
selected.lease.retry_succeeded();
assert_eq!(state.requests.load(Ordering::Relaxed), 1);
assert_eq!(state.errors.load(Ordering::Relaxed), 1);
assert_eq!(state.timeouts_total.load(Ordering::Relaxed), 1);
assert_eq!(state.io_errors_total.load(Ordering::Relaxed), 1);
assert_eq!(state.bytes_in.load(Ordering::Relaxed), 50);
assert_eq!(state.bytes_out.load(Ordering::Relaxed), 100);
assert_eq!(state.retry_attempts_total.load(Ordering::Relaxed), 1);
assert_eq!(state.retry_successes_total.load(Ordering::Relaxed), 1);
assert!(pool
.prometheus()
.contains("rust_proxy_backend_requests_total{vhost=\"app\",backend=\"a\"} 1"));
assert!(pool
.prometheus()
.contains("rust_proxy_backend_retry_attempts_total{vhost=\"app\",backend=\"a\"} 1"));
}
#[tokio::test]
async fn slow_5xx_counts_both_observability_classes_when_passive_is_off() {
let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
let selected = select(&pool);
selected.lease.response(500, Duration::from_secs(20), true);
let state = &selected.lease.member.state;
assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 1);
assert_eq!(state.slow_responses_total.load(Ordering::Relaxed), 1);
assert_eq!(state.passive_health_failures_total.load(Ordering::Relaxed), 0);
assert!(state.healthy());
}
#[tokio::test]
async fn opaque_cookie_is_removed_from_origin_requests_and_protected_on_responses() {
let pool = pool(Algorithm::RoundRobin, StickyMode::Cookie);
let first = select(&pool);
let cookie = first.cookie.unwrap().to_str().unwrap().to_owned();
assert!(
cookie.contains("HttpOnly")
&& cookie.contains("SameSite=Lax")
&& !cookie.contains("Secure")
);
let cookie = cookie.split(';').next().unwrap();
let mut headers = HeaderMap::new();
headers.insert("cookie", format!("{cookie}; app=ok").parse().unwrap());
let next = pool
.select("10.0.0.1".parse().unwrap(), &headers, false)
.unwrap();
assert_eq!(
next.lease.member.definition.id,
first.lease.member.definition.id
);
assert!(next.cookie.is_none());
pool.strip_affinity(&mut headers);
assert_eq!(headers["cookie"], "app=ok");
headers.append(
"set-cookie",
format!("{}=origin-spoof", pool.cookie_name)
.parse()
.unwrap(),
);
headers.append("set-cookie", "app=ok".parse().unwrap());
pool.strip_affinity_response(&mut headers);
assert_eq!(headers.get_all("set-cookie").iter().count(), 1);
assert_eq!(headers["set-cookie"], "app=ok");
let secure = pool
.select("127.0.0.1".parse().unwrap(), &HeaderMap::new(), true)
.unwrap();
assert!(secure
.cookie
.unwrap()
.to_str()
.unwrap()
.contains("; Secure"));
}
#[tokio::test]
async fn failed_sticky_member_is_replaced_and_duplicate_cookies_are_ignored() {
let pool = pool(Algorithm::RoundRobin, StickyMode::Cookie);
let first = select(&pool);
let cookie = first
.cookie
.unwrap()
.to_str()
.unwrap()
.split(';')
.next()
.unwrap()
.to_owned();
let mut headers = HeaderMap::new();
headers.insert("cookie", format!("{cookie}; {cookie}").parse().unwrap());
assert!(pool.affinity(&headers).is_none());
headers.insert("cookie", cookie.parse().unwrap());
lock(&first.lease.member.state.health).healthy = false;
let next = pool
.select("127.0.0.1".parse().unwrap(), &headers, false)
.unwrap();
assert_ne!(
next.lease.member.definition.id,
first.lease.member.definition.id
);
assert!(next.cookie.is_some());
lock(&next.lease.member.state.health).healthy = false;
assert!(pool
.select("127.0.0.1".parse().unwrap(), &headers, false)
.is_none());
}
#[tokio::test]
async fn slow_start_ramps_weight_and_error_is_counted_once_per_exchange() {
let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
let selected = select(&pool);
let state = &selected.lease.member.state;
assert!(state.weight(100, 100) < 100.0);
assert_eq!(state.weight(100, 0), 100.0);
selected.lease.started();
selected.lease.bytes(50, true);
selected.lease.bytes(100, false);
selected.lease.error("timeout", Duration::from_secs(1));
selected.lease.error("body", Duration::from_secs(2));
selected.lease.retry_attempted();
selected.lease.retry_succeeded();
assert_eq!(state.requests.load(Ordering::Relaxed), 1);
assert_eq!(state.errors.load(Ordering::Relaxed), 1);
assert_eq!(state.timeouts_total.load(Ordering::Relaxed), 1);
assert_eq!(state.io_errors_total.load(Ordering::Relaxed), 1);
assert_eq!(state.bytes_in.load(Ordering::Relaxed), 50);
assert_eq!(state.bytes_out.load(Ordering::Relaxed), 100);
assert_eq!(state.retry_attempts_total.load(Ordering::Relaxed), 1);
assert_eq!(state.retry_successes_total.load(Ordering::Relaxed), 1);
assert!(pool
.prometheus()
.contains("rust_proxy_backend_requests_total{vhost=\"app\",backend=\"a\"} 1"));
assert!(pool
.prometheus()
.contains("rust_proxy_backend_retry_attempts_total{vhost=\"app\",backend=\"a\"} 1"));
}
#[tokio::test]
async fn slow_5xx_counts_both_observability_classes_when_passive_is_off() {
let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
let selected = select(&pool);
selected.lease.response(500, Duration::from_secs(20), true);
let state = &selected.lease.member.state;
assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 1);
assert_eq!(state.slow_responses_total.load(Ordering::Relaxed), 1);
assert_eq!(
state.passive_health_failures_total.load(Ordering::Relaxed),
0
);
assert!(state.healthy());
}
#[tokio::test]
async fn client_upload_duration_does_not_trigger_passive_slow_failure() {
let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
let upload = select(&pool);
upload.lease.response(200, Duration::from_secs(20), false);
assert_eq!(upload.lease.member.state.errors.load(Ordering::Relaxed), 0);
let read = select(&pool);
read.lease.response(200, Duration::from_secs(20), true);
assert_eq!(read.lease.member.state.errors.load(Ordering::Relaxed), 1);
}
#[tokio::test]
async fn backup_is_only_new_assignment_when_no_primary_is_eligible() {
let mut pool = pool(Algorithm::RoundRobin, StickyMode::Off);
Arc::get_mut(&mut Arc::get_mut(&mut pool).unwrap().members[1])
.unwrap()
.definition
.backup = true;
for _ in 0..8 {
assert_eq!(select(&pool).lease.member.definition.id, "a");
}
lock(&pool.members[0].state.health).healthy = false;
assert_eq!(select(&pool).lease.member.definition.id, "b");
lock(&pool.members[0].state.health).healthy = true;
#[tokio::test]
async fn client_upload_duration_does_not_trigger_passive_slow_failure() {
let pool = pool(Algorithm::LeastConnections, StickyMode::Off);
let upload = select(&pool);
upload.lease.response(200, Duration::from_secs(20), false);
assert_eq!(upload.lease.member.state.errors.load(Ordering::Relaxed), 0);
let read = select(&pool);
read.lease.response(200, Duration::from_secs(20), true);
assert_eq!(read.lease.member.state.errors.load(Ordering::Relaxed), 1);
}
#[tokio::test]
async fn backup_is_only_new_assignment_when_no_primary_is_eligible() {
let mut pool = pool(Algorithm::RoundRobin, StickyMode::Off);
Arc::get_mut(&mut Arc::get_mut(&mut pool).unwrap().members[1])
.unwrap()
.definition
.backup = true;
for _ in 0..8 {
assert_eq!(select(&pool).lease.member.definition.id, "a");
}
#[tokio::test]
async fn drain_rejects_new_sessions_but_preserves_valid_existing_affinity() {
let mut pool = pool(Algorithm::RoundRobin, StickyMode::Cookie);
Arc::get_mut(&mut Arc::get_mut(&mut pool).unwrap().members[0])
lock(&pool.members[0].state.health).healthy = false;
assert_eq!(select(&pool).lease.member.definition.id, "b");
lock(&pool.members[0].state.health).healthy = true;
assert_eq!(select(&pool).lease.member.definition.id, "a");
}
#[tokio::test]
async fn drain_rejects_new_sessions_but_preserves_valid_existing_affinity() {
let mut pool = pool(Algorithm::RoundRobin, StickyMode::Cookie);
Arc::get_mut(&mut Arc::get_mut(&mut pool).unwrap().members[0])
.unwrap()
.definition
.drain = true;
assert_eq!(select(&pool).lease.member.definition.id, "b");
let mut headers = HeaderMap::new();
headers.insert(
"cookie",
format!("{}={}", pool.cookie_name, pool.members[0].token)
.parse()
.unwrap(),
);
let existing = pool
.select("127.0.0.1".parse().unwrap(), &headers, false)
.unwrap();
assert_eq!(existing.lease.member.definition.id, "a");
lock(&pool.members[0].state.health).healthy = false;
assert_eq!(
pool.select("127.0.0.1".parse().unwrap(), &headers, false)
.unwrap()
.lease
.member
.definition
.drain = true;
assert_eq!(select(&pool).lease.member.definition.id, "b");
let mut headers = HeaderMap::new();
headers.insert(
"cookie",
format!("{}={}", pool.cookie_name, pool.members[0].token)
.parse()
.unwrap(),
);
let existing = pool
.select("127.0.0.1".parse().unwrap(), &headers, false)
.unwrap();
assert_eq!(existing.lease.member.definition.id, "a");
lock(&pool.members[0].state.health).healthy = false;
assert_eq!(
pool.select("127.0.0.1".parse().unwrap(), &headers, false)
.unwrap()
.lease
.member
.definition
.id,
"b"
);
}
.id,
"b"
);
}