v0.25.1
This commit is contained in:
Mateusz Gruszczyński committed
1 parent
87e571c170
commit
fa7abc9e53
308 files changed
+49746
-50237
No files matched your search
@@ -0,0 +1,353 @@
|
||||
impl BackendState {
|
||||
fn new(mode: HealthMode) -> Self {
|
||||
let now = Instant::now();
|
||||
Self {
|
||||
health: Mutex::new(Health {
|
||||
healthy: mode == HealthMode::Off,
|
||||
active_healthy: mode == HealthMode::Off,
|
||||
passive_healthy: true,
|
||||
checked: false,
|
||||
failures: 0,
|
||||
passive_failures: 0,
|
||||
successes: 0,
|
||||
passive_successes: 0,
|
||||
next_probe: now,
|
||||
probing: false,
|
||||
healthy_since: now,
|
||||
result: if mode == HealthMode::Off {
|
||||
"disabled"
|
||||
} else {
|
||||
"checking"
|
||||
},
|
||||
passive_result: "none",
|
||||
last_probe_ms: 0,
|
||||
latency_us: 0,
|
||||
ewma_us: 0,
|
||||
transitions: 0,
|
||||
unhealthy_source: if mode == HealthMode::Off { "none" } else { "active" },
|
||||
unhealthy_reason: if mode == HealthMode::Off { "none" } else { "checking" },
|
||||
active_unhealthy_source: if mode == HealthMode::Off { "none" } else { "active" },
|
||||
active_unhealthy_reason: if mode == HealthMode::Off { "none" } else { "checking" },
|
||||
passive_unhealthy_source: "none",
|
||||
passive_unhealthy_reason: "none",
|
||||
}),
|
||||
traffic: Mutex::new(Traffic {
|
||||
at: now,
|
||||
total: 0,
|
||||
rate: 0.0,
|
||||
}),
|
||||
last_failure: Mutex::new(FailureObservation::default()),
|
||||
active: AtomicU64::new(0),
|
||||
requests: AtomicU64::new(0),
|
||||
errors: AtomicU64::new(0),
|
||||
bytes_in: AtomicU64::new(0),
|
||||
bytes_out: AtomicU64::new(0),
|
||||
last_status: AtomicU16::new(0),
|
||||
active_health_failures_total: AtomicU64::new(0),
|
||||
passive_health_failures_total: AtomicU64::new(0),
|
||||
http_5xx_total: AtomicU64::new(0),
|
||||
timeouts_total: AtomicU64::new(0),
|
||||
connect_errors_total: AtomicU64::new(0),
|
||||
io_errors_total: AtomicU64::new(0),
|
||||
slow_responses_total: AtomicU64::new(0),
|
||||
retry_attempts_total: AtomicU64::new(0),
|
||||
retry_successes_total: AtomicU64::new(0),
|
||||
capacity_rejections_total: AtomicU64::new(0),
|
||||
queue_timeouts_total: AtomicU64::new(0),
|
||||
dns_errors_total: AtomicU64::new(0),
|
||||
tls_errors_total: AtomicU64::new(0),
|
||||
response_header_timeouts_total: AtomicU64::new(0),
|
||||
read_timeouts_total: AtomicU64::new(0),
|
||||
send_timeouts_total: AtomicU64::new(0),
|
||||
last_ejected_ms: AtomicU64::new(0),
|
||||
last_recovered_ms: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
fn has_capacity(&self, limit: u32) -> bool {
|
||||
limit == 0 || self.active.load(Ordering::Relaxed) < u64::from(limit)
|
||||
}
|
||||
fn try_acquire_active(&self, limit: u32) -> bool {
|
||||
if limit == 0 {
|
||||
self.active.fetch_add(1, Ordering::Relaxed);
|
||||
return true;
|
||||
}
|
||||
let limit = u64::from(limit);
|
||||
let mut current = self.active.load(Ordering::Relaxed);
|
||||
loop {
|
||||
if current >= limit {
|
||||
return false;
|
||||
}
|
||||
match self.active.compare_exchange_weak(
|
||||
current,
|
||||
current + 1,
|
||||
Ordering::AcqRel,
|
||||
Ordering::Relaxed,
|
||||
) {
|
||||
Ok(_) => return true,
|
||||
Err(next) => current = next,
|
||||
}
|
||||
}
|
||||
}
|
||||
pub fn healthy(&self) -> bool {
|
||||
lock(&self.health).healthy
|
||||
}
|
||||
fn weight(&self, configured: u16, slow_start: u64) -> f64 {
|
||||
let h = lock(&self.health);
|
||||
if slow_start == 0 {
|
||||
return configured as f64;
|
||||
}
|
||||
let ratio = (h.healthy_since.elapsed().as_secs_f64() / slow_start as f64).clamp(0.05, 1.0);
|
||||
(configured as f64 * ratio).max(0.05)
|
||||
}
|
||||
fn latency(&self) -> f64 {
|
||||
lock(&self.health).ewma_us.max(1000) as f64 / 1000.0
|
||||
}
|
||||
fn traffic_rate(&self) -> f64 {
|
||||
lock(&self.traffic).rate
|
||||
}
|
||||
fn sample_traffic(&self) {
|
||||
let mut sample = lock(&self.traffic);
|
||||
let elapsed = sample.at.elapsed().as_secs_f64();
|
||||
if elapsed < 1.0 {
|
||||
return;
|
||||
}
|
||||
let total = self
|
||||
.bytes_in
|
||||
.load(Ordering::Relaxed)
|
||||
.saturating_add(self.bytes_out.load(Ordering::Relaxed));
|
||||
let delta = total.saturating_sub(sample.total);
|
||||
if delta == 0 {
|
||||
// No bytes moved in this sampling window: report an actual idle backend
|
||||
// instead of keeping a decaying EWMA tail that looks like live traffic.
|
||||
sample.rate = 0.0;
|
||||
} else {
|
||||
let rate = delta as f64 / elapsed;
|
||||
sample.rate = sample.rate * 0.5 + rate * 0.5;
|
||||
}
|
||||
sample.total = total;
|
||||
sample.at = Instant::now();
|
||||
}
|
||||
fn claim_probe(&self, h: &HealthCheck) -> bool {
|
||||
if h.mode == HealthMode::Off {
|
||||
return false;
|
||||
}
|
||||
let mut state = lock(&self.health);
|
||||
if state.probing || Instant::now() < state.next_probe {
|
||||
return false;
|
||||
}
|
||||
state.probing = true;
|
||||
state.next_probe = Instant::now() + Duration::from_millis(h.interval_ms);
|
||||
true
|
||||
}
|
||||
fn update_latency(h: &mut Health, elapsed: Duration) {
|
||||
let latency = elapsed.as_micros().min(u64::MAX as u128) as u64;
|
||||
h.latency_us = latency;
|
||||
h.ewma_us = if h.ewma_us == 0 {
|
||||
latency
|
||||
} else {
|
||||
h.ewma_us.saturating_mul(7).saturating_add(latency) / 8
|
||||
};
|
||||
}
|
||||
|
||||
fn finish_transition(&self, h: &mut Health, previous: bool) {
|
||||
h.healthy = h.active_healthy && h.passive_healthy;
|
||||
if previous != h.healthy {
|
||||
h.transitions = h.transitions.saturating_add(1);
|
||||
if h.healthy {
|
||||
self.last_recovered_ms
|
||||
.store(crate::logging::now_ms(), Ordering::Relaxed);
|
||||
h.healthy_since = Instant::now();
|
||||
h.unhealthy_source = "none";
|
||||
h.unhealthy_reason = "none";
|
||||
return;
|
||||
} else {
|
||||
self.last_ejected_ms
|
||||
.store(crate::logging::now_ms(), Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
if h.healthy {
|
||||
return;
|
||||
}
|
||||
|
||||
let current_source_is_still_unhealthy =
|
||||
(h.unhealthy_source.starts_with("active") && !h.active_healthy)
|
||||
|| (h.unhealthy_source.starts_with("passive") && !h.passive_healthy);
|
||||
if current_source_is_still_unhealthy {
|
||||
return;
|
||||
}
|
||||
if !h.active_healthy {
|
||||
h.unhealthy_source = h.active_unhealthy_source;
|
||||
h.unhealthy_reason = h.active_unhealthy_reason;
|
||||
} else if !h.passive_healthy {
|
||||
h.unhealthy_source = h.passive_unhealthy_source;
|
||||
h.unhealthy_reason = h.passive_unhealthy_reason;
|
||||
}
|
||||
}
|
||||
|
||||
fn observe_active(&self, settings: &HealthCheck, result: &'static str, elapsed: Duration) {
|
||||
if settings.mode == HealthMode::Off {
|
||||
return;
|
||||
}
|
||||
let mut h = lock(&self.health);
|
||||
let previous = h.healthy;
|
||||
Self::update_latency(&mut h, elapsed);
|
||||
h.probing = false;
|
||||
h.checked = true;
|
||||
h.last_probe_ms = crate::logging::now_ms();
|
||||
h.result = result;
|
||||
if result == "ok" {
|
||||
h.failures = 0;
|
||||
h.successes = h.successes.saturating_add(1);
|
||||
if h.successes >= settings.rise {
|
||||
h.active_healthy = true;
|
||||
}
|
||||
} else {
|
||||
self.active_health_failures_total.fetch_add(1, Ordering::Relaxed);
|
||||
h.successes = 0;
|
||||
h.failures = h.failures.saturating_add(1);
|
||||
if h.failures >= settings.fall {
|
||||
h.active_healthy = false;
|
||||
}
|
||||
}
|
||||
let source = match settings.mode {
|
||||
HealthMode::Tcp => "active_tcp",
|
||||
HealthMode::Http => "active_http",
|
||||
HealthMode::Off => "active",
|
||||
};
|
||||
if result != "ok" {
|
||||
self.record_failure("active_health", &format!("{source}:{result}"));
|
||||
}
|
||||
if h.active_healthy {
|
||||
h.active_unhealthy_source = "none";
|
||||
h.active_unhealthy_reason = "none";
|
||||
} else {
|
||||
h.active_unhealthy_source = source;
|
||||
h.active_unhealthy_reason = result;
|
||||
if previous || h.unhealthy_source == "active" || h.unhealthy_source == "none" {
|
||||
h.unhealthy_source = source;
|
||||
h.unhealthy_reason = result;
|
||||
}
|
||||
}
|
||||
self.finish_transition(&mut h, previous);
|
||||
}
|
||||
|
||||
fn record_failure(&self, class: &'static str, detail: &str) {
|
||||
match class {
|
||||
"dns" => {
|
||||
self.dns_errors_total.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
"tls" => {
|
||||
self.tls_errors_total.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
"response_header_timeout" => {
|
||||
self.response_header_timeouts_total
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
"read_timeout" => {
|
||||
self.read_timeouts_total.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
"send_timeout" => {
|
||||
self.send_timeouts_total.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
let mut last = lock(&self.last_failure);
|
||||
last.at_ms = crate::logging::now_ms();
|
||||
last.class = class;
|
||||
last.detail.clear();
|
||||
last.detail.extend(detail.chars().take(1024));
|
||||
}
|
||||
|
||||
fn record_passive_observation(&self, reason: &'static str) {
|
||||
match reason {
|
||||
"http_5xx" => { self.http_5xx_total.fetch_add(1, Ordering::Relaxed); }
|
||||
"timeout" => { self.timeouts_total.fetch_add(1, Ordering::Relaxed); }
|
||||
"connect_error" => { self.connect_errors_total.fetch_add(1, Ordering::Relaxed); }
|
||||
"io_error" => { self.io_errors_total.fetch_add(1, Ordering::Relaxed); }
|
||||
"slow_response" => { self.slow_responses_total.fetch_add(1, Ordering::Relaxed); }
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn observe_passive(
|
||||
&self,
|
||||
active: &HealthCheck,
|
||||
passive: &PassiveHealth,
|
||||
reason: &'static str,
|
||||
elapsed: Duration,
|
||||
) {
|
||||
self.record_passive_observation(reason);
|
||||
{
|
||||
let mut h = lock(&self.health);
|
||||
Self::update_latency(&mut h, elapsed);
|
||||
h.passive_result = reason;
|
||||
}
|
||||
if !passive.enabled {
|
||||
return;
|
||||
}
|
||||
let mut h = lock(&self.health);
|
||||
let previous = h.healthy;
|
||||
if reason == "ok" {
|
||||
h.passive_failures = 0;
|
||||
h.passive_successes = h.passive_successes.saturating_add(1);
|
||||
if h.passive_successes >= passive.rise(active) {
|
||||
h.passive_healthy = true;
|
||||
h.passive_unhealthy_source = "none";
|
||||
h.passive_unhealthy_reason = "none";
|
||||
}
|
||||
self.finish_transition(&mut h, previous);
|
||||
return;
|
||||
}
|
||||
if !passive.counts(reason) {
|
||||
return;
|
||||
}
|
||||
self.passive_health_failures_total.fetch_add(1, Ordering::Relaxed);
|
||||
h.passive_successes = 0;
|
||||
h.passive_failures = h.passive_failures.saturating_add(1);
|
||||
if h.passive_failures >= passive.fall(active) {
|
||||
h.passive_healthy = false;
|
||||
}
|
||||
let source = match reason {
|
||||
"connect_error" => "passive_connect_error",
|
||||
"timeout" => "passive_timeout",
|
||||
"io_error" => "passive_io_error",
|
||||
"http_5xx" => "passive_http_5xx",
|
||||
"slow_response" => "passive_slow",
|
||||
_ => "passive",
|
||||
};
|
||||
if !h.passive_healthy {
|
||||
h.passive_unhealthy_source = source;
|
||||
h.passive_unhealthy_reason = reason;
|
||||
if previous {
|
||||
h.unhealthy_source = source;
|
||||
h.unhealthy_reason = reason;
|
||||
}
|
||||
}
|
||||
self.finish_transition(&mut h, previous);
|
||||
}
|
||||
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub struct Member {
|
||||
pub backend: Arc<CompiledBackend>,
|
||||
pub definition: PoolMember,
|
||||
pub state: Arc<BackendState>,
|
||||
token: String,
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub struct Pool {
|
||||
pub id: String,
|
||||
pub settings: Settings,
|
||||
pub members: Vec<Arc<Member>>,
|
||||
pub fingerprint: Vec<u8>,
|
||||
cookie_name: String,
|
||||
cursor: AtomicU64,
|
||||
scheduling: Mutex<std::collections::HashMap<String, f64>>,
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub struct Lease {
|
||||
pub member: Arc<Member>,
|
||||
health: HealthCheck,
|
||||
passive_health: PassiveHealth,
|
||||
error_recorded: AtomicBool,
|
||||
}
|
||||
Reference in new issue
Block a user