0.32.7

src/balance/health.rs

@@ -46,6 +46,12 @@ impl BackendState {
                 },
                 passive_unhealthy_source: "none",
                 passive_unhealthy_reason: "none",
+                passive_first_failure: None,
+                active_first_failure: None,
+                passive_probe_successes: 0,
+                passive_ejected_at: None,
+                passive_cooldown: Duration::from_secs(5),
+                last_io_counted_at: None,
             }),
             traffic: Mutex::new(Traffic {
                 at: now,
@@ -105,7 +111,29 @@ impl BackendState {
         }
     }
     pub fn healthy(&self) -> bool {
-        lock(&self.health).healthy
+        let mut h = lock(&self.health);
+        // Passive-only monitoring has no active probes to recover an ejected
+        // backend. After a bounded quarantine, allow one new traffic attempt.
+        if !h.passive_healthy && h.active_healthy && !h.checked {
+            if h.passive_ejected_at
+                .is_some_and(|at| at.elapsed() >= h.passive_cooldown)
+            {
+                let previous = h.healthy;
+                Self::clear_passive_ejection(&mut h);
+                self.finish_transition(&mut h, previous);
+            }
+        }
+        h.healthy
+    }
+    fn clear_passive_ejection(h: &mut Health) {
+        h.passive_healthy = true;
+        h.passive_failures = 0;
+        h.passive_successes = 0;
+        h.passive_probe_successes = 0;
+        h.passive_first_failure = None;
+        h.passive_ejected_at = None;
+        h.passive_unhealthy_source = "none";
+        h.passive_unhealthy_reason = "none";
     }
     fn weight(&self, configured: u16, slow_start: u64) -> f64 {
         let h = lock(&self.health);
@@ -212,12 +240,27 @@ impl BackendState {
         h.last_probe_ms = crate::logging::now_ms();
         h.result = result;
         if result == "ok" {
+            h.active_first_failure = None;
             h.failures = 0;
             h.successes = h.successes.saturating_add(1);
             if h.successes >= settings.rise {
                 h.active_healthy = true;
             }
+            if !h.passive_healthy && h.passive_ejected_at.is_some() && h.active_healthy {
+                h.passive_probe_successes = h.passive_probe_successes.saturating_add(1);
+                if h.passive_probe_successes >= settings.rise.max(2) {
+                    Self::clear_passive_ejection(&mut h);
+                }
+            }
         } else {
+            h.passive_probe_successes = 0;
+            if h.active_first_failure.is_none() {
+                h.active_first_failure = Some(FailureObservation {
+                    at_ms: crate::logging::now_ms(),
+                    class: "active_health",
+                    detail: result.into(),
+                });
+            }
             self.active_health_failures_total
                 .fetch_add(1, Ordering::Relaxed);
             h.successes = 0;
@@ -248,6 +291,10 @@ impl BackendState {
         self.finish_transition(&mut h, previous);
     }
 
+    fn first_active_failure(&self) -> Option<FailureObservation> {
+        lock(&self.health).active_first_failure.clone()
+    }
+
     fn record_failure(&self, class: &'static str, detail: &str) {
         match class {
             "dns" => {
@@ -303,37 +350,69 @@ impl BackendState {
         reason: &'static str,
         elapsed: Duration,
     ) {
+        self.observe_passive_with_detail(active, passive, reason, reason, reason, elapsed);
+    }
+
+    /// Returns the first failure in a consecutive counted streak only when
+    /// this observation actually removes the backend from the traffic pool.
+    fn observe_passive_with_detail(
+        &self,
+        active: &HealthCheck,
+        passive: &PassiveHealth,
+        reason: &'static str,
+        class: &'static str,
+        detail: &str,
+        elapsed: Duration,
+    ) -> Option<(FailureObservation, u16)> {
         self.record_passive_observation(reason);
-        {
-            let mut h = lock(&self.health);
-            Self::update_latency(&mut h, elapsed);
-            h.passive_result = reason;
-        }
-        if !passive.enabled {
-            return;
-        }
         let mut h = lock(&self.health);
+        Self::update_latency(&mut h, elapsed);
+        h.passive_result = reason;
+        if !passive.enabled {
+            return None;
+        }
         let previous = h.healthy;
         if reason == "ok" {
             h.passive_failures = 0;
+            h.passive_first_failure = None;
+            h.last_io_counted_at = None;
             h.passive_successes = h.passive_successes.saturating_add(1);
             if h.passive_successes >= passive.rise(active) {
-                h.passive_healthy = true;
-                h.passive_unhealthy_source = "none";
-                h.passive_unhealthy_reason = "none";
+                Self::clear_passive_ejection(&mut h);
             }
             self.finish_transition(&mut h, previous);
-            return;
+            return None;
         }
         if !passive.counts(reason) {
-            return;
+            return None;
+        }
+        // A fan-out of failures from one expired H2 connection must not count
+        // as independent failed backend observations in a single instant.
+        if reason == "io_error" {
+            let now = Instant::now();
+            if h.last_io_counted_at
+                .is_some_and(|last| now.duration_since(last) < Duration::from_millis(250))
+            {
+                return None;
+            }
+            h.last_io_counted_at = Some(now);
         }
         self.passive_health_failures_total
             .fetch_add(1, Ordering::Relaxed);
+        if h.passive_failures == 0 {
+            h.passive_first_failure = Some(FailureObservation {
+                at_ms: crate::logging::now_ms(),
+                class,
+                detail: detail.chars().take(1024).collect(),
+            });
+        }
         h.passive_successes = 0;
+        h.passive_probe_successes = 0;
         h.passive_failures = h.passive_failures.saturating_add(1);
-        if h.passive_failures >= passive.fall(active) {
+        if h.passive_failures >= passive.fall(active) && h.passive_healthy {
             h.passive_healthy = false;
+            h.passive_ejected_at = Some(Instant::now());
+            h.passive_cooldown = Duration::from_millis(active.interval_ms.clamp(5_000, 30_000));
         }
         let source = match reason {
             "connect_error" => "passive_connect_error",
@@ -352,6 +431,13 @@ impl BackendState {
             }
         }
         self.finish_transition(&mut h, previous);
+        if previous && !h.healthy {
+            h.passive_first_failure
+                .clone()
+                .map(|first| (first, h.passive_failures))
+        } else {
+            None
+        }
     }
 }
 #[derive(Debug)]