0.32.7

src/balance/health.rs

@@ -46,6 +46,12 @@ impl BackendState {
                 },
                 passive_unhealthy_source: "none",
                 passive_unhealthy_reason: "none",
+                passive_first_failure: None,
+                active_first_failure: None,
+                passive_probe_successes: 0,
+                passive_ejected_at: None,
+                passive_cooldown: Duration::from_secs(5),
+                last_io_counted_at: None,
             }),
             traffic: Mutex::new(Traffic {
                 at: now,
@@ -105,7 +111,29 @@ impl BackendState {
         }
     }
     pub fn healthy(&self) -> bool {
-        lock(&self.health).healthy
+        let mut h = lock(&self.health);
+        // Passive-only monitoring has no active probes to recover an ejected
+        // backend. After a bounded quarantine, allow one new traffic attempt.
+        if !h.passive_healthy && h.active_healthy && !h.checked {
+            if h.passive_ejected_at
+                .is_some_and(|at| at.elapsed() >= h.passive_cooldown)
+            {
+                let previous = h.healthy;
+                Self::clear_passive_ejection(&mut h);
+                self.finish_transition(&mut h, previous);
+            }
+        }
+        h.healthy
+    }
+    fn clear_passive_ejection(h: &mut Health) {
+        h.passive_healthy = true;
+        h.passive_failures = 0;
+        h.passive_successes = 0;
+        h.passive_probe_successes = 0;
+        h.passive_first_failure = None;
+        h.passive_ejected_at = None;
+        h.passive_unhealthy_source = "none";
+        h.passive_unhealthy_reason = "none";
     }
     fn weight(&self, configured: u16, slow_start: u64) -> f64 {
         let h = lock(&self.health);
@@ -212,12 +240,27 @@ impl BackendState {
         h.last_probe_ms = crate::logging::now_ms();
         h.result = result;
         if result == "ok" {
+            h.active_first_failure = None;
             h.failures = 0;
             h.successes = h.successes.saturating_add(1);
             if h.successes >= settings.rise {
                 h.active_healthy = true;
             }
+            if !h.passive_healthy && h.passive_ejected_at.is_some() && h.active_healthy {
+                h.passive_probe_successes = h.passive_probe_successes.saturating_add(1);
+                if h.passive_probe_successes >= settings.rise.max(2) {
+                    Self::clear_passive_ejection(&mut h);
+                }
+            }
         } else {
+            h.passive_probe_successes = 0;
+            if h.active_first_failure.is_none() {
+                h.active_first_failure = Some(FailureObservation {
+                    at_ms: crate::logging::now_ms(),
+                    class: "active_health",
+                    detail: result.into(),
+                });
+            }
             self.active_health_failures_total
                 .fetch_add(1, Ordering::Relaxed);
             h.successes = 0;
@@ -248,6 +291,10 @@ impl BackendState {
         self.finish_transition(&mut h, previous);
     }
 
+    fn first_active_failure(&self) -> Option<FailureObservation> {
+        lock(&self.health).active_first_failure.clone()
+    }
+
     fn record_failure(&self, class: &'static str, detail: &str) {
         match class {
             "dns" => {
@@ -303,37 +350,69 @@ impl BackendState {
         reason: &'static str,
         elapsed: Duration,
     ) {
+        self.observe_passive_with_detail(active, passive, reason, reason, reason, elapsed);
+    }
+
+    /// Returns the first failure in a consecutive counted streak only when
+    /// this observation actually removes the backend from the traffic pool.
+    fn observe_passive_with_detail(
+        &self,
+        active: &HealthCheck,
+        passive: &PassiveHealth,
+        reason: &'static str,
+        class: &'static str,
+        detail: &str,
+        elapsed: Duration,
+    ) -> Option<(FailureObservation, u16)> {
         self.record_passive_observation(reason);
-        {
-            let mut h = lock(&self.health);
-            Self::update_latency(&mut h, elapsed);
-            h.passive_result = reason;
-        }
-        if !passive.enabled {
-            return;
-        }
         let mut h = lock(&self.health);
+        Self::update_latency(&mut h, elapsed);
+        h.passive_result = reason;
+        if !passive.enabled {
+            return None;
+        }
         let previous = h.healthy;
         if reason == "ok" {
             h.passive_failures = 0;
+            h.passive_first_failure = None;
+            h.last_io_counted_at = None;
             h.passive_successes = h.passive_successes.saturating_add(1);
             if h.passive_successes >= passive.rise(active) {
-                h.passive_healthy = true;
-                h.passive_unhealthy_source = "none";
-                h.passive_unhealthy_reason = "none";
+                Self::clear_passive_ejection(&mut h);
             }
             self.finish_transition(&mut h, previous);
-            return;
+            return None;
         }
         if !passive.counts(reason) {
-            return;
+            return None;
+        }
+        // A fan-out of failures from one expired H2 connection must not count
+        // as independent failed backend observations in a single instant.
+        if reason == "io_error" {
+            let now = Instant::now();
+            if h.last_io_counted_at
+                .is_some_and(|last| now.duration_since(last) < Duration::from_millis(250))
+            {
+                return None;
+            }
+            h.last_io_counted_at = Some(now);
         }
         self.passive_health_failures_total
             .fetch_add(1, Ordering::Relaxed);
+        if h.passive_failures == 0 {
+            h.passive_first_failure = Some(FailureObservation {
+                at_ms: crate::logging::now_ms(),
+                class,
+                detail: detail.chars().take(1024).collect(),
+            });
+        }
         h.passive_successes = 0;
+        h.passive_probe_successes = 0;
         h.passive_failures = h.passive_failures.saturating_add(1);
-        if h.passive_failures >= passive.fall(active) {
+        if h.passive_failures >= passive.fall(active) && h.passive_healthy {
             h.passive_healthy = false;
+            h.passive_ejected_at = Some(Instant::now());
+            h.passive_cooldown = Duration::from_millis(active.interval_ms.clamp(5_000, 30_000));
         }
         let source = match reason {
             "connect_error" => "passive_connect_error",
@@ -352,6 +431,13 @@ impl BackendState {
             }
         }
         self.finish_transition(&mut h, previous);
+        if previous && !h.healthy {
+            h.passive_first_failure
+                .clone()
+                .map(|first| (first, h.passive_failures))
+        } else {
+            None
+        }
     }
 }
 #[derive(Debug)]

src/balance/lease_runtime.rs

@@ -40,9 +40,26 @@ impl Lease {
             "ok"
         };
         let before = self.member.state.healthy();
-        self.member
-            .state
-            .observe_passive(&self.health, &self.passive_health, reason, elapsed);
+        let failure_detail = if is_5xx {
+            format!("status={status}")
+        } else {
+            reason.into()
+        };
+        let ejected = self.member.state.observe_passive_with_detail(
+            &self.health,
+            &self.passive_health,
+            reason,
+            reason,
+            &failure_detail,
+            elapsed,
+        );
+        if let Some((first, failures)) = ejected {
+            tracing::warn!(backend = %self.member.definition.id, origin = %self.member.backend.origin, source = "passive",
+                reason, first_error_class = first.class,
+                first_error_at_ms = first.at_ms, first_error_detail = %first.detail,
+                trigger_error_detail = %failure_detail, failures,
+                "backend excluded from traffic");
+        }
         let after = self.member.state.healthy();
         if before != after {
             tracing::info!(
@@ -62,6 +79,11 @@ impl Lease {
             self.member.state.errors.fetch_add(1, Ordering::Relaxed);
         }
         self.member.state.record_failure(class, detail);
+        // GOAWAY(NO_ERROR) is a graceful H2 connection shutdown, not an
+        // origin failure. It can still be surfaced in access diagnostics.
+        if matches!(class, "h2_graceful_goaway" | "h2_graceful_close") {
+            return;
+        }
         let passive_reason = match class {
             "connect" | "dns" | "tls" => "connect_error",
             "timeout"
@@ -72,12 +94,21 @@ impl Lease {
             _ => "io_error",
         };
         let before = self.member.state.healthy();
-        self.member.state.observe_passive(
+        let ejected = self.member.state.observe_passive_with_detail(
             &self.health,
             &self.passive_health,
             passive_reason,
+            class,
+            detail,
             elapsed,
         );
+        if let Some((first, failures)) = ejected {
+            tracing::warn!(backend = %self.member.definition.id, origin = %self.member.backend.origin, source = "passive",
+                reason = passive_reason, first_error_class = first.class,
+                first_error_at_ms = first.at_ms, first_error_detail = %first.detail,
+                trigger_error_detail = %detail, failures,
+                "backend excluded from traffic");
+        }
         let after = self.member.state.healthy();
         if before != after {
             tracing::info!(
@@ -326,8 +357,18 @@ async fn probe(member: Arc<Member>, settings: HealthCheck) {
     };
     let before = member.state.healthy();
     member.state.observe_active(&settings, result, elapsed);
-    if before != member.state.healthy() {
-        tracing::info!(backend = %member.definition.id, healthy = member.state.healthy(), reason = result, "backend health changed");
+    let after = member.state.healthy();
+    if before && !after {
+        let first = member.state.first_active_failure();
+        tracing::warn!(backend = %member.definition.id, origin = %member.backend.origin, source = "active",
+            reason = result,
+            first_error_class = first.as_ref().map_or("active_health", |f| f.class),
+            first_error_at_ms = first.as_ref().map_or(0, |f| f.at_ms),
+            first_error_detail = %first.as_ref().map_or(result, |f| f.detail.as_str()),
+            "backend excluded from traffic");
+    }
+    if before != after {
+        tracing::info!(backend = %member.definition.id, healthy = after, reason = result, "backend health changed");
     }
 }
 /// One supervisor, at most 16 probes, at most one outstanding probe per member.

src/balance/model.rs

@@ -355,7 +355,14 @@ struct Health {
     active_unhealthy_reason: &'static str,
     passive_unhealthy_source: &'static str,
     passive_unhealthy_reason: &'static str,
+    passive_first_failure: Option<FailureObservation>,
+    active_first_failure: Option<FailureObservation>,
+    passive_probe_successes: u16,
+    passive_ejected_at: Option<Instant>,
+    passive_cooldown: Duration,
+    last_io_counted_at: Option<Instant>,
 }
+
 #[derive(Debug)]
 struct Traffic {
     at: Instant,