0.32.7

src/admin/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.0.3",
   "info": {
     "title": "RustProxy Administration API",
-    "version": "0.32.6",
+    "version": "0.32.7",
     "description": "Administration API for the active panel origin. Sign in through the panel; the same-origin HttpOnly cookie authenticates all private operations. Mutations also require the exact Origin, X-RustProxy-Request: 1 and a session-bound X-CSRF-Token. Loopback HTTP is the default; Settings can enable HTTPS before using a non-loopback administration bind. This document contains no live configuration or credentials. Object schemas marked extensible are summaries: full configuration validation is authoritative in Rust."
   },
   "servers": [

src/balance/health.rs

@@ -46,6 +46,12 @@ impl BackendState {
                 },
                 passive_unhealthy_source: "none",
                 passive_unhealthy_reason: "none",
+                passive_first_failure: None,
+                active_first_failure: None,
+                passive_probe_successes: 0,
+                passive_ejected_at: None,
+                passive_cooldown: Duration::from_secs(5),
+                last_io_counted_at: None,
             }),
             traffic: Mutex::new(Traffic {
                 at: now,
@@ -105,7 +111,29 @@ impl BackendState {
         }
     }
     pub fn healthy(&self) -> bool {
-        lock(&self.health).healthy
+        let mut h = lock(&self.health);
+        // Passive-only monitoring has no active probes to recover an ejected
+        // backend. After a bounded quarantine, allow one new traffic attempt.
+        if !h.passive_healthy && h.active_healthy && !h.checked {
+            if h.passive_ejected_at
+                .is_some_and(|at| at.elapsed() >= h.passive_cooldown)
+            {
+                let previous = h.healthy;
+                Self::clear_passive_ejection(&mut h);
+                self.finish_transition(&mut h, previous);
+            }
+        }
+        h.healthy
+    }
+    fn clear_passive_ejection(h: &mut Health) {
+        h.passive_healthy = true;
+        h.passive_failures = 0;
+        h.passive_successes = 0;
+        h.passive_probe_successes = 0;
+        h.passive_first_failure = None;
+        h.passive_ejected_at = None;
+        h.passive_unhealthy_source = "none";
+        h.passive_unhealthy_reason = "none";
     }
     fn weight(&self, configured: u16, slow_start: u64) -> f64 {
         let h = lock(&self.health);
@@ -212,12 +240,27 @@ impl BackendState {
         h.last_probe_ms = crate::logging::now_ms();
         h.result = result;
         if result == "ok" {
+            h.active_first_failure = None;
             h.failures = 0;
             h.successes = h.successes.saturating_add(1);
             if h.successes >= settings.rise {
                 h.active_healthy = true;
             }
+            if !h.passive_healthy && h.passive_ejected_at.is_some() && h.active_healthy {
+                h.passive_probe_successes = h.passive_probe_successes.saturating_add(1);
+                if h.passive_probe_successes >= settings.rise.max(2) {
+                    Self::clear_passive_ejection(&mut h);
+                }
+            }
         } else {
+            h.passive_probe_successes = 0;
+            if h.active_first_failure.is_none() {
+                h.active_first_failure = Some(FailureObservation {
+                    at_ms: crate::logging::now_ms(),
+                    class: "active_health",
+                    detail: result.into(),
+                });
+            }
             self.active_health_failures_total
                 .fetch_add(1, Ordering::Relaxed);
             h.successes = 0;
@@ -248,6 +291,10 @@ impl BackendState {
         self.finish_transition(&mut h, previous);
     }
 
+    fn first_active_failure(&self) -> Option<FailureObservation> {
+        lock(&self.health).active_first_failure.clone()
+    }
+
     fn record_failure(&self, class: &'static str, detail: &str) {
         match class {
             "dns" => {
@@ -303,37 +350,69 @@ impl BackendState {
         reason: &'static str,
         elapsed: Duration,
     ) {
+        self.observe_passive_with_detail(active, passive, reason, reason, reason, elapsed);
+    }
+
+    /// Returns the first failure in a consecutive counted streak only when
+    /// this observation actually removes the backend from the traffic pool.
+    fn observe_passive_with_detail(
+        &self,
+        active: &HealthCheck,
+        passive: &PassiveHealth,
+        reason: &'static str,
+        class: &'static str,
+        detail: &str,
+        elapsed: Duration,
+    ) -> Option<(FailureObservation, u16)> {
         self.record_passive_observation(reason);
-        {
-            let mut h = lock(&self.health);
-            Self::update_latency(&mut h, elapsed);
-            h.passive_result = reason;
-        }
-        if !passive.enabled {
-            return;
-        }
         let mut h = lock(&self.health);
+        Self::update_latency(&mut h, elapsed);
+        h.passive_result = reason;
+        if !passive.enabled {
+            return None;
+        }
         let previous = h.healthy;
         if reason == "ok" {
             h.passive_failures = 0;
+            h.passive_first_failure = None;
+            h.last_io_counted_at = None;
             h.passive_successes = h.passive_successes.saturating_add(1);
             if h.passive_successes >= passive.rise(active) {
-                h.passive_healthy = true;
-                h.passive_unhealthy_source = "none";
-                h.passive_unhealthy_reason = "none";
+                Self::clear_passive_ejection(&mut h);
             }
             self.finish_transition(&mut h, previous);
-            return;
+            return None;
         }
         if !passive.counts(reason) {
-            return;
+            return None;
+        }
+        // A fan-out of failures from one expired H2 connection must not count
+        // as independent failed backend observations in a single instant.
+        if reason == "io_error" {
+            let now = Instant::now();
+            if h.last_io_counted_at
+                .is_some_and(|last| now.duration_since(last) < Duration::from_millis(250))
+            {
+                return None;
+            }
+            h.last_io_counted_at = Some(now);
         }
         self.passive_health_failures_total
             .fetch_add(1, Ordering::Relaxed);
+        if h.passive_failures == 0 {
+            h.passive_first_failure = Some(FailureObservation {
+                at_ms: crate::logging::now_ms(),
+                class,
+                detail: detail.chars().take(1024).collect(),
+            });
+        }
         h.passive_successes = 0;
+        h.passive_probe_successes = 0;
         h.passive_failures = h.passive_failures.saturating_add(1);
-        if h.passive_failures >= passive.fall(active) {
+        if h.passive_failures >= passive.fall(active) && h.passive_healthy {
             h.passive_healthy = false;
+            h.passive_ejected_at = Some(Instant::now());
+            h.passive_cooldown = Duration::from_millis(active.interval_ms.clamp(5_000, 30_000));
         }
         let source = match reason {
             "connect_error" => "passive_connect_error",
@@ -352,6 +431,13 @@ impl BackendState {
             }
         }
         self.finish_transition(&mut h, previous);
+        if previous && !h.healthy {
+            h.passive_first_failure
+                .clone()
+                .map(|first| (first, h.passive_failures))
+        } else {
+            None
+        }
     }
 }
 #[derive(Debug)]

src/balance/lease_runtime.rs

@@ -40,9 +40,26 @@ impl Lease {
             "ok"
         };
         let before = self.member.state.healthy();
-        self.member
-            .state
-            .observe_passive(&self.health, &self.passive_health, reason, elapsed);
+        let failure_detail = if is_5xx {
+            format!("status={status}")
+        } else {
+            reason.into()
+        };
+        let ejected = self.member.state.observe_passive_with_detail(
+            &self.health,
+            &self.passive_health,
+            reason,
+            reason,
+            &failure_detail,
+            elapsed,
+        );
+        if let Some((first, failures)) = ejected {
+            tracing::warn!(backend = %self.member.definition.id, origin = %self.member.backend.origin, source = "passive",
+                reason, first_error_class = first.class,
+                first_error_at_ms = first.at_ms, first_error_detail = %first.detail,
+                trigger_error_detail = %failure_detail, failures,
+                "backend excluded from traffic");
+        }
         let after = self.member.state.healthy();
         if before != after {
             tracing::info!(
@@ -62,6 +79,11 @@ impl Lease {
             self.member.state.errors.fetch_add(1, Ordering::Relaxed);
         }
         self.member.state.record_failure(class, detail);
+        // GOAWAY(NO_ERROR) is a graceful H2 connection shutdown, not an
+        // origin failure. It can still be surfaced in access diagnostics.
+        if matches!(class, "h2_graceful_goaway" | "h2_graceful_close") {
+            return;
+        }
         let passive_reason = match class {
             "connect" | "dns" | "tls" => "connect_error",
             "timeout"
@@ -72,12 +94,21 @@ impl Lease {
             _ => "io_error",
         };
         let before = self.member.state.healthy();
-        self.member.state.observe_passive(
+        let ejected = self.member.state.observe_passive_with_detail(
             &self.health,
             &self.passive_health,
             passive_reason,
+            class,
+            detail,
             elapsed,
         );
+        if let Some((first, failures)) = ejected {
+            tracing::warn!(backend = %self.member.definition.id, origin = %self.member.backend.origin, source = "passive",
+                reason = passive_reason, first_error_class = first.class,
+                first_error_at_ms = first.at_ms, first_error_detail = %first.detail,
+                trigger_error_detail = %detail, failures,
+                "backend excluded from traffic");
+        }
         let after = self.member.state.healthy();
         if before != after {
             tracing::info!(
@@ -326,8 +357,18 @@ async fn probe(member: Arc<Member>, settings: HealthCheck) {
     };
     let before = member.state.healthy();
     member.state.observe_active(&settings, result, elapsed);
-    if before != member.state.healthy() {
-        tracing::info!(backend = %member.definition.id, healthy = member.state.healthy(), reason = result, "backend health changed");
+    let after = member.state.healthy();
+    if before && !after {
+        let first = member.state.first_active_failure();
+        tracing::warn!(backend = %member.definition.id, origin = %member.backend.origin, source = "active",
+            reason = result,
+            first_error_class = first.as_ref().map_or("active_health", |f| f.class),
+            first_error_at_ms = first.as_ref().map_or(0, |f| f.at_ms),
+            first_error_detail = %first.as_ref().map_or(result, |f| f.detail.as_str()),
+            "backend excluded from traffic");
+    }
+    if before != after {
+        tracing::info!(backend = %member.definition.id, healthy = after, reason = result, "backend health changed");
     }
 }
 /// One supervisor, at most 16 probes, at most one outstanding probe per member.

src/balance/model.rs

@@ -355,7 +355,14 @@ struct Health {
     active_unhealthy_reason: &'static str,
     passive_unhealthy_source: &'static str,
     passive_unhealthy_reason: &'static str,
+    passive_first_failure: Option<FailureObservation>,
+    active_first_failure: Option<FailureObservation>,
+    passive_probe_successes: u16,
+    passive_ejected_at: Option<Instant>,
+    passive_cooldown: Duration,
+    last_io_counted_at: Option<Instant>,
 }
+
 #[derive(Debug)]
 struct Traffic {
     at: Instant,

src/proxy/forward/response.rs

@@ -28,7 +28,7 @@
             }
             if let Some(lease) = &active_lease {
                 lease.error_detail(
-                    upstream_error_class(&cause, connect_failure),
+                    upstream_health_error_class(&cause, connect_failure, &active_backend),
                     &detail,
                     final_attempt_elapsed,
                 );

src/proxy/forward/upstream.rs

@@ -141,6 +141,7 @@ let mut active_uri = upstream_target;
 let mut request_attempt = Some(request);
 let mut attempted_backends = Vec::<String>::new();
 let mut retries = 0u8;
+let mut graceful_retry_done = false;
 let upstream_result = loop {
     let client = if handshake.is_some() && !websocket_upstream_h2 {
         &active_backend.websocket_client
@@ -153,6 +154,36 @@ let upstream_result = loop {
     )
     .await;
     let attempt_elapsed = attempt_started.elapsed();
+    if let Ok(Err(cause)) = &result {
+        if !graceful_retry_done
+            && !admin_control_plane
+            && handshake.is_none()
+            && request_body_empty
+            && retry_safe_method(&retry_method)
+            && (is_graceful_h2_goaway(cause)
+                || is_related_h2_close(cause, &active_backend))
+        {
+            if let Some(next_request) = retry_request(
+                &retry_method,
+                upstream_request_version(&active_backend, false),
+                &retry_headers,
+                &active_backend,
+                &original_host,
+                &origin_suffix,
+                retry_host_follows_backend,
+            ) {
+                graceful_retry_done = true;
+                if let Some(lease) = &active_lease {
+                    lease.retry_attempted();
+                }
+                tracing::debug!(backend = %active_backend.id,
+                    "retrying safe empty request after graceful upstream H2 shutdown");
+                request_attempt = Some(next_request);
+                attempt_started = Instant::now();
+                continue;
+            }
+        }
+    }
     let Some(policy) = retry_policy.as_ref() else {
         break result;
     };
@@ -220,7 +251,7 @@ let upstream_result = loop {
                     connect_failure,
                 );
                 lease.error_detail(
-                    upstream_error_class(cause, connect_failure),
+                    upstream_health_error_class(cause, connect_failure, &active_backend),
                     &detail,
                     attempt_elapsed,
                 );

src/proxy/h2_graceful.rs

@@ -0,0 +1,27 @@
+fn is_graceful_h2_goaway(error: &(dyn std::error::Error + 'static)) -> bool {
+    crate::upstream::is_graceful_h2_goaway(error)
+}
+
+fn is_related_h2_close(
+    error: &(dyn std::error::Error + 'static),
+    backend: &crate::config::CompiledBackend,
+) -> bool {
+    let chain = format!("{error:?}");
+    backend.client.recently_saw_graceful_goaway()
+        && chain.contains("Canceled")
+        && chain.contains("connection closed")
+}
+
+fn upstream_health_error_class(
+    error: &(dyn std::error::Error + 'static),
+    connect_failure: bool,
+    backend: &crate::config::CompiledBackend,
+) -> &'static str {
+    if is_graceful_h2_goaway(error) {
+        "h2_graceful_goaway"
+    } else if is_related_h2_close(error, backend) {
+        "h2_graceful_close"
+    } else {
+        upstream_error_class(error, connect_failure)
+    }
+}

src/proxy/tests/authority_tests.rs

@@ -571,3 +571,28 @@ mod authority_tests {
         }
     }
 }
+
+#[cfg(test)]
+mod graceful_h2_tests {
+    use super::*;
+    use std::fmt;
+
+    #[derive(Debug)]
+    struct H2DebugError(&'static str);
+    impl fmt::Display for H2DebugError {
+        fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+            write!(f, "client error (SendRequest): {}", self.0)
+        }
+    }
+    impl std::error::Error for H2DebugError {}
+
+    #[test]
+    fn only_graceful_remote_h2_goaway_is_neutral() {
+        let valid = H2DebugError("GoAway(b\"\", NO_ERROR, Remote)");
+        let protocol = H2DebugError("GoAway(b\"\", PROTOCOL_ERROR, Remote)");
+        let local = H2DebugError("GoAway(b\"\", NO_ERROR, Local)");
+        assert!(is_graceful_h2_goaway(&valid));
+        assert!(!is_graceful_h2_goaway(&protocol));
+        assert!(!is_graceful_h2_goaway(&local));
+    }
+}

src/proxy/types.rs

@@ -138,6 +138,8 @@ fn upstream_error_class(
     }
 }
 
+include!("h2_graceful.rs");
+
 fn retry_safe_method(method: &Method) -> bool {
     method == Method::GET || method == Method::HEAD || method == Method::OPTIONS
 }
@@ -171,11 +173,12 @@ fn upstream_request_target(
     websocket_h1: bool,
     websocket_h2: bool,
 ) -> Option<Uri> {
-    let h2_capable = !websocket_h1 && match backend.protocol {
-        crate::config::UpstreamProtocol::Http1 => false,
-        crate::config::UpstreamProtocol::Auto => backend.origin.starts_with("https://"),
-        crate::config::UpstreamProtocol::Http2 | crate::config::UpstreamProtocol::H2 => true,
-    };
+    let h2_capable = !websocket_h1
+        && match backend.protocol {
+            crate::config::UpstreamProtocol::Http1 => false,
+            crate::config::UpstreamProtocol::Auto => backend.origin.starts_with("https://"),
+            crate::config::UpstreamProtocol::Http2 | crate::config::UpstreamProtocol::H2 => true,
+        };
     if !h2_capable {
         return Some(uri.clone());
     }

src/upstream.rs

@@ -32,7 +32,8 @@ pub type BoxError = Box<dyn std::error::Error + Send + Sync>;
 
 mod client;
 pub use client::{
-    client, client_with_keepalive, client_with_timeout, ClientRotationStats, ProxyClient,
+    client, client_with_keepalive, client_with_timeout, is_graceful_h2_goaway, ClientRotationStats,
+    ProxyClient,
 };
 
 #[derive(Clone, Debug, Default, PartialEq, Eq)]

src/upstream/client.rs

@@ -22,6 +22,13 @@ use std::{
     time::{Duration, Instant},
 };
 
+/// Hyper's public error chain omits the GOAWAY reason, but Debug retains
+/// the peer-initiated H2 `NO_ERROR` variant. Restrict this to graceful closure.
+pub fn is_graceful_h2_goaway(error: &(dyn std::error::Error + 'static)) -> bool {
+    let chain = format!("{error:?}");
+    chain.contains("GoAway(") && chain.contains("NO_ERROR") && chain.contains("Remote")
+}
+
 static MONOTONIC_START: OnceLock<Instant> = OnceLock::new();
 type RawProxyClient = Client<super::TimedConnector, Body>;
 
@@ -54,6 +61,7 @@ struct RotatingClient {
     observed_at_ms: AtomicU64,
     observed_h1: AtomicU64,
     observed_h2: AtomicU64,
+    graceful_goaway_at_ms: AtomicU64,
 }
 
 /// Backend HTTP client with bounded generation lifetime.
@@ -107,19 +115,43 @@ impl ProxyClient {
                 observed_at_ms: AtomicU64::new(0),
                 observed_h1: AtomicU64::new(0),
                 observed_h2: AtomicU64::new(0),
+                graceful_goaway_at_ms: AtomicU64::new(0),
             }),
         }
     }
 
     pub async fn request(&self, request: Request<Body>) -> Result<Response<Incoming>, ClientError> {
         let client = self.inner.client_for_request();
+        let generation = self.inner.generation.load(Ordering::Acquire);
         let response = client.request(request).await;
-        if let Ok(value) = &response {
-            self.inner.observe_protocol(value.version());
+        match &response {
+            Ok(value) => self.inner.observe_protocol(value.version()),
+            Err(error) if is_graceful_h2_goaway(error) => {
+                // Also applies to cache refresh and other backend HTTP callers,
+                // not only the regular forward loop.
+                self.note_graceful_goaway(generation);
+            }
+            Err(_) => {}
         }
         response
     }
 
+    /// A peer may gracefully close a reused H2 connection. Discard only the
+    /// affected client generation; concurrent requests must not rotate again.
+    pub fn note_graceful_goaway(&self, generation: u64) {
+        self.inner
+            .graceful_goaway_at_ms
+            .store(monotonic_millis().saturating_add(1), Ordering::Release);
+        self.inner.rotate(generation, "h2_goaway_no_error", false);
+    }
+
+    /// A cancellation immediately following GOAWAY may be another stream on
+    /// the closing H2 connection. Keep this correlation window short.
+    pub fn recently_saw_graceful_goaway(&self) -> bool {
+        let at = self.inner.graceful_goaway_at_ms.load(Ordering::Acquire);
+        at != 0 && monotonic_millis().saturating_add(1).saturating_sub(at) <= 1_000
+    }
+
     pub fn protocol_stats(&self) -> UpstreamProtocolStats {
         self.inner.protocol_stats()
     }
@@ -425,14 +457,18 @@ fn build_raw_client(factory: &ClientFactory) -> Result<RawProxyClient> {
         url::Host::Ipv6(ip) => format!("[{ip}]"),
         host => host.to_string(),
     };
-    let port = origin.port_or_known_default().context("backend port missing")?;
-    let tls_name = backend.tls_server_name
+    let port = origin
+        .port_or_known_default()
+        .context("backend port missing")?;
+    let tls_name = backend
+        .tls_server_name
         .as_deref()
         .map(crate::config::normalize_host)
         .transpose()?
         .unwrap_or(network_host);
     let destination: http::Uri = format!("{}://{tls_name}:{port}", origin.scheme())
-        .parse().context("invalid configured backend transport authority")?;
+        .parse()
+        .context("invalid configured backend transport authority")?;
     Ok(builder.build(TimedConnector {
         inner,
         policy: policy.clone(),