This commit is contained in:
Mateusz Gruszczyński committed 2026-10-14 14:54:22 +02:00
1 parent 1b9135117c
commit 4f36de240c
32 files changed
+1175 -704

No files matched your search

+47 -6
View File
@@ -40,9 +40,26 @@ impl Lease {
"ok"
};
let before = self.member.state.healthy();
self.member
.state
.observe_passive(&self.health, &self.passive_health, reason, elapsed);
let failure_detail = if is_5xx {
format!("status={status}")
} else {
reason.into()
};
let ejected = self.member.state.observe_passive_with_detail(
&self.health,
&self.passive_health,
reason,
reason,
&failure_detail,
elapsed,
);
if let Some((first, failures)) = ejected {
tracing::warn!(backend = %self.member.definition.id, origin = %self.member.backend.origin, source = "passive",
reason, first_error_class = first.class,
first_error_at_ms = first.at_ms, first_error_detail = %first.detail,
trigger_error_detail = %failure_detail, failures,
"backend excluded from traffic");
}
let after = self.member.state.healthy();
if before != after {
tracing::info!(
@@ -62,6 +79,11 @@ impl Lease {
self.member.state.errors.fetch_add(1, Ordering::Relaxed);
}
self.member.state.record_failure(class, detail);
// GOAWAY(NO_ERROR) is a graceful H2 connection shutdown, not an
// origin failure. It can still be surfaced in access diagnostics.
if matches!(class, "h2_graceful_goaway" | "h2_graceful_close") {
return;
}
let passive_reason = match class {
"connect" | "dns" | "tls" => "connect_error",
"timeout"
@@ -72,12 +94,21 @@ impl Lease {
_ => "io_error",
};
let before = self.member.state.healthy();
self.member.state.observe_passive(
let ejected = self.member.state.observe_passive_with_detail(
&self.health,
&self.passive_health,
passive_reason,
class,
detail,
elapsed,
);
if let Some((first, failures)) = ejected {
tracing::warn!(backend = %self.member.definition.id, origin = %self.member.backend.origin, source = "passive",
reason = passive_reason, first_error_class = first.class,
first_error_at_ms = first.at_ms, first_error_detail = %first.detail,
trigger_error_detail = %detail, failures,
"backend excluded from traffic");
}
let after = self.member.state.healthy();
if before != after {
tracing::info!(
@@ -326,8 +357,18 @@ async fn probe(member: Arc<Member>, settings: HealthCheck) {
};
let before = member.state.healthy();
member.state.observe_active(&settings, result, elapsed);
if before != member.state.healthy() {
tracing::info!(backend = %member.definition.id, healthy = member.state.healthy(), reason = result, "backend health changed");
let after = member.state.healthy();
if before && !after {
let first = member.state.first_active_failure();
tracing::warn!(backend = %member.definition.id, origin = %member.backend.origin, source = "active",
reason = result,
first_error_class = first.as_ref().map_or("active_health", |f| f.class),
first_error_at_ms = first.as_ref().map_or(0, |f| f.at_ms),
first_error_detail = %first.as_ref().map_or(result, |f| f.detail.as_str()),
"backend excluded from traffic");
}
if before != after {
tracing::info!(backend = %member.definition.id, healthy = after, reason = result, "backend health changed");
}
}
/// One supervisor, at most 16 probes, at most one outstanding probe per member.