v0.25.1
This commit is contained in:
Mateusz Gruszczyński committed
1 parent
87e571c170
commit
fa7abc9e53
308 files changed
+49746
-50237
No files matched your search
@@ -0,0 +1,294 @@
|
||||
impl Lease {
|
||||
pub fn started(&self) {
|
||||
self.member.state.requests.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
pub fn response(&self, status: u16, elapsed: Duration, request_body_empty: bool) {
|
||||
self.member
|
||||
.state
|
||||
.last_status
|
||||
.store(status, Ordering::Relaxed);
|
||||
let is_5xx = status >= 500;
|
||||
let is_slow = request_body_empty
|
||||
&& self.health.slow_response_ms > 0
|
||||
&& elapsed > Duration::from_millis(self.health.slow_response_ms);
|
||||
if (is_5xx || is_slow) && !self.error_recorded.swap(true, Ordering::Relaxed) {
|
||||
self.member.state.errors.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
if is_5xx {
|
||||
self.member
|
||||
.state
|
||||
.record_failure("http_5xx", &format!("status={status}"));
|
||||
}
|
||||
let reason = if is_5xx
|
||||
&& (!is_slow
|
||||
|| !self.passive_health.enabled
|
||||
|| self.passive_health.count_http_5xx
|
||||
|| !self.passive_health.count_slow_responses)
|
||||
{
|
||||
if is_slow {
|
||||
self.member.state.record_passive_observation("slow_response");
|
||||
}
|
||||
"http_5xx"
|
||||
} else if is_slow {
|
||||
if is_5xx {
|
||||
self.member.state.record_passive_observation("http_5xx");
|
||||
}
|
||||
"slow_response"
|
||||
} else {
|
||||
"ok"
|
||||
};
|
||||
let before = self.member.state.healthy();
|
||||
self.member
|
||||
.state
|
||||
.observe_passive(&self.health, &self.passive_health, reason, elapsed);
|
||||
let after = self.member.state.healthy();
|
||||
if before != after {
|
||||
tracing::info!(
|
||||
backend = %self.member.definition.id,
|
||||
healthy = after,
|
||||
source = "passive",
|
||||
reason,
|
||||
"backend health changed"
|
||||
);
|
||||
}
|
||||
}
|
||||
pub fn error(&self, reason: &'static str, elapsed: Duration) {
|
||||
self.error_detail(reason, reason, elapsed);
|
||||
}
|
||||
pub fn error_detail(&self, class: &'static str, detail: &str, elapsed: Duration) {
|
||||
if !self.error_recorded.swap(true, Ordering::Relaxed) {
|
||||
self.member.state.errors.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
self.member.state.record_failure(class, detail);
|
||||
let passive_reason = match class {
|
||||
"connect" | "dns" | "tls" => "connect_error",
|
||||
"timeout" | "body_timeout" | "response_header_timeout" | "read_timeout"
|
||||
| "send_timeout" => "timeout",
|
||||
_ => "io_error",
|
||||
};
|
||||
let before = self.member.state.healthy();
|
||||
self.member.state.observe_passive(
|
||||
&self.health,
|
||||
&self.passive_health,
|
||||
passive_reason,
|
||||
elapsed,
|
||||
);
|
||||
let after = self.member.state.healthy();
|
||||
if before != after {
|
||||
tracing::info!(
|
||||
backend = %self.member.definition.id,
|
||||
healthy = after,
|
||||
source = "passive",
|
||||
reason = passive_reason,
|
||||
"backend health changed"
|
||||
);
|
||||
}
|
||||
}
|
||||
pub fn bytes(&self, count: u64, incoming: bool) {
|
||||
if incoming {
|
||||
self.member
|
||||
.state
|
||||
.bytes_in
|
||||
.fetch_add(count, Ordering::Relaxed);
|
||||
} else {
|
||||
self.member
|
||||
.state
|
||||
.bytes_out
|
||||
.fetch_add(count, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
pub fn retry_attempted(&self) {
|
||||
self.member
|
||||
.state
|
||||
.retry_attempts_total
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
pub fn retry_succeeded(&self) {
|
||||
self.member
|
||||
.state
|
||||
.retry_successes_total
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
impl Drop for Lease {
|
||||
fn drop(&mut self) {
|
||||
self.member.state.active.fetch_sub(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
pub struct Selection {
|
||||
pub lease: Arc<Lease>,
|
||||
pub cookie: Option<HeaderValue>,
|
||||
}
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum SelectionFailure {
|
||||
Unavailable,
|
||||
Capacity,
|
||||
}
|
||||
|
||||
impl Pool {
|
||||
pub fn prometheus(&self) -> String {
|
||||
use std::fmt::Write;
|
||||
let mut text = String::new();
|
||||
for member in &self.members {
|
||||
let state = &member.state;
|
||||
let labels = format!("vhost=\"{}\",backend=\"{}\"", self.id, member.definition.id);
|
||||
for (metric, value) in [
|
||||
(
|
||||
"up",
|
||||
u64::from(state.healthy() && member.definition.enabled),
|
||||
),
|
||||
("draining", u64::from(member.definition.drain)),
|
||||
("active_exchanges", state.active.load(Ordering::Relaxed)),
|
||||
("requests_total", state.requests.load(Ordering::Relaxed)),
|
||||
("errors_total", state.errors.load(Ordering::Relaxed)),
|
||||
("bytes_in_total", state.bytes_in.load(Ordering::Relaxed)),
|
||||
("bytes_out_total", state.bytes_out.load(Ordering::Relaxed)),
|
||||
] {
|
||||
let _ = writeln!(text, "rust_proxy_backend_{metric}{{{labels}}} {value}");
|
||||
}
|
||||
for (metric, value) in [
|
||||
("active_health_failures_total", state.active_health_failures_total.load(Ordering::Relaxed)),
|
||||
("passive_health_failures_total", state.passive_health_failures_total.load(Ordering::Relaxed)),
|
||||
("http_5xx_total", state.http_5xx_total.load(Ordering::Relaxed)),
|
||||
("passive_http_5xx_total", state.http_5xx_total.load(Ordering::Relaxed)),
|
||||
("timeouts_total", state.timeouts_total.load(Ordering::Relaxed)),
|
||||
("passive_timeouts_total", state.timeouts_total.load(Ordering::Relaxed)),
|
||||
("connect_errors_total", state.connect_errors_total.load(Ordering::Relaxed)),
|
||||
("passive_connect_errors_total", state.connect_errors_total.load(Ordering::Relaxed)),
|
||||
("io_errors_total", state.io_errors_total.load(Ordering::Relaxed)),
|
||||
("passive_io_errors_total", state.io_errors_total.load(Ordering::Relaxed)),
|
||||
("slow_responses_total", state.slow_responses_total.load(Ordering::Relaxed)),
|
||||
("passive_slow_total", state.slow_responses_total.load(Ordering::Relaxed)),
|
||||
("retry_attempts_total", state.retry_attempts_total.load(Ordering::Relaxed)),
|
||||
("retry_successes_total", state.retry_successes_total.load(Ordering::Relaxed)),
|
||||
("capacity_rejections_total", state.capacity_rejections_total.load(Ordering::Relaxed)),
|
||||
("queue_timeouts_total", state.queue_timeouts_total.load(Ordering::Relaxed)),
|
||||
("dns_errors_total", state.dns_errors_total.load(Ordering::Relaxed)),
|
||||
("tls_errors_total", state.tls_errors_total.load(Ordering::Relaxed)),
|
||||
("response_header_timeouts_total", state.response_header_timeouts_total.load(Ordering::Relaxed)),
|
||||
("read_timeouts_total", state.read_timeouts_total.load(Ordering::Relaxed)),
|
||||
("send_timeouts_total", state.send_timeouts_total.load(Ordering::Relaxed)),
|
||||
] {
|
||||
let _ = writeln!(text, "rust_proxy_backend_{metric}{{{labels}}} {value}");
|
||||
}
|
||||
for (metric, value) in [
|
||||
("last_ejected_timestamp_milliseconds", state.last_ejected_ms.load(Ordering::Relaxed)),
|
||||
("last_recovered_timestamp_milliseconds", state.last_recovered_ms.load(Ordering::Relaxed)),
|
||||
] {
|
||||
let _ = writeln!(text, "rust_proxy_backend_{metric}{{{labels}}} {value}");
|
||||
}
|
||||
let _ = writeln!(
|
||||
text,
|
||||
"rust_proxy_backend_traffic_bytes_per_second{{{labels}}} {:.3}",
|
||||
state.traffic_rate()
|
||||
);
|
||||
let _ = writeln!(
|
||||
text,
|
||||
"rust_proxy_backend_latency_milliseconds{{{labels}}} {:.3}",
|
||||
state.latency()
|
||||
);
|
||||
}
|
||||
text
|
||||
}
|
||||
}
|
||||
async fn probe(member: Arc<Member>, settings: HealthCheck) {
|
||||
let start = Instant::now();
|
||||
let result = tokio::time::timeout(Duration::from_millis(settings.timeout_ms), async {
|
||||
match settings.mode {
|
||||
HealthMode::Tcp => {
|
||||
crate::upstream::tcp_probe(&member.backend.origin, &member.backend.address_policy)
|
||||
.await
|
||||
.map(|_| "ok")
|
||||
.unwrap_or("connect")
|
||||
}
|
||||
HealthMode::Http => {
|
||||
let request = Request::builder()
|
||||
.uri(format!(
|
||||
"{}{}",
|
||||
member.backend.request_origin, settings.path
|
||||
))
|
||||
.header(
|
||||
"host",
|
||||
settings
|
||||
.host
|
||||
.as_deref()
|
||||
.unwrap_or_else(|| member.backend.host_header.to_str().unwrap_or("")),
|
||||
)
|
||||
.header("connection", "close")
|
||||
.header(
|
||||
"user-agent",
|
||||
concat!("RustProxy-health/", env!("CARGO_PKG_VERSION")),
|
||||
)
|
||||
.body(Body::empty());
|
||||
match request {
|
||||
Ok(request) => match member.backend.websocket_client.request(request).await {
|
||||
Ok(response) => {
|
||||
if (settings.expected_status_min..=settings.expected_status_max)
|
||||
.contains(&response.status().as_u16())
|
||||
{
|
||||
"ok"
|
||||
} else {
|
||||
"http_status"
|
||||
}
|
||||
}
|
||||
Err(_) => "connect",
|
||||
},
|
||||
Err(_) => "request",
|
||||
}
|
||||
}
|
||||
HealthMode::Off => "ok",
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap_or("timeout");
|
||||
let elapsed = start.elapsed();
|
||||
let result = if result == "ok"
|
||||
&& settings.slow_response_ms > 0
|
||||
&& elapsed > Duration::from_millis(settings.slow_response_ms)
|
||||
{
|
||||
"slow"
|
||||
} else {
|
||||
result
|
||||
};
|
||||
let before = member.state.healthy();
|
||||
member.state.observe_active(&settings, result, elapsed);
|
||||
if before != member.state.healthy() {
|
||||
tracing::info!(backend = %member.definition.id, healthy = member.state.healthy(), reason = result, "backend health changed");
|
||||
}
|
||||
}
|
||||
/// One supervisor, at most 16 probes, at most one outstanding probe per member.
|
||||
/// DNS workers have their own shared bound and never resolve arbitrary URLs.
|
||||
pub async fn health_loop(app: Weak<crate::runtime::App>, shutdown: CancellationToken) {
|
||||
let mut interval = tokio::time::interval(Duration::from_millis(250));
|
||||
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||
let mut checks = JoinSet::new();
|
||||
let mut cursor = 0usize;
|
||||
loop {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = shutdown.cancelled() => break,
|
||||
_ = checks.join_next(), if !checks.is_empty() => {},
|
||||
_ = interval.tick() => {
|
||||
let Some(app) = app.upgrade() else { break; };
|
||||
let snapshot = app.config.load_full();
|
||||
let members: Vec<_> = snapshot.pools.iter().flat_map(|pool|
|
||||
pool.members.iter().map(move |member| (member, &pool.settings.health_check))).collect();
|
||||
for (member, _) in &members { member.state.sample_traffic(); }
|
||||
// Rotate the bounded probe window: slow first members must not
|
||||
// starve later pools when all intervals equal their timeouts.
|
||||
let start = cursor;
|
||||
for offset in 0..members.len() {
|
||||
if checks.len() >= 16 { break; }
|
||||
let index = (start + offset) % members.len();
|
||||
let (member, health) = members[index];
|
||||
if member.definition.enabled && member.state.claim_probe(health) {
|
||||
checks.spawn(probe(member.clone(), health.clone()));
|
||||
cursor = (index + 1) % members.len();
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
checks.abort_all();
|
||||
while checks.join_next().await.is_some() {}
|
||||
}
|
||||
Reference in new issue
Block a user