v0.25.1
This commit is contained in:
Mateusz Gruszczyński committed
1 parent
87e571c170
commit
fa7abc9e53
308 files changed
+49746
-50237
No files matched your search
@@ -0,0 +1,370 @@
|
||||
fn render_core(app: &App, snapshot: &crate::config::Snapshot, out: &mut String, f: &mut Families) {
|
||||
let config = &snapshot.envelope.config;
|
||||
let settings = &config.prometheus;
|
||||
f.add(
|
||||
"rust_proxy_admin_requests_total",
|
||||
"counter",
|
||||
"Admin HTTP requests, including denied requests and static assets.",
|
||||
"",
|
||||
app.metrics.admin_requests.load(Ordering::Relaxed),
|
||||
);
|
||||
for (index, counter) in app.metrics.admin_status.iter().enumerate() {
|
||||
f.add(
|
||||
"rust_proxy_admin_responses_total",
|
||||
"counter",
|
||||
"Admin response headers by status class.",
|
||||
&format!("class=\"{}xx\"", index + 1),
|
||||
counter.load(Ordering::Relaxed),
|
||||
);
|
||||
}
|
||||
out.push_str("# HELP rust_proxy_admin_header_duration_seconds Time to control-plane response headers; not SSE lifetime.\n# TYPE rust_proxy_admin_header_duration_seconds summary\n");
|
||||
let _ = writeln!(out, "rust_proxy_admin_header_duration_seconds_count {}\nrust_proxy_admin_header_duration_seconds_sum {:.6}",
|
||||
app.metrics.admin_response_headers.load(Ordering::Relaxed), app.metrics.admin_header_duration_us.load(Ordering::Relaxed) as f64/1e6);
|
||||
f.add(
|
||||
"rust_proxy_build_info",
|
||||
"gauge",
|
||||
"Build version.",
|
||||
&labels(&[("version", env!("CARGO_PKG_VERSION"))]),
|
||||
1,
|
||||
);
|
||||
f.add(
|
||||
"rust_proxy_uptime_seconds",
|
||||
"gauge",
|
||||
"Process monotonic uptime.",
|
||||
"",
|
||||
app.started.elapsed().as_secs_f64(),
|
||||
);
|
||||
f.add(
|
||||
"rust_proxy_vhosts_configured",
|
||||
"gauge",
|
||||
"Configured vhosts including disabled.",
|
||||
"",
|
||||
config.vhosts.len(),
|
||||
);
|
||||
f.add(
|
||||
"rust_proxy_redirect_hosts_configured",
|
||||
"gauge",
|
||||
"Configured redirect hosts including disabled.",
|
||||
"",
|
||||
config.redirects.len(),
|
||||
);
|
||||
f.add(
|
||||
"rust_proxy_not_found_hosts_configured",
|
||||
"gauge",
|
||||
"Configured local-response/404 hosts including disabled.",
|
||||
"",
|
||||
config.not_found_hosts.len(),
|
||||
);
|
||||
f.add(
|
||||
"rust_proxy_stream_hosts_configured",
|
||||
"gauge",
|
||||
"Configured Layer-4 stream hosts including disabled.",
|
||||
"",
|
||||
config.streams.len(),
|
||||
);
|
||||
f.add(
|
||||
"rust_proxy_acl_profiles_configured",
|
||||
"gauge",
|
||||
"Configured reusable ACL profiles.",
|
||||
"",
|
||||
config.acl_profiles.len(),
|
||||
);
|
||||
if settings.include_system {
|
||||
let geo = app.geoip.view();
|
||||
f.add(
|
||||
"rust_proxy_geoip_loaded",
|
||||
"gauge",
|
||||
"Whether the bounded country GeoIP database is loaded.",
|
||||
"",
|
||||
u8::from(geo.loaded),
|
||||
);
|
||||
f.add(
|
||||
"rust_proxy_geoip_ipv4_ranges",
|
||||
"gauge",
|
||||
"Loaded IPv4 country ranges.",
|
||||
"",
|
||||
geo.ipv4_ranges,
|
||||
);
|
||||
f.add(
|
||||
"rust_proxy_geoip_ipv6_ranges",
|
||||
"gauge",
|
||||
"Loaded IPv6 country ranges.",
|
||||
"",
|
||||
geo.ipv6_ranges,
|
||||
);
|
||||
}
|
||||
f.add(
|
||||
"rust_proxy_certificates_configured",
|
||||
"gauge",
|
||||
"Certificate inventory size, not validity.",
|
||||
"",
|
||||
config.tls.certificates.len(),
|
||||
);
|
||||
let renewals = app.acme.renewal_view();
|
||||
for certificate in &config.tls.certificates {
|
||||
let l = labels(&[("certificate", &certificate.id)]);
|
||||
let issuer = certificate.issuer.as_ref();
|
||||
f.add(
|
||||
"rust_proxy_certificate_auto_renew_enabled",
|
||||
"gauge",
|
||||
"Desired automatic renewal switch for a configured certificate.",
|
||||
&l,
|
||||
u8::from(issuer.is_some_and(|issuer| issuer.auto_renew)),
|
||||
);
|
||||
if let Some(issuer) = issuer {
|
||||
f.add(
|
||||
"rust_proxy_certificate_issuer_info",
|
||||
"gauge",
|
||||
"Bounded configured certificate issuer metadata.",
|
||||
&labels(&[
|
||||
("certificate", &certificate.id),
|
||||
("provider", &issuer.provider),
|
||||
("environment", &issuer.environment),
|
||||
]),
|
||||
1,
|
||||
);
|
||||
}
|
||||
if let Some(renewal) = renewals.get(&certificate.id) {
|
||||
for (name, value) in [
|
||||
("last_checked_timestamp_seconds", Some(renewal.last_checked)),
|
||||
("next_check_timestamp_seconds", Some(renewal.next_check)),
|
||||
("last_renewed_timestamp_seconds", renewal.last_renewed),
|
||||
("not_after_timestamp_seconds", renewal.not_after),
|
||||
] {
|
||||
if let Some(value) = value.filter(|value| *value > 0) {
|
||||
f.add(
|
||||
&format!("rust_proxy_certificate_renewal_{name}"),
|
||||
"gauge",
|
||||
"Certificate renewal observation timestamp.",
|
||||
&l,
|
||||
value,
|
||||
);
|
||||
}
|
||||
}
|
||||
f.add(
|
||||
"rust_proxy_certificate_renewal_failures",
|
||||
"gauge",
|
||||
"Consecutive observed automatic renewal failures.",
|
||||
&l,
|
||||
renewal.failures,
|
||||
);
|
||||
if !renewal.status.is_empty() {
|
||||
f.add(
|
||||
"rust_proxy_certificate_renewal_status_info",
|
||||
"gauge",
|
||||
"Latest bounded certificate renewal status.",
|
||||
&labels(&[
|
||||
("certificate", &certificate.id),
|
||||
("status", &renewal.status),
|
||||
]),
|
||||
1,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
f.add(
|
||||
"rust_proxy_listeners_active",
|
||||
"gauge",
|
||||
"Bound frontend endpoints.",
|
||||
"",
|
||||
app.active_listeners.len(),
|
||||
);
|
||||
f.add(
|
||||
"rust_proxy_restart_required",
|
||||
"gauge",
|
||||
"Listener/exporter intent differs from startup state.",
|
||||
"",
|
||||
u8::from(
|
||||
snapshot.listeners != app.active_listeners
|
||||
|| settings.restart_differs(&app.active_prometheus),
|
||||
),
|
||||
);
|
||||
f.add(
|
||||
"rust_proxy_request_capacity",
|
||||
"gauge",
|
||||
"Configured maximum concurrent admitted exchanges.",
|
||||
"",
|
||||
app.max_inflight,
|
||||
);
|
||||
f.add(
|
||||
"rust_proxy_request_permits_available",
|
||||
"gauge",
|
||||
"Currently available exchange permits.",
|
||||
"",
|
||||
app.inflight.available_permits(),
|
||||
);
|
||||
if let Some(job) = app.acme.view() {
|
||||
f.add(
|
||||
"rust_proxy_acme_job_state",
|
||||
"gauge",
|
||||
"Latest ACME job state; no domains or secrets.",
|
||||
&labels(&[("provider", &job.provider), ("state", job.status)]),
|
||||
1,
|
||||
);
|
||||
}
|
||||
for (index, listener) in app.active_listeners.iter().enumerate() {
|
||||
let index = index.to_string();
|
||||
let address = listener.address.to_string();
|
||||
let l = labels(&[("listener", &index), ("address", &address)]);
|
||||
for (protocol, enabled) in [
|
||||
("http1", true),
|
||||
("http2", listener.tls && listener.http2),
|
||||
("http3", listener.http3.is_some()),
|
||||
("tls", listener.tls),
|
||||
] {
|
||||
f.add(
|
||||
"rust_proxy_listener_protocol_enabled",
|
||||
"gauge",
|
||||
"Protocols active per listener.",
|
||||
&format!("{l},protocol=\"{protocol}\""),
|
||||
u8::from(enabled),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
fn render_vhosts(snapshot: &crate::config::Snapshot, f: &mut Families) {
|
||||
let config = &snapshot.envelope.config;
|
||||
let settings = &config.prometheus;
|
||||
if settings.include_vhosts {
|
||||
for redirect in &config.redirects {
|
||||
let l = labels(&[("redirect", &redirect.id)]);
|
||||
f.add(
|
||||
"rust_proxy_redirect_host_enabled",
|
||||
"gauge",
|
||||
"Desired redirect-host enable flag.",
|
||||
&l,
|
||||
u8::from(redirect.enabled),
|
||||
);
|
||||
f.add(
|
||||
"rust_proxy_redirect_host_domains",
|
||||
"gauge",
|
||||
"Configured DNS names for a redirect host.",
|
||||
&l,
|
||||
redirect.frontend.hosts.len(),
|
||||
);
|
||||
f.add(
|
||||
"rust_proxy_redirect_host_rules",
|
||||
"gauge",
|
||||
"Configured redirect rules.",
|
||||
&l,
|
||||
redirect.rules.len(),
|
||||
);
|
||||
}
|
||||
for host in &config.not_found_hosts {
|
||||
let l = labels(&[("host", &host.id)]);
|
||||
f.add(
|
||||
"rust_proxy_not_found_host_enabled",
|
||||
"gauge",
|
||||
"Desired local-response/404-host enable flag.",
|
||||
&l,
|
||||
u8::from(host.enabled),
|
||||
);
|
||||
f.add(
|
||||
"rust_proxy_not_found_host_domains",
|
||||
"gauge",
|
||||
"Configured DNS names for a local-response/404 host.",
|
||||
&l,
|
||||
host.frontend.hosts.len(),
|
||||
);
|
||||
f.add(
|
||||
"rust_proxy_not_found_host_status",
|
||||
"gauge",
|
||||
"Configured local-response HTTP status.",
|
||||
&l,
|
||||
host.page.status.unwrap_or(404),
|
||||
);
|
||||
}
|
||||
for host in &config.vhosts {
|
||||
let l = labels(&[("vhost", &host.id)]);
|
||||
f.add(
|
||||
"rust_proxy_vhost_enabled",
|
||||
"gauge",
|
||||
"Desired vhost enable flag.",
|
||||
&l,
|
||||
u8::from(host.enabled),
|
||||
);
|
||||
f.add(
|
||||
"rust_proxy_vhost_domains",
|
||||
"gauge",
|
||||
"Configured DNS names for a vhost.",
|
||||
&l,
|
||||
host.frontend.hosts.len(),
|
||||
);
|
||||
f.add(
|
||||
"rust_proxy_vhost_paths",
|
||||
"gauge",
|
||||
"Configured path rules for a vhost.",
|
||||
&l,
|
||||
host.routes.len(),
|
||||
);
|
||||
if let Ok(cache) = host.cache.resolve(&config.cache) {
|
||||
f.add(
|
||||
"rust_proxy_vhost_cache_enabled",
|
||||
"gauge",
|
||||
"Desired effective cache switch; request privacy rules still apply.",
|
||||
&l,
|
||||
u8::from(cache.enabled),
|
||||
);
|
||||
f.add(
|
||||
"rust_proxy_vhost_cache_max_object_bytes",
|
||||
"gauge",
|
||||
"Effective maximum cache object size.",
|
||||
&l,
|
||||
cache.max_object_bytes,
|
||||
);
|
||||
f.add(
|
||||
"rust_proxy_vhost_cache_max_ttl_seconds",
|
||||
"gauge",
|
||||
"Effective maximum cache TTL.",
|
||||
&l,
|
||||
cache.max_ttl_seconds,
|
||||
);
|
||||
}
|
||||
for domain in &host.frontend.hosts {
|
||||
f.add(
|
||||
"rust_proxy_vhost_domain_info",
|
||||
"gauge",
|
||||
"Configured name ownership.",
|
||||
&labels(&[("vhost", &host.id), ("domain", domain)]),
|
||||
1,
|
||||
);
|
||||
}
|
||||
let m = &snapshot.vhost_metrics[&host.id];
|
||||
for (name, counter) in [
|
||||
("requests_total", &m.requests),
|
||||
("completed_total", &m.completed),
|
||||
("errors_total", &m.errors),
|
||||
("rejected_total", &m.rejected),
|
||||
("request_bytes_total", &m.bytes_in),
|
||||
("response_bytes_total", &m.bytes_out),
|
||||
] {
|
||||
f.add(
|
||||
&format!("rust_proxy_vhost_{name}"),
|
||||
"counter",
|
||||
"Vhost exchange counter; reset on process restart or vhost removal.",
|
||||
&l,
|
||||
counter.load(Ordering::Relaxed),
|
||||
);
|
||||
}
|
||||
f.add(
|
||||
"rust_proxy_vhost_active",
|
||||
"gauge",
|
||||
"Admitted unfinished exchanges.",
|
||||
&l,
|
||||
m.active.load(Ordering::Relaxed),
|
||||
);
|
||||
for (index, counter) in m.status.iter().enumerate() {
|
||||
f.add(
|
||||
"rust_proxy_vhost_responses_total",
|
||||
"counter",
|
||||
"Completed exchanges by status class.",
|
||||
&format!("{l},class=\"{}xx\"", index + 1),
|
||||
counter.load(Ordering::Relaxed),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,245 @@
|
||||
fn render_misc(app: &App, snapshot: &crate::config::Snapshot, f: &mut Families) {
|
||||
let config = &snapshot.envelope.config;
|
||||
let settings = &config.prometheus;
|
||||
if settings.include_streams {
|
||||
for host in &config.streams {
|
||||
let protocol = match host.protocol {
|
||||
crate::stream::Protocol::Tcp => "tcp",
|
||||
crate::stream::Protocol::Udp => "udp",
|
||||
};
|
||||
f.add(
|
||||
"rust_proxy_stream_enabled",
|
||||
"gauge",
|
||||
"Desired Layer-4 stream enable flag.",
|
||||
&labels(&[("stream", &host.id), ("protocol", protocol)]),
|
||||
u8::from(host.enabled),
|
||||
);
|
||||
}
|
||||
for host in &app.active_streams {
|
||||
let Some(metrics) = app.stream_metrics.get(&host.id) else {
|
||||
continue;
|
||||
};
|
||||
let protocol = match host.protocol {
|
||||
crate::stream::Protocol::Tcp => "tcp",
|
||||
crate::stream::Protocol::Udp => "udp",
|
||||
};
|
||||
let l = labels(&[("stream", &host.id), ("protocol", protocol)]);
|
||||
let view = metrics.view();
|
||||
for (name, kind, value) in [
|
||||
("sessions_total", "counter", view.sessions_total),
|
||||
(
|
||||
"sessions_completed_total",
|
||||
"counter",
|
||||
view.sessions_completed,
|
||||
),
|
||||
("active", "gauge", view.active),
|
||||
("rejected_total", "counter", view.rejected),
|
||||
("errors_total", "counter", view.errors),
|
||||
("bytes_in_total", "counter", view.bytes_in),
|
||||
("bytes_out_total", "counter", view.bytes_out),
|
||||
("packets_in_total", "counter", view.packets_in),
|
||||
("packets_out_total", "counter", view.packets_out),
|
||||
] {
|
||||
f.add(
|
||||
&format!("rust_proxy_stream_{name}"),
|
||||
kind,
|
||||
"Layer-4 stream observation; reset on process restart.",
|
||||
&l,
|
||||
value,
|
||||
);
|
||||
}
|
||||
f.add(
|
||||
"rust_proxy_stream_max_connections",
|
||||
"gauge",
|
||||
"Configured stream connection/session capacity.",
|
||||
&l,
|
||||
host.max_connections,
|
||||
);
|
||||
}
|
||||
}
|
||||
if settings.include_logging {
|
||||
let stats = app.logs.stats();
|
||||
f.add(
|
||||
"rust_proxy_log_sink_queue_capacity",
|
||||
"gauge",
|
||||
"Structured log sink queue capacity.",
|
||||
"",
|
||||
stats.queue_capacity,
|
||||
);
|
||||
f.add(
|
||||
"rust_proxy_log_sink_queue_dropped_total",
|
||||
"counter",
|
||||
"Structured log entries dropped because the sink queue was full.",
|
||||
"",
|
||||
stats.queue_dropped,
|
||||
);
|
||||
f.add(
|
||||
"rust_proxy_log_sink_sent_total",
|
||||
"counter",
|
||||
"Structured log entries delivered to configured sinks.",
|
||||
"",
|
||||
stats.sent,
|
||||
);
|
||||
f.add(
|
||||
"rust_proxy_log_sink_errors_total",
|
||||
"counter",
|
||||
"Structured log sink delivery errors.",
|
||||
"",
|
||||
stats.errors,
|
||||
);
|
||||
}
|
||||
if settings.include_cache_domains {
|
||||
// Bounded resident inspector. Counter families are gauges deliberately:
|
||||
// these per-domain values reset on config publication, unlike globals.
|
||||
let dashboard = app.cache.dashboard(None, 0, 1);
|
||||
f.add(
|
||||
"rust_proxy_cache_statistics_revision",
|
||||
"gauge",
|
||||
"Revision that owns domain statistics.",
|
||||
"",
|
||||
dashboard.statistics_revision,
|
||||
);
|
||||
for domain in dashboard.domains {
|
||||
// Only configured names, never arbitrary Host input or unknown scope.
|
||||
if !config
|
||||
.vhosts
|
||||
.iter()
|
||||
.any(|h| h.id == domain.vhost && h.frontend.hosts.contains(&domain.domain))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let l = labels(&[("vhost", &domain.vhost), ("domain", &domain.domain)]);
|
||||
for (name, n) in [
|
||||
("objects", domain.entries as u64),
|
||||
("hot_objects", domain.hot_entries as u64),
|
||||
("resident_bytes", domain.resident_bytes as u64),
|
||||
("cold_objects", domain.cold_entries as u64),
|
||||
("cold_bytes", domain.cold_bytes),
|
||||
("hits", domain.counters.hits),
|
||||
("misses", domain.counters.misses),
|
||||
("bypasses", domain.counters.bypasses),
|
||||
("stored", domain.counters.stored),
|
||||
("saved_body_bytes", domain.counters.saved_body_bytes),
|
||||
] {
|
||||
f.add(
|
||||
&format!("rust_proxy_cache_domain_{name}"),
|
||||
"gauge",
|
||||
"Domain cache observation since current configuration revision.",
|
||||
&l,
|
||||
n,
|
||||
);
|
||||
}
|
||||
f.add(
|
||||
"rust_proxy_cache_domain_hit_ratio",
|
||||
"gauge",
|
||||
"Domain HIT ratio 0..1.",
|
||||
&l,
|
||||
domain.hit_ratio,
|
||||
);
|
||||
for (reason, count) in domain.counters.reasons {
|
||||
f.add(
|
||||
"rust_proxy_cache_domain_decisions",
|
||||
"gauge",
|
||||
"Decision reasons since config publication.",
|
||||
&format!("{l},reason=\"{}\"", escaped(&reason)),
|
||||
count,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
fn render_histograms(snapshot: &crate::config::Snapshot, docker_targets: &DockerTargets<'_>, out: &mut String) {
|
||||
let config = &snapshot.envelope.config;
|
||||
let settings = &config.prometheus;
|
||||
if settings.include_vhosts {
|
||||
out.push_str("# HELP rust_proxy_vhost_exchange_duration_seconds Full exchange lifetime including streaming and cancellation.\n# TYPE rust_proxy_vhost_exchange_duration_seconds histogram\n");
|
||||
for (id, m) in &snapshot.vhost_metrics {
|
||||
if docker_targets.contains_key(id) {
|
||||
continue;
|
||||
}
|
||||
let l = labels(&[("vhost", id)]);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"rust_proxy_vhost_exchange_duration_seconds_sum{{{l}}} {:.6}",
|
||||
m.duration_us.load(Ordering::Relaxed) as f64 / 1e6
|
||||
);
|
||||
// Read the cumulative last bucket once for +Inf and count.
|
||||
let count =
|
||||
m.buckets[crate::metrics::DURATION_BUCKETS_US.len()].load(Ordering::Relaxed);
|
||||
let mut previous = 0;
|
||||
for (i, threshold) in crate::metrics::DURATION_BUCKETS_US.iter().enumerate() {
|
||||
let value = m.buckets[i]
|
||||
.load(Ordering::Relaxed)
|
||||
.max(previous)
|
||||
.min(count);
|
||||
previous = value;
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"rust_proxy_vhost_exchange_duration_seconds_bucket{{{l},le=\"{}\"}} {value}",
|
||||
*threshold as f64 / 1e6
|
||||
);
|
||||
}
|
||||
let _ = writeln!(out, "rust_proxy_vhost_exchange_duration_seconds_bucket{{{l},le=\"+Inf\"}} {count}\nrust_proxy_vhost_exchange_duration_seconds_count{{{l}}} {count}");
|
||||
}
|
||||
}
|
||||
if settings.include_docker_vhosts {
|
||||
out.push_str("# HELP rust_proxy_docker_vhost_exchange_duration_seconds Full Docker-vhost exchange lifetime including streaming and cancellation.\n# TYPE rust_proxy_docker_vhost_exchange_duration_seconds histogram\n");
|
||||
for (id, m) in &snapshot.vhost_metrics {
|
||||
let Some((agent, assignment)) = docker_targets.get(id) else {
|
||||
continue;
|
||||
};
|
||||
let l = labels(&[
|
||||
("agent", &agent.id),
|
||||
("vhost", id),
|
||||
("service", &assignment.service_key),
|
||||
]);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"rust_proxy_docker_vhost_exchange_duration_seconds_sum{{{l}}} {:.6}",
|
||||
m.duration_us.load(Ordering::Relaxed) as f64 / 1e6
|
||||
);
|
||||
let count =
|
||||
m.buckets[crate::metrics::DURATION_BUCKETS_US.len()].load(Ordering::Relaxed);
|
||||
let mut previous = 0;
|
||||
for (i, threshold) in crate::metrics::DURATION_BUCKETS_US.iter().enumerate() {
|
||||
let value = m.buckets[i]
|
||||
.load(Ordering::Relaxed)
|
||||
.max(previous)
|
||||
.min(count);
|
||||
previous = value;
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"rust_proxy_docker_vhost_exchange_duration_seconds_bucket{{{l},le=\"{}\"}} {value}",
|
||||
*threshold as f64 / 1e6
|
||||
);
|
||||
}
|
||||
let _ = writeln!(out, "rust_proxy_docker_vhost_exchange_duration_seconds_bucket{{{l},le=\"+Inf\"}} {count}\nrust_proxy_docker_vhost_exchange_duration_seconds_count{{{l}}} {count}");
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
pub fn render(app: &App) -> String {
|
||||
let snapshot = app.config.load_full();
|
||||
let config = &snapshot.envelope.config;
|
||||
let docker_targets = docker_targets(config);
|
||||
let mut out = app.metrics.prometheus(
|
||||
app.logs.dropped.load(Ordering::Relaxed),
|
||||
snapshot.envelope.revision,
|
||||
);
|
||||
out.push_str(&app.cache.prometheus());
|
||||
if config.prometheus.include_system {
|
||||
out.push_str(&app.diagnostics.prometheus());
|
||||
}
|
||||
let mut families = Families::default();
|
||||
render_core(app, &snapshot, &mut out, &mut families);
|
||||
render_vhosts(&snapshot, &mut families);
|
||||
render_backends(&snapshot, &docker_targets, &mut families);
|
||||
render_docker(&snapshot, &docker_targets, &mut families);
|
||||
render_misc(app, &snapshot, &mut families);
|
||||
families.finish(&mut out);
|
||||
render_histograms(&snapshot, &docker_targets, &mut out);
|
||||
out
|
||||
}
|
||||
@@ -0,0 +1,268 @@
|
||||
use crate::{runtime::App, server::PeerAddr};
|
||||
use anyhow::{ensure, Context, Result};
|
||||
use axum::{
|
||||
extract::{Request, State},
|
||||
response::{IntoResponse, Response},
|
||||
Router,
|
||||
};
|
||||
use http::StatusCode;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
fmt::Write,
|
||||
net::SocketAddr,
|
||||
sync::{atomic::Ordering, Arc},
|
||||
};
|
||||
use zeroize::Zeroizing;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(default, deny_unknown_fields)]
|
||||
pub struct Settings {
|
||||
pub enabled: bool,
|
||||
pub listen: SocketAddr,
|
||||
pub path: String,
|
||||
pub allowed_cidrs: Vec<String>,
|
||||
/// Only a variable NAME is persisted; its value is never in config/API.
|
||||
pub bearer_token_env: Option<String>,
|
||||
/// Allow clients already matched by `allowed_cidrs` to scrape without a bearer token.
|
||||
/// Intended only for private/trusted monitoring networks.
|
||||
pub allow_trusted_network_without_token: bool,
|
||||
pub include_system: bool,
|
||||
pub include_vhosts: bool,
|
||||
pub include_backends: bool,
|
||||
pub include_docker_vhosts: bool,
|
||||
pub include_cache_domains: bool,
|
||||
pub include_streams: bool,
|
||||
pub include_logging: bool,
|
||||
}
|
||||
impl Default for Settings {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
listen: "127.0.0.1:9191".parse().expect("static socket"),
|
||||
path: "/metrics".into(),
|
||||
allowed_cidrs: vec!["127.0.0.0/8".into(), "::1/128".into()],
|
||||
bearer_token_env: None,
|
||||
allow_trusted_network_without_token: false,
|
||||
include_system: true,
|
||||
include_vhosts: true,
|
||||
include_backends: true,
|
||||
include_docker_vhosts: true,
|
||||
include_cache_domains: true,
|
||||
include_streams: true,
|
||||
include_logging: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
impl Settings {
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
ensure!(
|
||||
self.listen.port() > 0 && !self.listen.ip().is_multicast(),
|
||||
"metrics: listen needs a unicast/loopback IP and nonzero port"
|
||||
);
|
||||
ensure!(
|
||||
self.path.starts_with('/')
|
||||
&& self.path.len() <= 128
|
||||
&& !self.path.contains("//")
|
||||
&& !self.path.contains("..")
|
||||
&& self
|
||||
.path
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_alphanumeric() || b"/_-.".contains(&b)),
|
||||
"metrics: path must be a plain absolute HTTP path without a query"
|
||||
);
|
||||
ensure!(
|
||||
!self.allowed_cidrs.is_empty() && self.allowed_cidrs.len() <= 32,
|
||||
"metrics: specify 1..32 allowed client CIDRs"
|
||||
);
|
||||
for cidr in &self.allowed_cidrs {
|
||||
let network = cidr
|
||||
.parse::<ipnet::IpNet>()
|
||||
.context("metrics: invalid client CIDR")?;
|
||||
if self.allow_trusted_network_without_token {
|
||||
ensure!(
|
||||
network.prefix_len() > 0,
|
||||
"metrics: token-free trusted networks cannot use a /0 CIDR"
|
||||
);
|
||||
}
|
||||
}
|
||||
if let Some(name) = &self.bearer_token_env {
|
||||
ensure!(
|
||||
!name.is_empty()
|
||||
&& name.len() <= 128
|
||||
&& name
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_uppercase() || b.is_ascii_digit() || b == b'_')
|
||||
&& !name.as_bytes()[0].is_ascii_digit(),
|
||||
"metrics: token environment variable must use A-Z, 0-9, underscore"
|
||||
);
|
||||
}
|
||||
ensure!(
|
||||
!self.enabled
|
||||
|| self.listen.ip().is_loopback()
|
||||
|| self.bearer_token_env.is_some()
|
||||
|| self.allow_trusted_network_without_token,
|
||||
"metrics: a non-loopback bind requires a bearer token or explicit trusted-network token bypass; protect HTTP with a private network/TLS tunnel"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
pub fn restart_differs(&self, other: &Self) -> bool {
|
||||
self.enabled != other.enabled
|
||||
|| self.listen != other.listen
|
||||
|| self.bearer_token_env != other.bearer_token_env
|
||||
}
|
||||
pub fn token(&self) -> Result<Option<Zeroizing<String>>> {
|
||||
let Some(name) = &self.bearer_token_env else {
|
||||
return Ok(None);
|
||||
};
|
||||
let value = Zeroizing::new(
|
||||
std::env::var(name).context("metrics: configured token variable is missing")?,
|
||||
);
|
||||
ensure!(
|
||||
value.len() >= 32 && value.len() <= 4096 && value.bytes().all(|b| b.is_ascii_graphic()),
|
||||
"metrics: token must contain 32..4096 non-whitespace ASCII characters"
|
||||
);
|
||||
Ok(Some(value))
|
||||
}
|
||||
}
|
||||
struct Exporter {
|
||||
app: Arc<App>,
|
||||
token: Option<Zeroizing<String>>,
|
||||
slots: tokio::sync::Semaphore,
|
||||
}
|
||||
pub fn router(app: Arc<App>, token: Option<Zeroizing<String>>) -> Router {
|
||||
Router::new()
|
||||
.fallback(scrape)
|
||||
.with_state(Arc::new(Exporter {
|
||||
app,
|
||||
token,
|
||||
slots: tokio::sync::Semaphore::new(4),
|
||||
}))
|
||||
}
|
||||
async fn scrape(State(exporter): State<Arc<Exporter>>, request: Request) -> Response {
|
||||
let config = exporter.app.config.load_full();
|
||||
let settings = &config.envelope.config.prometheus;
|
||||
// Revocation is immediate even when changing bind/credentials needs restart.
|
||||
if !settings.enabled
|
||||
|| settings.restart_differs(&exporter.app.active_prometheus)
|
||||
|| request.uri().path() != settings.path
|
||||
|| request.uri().query().is_some()
|
||||
{
|
||||
return StatusCode::NOT_FOUND.into_response();
|
||||
}
|
||||
if request.method() != http::Method::GET {
|
||||
return (StatusCode::METHOD_NOT_ALLOWED, [("allow", "GET")]).into_response();
|
||||
}
|
||||
let allowed = request.extensions().get::<PeerAddr>().is_some_and(|peer| {
|
||||
let peer = crate::vhost::canonical_ip(peer.0.ip());
|
||||
settings
|
||||
.allowed_cidrs
|
||||
.iter()
|
||||
.filter_map(|v| v.parse::<ipnet::IpNet>().ok())
|
||||
.any(|cidr| cidr.contains(&peer))
|
||||
});
|
||||
if !allowed {
|
||||
return StatusCode::FORBIDDEN.into_response();
|
||||
}
|
||||
if let Some(expected) = &exporter.token {
|
||||
if settings.allow_trusted_network_without_token {
|
||||
// The actual transport peer already matched an administrator-configured
|
||||
// CIDR above. Forwarded headers never participate in this decision.
|
||||
} else {
|
||||
let values: Vec<_> = request
|
||||
.headers()
|
||||
.get_all(http::header::AUTHORIZATION)
|
||||
.iter()
|
||||
.collect();
|
||||
let valid = values.len() == 1
|
||||
&& values[0]
|
||||
.to_str()
|
||||
.ok()
|
||||
.and_then(|v| v.strip_prefix("Bearer "))
|
||||
.is_some_and(|value| crate::auth::equal_secret(value, expected));
|
||||
if !valid {
|
||||
return (
|
||||
StatusCode::UNAUTHORIZED,
|
||||
[
|
||||
("www-authenticate", "Bearer"),
|
||||
("cache-control", "no-store"),
|
||||
],
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
}
|
||||
}
|
||||
let Ok(_permit) = exporter.slots.try_acquire() else {
|
||||
return StatusCode::TOO_MANY_REQUESTS.into_response();
|
||||
};
|
||||
response(&exporter.app)
|
||||
}
|
||||
pub fn response(app: &App) -> Response {
|
||||
(
|
||||
[
|
||||
("content-type", "text/plain; version=0.0.4; charset=utf-8"),
|
||||
("cache-control", "no-store"),
|
||||
("x-content-type-options", "nosniff"),
|
||||
],
|
||||
render(app),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
fn escaped(value: &str) -> String {
|
||||
value
|
||||
.replace('\\', "\\\\")
|
||||
.replace('\n', "\\n")
|
||||
.replace('"', "\\\"")
|
||||
}
|
||||
fn labels(items: &[(&str, &str)]) -> String {
|
||||
items
|
||||
.iter()
|
||||
.map(|(k, v)| format!("{k}=\"{}\"", escaped(v)))
|
||||
.collect::<Vec<_>>()
|
||||
.join(",")
|
||||
}
|
||||
#[derive(Default)]
|
||||
struct Families(BTreeMap<String, (&'static str, &'static str, Vec<String>)>);
|
||||
impl Families {
|
||||
fn add(
|
||||
&mut self,
|
||||
name: &str,
|
||||
kind: &'static str,
|
||||
help: &'static str,
|
||||
labels: &str,
|
||||
value: impl std::fmt::Display,
|
||||
) {
|
||||
let family = self
|
||||
.0
|
||||
.entry(name.into())
|
||||
.or_insert((kind, help, Vec::new()));
|
||||
family.2.push(if labels.is_empty() {
|
||||
format!("{name} {value}\n")
|
||||
} else {
|
||||
format!("{name}{{{labels}}} {value}\n")
|
||||
});
|
||||
}
|
||||
fn finish(self, out: &mut String) {
|
||||
for (name, (kind, help, samples)) in self.0 {
|
||||
let _ = writeln!(out, "# HELP {name} {help}\n# TYPE {name} {kind}");
|
||||
for sample in samples {
|
||||
out.push_str(&sample);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type DockerTargets<'a> = BTreeMap<String, (&'a crate::docker::Agent, &'a crate::docker::Assignment)>;
|
||||
|
||||
fn docker_targets(config: &crate::config::Config) -> DockerTargets<'_> {
|
||||
let mut targets = BTreeMap::new();
|
||||
for agent in &config.docker_agents.agents {
|
||||
for assignment in &agent.assignments {
|
||||
targets.insert(
|
||||
crate::docker::generated_id(&agent.id, &assignment.service_key),
|
||||
(agent, assignment),
|
||||
);
|
||||
}
|
||||
}
|
||||
targets
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn secure_defaults_and_paths() {
|
||||
let mut settings = Settings::default();
|
||||
assert!(!settings.enabled);
|
||||
settings.validate().unwrap();
|
||||
settings.enabled = true;
|
||||
settings.listen = "0.0.0.0:9191".parse().unwrap();
|
||||
assert!(settings.validate().is_err());
|
||||
settings.bearer_token_env = Some("RUST_PROXY_METRICS_TOKEN".into());
|
||||
settings.validate().unwrap();
|
||||
settings.bearer_token_env = None;
|
||||
settings.allow_trusted_network_without_token = true;
|
||||
settings.allowed_cidrs = vec!["10.20.30.0/24".into()];
|
||||
settings.validate().unwrap();
|
||||
settings.allowed_cidrs = vec!["0.0.0.0/0".into()];
|
||||
assert!(settings.validate().is_err());
|
||||
settings.allowed_cidrs = vec!["10.20.30.0/24".into()];
|
||||
for path in [
|
||||
"metrics",
|
||||
"/metrics?secret=x",
|
||||
"/../metrics",
|
||||
"/m\nTYPE fake gauge",
|
||||
] {
|
||||
settings.path = path.into();
|
||||
assert!(settings.validate().is_err());
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn escaping_and_grouping() {
|
||||
assert_eq!(escaped("a\"b\\c\nd"), "a\\\"b\\\\c\\nd");
|
||||
let mut f = Families::default();
|
||||
f.add("x", "gauge", "Test.", "a=\"1\"", 1);
|
||||
f.add("x", "gauge", "Test.", "a=\"2\"", 2);
|
||||
let mut text = String::new();
|
||||
f.finish(&mut text);
|
||||
assert_eq!(text.matches("# TYPE x").count(), 1);
|
||||
assert!(text.ends_with('\n'));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,389 @@
|
||||
fn render_backends(snapshot: &crate::config::Snapshot, docker_targets: &DockerTargets<'_>, f: &mut Families) {
|
||||
let config = &snapshot.envelope.config;
|
||||
let settings = &config.prometheus;
|
||||
if settings.include_backends {
|
||||
for pool in &snapshot.pools {
|
||||
if docker_targets.contains_key(&pool.id) {
|
||||
continue;
|
||||
}
|
||||
let view = pool.view();
|
||||
if let (Some(algorithm), Some(health_mode), Some(sticky_mode)) = (
|
||||
view["algorithm"].as_str(),
|
||||
view["health_mode"].as_str(),
|
||||
view["sticky_mode"].as_str(),
|
||||
) {
|
||||
f.add(
|
||||
"rust_proxy_backend_pool_info",
|
||||
"gauge",
|
||||
"Bounded backend-pool selection and health policy.",
|
||||
&labels(&[
|
||||
("vhost", &pool.id),
|
||||
("algorithm", algorithm),
|
||||
("health_mode", health_mode),
|
||||
("sticky_mode", sticky_mode),
|
||||
]),
|
||||
1,
|
||||
);
|
||||
}
|
||||
f.add(
|
||||
"rust_proxy_backend_pool_ready",
|
||||
"gauge",
|
||||
"At least one healthy eligible pool member.",
|
||||
&labels(&[("vhost", &pool.id)]),
|
||||
u8::from(pool.ready()),
|
||||
);
|
||||
if let Some(members) = view["members"].as_array() {
|
||||
for member in members {
|
||||
let l = labels(&[
|
||||
("vhost", &pool.id),
|
||||
("backend", member["id"].as_str().unwrap_or("unknown")),
|
||||
]);
|
||||
let status = member["status"].as_str().unwrap_or("unknown");
|
||||
f.add(
|
||||
"rust_proxy_backend_status_info",
|
||||
"gauge",
|
||||
"Latest bounded backend selection/health state.",
|
||||
&format!("{l},status=\"{}\"", escaped(status)),
|
||||
1,
|
||||
);
|
||||
for (name, flag) in [
|
||||
(
|
||||
"up",
|
||||
status != "disabled" && member["healthy"].as_bool().unwrap_or(false),
|
||||
),
|
||||
("draining", status == "draining"),
|
||||
("backup", member["backup"].as_bool().unwrap_or(false)),
|
||||
] {
|
||||
f.add(
|
||||
&format!("rust_proxy_backend_{name}"),
|
||||
"gauge",
|
||||
"Backend health or selection state.",
|
||||
&l,
|
||||
u8::from(flag),
|
||||
);
|
||||
}
|
||||
for (field, metric, kind) in [
|
||||
("active", "active_exchanges", "gauge"),
|
||||
("requests", "requests_total", "counter"),
|
||||
("errors", "errors_total", "counter"),
|
||||
("bytes_in", "bytes_in_total", "counter"),
|
||||
("bytes_out", "bytes_out_total", "counter"),
|
||||
("weight", "weight", "gauge"),
|
||||
("failures", "consecutive_failures", "gauge"),
|
||||
("passive_failures", "passive_failures", "gauge"),
|
||||
("successes", "consecutive_successes", "gauge"),
|
||||
("passive_successes", "passive_successes", "gauge"),
|
||||
("transitions", "health_transitions_total", "counter"),
|
||||
("active_health_failures_total", "active_health_failures_total", "counter"),
|
||||
("passive_health_failures_total", "passive_health_failures_total", "counter"),
|
||||
("http_5xx_total", "http_5xx_total", "counter"),
|
||||
("http_5xx_total", "passive_http_5xx_total", "counter"),
|
||||
("timeouts_total", "timeouts_total", "counter"),
|
||||
("timeouts_total", "passive_timeouts_total", "counter"),
|
||||
("connect_errors_total", "connect_errors_total", "counter"),
|
||||
("connect_errors_total", "passive_connect_errors_total", "counter"),
|
||||
("io_errors_total", "io_errors_total", "counter"),
|
||||
("io_errors_total", "passive_io_errors_total", "counter"),
|
||||
("slow_responses_total", "slow_responses_total", "counter"),
|
||||
("slow_responses_total", "passive_slow_total", "counter"),
|
||||
("capacity_rejections_total", "capacity_rejections_total", "counter"),
|
||||
("queue_timeouts_total", "queue_timeouts_total", "counter"),
|
||||
("dns_errors_total", "dns_errors_total", "counter"),
|
||||
("tls_errors_total", "tls_errors_total", "counter"),
|
||||
("response_header_timeouts_total", "response_header_timeouts_total", "counter"),
|
||||
("read_timeouts_total", "read_timeouts_total", "counter"),
|
||||
("send_timeouts_total", "send_timeouts_total", "counter"),
|
||||
("max_concurrent_requests", "max_concurrent_requests", "gauge"),
|
||||
("queue_timeout_ms", "queue_timeout_milliseconds", "gauge"),
|
||||
("last_status", "last_http_status", "gauge"),
|
||||
] {
|
||||
if let Some(n) = member[field].as_u64() {
|
||||
f.add(
|
||||
&format!("rust_proxy_backend_{metric}"),
|
||||
kind,
|
||||
"Backend observation.",
|
||||
&l,
|
||||
n,
|
||||
);
|
||||
}
|
||||
}
|
||||
for (field, metric, divisor) in [
|
||||
("latency_ms", "latency_seconds", 1000.0),
|
||||
("ewma_latency_ms", "ewma_latency_seconds", 1000.0),
|
||||
("last_probe_ms", "last_probe_timestamp_seconds", 1000.0),
|
||||
("last_failure_at_ms", "last_failure_timestamp_seconds", 1000.0),
|
||||
("last_ejected_ms", "last_ejected_timestamp_seconds", 1000.0),
|
||||
("last_recovered_ms", "last_recovered_timestamp_seconds", 1000.0),
|
||||
("traffic_bytes_per_second", "traffic_bytes_per_second", 1.0),
|
||||
] {
|
||||
if let Some(n) = member[field].as_f64() {
|
||||
f.add(
|
||||
&format!("rust_proxy_backend_{metric}"),
|
||||
"gauge",
|
||||
"Backend moving observation.",
|
||||
&l,
|
||||
n / divisor,
|
||||
);
|
||||
}
|
||||
}
|
||||
if let Some(result) = member["last_result"]
|
||||
.as_str()
|
||||
.filter(|value| !value.is_empty())
|
||||
{
|
||||
f.add(
|
||||
"rust_proxy_backend_last_probe_result_info",
|
||||
"gauge",
|
||||
"Latest bounded backend health-probe result.",
|
||||
&format!("{l},result=\"{}\"", escaped(result)),
|
||||
1,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
fn render_docker(snapshot: &crate::config::Snapshot, docker_targets: &DockerTargets<'_>, f: &mut Families) {
|
||||
let config = &snapshot.envelope.config;
|
||||
let settings = &config.prometheus;
|
||||
if settings.include_docker_vhosts {
|
||||
f.add(
|
||||
"rust_proxy_docker_agents_configured",
|
||||
"gauge",
|
||||
"Configured Docker discovery agents including disabled agents.",
|
||||
"",
|
||||
config.docker_agents.agents.len(),
|
||||
);
|
||||
f.add(
|
||||
"rust_proxy_docker_vhosts_configured",
|
||||
"gauge",
|
||||
"Configured Docker vhost assignments including disabled assignments.",
|
||||
"",
|
||||
docker_targets.len(),
|
||||
);
|
||||
for agent in &config.docker_agents.agents {
|
||||
let agent_labels = labels(&[("agent", &agent.id)]);
|
||||
f.add(
|
||||
"rust_proxy_docker_agent_enabled",
|
||||
"gauge",
|
||||
"Desired Docker agent enable flag.",
|
||||
&agent_labels,
|
||||
u8::from(agent.enabled),
|
||||
);
|
||||
f.add(
|
||||
"rust_proxy_docker_agent_assignments",
|
||||
"gauge",
|
||||
"Configured Docker vhost assignments for an agent.",
|
||||
&agent_labels,
|
||||
agent.assignments.len(),
|
||||
);
|
||||
for assignment in &agent.assignments {
|
||||
let vhost = crate::docker::generated_id(&agent.id, &assignment.service_key);
|
||||
let base_labels = labels(&[
|
||||
("agent", &agent.id),
|
||||
("vhost", &vhost),
|
||||
("service", &assignment.service_key),
|
||||
]);
|
||||
let domains = crate::docker::assignment_hosts(assignment, &agent.base_domain)
|
||||
.unwrap_or_default();
|
||||
f.add(
|
||||
"rust_proxy_docker_vhost_enabled",
|
||||
"gauge",
|
||||
"Desired Docker vhost assignment enable flag.",
|
||||
&base_labels,
|
||||
u8::from(agent.enabled && assignment.enabled),
|
||||
);
|
||||
f.add(
|
||||
"rust_proxy_docker_vhost_domains",
|
||||
"gauge",
|
||||
"Configured DNS names for a Docker vhost assignment.",
|
||||
&base_labels,
|
||||
domains.len(),
|
||||
);
|
||||
for domain in domains {
|
||||
f.add(
|
||||
"rust_proxy_docker_vhost_domain_info",
|
||||
"gauge",
|
||||
"Configured Docker vhost name ownership.",
|
||||
&format!("{base_labels},domain=\"{}\"", escaped(&domain)),
|
||||
1,
|
||||
);
|
||||
}
|
||||
if let Some(m) = snapshot.vhost_metrics.get(&vhost) {
|
||||
for (name, counter) in [
|
||||
("requests_total", &m.requests),
|
||||
("completed_total", &m.completed),
|
||||
("errors_total", &m.errors),
|
||||
("rejected_total", &m.rejected),
|
||||
("request_bytes_total", &m.bytes_in),
|
||||
("response_bytes_total", &m.bytes_out),
|
||||
] {
|
||||
f.add(
|
||||
&format!("rust_proxy_docker_vhost_{name}"),
|
||||
"counter",
|
||||
"Docker vhost exchange counter; reset on process restart or assignment removal.",
|
||||
&base_labels,
|
||||
counter.load(Ordering::Relaxed),
|
||||
);
|
||||
}
|
||||
f.add(
|
||||
"rust_proxy_docker_vhost_active",
|
||||
"gauge",
|
||||
"Admitted unfinished Docker-vhost exchanges.",
|
||||
&base_labels,
|
||||
m.active.load(Ordering::Relaxed),
|
||||
);
|
||||
for (index, counter) in m.status.iter().enumerate() {
|
||||
f.add(
|
||||
"rust_proxy_docker_vhost_responses_total",
|
||||
"counter",
|
||||
"Completed Docker-vhost exchanges by status class.",
|
||||
&format!("{base_labels},class=\"{}xx\"", index + 1),
|
||||
counter.load(Ordering::Relaxed),
|
||||
);
|
||||
}
|
||||
}
|
||||
let Some(pool) = snapshot.pools.iter().find(|pool| pool.id == vhost) else {
|
||||
continue;
|
||||
};
|
||||
let view = pool.view();
|
||||
if let (Some(algorithm), Some(health_mode), Some(sticky_mode)) = (
|
||||
view["algorithm"].as_str(),
|
||||
view["health_mode"].as_str(),
|
||||
view["sticky_mode"].as_str(),
|
||||
) {
|
||||
f.add(
|
||||
"rust_proxy_docker_backend_pool_info",
|
||||
"gauge",
|
||||
"Bounded Docker backend-pool selection and health policy.",
|
||||
&format!(
|
||||
"{base_labels},algorithm=\"{}\",health_mode=\"{}\",sticky_mode=\"{}\"",
|
||||
escaped(algorithm),
|
||||
escaped(health_mode),
|
||||
escaped(sticky_mode)
|
||||
),
|
||||
1,
|
||||
);
|
||||
}
|
||||
f.add(
|
||||
"rust_proxy_docker_backend_pool_ready",
|
||||
"gauge",
|
||||
"At least one healthy eligible Docker backend member.",
|
||||
&base_labels,
|
||||
u8::from(pool.ready()),
|
||||
);
|
||||
if let Some(members) = view["members"].as_array() {
|
||||
for member in members {
|
||||
let backend = member["id"].as_str().unwrap_or("unknown");
|
||||
let member_labels =
|
||||
format!("{base_labels},backend=\"{}\"", escaped(backend));
|
||||
let status = member["status"].as_str().unwrap_or("unknown");
|
||||
f.add(
|
||||
"rust_proxy_docker_backend_status_info",
|
||||
"gauge",
|
||||
"Latest bounded Docker backend selection/health state.",
|
||||
&format!("{member_labels},status=\"{}\"", escaped(status)),
|
||||
1,
|
||||
);
|
||||
for (name, flag) in [
|
||||
(
|
||||
"up",
|
||||
status != "disabled"
|
||||
&& member["healthy"].as_bool().unwrap_or(false),
|
||||
),
|
||||
("draining", status == "draining"),
|
||||
("backup", member["backup"].as_bool().unwrap_or(false)),
|
||||
] {
|
||||
f.add(
|
||||
&format!("rust_proxy_docker_backend_{name}"),
|
||||
"gauge",
|
||||
"Docker backend health or selection state.",
|
||||
&member_labels,
|
||||
u8::from(flag),
|
||||
);
|
||||
}
|
||||
for (field, metric, kind) in [
|
||||
("active", "active_exchanges", "gauge"),
|
||||
("requests", "requests_total", "counter"),
|
||||
("errors", "errors_total", "counter"),
|
||||
("bytes_in", "bytes_in_total", "counter"),
|
||||
("bytes_out", "bytes_out_total", "counter"),
|
||||
("weight", "weight", "gauge"),
|
||||
("failures", "consecutive_failures", "gauge"),
|
||||
("passive_failures", "passive_failures", "gauge"),
|
||||
("successes", "consecutive_successes", "gauge"),
|
||||
("passive_successes", "passive_successes", "gauge"),
|
||||
("transitions", "health_transitions_total", "counter"),
|
||||
("active_health_failures_total", "active_health_failures_total", "counter"),
|
||||
("passive_health_failures_total", "passive_health_failures_total", "counter"),
|
||||
("http_5xx_total", "http_5xx_total", "counter"),
|
||||
("http_5xx_total", "passive_http_5xx_total", "counter"),
|
||||
("timeouts_total", "timeouts_total", "counter"),
|
||||
("timeouts_total", "passive_timeouts_total", "counter"),
|
||||
("connect_errors_total", "connect_errors_total", "counter"),
|
||||
("connect_errors_total", "passive_connect_errors_total", "counter"),
|
||||
("io_errors_total", "io_errors_total", "counter"),
|
||||
("io_errors_total", "passive_io_errors_total", "counter"),
|
||||
("slow_responses_total", "slow_responses_total", "counter"),
|
||||
("slow_responses_total", "passive_slow_total", "counter"),
|
||||
("capacity_rejections_total", "capacity_rejections_total", "counter"),
|
||||
("queue_timeouts_total", "queue_timeouts_total", "counter"),
|
||||
("dns_errors_total", "dns_errors_total", "counter"),
|
||||
("tls_errors_total", "tls_errors_total", "counter"),
|
||||
("response_header_timeouts_total", "response_header_timeouts_total", "counter"),
|
||||
("read_timeouts_total", "read_timeouts_total", "counter"),
|
||||
("send_timeouts_total", "send_timeouts_total", "counter"),
|
||||
("max_concurrent_requests", "max_concurrent_requests", "gauge"),
|
||||
("queue_timeout_ms", "queue_timeout_milliseconds", "gauge"),
|
||||
("last_status", "last_http_status", "gauge"),
|
||||
] {
|
||||
if let Some(n) = member[field].as_u64() {
|
||||
f.add(
|
||||
&format!("rust_proxy_docker_backend_{metric}"),
|
||||
kind,
|
||||
"Docker backend observation.",
|
||||
&member_labels,
|
||||
n,
|
||||
);
|
||||
}
|
||||
}
|
||||
for (field, metric, divisor) in [
|
||||
("latency_ms", "latency_seconds", 1000.0),
|
||||
("ewma_latency_ms", "ewma_latency_seconds", 1000.0),
|
||||
("last_probe_ms", "last_probe_timestamp_seconds", 1000.0),
|
||||
("last_failure_at_ms", "last_failure_timestamp_seconds", 1000.0),
|
||||
("last_ejected_ms", "last_ejected_timestamp_seconds", 1000.0),
|
||||
("last_recovered_ms", "last_recovered_timestamp_seconds", 1000.0),
|
||||
("traffic_bytes_per_second", "traffic_bytes_per_second", 1.0),
|
||||
] {
|
||||
if let Some(n) = member[field].as_f64() {
|
||||
f.add(
|
||||
&format!("rust_proxy_docker_backend_{metric}"),
|
||||
"gauge",
|
||||
"Docker backend moving observation.",
|
||||
&member_labels,
|
||||
n / divisor,
|
||||
);
|
||||
}
|
||||
}
|
||||
if let Some(result) = member["last_result"]
|
||||
.as_str()
|
||||
.filter(|value| !value.is_empty())
|
||||
{
|
||||
f.add(
|
||||
"rust_proxy_docker_backend_last_probe_result_info",
|
||||
"gauge",
|
||||
"Latest bounded Docker backend health-probe result.",
|
||||
&format!("{member_labels},result=\"{}\"", escaped(result)),
|
||||
1,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Reference in new issue
Block a user