fn render_backends( snapshot: &crate::config::Snapshot, docker_targets: &DockerTargets<'_>, f: &mut Families, ) { let config = &snapshot.envelope.config; let settings = &config.prometheus; if settings.include_backends { for pool in &snapshot.pools { if docker_targets.contains_key(&pool.id) { continue; } let view = pool.view(); if let (Some(algorithm), Some(health_mode), Some(sticky_mode)) = ( view["algorithm"].as_str(), view["health_mode"].as_str(), view["sticky_mode"].as_str(), ) { f.add( "rust_proxy_backend_pool_info", "gauge", "Bounded backend-pool selection and health policy.", &labels(&[ ("vhost", &pool.id), ("algorithm", algorithm), ("health_mode", health_mode), ("sticky_mode", sticky_mode), ]), 1, ); } f.add( "rust_proxy_backend_pool_ready", "gauge", "At least one healthy eligible pool member.", &labels(&[("vhost", &pool.id)]), u8::from(pool.ready()), ); if let Some(members) = view["members"].as_array() { for member in members { let l = labels(&[ ("vhost", &pool.id), ("backend", member["id"].as_str().unwrap_or("unknown")), ]); let status = member["status"].as_str().unwrap_or("unknown"); f.add( "rust_proxy_backend_status_info", "gauge", "Latest bounded backend selection/health state.", &format!("{l},status=\"{}\"", escaped(status)), 1, ); for (name, flag) in [ ( "up", status != "disabled" && member["healthy"].as_bool().unwrap_or(false), ), ("draining", status == "draining"), ("backup", member["backup"].as_bool().unwrap_or(false)), ] { f.add( &format!("rust_proxy_backend_{name}"), "gauge", "Backend health or selection state.", &l, u8::from(flag), ); } for (field, metric, kind) in [ ("active", "active_exchanges", "gauge"), ("requests", "requests_total", "counter"), ("errors", "errors_total", "counter"), ("bytes_in", "bytes_in_total", "counter"), ("bytes_out", "bytes_out_total", "counter"), ("weight", "weight", "gauge"), ("failures", "consecutive_failures", "gauge"), ("passive_failures", "passive_failures", "gauge"), ("successes", "consecutive_successes", "gauge"), ("passive_successes", "passive_successes", "gauge"), ("transitions", "health_transitions_total", "counter"), ( "active_health_failures_total", "active_health_failures_total", "counter", ), ( "passive_health_failures_total", "passive_health_failures_total", "counter", ), ("http_5xx_total", "http_5xx_total", "counter"), ("http_5xx_total", "passive_http_5xx_total", "counter"), ("timeouts_total", "timeouts_total", "counter"), ("timeouts_total", "passive_timeouts_total", "counter"), ("connect_errors_total", "connect_errors_total", "counter"), ( "connect_errors_total", "passive_connect_errors_total", "counter", ), ("io_errors_total", "io_errors_total", "counter"), ("io_errors_total", "passive_io_errors_total", "counter"), ("slow_responses_total", "slow_responses_total", "counter"), ("slow_responses_total", "passive_slow_total", "counter"), ("retry_attempts_total", "retry_attempts_total", "counter"), ("retry_successes_total", "retry_successes_total", "counter"), ( "capacity_rejections_total", "capacity_rejections_total", "counter", ), ("queue_timeouts_total", "queue_timeouts_total", "counter"), ("dns_errors_total", "dns_errors_total", "counter"), ("tls_errors_total", "tls_errors_total", "counter"), ( "response_header_timeouts_total", "response_header_timeouts_total", "counter", ), ("read_timeouts_total", "read_timeouts_total", "counter"), ("send_timeouts_total", "send_timeouts_total", "counter"), ( "upstream_client_generation", "upstream_client_generation", "gauge", ), ( "upstream_client_generation_requests", "upstream_client_generation_requests", "gauge", ), ( "upstream_client_rotations_total", "upstream_client_rotations_total", "counter", ), ( "max_concurrent_requests", "max_concurrent_requests", "gauge", ), ("queue_timeout_ms", "queue_timeout_milliseconds", "gauge"), ("last_status", "last_http_status", "gauge"), ] { if let Some(n) = member[field].as_u64() { f.add( &format!("rust_proxy_backend_{metric}"), kind, "Backend observation.", &l, n, ); } } for (field, metric, divisor) in [ ("latency_ms", "latency_seconds", 1000.0), ("ewma_latency_ms", "ewma_latency_seconds", 1000.0), ("last_probe_ms", "last_probe_timestamp_seconds", 1000.0), ( "last_failure_at_ms", "last_failure_timestamp_seconds", 1000.0, ), ("last_ejected_ms", "last_ejected_timestamp_seconds", 1000.0), ( "last_recovered_ms", "last_recovered_timestamp_seconds", 1000.0, ), ("traffic_bytes_per_second", "traffic_bytes_per_second", 1.0), ( "upstream_client_generation_age_seconds", "upstream_client_generation_age_seconds", 1.0, ), ] { if let Some(n) = member[field].as_f64() { f.add( &format!("rust_proxy_backend_{metric}"), "gauge", "Backend moving observation.", &l, n / divisor, ); } } if let Some(result) = member["last_result"] .as_str() .filter(|value| !value.is_empty()) { f.add( "rust_proxy_backend_last_probe_result_info", "gauge", "Latest bounded backend health-probe result.", &format!("{l},result=\"{}\"", escaped(result)), 1, ); } } } } } } fn render_docker( snapshot: &crate::config::Snapshot, docker_targets: &DockerTargets<'_>, f: &mut Families, ) { let config = &snapshot.envelope.config; let settings = &config.prometheus; if settings.include_docker_vhosts { f.add( "rust_proxy_docker_agents_configured", "gauge", "Configured Docker discovery agents including disabled agents.", "", config.docker_agents.agents.len(), ); f.add( "rust_proxy_docker_vhosts_configured", "gauge", "Configured Docker vhost assignments including disabled assignments.", "", docker_targets.len(), ); for agent in &config.docker_agents.agents { let agent_labels = labels(&[("agent", &agent.id)]); f.add( "rust_proxy_docker_agent_enabled", "gauge", "Desired Docker agent enable flag.", &agent_labels, u8::from(agent.enabled), ); f.add( "rust_proxy_docker_agent_assignments", "gauge", "Configured Docker vhost assignments for an agent.", &agent_labels, agent.assignments.len(), ); for assignment in &agent.assignments { let vhost = crate::docker::generated_id(&agent.id, &assignment.service_key); let base_labels = labels(&[ ("agent", &agent.id), ("vhost", &vhost), ("service", &assignment.service_key), ]); let domains = crate::docker::assignment_hosts(assignment, &agent.base_domain) .unwrap_or_default(); f.add( "rust_proxy_docker_vhost_enabled", "gauge", "Desired Docker vhost assignment enable flag.", &base_labels, u8::from(agent.enabled && assignment.enabled), ); f.add( "rust_proxy_docker_vhost_domains", "gauge", "Configured DNS names for a Docker vhost assignment.", &base_labels, domains.len(), ); for domain in domains { f.add( "rust_proxy_docker_vhost_domain_info", "gauge", "Configured Docker vhost name ownership.", &format!("{base_labels},domain=\"{}\"", escaped(&domain)), 1, ); } if let Some(m) = snapshot.vhost_metrics.get(&vhost) { for (name, counter) in [ ("requests_total", &m.requests), ("completed_total", &m.completed), ("errors_total", &m.errors), ("rejected_total", &m.rejected), ("request_bytes_total", &m.bytes_in), ("response_bytes_total", &m.bytes_out), ] { f.add( &format!("rust_proxy_docker_vhost_{name}"), "counter", "Docker vhost exchange counter; reset on process restart or assignment removal.", &base_labels, counter.load(Ordering::Relaxed), ); } f.add( "rust_proxy_docker_vhost_active", "gauge", "Admitted unfinished Docker-vhost exchanges.", &base_labels, m.active.load(Ordering::Relaxed), ); for (index, counter) in m.status.iter().enumerate() { f.add( "rust_proxy_docker_vhost_responses_total", "counter", "Completed Docker-vhost exchanges by status class.", &format!("{base_labels},class=\"{}xx\"", index + 1), counter.load(Ordering::Relaxed), ); } } let Some(pool) = snapshot.pools.iter().find(|pool| pool.id == vhost) else { continue; }; let view = pool.view(); if let (Some(algorithm), Some(health_mode), Some(sticky_mode)) = ( view["algorithm"].as_str(), view["health_mode"].as_str(), view["sticky_mode"].as_str(), ) { f.add( "rust_proxy_docker_backend_pool_info", "gauge", "Bounded Docker backend-pool selection and health policy.", &format!( "{base_labels},algorithm=\"{}\",health_mode=\"{}\",sticky_mode=\"{}\"", escaped(algorithm), escaped(health_mode), escaped(sticky_mode) ), 1, ); } f.add( "rust_proxy_docker_backend_pool_ready", "gauge", "At least one healthy eligible Docker backend member.", &base_labels, u8::from(pool.ready()), ); if let Some(members) = view["members"].as_array() { for member in members { let backend = member["id"].as_str().unwrap_or("unknown"); let member_labels = format!("{base_labels},backend=\"{}\"", escaped(backend)); let status = member["status"].as_str().unwrap_or("unknown"); f.add( "rust_proxy_docker_backend_status_info", "gauge", "Latest bounded Docker backend selection/health state.", &format!("{member_labels},status=\"{}\"", escaped(status)), 1, ); for (name, flag) in [ ( "up", status != "disabled" && member["healthy"].as_bool().unwrap_or(false), ), ("draining", status == "draining"), ("backup", member["backup"].as_bool().unwrap_or(false)), ] { f.add( &format!("rust_proxy_docker_backend_{name}"), "gauge", "Docker backend health or selection state.", &member_labels, u8::from(flag), ); } for (field, metric, kind) in [ ("active", "active_exchanges", "gauge"), ("requests", "requests_total", "counter"), ("errors", "errors_total", "counter"), ("bytes_in", "bytes_in_total", "counter"), ("bytes_out", "bytes_out_total", "counter"), ("weight", "weight", "gauge"), ("failures", "consecutive_failures", "gauge"), ("passive_failures", "passive_failures", "gauge"), ("successes", "consecutive_successes", "gauge"), ("passive_successes", "passive_successes", "gauge"), ("transitions", "health_transitions_total", "counter"), ( "active_health_failures_total", "active_health_failures_total", "counter", ), ( "passive_health_failures_total", "passive_health_failures_total", "counter", ), ("http_5xx_total", "http_5xx_total", "counter"), ("http_5xx_total", "passive_http_5xx_total", "counter"), ("timeouts_total", "timeouts_total", "counter"), ("timeouts_total", "passive_timeouts_total", "counter"), ("connect_errors_total", "connect_errors_total", "counter"), ( "connect_errors_total", "passive_connect_errors_total", "counter", ), ("io_errors_total", "io_errors_total", "counter"), ("io_errors_total", "passive_io_errors_total", "counter"), ("slow_responses_total", "slow_responses_total", "counter"), ("slow_responses_total", "passive_slow_total", "counter"), ("retry_attempts_total", "retry_attempts_total", "counter"), ("retry_successes_total", "retry_successes_total", "counter"), ( "capacity_rejections_total", "capacity_rejections_total", "counter", ), ("queue_timeouts_total", "queue_timeouts_total", "counter"), ("dns_errors_total", "dns_errors_total", "counter"), ("tls_errors_total", "tls_errors_total", "counter"), ( "response_header_timeouts_total", "response_header_timeouts_total", "counter", ), ("read_timeouts_total", "read_timeouts_total", "counter"), ("send_timeouts_total", "send_timeouts_total", "counter"), ( "upstream_client_generation", "upstream_client_generation", "gauge", ), ( "upstream_client_generation_requests", "upstream_client_generation_requests", "gauge", ), ( "upstream_client_rotations_total", "upstream_client_rotations_total", "counter", ), ( "max_concurrent_requests", "max_concurrent_requests", "gauge", ), ("queue_timeout_ms", "queue_timeout_milliseconds", "gauge"), ("last_status", "last_http_status", "gauge"), ] { if let Some(n) = member[field].as_u64() { f.add( &format!("rust_proxy_docker_backend_{metric}"), kind, "Docker backend observation.", &member_labels, n, ); } } for (field, metric, divisor) in [ ("latency_ms", "latency_seconds", 1000.0), ("ewma_latency_ms", "ewma_latency_seconds", 1000.0), ("last_probe_ms", "last_probe_timestamp_seconds", 1000.0), ( "last_failure_at_ms", "last_failure_timestamp_seconds", 1000.0, ), ("last_ejected_ms", "last_ejected_timestamp_seconds", 1000.0), ( "last_recovered_ms", "last_recovered_timestamp_seconds", 1000.0, ), ("traffic_bytes_per_second", "traffic_bytes_per_second", 1.0), ( "upstream_client_generation_age_seconds", "upstream_client_generation_age_seconds", 1.0, ), ] { if let Some(n) = member[field].as_f64() { f.add( &format!("rust_proxy_docker_backend_{metric}"), "gauge", "Docker backend moving observation.", &member_labels, n / divisor, ); } } if let Some(result) = member["last_result"] .as_str() .filter(|value| !value.is_empty()) { f.add( "rust_proxy_docker_backend_last_probe_result_info", "gauge", "Latest bounded Docker backend health-probe result.", &format!("{member_labels},result=\"{}\"", escaped(result)), 1, ); } } } } } } }