/// Certificate expiry read from the certificate files, refreshed at most every five minutes. fn certificate_expiry( app: &App, certificates: &[crate::tls::Certificate], ) -> std::collections::HashMap { type Cached = (std::time::Instant, Vec<(String, String)>, std::collections::HashMap); static CACHE: std::sync::Mutex> = std::sync::Mutex::new(None); let wanted: Vec<(String, String)> = certificates .iter() .map(|cert| (cert.id.clone(), cert.cert_file.clone())) .collect(); let mut cache = CACHE.lock().unwrap_or_else(|e| e.into_inner()); if let Some((at, files, values)) = cache.as_ref() { if at.elapsed() < std::time::Duration::from_secs(300) && *files == wanted { return values.clone(); } } let values = read_certificate_expiry(app.policy.cert_dir.as_deref(), &wanted); *cache = Some((std::time::Instant::now(), wanted, values.clone())); values } fn read_certificate_expiry( directory: Option<&std::path::Path>, certificates: &[(String, String)], ) -> std::collections::HashMap { let Some(directory) = directory else { return Default::default() }; certificates .iter() .filter_map(|(id, file)| { crate::tls::certificate_metadata(directory, file) .ok() .and_then(|metadata| metadata.not_after_unix) .map(|not_after| (id.clone(), not_after)) }) .collect() } fn render_core(app: &App, snapshot: &crate::config::Snapshot, out: &mut String, f: &mut Families) { let config = &snapshot.envelope.config; let settings = &config.prometheus; f.add( "rust_proxy_admin_requests_total", "counter", "Admin HTTP requests, including denied requests and static assets.", "", app.metrics.admin_requests.load(Ordering::Relaxed), ); for (index, counter) in app.metrics.admin_status.iter().enumerate() { f.add( "rust_proxy_admin_responses_total", "counter", "Admin response headers by status class.", &format!("class=\"{}xx\"", index + 1), counter.load(Ordering::Relaxed), ); } out.push_str("# HELP rust_proxy_admin_header_duration_seconds Time to control-plane response headers; not SSE lifetime.\n# TYPE rust_proxy_admin_header_duration_seconds summary\n"); let _ = writeln!(out, "rust_proxy_admin_header_duration_seconds_count {}\nrust_proxy_admin_header_duration_seconds_sum {:.6}", app.metrics.admin_response_headers.load(Ordering::Relaxed), app.metrics.admin_header_duration_us.load(Ordering::Relaxed) as f64/1e6); f.add( "rust_proxy_build_info", "gauge", "Build version.", &labels(&[("version", env!("CARGO_PKG_VERSION"))]), 1, ); f.add( "rust_proxy_uptime_seconds", "gauge", "Process monotonic uptime.", "", app.started.elapsed().as_secs_f64(), ); f.add( "rust_proxy_vhosts_configured", "gauge", "Configured vhosts including disabled.", "", config.vhosts.len(), ); f.add( "rust_proxy_redirect_hosts_configured", "gauge", "Configured redirect hosts including disabled.", "", config.redirects.len(), ); f.add( "rust_proxy_not_found_hosts_configured", "gauge", "Configured local-response/404 hosts including disabled.", "", config.not_found_hosts.len(), ); f.add( "rust_proxy_stream_hosts_configured", "gauge", "Configured Layer-4 stream hosts including disabled.", "", config.streams.len(), ); f.add( "rust_proxy_acl_profiles_configured", "gauge", "Configured reusable ACL profiles.", "", config.acl_profiles.len(), ); if settings.include_system { let geo = app.geoip.load().view(); f.add( "rust_proxy_geoip_loaded", "gauge", "Whether the bounded country GeoIP database is loaded.", "", u8::from(geo.loaded), ); f.add( "rust_proxy_geoip_ipv4_ranges", "gauge", "Loaded IPv4 country ranges.", "", geo.ipv4_ranges, ); f.add( "rust_proxy_geoip_ipv6_ranges", "gauge", "Loaded IPv6 country ranges.", "", geo.ipv6_ranges, ); } f.add( "rust_proxy_certificates_configured", "gauge", "Certificate inventory size, not validity.", "", config.tls.certificates.len(), ); let renewals = app.acme.renewal_view(); let expiry = certificate_expiry(app, &config.tls.certificates); for certificate in &config.tls.certificates { let l = labels(&[("certificate", &certificate.id)]); if let Some(not_after) = expiry.get(&certificate.id).copied() { f.add( "rust_proxy_certificate_not_after_timestamp_seconds", "gauge", "Expiry time of the installed certificate (Unix seconds).", &l, not_after, ); let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|elapsed| elapsed.as_secs() as i64) .unwrap_or_default(); f.add( "rust_proxy_certificate_days_remaining", "gauge", "Whole days until the installed certificate expires; negative when expired.", &l, (not_after - now).div_euclid(86_400), ); } let issuer = certificate.issuer.as_ref(); f.add( "rust_proxy_certificate_auto_renew_enabled", "gauge", "Desired automatic renewal switch for a configured certificate.", &l, u8::from(issuer.is_some_and(|issuer| issuer.auto_renew)), ); if let Some(issuer) = issuer { f.add( "rust_proxy_certificate_issuer_info", "gauge", "Bounded configured certificate issuer metadata.", &labels(&[ ("certificate", &certificate.id), ("provider", &issuer.provider), ("environment", &issuer.environment), ]), 1, ); } if let Some(renewal) = renewals.get(&certificate.id) { for (name, value) in [ ("last_checked_timestamp_seconds", Some(renewal.last_checked)), ("next_check_timestamp_seconds", Some(renewal.next_check)), ("last_renewed_timestamp_seconds", renewal.last_renewed), ("not_after_timestamp_seconds", renewal.not_after), ] { if let Some(value) = value.filter(|value| *value > 0) { f.add( &format!("rust_proxy_certificate_renewal_{name}"), "gauge", "Certificate renewal observation timestamp.", &l, value, ); } } f.add( "rust_proxy_certificate_renewal_failures", "gauge", "Consecutive observed automatic renewal failures.", &l, renewal.failures, ); if !renewal.status.is_empty() { f.add( "rust_proxy_certificate_renewal_status_info", "gauge", "Latest bounded certificate renewal status.", &labels(&[ ("certificate", &certificate.id), ("status", &renewal.status), ]), 1, ); } } } f.add( "rust_proxy_listeners_active", "gauge", "Bound frontend endpoints.", "", app.active_listeners.len(), ); f.add( "rust_proxy_restart_required", "gauge", "Listener/exporter changes requiring a restart.", "", u8::from( crate::http3::listeners_require_restart(&app.active_listeners, &snapshot.listeners) || settings.restart_differs(&app.active_prometheus), ), ); f.add( "rust_proxy_request_capacity", "gauge", "Configured maximum concurrent admitted exchanges.", "", app.max_inflight, ); f.add( "rust_proxy_request_permits_available", "gauge", "Currently available exchange permits.", "", app.inflight.available_permits(), ); if let Some(job) = app.acme.view() { f.add( "rust_proxy_acme_job_state", "gauge", "Latest ACME job state; no domains or secrets.", &labels(&[("provider", &job.provider), ("state", job.status)]), 1, ); } for (index, listener) in app.active_listeners.iter().enumerate() { let index = index.to_string(); let address = listener.address.to_string(); let l = labels(&[("listener", &index), ("address", &address)]); for (protocol, enabled) in [ ("http1", true), ("http2", listener.tls && listener.http2), ("http3", listener.http3.is_some()), ("tls", listener.tls), ] { f.add( "rust_proxy_listener_protocol_enabled", "gauge", "Protocols active per listener.", &format!("{l},protocol=\"{protocol}\""), u8::from(enabled), ); } } } fn render_vhosts(snapshot: &crate::config::Snapshot, f: &mut Families) { let config = &snapshot.envelope.config; let settings = &config.prometheus; if settings.include_vhosts { f.add( "rust_proxy_web_services_configured", "gauge", "Configured built-in Web Server services.", "", config.web_services.len(), ); for service in &config.web_services { let php = if service.php.enabled { "true" } else { "false" }; let directory_listing = if service.directory_listing { "true" } else { "false" }; f.add( "rust_proxy_web_service_info", "gauge", "Bounded built-in Web Server service configuration metadata.", &labels(&[ ("service", &service.id), ("php", php), ("directory_listing", directory_listing), ]), 1, ); } for redirect in &config.redirects { let l = labels(&[("redirect", &redirect.id)]); f.add( "rust_proxy_redirect_host_enabled", "gauge", "Desired redirect-host enable flag.", &l, u8::from(redirect.enabled), ); f.add( "rust_proxy_redirect_host_domains", "gauge", "Configured DNS names for a redirect host.", &l, redirect.frontend.hosts.len(), ); f.add( "rust_proxy_redirect_host_rules", "gauge", "Configured redirect rules.", &l, redirect.rules.len(), ); } for host in &config.not_found_hosts { let l = labels(&[("host", &host.id)]); f.add( "rust_proxy_not_found_host_enabled", "gauge", "Desired local-response/404-host enable flag.", &l, u8::from(host.enabled), ); f.add( "rust_proxy_not_found_host_domains", "gauge", "Configured DNS names for a local-response/404 host.", &l, host.frontend.hosts.len(), ); f.add( "rust_proxy_not_found_host_status", "gauge", "Configured local-response HTTP status.", &l, host.page.status.unwrap_or(404), ); } for host in &config.vhosts { let l = labels(&[("vhost", &host.id)]); let (source, web_service) = match &host.source { crate::vhost::Source::ReverseProxy => ("reverse_proxy", ""), crate::vhost::Source::WebServer { web_service_id } => { ("web_server", web_service_id.as_str()) } }; f.add( "rust_proxy_vhost_source_info", "gauge", "Configured main serving source for a vhost.", &format!( "{l},source=\"{source}\",web_service=\"{}\"", escaped(web_service) ), 1, ); for route in host.routes.iter().filter(|route| route.enabled) { let (route_source, route_service) = if route.response.is_some() { ("local_response", "") } else { match &route.source { crate::vhost::Source::ReverseProxy => ("reverse_proxy", ""), crate::vhost::Source::WebServer { web_service_id } => { ("web_server", web_service_id.as_str()) } } }; f.add( "rust_proxy_vhost_route_source_info", "gauge", "Configured serving source for an enabled vhost route.", &format!( "{l},route=\"{}\",source=\"{route_source}\",web_service=\"{}\"", escaped(&route.id), escaped(route_service) ), 1, ); } f.add( "rust_proxy_vhost_enabled", "gauge", "Desired vhost enable flag.", &l, u8::from(host.enabled), ); f.add( "rust_proxy_vhost_domains", "gauge", "Configured DNS names for a vhost.", &l, host.frontend.hosts.len(), ); f.add( "rust_proxy_vhost_paths", "gauge", "Configured path rules for a vhost.", &l, host.routes.len(), ); if let Ok(cache) = host.cache.resolve(&config.cache) { f.add( "rust_proxy_vhost_cache_enabled", "gauge", "Desired effective cache switch; request privacy rules still apply.", &l, u8::from(cache.enabled), ); f.add( "rust_proxy_vhost_cache_max_object_bytes", "gauge", "Effective maximum cache object size.", &l, cache.max_object_bytes, ); f.add( "rust_proxy_vhost_cache_max_ttl_seconds", "gauge", "Effective maximum cache TTL.", &l, cache.max_ttl_seconds, ); } for domain in &host.frontend.hosts { f.add( "rust_proxy_vhost_domain_info", "gauge", "Configured name ownership.", &labels(&[("vhost", &host.id), ("domain", domain)]), 1, ); } let m = &snapshot.vhost_metrics[&host.id]; for (name, counter) in [ ("requests_total", &m.requests), ("completed_total", &m.completed), ("errors_total", &m.errors), ("rejected_total", &m.rejected), ("request_bytes_total", &m.bytes_in), ("response_bytes_total", &m.bytes_out), ] { f.add( &format!("rust_proxy_vhost_{name}"), "counter", "Vhost exchange counter; reset on process restart or vhost removal.", &l, counter.load(Ordering::Relaxed), ); } f.add( "rust_proxy_vhost_active", "gauge", "Admitted unfinished exchanges.", &l, m.active.load(Ordering::Relaxed), ); for (index, counter) in m.status.iter().enumerate() { f.add( "rust_proxy_vhost_responses_total", "counter", "Completed exchanges by status class.", &format!("{l},class=\"{}xx\"", index + 1), counter.load(Ordering::Relaxed), ); } } } }