493 lines
17 KiB
Rust
493 lines
17 KiB
Rust
/// Certificate expiry read from the certificate files, refreshed at most every five minutes.
|
|
fn certificate_expiry(
|
|
app: &App,
|
|
certificates: &[crate::tls::Certificate],
|
|
) -> std::collections::HashMap<String, i64> {
|
|
type Cached = (std::time::Instant, Vec<(String, String)>, std::collections::HashMap<String, i64>);
|
|
static CACHE: std::sync::Mutex<Option<Cached>> = std::sync::Mutex::new(None);
|
|
let wanted: Vec<(String, String)> = certificates
|
|
.iter()
|
|
.map(|cert| (cert.id.clone(), cert.cert_file.clone()))
|
|
.collect();
|
|
let mut cache = CACHE.lock().unwrap_or_else(|e| e.into_inner());
|
|
if let Some((at, files, values)) = cache.as_ref() {
|
|
if at.elapsed() < std::time::Duration::from_secs(300) && *files == wanted {
|
|
return values.clone();
|
|
}
|
|
}
|
|
let values = read_certificate_expiry(app.policy.cert_dir.as_deref(), &wanted);
|
|
*cache = Some((std::time::Instant::now(), wanted, values.clone()));
|
|
values
|
|
}
|
|
|
|
fn read_certificate_expiry(
|
|
directory: Option<&std::path::Path>,
|
|
certificates: &[(String, String)],
|
|
) -> std::collections::HashMap<String, i64> {
|
|
let Some(directory) = directory else { return Default::default() };
|
|
certificates
|
|
.iter()
|
|
.filter_map(|(id, file)| {
|
|
crate::tls::certificate_metadata(directory, file)
|
|
.ok()
|
|
.and_then(|metadata| metadata.not_after_unix)
|
|
.map(|not_after| (id.clone(), not_after))
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
fn render_core(app: &App, snapshot: &crate::config::Snapshot, out: &mut String, f: &mut Families) {
|
|
let config = &snapshot.envelope.config;
|
|
let settings = &config.prometheus;
|
|
f.add(
|
|
"rust_proxy_admin_requests_total",
|
|
"counter",
|
|
"Admin HTTP requests, including denied requests and static assets.",
|
|
"",
|
|
app.metrics.admin_requests.load(Ordering::Relaxed),
|
|
);
|
|
for (index, counter) in app.metrics.admin_status.iter().enumerate() {
|
|
f.add(
|
|
"rust_proxy_admin_responses_total",
|
|
"counter",
|
|
"Admin response headers by status class.",
|
|
&format!("class=\"{}xx\"", index + 1),
|
|
counter.load(Ordering::Relaxed),
|
|
);
|
|
}
|
|
out.push_str("# HELP rust_proxy_admin_header_duration_seconds Time to control-plane response headers; not SSE lifetime.\n# TYPE rust_proxy_admin_header_duration_seconds summary\n");
|
|
let _ = writeln!(out, "rust_proxy_admin_header_duration_seconds_count {}\nrust_proxy_admin_header_duration_seconds_sum {:.6}",
|
|
app.metrics.admin_response_headers.load(Ordering::Relaxed), app.metrics.admin_header_duration_us.load(Ordering::Relaxed) as f64/1e6);
|
|
f.add(
|
|
"rust_proxy_build_info",
|
|
"gauge",
|
|
"Build version.",
|
|
&labels(&[("version", env!("CARGO_PKG_VERSION"))]),
|
|
1,
|
|
);
|
|
f.add(
|
|
"rust_proxy_uptime_seconds",
|
|
"gauge",
|
|
"Process monotonic uptime.",
|
|
"",
|
|
app.started.elapsed().as_secs_f64(),
|
|
);
|
|
f.add(
|
|
"rust_proxy_vhosts_configured",
|
|
"gauge",
|
|
"Configured vhosts including disabled.",
|
|
"",
|
|
config.vhosts.len(),
|
|
);
|
|
f.add(
|
|
"rust_proxy_redirect_hosts_configured",
|
|
"gauge",
|
|
"Configured redirect hosts including disabled.",
|
|
"",
|
|
config.redirects.len(),
|
|
);
|
|
f.add(
|
|
"rust_proxy_not_found_hosts_configured",
|
|
"gauge",
|
|
"Configured local-response/404 hosts including disabled.",
|
|
"",
|
|
config.not_found_hosts.len(),
|
|
);
|
|
f.add(
|
|
"rust_proxy_stream_hosts_configured",
|
|
"gauge",
|
|
"Configured Layer-4 stream hosts including disabled.",
|
|
"",
|
|
config.streams.len(),
|
|
);
|
|
f.add(
|
|
"rust_proxy_acl_profiles_configured",
|
|
"gauge",
|
|
"Configured reusable ACL profiles.",
|
|
"",
|
|
config.acl_profiles.len(),
|
|
);
|
|
if settings.include_system {
|
|
let geo = app.geoip.load().view();
|
|
f.add(
|
|
"rust_proxy_geoip_loaded",
|
|
"gauge",
|
|
"Whether the bounded country GeoIP database is loaded.",
|
|
"",
|
|
u8::from(geo.loaded),
|
|
);
|
|
f.add(
|
|
"rust_proxy_geoip_ipv4_ranges",
|
|
"gauge",
|
|
"Loaded IPv4 country ranges.",
|
|
"",
|
|
geo.ipv4_ranges,
|
|
);
|
|
f.add(
|
|
"rust_proxy_geoip_ipv6_ranges",
|
|
"gauge",
|
|
"Loaded IPv6 country ranges.",
|
|
"",
|
|
geo.ipv6_ranges,
|
|
);
|
|
}
|
|
f.add(
|
|
"rust_proxy_certificates_configured",
|
|
"gauge",
|
|
"Certificate inventory size, not validity.",
|
|
"",
|
|
config.tls.certificates.len(),
|
|
);
|
|
let renewals = app.acme.renewal_view();
|
|
let expiry = certificate_expiry(app, &config.tls.certificates);
|
|
for certificate in &config.tls.certificates {
|
|
let l = labels(&[("certificate", &certificate.id)]);
|
|
if let Some(not_after) = expiry.get(&certificate.id).copied() {
|
|
f.add(
|
|
"rust_proxy_certificate_not_after_timestamp_seconds",
|
|
"gauge",
|
|
"Expiry time of the installed certificate (Unix seconds).",
|
|
&l,
|
|
not_after,
|
|
);
|
|
let now = std::time::SystemTime::now()
|
|
.duration_since(std::time::UNIX_EPOCH)
|
|
.map(|elapsed| elapsed.as_secs() as i64)
|
|
.unwrap_or_default();
|
|
f.add(
|
|
"rust_proxy_certificate_days_remaining",
|
|
"gauge",
|
|
"Whole days until the installed certificate expires; negative when expired.",
|
|
&l,
|
|
(not_after - now).div_euclid(86_400),
|
|
);
|
|
}
|
|
let issuer = certificate.issuer.as_ref();
|
|
f.add(
|
|
"rust_proxy_certificate_auto_renew_enabled",
|
|
"gauge",
|
|
"Desired automatic renewal switch for a configured certificate.",
|
|
&l,
|
|
u8::from(issuer.is_some_and(|issuer| issuer.auto_renew)),
|
|
);
|
|
if let Some(issuer) = issuer {
|
|
f.add(
|
|
"rust_proxy_certificate_issuer_info",
|
|
"gauge",
|
|
"Bounded configured certificate issuer metadata.",
|
|
&labels(&[
|
|
("certificate", &certificate.id),
|
|
("provider", &issuer.provider),
|
|
("environment", &issuer.environment),
|
|
]),
|
|
1,
|
|
);
|
|
}
|
|
if let Some(renewal) = renewals.get(&certificate.id) {
|
|
for (name, value) in [
|
|
("last_checked_timestamp_seconds", Some(renewal.last_checked)),
|
|
("next_check_timestamp_seconds", Some(renewal.next_check)),
|
|
("last_renewed_timestamp_seconds", renewal.last_renewed),
|
|
("not_after_timestamp_seconds", renewal.not_after),
|
|
] {
|
|
if let Some(value) = value.filter(|value| *value > 0) {
|
|
f.add(
|
|
&format!("rust_proxy_certificate_renewal_{name}"),
|
|
"gauge",
|
|
"Certificate renewal observation timestamp.",
|
|
&l,
|
|
value,
|
|
);
|
|
}
|
|
}
|
|
f.add(
|
|
"rust_proxy_certificate_renewal_failures",
|
|
"gauge",
|
|
"Consecutive observed automatic renewal failures.",
|
|
&l,
|
|
renewal.failures,
|
|
);
|
|
if !renewal.status.is_empty() {
|
|
f.add(
|
|
"rust_proxy_certificate_renewal_status_info",
|
|
"gauge",
|
|
"Latest bounded certificate renewal status.",
|
|
&labels(&[
|
|
("certificate", &certificate.id),
|
|
("status", &renewal.status),
|
|
]),
|
|
1,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
f.add(
|
|
"rust_proxy_listeners_active",
|
|
"gauge",
|
|
"Bound frontend endpoints.",
|
|
"",
|
|
app.active_listeners.len(),
|
|
);
|
|
f.add(
|
|
"rust_proxy_restart_required",
|
|
"gauge",
|
|
"Listener/exporter changes requiring a restart.",
|
|
"",
|
|
u8::from(
|
|
crate::http3::listeners_require_restart(&app.active_listeners, &snapshot.listeners)
|
|
|| settings.restart_differs(&app.active_prometheus),
|
|
),
|
|
);
|
|
f.add(
|
|
"rust_proxy_request_capacity",
|
|
"gauge",
|
|
"Configured maximum concurrent admitted exchanges.",
|
|
"",
|
|
app.max_inflight,
|
|
);
|
|
f.add(
|
|
"rust_proxy_request_permits_available",
|
|
"gauge",
|
|
"Currently available exchange permits.",
|
|
"",
|
|
app.inflight.available_permits(),
|
|
);
|
|
if let Some(job) = app.acme.view() {
|
|
f.add(
|
|
"rust_proxy_acme_job_state",
|
|
"gauge",
|
|
"Latest ACME job state; no domains or secrets.",
|
|
&labels(&[("provider", &job.provider), ("state", job.status)]),
|
|
1,
|
|
);
|
|
}
|
|
for (index, listener) in app.active_listeners.iter().enumerate() {
|
|
let index = index.to_string();
|
|
let address = listener.address.to_string();
|
|
let l = labels(&[("listener", &index), ("address", &address)]);
|
|
for (protocol, enabled) in [
|
|
("http1", true),
|
|
("http2", listener.tls && listener.http2),
|
|
("http3", listener.http3.is_some()),
|
|
("tls", listener.tls),
|
|
] {
|
|
f.add(
|
|
"rust_proxy_listener_protocol_enabled",
|
|
"gauge",
|
|
"Protocols active per listener.",
|
|
&format!("{l},protocol=\"{protocol}\""),
|
|
u8::from(enabled),
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
fn render_vhosts(snapshot: &crate::config::Snapshot, f: &mut Families) {
|
|
let config = &snapshot.envelope.config;
|
|
let settings = &config.prometheus;
|
|
if settings.include_vhosts {
|
|
f.add(
|
|
"rust_proxy_web_services_configured",
|
|
"gauge",
|
|
"Configured built-in Web Server services.",
|
|
"",
|
|
config.web_services.len(),
|
|
);
|
|
for service in &config.web_services {
|
|
let php = if service.php.enabled { "true" } else { "false" };
|
|
let directory_listing = if service.directory_listing {
|
|
"true"
|
|
} else {
|
|
"false"
|
|
};
|
|
f.add(
|
|
"rust_proxy_web_service_info",
|
|
"gauge",
|
|
"Bounded built-in Web Server service configuration metadata.",
|
|
&labels(&[
|
|
("service", &service.id),
|
|
("php", php),
|
|
("directory_listing", directory_listing),
|
|
]),
|
|
1,
|
|
);
|
|
}
|
|
for redirect in &config.redirects {
|
|
let l = labels(&[("redirect", &redirect.id)]);
|
|
f.add(
|
|
"rust_proxy_redirect_host_enabled",
|
|
"gauge",
|
|
"Desired redirect-host enable flag.",
|
|
&l,
|
|
u8::from(redirect.enabled),
|
|
);
|
|
f.add(
|
|
"rust_proxy_redirect_host_domains",
|
|
"gauge",
|
|
"Configured DNS names for a redirect host.",
|
|
&l,
|
|
redirect.frontend.hosts.len(),
|
|
);
|
|
f.add(
|
|
"rust_proxy_redirect_host_rules",
|
|
"gauge",
|
|
"Configured redirect rules.",
|
|
&l,
|
|
redirect.rules.len(),
|
|
);
|
|
}
|
|
for host in &config.not_found_hosts {
|
|
let l = labels(&[("host", &host.id)]);
|
|
f.add(
|
|
"rust_proxy_not_found_host_enabled",
|
|
"gauge",
|
|
"Desired local-response/404-host enable flag.",
|
|
&l,
|
|
u8::from(host.enabled),
|
|
);
|
|
f.add(
|
|
"rust_proxy_not_found_host_domains",
|
|
"gauge",
|
|
"Configured DNS names for a local-response/404 host.",
|
|
&l,
|
|
host.frontend.hosts.len(),
|
|
);
|
|
f.add(
|
|
"rust_proxy_not_found_host_status",
|
|
"gauge",
|
|
"Configured local-response HTTP status.",
|
|
&l,
|
|
host.page.status.unwrap_or(404),
|
|
);
|
|
}
|
|
for host in &config.vhosts {
|
|
let l = labels(&[("vhost", &host.id)]);
|
|
let (source, web_service) = match &host.source {
|
|
crate::vhost::Source::ReverseProxy => ("reverse_proxy", ""),
|
|
crate::vhost::Source::WebServer { web_service_id } => {
|
|
("web_server", web_service_id.as_str())
|
|
}
|
|
};
|
|
f.add(
|
|
"rust_proxy_vhost_source_info",
|
|
"gauge",
|
|
"Configured main serving source for a vhost.",
|
|
&format!(
|
|
"{l},source=\"{source}\",web_service=\"{}\"",
|
|
escaped(web_service)
|
|
),
|
|
1,
|
|
);
|
|
for route in host.routes.iter().filter(|route| route.enabled) {
|
|
let (route_source, route_service) = if route.response.is_some() {
|
|
("local_response", "")
|
|
} else {
|
|
match &route.source {
|
|
crate::vhost::Source::ReverseProxy => ("reverse_proxy", ""),
|
|
crate::vhost::Source::WebServer { web_service_id } => {
|
|
("web_server", web_service_id.as_str())
|
|
}
|
|
}
|
|
};
|
|
f.add(
|
|
"rust_proxy_vhost_route_source_info",
|
|
"gauge",
|
|
"Configured serving source for an enabled vhost route.",
|
|
&format!(
|
|
"{l},route=\"{}\",source=\"{route_source}\",web_service=\"{}\"",
|
|
escaped(&route.id),
|
|
escaped(route_service)
|
|
),
|
|
1,
|
|
);
|
|
}
|
|
f.add(
|
|
"rust_proxy_vhost_enabled",
|
|
"gauge",
|
|
"Desired vhost enable flag.",
|
|
&l,
|
|
u8::from(host.enabled),
|
|
);
|
|
f.add(
|
|
"rust_proxy_vhost_domains",
|
|
"gauge",
|
|
"Configured DNS names for a vhost.",
|
|
&l,
|
|
host.frontend.hosts.len(),
|
|
);
|
|
f.add(
|
|
"rust_proxy_vhost_paths",
|
|
"gauge",
|
|
"Configured path rules for a vhost.",
|
|
&l,
|
|
host.routes.len(),
|
|
);
|
|
if let Ok(cache) = host.cache.resolve(&config.cache) {
|
|
f.add(
|
|
"rust_proxy_vhost_cache_enabled",
|
|
"gauge",
|
|
"Desired effective cache switch; request privacy rules still apply.",
|
|
&l,
|
|
u8::from(cache.enabled),
|
|
);
|
|
f.add(
|
|
"rust_proxy_vhost_cache_max_object_bytes",
|
|
"gauge",
|
|
"Effective maximum cache object size.",
|
|
&l,
|
|
cache.max_object_bytes,
|
|
);
|
|
f.add(
|
|
"rust_proxy_vhost_cache_max_ttl_seconds",
|
|
"gauge",
|
|
"Effective maximum cache TTL.",
|
|
&l,
|
|
cache.max_ttl_seconds,
|
|
);
|
|
}
|
|
for domain in &host.frontend.hosts {
|
|
f.add(
|
|
"rust_proxy_vhost_domain_info",
|
|
"gauge",
|
|
"Configured name ownership.",
|
|
&labels(&[("vhost", &host.id), ("domain", domain)]),
|
|
1,
|
|
);
|
|
}
|
|
let m = &snapshot.vhost_metrics[&host.id];
|
|
for (name, counter) in [
|
|
("requests_total", &m.requests),
|
|
("completed_total", &m.completed),
|
|
("errors_total", &m.errors),
|
|
("rejected_total", &m.rejected),
|
|
("request_bytes_total", &m.bytes_in),
|
|
("response_bytes_total", &m.bytes_out),
|
|
] {
|
|
f.add(
|
|
&format!("rust_proxy_vhost_{name}"),
|
|
"counter",
|
|
"Vhost exchange counter; reset on process restart or vhost removal.",
|
|
&l,
|
|
counter.load(Ordering::Relaxed),
|
|
);
|
|
}
|
|
f.add(
|
|
"rust_proxy_vhost_active",
|
|
"gauge",
|
|
"Admitted unfinished exchanges.",
|
|
&l,
|
|
m.active.load(Ordering::Relaxed),
|
|
);
|
|
for (index, counter) in m.status.iter().enumerate() {
|
|
f.add(
|
|
"rust_proxy_vhost_responses_total",
|
|
"counter",
|
|
"Completed exchanges by status class.",
|
|
&format!("{l},class=\"{}xx\"", index + 1),
|
|
counter.load(Ordering::Relaxed),
|
|
);
|
|
}
|
|
}
|
|
}
|
|
}
|