This commit is contained in:
Mateusz Gruszczyński committed 2026-11-06 23:16:36 +01:00
1 parent 09414dfb09
commit f4606cce18
36 files changed
+641 -74

No files matched your search

+30 -1
View File
@@ -2,7 +2,7 @@
"openapi": "3.0.3",
"info": {
"title": "RustProxy Administration API",
"version": "0.35.1",
"version": "0.35.2",
"description": "Administration API for the active panel origin. Sign in through the panel; the same-origin HttpOnly cookie authenticates all private operations. Mutations also require the exact Origin, X-RustProxy-Request: 1 and a session-bound X-CSRF-Token. Loopback HTTP is the default; Settings can enable HTTPS before using a non-loopback administration bind. This document contains no live configuration or credentials. Object schemas marked extensible are summaries: full configuration validation is authoritative in Rust. The API also permits explicitly scoped GeneralApiBearer tokens on a narrow route allowlist, with RO/RW and source CIDR checks; API-token management and Prometheus-token management are browser-session-only."
},
"servers": [
@@ -3434,6 +3434,19 @@
"nullable": true,
"description": "Optional HTTP Host override. When null for an HTTP vhost probe, RustProxy uses the first exact vhost DNS hostname; if none exists it falls back to the backend host."
},
"port": {
"type": "integer",
"minimum": 1,
"maximum": 65535,
"nullable": true,
"description": "Optional port for active probes. When null or equal to the backend port, probes use the backend address; otherwise TCP and HTTP probes connect to the same host on this port with the same TLS settings and address restrictions."
},
"expect_body": {
"type": "string",
"nullable": true,
"maxLength": 256,
"description": "Optional text that must appear in the first 64 KiB of an HTTP probe response body. A response with an accepted status but without this text counts as a failed probe (result http_body)."
},
"interval_ms": {
"type": "integer",
"minimum": 250,
@@ -3766,6 +3779,10 @@
"extra_h3_share",
"extra_cache_miss_rate",
"extra_backend_active",
"extra_backends_healthy",
"extra_error_share",
"extra_rejected_rate",
"extra_cert_expiry",
"wide_protocol_rates",
"wide_cpu",
"wide_memory",
@@ -3847,6 +3864,10 @@
"extra_h3_share",
"extra_cache_miss_rate",
"extra_backend_active",
"extra_backends_healthy",
"extra_error_share",
"extra_rejected_rate",
"extra_cert_expiry",
"wide_protocol_rates",
"wide_cpu",
"wide_memory",
@@ -3935,6 +3956,10 @@
"extra_h3_share",
"extra_cache_miss_rate",
"extra_backend_active",
"extra_backends_healthy",
"extra_error_share",
"extra_rejected_rate",
"extra_cert_expiry",
"wide_protocol_rates",
"wide_cpu",
"wide_memory",
@@ -4004,6 +4029,10 @@
"extra_h3_share",
"extra_cache_miss_rate",
"extra_backend_active",
"extra_backends_healthy",
"extra_error_share",
"extra_rejected_rate",
"extra_cert_expiry",
"wide_protocol_rates",
"wide_cpu",
"wide_memory",
+18 -2
View File
@@ -165,7 +165,7 @@ function setPanel(name, navigate = true) {
if (name === "filelogs") void window.rustProxyFileLogs?.refresh();
if (name === "tls") void refreshCertificateInventory().then(() => { if (authenticated && currentPanel === "tls") renderCertificates(); });
}
function bytes(value) { if (value < 1024) return `${value} B`; for (const unit of ["KiB", "MiB", "GiB", "TiB"]) { value /= 1024; if (value < 1024 || unit === "TiB") return `${value.toFixed(value < 10 ? 1 : 0)} ${unit}`; } }
function bytes(value) { if (value < 1024) return `${Math.round(value)} B`; for (const unit of ["KiB", "MiB", "GiB", "TiB"]) { value /= 1024; if (value < 1024 || unit === "TiB") return `${value.toFixed(value < 10 ? 1 : 0)} ${unit}`; } }
function uptime(value) {
let seconds = Math.max(0, Math.floor(Number(value) || 0));
if (seconds < 60) return `${seconds}s`;
@@ -232,7 +232,23 @@ window.rustProxyOverviewSamples.onChange(() => {
}
redrawOverviewHistoryCharts();
});
function renderStatus(s, systemSample = false) {
// Safari has no CSS scroll anchoring: keep the element under the viewport centre in place
// when a live refresh changes the height of content above it.
const nativeScrollAnchoring = typeof CSS !== "undefined" && CSS.supports?.("overflow-anchor", "auto");
function preserveScrollAnchor(render) {
if (nativeScrollAnchoring || window.scrollY <= 0) return render();
const probe = document.elementFromPoint(window.innerWidth / 2, window.innerHeight / 2);
const anchor = probe?.closest?.("article, tr, .mini-live-chart, .network-interface-row") || probe;
const before = anchor?.isConnected ? anchor.getBoundingClientRect().top : null;
const result = render();
if (before !== null && anchor.isConnected) {
const shift = anchor.getBoundingClientRect().top - before;
if (Math.abs(shift) >= 1) window.scrollBy(0, shift);
}
return result;
}
function renderStatus(s, systemSample = false) { return preserveScrollAnchor(() => renderStatusNow(s, systemSample)); }
function renderStatusNow(s, systemSample = false) {
const firstStatus = latestStatus === null;
latestStatus = {
...(latestStatus || {}), ...s,
+39 -2
View File
@@ -8,10 +8,12 @@ window.rustProxyExtraOverview = (() => {
"extra_response_latency", "extra_backend_unhealthy",
"extra_total_rps", "extra_active_exchanges", "extra_h1_share", "extra_h3_share",
"extra_cache_miss_rate", "extra_backend_active",
"extra_backends_healthy", "extra_error_share", "extra_rejected_rate", "extra_cert_expiry",
];
const bounded = new Set([
"extra_process_fds", "extra_cache_memory", "extra_cold_queue_memory", "extra_cold_storage",
"extra_h1_share", "extra_h2_share", "extra_h3_share",
"extra_backends_healthy", "extra_cert_expiry",
]);
const histories = new Map(), lastCounter = new Map(), lastValue = new Map();
const lastUnits = new Map(), lastSegments = new Map();
@@ -98,11 +100,36 @@ window.rustProxyExtraOverview = (() => {
unit: "count", detail: ""
};
}
if (id === "extra_backends_healthy") {
const members = Array.isArray(status.backend_pools)
? status.backend_pools.flatMap(pool => (pool.members || []).map(member => ({ pool: pool.id, ...member }))) : null;
if (!members) return { value: null, unit: "count", detail: "" };
const monitored = members.filter(member => member.health_check_enabled !== false && member.health_mode !== "off");
const down = monitored.filter(member => member.status === "down" || member.healthy === false);
const up = monitored.length - down.length;
return {
value: monitored.length ? `${fmt(up)} / ${fmt(monitored.length)}` : null, unit: "text",
meterPercent: monitored.length ? up / monitored.length * 100 : null,
detail: !monitored.length ? t("overviewExtraBackendsNone")
: down.length ? `${t("overviewExtraBackendsDown")}: ${down.slice(0, 3).map(member => `${member.pool}/${member.id}`).join(", ")}${down.length > 3 ? " …" : ""}`
: t("overviewExtraBackendsAllUp"),
};
}
if (id === "extra_cert_expiry") {
const inventory = typeof certificateInventory !== "undefined" && certificateInventory instanceof Map ? [...certificateInventory.values()] : [];
const dated = inventory.filter(cert => Number.isFinite(Number(cert?.days_remaining)))
.sort((a, b) => Number(a.days_remaining) - Number(b.days_remaining));
if (!dated.length) return { value: null, unit: "days", detail: t("overviewExtraCertNone") };
const nearest = dated[0], days = Number(nearest.days_remaining);
const name = (Array.isArray(nearest.covered_hosts) && nearest.covered_hosts[0]) || (Array.isArray(nearest.hosts) && nearest.hosts[0]) || nearest.id;
return { value: days, unit: "days", meterPercent: Math.max(0, Math.min(100, days / 90 * 100)), detail: `${name} · ${fmt(dated.length)} ${t("overviewExtraCertCount")}` };
}
if (id === "extra_cache_memory") return ramBudget(c.resident_bytes, c.hot_limit_bytes);
if (id === "extra_cold_queue_memory") return ramBudget(c.cold_queue_bytes, c.cold_queue_limit_bytes);
if (id === "extra_cold_storage") return ramBudget(c.cold_bytes, c.cold_limit_bytes);
if (!["extra_tls_error_rate", "extra_request_error_rate", "extra_cache_hit_rate", "extra_h2_share",
"extra_total_rps", "extra_h1_share", "extra_h3_share", "extra_cache_miss_rate"].includes(id))
"extra_total_rps", "extra_h1_share", "extra_h3_share", "extra_cache_miss_rate",
"extra_error_share", "extra_rejected_rate"].includes(id))
return { value: null, unit: "count", detail: "" };
const fields = ({
extra_tls_error_rate: [m.tls_handshake_errors],
@@ -113,8 +140,10 @@ window.rustProxyExtraOverview = (() => {
extra_h3_share: [m.http3_requests, m.requests],
extra_total_rps: [m.requests],
extra_cache_miss_rate: [c.misses],
extra_error_share: [m.errors, m.requests],
extra_rejected_rate: [m.rejected_requests, m.rejected_connections],
})[id].map(positive);
const unit = ["extra_cache_hit_rate", "extra_h2_share", "extra_h1_share", "extra_h3_share"].includes(id)
const unit = ["extra_cache_hit_rate", "extra_h2_share", "extra_h1_share", "extra_h3_share", "extra_error_share"].includes(id)
? "percent" : "rate";
if (fields.some(v => v === null)) {
lastCounter.delete(id);
@@ -124,6 +153,12 @@ window.rustProxyExtraOverview = (() => {
const result = elapsedDelta(id, fields, now);
if (!result) return { value: null, unit, detail: "", slices: null };
const { deltas, seconds } = result;
if (id === "extra_rejected_rate") return {
value: (deltas[0] + deltas[1]) / seconds, unit, detail: `${t("overviewExtraRejectedDetail")}: ${fmt(fields[0])} / ${fmt(fields[1])}`
};
if (id === "extra_error_share") return {
value: deltas[1] > 0 ? Math.min(100, deltas[0] / deltas[1] * 100) : 0, unit, detail: `${t("overviewExtraErrorShareDetail")}: ${fmt(deltas[0])} / ${fmt(deltas[1])}`
};
if (unit === "rate") return { value: deltas[0] / seconds, unit, detail: "" };
const sum = id === "extra_cache_hit_rate" ? deltas[0] + deltas[1] : deltas[1];
return {
@@ -137,6 +172,8 @@ window.rustProxyExtraOverview = (() => {
if (unit === "bytes") return fmtBytes(value);
if (unit === "ms") return `${fmt(value, 1)} ms`;
if (unit === "rate") return `${fmt(value, 1)}/s`;
if (unit === "days") return `${fmt(value)} ${t("overviewExtraDaysUnit")}`;
if (unit === "text") return String(value);
return fmt(value);
}
function points(id, history, unit) {
+1 -1
View File
@@ -182,7 +182,7 @@ const rustProxyRefresh = (() => {
const value = await api("/api/system-info", { signal });
if (!isCurrent()) return;
latestStatus = { ...(latestStatus || {}), ...value };
if (latestStatus.metrics) renderDiagnostics(latestStatus, false, true);
if (latestStatus.metrics) preserveScrollAnchor(() => renderDiagnostics(latestStatus, false, true));
window.rustProxySettings?.render();
}
});
+14
View File
@@ -2119,3 +2119,17 @@ html[data-dashboard-density] #panel-overview .stat :is(.stat-chart-legend, .over
color: var(--muted);
font: 400 10px var(--mono);
}
/* Live values stay on one line so refreshes do not change card heights. */
.mini-live-chart-head> :last-child,
.network-interface-row-head .network-interface-head-actions {
flex: 0 0 auto;
white-space: nowrap;
}
.mini-live-chart-head>span:first-child {
min-width: 0;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
+4
View File
@@ -5,6 +5,10 @@
"bandwidth",
"errors",
"memory",
"extra_backends_healthy",
"extra_error_share",
"extra_rejected_rate",
"extra_cert_expiry",
"websocket",
"backend_traffic",
"extra_response_latency",
+44
View File
@@ -867,5 +867,49 @@
"category": "web_server",
"default_hidden": true,
"display_type": "chart"
},
{
"id": "extra_backends_healthy",
"label_key": "overviewExtraBackendsHealthy",
"description_key": "overviewExtraBackendsHealthyDescription",
"element_selector": "[data-dashboard-widget=\"extra_backends_healthy\"]",
"data_source": "backend_status",
"no_data": "neutral_dash",
"category": "backends",
"default_hidden": false,
"display_type": "value_chart"
},
{
"id": "extra_error_share",
"label_key": "overviewExtraErrorShare",
"description_key": "overviewExtraErrorShareDescription",
"element_selector": "[data-dashboard-widget=\"extra_error_share\"]",
"data_source": "status_metrics",
"no_data": "neutral_dash",
"category": "performance",
"default_hidden": false,
"display_type": "value_chart"
},
{
"id": "extra_rejected_rate",
"label_key": "overviewExtraRejectedRate",
"description_key": "overviewExtraRejectedRateDescription",
"element_selector": "[data-dashboard-widget=\"extra_rejected_rate\"]",
"data_source": "status_metrics",
"no_data": "neutral_dash",
"category": "performance",
"default_hidden": false,
"display_type": "value_chart"
},
{
"id": "extra_cert_expiry",
"label_key": "overviewExtraCertExpiry",
"description_key": "overviewExtraCertExpiryDescription",
"element_selector": "[data-dashboard-widget=\"extra_cert_expiry\"]",
"data_source": "system_status",
"no_data": "neutral_dash",
"category": "overview",
"default_hidden": false,
"display_type": "value_chart"
}
]
@@ -365,6 +365,36 @@
</polyline>
</svg><small id="overview-extra-extra_backend_unhealthy-detail">-</small>
</article>
<article class="stat stat-meter" data-dashboard-widget="extra_backends_healthy" hidden>
<span data-i18n="overviewExtraBackendsHealthy">overviewExtraBackendsHealthy</span><strong id="overview-extra-extra_backends_healthy">-</strong>
<div class="overview-meter" id="overview-extra-extra_backends_healthy-meter" role="progressbar"
data-i18n-aria="overviewExtraBackendsHealthy" aria-valuemin="0" aria-valuemax="100" aria-valuetext="-">
<span class="overview-meter-fill" id="overview-extra-extra_backends_healthy-meter-fill"></span>
</div><small id="overview-extra-extra_backends_healthy-detail">-</small>
</article>
<article class="stat stat-chart" data-dashboard-widget="extra_error_share" hidden>
<span data-i18n="overviewExtraErrorShare">overviewExtraErrorShare</span><strong id="overview-extra-extra_error_share">-</strong>
<svg preserveAspectRatio="none" data-i18n-aria="overviewExtraErrorShare" role="img" viewBox="0 0 300 60">
<path class="chart-grid" d="M0 15H300 M0 30H300 M0 45H300"></path>
<polyline class="diagnostic-line" id="overview-extra-extra_error_share-line" points="0,55 300,55">
</polyline>
</svg><small id="overview-extra-extra_error_share-detail">-</small>
</article>
<article class="stat stat-chart" data-dashboard-widget="extra_rejected_rate" hidden>
<span data-i18n="overviewExtraRejectedRate">overviewExtraRejectedRate</span><strong id="overview-extra-extra_rejected_rate">-</strong>
<svg preserveAspectRatio="none" data-i18n-aria="overviewExtraRejectedRate" role="img" viewBox="0 0 300 60">
<path class="chart-grid" d="M0 15H300 M0 30H300 M0 45H300"></path>
<polyline class="diagnostic-line" id="overview-extra-extra_rejected_rate-line" points="0,55 300,55">
</polyline>
</svg><small id="overview-extra-extra_rejected_rate-detail">-</small>
</article>
<article class="stat stat-meter" data-dashboard-widget="extra_cert_expiry" hidden>
<span data-i18n="overviewExtraCertExpiry">overviewExtraCertExpiry</span><strong id="overview-extra-extra_cert_expiry">-</strong>
<div class="overview-meter" id="overview-extra-extra_cert_expiry-meter" role="progressbar"
data-i18n-aria="overviewExtraCertExpiry" aria-valuemin="0" aria-valuemax="100" aria-valuetext="-">
<span class="overview-meter-fill" id="overview-extra-extra_cert_expiry-meter-fill"></span>
</div><small id="overview-extra-extra_cert_expiry-detail">-</small>
</article>
<article class="stat stat-chart" data-dashboard-widget="extra_total_rps" hidden>
<span data-i18n="overviewExtraTotalRps">HTTP requests / s</span><strong
id="overview-extra-extra_total_rps">-</strong>
+3
View File
@@ -38,6 +38,7 @@ const num = (name, scope) => { const value = Number(str(name, scope)); if (!Numb
const yes = (name, scope) => read(name, scope).checked;
const lines = value => value.split(/[\n,]+/).map(s => s.trim()).filter(Boolean);
const nullable = (name, scope) => str(name, scope) || null;
const optionalPort = (name, scope) => { const value = str(name, scope); return value ? Number(value) : null; };
const defaultProxyTimeouts = () => ({ connect_timeout_seconds: 10, response_header_timeout_seconds: 120, read_timeout_seconds: 60, send_timeout_seconds: 60 });
const changed = () => { dirty = true; };
const fillWebServices = (select, chosen = "") => {
@@ -236,6 +237,8 @@ function memberRow(parent, value = { id: `origin-${parent.children.length + 1}`,
field(healthFields, "member-health-mode", "vhHealthMode", health.mode || "tcp", "select", { values: [["off", "vhOff"], ["tcp", "vhTcp"], ["http", "vhHttp"]] });
field(healthFields, "member-health-path", "vhHealthPath", health.path ?? "/", "text", { maxLength: 1024 });
field(healthFields, "member-health-host", "vhHealthHost", health.host ?? "", "text", { maxLength: 253 });
field(healthFields, "member-health-port", "vhHealthPort", health.port ?? "", "number", { min: 1, max: 65535, step: 1 }).placeholder = t("vhHealthPortPlaceholder");
field(healthFields, "member-health-expect-body", "vhHealthExpectBody", health.expect_body ?? "", "text", { maxLength: 256 });
field(healthFields, "member-health-interval", "vhInterval", health.interval_ms ?? 5000, "number", { min: 250, max: 300000, required: true });
field(healthFields, "member-health-timeout", "vhTimeout", health.timeout_ms ?? 2000, "number", { min: 50, max: 60000, required: true });
field(healthFields, "member-health-slow", "vhSlow", health.slow_response_ms ?? 1500, "number", { min: 0, max: 60000, required: true });
+4 -2
View File
@@ -238,6 +238,8 @@ function load(value = defaults(), migrationIds = [], show = true) {
const healthOffWarning = note(g, "vhHealthOffWarning"); const syncHealthWarning = () => { healthOffWarning.hidden = healthMode.value !== "off"; }; healthMode.addEventListener("change", syncHealthWarning); syncHealthWarning();
field(g, "health-path", "vhHealthPath", health.path || "/", "text", { required: true, maxLength: 1024 });
field(g, "health-host", "vhHealthHost", health.host || "", "text", { maxLength: 260 });
field(g, "health-port", "vhHealthPort", health.port ?? "", "number", { min: 1, max: 65535, step: 1 }).placeholder = t("vhHealthPortPlaceholder");
field(g, "health-expect-body", "vhHealthExpectBody", health.expect_body || "", "text", { maxLength: 256 });
for (const [name, key, val, min, max] of [["interval", "vhInterval", health.interval_ms ?? 5000, 250, 300000], ["timeout", "vhTimeout", health.timeout_ms ?? 2000, 50, 60000], ["slow", "vhSlow", health.slow_response_ms ?? 1500, 0, 60000], ["fall", "vhFall", health.fall ?? 3, 1, 20], ["rise", "vhRise", health.rise ?? 2, 1, 20], ["status-min", "vhStatusMin", health.expected_status_min ?? 200, 100, 599], ["status-max", "vhStatusMax", health.expected_status_max ?? 399, 100, 599]]) field(g, name, key, val, "number", { required: true, min, max }); note(g, "vhHealthNote");
backendHealthEditorControl = backendHealthEditor(g, members);
note(g, "backendHealthOverridesHelp");
@@ -560,7 +562,7 @@ function collect() {
pool_eligible: yes("member-pool", row), backup: yes("member-backup", row), drain: yes("member-drain", row),
health_check_enabled: yes("member-health-enabled", row),
health_check: yes("member-health-custom", row.memberHealthPanel) ? {
mode: str("member-health-mode", row.memberHealthPanel), path: str("member-health-path", row.memberHealthPanel), host: nullable("member-health-host", row.memberHealthPanel),
mode: str("member-health-mode", row.memberHealthPanel), path: str("member-health-path", row.memberHealthPanel), host: nullable("member-health-host", row.memberHealthPanel), port: optionalPort("member-health-port", row.memberHealthPanel), expect_body: nullable("member-health-expect-body", row.memberHealthPanel),
interval_ms: num("member-health-interval", row.memberHealthPanel), timeout_ms: num("member-health-timeout", row.memberHealthPanel), slow_response_ms: num("member-health-slow", row.memberHealthPanel),
fall: num("member-health-fall", row.memberHealthPanel), rise: num("member-health-rise", row.memberHealthPanel),
expected_status_min: num("member-health-min", row.memberHealthPanel), expected_status_max: num("member-health-max", row.memberHealthPanel)
@@ -630,7 +632,7 @@ function collect() {
load_balancing: {
algorithm: str("algorithm"), slow_start_seconds: num("slow-start"), keepalive: { enabled: yes("keepalive"), idle_timeout_seconds: num("idle-timeout"), max_idle_per_host: num("max-idle"), tcp_keepalive_seconds: num("tcp-keepalive"), pool_recycle_requests: num("pool-recycle-requests"), pool_recycle_seconds: num("pool-recycle-seconds"), http1_max_buffer_bytes: num("http1-max-buffer") },
retry: { max_retries: num("retry-max"), retry_connect_errors: yes("retry-connect"), retry_timeouts: yes("retry-timeouts"), retry_statuses: str("retry-statuses").split(/[\s,]+/).filter(Boolean).map(value => Number(value)) },
health_check: { mode: str("health-mode"), path: str("health-path"), host: nullable("health-host"), interval_ms: num("interval"), timeout_ms: num("timeout"), slow_response_ms: num("slow"), fall: num("fall"), rise: num("rise"), expected_status_min: num("status-min"), expected_status_max: num("status-max") },
health_check: { mode: str("health-mode"), path: str("health-path"), host: nullable("health-host"), port: optionalPort("health-port"), expect_body: nullable("health-expect-body"), interval_ms: num("interval"), timeout_ms: num("timeout"), slow_response_ms: num("slow"), fall: num("fall"), rise: num("rise"), expected_status_min: num("status-min"), expected_status_max: num("status-max") },
passive_health: {
enabled: yes("passive-enabled"),
count_connect_errors: yes("passive-connect"),
+1 -1
View File
@@ -2,7 +2,7 @@
function inheritBackendHealthInputs(panel) {
for (const [local, shared] of [
["member-health-mode", "health-mode"], ["member-health-path", "health-path"],
["member-health-host", "health-host"], ["member-health-interval", "interval"],
["member-health-host", "health-host"], ["member-health-port", "health-port"], ["member-health-expect-body", "health-expect-body"], ["member-health-interval", "interval"],
["member-health-timeout", "timeout"], ["member-health-slow", "slow"],
["member-health-fall", "fall"], ["member-health-rise", "rise"],
["member-health-min", "status-min"], ["member-health-max", "status-max"]
+1
View File
@@ -450,6 +450,7 @@ pub struct Member {
pub backend: Arc<CompiledBackend>,
pub definition: PoolMember,
pub state: Arc<BackendState>,
health_probe: Option<Arc<CompiledBackend>>,
health_settings: Arc<HealthCheck>,
passive_settings: Arc<PassiveHealth>,
token: String,
+37 -7
View File
@@ -291,12 +291,36 @@ impl Pool {
text
}
}
const PROBE_BODY_LIMIT: usize = 65_536;
async fn probe_body_contains(mut body: hyper::body::Incoming, expected: &[u8]) -> bool {
use http_body_util::BodyExt;
let mut seen = Vec::new();
while seen.len() < PROBE_BODY_LIMIT {
match body.frame().await {
Some(Ok(frame)) => {
if let Ok(data) = frame.into_data() {
let room = PROBE_BODY_LIMIT - seen.len();
seen.extend_from_slice(&data[..data.len().min(room)]);
if seen.windows(expected.len()).any(|window| window == expected) {
return true;
}
}
}
Some(Err(_)) => return false,
None => break,
}
}
seen.windows(expected.len()).any(|window| window == expected)
}
async fn probe(member: Arc<Member>, settings: Arc<HealthCheck>) {
let start = Instant::now();
let target = member.health_probe.as_deref().unwrap_or(&member.backend);
let result = tokio::time::timeout(Duration::from_millis(settings.timeout_ms), async {
match settings.mode {
HealthMode::Tcp => {
crate::upstream::tcp_probe(&member.backend.origin, &member.backend.address_policy)
crate::upstream::tcp_probe(&target.origin, &target.address_policy)
.await
.map(|_| "ok")
.unwrap_or("connect")
@@ -305,14 +329,14 @@ async fn probe(member: Arc<Member>, settings: Arc<HealthCheck>) {
let request = Request::builder()
.uri(format!(
"{}{}",
member.backend.request_origin, settings.path
target.request_origin, settings.path
))
.header(
"host",
settings
.host
.as_deref()
.unwrap_or_else(|| member.backend.host_header.to_str().unwrap_or("")),
.unwrap_or_else(|| target.host_header.to_str().unwrap_or("")),
)
.header("connection", "close")
.header(
@@ -321,14 +345,20 @@ async fn probe(member: Arc<Member>, settings: Arc<HealthCheck>) {
)
.body(Body::empty());
match request {
Ok(request) => match member.backend.websocket_client.request(request).await {
Ok(request) => match target.websocket_client.request(request).await {
Ok(response) => {
if (settings.expected_status_min..=settings.expected_status_max)
if !(settings.expected_status_min..=settings.expected_status_max)
.contains(&response.status().as_u16())
{
"ok"
} else {
"http_status"
} else if let Some(expected) = settings.expect_body.as_deref() {
if probe_body_contains(response.into_body(), expected.as_bytes()).await {
"ok"
} else {
"http_body"
}
} else {
"ok"
}
}
Err(_) => "connect",
+11
View File
@@ -99,6 +99,8 @@ pub struct HealthCheck {
pub mode: HealthMode,
pub path: String,
pub host: Option<String>,
pub port: Option<u16>,
pub expect_body: Option<String>,
pub interval_ms: u64,
pub timeout_ms: u64,
pub slow_response_ms: u64,
@@ -113,6 +115,8 @@ impl Default for HealthCheck {
mode: HealthMode::Tcp,
path: "/".into(),
host: None,
port: None,
expect_body: None,
interval_ms: 5000,
timeout_ms: 2000,
slow_response_ms: 1500,
@@ -261,6 +265,13 @@ impl Settings {
&& !h.path.bytes().any(|b| b <= 32 || b == 127),
"invalid health HTTP path"
);
ensure!(h.port != Some(0), "health check port must be 1..65535");
if let Some(expected) = &h.expect_body {
ensure!(
!expected.is_empty() && expected.len() <= 256 && !expected.chars().any(char::is_control),
"health check expected body text must be 1..256 printable characters"
);
}
if let Some(host) = &h.host {
crate::config::request_host(host)?;
let _: HeaderValue = host.parse()?;
+1
View File
@@ -24,6 +24,7 @@ impl Pool {
let passive_settings = Arc::new(settings.passive_for(&definition));
Arc::new(Member {
state: Arc::new(BackendState::new(health_settings.mode)),
health_probe: backend.health_probe.clone(),
definition,
backend,
health_settings,
+129
View File
@@ -832,4 +832,133 @@ mod tests {
assert_eq!(state.last_status.load(Ordering::Relaxed), 503);
assert_eq!(lock(&state.last_failure).detail, "status=503");
}
fn health_port_config(backend_port: u16, health: serde_json::Value) -> crate::config::Config {
serde_json::from_value(serde_json::json!({
"listeners":{"endpoints":[{"id":"http","binding":{"address":"127.0.0.1:8080"}}]},
"vhosts":[{
"id":"app","frontend":{"hosts":["app.localhost"],"listener_ids":["http"]},
"backends":[{"id":"a","url":format!("http://127.0.0.1:{backend_port}"),"allow_private":true}],
"load_balancing":{"health_check":health}
}]})).unwrap()
}
fn health_port_policy() -> crate::config::ValidationPolicy {
crate::config::ValidationPolicy {
proxy_addr: "127.0.0.1:8080".parse().unwrap(),
admin_addr: "127.0.0.1:9090".parse().unwrap(),
tls_addr: None,
cert_dir: None,
data_dir: None,
}
}
#[test]
fn health_check_port_compiles_separate_probe_target() {
let policy = health_port_policy();
let snapshot = crate::config::compile(
health_port_config(9000, serde_json::json!({"mode":"tcp","port":8079})), 0, &policy).unwrap();
let backend = &snapshot.pools[0].members[0].backend;
assert_eq!(backend.origin, "http://127.0.0.1:9000");
assert_eq!(backend.health_probe.as_ref().unwrap().origin, "http://127.0.0.1:8079");
for health in [
serde_json::json!({"mode":"tcp","port":9000}),
serde_json::json!({"mode":"tcp"}),
serde_json::json!({"mode":"off","port":8079}),
] {
let snapshot = crate::config::compile(health_port_config(9000, health), 0, &policy).unwrap();
assert!(snapshot.pools[0].members[0].backend.health_probe.is_none());
}
assert!(crate::config::compile(
health_port_config(9000, serde_json::json!({"mode":"tcp","port":0})), 0, &policy).is_err());
assert!(crate::config::compile(
health_port_config(9000, serde_json::json!({"mode":"http","port":9090})), 0, &policy).is_err());
}
#[tokio::test]
async fn health_check_probes_configured_port_instead_of_traffic_port() {
let traffic = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let traffic_port = traffic.local_addr().unwrap().port();
let closed_port = {
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
listener.local_addr().unwrap().port()
};
let policy = health_port_policy();
let config = health_port_config(traffic_port, serde_json::json!({
"mode":"tcp","port":closed_port,"fall":1,"rise":1,"interval_ms":250,"timeout_ms":200,"slow_response_ms":0}));
let snapshot = crate::config::compile(config, 0, &policy).unwrap();
let member = snapshot.pools[0].members[0].clone();
let settings = member.health_settings.clone();
probe(member.clone(), settings.clone()).await;
assert!(!member.state.healthy(), "probe must use the closed health port");
let health = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let health_port = health.local_addr().unwrap().port();
drop(traffic);
let config = health_port_config(traffic_port, serde_json::json!({
"mode":"tcp","port":health_port,"fall":1,"rise":1,"interval_ms":250,"timeout_ms":200,"slow_response_ms":0}));
let snapshot = crate::config::compile(config, 0, &policy).unwrap();
let member = snapshot.pools[0].members[0].clone();
let settings = member.health_settings.clone();
probe(member.clone(), settings).await;
assert!(member.state.healthy(), "probe must succeed on the open health port while traffic port is closed");
tokio::time::timeout(Duration::from_secs(1), health.accept())
.await
.expect("health port must receive the probe connection")
.unwrap();
}
async fn one_shot_http(body: &'static str) -> (u16, tokio::task::JoinHandle<()>) {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
let task = tokio::spawn(async move {
while let Ok((mut socket, _)) = listener.accept().await {
let mut buffer = [0u8; 2048];
let _ = socket.read(&mut buffer).await;
let response = format!(
"HTTP/1.1 200 OK\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
body.len(), body
);
let _ = socket.write_all(response.as_bytes()).await;
}
});
(port, task)
}
#[tokio::test]
async fn http_health_check_requires_expected_body_text() {
let policy = health_port_policy();
for (body, healthy) in [("{\"status\":\"ok\"}", true), ("{\"status\":\"degraded\"}", false)] {
let (port, server) = one_shot_http(body).await;
let config = health_port_config(port, serde_json::json!({
"mode":"http","path":"/health","expect_body":"\"status\":\"ok\"",
"fall":1,"rise":1,"interval_ms":1000,"timeout_ms":800,"slow_response_ms":0}));
let snapshot = crate::config::compile(config, 0, &policy).unwrap();
let member = snapshot.pools[0].members[0].clone();
probe(member.clone(), member.health_settings.clone()).await;
assert_eq!(member.state.healthy(), healthy, "body {body}");
if !healthy {
assert_eq!(lock(&member.state.health).result, "http_body");
}
server.abort();
}
for invalid in ["", "line\nbreak"] {
assert!(crate::config::compile(
health_port_config(9000, serde_json::json!({"mode":"http","expect_body":invalid})), 0, &policy).is_err());
}
}
#[tokio::test]
async fn health_check_port_survives_live_reload_with_reused_clients() {
let policy = health_port_policy();
let defaults = (policy.proxy_addr, policy.tls_addr);
let first = crate::config::compile_reusing(
health_port_config(9000, serde_json::json!({"mode":"tcp"})), 0, &policy, defaults, None).unwrap();
assert!(first.pools[0].members[0].health_probe.is_none());
let second = crate::config::compile_reusing(
health_port_config(9000, serde_json::json!({"mode":"tcp","port":8079})), 1, &policy, defaults, Some(&first)).unwrap();
let member = &second.pools[0].members[0];
assert!(Arc::ptr_eq(&member.backend, &first.pools[0].members[0].backend), "traffic clients are reused");
assert_eq!(member.health_probe.as_ref().unwrap().origin, "http://127.0.0.1:8079");
let third = crate::config::compile_reusing(
health_port_config(9000, serde_json::json!({"mode":"tcp"})), 2, &policy, defaults, Some(&second)).unwrap();
assert!(third.pools[0].members[0].health_probe.is_none());
}
}
+15
View File
@@ -104,8 +104,23 @@ pub(crate) fn compile_backend_with_timeout(
targets_admin_listener,
client,
websocket_client,
health_probe: None,
})
}
/// Backend copy used only by active health checks that probe a different port.
pub(crate) fn health_probe_backend(backend: &Backend, port: Option<u16>) -> Result<Option<Backend>> {
let Some(port) = port else { return Ok(None) };
let mut url = Url::parse(&backend.url).context("invalid backend URL")?;
if url.port_or_known_default() == Some(port) {
return Ok(None);
}
url.set_port(Some(port))
.map_err(|_| anyhow::anyhow!("backend URL does not accept a health check port"))?;
let mut probe = backend.clone();
probe.url = url.as_str().trim_end_matches('/').to_owned() + "/";
Ok(Some(probe))
}
pub fn compile(config: Config, revision: u64, policy: &ValidationPolicy) -> Result<Snapshot> {
compile_with_defaults(
config,
+1
View File
@@ -224,6 +224,7 @@ pub struct CompiledBackend {
pub tls_verify: bool,
pub accept_local_listener_risk: bool,
pub targets_admin_listener: bool,
pub health_probe: Option<Arc<CompiledBackend>>,
}
#[derive(Debug, Clone)]
pub enum CompiledSource {
+24 -3
View File
@@ -99,17 +99,38 @@ fn compile_routing(
});
let mut compiled_members = Vec::new();
for definition in host.backends.iter().filter(|_| needs_proxy_pool) {
let connect_timeout = host
.proxy_timeouts
.connect_timeout()
.unwrap_or(std::time::Duration::from_secs(10));
let mut backend = compile_backend_with_timeout(
&definition.backend(),
desired_policy,
guarded_bindings,
&host.load_balancing.keepalive,
host.proxy_timeouts
.connect_timeout()
.unwrap_or(std::time::Duration::from_secs(10)),
connect_timeout,
)
.with_context(|| format!("vhost {} backend {}", host.id, definition.id))?;
backend.id = format!("{}/{}", host.id, definition.id);
let health = host.load_balancing.health_for(definition);
if health.mode != crate::balance::HealthMode::Off {
if let Some(probe) =
health_probe_backend(&definition.backend(), health.port)?
{
let mut probe = compile_backend_with_timeout(
&probe,
desired_policy,
guarded_bindings,
&host.load_balancing.keepalive,
connect_timeout,
)
.with_context(|| {
format!("vhost {} backend {} health check port", host.id, definition.id)
})?;
probe.id = format!("{}/{}#health", host.id, definition.id);
backend.health_probe = Some(Arc::new(probe));
}
}
compiled_members.push(Arc::new(backend));
}
let pool = if compiled_members.is_empty() {
+1
View File
@@ -1,6 +1,7 @@
#![allow(clippy::too_many_arguments)]
#![cfg_attr(test, allow(clippy::field_reassign_with_default))]
#[cfg(feature = "mimalloc")]
#[global_allocator]
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
+59
View File
@@ -1,3 +1,41 @@
/// Certificate expiry read from the certificate files, refreshed at most every five minutes.
fn certificate_expiry(
app: &App,
certificates: &[crate::tls::Certificate],
) -> std::collections::HashMap<String, i64> {
type Cached = (std::time::Instant, Vec<(String, String)>, std::collections::HashMap<String, i64>);
static CACHE: std::sync::Mutex<Option<Cached>> = std::sync::Mutex::new(None);
let wanted: Vec<(String, String)> = certificates
.iter()
.map(|cert| (cert.id.clone(), cert.cert_file.clone()))
.collect();
let mut cache = CACHE.lock().unwrap_or_else(|e| e.into_inner());
if let Some((at, files, values)) = cache.as_ref() {
if at.elapsed() < std::time::Duration::from_secs(300) && *files == wanted {
return values.clone();
}
}
let values = read_certificate_expiry(app.policy.cert_dir.as_deref(), &wanted);
*cache = Some((std::time::Instant::now(), wanted, values.clone()));
values
}
fn read_certificate_expiry(
directory: Option<&std::path::Path>,
certificates: &[(String, String)],
) -> std::collections::HashMap<String, i64> {
let Some(directory) = directory else { return Default::default() };
certificates
.iter()
.filter_map(|(id, file)| {
crate::tls::certificate_metadata(directory, file)
.ok()
.and_then(|metadata| metadata.not_after_unix)
.map(|not_after| (id.clone(), not_after))
})
.collect()
}
fn render_core(app: &App, snapshot: &crate::config::Snapshot, out: &mut String, f: &mut Families) {
let config = &snapshot.envelope.config;
let settings = &config.prometheus;
@@ -101,8 +139,29 @@ fn render_core(app: &App, snapshot: &crate::config::Snapshot, out: &mut String,
config.tls.certificates.len(),
);
let renewals = app.acme.renewal_view();
let expiry = certificate_expiry(app, &config.tls.certificates);
for certificate in &config.tls.certificates {
let l = labels(&[("certificate", &certificate.id)]);
if let Some(not_after) = expiry.get(&certificate.id).copied() {
f.add(
"rust_proxy_certificate_not_after_timestamp_seconds",
"gauge",
"Expiry time of the installed certificate (Unix seconds).",
&l,
not_after,
);
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|elapsed| elapsed.as_secs() as i64)
.unwrap_or_default();
f.add(
"rust_proxy_certificate_days_remaining",
"gauge",
"Whole days until the installed certificate expires; negative when expired.",
&l,
(not_after - now).div_euclid(86_400),
);
}
let issuer = certificate.issuer.as_ref();
f.add(
"rust_proxy_certificate_auto_renew_enabled",
+14
View File
@@ -218,4 +218,18 @@ mod tests {
assert_eq!(response.status(), StatusCode::OK);
store.close().await;
}
#[test]
fn certificate_expiry_is_read_from_installed_certificate_files() {
let dir = tempfile::tempdir().unwrap();
let generated = rcgen::generate_simple_self_signed(vec!["expiry.localhost".into()]).unwrap();
std::fs::write(dir.path().join("site.pem"), generated.cert.pem()).unwrap();
let values = read_certificate_expiry(
Some(dir.path()),
&[("site".into(), "site.pem".into()), ("missing".into(), "missing.pem".into())],
);
assert!(values.get("site").is_some_and(|not_after| *not_after > 0));
assert!(!values.contains_key("missing"));
assert!(read_certificate_expiry(None, &[("site".into(), "site.pem".into())]).is_empty());
}
}