gru/RustProxy
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.31.5

Browse Source
This commit is contained in:
Mateusz Gruszczyński committed 2026-10-11 17:10:04 +02:00
1 parent d942c1a35d
commit 75104f87ba
103 files changed
+1344 -370
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/admin/file_logs.rs
+16 -188
View File
@@ -4,12 +4,15 @@ const FILE_LOG_SCAN_LINES: usize = 20_000;
const FILE_LOG_MAX_SOURCES: usize = 32;
const FILE_LOG_MAX_LIMIT: usize = 500;
const FILE_LOG_MAX_OFFSET: usize = 10_000;
const FILE_LOG_MAX_AGGREGATE_RESULTS: usize = 200;
const FILE_LOG_MAX_AGGREGATE_VALUES: usize = 4096;
const FILE_LOG_GZIP_INPUT_MAX: u64 = 8 * 1024 * 1024;
const FILE_LOG_GZIP_OUTPUT_MAX: u64 = 64 * 1024 * 1024;
#[derive(Clone)]
struct FileLogSource {
id: String,
label: String,
group_id: String,
kind: &'static str,
path: std::path::PathBuf,
enabled: bool,
@@ -28,6 +31,10 @@ struct FileLogQuery {
status_class: Option<String>,
client_ip: Option<String>,
vhost: Option<String>,
consolidate_vhosts: Option<bool>,
aggregate_field: Option<String>,
aggregate_limit: Option<usize>,
aggregate_min_count: Option<usize>,
offset: Option<usize>,
limit: Option<usize>,
}
@@ -42,6 +49,7 @@ fn file_modified_ms(metadata: &std::fs::Metadata) -> u64 {
fn source_from_path(
id: String,
label: String,
group_id: String,
kind: &'static str,
path: std::path::PathBuf,
enabled: bool,
@@ -58,6 +66,7 @@ fn source_from_path(
FileLogSource {
id,
label,
group_id,
kind,
path,
enabled,
@@ -109,9 +118,12 @@ fn configured_file_log_sources(config: &crate::config::Config) -> Vec<FileLogSou
} else {
format!("vhost-gz:{name}")
};
let vhost_name = name.split_once(".jsonl").map_or(name.as_str(), |(prefix, _)| prefix);
let group_id = format!("vhost:{vhost_name}");
sources.push(source_from_path(
id,
name,
group_id,
"access",
path,
config.logging.file.enabled,
@@ -124,6 +136,7 @@ fn configured_file_log_sources(config: &crate::config::Config) -> Vec<FileLogSou
sources.push(source_from_path(
"access".into(),
"Access log".into(),
"access".into(),
"access",
access_path.clone(),
config.logging.file.enabled,
@@ -139,6 +152,7 @@ fn configured_file_log_sources(config: &crate::config::Config) -> Vec<FileLogSou
sources.push(source_from_path(
"error".into(),
"Error log".into(),
"error".into(),
"error",
error_path.clone(),
config.error_log.enabled,
@@ -155,6 +169,7 @@ fn file_log_sources_json(sources: &[FileLogSource]) -> Value {
json!({
"id": source.id,
"label": source.label,
"group_id": source.group_id,
"kind": source.kind,
"path": source.path.to_string_lossy(),
"enabled": source.enabled,
@@ -281,191 +296,4 @@ fn normalized_record(
value
}
fn contains_ci(value: Option<&Value>, needle: &str) -> bool {
value
.and_then(Value::as_str)
.is_some_and(|text| text.to_ascii_lowercase().contains(needle))
}
fn record_matches(value: &Value, query: &FileLogQuery, search: &[FileLogSearchClause]) -> bool {
let Some(object) = value.as_object() else {
return false;
};
if !record_matches_search(value, search) {
return false;
}
if let Some(event) = query
.event
.as_deref()
.filter(|v| !v.is_empty() && *v != "all")
{
if object.get("event").and_then(Value::as_str) != Some(event) {
return false;
}
}
if let Some(class) = query
.status_class
.as_deref()
.filter(|v| !v.is_empty() && *v != "all")
{
let Some(status) = object.get("status").and_then(Value::as_u64) else {
return false;
};
let expected = class
.as_bytes()
.first()
.and_then(|v| v.checked_sub(b'0'))
.map(u64::from);
if expected != Some(status / 100) {
return false;
}
}
if let Some(client) = query.client_ip.as_deref().filter(|v| !v.is_empty()) {
if !contains_ci(object.get("client_ip"), &client.to_ascii_lowercase()) {
return false;
}
}
if let Some(vhost) = query.vhost.as_deref().filter(|v| !v.is_empty()) {
if !contains_ci(object.get("vhost"), &vhost.to_ascii_lowercase()) {
return false;
}
}
true
}
fn validate_file_log_query(query: &FileLogQuery) -> Result<(), ApiError> {
let text_ok = |value: &Option<String>, max: usize| {
value
.as_ref()
.is_none_or(|v| v.len() <= max && !v.chars().any(char::is_control))
};
if !text_ok(&query.source, 192)
|| !text_ok(&query.q, 512)
|| !text_ok(&query.client_ip, 128)
|| !text_ok(&query.vhost, 256)
|| query.offset.unwrap_or(0) > FILE_LOG_MAX_OFFSET
|| query
.limit
.is_some_and(|v| v == 0 || v > FILE_LOG_MAX_LIMIT)
|| query.event.as_ref().is_some_and(|v| {
!matches!(
v.as_str(),
"" | "all" | "request" | "protocol_error" | "error"
)
})
|| query.status_class.as_ref().is_some_and(|v| {
!matches!(
v.as_str(),
"" | "all" | "1xx" | "2xx" | "3xx" | "4xx" | "5xx"
)
})
{
return Err(ApiError(
StatusCode::BAD_REQUEST,
"invalid file log filter".into(),
));
}
Ok(())
}
async fn file_logs_query(
State(state): State<Arc<App>>,
Query(query): Query<FileLogQuery>,
) -> Result<Json<Value>, ApiError> {
validate_file_log_query(&query)?;
let search = parse_file_log_search(query.q.as_deref().unwrap_or(""))?;
let config = state.config.load().envelope.config.clone();
tokio::task::spawn_blocking(move || {
let mut sources = configured_file_log_sources(&config);
let log_format = config.logging.format.clone();
let requested = query.source.as_deref().unwrap_or("all");
let mut truncated = false;
if requested != "all" {
sources.retain(|source| source.id == requested);
if sources.is_empty() {
return Err(ApiError(
StatusCode::BAD_REQUEST,
"unknown file log source".into(),
));
}
} else {
sources.sort_by(|a, b| b.modified_ms.cmp(&a.modified_ms));
if sources.len() > FILE_LOG_MAX_SOURCES {
truncated = true;
sources.truncate(FILE_LOG_MAX_SOURCES);
}
}
let offset = query.offset.unwrap_or(0);
let limit = query.limit.unwrap_or(100).clamp(1, FILE_LOG_MAX_LIMIT);
let mut remaining_bytes = FILE_LOG_SCAN_BYTES_TOTAL;
let mut scanned_bytes = 0u64;
let mut remaining_lines = FILE_LOG_SCAN_LINES;
let mut scanned_lines = 0usize;
let mut records = Vec::new();
let mut fields = std::collections::BTreeSet::new();
for (index, source) in sources.iter().enumerate() {
if remaining_bytes == 0 || remaining_lines == 0 {
truncated = true;
break;
}
let sources_left = sources.len().saturating_sub(index).max(1);
let line_budget = (remaining_lines / sources_left).max(1);
match read_tail(source, remaining_bytes, line_budget) {
Ok((lines, bytes, source_truncated)) => {
remaining_bytes = remaining_bytes.saturating_sub(bytes);
remaining_lines = remaining_lines.saturating_sub(lines.len());
scanned_bytes += bytes;
scanned_lines += lines.len();
truncated |= source_truncated;
for line in lines {
if line.is_empty() {
continue;
}
let value = normalized_record(source, &line, &log_format);
if record_matches(&value, &query, &search) {
collect_file_log_fields(&value, &mut fields);
records.push(value);
}
}
}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
Err(_) => truncated = true,
}
}
records.sort_by(|a, b| {
let numeric = |value: &Value| value.get("time_ms").and_then(Value::as_u64);
match (numeric(a), numeric(b)) {
(Some(left), Some(right)) => right.cmp(&left),
_ => b
.get("timestamp")
.and_then(Value::as_str)
.unwrap_or("")
.cmp(a.get("timestamp").and_then(Value::as_str).unwrap_or("")),
}
});
let total = records.len();
let page = records
.into_iter()
.skip(offset)
.take(limit)
.collect::<Vec<_>>();
Ok(Json(json!({
"entries": page,
"total": total,
"offset": offset,
"limit": limit,
"truncated": truncated,
"scanned_bytes": scanned_bytes,
"scanned_lines": scanned_lines,
"fields": fields,
"sources": file_log_sources_json(&sources),
})))
})
.await
.map_err(|_| {
ApiError(
StatusCode::INTERNAL_SERVER_ERROR,
"file log query worker failed".into(),
)
})?
}
include!("file_logs_query.rs");