302 lines
10 KiB
Rust
302 lines
10 KiB
Rust
const FILE_LOG_SCAN_BYTES_PER_SOURCE: u64 = 2 * 1024 * 1024;
|
|
const FILE_LOG_SCAN_BYTES_TOTAL: u64 = 8 * 1024 * 1024;
|
|
const FILE_LOG_SCAN_LINES: usize = 20_000;
|
|
const FILE_LOG_MAX_SOURCES: usize = 32;
|
|
const FILE_LOG_MAX_LIMIT: usize = 500;
|
|
const FILE_LOG_MAX_OFFSET: usize = 10_000;
|
|
const FILE_LOG_MAX_AGGREGATE_RESULTS: usize = 200;
|
|
const FILE_LOG_MAX_AGGREGATE_VALUES: usize = 4096;
|
|
const FILE_LOG_GZIP_INPUT_MAX: u64 = 8 * 1024 * 1024;
|
|
const FILE_LOG_GZIP_OUTPUT_MAX: u64 = 64 * 1024 * 1024;
|
|
#[derive(Clone)]
|
|
struct FileLogSource {
|
|
id: String,
|
|
label: String,
|
|
group_id: String,
|
|
kind: &'static str,
|
|
path: std::path::PathBuf,
|
|
enabled: bool,
|
|
format: &'static str,
|
|
exists: bool,
|
|
size: u64,
|
|
modified_ms: u64,
|
|
compressed: bool,
|
|
}
|
|
#[derive(Default, Deserialize)]
|
|
#[serde(default, deny_unknown_fields)]
|
|
struct FileLogQuery {
|
|
source: Option<String>,
|
|
q: Option<String>,
|
|
event: Option<String>,
|
|
status_class: Option<String>,
|
|
client_ip: Option<String>,
|
|
vhost: Option<String>,
|
|
consolidate_vhosts: Option<bool>,
|
|
aggregate_field: Option<String>,
|
|
aggregate_limit: Option<usize>,
|
|
aggregate_min_count: Option<usize>,
|
|
sort_by: Option<String>,
|
|
sort_descending: Option<bool>,
|
|
offset: Option<usize>,
|
|
limit: Option<usize>,
|
|
}
|
|
fn file_modified_ms(metadata: &std::fs::Metadata) -> u64 {
|
|
metadata
|
|
.modified()
|
|
.ok()
|
|
.and_then(|time| time.duration_since(std::time::UNIX_EPOCH).ok())
|
|
.map(|duration| duration.as_millis() as u64)
|
|
.unwrap_or(0)
|
|
}
|
|
fn source_from_path(
|
|
id: String,
|
|
label: String,
|
|
group_id: String,
|
|
kind: &'static str,
|
|
path: std::path::PathBuf,
|
|
enabled: bool,
|
|
format: &'static str,
|
|
) -> FileLogSource {
|
|
let metadata = std::fs::symlink_metadata(&path).ok();
|
|
let safe_regular = metadata
|
|
.as_ref()
|
|
.is_some_and(|meta| meta.file_type().is_file() && !meta.file_type().is_symlink());
|
|
let compressed = path
|
|
.extension()
|
|
.and_then(|ext| ext.to_str())
|
|
.is_some_and(|ext| ext.eq_ignore_ascii_case("gz"));
|
|
FileLogSource {
|
|
id,
|
|
label,
|
|
group_id,
|
|
kind,
|
|
path,
|
|
enabled,
|
|
format,
|
|
exists: safe_regular,
|
|
size: metadata
|
|
.as_ref()
|
|
.filter(|_| safe_regular)
|
|
.map_or(0, |m| m.len()),
|
|
modified_ms: metadata
|
|
.as_ref()
|
|
.filter(|_| safe_regular)
|
|
.map_or(0, file_modified_ms),
|
|
compressed,
|
|
}
|
|
}
|
|
include!("file_logs_io.rs");
|
|
|
|
fn configured_file_log_sources(config: &crate::config::Config) -> Vec<FileLogSource> {
|
|
let mut sources = Vec::new();
|
|
if config.logging.file.per_vhost {
|
|
let directory = std::path::PathBuf::from(&config.logging.file.directory);
|
|
let directory_safe = std::fs::symlink_metadata(&directory)
|
|
.ok()
|
|
.is_some_and(|meta| meta.file_type().is_dir() && !meta.file_type().is_symlink());
|
|
if directory_safe {
|
|
let entries = newest_log_files(std::fs::read_dir(&directory)
|
|
.ok()
|
|
.into_iter()
|
|
.flat_map(|iter| iter.filter_map(|entry| entry.ok()))
|
|
.filter_map(|entry| {
|
|
let path = entry.path();
|
|
let name = entry.file_name().to_string_lossy().into_owned();
|
|
let plain = name.ends_with(".jsonl");
|
|
let rotated = name.contains(".jsonl.") && name.ends_with(".gz");
|
|
if (!plain && !rotated) || name.len() > 180 {
|
|
return None;
|
|
}
|
|
let file_type = entry.file_type().ok()?;
|
|
if !file_type.is_file() || file_type.is_symlink() {
|
|
return None;
|
|
}
|
|
let meta = entry.metadata().ok()?;
|
|
Some((name, path, file_modified_ms(&meta)))
|
|
}), 256);
|
|
for (name, path, _) in entries {
|
|
let id = if name.ends_with(".jsonl") {
|
|
format!("vhost:{}", name.trim_end_matches(".jsonl"))
|
|
} else {
|
|
format!("vhost-gz:{name}")
|
|
};
|
|
let vhost_name = name.split_once(".jsonl").map_or(name.as_str(), |(prefix, _)| prefix);
|
|
let group_id = format!("vhost:{vhost_name}");
|
|
sources.push(source_from_path(
|
|
id,
|
|
name,
|
|
group_id,
|
|
"access",
|
|
path,
|
|
config.logging.file.enabled,
|
|
"JSONL",
|
|
));
|
|
}
|
|
}
|
|
} else {
|
|
let access_path = std::path::PathBuf::from(&config.logging.file.path);
|
|
sources.push(source_from_path(
|
|
"access".into(),
|
|
"Access log".into(),
|
|
"access".into(),
|
|
"access",
|
|
access_path.clone(),
|
|
config.logging.file.enabled,
|
|
"JSONL",
|
|
));
|
|
sources.extend(rotated_gzip_sources(&access_path, "access", "Access log", "access", config.logging.file.enabled, "JSONL"));
|
|
}
|
|
let error_format = match config.error_log.format {
|
|
crate::logging::ErrorLogFormat::Json => "JSONL",
|
|
crate::logging::ErrorLogFormat::Raw => "RAW",
|
|
};
|
|
let error_path = std::path::PathBuf::from(&config.error_log.path);
|
|
sources.push(source_from_path(
|
|
"error".into(),
|
|
"Error log".into(),
|
|
"error".into(),
|
|
"error",
|
|
error_path.clone(),
|
|
config.error_log.enabled,
|
|
error_format,
|
|
));
|
|
sources.extend(rotated_gzip_sources(&error_path, "error", "Error log", "error", config.error_log.enabled, error_format));
|
|
sources
|
|
}
|
|
fn file_log_sources_json(sources: &[FileLogSource]) -> Value {
|
|
Value::Array(
|
|
sources
|
|
.iter()
|
|
.map(|source| {
|
|
json!({
|
|
"id": source.id,
|
|
"label": source.label,
|
|
"group_id": source.group_id,
|
|
"kind": source.kind,
|
|
"path": source.path.to_string_lossy(),
|
|
"enabled": source.enabled,
|
|
"format": source.format,
|
|
"exists": source.exists,
|
|
"size": source.size,
|
|
"modified_ms": source.modified_ms,
|
|
"compressed": source.compressed,
|
|
})
|
|
})
|
|
.collect(),
|
|
)
|
|
}
|
|
async fn file_logs(State(state): State<Arc<App>>) -> Result<Json<Value>, ApiError> {
|
|
let config = state.config.load().envelope.config.clone();
|
|
let sources = tokio::task::spawn_blocking(move || configured_file_log_sources(&config))
|
|
.await
|
|
.map_err(|_| {
|
|
ApiError(
|
|
StatusCode::INTERNAL_SERVER_ERROR,
|
|
"file log inventory worker failed".into(),
|
|
)
|
|
})?;
|
|
Ok(Json(json!({
|
|
"sources": file_log_sources_json(&sources),
|
|
"limits": {
|
|
"max_limit": FILE_LOG_MAX_LIMIT,
|
|
"max_offset": FILE_LOG_MAX_OFFSET,
|
|
"scan_bytes_per_source": FILE_LOG_SCAN_BYTES_PER_SOURCE,
|
|
"scan_bytes_total": FILE_LOG_SCAN_BYTES_TOTAL,
|
|
"scan_lines": FILE_LOG_SCAN_LINES,
|
|
"max_sources_per_query": FILE_LOG_MAX_SOURCES,
|
|
}
|
|
})))
|
|
}
|
|
fn copy_mapped_field(
|
|
object: &mut serde_json::Map<String, Value>,
|
|
format: &crate::logging::JsonFormat,
|
|
canonical: &str,
|
|
) {
|
|
if object.contains_key(canonical) {
|
|
return;
|
|
}
|
|
let mapped = format
|
|
.fields
|
|
.iter()
|
|
.find_map(|(output, source)| (source == canonical).then_some(output));
|
|
if let Some(value) = mapped.and_then(|output| object.get(output)).cloned() {
|
|
object.insert(canonical.to_owned(), value);
|
|
}
|
|
}
|
|
|
|
fn normalized_record(
|
|
source: &FileLogSource,
|
|
line: &str,
|
|
format: &crate::logging::JsonFormat,
|
|
) -> Value {
|
|
let mut value = if source.format == "RAW" {
|
|
parse_raw_error_record(line)
|
|
} else {
|
|
serde_json::from_str::<Value>(line).unwrap_or_else(|_| json!({"raw": line}))
|
|
};
|
|
if !value.is_object() {
|
|
value = json!({"value": value});
|
|
}
|
|
let object = value.as_object_mut().expect("record normalized to object");
|
|
if source.kind == "access" {
|
|
for field in [
|
|
"event",
|
|
"time_ms",
|
|
"status",
|
|
"client_ip",
|
|
"vhost",
|
|
"method",
|
|
"host",
|
|
"path",
|
|
"transport",
|
|
"http_version",
|
|
"protocol_error",
|
|
"error_detail",
|
|
"tls_sni",
|
|
"reason",
|
|
"error",
|
|
"sni",
|
|
"authority",
|
|
"matched_by",
|
|
] {
|
|
copy_mapped_field(object, format, field);
|
|
}
|
|
if !object.contains_key("client_ip") {
|
|
if let Some(value) = object.get("source_ip").cloned() {
|
|
object.insert("client_ip".into(), value);
|
|
}
|
|
}
|
|
if !object.contains_key("reason") {
|
|
if let Some(value) = object.get("protocol_error").cloned() {
|
|
object.insert("reason".into(), value);
|
|
}
|
|
}
|
|
if !object.contains_key("error") {
|
|
if let Some(value) = object.get("error_detail").cloned() {
|
|
object.insert("error".into(), value);
|
|
}
|
|
}
|
|
if !object.contains_key("sni") {
|
|
if let Some(value) = object.get("tls_sni").cloned() {
|
|
object.insert("sni".into(), value);
|
|
}
|
|
}
|
|
}
|
|
object.entry("event").or_insert_with(|| {
|
|
Value::String(
|
|
if source.kind == "error" {
|
|
"error"
|
|
} else {
|
|
"request"
|
|
}
|
|
.into(),
|
|
)
|
|
});
|
|
object.insert("_source".into(), Value::String(source.id.clone()));
|
|
object.insert("_source_kind".into(), Value::String(source.kind.into()));
|
|
object.insert("_source_format".into(), Value::String(source.format.into()));
|
|
value
|
|
}
|
|
|
|
include!("file_logs_query.rs");
|