v0.26.0

src/admin/file_logs.rs

@@ -0,0 +1,426 @@
+const FILE_LOG_SCAN_BYTES_PER_SOURCE: u64 = 2 * 1024 * 1024;
+const FILE_LOG_SCAN_BYTES_TOTAL: u64 = 8 * 1024 * 1024;
+const FILE_LOG_SCAN_LINES: usize = 20_000;
+const FILE_LOG_MAX_SOURCES: usize = 32;
+const FILE_LOG_MAX_LIMIT: usize = 500;
+const FILE_LOG_MAX_OFFSET: usize = 10_000;
+
+#[derive(Clone)]
+struct FileLogSource {
+    id: String,
+    label: String,
+    kind: &'static str,
+    path: std::path::PathBuf,
+    enabled: bool,
+    format: &'static str,
+    exists: bool,
+    size: u64,
+    modified_ms: u64,
+}
+
+#[derive(Default, Deserialize)]
+#[serde(default, deny_unknown_fields)]
+struct FileLogQuery {
+    source: Option<String>,
+    q: Option<String>,
+    event: Option<String>,
+    status_class: Option<String>,
+    client_ip: Option<String>,
+    vhost: Option<String>,
+    offset: Option<usize>,
+    limit: Option<usize>,
+}
+
+fn file_modified_ms(metadata: &std::fs::Metadata) -> u64 {
+    metadata
+        .modified()
+        .ok()
+        .and_then(|time| time.duration_since(std::time::UNIX_EPOCH).ok())
+        .map(|duration| duration.as_millis() as u64)
+        .unwrap_or(0)
+}
+
+fn source_from_path(
+    id: String,
+    label: String,
+    kind: &'static str,
+    path: std::path::PathBuf,
+    enabled: bool,
+    format: &'static str,
+) -> FileLogSource {
+    let metadata = std::fs::symlink_metadata(&path).ok();
+    let safe_regular = metadata
+        .as_ref()
+        .is_some_and(|meta| meta.file_type().is_file() && !meta.file_type().is_symlink());
+    FileLogSource {
+        id,
+        label,
+        kind,
+        path,
+        enabled,
+        format,
+        exists: safe_regular,
+        size: metadata
+            .as_ref()
+            .filter(|_| safe_regular)
+            .map_or(0, |m| m.len()),
+        modified_ms: metadata
+            .as_ref()
+            .filter(|_| safe_regular)
+            .map_or(0, file_modified_ms),
+    }
+}
+
+fn configured_file_log_sources(config: &crate::config::Config) -> Vec<FileLogSource> {
+    let mut sources = Vec::new();
+    if config.logging.file.per_vhost {
+        let directory = std::path::PathBuf::from(&config.logging.file.directory);
+        let directory_safe = std::fs::symlink_metadata(&directory)
+            .ok()
+            .is_some_and(|meta| meta.file_type().is_dir() && !meta.file_type().is_symlink());
+        if directory_safe {
+            let mut entries = std::fs::read_dir(&directory)
+                .ok()
+                .into_iter()
+                .flat_map(|iter| iter.filter_map(|entry| entry.ok()))
+                .filter_map(|entry| {
+                    let path = entry.path();
+                    let name = entry.file_name().to_string_lossy().into_owned();
+                    if !name.ends_with(".jsonl") || name.len() > 160 {
+                        return None;
+                    }
+                    let file_type = entry.file_type().ok()?;
+                    if !file_type.is_file() || file_type.is_symlink() {
+                        return None;
+                    }
+                    let meta = entry.metadata().ok()?;
+                    Some((name, path, file_modified_ms(&meta)))
+                })
+                .collect::<Vec<_>>();
+            entries.sort_by(|a, b| b.2.cmp(&a.2).then_with(|| a.0.cmp(&b.0)));
+            entries.truncate(256);
+            for (name, path, _) in entries {
+                let id = format!("vhost:{}", name.trim_end_matches(".jsonl"));
+                sources.push(source_from_path(
+                    id,
+                    name,
+                    "access",
+                    path,
+                    config.logging.file.enabled,
+                    "JSONL",
+                ));
+            }
+        }
+    } else {
+        sources.push(source_from_path(
+            "access".into(),
+            "Access log".into(),
+            "access",
+            std::path::PathBuf::from(&config.logging.file.path),
+            config.logging.file.enabled,
+            "JSONL",
+        ));
+    }
+    sources.push(source_from_path(
+        "error".into(),
+        "Error log".into(),
+        "error",
+        std::path::PathBuf::from(&config.error_log.path),
+        config.error_log.enabled,
+        match config.error_log.format {
+            crate::logging::ErrorLogFormat::Json => "JSONL",
+            crate::logging::ErrorLogFormat::Raw => "RAW",
+        },
+    ));
+    sources
+}
+
+fn file_log_sources_json(sources: &[FileLogSource]) -> Value {
+    Value::Array(
+        sources
+            .iter()
+            .map(|source| {
+                json!({
+                    "id": source.id,
+                    "label": source.label,
+                    "kind": source.kind,
+                    "path": source.path.to_string_lossy(),
+                    "enabled": source.enabled,
+                    "format": source.format,
+                    "exists": source.exists,
+                    "size": source.size,
+                    "modified_ms": source.modified_ms,
+                })
+            })
+            .collect(),
+    )
+}
+
+async fn file_logs(State(state): State<Arc<App>>) -> Result<Json<Value>, ApiError> {
+    let config = state.config.load().envelope.config.clone();
+    let sources = tokio::task::spawn_blocking(move || configured_file_log_sources(&config))
+        .await
+        .map_err(|_| {
+            ApiError(
+                StatusCode::INTERNAL_SERVER_ERROR,
+                "file log inventory worker failed".into(),
+            )
+        })?;
+    Ok(Json(json!({
+        "sources": file_log_sources_json(&sources),
+        "limits": {
+            "max_limit": FILE_LOG_MAX_LIMIT,
+            "max_offset": FILE_LOG_MAX_OFFSET,
+            "scan_bytes_per_source": FILE_LOG_SCAN_BYTES_PER_SOURCE,
+            "scan_bytes_total": FILE_LOG_SCAN_BYTES_TOTAL,
+            "scan_lines": FILE_LOG_SCAN_LINES,
+            "max_sources_per_query": FILE_LOG_MAX_SOURCES,
+        }
+    })))
+}
+
+fn open_file_log(path: &std::path::Path) -> std::io::Result<std::fs::File> {
+    let mut options = std::fs::OpenOptions::new();
+    options.read(true);
+    #[cfg(unix)]
+    {
+        use std::os::unix::fs::OpenOptionsExt;
+        options.custom_flags(libc::O_NOFOLLOW);
+    }
+    let file = options.open(path)?;
+    let metadata = file.metadata()?;
+    if !metadata.file_type().is_file() {
+        return Err(std::io::Error::new(
+            std::io::ErrorKind::InvalidInput,
+            "configured log path is not a regular file",
+        ));
+    }
+    Ok(file)
+}
+
+fn read_tail(
+    source: &FileLogSource,
+    budget: u64,
+    line_budget: usize,
+) -> std::io::Result<(Vec<String>, u64, bool)> {
+    use std::io::{Read, Seek, SeekFrom};
+    let mut file = open_file_log(&source.path)?;
+    let length = file.metadata()?.len();
+    let bytes = length.min(budget).min(FILE_LOG_SCAN_BYTES_PER_SOURCE);
+    let start = length.saturating_sub(bytes);
+    file.seek(SeekFrom::Start(start))?;
+    let mut content = Vec::with_capacity(bytes as usize);
+    file.take(bytes).read_to_end(&mut content)?;
+    let truncated = start > 0;
+    if truncated {
+        if let Some(index) = content.iter().position(|byte| *byte == b'\n') {
+            content.drain(..=index);
+        } else {
+            content.clear();
+        }
+    }
+    let text = String::from_utf8_lossy(&content);
+    let mut iter = text.lines().rev();
+    let lines = iter
+        .by_ref()
+        .take(line_budget)
+        .map(str::to_owned)
+        .collect::<Vec<_>>();
+    Ok((lines, bytes, truncated || iter.next().is_some()))
+}
+
+fn normalized_record(source: &FileLogSource, line: &str) -> Value {
+    let mut value = serde_json::from_str::<Value>(line).unwrap_or_else(|_| json!({"raw": line}));
+    if !value.is_object() {
+        value = json!({"value": value});
+    }
+    let object = value.as_object_mut().expect("record normalized to object");
+    object.entry("event").or_insert_with(|| {
+        Value::String(
+            if source.kind == "error" {
+                "error"
+            } else {
+                "request"
+            }
+            .into(),
+        )
+    });
+    object.insert("_source".into(), Value::String(source.id.clone()));
+    object.insert("_source_kind".into(), Value::String(source.kind.into()));
+    value
+}
+
+fn contains_ci(value: Option<&Value>, needle: &str) -> bool {
+    value
+        .and_then(Value::as_str)
+        .is_some_and(|text| text.to_ascii_lowercase().contains(needle))
+}
+
+fn record_matches(value: &Value, query: &FileLogQuery, q: &str) -> bool {
+    let Some(object) = value.as_object() else {
+        return false;
+    };
+    if !q.is_empty() && !value.to_string().to_ascii_lowercase().contains(q) {
+        return false;
+    }
+    if let Some(event) = query
+        .event
+        .as_deref()
+        .filter(|v| !v.is_empty() && *v != "all")
+    {
+        if object.get("event").and_then(Value::as_str) != Some(event) {
+            return false;
+        }
+    }
+    if let Some(class) = query
+        .status_class
+        .as_deref()
+        .filter(|v| !v.is_empty() && *v != "all")
+    {
+        let Some(status) = object.get("status").and_then(Value::as_u64) else {
+            return false;
+        };
+        let expected = class
+            .as_bytes()
+            .first()
+            .and_then(|v| v.checked_sub(b'0'))
+            .map(u64::from);
+        if expected != Some(status / 100) {
+            return false;
+        }
+    }
+    if let Some(client) = query.client_ip.as_deref().filter(|v| !v.is_empty()) {
+        if !contains_ci(object.get("client_ip"), &client.to_ascii_lowercase()) {
+            return false;
+        }
+    }
+    if let Some(vhost) = query.vhost.as_deref().filter(|v| !v.is_empty()) {
+        if !contains_ci(object.get("vhost"), &vhost.to_ascii_lowercase()) {
+            return false;
+        }
+    }
+    true
+}
+
+fn validate_file_log_query(query: &FileLogQuery) -> Result<(), ApiError> {
+    let text_ok = |value: &Option<String>, max: usize| {
+        value
+            .as_ref()
+            .is_none_or(|v| v.len() <= max && !v.chars().any(char::is_control))
+    };
+    if !text_ok(&query.source, 192)
+        || !text_ok(&query.q, 256)
+        || !text_ok(&query.client_ip, 128)
+        || !text_ok(&query.vhost, 256)
+        || query.offset.unwrap_or(0) > FILE_LOG_MAX_OFFSET
+        || query
+            .limit
+            .is_some_and(|v| v == 0 || v > FILE_LOG_MAX_LIMIT)
+        || query.event.as_ref().is_some_and(|v| {
+            !matches!(
+                v.as_str(),
+                "" | "all" | "request" | "protocol_error" | "error"
+            )
+        })
+        || query.status_class.as_ref().is_some_and(|v| {
+            !matches!(
+                v.as_str(),
+                "" | "all" | "1xx" | "2xx" | "3xx" | "4xx" | "5xx"
+            )
+        })
+    {
+        return Err(ApiError(
+            StatusCode::BAD_REQUEST,
+            "invalid file log filter".into(),
+        ));
+    }
+    Ok(())
+}
+
+async fn file_logs_query(
+    State(state): State<Arc<App>>,
+    Query(query): Query<FileLogQuery>,
+) -> Result<Json<Value>, ApiError> {
+    validate_file_log_query(&query)?;
+    let config = state.config.load().envelope.config.clone();
+    tokio::task::spawn_blocking(move || {
+        let mut sources = configured_file_log_sources(&config);
+        let requested = query.source.as_deref().unwrap_or("all");
+        if requested != "all" {
+            sources.retain(|source| source.id == requested);
+            if sources.is_empty() {
+                return Err(ApiError(
+                    StatusCode::BAD_REQUEST,
+                    "unknown file log source".into(),
+                ));
+            }
+        } else {
+            sources.sort_by(|a, b| b.modified_ms.cmp(&a.modified_ms));
+            sources.truncate(FILE_LOG_MAX_SOURCES);
+        }
+        let q = query.q.clone().unwrap_or_default().to_ascii_lowercase();
+        let offset = query.offset.unwrap_or(0);
+        let limit = query.limit.unwrap_or(100).clamp(1, FILE_LOG_MAX_LIMIT);
+        let mut remaining_bytes = FILE_LOG_SCAN_BYTES_TOTAL;
+        let mut scanned_bytes = 0u64;
+        let mut remaining_lines = FILE_LOG_SCAN_LINES;
+        let mut scanned_lines = 0usize;
+        let mut truncated = false;
+        let mut records = Vec::new();
+        for (index, source) in sources.iter().enumerate() {
+            if remaining_bytes == 0 || remaining_lines == 0 {
+                truncated = true;
+                break;
+            }
+            let sources_left = sources.len().saturating_sub(index).max(1);
+            let line_budget = (remaining_lines / sources_left).max(1);
+            match read_tail(source, remaining_bytes, line_budget) {
+                Ok((lines, bytes, source_truncated)) => {
+                    remaining_bytes = remaining_bytes.saturating_sub(bytes);
+                    remaining_lines = remaining_lines.saturating_sub(lines.len());
+                    scanned_bytes += bytes;
+                    scanned_lines += lines.len();
+                    truncated |= source_truncated;
+                    for line in lines {
+                        if line.is_empty() {
+                            continue;
+                        }
+                        let value = normalized_record(source, &line);
+                        if record_matches(&value, &query, &q) {
+                            records.push(value);
+                        }
+                    }
+                }
+                Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
+                Err(_) => truncated = true,
+            }
+        }
+        records.sort_by(|a, b| {
+            let time = |value: &Value| value.get("time_ms").and_then(Value::as_u64).unwrap_or(0);
+            time(b).cmp(&time(a))
+        });
+        let total = records.len();
+        let page = records
+            .into_iter()
+            .skip(offset)
+            .take(limit)
+            .collect::<Vec<_>>();
+        Ok(Json(json!({
+            "entries": page,
+            "total": total,
+            "offset": offset,
+            "limit": limit,
+            "truncated": truncated,
+            "scanned_bytes": scanned_bytes,
+            "scanned_lines": scanned_lines,
+            "sources": file_log_sources_json(&sources),
+        })))
+    })
+    .await
+    .map_err(|_| {
+        ApiError(
+            StatusCode::INTERNAL_SERVER_ERROR,
+            "file log query worker failed".into(),
+        )
+    })?
+}