This commit is contained in:
Mateusz Gruszczyński committed 2026-10-31 14:36:07 +01:00
1 parent fc81ecafd6
commit dc57aa7089
89 files changed
+5290 -1551

No files matched your search

+2 -2
View File
@@ -47,7 +47,7 @@ fn extract_cookie_value(req: &Request, cookie_name: &str) -> Option<String> {
};
for segment in header_str.split(';') {
let trimmed = segment.trim();
let prefix = format!("{}=", cookie_name);
let prefix = format!("{cookie_name}=");
if let Some(cookie_value) = trimmed.strip_prefix(&prefix) {
let cookie_value = cookie_value.trim();
if !cookie_value.is_empty() {
@@ -60,7 +60,7 @@ fn extract_cookie_value(req: &Request, cookie_name: &str) -> Option<String> {
}
fn extract_session_cookie(req: &Request, account_id: &str) -> Option<String> {
let cookie_name = format!("rustbox_session_{}", account_id);
let cookie_name = format!("rustbox_session_{account_id}");
extract_cookie_value(req, &cookie_name)
}
+11
View File
@@ -1,4 +1,6 @@
#[cfg(test)]
use std::fs;
#[cfg(test)]
use std::path::PathBuf;
use sha2::{Digest, Sha256};
@@ -11,10 +13,19 @@ pub fn hash_email(email: &str) -> String {
result.iter().map(|b| format!("{b:02x}")).collect()
}
#[cfg(test)]
pub fn provision_user_data(data_dir: &str, user_hash: &str) -> Result<PathBuf, String> {
provision_user_data_with_search(data_dir, user_hash, true)
}
pub fn provision_user_database(data_dir: &str, user_hash: &str) -> Result<(), String> {
let conn = crate::db::pool::open_user_db(data_dir, user_hash)
.map_err(|e| format!("failed to initialize user database: {e}"))?;
drop(conn);
Ok(())
}
#[cfg(test)]
pub fn provision_user_data_with_search(
data_dir: &str,
user_hash: &str,
+425 -18
View File
@@ -7,6 +7,118 @@ use figment::{
use ipnet::IpNet;
use serde::Deserialize;
#[derive(Debug, Deserialize, Clone)]
pub struct StorageConfig {
#[serde(default = "default_storage_backend")]
pub backend: String,
#[serde(default)]
pub path: Option<String>,
#[serde(default)]
pub shared: bool,
#[serde(default)]
pub bucket: Option<String>,
#[serde(default = "default_s3_region")]
pub region: String,
#[serde(default)]
pub endpoint: Option<String>,
#[serde(default)]
pub prefix: String,
#[serde(default)]
pub path_style: bool,
#[serde(default)]
pub allow_http: bool,
#[serde(default)]
pub access_key_id: Option<String>,
#[serde(default)]
pub secret_access_key: Option<String>,
#[serde(default)]
pub session_token: Option<String>,
#[serde(default = "default_storage_request_timeout_secs")]
pub request_timeout_secs: u64,
#[serde(default = "default_storage_connect_timeout_secs")]
pub connect_timeout_secs: u64,
#[serde(default = "default_storage_max_retries")]
pub max_retries: usize,
#[serde(default = "default_storage_retry_timeout_secs")]
pub retry_timeout_secs: u64,
#[serde(default)]
pub server_side_encryption: Option<String>,
#[serde(default)]
pub kms_key_id: Option<String>,
}
impl Default for StorageConfig {
fn default() -> Self {
Self {
backend: default_storage_backend(),
path: None,
shared: false,
bucket: None,
region: default_s3_region(),
endpoint: None,
prefix: String::new(),
path_style: false,
allow_http: false,
access_key_id: None,
secret_access_key: None,
session_token: None,
request_timeout_secs: default_storage_request_timeout_secs(),
connect_timeout_secs: default_storage_connect_timeout_secs(),
max_retries: default_storage_max_retries(),
retry_timeout_secs: default_storage_retry_timeout_secs(),
server_side_encryption: None,
kms_key_id: None,
}
}
}
#[derive(Debug, Deserialize, Clone)]
pub struct SearchConfig {
#[serde(default = "default_search_backend")]
pub backend: String,
#[serde(default)]
pub path: Option<String>,
#[serde(default)]
pub urls: Vec<String>,
#[serde(default = "default_search_index_prefix")]
pub index_prefix: String,
#[serde(default)]
pub username: Option<String>,
#[serde(default)]
pub password: Option<String>,
#[serde(default)]
pub required: bool,
#[serde(default)]
pub allow_insecure_http: bool,
#[serde(default = "default_search_request_timeout_secs")]
pub request_timeout_secs: u64,
#[serde(default = "default_search_worker_interval_secs")]
pub worker_interval_secs: u64,
#[serde(default = "default_search_worker_batch_size")]
pub worker_batch_size: usize,
#[serde(default = "default_search_worker_max_attempts")]
pub worker_max_attempts: u32,
}
impl Default for SearchConfig {
fn default() -> Self {
Self {
backend: default_search_backend(),
path: None,
urls: Vec::new(),
index_prefix: default_search_index_prefix(),
username: None,
password: None,
required: false,
allow_insecure_http: false,
request_timeout_secs: default_search_request_timeout_secs(),
worker_interval_secs: default_search_worker_interval_secs(),
worker_batch_size: default_search_worker_batch_size(),
worker_max_attempts: default_search_worker_max_attempts(),
}
}
}
#[derive(Debug, Deserialize, Clone)]
#[allow(dead_code)]
pub struct AppConfig {
@@ -80,10 +192,10 @@ pub struct AppConfig {
pub cluster_mode: bool,
#[serde(default)]
pub cluster_shared_data_dir: bool,
pub storage: StorageConfig,
#[serde(default = "default_search_index_backend")]
pub search_index_backend: String,
#[serde(default)]
pub search: SearchConfig,
#[serde(default = "default_session_timeout_hours")]
pub session_timeout_hours: u64,
@@ -211,6 +323,46 @@ fn default_data_dir() -> String {
"/data".to_string()
}
fn default_s3_region() -> String {
"us-east-1".to_string()
}
fn default_storage_request_timeout_secs() -> u64 {
30
}
fn default_storage_connect_timeout_secs() -> u64 {
5
}
fn default_storage_max_retries() -> usize {
3
}
fn default_storage_retry_timeout_secs() -> u64 {
60
}
fn default_search_index_prefix() -> String {
"rustbox".to_string()
}
fn default_search_request_timeout_secs() -> u64 {
10
}
fn default_search_worker_interval_secs() -> u64 {
2
}
fn default_search_worker_batch_size() -> usize {
100
}
fn default_search_worker_max_attempts() -> u32 {
12
}
fn default_log_level() -> String {
"info".to_string()
}
@@ -223,8 +375,12 @@ fn default_session_backend() -> String {
"database".to_string()
}
fn default_search_index_backend() -> String {
"tantivy".to_string()
fn default_storage_backend() -> String {
"filesystem".to_string()
}
fn default_search_backend() -> String {
"local_tantivy".to_string()
}
fn default_session_timeout_hours() -> u64 {
@@ -305,8 +461,8 @@ impl Default for AppConfig {
redis_password: None,
redis_sentinel_password: None,
cluster_mode: false,
cluster_shared_data_dir: false,
search_index_backend: default_search_index_backend(),
storage: StorageConfig::default(),
search: SearchConfig::default(),
session_timeout_hours: default_session_timeout_hours(),
remember_me_enabled: default_remember_me_enabled(),
remember_me_days: default_remember_me_days(),
@@ -450,7 +606,43 @@ impl AppConfig {
figment = figment.merge(Toml::file(path));
}
figment.merge(Env::raw()).extract()
let figment = figment.merge(Env::raw().split("__"));
let data_dir_explicit = figment.find_value("data_dir").is_ok();
let mut config: Self = figment.extract()?;
config.apply_cluster_data_dir_default(data_dir_explicit);
Ok(config)
}
fn apply_cluster_data_dir_default(&mut self, data_dir_explicit: bool) {
if self.cluster_mode && !data_dir_explicit {
self.data_dir.clear();
}
}
pub fn effective_storage_backend(&self) -> String {
self.storage.backend.trim().to_ascii_lowercase()
}
pub fn effective_search_backend(&self) -> String {
self.search.backend.trim().to_ascii_lowercase()
}
pub fn effective_storage_path(&self) -> &str {
self.storage
.path
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
.unwrap_or(&self.data_dir)
}
pub fn effective_search_path(&self) -> &str {
self.search
.path
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
.unwrap_or(&self.data_dir)
}
pub fn validate(&self) -> Result<(), String> {
@@ -493,10 +685,7 @@ impl AppConfig {
if self.max_attachment_size_mb > self.max_message_size_mb {
return Err("MAX_ATTACHMENT_SIZE_MB cannot exceed MAX_MESSAGE_SIZE_MB".to_string());
}
if !matches!(
self.database_backend.as_str(),
"sqlite" | "mysql" | "pgsql" | "postgres" | "postgresql"
) {
if !matches!(self.database_backend.as_str(), "sqlite" | "mysql" | "pgsql") {
return Err("DATABASE_BACKEND must be one of: sqlite, mysql, pgsql".to_string());
}
if self.database_backend != "sqlite"
@@ -530,8 +719,133 @@ impl AppConfig {
return Err("Redis sessions require REDIS_URL or REDIS_SENTINEL_NODES + REDIS_SENTINEL_MASTER".to_string());
}
}
if !matches!(self.search_index_backend.as_str(), "tantivy" | "disabled") {
return Err("SEARCH_INDEX_BACKEND must be one of: tantivy, disabled".to_string());
let storage_backend = self.effective_storage_backend();
if !matches!(storage_backend.as_str(), "filesystem" | "s3") {
return Err("storage.backend must be one of: filesystem, s3".to_string());
}
if storage_backend == "filesystem" {
let path = self.effective_storage_path();
if path.trim().is_empty() {
return Err(
"storage.path/data_dir cannot be empty for filesystem storage".to_string(),
);
}
if self.cluster_mode && !std::path::Path::new(path).is_absolute() {
return Err("cluster filesystem storage.path must be an absolute path".to_string());
}
} else {
if self
.storage
.bucket
.as_deref()
.map(str::trim)
.filter(|v| !v.is_empty())
.is_none()
{
return Err("storage.bucket is required when storage.backend=s3".to_string());
}
if self.storage.region.trim().is_empty() {
return Err("storage.region cannot be empty".to_string());
}
if let Some(endpoint) = self
.storage
.endpoint
.as_deref()
.map(str::trim)
.filter(|v| !v.is_empty())
{
let url = url::Url::parse(endpoint)
.map_err(|e| format!("invalid storage.endpoint: {e}"))?;
if url.scheme() != "https" && !(self.storage.allow_http && url.scheme() == "http") {
return Err(
"storage.endpoint must use HTTPS unless storage.allow_http=true"
.to_string(),
);
}
}
}
if self.storage.request_timeout_secs == 0
|| self.storage.connect_timeout_secs == 0
|| self.storage.retry_timeout_secs == 0
{
return Err("storage timeouts must be greater than zero".to_string());
}
if let Some(mode) = self
.storage
.server_side_encryption
.as_deref()
.map(str::trim)
.filter(|v| !v.is_empty())
{
if !matches!(mode, "AES256" | "aws:kms") {
return Err("storage.server_side_encryption must be AES256 or aws:kms".to_string());
}
if mode == "aws:kms"
&& self
.storage
.kms_key_id
.as_deref()
.map(str::trim)
.filter(|v| !v.is_empty())
.is_none()
{
return Err("storage.kms_key_id is required for aws:kms encryption".to_string());
}
}
let search_backend = self.effective_search_backend();
if !matches!(
search_backend.as_str(),
"local_tantivy" | "disabled" | "opensearch"
) {
return Err(
"search.backend must be one of: disabled, local_tantivy, opensearch".to_string(),
);
}
if search_backend == "local_tantivy" {
let path = self.effective_search_path();
if path.trim().is_empty() {
return Err("search.path/data_dir cannot be empty for local_tantivy".to_string());
}
}
if search_backend == "opensearch" {
if self.search.urls.is_empty() {
return Err("search.urls must contain at least one URL for opensearch".to_string());
}
for raw in &self.search.urls {
let parsed = url::Url::parse(raw.trim())
.map_err(|e| format!("invalid OpenSearch URL '{raw}': {e}"))?;
if parsed.scheme() != "https"
&& !(self.search.allow_insecure_http && parsed.scheme() == "http")
{
return Err(
"OpenSearch URLs must use HTTPS unless search.allow_insecure_http=true"
.to_string(),
);
}
if !parsed.username().is_empty() || parsed.password().is_some() {
return Err("OpenSearch credentials must not be embedded in URLs".to_string());
}
}
let prefix = self.search.index_prefix.as_str();
if prefix.is_empty()
|| !prefix
.chars()
.next()
.is_some_and(|c| c.is_ascii_lowercase() || c.is_ascii_digit())
|| !prefix
.chars()
.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || matches!(c, '-' | '_'))
{
return Err("search.index_prefix must start with a lowercase ASCII letter/digit and contain only lowercase ASCII letters, digits, '-' or '_'".to_string());
}
}
if self.search.request_timeout_secs == 0
|| self.search.worker_interval_secs == 0
|| self.search.worker_batch_size == 0
|| self.search.worker_max_attempts == 0
{
return Err("search timeout/worker settings must be greater than zero".to_string());
}
if self.cluster_mode && self.database_backend == "sqlite" {
return Err(
@@ -539,15 +853,17 @@ impl AppConfig {
.to_string(),
);
}
if self.cluster_mode && !self.cluster_shared_data_dir {
let storage_cluster_safe =
storage_backend == "s3" || (storage_backend == "filesystem" && self.storage.shared);
if self.cluster_mode && !storage_cluster_safe {
return Err(
"CLUSTER_MODE requires CLUSTER_SHARED_DATA_DIR=true after data_dir is mounted as shared storage on every node; draft attachments are stored below data_dir"
"CLUSTER_MODE requires cluster-safe blob storage: storage.backend=s3 or storage.backend=filesystem with storage.shared=true"
.to_string(),
);
}
if self.cluster_mode && self.search_index_backend != "disabled" {
if self.cluster_mode && search_backend == "local_tantivy" {
return Err(
"CLUSTER_MODE requires SEARCH_INDEX_BACKEND=disabled; the built-in Tantivy index is local and does not coordinate writers across nodes"
"CLUSTER_MODE forbids search.backend=local_tantivy; use disabled or an external indexer"
.to_string(),
);
}
@@ -669,6 +985,8 @@ mod tests {
assert!(config.redis_sentinel_master.is_none());
assert!(config.redis_sentinel_password.is_none());
assert!(!config.cluster_mode);
assert_eq!(config.storage.backend, "filesystem");
assert_eq!(config.search.backend, "local_tantivy");
assert_eq!(config.session_timeout_hours, 24);
assert!(config.remember_me_enabled);
assert_eq!(config.remember_me_days, 30);
@@ -869,6 +1187,95 @@ mod tests {
);
}
#[test]
fn single_node_pgsql_can_still_use_data_dir_backends() {
let config = AppConfig {
imap_host: Some("imap.example.com".to_string()),
database_backend: "pgsql".to_string(),
database_url: Some("postgres://rustbox:secret@db/rustbox".to_string()),
session_encryption_key: Some("shared-key".to_string()),
data_dir: "/srv/rustbox".to_string(),
..AppConfig::default()
};
assert!(config.validate().is_ok());
assert_eq!(config.effective_storage_backend(), "filesystem");
assert_eq!(config.effective_search_backend(), "local_tantivy");
}
#[test]
fn cluster_mode_disables_implicit_data_dir_but_preserves_explicit_value() {
let mut implicit = AppConfig {
cluster_mode: true,
data_dir: "/data".to_string(),
..AppConfig::default()
};
implicit.apply_cluster_data_dir_default(false);
assert!(implicit.data_dir.is_empty());
let mut explicit = AppConfig {
cluster_mode: true,
data_dir: "/mnt/shared-rustbox".to_string(),
..AppConfig::default()
};
explicit.apply_cluster_data_dir_default(true);
assert_eq!(explicit.data_dir, "/mnt/shared-rustbox");
}
#[test]
fn cluster_with_s3_and_opensearch_does_not_require_data_dir() {
let mut config = AppConfig {
imap_host: Some("imap.example.com".to_string()),
database_backend: "pgsql".to_string(),
database_url: Some("postgres://rustbox:secret@db/rustbox".to_string()),
session_encryption_key: Some("shared-key".to_string()),
cluster_mode: true,
pgp_enabled: false,
data_dir: String::new(),
..AppConfig::default()
};
config.storage.backend = "s3".to_string();
config.storage.bucket = Some("rustbox".to_string());
config.storage.endpoint = Some("https://s3.example.com".to_string());
config.search.backend = "opensearch".to_string();
config.search.urls = vec!["https://search.example.com".to_string()];
assert!(config.validate().is_ok());
}
#[test]
fn cluster_accepts_shared_posix_storage_without_s3() {
let mut config = AppConfig {
imap_host: Some("imap.example.com".to_string()),
database_backend: "mysql".to_string(),
database_url: Some("mysql://rustbox:secret@db/rustbox".to_string()),
session_encryption_key: Some("shared-key".to_string()),
cluster_mode: true,
pgp_enabled: false,
..AppConfig::default()
};
config.storage.backend = "filesystem".to_string();
config.storage.path = Some("/mnt/rustbox-shared".to_string());
config.storage.shared = true;
config.search.backend = "disabled".to_string();
assert!(config.validate().is_ok());
}
#[test]
fn pre_0_2_backend_aliases_are_rejected() {
let mut config = AppConfig {
imap_host: Some("imap.example.com".to_string()),
..AppConfig::default()
};
config.search.backend = "tantivy".to_string();
assert!(config.validate().unwrap_err().contains("search.backend"));
config.search.backend = "local_tantivy".to_string();
for backend in ["postgres", "postgresql"] {
config.database_backend = backend.to_string();
assert!(config.validate().unwrap_err().contains("DATABASE_BACKEND"));
}
}
#[test]
fn cluster_mode_rejects_local_sqlite_even_with_redis_sessions() {
let config = AppConfig {
+1 -1
View File
@@ -491,7 +491,7 @@ pub fn update_preferences(
set_bool!(attachment_reminder, "attachment_reminder");
set_bool!(deep_index, "deep_index");
if let Some(ref value) = data.animation_mode {
sets.push(format!("animation_mode = ?{}", idx));
sets.push(format!("animation_mode = ?{idx}"));
values.push(value.to_db_value());
idx += 1;
}
+85 -40
View File
@@ -33,7 +33,8 @@ pub struct DraftAttachment {
pub filename: String,
pub content_type: String,
pub size: i64,
pub file_path: String,
pub storage_key: String,
pub checksum_sha256: Option<String>,
pub created_at: String,
}
@@ -196,18 +197,31 @@ pub fn delete_draft(conn: &Connection, id: &str) -> Result<bool, String> {
Ok(deleted > 0)
}
pub struct NewDraftAttachment<'a> {
pub id: &'a str,
pub draft_id: &'a str,
pub filename: &'a str,
pub content_type: &'a str,
pub size: i64,
pub storage_key: &'a str,
pub checksum_sha256: Option<&'a str>,
}
pub fn add_draft_attachment(
conn: &Connection,
id: &str,
draft_id: &str,
filename: &str,
content_type: &str,
size: i64,
file_path: &str,
attachment: &NewDraftAttachment<'_>,
) -> Result<(), String> {
conn.execute(
crate::dbq!(db_drafts::Q_006),
params![id, draft_id, filename, content_type, size, file_path],
params![
attachment.id,
attachment.draft_id,
attachment.filename,
attachment.content_type,
attachment.size,
attachment.storage_key,
attachment.checksum_sha256
],
)
.map_err(|e| format!("Failed to add draft attachment: {e}"))?;
Ok(())
@@ -229,8 +243,9 @@ pub fn get_draft_attachments(
filename: row.get(2)?,
content_type: row.get(3)?,
size: row.get(4)?,
file_path: row.get(5)?,
created_at: row.get(6)?,
storage_key: row.get(5)?,
checksum_sha256: row.get(6)?,
created_at: row.get(7)?,
})
})
.map_err(|e| format!("Failed to query draft attachments: {e}"))?;
@@ -249,6 +264,21 @@ pub fn delete_draft_attachment(conn: &Connection, id: &str) -> Result<bool, Stri
Ok(deleted > 0)
}
pub fn update_attachment_storage(
conn: &Connection,
id: &str,
storage_key: &str,
checksum_sha256: Option<&str>,
) -> Result<bool, String> {
let updated = conn
.execute(
crate::dbq!(db_drafts::Q_009),
params![storage_key, checksum_sha256, id],
)
.map_err(|e| format!("Failed to update draft attachment storage: {e}"))?;
Ok(updated > 0)
}
#[cfg(test)]
mod tests {
use super::*;
@@ -431,23 +461,29 @@ mod tests {
add_draft_attachment(
&conn,
"att-1",
"draft-1",
"file.pdf",
"application/pdf",
1024,
"/data/abc/attachments/draft-1/att-1",
&NewDraftAttachment {
id: "att-1",
draft_id: "draft-1",
filename: "file.pdf",
content_type: "application/pdf",
size: 1024,
storage_key: "/data/abc/attachments/draft-1/att-1",
checksum_sha256: None,
},
)
.unwrap();
add_draft_attachment(
&conn,
"att-2",
"draft-1",
"photo.jpg",
"image/jpeg",
2048,
"/data/abc/attachments/draft-1/att-2",
&NewDraftAttachment {
id: "att-2",
draft_id: "draft-1",
filename: "photo.jpg",
content_type: "image/jpeg",
size: 2048,
storage_key: "/data/abc/attachments/draft-1/att-2",
checksum_sha256: None,
},
)
.unwrap();
@@ -469,12 +505,15 @@ mod tests {
.unwrap();
add_draft_attachment(
&conn,
"att-1",
"draft-1",
"file.pdf",
"application/pdf",
100,
"/path",
&NewDraftAttachment {
id: "att-1",
draft_id: "draft-1",
filename: "file.pdf",
content_type: "application/pdf",
size: 100,
storage_key: "/path",
checksum_sha256: None,
},
)
.unwrap();
@@ -496,22 +535,28 @@ mod tests {
.unwrap();
add_draft_attachment(
&conn,
"att-1",
"draft-1",
"file.pdf",
"application/pdf",
100,
"/path",
&NewDraftAttachment {
id: "att-1",
draft_id: "draft-1",
filename: "file.pdf",
content_type: "application/pdf",
size: 100,
storage_key: "/path",
checksum_sha256: None,
},
)
.unwrap();
add_draft_attachment(
&conn,
"att-2",
"draft-1",
"pic.jpg",
"image/jpeg",
200,
"/path2",
&NewDraftAttachment {
id: "att-2",
draft_id: "draft-1",
filename: "pic.jpg",
content_type: "image/jpeg",
size: 200,
storage_key: "/path2",
checksum_sha256: None,
},
)
.unwrap();
+1 -1
View File
@@ -232,7 +232,7 @@ pub fn update_identity(
boolean!(signature_forward, "signature_forward");
text!(signature_position, "signature_position");
if let Some(ref value) = data.pgp_fingerprint {
sets.push(format!("pgp_fingerprint = ?{}", idx));
sets.push(format!("pgp_fingerprint = ?{idx}"));
values.push(value.to_db_value());
idx += 1;
}
+15 -11
View File
@@ -1,5 +1,5 @@
use crate::db::pool::{
Connection, DatabaseBackend, DbValue, ToDbValue, configured_backend, params,
Connection, DatabaseBackend, DbValue, ToDbValue, Transaction, configured_backend, params,
};
use crate::queries::sqlite::queries::db_messages::MSG_SELECT_COLS;
use serde::Serialize;
@@ -108,11 +108,20 @@ pub fn upsert_message(
Ok(())
}
pub fn remove_after_mutation_in_transaction(
transaction: &Transaction,
folder: &str,
uid: u32,
) -> Result<(), String> {
delete_message(transaction, folder, uid)?;
super::header_sync::invalidate(transaction)?;
super::folders::invalidate_folder_freshness(transaction, folder)?;
Ok(())
}
pub fn remove_after_mutation(conn: &Connection, folder: &str, uid: u32) -> Result<(), String> {
let transaction = conn.unchecked_transaction().map_err(|e| e.to_string())?;
delete_message(&transaction, folder, uid)?;
super::header_sync::invalidate(&transaction)?;
super::folders::invalidate_folder_freshness(&transaction, folder)?;
remove_after_mutation_in_transaction(&transaction, folder, uid)?;
transaction.commit().map_err(|e| e.to_string())
}
@@ -495,7 +504,7 @@ pub fn get_thread_messages(
.replace('!', "!!")
.replace("%", "!%")
.replace("_", "!_");
let like_pattern = format!("%{}%", escaped_target);
let like_pattern = format!("%{escaped_target}%");
let sql = crate::dbq!(db_messages::FORMAT_004).replace("{MSG_SELECT_COLS}", MSG_SELECT_COLS);
let mut stmt = conn
@@ -815,12 +824,7 @@ mod tests {
ensure_folder(&conn, "INBOX");
for uid in 1..=5 {
insert_sample(
&conn,
"INBOX",
uid,
&format!("2024-01-{:02}T10:00:00Z", uid),
);
insert_sample(&conn, "INBOX", uid, &format!("2024-01-{uid:02}T10:00:00Z"));
}
let page0 = get_messages(&conn, "INBOX", 0, 2).unwrap();
+1
View File
@@ -13,6 +13,7 @@ pub mod notification_preferences;
pub mod pgp;
pub mod pool;
pub mod response_templates;
pub mod search_outbox;
pub mod special_folders;
pub mod tags;
pub mod trusted_senders;
+78 -19
View File
@@ -20,10 +20,20 @@ const SQLITE_SCHEMA: &str = include_str!("../../migrations/sqlite/schema.sql");
const MYSQL_SCHEMA: &str = include_str!("../../migrations/mysql/schema.sql");
const PGSQL_SCHEMA: &str = include_str!("../../migrations/pgsql/schema.sql");
// Future migrations start at V047. Each entry must exist for every supported engine.
const SQLITE_MIGRATIONS: &[(u32, &str)] = &[];
const MYSQL_MIGRATIONS: &[(u32, &str)] = &[];
const PGSQL_MIGRATIONS: &[(u32, &str)] = &[];
// The squashed legacy baseline is V046. Runtime migrations start at V047 and
// are stored as real files for every supported SQL engine.
const SQLITE_MIGRATIONS: &[(u32, &str)] = &[(
47,
include_str!("../../migrations/sqlite/V047__blob_keys_and_search_outbox.sql"),
)];
const MYSQL_MIGRATIONS: &[(u32, &str)] = &[(
47,
include_str!("../../migrations/mysql/V047__blob_keys_and_search_outbox.sql"),
)];
const PGSQL_MIGRATIONS: &[(u32, &str)] = &[(
47,
include_str!("../../migrations/pgsql/V047__blob_keys_and_search_outbox.sql"),
)];
const TENANT_TABLES: &[&str] = &[
"calendar_events",
@@ -48,6 +58,7 @@ const TENANT_TABLES: &[&str] = &[
"pgp_preferences",
"response_templates",
"saved_searches",
"search_outbox",
"special_folders",
"tags",
"trusted_senders",
@@ -79,6 +90,7 @@ const IMPORT_TABLES: &[&str] = &[
"pgp_preferences",
"response_templates",
"saved_searches",
"search_outbox",
"special_folders",
"trusted_senders",
"user_meta",
@@ -96,7 +108,7 @@ impl DatabaseBackend {
match value.trim().to_ascii_lowercase().as_str() {
"sqlite" => Ok(Self::Sqlite),
"mysql" => Ok(Self::Mysql),
"pgsql" | "postgres" | "postgresql" => Ok(Self::Pgsql),
"pgsql" => Ok(Self::Pgsql),
other => Err(format!("unsupported database backend: {other}")),
}
}
@@ -291,6 +303,46 @@ pub fn configured_backend() -> DatabaseBackend {
}
}
pub fn list_account_user_hashes() -> Result<Vec<String>, String> {
match DATABASE.get() {
Some(ConfiguredDatabase::Sqlite { path }) => {
let conn = rusqlite::Connection::open(path)
.map_err(|e| format!("failed to open SQLite account registry: {e}"))?;
let mut stmt = conn
.prepare("SELECT user_hash FROM rustbox_accounts ORDER BY user_hash")
.map_err(|e| format!("failed to prepare account registry query: {e}"))?;
let rows = stmt
.query_map([], |row| row.get::<_, String>(0))
.map_err(|e| format!("failed to query account registry: {e}"))?;
rows.map(|row| row.map_err(|e| format!("failed to read account registry row: {e}")))
.collect()
}
Some(ConfiguredDatabase::Mysql { pool, runtime }) => {
let pool = pool.clone();
runtime_wait(runtime, async move {
sqlx::query_scalar::<_, String>(
"SELECT user_hash FROM rustbox_accounts ORDER BY user_hash",
)
.fetch_all(&pool)
.await
.map_err(|e| format!("failed to query MySQL account registry: {e}"))
})
}
Some(ConfiguredDatabase::Pgsql { pool, runtime }) => {
let pool = pool.clone();
runtime_wait(runtime, async move {
sqlx::query_scalar::<_, String>(
"SELECT user_hash FROM rustbox_accounts ORDER BY user_hash",
)
.fetch_all(&pool)
.await
.map_err(|e| format!("failed to query PostgreSQL account registry: {e}"))
})
}
None => Err("database is not configured".to_string()),
}
}
#[macro_export]
macro_rules! dbq {
($module:ident::$name:ident) => {{
@@ -1562,12 +1614,16 @@ fn read_legacy_sqlite(path: &Path) -> Result<Vec<LegacyTableData>, String> {
.map_err(|e| format!("failed to read legacy SQLite schema version: {e}"))?;
if version < BASE_SCHEMA_VERSION {
return Err(format!(
"legacy SQLite schema version {version} is older than the RustBox 0.1.2 baseline {BASE_SCHEMA_VERSION}; upgrade with RustBox 0.1.0 first"
"legacy SQLite schema version {version} is older than the legacy RustBox baseline {BASE_SCHEMA_VERSION}; upgrade with RustBox 0.1.0 first"
));
}
if version > BASE_SCHEMA_VERSION {
let latest_supported_sqlite = SQLITE_MIGRATIONS
.last()
.map(|(migration_version, _)| *migration_version)
.unwrap_or(BASE_SCHEMA_VERSION);
if version > latest_supported_sqlite {
return Err(format!(
"legacy SQLite schema version {version} is newer than the RustBox 0.1.2 baseline {BASE_SCHEMA_VERSION}"
"legacy SQLite schema version {version} is newer than the latest SQLite schema supported by this build ({latest_supported_sqlite})"
));
}
let mut output = Vec::new();
@@ -1904,7 +1960,7 @@ fn ensure_sqlite_tenant(
.map_err(|e| format!("failed to initialize SQLite tenant defaults: {e}"))?;
tx.execute(
"INSERT INTO rustbox_accounts(user_hash, table_prefix, schema_version) VALUES(?1, ?2, ?3)",
rusqlite::params![user_hash, prefix, BASE_SCHEMA_VERSION],
rusqlite::params![user_hash, prefix, latest_schema_version()],
)
.map_err(|e| format!("failed to register SQLite account schema: {e}"))?;
tx.commit().map_err(|e| e.to_string())?;
@@ -1922,7 +1978,7 @@ fn apply_sqlite_migrations(
) -> Result<(), String> {
if current < BASE_SCHEMA_VERSION {
return Err(format!(
"database schema version {current} is older than the RustBox 0.1.2 baseline {BASE_SCHEMA_VERSION}; upgrade with RustBox 0.1.0 first"
"database schema version {current} is older than the legacy RustBox baseline {BASE_SCHEMA_VERSION}; upgrade with RustBox 0.1.0 first"
));
}
let latest = latest_schema_version();
@@ -1967,7 +2023,8 @@ fn ensure_server_tenant(
user_hash: &str,
prefix: &str,
) -> Result<(), String> {
let legacy_path = Path::new(data_dir).join(user_hash).join("db.sqlite");
let legacy_path = (!data_dir.trim().is_empty())
.then(|| Path::new(data_dir).join(user_hash).join("db.sqlite"));
let imported_legacy = match configured {
ConfiguredDatabase::Mysql { pool, runtime } => {
@@ -2015,8 +2072,8 @@ fn ensure_server_tenant(
.bind(&prefix).fetch_optional(&pool).await.map_err(|e| e.to_string())?;
if collision.is_some() { return Err("database account-prefix collision detected".into()); }
let legacy_data = if legacy_path.is_file() {
read_legacy_sqlite(&legacy_path)?
let legacy_data = if let Some(path) = legacy_path.as_ref().filter(|path| path.is_file()) {
read_legacy_sqlite(path)?
} else {
Vec::new()
};
@@ -2035,7 +2092,7 @@ fn ensure_server_tenant(
return Err(format!("failed to initialize MySQL tenant defaults: {error}"));
}
if let Err(error) = sqlx::query("INSERT INTO rustbox_accounts(user_hash, table_prefix, schema_version) VALUES(?, ?, ?)")
.bind(&user_hash).bind(&prefix).bind(i64::from(BASE_SCHEMA_VERSION)).execute(&pool).await
.bind(&user_hash).bind(&prefix).bind(i64::from(latest_schema_version())).execute(&pool).await
{
drop_mysql_tenant(&pool, &prefix).await;
return Err(format!("failed to register MySQL account schema: {error}"));
@@ -2090,8 +2147,8 @@ fn ensure_server_tenant(
.bind(&prefix).fetch_optional(&pool).await.map_err(|e| e.to_string())?;
if collision.is_some() { return Err("database account-prefix collision detected".into()); }
let legacy_data = if legacy_path.is_file() {
read_legacy_sqlite(&legacy_path)?
let legacy_data = if let Some(path) = legacy_path.as_ref().filter(|path| path.is_file()) {
read_legacy_sqlite(path)?
} else {
Vec::new()
};
@@ -2110,7 +2167,7 @@ fn ensure_server_tenant(
return Err(format!("failed to initialize PostgreSQL tenant defaults: {error}"));
}
if let Err(error) = sqlx::query("INSERT INTO rustbox_accounts(user_hash, table_prefix, schema_version) VALUES($1, $2, $3)")
.bind(&user_hash).bind(&prefix).bind(i64::from(BASE_SCHEMA_VERSION)).execute(&pool).await
.bind(&user_hash).bind(&prefix).bind(i64::from(latest_schema_version())).execute(&pool).await
{
drop_pgsql_tenant(&pool, &prefix).await;
return Err(format!("failed to register PostgreSQL account schema: {error}"));
@@ -2128,7 +2185,9 @@ fn ensure_server_tenant(
};
if imported_legacy {
backup_legacy_sqlite(&legacy_path);
if let Some(path) = legacy_path.as_ref() {
backup_legacy_sqlite(path);
}
}
Ok(())
}
@@ -2215,7 +2274,7 @@ pub fn open_test_db() -> Connection {
.unwrap();
raw.execute(
"INSERT INTO rustbox_accounts(user_hash, table_prefix, schema_version) VALUES(?1, ?2, ?3)",
rusqlite::params![user_hash, prefix, BASE_SCHEMA_VERSION],
rusqlite::params![user_hash, prefix, latest_schema_version()],
)
.unwrap();
Connection {
+258
View File
@@ -0,0 +1,258 @@
use std::time::{SystemTime, UNIX_EPOCH};
use crate::db::pool::{Connection, params};
#[derive(Debug, Clone)]
pub struct OutboxEvent {
pub id: String,
pub generation: String,
pub event_type: String,
pub message_uid: u32,
pub folder: String,
pub payload: Option<String>,
pub attempts: u32,
}
#[derive(Debug, Clone, Default)]
pub struct OutboxStats {
pub total: usize,
pub dead_letters: usize,
pub lag_seconds: u64,
}
pub fn now_epoch() -> i64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
.min(i64::MAX as u64) as i64
}
pub fn enqueue(
conn: &Connection,
id: &str,
event_type: &str,
message_uid: u32,
folder: &str,
payload: Option<&str>,
) -> Result<String, String> {
let generation = uuid::Uuid::new_v4().to_string();
let now = now_epoch();
let result = match crate::db::pool::configured_backend() {
crate::db::pool::DatabaseBackend::Mysql => conn.execute(
crate::dbq!(db_search_outbox::Q_001),
params![
id,
&generation,
event_type,
message_uid,
folder,
payload,
now,
now
],
),
_ => conn.execute(
crate::dbq!(db_search_outbox::Q_001),
params![
id,
&generation,
event_type,
message_uid,
folder,
payload,
now
],
),
};
result.map_err(|e| format!("failed to enqueue search event: {e}"))?;
Ok(generation)
}
pub fn list_ready(conn: &Connection, limit: usize) -> Result<Vec<OutboxEvent>, String> {
let now = now_epoch();
let mut stmt = conn
.prepare(crate::dbq!(db_search_outbox::Q_002))
.map_err(|e| format!("failed to prepare search outbox query: {e}"))?;
let ready_params = match crate::db::pool::configured_backend() {
crate::db::pool::DatabaseBackend::Mysql => params![now, now, limit],
_ => params![now, limit],
};
let rows = stmt
.query_map(ready_params, |row| {
Ok(OutboxEvent {
id: row.get(0)?,
generation: row.get(1)?,
event_type: row.get(2)?,
message_uid: row.get::<_, i64>(3)?.max(0) as u32,
folder: row.get(4)?,
payload: row.get(5)?,
attempts: row.get::<_, i64>(6)?.max(0) as u32,
})
})
.map_err(|e| format!("failed to query search outbox: {e}"))?;
rows.map(|row| row.map_err(|e| format!("failed to read search outbox row: {e}")))
.collect()
}
pub fn claim(conn: &Connection, event: &OutboxEvent, lease_secs: u64) -> Result<bool, String> {
let now = now_epoch();
let lease_until = now.saturating_add(lease_secs.max(1) as i64);
let changed = match crate::db::pool::configured_backend() {
crate::db::pool::DatabaseBackend::Mysql => conn.execute(
crate::dbq!(db_search_outbox::Q_006),
params![lease_until, &event.id, &event.generation, now, now],
),
_ => conn.execute(
crate::dbq!(db_search_outbox::Q_006),
params![&event.id, &event.generation, lease_until, now],
),
}
.map_err(|e| format!("failed to claim search event: {e}"))?;
Ok(changed > 0)
}
pub fn mark_success(conn: &Connection, event: &OutboxEvent) -> Result<bool, String> {
let changed = conn
.execute(
crate::dbq!(db_search_outbox::Q_003),
params![&event.id, &event.generation],
)
.map_err(|e| format!("failed to acknowledge search event: {e}"))?;
Ok(changed > 0)
}
pub fn mark_failure(
conn: &Connection,
event: &OutboxEvent,
error: &str,
max_attempts: u32,
) -> Result<bool, String> {
let attempts = event.attempts.saturating_add(1);
let dead = attempts >= max_attempts;
let backoff = 2u64.saturating_pow(attempts.min(10)).min(900);
let available_at = now_epoch().saturating_add(backoff as i64);
let now = now_epoch();
let truncated: String = error.chars().take(2048).collect();
let changed = match crate::db::pool::configured_backend() {
crate::db::pool::DatabaseBackend::Mysql => conn.execute(
crate::dbq!(db_search_outbox::Q_004),
params![
attempts,
available_at,
dead,
&truncated,
now,
&event.id,
&event.generation
],
),
_ => conn.execute(
crate::dbq!(db_search_outbox::Q_004),
params![
&event.id,
&event.generation,
attempts,
available_at,
dead,
&truncated,
now
],
),
}
.map_err(|e| format!("failed to update search retry state: {e}"))?;
Ok(changed > 0)
}
pub fn clear_before(conn: &Connection, updated_before: i64) -> Result<usize, String> {
conn.execute(
crate::dbq!(db_search_outbox::Q_007),
params![updated_before],
)
.map_err(|e| format!("failed to clear superseded search events: {e}"))
}
pub fn stats(conn: &Connection) -> Result<OutboxStats, String> {
let (total, dead, oldest): (i64, i64, i64) = conn
.query_row(crate::dbq!(db_search_outbox::Q_005), [], |row| {
Ok((row.get(0)?, row.get(1)?, row.get(2)?))
})
.map_err(|e| format!("failed to read search outbox stats: {e}"))?;
let now = now_epoch();
Ok(OutboxStats {
total: total.max(0) as usize,
dead_letters: dead.max(0) as usize,
lag_seconds: if oldest > 0 {
now.saturating_sub(oldest) as u64
} else {
0
},
})
}
#[cfg(test)]
mod tests {
use super::*;
use crate::db::pool::open_test_db;
#[test]
fn enqueue_is_idempotent_per_document_and_generation_safe() {
let conn = open_test_db();
let first =
enqueue(&conn, "doc-1", "upsert", 7, "INBOX", Some("{}")).expect("first enqueue");
let second = enqueue(&conn, "doc-1", "delete", 7, "INBOX", None).expect("second enqueue");
assert_ne!(first, second);
let events = list_ready(&conn, 10).unwrap();
assert_eq!(events.len(), 1);
assert_eq!(events[0].event_type, "delete");
let stale = OutboxEvent {
generation: first,
..events[0].clone()
};
assert!(!mark_success(&conn, &stale).unwrap());
assert!(mark_success(&conn, &events[0]).unwrap());
}
#[test]
fn enqueue_participates_in_caller_transaction() {
let conn = open_test_db();
{
let transaction = conn.unchecked_transaction().unwrap();
enqueue(&transaction, "doc-tx", "delete", 11, "INBOX", None).unwrap();
// Drop without commit: the message mutation and its search event must roll back together.
}
assert!(list_ready(&conn, 10).unwrap().is_empty());
}
#[test]
fn event_claim_is_exclusive_until_lease_expires() {
let conn = open_test_db();
enqueue(&conn, "doc-claim", "delete", 3, "INBOX", None).unwrap();
let event = list_ready(&conn, 1).unwrap().remove(0);
assert!(claim(&conn, &event, 60).unwrap());
assert!(!claim(&conn, &event, 60).unwrap());
assert!(list_ready(&conn, 10).unwrap().is_empty());
}
#[test]
fn reenqueue_preserves_active_claim_lease() {
let conn = open_test_db();
enqueue(&conn, "doc-lease", "upsert", 4, "INBOX", Some("{}")).expect("initial enqueue");
let event = list_ready(&conn, 1).unwrap().remove(0);
assert!(claim(&conn, &event, 60).unwrap());
enqueue(&conn, "doc-lease", "delete", 4, "INBOX", None).expect("replacement enqueue");
assert!(list_ready(&conn, 10).unwrap().is_empty());
}
#[test]
fn failures_backoff_and_dead_letter() {
let conn = open_test_db();
enqueue(&conn, "doc-2", "delete", 9, "Trash", None).unwrap();
let event = list_ready(&conn, 1).unwrap().remove(0);
assert!(mark_failure(&conn, &event, "boom", 1).unwrap());
let stats = stats(&conn).unwrap();
assert_eq!(stats.total, 1);
assert_eq!(stats.dead_letters, 1);
}
}
+2 -4
View File
@@ -751,8 +751,7 @@ mod tests {
let colors = extract_background_colors(html);
assert!(
colors.iter().any(|c| c.color == "whitesmoke"),
"expected whitesmoke, got: {:?}",
colors
"expected whitesmoke, got: {colors:?}"
);
assert!(
!colors.iter().any(|c| c.color == "white"),
@@ -789,8 +788,7 @@ mod tests {
let colors = extract_background_colors(html);
assert!(
colors.is_empty(),
"Fully transparent rgba should produce no colors, got {:?}",
colors
"Fully transparent rgba should produce no colors, got {colors:?}"
);
}
+1 -4
View File
@@ -1111,10 +1111,7 @@ impl ImapClient for RealImapClient {
let items = {
let mut fetch_stream = session
.uid_fetch(
"1:*",
format!("(UID FLAGS) (CHANGEDSINCE {})", since_modseq),
)
.uid_fetch("1:*", format!("(UID FLAGS) (CHANGEDSINCE {since_modseq})"))
.await
.map_err(map_imap_error)?;
+73 -13
View File
@@ -13,6 +13,7 @@ mod search;
mod session_store;
mod sieve;
mod smtp;
mod storage;
mod ui;
use std::fs::OpenOptions;
@@ -58,6 +59,33 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.await
.map_err(std::io::Error::other)?;
let config = Arc::new(loaded_config);
let storage = storage::from_config(&config).map_err(std::io::Error::other)?;
if let Some(options) = storage::parse_migration_args(&args).map_err(std::io::Error::other)? {
let report = storage::migrate_attachments(&config, storage.clone(), &options)
.await
.map_err(std::io::Error::other)?;
println!(
"storage migration complete: scanned={} migrated={} skipped={} failed={}",
report.scanned, report.migrated, report.skipped, report.failed
);
if report.failed > 0 {
return Err(std::io::Error::other("storage migration completed with failures").into());
}
return Ok(());
}
let search_engine = Arc::new(
search::engine::SearchEngine::from_config(&config).map_err(std::io::Error::other)?,
);
if args.get(1).map(String::as_str) == Some("search")
&& args.get(2).map(String::as_str) == Some("reindex")
{
run_search_reindex(&config, &search_engine)
.await
.map_err(std::io::Error::other)?;
return Ok(());
}
let persistence = session_store::from_config(&config)
.await
@@ -86,24 +114,28 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let imap_client: Arc<dyn imap::client::ImapClient> = Arc::new(RealImapClient::default());
let smtp_client: Arc<dyn smtp::client::SmtpClient> = Arc::new(RealSmtpClient);
let search_base = std::path::PathBuf::from(&config.data_dir);
let search_engine = Arc::new(if config.search_index_backend == "disabled" {
search::engine::SearchEngine::disabled(search_base)
} else {
search::engine::SearchEngine::new(search_base)
});
if config.search.required {
search_engine
.health()
.await
.map_err(std::io::Error::other)?;
}
let _search_worker = search_engine.start_worker();
let event_bus = Arc::new(realtime::events::EventBus::new());
let idle_manager = Arc::new(realtime::idle::IdleManager::new());
let app = routes::create_router(
let app = routes::create_router_with_storage(
config.clone(),
store,
imap_client,
smtp_client,
search_engine,
event_bus,
idle_manager.clone(),
routes::RouterServices {
store,
imap_client,
smtp_client,
search_engine,
storage,
event_bus,
idle_manager: idle_manager.clone(),
},
);
let bind_addr = format!("{}:{}", config.host, config.port);
@@ -126,6 +158,34 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
Ok(())
}
async fn run_search_reindex(
config: &AppConfig,
search_engine: &search::engine::SearchEngine,
) -> Result<(), String> {
if !search_engine.is_enabled() {
return Err("search reindex requires an enabled search backend".to_string());
}
search_engine.health().await?;
let reindex_started_at = db::search_outbox::now_epoch();
let mut users = 0usize;
let mut source_messages = 0usize;
let mut indexed_messages = 0usize;
for user_hash in db::pool::list_account_user_hashes()? {
let conn = db::pool::open_user_db(&config.data_dir, &user_hash)?;
let sources = db::messages::list_index_source_messages(&conn)?;
source_messages += sources.len();
let messages = routes::diagnostics::index_sources_to_messages(sources);
let index = search_engine.open_user_index(&user_hash)?;
indexed_messages += index.rebuild_from_messages(&messages).await?;
db::search_outbox::clear_before(&conn, reindex_started_at)?;
users += 1;
}
println!(
"search reindex complete: users={users} source_messages={source_messages} indexed_messages={indexed_messages}"
);
Ok(())
}
fn init_tracing(log_dir: Option<&str>, log_level: &str) -> Result<(), Box<dyn std::error::Error>> {
let env_filter = EnvFilter::try_from_default_env()
.unwrap_or_else(|_| EnvFilter::new(log_level.trim().to_ascii_lowercase()));
+1 -1
View File
@@ -126,7 +126,7 @@ async fn https_get_address(
format!("{host}:{port}")
};
let request = format!(
"GET {target} HTTP/1.1\r\nHost: {host_header}\r\nUser-Agent: RustBox/0.1.2\r\nAccept: application/octet-stream, application/pgp-keys, text/plain\r\nConnection: close\r\n\r\n"
"GET {target} HTTP/1.1\r\nHost: {host_header}\r\nUser-Agent: RustBox/0.2.0\r\nAccept: application/octet-stream, application/pgp-keys, text/plain\r\nConnection: close\r\n\r\n"
);
timeout(duration, stream.write_all(request.as_bytes()))
.await
+1 -2
View File
@@ -1090,8 +1090,7 @@ mod tests {
let encrypted =
encrypt_bytes(clear, std::slice::from_ref(&key.public_armor)).expect("encrypt mime");
let encrypted_mime = format!(
"Content-Type: multipart/encrypted; protocol=\"application/pgp-encrypted\"; boundary=\"rbe\"\r\n\r\n--rbe\r\nContent-Type: application/pgp-encrypted\r\n\r\nVersion: 1\r\n--rbe\r\nContent-Type: application/octet-stream\r\n\r\n{}\r\n--rbe--\r\n",
encrypted
"Content-Type: multipart/encrypted; protocol=\"application/pgp-encrypted\"; boundary=\"rbe\"\r\n\r\n--rbe\r\nContent-Type: application/pgp-encrypted\r\n\r\nVersion: 1\r\n--rbe\r\nContent-Type: application/octet-stream\r\n\r\n{encrypted}\r\n--rbe--\r\n"
);
let secrets = vec![secret.to_string()];
let processed = process_message(encrypted_mime.as_bytes(), &secrets, "", &public)
+15 -3
View File
@@ -159,12 +159,14 @@ pub mod db_drafts {
pub const Q_004: &str = "UPDATE drafts SET remote_folder = ?2, remote_uid = ?3, remote_uid_validity = ?4 WHERE id = ?1";
pub const Q_005: &str = "DELETE FROM drafts WHERE id = ?";
pub const Q_006: &str =
"INSERT INTO draft_attachments (id, draft_id, filename, content_type, size, file_path)
VALUES (?, ?, ?, ?, ?, ?)";
"INSERT INTO draft_attachments (id, draft_id, filename, content_type, size, file_path, checksum_sha256)
VALUES (?, ?, ?, ?, ?, ?, ?)";
pub const Q_007: &str =
"SELECT id, draft_id, filename, content_type, size, file_path, created_at
"SELECT id, draft_id, filename, content_type, size, file_path, checksum_sha256, created_at
FROM draft_attachments WHERE draft_id = ? ORDER BY created_at ASC";
pub const Q_008: &str = "DELETE FROM draft_attachments WHERE id = ?";
pub const Q_009: &str =
"UPDATE draft_attachments SET file_path=?, checksum_sha256=? WHERE id=?";
}
pub mod db_folder_preferences {
@@ -496,3 +498,13 @@ pub mod sessions {
pub const DELETE_EXPIRED: &str =
"DELETE FROM sessions WHERE last_accessed_unix + timeout_secs < ?";
}
pub mod db_search_outbox {
pub const Q_001: &str = "INSERT INTO search_outbox (id, generation, event_type, message_uid, folder, payload, attempts, available_at, dead_letter, last_error, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, 0, 0, 0, NULL, ?, ?) ON DUPLICATE KEY UPDATE generation=VALUES(generation), event_type=VALUES(event_type), message_uid=VALUES(message_uid), folder=VALUES(folder), payload=VALUES(payload), attempts=0, available_at=0, dead_letter=0, last_error=NULL, created_at=VALUES(created_at), updated_at=VALUES(updated_at)";
pub const Q_002: &str = "SELECT id, generation, event_type, message_uid, folder, payload, attempts, created_at, updated_at FROM search_outbox WHERE dead_letter=0 AND available_at <= ? AND claimed_until <= ? ORDER BY updated_at ASC LIMIT ?";
pub const Q_003: &str = "DELETE FROM search_outbox WHERE id=? AND generation=?";
pub const Q_004: &str = "UPDATE search_outbox SET attempts=?, available_at=?, claimed_until=0, dead_letter=?, last_error=?, updated_at=? WHERE id=? AND generation=?";
pub const Q_005: &str = "SELECT COUNT(*), COALESCE(SUM(CASE WHEN dead_letter <> 0 THEN 1 ELSE 0 END), 0), COALESCE(MIN(CASE WHEN dead_letter = 0 THEN created_at ELSE NULL END), 0) FROM search_outbox";
pub const Q_006: &str = "UPDATE search_outbox SET claimed_until=? WHERE id=? AND generation=? AND dead_letter=0 AND available_at <= ? AND claimed_until <= ?";
pub const Q_007: &str = "DELETE FROM search_outbox WHERE updated_at < ?";
}
+15 -3
View File
@@ -160,12 +160,14 @@ pub mod db_drafts {
pub const Q_004: &str = "UPDATE drafts SET remote_folder = $2, remote_uid = $3, remote_uid_validity = $4 WHERE id = $1";
pub const Q_005: &str = "DELETE FROM drafts WHERE id = $1";
pub const Q_006: &str =
"INSERT INTO draft_attachments (id, draft_id, filename, content_type, size, file_path)
VALUES ($1, $2, $3, $4, $5, $6)";
"INSERT INTO draft_attachments (id, draft_id, filename, content_type, size, file_path, checksum_sha256)
VALUES ($1, $2, $3, $4, $5, $6, $7)";
pub const Q_007: &str =
"SELECT id, draft_id, filename, content_type, size, file_path, created_at
"SELECT id, draft_id, filename, content_type, size, file_path, checksum_sha256, created_at
FROM draft_attachments WHERE draft_id = $1 ORDER BY created_at ASC";
pub const Q_008: &str = "DELETE FROM draft_attachments WHERE id = $1";
pub const Q_009: &str =
"UPDATE draft_attachments SET file_path=$1, checksum_sha256=$2 WHERE id=$3";
}
pub mod db_folder_preferences {
@@ -499,3 +501,13 @@ pub mod sessions {
pub const DELETE_EXPIRED: &str =
"DELETE FROM sessions WHERE last_accessed_unix + timeout_secs < $1";
}
pub mod db_search_outbox {
pub const Q_001: &str = "INSERT INTO search_outbox (id, generation, event_type, message_uid, folder, payload, attempts, available_at, dead_letter, last_error, created_at, updated_at) VALUES ($1, $2, $3, $4, $5, $6, 0, 0, 0, NULL, $7, $7) ON CONFLICT(id) DO UPDATE SET generation=excluded.generation, event_type=excluded.event_type, message_uid=excluded.message_uid, folder=excluded.folder, payload=excluded.payload, attempts=0, available_at=0, dead_letter=0, last_error=NULL, created_at=excluded.created_at, updated_at=excluded.updated_at";
pub const Q_002: &str = "SELECT id, generation, event_type, message_uid, folder, payload, attempts, created_at, updated_at FROM search_outbox WHERE dead_letter=0 AND available_at <= $1 AND claimed_until <= $1 ORDER BY updated_at ASC LIMIT $2";
pub const Q_003: &str = "DELETE FROM search_outbox WHERE id=$1 AND generation=$2";
pub const Q_004: &str = "UPDATE search_outbox SET attempts=$3, available_at=$4, claimed_until=0, dead_letter=$5, last_error=$6, updated_at=$7 WHERE id=$1 AND generation=$2";
pub const Q_005: &str = "SELECT COUNT(*), COALESCE(SUM(CASE WHEN dead_letter <> 0 THEN 1 ELSE 0 END), 0), COALESCE(MIN(CASE WHEN dead_letter = 0 THEN created_at ELSE NULL END), 0) FROM search_outbox";
pub const Q_006: &str = "UPDATE search_outbox SET claimed_until=$3 WHERE id=$1 AND generation=$2 AND dead_letter=0 AND available_at <= $4 AND claimed_until <= $4";
pub const Q_007: &str = "DELETE FROM search_outbox WHERE updated_at < $1";
}
+15 -3
View File
@@ -157,12 +157,14 @@ pub mod db_drafts {
pub const Q_004: &str = "UPDATE drafts SET remote_folder = ?2, remote_uid = ?3, remote_uid_validity = ?4 WHERE id = ?1";
pub const Q_005: &str = "DELETE FROM drafts WHERE id = ?1";
pub const Q_006: &str =
"INSERT INTO draft_attachments (id, draft_id, filename, content_type, size, file_path)
VALUES (?1, ?2, ?3, ?4, ?5, ?6)";
"INSERT INTO draft_attachments (id, draft_id, filename, content_type, size, file_path, checksum_sha256)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)";
pub const Q_007: &str =
"SELECT id, draft_id, filename, content_type, size, file_path, created_at
"SELECT id, draft_id, filename, content_type, size, file_path, checksum_sha256, created_at
FROM draft_attachments WHERE draft_id = ?1 ORDER BY created_at ASC";
pub const Q_008: &str = "DELETE FROM draft_attachments WHERE id = ?1";
pub const Q_009: &str =
"UPDATE draft_attachments SET file_path=?1, checksum_sha256=?2 WHERE id=?3";
}
pub mod db_folder_preferences {
@@ -747,3 +749,13 @@ pub mod sessions {
pub const DELETE_EXPIRED: &str =
"DELETE FROM sessions WHERE last_accessed_unix + timeout_secs < ?";
}
pub mod db_search_outbox {
pub const Q_001: &str = "INSERT INTO search_outbox (id, generation, event_type, message_uid, folder, payload, attempts, available_at, dead_letter, last_error, created_at, updated_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, 0, 0, 0, NULL, ?7, ?7) ON CONFLICT(id) DO UPDATE SET generation=excluded.generation, event_type=excluded.event_type, message_uid=excluded.message_uid, folder=excluded.folder, payload=excluded.payload, attempts=0, available_at=0, dead_letter=0, last_error=NULL, created_at=excluded.created_at, updated_at=excluded.updated_at";
pub const Q_002: &str = "SELECT id, generation, event_type, message_uid, folder, payload, attempts, created_at, updated_at FROM search_outbox WHERE dead_letter=0 AND available_at <= ?1 AND claimed_until <= ?1 ORDER BY updated_at ASC LIMIT ?2";
pub const Q_003: &str = "DELETE FROM search_outbox WHERE id=?1 AND generation=?2";
pub const Q_004: &str = "UPDATE search_outbox SET attempts=?3, available_at=?4, claimed_until=0, dead_letter=?5, last_error=?6, updated_at=?7 WHERE id=?1 AND generation=?2";
pub const Q_005: &str = "SELECT COUNT(*), COALESCE(SUM(CASE WHEN dead_letter <> 0 THEN 1 ELSE 0 END), 0), COALESCE(MIN(CASE WHEN dead_letter = 0 THEN created_at ELSE NULL END), 0) FROM search_outbox";
pub const Q_006: &str = "UPDATE search_outbox SET claimed_until=?3 WHERE id=?1 AND generation=?2 AND dead_letter=0 AND available_at <= ?4 AND claimed_until <= ?4";
pub const Q_007: &str = "DELETE FROM search_outbox WHERE updated_at < ?1";
}
+86 -9
View File
@@ -9,6 +9,7 @@ use crate::imap::client::ImapCredentials;
use crate::imap::connection::ImapStream;
use crate::imap::parse::{decode_rfc2047, flag_to_string, has_attachments, imap_address_to_email};
use crate::realtime::events::{EventBus, MailEvent};
use crate::search::engine::SearchEngine;
use crate::{db, imap};
pub struct IdleManager {
@@ -34,6 +35,7 @@ impl IdleManager {
creds: ImapCredentials,
event_bus: Arc<EventBus>,
config: Arc<AppConfig>,
search_engine: Arc<SearchEngine>,
) {
use dashmap::mapref::entry::Entry;
@@ -43,7 +45,15 @@ impl IdleManager {
}
Entry::Vacant(entry) => {
let handle = tokio::spawn(async move {
idle_loop(&user_hash, &folder, &creds, &event_bus, &config).await;
idle_loop(
&user_hash,
&folder,
&creds,
&event_bus,
&config,
&search_engine,
)
.await;
});
entry.insert(IdleTask {
handle,
@@ -94,12 +104,13 @@ async fn idle_loop(
creds: &ImapCredentials,
event_bus: &EventBus,
config: &AppConfig,
search_engine: &SearchEngine,
) {
let mut backoff = std::time::Duration::from_secs(1);
let max_backoff = std::time::Duration::from_secs(60);
loop {
match run_idle_session(user_hash, folder, creds, event_bus, config).await {
match run_idle_session(user_hash, folder, creds, event_bus, config, search_engine).await {
Ok(()) => {
tracing::info!(user_hash = %user_hash, folder = %folder, "IDLE session ended normally");
break;
@@ -125,6 +136,7 @@ async fn run_idle_session(
creds: &ImapCredentials,
event_bus: &EventBus,
config: &AppConfig,
search_engine: &SearchEngine,
) -> Result<(), String> {
use imap::client::connect;
@@ -173,7 +185,8 @@ async fn run_idle_session(
let result;
(session, result) =
fetch_new_after_idle(session, user_hash, folder, config).await;
fetch_new_after_idle(session, user_hash, folder, config, search_engine)
.await;
if result.new_count > 0 {
event_bus
@@ -256,6 +269,7 @@ async fn fetch_new_after_idle(
user_hash: &str,
folder: &str,
config: &AppConfig,
search_engine: &SearchEngine,
) -> (async_imap::Session<ImapStream>, ReconcileResult) {
let (cached_modseq, max_cached_uid, cached_count, cached_uid_validity, header_sync) =
match db::pool::open_user_db(&config.data_dir, user_hash) {
@@ -310,7 +324,7 @@ async fn fetch_new_after_idle(
}
if cached_modseq > 0 && server_modseq > 0 {
let fetch_query = format!("(UID FLAGS) (CHANGEDSINCE {})", cached_modseq);
let fetch_query = format!("(UID FLAGS) (CHANGEDSINCE {cached_modseq})");
let changed_items: Option<Vec<(u32, Vec<String>)>> =
match session.uid_fetch("1:*", &fetch_query).await {
Ok(stream) => {
@@ -396,8 +410,14 @@ async fn fetch_new_after_idle(
let mut deleted_count = 0u32;
if server_exists < cached_count {
deleted_count =
fetch_uids_and_remove_deleted(&mut session, user_hash, folder, config).await;
deleted_count = fetch_uids_and_remove_deleted(
&mut session,
user_hash,
folder,
config,
search_engine,
)
.await;
}
let (latest_uid, latest_sender, latest_subject) = if new_count > 0 {
@@ -514,7 +534,7 @@ async fn fetch_new_after_idle(
&& let Ok(conn) = db::pool::open_user_db(&config.data_dir, user_hash)
{
for uid in &deleted_uids {
if db::messages::delete_message(&conn, folder, *uid).is_ok() {
if delete_cached_message(&conn, search_engine, user_hash, folder, *uid) {
deleted_count += 1;
}
}
@@ -794,11 +814,55 @@ async fn fetch_and_store_headers(
new_count
}
fn delete_cached_message(
conn: &db::pool::Connection,
search_engine: &SearchEngine,
user_hash: &str,
folder: &str,
uid: u32,
) -> bool {
let transaction = match conn.unchecked_transaction() {
Ok(transaction) => transaction,
Err(error) => {
tracing::warn!(uid, folder = %folder, error = %error, "IDLE delete: failed to start transaction");
return false;
}
};
if let Err(error) = db::messages::delete_message(&transaction, folder, uid) {
tracing::warn!(uid, folder = %folder, error = %error, "IDLE delete: failed to remove cached message");
return false;
}
if search_engine.is_enabled() {
let index = match search_engine.open_user_index(user_hash) {
Ok(index) => index,
Err(error) => {
tracing::warn!(uid, folder = %folder, error = %error, "IDLE delete: failed to open search index");
return false;
}
};
if let Err(error) = index.delete_message_with_connection(&transaction, uid, folder) {
tracing::warn!(uid, folder = %folder, error = %error, "IDLE delete: failed to enqueue search deletion");
return false;
}
}
match transaction.commit() {
Ok(()) => true,
Err(error) => {
tracing::warn!(uid, folder = %folder, error = %error, "IDLE delete: failed to commit transaction");
false
}
}
}
async fn fetch_uids_and_remove_deleted(
session: &mut async_imap::Session<ImapStream>,
user_hash: &str,
folder: &str,
config: &AppConfig,
search_engine: &SearchEngine,
) -> u32 {
let imap_uids: Option<std::collections::HashSet<u32>> =
match session.uid_fetch("1:*", "UID").await {
@@ -840,7 +904,8 @@ async fn fetch_uids_and_remove_deleted(
if let Ok(conn) = db::pool::open_user_db(&config.data_dir, user_hash) {
let cached = db::messages::get_all_uids_and_flags(&conn, folder).unwrap_or_default();
for (uid, _) in &cached {
if !imap_uids.contains(uid) && db::messages::delete_message(&conn, folder, *uid).is_ok()
if !imap_uids.contains(uid)
&& delete_cached_message(&conn, search_engine, user_hash, folder, *uid)
{
deleted_count += 1;
}
@@ -859,6 +924,9 @@ mod tests {
let manager = IdleManager::new();
let config = Arc::new(serde_json::from_str::<AppConfig>("{}").unwrap());
let event_bus = Arc::new(EventBus::new());
let search_engine = Arc::new(SearchEngine::disabled(std::path::PathBuf::from(
"/tmp/rustbox-idle-test-search",
)));
let creds = ImapCredentials {
host: "localhost".into(),
port: 143,
@@ -876,6 +944,7 @@ mod tests {
creds.clone(),
event_bus.clone(),
config.clone(),
search_engine.clone(),
)
.await;
let original_task = manager.tasks.get(&key).unwrap().handle.id();
@@ -887,6 +956,7 @@ mod tests {
creds.clone(),
event_bus.clone(),
config.clone(),
search_engine.clone(),
)
.await;
assert_eq!(manager.tasks.len(), 1);
@@ -900,7 +970,14 @@ mod tests {
assert!(manager.tasks.is_empty());
manager
.start_idle(key.0.clone(), key.1.clone(), creds, event_bus, config)
.start_idle(
key.0.clone(),
key.1.clone(),
creds,
event_bus,
config,
search_engine,
)
.await;
assert_ne!(manager.tasks.get(&key).unwrap().handle.id(), original_task);
manager.stop_idle(&key.0, &key.1).await;
+38 -13
View File
@@ -127,6 +127,7 @@ async fn run_sync(
cached_modseq,
&status,
event_bus,
&search_engine,
)
.await?;
if folder_changed {
@@ -143,6 +144,7 @@ async fn run_sync(
folder_name,
&status,
event_bus,
&search_engine,
)
.await?;
if folder_changed {
@@ -208,6 +210,7 @@ async fn index_message_bodies(
let hash = user_hash.to_owned();
database
.run(move |conn| {
let transaction = conn.unchecked_transaction().map_err(|e| e.to_string())?;
let att_json = serde_json::to_string(&body.attachments).ok();
let detected_theme = body
.text_html
@@ -215,7 +218,7 @@ async fn index_message_bodies(
.and_then(|h| email_theme::detect_email_theme(h))
.map(|t| t.as_i32());
db::messages::cache_message_body(
conn,
&transaction,
&folder,
uid,
body.text_html.as_deref(),
@@ -225,22 +228,26 @@ async fn index_message_bodies(
detected_theme,
)?;
if let Some(text) = body.text_plain
&& let Some(msg) = db::messages::get_single_message(conn, &folder, uid)?
&& let Some(msg) = db::messages::get_single_message(&transaction, &folder, uid)?
{
engine
.open_user_index(&hash)?
.index_message(&IndexableMessage {
uid: msg.uid,
folder: msg.folder,
subject: msg.subject,
from_address: msg.from_address,
from_name: msg.from_name,
to_addresses: msg.to_addresses,
body_text: text,
date_epoch: db::messages::parse_date_to_epoch_public(&msg.date),
has_attachments: msg.has_attachments,
})?;
.index_message_with_connection(
&transaction,
&IndexableMessage {
uid: msg.uid,
folder: msg.folder,
subject: msg.subject,
from_address: msg.from_address,
from_name: msg.from_name,
to_addresses: msg.to_addresses,
body_text: text,
date_epoch: db::messages::parse_date_to_epoch_public(&msg.date),
has_attachments: msg.has_attachments,
},
)?;
}
transaction.commit().map_err(|e| e.to_string())?;
Ok(())
})
.await?;
@@ -266,6 +273,7 @@ async fn sync_condstore(
cached_modseq: u64,
status: &crate::imap::types::FolderStatusExtended,
event_bus: &EventBus,
search_engine: &Arc<SearchEngine>,
) -> Result<bool, String> {
let (changed, new_modseq) = match imap_client
.fetch_changed_flags(creds, folder_name, cached_modseq)
@@ -286,6 +294,7 @@ async fn sync_condstore(
folder_name,
status,
event_bus,
search_engine,
)
.await;
}
@@ -311,6 +320,8 @@ async fn sync_condstore(
};
let name = folder_name.to_owned();
let status = status.clone();
let search_engine = Arc::clone(search_engine);
let search_user_hash = user_hash.to_owned();
let (folder_changed, new_messages) = database
.run(move |conn| {
let sync_keywords = db::mail_preferences::get_preferences(conn)
@@ -329,6 +340,9 @@ async fn sync_condstore(
for (uid, _) in db::messages::get_all_uids_and_flags(&transaction, &name)? {
if !imap_uids.contains(&uid) {
db::messages::delete_message(&transaction, &name, uid)?;
if let Ok(index) = search_engine.open_user_index(&search_user_hash) {
index.delete_message_with_connection(&transaction, uid, &name)?;
}
folder_changed = true;
}
}
@@ -384,6 +398,7 @@ async fn sync_condstore(
Ok(folder_changed)
}
#[allow(clippy::too_many_arguments)]
async fn sync_full(
config: &AppConfig,
user_hash: &str,
@@ -392,6 +407,7 @@ async fn sync_full(
folder_name: &str,
status: &crate::imap::types::FolderStatusExtended,
event_bus: &EventBus,
search_engine: &Arc<SearchEngine>,
) -> Result<bool, String> {
let database = db::pool::UserDb::new(&config.data_dir, user_hash);
let name = folder_name.to_owned();
@@ -429,6 +445,8 @@ async fn sync_full(
let name = folder_name.to_owned();
let status = status.clone();
let search_engine = Arc::clone(search_engine);
let search_user_hash = user_hash.to_owned();
let folder_changed = database
.run(move |conn| {
let folder_name = &name;
@@ -451,6 +469,13 @@ async fn sync_full(
match imap_map.get(uid) {
None => {
db::messages::delete_message(&transaction, folder_name, *uid)?;
if let Ok(index) = search_engine.open_user_index(&search_user_hash) {
index.delete_message_with_connection(
&transaction,
*uid,
folder_name,
)?;
}
folder_changed = true;
tracing::debug!(
folder = %folder_name,
+1
View File
@@ -135,6 +135,7 @@ async fn handle_socket_multi_account(
creds.clone(),
event_bus.clone(),
config.clone(),
search_engine.clone(),
)
.await;
+54 -87
View File
@@ -1,46 +1,17 @@
use std::path::Path;
use std::sync::Arc;
use std::time::{Duration, SystemTime};
use axum::extract::Multipart;
use axum::extract::Path as AxumPath;
use axum::response::{IntoResponse, Response};
use axum::{Extension, Json};
use serde::Serialize;
use tokio::io::AsyncWriteExt;
use sha2::{Digest, Sha256};
use crate::auth::session::SessionState;
use crate::config::AppConfig;
use crate::db;
use crate::error::AppError;
const STALE_PART_MAX_AGE: Duration = Duration::from_secs(24 * 60 * 60);
async fn cleanup_stale_upload_parts(dir: &Path) {
let Ok(mut entries) = tokio::fs::read_dir(dir).await else {
return;
};
while let Ok(Some(entry)) = entries.next_entry().await {
let path = entry.path();
let is_part = path
.file_name()
.and_then(|name| name.to_str())
.is_some_and(|name| name.ends_with(".part"));
if !is_part {
continue;
}
let stale = entry
.metadata()
.await
.ok()
.and_then(|meta| meta.modified().ok())
.and_then(|modified| SystemTime::now().duration_since(modified).ok())
.is_some_and(|age| age >= STALE_PART_MAX_AGE);
if stale {
let _ = tokio::fs::remove_file(path).await;
}
}
}
use crate::storage::{BlobStorage, attachment_key};
#[derive(Debug, Serialize)]
struct UploadResponse {
@@ -58,6 +29,7 @@ struct DeleteResponse {
pub async fn upload_attachment(
Extension(session): Extension<SessionState>,
Extension(config): Extension<Arc<AppConfig>>,
Extension(storage): Extension<Arc<dyn BlobStorage>>,
AxumPath(draft_id): AxumPath<String>,
mut multipart: Multipart,
) -> Result<Response, AppError> {
@@ -74,15 +46,6 @@ pub async fn upload_attachment(
.map_err(AppError::InternalError)?;
}
let att_dir = Path::new(&config.data_dir)
.join(&session.user_hash)
.join("attachments")
.join(&draft_id);
tokio::fs::create_dir_all(&att_dir)
.await
.map_err(|e| AppError::InternalError(format!("Failed to create attachment dir: {e}")))?;
cleanup_stale_upload_parts(&att_dir).await;
let mut uploaded = Vec::new();
let max_attachment_size = config.max_attachment_size_bytes();
@@ -97,19 +60,20 @@ pub async fn upload_attachment(
.unwrap_or("application/octet-stream")
.to_string();
let att_id = uuid::Uuid::new_v4().to_string();
let file_path = att_dir.join(&att_id);
let part_path = att_dir.join(format!(".{att_id}.part"));
let mut file = tokio::fs::File::create(&part_path).await.map_err(|e| {
AppError::InternalError(format!("Failed to create attachment temp file: {e}"))
})?;
let key = attachment_key(&session.user_hash, &draft_id, &att_id)
.map_err(AppError::InternalError)?;
let mut writer = storage
.begin_upload(&key)
.await
.map_err(|e| AppError::ServiceUnavailable(format!("Storage unavailable: {e}")))?;
let mut size: usize = 0;
let mut checksum = Sha256::new();
loop {
let chunk = match field.chunk().await {
Ok(chunk) => chunk,
Err(e) => {
drop(file);
let _ = tokio::fs::remove_file(&part_path).await;
let _ = writer.abort().await;
return Err(AppError::BadRequest(format!(
"Failed to read field data: {e}"
)));
@@ -118,47 +82,43 @@ pub async fn upload_attachment(
let Some(chunk) = chunk else { break };
size = size.saturating_add(chunk.len());
if size > max_attachment_size {
drop(file);
let _ = tokio::fs::remove_file(&part_path).await;
let _ = writer.abort().await;
return Err(AppError::BadRequest(format!(
"File '{}' exceeds maximum size of {} MiB",
filename, config.max_attachment_size_mb
)));
}
if let Err(e) = file.write_all(&chunk).await {
drop(file);
let _ = tokio::fs::remove_file(&part_path).await;
return Err(AppError::InternalError(format!(
"Failed to write attachment: {e}"
checksum.update(&chunk);
if let Err(e) = writer.write(chunk).await {
let _ = writer.abort().await;
return Err(AppError::ServiceUnavailable(format!(
"Failed to upload attachment: {e}"
)));
}
}
if let Err(e) = file.flush().await {
drop(file);
let _ = tokio::fs::remove_file(&part_path).await;
return Err(AppError::InternalError(format!(
"Failed to flush attachment: {e}"
)));
}
drop(file);
if let Err(e) = tokio::fs::rename(&part_path, &file_path).await {
let _ = tokio::fs::remove_file(&part_path).await;
return Err(AppError::InternalError(format!(
"Failed to finalize attachment: {e}"
)));
}
let stat = writer.finish().await.map_err(|e| {
AppError::ServiceUnavailable(format!("Failed to finalize attachment upload: {e}"))
})?;
if stat.size != size as u64 {
let _ = storage.delete(&key).await;
return Err(AppError::InternalError(format!(
"Attachment size verification failed for '{filename}'"
)));
}
let checksum = format!("{:x}", checksum.finalize());
let size = size as i64;
if let Err(e) = db::drafts::add_draft_attachment(
&conn,
&att_id,
&draft_id,
&filename,
&content_type,
let attachment = db::drafts::NewDraftAttachment {
id: &att_id,
draft_id: &draft_id,
filename: &filename,
content_type: &content_type,
size,
file_path.to_str().unwrap_or(""),
) {
let _ = tokio::fs::remove_file(&file_path).await;
storage_key: &key,
checksum_sha256: Some(&checksum),
};
if let Err(e) = db::drafts::add_draft_attachment(&conn, &attachment) {
let _ = storage.delete(&key).await;
return Err(AppError::InternalError(e));
}
@@ -183,6 +143,7 @@ pub async fn upload_attachment(
pub async fn get_attachment_content(
Extension(session): Extension<SessionState>,
Extension(config): Extension<Arc<AppConfig>>,
Extension(storage): Extension<Arc<dyn BlobStorage>>,
AxumPath((draft_id, attachment_id)): AxumPath<(String, String)>,
) -> Result<Response, AppError> {
db::drafts::validate_draft_id(&draft_id).map_err(AppError::BadRequest)?;
@@ -191,26 +152,28 @@ pub async fn get_attachment_content(
let attachments =
db::drafts::get_draft_attachments(&conn, &draft_id).map_err(AppError::InternalError)?;
let attachment = attachments
.iter()
.find(|a| a.id == attachment_id)
.ok_or_else(|| AppError::NotFound("Attachment not found".to_string()))?;
let data = tokio::fs::read(&attachment.file_path)
let stream = storage
.get(&attachment.storage_key)
.await
.map_err(|e| AppError::InternalError(format!("Failed to read attachment file: {e}")))?;
.map_err(|e| AppError::ServiceUnavailable(format!("Storage read failed: {e}")))?;
Ok(Response::builder()
.header("content-type", &attachment.content_type)
.header("content-length", attachment.size.max(0).to_string())
.header("cache-control", "private, max-age=3600")
.body(axum::body::Body::from(data))
.body(axum::body::Body::from_stream(stream))
.unwrap())
}
pub async fn delete_attachment(
Extension(session): Extension<SessionState>,
Extension(config): Extension<Arc<AppConfig>>,
Extension(storage): Extension<Arc<dyn BlobStorage>>,
AxumPath((draft_id, attachment_id)): AxumPath<(String, String)>,
) -> Result<Response, AppError> {
db::drafts::validate_draft_id(&draft_id).map_err(AppError::BadRequest)?;
@@ -219,18 +182,22 @@ pub async fn delete_attachment(
let attachments =
db::drafts::get_draft_attachments(&conn, &draft_id).map_err(AppError::InternalError)?;
let attachment = attachments
.iter()
.find(|a| a.id == attachment_id)
.ok_or_else(|| AppError::NotFound("Attachment not found".to_string()))?;
let file_path = attachment.file_path.clone();
if let Err(e) = tokio::fs::remove_file(&file_path).await {
tracing::warn!(error = %e, path = %file_path, "Failed to delete attachment file from disk");
let storage_key = attachment.storage_key.clone();
let deleted = db::drafts::delete_draft_attachment(&conn, &attachment_id)
.map_err(AppError::InternalError)?;
if !deleted {
return Err(AppError::NotFound("Attachment not found".to_string()));
}
if let Err(error) = storage.delete(&storage_key).await {
// SQL is authoritative. A failed physical delete leaves an orphaned blob,
// which is safer than leaving a DB row pointing at a missing attachment.
tracing::warn!(error = %error, storage_key = %storage_key, "Failed to remove deleted attachment blob");
}
db::drafts::delete_draft_attachment(&conn, &attachment_id).map_err(AppError::InternalError)?;
Ok(Json(DeleteResponse {
status: "deleted".to_string(),
+6 -14
View File
@@ -56,8 +56,7 @@ fn browser_cookie(
) -> String {
let secure_flag = if secure { " Secure;" } else { "" };
format!(
"{}={};{} HttpOnly; SameSite={}; Path={}; Max-Age={}",
BROWSER_COOKIE, browser_id, secure_flag, same_site, path, max_age_secs
"{BROWSER_COOKIE}={browser_id};{secure_flag} HttpOnly; SameSite={same_site}; Path={path}; Max-Age={max_age_secs}"
)
}
@@ -71,24 +70,21 @@ fn account_session_cookie(
) -> String {
let secure_flag = if secure { " Secure;" } else { "" };
format!(
"rustbox_session_{}={};{} HttpOnly; SameSite={}; Path={}; Max-Age={}",
account_id, token, secure_flag, same_site, path, max_age_secs
"rustbox_session_{account_id}={token};{secure_flag} HttpOnly; SameSite={same_site}; Path={path}; Max-Age={max_age_secs}"
)
}
fn clearing_browser_cookie(secure: bool, same_site: &str, path: &str) -> String {
let secure_flag = if secure { " Secure;" } else { "" };
format!(
"{}=;{} HttpOnly; SameSite={}; Path={}; Max-Age=0",
BROWSER_COOKIE, secure_flag, same_site, path
"{BROWSER_COOKIE}=;{secure_flag} HttpOnly; SameSite={same_site}; Path={path}; Max-Age=0"
)
}
fn clearing_account_cookie(account_id: &str, secure: bool, same_site: &str, path: &str) -> String {
let secure_flag = if secure { " Secure;" } else { "" };
format!(
"rustbox_session_{}=;{} HttpOnly; SameSite={}; Path={}; Max-Age=0",
account_id, secure_flag, same_site, path
"rustbox_session_{account_id}=;{secure_flag} HttpOnly; SameSite={same_site}; Path={path}; Max-Age=0"
)
}
@@ -237,13 +233,9 @@ pub async fn login(
}
let user_hash = user_data::hash_email(&body.email);
let provision_result = if config.search_index_backend == "tantivy" {
user_data::provision_user_data(&config.data_dir, &user_hash)
} else {
user_data::provision_user_data_with_search(&config.data_dir, &user_hash, false)
};
let provision_result = user_data::provision_user_database(&config.data_dir, &user_hash);
if let Err(e) = provision_result {
tracing::error!(error = %e, "failed to provision user data directory");
tracing::error!(error = %e, "failed to provision user database");
return (
StatusCode::INTERNAL_SERVER_ERROR,
[("content-type", "application/json")],
+2 -2
View File
@@ -144,7 +144,7 @@ fn append_ics_event(out: &mut String, event: &db::calendar::CalendarEvent) {
}
if let Some(minutes) = event.reminder_minutes.filter(|value| *value >= 0) {
out.push_str("BEGIN:VALARM\r\nACTION:DISPLAY\r\n");
out.push_str(&format!("TRIGGER:-PT{}M\r\n", minutes));
out.push_str(&format!("TRIGGER:-PT{minutes}M\r\n"));
out.push_str("DESCRIPTION:Reminder\r\nEND:VALARM\r\n");
}
out.push_str("END:VEVENT\r\n");
@@ -174,7 +174,7 @@ pub async fn export_ics(
.map_err(|e| AppError::InternalError(format!("Database error: {e}")))?;
let mut body = String::from(
"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//RustBox//Calendar 0.1.2//EN\r\nCALSCALE:GREGORIAN\r\n",
"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//RustBox//Calendar 0.2.0//EN\r\nCALSCALE:GREGORIAN\r\n",
);
for event in &events {
append_ics_event(&mut body, event);
+51 -30
View File
@@ -44,10 +44,14 @@ pub struct SmtpDiagnosticsResponse {
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct SearchDiagnosticsResponse {
backend: String,
indexed_documents: usize,
cached_messages: usize,
cached_bodies: usize,
excluded_messages: usize,
queue_depth: usize,
dead_letters: usize,
queue_lag_seconds: u64,
}
#[derive(Debug, Serialize)]
@@ -181,7 +185,7 @@ pub async fn reconnect_imap(
.into_response())
}
fn index_sources_to_messages(
pub(crate) fn index_sources_to_messages(
sources: Vec<db::messages::IndexSourceMessage>,
) -> Vec<IndexableMessage> {
sources
@@ -254,11 +258,11 @@ pub async fn search_diagnostics(
) -> Result<Response, AppError> {
let data_dir = config.data_dir.clone();
let user_hash = session.user_hash.clone();
let engine = search_engine.clone();
let db_user_hash = user_hash.clone();
let stats =
tokio::task::spawn_blocking(move || -> Result<SearchDiagnosticsResponse, String> {
let conn = db::pool::open_user_db(&data_dir, &user_hash)
let (cached_messages, cached_bodies, excluded_messages, outbox) =
tokio::task::spawn_blocking(move || -> Result<_, String> {
let conn = db::pool::open_user_db(&data_dir, &db_user_hash)
.map_err(|e| format!("Database error: {e}"))?;
let sources = db::messages::list_index_source_messages(&conn)?;
let cached_messages = sources.len();
@@ -269,23 +273,35 @@ pub async fn search_diagnostics(
let cached_bodies: usize = conn
.query_row(crate::dbq!(routes_diagnostics::Q_003), [], |row| row.get(0))
.map_err(|e| format!("Failed to count cached bodies: {e}"))?;
let indexed_documents = if engine.is_enabled() {
engine.open_user_index(&user_hash)?.document_count()
} else {
0
};
Ok(SearchDiagnosticsResponse {
indexed_documents,
cached_messages,
cached_bodies,
excluded_messages,
})
let outbox = db::search_outbox::stats(&conn).unwrap_or_default();
Ok((cached_messages, cached_bodies, excluded_messages, outbox))
})
.await
.map_err(|e| AppError::InternalError(format!("Search diagnostics task failed: {e}")))?
.map_err(AppError::InternalError)?;
let indexed_documents = if search_engine.is_enabled() {
search_engine
.open_user_index(&user_hash)
.map_err(AppError::InternalError)?
.document_count()
.await
.map_err(AppError::InternalError)?
} else {
0
};
let stats = SearchDiagnosticsResponse {
backend: search_engine.backend_name().to_string(),
indexed_documents,
cached_messages,
cached_bodies,
excluded_messages,
queue_depth: outbox.total.saturating_sub(outbox.dead_letters),
dead_letters: outbox.dead_letters,
queue_lag_seconds: outbox.lag_seconds,
};
Ok(Json(stats).into_response())
}
@@ -301,26 +317,31 @@ pub async fn rebuild_search_index(
}
let data_dir = config.data_dir.clone();
let user_hash = session.user_hash.clone();
let engine = search_engine.clone();
let db_user_hash = user_hash.clone();
let result = tokio::task::spawn_blocking(move || -> Result<SearchRebuildResponse, String> {
let conn = db::pool::open_user_db(&data_dir, &user_hash)
let (source_messages, messages) = tokio::task::spawn_blocking(move || -> Result<_, String> {
let conn = db::pool::open_user_db(&data_dir, &db_user_hash)
.map_err(|e| format!("Database error: {e}"))?;
let sources = db::messages::list_index_source_messages(&conn)?;
let source_messages = sources.len();
let messages = index_sources_to_messages(sources);
let index = engine.open_user_index(&user_hash)?;
let indexed_documents = index.rebuild_from_messages(&messages)?;
Ok(SearchRebuildResponse {
status: "rebuilt",
indexed_documents,
source_messages,
})
Ok((source_messages, index_sources_to_messages(sources)))
})
.await
.map_err(|e| AppError::InternalError(format!("Search rebuild task failed: {e}")))?
.map_err(AppError::InternalError)?;
Ok(Json(result).into_response())
let index = search_engine
.open_user_index(&user_hash)
.map_err(AppError::InternalError)?;
let indexed_documents = index
.rebuild_from_messages(&messages)
.await
.map_err(AppError::InternalError)?;
Ok(Json(SearchRebuildResponse {
status: "rebuilt",
indexed_documents,
source_messages,
})
.into_response())
}
+30 -25
View File
@@ -1,4 +1,3 @@
use std::path::Path;
use std::sync::Arc;
use axum::extract::Path as AxumPath;
@@ -12,6 +11,7 @@ use crate::db;
use crate::error::AppError;
use crate::imap::client::{ImapClient, ImapCredentials};
use crate::smtp::client::AttachmentData;
use crate::storage::{self, BlobStorage};
#[derive(Debug, Deserialize)]
pub struct UpsertDraftRequest {
@@ -115,6 +115,7 @@ pub async fn upsert_draft_handler(
Extension(session): Extension<SessionState>,
Extension(config): Extension<Arc<AppConfig>>,
Extension(imap_client): Extension<Arc<dyn ImapClient>>,
Extension(storage): Extension<Arc<dyn BlobStorage>>,
Json(req): Json<UpsertDraftRequest>,
) -> Result<Response, AppError> {
db::drafts::validate_draft_id(&req.id).map_err(AppError::BadRequest)?;
@@ -172,7 +173,8 @@ pub async fn upsert_draft_handler(
.unwrap_or_else(|| "Drafts".to_string());
let previous = db::drafts::get_draft(&conn, &req.id).map_err(AppError::InternalError)?;
let snapshot_attachments = load_snapshot_attachments(&conn, &req.id);
let snapshot_attachments =
load_snapshot_attachments(&conn, &req.id, storage.as_ref()).await;
match snapshot_attachments
.and_then(|attachments| build_draft_rfc822(&req, &session.email, &attachments))
{
@@ -315,28 +317,30 @@ pub(crate) async fn remove_remote_draft_snapshot(
}
}
fn load_snapshot_attachments(
async fn load_snapshot_attachments(
conn: &crate::db::pool::Connection,
draft_id: &str,
storage: &dyn BlobStorage,
) -> Result<Vec<AttachmentData>, String> {
let records = db::drafts::get_draft_attachments(conn, draft_id)?;
records
.into_iter()
.map(|record| {
let data = std::fs::read(&record.file_path).map_err(|e| {
let mut attachments = Vec::with_capacity(records.len());
for record in records {
let data = storage::read_all(storage, &record.storage_key)
.await
.map_err(|e| {
format!(
"Failed to read draft attachment '{}' for IMAP snapshot: {e}",
record.filename
)
})?;
Ok(AttachmentData {
filename: record.filename,
content_type: record.content_type,
data,
content_id: Some(record.id),
})
})
.collect()
attachments.push(AttachmentData {
filename: record.filename,
content_type: record.content_type,
data,
content_id: Some(record.id),
});
}
Ok(attachments)
}
fn build_draft_rfc822(
@@ -535,6 +539,7 @@ pub async fn delete_draft_handler(
Extension(session): Extension<SessionState>,
Extension(config): Extension<Arc<AppConfig>>,
Extension(imap_client): Extension<Arc<dyn ImapClient>>,
Extension(storage): Extension<Arc<dyn BlobStorage>>,
AxumPath(id): AxumPath<String>,
) -> Result<Response, AppError> {
db::drafts::validate_draft_id(&id).map_err(AppError::BadRequest)?;
@@ -550,20 +555,20 @@ pub async fn delete_draft_handler(
remove_remote_draft_snapshot(imap_client.as_ref(), &creds, &existing).await;
}
let attachments =
db::drafts::get_draft_attachments(&conn, &id).map_err(AppError::InternalError)?;
let deleted = db::drafts::delete_draft(&conn, &id).map_err(AppError::InternalError)?;
if !deleted {
return Err(AppError::NotFound("Draft not found".to_string()));
}
let att_dir = Path::new(&config.data_dir)
.join(&session.user_hash)
.join("attachments")
.join(&id);
if att_dir.exists()
&& let Err(e) = tokio::fs::remove_dir_all(&att_dir).await
{
tracing::warn!(error = %e, path = %att_dir.display(), "Failed to clean up attachment directory");
for attachment in &attachments {
if let Err(error) = storage.delete(&attachment.storage_key).await {
tracing::warn!(
error = %error,
storage_key = %attachment.storage_key,
"Failed to remove deleted draft attachment blob"
);
}
}
Ok(Json(DeleteResponse {
+13 -6
View File
@@ -224,8 +224,11 @@ pub async fn empty_folder(
let conn = db::pool::open_user_db(&config.data_dir, &session.user_hash)
.map_err(|e| AppError::InternalError(format!("Database error: {e}")))?;
let transaction = conn
.unchecked_transaction()
.map_err(|e| AppError::InternalError(format!("Database transaction error: {e}")))?;
let uids: Vec<u32> = {
let mut statement = conn
let mut statement = transaction
.prepare(crate::dbq!(routes_folder_mgmt::Q_003))
.map_err(|e| AppError::InternalError(format!("Database error: {e}")))?;
let rows = statement
@@ -233,18 +236,22 @@ pub async fn empty_folder(
.map_err(|e| AppError::InternalError(format!("Database error: {e}")))?;
rows.filter_map(Result::ok).collect()
};
db::messages::delete_folder_messages(&conn, &name)
db::messages::delete_folder_messages(&transaction, &name)
.map_err(|e| AppError::InternalError(format!("Database error: {e}")))?;
conn.execute(crate::dbq!(routes_folder_mgmt::Q_004), [&name])
transaction
.execute(crate::dbq!(routes_folder_mgmt::Q_004), [&name])
.map_err(|e| AppError::InternalError(format!("Database error: {e}")))?;
if let Ok(index) = search_engine.open_user_index(&session.user_hash) {
for uid in uids {
if let Err(error) = index.delete_message(uid, &name) {
tracing::warn!(uid, folder = %name, %error, "Failed to remove emptied message from search index");
}
index
.delete_message_with_connection(&transaction, uid, &name)
.map_err(|e| AppError::InternalError(format!("Search outbox error: {e}")))?;
}
}
transaction
.commit()
.map_err(|e| AppError::InternalError(format!("Database commit error: {e}")))?;
Ok(Json(MessageResponse { status: "emptied" }).into_response())
}
+28 -7
View File
@@ -1,4 +1,3 @@
use std::fs;
use std::sync::Arc;
use axum::http::StatusCode;
@@ -7,22 +6,41 @@ use axum::{Extension, Json};
use serde::Serialize;
use crate::config::AppConfig;
use crate::search::engine::SearchEngine;
use crate::storage::BlobStorage;
#[derive(Serialize)]
pub struct HealthResponse {
pub status: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub storage: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub search: Option<String>,
}
pub async fn health_check() -> Json<HealthResponse> {
Json(HealthResponse {
status: "ok".to_string(),
storage: None,
search: None,
})
}
pub async fn readiness_check(Extension(config): Extension<Arc<AppConfig>>) -> Response {
let ready = fs::create_dir_all(&config.data_dir).is_ok()
&& fs::metadata(&config.data_dir).is_ok_and(|metadata| metadata.is_dir());
let status = if ready { "ready" } else { "not_ready" };
pub async fn readiness_check(
Extension(config): Extension<Arc<AppConfig>>,
Extension(storage): Extension<Arc<dyn BlobStorage>>,
Extension(search): Extension<Arc<SearchEngine>>,
) -> Response {
let storage_status = match storage.health().await {
Ok(()) => "ok".to_string(),
Err(error) => format!("error: {error}"),
};
let search_status = match search.health().await {
Ok(()) => "ok".to_string(),
Err(error) if !config.search.required => format!("degraded: {error}"),
Err(error) => format!("error: {error}"),
};
let ready = storage_status == "ok" && (!config.search.required || search_status == "ok");
(
if ready {
StatusCode::OK
@@ -30,7 +48,9 @@ pub async fn readiness_check(Extension(config): Extension<Arc<AppConfig>>) -> Re
StatusCode::SERVICE_UNAVAILABLE
},
Json(HealthResponse {
status: status.to_string(),
status: if ready { "ready" } else { "not_ready" }.to_string(),
storage: Some(storage_status),
search: Some(search_status),
}),
)
.into_response()
@@ -118,7 +138,7 @@ mod tests {
}
#[tokio::test]
async fn readiness_creates_data_directory_and_returns_ready() {
async fn readiness_checks_storage_without_requiring_node_local_data_dir_in_cluster_shape() {
let temp = tempfile::tempdir().unwrap();
let data_dir = temp.path().join("nested").join("rustbox");
let app = test_router(test_config(data_dir.to_string_lossy().into_owned()));
@@ -136,6 +156,7 @@ mod tests {
let body = response.into_body().collect().await.unwrap().to_bytes();
let json: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(json["status"], "ready");
assert_eq!(json["storage"], "ok");
assert!(data_dir.is_dir());
}
}
+78 -58
View File
@@ -402,7 +402,9 @@ fn upsert_and_index_headers(
}
})
.collect();
let _ = user_index.index_messages_batch(&indexable);
user_index
.index_messages_batch_with_connection(conn, &indexable)
.map_err(|e| AppError::InternalError(format!("Search outbox error: {e}")))?;
}
Ok(())
@@ -545,7 +547,6 @@ pub async fn get_message(
.map_err(|e| AppError::InternalError(format!("Database error: {e}")))?;
let usable_cache = cached_body.filter(|c| c.attachments_json.is_some());
let fetched_body = usable_cache.is_none();
let (body_html, body_text, attachments, raw_headers, email_theme) = if let Some(cached) =
usable_cache
@@ -585,18 +586,32 @@ pub async fn get_message(
.await
.map_err(|e| match e {
crate::imap::client::ImapError::MessageNotFound { .. } => {
let _ = db::messages::delete_message(&conn, &folder, uid);
let _ = db::folders::invalidate_folder_freshness(&conn, &folder);
if let Ok(user_index) = search_engine.open_user_index(&session.user_hash) {
let _ = user_index.delete_message(uid, &folder);
if let Ok(transaction) = conn.unchecked_transaction() {
let _ = db::messages::delete_message(&transaction, &folder, uid);
let _ = db::folders::invalidate_folder_freshness(&transaction, &folder);
if let Ok(user_index) = search_engine.open_user_index(&session.user_hash) {
let _ = user_index.delete_message_with_connection(
&transaction,
uid,
&folder,
);
}
let _ = transaction.commit();
}
AppError::NotFound(format!("Message UID {uid} not found in folder {folder}"))
}
other => {
let _ = db::messages::delete_message(&conn, &folder, uid);
let _ = db::folders::invalidate_folder_freshness(&conn, &folder);
if let Ok(user_index) = search_engine.open_user_index(&session.user_hash) {
let _ = user_index.delete_message(uid, &folder);
if let Ok(transaction) = conn.unchecked_transaction() {
let _ = db::messages::delete_message(&transaction, &folder, uid);
let _ = db::folders::invalidate_folder_freshness(&transaction, &folder);
if let Ok(user_index) = search_engine.open_user_index(&session.user_hash) {
let _ = user_index.delete_message_with_connection(
&transaction,
uid,
&folder,
);
}
let _ = transaction.commit();
}
AppError::ServiceUnavailable(format!("IMAP error: {other}"))
}
@@ -639,8 +654,11 @@ pub async fn get_message(
.and_then(|h| email_theme::detect_email_theme(h))
.map(|t| t.as_i32());
let transaction = conn
.unchecked_transaction()
.map_err(|e| AppError::InternalError(format!("Database transaction error: {e}")))?;
db::messages::cache_message_body(
&conn,
&transaction,
&folder,
uid,
resolved_html.as_deref(),
@@ -651,6 +669,31 @@ pub async fn get_message(
)
.map_err(|e| AppError::InternalError(format!("Database error: {e}")))?;
if let Some(text) = body.text_plain.as_ref()
&& !UserIndex::is_excluded_folder(&folder)
&& let Some(msg) = db::messages::get_single_message(&transaction, &folder, uid)
.map_err(|e| AppError::InternalError(format!("Database error: {e}")))?
&& let Ok(user_index) = search_engine.open_user_index(&session.user_hash)
{
let indexable = IndexableMessage {
uid: msg.uid,
folder: msg.folder,
subject: msg.subject,
from_address: msg.from_address,
from_name: msg.from_name,
to_addresses: msg.to_addresses,
body_text: text.clone(),
date_epoch: crate::db::messages::parse_date_to_epoch_public(&msg.date),
has_attachments: msg.has_attachments,
};
user_index
.index_message_with_connection(&transaction, &indexable)
.map_err(|e| AppError::InternalError(format!("Search outbox error: {e}")))?;
}
transaction
.commit()
.map_err(|e| AppError::InternalError(format!("Database commit error: {e}")))?;
(
resolved_html,
body.text_plain,
@@ -742,30 +785,6 @@ pub async fn get_message(
}
};
if fetched_body
&& let Some(ref text) = body_text
&& !UserIndex::is_excluded_folder(&folder)
{
let indexable = IndexableMessage {
uid: msg.uid,
folder: msg.folder.clone(),
subject: msg.subject.clone(),
from_address: msg.from_address.clone(),
from_name: msg.from_name.clone(),
to_addresses: msg.to_addresses.clone(),
body_text: text.clone(),
date_epoch: crate::db::messages::parse_date_to_epoch_public(&msg.date),
has_attachments: msg.has_attachments,
};
let search_engine = Arc::clone(&search_engine);
let user_hash = session.user_hash.clone();
tokio::task::spawn_blocking(move || {
if let Ok(user_index) = search_engine.open_user_index(&user_hash) {
let _ = user_index.index_message(&indexable);
}
});
}
let thread_messages = if let Some(ref message_id) = msg.message_id {
db::messages::get_full_thread(&conn, message_id, msg.references_header.as_deref())
.unwrap_or_default()
@@ -973,34 +992,31 @@ pub async fn move_message_handler(
let conn = db::pool::open_user_db(&config.data_dir, &session.user_hash)
.map_err(|e| AppError::InternalError(format!("Database error: {e}")))?;
db::messages::remove_after_mutation(&conn, &body.from_folder, body.uid)
let transaction = conn
.unchecked_transaction()
.map_err(|e| AppError::InternalError(format!("Database transaction error: {e}")))?;
db::messages::remove_after_mutation_in_transaction(&transaction, &body.from_folder, body.uid)
.map_err(|e| AppError::InternalError(format!("Database error: {e}")))?;
let user_hash = session.user_hash.clone();
let from_folder = body.from_folder.clone();
let uid = body.uid;
let search_result = tokio::task::spawn_blocking(move || {
let index = search_engine.open_user_index(&user_hash)?;
index.delete_message(uid, &from_folder)
})
.await;
if !matches!(search_result, Ok(Ok(()))) {
tracing::warn!(
?search_result,
"Failed to remove moved message from search index"
);
if let Ok(index) = search_engine.open_user_index(&session.user_hash) {
index
.delete_message_with_connection(&transaction, body.uid, &body.from_folder)
.map_err(|e| AppError::InternalError(format!("Search outbox error: {e}")))?;
}
db::folders::refresh_unread_count(&conn, &body.from_folder)
db::folders::refresh_unread_count(&transaction, &body.from_folder)
.map_err(|e| AppError::InternalError(format!("Database error: {e}")))?;
if was_unread {
db::folders::adjust_unread_count(&conn, &body.to_folder, 1)
db::folders::adjust_unread_count(&transaction, &body.to_folder, 1)
.map_err(|e| AppError::InternalError(format!("Database error: {e}")))?;
}
db::folders::invalidate_folder_freshness(&conn, &body.to_folder)
db::folders::invalidate_folder_freshness(&transaction, &body.to_folder)
.map_err(|e| AppError::InternalError(format!("Database error: {e}")))?;
transaction
.commit()
.map_err(|e| AppError::InternalError(format!("Database commit error: {e}")))?;
let undo_move = undo_destination_uid.map(|destination_uid| {
serde_json::json!({
@@ -1454,6 +1470,7 @@ mod sync_tests {
limit: 10,
..Default::default()
})
.await
.unwrap()
.0
.is_empty()
@@ -1646,12 +1663,14 @@ mod sync_tests {
.is_none()
);
assert!(db::folders::is_folder_fresh(&conn, "INBOX", 30).unwrap());
let requests = fresh_client.header_requests.lock().unwrap();
assert_eq!(requests.len(), 4);
assert!(requests.iter().all(|(_, uids)| {
uids.split(',')
.all(|uid| uid.parse::<u32>().unwrap() <= 900)
}));
{
let requests = fresh_client.header_requests.lock().unwrap();
assert_eq!(requests.len(), 4);
assert!(requests.iter().all(|(_, uids)| {
uids.split(',')
.all(|uid| uid.parse::<u32>().unwrap() <= 900)
}));
}
let results = engine
.open_user_index(user_hash)
.unwrap()
@@ -1661,6 +1680,7 @@ mod sync_tests {
limit: 1100,
..Default::default()
})
.await
.unwrap()
.0;
assert!(results.iter().any(|message| message.uid == 1));
+38
View File
@@ -101,6 +101,7 @@ impl KeyExtractor for ProxyAwareIpExtractor {
}
}
#[cfg(test)]
pub fn create_router(
config: Arc<AppConfig>,
store: Arc<SessionStore>,
@@ -110,6 +111,42 @@ pub fn create_router(
event_bus: Arc<EventBus>,
idle_manager: Arc<IdleManager>,
) -> Router {
let storage =
crate::storage::from_config(&config).expect("validated RustBox storage configuration");
create_router_with_storage(
config,
RouterServices {
store,
imap_client,
smtp_client,
search_engine,
storage,
event_bus,
idle_manager,
},
)
}
pub struct RouterServices {
pub store: Arc<SessionStore>,
pub imap_client: Arc<dyn ImapClient>,
pub smtp_client: Arc<dyn SmtpClient>,
pub search_engine: Arc<crate::search::engine::SearchEngine>,
pub storage: Arc<dyn crate::storage::BlobStorage>,
pub event_bus: Arc<EventBus>,
pub idle_manager: Arc<IdleManager>,
}
pub fn create_router_with_storage(config: Arc<AppConfig>, services: RouterServices) -> Router {
let RouterServices {
store,
imap_client,
smtp_client,
search_engine,
storage,
event_bus,
idle_manager,
} = services;
let governor_conf = GovernorConfigBuilder::default()
.key_extractor(ProxyAwareIpExtractor {
trusted_proxies: config.parsed_trusted_proxies(),
@@ -477,6 +514,7 @@ pub fn create_router(
.layer(Extension(event_bus))
.layer(Extension(smtp_client))
.layer(Extension(search_engine))
.layer(Extension(storage))
.layer(Extension(imap_client))
.layer(Extension(store))
.layer(Extension(config.clone()))
+14 -12
View File
@@ -362,18 +362,20 @@ pub async fn search_messages(
if !parsed.text.is_empty()
&& results.len() < limit
&& let Ok(user_index) = search_engine.open_user_index(&session.user_hash)
&& let Ok((tantivy_results, _)) = user_index.search(&SearchQuery {
text: parsed.text,
subject_only: subject,
folder,
from,
to,
date_from,
date_to,
has_attachment,
limit,
offset: if select_all { 0 } else { search_offset },
})
&& let Ok((tantivy_results, _)) = user_index
.search(&SearchQuery {
text: parsed.text,
subject_only: subject,
folder,
from,
to,
date_from,
date_to,
has_attachment,
limit,
offset: if select_all { 0 } else { search_offset },
})
.await
{
let data_dir2 = config.data_dir.clone();
let user_hash2 = session.user_hash.clone();
+49 -25
View File
@@ -1,4 +1,3 @@
use std::path::Path;
use std::sync::Arc;
use axum::response::{IntoResponse, Response};
@@ -12,6 +11,7 @@ use crate::error::AppError;
use crate::imap::client::{ImapClient, ImapCredentials};
use crate::smtp::client::{AttachmentData, SendableMessage, SmtpClient, SmtpCredentials};
use crate::smtp::types::{PgpMode, PgpSendConfig};
use crate::storage::{self, BlobStorage};
#[derive(Deserialize)]
pub struct SendRequest {
@@ -72,6 +72,7 @@ pub async fn send_message_handler(
Extension(config): Extension<Arc<AppConfig>>,
Extension(smtp_client): Extension<Arc<dyn SmtpClient>>,
Extension(imap_client): Extension<Arc<dyn ImapClient>>,
Extension(storage): Extension<Arc<dyn BlobStorage>>,
Json(req): Json<SendRequest>,
) -> Result<Response, AppError> {
if let Some(ref draft_id) = req.draft_id {
@@ -242,7 +243,13 @@ pub async fn send_message_handler(
}
let mut attachments = if let Some(ref draft_id) = req.draft_id {
load_draft_attachments(&config.data_dir, &session.user_hash, draft_id)?
load_draft_attachments(
&config.data_dir,
&session.user_hash,
draft_id,
storage.as_ref(),
)
.await?
} else {
vec![]
};
@@ -360,7 +367,13 @@ pub async fn send_message_handler(
)
.await;
}
cleanup_draft(&config.data_dir, &session.user_hash, draft_id).await;
cleanup_draft(
&config.data_dir,
&session.user_hash,
draft_id,
storage.as_ref(),
)
.await;
}
Ok(Json(SendResponse {
@@ -370,10 +383,11 @@ pub async fn send_message_handler(
.into_response())
}
fn load_draft_attachments(
async fn load_draft_attachments(
data_dir: &str,
user_hash: &str,
draft_id: &str,
storage: &dyn BlobStorage,
) -> Result<Vec<AttachmentData>, AppError> {
let conn = db::pool::open_user_db(data_dir, user_hash)
.map_err(|e| AppError::InternalError(format!("Failed to open database: {e}")))?;
@@ -381,14 +395,16 @@ fn load_draft_attachments(
let db_attachments =
db::drafts::get_draft_attachments(&conn, draft_id).map_err(AppError::InternalError)?;
let mut attachments = Vec::new();
let mut attachments = Vec::with_capacity(db_attachments.len());
for att in db_attachments {
let data = std::fs::read(&att.file_path).map_err(|e| {
AppError::InternalError(format!(
"Failed to read attachment file '{}': {e}",
att.filename
))
})?;
let data = storage::read_all(storage, &att.storage_key)
.await
.map_err(|e| {
AppError::InternalError(format!(
"Failed to read attachment '{}': {e}",
att.filename
))
})?;
attachments.push(AttachmentData {
filename: att.filename,
content_type: att.content_type,
@@ -399,21 +415,29 @@ fn load_draft_attachments(
Ok(attachments)
}
async fn cleanup_draft(data_dir: &str, user_hash: &str, draft_id: &str) {
if let Ok(conn) = db::pool::open_user_db(data_dir, user_hash)
&& let Err(e) = db::drafts::delete_draft(&conn, draft_id)
{
tracing::warn!(error = %e, "Failed to delete draft after send");
async fn cleanup_draft(data_dir: &str, user_hash: &str, draft_id: &str, storage: &dyn BlobStorage) {
let Ok(conn) = db::pool::open_user_db(data_dir, user_hash) else {
return;
};
let attachments = db::drafts::get_draft_attachments(&conn, draft_id).unwrap_or_default();
let deleted = match db::drafts::delete_draft(&conn, draft_id) {
Ok(deleted) => deleted,
Err(e) => {
tracing::warn!(error = %e, "Failed to delete draft after send");
return;
}
};
if !deleted {
return;
}
let att_dir = Path::new(data_dir)
.join(user_hash)
.join("attachments")
.join(draft_id);
if att_dir.exists()
&& let Err(e) = tokio::fs::remove_dir_all(&att_dir).await
{
tracing::warn!(error = %e, path = %att_dir.display(), "Failed to clean up attachment directory");
for attachment in attachments {
if let Err(e) = storage.delete(&attachment.storage_key).await {
tracing::warn!(
error = %e,
storage_key = %attachment.storage_key,
"Failed to delete draft attachment after send"
);
}
}
}
+1 -1
View File
@@ -771,7 +771,7 @@ async fn logout_clears_cookie() {
let set_cookie = response.headers().get_all("set-cookie");
let cookie_str = format!("{:?}", set_cookie);
let cookie_str = format!("{set_cookie:?}");
assert!(cookie_str.contains("rustbox_browser=;") || cookie_str.contains("Max-Age=0"));
}
+1 -1
View File
@@ -228,7 +228,7 @@ async fn post_one_click_to_address(
format!("{host}:{port}")
};
let request = format!(
"POST {path} HTTP/1.1\r\nHost: {host_header}\r\nUser-Agent: RustBox/0.1.2\r\nContent-Type: application/x-www-form-urlencoded\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{ONE_CLICK_BODY}",
"POST {path} HTTP/1.1\r\nHost: {host_header}\r\nUser-Agent: RustBox/0.2.0\r\nContent-Type: application/x-www-form-urlencoded\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{ONE_CLICK_BODY}",
ONE_CLICK_BODY.len()
);
+763 -59
View File
@@ -1,7 +1,12 @@
use std::ops::Bound;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
use serde::Serialize;
use reqwest::Method;
use serde::{Deserialize, Serialize};
use serde_json::{Value as JsonValue, json};
use sha2::{Digest, Sha256};
use tantivy::collector::{Count, TopDocs};
use tantivy::query::{BooleanQuery, Occur, QueryParser, RangeQuery, TermQuery};
use tantivy::schema::Value;
@@ -11,6 +16,7 @@ use tantivy::schema::{
use tantivy::snippet::SnippetGenerator;
use tantivy::{Index, IndexReader, IndexWriter, ReloadPolicy, TantivyDocument, Term, doc};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct IndexableMessage {
pub uid: u32,
pub folder: String,
@@ -46,8 +52,36 @@ pub struct SearchQuery {
}
pub struct SearchEngine {
base_dir: PathBuf,
enabled: bool,
backend: SearchBackend,
}
enum SearchBackend {
Disabled,
LocalTantivy {
base_dir: PathBuf,
},
OpenSearch {
backend: Arc<OpenSearchBackend>,
data_dir: String,
worker_interval_secs: u64,
worker_batch_size: usize,
worker_max_attempts: u32,
},
}
#[derive(Clone)]
struct OpenSearchBackend {
client: reqwest::Client,
urls: Vec<String>,
index_name: String,
username: Option<String>,
password: Option<String>,
request_timeout_secs: u64,
}
struct OpenSearchUserIndex {
backend: Arc<OpenSearchBackend>,
user_hash: String,
}
struct SchemaFields {
@@ -62,7 +96,7 @@ struct SchemaFields {
has_attachments: Field,
}
pub struct UserIndex {
struct LocalUserIndex {
index: Index,
reader: IndexReader,
#[allow(dead_code)]
@@ -110,56 +144,261 @@ fn build_schema() -> (Schema, SchemaFields) {
impl SearchEngine {
pub fn new(base_dir: PathBuf) -> Self {
Self {
base_dir,
enabled: true,
backend: SearchBackend::LocalTantivy { base_dir },
}
}
pub fn disabled(base_dir: PathBuf) -> Self {
pub fn disabled(_base_dir: PathBuf) -> Self {
Self {
base_dir,
enabled: false,
backend: SearchBackend::Disabled,
}
}
pub fn from_config(config: &crate::config::AppConfig) -> Result<Self, String> {
match config.effective_search_backend().as_str() {
"disabled" => Ok(Self::disabled(PathBuf::from(
config.effective_search_path(),
))),
"local_tantivy" => Ok(Self::new(PathBuf::from(config.effective_search_path()))),
"opensearch" => {
let client = reqwest::Client::builder()
.connect_timeout(Duration::from_secs(
config.search.request_timeout_secs.min(30),
))
.timeout(Duration::from_secs(config.search.request_timeout_secs))
.build()
.map_err(|e| format!("failed to build OpenSearch client: {e}"))?;
let backend = OpenSearchBackend {
client,
urls: config
.search
.urls
.iter()
.map(|url| url.trim_end_matches('/').to_string())
.collect(),
index_name: format!("{}-messages-v1", config.search.index_prefix),
username: config.search.username.clone(),
password: config.search.password.clone(),
request_timeout_secs: config.search.request_timeout_secs,
};
Ok(Self {
backend: SearchBackend::OpenSearch {
backend: Arc::new(backend),
data_dir: config.data_dir.clone(),
worker_interval_secs: config.search.worker_interval_secs,
worker_batch_size: config.search.worker_batch_size,
worker_max_attempts: config.search.worker_max_attempts,
},
})
}
other => Err(format!("unsupported search backend: {other}")),
}
}
pub fn is_enabled(&self) -> bool {
self.enabled
!matches!(&self.backend, SearchBackend::Disabled)
}
pub fn backend_name(&self) -> &'static str {
match &self.backend {
SearchBackend::Disabled => "disabled",
SearchBackend::LocalTantivy { .. } => "local_tantivy",
SearchBackend::OpenSearch { .. } => "opensearch",
}
}
pub async fn health(&self) -> Result<(), String> {
match &self.backend {
SearchBackend::Disabled => Ok(()),
SearchBackend::LocalTantivy { base_dir } => {
std::fs::create_dir_all(base_dir)
.map_err(|e| format!("failed to prepare local search directory: {e}"))?;
Ok(())
}
SearchBackend::OpenSearch { backend, .. } => backend.health().await,
}
}
pub fn open_user_index(&self, user_hash: &str) -> Result<UserIndex, String> {
if !self.enabled {
return Err("Tantivy search index is disabled by configuration".to_string());
match &self.backend {
SearchBackend::Disabled => Err("search index is disabled by configuration".to_string()),
SearchBackend::LocalTantivy { base_dir } => {
let index_dir = base_dir.join(user_hash).join("tantivy");
std::fs::create_dir_all(&index_dir)
.map_err(|e| format!("Failed to create index directory: {e}"))?;
let (schema, fields) = build_schema();
let index = if Index::open_in_dir(&index_dir).is_ok() {
Index::open_in_dir(&index_dir)
.map_err(|e| format!("Failed to open existing index: {e}"))?
} else {
Index::create_in_dir(&index_dir, schema.clone())
.map_err(|e| format!("Failed to create index: {e}"))?
};
let reader = index
.reader_builder()
.reload_policy(ReloadPolicy::OnCommitWithDelay)
.try_into()
.map_err(|e| format!("Failed to create index reader: {e}"))?;
Ok(UserIndex {
inner: UserIndexBackend::Local(LocalUserIndex {
index,
reader,
schema,
fields,
}),
})
}
SearchBackend::OpenSearch { backend, .. } => Ok(UserIndex {
inner: UserIndexBackend::OpenSearch(OpenSearchUserIndex {
backend: backend.clone(),
user_hash: user_hash.to_string(),
}),
}),
}
let index_dir = self.base_dir.join(user_hash).join("tantivy");
std::fs::create_dir_all(&index_dir)
.map_err(|e| format!("Failed to create index directory: {e}"))?;
}
let (schema, fields) = build_schema();
let index = if Index::open_in_dir(&index_dir).is_ok() {
Index::open_in_dir(&index_dir)
.map_err(|e| format!("Failed to open existing index: {e}"))?
} else {
Index::create_in_dir(&index_dir, schema.clone())
.map_err(|e| format!("Failed to create index: {e}"))?
pub fn start_worker(&self) -> Option<tokio::task::JoinHandle<()>> {
let SearchBackend::OpenSearch {
backend,
data_dir,
worker_interval_secs,
worker_batch_size,
worker_max_attempts,
} = &self.backend
else {
return None;
};
let reader = index
.reader_builder()
.reload_policy(ReloadPolicy::OnCommitWithDelay)
.try_into()
.map_err(|e| format!("Failed to create index reader: {e}"))?;
Ok(UserIndex {
index,
reader,
schema,
fields,
})
let backend = backend.clone();
let data_dir = data_dir.clone();
let interval_secs = *worker_interval_secs;
let batch_size = *worker_batch_size;
let max_attempts = *worker_max_attempts;
Some(tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(interval_secs));
loop {
interval.tick().await;
match process_search_outbox(&backend, &data_dir, batch_size, max_attempts).await {
Ok(processed) if processed > 0 => {
tracing::debug!(processed, "processed OpenSearch outbox events");
}
Ok(_) => {}
Err(error) => tracing::warn!(%error, "OpenSearch outbox worker pass failed"),
}
}
}))
}
}
pub struct UserIndex {
inner: UserIndexBackend,
}
enum UserIndexBackend {
Local(LocalUserIndex),
OpenSearch(OpenSearchUserIndex),
}
impl UserIndex {
pub fn is_excluded_folder(folder: &str) -> bool {
LocalUserIndex::is_excluded_folder(folder)
}
#[cfg(test)]
pub fn index_message(&self, msg: &IndexableMessage) -> Result<(), String> {
match &self.inner {
UserIndexBackend::Local(index) => index.index_message(msg),
UserIndexBackend::OpenSearch(_) => {
Err("test-only direct indexing is unavailable for OpenSearch".to_string())
}
}
}
pub fn index_message_with_connection(
&self,
conn: &crate::db::pool::Connection,
msg: &IndexableMessage,
) -> Result<(), String> {
match &self.inner {
UserIndexBackend::Local(index) => index.index_message(msg),
UserIndexBackend::OpenSearch(index) => index.enqueue_upsert_on(conn, msg),
}
}
pub async fn document_count(&self) -> Result<usize, String> {
match &self.inner {
UserIndexBackend::Local(index) => Ok(index.document_count()),
UserIndexBackend::OpenSearch(index) => index.document_count().await,
}
}
pub async fn rebuild_from_messages(
&self,
messages: &[IndexableMessage],
) -> Result<usize, String> {
match &self.inner {
UserIndexBackend::Local(index) => index.rebuild_from_messages(messages),
UserIndexBackend::OpenSearch(index) => index.rebuild_from_messages(messages).await,
}
}
#[cfg(test)]
pub fn index_messages_batch(&self, messages: &[IndexableMessage]) -> Result<(), String> {
match &self.inner {
UserIndexBackend::Local(index) => index.index_messages_batch(messages),
UserIndexBackend::OpenSearch(_) => {
Err("test-only direct batch indexing is unavailable for OpenSearch".to_string())
}
}
}
pub fn index_messages_batch_with_connection(
&self,
conn: &crate::db::pool::Connection,
messages: &[IndexableMessage],
) -> Result<(), String> {
match &self.inner {
UserIndexBackend::Local(index) => index.index_messages_batch(messages),
UserIndexBackend::OpenSearch(index) => {
for message in messages {
index.enqueue_upsert_on(conn, message)?;
}
Ok(())
}
}
}
#[cfg(test)]
pub fn delete_message(&self, uid: u32, folder: &str) -> Result<(), String> {
match &self.inner {
UserIndexBackend::Local(index) => index.delete_message(uid, folder),
UserIndexBackend::OpenSearch(_) => {
Err("test-only direct deletion is unavailable for OpenSearch".to_string())
}
}
}
pub fn delete_message_with_connection(
&self,
conn: &crate::db::pool::Connection,
uid: u32,
folder: &str,
) -> Result<(), String> {
match &self.inner {
UserIndexBackend::Local(index) => index.delete_message(uid, folder),
UserIndexBackend::OpenSearch(index) => index.enqueue_delete_on(conn, uid, folder),
}
}
pub async fn search(&self, query: &SearchQuery) -> Result<(Vec<SearchResult>, usize), String> {
match &self.inner {
UserIndexBackend::Local(index) => index.search(query),
UserIndexBackend::OpenSearch(index) => index.search(query).await,
}
}
}
impl LocalUserIndex {
pub fn is_excluded_folder(folder: &str) -> bool {
let lower = folder.to_lowercase();
lower == "trash"
@@ -432,6 +671,426 @@ impl UserIndex {
}
}
impl OpenSearchBackend {
fn event_lease_secs(&self) -> u64 {
// A mutation can probe/create the index and then write the document.
// Account for failover across all configured URLs and keep a safety margin so
// another worker cannot reclaim the same document while this request is alive.
self.request_timeout_secs
.saturating_mul(self.urls.len().max(1) as u64)
.saturating_mul(4)
.saturating_add(30)
.max(60)
}
async fn request(
&self,
method: Method,
path: &str,
body: Option<&JsonValue>,
) -> Result<(reqwest::StatusCode, String), String> {
let mut last_error = None;
for base in &self.urls {
let url = format!(
"{}/{}",
base.trim_end_matches('/'),
path.trim_start_matches('/')
);
let mut request = self.client.request(method.clone(), &url);
if let Some(username) = self.username.as_deref() {
request = request.basic_auth(username, self.password.as_deref());
}
if let Some(body) = body {
request = request.json(body);
}
match request.send().await {
Ok(response) => {
let status = response.status();
let text = response.text().await.unwrap_or_default();
if status.is_server_error() {
last_error = Some(format!("OpenSearch {url} returned {status}: {text}"));
continue;
}
return Ok((status, text));
}
Err(error) => {
last_error = Some(format!("OpenSearch request to {url} failed: {error}"))
}
}
}
Err(last_error.unwrap_or_else(|| "no OpenSearch URLs configured".to_string()))
}
async fn ensure_index(&self) -> Result<(), String> {
let (status, body) = self.request(Method::HEAD, &self.index_name, None).await?;
if status.is_success() {
return Ok(());
}
if status != reqwest::StatusCode::NOT_FOUND {
return Err(format!(
"OpenSearch index probe failed with {status}: {body}"
));
}
let mapping = json!({
"mappings": {
"dynamic": "strict",
"properties": {
"user_hash": {"type": "keyword"},
"uid": {"type": "long"},
"folder": {"type": "keyword"},
"subject": {"type": "text"},
"from_address": {"type": "keyword"},
"from_name": {"type": "text"},
"to_addresses": {"type": "text"},
"body_text": {"type": "text"},
"date_epoch": {"type": "long"},
"has_attachments": {"type": "boolean"}
}
}
});
let (status, body) = self
.request(Method::PUT, &self.index_name, Some(&mapping))
.await?;
if status.is_success()
|| (status == reqwest::StatusCode::BAD_REQUEST
&& body.contains("resource_already_exists_exception"))
{
Ok(())
} else {
Err(format!(
"failed to create OpenSearch index ({status}): {body}"
))
}
}
async fn health(&self) -> Result<(), String> {
let (status, body) = self
.request(Method::GET, "_cluster/health?local=true", None)
.await?;
if status.is_success() {
self.ensure_index().await
} else {
Err(format!("OpenSearch health check failed ({status}): {body}"))
}
}
fn document_id(user_hash: &str, uid: u32, folder: &str) -> String {
format!(
"{:x}",
Sha256::digest(format!("{user_hash}\0{folder}\0{uid}").as_bytes())
)
}
async fn index_direct(&self, user_hash: &str, msg: &IndexableMessage) -> Result<(), String> {
if LocalUserIndex::is_excluded_folder(&msg.folder) {
return self.delete_direct(user_hash, msg.uid, &msg.folder).await;
}
self.ensure_index().await?;
let id = Self::document_id(user_hash, msg.uid, &msg.folder);
let path = format!("{}/_doc/{}?routing={}", self.index_name, id, user_hash);
let body = json!({
"user_hash": user_hash,
"uid": msg.uid,
"folder": &msg.folder,
"subject": &msg.subject,
"from_address": &msg.from_address,
"from_name": &msg.from_name,
"to_addresses": &msg.to_addresses,
"body_text": &msg.body_text,
"date_epoch": msg.date_epoch,
"has_attachments": msg.has_attachments
});
let (status, response) = self.request(Method::PUT, &path, Some(&body)).await?;
if status.is_success() {
Ok(())
} else {
Err(format!(
"OpenSearch document upsert failed ({status}): {response}"
))
}
}
async fn delete_direct(&self, user_hash: &str, uid: u32, folder: &str) -> Result<(), String> {
self.ensure_index().await?;
let id = Self::document_id(user_hash, uid, folder);
let path = format!("{}/_doc/{}?routing={}", self.index_name, id, user_hash);
let (status, response) = self.request(Method::DELETE, &path, None).await?;
if status.is_success() || status == reqwest::StatusCode::NOT_FOUND {
Ok(())
} else {
Err(format!(
"OpenSearch document delete failed ({status}): {response}"
))
}
}
async fn delete_user_documents(&self, user_hash: &str) -> Result<(), String> {
self.ensure_index().await?;
let path = format!(
"{}/_delete_by_query?refresh=true&routing={}",
self.index_name, user_hash
);
let body = json!({"query": {"term": {"user_hash": user_hash}}});
let (status, response) = self.request(Method::POST, &path, Some(&body)).await?;
if status.is_success() {
Ok(())
} else {
Err(format!(
"OpenSearch user reindex cleanup failed ({status}): {response}"
))
}
}
async fn refresh(&self) -> Result<(), String> {
self.ensure_index().await?;
let path = format!("{}/_refresh", self.index_name);
let (status, response) = self.request(Method::POST, &path, None).await?;
if status.is_success() {
Ok(())
} else {
Err(format!("OpenSearch refresh failed ({status}): {response}"))
}
}
async fn count(&self, user_hash: &str) -> Result<usize, String> {
self.ensure_index().await?;
let path = format!("{}/_count?routing={}", self.index_name, user_hash);
let body = json!({"query": {"term": {"user_hash": user_hash}}});
let (status, response) = self.request(Method::POST, &path, Some(&body)).await?;
if !status.is_success() {
return Err(format!("OpenSearch count failed ({status}): {response}"));
}
let parsed: JsonValue = serde_json::from_str(&response)
.map_err(|e| format!("invalid OpenSearch count response: {e}"))?;
Ok(parsed.get("count").and_then(JsonValue::as_u64).unwrap_or(0) as usize)
}
async fn search(
&self,
user_hash: &str,
query: &SearchQuery,
) -> Result<(Vec<SearchResult>, usize), String> {
self.ensure_index().await?;
let body = build_opensearch_query(user_hash, query);
let path = format!("{}/_search?routing={}", self.index_name, user_hash);
let (status, response) = self.request(Method::POST, &path, Some(&body)).await?;
if !status.is_success() {
return Err(format!("OpenSearch search failed ({status}): {response}"));
}
let parsed: JsonValue = serde_json::from_str(&response)
.map_err(|e| format!("invalid OpenSearch search response: {e}"))?;
let total = parsed
.pointer("/hits/total/value")
.and_then(JsonValue::as_u64)
.unwrap_or(0) as usize;
let mut results = Vec::new();
if let Some(hits) = parsed.pointer("/hits/hits").and_then(JsonValue::as_array) {
for hit in hits {
let source = hit.get("_source").unwrap_or(&JsonValue::Null);
let uid = source.get("uid").and_then(JsonValue::as_u64).unwrap_or(0) as u32;
let folder = source
.get("folder")
.and_then(JsonValue::as_str)
.unwrap_or("")
.to_string();
let score = hit.get("_score").and_then(JsonValue::as_f64).unwrap_or(0.0) as f32;
let snippet = hit
.pointer("/highlight/body_text/0")
.or_else(|| hit.pointer("/highlight/subject/0"))
.and_then(JsonValue::as_str)
.or_else(|| source.get("subject").and_then(JsonValue::as_str))
.unwrap_or("")
.to_string();
results.push(SearchResult {
uid,
folder,
score,
snippet,
});
}
}
Ok((results, total))
}
}
fn build_opensearch_query(user_hash: &str, query: &SearchQuery) -> JsonValue {
let mut must = Vec::new();
let mut filter = vec![json!({"term": {"user_hash": user_hash}})];
if !query.text.trim().is_empty() {
must.push(json!({
"multi_match": {
"query": query.text.as_str(),
"fields": ["subject^2", "body_text", "from_name", "to_addresses"]
}
}));
}
if let Some(subject) = query.subject_only.as_deref() {
must.push(json!({"match": {"subject": subject}}));
}
if let Some(folder) = query.folder.as_deref() {
filter.push(json!({"term": {"folder": folder}}));
}
if let Some(from) = query.from.as_deref() {
if from.contains('@') {
filter.push(json!({"term": {"from_address": from}}));
} else {
must.push(json!({"match": {"from_name": from}}));
}
}
if let Some(to) = query.to.as_deref() {
must.push(json!({"match": {"to_addresses": to}}));
}
if query.date_from.is_some() || query.date_to.is_some() {
let mut range = serde_json::Map::new();
if let Some(value) = query.date_from {
range.insert("gte".to_string(), json!(value));
}
if let Some(value) = query.date_to {
range.insert("lte".to_string(), json!(value));
}
filter.push(json!({"range": {"date_epoch": JsonValue::Object(range)}}));
}
if let Some(value) = query.has_attachment {
filter.push(json!({"term": {"has_attachments": value}}));
}
let query_body = if must.is_empty() {
json!({"bool": {"filter": filter}})
} else {
json!({"bool": {"must": must, "filter": filter}})
};
json!({
"from": query.offset,
"size": query.limit.max(1),
"track_total_hits": true,
"query": query_body,
"highlight": {
"pre_tags": ["<b>"],
"post_tags": ["</b>"],
"fields": {"subject": {}, "body_text": {"fragment_size": 200, "number_of_fragments": 1}}
}
})
}
impl OpenSearchUserIndex {
fn event_id(&self, uid: u32, folder: &str) -> String {
OpenSearchBackend::document_id(&self.user_hash, uid, folder)
}
fn enqueue_upsert_on(
&self,
conn: &crate::db::pool::Connection,
msg: &IndexableMessage,
) -> Result<(), String> {
let id = self.event_id(msg.uid, &msg.folder);
if LocalUserIndex::is_excluded_folder(&msg.folder) {
crate::db::search_outbox::enqueue(conn, &id, "delete", msg.uid, &msg.folder, None)?;
} else {
let payload = serde_json::to_string(msg)
.map_err(|e| format!("failed to serialize search document: {e}"))?;
crate::db::search_outbox::enqueue(
conn,
&id,
"upsert",
msg.uid,
&msg.folder,
Some(&payload),
)?;
}
Ok(())
}
fn enqueue_delete_on(
&self,
conn: &crate::db::pool::Connection,
uid: u32,
folder: &str,
) -> Result<(), String> {
let id = self.event_id(uid, folder);
crate::db::search_outbox::enqueue(conn, &id, "delete", uid, folder, None)?;
Ok(())
}
async fn document_count(&self) -> Result<usize, String> {
self.backend.count(&self.user_hash).await
}
async fn search(&self, query: &SearchQuery) -> Result<(Vec<SearchResult>, usize), String> {
self.backend.search(&self.user_hash, query).await
}
async fn rebuild_from_messages(&self, messages: &[IndexableMessage]) -> Result<usize, String> {
self.backend.delete_user_documents(&self.user_hash).await?;
let mut indexed = 0;
for message in messages {
if LocalUserIndex::is_excluded_folder(&message.folder) {
continue;
}
self.backend.index_direct(&self.user_hash, message).await?;
indexed += 1;
}
self.backend.refresh().await?;
Ok(indexed)
}
}
async fn process_search_outbox(
backend: &OpenSearchBackend,
data_dir: &str,
batch_size: usize,
max_attempts: u32,
) -> Result<usize, String> {
let mut processed = 0;
for user_hash in crate::db::pool::list_account_user_hashes()? {
let conn = match crate::db::pool::open_user_db(data_dir, &user_hash) {
Ok(conn) => conn,
Err(error) => {
tracing::warn!(%error, %user_hash, "failed to open tenant DB for search outbox");
continue;
}
};
let events = crate::db::search_outbox::list_ready(&conn, batch_size)?;
let lease_secs = backend.event_lease_secs();
for event in events {
if !crate::db::search_outbox::claim(&conn, &event, lease_secs)? {
continue;
}
let result = match event.event_type.as_str() {
"upsert" => {
let message = event
.payload
.as_deref()
.ok_or_else(|| "search upsert event is missing payload".to_string())
.and_then(|payload| {
serde_json::from_str::<IndexableMessage>(payload)
.map_err(|e| format!("invalid search outbox payload: {e}"))
});
match message {
Ok(message) => backend.index_direct(&user_hash, &message).await,
Err(error) => Err(error),
}
}
"delete" => {
backend
.delete_direct(&user_hash, event.message_uid, &event.folder)
.await
}
other => Err(format!("unsupported search outbox event type: {other}")),
};
match result {
Ok(()) => {
crate::db::search_outbox::mark_success(&conn, &event)?;
processed += 1;
}
Err(error) => {
crate::db::search_outbox::mark_failure(&conn, &event, &error, max_attempts)?;
tracing::warn!(%error, event_id = %event.id, %user_hash, "OpenSearch outbox event failed");
}
}
}
}
Ok(processed)
}
#[cfg(test)]
mod tests {
use super::*;
@@ -446,6 +1105,26 @@ mod tests {
(tmp, user_index)
}
#[test]
fn opensearch_event_lease_covers_timeout_and_failover_budget() {
let backend = OpenSearchBackend {
client: reqwest::Client::new(),
urls: vec!["http://os-a".to_string(), "http://os-b".to_string()],
index_name: "rustbox-test".to_string(),
username: None,
password: None,
request_timeout_secs: 15,
};
assert_eq!(backend.event_lease_secs(), 150);
let single = OpenSearchBackend {
urls: vec!["http://os-a".to_string()],
request_timeout_secs: 5,
..backend
};
assert_eq!(single.event_lease_secs(), 60);
}
fn make_message(uid: u32, folder: &str, subject: &str, body: &str) -> IndexableMessage {
IndexableMessage {
uid,
@@ -460,8 +1139,8 @@ mod tests {
}
}
#[test]
fn index_and_search_by_subject() {
#[tokio::test]
async fn index_and_search_by_subject() {
let (_tmp, idx) = setup();
let msg1 = make_message(1, "INBOX", "Meeting tomorrow morning", "Let us meet.");
@@ -475,7 +1154,7 @@ mod tests {
limit: 10,
..Default::default()
};
let (results, total) = idx.search(&query).unwrap();
let (results, total) = idx.search(&query).await.unwrap();
assert_eq!(total, 1);
assert_eq!(results.len(), 1);
@@ -483,8 +1162,8 @@ mod tests {
assert_eq!(results[0].folder, "INBOX");
}
#[test]
fn index_and_search_by_body() {
#[tokio::test]
async fn index_and_search_by_body() {
let (_tmp, idx) = setup();
let msg1 = make_message(1, "INBOX", "Hello", "The quarterly report is attached.");
@@ -502,15 +1181,15 @@ mod tests {
limit: 10,
..Default::default()
};
let (results, total) = idx.search(&query).unwrap();
let (results, total) = idx.search(&query).await.unwrap();
assert_eq!(total, 1);
assert_eq!(results.len(), 1);
assert_eq!(results[0].uid, 1);
}
#[test]
fn search_with_folder_filter() {
#[tokio::test]
async fn search_with_folder_filter() {
let (_tmp, idx) = setup();
let msg1 = make_message(1, "INBOX", "Project update", "Status of the project.");
@@ -524,7 +1203,7 @@ mod tests {
limit: 10,
..Default::default()
};
let (results, total) = idx.search(&query).unwrap();
let (results, total) = idx.search(&query).await.unwrap();
assert_eq!(total, 1);
assert_eq!(results.len(), 1);
@@ -532,8 +1211,8 @@ mod tests {
assert_eq!(results[0].folder, "Sent");
}
#[test]
fn search_with_date_range() {
#[tokio::test]
async fn search_with_date_range() {
let (_tmp, idx) = setup();
let mut msg1 = make_message(1, "INBOX", "January news", "News from January.");
@@ -554,15 +1233,15 @@ mod tests {
limit: 10,
..Default::default()
};
let (results, total) = idx.search(&query).unwrap();
let (results, total) = idx.search(&query).await.unwrap();
assert_eq!(total, 1);
assert_eq!(results.len(), 1);
assert_eq!(results[0].uid, 2);
}
#[test]
fn delete_removes_from_index() {
#[tokio::test]
async fn delete_removes_from_index() {
let (_tmp, idx) = setup();
let msg1 = make_message(1, "INBOX", "Important email", "Very important content.");
@@ -575,19 +1254,19 @@ mod tests {
limit: 10,
..Default::default()
};
let (_, total) = idx.search(&query).unwrap();
let (_, total) = idx.search(&query).await.unwrap();
assert_eq!(total, 2);
idx.delete_message(1, "INBOX").unwrap();
let (results, total) = idx.search(&query).unwrap();
let (results, total) = idx.search(&query).await.unwrap();
assert_eq!(total, 1);
assert_eq!(results.len(), 1);
assert_eq!(results[0].uid, 2);
}
#[test]
fn search_empty_text_returns_all() {
#[tokio::test]
async fn search_empty_text_returns_all() {
let (_tmp, idx) = setup();
let msg = make_message(1, "INBOX", "Test subject", "Test body text.");
@@ -598,14 +1277,39 @@ mod tests {
limit: 10,
..Default::default()
};
let (results, total) = idx.search(&query).unwrap();
let (results, total) = idx.search(&query).await.unwrap();
assert_eq!(total, 1);
assert_eq!(results.len(), 1);
}
#[test]
fn search_pagination() {
#[tokio::test]
async fn opensearch_query_always_scopes_to_tenant() {
let query = SearchQuery {
text: "invoice".to_string(),
folder: Some("INBOX".to_string()),
limit: 25,
..Default::default()
};
let body = build_opensearch_query("tenant-a", &query);
let filters = body
.pointer("/query/bool/filter")
.and_then(JsonValue::as_array)
.expect("OpenSearch filter array");
assert!(filters.iter().any(|filter| {
filter
.pointer("/term/user_hash")
.and_then(JsonValue::as_str)
== Some("tenant-a")
}));
assert!(filters.iter().any(|filter| {
filter.pointer("/term/folder").and_then(JsonValue::as_str) == Some("INBOX")
}));
}
#[tokio::test]
async fn search_pagination() {
let (_tmp, idx) = setup();
let messages: Vec<IndexableMessage> = (1..=5)
@@ -627,7 +1331,7 @@ mod tests {
offset: 0,
..Default::default()
};
let (results1, total1) = idx.search(&query_page1).unwrap();
let (results1, total1) = idx.search(&query_page1).await.unwrap();
assert_eq!(total1, 5);
assert_eq!(results1.len(), 2);
@@ -637,7 +1341,7 @@ mod tests {
offset: 2,
..Default::default()
};
let (results2, total2) = idx.search(&query_page2).unwrap();
let (results2, total2) = idx.search(&query_page2).await.unwrap();
assert_eq!(total2, 5);
assert_eq!(results2.len(), 2);
+1 -1
View File
@@ -178,7 +178,7 @@ impl DatabaseSessionStore {
.map_err(|e| format!("MySQL database schema initialization failed: {e}"))?;
Pool::Mysql(pool)
}
"pgsql" | "postgres" | "postgresql" => {
"pgsql" => {
let url = config
.database_url
.as_deref()
+2 -3
View File
@@ -122,7 +122,7 @@ impl ManageSieveClient {
let plain = format!("\0{}\0{}", creds.username, creds.password);
let encoded = base64::engine::general_purpose::STANDARD.encode(plain.as_bytes());
let command = format!("AUTHENTICATE \"PLAIN\" \"{}\"\r\n", encoded);
let command = format!("AUTHENTICATE \"PLAIN\" \"{encoded}\"\r\n");
write_all_timeout(&mut stream, command.as_bytes(), creds.timeout).await?;
let auth = read_response(&mut stream, creds.timeout).await?;
if auth.status == ResponseStatus::No || auth.status == ResponseStatus::Bye {
@@ -421,8 +421,7 @@ async fn read_wire_line<S: AsyncRead + Unpin>(stream: &mut S) -> Result<WireLine
if size > MAX_LITERAL_BYTES {
return Err(SieveError::Protocol(format!(
"server literal exceeds {} bytes",
MAX_LITERAL_BYTES
"server literal exceeds {MAX_LITERAL_BYTES} bytes"
)));
}
let mut literal = vec![0u8; size];
+1 -1
View File
@@ -29,7 +29,7 @@ impl fmt::Display for SieveError {
write!(f, "ManageSieve server rejected the command: {msg}")
}
Self::Timeout => write!(f, "ManageSieve operation timed out"),
Self::ScriptTooLarge => write!(f, "Sieve script exceeds the RustBox 0.1.2 size limit"),
Self::ScriptTooLarge => write!(f, "Sieve script exceeds the RustBox 0.2.0 size limit"),
}
}
}
+633
View File
@@ -0,0 +1,633 @@
use std::path::{Path as FsPath, PathBuf};
use std::pin::Pin;
use std::sync::Arc;
use std::time::Duration;
use async_trait::async_trait;
use bytes::Bytes;
use futures::{Stream, StreamExt};
use object_store::aws::{AmazonS3Builder, Checksum};
use object_store::buffered::BufWriter;
use object_store::client::ClientOptions;
use object_store::local::LocalFileSystem;
use object_store::path::Path;
use object_store::{ObjectStore, ObjectStoreExt, RetryConfig};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use crate::config::{AppConfig, StorageConfig};
pub type BlobStream = Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BlobStat {
pub size: u64,
pub e_tag: Option<String>,
}
#[async_trait]
pub trait BlobUpload: Send {
async fn write(&mut self, chunk: Bytes) -> Result<(), String>;
async fn finish(self: Box<Self>) -> Result<BlobStat, String>;
async fn abort(self: Box<Self>) -> Result<(), String>;
}
#[async_trait]
pub trait BlobStorage: Send + Sync {
async fn begin_upload(&self, key: &str) -> Result<Box<dyn BlobUpload>, String>;
async fn get(&self, key: &str) -> Result<BlobStream, String>;
async fn delete(&self, key: &str) -> Result<(), String>;
async fn exists(&self, key: &str) -> Result<bool, String>;
async fn health(&self) -> Result<(), String>;
}
#[derive(Clone)]
struct ObjectStorage {
store: Arc<dyn ObjectStore>,
prefix: String,
}
impl ObjectStorage {
fn validate_logical_key(key: &str) -> Result<(), String> {
if key.is_empty() || key.starts_with('/') || key.contains('\\') {
return Err("invalid blob key".to_string());
}
if key
.split('/')
.any(|part| part.is_empty() || part == "." || part == "..")
{
return Err("invalid blob key".to_string());
}
Ok(())
}
fn object_path(&self, key: &str) -> Result<Path, String> {
Self::validate_logical_key(key)?;
let full = if self.prefix.is_empty() {
key.to_string()
} else {
format!("{}/{}", self.prefix.trim_matches('/'), key)
};
Path::parse(full).map_err(|e| format!("invalid object key: {e}"))
}
fn stat_from_meta(meta: object_store::ObjectMeta) -> BlobStat {
BlobStat {
size: meta.size,
e_tag: meta.e_tag,
}
}
}
struct ObjectUpload {
writer: BufWriter,
store: Arc<dyn ObjectStore>,
path: Path,
}
#[async_trait]
impl BlobUpload for ObjectUpload {
async fn write(&mut self, chunk: Bytes) -> Result<(), String> {
self.writer
.put(chunk)
.await
.map_err(|e| format!("blob upload failed: {e}"))
}
async fn finish(mut self: Box<Self>) -> Result<BlobStat, String> {
self.writer
.shutdown()
.await
.map_err(|e| format!("blob upload finalization failed: {e}"))?;
let meta = self
.store
.head(&self.path)
.await
.map_err(|e| format!("blob stat after upload failed: {e}"))?;
Ok(ObjectStorage::stat_from_meta(meta))
}
async fn abort(mut self: Box<Self>) -> Result<(), String> {
self.writer
.abort()
.await
.map_err(|e| format!("blob upload abort failed: {e}"))
}
}
#[async_trait]
impl BlobStorage for ObjectStorage {
async fn begin_upload(&self, key: &str) -> Result<Box<dyn BlobUpload>, String> {
let path = self.object_path(key)?;
let writer = BufWriter::with_capacity(self.store.clone(), path.clone(), 8 * 1024 * 1024)
.with_max_concurrency(4);
Ok(Box::new(ObjectUpload {
writer,
store: self.store.clone(),
path,
}))
}
async fn get(&self, key: &str) -> Result<BlobStream, String> {
let path = self.object_path(key)?;
let result = self
.store
.get(&path)
.await
.map_err(|e| format!("blob read failed: {e}"))?;
Ok(Box::pin(
result
.into_stream()
.map(|item| item.map_err(std::io::Error::other)),
))
}
async fn delete(&self, key: &str) -> Result<(), String> {
let path = self.object_path(key)?;
match self.store.delete(&path).await {
Ok(()) => Ok(()),
Err(object_store::Error::NotFound { .. }) => Ok(()),
Err(e) => Err(format!("blob delete failed: {e}")),
}
}
async fn exists(&self, key: &str) -> Result<bool, String> {
let path = self.object_path(key)?;
match self.store.head(&path).await {
Ok(_) => Ok(true),
Err(object_store::Error::NotFound { .. }) => Ok(false),
Err(e) => Err(format!("blob exists check failed: {e}")),
}
}
async fn health(&self) -> Result<(), String> {
let key = format!("_health/{}", uuid::Uuid::new_v4());
let mut upload = self.begin_upload(&key).await?;
upload.write(Bytes::from_static(b"ok")).await?;
let stat = upload.finish().await?;
if stat.size != 2 {
let _ = self.delete(&key).await;
return Err("storage health probe returned unexpected size".to_string());
}
self.delete(&key).await?;
Ok(())
}
}
fn normalize_prefix(prefix: &str) -> Result<String, String> {
let trimmed = prefix.trim_matches('/');
if trimmed.is_empty() {
return Ok(String::new());
}
ObjectStorage::validate_logical_key(trimmed)?;
Ok(trimmed.to_string())
}
fn filesystem_storage(
config: &AppConfig,
storage: &StorageConfig,
) -> Result<Arc<dyn BlobStorage>, String> {
let path = storage
.path
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
.unwrap_or(&config.data_dir);
let configured_root = PathBuf::from(path);
let root = if configured_root.is_absolute() {
configured_root
} else {
std::env::current_dir()
.map_err(|e| format!("failed to resolve relative storage path: {e}"))?
.join(configured_root)
};
std::fs::create_dir_all(&root).map_err(|e| {
format!(
"failed to create storage directory '{}': {e}",
root.display()
)
})?;
let store = LocalFileSystem::new_with_prefix(&root)
.map_err(|e| format!("failed to initialize filesystem storage: {e}"))?;
let prefix = normalize_prefix(&storage.prefix)?;
Ok(Arc::new(ObjectStorage {
store: Arc::new(store),
prefix,
}))
}
fn s3_storage(storage: &StorageConfig) -> Result<Arc<dyn BlobStorage>, String> {
let bucket = storage
.bucket
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
.ok_or_else(|| "storage.bucket is required for S3".to_string())?;
let client_options = ClientOptions::new()
.with_connect_timeout(Duration::from_secs(storage.connect_timeout_secs))
.with_timeout(Duration::from_secs(storage.request_timeout_secs));
let retry = RetryConfig {
max_retries: storage.max_retries,
retry_timeout: Duration::from_secs(storage.retry_timeout_secs),
..RetryConfig::default()
};
let mut builder = AmazonS3Builder::from_env()
.with_bucket_name(bucket)
.with_region(storage.region.trim())
.with_virtual_hosted_style_request(!storage.path_style)
.with_allow_http(storage.allow_http)
.with_client_options(client_options)
.with_retry(retry)
.with_checksum_algorithm(Checksum::SHA256);
if let Some(endpoint) = storage
.endpoint
.as_deref()
.map(str::trim)
.filter(|v| !v.is_empty())
{
builder = builder.with_endpoint(endpoint);
}
if let Some(value) = storage
.access_key_id
.as_deref()
.map(str::trim)
.filter(|v| !v.is_empty())
{
builder = builder.with_access_key_id(value);
}
if let Some(value) = storage
.secret_access_key
.as_deref()
.map(str::trim)
.filter(|v| !v.is_empty())
{
builder = builder.with_secret_access_key(value);
}
if let Some(value) = storage
.session_token
.as_deref()
.map(str::trim)
.filter(|v| !v.is_empty())
{
builder = builder.with_token(value);
}
match storage
.server_side_encryption
.as_deref()
.map(str::trim)
.filter(|v| !v.is_empty())
{
Some("AES256") => {
let key: object_store::aws::AmazonS3ConfigKey = "aws_server_side_encryption"
.parse()
.map_err(|e| format!("invalid S3 encryption config key: {e}"))?;
builder = builder.with_config(key, "AES256");
}
Some("aws:kms") => {
builder =
builder.with_sse_kms_encryption(storage.kms_key_id.clone().unwrap_or_default());
}
Some(other) => {
return Err(format!(
"unsupported S3 server-side encryption mode: {other}"
));
}
None => {}
}
let store = builder
.build()
.map_err(|e| format!("failed to initialize S3 storage: {e}"))?;
let prefix = normalize_prefix(&storage.prefix)?;
Ok(Arc::new(ObjectStorage {
store: Arc::new(store),
prefix,
}))
}
pub fn from_config(config: &AppConfig) -> Result<Arc<dyn BlobStorage>, String> {
match config.effective_storage_backend().as_str() {
"filesystem" => filesystem_storage(config, &config.storage),
"s3" => s3_storage(&config.storage),
other => Err(format!("unsupported storage backend: {other}")),
}
}
pub async fn read_all(storage: &dyn BlobStorage, key: &str) -> Result<Vec<u8>, String> {
let mut stream = storage.get(key).await?;
let mut data = Vec::new();
while let Some(chunk) = stream.next().await {
let chunk = chunk.map_err(|e| format!("blob stream read failed: {e}"))?;
data.extend_from_slice(&chunk);
}
Ok(data)
}
pub fn attachment_key(
user_hash: &str,
draft_id: &str,
attachment_id: &str,
) -> Result<String, String> {
for value in [user_hash, draft_id, attachment_id] {
if value.is_empty()
|| !value
.chars()
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_'))
{
return Err("invalid attachment key component".to_string());
}
}
Ok(format!(
"users/{user_hash}/drafts/{draft_id}/attachments/{attachment_id}"
))
}
#[derive(Debug, Clone)]
pub struct MigrationOptions {
pub from_filesystem: PathBuf,
pub dry_run: bool,
pub verify: bool,
}
#[derive(Debug, Default, Clone)]
pub struct MigrationReport {
pub scanned: usize,
pub migrated: usize,
pub skipped: usize,
pub failed: usize,
}
pub fn parse_migration_args(args: &[String]) -> Result<Option<MigrationOptions>, String> {
if args.get(1).map(String::as_str) != Some("storage")
|| args.get(2).map(String::as_str) != Some("migrate")
{
return Ok(None);
}
let mut from = None;
let mut dry_run = false;
let mut verify = false;
let mut index = 3;
while index < args.len() {
match args[index].as_str() {
"--from" => {
index += 1;
let value = args
.get(index)
.ok_or_else(|| "--from requires filesystem:/path".to_string())?;
let path = value.strip_prefix("filesystem:").ok_or_else(|| {
"--from currently supports filesystem:/absolute/path".to_string()
})?;
let path = PathBuf::from(path);
if !path.is_absolute() {
return Err(
"storage migration source must be an absolute filesystem path".to_string(),
);
}
from = Some(path);
}
"--to" => {
index += 1;
let value = args
.get(index)
.ok_or_else(|| "--to requires configured".to_string())?;
if value != "configured" {
return Err("--to currently accepts only 'configured'; configure [storage] for the destination backend".to_string());
}
}
"--dry-run" => dry_run = true,
"--verify" => verify = true,
other => return Err(format!("unknown storage migrate argument: {other}")),
}
index += 1;
}
Ok(Some(MigrationOptions {
from_filesystem: from.ok_or_else(|| {
"storage migrate requires --from filesystem:/absolute/path".to_string()
})?,
dry_run,
verify,
}))
}
async fn checksum_stream(storage: &dyn BlobStorage, key: &str) -> Result<String, String> {
use sha2::{Digest, Sha256};
let mut stream = storage.get(key).await?;
let mut hasher = Sha256::new();
while let Some(chunk) = stream.next().await {
let chunk = chunk.map_err(|e| format!("failed to verify migrated blob: {e}"))?;
hasher.update(&chunk);
}
Ok(format!("{:x}", hasher.finalize()))
}
fn migration_source_path(root: &FsPath, stored_key: &str) -> Result<PathBuf, String> {
let candidate = if FsPath::new(stored_key).is_absolute() {
PathBuf::from(stored_key)
} else {
ObjectStorage::validate_logical_key(stored_key)?;
root.join(stored_key)
};
let canonical_root = root.canonicalize().map_err(|e| {
format!(
"failed to resolve migration source '{}': {e}",
root.display()
)
})?;
let canonical = candidate.canonicalize().map_err(|e| {
format!(
"failed to resolve source blob '{}': {e}",
candidate.display()
)
})?;
if !canonical.starts_with(&canonical_root) {
return Err(format!(
"source blob '{}' escapes migration root",
candidate.display()
));
}
Ok(canonical)
}
pub async fn migrate_attachments(
config: &AppConfig,
destination: Arc<dyn BlobStorage>,
options: &MigrationOptions,
) -> Result<MigrationReport, String> {
if !options.from_filesystem.is_dir() {
return Err(format!(
"storage migration source '{}' is not a directory",
options.from_filesystem.display()
));
}
let mut report = MigrationReport::default();
for user_hash in crate::db::pool::list_account_user_hashes()? {
let conn = crate::db::pool::open_user_db(&config.data_dir, &user_hash)?;
for draft in crate::db::drafts::list_drafts(&conn)? {
for attachment in crate::db::drafts::get_draft_attachments(&conn, &draft.id)? {
report.scanned += 1;
let target_key = attachment_key(&user_hash, &draft.id, &attachment.id)?;
let source_path = match migration_source_path(
&options.from_filesystem,
&attachment.storage_key,
) {
Ok(path) => path,
Err(error) => {
// A logical DB key may already point at the configured destination after a previous run.
if attachment.storage_key == target_key
&& destination.exists(&target_key).await?
{
report.skipped += 1;
println!("skip {target_key}: already migrated");
continue;
}
report.failed += 1;
eprintln!("fail {target_key}: {error}");
continue;
}
};
let source_meta = match tokio::fs::metadata(&source_path).await {
Ok(meta) if meta.is_file() => meta,
Ok(_) => {
report.failed += 1;
eprintln!("fail {target_key}: source is not a regular file");
continue;
}
Err(error) => {
report.failed += 1;
eprintln!("fail {target_key}: source stat failed: {error}");
continue;
}
};
if options.dry_run {
report.migrated += 1;
println!(
"dry-run {} -> {} ({} bytes)",
source_path.display(),
target_key,
source_meta.len()
);
continue;
}
let mut source = tokio::fs::File::open(&source_path).await.map_err(|e| {
format!("failed to open source '{}': {e}", source_path.display())
})?;
let mut upload = destination.begin_upload(&target_key).await?;
use sha2::Digest as _;
let mut hasher = sha2::Sha256::new();
let mut buffer = vec![0u8; 1024 * 1024];
loop {
let read = match source.read(&mut buffer).await {
Ok(read) => read,
Err(error) => {
let _ = upload.abort().await;
return Err(format!(
"failed reading source '{}': {error}",
source_path.display()
));
}
};
if read == 0 {
break;
}
hasher.update(&buffer[..read]);
if let Err(error) = upload.write(Bytes::copy_from_slice(&buffer[..read])).await
{
let _ = upload.abort().await;
return Err(error);
}
}
let source_checksum = format!("{:x}", hasher.finalize());
let target_stat = upload.finish().await?;
if target_stat.size != source_meta.len() {
let _ = destination.delete(&target_key).await;
report.failed += 1;
eprintln!("fail {target_key}: size verification failed");
continue;
}
if options.verify {
let target_checksum = match checksum_stream(destination.as_ref(), &target_key)
.await
{
Ok(checksum) => checksum,
Err(error) => {
let _ = destination.delete(&target_key).await;
report.failed += 1;
eprintln!("fail {target_key}: checksum verification failed: {error}");
continue;
}
};
if target_checksum != source_checksum {
let _ = destination.delete(&target_key).await;
report.failed += 1;
eprintln!("fail {target_key}: checksum verification failed");
continue;
}
}
if let Err(error) = crate::db::drafts::update_attachment_storage(
&conn,
&attachment.id,
&target_key,
Some(&source_checksum),
) {
let _ = destination.delete(&target_key).await;
return Err(error);
}
report.migrated += 1;
println!("migrated {} -> {}", source_path.display(), target_key);
}
}
}
Ok(report)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::config::AppConfig;
use tempfile::TempDir;
#[tokio::test]
async fn filesystem_backend_streams_round_trip() {
let temp = TempDir::new().expect("temp dir");
let config = AppConfig {
data_dir: temp.path().to_string_lossy().into_owned(),
storage: StorageConfig {
backend: "filesystem".to_string(),
path: Some(temp.path().join("blobs").to_string_lossy().into_owned()),
..StorageConfig::default()
},
..AppConfig::default()
};
let storage = from_config(&config).expect("filesystem storage");
let key = attachment_key("user_1", "draft-1", "attachment-1").expect("key");
let mut upload = storage.begin_upload(&key).await.expect("upload");
upload
.write(Bytes::from_static(b"hello "))
.await
.expect("chunk 1");
upload
.write(Bytes::from_static(b"world"))
.await
.expect("chunk 2");
let stat = upload.finish().await.expect("finish");
assert_eq!(stat.size, 11);
assert!(storage.exists(&key).await.expect("exists"));
assert_eq!(
read_all(storage.as_ref(), &key).await.expect("read"),
b"hello world"
);
storage.delete(&key).await.expect("delete");
assert!(!storage.exists(&key).await.expect("missing"));
}
#[test]
fn attachment_keys_are_logical_and_path_safe() {
assert_eq!(
attachment_key("abc123", "draft_1", "att-1").unwrap(),
"users/abc123/drafts/draft_1/attachments/att-1"
);
assert!(attachment_key("../user", "draft", "att").is_err());
assert!(attachment_key("user", "draft/other", "att").is_err());
}
}
-1
View File
@@ -1094,4 +1094,3 @@ html[data-interface-skin="notion"][data-theme="dark"] {
--field: #191919; --selected: #353535; --hover: #2b2b2b; --shadow: rgba(0, 0, 0, .38);
--folder-bg: #202020; --message-bg: #202020; --reader-bg: #202020;
}
-1
View File
@@ -162,4 +162,3 @@
function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
-1
View File
@@ -344,4 +344,3 @@ function setBusy(button, busy) {
button.disabled = busy;
button.setAttribute("aria-busy", String(busy));
}
-1
View File
@@ -690,4 +690,3 @@ async function logout() {
localStorage.removeItem("rustbox.activeAccount");
showLogin();
}
-1
View File
@@ -450,4 +450,3 @@ async function bulkAddTag(tagId) {
toast(`Label “${tag.name}” added`);
} catch (error) { toast(error.message, "error"); }
}
-1
View File
@@ -849,4 +849,3 @@ function closeReader() {
$("#reader-empty").classList.remove("hidden");
$("#reader-content").classList.add("hidden");
}
-1
View File
@@ -1227,4 +1227,3 @@ async function sendMessage(event) {
errorBox.classList.remove("hidden");
} finally { setBusy(send, false); }
}
-1
View File
@@ -1145,4 +1145,3 @@ async function createFilterFromCurrentMessage(kind = "sender") {
dialog.showModal();
setTimeout(() => value.focus(), 20);
}
-1
View File
@@ -291,4 +291,3 @@ function composeToContact() {
openCompose("new");
$("#compose-to").value = email;
}
-1
View File
@@ -266,4 +266,3 @@ async function deleteCalendarEvent() {
try { await request(`/calendar/events/${encodeURIComponent(id)}`, { method: "DELETE" }); $("#calendar-form").classList.add("hidden"); await loadCalendar(); toast("Event deleted"); }
catch (error) { toast(error.message, "error"); }
}
-1
View File
@@ -554,4 +554,3 @@ async function autoDetectSpecialFolders() {
}
await saveSpecialFolders();
}
-1
View File
@@ -465,4 +465,3 @@ async function rebuildSearchIndex() {
button.disabled = false;
}
}
-1
View File
@@ -1022,4 +1022,3 @@ async function deleteSieve() {
toast(`Sieve script “${name}” deleted`, "success");
} catch (error) { toast(error.message, "error"); }
}