v0.2.13
RustBox CI / rustbox (push) Canceled after 0s

This commit is contained in:
Mateusz Gruszczyński committed 2026-11-02 14:23:13 +01:00
1 parent 83440f20e2
commit 12e0c99850
32 files changed
+729 -109

No files matched your search

+155
View File
@@ -0,0 +1,155 @@
use std::time::{SystemTime, UNIX_EPOCH};
use crate::db::pool::{Connection, params};
#[derive(Debug, Clone)]
pub struct BlobDeleteEvent {
pub id: String,
pub storage_key: String,
pub attempts: u32,
}
pub fn now_epoch() -> i64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
.min(i64::MAX as u64) as i64
}
pub fn enqueue(conn: &Connection, storage_key: &str) -> Result<BlobDeleteEvent, String> {
if storage_key.trim().is_empty() {
return Err("cannot enqueue an empty blob storage key".to_string());
}
let id = uuid::Uuid::new_v4().to_string();
let now = now_epoch();
let result = match crate::db::pool::configured_backend() {
crate::db::pool::DatabaseBackend::Mysql => conn.execute(
crate::dbq!(db_blob_delete_outbox::Q_001),
params![&id, storage_key, now, now],
),
_ => conn.execute(
crate::dbq!(db_blob_delete_outbox::Q_001),
params![&id, storage_key, now],
),
};
result.map_err(|e| format!("failed to enqueue blob deletion: {e}"))?;
Ok(BlobDeleteEvent {
id,
storage_key: storage_key.to_string(),
attempts: 0,
})
}
pub fn list_ready(conn: &Connection, limit: usize) -> Result<Vec<BlobDeleteEvent>, String> {
let now = now_epoch();
let mut stmt = conn
.prepare(crate::dbq!(db_blob_delete_outbox::Q_002))
.map_err(|e| format!("failed to prepare blob deletion outbox query: {e}"))?;
let query_params = match crate::db::pool::configured_backend() {
crate::db::pool::DatabaseBackend::Mysql => params![now, now, limit],
_ => params![now, limit],
};
let rows = stmt
.query_map(query_params, |row| {
Ok(BlobDeleteEvent {
id: row.get(0)?,
storage_key: row.get(1)?,
attempts: row.get::<_, i64>(2)?.max(0) as u32,
})
})
.map_err(|e| format!("failed to query blob deletion outbox: {e}"))?;
rows.map(|row| row.map_err(|e| format!("failed to read blob deletion outbox row: {e}")))
.collect()
}
pub fn claim(conn: &Connection, event: &BlobDeleteEvent, lease_secs: u64) -> Result<bool, String> {
let now = now_epoch();
let lease_until = now.saturating_add(lease_secs.max(1) as i64);
let changed = match crate::db::pool::configured_backend() {
crate::db::pool::DatabaseBackend::Mysql => conn.execute(
crate::dbq!(db_blob_delete_outbox::Q_003),
params![lease_until, &event.id, now, now],
),
_ => conn.execute(
crate::dbq!(db_blob_delete_outbox::Q_003),
params![&event.id, lease_until, now],
),
}
.map_err(|e| format!("failed to claim blob deletion: {e}"))?;
Ok(changed > 0)
}
pub fn mark_success(conn: &Connection, event: &BlobDeleteEvent) -> Result<bool, String> {
let changed = conn
.execute(
crate::dbq!(db_blob_delete_outbox::Q_004),
params![&event.id],
)
.map_err(|e| format!("failed to acknowledge blob deletion: {e}"))?;
Ok(changed > 0)
}
pub fn mark_failure(
conn: &Connection,
event: &BlobDeleteEvent,
error: &str,
) -> Result<bool, String> {
let attempts = event.attempts.saturating_add(1);
let backoff = 2u64.saturating_pow(attempts.min(12)).min(3600);
let available_at = now_epoch().saturating_add(backoff as i64);
let now = now_epoch();
let truncated: String = error.chars().take(2048).collect();
let changed = match crate::db::pool::configured_backend() {
crate::db::pool::DatabaseBackend::Mysql => conn.execute(
crate::dbq!(db_blob_delete_outbox::Q_005),
params![attempts, available_at, &truncated, now, &event.id],
),
_ => conn.execute(
crate::dbq!(db_blob_delete_outbox::Q_005),
params![&event.id, attempts, available_at, &truncated, now],
),
}
.map_err(|e| format!("failed to update blob deletion retry state: {e}"))?;
Ok(changed > 0)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::db::pool::open_test_db;
#[test]
fn enqueue_participates_in_caller_transaction() {
let conn = open_test_db();
{
let transaction = conn.unchecked_transaction().unwrap();
enqueue(&transaction, "users/test/drafts/d/attachments/a").unwrap();
}
assert!(list_ready(&conn, 10).unwrap().is_empty());
}
#[test]
fn failed_deletions_are_retried_instead_of_dead_lettered() {
let conn = open_test_db();
let event = enqueue(&conn, "users/test/drafts/d/attachments/a").unwrap();
assert!(claim(&conn, &event, 60).unwrap());
assert!(mark_failure(&conn, &event, "storage unavailable").unwrap());
let count: i64 = conn
.query_row(
"SELECT COUNT(*) FROM blob_delete_outbox WHERE id=?1",
params![&event.id],
|row| row.get(0),
)
.unwrap();
assert_eq!(count, 1);
}
#[test]
fn successful_delete_is_acknowledged() {
let conn = open_test_db();
let event = enqueue(&conn, "users/test/drafts/d/attachments/a").unwrap();
assert!(mark_success(&conn, &event).unwrap());
assert!(list_ready(&conn, 10).unwrap().is_empty());
}
}
+61
View File
@@ -257,6 +257,67 @@ pub fn get_draft_attachments(
Ok(attachments)
}
pub fn lock_draft_for_update(conn: &Connection, id: &str) -> Result<bool, String> {
match conn.query_row(crate::dbq!(db_drafts::Q_010), params![id], |_row| Ok(())) {
Ok(()) => Ok(true),
Err(crate::db::pool::DbError::QueryReturnedNoRows) => Ok(false),
Err(e) => Err(format!("Failed to lock draft: {e}")),
}
}
pub fn get_draft_attachment_for_update(
conn: &Connection,
draft_id: &str,
attachment_id: &str,
) -> Result<Option<DraftAttachment>, String> {
let result = conn.query_row(
crate::dbq!(db_drafts::Q_011),
params![draft_id, attachment_id],
|row| {
Ok(DraftAttachment {
id: row.get(0)?,
draft_id: row.get(1)?,
filename: row.get(2)?,
content_type: row.get(3)?,
size: row.get(4)?,
storage_key: row.get(5)?,
checksum_sha256: row.get(6)?,
created_at: row.get(7)?,
})
},
);
match result {
Ok(attachment) => Ok(Some(attachment)),
Err(crate::db::pool::DbError::QueryReturnedNoRows) => Ok(None),
Err(e) => Err(format!("Failed to lock draft attachment: {e}")),
}
}
pub fn get_draft_attachments_for_update(
conn: &Connection,
draft_id: &str,
) -> Result<Vec<DraftAttachment>, String> {
let mut stmt = conn
.prepare(crate::dbq!(db_drafts::Q_012))
.map_err(|e| format!("Failed to prepare locked draft attachments query: {e}"))?;
let rows = stmt
.query_map(params![draft_id], |row| {
Ok(DraftAttachment {
id: row.get(0)?,
draft_id: row.get(1)?,
filename: row.get(2)?,
content_type: row.get(3)?,
size: row.get(4)?,
storage_key: row.get(5)?,
checksum_sha256: row.get(6)?,
created_at: row.get(7)?,
})
})
.map_err(|e| format!("Failed to query locked draft attachments: {e}"))?;
rows.map(|row| row.map_err(|e| format!("Failed to read locked attachment row: {e}")))
.collect()
}
pub fn delete_draft_attachment(conn: &Connection, id: &str) -> Result<bool, String> {
let deleted = conn
.execute(crate::dbq!(db_drafts::Q_008), params![id])
+1
View File
@@ -1,3 +1,4 @@
pub mod blob_delete_outbox;
pub mod calendar;
pub mod contact_groups;
pub mod contacts;
+14
View File
@@ -30,6 +30,10 @@ const SQLITE_MIGRATIONS: &[(u32, &str)] = &[
48,
include_str!("../../migrations/sqlite/V048__folder_counter_generation.sql"),
),
(
49,
include_str!("../../migrations/sqlite/V049__blob_delete_outbox.sql"),
),
];
const MYSQL_MIGRATIONS: &[(u32, &str)] = &[
(
@@ -40,6 +44,10 @@ const MYSQL_MIGRATIONS: &[(u32, &str)] = &[
48,
include_str!("../../migrations/mysql/V048__folder_counter_generation.sql"),
),
(
49,
include_str!("../../migrations/mysql/V049__blob_delete_outbox.sql"),
),
];
const PGSQL_MIGRATIONS: &[(u32, &str)] = &[
(
@@ -50,9 +58,14 @@ const PGSQL_MIGRATIONS: &[(u32, &str)] = &[
48,
include_str!("../../migrations/pgsql/V048__folder_counter_generation.sql"),
),
(
49,
include_str!("../../migrations/pgsql/V049__blob_delete_outbox.sql"),
),
];
const TENANT_TABLES: &[&str] = &[
"blob_delete_outbox",
"calendar_events",
"calendar_settings",
"contact_group_members",
@@ -83,6 +96,7 @@ const TENANT_TABLES: &[&str] = &[
];
const IMPORT_TABLES: &[&str] = &[
"blob_delete_outbox",
"calendar_events",
"calendar_settings",
"contact_groups",
+2
View File
@@ -121,6 +121,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.map_err(std::io::Error::other)?;
}
let _search_worker = search_engine.start_worker();
let _blob_delete_worker =
crate::storage::start_blob_delete_worker(config.clone(), storage.clone());
let event_bus = Arc::new(realtime::events::EventBus::new());
let idle_manager = Arc::new(realtime::idle::IdleManager::new());
+11
View File
@@ -167,6 +167,9 @@ pub mod db_drafts {
pub const Q_008: &str = "DELETE FROM draft_attachments WHERE id = ?";
pub const Q_009: &str =
"UPDATE draft_attachments SET file_path=?, checksum_sha256=? WHERE id=?";
pub const Q_010: &str = "SELECT id FROM drafts WHERE id=? FOR UPDATE";
pub const Q_011: &str = "SELECT id, draft_id, filename, content_type, size, file_path, checksum_sha256, created_at FROM draft_attachments WHERE draft_id=? AND id=? FOR UPDATE";
pub const Q_012: &str = "SELECT id, draft_id, filename, content_type, size, file_path, checksum_sha256, created_at FROM draft_attachments WHERE draft_id=? ORDER BY created_at ASC FOR UPDATE";
}
pub mod db_folder_preferences {
@@ -527,3 +530,11 @@ pub mod db_search_outbox {
pub const Q_006: &str = "UPDATE search_outbox SET claimed_until=? WHERE id=? AND generation=? AND dead_letter=0 AND available_at <= ? AND claimed_until <= ?";
pub const Q_007: &str = "DELETE FROM search_outbox WHERE updated_at < ?";
}
pub mod db_blob_delete_outbox {
pub const Q_001: &str = "INSERT INTO blob_delete_outbox (id, storage_key, attempts, available_at, claimed_until, last_error, created_at, updated_at) VALUES (?, ?, 0, 0, 0, NULL, ?, ?)";
pub const Q_002: &str = "SELECT id, storage_key, attempts FROM blob_delete_outbox WHERE available_at <= ? AND claimed_until <= ? ORDER BY updated_at ASC LIMIT ?";
pub const Q_003: &str = "UPDATE blob_delete_outbox SET claimed_until=? WHERE id=? AND available_at <= ? AND claimed_until <= ?";
pub const Q_004: &str = "DELETE FROM blob_delete_outbox WHERE id=?";
pub const Q_005: &str = "UPDATE blob_delete_outbox SET attempts=?, available_at=?, claimed_until=0, last_error=?, updated_at=? WHERE id=?";
}
+11
View File
@@ -168,6 +168,9 @@ pub mod db_drafts {
pub const Q_008: &str = "DELETE FROM draft_attachments WHERE id = $1";
pub const Q_009: &str =
"UPDATE draft_attachments SET file_path=$1, checksum_sha256=$2 WHERE id=$3";
pub const Q_010: &str = "SELECT id FROM drafts WHERE id=$1 FOR UPDATE";
pub const Q_011: &str = "SELECT id, draft_id, filename, content_type, size, file_path, checksum_sha256, created_at FROM draft_attachments WHERE draft_id=$1 AND id=$2 FOR UPDATE";
pub const Q_012: &str = "SELECT id, draft_id, filename, content_type, size, file_path, checksum_sha256, created_at FROM draft_attachments WHERE draft_id=$1 ORDER BY created_at ASC FOR UPDATE";
}
pub mod db_folder_preferences {
@@ -528,3 +531,11 @@ pub mod db_search_outbox {
pub const Q_006: &str = "UPDATE search_outbox SET claimed_until=$3 WHERE id=$1 AND generation=$2 AND dead_letter=0 AND available_at <= $4 AND claimed_until <= $4";
pub const Q_007: &str = "DELETE FROM search_outbox WHERE updated_at < $1";
}
pub mod db_blob_delete_outbox {
pub const Q_001: &str = "INSERT INTO blob_delete_outbox (id, storage_key, attempts, available_at, claimed_until, last_error, created_at, updated_at) VALUES ($1, $2, 0, 0, 0, NULL, $3, $3)";
pub const Q_002: &str = "SELECT id, storage_key, attempts FROM blob_delete_outbox WHERE available_at <= $1 AND claimed_until <= $1 ORDER BY updated_at ASC LIMIT $2";
pub const Q_003: &str = "UPDATE blob_delete_outbox SET claimed_until=$2 WHERE id=$1 AND available_at <= $3 AND claimed_until <= $3";
pub const Q_004: &str = "DELETE FROM blob_delete_outbox WHERE id=$1";
pub const Q_005: &str = "UPDATE blob_delete_outbox SET attempts=$2, available_at=$3, claimed_until=0, last_error=$4, updated_at=$5 WHERE id=$1";
}
+11
View File
@@ -165,6 +165,9 @@ pub mod db_drafts {
pub const Q_008: &str = "DELETE FROM draft_attachments WHERE id = ?1";
pub const Q_009: &str =
"UPDATE draft_attachments SET file_path=?1, checksum_sha256=?2 WHERE id=?3";
pub const Q_010: &str = "SELECT id FROM drafts WHERE id=?1";
pub const Q_011: &str = "SELECT id, draft_id, filename, content_type, size, file_path, checksum_sha256, created_at FROM draft_attachments WHERE draft_id=?1 AND id=?2";
pub const Q_012: &str = "SELECT id, draft_id, filename, content_type, size, file_path, checksum_sha256, created_at FROM draft_attachments WHERE draft_id=?1 ORDER BY created_at ASC";
}
pub mod db_folder_preferences {
@@ -782,3 +785,11 @@ pub mod db_search_outbox {
pub const Q_006: &str = "UPDATE search_outbox SET claimed_until=?3 WHERE id=?1 AND generation=?2 AND dead_letter=0 AND available_at <= ?4 AND claimed_until <= ?4";
pub const Q_007: &str = "DELETE FROM search_outbox WHERE updated_at < ?1";
}
pub mod db_blob_delete_outbox {
pub const Q_001: &str = "INSERT INTO blob_delete_outbox (id, storage_key, attempts, available_at, claimed_until, last_error, created_at, updated_at) VALUES (?1, ?2, 0, 0, 0, NULL, ?3, ?3)";
pub const Q_002: &str = "SELECT id, storage_key, attempts FROM blob_delete_outbox WHERE available_at <= ?1 AND claimed_until <= ?1 ORDER BY updated_at ASC LIMIT ?2";
pub const Q_003: &str = "UPDATE blob_delete_outbox SET claimed_until=?2 WHERE id=?1 AND available_at <= ?3 AND claimed_until <= ?3";
pub const Q_004: &str = "DELETE FROM blob_delete_outbox WHERE id=?1";
pub const Q_005: &str = "UPDATE blob_delete_outbox SET attempts=?2, available_at=?3, claimed_until=0, last_error=?4, updated_at=?5 WHERE id=?1";
}
+62 -19
View File
@@ -11,7 +11,7 @@ use crate::auth::session::SessionState;
use crate::config::AppConfig;
use crate::db;
use crate::error::AppError;
use crate::storage::{BlobStorage, attachment_key};
use crate::storage::{self, BlobStorage, attachment_key};
#[derive(Debug, Serialize)]
struct UploadResponse {
@@ -73,7 +73,13 @@ pub async fn upload_attachment(
let chunk = match field.chunk().await {
Ok(chunk) => chunk,
Err(e) => {
let _ = writer.abort().await;
if writer.abort().await.is_err()
&& let Err(cleanup_error) =
storage::enqueue_unreferenced_delete(&conn, storage.as_ref(), &key)
.await
{
tracing::error!(error = %cleanup_error, storage_key = %key, "Failed to queue aborted attachment cleanup");
}
return Err(AppError::BadRequest(format!(
"Failed to read field data: {e}"
)));
@@ -82,7 +88,12 @@ pub async fn upload_attachment(
let Some(chunk) = chunk else { break };
size = size.saturating_add(chunk.len());
if size > max_attachment_size {
let _ = writer.abort().await;
if writer.abort().await.is_err()
&& let Err(cleanup_error) =
storage::enqueue_unreferenced_delete(&conn, storage.as_ref(), &key).await
{
tracing::error!(error = %cleanup_error, storage_key = %key, "Failed to queue oversized attachment cleanup");
}
return Err(AppError::BadRequest(format!(
"File '{}' exceeds maximum size of {} MiB",
filename, config.max_attachment_size_mb
@@ -90,18 +101,37 @@ pub async fn upload_attachment(
}
checksum.update(&chunk);
if let Err(e) = writer.write(chunk).await {
let _ = writer.abort().await;
if writer.abort().await.is_err()
&& let Err(cleanup_error) =
storage::enqueue_unreferenced_delete(&conn, storage.as_ref(), &key).await
{
tracing::error!(error = %cleanup_error, storage_key = %key, "Failed to queue interrupted attachment cleanup");
}
return Err(AppError::ServiceUnavailable(format!(
"Failed to upload attachment: {e}"
)));
}
}
let stat = writer.finish().await.map_err(|e| {
AppError::ServiceUnavailable(format!("Failed to finalize attachment upload: {e}"))
})?;
let stat = match writer.finish().await {
Ok(stat) => stat,
Err(e) => {
if let Err(cleanup_error) =
storage::enqueue_unreferenced_delete(&conn, storage.as_ref(), &key).await
{
tracing::error!(error = %cleanup_error, storage_key = %key, "Failed to queue failed-finalize attachment cleanup");
}
return Err(AppError::ServiceUnavailable(format!(
"Failed to finalize attachment upload: {e}"
)));
}
};
if stat.size != size as u64 {
let _ = storage.delete(&key).await;
if let Err(cleanup_error) =
storage::enqueue_unreferenced_delete(&conn, storage.as_ref(), &key).await
{
tracing::error!(error = %cleanup_error, storage_key = %key, "Failed to queue invalid attachment cleanup");
}
return Err(AppError::InternalError(format!(
"Attachment size verification failed for '{filename}'"
)));
@@ -118,7 +148,11 @@ pub async fn upload_attachment(
checksum_sha256: Some(&checksum),
};
if let Err(e) = db::drafts::add_draft_attachment(&conn, &attachment) {
let _ = storage.delete(&key).await;
if let Err(cleanup_error) =
storage::enqueue_unreferenced_delete(&conn, storage.as_ref(), &key).await
{
tracing::error!(error = %cleanup_error, storage_key = %key, "Failed to queue unattached blob cleanup");
}
return Err(AppError::InternalError(e));
}
@@ -180,21 +214,30 @@ pub async fn delete_attachment(
let conn = db::pool::open_user_db(&config.data_dir, &session.user_hash)
.map_err(|e| AppError::InternalError(format!("Failed to open database: {e}")))?;
let attachments =
db::drafts::get_draft_attachments(&conn, &draft_id).map_err(AppError::InternalError)?;
let attachment = attachments
.iter()
.find(|a| a.id == attachment_id)
.ok_or_else(|| AppError::NotFound("Attachment not found".to_string()))?;
let transaction = conn.unchecked_transaction().map_err(|e| {
AppError::InternalError(format!(
"Failed to start attachment delete transaction: {e}"
))
})?;
let attachment =
db::drafts::get_draft_attachment_for_update(&transaction, &draft_id, &attachment_id)
.map_err(AppError::InternalError)?
.ok_or_else(|| AppError::NotFound("Attachment not found".to_string()))?;
let storage_key = attachment.storage_key.clone();
let deleted = db::drafts::delete_draft_attachment(&conn, &attachment_id)
let cleanup_event = db::blob_delete_outbox::enqueue(&transaction, &storage_key)
.map_err(AppError::InternalError)?;
let deleted = db::drafts::delete_draft_attachment(&transaction, &attachment_id)
.map_err(AppError::InternalError)?;
if !deleted {
return Err(AppError::NotFound("Attachment not found".to_string()));
}
if let Err(error) = storage.delete(&storage_key).await {
tracing::warn!(error = %error, storage_key = %storage_key, "Failed to remove deleted attachment blob");
transaction.commit().map_err(|e| {
AppError::InternalError(format!("Failed to commit attachment deletion: {e}"))
})?;
if let Err(error) =
storage::process_queued_delete(&conn, storage.as_ref(), &cleanup_event).await
{
tracing::warn!(error = %error, storage_key = %storage_key, "Failed to process queued attachment blob deletion");
}
Ok(Json(DeleteResponse {
+31 -12
View File
@@ -1179,25 +1179,44 @@ pub async fn delete_draft_handler(
delete_imap_draft_source(imap_client.as_ref(), &creds, &drafts_folder, source).await?;
}
let attachments = if existing.is_some() {
db::drafts::get_draft_attachments(&conn, &id).map_err(AppError::InternalError)?
} else {
Vec::new()
};
if existing.is_some() {
let deleted = db::drafts::delete_draft(&conn, &id).map_err(AppError::InternalError)?;
let (attachments, cleanup_events) = if existing.is_some() {
let transaction = conn.unchecked_transaction().map_err(|e| {
AppError::InternalError(format!("Failed to start draft delete transaction: {e}"))
})?;
if !db::drafts::lock_draft_for_update(&transaction, &id).map_err(AppError::InternalError)? {
return Err(AppError::NotFound("Draft not found".to_string()));
}
let attachments = db::drafts::get_draft_attachments_for_update(&transaction, &id)
.map_err(AppError::InternalError)?;
let mut events = Vec::with_capacity(attachments.len());
for attachment in &attachments {
events.push(
db::blob_delete_outbox::enqueue(&transaction, &attachment.storage_key)
.map_err(AppError::InternalError)?,
);
}
let deleted =
db::drafts::delete_draft(&transaction, &id).map_err(AppError::InternalError)?;
if !deleted {
return Err(AppError::NotFound("Draft not found".to_string()));
}
}
transaction.commit().map_err(|e| {
AppError::InternalError(format!("Failed to commit draft deletion: {e}"))
})?;
(attachments, events)
} else {
(Vec::new(), Vec::new())
};
let mut attachments_deleted = 0usize;
for attachment in &attachments {
match storage.delete(&attachment.storage_key).await {
Ok(()) => attachments_deleted += 1,
for (attachment, event) in attachments.iter().zip(cleanup_events.iter()) {
match storage::process_queued_delete(&conn, storage.as_ref(), event).await {
Ok(true) => attachments_deleted += 1,
Ok(false) => {}
Err(error) => tracing::warn!(
error = %error,
storage_key = %attachment.storage_key,
"Failed to remove deleted draft attachment blob"
"Failed to process queued draft attachment deletion"
),
}
}
+49 -10
View File
@@ -457,26 +457,65 @@ async fn load_draft_attachments(
}
async fn cleanup_draft(data_dir: &str, user_hash: &str, draft_id: &str, storage: &dyn BlobStorage) {
let Ok(conn) = db::pool::open_user_db(data_dir, user_hash) else {
return;
let conn = match db::pool::open_user_db(data_dir, user_hash) {
Ok(conn) => conn,
Err(error) => {
tracing::warn!(error = %error, "Failed to open database for draft cleanup after send");
return;
}
};
let attachments = db::drafts::get_draft_attachments(&conn, draft_id).unwrap_or_default();
let deleted = match db::drafts::delete_draft(&conn, draft_id) {
let transaction = match conn.unchecked_transaction() {
Ok(transaction) => transaction,
Err(error) => {
tracing::warn!(error = %error, "Failed to start draft cleanup transaction after send");
return;
}
};
match db::drafts::lock_draft_for_update(&transaction, draft_id) {
Ok(true) => {}
Ok(false) => return,
Err(error) => {
tracing::warn!(error = %error, "Failed to lock draft for cleanup after send");
return;
}
}
let attachments = match db::drafts::get_draft_attachments_for_update(&transaction, draft_id) {
Ok(attachments) => attachments,
Err(error) => {
tracing::warn!(error = %error, "Failed to load draft attachments for cleanup after send");
return;
}
};
let mut cleanup_events = Vec::with_capacity(attachments.len());
for attachment in &attachments {
match db::blob_delete_outbox::enqueue(&transaction, &attachment.storage_key) {
Ok(event) => cleanup_events.push(event),
Err(error) => {
tracing::warn!(error = %error, storage_key = %attachment.storage_key, "Failed to queue draft attachment cleanup after send");
return;
}
}
}
let deleted = match db::drafts::delete_draft(&transaction, draft_id) {
Ok(deleted) => deleted,
Err(e) => {
tracing::warn!(error = %e, "Failed to delete draft after send");
Err(error) => {
tracing::warn!(error = %error, "Failed to delete draft after send");
return;
}
};
if !deleted {
return;
}
for attachment in attachments {
if let Err(e) = storage.delete(&attachment.storage_key).await {
if let Err(error) = transaction.commit() {
tracing::warn!(error = %error, "Failed to commit draft cleanup after send");
return;
}
for (attachment, event) in attachments.iter().zip(cleanup_events.iter()) {
if let Err(error) = storage::process_queued_delete(&conn, storage, event).await {
tracing::warn!(
error = %e,
error = %error,
storage_key = %attachment.storage_key,
"Failed to delete draft attachment after send"
"Failed to process queued draft attachment cleanup after send"
);
}
}
+117 -7
View File
@@ -160,13 +160,17 @@ impl BlobStorage for ObjectStorage {
}
async fn health(&self) -> Result<(), String> {
let key = format!("_health/{}", uuid::Uuid::new_v4());
let key = "_health/probe".to_string();
let mut upload = self.begin_upload(&key).await?;
upload.write(Bytes::from_static(b"ok")).await?;
let stat = upload.finish().await?;
if stat.size != 2 {
let _ = self.delete(&key).await;
return Err("storage health probe returned unexpected size".to_string());
return match self.delete(&key).await {
Ok(()) => Err("storage health probe returned unexpected size".to_string()),
Err(cleanup_error) => Err(format!(
"storage health probe returned unexpected size and cleanup failed: {cleanup_error}"
)),
};
}
self.delete(&key).await?;
Ok(())
@@ -311,6 +315,89 @@ pub fn from_config(config: &AppConfig) -> Result<Arc<dyn BlobStorage>, String> {
}
}
const BLOB_DELETE_BATCH_SIZE: usize = 100;
const BLOB_DELETE_LEASE_SECS: u64 = 60;
const BLOB_DELETE_WORKER_INTERVAL_SECS: u64 = 30;
pub async fn process_queued_delete(
conn: &crate::db::pool::Connection,
storage: &dyn BlobStorage,
event: &crate::db::blob_delete_outbox::BlobDeleteEvent,
) -> Result<bool, String> {
if !crate::db::blob_delete_outbox::claim(conn, event, BLOB_DELETE_LEASE_SECS)? {
return Ok(false);
}
match storage.delete(&event.storage_key).await {
Ok(()) => {
crate::db::blob_delete_outbox::mark_success(conn, event)?;
Ok(true)
}
Err(error) => {
crate::db::blob_delete_outbox::mark_failure(conn, event, &error)?;
tracing::warn!(
error = %error,
storage_key = %event.storage_key,
attempts = event.attempts.saturating_add(1),
"Blob deletion failed; cleanup remains queued"
);
Ok(false)
}
}
}
pub async fn enqueue_unreferenced_delete(
conn: &crate::db::pool::Connection,
storage: &dyn BlobStorage,
storage_key: &str,
) -> Result<bool, String> {
match crate::db::blob_delete_outbox::enqueue(conn, storage_key) {
Ok(event) => process_queued_delete(conn, storage, &event).await,
Err(queue_error) => match storage.delete(storage_key).await {
Ok(()) => Ok(true),
Err(delete_error) => Err(format!(
"failed to persist blob cleanup ({queue_error}); direct delete also failed: {delete_error}"
)),
},
}
}
pub async fn process_blob_delete_outbox(
config: &AppConfig,
storage: &dyn BlobStorage,
) -> Result<usize, String> {
let mut processed = 0usize;
for user_hash in crate::db::pool::list_account_user_hashes()? {
let conn = crate::db::pool::open_user_db(&config.data_dir, &user_hash)?;
let events = crate::db::blob_delete_outbox::list_ready(&conn, BLOB_DELETE_BATCH_SIZE)?;
for event in events {
if process_queued_delete(&conn, storage, &event).await? {
processed = processed.saturating_add(1);
}
}
}
Ok(processed)
}
pub fn start_blob_delete_worker(
config: Arc<AppConfig>,
storage: Arc<dyn BlobStorage>,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
loop {
match process_blob_delete_outbox(&config, storage.as_ref()).await {
Ok(processed) if processed > 0 => {
tracing::debug!(processed, "Processed queued blob deletions");
}
Ok(_) => {}
Err(error) => {
tracing::warn!(%error, "Blob deletion worker pass failed");
}
}
tokio::time::sleep(Duration::from_secs(BLOB_DELETE_WORKER_INTERVAL_SECS)).await;
}
})
}
pub async fn read_all(storage: &dyn BlobStorage, key: &str) -> Result<Vec<u8>, String> {
let mut stream = storage.get(key).await?;
let mut data = Vec::new();
@@ -537,7 +624,11 @@ pub async fn migrate_attachments(
let source_checksum = format!("{:x}", hasher.finalize());
let target_stat = upload.finish().await?;
if target_stat.size != source_meta.len() {
let _ = destination.delete(&target_key).await;
if let Err(cleanup_error) =
enqueue_unreferenced_delete(&conn, destination.as_ref(), &target_key).await
{
eprintln!("fail {target_key}: cleanup queue failed: {cleanup_error}");
}
report.failed += 1;
eprintln!("fail {target_key}: size verification failed");
continue;
@@ -548,14 +639,29 @@ pub async fn migrate_attachments(
{
Ok(checksum) => checksum,
Err(error) => {
let _ = destination.delete(&target_key).await;
if let Err(cleanup_error) = enqueue_unreferenced_delete(
&conn,
destination.as_ref(),
&target_key,
)
.await
{
eprintln!(
"fail {target_key}: cleanup queue failed: {cleanup_error}"
);
}
report.failed += 1;
eprintln!("fail {target_key}: checksum verification failed: {error}");
continue;
}
};
if target_checksum != source_checksum {
let _ = destination.delete(&target_key).await;
if let Err(cleanup_error) =
enqueue_unreferenced_delete(&conn, destination.as_ref(), &target_key)
.await
{
eprintln!("fail {target_key}: cleanup queue failed: {cleanup_error}");
}
report.failed += 1;
eprintln!("fail {target_key}: checksum verification failed");
continue;
@@ -567,7 +673,11 @@ pub async fn migrate_attachments(
&target_key,
Some(&source_checksum),
) {
let _ = destination.delete(&target_key).await;
if let Err(cleanup_error) =
enqueue_unreferenced_delete(&conn, destination.as_ref(), &target_key).await
{
tracing::error!(error = %cleanup_error, storage_key = %target_key, "Failed to queue storage migration cleanup");
}
return Err(error);
}
report.migrated += 1;