1 parent
0d9f3ff447
commit
95c0587c1e
26 files changed
+237
-52
No files matched your search
@@ -164,6 +164,45 @@ pub fn mark_failure(
|
||||
Ok(changed > 0)
|
||||
}
|
||||
|
||||
pub fn defer_transient_failure(
|
||||
conn: &Connection,
|
||||
event: &OutboxEvent,
|
||||
error: &str,
|
||||
delay_secs: u64,
|
||||
) -> Result<bool, String> {
|
||||
let available_at = now_epoch().saturating_add(delay_secs.max(1) as i64);
|
||||
let now = now_epoch();
|
||||
let truncated: String = error.chars().take(2048).collect();
|
||||
let changed = match crate::db::pool::configured_backend() {
|
||||
crate::db::pool::DatabaseBackend::Mysql => conn.execute(
|
||||
crate::dbq!(db_search_outbox::Q_004),
|
||||
params![
|
||||
event.attempts,
|
||||
available_at,
|
||||
false,
|
||||
&truncated,
|
||||
now,
|
||||
&event.id,
|
||||
&event.generation
|
||||
],
|
||||
),
|
||||
_ => conn.execute(
|
||||
crate::dbq!(db_search_outbox::Q_004),
|
||||
params![
|
||||
&event.id,
|
||||
&event.generation,
|
||||
event.attempts,
|
||||
available_at,
|
||||
false,
|
||||
&truncated,
|
||||
now
|
||||
],
|
||||
),
|
||||
}
|
||||
.map_err(|e| format!("failed to defer transient search event: {e}"))?;
|
||||
Ok(changed > 0)
|
||||
}
|
||||
|
||||
pub fn clear_before(conn: &Connection, updated_before: i64) -> Result<usize, String> {
|
||||
conn.execute(
|
||||
crate::dbq!(db_search_outbox::Q_007),
|
||||
|
||||
@@ -20,6 +20,7 @@ pub struct AuthPolicyResponse {
|
||||
remember_me_enabled: bool,
|
||||
remember_me_days: u64,
|
||||
session_timeout_hours: u64,
|
||||
pgp_enabled: bool,
|
||||
}
|
||||
|
||||
pub async fn auth_policy(Extension(config): Extension<Arc<AppConfig>>) -> Json<AuthPolicyResponse> {
|
||||
@@ -27,6 +28,7 @@ pub async fn auth_policy(Extension(config): Extension<Arc<AppConfig>>) -> Json<A
|
||||
remember_me_enabled: config.remember_me_enabled,
|
||||
remember_me_days: config.remember_me_days,
|
||||
session_timeout_hours: config.session_timeout_hours,
|
||||
pgp_enabled: config.pgp_enabled,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
+62
-8
@@ -275,16 +275,38 @@ impl SearchEngine {
|
||||
let batch_size = *worker_batch_size;
|
||||
let max_attempts = *worker_max_attempts;
|
||||
Some(tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(Duration::from_secs(interval_secs));
|
||||
let mut failure_streak = 0u32;
|
||||
loop {
|
||||
interval.tick().await;
|
||||
match process_search_outbox(&backend, &data_dir, batch_size, max_attempts).await {
|
||||
Ok(processed) if processed > 0 => {
|
||||
tracing::debug!(processed, "processed OpenSearch outbox events");
|
||||
let delay_secs = match process_search_outbox(
|
||||
&backend,
|
||||
&data_dir,
|
||||
batch_size,
|
||||
max_attempts,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(processed) => {
|
||||
failure_streak = 0;
|
||||
if processed > 0 {
|
||||
tracing::debug!(processed, "processed OpenSearch outbox events");
|
||||
}
|
||||
interval_secs
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(error) => tracing::warn!(%error, "OpenSearch outbox worker pass failed"),
|
||||
}
|
||||
Err(error) => {
|
||||
failure_streak = failure_streak.saturating_add(1);
|
||||
let multiplier = 2u64.saturating_pow(failure_streak.min(5));
|
||||
let retry_after_secs = interval_secs
|
||||
.saturating_mul(multiplier)
|
||||
.clamp(interval_secs, 60);
|
||||
tracing::warn!(
|
||||
%error,
|
||||
retry_after_secs,
|
||||
"OpenSearch outbox worker pass failed; backing off"
|
||||
);
|
||||
retry_after_secs
|
||||
}
|
||||
};
|
||||
tokio::time::sleep(Duration::from_secs(delay_secs)).await;
|
||||
}
|
||||
}))
|
||||
}
|
||||
@@ -1033,6 +1055,14 @@ impl OpenSearchUserIndex {
|
||||
}
|
||||
}
|
||||
|
||||
fn is_opensearch_backpressure_error(error: &str) -> bool {
|
||||
let lower = error.to_ascii_lowercase();
|
||||
lower.contains("429 too many requests")
|
||||
|| lower.contains("status 429")
|
||||
|| lower.contains("flood-stage watermark")
|
||||
|| lower.contains("read-only-allow-delete")
|
||||
}
|
||||
|
||||
async fn process_search_outbox(
|
||||
backend: &OpenSearchBackend,
|
||||
data_dir: &str,
|
||||
@@ -1082,6 +1112,17 @@ async fn process_search_outbox(
|
||||
processed += 1;
|
||||
}
|
||||
Err(error) => {
|
||||
if is_opensearch_backpressure_error(&error) {
|
||||
crate::db::search_outbox::defer_transient_failure(
|
||||
&conn,
|
||||
&event,
|
||||
&error,
|
||||
30,
|
||||
)?;
|
||||
return Err(format!(
|
||||
"OpenSearch temporarily rejected indexing for tenant {user_hash}: {error}"
|
||||
));
|
||||
}
|
||||
crate::db::search_outbox::mark_failure(&conn, &event, &error, max_attempts)?;
|
||||
tracing::warn!(%error, event_id = %event.id, %user_hash, "OpenSearch outbox event failed");
|
||||
}
|
||||
@@ -1105,6 +1146,19 @@ mod tests {
|
||||
(tmp, user_index)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn opensearch_backpressure_classifier_detects_flood_stage_and_429() {
|
||||
assert!(is_opensearch_backpressure_error(
|
||||
"OpenSearch document upsert failed (429 Too Many Requests): flood-stage watermark"
|
||||
));
|
||||
assert!(is_opensearch_backpressure_error(
|
||||
"index blocked by read-only-allow-delete block"
|
||||
));
|
||||
assert!(!is_opensearch_backpressure_error(
|
||||
"OpenSearch document upsert failed (500 Internal Server Error)"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn opensearch_event_lease_covers_timeout_and_failover_budget() {
|
||||
let backend = OpenSearchBackend {
|
||||
|
||||
+2
-2
@@ -943,7 +943,7 @@
|
||||
</div><button id="identity-new" class="button primary" type="button">New identity</button>
|
||||
</div>
|
||||
<div class="identity-overview">
|
||||
<div class="identity-overview-hint"><strong>Configured identities</strong><span class="muted">Select an
|
||||
<div class="identity-overview-hint"><strong>Configured identities</strong><span id="identity-overview-help" class="muted">Select an
|
||||
identity to edit its sender data, text/HTML footer, image and OpenPGP defaults.</span></div>
|
||||
<div id="identity-list" class="identity-list"></div>
|
||||
</div>
|
||||
@@ -1249,7 +1249,7 @@
|
||||
</div>
|
||||
</section>
|
||||
<section class="identity-modal-section identity-modal-options">
|
||||
<div class="identity-modal-heading"><strong>Defaults</strong><span class="muted">Choose where this signature
|
||||
<div class="identity-modal-heading"><strong>Defaults</strong><span id="identity-defaults-help" class="muted">Choose where this signature
|
||||
is inserted and, when OpenPGP is configured, its default behavior.</span></div>
|
||||
<div class="identity-toggle-grid">
|
||||
<label class="checkbox-row"><input id="identity-signature-new" type="checkbox"><span>Signature on new
|
||||
|
||||
@@ -33,7 +33,7 @@
|
||||
pgpKeys: [],
|
||||
pgpPrefs: { wkdEnabled: true, hkpEnabled: false, requireTrustedEncryption: false },
|
||||
pgpRecipientStatus: [],
|
||||
authPolicy: { rememberMeEnabled: true, rememberMeDays: 30, sessionTimeoutHours: 24 },
|
||||
authPolicy: { rememberMeEnabled: true, rememberMeDays: 30, sessionTimeoutHours: 24, pgpEnabled: true },
|
||||
contacts: [],
|
||||
trustedSenders: new Set(),
|
||||
contactGroups: [],
|
||||
|
||||
@@ -462,8 +462,9 @@ async function loadAuthPolicy() {
|
||||
try {
|
||||
state.authPolicy = await request("/auth/policy", { method: "GET" });
|
||||
} catch (_) {
|
||||
state.authPolicy = { rememberMeEnabled: true, rememberMeDays: 30, sessionTimeoutHours: 24 };
|
||||
state.authPolicy = { rememberMeEnabled: true, rememberMeDays: 30, sessionTimeoutHours: 24, pgpEnabled: true };
|
||||
}
|
||||
setOpenPgpAvailability(state.authPolicy.pgpEnabled !== false);
|
||||
const enabled = state.authPolicy.rememberMeEnabled !== false;
|
||||
["#login-remember-row", "#add-account-remember-row"].forEach((selector) => $(selector)?.classList.toggle("hidden", !enabled));
|
||||
["#login-remember", "#add-account-remember"].forEach((selector) => { const input = $(selector); if (input && !enabled) { input.checked = false; input.disabled = true; } });
|
||||
|
||||
@@ -470,7 +470,7 @@ function renderPgpReaderState(detail) {
|
||||
const title = $("#pgp-reader-title");
|
||||
const find = $("#pgp-reader-discover");
|
||||
const kind = pgpMessageKind(detail);
|
||||
const configured = state.pgpKeys.length > 0;
|
||||
const configured = state.authPolicy.pgpEnabled !== false && state.pgpKeys.length > 0;
|
||||
bar.classList.toggle("hidden", !kind.pgp || !configured);
|
||||
find.classList.add("hidden");
|
||||
if (!kind.pgp || !configured) return;
|
||||
|
||||
@@ -1,5 +1,41 @@
|
||||
|
||||
|
||||
function setOpenPgpAvailability(enabled) {
|
||||
const available = Boolean(enabled);
|
||||
state.authPolicy.pgpEnabled = available;
|
||||
document.querySelector('.settings-tab[data-tab="openpgp"]')?.classList.toggle("hidden", !available);
|
||||
const settingsPanel = document.querySelector('.settings-panel[data-panel="openpgp"]');
|
||||
if (settingsPanel && !available) settingsPanel.classList.add("hidden");
|
||||
$("#compose-pgp-option")?.classList.toggle("hidden", !available || state.pgpKeys.length === 0);
|
||||
$("#compose-pgp-panel")?.classList.toggle("hidden", !available || state.pgpKeys.length === 0);
|
||||
if (!available) $("#pgp-reader-bar")?.classList.add("hidden");
|
||||
$("#identity-pgp-key-row")?.classList.toggle("hidden", !available);
|
||||
$$(".identity-pgp-default").forEach((row) => row.classList.toggle("hidden", !available));
|
||||
const overview = $("#identity-overview-help");
|
||||
if (overview) overview.textContent = available
|
||||
? "Select an identity to edit its sender data, text/HTML footer, image and OpenPGP defaults."
|
||||
: "Select an identity to edit its sender data, text/HTML footer and image.";
|
||||
const defaultsHelp = $("#identity-defaults-help");
|
||||
if (defaultsHelp) defaultsHelp.textContent = available
|
||||
? "Choose where this signature is inserted and, when OpenPGP is configured, its default behavior."
|
||||
: "Choose where this signature is inserted.";
|
||||
if (!available) {
|
||||
state.pgpKeys = [];
|
||||
state.pgpRecipientStatus = [];
|
||||
const mode = $("#compose-pgp-mode");
|
||||
if (mode) mode.value = "none";
|
||||
const panel = $("#compose-pgp-panel");
|
||||
if (panel) panel.open = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function loadOpenPgp() {
|
||||
if (state.authPolicy.pgpEnabled === false) {
|
||||
setOpenPgpAvailability(false);
|
||||
updateComposePgpUi();
|
||||
if (state.currentDetail) renderPgpReaderState(state.currentDetail);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const [keys, prefs] = await Promise.all([
|
||||
request("/pgp/keys", { method: "GET" }),
|
||||
@@ -13,9 +49,11 @@ async function loadOpenPgp() {
|
||||
if (state.currentDetail) renderPgpReaderState(state.currentDetail);
|
||||
} catch (error) {
|
||||
state.pgpKeys = [];
|
||||
const disabled = String(error?.message || "").toLowerCase().includes("openpgp is disabled");
|
||||
if (disabled) setOpenPgpAvailability(false);
|
||||
updateComposePgpUi();
|
||||
if (state.currentDetail) renderPgpReaderState(state.currentDetail);
|
||||
toast(error.message, "error");
|
||||
if (!disabled) toast(error.message, "error");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -177,7 +215,8 @@ function updateComposePgpUi(applyDefaults = false) {
|
||||
const panel = $("#compose-pgp-panel");
|
||||
const option = $("#compose-pgp-option");
|
||||
if (!modeSelect || !panel) return;
|
||||
const configured = state.pgpKeys.length > 0;
|
||||
const available = state.authPolicy.pgpEnabled !== false;
|
||||
const configured = available && state.pgpKeys.length > 0;
|
||||
option?.classList.toggle("hidden", !configured);
|
||||
panel.classList.toggle("hidden", !configured);
|
||||
if (!configured) {
|
||||
|
||||
@@ -45,11 +45,12 @@ async function loadQuota() {
|
||||
}
|
||||
|
||||
function openSettings(tab = "appearance") {
|
||||
if (tab === "openpgp" && state.authPolicy.pgpEnabled === false) tab = "appearance";
|
||||
selectSettingsTab(tab);
|
||||
$("#settings-dialog").showModal();
|
||||
if (tab === "sieve") loadSieve();
|
||||
if (tab === "identities") { loadIdentities(); loadOpenPgp(); }
|
||||
if (tab === "openpgp") loadOpenPgp();
|
||||
if (tab === "identities") { loadIdentities(); if (state.authPolicy.pgpEnabled !== false) loadOpenPgp(); }
|
||||
if (tab === "openpgp" && state.authPolicy.pgpEnabled !== false) loadOpenPgp();
|
||||
if (tab === "notifications") loadNotificationPreferences();
|
||||
if (tab === "mailbox") loadMailPreferences();
|
||||
if (tab === "diagnostics") loadDiagnostics();
|
||||
@@ -58,6 +59,7 @@ function openSettings(tab = "appearance") {
|
||||
}
|
||||
|
||||
function selectSettingsTab(name) {
|
||||
if (name === "openpgp" && state.authPolicy.pgpEnabled === false) name = "appearance";
|
||||
$$(".settings-tab").forEach((button) => button.classList.toggle("active", button.dataset.tab === name));
|
||||
$$(".settings-panel").forEach((panel) => panel.classList.toggle("hidden", panel.dataset.panel !== name));
|
||||
}
|
||||
|
||||
@@ -292,7 +292,7 @@ function bindEvents() {
|
||||
$("#pgp-import-file").addEventListener("click", () => $("#pgp-import-file-input").click());
|
||||
$("#pgp-import-file-input").addEventListener("change", async (event) => { const file = event.target.files?.[0]; if (file) await importPgpFile(file); event.target.value = ""; });
|
||||
$("#pgp-discover").addEventListener("click", () => discoverPgpEmail());
|
||||
$$(".settings-tab").forEach((button) => button.addEventListener("click", () => { selectSettingsTab(button.dataset.tab); if (button.dataset.tab === "sieve") loadSieve(); if (button.dataset.tab === "identities") { loadIdentities(); loadOpenPgp(); } if (button.dataset.tab === "openpgp") loadOpenPgp(); if (button.dataset.tab === "notifications") loadNotificationPreferences(); if (button.dataset.tab === "mailbox") loadMailPreferences(); if (button.dataset.tab === "composing") loadResponseTemplates(); if (button.dataset.tab === "diagnostics") loadDiagnostics(); if (button.dataset.tab === "account") { renderAccountSettings(); loadActiveSessions(); } if (button.dataset.tab === "folders") { loadSpecialFolders(); loadQuota(); } }));
|
||||
$$(".settings-tab").forEach((button) => button.addEventListener("click", () => { selectSettingsTab(button.dataset.tab); if (button.dataset.tab === "sieve") loadSieve(); if (button.dataset.tab === "identities") { loadIdentities(); if (state.authPolicy.pgpEnabled !== false) loadOpenPgp(); } if (button.dataset.tab === "openpgp" && state.authPolicy.pgpEnabled !== false) loadOpenPgp(); if (button.dataset.tab === "notifications") loadNotificationPreferences(); if (button.dataset.tab === "mailbox") loadMailPreferences(); if (button.dataset.tab === "composing") loadResponseTemplates(); if (button.dataset.tab === "diagnostics") loadDiagnostics(); if (button.dataset.tab === "account") { renderAccountSettings(); loadActiveSessions(); } if (button.dataset.tab === "folders") { loadSpecialFolders(); loadQuota(); } }));
|
||||
$("#pref-theme").addEventListener("change", (event) => savePreference("theme", event.target.value));
|
||||
$("#pref-interface-skin").addEventListener("change", (event) => savePreference("interfaceSkin", event.target.value));
|
||||
$("#pref-preset-scale-unlocked").addEventListener("change", (event) => savePreference("presetScaleUnlocked", event.target.checked));
|
||||
@@ -515,7 +515,7 @@ async function boot() {
|
||||
await loadAuthPolicy();
|
||||
const authenticated = await loadAccounts();
|
||||
if (authenticated) {
|
||||
await loadPreferences(); await loadMailPreferences(); await loadOpenPgp(); await loadIdentities(); await loadResponseTemplates(); await loadFolders(true); await loadSpecialFolders(); await loadQuota(); await loadTags(); await loadNotificationPreferences(); await loadTrustedSenders(); startCalendarReminderWatcher(); connectRealtime();
|
||||
await loadPreferences(); await loadMailPreferences(); if (state.authPolicy.pgpEnabled !== false) await loadOpenPgp(); else setOpenPgpAvailability(false); await loadIdentities(); await loadResponseTemplates(); await loadFolders(true); await loadSpecialFolders(); await loadQuota(); await loadTags(); await loadNotificationPreferences(); await loadTrustedSenders(); startCalendarReminderWatcher(); connectRealtime();
|
||||
const params = new URLSearchParams(location.search);
|
||||
const folder = params.get("mail_folder");
|
||||
const uid = Number(params.get("mail_uid"));
|
||||
|
||||
Reference in new issue
Block a user