diff --git a/.env.example b/.env.example index f6989be..0d02e38 100644 --- a/.env.example +++ b/.env.example @@ -20,7 +20,7 @@ SIEVE_ALLOW_INSECURE_AUTH=false SIEVE_TIMEOUT_SECS=10 TRUSTED_PROXIES=127.0.0.1,::1 -RUST_LOG=info +RUST_LOG=warn,rustbox=debug,tower_http=info # RUSTBOX_CONFIG=/etc/rustbox/rustbox.toml # RustBox 0.2 storage/search nested overrides use double underscores. diff --git a/CHANGELOG.md b/CHANGELOG.md index 98fd6dd..43f3b2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,36 @@ # RustBox changelog +### 0.2.8 + +- Scoped default logging so RustBox can run at DEBUG/INFO without enabling verbose dependency logs; Tantivy merge/GC/file-watcher noise and its internal filesystem/source paths are hidden unless explicitly enabled through a custom `RUST_LOG` target filter. Development and deployment examples now use target-scoped filters. +- Kept administrator diagnostics process-local and cross-platform: removed host CPU/RAM/load-average and Linux `/proc` sampling; runtime diagnostics now rely on portable RustBox in-memory counters plus the process PID. +- Added a disabled-by-default administrator system-diagnostics endpoint with loopback-only CIDR allowlisting, mandatory token or Basic authentication, fail-closed trusted-proxy handling, secret redaction, live SQL/session/storage/search probes, active IMAP IDLE/session/runtime state and bounded security telemetry for failed logins, rate limiting, access denials and server errors. +- Moved the synchronization/index activity indicator to the left of Contacts. +- Corrected production TOML section ordering so intended top-level database/session/mail/security/special-folder settings are no longer accidentally parsed as search settings. +- Refined folder counters so only the unread value is emphasized, hid the sync/index activity icon when idle, and made manual search-index rebuilds drive the same live activity indicator as background indexing. +- Made folder totals/unread counts server-authoritative: IMAP `STATUS MESSAGES/UNSEEN` drives badges, selected-mailbox fallback counts `UID SEARCH UNSEEN`, periodic reconciliation repairs counters even when `HIGHESTMODSEQ` is unchanged, and mutation-time compare-and-set/retry prevents out-of-order STATUS responses from rolling counters backwards. +- Decoupled counter refresh from folder-tree discovery: a forced counter refresh reuses a fresh cached folder tree and batches STATUS requests instead of forcing another IMAP LIST. +- Changed read/unread/flag updates to patch the affected message row and folder badge live; ordinary flag changes no longer rebuild the message list, removing the reader/list jump during mark-read operations. +- Added responsive folder counters (`total / unread` when unread is non-zero, total-only at zero, compact mode in narrow sidebars) and surfaced IMAP header-cache plus search-index backlog in the top-right activity indicator/tooltip. +- Extended the opt-in security-header middleware with a private `/api` profile (`no-store`, `nosniff`, same-origin CORP, `frame-ancestors 'none'` and `X-Frame-Options: DENY`), while keeping Permissions-Policy/COOP and the full document CSP on HTML and HSTS at the public HTTPS edge; disabling `security_headers_enabled` removes the whole RustBox-added security profile. +- Minimized unauthenticated `/api/auth/policy` to `rememberMeEnabled`; optional feature policy moved behind authenticated `/api/auth/session-policy`, and session/remember durations are no longer exposed by policy endpoints. +- Added optimistic read/unread updates in the browser with guarded rollback, followed by authoritative server reconciliation; normal read/unread operations no longer replace the message list. +- Fixed realtime WebSocket deduplication so unique server `eventId` values always win over payload fingerprints; rapid legitimate counter cycles such as `1 → 0 → 1` are no longer dropped as duplicates. Active-folder total text is patched live together with the sidebar counter. +- Hardened IDLE/CONDSTORE reconciliation: counter STATUS no longer advances the flag-sync cursor, non-CONDSTORE servers fall back to a full `UID FLAGS` comparison, partial IMAP fetch streams are discarded instead of being treated as complete, and IDLE header/index writes participate in the same cache-generation transaction. +- Fixed UIDVALIDITY reset writes to stay inside the same database transaction on SQLite, MySQL and PostgreSQL instead of mixing transaction/base-connection writes. +- Tightened the HTML CSP to `style-src 'self'` plus the narrower `style-src-attr 'unsafe-inline'`; inline JavaScript remains disallowed. +- Added regression coverage for authoritative unread refresh, API/document/static header profiles and pre-login policy minimization. +- Added database migration V048 on SQLite, MySQL and PostgreSQL: folder counters now carry a monotonic `counter_generation`, so delayed STATUS responses cannot win an ABA race after newer read/unread activity. V049 is the next migration number. +- Bumped the application/source release to 0.2.8. + +### 0.2.7 +- Made opt-in browser security headers content-type-aware instead of attaching document-only policies to every response. +- HTML documents keep the full CSP, clickjacking, permissions, COOP, referrer and MIME-sniffing protections. +- CSS responses now receive only the applicable `Referrer-Policy` and `X-Content-Type-Options`; JavaScript, JSON, SVG, API and other non-document responses retain `X-Content-Type-Options: nosniff` without redundant document policies. +- Added regression coverage for HTML, CSS and API header sets while preserving the disabled-by-default compatibility switch. +- No schema migration is required; runtime schema remains V047 and V048 remains the next migration number. +- Bumped the application/source release to 0.2.7. + ### 0.2.6 - Fixed iPhone/iOS reader toolbar popovers (`Message details` and `Move / copy`) by rendering the open menu outside the horizontally scrolling toolbar and positioning it against the visual viewport. - Made folder counters and message-list refreshes more responsive: realtime sync stores the server-reported unread count, folder hierarchy refresh avoids unnecessary IMAP LIST calls, and manual refresh forces the active-folder status check in parallel with counter refresh. diff --git a/Cargo.lock b/Cargo.lock index 1b970bf..b5e63d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3506,7 +3506,7 @@ dependencies = [ [[package]] name = "rustbox" -version = "0.2.6" +version = "0.2.8" dependencies = [ "aes-gcm", "async-imap", diff --git a/Cargo.toml b/Cargo.toml index f4d9155..fc9cf45 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rustbox" -version = "0.2.6" +version = "0.2.8" edition = "2024" rust-version = "1.99.1" default-run = "rustbox" diff --git a/NOTICE.md b/NOTICE.md index 72882da..21058cf 100644 --- a/NOTICE.md +++ b/NOTICE.md @@ -1,6 +1,6 @@ # RustBox attribution / notice -RustBox 0.2.6 is derived from the Oxi webmail project: +RustBox 0.2.8 is derived from the Oxi webmail project: - Upstream: `c0h1b4/oxi` - Base snapshot: `7f1507c486b78cf33e54beb701dd460d3bd2bd5d` diff --git a/README.md b/README.md index c12e5a9..03e1ca0 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# RustBox 0.2.6 +# RustBox 0.2.8 RustBox is a webmail derived from `c0h1b4/oxi`. The consumed Oxi revision is recorded in `UPSTREAM_BASE`. @@ -34,7 +34,7 @@ rustbox/ `build.rs` combines the CSS modules from `src/web/css/manifest.txt` and JavaScript modules from `src/web/js/manifest.txt`. Cargo embeds the generated CSS/JS bundles, HTML and SVG sprite in the RustBox executable. There is no standalone source `src/web/app.js`; `src/web/` is the single source of truth and Cargo generates the embedded bundles in `OUT_DIR`. -## Storage and search in 0.2.6 +## Storage and search in 0.2.8 RustBox separates three concerns: SQL metadata, blob storage and search. The SQL engine does **not** decide whether `data_dir` may be used. A single-node PostgreSQL/MySQL deployment may still use filesystem blobs and local Tantivy. In cluster mode, local node state is rejected unless the configured backend is explicitly shared. @@ -51,6 +51,8 @@ Search backends: In a cluster using shared SQL + S3/shared POSIX + OpenSearch/disabled search + `pgp_enabled=false`, `data_dir` is not required as persistent node state. See `docker/README.md` for ready Compose variants. +Administrator-only deep runtime diagnostics can be enabled separately under `[diagnostics_endpoint]`. It is disabled and loopback-only by default, supports a network allowlist plus token or Basic Auth, redacts secrets, and probes SQL/Redis/storage/search/session/runtime/security state. See [`docs/SYSTEM_DIAGNOSTICS.md`](docs/SYSTEM_DIAGNOSTICS.md). + ## Configure mail servers Mail endpoints are configured by the administrator, not on the login screen. Users enter only email and password. @@ -60,7 +62,10 @@ Edit `config.dev.toml` before the first run: ```toml default_theme = "system" # system | dark | light default_interface_skin = "rustbox" # rustbox | gmail | proton | zoho | outlook | fastmail | apple-mail | yahoo | hey | nord | github | gitlab | discord | slack | reddit | notion | ... -log_level = "info" # trace | debug | info | warn | error | off; RUST_LOG overrides it +log_level = "info" # RustBox app level; dependencies default to WARN. RUST_LOG can override the full filter. +# Recommended dev filter: warn,rustbox=debug,tower_http=info +# Avoid a global RUST_LOG=debug/info unless diagnosing a dependency: third-party +# crates may emit internal filesystem/source paths in verbose records. # Mail hosts may be omitted: IMAP defaults to localhost:993 and SMTP to localhost:587. # imap_host = "localhost" # smtp_host = "localhost" @@ -75,7 +80,7 @@ max_message_size_mb = 50 sieve_enabled = true diagnostics_enabled = true about_enabled = true -# Adds CSP/frame/referrer/permissions/COOP response headers when enabled. +# Adds content-type-aware browser security headers when enabled: full document policy on HTML, private no-store/CORP/anti-framing policy on /api, Referrer-Policy on HTML/CSS, and nosniff broadly. security_headers_enabled = false # Optional; defaults to imap_host. Ignored when sieve_enabled=false. @@ -125,10 +130,16 @@ bucket = "rustbox" region = "eu-central-1" # endpoint = "https://s3.example.com" # MinIO/Ceph RGW/etc. +[sync] +interval_secs = 30 +deep_index_batch_size = 10 + [search] backend = "opensearch" urls = ["https://opensearch.example.com:9200"] required = true +# Production-safe default: do not proactively download every full message body. +deep_index = false ``` Shared POSIX is also supported without S3: @@ -155,7 +166,7 @@ rustbox search reindex Ready Docker examples are in `docker/`: standalone SQLite, PostgreSQL + SILO (S3-compatible) + OpenSearch, and MySQL + NFS + OpenSearch. -All executable application SQL is centralized by dialect under `src/queries/{sqlite,mysql,pgsql}/queries.rs`. The squashed compatibility baseline remains V046; RustBox 0.2.1 adds runtime migration V047 for attachment checksums/logical blob keys and the durable search outbox. The next migration number is V048. See `migrations/README.md`. +All executable application SQL is centralized by dialect under `src/queries/{sqlite,mysql,pgsql}/queries.rs`. The squashed compatibility baseline remains V046; RustBox 0.2.1 adds runtime migration V047 for attachment checksums/logical blob keys and the durable search outbox, while RustBox 0.2.8 adds V048 for monotonic folder-counter generations used by race-safe IMAP counter reconciliation. The next migration number is V049. See `migrations/README.md`. ## Development @@ -224,7 +235,7 @@ Initialize Git after extracting the release archive: ```bash git init git add . -git commit -m "RustBox 0.2.6 release" +git commit -m "RustBox 0.2.8 release" ./scripts/setup-upstream.sh ``` diff --git a/RELEASE-MANIFEST.txt b/RELEASE-MANIFEST.txt index 6865c8e..ff4b32c 100644 --- a/RELEASE-MANIFEST.txt +++ b/RELEASE-MANIFEST.txt @@ -1,15 +1,20 @@ -RustBox 0.2.6 release manifest +RustBox 0.2.8 release manifest -0.2.6 feature/configuration changes: -- fixes iPhone/iOS Safari reader action menus by rendering open Details / Move / copy popovers outside the scrolling toolbar -- refreshes unread counters with server-reported unread counts and avoids frequent IMAP folder-list refreshes -- watches the currently viewed folder with IMAP IDLE and synchronizes remaining folders on a 30-second fallback cadence -- stores new message headers before realtime events, allowing newly arrived messages to appear live in the active list -- shows an in-app toast for new mail in any folder and deduplicates browser notifications per message -- shows an animated indexing indicator beside Contacts while deep indexing is active -- excludes Trash/Junk/Spam from the deep-index selection query -- manual refresh forces current-folder message/count refresh while folder counters refresh in parallel -- no schema migration; V047 remains current and V048 remains next +0.2.8 feature/configuration changes: +- fixes folder unread counts by treating IMAP STATUS UNSEEN as authoritative and never using SELECT/EXAMINE UNSEEN as a count +- falls back to UID SEARCH UNSEEN when STATUS is unavailable for the selected mailbox +- reconciles MESSAGES/UNSEEN even when HIGHESTMODSEQ is unchanged and rejects out-of-order stale counter writes with compare-and-set/retry +- updates individual message rows and folder counters optimistically/live for read/unread/flag events instead of rebuilding the message list, then reconciles them to authoritative IMAP state +- shows wide folder badges as total / unread, hides the unread suffix at zero, and switches to compact counters in narrow sidebars +- exposes IMAP header-cache and deep-index backlog through the activity indicator and tooltip +- keeps counter STATUS separate from the CONDSTORE cursor, falls back to full UID/FLAGS reconciliation when needed, and refuses to apply partial IMAP streams +- keeps security_headers_enabled opt-in, centralizes the RustBox-added browser security profile behind that switch, and applies the full document-only profile only to HTML/XHTML +- uses external same-origin JavaScript with `script-src 'self'`, disables inline script attributes, restricts stylesheets to `style-src 'self'`, and permits only dynamic style attributes via `style-src-attr 'unsafe-inline'` +- applies no-store, nosniff, same-origin CORP and an anti-framing `frame-ancestors 'none'`/X-Frame-Options policy to private API responses; the full document CSP plus Permissions-Policy/COOP remain HTML-only, and assets receive same-origin CORP where applicable +- leaves HSTS to the HTTPS/TLS-terminating deployment layer +- minimizes the public auth policy to rememberMeEnabled and moves optional feature availability behind authenticated /api/auth/session-policy +- adds/extends regression coverage and offline release checks for the changed behavior +- adds runtime migration V048 on SQLite/MySQL/PostgreSQL for monotonic folder-counter generations; V049 remains next Architecture: - database_backend = sqlite | mysql | pgsql selects authoritative SQL only @@ -36,31 +41,8 @@ Database/schema: - migrations/{sqlite,mysql,pgsql}/schema.sql contain the current fresh schema - compatibility baseline remains V046 - runtime migration V047 adds attachment checksum metadata and search_outbox -- next migration number: V048 - -PostgreSQL message mutation fix (0.2.2): -- header_sync_generation UPSERT qualifies the current tenant user_meta.value on the UPDATE RHS -- this prevents PostgreSQL from treating `value` as ambiguous during ON CONFLICT DO UPDATE -- SQLite/MySQL equivalents were reviewed and require no matching SQL change -- no schema migration is required - -PostgreSQL startup compile fix (0.2.2): -- multi-statement global schema DDL uses `sqlx::Executor::execute()` on the transaction connection -- avoids SQLx raw_sql mutable-connection lifetime inference failures (`Executor is not general enough`) -- retains the startup transaction and `SET LOCAL client_min_messages TO WARNING` -- no schema migration is required - -Database/session startup (0.2.2): -- db::pool::configure() is the single owner of global schema initialization -- database-backed session persistence does not execute schema.sql a second time -- the fix applies uniformly to SQLite, MySQL and PostgreSQL -- PostgreSQL expected IF NOT EXISTS notices are suppressed only inside the startup DDL transaction -- no SQL migration is required; runtime schema remains V047 - -SQLite test isolation (0.2.2): -- test builds initialize each requested SQLite database but do not publish a TempDir-backed path into the process-wide database OnceLock -- account/database tests therefore remain isolated when Rust runs them in parallel -- a regression test covers deleting the first TempDir before opening a second test database +- runtime migration V048 adds `folders.counter_generation` for race-safe authoritative IMAP counter writes +- next migration number: V049 Docker examples: - docker/compose-single-sqlite.yml @@ -69,14 +51,14 @@ Docker examples: - docker/compose-mysql-nfs-opensearch.yml (MySQL + real NFS volume + OpenSearch) Operations/docs: -- README.md documents 0.2.6 storage/search configuration +- README.md documents 0.2.8 configuration - docker/README.md documents Compose variants - docs/STORAGE_SEARCH_OPERATIONS.md documents failure modes, shared-storage requirements and migration behavior - TODO.md leaves live SILO/MinIO/AWS/Ceph and real multi-node acceptance validation explicitly pending Build/release metadata: - `scripts/package-release.sh` runs the full Rust release gate before creating ZIP/SHA256 artifacts -- Cargo.toml version = 0.2.6 +- Cargo.toml version = 0.2.8 - rust-toolchain.toml pins Rust 1.99.1 - Cargo.lock is retained for reproducible application builds; the release gate generates it only when missing - CI and scripts/build-release.sh use the retained Cargo.lock (generating it only if missing), then run locked tests/Clippy/release build diff --git a/RELEASE-VALIDATION.txt b/RELEASE-VALIDATION.txt index cc8d6dd..c04933b 100644 --- a/RELEASE-VALIDATION.txt +++ b/RELEASE-VALIDATION.txt @@ -1,46 +1,29 @@ -RustBox 0.2.6 - final source validation +RustBox 0.2.8 - final source validation -0.2.6 scope: -- iOS-safe reader action popovers for Details and Move / copy, portaled outside the horizontally scrolling toolbar -- faster folder-counter refresh using a server-authoritative realtime unread counts and a longer folder-list cache TTL -- live refresh for the currently viewed folder via IMAP IDLE plus a 30-second fallback sync for other folders -- periodic sync stores newly arrived message headers before emitting realtime events, so new messages can appear in the list immediately -- in-app toast for new mail in any folder and per-message browser-notification deduplication -- animated indexing indicator beside Contacts while deep indexing is active -- deep indexing excludes Trash/Junk/Spam at query time so excluded folders cannot block the indexing queue -- manual refresh forces current-folder status refresh while folder-list/count refresh runs in parallel -- no database migration; V047 remains current and V048 remains next -- version metadata updated to 0.2.6 +0.2.8 scope: +- authoritative IMAP folder totals/unread counters from server state, including selected-mailbox fallback via UID SEARCH UNSEEN +- optimistic/targeted realtime read/unread/flag updates without full message-list rerenders +- responsive total/unread folder badges, authoritative FolderStatus updates, and separate counter/CONDSTORE cursor handling +- IMAP cache/deep-index progress surfaced through the activity indicator +- content-type-aware opt-in security headers with external-script `script-src 'self'` CSP and stylesheet-only `style-src 'self'` +- private API no-store/nosniff/same-origin-CORP plus anti-framing CSP/X-Frame-Options; full document CSP, Permissions-Policy and COOP remain HTML-only +- pre-login auth policy minimized to rememberMeEnabled; optional feature policy requires authentication +- runtime migration V048 adds monotonic folder-counter generations on SQLite/MySQL/PostgreSQL; V049 remains next +- version metadata updated to 0.2.8 Completed in this environment: -- offline release validator: scripts/validate-release-static.py (212/212 checks passed) -- all maintained TOML examples parse successfully -- Docker Compose topology/config references are validated by the offline release validator -- JSON files parse successfully -- browser JavaScript assembled in src/web/js/manifest.txt order passes node --check -- shell scripts pass bash -n +- offline release validator: scripts/validate-release-static.py (279/279 checks passed) +- browser JavaScript assembled in manifest order passes `node --check` +- all JSON files parse successfully - Python helper scripts compile successfully -- targeted source checks validate the PostgreSQL header-sync UPSERT and startup schema-initialization control flow -- the PostgreSQL startup DDL path now uses `Executor::execute()` instead of the lifetime-problematic `raw_sql(...).execute(&mut PgConnection)` path; compiler validation remains part of the Rust-host release gate -- whitespace/patch-artifact checks pass on the final source tree (this packaged worktree has no `.git` metadata) -- Cargo.lock is retained for reproducible builds; no *.orig, *.rej, temporary FIX/APPLY scripts, or patch leftovers are shipped -- regression source checks retain the PostgreSQL header-sync qualification audit and now reject the SQLx raw_sql mutable-connection lifetime regression in startup DDL -- SQLite unit-test configuration no longer leaks a temporary database path through the process-wide OnceLock; a dedicated regression test covers deleted TempDir reuse -- the static release contract now verifies that build-release.sh requires cargo test, Clippy with -D warnings, and a locked release build -- scripts/package-release.sh runs the full build gate before creating the source ZIP and SHA256, so packaging cannot succeed after a failed Rust test/Clippy/build gate +- shell scripts pass `bash -n` +- release metadata consistently identifies 0.2.8 +- V048 migration files/catalog registration and SQLite upgrade execution are validated offline -0.2.2 fixes included: -- PostgreSQL startup schema initialization no longer triggers SQLx `Executor is not general enough`; raw multi-statement DDL is executed directly through `Executor::execute()` inside the existing transaction -- PostgreSQL header-sync invalidation qualifies the existing tenant user_meta.value in ON CONFLICT DO UPDATE, eliminating the ambiguous-column 500 seen during message mutations such as marking spam -- tenant SQL rewriting has a regression test proving the qualified user_meta reference is rewritten to the tenant table -- SQLite and MySQL equivalents were reviewed and do not have the PostgreSQL ambiguity -- 0.2.2 single-schema-owner and PostgreSQL NOTICE-noise fixes remain in place -- no database migration is required; V047 remains current and V048 remains next -- 0.2.1 OpenPGP visibility, OpenSearch backpressure and HTTP/dev cookie fixes remain in place -Not executed in this packaging environment: -- cargo fmt/test/clippy/build with Rust 1.99.1, because Rust/Cargo is not installed in this validation environment -- live PostgreSQL/MySQL/S3/NFS/OpenSearch integration tests +Not executable in this packaging environment: +- cargo fmt/test/clippy/build with Rust 1.99.1, because Rust/Cargo is not installed +- live PostgreSQL/MySQL/S3/NFS/OpenSearch/IMAP integration tests The final release gate on a Rust host is: bash scripts/build-release.sh diff --git a/build.rs b/build.rs index cfd56c0..2f06b3c 100644 --- a/build.rs +++ b/build.rs @@ -58,9 +58,11 @@ fn main() { let out = PathBuf::from(env::var_os("OUT_DIR").expect("OUT_DIR")); let css_dir = root.join("src/web/css"); let js_dir = root.join("src/web/js"); + let theme_bootstrap_path = root.join("src/web/theme-bootstrap.js"); for path in [ root.join("src/web/index.html"), + theme_bootstrap_path.clone(), root.join("src/web/icons.svg"), root.join("src/web/i18n/en.json"), root.join("src/web/i18n/pl.json"), @@ -70,8 +72,14 @@ fn main() { let css_bundle = bundle_manifest(&root, &css_dir, &css_dir.join("manifest.txt"), ".css"); let js_bundle = bundle_manifest(&root, &js_dir, &js_dir.join("manifest.txt"), ".js"); + let theme_bootstrap = fs::read_to_string(&theme_bootstrap_path) + .unwrap_or_else(|error| panic!("{}: {error}", theme_bootstrap_path.display())); - let asset_id = fnv1a64(&[css_bundle.as_bytes(), js_bundle.as_bytes()]); + let asset_id = fnv1a64(&[ + css_bundle.as_bytes(), + js_bundle.as_bytes(), + theme_bootstrap.as_bytes(), + ]); println!("cargo:rustc-env=RUSTBOX_ASSET_ID={asset_id:016x}"); fs::write(out.join("rustbox-ui.css"), css_bundle).expect("write RustBox CSS bundle"); diff --git a/config.dev.toml b/config.dev.toml index 335e6f5..1080e63 100644 --- a/config.dev.toml +++ b/config.dev.toml @@ -52,3 +52,51 @@ sieve_allow_insecure_auth = false sieve_timeout_secs = 10 trusted_proxies = "127.0.0.1,::1" +[sync] +interval_secs = 30 +# Maximum number of full message bodies fetched from IMAP during one sync pass when +# automatic deep indexing is enabled. Higher values fill the cache/index faster but +# increase network traffic, application database/search-storage usage and indexing CPU/I/O bursts. +deep_index_batch_size = 10 + +# Development search/cache profile. +[search] +backend = "local_tantivy" +# Force automatic full-body cache/index for every account in this development instance. +# Each periodic sync may download up to sync.deep_index_batch_size full MIME bodies. +# This is intentionally ON in dev so cache/index behaviour is visible during testing. +# Setting it to false falls back to the existing per-account deep_index preference. +deep_index = true +# Search outbox worker cadence/batch; these do NOT control IMAP body downloads. +worker_interval_secs = 2 +worker_batch_size = 100 +worker_max_attempts = 12 + + +# Periodic IMAP reconciliation. Active folders still use IMAP IDLE for near-real-time +# updates; this interval controls the fallback/background pass for counters, flags, +# new headers in other folders, and the automatic deep-index pass. Lower values make +# background state converge faster but increase IMAP traffic and server load. + +# Administrator-only system diagnostics endpoint. Disabled by default. +# When enabled it exposes a redacted runtime/configuration report plus live probes +# for SQL, session persistence (database/Redis), blob storage (filesystem/S3), +# search (Tantivy/OpenSearch), IMAP IDLE counters and security telemetry. +# Access requires BOTH an allowed source network and the configured auth method. +# Secrets (passwords, tokens, S3 keys, session keys and URL credentials) are never returned. +[diagnostics_endpoint] +enabled = false +path = "/api/system/diagnostics" +# Default/fail-safe allowlist: only this host. Add trusted LAN/pod/monitoring CIDRs explicitly. +allowed_networks = ["127.0.0.1/32", "::1/128"] +# token | basic. Credentials should normally come from .env / secret manager, not TOML. +auth = "token" +# token = "replace-with-at-least-24-random-characters" +# basic_username = "rustbox-monitor" +# basic_password = "replace-with-a-long-random-password" +# E-mail addresses and stable user hashes are omitted by default; aggregate counts remain available. +include_user_identifiers = false +# Timeout applied independently to database/session/storage/search probes. +probe_timeout_secs = 5 +# Rolling in-memory window used for login failures, 403, 429 and 5xx attack/problem signals. +security_window_secs = 900 diff --git a/config.example.toml b/config.example.toml index 3b4ba68..1ca30a0 100644 --- a/config.example.toml +++ b/config.example.toml @@ -1,4 +1,4 @@ -# RustBox 0.2.6 +# RustBox 0.2.8 host = "127.0.0.1" port = 3001 environment = "production" @@ -59,7 +59,7 @@ max_message_size_mb = 50 sieve_enabled = true diagnostics_enabled = true about_enabled = true -# Adds browser-facing security headers (CSP, frame/referrer/permissions policies). +# Adds content-type-aware browser security headers: document policies on HTML, private no-store/framing/CORP policy on /api, Referrer-Policy on HTML/CSS, and nosniff broadly. security_headers_enabled = false # ManageSieve. If sieve_host is omitted, imap_host from the authenticated @@ -72,7 +72,6 @@ sieve_allow_insecure_auth = false sieve_timeout_secs = 10 trusted_proxies = "127.0.0.1,::1" - # Optional server-wide defaults for special IMAP folders. These are used when # a mailbox has no per-account override and the named folder exists. # default_sent_folder = "Sent" @@ -96,10 +95,6 @@ pgp_max_key_bytes = 1048576 # Private-key export is deliberately disabled unless the administrator opts in. pgp_allow_private_export = false -# RustBox 0.2 blob/search backends -# Blob storage is independent from the SQL database. PostgreSQL/MySQL can still use -# local filesystem storage in single-node mode. In cluster mode use S3-compatible -# storage or a genuinely shared POSIX filesystem (NFS/CephFS/GlusterFS). [storage] backend = "filesystem" # filesystem | s3 # path = "/var/lib/rustbox/blobs" # defaults to data_dir for filesystem @@ -120,6 +115,17 @@ retry_timeout_secs = 60 # server_side_encryption = "AES256" # or "aws:kms" # kms_key_id = "..." +# Periodic IMAP reconciliation. Active folders still use IMAP IDLE; this is the +# background/fallback cadence for folder counters, flags, new headers in other +# folders, and the automatic deep-index pass. Lower values improve convergence but +# increase IMAP traffic and mail-server load. +[sync] +interval_secs = 30 +# Maximum full message bodies downloaded from IMAP per periodic pass when deep index +# is effective. Larger batches catch up faster at the cost of network traffic, +# application-database/search-storage usage and indexing CPU/I/O. +deep_index_batch_size = 10 + # Search is independent from blob storage. Local Tantivy is single-node only. [search] backend = "local_tantivy" # disabled | local_tantivy | opensearch @@ -129,8 +135,45 @@ index_prefix = "rustbox" required = false allow_insecure_http = false request_timeout_secs = 10 +# Server-wide automatic full-body cache/index switch. When true, RustBox proactively +# downloads message bodies from IMAP and indexes them for every account, up to +# sync.deep_index_batch_size per sync pass. This increases IMAP traffic, +# application-database/search-storage usage and indexing load. Production default +# is false. When false, +# the existing per-account deep_index preference may still opt an account in. +deep_index = false +# Search outbox worker cadence/batch; these do NOT control IMAP body downloads. worker_interval_secs = 2 worker_batch_size = 100 worker_max_attempts = 8 # username = "rustbox" # password = "..." + + +# RustBox 0.2 blob/search backends +# Blob storage is independent from the SQL database. PostgreSQL/MySQL can still use +# local filesystem storage in single-node mode. In cluster mode use S3-compatible +# storage or a genuinely shared POSIX filesystem (NFS/CephFS/GlusterFS). + +# Administrator-only system diagnostics endpoint. Disabled by default. +# When enabled it exposes a redacted runtime/configuration report plus live probes +# for SQL, session persistence (database/Redis), blob storage (filesystem/S3), +# search (Tantivy/OpenSearch), IMAP IDLE counters and security telemetry. +# Access requires BOTH an allowed source network and the configured auth method. +# Secrets (passwords, tokens, S3 keys, session keys and URL credentials) are never returned. +[diagnostics_endpoint] +enabled = false +path = "/api/system/diagnostics" +# Default/fail-safe allowlist: only this host. Add trusted LAN/pod/monitoring CIDRs explicitly. +allowed_networks = ["127.0.0.1/32", "::1/128"] +# token | basic. Credentials should normally come from .env / secret manager, not TOML. +auth = "token" +# token = "replace-with-at-least-24-random-characters" +# basic_username = "rustbox-monitor" +# basic_password = "replace-with-a-long-random-password" +# E-mail addresses and stable user hashes are omitted by default; aggregate counts remain available. +include_user_identifiers = false +# Timeout applied independently to database/session/storage/search probes. +probe_timeout_secs = 5 +# Rolling in-memory window used for login failures, 403, 429 and 5xx attack/problem signals. +security_window_secs = 900 diff --git a/dev.sh b/dev.sh index bb81d8c..4bd239d 100755 --- a/dev.sh +++ b/dev.sh @@ -4,6 +4,17 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" cd "$ROOT" +# Local development settings live in .env. RustBox reads environment variables +# from the process; it does not parse dotenv files itself. Export the file here +# before applying development defaults so explicit .env values always win. +ENV_FILE="${RUSTBOX_ENV_FILE:-$ROOT/.env}" +if [ -f "$ENV_FILE" ]; then + set -a + # shellcheck disable=SC1090 + . "$ENV_FILE" + set +a +fi + if ! command -v cargo >/dev/null 2>&1; then echo "error: Rust/Cargo is required" >&2 exit 1 @@ -12,7 +23,28 @@ fi mkdir -p "$ROOT/.dev/data" export RUSTBOX_CONFIG="${RUSTBOX_CONFIG:-$ROOT/config.dev.toml}" export DATA_DIR="${DATA_DIR:-$ROOT/.dev/data}" -export RUST_LOG="${RUST_LOG:-rustbox=debug,tower_http=info}" + +# Keep application debugging useful without enabling verbose INFO/DEBUG output +# from every dependency. A legacy/simple RUST_LOG level from .env is scoped to +# RustBox automatically; explicit directive filters (containing '=' or ',') are +# respected exactly as provided. +case "${RUST_LOG:-}" in + ""|debug) + export RUST_LOG="warn,rustbox=debug,tower_http=info" + ;; + trace) + export RUST_LOG="warn,rustbox=trace,tower_http=debug" + ;; + info) + export RUST_LOG="warn,rustbox=info,tower_http=info" + ;; + warn|error|off) + export RUST_LOG="${RUST_LOG}" + ;; + *) + export RUST_LOG="${RUST_LOG}" + ;; +esac if [ ! -f "$RUSTBOX_CONFIG" ]; then echo "error: config not found: $RUSTBOX_CONFIG" >&2 @@ -20,8 +52,22 @@ if [ ! -f "$RUSTBOX_CONFIG" ]; then fi echo "RustBox: http://127.0.0.1:3001" -echo "Config: $RUSTBOX_CONFIG" -echo "Data: $DATA_DIR" +display_path() { + local value="$1" + if [[ "$value" == "$ROOT"/* ]]; then + printf './%s' "${value#"$ROOT"/}" + else + printf '%s' "$(basename "$value")" + fi +} + +echo "Config: $(display_path "$RUSTBOX_CONFIG")" +echo "Data: $(display_path "$DATA_DIR")" +if [ -f "$ENV_FILE" ]; then + echo "Env: $(display_path "$ENV_FILE")" +fi +echo "IMAP: ${IMAP_HOST:-localhost}:${IMAP_PORT:-993}" +echo "SMTP: ${SMTP_HOST:-${IMAP_HOST:-localhost}}:${SMTP_PORT:-587}" echo cargo run --quiet --bin rustbox -- --check-config diff --git a/docker/.env.example b/docker/.env.example index 15d9014..3bcb5b4 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -15,7 +15,7 @@ SMTP_PORT=587 TLS_ENABLED=true # Application logging. -RUST_LOG=info +RUST_LOG=warn,rustbox=info,tower_http=info # Shared session encryption key. Required to be identical on all RustBox nodes. # Replace for any non-local deployment. @@ -52,3 +52,12 @@ COMPOSE_PROJECT_NAME=rustbox # development = cookies work over plain HTTP; production = Secure cookies, requires HTTPS ENVIRONMENT=development + +# Administrator-only diagnostics endpoint (requires corresponding TOML section). +# DIAGNOSTICS_ENDPOINT__ENABLED=true +# DIAGNOSTICS_ENDPOINT__AUTH=token +# DIAGNOSTICS_ENDPOINT__TOKEN=replace-with-at-least-24-random-characters +# Alternative Basic Auth: +# DIAGNOSTICS_ENDPOINT__AUTH=basic +# DIAGNOSTICS_ENDPOINT__BASIC_USERNAME=rustbox-monitor +# DIAGNOSTICS_ENDPOINT__BASIC_PASSWORD=replace-with-a-long-random-password diff --git a/docker/README.md b/docker/README.md index 6eba0fe..ba31da9 100644 --- a/docker/README.md +++ b/docker/README.md @@ -80,7 +80,7 @@ cp .env.example .env | `SMTP_HOST` | `mail.example.com` | all | SMTP server hostname used for sending mail. | | `SMTP_PORT` | `587` | all | SMTP submission port. | | `TLS_ENABLED` | `true` | all | Enables TLS for mail connections according to RustBox configuration. | -| `RUST_LOG` | `info` | all | Rust logging filter/level. | +| `RUST_LOG` | `warn,rustbox=info,tower_http=info` | all | Rust logging filter. Keeps dependencies quiet by default; use explicit target directives for dependency debugging. | | `SESSION_ENCRYPTION_KEY` | development key | PostgreSQL, MySQL | Shared key used to encrypt session data. All application nodes must use the same value. Replace it outside local testing. | | `POSTGRES_PASSWORD` | `rustbox-dev-password` | PostgreSQL | Password for PostgreSQL user `rustbox`; also used in `DATABASE_URL`. | | `S3_ACCESS_KEY` | `rustbox` | PostgreSQL/S3 | Access key for the bundled S3-compatible SILO service and RustBox S3 client. | @@ -201,4 +201,4 @@ docker compose -p rustbox -f compose-postgres-s3-opensearch.yml exec opensearch -d '{"index.blocks.read_only_allow_delete":null}' ``` -RustBox 0.2.6 retains the OpenSearch outbox backoff introduced in 0.2.2, so these 429/flood-stage responses no longer produce one warning per queued event on a full disk. +RustBox 0.2.8 retains the OpenSearch outbox backoff introduced in 0.2.2, so these 429/flood-stage responses no longer produce one warning per queued event on a full disk. diff --git a/docker/compose-ha-garage.yml b/docker/compose-ha-garage.yml index 6df3f04..b0e55ee 100644 --- a/docker/compose-ha-garage.yml +++ b/docker/compose-ha-garage.yml @@ -23,7 +23,7 @@ x-rustbox-common: &rustbox-common SMTP_HOST: ${SMTP_HOST:-mail.example.com} SMTP_PORT: ${SMTP_PORT:-587} TLS_ENABLED: ${TLS_ENABLED:-true} - RUST_LOG: ${RUST_LOG:-info} + RUST_LOG: ${RUST_LOG:-warn,rustbox=info,tower_http=info} DATABASE_URL: postgres://rustbox:${POSTGRES_PASSWORD:-rustbox-dev-password}@postgres:5432/rustbox SESSION_ENCRYPTION_KEY: ${SESSION_ENCRYPTION_KEY:-MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=} AWS_ACCESS_KEY_ID: ${GARAGE_ACCESS_KEY_ID:-GKb69252a3b0643e8bd08d4cd4} diff --git a/docker/compose-mysql-nfs-opensearch.yml b/docker/compose-mysql-nfs-opensearch.yml index 98ab485..2300d80 100644 --- a/docker/compose-mysql-nfs-opensearch.yml +++ b/docker/compose-mysql-nfs-opensearch.yml @@ -52,7 +52,7 @@ services: SMTP_HOST: ${SMTP_HOST:-mail.example.com} SMTP_PORT: ${SMTP_PORT:-587} TLS_ENABLED: ${TLS_ENABLED:-true} - RUST_LOG: ${RUST_LOG:-info} + RUST_LOG: ${RUST_LOG:-warn,rustbox=info,tower_http=info} DATABASE_URL: mysql://rustbox:${MYSQL_PASSWORD:-rustbox-dev-password}@mysql:3306/rustbox SESSION_ENCRYPTION_KEY: ${SESSION_ENCRYPTION_KEY:-MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=} volumes: diff --git a/docker/compose-postgres-s3-opensearch.yml b/docker/compose-postgres-s3-opensearch.yml index 55d9e0a..e1b498d 100644 --- a/docker/compose-postgres-s3-opensearch.yml +++ b/docker/compose-postgres-s3-opensearch.yml @@ -76,7 +76,7 @@ services: SMTP_HOST: ${SMTP_HOST:-mail.example.com} SMTP_PORT: ${SMTP_PORT:-587} TLS_ENABLED: ${TLS_ENABLED:-true} - RUST_LOG: ${RUST_LOG:-info} + RUST_LOG: ${RUST_LOG:-warn,rustbox=info,tower_http=info} DATABASE_URL: postgres://rustbox:${POSTGRES_PASSWORD:-rustbox-dev-password}@postgres:5432/rustbox SESSION_ENCRYPTION_KEY: ${SESSION_ENCRYPTION_KEY:-MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=} AWS_ACCESS_KEY_ID: ${S3_ACCESS_KEY:-rustbox} diff --git a/docker/compose-single-sqlite.yml b/docker/compose-single-sqlite.yml index e16276f..fe390c2 100644 --- a/docker/compose-single-sqlite.yml +++ b/docker/compose-single-sqlite.yml @@ -17,7 +17,7 @@ services: SMTP_HOST: ${SMTP_HOST:-mail.example.com} SMTP_PORT: ${SMTP_PORT:-587} TLS_ENABLED: ${TLS_ENABLED:-true} - RUST_LOG: ${RUST_LOG:-info} + RUST_LOG: ${RUST_LOG:-warn,rustbox=info,tower_http=info} volumes: - ./configs/single-sqlite.toml:/etc/rustbox/rustbox.toml:ro - rustbox-data:/var/lib/rustbox diff --git a/docker/configs/ha-garage.toml b/docker/configs/ha-garage.toml index b6401bb..6a3e848 100644 --- a/docker/configs/ha-garage.toml +++ b/docker/configs/ha-garage.toml @@ -21,8 +21,16 @@ path_style = true allow_http = true prefix = "mail" +[sync] +# Background IMAP reconciliation cadence; active folders still use IDLE. +interval_secs = 30 +# Full bodies fetched per pass when search.deep_index is effective. +deep_index_batch_size = 10 + [search] backend = "opensearch" +# Dev: proactively cache/index full message bodies for testing. +deep_index = true urls = ["http://opensearch:9200"] index_prefix = "rustbox" allow_insecure_http = true @@ -30,3 +38,16 @@ required = true worker_interval_secs = 2 worker_batch_size = 100 worker_max_attempts = 8 + +# Administrator-only system diagnostics. Disabled by default and loopback-only when enabled. +# For monitoring in Docker/Kubernetes, explicitly add the monitoring/pod CIDR and keep +# token/basic credentials in the orchestrator secret store. +[diagnostics_endpoint] +enabled = false +path = "/api/system/diagnostics" +allowed_networks = ["127.0.0.1/32", "::1/128"] +auth = "token" +include_user_identifiers = false +probe_timeout_secs = 5 +security_window_secs = 900 + diff --git a/docker/configs/mysql-nfs-opensearch.toml b/docker/configs/mysql-nfs-opensearch.toml index a990a4f..4401a1e 100644 --- a/docker/configs/mysql-nfs-opensearch.toml +++ b/docker/configs/mysql-nfs-opensearch.toml @@ -11,8 +11,16 @@ backend = "filesystem" path = "/mnt/rustbox-shared" shared = true +[sync] +# Background IMAP reconciliation cadence; active folders still use IDLE. +interval_secs = 30 +# Full bodies fetched per pass when search.deep_index is effective. +deep_index_batch_size = 10 + [search] backend = "opensearch" +# Production: do not proactively download/store every full message body. +deep_index = false urls = ["http://opensearch:9200"] index_prefix = "rustbox" allow_insecure_http = true @@ -20,3 +28,16 @@ required = true worker_interval_secs = 2 worker_batch_size = 100 worker_max_attempts = 8 + +# Administrator-only system diagnostics. Disabled by default and loopback-only when enabled. +# For monitoring in Docker/Kubernetes, explicitly add the monitoring/pod CIDR and keep +# token/basic credentials in the orchestrator secret store. +[diagnostics_endpoint] +enabled = false +path = "/api/system/diagnostics" +allowed_networks = ["127.0.0.1/32", "::1/128"] +auth = "token" +include_user_identifiers = false +probe_timeout_secs = 5 +security_window_secs = 900 + diff --git a/docker/configs/postgres-s3-opensearch.toml b/docker/configs/postgres-s3-opensearch.toml index ef99ad7..6870cc6 100644 --- a/docker/configs/postgres-s3-opensearch.toml +++ b/docker/configs/postgres-s3-opensearch.toml @@ -15,8 +15,16 @@ path_style = true allow_http = true prefix = "mail" +[sync] +# Background IMAP reconciliation cadence; active folders still use IDLE. +interval_secs = 30 +# Full bodies fetched per pass when search.deep_index is effective. +deep_index_batch_size = 10 + [search] backend = "opensearch" +# Production: do not proactively download/store every full message body. +deep_index = false urls = ["http://opensearch:9200"] index_prefix = "rustbox" allow_insecure_http = true @@ -24,3 +32,16 @@ required = true worker_interval_secs = 2 worker_batch_size = 100 worker_max_attempts = 8 + +# Administrator-only system diagnostics. Disabled by default and loopback-only when enabled. +# For monitoring in Docker/Kubernetes, explicitly add the monitoring/pod CIDR and keep +# token/basic credentials in the orchestrator secret store. +[diagnostics_endpoint] +enabled = false +path = "/api/system/diagnostics" +allowed_networks = ["127.0.0.1/32", "::1/128"] +auth = "token" +include_user_identifiers = false +probe_timeout_secs = 5 +security_window_secs = 900 + diff --git a/docker/configs/single-sqlite.toml b/docker/configs/single-sqlite.toml index aafddd1..a629257 100644 --- a/docker/configs/single-sqlite.toml +++ b/docker/configs/single-sqlite.toml @@ -12,6 +12,27 @@ backend = "filesystem" path = "/var/lib/rustbox/blobs" shared = false +[sync] +# Background IMAP reconciliation cadence; active folders still use IDLE. +interval_secs = 30 +# Full bodies fetched per pass when search.deep_index is effective. +deep_index_batch_size = 10 + [search] backend = "local_tantivy" +# Production: do not proactively download/store every full message body. +deep_index = false path = "/var/lib/rustbox/search" + +# Administrator-only system diagnostics. Disabled by default and loopback-only when enabled. +# For monitoring in Docker/Kubernetes, explicitly add the monitoring/pod CIDR and keep +# token/basic credentials in the orchestrator secret store. +[diagnostics_endpoint] +enabled = false +path = "/api/system/diagnostics" +allowed_networks = ["127.0.0.1/32", "::1/128"] +auth = "token" +include_user_identifiers = false +probe_timeout_secs = 5 +security_window_secs = 900 + diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index a611c8f..9db52c5 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -1,8 +1,18 @@ -# RustBox 0.2.6 - implementation status +# RustBox 0.2.8 - implementation status -This file tracks the `0.2.6` source snapshot. Storage/search scope and outstanding acceptance work are tracked in `TODO.md`. A feature is only listed as implemented when working code exists in this source tree; partial items are called out explicitly. +This file tracks the `0.2.8` source snapshot. Storage/search scope and outstanding acceptance work are tracked in `TODO.md`. A feature is only listed as implemented when working code exists in this source tree; partial items are called out explicitly. -## 0.2.6 storage/search architecture +## 0.2.8 browser security headers + +- `security_headers_enabled` remains opt-in and disabled by default for compatibility; optional browser-security headers are centralized in this middleware, so disabling it removes the RustBox-added security profile while normal response semantics such as `Content-Type` and route-specific caching remain. +- HTML/XHTML documents receive the full document profile: CSP, `X-Frame-Options: DENY`, `Permissions-Policy`, `Cross-Origin-Opener-Policy: same-origin`, `Referrer-Policy: no-referrer` and `X-Content-Type-Options: nosniff`. +- The HTML CSP allows only same-origin scripts (`script-src 'self'`) and disables inline script attributes; the theme bootstrap is an external fingerprinted asset. Stylesheets are same-origin only (`style-src 'self'`), while `style-src-attr 'unsafe-inline'` is narrowly retained because the current UI still sets per-element style attributes dynamically. +- Private `/api` responses receive `Cache-Control: no-store`, `X-Content-Type-Options: nosniff`, `Cross-Origin-Resource-Policy: same-origin`, `Content-Security-Policy: frame-ancestors 'none'` and legacy-compatible `X-Frame-Options: DENY`. API JSON does not receive the full document CSP, `Permissions-Policy`, COOP or `Referrer-Policy`. +- CSS receives `Referrer-Policy: no-referrer` and `X-Content-Type-Options: nosniff`; application assets also receive same-origin CORP without redundant HTML document policies. +- HSTS is deliberately not synthesized by RustBox because the application may sit behind a TLS-terminating reverse proxy and cannot reliably determine the public HTTPS boundary; HSTS belongs at that HTTPS edge. +- The public `/api/auth/policy` is data-minimized to the single login-form capability `rememberMeEnabled`. Optional feature availability (OpenPGP, Sieve, diagnostics, About) is returned only from authenticated `/api/auth/session-policy`; session/remember durations are not exposed to the browser policy endpoints. + +## 0.2.8 storage/search architecture - SQL backend selection (`sqlite`/`mysql`/`pgsql`) is independent from blob storage and search backend selection. - Cluster mode clears an implicit/default `data_dir`; PostgreSQL/MySQL single-node deployments may still intentionally use `data_dir`-backed filesystem storage and local Tantivy. @@ -10,7 +20,7 @@ This file tracks the `0.2.6` source snapshot. Storage/search scope and outstandi - Draft attachment upload/read/delete/snapshot/send paths use the `BlobStorage` abstraction and logical object keys instead of assuming node-local paths. - OpenSearch is available as the cluster-capable indexer. Local Tantivy remains single-node only; search may also be disabled. - Search mutations use a durable SQL outbox with claim/lease, retry/backoff and dead-letter state; the index remains rebuildable derived state. -- V047 adds attachment checksum metadata and the search outbox across SQLite, MySQL and PostgreSQL. +- V047 adds attachment checksum metadata and the search outbox across SQLite, MySQL and PostgreSQL. V048 adds monotonic folder-counter generations used by compare-and-set reconciliation of authoritative IMAP counts; V049 is reserved as the next migration number. - SQL global-schema initialization is centralized in `db::pool::configure()`; database-backed sessions do not repeat startup DDL for SQLite, MySQL or PostgreSQL. - Docker Compose examples cover standalone SQLite, PostgreSQL + SILO (S3-compatible) + OpenSearch, and MySQL + NFS + OpenSearch. - Live SILO/MinIO/AWS S3/Ceph RGW, live NFS multi-host, live OpenSearch and full multi-node failover acceptance remain release-environment validation items and are not represented as completed here. @@ -26,6 +36,11 @@ This file tracks the `0.2.6` source snapshot. Storage/search scope and outstandi ### Mail and IMAP workflow +- Folder totals and unread counters use authoritative IMAP `STATUS (... MESSAGES UNSEEN ...)` data. If STATUS is unavailable for a selected mailbox, RustBox falls back to `UID SEARCH UNSEEN`; it never treats SELECT/EXAMINE `UNSEEN` (the first unseen sequence number) as an unread count. +- Folder counter reconciliation runs independently from content synchronization and repairs `MESSAGES/UNSEEN` even when `HIGHESTMODSEQ` is unchanged. A manual counter refresh reuses the cached folder tree while it is fresh and batches STATUS calls on one IMAP session instead of forcing another LIST. +- Read/unread/flag mutations update the affected row/badge optimistically, then reconcile against authoritative IMAP state and publish targeted flag/`FolderStatus` events without replacing the whole message list. The active folder is watched through IMAP IDLE; other folders are reconciled every 30 seconds. +- Large folders may hydrate message headers incrementally, but the sidebar total/unread badge remains server-authoritative. IDLE/periodic reconciliation keeps the CONDSTORE cursor separate from counter refresh, falls back to a full `UID FLAGS` comparison when CONDSTORE is unavailable, and discards partial IMAP fetch streams rather than applying incomplete state. The top-right activity indicator distinguishes active synchronization/indexing from an incomplete local IMAP header cache and exposes backlog details in its tooltip. +- Wide folder badges display `total / unread` while unread is non-zero and total alone at zero; narrow sidebars switch to the compact existing counter presentation. - IMAP folder list, hierarchy, create, create subfolder, rename, delete, subscribe/unsubscribe, plus collapsible branch navigation with per-account remembered expansion state. - Special folders stored per account: Sent, Drafts, Trash, Junk and Archive. Spam/Junk additionally supports a browser-wide default destination with a per-account override and IMAP special-use/name fallback. - Special-folder references follow folder renames and are cleared when a referenced folder is deleted. @@ -98,6 +113,7 @@ This file tracks the `0.2.6` source snapshot. Storage/search scope and outstandi - Account switch/remove/add workflow for multiple authenticated accounts in one browser session. - Active-session list across browsers while the RustBox process is running, with creation/last-active timestamps, friendly browser/device labels, full User-Agent hover detail, individual revoke and revoke-all-others. - Diagnostics panel exposing IMAP capabilities (including persistent custom-keyword support), authenticated SMTP connectivity, ManageSieve status, browser WebSocket state, local synchronization-cache timestamps/counts and live IMAP IDLE task/subscriber counts without returning server host names or credentials. Public liveness (`/api/health`) and local-storage readiness (`/api/ready`) endpoints are available for service supervision. +- Optional administrator system diagnostics endpoint (disabled by default) supports explicit IP/CIDR allowlisting plus token/Basic authentication, redacts configured secrets, probes SQL/session/storage/search backends, reports active sessions and IMAP IDLE state, and surfaces bounded node-local security signals for failed logins, 403/429 responses and 5xx failures. - Search-index diagnostics expose cached/indexed counts, with a rebuild operation from the configured account-data backend. - IMAP reconnect control drops reusable request-path sessions and verifies a fresh authenticated connection. - Graceful SIGTERM/Ctrl+C shutdown stops accepting new requests and aborts dedicated IMAP IDLE tasks before process exit. diff --git a/docs/RELEASING.md b/docs/RELEASING.md index 93a525b..4e12746 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -26,7 +26,7 @@ RUSTBOX_CONFIG=./config.dev.toml ./target/release/rustbox --check-config Browser JavaScript source lives in functional modules under `src/web/js/`, ordered by `src/web/js/manifest.txt`. `build.rs` concatenates those modules into the embedded `rustbox-ui.js` asset during every Cargo build. The repository has no duplicate `ui/` snapshot; `src/web/` is the only browser UI source. -## Updating an existing checkout to 0.2.6 +## Updating an existing checkout to 0.2.8 Do not overlay the release with `cp *` or another command that skips dotfiles. That can retain stale `.github`, `.env.example`, or `Cargo.lock` files. diff --git a/docs/STORAGE_SEARCH_OPERATIONS.md b/docs/STORAGE_SEARCH_OPERATIONS.md index 6e325a6..fc627e9 100644 --- a/docs/STORAGE_SEARCH_OPERATIONS.md +++ b/docs/STORAGE_SEARCH_OPERATIONS.md @@ -1,6 +1,6 @@ -# RustBox 0.2.6 storage/search operations +# RustBox 0.2.8 storage/search operations -RustBox 0.2.6 treats SQL, blob storage and search as independent capabilities. The database engine does not decide whether `data_dir` is allowed. +RustBox 0.2.8 treats SQL, blob storage and search as independent capabilities. The database engine does not decide whether `data_dir` is allowed. ## Deployment matrix @@ -50,6 +50,14 @@ The rebuild removes superseded outbox events older than the rebuild start and ke If `search.required=false`, an unavailable OpenSearch backend may leave the application usable while the outbox grows; readiness reports degraded search and workers retry later. With `search.required=true`, search health is a startup/readiness requirement. +### Automatic body cache / deep index + +Automatic full-body caching is controlled by `search.deep_index`. When it is `true` and the search backend is enabled, every periodic IMAP reconciliation may fetch full MIME bodies for messages that have headers cached but no body/index yet. `sync.deep_index_batch_size` caps the number of bodies fetched in one pass. Full bodies are stored in the per-account application database/cache and then queued/indexed, so enabling this increases IMAP bandwidth, application-database/search-storage use and indexing CPU/I/O. + +`search.deep_index=false` is the production default. It prevents a server-wide forced full-mailbox download, but preserves the existing per-account `deep_index` opt-in. Development configuration sets `search.deep_index=true` so cache/index behavior is exercised without an additional preference change. + +`sync.interval_secs` controls the periodic reconciliation cadence. Active folders still use IMAP IDLE for near-real-time events; the periodic pass is the fallback/background reconciliation for counters, flags and headers in other folders and is also the cadence for automatic deep-index batches. Lower values converge faster but create more IMAP requests. + ## Failure behavior - SQL unavailable: authoritative mailbox/session/account operations cannot proceed. diff --git a/docs/SYSTEM_DIAGNOSTICS.md b/docs/SYSTEM_DIAGNOSTICS.md new file mode 100644 index 0000000..949eb8a --- /dev/null +++ b/docs/SYSTEM_DIAGNOSTICS.md @@ -0,0 +1,149 @@ +# Administrator system diagnostics endpoint + +RustBox can expose a single administrator-only JSON endpoint for operational and security diagnostics. It is intended for local troubleshooting, monitoring agents and cluster operations. It is **disabled by default** and, when enabled without changing the allowlist, accepts only loopback clients. + +The endpoint is separate from the user-facing `/api/diagnostics/*` routes. It does not require a RustBox mailbox session. Instead, access requires **both**: + +1. the effective client IP to match `diagnostics_endpoint.allowed_networks`, and +2. the configured diagnostics token or Basic Auth credentials. + +Requests from a non-allowed address receive HTTP `403`. Requests from an allowed address with missing/invalid diagnostics credentials receive HTTP `401`. Critical backend probe failures return HTTP `503`; healthy or degraded reports return HTTP `200` with `status` set to `ok` or `degraded`. + +## Configuration + +```toml +[diagnostics_endpoint] +enabled = false +path = "/api/system/diagnostics" +allowed_networks = ["127.0.0.1/32", "::1/128"] +auth = "token" # token | basic +# token = "replace-with-at-least-24-random-characters" +# basic_username = "rustbox-monitor" +# basic_password = "replace-with-a-long-random-password" +include_user_identifiers = false +probe_timeout_secs = 5 +security_window_secs = 900 +``` + +`path` must remain below `/api/system/` so it cannot collide with normal application API routes. + +### Token authentication + +Prefer keeping the token outside TOML, for example in `.env` or an orchestrator secret: + +```bash +DIAGNOSTICS_ENDPOINT__ENABLED=true +DIAGNOSTICS_ENDPOINT__AUTH=token +DIAGNOSTICS_ENDPOINT__TOKEN='a-long-random-secret-at-least-24-characters' +``` + +Example request: + +```bash +curl -fsS \ + -H "Authorization: Bearer $DIAGNOSTICS_TOKEN" \ + http://127.0.0.1:3001/api/system/diagnostics +``` + +`X-RustBox-Diagnostics-Token` is accepted as an alternative to the Bearer header for monitoring systems that cannot set an `Authorization` header. + +### Basic authentication + +```toml +[diagnostics_endpoint] +enabled = true +allowed_networks = ["127.0.0.1/32", "::1/128"] +auth = "basic" +``` + +```bash +DIAGNOSTICS_ENDPOINT__BASIC_USERNAME='rustbox-monitor' +DIAGNOSTICS_ENDPOINT__BASIC_PASSWORD='a-long-random-monitoring-password' +``` + +Basic Auth must only be used over HTTPS/TLS (directly or at a trusted reverse proxy), because the credentials are not encrypted by the Basic scheme itself. + +## Networks and reverse proxies + +The network allowlist accepts IPv4/IPv6 addresses and CIDRs. Single addresses may also be specified in TOML. + +Example monitoring network: + +```toml +[diagnostics_endpoint] +enabled = true +allowed_networks = [ + "127.0.0.1/32", + "::1/128", + "10.42.20.0/24", + "2001:db8:42::/64", +] +auth = "token" +``` + +For a reverse proxy, configure `trusted_proxies` separately. RustBox only accepts `X-Forwarded-For` for diagnostics when the directly connected proxy is trusted. If proxy headers arrive from an untrusted peer, diagnostics fails closed with HTTP `403` rather than treating the proxy address as the real client. + +```toml +trusted_proxies = "127.0.0.1/32,10.42.0.0/16" + +[diagnostics_endpoint] +enabled = true +allowed_networks = ["10.42.20.0/24"] +auth = "token" +``` + +A trusted proxy should provide a valid `X-Forwarded-For`. `Forwarded` or `X-Real-IP` without `X-Forwarded-For` is deliberately rejected for this endpoint. In production, also restrict the diagnostics path at the firewall, ingress or reverse-proxy layer. + +## Returned data + +The report contains: + +- application version, process uptime/start time and PID; host-wide CPU, memory, load-average and other machine statistics are deliberately not collected; +- effective non-secret configuration and feature switches; +- IMAP/SMTP/Sieve endpoint configuration without credentials; +- SQL backend, engine version, query latency, account/schema versions, SQLite file size or SQL pool size/idle connection count; +- session persistence health and aggregate active session/user/browser counts for SQL or Redis; +- local-node cached sessions, timestamps and mail endpoints; e-mail/user hash identifiers appear only when `include_user_identifiers=true`; +- filesystem/S3 configuration plus a live storage health probe; +- Tantivy/OpenSearch configuration; OpenSearch adds cluster health, node/shard counts and index document count; +- active IMAP IDLE task/subscriber counts; +- request counters and a bounded in-memory security window for failed logins, access denials, rate-limit responses and server errors; +- configuration warnings for risky settings such as disabled TLS verification, HTTP OpenSearch/S3, global diagnostics allowlists or production browser security headers being disabled. + +## Secret handling + +The endpoint never returns diagnostics credentials, mailbox passwords, session encryption keys, Redis passwords, S3 access/secret/session keys, OpenSearch passwords or full credential-bearing DSNs. URLs are reduced to scheme/host/port/path and query strings/userinfo are removed. Secret fields are represented only as `...Configured: true|false` where useful. + +`include_user_identifiers=false` is the default. With that setting, local session details omit e-mail addresses and stable user hashes while aggregate user/session counts remain visible. + +All endpoint responses, including `401` and `403`, carry `Cache-Control: no-store`, `Pragma: no-cache`, `X-Content-Type-Options: nosniff` and `Cross-Origin-Resource-Policy: same-origin` independently of the general browser security-header switch. + +## Security signals + +The telemetry buffer is bounded and in-memory; it stores only timestamp, event category, source IP and request path. It does **not** store credentials, Authorization headers, cookies, tokens, request bodies or mail content. + +Within `security_window_secs` the report includes counts and top source IPs for: + +- rejected login attempts; +- HTTP `429` rate limiting; +- HTTP `403` access denials; +- HTTP `5xx` server failures; +- denied diagnostics access. + +`bruteForceSignal` becomes true after repeated login rejection/rate-limit activity, while `diagnosticsAttackSignal` flags repeated attempts against the management endpoint. These are preliminary operational signals, not an IDS or an account-lockout mechanism. In a multi-node deployment the counters are node-local; aggregate application/security logs in a central monitoring/SIEM system for cluster-wide detection. + +## Process and host metrics + +The diagnostics endpoint deliberately avoids host-wide metrics and platform-specific process probes. It does not read `/proc`, query system CPU/RAM/load averages, enumerate host resources or add a system-information dependency. The portable process section is limited to the RustBox PID, while process uptime/start time and request/security counters come from RustBox's own in-memory runtime state. This keeps the endpoint low-overhead and portable across supported operating systems. + +## Probe cost + +The endpoint performs live backend checks, so it should not be polled at sub-second frequency: + +- SQL executes lightweight health/version/schema queries; +- Redis uses `PING` plus bounded `SCAN`/`MGET` of at most 5000 session records and reports `truncated=true` if the cap is hit; +- S3/filesystem health writes and deletes a tiny temporary `_health/...` object; +- OpenSearch queries cluster health and the configured index count; +- each backend probe has its own `probe_timeout_secs` timeout. + +A polling interval around tens of seconds is appropriate for typical monitoring. Use the normal `/api/health` or `/api/ready` endpoints for very frequent liveness/readiness checks. diff --git a/migrations/README.md b/migrations/README.md index a05dcca..e38d132 100644 --- a/migrations/README.md +++ b/migrations/README.md @@ -1,6 +1,6 @@ # Database schemas and migrations -RustBox 0.2.6 keeps SQL, blob storage and search as independent concerns: +RustBox 0.2.8 keeps SQL, blob storage and search as independent concerns: ```text database_backend = sqlite | mysql | pgsql @@ -17,16 +17,19 @@ When `session_backend = "database"`, sessions live in the selected SQL database. migrations/ ├── sqlite/ │ ├── schema.sql -│ └── V047__blob_keys_and_search_outbox.sql +│ ├── V047__blob_keys_and_search_outbox.sql +│ └── V048__folder_counter_generation.sql ├── mysql/ │ ├── schema.sql -│ └── V047__blob_keys_and_search_outbox.sql +│ ├── V047__blob_keys_and_search_outbox.sql +│ └── V048__folder_counter_generation.sql └── pgsql/ ├── schema.sql - └── V047__blob_keys_and_search_outbox.sql + ├── V047__blob_keys_and_search_outbox.sql + └── V048__folder_counter_generation.sql ``` -Each `schema.sql` is the current complete schema for fresh installations. The historical compatibility baseline remains **V046**. RustBox 0.2.1 adds **V047**, which adds attachment SHA-256 metadata and the durable search outbox used by external indexing. +Each `schema.sql` is the current complete schema for fresh installations. The historical compatibility baseline remains **V046**. RustBox 0.2.1 adds **V047**, which adds attachment SHA-256 metadata and the durable search outbox used by external indexing. RustBox 0.2.8 adds **V048**, which adds the monotonic `folders.counter_generation` used to reject stale IMAP counter writes even when the numeric counts temporarily return to an earlier value. Account data is isolated inside the selected physical database with a deterministic, collision-checked table prefix derived from the account hash. The `{{p}}` marker in tenant SQL is replaced by RustBox at provisioning/migration time. Sessions are global and therefore are not prefixed. @@ -34,7 +37,7 @@ Account data is isolated inside the selected physical database with a determinis Every schema-changing release must add the same logical migration for all supported SQL engines and register it in `src/db/pool.rs`. Do not update only `schema.sql`: fresh installs use the baseline while existing installations use the numbered migration path. -The next migration number after RustBox 0.2.6 is **V048**. +The next migration number after RustBox 0.2.8 is **V049**. Redis and Redis Sentinel do not use SQL migrations. diff --git a/migrations/mysql/V048__folder_counter_generation.sql b/migrations/mysql/V048__folder_counter_generation.sql new file mode 100644 index 0000000..8b6e516 --- /dev/null +++ b/migrations/mysql/V048__folder_counter_generation.sql @@ -0,0 +1 @@ +ALTER TABLE {{p}}folders ADD COLUMN counter_generation BIGINT NOT NULL DEFAULT 0; diff --git a/migrations/mysql/schema.sql b/migrations/mysql/schema.sql index 132c830..2ba713f 100644 --- a/migrations/mysql/schema.sql +++ b/migrations/mysql/schema.sql @@ -1,7 +1,7 @@ -- RustBox 0.2.0 MySQL schema. -- One physical database contains global sessions plus isolated per-account tables. -- Application tables are materialized by replacing {{p}} with a collision-checked prefix. --- Squashed compatibility baseline: V046. Current runtime schema: V047. +-- Squashed compatibility baseline: V046. Current runtime schema: V048. -- RUSTBOX:GLOBAL-BEGIN CREATE TABLE IF NOT EXISTS rustbox_accounts ( @@ -247,6 +247,7 @@ CREATE TABLE {{p}}folders ( unread_count BIGINT NOT NULL DEFAULT 0, uid_validity BIGINT NOT NULL DEFAULT 0, highest_modseq BIGINT NOT NULL DEFAULT 0, + counter_generation BIGINT NOT NULL DEFAULT 0, updated_at VARCHAR(64) NOT NULL DEFAULT (CAST(CURRENT_TIMESTAMP AS CHAR)), messages_updated_at VARCHAR(64) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/migrations/pgsql/V048__folder_counter_generation.sql b/migrations/pgsql/V048__folder_counter_generation.sql new file mode 100644 index 0000000..8b6e516 --- /dev/null +++ b/migrations/pgsql/V048__folder_counter_generation.sql @@ -0,0 +1 @@ +ALTER TABLE {{p}}folders ADD COLUMN counter_generation BIGINT NOT NULL DEFAULT 0; diff --git a/migrations/pgsql/schema.sql b/migrations/pgsql/schema.sql index 6357d72..4506f92 100644 --- a/migrations/pgsql/schema.sql +++ b/migrations/pgsql/schema.sql @@ -1,7 +1,7 @@ -- RustBox 0.2.0 PostgreSQL schema. -- One physical database contains global sessions plus isolated per-account tables. -- Application tables are materialized by replacing {{p}} with a collision-checked prefix. --- Squashed compatibility baseline: V046. Current runtime schema: V047. +-- Squashed compatibility baseline: V046. Current runtime schema: V048. -- RUSTBOX:GLOBAL-BEGIN CREATE TABLE IF NOT EXISTS rustbox_accounts ( @@ -248,6 +248,7 @@ CREATE TABLE {{p}}folders ( unread_count BIGINT NOT NULL DEFAULT 0, uid_validity BIGINT NOT NULL DEFAULT 0, highest_modseq BIGINT NOT NULL DEFAULT 0, + counter_generation BIGINT NOT NULL DEFAULT 0, updated_at TEXT NOT NULL DEFAULT (CURRENT_TIMESTAMP::text), messages_updated_at TEXT ); diff --git a/migrations/sqlite/V048__folder_counter_generation.sql b/migrations/sqlite/V048__folder_counter_generation.sql new file mode 100644 index 0000000..0b92a5a --- /dev/null +++ b/migrations/sqlite/V048__folder_counter_generation.sql @@ -0,0 +1 @@ +ALTER TABLE {{p}}folders ADD COLUMN counter_generation INTEGER NOT NULL DEFAULT 0; diff --git a/migrations/sqlite/schema.sql b/migrations/sqlite/schema.sql index eb511e8..8e85cb1 100644 --- a/migrations/sqlite/schema.sql +++ b/migrations/sqlite/schema.sql @@ -1,7 +1,7 @@ -- RustBox 0.2.0 SQLite schema. -- One physical database contains global sessions plus isolated per-account tables. -- Application tables are materialized from the tenant section by replacing {{p}}. --- Squashed compatibility baseline: V046. Current runtime schema: V047. +-- Squashed compatibility baseline: V046. Current runtime schema: V048. -- RUSTBOX:GLOBAL-BEGIN CREATE TABLE IF NOT EXISTS rustbox_accounts ( @@ -184,6 +184,7 @@ CREATE TABLE {{p}}folders ( unread_count INTEGER NOT NULL DEFAULT 0, uid_validity INTEGER NOT NULL DEFAULT 0, highest_modseq INTEGER NOT NULL DEFAULT 0, + counter_generation INTEGER NOT NULL DEFAULT 0, updated_at TEXT NOT NULL DEFAULT (datetime('now')), messages_updated_at TEXT); diff --git a/packaging/rustbox.toml b/packaging/rustbox.toml index dd76827..86ea358 100644 --- a/packaging/rustbox.toml +++ b/packaging/rustbox.toml @@ -1,4 +1,4 @@ -# RustBox 0.2.6 - packaged production configuration (DEB/RPM) +# RustBox 0.2.8 - packaged production configuration (DEB/RPM) # # This file contains all supported RustBox configuration options. # Commented options are optional and do not change the packaged defaults. @@ -39,10 +39,8 @@ default_interface_skin = "apple-mail" sieve_enabled = true diagnostics_enabled = true about_enabled = true -# Browser-facing CSP/frame/referrer/permissions headers. Off by default for compatibility. +# Content-type-aware browser security headers. Off by default for compatibility. security_headers_enabled = false - - # ----------------------------------------------------------------------------- # Database # ----------------------------------------------------------------------------- @@ -212,20 +210,6 @@ pgp_allow_private_export = true # # pgp_enabled = false - -# ----------------------------------------------------------------------------- -# Blob storage -# ----------------------------------------------------------------------------- -# -# Blob storage is independent from database_backend. -# -# PostgreSQL/MySQL can use a local filesystem in single-node mode. -# -# Cluster deployments must use: -# - S3-compatible object storage, or -# - a genuinely shared POSIX filesystem such as NFS/CephFS/GlusterFS. -# - [storage] # filesystem | s3 @@ -284,6 +268,23 @@ retry_timeout_secs = 60 # kms_key_id = "..." +# ----------------------------------------------------------------------------- +# IMAP synchronization / automatic body cache +# ----------------------------------------------------------------------------- + +[sync] + +# Periodic fallback/background IMAP reconciliation. Active folders use IMAP IDLE, +# while this pass refreshes counters/flags/headers in the rest of the mailbox and +# also runs automatic deep indexing when enabled. Lower values increase IMAP load. +interval_secs = 30 + +# Maximum full message bodies fetched from IMAP per periodic pass when deep index is +# effective. Larger batches catch up faster but increase bandwidth, application database/ +# search-storage usage and indexing CPU/I/O bursts. +deep_index_batch_size = 10 + + # ----------------------------------------------------------------------------- # Search # ----------------------------------------------------------------------------- @@ -327,6 +328,13 @@ allow_insecure_http = false request_timeout_secs = 10 +# Server-wide automatic full-body cache/index switch. Production default is false +# because enabling it proactively downloads and stores full message bodies. When +# true, all accounts are deep-indexed; when false, an account may still opt in via +# its existing deep_index preference. +deep_index = false + +# Search outbox worker cadence/batch; these do NOT control IMAP body downloads. worker_interval_secs = 2 worker_batch_size = 100 worker_max_attempts = 12 @@ -378,6 +386,7 @@ worker_max_attempts = 12 # required = true # allow_insecure_http = false # request_timeout_secs = 10 +# deep_index = false # worker_interval_secs = 2 # worker_batch_size = 100 # worker_max_attempts = 12 @@ -430,4 +439,40 @@ worker_max_attempts = 12 # ] # redis_sentinel_master = "mymaster" # redis_sentinel_password = "sentinel-secret" -# redis_password = "secret" \ No newline at end of file +# redis_password = "secret" + + +# ----------------------------------------------------------------------------- +# Blob storage +# ----------------------------------------------------------------------------- +# +# Blob storage is independent from database_backend. +# +# PostgreSQL/MySQL can use a local filesystem in single-node mode. +# +# Cluster deployments must use: +# - S3-compatible object storage, or +# - a genuinely shared POSIX filesystem such as NFS/CephFS/GlusterFS. +# + +# ----------------------------------------------------------------------------- +# Administrator system diagnostics endpoint +# ----------------------------------------------------------------------------- +# Disabled by default. When enabled, the endpoint performs live probes of SQL, +# session persistence/Redis, filesystem/S3, Tantivy/OpenSearch and reports +# process/runtime/IMAP-IDLE/security telemetry. Returned configuration is redacted: +# passwords, auth tokens, S3 secrets, session keys and URL credentials are never emitted. +# Access is an AND condition: source IP/CIDR must be allowed and token/basic auth must pass. +[diagnostics_endpoint] +enabled = false +path = "/api/system/diagnostics" +# Fail-safe default. Add only monitoring/admin networks that really need access. +allowed_networks = ["127.0.0.1/32", "::1/128"] +auth = "token" # token | basic +# Prefer injecting these with a secret manager/environment rather than storing here. +# token = "at-least-24-random-characters" +# basic_username = "rustbox-monitor" +# basic_password = "long-random-password" +include_user_identifiers = false +probe_timeout_secs = 5 +security_window_secs = 900 diff --git a/packaging/systemd/rustbox.service b/packaging/systemd/rustbox.service index 3cc5d30..7b5cbc6 100644 --- a/packaging/systemd/rustbox.service +++ b/packaging/systemd/rustbox.service @@ -8,7 +8,7 @@ Type=simple User=rustbox Group=rustbox Environment=RUSTBOX_CONFIG=/etc/rustbox/rustbox.toml -#Environment=RUST_LOG=info +#Environment=RUST_LOG=warn,rustbox=info,tower_http=info WorkingDirectory=/var/lib/rustbox ExecStart=/usr/local/bin/rustbox Restart=on-failure diff --git a/scripts/validate-release-static.py b/scripts/validate-release-static.py index 8a95dff..4aa9e3f 100755 --- a/scripts/validate-release-static.py +++ b/scripts/validate-release-static.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Static/offline release checks for RustBox 0.2.6. +"""Static/offline release checks for RustBox 0.2.8. These checks intentionally avoid pretending to replace `cargo test`: they validate configuration, SQL migrations, Docker examples, query-catalog parity and release @@ -49,14 +49,14 @@ def extract_section(text: str, begin: str, end: str) -> str: # Release/toolchain metadata. cargo = tomllib.loads(read("Cargo.toml")) -require(cargo["package"]["version"] == "0.2.6", "Cargo package version is 0.2.6") +require(cargo["package"]["version"] == "0.2.8", "Cargo package version is 0.2.8") require(cargo["package"]["rust-version"] == "1.99.1", "Cargo rust-version is 1.99.1") toolchain = tomllib.loads(read("rust-toolchain.toml")) require(toolchain["toolchain"]["channel"] == "1.99.1", "rust-toolchain pins Rust 1.99.1") require((ROOT / "Cargo.lock").is_file(), "Cargo.lock is retained for reproducible application builds") lock = tomllib.loads(read("Cargo.lock")) lock_rustbox = next((pkg for pkg in lock.get("package", []) if pkg.get("name") == "rustbox"), {}) -require(lock_rustbox.get("version") == "0.2.6", "Cargo.lock RustBox package version is 0.2.6") +require(lock_rustbox.get("version") == "0.2.8", "Cargo.lock RustBox package version is 0.2.8") require("cargo generate-lockfile" in read("scripts/build-release.sh"), "release script generates Cargo.lock when missing") release_script = read("scripts/build-release.sh") release_script_flat = " ".join(release_script.replace("\\", " ").split()) @@ -69,14 +69,24 @@ require("make_zip.py" in package_script and "sha256sum" in package_script, "rele require("cargo generate-lockfile" in read(".github/workflows/ci.yml"), "CI regenerates Cargo.lock before --locked checks") require("validate-release-static.py" in read("scripts/build-release.sh"), "release script runs offline/static release gate") require("validate-release-static.py" in read(".github/workflows/ci.yml"), "CI runs offline/static release gate") -require(read("README.md").startswith("# RustBox 0.2.6"), "README identifies RustBox 0.2.6") -require(read("RELEASE-MANIFEST.txt").startswith("RustBox 0.2.6"), "release manifest identifies RustBox 0.2.6") -require(read("RELEASE-VALIDATION.txt").startswith("RustBox 0.2.6"), "release validation identifies RustBox 0.2.6") -require(read("docs/IMPLEMENTATION_STATUS.md").startswith("# RustBox 0.2.6"), "implementation status identifies RustBox 0.2.6") +require(read("README.md").startswith("# RustBox 0.2.8"), "README identifies RustBox 0.2.8") +require(read("RELEASE-MANIFEST.txt").startswith("RustBox 0.2.8"), "release manifest identifies RustBox 0.2.8") +require(read("RELEASE-VALIDATION.txt").startswith("RustBox 0.2.8"), "release validation identifies RustBox 0.2.8") +require(read("docs/IMPLEMENTATION_STATUS.md").startswith("# RustBox 0.2.8"), "implementation status identifies RustBox 0.2.8") +require("V048" in read("RELEASE-MANIFEST.txt") and "V049" in read("RELEASE-MANIFEST.txt"), "release manifest documents V048 and reserves V049") +require("V048" in read("migrations/README.md") and "V049" in read("migrations/README.md"), "migration guide documents V048 and reserves V049") require((ROOT / "docs/STORAGE_SEARCH_OPERATIONS.md").is_file(), "storage/search operations guide exists") require('imap_host = ""' not in read("config.dev.toml"), "development config does not override IMAP host with an invalid empty value") require(not re.search(r"(?m)^IMAP_HOST=\s*$", read(".env.example")), "environment example does not set an invalid empty IMAP host") -require("pgp_enabled: bool" in read("src/routes/auth.rs"), "auth policy exposes OpenPGP availability") +dev_sh = read("dev.sh") +require('ENV_FILE="${RUSTBOX_ENV_FILE:-$ROOT/.env}"' in dev_sh and '. "$ENV_FILE"' in dev_sh, "development launcher loads project .env") +require(dev_sh.find('. "$ENV_FILE"') < dev_sh.find('cargo run --quiet --bin rustbox -- --check-config'), "development launcher loads .env before config validation") +require('warn,rustbox=debug,tower_http=info' in dev_sh, "development launcher keeps dependencies at WARN while RustBox stays DEBUG") +require('case "${RUST_LOG:-}" in' in dev_sh and 'trace)' in dev_sh and 'info)' in dev_sh, "development launcher scopes legacy/simple global RUST_LOG levels") +require('RUST_LOG=warn,rustbox=debug,tower_http=info' in read(".env.example"), "development environment example does not enable noisy dependency INFO logs") +main_rs = read("src/main.rs") +require('"debug" => "warn,rustbox=debug,tower_http=info"' in main_rs and 'EnvFilter::new(default_log_filter(log_level))' in main_rs, "config log_level controls RustBox verbosity without globally enabling dependency DEBUG/INFO") +require("pub struct SessionPolicyResponse" in read("src/routes/auth.rs") and "pgp_enabled: bool" in read("src/routes/auth.rs"), "authenticated session policy exposes OpenPGP availability") require("setOpenPgpAvailability" in read("src/web/js/72-openpgp-identities.js"), "UI hides administratively disabled OpenPGP") storage_rs = read("src/storage/mod.rs") require( @@ -95,9 +105,143 @@ config_rs = read("src/config.rs") routes_mod = read("src/routes/mod.rs") require("pub sieve_enabled: bool" in config_rs and "pub diagnostics_enabled: bool" in config_rs and "pub about_enabled: bool" in config_rs, "configuration exposes Sieve, diagnostics and About feature gates") require("pub security_headers_enabled: bool" in config_rs and "security_headers_enabled: false" in config_rs, "security headers are configurable and disabled by default") +require("pub struct SyncConfig" in config_rs and "pub interval_secs: u64" in config_rs and "pub deep_index_batch_size: u32" in config_rs, "IMAP sync cadence and deep-index batch size are configurable") +require("pub deep_index: bool" in config_rs and "deep_index: false" in config_rs, "server-wide automatic deep index is configurable and production-safe by default") +system_diag_rs = read("src/routes/system_diagnostics.rs") +session_rs = read("src/auth/session.rs") +session_persistence_rs = read("src/session_store/mod.rs") +require("pub struct DiagnosticsEndpointConfig" in config_rs and "pub diagnostics_endpoint: DiagnosticsEndpointConfig" in config_rs, "administrator diagnostics endpoint has a dedicated configuration block") +require('default_diagnostics_allowed_networks' in config_rs and '"127.0.0.1/32"' in config_rs and '"::1/128"' in config_rs, "administrator diagnostics allowlist defaults to IPv4/IPv6 loopback") +require('diagnostics_endpoint.path must be an absolute /api/system/' in config_rs and 'diagnostics_endpoint.token must contain at least 24 characters' in config_rs and 'diagnostics_endpoint.basic_password must contain at least 16 characters' in config_rs, "administrator diagnostics validates namespace and authentication strength") +require('if config.diagnostics_endpoint.enabled' in routes_mod and 'get(system_diagnostics::system_diagnostics)' in routes_mod, "administrator diagnostics route is registered only when enabled") +require('diagnostics_client_ip' in system_diag_rs and 'proxy headers received from an untrusted peer' in system_diag_rs, "administrator diagnostics fails closed on untrusted reverse-proxy headers") +require('parsed_diagnostics_allowed_networks' in system_diag_rs and 'return forbidden();' in system_diag_rs, "administrator diagnostics enforces source IP/CIDR allowlist before probes") +require('constant_time_secret_eq' in system_diag_rs and 'basic_credentials' in system_diag_rs and 'bearer_token' in system_diag_rs, "administrator diagnostics supports protected token and Basic authentication") +require('HeaderValue::from_static("no-store")' in system_diag_rs and 'harden_diagnostics_response' in system_diag_rs, "administrator diagnostics responses are explicitly non-cacheable") +require('"database": db_diag' in system_diag_rs and '"storage": {' in system_diag_rs and '"search": search_status' in system_diag_rs and '"sessions": {' in system_diag_rs, "administrator diagnostics reports database, storage, search and session state") +require('bruteForceSignal' in system_diag_rs and 'diagnosticsAttackSignal' in system_diag_rs and 'MAX_SECURITY_EVENTS' in system_diag_rs, "administrator diagnostics includes bounded brute-force/access-denial telemetry") +require('pub struct SessionPersistenceDiagnostics' in session_persistence_rs and 'diagnostics_snapshot' in session_rs, "administrator diagnostics reports local and persisted session state") +require('user_hash: Option' in session_rs and 'include_user_identifiers.then' in session_rs, "administrator diagnostics omits stable user identifiers unless explicitly enabled") +require('schema_aligned: Option' in read("src/db/pool.rs") and 'expected_schema_version' in read("src/db/pool.rs"), "administrator diagnostics reports database schema alignment") +require('connected_clients' in read("src/session_store/redis_store.rs") and 'used_memory_bytes' in read("src/session_store/redis_store.rs") and 'master_link_status' in read("src/session_store/redis_store.rs"), "administrator diagnostics reports Redis runtime INFO without credentials") +require('"idleConnections"' in system_diag_rs and 'diagnostics_snapshot' in read("src/realtime/idle.rs"), "administrator diagnostics reports active IMAP IDLE connections") +require('cluster_status.eq_ignore_ascii_case("green")' in read("src/search/engine.rs") and 'cluster_status.eq_ignore_ascii_case("red")' in read("src/search/engine.rs"), "administrator diagnostics treats OpenSearch health conservatively") +require('middleware::from_fn(system_diagnostics::observe_requests)' in routes_mod and routes_mod.count('if config.diagnostics_endpoint.enabled') >= 2, "administrator diagnostics telemetry middleware is installed only when the endpoint is enabled") +require('sanitize_url' in system_diag_rs and 'secretKeyConfigured' in system_diag_rs and 'passwordConfigured' in system_diag_rs and 'credentialConfigured' in system_diag_rs, "administrator diagnostics redacts configured credentials and DSN userinfo") +require('/proc/' not in system_diag_rs and 'systemMemoryTotalKiB' not in system_diag_rs and 'loadAverage' not in system_diag_rs and 'availableProcessors' not in system_diag_rs, "administrator diagnostics avoids host-specific /proc and machine-wide resource metrics") +require((ROOT / "docs/SYSTEM_DIAGNOSTICS.md").is_file() and "disabled by default" in read("docs/SYSTEM_DIAGNOSTICS.md") and "HTTP `403`" in read("docs/SYSTEM_DIAGNOSTICS.md"), "administrator diagnostics security and operations are documented") require("if config.sieve_enabled" in routes_mod and "if config.diagnostics_enabled" in routes_mod, "disabled Sieve and diagnostics routes are not registered") require(".fallback(|| async { axum::http::StatusCode::NOT_FOUND })" in routes_mod, "API namespace has an explicit 404 fallback before the SPA fallback") require("if config.security_headers_enabled" in routes_mod and "add_security_headers" in routes_mod, "security-header middleware is opt-in") +ui_rs = read("src/ui.rs") +pgp_rs = read("src/routes/pgp.rs") +contacts_rs = read("src/routes/contacts.rs") +require("X_CONTENT_TYPE_OPTIONS" not in ui_rs and "x-content-type-options" not in pgp_rs and "x-content-type-options" not in contacts_rs, "security headers are centralized behind the opt-in middleware") +require("response_content_type_is" in routes_mod, "security-header middleware classifies response content types") +require("headers.insert(\n headers.insert(" not in routes_mod, "security-header middleware has no duplicated insert fragment") +require("if is_html || is_css" in routes_mod, "Referrer-Policy is limited to HTML/CSS responses") +require("if is_html {" in routes_mod, "document-only browser policies are limited to HTML responses") +require("script-src 'self'; script-src-attr 'none'" in routes_mod, "HTML CSP permits only same-origin external scripts and disables script attributes") +require("style-src 'self'; style-src-attr 'unsafe-inline'" in routes_mod and "style-src 'self' 'unsafe-inline'" not in routes_mod, "HTML CSP limits unsafe-inline to dynamic style attributes rather than stylesheet blocks") +require("sha256-" not in routes_mod and "script-src 'self' 'unsafe-inline'" not in routes_mod, "HTML CSP does not rely on inline-script hashes or unsafe-inline") +require('headers.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store"))' in routes_mod, "API security middleware forces private responses to no-store") +require( + "HeaderValue::from_static(\"frame-ancestors 'none'\")" in routes_mod + and 'headers.insert("x-frame-options", HeaderValue::from_static("DENY"))' in routes_mod, + "browser API security profile includes explicit frame denial", +) +html_security_start = routes_mod.find("if is_html {") +html_security_end = routes_mod.find("\n response", html_security_start) +require( + html_security_start >= 0 + and html_security_end > html_security_start + and routes_mod.count('"permissions-policy"') == 1 + and routes_mod.count('"cross-origin-opener-policy"') == 1 + and html_security_start < routes_mod.find('"permissions-policy"') < html_security_end + and html_security_start < routes_mod.find('"cross-origin-opener-policy"') < html_security_end, + "API security profile keeps Permissions-Policy and COOP on documents only", +) +require('"cross-origin-resource-policy"' in routes_mod and 'HeaderValue::from_static("same-origin")' in routes_mod, "security middleware applies same-origin CORP where appropriate") +route_tests = read("src/routes/tests.rs") +auth_routes = read("src/routes/auth.rs") +auth_js = read("src/web/js/20-preferences-auth.js") +require("security_headers_are_opt_in_and_content_type_aware" in route_tests, "security-header regression test covers content-type-aware behavior") +require("auth_policy_exposes_only_login_requirements_before_authentication" in route_tests, "public auth-policy minimization has a regression test") +require("pub struct SessionPolicyResponse" in auth_routes and '.route("/session-policy", get(auth::session_policy))' in routes_mod, "feature policy is exposed only behind the authenticated session route") +require('request("/auth/session-policy"' in auth_js and "async function loadSessionPolicy()" in auth_js, "frontend loads optional feature policy only after authentication") +require("script-src 'self' 'unsafe-inline'" in route_tests, "security regression test explicitly rejects unsafe-inline script policy") +security_test_start = route_tests.find("async fn security_headers_are_opt_in_and_content_type_aware()") +security_test_end = route_tests.find("\n#[tokio::test]", security_test_start + 1) +security_test = route_tests[security_test_start:security_test_end] +require( + 'api.headers()["content-security-policy"]' in security_test + and "frame-ancestors 'none'" in security_test + and 'api.headers()["x-frame-options"]' in security_test + and '"DENY"' in security_test, + "API regression test enforces browser-facing anti-framing headers", +) +require( + 'api.headers().get("permissions-policy").is_none()' in route_tests + and 'api.headers().get("cross-origin-opener-policy").is_none()' in route_tests, + "API regression test omits HTML-only Permissions-Policy and COOP", +) +imap_client_rs = read("src/imap/client.rs") +idle_rs = read("src/realtime/idle.rs") +realtime_sync_rs = read("src/realtime/sync.rs") +require("config.sync.interval_secs" in realtime_sync_rs and "SYNC_INTERVAL_SECS" not in realtime_sync_rs, "periodic IMAP sync interval comes from configuration") +require("config.sync.deep_index_batch_size" in realtime_sync_rs and "DEEP_INDEX_BATCH" not in realtime_sync_rs, "automatic body-cache batch size comes from configuration") +require("config.search.deep_index" in realtime_sync_rs and "force_deep_index" in realtime_sync_rs, "server-wide deep-index setting participates in effective account behavior") +messages_route_rs = read("src/routes/messages/mod.rs") +folders_route_rs = read("src/routes/folders.rs") +folders_js = read("src/web/js/30-folders-tags.js") +reader_js = read("src/web/js/40-messages-reader.js") +realtime_js = read("src/web/js/80-settings-realtime.js") +require('uid_search("UNSEEN")' in imap_client_rs and "folder_status_extended_on_session" in imap_client_rs, "IMAP folder status has an authoritative UNSEEN search fallback") +require('uid_search("UNSEEN")' in idle_rs and "server_unseen" in idle_rs, "IDLE reconciliation counts unseen UIDs instead of using SELECT UNSEEN as a count") +require("tracing::info!(\n tracing::info!(" not in idle_rs, "IDLE handler has no duplicated tracing macro fragment") +require("set_folder_counts" in realtime_sync_rs and "status.unseen != folder.unread_count" in realtime_sync_rs, "periodic sync repairs server folder counters before fast-path skips") +require("MailEvent::FolderStatus" in messages_route_rs and "MessageFlagChange" in messages_route_rs, "message flag mutations publish targeted flag and folder-status events") +folder_counts_route_rs = read("src/routes/folder_counts.rs") +folders_db_rs = read("src/db/folders.rs") +require("pub counter_generation: u64" in folders_db_rs and "expected_generation: u64" in folders_db_rs, "folder counter CAS uses a monotonic database generation") +require("counter_generation_blocks_aba_stale_writer" in folders_db_rs, "folder counter generation has an ABA stale-writer regression test") +require("set_folder_counts_if_unchanged" in folder_counts_route_rs and "for attempt in 0..2" in folder_counts_route_rs, "post-mutation folder counters use CAS with a fresh STATUS retry") +require("reconcile_after_mutation" in messages_route_rs and "counter_snapshot" in messages_route_rs, "message mutations reconcile authoritative counters without stale overwrites") +folder_mgmt_rs = read("src/routes/folder_mgmt.rs") +require("reconcile_after_mutation" in folder_mgmt_rs, "mark-all-read uses the race-safe counter reconciler") +require("get_all_uids_and_flags" in folder_mgmt_rs and "through_uid" in folder_mgmt_rs, "mark-all-read snapshots a UID cutoff so later arrivals stay unread locally") +require("cached_uids" in folder_mgmt_rs and "for uid in &cached_uids" in folder_mgmt_rs, "empty-folder cleanup removes only messages captured before the server mutation") +require("routes_folder_mgmt::Q_002" not in folder_mgmt_rs, "mark-all-read no longer blindly overwrites unread counters with zero") +require("set_folder_counts_if_unchanged" in folders_route_rs, "forced folder refresh protects authoritative counters with compare-and-set") +require("folder_statuses_extended" in folders_route_rs and "pending_sync_count" in folders_route_rs, "folder refresh batches authoritative IMAP status and exposes sync backlog") +require("folder_refresh_uses_server_counts_without_relisting_fresh_folder_tree" in route_tests, "folder refresh regression test covers stale unread count versus authoritative IMAP STATUS") +require("if !cache_fresh" in folders_route_rs and "if query.refresh || !cache_fresh" in folders_route_rs, "folder tree LIST uses TTL while forced refresh updates counters through STATUS") +require("is_noselect" in folders_route_rs and "is_noselect" in realtime_sync_rs, "folder status refresh skips IMAP N\\Noselect hierarchy nodes") +require("folder-count-wide" in folders_js and "folder-count-compact" in folders_js, "folder UI exposes responsive total/unread counters") +index_html = read("src/web/index.html") +require(index_html.find('id="indexing-indicator"') < index_html.find('id="contacts-btn"'), "synchronization/index activity indicator is positioned left of Contacts") +require("folder-separator" in folders_js and ".folder-count .folder-separator" in read("src/web/css/30-mail.css"), "folder total/unread separator is styled independently from the unread value") +require("#message-count" in folders_js and "folderName === state.folder" in folders_js, "authoritative folder totals update the active-folder message count without a list rerender") +require("applyLocalMessageFlags" in reader_js and "updateMessageRowState" in reader_js, "reader flag updates patch individual message rows") +require("applyOptimisticMessageFlag" in reader_js and "rollback = applyOptimisticMessageFlag" in reader_js, "read/unread actions update rows and counters optimistically with guarded rollback") +require("if (payload.eventId)" in realtime_js and "const duplicate = state.realtimeSeenEvents.has(idKey);" in realtime_js and realtime_js.find("if (payload.eventId)") < realtime_js.find("const fingerprint ="), "realtime event dedupe trusts unique eventId before fingerprint fallback") +require("\"eventId\": uuid::Uuid::new_v4().to_string()" in read("src/realtime/ws.rs"), "websocket envelopes carry unique eventId values for lossless client dedupe") +require("MailEvent::FolderAllRead" in read("src/routes/folder_mgmt.rs") and "applyRealtimeFolderAllRead" in realtime_js, "mark-all-read propagates as a targeted realtime event instead of forcing a list reload") +require("update_folder_sync_cursor" in read("src/db/folders.rs") and "Q_031" in read("src/queries/sqlite/queries.rs"), "folder flag-sync cursor is stored independently from authoritative counters") +require('uid_fetch("1:*", "UID FLAGS")' in idle_rs and idle_rs.count("complete.then_some") >= 3, "IDLE fallback reconciles flags and rejects partial IMAP streams") +require(re.search(r"index_message_with_connection\s*\(\s*&transaction", idle_rs) is not None, "IDLE header ingestion enqueues search indexing in the same database transaction") +require("delete_folder_messages(&transaction" in messages_route_rs and "db::header_sync::invalidate(&transaction)" in messages_route_rs and re.search(r"update_folder_sync_status\s*\(\s*&transaction", messages_route_rs) is not None, "UIDVALIDITY reset keeps cache invalidation and folder metadata in one database transaction") +require( + "db::folders::get_folder(conn, &status_name)" in messages_route_rs + and "total_count: folder_state.total_count" in messages_route_rs + and "unread_count: folder_state.unread_count" in messages_route_rs, + "background sync publishes committed folder counters instead of stale network snapshots", +) +require("IMAP cache incomplete" in realtime_js and "Search indexing" in realtime_js, "activity indicator reports IMAP cache and search-index backlog") +require('indicator.classList.toggle("hidden", !busy)' in realtime_js, "activity indicator is hidden whenever synchronization/indexing is idle") +require("setIndexingIndicator(true, 0)" in realtime_js and "setIndexingIndicator(false, 0)" in realtime_js, "manual search reindex drives the activity indicator immediately") +require("MailEvent::IndexingStatus" in read("src/routes/diagnostics.rs") and "Extension(event_bus): Extension>" in read("src/routes/diagnostics.rs"), "manual search reindex broadcasts indexing activity to connected tabs") +require("deep_index_enabled" in read("src/routes/diagnostics.rs") and "body cache disabled" in realtime_js, "search diagnostics explain whether automatic body caching is enabled") require('.route("/settings/reset", post(settings::reset_all_settings))' in routes_mod, "authenticated settings reset route is registered") require("db::settings::reset_all" in read("src/routes/settings.rs"), "settings reset route delegates preference reset to the DB layer") require('$("#quick-filter-btn")?.classList.toggle("hidden", state.authPolicy.sieveEnabled === false)' in read("src/web/js/20-preferences-auth.js"), "Sieve quick-filter UI disappears when Sieve is disabled") @@ -233,11 +377,41 @@ require(pg_s3["storage"]["backend"] == "s3" and pg_s3["search"]["backend"] == "o require(my_nfs["cluster_mode"] is True and "data_dir" not in my_nfs, "MySQL/NFS cluster config omits data_dir") require(my_nfs["storage"]["backend"] == "filesystem" and my_nfs["storage"]["shared"] is True, "MySQL cluster marks POSIX/NFS storage shared") require(my_nfs["search"]["backend"] == "opensearch", "MySQL/NFS cluster uses OpenSearch") +dev_config = tomllib.loads(read("config.dev.toml")) +prod_config = tomllib.loads(read("config.example.toml")) +packaged_config = tomllib.loads(read("packaging/rustbox.toml")) +require(dev_config["search"]["deep_index"] is True, "development config enables automatic full-body cache/index") +require(prod_config["search"]["deep_index"] is False and packaged_config["search"]["deep_index"] is False, "production configs keep automatic full-body cache/index disabled") +require(prod_config.get("pgp_enabled") is True and "pgp_enabled" not in prod_config.get("search", {}), "production example keeps root-level settings outside TOML search table") +require(packaged_config.get("database_backend") == "sqlite" and packaged_config.get("max_attachment_size_mb") == 50 and packaged_config.get("sieve_tls_mode") == "none" and "database_backend" not in packaged_config.get("search", {}), "packaged production config preserves top-level runtime settings outside TOML search table") +for diagnostics_config in [ + "config.dev.toml", + "config.example.toml", + "packaging/rustbox.toml", + "docker/configs/single-sqlite.toml", + "docker/configs/postgres-s3-opensearch.toml", + "docker/configs/ha-garage.toml", + "docker/configs/mysql-nfs-opensearch.toml", +]: + parsed = tomllib.loads(read(diagnostics_config)) + diagnostics = parsed.get("diagnostics_endpoint", {}) + require( + diagnostics.get("enabled") is False + and diagnostics.get("allowed_networks") == ["127.0.0.1/32", "::1/128"] + and diagnostics.get("path") == "/api/system/diagnostics", + f"diagnostics config is disabled and loopback-only by default: {diagnostics_config}", + ) +require(dev_config["sync"]["interval_secs"] == 30 and dev_config["sync"]["deep_index_batch_size"] == 10, "development config documents 30s/10-message sync defaults") +require(prod_config["sync"]["interval_secs"] == 30 and prod_config["sync"]["deep_index_batch_size"] == 10, "production example documents 30s/10-message sync defaults") +require(packaged_config["sync"]["interval_secs"] == 30 and packaged_config["sync"]["deep_index_batch_size"] == 10, "packaged production config documents 30s/10-message sync defaults") ha = tomllib.loads(read("docker/configs/ha-garage.toml")) require(ha["cluster_mode"] is True and ha["session_backend"] == "redis", "HA/Garage config enables cluster mode with Redis-compatible sessions") require(ha["redis_url"] == "redis://valkey:6379/", "HA/Garage config targets Valkey for shared sessions") require(ha["storage"]["backend"] == "s3" and ha["storage"]["endpoint"] == "http://garage:3900", "HA/Garage config stores blobs in Garage S3") require(ha["search"]["backend"] == "opensearch", "HA/Garage config uses OpenSearch") +require(ha["search"]["deep_index"] is True, "development HA example enables automatic deep index") +require(single["search"]["deep_index"] is False and pg_s3["search"]["deep_index"] is False and my_nfs["search"]["deep_index"] is False, "production Docker configs disable automatic full-body cache/index by default") +require(all(cfg["sync"]["interval_secs"] > 0 and cfg["sync"]["deep_index_batch_size"] > 0 for cfg in (single, pg_s3, my_nfs, ha)), "Docker configs provide positive IMAP sync/deep-index tuning values") dockerfile = read("docker/Dockerfile") require("FROM rust:1.99.1-trixie AS builder" in dockerfile, "Docker builder pins Rust 1.99.1 on Debian Trixie") require("FROM debian:trixie-slim" in dockerfile, "Docker runtime uses Debian Trixie slim") @@ -295,6 +469,8 @@ try: require(outbox is not None, "fresh SQLite schema has search_outbox") outbox_columns = {row[1] for row in conn.execute("PRAGMA table_info(u_test__search_outbox)")} require("claimed_until" in outbox_columns, "fresh SQLite search_outbox separates worker lease from retry availability") + folder_columns = {row[1] for row in conn.execute("PRAGMA table_info(u_test__folders)")} + require("counter_generation" in folder_columns, "fresh SQLite schema has V048 folder counter generation") sqlite_header_sync = "INSERT INTO u_test__user_meta (key, value) VALUES ('header_sync_generation', '1') ON CONFLICT(key) DO UPDATE SET value = CAST(value AS INTEGER) + 1" conn.execute(sqlite_header_sync) conn.execute(sqlite_header_sync) @@ -338,10 +514,44 @@ try: except Exception as exc: errors.append(f"SQLite V047 migration execution failed: {exc}") +# V048 upgrade execution from a V047-style folders table, preserving existing counters. +try: + conn = sqlite3.connect(":memory:") + conn.executescript( + "CREATE TABLE u_test__folders (" + "name TEXT PRIMARY KEY, delimiter TEXT, parent TEXT, flags TEXT NOT NULL DEFAULT '', " + "is_subscribed INTEGER NOT NULL DEFAULT 1, total_count INTEGER NOT NULL DEFAULT 0, " + "unread_count INTEGER NOT NULL DEFAULT 0, uid_validity INTEGER NOT NULL DEFAULT 0, " + "highest_modseq INTEGER NOT NULL DEFAULT 0, updated_at TEXT NOT NULL, messages_updated_at TEXT);" + "INSERT INTO u_test__folders " + "(name, total_count, unread_count, uid_validity, highest_modseq, updated_at) VALUES " + "('INBOX', 3444, 12, 77, 9001, '2026-11-01T00:00:00Z');" + ) + migration = read("migrations/sqlite/V048__folder_counter_generation.sql").replace("{{p}}", "u_test__") + conn.executescript(migration) + columns = {row[1] for row in conn.execute("PRAGMA table_info(u_test__folders)")} + require("counter_generation" in columns, "V048 adds counter_generation to SQLite folders") + row = conn.execute( + "SELECT name, total_count, unread_count, uid_validity, highest_modseq, counter_generation " + "FROM u_test__folders WHERE name='INBOX'" + ).fetchone() + require( + row == ('INBOX', 3444, 12, 77, 9001, 0), + "V048 preserves existing folder counters and initializes generation at zero", + ) + conn.close() +except Exception as exc: + errors.append(f"SQLite V048 migration execution failed: {exc}") + # Migration/query parity across SQL engines. for dialect in ("sqlite", "mysql", "pgsql"): require((ROOT / f"migrations/{dialect}/V047__blob_keys_and_search_outbox.sql").is_file(), f"V047 migration exists for {dialect}") + require((ROOT / f"migrations/{dialect}/V048__folder_counter_generation.sql").is_file(), f"V048 migration exists for {dialect}") + schema_text = read(f"migrations/{dialect}/schema.sql") + require("Current runtime schema: V048" in schema_text and "counter_generation" in schema_text, f"fresh {dialect} schema includes V048 folder counter generation") queries = read(f"src/queries/{dialect}/queries.rs") + folder_block = re.search(r"pub mod db_folders\s*\{(.*?)\n\}", queries, re.S) + require(folder_block is not None and "Q_032" in folder_block.group(1) and "counter_generation = counter_generation + 1" in folder_block.group(1), f"folder counter CAS increments generation for {dialect}") block_match = re.search(r"pub mod db_search_outbox\s*\{(.*?)\n\}", queries, re.S) require(block_match is not None, f"search outbox query module exists for {dialect}") if block_match: @@ -351,6 +561,7 @@ for dialect in ("sqlite", "mysql", "pgsql"): pool = read("src/db/pool.rs") for dialect in ("SQLITE", "MYSQL", "PGSQL"): require(f"{dialect}_MIGRATIONS" in pool and "V047__blob_keys_and_search_outbox.sql" in pool, f"runtime migration catalog includes V047 ({dialect.lower()})") + require(f"{dialect}_MIGRATIONS" in pool and "V048__folder_counter_generation.sql" in pool, f"runtime migration catalog includes V048 ({dialect.lower()})") require("const BASE_SCHEMA_VERSION: u32 = 46" in pool, "legacy squashed baseline stays V046") require("latest_schema_version" in pool, "runtime schema version is computed from migrations") require( diff --git a/src/auth/session.rs b/src/auth/session.rs index dc399a4..364bb9c 100644 --- a/src/auth/session.rs +++ b/src/auth/session.rs @@ -1,3 +1,4 @@ +use std::collections::HashSet; use std::sync::Arc; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -5,9 +6,10 @@ use base64::Engine; use base64::engine::general_purpose::URL_SAFE_NO_PAD; use dashmap::DashMap; use rand::RngCore; +use serde::Serialize; use crate::session_store::crypto::SessionCrypto; -use crate::session_store::{PersistedSession, SessionPersistence}; +use crate::session_store::{PersistedSession, SessionPersistence, SessionPersistenceDiagnostics}; pub type SessionId = String; pub type BrowserId = String; @@ -54,6 +56,29 @@ pub struct AccountSession { pub user_agent: String, } +#[derive(Debug, Clone, Serialize)] +pub struct DiagnosticSession { + #[serde(skip_serializing_if = "Option::is_none")] + pub email: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub user_hash: Option, + pub imap_host: String, + pub imap_port: u16, + pub smtp_host: String, + pub smtp_port: u16, + pub created_at_unix: u64, + pub last_accessed_unix: u64, + pub user_agent: String, +} + +#[derive(Debug, Clone, Serialize)] +pub struct SessionStoreDiagnostics { + pub local_cached_sessions: usize, + pub local_cached_browsers: usize, + pub local_unique_users: usize, + pub sessions: Vec, +} + #[derive(Clone)] pub struct SessionStore { sessions: Arc>, @@ -480,6 +505,34 @@ impl SessionStore { self.sessions.is_empty() } + pub fn diagnostics_snapshot(&self, include_user_identifiers: bool) -> SessionStoreDiagnostics { + let mut users = HashSet::new(); + let mut sessions = Vec::new(); + for entry in self.sessions.iter() { + let session = entry.value(); + users.insert(session.user_hash.clone()); + sessions.push(DiagnosticSession { + email: include_user_identifiers.then(|| session.email.clone()), + user_hash: include_user_identifiers.then(|| session.user_hash.clone()), + imap_host: session.imap_host.clone(), + imap_port: session.imap_port, + smtp_host: session.smtp_host.clone(), + smtp_port: session.smtp_port, + created_at_unix: session.created_at_unix, + last_accessed_unix: session.last_accessed_unix, + user_agent: session.user_agent.chars().take(256).collect(), + }); + } + sessions.sort_by_key(|session| std::cmp::Reverse(session.last_accessed_unix)); + sessions.truncate(100); + SessionStoreDiagnostics { + local_cached_sessions: self.sessions.len(), + local_cached_browsers: self.browsers.len(), + local_unique_users: users.len(), + sessions, + } + } + fn cache_persisted(&self, record: &PersistedSession) -> Result { let crypto = self .crypto @@ -747,6 +800,14 @@ impl SessionStore { removed } + pub async fn persistence_diagnostics(&self) -> Result { + let persistence = self + .persistence + .as_ref() + .ok_or_else(|| "session persistence is not configured".to_string())?; + persistence.diagnostics(Self::unix_now()).await + } + pub async fn purge_expired_shared(&self) { self.purge_expired(); if let Some(persistence) = self.persistence.as_ref() diff --git a/src/config.rs b/src/config.rs index cfa804f..d8827c8 100644 --- a/src/config.rs +++ b/src/config.rs @@ -4,7 +4,7 @@ use figment::{ Figment, providers::{Env, Format, Toml}, }; -use ipnet::IpNet; +use ipnet::{IpNet, Ipv4Net, Ipv6Net}; use serde::Deserialize; #[derive(Debug, Deserialize, Clone)] @@ -72,6 +72,64 @@ impl Default for StorageConfig { } } +#[derive(Debug, Deserialize, Clone)] +pub struct SyncConfig { + #[serde(default = "default_sync_interval_secs")] + pub interval_secs: u64, + #[serde(default = "default_deep_index_batch_size")] + pub deep_index_batch_size: u32, +} + +impl Default for SyncConfig { + fn default() -> Self { + Self { + interval_secs: default_sync_interval_secs(), + deep_index_batch_size: default_deep_index_batch_size(), + } + } +} + +#[derive(Debug, Deserialize, Clone)] +pub struct DiagnosticsEndpointConfig { + #[serde(default)] + pub enabled: bool, + #[serde(default = "default_diagnostics_endpoint_path")] + pub path: String, + #[serde(default = "default_diagnostics_allowed_networks")] + pub allowed_networks: Vec, + #[serde(default = "default_diagnostics_auth_mode")] + pub auth: String, + #[serde(default)] + pub token: Option, + #[serde(default)] + pub basic_username: Option, + #[serde(default)] + pub basic_password: Option, + #[serde(default)] + pub include_user_identifiers: bool, + #[serde(default = "default_diagnostics_probe_timeout_secs")] + pub probe_timeout_secs: u64, + #[serde(default = "default_diagnostics_security_window_secs")] + pub security_window_secs: u64, +} + +impl Default for DiagnosticsEndpointConfig { + fn default() -> Self { + Self { + enabled: false, + path: default_diagnostics_endpoint_path(), + allowed_networks: default_diagnostics_allowed_networks(), + auth: default_diagnostics_auth_mode(), + token: None, + basic_username: None, + basic_password: None, + include_user_identifiers: false, + probe_timeout_secs: default_diagnostics_probe_timeout_secs(), + security_window_secs: default_diagnostics_security_window_secs(), + } + } +} + #[derive(Debug, Deserialize, Clone)] pub struct SearchConfig { #[serde(default = "default_search_backend")] @@ -92,6 +150,8 @@ pub struct SearchConfig { pub allow_insecure_http: bool, #[serde(default = "default_search_request_timeout_secs")] pub request_timeout_secs: u64, + #[serde(default)] + pub deep_index: bool, #[serde(default = "default_search_worker_interval_secs")] pub worker_interval_secs: u64, #[serde(default = "default_search_worker_batch_size")] @@ -112,6 +172,7 @@ impl Default for SearchConfig { required: false, allow_insecure_http: false, request_timeout_secs: default_search_request_timeout_secs(), + deep_index: false, worker_interval_secs: default_search_worker_interval_secs(), worker_batch_size: default_search_worker_batch_size(), worker_max_attempts: default_search_worker_max_attempts(), @@ -197,6 +258,12 @@ pub struct AppConfig { #[serde(default)] pub search: SearchConfig, + #[serde(default)] + pub sync: SyncConfig, + + #[serde(default)] + pub diagnostics_endpoint: DiagnosticsEndpointConfig, + #[serde(default = "default_session_timeout_hours")] pub session_timeout_hours: u64, @@ -363,6 +430,34 @@ fn default_search_request_timeout_secs() -> u64 { 10 } +fn default_sync_interval_secs() -> u64 { + 30 +} + +fn default_deep_index_batch_size() -> u32 { + 10 +} + +fn default_diagnostics_endpoint_path() -> String { + "/api/system/diagnostics".to_string() +} + +fn default_diagnostics_allowed_networks() -> Vec { + vec!["127.0.0.1/32".to_string(), "::1/128".to_string()] +} + +fn default_diagnostics_auth_mode() -> String { + "token".to_string() +} + +fn default_diagnostics_probe_timeout_secs() -> u64 { + 5 +} + +fn default_diagnostics_security_window_secs() -> u64 { + 900 +} + fn default_search_worker_interval_secs() -> u64 { 2 } @@ -479,6 +574,8 @@ impl Default for AppConfig { cluster_mode: false, storage: StorageConfig::default(), search: SearchConfig::default(), + sync: SyncConfig::default(), + diagnostics_endpoint: DiagnosticsEndpointConfig::default(), session_timeout_hours: default_session_timeout_hours(), remember_me_enabled: default_remember_me_enabled(), remember_me_days: default_remember_me_days(), @@ -565,6 +662,30 @@ impl AppConfig { .collect() } + pub fn parsed_diagnostics_allowed_networks(&self) -> Result, String> { + self.diagnostics_endpoint + .allowed_networks + .iter() + .map(|entry| { + let trimmed = entry.trim(); + if let Ok(network) = trimmed.parse::() { + return Ok(network); + } + let ip = trimmed + .parse::() + .map_err(|e| format!("invalid diagnostics allowed network '{trimmed}': {e}"))?; + Ok(match ip { + IpAddr::V4(value) => { + IpNet::V4(Ipv4Net::new(value, 32).expect("valid IPv4 prefix")) + } + IpAddr::V6(value) => { + IpNet::V6(Ipv6Net::new(value, 128).expect("valid IPv6 prefix")) + } + }) + }) + .collect() + } + pub fn parsed_outgoing_headers(&self) -> Result, String> { const RESERVED: &[&str] = &[ "from", @@ -743,6 +864,86 @@ impl AppConfig { return Err("Redis sessions require REDIS_URL or REDIS_SENTINEL_NODES + REDIS_SENTINEL_MASTER".to_string()); } } + if self.diagnostics_endpoint.enabled { + let path = self.diagnostics_endpoint.path.trim(); + if !path.starts_with("/api/system/") || path.contains('?') || path.contains('#') { + return Err("diagnostics_endpoint.path must be an absolute /api/system/... path without query/fragment".to_string()); + } + if self.diagnostics_endpoint.allowed_networks.is_empty() { + return Err( + "diagnostics_endpoint.allowed_networks cannot be empty when enabled" + .to_string(), + ); + } + self.parsed_diagnostics_allowed_networks()?; + if self.diagnostics_endpoint.probe_timeout_secs == 0 + || self.diagnostics_endpoint.probe_timeout_secs > 60 + { + return Err( + "diagnostics_endpoint.probe_timeout_secs must be between 1 and 60".to_string(), + ); + } + if self.diagnostics_endpoint.security_window_secs < 60 + || self.diagnostics_endpoint.security_window_secs > 86_400 + { + return Err( + "diagnostics_endpoint.security_window_secs must be between 60 and 86400" + .to_string(), + ); + } + match self + .diagnostics_endpoint + .auth + .trim() + .to_ascii_lowercase() + .as_str() + { + "token" => { + let token = self + .diagnostics_endpoint + .token + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .ok_or("diagnostics_endpoint.token is required when auth=token")?; + if token.len() < 24 { + return Err( + "diagnostics_endpoint.token must contain at least 24 characters" + .to_string(), + ); + } + } + "basic" => { + let username = self + .diagnostics_endpoint + .basic_username + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .ok_or("diagnostics_endpoint.basic_username is required when auth=basic")?; + let password = self + .diagnostics_endpoint + .basic_password + .as_deref() + .filter(|value| !value.is_empty()) + .ok_or("diagnostics_endpoint.basic_password is required when auth=basic")?; + if username.contains(':') { + return Err( + "diagnostics_endpoint.basic_username cannot contain ':'".to_string() + ); + } + if password.len() < 16 { + return Err("diagnostics_endpoint.basic_password must contain at least 16 characters".to_string()); + } + } + _ => { + return Err( + "diagnostics_endpoint.auth must be one of: token, basic".to_string() + ); + } + } + } + let storage_backend = self.effective_storage_backend(); if !matches!(storage_backend.as_str(), "filesystem" | "s3") { return Err("storage.backend must be one of: filesystem, s3".to_string()); @@ -871,6 +1072,12 @@ impl AppConfig { { return Err("search timeout/worker settings must be greater than zero".to_string()); } + if self.sync.interval_secs == 0 || self.sync.deep_index_batch_size == 0 { + return Err( + "sync.interval_secs and sync.deep_index_batch_size must be greater than zero" + .to_string(), + ); + } if self.cluster_mode && self.database_backend == "sqlite" { return Err( "CLUSTER_MODE requires a shared mysql/pgsql application database; Redis only replaces session persistence" diff --git a/src/db/folders.rs b/src/db/folders.rs index 0de1854..fd0cd48 100644 --- a/src/db/folders.rs +++ b/src/db/folders.rs @@ -12,6 +12,7 @@ pub struct CachedFolder { pub unread_count: u32, pub uid_validity: u32, pub highest_modseq: u64, + pub counter_generation: u64, } #[allow(clippy::too_many_arguments)] @@ -68,6 +69,7 @@ pub fn get_all_folders(conn: &Connection) -> Result, String> { .query_map([], |row| { let is_subscribed_int: i32 = row.get(4)?; let highest_modseq_int: i64 = row.get(8)?; + let counter_generation_int: i64 = row.get(9)?; Ok(CachedFolder { name: row.get(0)?, delimiter: row.get(1)?, @@ -78,6 +80,7 @@ pub fn get_all_folders(conn: &Connection) -> Result, String> { unread_count: row.get(6)?, uid_validity: row.get(7)?, highest_modseq: highest_modseq_int as u64, + counter_generation: counter_generation_int as u64, }) }) .map_err(|e| format!("Failed to query folders: {e}"))?; @@ -118,6 +121,7 @@ pub fn touch_all_folders(conn: &Connection) -> Result<(), String> { Ok(()) } +#[cfg(test)] pub fn update_folder_status( conn: &Connection, name: &str, @@ -147,21 +151,6 @@ pub fn update_folder_sync_status( Ok(()) } -pub fn refresh_unread_count(conn: &Connection, folder_name: &str) -> Result<(), String> { - let unread: u32 = conn - .query_row( - crate::dbq!(db_folders::Q_008), - params![folder_name], - |row| row.get(0), - ) - .map_err(|e| format!("Failed to count unread messages: {e}"))?; - - conn.execute(crate::dbq!(db_folders::Q_009), params![unread, folder_name]) - .map_err(|e| format!("Failed to update unread_count: {e}"))?; - - Ok(()) -} - pub fn set_unread_count( conn: &Connection, folder_name: &str, @@ -175,15 +164,74 @@ pub fn set_unread_count( Ok(()) } -pub fn mark_folder_messages_fresh(conn: &Connection, folder_name: &str) -> Result<(), String> { - conn.execute(crate::dbq!(db_folders::Q_029), params![folder_name]) - .map_err(|e| format!("Failed to mark folder messages fresh: {e}"))?; +pub fn set_folder_counts( + conn: &Connection, + folder_name: &str, + total_count: u32, + unread_count: u32, +) -> Result<(), String> { + conn.execute( + crate::dbq!(db_folders::Q_030), + params![total_count, unread_count, folder_name], + ) + .map_err(|e| format!("Failed to set folder counts: {e}"))?; Ok(()) } -pub fn adjust_unread_count(conn: &Connection, folder_name: &str, delta: i32) -> Result<(), String> { - conn.execute(crate::dbq!(db_folders::Q_010), params![delta, folder_name]) - .map_err(|e| format!("Failed to adjust unread_count: {e}"))?; +pub fn update_folder_sync_cursor( + conn: &Connection, + folder_name: &str, + uid_validity: u32, + highest_modseq: u64, +) -> Result<(), String> { + conn.execute( + crate::dbq!(db_folders::Q_031), + params![uid_validity, highest_modseq as i64, folder_name], + ) + .map_err(|e| format!("Failed to update folder sync cursor: {e}"))?; + Ok(()) +} + +pub fn set_folder_counts_if_unchanged( + conn: &Connection, + folder_name: &str, + expected_generation: u64, + total_count: u32, + unread_count: u32, +) -> Result { + let changed = conn + .execute( + crate::dbq!(db_folders::Q_032), + params![ + total_count, + unread_count, + folder_name, + expected_generation as i64 + ], + ) + .map_err(|e| format!("Failed to conditionally set folder counts: {e}"))?; + Ok(changed > 0) +} + +pub fn pending_message_sync_count(conn: &Connection) -> Result { + let mut pending = 0u64; + for folder in get_all_folders(conn)? { + if folder + .flags + .split(',') + .any(|flag| flag.trim().eq_ignore_ascii_case("\\Noselect")) + { + continue; + } + let cached = crate::db::messages::count_messages(conn, &folder.name)?; + pending = pending.saturating_add(folder.total_count.saturating_sub(cached) as u64); + } + Ok(pending.min(u32::MAX as u64) as u32) +} + +pub fn mark_folder_messages_fresh(conn: &Connection, folder_name: &str) -> Result<(), String> { + conn.execute(crate::dbq!(db_folders::Q_029), params![folder_name]) + .map_err(|e| format!("Failed to mark folder messages fresh: {e}"))?; Ok(()) } @@ -237,6 +285,7 @@ pub fn get_folder(conn: &Connection, name: &str) -> Result, let result = conn.query_row(crate::dbq!(db_folders::Q_015), params![name], |row| { let is_subscribed_int: i32 = row.get(4)?; let highest_modseq_int: i64 = row.get(8)?; + let counter_generation_int: i64 = row.get(9)?; Ok(CachedFolder { name: row.get(0)?, delimiter: row.get(1)?, @@ -247,6 +296,7 @@ pub fn get_folder(conn: &Connection, name: &str) -> Result, unread_count: row.get(6)?, uid_validity: row.get(7)?, highest_modseq: highest_modseq_int as u64, + counter_generation: counter_generation_int as u64, }) }); @@ -433,6 +483,59 @@ mod tests { assert!(!is_folder_fresh(&conn, "Sent", 30).unwrap()); } + #[test] + fn conditional_folder_counts_reject_stale_writer() { + let conn = open_test_db(); + upsert_folder(&conn, "INBOX", None, None, "", true, 3444, 12, 1, 50).unwrap(); + + let generation = get_folder(&conn, "INBOX") + .unwrap() + .unwrap() + .counter_generation; + assert!(set_folder_counts_if_unchanged(&conn, "INBOX", generation, 3444, 11).unwrap()); + assert!(!set_folder_counts_if_unchanged(&conn, "INBOX", generation, 3444, 10).unwrap()); + + let folder = get_folder(&conn, "INBOX").unwrap().unwrap(); + assert_eq!(folder.total_count, 3444); + assert_eq!(folder.unread_count, 11); + } + + #[test] + fn counter_generation_blocks_aba_stale_writer() { + let conn = open_test_db(); + upsert_folder(&conn, "INBOX", None, None, "", true, 3444, 12, 1, 50).unwrap(); + + let generation_12 = get_folder(&conn, "INBOX") + .unwrap() + .unwrap() + .counter_generation; + assert!(set_folder_counts_if_unchanged(&conn, "INBOX", generation_12, 3444, 11).unwrap()); + + let generation_11 = get_folder(&conn, "INBOX") + .unwrap() + .unwrap() + .counter_generation; + assert!(set_folder_counts_if_unchanged(&conn, "INBOX", generation_11, 3444, 12).unwrap()); + assert!(!set_folder_counts_if_unchanged(&conn, "INBOX", generation_12, 3444, 10).unwrap()); + let folder = get_folder(&conn, "INBOX").unwrap().unwrap(); + assert_eq!((folder.total_count, folder.unread_count), (3444, 12)); + assert!(folder.counter_generation > generation_12); + } + + #[test] + fn sync_cursor_update_does_not_change_authoritative_counts() { + let conn = open_test_db(); + upsert_folder(&conn, "INBOX", None, None, "", true, 3444, 12, 1, 50).unwrap(); + + update_folder_sync_cursor(&conn, "INBOX", 2, 75).unwrap(); + + let folder = get_folder(&conn, "INBOX").unwrap().unwrap(); + assert_eq!(folder.total_count, 3444); + assert_eq!(folder.unread_count, 12); + assert_eq!(folder.uid_validity, 2); + assert_eq!(folder.highest_modseq, 75); + } + #[test] fn test_remove_stale_folders() { let conn = open_test_db(); diff --git a/src/db/messages.rs b/src/db/messages.rs index ba4f76a..77268b6 100644 --- a/src/db/messages.rs +++ b/src/db/messages.rs @@ -476,6 +476,11 @@ pub fn get_unindexed_messages(conn: &Connection, limit: u32) -> Result Result { + conn.query_row(crate::dbq!(db_messages::Q_015), [], |row| row.get(0)) + .map_err(|e| format!("Failed to count unindexed messages: {e}")) +} + pub fn get_all_uids_and_flags( conn: &Connection, folder: &str, diff --git a/src/db/pool.rs b/src/db/pool.rs index 6cff381..a6f0242 100644 --- a/src/db/pool.rs +++ b/src/db/pool.rs @@ -7,6 +7,7 @@ use std::sync::{Arc, Mutex, OnceLock, mpsc}; use rusqlite::OptionalExtension as _; use rusqlite::types::{ToSql, ToSqlOutput, Value}; +use serde::Serialize; use sha2::{Digest, Sha256}; use sqlx::pool::PoolConnection; use sqlx::{Executor as _, MySql, MySqlPool, PgPool, Postgres, Row as SqlxRow}; @@ -20,18 +21,36 @@ const SQLITE_SCHEMA: &str = include_str!("../../migrations/sqlite/schema.sql"); const MYSQL_SCHEMA: &str = include_str!("../../migrations/mysql/schema.sql"); const PGSQL_SCHEMA: &str = include_str!("../../migrations/pgsql/schema.sql"); -const SQLITE_MIGRATIONS: &[(u32, &str)] = &[( - 47, - include_str!("../../migrations/sqlite/V047__blob_keys_and_search_outbox.sql"), -)]; -const MYSQL_MIGRATIONS: &[(u32, &str)] = &[( - 47, - include_str!("../../migrations/mysql/V047__blob_keys_and_search_outbox.sql"), -)]; -const PGSQL_MIGRATIONS: &[(u32, &str)] = &[( - 47, - include_str!("../../migrations/pgsql/V047__blob_keys_and_search_outbox.sql"), -)]; +const SQLITE_MIGRATIONS: &[(u32, &str)] = &[ + ( + 47, + include_str!("../../migrations/sqlite/V047__blob_keys_and_search_outbox.sql"), + ), + ( + 48, + include_str!("../../migrations/sqlite/V048__folder_counter_generation.sql"), + ), +]; +const MYSQL_MIGRATIONS: &[(u32, &str)] = &[ + ( + 47, + include_str!("../../migrations/mysql/V047__blob_keys_and_search_outbox.sql"), + ), + ( + 48, + include_str!("../../migrations/mysql/V048__folder_counter_generation.sql"), + ), +]; +const PGSQL_MIGRATIONS: &[(u32, &str)] = &[ + ( + 47, + include_str!("../../migrations/pgsql/V047__blob_keys_and_search_outbox.sql"), + ), + ( + 48, + include_str!("../../migrations/pgsql/V048__folder_counter_generation.sql"), + ), +]; const TENANT_TABLES: &[&str] = &[ "calendar_events", @@ -93,6 +112,26 @@ const IMPORT_TABLES: &[&str] = &[ "user_meta", ]; +#[derive(Debug, Clone, Serialize)] +pub struct DatabaseDiagnostics { + pub backend: String, + pub status: String, + pub latency_ms: u128, + pub pool_size: Option, + pub idle_connections: Option, + pub database_bytes: Option, + pub engine_version: Option, + pub account_count: Option, + pub schema_version_min: Option, + pub schema_version_max: Option, + pub schema_aligned: Option, + pub expected_schema_version: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +type SqliteDiagnosticsProbe = (u64, Option, Option, Option, String); + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum DatabaseBackend { Sqlite, @@ -312,6 +351,188 @@ pub fn configured_backend() -> DatabaseBackend { } } +pub async fn diagnostics() -> DatabaseDiagnostics { + let started = std::time::Instant::now(); + let expected = latest_schema_version(); + match DATABASE.get() { + Some(ConfiguredDatabase::Sqlite { path }) => { + let path = path.clone(); + let result = tokio::task::spawn_blocking(move || -> Result { + let conn = rusqlite::Connection::open(&path) + .map_err(|e| format!("failed to open SQLite diagnostics connection: {e}"))?; + conn.query_row("SELECT 1", [], |_row| Ok(())) + .map_err(|e| format!("SQLite probe failed: {e}"))?; + let engine_version: String = conn + .query_row("SELECT sqlite_version()", [], |row| row.get(0)) + .map_err(|e| format!("SQLite version probe failed: {e}"))?; + let (count, min_version, max_version): (i64, Option, Option) = conn + .query_row( + "SELECT COUNT(*), MIN(schema_version), MAX(schema_version) FROM rustbox_accounts", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .map_err(|e| format!("SQLite account diagnostics failed: {e}"))?; + let bytes = std::fs::metadata(&path).ok().map(|meta| meta.len()); + Ok(( + count.max(0) as u64, + min_version.and_then(|value| u32::try_from(value).ok()), + max_version.and_then(|value| u32::try_from(value).ok()), + bytes, + engine_version, + )) + }).await; + match result { + Ok(Ok((count, min_version, max_version, bytes, engine_version))) => { + DatabaseDiagnostics { + backend: "sqlite".to_string(), + status: "ok".to_string(), + latency_ms: started.elapsed().as_millis(), + pool_size: None, + idle_connections: None, + database_bytes: bytes, + engine_version: Some(engine_version), + account_count: Some(count), + schema_version_min: min_version, + schema_version_max: max_version, + schema_aligned: Some( + count == 0 + || (min_version == Some(expected) && max_version == Some(expected)), + ), + expected_schema_version: expected, + error: None, + } + } + other => DatabaseDiagnostics { + backend: "sqlite".to_string(), + status: "error".to_string(), + latency_ms: started.elapsed().as_millis(), + pool_size: None, + idle_connections: None, + database_bytes: None, + engine_version: None, + account_count: None, + schema_version_min: None, + schema_version_max: None, + schema_aligned: None, + expected_schema_version: expected, + error: Some(match other { + Ok(Err(error)) => error, + Err(error) => format!("SQLite diagnostics task failed: {error}"), + Ok(Ok(_)) => unreachable!(), + }), + }, + } + } + Some(ConfiguredDatabase::Mysql { pool, .. }) => { + let result: Result<(i64, Option, Option, String), String> = async { + sqlx::query_scalar::<_, i32>("SELECT 1") + .fetch_one(pool) + .await + .map_err(|e| format!("MySQL probe failed: {e}"))?; + let engine_version: String = sqlx::query_scalar("SELECT VERSION()") + .fetch_one(pool) + .await + .map_err(|e| format!("MySQL version probe failed: {e}"))?; + let (count, min, max): (i64, Option, Option) = sqlx::query_as("SELECT COUNT(*), MIN(schema_version), MAX(schema_version) FROM rustbox_accounts") + .fetch_one(pool) + .await + .map_err(|e| format!("MySQL account diagnostics failed: {e}"))?; + Ok((count, min, max, engine_version)) + }.await; + let (status, account_count, min_version, max_version, engine_version, error) = + match result { + Ok((count, min, max, version)) => ( + "ok", + Some(count.max(0) as u64), + min.and_then(|v| u32::try_from(v).ok()), + max.and_then(|v| u32::try_from(v).ok()), + Some(version), + None, + ), + Err(error) => ("error", None, None, None, None, Some(error)), + }; + DatabaseDiagnostics { + backend: "mysql".to_string(), + status: status.to_string(), + latency_ms: started.elapsed().as_millis(), + pool_size: Some(pool.size()), + idle_connections: Some(pool.num_idle()), + database_bytes: None, + engine_version, + account_count, + schema_version_min: min_version, + schema_version_max: max_version, + schema_aligned: account_count.map(|count| { + count == 0 || (min_version == Some(expected) && max_version == Some(expected)) + }), + expected_schema_version: expected, + error, + } + } + Some(ConfiguredDatabase::Pgsql { pool, .. }) => { + let result: Result<(i64, Option, Option, String), String> = async { + sqlx::query_scalar::<_, i32>("SELECT 1") + .fetch_one(pool) + .await + .map_err(|e| format!("PostgreSQL probe failed: {e}"))?; + let engine_version: String = sqlx::query_scalar("SHOW server_version") + .fetch_one(pool) + .await + .map_err(|e| format!("PostgreSQL version probe failed: {e}"))?; + let (count, min, max): (i64, Option, Option) = sqlx::query_as("SELECT COUNT(*), MIN(schema_version), MAX(schema_version) FROM rustbox_accounts") + .fetch_one(pool) + .await + .map_err(|e| format!("PostgreSQL account diagnostics failed: {e}"))?; + Ok((count, min, max, engine_version)) + }.await; + let (status, account_count, min_version, max_version, engine_version, error) = + match result { + Ok((count, min, max, version)) => ( + "ok", + Some(count.max(0) as u64), + min.and_then(|v| u32::try_from(v).ok()), + max.and_then(|v| u32::try_from(v).ok()), + Some(version), + None, + ), + Err(error) => ("error", None, None, None, None, Some(error)), + }; + DatabaseDiagnostics { + backend: "pgsql".to_string(), + status: status.to_string(), + latency_ms: started.elapsed().as_millis(), + pool_size: Some(pool.size()), + idle_connections: Some(pool.num_idle()), + database_bytes: None, + engine_version, + account_count, + schema_version_min: min_version, + schema_version_max: max_version, + schema_aligned: account_count.map(|count| { + count == 0 || (min_version == Some(expected) && max_version == Some(expected)) + }), + expected_schema_version: expected, + error, + } + } + None => DatabaseDiagnostics { + backend: "unconfigured".to_string(), + status: "error".to_string(), + latency_ms: started.elapsed().as_millis(), + pool_size: None, + idle_connections: None, + database_bytes: None, + engine_version: None, + account_count: None, + schema_version_min: None, + schema_version_max: None, + schema_aligned: None, + expected_schema_version: expected, + error: Some("database is not configured".to_string()), + }, + } +} + pub fn list_account_user_hashes() -> Result, String> { match DATABASE.get() { Some(ConfiguredDatabase::Sqlite { path }) => { @@ -1838,7 +2059,7 @@ fn tenant_prefix(user_hash: &str) -> Result { Ok(format!("u_{}__", &normalized[..24])) } -fn latest_schema_version() -> u32 { +pub fn latest_schema_version() -> u32 { let migrations = match configured_backend() { DatabaseBackend::Sqlite => SQLITE_MIGRATIONS, DatabaseBackend::Mysql => MYSQL_MIGRATIONS, @@ -2108,8 +2329,6 @@ fn ensure_server_tenant( let tenant = tenant_schema(PGSQL_SCHEMA, &prefix)?; let legacy_path = legacy_path.clone(); runtime_wait(runtime, async move { - // PostgreSQL advisory locks are session-scoped, so they work across all RustBox nodes - // that share this database and disappear automatically with the connection. let lock_name = format!("rustbox:{prefix}"); let mut lock_conn = pool.acquire().await.map_err(|e| { format!("failed to acquire PostgreSQL provisioning connection: {e}") diff --git a/src/imap/client.rs b/src/imap/client.rs index 419b917..a82ac7a 100644 --- a/src/imap/client.rs +++ b/src/imap/client.rs @@ -19,6 +19,60 @@ pub(crate) use super::connection::connect; #[path = "mock.rs"] pub mod mock; +async fn folder_status_extended_on_session( + session: &mut async_imap::Session, + folder: &str, +) -> Result { + if let Ok(mailbox) = session + .status( + folder, + "(MESSAGES UIDNEXT UIDVALIDITY UNSEEN HIGHESTMODSEQ)", + ) + .await + && let Some(unseen) = mailbox.unseen + { + return Ok(FolderStatusExtended { + uid_validity: mailbox.uid_validity.unwrap_or(0), + exists: mailbox.exists, + uid_next: mailbox.uid_next.unwrap_or(0), + unseen, + highest_modseq: mailbox.highest_modseq.unwrap_or(0), + }); + } + + if let Ok(mailbox) = session + .status(folder, "(MESSAGES UIDNEXT UIDVALIDITY UNSEEN)") + .await + && let Some(unseen) = mailbox.unseen + { + return Ok(FolderStatusExtended { + uid_validity: mailbox.uid_validity.unwrap_or(0), + exists: mailbox.exists, + uid_next: mailbox.uid_next.unwrap_or(0), + unseen, + highest_modseq: mailbox.highest_modseq.unwrap_or(0), + }); + } + + let mailbox = match session.select_condstore(folder).await { + Ok(mailbox) => mailbox, + Err(_) => session.select(folder).await.map_err(map_imap_error)?, + }; + let unseen = session + .uid_search("UNSEEN") + .await + .map_err(map_imap_error)? + .len() as u32; + + Ok(FolderStatusExtended { + uid_validity: mailbox.uid_validity.unwrap_or(0), + exists: mailbox.exists, + uid_next: mailbox.uid_next.unwrap_or(0), + unseen, + highest_modseq: mailbox.highest_modseq.unwrap_or(0), + }) +} + #[allow(dead_code)] #[allow(clippy::double_must_use)] #[async_trait] @@ -170,6 +224,21 @@ pub trait ImapClient: Send + Sync { folder: &str, ) -> Result; + async fn folder_statuses_extended( + &self, + creds: &ImapCredentials, + folders: &[String], + ) -> Result, ImapError> { + let mut statuses = Vec::with_capacity(folders.len()); + for folder in folders { + statuses.push(( + folder.clone(), + self.folder_status_extended(creds, folder).await?, + )); + } + Ok(statuses) + } + async fn fetch_changed_flags( &self, creds: &ImapCredentials, @@ -1073,25 +1142,32 @@ impl ImapClient for RealImapClient { folder: &str, ) -> Result { let mut session = self.pool.checkout(creds).await?; + let result = folder_status_extended_on_session(&mut session, folder).await; + if result.is_ok() { + session.recycle(); + } + result + } - let mailbox = session - .status( - folder, - "(MESSAGES UIDNEXT UIDVALIDITY UNSEEN HIGHESTMODSEQ)", - ) - .await - .map_err(map_imap_error)?; + async fn folder_statuses_extended( + &self, + creds: &ImapCredentials, + folders: &[String], + ) -> Result, ImapError> { + let mut session = self.pool.checkout(creds).await?; + let mut statuses = Vec::with_capacity(folders.len()); - let result = FolderStatusExtended { - uid_validity: mailbox.uid_validity.unwrap_or(0), - exists: mailbox.exists, - uid_next: mailbox.uid_next.unwrap_or(0), - unseen: mailbox.unseen.unwrap_or(0), - highest_modseq: mailbox.highest_modseq.unwrap_or(0), - }; + for folder in folders { + match folder_status_extended_on_session(&mut session, folder).await { + Ok(status) => statuses.push((folder.clone(), status)), + Err(error) => { + tracing::debug!(folder = %folder, error = %error, "Skipping folder STATUS refresh"); + } + } + } session.recycle(); - Ok(result) + Ok(statuses) } async fn fetch_changed_flags( diff --git a/src/main.rs b/src/main.rs index d8d73a9..bc0c5f1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -186,9 +186,22 @@ async fn run_search_reindex( Ok(()) } +fn default_log_filter(log_level: &str) -> String { + let app_level = log_level.trim().to_ascii_lowercase(); + match app_level.as_str() { + "trace" => "warn,rustbox=trace,tower_http=debug".to_string(), + "debug" => "warn,rustbox=debug,tower_http=info".to_string(), + "info" => "warn,rustbox=info,tower_http=info".to_string(), + "warn" => "warn,rustbox=warn,tower_http=warn".to_string(), + "error" => "error,rustbox=error,tower_http=error".to_string(), + "off" => "off".to_string(), + _ => "warn,rustbox=info,tower_http=info".to_string(), + } +} + fn init_tracing(log_dir: Option<&str>, log_level: &str) -> Result<(), Box> { let env_filter = EnvFilter::try_from_default_env() - .unwrap_or_else(|_| EnvFilter::new(log_level.trim().to_ascii_lowercase())); + .unwrap_or_else(|_| EnvFilter::new(default_log_filter(log_level))); if let Some(log_dir) = log_dir.map(str::trim).filter(|value| !value.is_empty()) { std::fs::create_dir_all(log_dir)?; let log_path = std::path::Path::new(log_dir).join("rustbox.log"); diff --git a/src/queries/mysql/queries.rs b/src/queries/mysql/queries.rs index ab00fc6..2373a6e 100644 --- a/src/queries/mysql/queries.rs +++ b/src/queries/mysql/queries.rs @@ -187,24 +187,25 @@ pub mod db_folders { ON DUPLICATE KEY UPDATE delimiter=VALUES(delimiter), parent=VALUES(parent), flags=VALUES(flags), is_subscribed=VALUES(is_subscribed), total_count=VALUES(total_count), unread_count=VALUES(unread_count), - uid_validity=VALUES(uid_validity), highest_modseq=VALUES(highest_modseq), updated_at=CURRENT_TIMESTAMP"; + uid_validity=VALUES(uid_validity), highest_modseq=VALUES(highest_modseq), + counter_generation=counter_generation + 1, updated_at=CURRENT_TIMESTAMP"; pub const Q_002: &str = "INSERT IGNORE INTO folders (name, delimiter, flags, is_subscribed, updated_at) VALUES (?, ?, ?, 1, CURRENT_TIMESTAMP)"; pub const Q_003: &str = "SELECT name, delimiter, parent, flags, is_subscribed, - total_count, unread_count, uid_validity, highest_modseq + total_count, unread_count, uid_validity, highest_modseq, counter_generation FROM folders ORDER BY name"; pub const Q_004: &str = "DELETE FROM folders"; pub const Q_005: &str = "UPDATE folders SET updated_at = CURRENT_TIMESTAMP"; - pub const Q_006: &str = "UPDATE folders SET uid_validity = ?, total_count = ?, messages_updated_at = CURRENT_TIMESTAMP + pub const Q_006: &str = "UPDATE folders SET uid_validity = ?, total_count = ?, counter_generation = counter_generation + 1, messages_updated_at = CURRENT_TIMESTAMP WHERE name = ?"; pub const Q_007: &str = - "UPDATE folders SET uid_validity = ?, total_count = ?, highest_modseq = ? + "UPDATE folders SET uid_validity = ?, total_count = ?, highest_modseq = ?, counter_generation = counter_generation + 1 WHERE name = ?"; pub const Q_008: &str = "SELECT CAST(COUNT(*) AS SIGNED) FROM messages WHERE folder = ? AND LOCATE(CONCAT(CHAR(92), 'Seen'), flags) = 0"; - pub const Q_009: &str = "UPDATE folders SET unread_count = ? WHERE name = ?"; - pub const Q_010: &str = "UPDATE folders SET unread_count = GREATEST(0, CAST(unread_count AS SIGNED) + ?) WHERE name = ?"; + pub const Q_009: &str = "UPDATE folders SET unread_count = ?, counter_generation = counter_generation + 1 WHERE name = ?"; + pub const Q_010: &str = "UPDATE folders SET unread_count = GREATEST(0, CAST(unread_count AS SIGNED) + ?), counter_generation = counter_generation + 1 WHERE name = ?"; pub const Q_011: &str = "SELECT EXISTS( SELECT 1 FROM folders WHERE updated_at > DATE_FORMAT(DATE_SUB(CURRENT_TIMESTAMP, INTERVAL ? SECOND), '%Y-%m-%d %H:%i:%s') @@ -216,14 +217,14 @@ pub mod db_folders { pub const Q_013: &str = "SELECT messages_updated_at IS NULL FROM folders WHERE name = ?"; pub const Q_014: &str = "UPDATE folders SET messages_updated_at = NULL WHERE name = ?"; pub const Q_015: &str = "SELECT name, delimiter, parent, flags, is_subscribed, - total_count, unread_count, uid_validity, highest_modseq + total_count, unread_count, uid_validity, highest_modseq, counter_generation FROM folders WHERE name = ?"; pub const Q_016: &str = "DELETE FROM messages WHERE folder = ?"; pub const Q_017: &str = "DELETE FROM folders WHERE name = ?"; pub const Q_018: &str = "SELECT 1"; - pub const Q_019: &str = "INSERT INTO folders (name, delimiter, parent, flags, is_subscribed, total_count, unread_count, uid_validity, highest_modseq, updated_at, messages_updated_at) - SELECT ?, delimiter, parent, flags, is_subscribed, total_count, unread_count, uid_validity, highest_modseq, CURRENT_TIMESTAMP, messages_updated_at + pub const Q_019: &str = "INSERT INTO folders (name, delimiter, parent, flags, is_subscribed, total_count, unread_count, uid_validity, highest_modseq, counter_generation, updated_at, messages_updated_at) + SELECT ?, delimiter, parent, flags, is_subscribed, total_count, unread_count, uid_validity, highest_modseq, counter_generation, CURRENT_TIMESTAMP, messages_updated_at FROM folders WHERE name = ?"; pub const Q_020: &str = "UPDATE messages SET folder = ? WHERE folder = ?"; pub const Q_021: &str = "UPDATE message_tags SET message_folder = ? WHERE message_folder = ?"; @@ -235,9 +236,13 @@ pub mod db_folders { pub const Q_026: &str = "UPDATE folders SET messages_updated_at = '2000-01-01 00:00:00' WHERE name = 'Sent'"; pub const Q_027: &str = "SELECT messages_updated_at FROM folders WHERE name = 'Sent'"; - pub const Q_028: &str = "UPDATE folders SET unread_count = ? WHERE name = ?"; + pub const Q_028: &str = "UPDATE folders SET unread_count = ?, counter_generation = counter_generation + 1 WHERE name = ?"; pub const Q_029: &str = "UPDATE folders SET messages_updated_at = CURRENT_TIMESTAMP WHERE name = ?"; + pub const Q_030: &str = "UPDATE folders SET total_count = ?, unread_count = ?, counter_generation = counter_generation + 1 WHERE name = ?"; + pub const Q_031: &str = + "UPDATE folders SET uid_validity = ?, highest_modseq = ? WHERE name = ?"; + pub const Q_032: &str = "UPDATE folders SET total_count = ?, unread_count = ?, counter_generation = counter_generation + 1 WHERE name = ? AND counter_generation = ?"; pub const FORMAT_001: &str = "DELETE FROM folders WHERE name NOT IN ({})"; } @@ -310,6 +315,12 @@ pub mod db_messages { LIMIT ?"; pub const Q_013: &str = "SELECT uid, flags FROM messages WHERE folder = ?"; pub const Q_014: &str = "SELECT CAST(SUM(size) AS SIGNED) FROM messages"; + pub const Q_015: &str = "SELECT COUNT(*) FROM messages + WHERE body_cached = 0 + AND lower(folder) NOT IN ('trash', 'junk', 'spam') + AND lower(folder) NOT LIKE '%/trash' + AND lower(folder) NOT LIKE '%/junk' + AND lower(folder) NOT LIKE '%/spam'"; pub const FORMAT_001: &str = "SELECT {MSG_SELECT_COLS} FROM messages WHERE folder = ? @@ -463,9 +474,9 @@ pub mod routes_diagnostics { pub mod routes_folder_mgmt { pub const Q_001: &str = "UPDATE messages SET flags = CASE WHEN flags = '' THEN ?2 ELSE CONCAT(flags, ',', ?2) END WHERE folder = ?1 AND LOCATE(?2, flags) = 0"; - pub const Q_002: &str = "UPDATE folders SET unread_count = 0 WHERE name = ?"; + pub const Q_002: &str = "UPDATE folders SET unread_count = 0, counter_generation = counter_generation + 1 WHERE name = ?"; pub const Q_003: &str = "SELECT uid FROM messages WHERE folder = ?"; - pub const Q_004: &str = "UPDATE folders SET total_count = 0, unread_count = 0, messages_updated_at = CURRENT_TIMESTAMP WHERE name = ?"; + pub const Q_004: &str = "UPDATE folders SET total_count = 0, unread_count = 0, counter_generation = counter_generation + 1, messages_updated_at = CURRENT_TIMESTAMP WHERE name = ?"; } pub mod routes_messages_mod { diff --git a/src/queries/pgsql/queries.rs b/src/queries/pgsql/queries.rs index 3ee45dd..dab21fa 100644 --- a/src/queries/pgsql/queries.rs +++ b/src/queries/pgsql/queries.rs @@ -187,25 +187,26 @@ pub mod db_folders { VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, CURRENT_TIMESTAMP::text) ON CONFLICT (name) DO UPDATE SET delimiter=excluded.delimiter, parent=excluded.parent, flags=excluded.flags, is_subscribed=excluded.is_subscribed, total_count=excluded.total_count, unread_count=excluded.unread_count, - uid_validity=excluded.uid_validity, highest_modseq=excluded.highest_modseq, updated_at=CURRENT_TIMESTAMP::text"; + uid_validity=excluded.uid_validity, highest_modseq=excluded.highest_modseq, + counter_generation=folders.counter_generation + 1, updated_at=CURRENT_TIMESTAMP::text"; pub const Q_002: &str = "INSERT INTO folders (name, delimiter, flags, is_subscribed, updated_at) VALUES ($1, $2, $3, 1, CURRENT_TIMESTAMP::text) ON CONFLICT (name) DO NOTHING"; pub const Q_003: &str = "SELECT name, delimiter, parent, flags, is_subscribed, - total_count, unread_count, uid_validity, highest_modseq + total_count, unread_count, uid_validity, highest_modseq, counter_generation FROM folders ORDER BY name"; pub const Q_004: &str = "DELETE FROM folders"; pub const Q_005: &str = "UPDATE folders SET updated_at = CURRENT_TIMESTAMP::text"; - pub const Q_006: &str = "UPDATE folders SET uid_validity = $1, total_count = $2, messages_updated_at = CURRENT_TIMESTAMP::text + pub const Q_006: &str = "UPDATE folders SET uid_validity = $1, total_count = $2, counter_generation = counter_generation + 1, messages_updated_at = CURRENT_TIMESTAMP::text WHERE name = $3"; pub const Q_007: &str = - "UPDATE folders SET uid_validity = $1, total_count = $2, highest_modseq = $3 + "UPDATE folders SET uid_validity = $1, total_count = $2, highest_modseq = $3, counter_generation = counter_generation + 1 WHERE name = $4"; pub const Q_008: &str = "SELECT COUNT(*) FROM messages WHERE folder = $1 AND strpos(flags, chr(92) || 'Seen') = 0"; - pub const Q_009: &str = "UPDATE folders SET unread_count = $1 WHERE name = $2"; - pub const Q_010: &str = "UPDATE folders SET unread_count = GREATEST(0, CAST(unread_count AS INTEGER) + $1) WHERE name = $2"; + pub const Q_009: &str = "UPDATE folders SET unread_count = $1, counter_generation = counter_generation + 1 WHERE name = $2"; + pub const Q_010: &str = "UPDATE folders SET unread_count = GREATEST(0, CAST(unread_count AS INTEGER) + $1), counter_generation = counter_generation + 1 WHERE name = $2"; pub const Q_011: &str = "SELECT EXISTS( SELECT 1 FROM folders WHERE updated_at > to_char(CURRENT_TIMESTAMP - ($1 * INTERVAL '1 second'), 'YYYY-MM-DD HH24:MI:SS') @@ -217,14 +218,14 @@ pub mod db_folders { pub const Q_013: &str = "SELECT messages_updated_at IS NULL FROM folders WHERE name = $1"; pub const Q_014: &str = "UPDATE folders SET messages_updated_at = NULL WHERE name = $1"; pub const Q_015: &str = "SELECT name, delimiter, parent, flags, is_subscribed, - total_count, unread_count, uid_validity, highest_modseq + total_count, unread_count, uid_validity, highest_modseq, counter_generation FROM folders WHERE name = $1"; pub const Q_016: &str = "DELETE FROM messages WHERE folder = $1"; pub const Q_017: &str = "DELETE FROM folders WHERE name = $1"; pub const Q_018: &str = "SELECT 1"; - pub const Q_019: &str = "INSERT INTO folders (name, delimiter, parent, flags, is_subscribed, total_count, unread_count, uid_validity, highest_modseq, updated_at, messages_updated_at) - SELECT $1, delimiter, parent, flags, is_subscribed, total_count, unread_count, uid_validity, highest_modseq, CURRENT_TIMESTAMP::text, messages_updated_at + pub const Q_019: &str = "INSERT INTO folders (name, delimiter, parent, flags, is_subscribed, total_count, unread_count, uid_validity, highest_modseq, counter_generation, updated_at, messages_updated_at) + SELECT $1, delimiter, parent, flags, is_subscribed, total_count, unread_count, uid_validity, highest_modseq, counter_generation, CURRENT_TIMESTAMP::text, messages_updated_at FROM folders WHERE name = $2"; pub const Q_020: &str = "UPDATE messages SET folder = $1 WHERE folder = $2"; pub const Q_021: &str = "UPDATE message_tags SET message_folder = $1 WHERE message_folder = $2"; @@ -236,9 +237,13 @@ pub mod db_folders { pub const Q_026: &str = "UPDATE folders SET messages_updated_at = '2000-01-01 00:00:00' WHERE name = 'Sent'"; pub const Q_027: &str = "SELECT messages_updated_at FROM folders WHERE name = 'Sent'"; - pub const Q_028: &str = "UPDATE folders SET unread_count = $1 WHERE name = $2"; + pub const Q_028: &str = "UPDATE folders SET unread_count = $1, counter_generation = counter_generation + 1 WHERE name = $2"; pub const Q_029: &str = "UPDATE folders SET messages_updated_at = CURRENT_TIMESTAMP::text WHERE name = $1"; + pub const Q_030: &str = "UPDATE folders SET total_count = $1, unread_count = $2, counter_generation = counter_generation + 1 WHERE name = $3"; + pub const Q_031: &str = + "UPDATE folders SET uid_validity = $1, highest_modseq = $2 WHERE name = $3"; + pub const Q_032: &str = "UPDATE folders SET total_count = $1, unread_count = $2, counter_generation = counter_generation + 1 WHERE name = $3 AND counter_generation = $4"; pub const FORMAT_001: &str = "DELETE FROM folders WHERE name NOT IN ({})"; } @@ -310,6 +315,12 @@ pub mod db_messages { LIMIT $1"; pub const Q_013: &str = "SELECT uid, flags FROM messages WHERE folder = $1"; pub const Q_014: &str = "SELECT SUM(size)::bigint FROM messages"; + pub const Q_015: &str = "SELECT COUNT(*) FROM messages + WHERE body_cached = 0 + AND lower(folder) NOT IN ('trash', 'junk', 'spam') + AND lower(folder) NOT LIKE '%/trash' + AND lower(folder) NOT LIKE '%/junk' + AND lower(folder) NOT LIKE '%/spam'"; pub const FORMAT_001: &str = "SELECT {MSG_SELECT_COLS} FROM messages WHERE folder = $1 @@ -467,9 +478,9 @@ pub mod routes_diagnostics { pub mod routes_folder_mgmt { pub const Q_001: &str = "UPDATE messages SET flags = CASE WHEN flags = '' THEN $2 ELSE flags || ',' || $2 END WHERE folder = $1 AND POSITION($2 IN flags) = 0"; - pub const Q_002: &str = "UPDATE folders SET unread_count = 0 WHERE name = $1"; + pub const Q_002: &str = "UPDATE folders SET unread_count = 0, counter_generation = counter_generation + 1 WHERE name = $1"; pub const Q_003: &str = "SELECT uid FROM messages WHERE folder = $1"; - pub const Q_004: &str = "UPDATE folders SET total_count = 0, unread_count = 0, messages_updated_at = CURRENT_TIMESTAMP::text WHERE name = $1"; + pub const Q_004: &str = "UPDATE folders SET total_count = 0, unread_count = 0, counter_generation = counter_generation + 1, messages_updated_at = CURRENT_TIMESTAMP::text WHERE name = $1"; } pub mod routes_messages_mod { diff --git a/src/queries/sqlite/queries.rs b/src/queries/sqlite/queries.rs index cc77986..fff6674 100644 --- a/src/queries/sqlite/queries.rs +++ b/src/queries/sqlite/queries.rs @@ -178,28 +178,33 @@ pub mod db_folder_preferences { } pub mod db_folders { - pub const Q_001: &str = "INSERT OR REPLACE INTO folders + pub const Q_001: &str = "INSERT INTO folders (name, delimiter, parent, flags, is_subscribed, total_count, unread_count, uid_validity, highest_modseq, updated_at) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, datetime('now'))"; + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, datetime('now')) + ON CONFLICT(name) DO UPDATE SET + delimiter=excluded.delimiter, parent=excluded.parent, flags=excluded.flags, + is_subscribed=excluded.is_subscribed, total_count=excluded.total_count, unread_count=excluded.unread_count, + uid_validity=excluded.uid_validity, highest_modseq=excluded.highest_modseq, + counter_generation=folders.counter_generation + 1, updated_at=datetime('now')"; pub const Q_002: &str = "INSERT OR IGNORE INTO folders (name, delimiter, flags, is_subscribed, updated_at) VALUES (?1, ?2, ?3, 1, datetime('now'))"; pub const Q_003: &str = "SELECT name, delimiter, parent, flags, is_subscribed, - total_count, unread_count, uid_validity, highest_modseq + total_count, unread_count, uid_validity, highest_modseq, counter_generation FROM folders ORDER BY name"; pub const Q_004: &str = "DELETE FROM folders"; pub const Q_005: &str = "UPDATE folders SET updated_at = datetime('now')"; - pub const Q_006: &str = "UPDATE folders SET uid_validity = ?1, total_count = ?2, messages_updated_at = datetime('now') + pub const Q_006: &str = "UPDATE folders SET uid_validity = ?1, total_count = ?2, counter_generation = counter_generation + 1, messages_updated_at = datetime('now') WHERE name = ?3"; pub const Q_007: &str = - "UPDATE folders SET uid_validity = ?1, total_count = ?2, highest_modseq = ?3 + "UPDATE folders SET uid_validity = ?1, total_count = ?2, highest_modseq = ?3, counter_generation = counter_generation + 1 WHERE name = ?4"; pub const Q_008: &str = "SELECT COUNT(*) FROM messages WHERE folder = ?1 AND flags NOT LIKE '%\\Seen%'"; - pub const Q_009: &str = "UPDATE folders SET unread_count = ?1 WHERE name = ?2"; - pub const Q_010: &str = "UPDATE folders SET unread_count = MAX(0, CAST(unread_count AS INTEGER) + ?1) WHERE name = ?2"; + pub const Q_009: &str = "UPDATE folders SET unread_count = ?1, counter_generation = counter_generation + 1 WHERE name = ?2"; + pub const Q_010: &str = "UPDATE folders SET unread_count = MAX(0, CAST(unread_count AS INTEGER) + ?1), counter_generation = counter_generation + 1 WHERE name = ?2"; pub const Q_011: &str = "SELECT EXISTS( SELECT 1 FROM folders WHERE updated_at > datetime('now', printf('-%d seconds', ?1)) @@ -211,14 +216,14 @@ pub mod db_folders { pub const Q_013: &str = "SELECT messages_updated_at IS NULL FROM folders WHERE name = ?1"; pub const Q_014: &str = "UPDATE folders SET messages_updated_at = NULL WHERE name = ?1"; pub const Q_015: &str = "SELECT name, delimiter, parent, flags, is_subscribed, - total_count, unread_count, uid_validity, highest_modseq + total_count, unread_count, uid_validity, highest_modseq, counter_generation FROM folders WHERE name = ?1"; pub const Q_016: &str = "DELETE FROM messages WHERE folder = ?1"; pub const Q_017: &str = "DELETE FROM folders WHERE name = ?1"; pub const Q_018: &str = "PRAGMA defer_foreign_keys = ON;"; - pub const Q_019: &str = "INSERT INTO folders (name, delimiter, parent, flags, is_subscribed, total_count, unread_count, uid_validity, highest_modseq, updated_at, messages_updated_at) - SELECT ?1, delimiter, parent, flags, is_subscribed, total_count, unread_count, uid_validity, highest_modseq, datetime('now'), messages_updated_at + pub const Q_019: &str = "INSERT INTO folders (name, delimiter, parent, flags, is_subscribed, total_count, unread_count, uid_validity, highest_modseq, counter_generation, updated_at, messages_updated_at) + SELECT ?1, delimiter, parent, flags, is_subscribed, total_count, unread_count, uid_validity, highest_modseq, counter_generation, datetime('now'), messages_updated_at FROM folders WHERE name = ?2"; pub const Q_020: &str = "UPDATE messages SET folder = ?1 WHERE folder = ?2"; pub const Q_021: &str = "UPDATE message_tags SET message_folder = ?1 WHERE message_folder = ?2"; @@ -230,9 +235,13 @@ pub mod db_folders { pub const Q_026: &str = "UPDATE folders SET messages_updated_at = '2000-01-01 00:00:00' WHERE name = 'Sent'"; pub const Q_027: &str = "SELECT messages_updated_at FROM folders WHERE name = 'Sent'"; - pub const Q_028: &str = "UPDATE folders SET unread_count = ?1 WHERE name = ?2"; + pub const Q_028: &str = "UPDATE folders SET unread_count = ?1, counter_generation = counter_generation + 1 WHERE name = ?2"; pub const Q_029: &str = "UPDATE folders SET messages_updated_at = datetime('now') WHERE name = ?1"; + pub const Q_030: &str = "UPDATE folders SET total_count = ?1, unread_count = ?2, counter_generation = counter_generation + 1 WHERE name = ?3"; + pub const Q_031: &str = + "UPDATE folders SET uid_validity = ?1, highest_modseq = ?2 WHERE name = ?3"; + pub const Q_032: &str = "UPDATE folders SET total_count = ?1, unread_count = ?2, counter_generation = counter_generation + 1 WHERE name = ?3 AND counter_generation = ?4"; pub const FORMAT_001: &str = "DELETE FROM folders WHERE name NOT IN ({})"; } @@ -299,6 +308,12 @@ pub mod db_messages { LIMIT ?1"; pub const Q_013: &str = "SELECT uid, flags FROM messages WHERE folder = ?1"; pub const Q_014: &str = "SELECT SUM(size) FROM messages"; + pub const Q_015: &str = "SELECT COUNT(*) FROM messages + WHERE body_cached = 0 + AND lower(folder) NOT IN ('trash', 'junk', 'spam') + AND lower(folder) NOT LIKE '%/trash' + AND lower(folder) NOT LIKE '%/junk' + AND lower(folder) NOT LIKE '%/spam'"; pub const FORMAT_001: &str = "SELECT {MSG_SELECT_COLS} FROM messages WHERE folder = ?1 @@ -457,9 +472,9 @@ pub mod routes_diagnostics { pub mod routes_folder_mgmt { pub const Q_001: &str = "UPDATE messages SET flags = CASE WHEN flags = '' THEN ?2 ELSE flags || ',' || ?2 END WHERE folder = ?1 AND instr(flags, ?2) = 0"; - pub const Q_002: &str = "UPDATE folders SET unread_count = 0 WHERE name = ?1"; + pub const Q_002: &str = "UPDATE folders SET unread_count = 0, counter_generation = counter_generation + 1 WHERE name = ?1"; pub const Q_003: &str = "SELECT uid FROM messages WHERE folder = ?1"; - pub const Q_004: &str = "UPDATE folders SET total_count = 0, unread_count = 0, messages_updated_at = datetime('now') WHERE name = ?1"; + pub const Q_004: &str = "UPDATE folders SET total_count = 0, unread_count = 0, counter_generation = counter_generation + 1, messages_updated_at = datetime('now') WHERE name = ?1"; } pub mod routes_messages_mod { diff --git a/src/realtime/events.rs b/src/realtime/events.rs index 4bd78cd..40e1846 100644 --- a/src/realtime/events.rs +++ b/src/realtime/events.rs @@ -3,6 +3,12 @@ use std::collections::HashMap; use serde::Serialize; use tokio::sync::{RwLock, broadcast}; +#[derive(Debug, Clone, Serialize)] +pub struct MessageFlagChange { + pub uid: u32, + pub flags: Vec, +} + #[derive(Debug, Clone, Serialize)] #[serde(tag = "type", content = "data")] #[allow(dead_code)] @@ -17,10 +23,32 @@ pub enum MailEvent { FlagsChanged { folder: String, + changes: Vec, + }, + + FolderAllRead { + folder: String, + through_uid: Option, + }, + + FolderStatus { + folder: String, + total_count: u32, + unread_count: u32, }, FolderUpdated, + SyncStatus { + active: bool, + folder: Option, + pending: u32, + }, + + SyncInventory { + pending: u32, + }, + IndexingStatus { active: bool, pending: u32, diff --git a/src/realtime/idle.rs b/src/realtime/idle.rs index 4b3c71e..cc19d6e 100644 --- a/src/realtime/idle.rs +++ b/src/realtime/idle.rs @@ -2,16 +2,25 @@ use std::sync::Arc; use dashmap::DashMap; use futures::StreamExt; +use serde::Serialize; use tokio::task::JoinHandle; use crate::config::AppConfig; use crate::imap::client::ImapCredentials; use crate::imap::connection::ImapStream; use crate::imap::parse::{decode_rfc2047, flag_to_string, has_attachments, imap_address_to_email}; -use crate::realtime::events::{EventBus, MailEvent}; -use crate::search::engine::SearchEngine; +use crate::realtime::events::{EventBus, MailEvent, MessageFlagChange}; +use crate::search::engine::{IndexableMessage, SearchEngine, UserIndex}; use crate::{db, imap}; +#[derive(Debug, Clone, Serialize)] +pub struct IdleTaskDiagnostics { + #[serde(skip_serializing_if = "Option::is_none")] + pub user_hash: Option, + pub folder: String, + pub subscribers: usize, +} + pub struct IdleManager { tasks: DashMap<(String, String), IdleTask>, } @@ -90,6 +99,21 @@ impl IdleManager { pub fn subscriber_count(&self) -> usize { self.tasks.iter().map(|task| task.value().subscribers).sum() } + + pub fn diagnostics_snapshot(&self, include_user_identifiers: bool) -> Vec { + let mut tasks = self + .tasks + .iter() + .map(|task| IdleTaskDiagnostics { + user_hash: include_user_identifiers.then(|| task.key().0.clone()), + folder: task.key().1.clone(), + subscribers: task.value().subscribers, + }) + .collect::>(); + tasks.sort_by(|left, right| left.folder.cmp(&right.folder)); + tasks.truncate(200); + tasks + } } impl Default for IdleManager { @@ -201,12 +225,28 @@ async fn run_idle_session( }, ) .await; - } else if result.flags_updated > 0 || result.deleted_count > 0 { + } + if result.flags_updated > 0 || result.deleted_count > 0 { event_bus .publish( user_hash, MailEvent::FlagsChanged { folder: folder.to_string(), + changes: result.flag_changes, + }, + ) + .await; + } + if let Ok(conn) = db::pool::open_user_db(&config.data_dir, user_hash) + && let Ok(Some(folder_state)) = db::folders::get_folder(&conn, folder) + { + event_bus + .publish( + user_hash, + MailEvent::FolderStatus { + folder: folder.to_string(), + total_count: folder_state.total_count, + unread_count: folder_state.unread_count, }, ) .await; @@ -249,6 +289,7 @@ struct ReconcileResult { latest_uid: Option, latest_sender: Option, latest_subject: Option, + flag_changes: Vec, } impl ReconcileResult { @@ -260,6 +301,7 @@ impl ReconcileResult { latest_uid: None, latest_sender: None, latest_subject: None, + flag_changes: Vec::new(), } } } @@ -271,28 +313,42 @@ async fn fetch_new_after_idle( config: &AppConfig, search_engine: &SearchEngine, ) -> (async_imap::Session, ReconcileResult) { - let (cached_modseq, max_cached_uid, cached_count, cached_uid_validity, header_sync) = - match db::pool::open_user_db(&config.data_dir, user_hash) { - Ok(conn) => { - let header_sync = match db::header_sync::HeaderSync::capture(&conn) { - Ok(sync) => sync, - Err(e) => { - tracing::warn!(error = %e, "IDLE fetch: failed to read header generation"); - return (session, ReconcileResult::empty()); - } - }; - let cached = db::folders::get_folder(&conn, folder).ok().flatten(); - let modseq = cached.as_ref().map(|f| f.highest_modseq).unwrap_or(0); - let validity = cached.as_ref().map(|f| f.uid_validity).unwrap_or(0); - let max_uid = db::messages::max_uid(&conn, folder).unwrap_or(0); - let count = db::messages::count_messages(&conn, folder).unwrap_or(0); - (modseq, max_uid, count, validity, header_sync) - } - Err(e) => { - tracing::warn!(error = %e, "IDLE fetch: failed to open DB"); - return (session, ReconcileResult::empty()); - } - }; + let ( + cached_modseq, + max_cached_uid, + cached_count, + cached_counter_generation, + cached_uid_validity, + header_sync, + ) = match db::pool::open_user_db(&config.data_dir, user_hash) { + Ok(conn) => { + let header_sync = match db::header_sync::HeaderSync::capture(&conn) { + Ok(sync) => sync, + Err(e) => { + tracing::warn!(error = %e, "IDLE fetch: failed to read header generation"); + return (session, ReconcileResult::empty()); + } + }; + let cached = db::folders::get_folder(&conn, folder).ok().flatten(); + let modseq = cached.as_ref().map(|f| f.highest_modseq).unwrap_or(0); + let validity = cached.as_ref().map(|f| f.uid_validity).unwrap_or(0); + let counter_generation = cached.as_ref().map(|f| f.counter_generation).unwrap_or(0); + let max_uid = db::messages::max_uid(&conn, folder).unwrap_or(0); + let count = db::messages::count_messages(&conn, folder).unwrap_or(0); + ( + modseq, + max_uid, + count, + counter_generation, + validity, + header_sync, + ) + } + Err(e) => { + tracing::warn!(error = %e, "IDLE fetch: failed to open DB"); + return (session, ReconcileResult::empty()); + } + }; let mailbox = match session.select_condstore(folder).await { Ok(mb) => mb, @@ -303,7 +359,13 @@ async fn fetch_new_after_idle( }; let server_exists = mailbox.exists; - let server_unseen = mailbox.unseen; + let server_unseen = match session.uid_search("UNSEEN").await { + Ok(uids) => Some(uids.len() as u32), + Err(error) => { + tracing::debug!(folder = %folder, error = %error, "IDLE UNSEEN search failed"); + None + } + }; let server_modseq = mailbox.highest_modseq.unwrap_or(0); let uid_validity = mailbox.uid_validity.unwrap_or(0); @@ -314,6 +376,18 @@ async fn fetch_new_after_idle( return (session, ReconcileResult::empty()); } + if let Some(unseen) = server_unseen + && let Ok(conn) = db::pool::open_user_db(&config.data_dir, user_hash) + { + let _ = db::folders::set_folder_counts_if_unchanged( + &conn, + folder, + cached_counter_generation, + server_exists, + unseen, + ); + } + if server_modseq > 0 && server_modseq == cached_modseq && server_exists == cached_count { tracing::debug!( user_hash = %user_hash, @@ -331,6 +405,7 @@ async fn fetch_new_after_idle( Ok(stream) => { let mut stream = std::pin::pin!(stream); let mut items = Vec::new(); + let mut complete = true; while let Some(result) = stream.next().await { match result { Ok(fetch) => { @@ -345,11 +420,12 @@ async fn fetch_new_after_idle( error = %e, "IDLE fetch: error reading CHANGEDSINCE stream" ); + complete = false; break; } } } - Some(items) + complete.then_some(items) } Err(e) => { tracing::warn!( @@ -378,18 +454,42 @@ async fn fetch_new_after_idle( let sync_keywords = db::mail_preferences::get_preferences(&conn) .map(|prefs| prefs.sync_imap_keywords) .unwrap_or(false); - for (uid, flags) in &flag_changes { - let mut sorted = flags.clone(); - sorted.sort(); - let flags_csv = sorted.join(","); - if db::messages::update_message_flags(&conn, folder, *uid, &flags_csv).is_ok() { - if sync_keywords - && let Err(error) = - db::tags::import_imap_keywords(&conn, *uid, folder, &sorted) - { - tracing::debug!(uid = uid, error = %error, "IDLE keyword import failed"); + match header_sync.begin_write(&conn) { + Ok(transaction) => { + for (uid, flags) in &flag_changes { + let mut sorted = flags.clone(); + sorted.sort(); + let flags_csv = sorted.join(","); + if db::messages::update_message_flags( + &transaction, + folder, + *uid, + &flags_csv, + ) + .is_ok() + { + if sync_keywords + && let Err(error) = db::tags::import_imap_keywords( + &transaction, + *uid, + folder, + &sorted, + ) + { + tracing::debug!(uid = uid, error = %error, "IDLE keyword import failed"); + } + flags_updated += 1; + } } - flags_updated += 1; + if let Err(error) = transaction.commit() { + tracing::debug!(error = %error, "IDLE flag reconciliation commit failed"); + flags_updated = 0; + flag_changes.clear(); + } + } + Err(error) => { + tracing::debug!(error = %error, "IDLE flag reconciliation discarded stale cache generation"); + flag_changes.clear(); } } } @@ -399,12 +499,15 @@ async fn fetch_new_after_idle( let uid_range = format!("{}:*", max_cached_uid + 1); new_count = fetch_and_store_headers( &mut session, - user_hash, - folder, - config, + HeaderFetchContext { + user_hash, + folder, + config, + search_engine, + header_sync, + }, &uid_range, max_cached_uid, - header_sync, ) .await; } @@ -430,22 +533,16 @@ async fn fetch_new_after_idle( if let Ok(conn) = db::pool::open_user_db(&config.data_dir, user_hash) && let Ok(transaction) = header_sync.begin_write(&conn) { - let _ = db::folders::update_folder_sync_status( - &conn, + let _ = db::folders::update_folder_sync_cursor( + &transaction, folder, uid_validity, - server_exists, server_modseq, ); - if let Some(unseen) = server_unseen { - let _ = db::folders::set_unread_count(&conn, folder, unseen); - } else { - let _ = db::folders::refresh_unread_count(&conn, folder); - } if has_new && new_count == 0 { - let _ = db::folders::invalidate_folder_freshness(&conn, folder); + let _ = db::folders::invalidate_folder_freshness(&transaction, folder); } else { - let _ = db::folders::mark_folder_messages_fresh(&conn, folder); + let _ = db::folders::mark_folder_messages_fresh(&transaction, folder); } let _ = transaction.commit(); } @@ -470,53 +567,136 @@ async fn fetch_new_after_idle( latest_uid, latest_sender, latest_subject, + flag_changes: flag_changes + .into_iter() + .map(|(uid, mut flags)| { + flags.sort(); + MessageFlagChange { uid, flags } + }) + .collect(), }, ); } } - let cached_uids: std::collections::HashSet = - match db::pool::open_user_db(&config.data_dir, user_hash) { - Ok(conn) => { - let uids_flags = - db::messages::get_all_uids_and_flags(&conn, folder).unwrap_or_default(); - uids_flags.iter().map(|(uid, _)| *uid).collect() - } - Err(e) => { - tracing::warn!(error = %e, "IDLE fetch: failed to open DB for fallback"); - return (session, ReconcileResult::empty()); - } - }; + let cached_state = match db::pool::open_user_db(&config.data_dir, user_hash) { + Ok(conn) => db::messages::get_all_uids_and_flags(&conn, folder).unwrap_or_default(), + Err(e) => { + tracing::warn!(error = %e, "IDLE fetch: failed to open DB for fallback"); + return (session, ReconcileResult::empty()); + } + }; + let cached_uids = cached_state + .iter() + .map(|(uid, _)| *uid) + .collect::>(); + let cached_flags = cached_state + .into_iter() + .map(|(uid, flags)| { + let mut normalized = flags + .split(',') + .filter(|flag| !flag.is_empty()) + .map(str::to_string) + .collect::>(); + normalized.sort(); + normalized.dedup(); + (uid, normalized) + }) + .collect::>(); - let imap_uids: Option> = - match session.uid_fetch("1:*", "UID").await { - Ok(stream) => { - let mut stream = std::pin::pin!(stream); - let mut uids = std::collections::HashSet::new(); - while let Some(result) = stream.next().await { - match result { - Ok(fetch) => { - if let Some(uid) = fetch.uid { - uids.insert(uid); - } - } - Err(e) => { - tracing::warn!(error = %e, "IDLE fetch: error reading UID stream"); - break; + let imap_state: Option>> = match session + .uid_fetch("1:*", "UID FLAGS") + .await + { + Ok(stream) => { + let mut stream = std::pin::pin!(stream); + let mut state = std::collections::HashMap::new(); + let mut complete = true; + while let Some(result) = stream.next().await { + match result { + Ok(fetch) => { + if let Some(uid) = fetch.uid { + let mut flags = fetch + .flags() + .map(|flag| flag_to_string(&flag)) + .collect::>(); + flags.sort(); + flags.dedup(); + state.insert(uid, flags); } } + Err(e) => { + tracing::warn!(error = %e, "IDLE fetch: error reading UID/FLAGS stream"); + complete = false; + break; + } } - Some(uids) } - Err(e) => { - tracing::warn!(error = %e, "IDLE fetch: uid_fetch(UIDs) failed"); - None - } - }; + complete.then_some(state) + } + Err(e) => { + tracing::warn!(error = %e, "IDLE fetch: uid_fetch(UID FLAGS) failed"); + None + } + }; - let Some(imap_uids) = imap_uids else { + let Some(imap_state) = imap_state else { return (session, ReconcileResult::empty()); }; + let imap_uids = imap_state + .keys() + .copied() + .collect::>(); + + let mut flag_changes = cached_flags + .iter() + .filter_map(|(uid, cached)| { + imap_state + .get(uid) + .filter(|server| *server != cached) + .map(|server| (*uid, server.clone())) + }) + .collect::>(); + flag_changes.sort_by_key(|(uid, _)| *uid); + let mut flags_updated = 0u32; + if !flag_changes.is_empty() + && let Ok(conn) = db::pool::open_user_db(&config.data_dir, user_hash) + { + let sync_keywords = db::mail_preferences::get_preferences(&conn) + .map(|prefs| prefs.sync_imap_keywords) + .unwrap_or(false); + match header_sync.begin_write(&conn) { + Ok(transaction) => { + for (uid, flags) in &flag_changes { + if db::messages::update_message_flags( + &transaction, + folder, + *uid, + &flags.join(","), + ) + .is_ok() + { + if sync_keywords + && let Err(error) = + db::tags::import_imap_keywords(&transaction, *uid, folder, flags) + { + tracing::debug!(uid = uid, error = %error, "IDLE fallback keyword import failed"); + } + flags_updated += 1; + } + } + if let Err(error) = transaction.commit() { + tracing::debug!(error = %error, "IDLE fallback flag reconciliation commit failed"); + flags_updated = 0; + flag_changes.clear(); + } + } + Err(error) => { + tracing::debug!(error = %error, "IDLE fallback flag reconciliation discarded stale cache generation"); + flag_changes.clear(); + } + } + } let has_new = imap_uids.iter().any(|&uid| uid > max_cached_uid); let mut new_count = 0u32; @@ -524,12 +704,15 @@ async fn fetch_new_after_idle( let uid_range = format!("{}:*", max_cached_uid + 1); new_count = fetch_and_store_headers( &mut session, - user_hash, - folder, - config, + HeaderFetchContext { + user_hash, + folder, + config, + search_engine, + header_sync, + }, &uid_range, max_cached_uid, - header_sync, ) .await; } @@ -553,22 +736,16 @@ async fn fetch_new_after_idle( if let Ok(conn) = db::pool::open_user_db(&config.data_dir, user_hash) && let Ok(transaction) = header_sync.begin_write(&conn) { - let _ = db::folders::update_folder_sync_status( - &conn, + let _ = db::folders::update_folder_sync_cursor( + &transaction, folder, uid_validity, - server_exists, server_modseq, ); - if let Some(unseen) = server_unseen { - let _ = db::folders::set_unread_count(&conn, folder, unseen); - } else { - let _ = db::folders::refresh_unread_count(&conn, folder); - } if has_new && new_count == 0 { - let _ = db::folders::invalidate_folder_freshness(&conn, folder); + let _ = db::folders::invalidate_folder_freshness(&transaction, folder); } else { - let _ = db::folders::mark_folder_messages_fresh(&conn, folder); + let _ = db::folders::mark_folder_messages_fresh(&transaction, folder); } let _ = transaction.commit(); } @@ -579,10 +756,11 @@ async fn fetch_new_after_idle( (None, None, None) }; - if new_count > 0 || deleted_count > 0 { + if flags_updated > 0 || new_count > 0 || deleted_count > 0 { tracing::info!( user_hash = %user_hash, folder = %folder, + flags_updated = flags_updated, new = new_count, deleted = deleted_count, "Reconciled cache after IDLE (full scan)" @@ -593,11 +771,15 @@ async fn fetch_new_after_idle( session, ReconcileResult { new_count, - flags_updated: 0, + flags_updated, deleted_count, latest_uid, latest_sender, latest_subject, + flag_changes: flag_changes + .into_iter() + .map(|(uid, flags)| MessageFlagChange { uid, flags }) + .collect(), }, ) } @@ -634,15 +816,27 @@ fn lookup_latest_message( } } +struct HeaderFetchContext<'a> { + user_hash: &'a str, + folder: &'a str, + config: &'a AppConfig, + search_engine: &'a SearchEngine, + header_sync: db::header_sync::HeaderSync, +} + async fn fetch_and_store_headers( session: &mut async_imap::Session, - user_hash: &str, - folder: &str, - config: &AppConfig, + context: HeaderFetchContext<'_>, uid_range: &str, min_uid: u32, - header_sync: db::header_sync::HeaderSync, ) -> u32 { + let HeaderFetchContext { + user_hash, + folder, + config, + search_engine, + header_sync, + } = context; let fetches: Option> = match session .uid_fetch( uid_range, @@ -653,16 +847,18 @@ async fn fetch_and_store_headers( Ok(stream) => { let mut stream = std::pin::pin!(stream); let mut items = Vec::new(); + let mut complete = true; while let Some(result) = stream.next().await { match result { Ok(fetch) => items.push(fetch), Err(e) => { tracing::warn!(error = %e, "IDLE fetch: error reading header stream"); + complete = false; break; } } } - Some(items) + complete.then_some(items) } Err(e) => { tracing::warn!(error = %e, "IDLE fetch: uid_fetch(headers) failed"); @@ -684,6 +880,11 @@ async fn fetch_and_store_headers( let sync_keywords = db::mail_preferences::get_preferences(&conn) .map(|prefs| prefs.sync_imap_keywords) .unwrap_or(false); + let user_index = if search_engine.is_enabled() && !UserIndex::is_excluded_folder(folder) { + search_engine.open_user_index(user_hash).ok() + } else { + None + }; for fetch in &fetches { let uid = match fetch.uid { Some(u) if u > min_uid => u, @@ -787,7 +988,7 @@ async fn fetch_and_store_headers( let flags_csv = flags.join(","); if let Err(e) = db::messages::upsert_message( - &conn, + &transaction, folder, uid, message_id.as_deref(), @@ -808,7 +1009,7 @@ async fn fetch_and_store_headers( tracing::warn!(uid = uid, error = %e, "IDLE fetch: failed to upsert message"); } else { if let Err(e) = db::contacts::populate_known_addresses( - &conn, + &transaction, from_address, from_name, &to_json, @@ -817,10 +1018,30 @@ async fn fetch_and_store_headers( tracing::warn!(uid = uid, error = %e, "IDLE fetch: failed to populate known addresses"); } if sync_keywords - && let Err(error) = db::tags::import_imap_keywords(&conn, uid, folder, &flags) + && let Err(error) = + db::tags::import_imap_keywords(&transaction, uid, folder, &flags) { tracing::debug!(uid = uid, error = %error, "IDLE keyword import failed"); } + if let Some(index) = user_index.as_ref() + && let Err(error) = index.index_message_with_connection( + &transaction, + &IndexableMessage { + uid, + folder: folder.to_string(), + subject: subject_str.to_string(), + from_address: from_address.to_string(), + from_name: from_name.to_string(), + to_addresses: to_json.clone(), + body_text: String::new(), + date_epoch: db::messages::parse_date_to_epoch_public(date_str), + has_attachments: has_attach, + }, + ) + { + tracing::warn!(uid = uid, error = %error, "IDLE fetch: failed to enqueue search indexing"); + return 0; + } new_count += 1; } } @@ -888,6 +1109,7 @@ async fn fetch_uids_and_remove_deleted( Ok(stream) => { let mut stream = std::pin::pin!(stream); let mut uids = std::collections::HashSet::new(); + let mut complete = true; while let Some(result) = stream.next().await { match result { Ok(fetch) => { @@ -900,11 +1122,12 @@ async fn fetch_uids_and_remove_deleted( error = %e, "IDLE fetch: error reading UID stream for deletion check" ); + complete = false; break; } } } - Some(uids) + complete.then_some(uids) } Err(e) => { tracing::warn!( diff --git a/src/realtime/sync.rs b/src/realtime/sync.rs index 6188dc0..feac0ee 100644 --- a/src/realtime/sync.rs +++ b/src/realtime/sync.rs @@ -5,12 +5,10 @@ use crate::config::AppConfig; use crate::db; use crate::email_theme; use crate::imap::client::{ImapClient, ImapCredentials}; -use crate::realtime::events::{EventBus, MailEvent}; +use crate::realtime::events::{EventBus, MailEvent, MessageFlagChange}; use crate::search::engine::UserIndex; use crate::search::engine::{IndexableMessage, SearchEngine}; -const SYNC_INTERVAL_SECS: u64 = 30; - pub async fn sync_loop( user_hash: String, creds: ImapCredentials, @@ -19,7 +17,8 @@ pub async fn sync_loop( event_bus: Arc, search_engine: Arc, ) { - let mut interval = tokio::time::interval(std::time::Duration::from_secs(SYNC_INTERVAL_SECS)); + let mut interval = + tokio::time::interval(std::time::Duration::from_secs(config.sync.interval_secs)); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); loop { @@ -62,28 +61,81 @@ async fn run_sync( uid_validity: f.uid_validity, highest_modseq: f.highest_modseq, total_count: f.total_count, + unread_count: f.unread_count, + counter_generation: f.counter_generation, + flags: f.flags, }) .collect::>()) }) .await?; let mut any_changes = false; + let folder_names = folder_snapshots + .iter() + .filter(|folder| !folder.is_noselect()) + .map(|folder| folder.name.clone()) + .collect::>(); + let mut batched_statuses = imap_client + .folder_statuses_extended(creds, &folder_names) + .await + .unwrap_or_default() + .into_iter() + .collect::>(); for folder in &folder_snapshots { let folder_name = &folder.name; + if folder.is_noselect() { + continue; + } - let status = match imap_client.folder_status_extended(creds, folder_name).await { - Ok(s) => s, - Err(e) => { - tracing::debug!( - folder = %folder_name, - error = %e, - "Skipping folder sync (STATUS failed)" - ); - continue; + let status = if let Some(status) = batched_statuses.remove(folder_name) { + status + } else { + match imap_client.folder_status_extended(creds, folder_name).await { + Ok(status) => status, + Err(error) => { + tracing::debug!( + folder = %folder_name, + error = %error, + "Skipping folder sync (STATUS failed)" + ); + continue; + } } }; + if status.exists != folder.total_count || status.unseen != folder.unread_count { + let name = folder_name.clone(); + let expected_generation = folder.counter_generation; + let total_count = status.exists; + let unread_count = status.unseen; + let applied = database + .run(move |conn| { + db::folders::set_folder_counts_if_unchanged( + conn, + &name, + expected_generation, + total_count, + unread_count, + ) + }) + .await?; + if applied { + event_bus + .publish( + user_hash, + MailEvent::FolderStatus { + folder: folder_name.clone(), + total_count, + unread_count, + }, + ) + .await; + } else { + tracing::debug!(folder = %folder_name, "Skipped stale periodic STATUS counters"); + } + } + if folder.uid_validity != 0 && status.uid_validity != folder.uid_validity { tracing::info!( folder = %folder_name, @@ -162,7 +214,13 @@ async fn run_sync( } if any_changes { - event_bus.publish(user_hash, MailEvent::FolderUpdated).await; + tracing::debug!(user_hash = %user_hash, "Periodic sync applied message changes"); + } + + if let Ok(pending) = database.run(db::folders::pending_message_sync_count).await { + event_bus + .publish(user_hash, MailEvent::SyncInventory { pending }) + .await; } if search_engine.is_enabled() @@ -182,8 +240,6 @@ async fn run_sync( Ok(()) } -const DEEP_INDEX_BATCH: u32 = 10; - async fn index_message_bodies( user_hash: &str, creds: &ImapCredentials, @@ -193,12 +249,18 @@ async fn index_message_bodies( search_engine: Arc, ) -> Result<(), String> { let database = db::pool::UserDb::new(&config.data_dir, user_hash); - let unindexed = database - .run(|conn| { - if !db::display_preferences::get_preferences(conn)?.deep_index { - return Ok(Vec::new()); + let force_deep_index = config.search.deep_index; + let deep_index_batch_size = config.sync.deep_index_batch_size; + let (pending_before, unindexed) = database + .run(move |conn| { + let account_deep_index = db::display_preferences::get_preferences(conn)?.deep_index; + if !force_deep_index && !account_deep_index { + return Ok((0, Vec::new())); } - db::messages::get_unindexed_messages(conn, DEEP_INDEX_BATCH) + Ok(( + db::messages::count_unindexed_messages(conn)?, + db::messages::get_unindexed_messages(conn, deep_index_batch_size)?, + )) }) .await?; @@ -208,7 +270,7 @@ async fn index_message_bodies( user_hash, MailEvent::IndexingStatus { active: false, - pending: 0, + pending: pending_before, }, ) .await; @@ -220,7 +282,7 @@ async fn index_message_bodies( user_hash, MailEvent::IndexingStatus { active: true, - pending: unindexed.len() as u32, + pending: pending_before, }, ) .await; @@ -297,12 +359,16 @@ async fn index_message_bodies( } .await; + let pending_after = database + .run(db::messages::count_unindexed_messages) + .await + .unwrap_or(pending_before); event_bus .publish( user_hash, MailEvent::IndexingStatus { active: false, - pending: 0, + pending: pending_after, }, ) .await; @@ -314,6 +380,17 @@ struct FolderSnapshot { uid_validity: u32, highest_modseq: u64, total_count: u32, + unread_count: u32, + counter_generation: u64, + flags: String, +} + +impl FolderSnapshot { + fn is_noselect(&self) -> bool { + self.flags + .split(',') + .any(|flag| flag.trim().eq_ignore_ascii_case("\\Noselect")) + } } fn store_headers( @@ -551,7 +628,7 @@ async fn sync_condstore( let search_engine_for_write = Arc::clone(search_engine); let search_user_hash = user_hash.to_owned(); let headers_for_write = new_headers; - let (folder_changed, non_new_change) = database + let (folder_changed, non_new_change, flag_changes) = database .run(move |conn| { let sync_keywords = db::mail_preferences::get_preferences(conn) .map(|prefs| prefs.sync_imap_keywords) @@ -560,9 +637,14 @@ async fn sync_condstore( .unchecked_transaction() .map_err(|error| error.to_string())?; let mut non_new_change = !changed.is_empty(); + let mut flag_changes = Vec::with_capacity(changed.len()); for (uid, mut flags) in changed { flags.sort(); db::messages::update_message_flags(&transaction, &name, uid, &flags.join(","))?; + flag_changes.push(MessageFlagChange { + uid, + flags: flags.clone(), + }); if sync_keywords { db::tags::import_imap_keywords(&transaction, uid, &name, &flags)?; } @@ -588,19 +670,18 @@ async fn sync_condstore( &search_engine_for_write, &search_user_hash, )?; + let modseq = if new_modseq > 0 { new_modseq } else { - status.highest_modseq + cached_modseq }; - db::folders::update_folder_sync_status( + db::folders::update_folder_sync_cursor( &transaction, &name, status.uid_validity, - status.exists, modseq, )?; - db::folders::set_unread_count(&transaction, &name, status.unseen)?; if has_unfetched_new { db::folders::invalidate_folder_freshness(&transaction, &name)?; } else if cache_initialized { @@ -610,6 +691,7 @@ async fn sync_condstore( Ok(( non_new_change || !headers_for_write.is_empty(), non_new_change, + flag_changes, )) }) .await?; @@ -635,6 +717,7 @@ async fn sync_condstore( user_hash, MailEvent::FlagsChanged { folder: folder_name.to_string(), + changes: flag_changes, }, ) .await; @@ -703,16 +786,21 @@ async fn sync_full( let status = status.clone(); database .run(move |conn| { - db::folders::update_folder_sync_status( + let safe_modseq = if status.exists == 0 { + status.highest_modseq + } else { + 0 + }; + db::folders::update_folder_sync_cursor( conn, &name, status.uid_validity, - status.exists, - status.highest_modseq, + safe_modseq, )?; - db::folders::set_unread_count(conn, &name, status.unseen)?; - if has_unfetched_new { + if has_unfetched_new || status.exists > 0 { db::folders::invalidate_folder_freshness(conn, &name)?; + } else { + db::folders::mark_folder_messages_fresh(conn, &name)?; } Ok(()) }) @@ -740,7 +828,7 @@ async fn sync_full( let search_engine_for_write = Arc::clone(search_engine); let search_user_hash = user_hash.to_owned(); let headers_for_write = new_headers; - let non_new_change = database + let (non_new_change, flag_changes) = database .run(move |conn| { let folder_name = &name; let sync_keywords = db::mail_preferences::get_preferences(conn) @@ -758,6 +846,7 @@ async fn sync_full( .collect(); let mut non_new_change = false; + let mut flag_changes = Vec::new(); for (uid, cached_flags_csv) in &cached { match imap_map.get(uid) { None => { @@ -788,6 +877,10 @@ async fn sync_full( *uid, &imap_flags_csv, )?; + flag_changes.push(MessageFlagChange { + uid: *uid, + flags: imap_flags.clone(), + }); non_new_change = true; } if sync_keywords { @@ -809,21 +902,20 @@ async fn sync_full( &search_engine_for_write, &search_user_hash, )?; - db::folders::update_folder_sync_status( + + db::folders::update_folder_sync_cursor( &transaction, folder_name, status.uid_validity, - status.exists, status.highest_modseq, )?; - db::folders::set_unread_count(&transaction, folder_name, status.unseen)?; if has_unfetched_new { db::folders::invalidate_folder_freshness(&transaction, folder_name)?; } else if cache_initialized { db::folders::mark_folder_messages_fresh(&transaction, folder_name)?; } transaction.commit().map_err(|error| error.to_string())?; - Ok(non_new_change) + Ok((non_new_change, flag_changes)) }) .await?; @@ -848,6 +940,7 @@ async fn sync_full( user_hash, MailEvent::FlagsChanged { folder: folder_name.to_string(), + changes: flag_changes, }, ) .await; diff --git a/src/routes/auth.rs b/src/routes/auth.rs index 118ee7b..a8c61c6 100644 --- a/src/routes/auth.rs +++ b/src/routes/auth.rs @@ -18,19 +18,27 @@ pub const BROWSER_COOKIE: &str = "rustbox_browser"; #[serde(rename_all = "camelCase")] pub struct AuthPolicyResponse { remember_me_enabled: bool, - remember_me_days: u64, - session_timeout_hours: u64, +} + +pub async fn auth_policy(Extension(config): Extension>) -> Json { + Json(AuthPolicyResponse { + remember_me_enabled: config.remember_me_enabled, + }) +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SessionPolicyResponse { pgp_enabled: bool, sieve_enabled: bool, diagnostics_enabled: bool, about_enabled: bool, } -pub async fn auth_policy(Extension(config): Extension>) -> Json { - Json(AuthPolicyResponse { - remember_me_enabled: config.remember_me_enabled, - remember_me_days: config.remember_me_days, - session_timeout_hours: config.session_timeout_hours, +pub async fn session_policy( + Extension(config): Extension>, +) -> Json { + Json(SessionPolicyResponse { pgp_enabled: config.pgp_enabled, sieve_enabled: config.sieve_enabled, diagnostics_enabled: config.diagnostics_enabled, diff --git a/src/routes/contacts.rs b/src/routes/contacts.rs index 99e2aa0..722da0e 100644 --- a/src/routes/contacts.rs +++ b/src/routes/contacts.rs @@ -624,7 +624,6 @@ pub async fn get_contact_photo_handler( Response::builder() .header("content-type", content_type) .header("cache-control", "private, max-age=3600") - .header("x-content-type-options", "nosniff") .body(axum::body::Body::from(data)) .map_err(|e| AppError::InternalError(format!("Failed to build photo response: {e}"))) } diff --git a/src/routes/diagnostics.rs b/src/routes/diagnostics.rs index 9b933cd..be5366a 100644 --- a/src/routes/diagnostics.rs +++ b/src/routes/diagnostics.rs @@ -9,6 +9,7 @@ use crate::config::AppConfig; use crate::db; use crate::error::AppError; use crate::imap::client::{ImapClient, ImapCredentials}; +use crate::realtime::events::{EventBus, MailEvent}; use crate::realtime::idle::IdleManager; use crate::search::engine::{IndexableMessage, SearchEngine, UserIndex}; use crate::smtp::client::{SmtpClient, SmtpCredentials}; @@ -45,6 +46,7 @@ pub struct SmtpDiagnosticsResponse { #[serde(rename_all = "camelCase")] pub struct SearchDiagnosticsResponse { backend: String, + deep_index_enabled: bool, indexed_documents: usize, cached_messages: usize, cached_bodies: usize, @@ -259,11 +261,14 @@ pub async fn search_diagnostics( let data_dir = config.data_dir.clone(); let user_hash = session.user_hash.clone(); let db_user_hash = user_hash.clone(); + let force_deep_index = config.search.deep_index; - let (cached_messages, cached_bodies, excluded_messages, outbox) = + let (deep_index_enabled, cached_messages, cached_bodies, excluded_messages, outbox) = tokio::task::spawn_blocking(move || -> Result<_, String> { let conn = db::pool::open_user_db(&data_dir, &db_user_hash) .map_err(|e| format!("Database error: {e}"))?; + let account_deep_index = db::display_preferences::get_preferences(&conn)?.deep_index; + let deep_index_enabled = force_deep_index || account_deep_index; let sources = db::messages::list_index_source_messages(&conn)?; let cached_messages = sources.len(); let excluded_messages = sources @@ -274,7 +279,13 @@ pub async fn search_diagnostics( .query_row(crate::dbq!(routes_diagnostics::Q_003), [], |row| row.get(0)) .map_err(|e| format!("Failed to count cached bodies: {e}"))?; let outbox = db::search_outbox::stats(&conn).unwrap_or_default(); - Ok((cached_messages, cached_bodies, excluded_messages, outbox)) + Ok(( + deep_index_enabled, + cached_messages, + cached_bodies, + excluded_messages, + outbox, + )) }) .await .map_err(|e| AppError::InternalError(format!("Search diagnostics task failed: {e}")))? @@ -293,6 +304,7 @@ pub async fn search_diagnostics( let stats = SearchDiagnosticsResponse { backend: search_engine.backend_name().to_string(), + deep_index_enabled, indexed_documents, cached_messages, cached_bodies, @@ -309,6 +321,7 @@ pub async fn rebuild_search_index( Extension(session): Extension, Extension(config): Extension>, Extension(search_engine): Extension>, + Extension(event_bus): Extension>, ) -> Result { if !search_engine.is_enabled() { return Err(AppError::ServiceUnavailable( @@ -330,13 +343,32 @@ pub async fn rebuild_search_index( .map_err(|e| AppError::InternalError(format!("Search rebuild task failed: {e}")))? .map_err(AppError::InternalError)?; - let index = search_engine - .open_user_index(&user_hash) - .map_err(AppError::InternalError)?; - let indexed_documents = index - .rebuild_from_messages(&messages) - .await - .map_err(AppError::InternalError)?; + event_bus + .publish( + &user_hash, + MailEvent::IndexingStatus { + active: true, + pending: source_messages.min(u32::MAX as usize) as u32, + }, + ) + .await; + + let rebuild_result = match search_engine.open_user_index(&user_hash) { + Ok(index) => index.rebuild_from_messages(&messages).await, + Err(error) => Err(error), + }; + + event_bus + .publish( + &user_hash, + MailEvent::IndexingStatus { + active: false, + pending: 0, + }, + ) + .await; + + let indexed_documents = rebuild_result.map_err(AppError::InternalError)?; Ok(Json(SearchRebuildResponse { status: "rebuilt", diff --git a/src/routes/folder_counts.rs b/src/routes/folder_counts.rs new file mode 100644 index 0000000..20fdfe2 --- /dev/null +++ b/src/routes/folder_counts.rs @@ -0,0 +1,120 @@ +use crate::config::AppConfig; +use crate::db; +use crate::error::AppError; +use crate::imap::client::{ImapClient, ImapCredentials}; + +pub(crate) type FolderCountSnapshot = Option<(u32, u32, u64)>; + +pub(crate) fn snapshot( + config: &AppConfig, + user_hash: &str, + folder: &str, +) -> Result { + let conn = db::pool::open_user_db(&config.data_dir, user_hash) + .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; + let state = db::folders::get_folder(&conn, folder) + .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; + Ok(state.map(|folder| { + ( + folder.total_count, + folder.unread_count, + folder.counter_generation, + ) + })) +} + +pub(crate) fn current_counts( + config: &AppConfig, + user_hash: &str, + folder: &str, +) -> Result, AppError> { + Ok(snapshot(config, user_hash, folder)?.map(|(total, unread, _generation)| (total, unread))) +} + +pub(crate) async fn reconcile_after_mutation( + config: &AppConfig, + user_hash: &str, + imap_client: &dyn ImapClient, + creds: &ImapCredentials, + folder: &str, + mut expected: FolderCountSnapshot, +) -> Result, AppError> { + for attempt in 0..2 { + let status = match imap_client.folder_status_extended(creds, folder).await { + Ok(status) => status, + Err(error) => { + tracing::debug!( + folder = %folder, + attempt, + error = %error, + "Could not refresh authoritative folder counters after IMAP mutation" + ); + return Ok(None); + } + }; + + let conn = db::pool::open_user_db(&config.data_dir, user_hash) + .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; + + let applied = + if let Some((_expected_total, _expected_unread, expected_generation)) = expected { + db::folders::set_folder_counts_if_unchanged( + &conn, + folder, + expected_generation, + status.exists, + status.unseen, + ) + .map_err(|e| AppError::InternalError(format!("Database error: {e}")))? + } else { + db::folders::set_folder_counts(&conn, folder, status.exists, status.unseen) + .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; + true + }; + + if applied { + return Ok(Some((status.exists, status.unseen))); + } + + expected = db::folders::get_folder(&conn, folder) + .map_err(|e| AppError::InternalError(format!("Database error: {e}")))? + .map(|state| { + ( + state.total_count, + state.unread_count, + state.counter_generation, + ) + }); + } + + current_counts(config, user_hash, folder) +} + +pub(crate) fn set_if_snapshot_matches( + config: &AppConfig, + user_hash: &str, + folder: &str, + expected: FolderCountSnapshot, + total_count: u32, + unread_count: u32, +) -> Result<(u32, u32), AppError> { + let conn = db::pool::open_user_db(&config.data_dir, user_hash) + .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; + + if let Some((_expected_total, _expected_unread, expected_generation)) = expected { + let _ = db::folders::set_folder_counts_if_unchanged( + &conn, + folder, + expected_generation, + total_count, + unread_count, + ) + .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; + } + + let state = db::folders::get_folder(&conn, folder) + .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; + Ok(state + .map(|state| (state.total_count, state.unread_count)) + .unwrap_or((total_count, unread_count))) +} diff --git a/src/routes/folder_mgmt.rs b/src/routes/folder_mgmt.rs index 6d6612b..dec5f73 100644 --- a/src/routes/folder_mgmt.rs +++ b/src/routes/folder_mgmt.rs @@ -10,6 +10,8 @@ use crate::config::AppConfig; use crate::db; use crate::error::AppError; use crate::imap::client::{ImapClient, ImapCredentials}; +use crate::realtime::events::{EventBus, MailEvent}; +use crate::routes::folder_counts; use crate::search::engine::SearchEngine; #[derive(Deserialize)] @@ -32,6 +34,14 @@ struct MessageResponse { status: &'static str, } +#[derive(Serialize)] +struct FolderCounterResponse { + status: &'static str, + total_count: u32, + unread_count: u32, + through_uid: Option, +} + const SYSTEM_FOLDERS: &[&str] = &["INBOX", "Sent", "Drafts", "Trash", "Junk", "Spam"]; fn is_system_folder(name: &str) -> bool { @@ -188,25 +198,96 @@ pub async fn mark_all_read( Extension(session): Extension, Extension(config): Extension>, Extension(imap_client): Extension>, + Extension(event_bus): Extension>, Path(name): Path, ) -> Result { let creds = build_creds(&session, &config)?; + let counter_snapshot = folder_counts::snapshot(&config, &session.user_hash, &name)?; + let cached_flags_snapshot = { + let conn = db::pool::open_user_db(&config.data_dir, &session.user_hash) + .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; + db::messages::get_all_uids_and_flags(&conn, &name) + .map_err(|e| AppError::InternalError(format!("Database error: {e}")))? + }; + let through_uid = cached_flags_snapshot.iter().map(|(uid, _)| *uid).max(); imap_client .mark_all_read(&creds, &name) .await .map_err(|e| AppError::ServiceUnavailable(format!("IMAP mark-all-read failed: {e}")))?; + let authoritative_counts = folder_counts::reconcile_after_mutation( + &config, + &session.user_hash, + imap_client.as_ref(), + &creds, + &name, + counter_snapshot, + ) + .await?; + let conn = db::pool::open_user_db(&config.data_dir, &session.user_hash) .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; - conn.execute( - crate::dbq!(routes_folder_mgmt::Q_001), - crate::db::pool::params![&name, "\\Seen"], - ) - .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; - conn.execute(crate::dbq!(routes_folder_mgmt::Q_002), [&name]) - .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; + for (uid, flags_csv) in cached_flags_snapshot { + let mut flags = flags_csv + .split(',') + .filter(|flag| !flag.is_empty()) + .map(str::to_string) + .collect::>(); + if !flags.iter().any(|flag| flag == "\\Seen") { + flags.push("\\Seen".to_string()); + flags.sort(); + flags.dedup(); + db::messages::update_message_flags(&conn, &name, uid, &flags.join(",")) + .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; + } + } + let (total_count, unread_count) = if let Some(counts) = authoritative_counts { + counts + } else if let Some((total_count, _unread_count, _generation)) = counter_snapshot { + folder_counts::set_if_snapshot_matches( + &config, + &session.user_hash, + &name, + counter_snapshot, + total_count, + 0, + )? + } else { + db::folders::get_folder(&conn, &name) + .map_err(|e| AppError::InternalError(format!("Database error: {e}")))? + .map(|folder| (folder.total_count, folder.unread_count)) + .unwrap_or((0, 0)) + }; + let (total_count, unread_count) = + folder_counts::current_counts(&config, &session.user_hash, &name)? + .unwrap_or((total_count, unread_count)); + event_bus + .publish( + &session.user_hash, + MailEvent::FolderStatus { + folder: name.clone(), + total_count, + unread_count, + }, + ) + .await; + event_bus + .publish( + &session.user_hash, + MailEvent::FolderAllRead { + folder: name, + through_uid, + }, + ) + .await; - Ok(Json(MessageResponse { status: "read" }).into_response()) + Ok(Json(FolderCounterResponse { + status: "read", + total_count, + unread_count, + through_uid, + }) + .into_response()) } pub async fn empty_folder( @@ -214,21 +295,16 @@ pub async fn empty_folder( Extension(config): Extension>, Extension(imap_client): Extension>, Extension(search_engine): Extension>, + Extension(event_bus): Extension>, Path(name): Path, ) -> Result { let creds = build_creds(&session, &config)?; - imap_client - .empty_folder(&creds, &name) - .await - .map_err(|e| AppError::ServiceUnavailable(format!("IMAP empty-folder failed: {e}")))?; + let counter_snapshot = folder_counts::snapshot(&config, &session.user_hash, &name)?; - let conn = db::pool::open_user_db(&config.data_dir, &session.user_hash) - .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; - let transaction = conn - .unchecked_transaction() - .map_err(|e| AppError::InternalError(format!("Database transaction error: {e}")))?; - let uids: Vec = { - let mut statement = transaction + let cached_uids: Vec = { + let conn = db::pool::open_user_db(&config.data_dir, &session.user_hash) + .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; + let mut statement = conn .prepare(crate::dbq!(routes_folder_mgmt::Q_003)) .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; let rows = statement @@ -236,16 +312,41 @@ pub async fn empty_folder( .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; rows.filter_map(Result::ok).collect() }; - db::messages::delete_folder_messages(&transaction, &name) + + imap_client + .empty_folder(&creds, &name) + .await + .map_err(|e| AppError::ServiceUnavailable(format!("IMAP empty-folder failed: {e}")))?; + + let authoritative_counts = folder_counts::reconcile_after_mutation( + &config, + &session.user_hash, + imap_client.as_ref(), + &creds, + &name, + counter_snapshot, + ) + .await?; + + let conn = db::pool::open_user_db(&config.data_dir, &session.user_hash) .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; - transaction - .execute(crate::dbq!(routes_folder_mgmt::Q_004), [&name]) + let transaction = conn + .unchecked_transaction() + .map_err(|e| AppError::InternalError(format!("Database transaction error: {e}")))?; + + for uid in &cached_uids { + db::messages::delete_message(&transaction, &name, *uid) + .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; + } + db::header_sync::invalidate(&transaction) + .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; + db::folders::invalidate_folder_freshness(&transaction, &name) .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; if let Ok(index) = search_engine.open_user_index(&session.user_hash) { - for uid in uids { + for uid in &cached_uids { index - .delete_message_with_connection(&transaction, uid, &name) + .delete_message_with_connection(&transaction, *uid, &name) .map_err(|e| AppError::InternalError(format!("Search outbox error: {e}")))?; } } @@ -253,6 +354,38 @@ pub async fn empty_folder( .commit() .map_err(|e| AppError::InternalError(format!("Database commit error: {e}")))?; + let (total_count, unread_count) = if let Some(counts) = authoritative_counts { + counts + } else if counter_snapshot.is_some() { + folder_counts::set_if_snapshot_matches( + &config, + &session.user_hash, + &name, + counter_snapshot, + 0, + 0, + )? + } else { + folder_counts::current_counts(&config, &session.user_hash, &name)?.unwrap_or((0, 0)) + }; + + let (total_count, unread_count) = + folder_counts::current_counts(&config, &session.user_hash, &name)? + .unwrap_or((total_count, unread_count)); + event_bus + .publish( + &session.user_hash, + MailEvent::FolderStatus { + folder: name.clone(), + total_count, + unread_count, + }, + ) + .await; + event_bus + .publish(&session.user_hash, MailEvent::FolderUpdated) + .await; + Ok(Json(MessageResponse { status: "emptied" }).into_response()) } diff --git a/src/routes/folders.rs b/src/routes/folders.rs index e0a4299..110462a 100644 --- a/src/routes/folders.rs +++ b/src/routes/folders.rs @@ -1,8 +1,9 @@ use std::sync::Arc; +use axum::extract::Query; use axum::response::{IntoResponse, Response}; use axum::{Extension, Json}; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use crate::auth::session::SessionState; use crate::config::AppConfig; @@ -13,6 +14,8 @@ use crate::imap::client::{ImapClient, ImapCredentials}; #[derive(Serialize)] struct FoldersResponse { folders: Vec, + pending_sync_count: u32, + pending_index_count: u32, } #[derive(Serialize)] @@ -27,10 +30,23 @@ struct FolderEntry { const FOLDER_LIST_TTL_SECS: u32 = 300; +fn is_noselect(flags: &str) -> bool { + flags + .split(',') + .any(|flag| flag.trim().eq_ignore_ascii_case("\\Noselect")) +} + +#[derive(Debug, Default, Deserialize)] +pub(crate) struct ListFoldersQuery { + #[serde(default)] + refresh: bool, +} + pub async fn list_folders( Extension(session): Extension, Extension(config): Extension>, Extension(imap_client): Extension>, + Query(query): Query, ) -> Result { let conn = db::pool::open_user_db(&config.data_dir, &session.user_hash) .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; @@ -38,16 +54,16 @@ pub async fn list_folders( let cache_fresh = db::folders::is_folder_cache_fresh(&conn, FOLDER_LIST_TTL_SECS) .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; - if !cache_fresh { - let creds = ImapCredentials { - host: session.imap_host.clone(), - port: session.imap_port, - tls: session.imap_tls, - tls_verify: session.imap_tls_verify, - email: session.email.clone(), - password: session.password.clone(), - }; + let creds = ImapCredentials { + host: session.imap_host.clone(), + port: session.imap_port, + tls: session.imap_tls, + tls_verify: session.imap_tls_verify, + email: session.email.clone(), + password: session.password.clone(), + }; + if !cache_fresh { let imap_folders = imap_client .list_folders(&creds) .await @@ -72,9 +88,84 @@ pub async fn list_folders( .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; } - let cached = db::folders::get_all_folders(&conn) + let mut cached = db::folders::get_all_folders(&conn) .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; + if query.refresh || !cache_fresh { + let names = cached + .iter() + .filter(|folder| !is_noselect(&folder.flags)) + .map(|folder| folder.name.clone()) + .collect::>(); + for folder in cached.iter().filter(|folder| is_noselect(&folder.flags)) { + db::folders::set_folder_counts(&conn, &folder.name, 0, 0) + .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; + } + let mut statuses = imap_client + .folder_statuses_extended(&creds, &names) + .await + .unwrap_or_default() + .into_iter() + .collect::>(); + + for name in &names { + let status = if let Some(status) = statuses.remove(name) { + Some(status) + } else { + match imap_client.folder_status_extended(&creds, name).await { + Ok(status) => Some(status), + Err(error) => { + tracing::debug!(folder = %name, error = %error, "Folder STATUS refresh failed; keeping cached counters"); + None + } + } + }; + let Some(status) = status else { continue }; + let cached_folder = cached + .iter() + .find(|folder| folder.name.as_str() == name.as_str()); + if let Some(cached_folder) = cached_folder + && cached_folder.uid_validity != 0 + && status.uid_validity != 0 + && cached_folder.uid_validity != status.uid_validity + { + db::folders::invalidate_folder_freshness(&conn, name) + .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; + } + + if let Some(cached_folder) = cached_folder { + let applied = db::folders::set_folder_counts_if_unchanged( + &conn, + name, + cached_folder.counter_generation, + status.exists, + status.unseen, + ) + .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; + if !applied { + tracing::debug!(folder = %name, "Skipped stale folder STATUS counters"); + } + } else { + db::folders::set_folder_counts(&conn, name, status.exists, status.unseen) + .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; + } + } + cached = db::folders::get_all_folders(&conn) + .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; + } + + let pending_sync_count = db::folders::pending_message_sync_count(&conn) + .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; + let account_deep_index = db::display_preferences::get_preferences(&conn) + .map_err(|e| AppError::InternalError(format!("Database error: {e}")))? + .deep_index; + let pending_index_count = if config.search.deep_index || account_deep_index { + db::messages::count_unindexed_messages(&conn) + .map_err(|e| AppError::InternalError(format!("Database error: {e}")))? + } else { + 0 + }; + let folders: Vec = cached .into_iter() .map(|f| { @@ -94,5 +185,10 @@ pub async fn list_folders( }) .collect(); - Ok(Json(FoldersResponse { folders }).into_response()) + Ok(Json(FoldersResponse { + folders, + pending_sync_count, + pending_index_count, + }) + .into_response()) } diff --git a/src/routes/messages/mod.rs b/src/routes/messages/mod.rs index 4208011..9c502b1 100644 --- a/src/routes/messages/mod.rs +++ b/src/routes/messages/mod.rs @@ -16,7 +16,8 @@ use crate::db::header_sync::HeaderSync; use crate::email_theme; use crate::error::AppError; use crate::imap::client::{ImapClient, ImapCredentials}; -use crate::realtime::events::{EventBus, MailEvent}; +use crate::realtime::events::{EventBus, MailEvent, MessageFlagChange}; +use crate::routes::folder_counts; use crate::search::engine::{IndexableMessage, SearchEngine, UserIndex}; fn build_creds(session: &SessionState, _config: &AppConfig) -> Result { @@ -85,47 +86,58 @@ pub async fn list_messages( let database = db::pool::UserDb::new(&config.data_dir, &session.user_hash); let cached_folder_name = folder.clone(); - let (folder_fresh, cached_uid_validity, cached_count, folder_invalidated, mut header_sync) = - database - .run(move |conn| { - let header_sync = HeaderSync::capture(conn)?; - let fresh = db::folders::is_folder_fresh( + let ( + folder_fresh, + cached_uid_validity, + cached_count, + cached_server_total, + cached_counter_generation, + cached_highest_modseq, + folder_invalidated, + mut header_sync, + ) = database + .run(move |conn| { + let header_sync = HeaderSync::capture(conn)?; + let fresh = + db::folders::is_folder_fresh(conn, &cached_folder_name, FOLDER_MESSAGES_TTL_SECS)?; + let cached = db::folders::get_folder(conn, &cached_folder_name)?; + let validity = cached.as_ref().map(|f| f.uid_validity).unwrap_or(0); + let server_total = cached.as_ref().map(|f| f.total_count).unwrap_or(0); + let counter_generation = cached.as_ref().map(|f| f.counter_generation).unwrap_or(0); + let highest_modseq = cached.as_ref().map(|f| f.highest_modseq).unwrap_or(0); + if cached.is_none() { + db::folders::upsert_folder( conn, &cached_folder_name, - FOLDER_MESSAGES_TTL_SECS, + None, + None, + "", + true, + 0, + 0, + 0, + 0, )?; - let cached = db::folders::get_folder(conn, &cached_folder_name)?; - let validity = cached.as_ref().map(|f| f.uid_validity).unwrap_or(0); - if cached.is_none() { - db::folders::upsert_folder( - conn, - &cached_folder_name, - None, - None, - "", - true, - 0, - 0, - 0, - 0, - )?; - } - Ok(( - fresh, - validity, - db::messages::count_messages(conn, &cached_folder_name)?, - db::folders::is_folder_messages_invalidated(conn, &cached_folder_name)?, - header_sync, - )) - }) - .await - .map_err(AppError::InternalError)?; + } + Ok(( + fresh, + validity, + db::messages::count_messages(conn, &cached_folder_name)?, + server_total, + counter_generation, + highest_modseq, + db::folders::is_folder_messages_invalidated(conn, &cached_folder_name)?, + header_sync, + )) + }) + .await + .map_err(AppError::InternalError)?; - if query.refresh || !folder_fresh { + if query.refresh || !folder_fresh || folder_invalidated || cached_count < cached_server_total { let creds = build_creds(&session, &config)?; let status = imap_client - .folder_status(&creds, &folder) + .folder_status_extended(&creds, &folder) .await .map_err(|e| AppError::ServiceUnavailable(format!("IMAP error: {e}")))?; @@ -135,28 +147,59 @@ pub async fn list_messages( imap_uid_validity = status.uid_validity, cached_count = cached_count, imap_exists = status.exists, + cached_highest_modseq = cached_highest_modseq, + imap_highest_modseq = status.highest_modseq, "list_messages: folder status check" ); let needs_full_sync = cached_uid_validity != 0 && cached_uid_validity != status.uid_validity; + let modseq_changed = !needs_full_sync + && cached_highest_modseq > 0 + && status.highest_modseq > 0 + && cached_highest_modseq != status.highest_modseq; + + if !needs_full_sync { + let name = folder.clone(); + let expected_generation = cached_counter_generation; + let total = status.exists; + let unread = status.unseen; + database + .run(move |conn| { + db::folders::set_folder_counts_if_unchanged( + conn, + &name, + expected_generation, + total, + unread, + ) + .map(|_| ()) + }) + .await + .map_err(AppError::InternalError)?; + } if needs_full_sync { tracing::info!(folder = %folder, "UIDVALIDITY changed, clearing cache"); let name = folder.clone(); + let uid_validity = status.uid_validity; + let exists = status.exists; + let highest_modseq = status.highest_modseq; + let unseen = status.unseen; header_sync = database .run(move |conn| { let transaction = header_sync.begin_write(conn)?; - db::messages::delete_folder_messages(conn, &name)?; + db::messages::delete_folder_messages(&transaction, &name)?; db::header_sync::invalidate(&transaction)?; db::folders::update_folder_sync_status( - conn, + &transaction, &name, - status.uid_validity, - status.exists, - 0, + uid_validity, + exists, + highest_modseq, )?; - let next = HeaderSync::capture(conn)?; + db::folders::set_unread_count(&transaction, &name, unseen)?; + let next = HeaderSync::capture(&transaction)?; transaction.commit().map_err(|e| e.to_string())?; Ok(next) }) @@ -167,7 +210,8 @@ pub async fn list_messages( let needs_sync = needs_full_sync || cached_count == 0 || status.exists != cached_count - || folder_invalidated; + || folder_invalidated + || modseq_changed; if needs_sync { if (needs_full_sync || cached_count == 0) && status.exists > 0 { @@ -187,14 +231,18 @@ pub async fn list_messages( let engine = search_engine.clone(); let hash = session.user_hash.clone(); let validity = status.uid_validity; - let exists = status.exists; + let highest_modseq = status.highest_modseq; database .run(move |conn| { let transaction = header_sync.begin_write(conn)?; upsert_and_index_headers(&transaction, &name, &headers, &engine, &hash) .map_err(|e| e.to_string())?; - db::folders::update_folder_status(conn, &name, validity, exists)?; - db::folders::refresh_unread_count(conn, &name)?; + db::folders::update_folder_sync_cursor( + &transaction, + &name, + validity, + highest_modseq, + )?; transaction.commit().map_err(|e| e.to_string()) }) .await @@ -213,7 +261,7 @@ pub async fn list_messages( uid_range: remaining_range, header_sync, uid_validity: status.uid_validity, - exists: status.exists, + highest_modseq: status.highest_modseq, })); syncing = true; } @@ -231,21 +279,24 @@ pub async fn list_messages( uid_range, header_sync, uid_validity: status.uid_validity, - exists: status.exists, + highest_modseq: status.highest_modseq, })); syncing = true; } } else { let name = folder.clone(); + let uid_validity = status.uid_validity; + let highest_modseq = status.highest_modseq; database .run(move |conn| { let transaction = header_sync.begin_write(conn)?; - db::folders::update_folder_status( - conn, + db::folders::update_folder_sync_cursor( + &transaction, &name, - status.uid_validity, - status.exists, + uid_validity, + highest_modseq, )?; + db::folders::mark_folder_messages_fresh(&transaction, &name)?; transaction.commit().map_err(|e| e.to_string()) }) .await @@ -450,7 +501,7 @@ struct BackgroundSyncParams { uid_range: (u32, u32), header_sync: HeaderSync, uid_validity: u32, - exists: u32, + highest_modseq: u64, } async fn sync_remaining_headers(params: BackgroundSyncParams) { @@ -465,35 +516,62 @@ async fn sync_remaining_headers(params: BackgroundSyncParams) { uid_range, header_sync, uid_validity, - exists, + highest_modseq, } = params; - let result: Result<(), String> = crate::imap::pool::background(async { + let result: Result< + ( + Option, + Vec, + bool, + ), + String, + > = crate::imap::pool::background(async { let database = db::pool::UserDb::new(&config.data_dir, &user_hash); + let mut server_state = imap_client + .fetch_uids_and_flags(&creds, &folder) + .await + .map_err(|e| format!("IMAP error: {e}"))?; + for (_, flags) in &mut server_state { + flags.sort(); + flags.dedup(); + } + + let name = folder.clone(); + let cached_state = database + .run(move |conn| db::messages::get_all_uids_and_flags(conn, &name)) + .await?; + let cached_uids = cached_state + .iter() + .map(|(uid, _)| *uid) + .collect::>(); + let mut uids = if uid_range.0 <= uid_range.1 { - imap_client - .fetch_uids_and_flags(&creds, &folder) - .await - .map_err(|e| format!("IMAP error: {e}"))? - .into_iter() - .map(|(uid, _)| uid) + server_state + .iter() + .map(|(uid, _)| *uid) .filter(|uid| (uid_range.0..=uid_range.1).contains(uid)) + .filter(|uid| !cached_uids.contains(uid)) .collect::>() } else { Vec::new() }; - let name = folder.clone(); - let cached_uids = database - .run(move |conn| { - Ok(db::messages::get_all_uids_and_flags(conn, &name)? - .into_iter() - .map(|(uid, _)| uid) - .collect::>()) - }) - .await?; - uids.retain(|uid| !cached_uids.contains(uid)); uids.sort_unstable(); uids.dedup(); + + event_bus + .publish( + &user_hash, + MailEvent::SyncStatus { + active: true, + folder: Some(folder.clone()), + pending: uids.len() as u32, + }, + ) + .await; + + let mut pending_headers = uids.len() as u32; for uid_set in header_chunks(&uids) { + let chunk_len = uid_set.split(',').count() as u32; let headers = imap_client .fetch_headers(&creds, &folder, &uid_set) .await @@ -509,27 +587,219 @@ async fn sync_remaining_headers(params: BackgroundSyncParams) { transaction.commit().map_err(|e| e.to_string()) }) .await?; + pending_headers = pending_headers.saturating_sub(chunk_len); + event_bus + .publish( + &user_hash, + MailEvent::SyncStatus { + active: true, + folder: Some(folder.clone()), + pending: pending_headers, + }, + ) + .await; } + + let expected_counts_name = folder.clone(); + let expected_counts = database + .run(move |conn| { + Ok( + db::folders::get_folder(conn, &expected_counts_name)?.map(|state| { + ( + state.total_count, + state.unread_count, + state.counter_generation, + ) + }), + ) + }) + .await?; + let final_status = imap_client + .folder_status_extended(&creds, &folder) + .await + .ok(); let name = folder.clone(); - database + let status_for_db = final_status.clone(); + let server_flags = server_state + .into_iter() + .collect::>>(); + let engine = search_engine.clone(); + let hash = user_hash.clone(); + let (flag_changes, structural_change) = database .run(move |conn| { let transaction = header_sync.begin_write(conn)?; - db::folders::update_folder_status(conn, &name, uid_validity, exists)?; - db::folders::refresh_unread_count(conn, &name)?; - transaction.commit().map_err(|e| e.to_string()) + let sync_keywords = db::mail_preferences::get_preferences(&transaction) + .map(|prefs| prefs.sync_imap_keywords) + .unwrap_or(false); + let current = db::messages::get_all_uids_and_flags(&transaction, &name)?; + let user_index = engine.open_user_index(&hash).ok(); + let mut flag_changes = Vec::new(); + let mut structural_change = false; + + for (uid, cached_flags_csv) in current { + let Some(server_flags_for_uid) = server_flags.get(&uid) else { + db::messages::delete_message(&transaction, &name, uid)?; + if let Some(index) = user_index.as_ref() { + index.delete_message_with_connection(&transaction, uid, &name)?; + } + structural_change = true; + continue; + }; + + let mut cached_flags = cached_flags_csv + .split(',') + .filter(|flag| !flag.is_empty()) + .map(str::to_string) + .collect::>(); + cached_flags.sort(); + cached_flags.dedup(); + if cached_flags.as_slice() != server_flags_for_uid.as_slice() { + db::messages::update_message_flags( + &transaction, + &name, + uid, + &server_flags_for_uid.join(","), + )?; + flag_changes.push(MessageFlagChange { + uid, + flags: server_flags_for_uid.clone(), + }); + } + if sync_keywords { + db::tags::import_imap_keywords( + &transaction, + uid, + &name, + server_flags_for_uid, + )?; + } + } + + if let Some(status) = status_for_db { + if uid_validity == 0 || status.uid_validity == uid_validity { + db::folders::update_folder_sync_cursor( + &transaction, + &name, + status.uid_validity, + highest_modseq, + )?; + } else { + db::folders::invalidate_folder_freshness(&transaction, &name)?; + } + + if let Some((_expected_total, _expected_unread, expected_generation)) = + expected_counts + { + let _ = db::folders::set_folder_counts_if_unchanged( + &transaction, + &name, + expected_generation, + status.exists, + status.unseen, + )?; + } else { + db::folders::set_folder_counts( + &transaction, + &name, + status.exists, + status.unseen, + )?; + } + + let cached_after = db::messages::count_messages(&transaction, &name)?; + if (uid_validity == 0 || status.uid_validity == uid_validity) + && cached_after == status.exists + { + db::folders::mark_folder_messages_fresh(&transaction, &name)?; + } else { + db::folders::invalidate_folder_freshness(&transaction, &name)?; + } + } else { + db::folders::invalidate_folder_freshness(&transaction, &name)?; + } + transaction.commit().map_err(|e| e.to_string())?; + Ok((flag_changes, structural_change)) }) - .await + .await?; + + Ok((final_status, flag_changes, structural_change)) }) .await; match result { - Ok(()) => { + Ok((_final_status, flag_changes, structural_change)) => { tracing::info!(folder = %folder, "Background sync: complete"); + if !flag_changes.is_empty() { + event_bus + .publish( + &user_hash, + MailEvent::FlagsChanged { + folder: folder.clone(), + changes: flag_changes, + }, + ) + .await; + } + if structural_change { + event_bus + .publish(&user_hash, MailEvent::FolderUpdated) + .await; + } + + let status_db = db::pool::UserDb::new(&config.data_dir, &user_hash); + let status_name = folder.clone(); + if let Ok(Some(folder_state)) = status_db + .run(move |conn| db::folders::get_folder(conn, &status_name)) + .await + { + event_bus + .publish( + &user_hash, + MailEvent::FolderStatus { + folder: folder.clone(), + total_count: folder_state.total_count, + unread_count: folder_state.unread_count, + }, + ) + .await; + } event_bus - .publish(&user_hash, MailEvent::FolderUpdated) + .publish( + &user_hash, + MailEvent::SyncStatus { + active: false, + folder: Some(folder.clone()), + pending: 0, + }, + ) .await; + let database = db::pool::UserDb::new(&config.data_dir, &user_hash); + if let Ok(pending) = database.run(db::folders::pending_message_sync_count).await { + event_bus + .publish(&user_hash, MailEvent::SyncInventory { pending }) + .await; + } } Err(e) => { + event_bus + .publish( + &user_hash, + MailEvent::SyncStatus { + active: false, + folder: Some(folder.clone()), + pending: 0, + }, + ) + .await; + let inventory_db = db::pool::UserDb::new(&config.data_dir, &user_hash); + if let Ok(pending) = inventory_db + .run(db::folders::pending_message_sync_count) + .await + { + event_bus + .publish(&user_hash, MailEvent::SyncInventory { pending }) + .await; + } tracing::warn!(folder = %folder, error = %e, "Background sync: failed (will retry on next request)"); let database = db::pool::UserDb::new(&config.data_dir, &user_hash); let name = folder.clone(); @@ -865,14 +1135,20 @@ pub async fn update_flags( Extension(session): Extension, Extension(config): Extension>, Extension(imap_client): Extension>, + Extension(event_bus): Extension>, Path((folder, uid)): Path<(String, u32)>, Json(body): Json, ) -> Result { validate_flags(&body.flags)?; let creds = build_creds(&session, &config)?; - let flag_refs: Vec<&str> = body.flags.iter().map(|s| s.as_str()).collect(); + let seen_changed = body.flags.iter().any(|flag| flag == "\\Seen"); + let counter_snapshot = if seen_changed { + folder_counts::snapshot(&config, &session.user_hash, &folder)? + } else { + None + }; if body.add { imap_client @@ -886,25 +1162,41 @@ pub async fn update_flags( .map_err(|e| AppError::ServiceUnavailable(format!("IMAP error: {e}")))?; } + let authoritative_counts = if seen_changed { + folder_counts::reconcile_after_mutation( + &config, + &session.user_hash, + imap_client.as_ref(), + &creds, + &folder, + counter_snapshot, + ) + .await? + } else { + None + }; + let conn = db::pool::open_user_db(&config.data_dir, &session.user_hash) .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; - let current_flags_csv: String = conn + let cached_flags: Option = conn .query_row( crate::dbq!(routes_messages_mod::Q_001), crate::db::pool::params![&folder, uid], |row| row.get(0), ) - .unwrap_or_default(); - + .ok(); + let current_flags_csv = cached_flags.clone().unwrap_or_default(); let mut current_flags: Vec = if current_flags_csv.is_empty() { vec![] } else { current_flags_csv .split(',') - .map(|s| s.to_string()) + .filter(|flag| !flag.is_empty()) + .map(|flag| flag.to_string()) .collect() }; + let was_seen = current_flags.iter().any(|flag| flag == "\\Seen"); if body.add { for flag in &body.flags { @@ -913,17 +1205,77 @@ pub async fn update_flags( } } } else { - current_flags.retain(|f| !body.flags.contains(f)); + current_flags.retain(|flag| !body.flags.contains(flag)); + } + current_flags.sort(); + current_flags.dedup(); + let is_seen = current_flags.iter().any(|flag| flag == "\\Seen"); + + db::messages::update_message_flags(&conn, &folder, uid, ¤t_flags.join(",")) + .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; + + if authoritative_counts.is_none() + && seen_changed + && cached_flags.is_some() + && was_seen != is_seen + && let Some((total_count, unread_count, _generation)) = counter_snapshot + { + let next_unread = if is_seen { + unread_count.saturating_sub(1) + } else { + unread_count.saturating_add(1) + }; + let _ = folder_counts::set_if_snapshot_matches( + &config, + &session.user_hash, + &folder, + counter_snapshot, + total_count, + next_unread, + )?; } - let new_flags_csv = current_flags.join(","); - db::messages::update_message_flags(&conn, &folder, uid, &new_flags_csv) + let folder_state = db::folders::get_folder(&conn, &folder) .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; + let total_count = folder_state + .as_ref() + .map(|state| state.total_count) + .unwrap_or(0); + let unread_count = folder_state + .as_ref() + .map(|state| state.unread_count) + .unwrap_or(0); - db::folders::refresh_unread_count(&conn, &folder) - .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; + event_bus + .publish( + &session.user_hash, + MailEvent::FlagsChanged { + folder: folder.clone(), + changes: vec![MessageFlagChange { + uid, + flags: current_flags.clone(), + }], + }, + ) + .await; + event_bus + .publish( + &session.user_hash, + MailEvent::FolderStatus { + folder: folder.clone(), + total_count, + unread_count, + }, + ) + .await; - Ok(Json(serde_json::json!({ "status": "ok" })).into_response()) + Ok(Json(serde_json::json!({ + "status": "ok", + "flags": current_flags, + "total_count": total_count, + "unread_count": unread_count, + })) + .into_response()) } pub async fn move_message_handler( @@ -931,6 +1283,7 @@ pub async fn move_message_handler( Extension(config): Extension>, Extension(imap_client): Extension>, Extension(search_engine): Extension>, + Extension(event_bus): Extension>, Json(body): Json, ) -> Result { if body.from_folder == body.to_folder { @@ -940,6 +1293,10 @@ pub async fn move_message_handler( } let creds = build_creds(&session, &config)?; + let source_counter_snapshot = + folder_counts::snapshot(&config, &session.user_hash, &body.from_folder)?; + let destination_counter_snapshot = + folder_counts::snapshot(&config, &session.user_hash, &body.to_folder)?; let source_snapshot = { let conn = db::pool::open_user_db(&config.data_dir, &session.user_hash) @@ -947,11 +1304,6 @@ pub async fn move_message_handler( db::messages::get_single_message(&conn, &body.from_folder, body.uid) .map_err(|e| AppError::InternalError(format!("Database error: {e}")))? }; - let was_unread = source_snapshot - .as_ref() - .map(|message| !message.flags.contains("\\Seen")) - .unwrap_or(false); - let destination_uid_floor = if source_snapshot .as_ref() .and_then(|message| message.message_id.as_deref()) @@ -971,6 +1323,27 @@ pub async fn move_message_handler( .await .map_err(|e| AppError::ServiceUnavailable(format!("IMAP error: {e}")))?; + let (source_counts, destination_counts) = tokio::join!( + folder_counts::reconcile_after_mutation( + &config, + &session.user_hash, + imap_client.as_ref(), + &creds, + &body.from_folder, + source_counter_snapshot, + ), + folder_counts::reconcile_after_mutation( + &config, + &session.user_hash, + imap_client.as_ref(), + &creds, + &body.to_folder, + destination_counter_snapshot, + ), + ); + let source_counts = source_counts?; + let destination_counts = destination_counts?; + let undo_destination_uid = if let (Some(uid_floor), Some(message_id)) = ( destination_uid_floor, source_snapshot @@ -1023,20 +1396,37 @@ pub async fn move_message_handler( .map_err(|e| AppError::InternalError(format!("Search outbox error: {e}")))?; } - db::folders::refresh_unread_count(&transaction, &body.from_folder) - .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; - - if was_unread { - db::folders::adjust_unread_count(&transaction, &body.to_folder, 1) - .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; - } - db::folders::invalidate_folder_freshness(&transaction, &body.to_folder) .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; transaction .commit() .map_err(|e| AppError::InternalError(format!("Database commit error: {e}")))?; + let source_counts = + folder_counts::current_counts(&config, &session.user_hash, &body.from_folder)? + .or(source_counts); + let destination_counts = + folder_counts::current_counts(&config, &session.user_hash, &body.to_folder)? + .or(destination_counts); + + for (folder_name, counts) in [ + (&body.from_folder, source_counts), + (&body.to_folder, destination_counts), + ] { + if let Some((total_count, unread_count)) = counts { + event_bus + .publish( + &session.user_hash, + MailEvent::FolderStatus { + folder: folder_name.to_string(), + total_count, + unread_count, + }, + ) + .await; + } + } + let undo_move = undo_destination_uid.map(|destination_uid| { serde_json::json!({ "from_folder": body.to_folder, @@ -1052,6 +1442,7 @@ pub async fn copy_message_handler( Extension(session): Extension, Extension(config): Extension>, Extension(imap_client): Extension>, + Extension(event_bus): Extension>, Json(body): Json, ) -> Result { if body.from_folder == body.to_folder { @@ -1060,15 +1451,43 @@ pub async fn copy_message_handler( )); } let creds = build_creds(&session, &config)?; + let destination_counter_snapshot = + folder_counts::snapshot(&config, &session.user_hash, &body.to_folder)?; imap_client .copy_message(&creds, &body.from_folder, body.uid, &body.to_folder) .await .map_err(|e| AppError::ServiceUnavailable(format!("IMAP error: {e}")))?; + let destination_counts = folder_counts::reconcile_after_mutation( + &config, + &session.user_hash, + imap_client.as_ref(), + &creds, + &body.to_folder, + destination_counter_snapshot, + ) + .await?; + let conn = db::pool::open_user_db(&config.data_dir, &session.user_hash) .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; db::folders::invalidate_folder_freshness(&conn, &body.to_folder) .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; + let destination_counts = + folder_counts::current_counts(&config, &session.user_hash, &body.to_folder)? + .or(destination_counts); + + if let Some((total_count, unread_count)) = destination_counts { + event_bus + .publish( + &session.user_hash, + MailEvent::FolderStatus { + folder: body.to_folder.clone(), + total_count, + unread_count, + }, + ) + .await; + } Ok(Json(serde_json::json!({ "status": "ok" })).into_response()) } @@ -1310,22 +1729,46 @@ pub async fn delete_message_handler( Extension(session): Extension, Extension(config): Extension>, Extension(imap_client): Extension>, + Extension(event_bus): Extension>, Path((folder, uid)): Path<(String, u32)>, ) -> Result { let creds = build_creds(&session, &config)?; + let counter_snapshot = folder_counts::snapshot(&config, &session.user_hash, &folder)?; imap_client .expunge_message(&creds, &folder, uid) .await .map_err(|e| AppError::ServiceUnavailable(format!("IMAP error: {e}")))?; + let authoritative_counts = folder_counts::reconcile_after_mutation( + &config, + &session.user_hash, + imap_client.as_ref(), + &creds, + &folder, + counter_snapshot, + ) + .await?; + let conn = db::pool::open_user_db(&config.data_dir, &session.user_hash) .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; db::messages::remove_after_mutation(&conn, &folder, uid) .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; + let authoritative_counts = folder_counts::current_counts(&config, &session.user_hash, &folder)? + .or(authoritative_counts); - db::folders::refresh_unread_count(&conn, &folder) - .map_err(|e| AppError::InternalError(format!("Database error: {e}")))?; + if let Some((total_count, unread_count)) = authoritative_counts { + event_bus + .publish( + &session.user_hash, + MailEvent::FolderStatus { + folder: folder.clone(), + total_count, + unread_count, + }, + ) + .await; + } Ok(Json(serde_json::json!({ "status": "ok" })).into_response()) } @@ -1335,6 +1778,34 @@ mod sync_tests { use super::*; use crate::imap::client::mock::MockImapClient; + fn drain_events(events: &mut tokio::sync::broadcast::Receiver) -> Vec { + let mut drained = Vec::new(); + while let Ok(event) = events.try_recv() { + drained.push(event); + } + drained + } + + async fn wait_for_sync_completion( + events: &mut tokio::sync::broadcast::Receiver, + folder: &str, + ) { + tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + match events.recv().await.unwrap() { + MailEvent::SyncStatus { + active: false, + folder: Some(event_folder), + .. + } if event_folder == folder => break, + _ => {} + } + } + }) + .await + .expect("background header sync did not complete in time"); + } + fn sync_header(uid: u32) -> crate::imap::types::ImapMessageHeader { crate::imap::types::ImapMessageHeader { uid, @@ -1430,11 +1901,11 @@ mod sync_tests { config: config.clone(), user_hash: user_hash.into(), search_engine: search_engine.clone(), - event_bus, + event_bus: event_bus.clone(), uid_range: (1, 100), header_sync, uid_validity: 1, - exists: 1, + highest_modseq: 0, })); gate.0.notified().await; @@ -1445,6 +1916,7 @@ mod sync_tests { Extension(config), Extension(imap_client), Extension(search_engine.clone()), + Extension(event_bus.clone()), Json(MoveMessageRequest { uid: 7, from_folder: "INBOX".into(), @@ -1458,6 +1930,7 @@ mod sync_tests { Extension(session), Extension(config), Extension(imap_client), + Extension(event_bus.clone()), Path(("INBOX".into(), 7)), ) .await @@ -1474,11 +1947,13 @@ mod sync_tests { .is_none() ); assert!(db::folders::is_folder_messages_invalidated(&conn, "INBOX").unwrap()); + let emitted = drain_events(&mut events); assert!( - matches!(events.try_recv().unwrap(), MailEvent::FolderUpdated), + emitted + .iter() + .any(|event| matches!(event, MailEvent::FolderUpdated)), "superseded sync must request fresh missing-UID reconciliation" ); - assert!(events.try_recv().is_err()); if move_message { let index = search_engine.open_user_index(user_hash).unwrap(); @@ -1560,7 +2035,7 @@ mod sync_tests { user_hash, ) .unwrap(); - db::folders::update_folder_status(&conn, "INBOX", 1, 1000).unwrap(); + db::folders::update_folder_status(&transaction, "INBOX", 1, 1000).unwrap(); transaction.commit().unwrap(); drop(conn); let creds = ImapCredentials { @@ -1612,7 +2087,7 @@ mod sync_tests { uid_range: (1, 900), header_sync, uid_validity: 1, - exists: 1000, + highest_modseq: 0, })); gate.0.notified().await; let folder = if other_folder { "Archive" } else { "INBOX" }; @@ -1620,6 +2095,7 @@ mod sync_tests { Extension(session.clone()), Extension(config.clone()), Extension(client.clone() as Arc), + Extension(bus.clone()), Path((folder.into(), 950)), ) .await @@ -1634,10 +2110,13 @@ mod sync_tests { .is_none() ); drop(conn); - assert!(matches!( - events.recv().await.unwrap(), - MailEvent::FolderUpdated - )); + let emitted = drain_events(&mut events); + assert!( + emitted + .iter() + .any(|event| matches!(event, MailEvent::FolderUpdated)), + "superseded backfill must request a fresh reconciliation" + ); let remaining = (1..=1000) .filter(|uid| other_folder || *uid != 950) @@ -1661,7 +2140,7 @@ mod sync_tests { .await .unwrap(); assert_eq!(response.status(), axum::http::StatusCode::OK); - events.recv().await.unwrap(); + wait_for_sync_completion(&mut events, "INBOX").await; let conn = db::pool::open_user_db(path, user_hash).unwrap(); assert_eq!( db::messages::count_messages(&conn, "INBOX").unwrap(), @@ -1716,10 +2195,10 @@ mod sync_tests { db::messages::remove_after_mutation(&conn, "INBOX", 7).unwrap(); let before_reset = HeaderSync::capture(&conn).unwrap(); let transaction = before_reset.begin_write(&conn).unwrap(); - db::messages::delete_folder_messages(&conn, "INBOX").unwrap(); + db::messages::delete_folder_messages(&transaction, "INBOX").unwrap(); db::header_sync::invalidate(&transaction).unwrap(); - db::folders::update_folder_sync_status(&conn, "INBOX", 2, 1, 0).unwrap(); - let new_namespace = HeaderSync::capture(&conn).unwrap(); + db::folders::update_folder_sync_status(&transaction, "INBOX", 2, 1, 0).unwrap(); + let new_namespace = HeaderSync::capture(&transaction).unwrap(); transaction.commit().unwrap(); assert!(stale.begin_write(&conn).is_err()); assert!(before_reset.begin_write(&conn).is_err()); @@ -1776,7 +2255,15 @@ mod sync_tests { let header_sync = HeaderSync::capture(&conn).unwrap(); drop(conn); - let client = Arc::new(MockImapClient::new()); + let client = Arc::new(MockImapClient::new().with_folder_status_extended( + crate::imap::types::FolderStatusExtended { + uid_validity: 42, + exists: 0, + uid_next: 1, + unseen: 0, + highest_modseq: 0, + }, + )); let config = Arc::new(AppConfig { data_dir: path.to_owned(), ..serde_json::from_str::("{}").unwrap() @@ -1802,7 +2289,7 @@ mod sync_tests { uid_range, header_sync, uid_validity: 42, - exists: 0, + highest_modseq: 0, }) .await; @@ -1814,11 +2301,23 @@ mod sync_tests { assert!(db::folders::is_folder_fresh(&conn, "Sent", 30).unwrap()); assert!(!db::folders::is_folder_messages_invalidated(&conn, "Sent").unwrap()); assert!(client.header_requests.lock().unwrap().is_empty()); - assert!(matches!( - events.try_recv().unwrap(), - MailEvent::FolderUpdated - )); - assert!(events.try_recv().is_err()); + let emitted = drain_events(&mut events); + assert!(emitted.iter().any(|event| matches!( + event, + MailEvent::FolderStatus { + folder, + total_count: 0, + unread_count: 0, + } if folder == "Sent" + ))); + assert!(emitted.iter().any(|event| matches!( + event, + MailEvent::SyncStatus { + active: false, + folder: Some(folder), + pending: 0, + } if folder == "Sent" + ))); } } @@ -1872,7 +2371,7 @@ mod sync_tests { uid_range: (1, 9_999_900), header_sync, uid_validity: 1, - exists: 3, + highest_modseq: 0, }) .await; @@ -1880,10 +2379,15 @@ mod sync_tests { *client.header_requests.lock().unwrap(), vec![("INBOX".into(), "1,9999899".into())] ); - assert!(matches!( - events.try_recv().unwrap(), - MailEvent::FolderUpdated - )); + let emitted = drain_events(&mut events); + assert!(emitted.iter().any(|event| matches!( + event, + MailEvent::SyncStatus { + active: false, + folder: Some(folder), + pending: 0, + } if folder == "INBOX" + ))); let conn = db::pool::open_user_db(path, user_hash).unwrap(); assert!( db::messages::get_single_message(&conn, "INBOX", 1) diff --git a/src/routes/mod.rs b/src/routes/mod.rs index 14fd507..4f416cb 100644 --- a/src/routes/mod.rs +++ b/src/routes/mod.rs @@ -6,6 +6,7 @@ pub mod contacts; pub mod diagnostics; pub mod display_preferences; pub mod drafts; +pub mod folder_counts; pub mod folder_mgmt; pub mod folder_preferences; pub mod folders; @@ -22,6 +23,7 @@ pub mod send; pub mod settings; pub mod sieve; pub mod special_folders; +pub mod system_diagnostics; pub mod tags; pub mod trusted_senders; pub mod unsubscribe; @@ -32,7 +34,7 @@ use std::time::Duration; use axum::body::Body; use axum::extract::{ConnectInfo, DefaultBodyLimit}; -use axum::http::{HeaderValue, Request}; +use axum::http::{HeaderValue, Request, header}; use axum::response::Response; use axum::routing::{delete, get, patch, post, put}; use axum::{Extension, Router, middleware}; @@ -104,29 +106,70 @@ impl KeyExtractor for ProxyAwareIpExtractor { } } +fn response_content_type_is(response: &Response, expected: &str) -> bool { + response + .headers() + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.split(';').next()) + .is_some_and(|value| value.trim().eq_ignore_ascii_case(expected)) +} + async fn add_security_headers(request: Request, next: middleware::Next) -> Response { + let request_path = request.uri().path().to_owned(); + let is_api = + request_path == "/api" || request_path.ends_with("/api") || request_path.contains("/api/"); let mut response = next.run(request).await; + let is_html = response_content_type_is(&response, "text/html") + || response_content_type_is(&response, "application/xhtml+xml"); + let is_css = response_content_type_is(&response, "text/css"); let headers = response.headers_mut(); + headers.insert( "x-content-type-options", HeaderValue::from_static("nosniff"), ); - headers.insert("x-frame-options", HeaderValue::from_static("DENY")); - headers.insert("referrer-policy", HeaderValue::from_static("no-referrer")); - headers.insert( - "permissions-policy", - HeaderValue::from_static("camera=(), microphone=(), geolocation=()"), - ); - headers.insert( - "cross-origin-opener-policy", - HeaderValue::from_static("same-origin"), - ); - headers.insert( - "content-security-policy", - HeaderValue::from_static( - "default-src 'self'; base-uri 'self'; object-src 'none'; frame-ancestors 'none'; form-action 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob: http: https:; font-src 'self' data:; connect-src 'self' ws: wss:", - ), - ); + + if is_api { + headers.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); + headers.insert( + "cross-origin-resource-policy", + HeaderValue::from_static("same-origin"), + ); + headers.insert("x-frame-options", HeaderValue::from_static("DENY")); + headers.insert( + "content-security-policy", + HeaderValue::from_static("frame-ancestors 'none'"), + ); + } else if is_html || is_css { + headers.insert("referrer-policy", HeaderValue::from_static("no-referrer")); + } + + if request_path.contains("/assets/") || request_path.ends_with("/favicon.svg") { + headers.insert( + "cross-origin-resource-policy", + HeaderValue::from_static("same-origin"), + ); + } + + if is_html { + headers.insert("x-frame-options", HeaderValue::from_static("DENY")); + headers.insert( + "permissions-policy", + HeaderValue::from_static("camera=(), microphone=(), geolocation=()"), + ); + headers.insert( + "cross-origin-opener-policy", + HeaderValue::from_static("same-origin"), + ); + headers.insert( + "content-security-policy", + HeaderValue::from_static( + "default-src 'self'; base-uri 'self'; object-src 'none'; frame-ancestors 'none'; form-action 'self'; script-src 'self'; script-src-attr 'none'; style-src 'self'; style-src-attr 'unsafe-inline'; img-src 'self' data: blob: http: https:; font-src 'self' data:; connect-src 'self' ws: wss:; frame-src 'self' blob:", + ), + ); + } + response } @@ -176,6 +219,8 @@ pub fn create_router_with_storage(config: Arc, services: RouterServic event_bus, idle_manager, } = services; + let system_diagnostics_telemetry = + Arc::new(system_diagnostics::SystemDiagnosticsTelemetry::new()); let governor_conf = GovernorConfigBuilder::default() .key_extractor(ProxyAwareIpExtractor { trusted_proxies: config.parsed_trusted_proxies(), @@ -200,6 +245,7 @@ pub fn create_router_with_storage(config: Arc, services: RouterServic let protected_auth = Router::new() .route("/session", get(auth::get_session)) + .route("/session-policy", get(auth::session_policy)) .route("/sessions", get(auth::list_active_sessions)) .route("/sessions/revoke-others", post(auth::revoke_other_sessions)) .route("/sessions/{id}", delete(auth::revoke_active_session)) @@ -532,13 +578,27 @@ pub fn create_router_with_storage(config: Arc, services: RouterServic let ws_route = Router::new().route("/ws", get(crate::realtime::ws::ws_handler)); - let api_router = Router::new() + let mut api_router = Router::new() .route("/health", get(health::health_check)) .route("/ready", get(health::readiness_check)) .nest("/auth", auth_router) .merge(ws_route) - .merge(protected_data) - .fallback(|| async { axum::http::StatusCode::NOT_FOUND }); + .merge(protected_data); + + if config.diagnostics_endpoint.enabled { + let diagnostics_path = config + .diagnostics_endpoint + .path + .trim() + .strip_prefix("/api") + .unwrap_or(config.diagnostics_endpoint.path.trim()); + api_router = api_router.route( + diagnostics_path, + get(system_diagnostics::system_diagnostics), + ); + } + + let api_router = api_router.fallback(|| async { axum::http::StatusCode::NOT_FOUND }); let inner = Router::new() .nest("/api", api_router) @@ -555,6 +615,24 @@ pub fn create_router_with_storage(config: Arc, services: RouterServic } }; + let router = if config.diagnostics_endpoint.enabled { + let system_diagnostics_dependencies = + Arc::new(system_diagnostics::SystemDiagnosticsDependencies::new( + config.clone(), + store.clone(), + search_engine.clone(), + storage.clone(), + idle_manager.clone(), + system_diagnostics_telemetry.clone(), + )); + router + .layer(middleware::from_fn(system_diagnostics::observe_requests)) + .layer(Extension(system_diagnostics_dependencies)) + .layer(Extension(system_diagnostics_telemetry)) + } else { + router + }; + let router = router .layer(Extension(idle_manager)) .layer(Extension(event_bus)) diff --git a/src/routes/pgp.rs b/src/routes/pgp.rs index 08490ed..e3619f1 100644 --- a/src/routes/pgp.rs +++ b/src/routes/pgp.rs @@ -378,10 +378,6 @@ pub async fn export_key( HeaderValue::from_static("application/pgp-keys; charset=utf-8"), ); headers.insert("cache-control", HeaderValue::from_static("no-store")); - headers.insert( - "x-content-type-options", - HeaderValue::from_static("nosniff"), - ); headers.insert( "content-disposition", HeaderValue::from_str(&format!("attachment; filename=\"{filename}\"")) diff --git a/src/routes/system_diagnostics.rs b/src/routes/system_diagnostics.rs new file mode 100644 index 0000000..5795130 --- /dev/null +++ b/src/routes/system_diagnostics.rs @@ -0,0 +1,896 @@ +use std::collections::{HashMap, VecDeque}; +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use axum::body::Body; +use axum::extract::ConnectInfo; +use axum::http::{HeaderMap, HeaderValue, Request, StatusCode, header}; +use axum::middleware; +use axum::response::{IntoResponse, Response}; +use axum::{Extension, Json}; +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD; +use serde::Serialize; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; + +use crate::auth::session::SessionStore; +use crate::config::{AppConfig, TrustedProxy}; +use crate::db; +use crate::realtime::idle::IdleManager; +use crate::search::engine::SearchEngine; +use crate::storage::BlobStorage; + +const MAX_SECURITY_EVENTS: usize = 4096; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize)] +#[serde(rename_all = "snake_case")] +enum SecurityEventKind { + LoginRejected, + RateLimited, + Forbidden, + ServerError, + DiagnosticsDenied, +} + +#[derive(Debug, Clone)] +struct SecurityEvent { + at_unix: u64, + kind: SecurityEventKind, + source_ip: IpAddr, + path: String, +} + +pub struct SystemDiagnosticsTelemetry { + started_at_unix: u64, + started: Instant, + total_requests: AtomicU64, + client_errors: AtomicU64, + server_errors: AtomicU64, + login_attempts: AtomicU64, + events: Mutex>, +} + +pub struct SystemDiagnosticsDependencies { + config: Arc, + store: Arc, + search: Arc, + storage: Arc, + idle_manager: Arc, + telemetry: Arc, +} + +impl SystemDiagnosticsDependencies { + pub fn new( + config: Arc, + store: Arc, + search: Arc, + storage: Arc, + idle_manager: Arc, + telemetry: Arc, + ) -> Self { + Self { + config, + store, + search, + storage, + idle_manager, + telemetry, + } + } +} + +impl SystemDiagnosticsTelemetry { + pub fn new() -> Self { + Self { + started_at_unix: now_unix(), + started: Instant::now(), + total_requests: AtomicU64::new(0), + client_errors: AtomicU64::new(0), + server_errors: AtomicU64::new(0), + login_attempts: AtomicU64::new(0), + events: Mutex::new(VecDeque::with_capacity(MAX_SECURITY_EVENTS)), + } + } + + fn record_event(&self, kind: SecurityEventKind, source_ip: IpAddr, path: &str) { + let mut events = self + .events + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if events.len() >= MAX_SECURITY_EVENTS { + events.pop_front(); + } + events.push_back(SecurityEvent { + at_unix: now_unix(), + kind, + source_ip, + path: path.chars().take(256).collect(), + }); + } + + fn security_snapshot(&self, window_secs: u64) -> Value { + let now = now_unix(); + let cutoff = now.saturating_sub(window_secs); + let events = self + .events + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let mut by_kind: HashMap = HashMap::new(); + let mut by_ip: HashMap = HashMap::new(); + let mut recent = Vec::new(); + for event in events.iter().filter(|event| event.at_unix >= cutoff) { + *by_kind.entry(event.kind).or_default() += 1; + *by_ip.entry(event.source_ip).or_default() += 1; + } + for event in events + .iter() + .rev() + .filter(|event| event.at_unix >= cutoff) + .take(50) + { + recent.push(json!({ + "timestampUnix": event.at_unix, + "kind": event.kind, + "sourceIp": event.source_ip.to_string(), + "path": event.path, + })); + } + let mut top_sources: Vec<(IpAddr, u64)> = by_ip.into_iter().collect(); + top_sources.sort_by_key(|source| std::cmp::Reverse(source.1)); + top_sources.truncate(20); + + let login_rejected = *by_kind.get(&SecurityEventKind::LoginRejected).unwrap_or(&0); + let rate_limited = *by_kind.get(&SecurityEventKind::RateLimited).unwrap_or(&0); + let forbidden = *by_kind.get(&SecurityEventKind::Forbidden).unwrap_or(&0); + let server_errors = *by_kind.get(&SecurityEventKind::ServerError).unwrap_or(&0); + let diagnostics_denied = *by_kind + .get(&SecurityEventKind::DiagnosticsDenied) + .unwrap_or(&0); + + let brute_force_signal = login_rejected >= 10 || rate_limited >= 3; + let diagnostics_attack_signal = diagnostics_denied >= 5; + json!({ + "windowSeconds": window_secs, + "loginRejected": login_rejected, + "rateLimited": rate_limited, + "forbidden": forbidden, + "serverErrors": server_errors, + "diagnosticsDenied": diagnostics_denied, + "bruteForceSignal": brute_force_signal, + "diagnosticsAttackSignal": diagnostics_attack_signal, + "securityAlert": brute_force_signal || diagnostics_attack_signal, + "topSources": top_sources.into_iter().map(|(ip, count)| json!({ + "ip": ip.to_string(), "events": count + })).collect::>(), + "recentEvents": recent, + "bufferCapacity": MAX_SECURITY_EVENTS, + "bufferUsed": events.len(), + }) + } +} + +impl Default for SystemDiagnosticsTelemetry { + fn default() -> Self { + Self::new() + } +} + +fn now_unix() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} + +fn trusted_contains(trusted: &[TrustedProxy], ip: &IpAddr) -> bool { + trusted.iter().any(|entry| entry.contains(ip)) +} + +fn effective_client_ip(config: &AppConfig, headers: &HeaderMap, peer_ip: IpAddr) -> IpAddr { + let trusted = config.parsed_trusted_proxies(); + if !trusted_contains(&trusted, &peer_ip) { + return peer_ip; + } + + let mut current = peer_ip; + for value in headers.get_all("x-forwarded-for").iter().rev() { + let Ok(value) = value.to_str() else { + return current; + }; + for entry in value.split(',').rev() { + if !trusted_contains(&trusted, ¤t) { + return current; + } + let Ok(parsed) = entry.trim().parse::() else { + return current; + }; + current = parsed; + } + } + current +} + +fn diagnostics_client_ip( + config: &AppConfig, + headers: &HeaderMap, + peer_ip: IpAddr, +) -> Result { + let has_xff = headers.contains_key("x-forwarded-for"); + let has_other_forwarding = + headers.contains_key("forwarded") || headers.contains_key("x-real-ip"); + if !has_xff && !has_other_forwarding { + return Ok(peer_ip); + } + + let trusted = config.parsed_trusted_proxies(); + if !trusted_contains(&trusted, &peer_ip) { + return Err("proxy headers received from an untrusted peer"); + } + if !has_xff { + return Err("diagnostics requires X-Forwarded-For when accessed through a trusted proxy"); + } + + let mut current = peer_ip; + let mut saw_forwarded_address = false; + for value in headers.get_all("x-forwarded-for").iter().rev() { + let value = value + .to_str() + .map_err(|_| "invalid X-Forwarded-For header encoding")?; + for entry in value.split(',').rev() { + if !trusted_contains(&trusted, ¤t) { + return Ok(current); + } + current = entry + .trim() + .parse::() + .map_err(|_| "invalid X-Forwarded-For address")?; + saw_forwarded_address = true; + } + } + if !saw_forwarded_address { + return Err("empty X-Forwarded-For header"); + } + Ok(current) +} + +pub async fn observe_requests(request: Request, next: middleware::Next) -> Response { + let telemetry = request + .extensions() + .get::>() + .cloned(); + let config = request.extensions().get::>().cloned(); + let path = request.uri().path().to_string(); + let peer_ip = request + .extensions() + .get::>() + .map(|value| value.0.ip()) + .unwrap_or(IpAddr::V4(Ipv4Addr::LOCALHOST)); + let source_ip = config + .as_deref() + .map(|cfg| effective_client_ip(cfg, request.headers(), peer_ip)) + .unwrap_or(peer_ip); + let is_login = path.ends_with("/api/auth/login"); + + if let Some(telemetry) = telemetry.as_ref() { + telemetry.total_requests.fetch_add(1, Ordering::Relaxed); + if is_login { + telemetry.login_attempts.fetch_add(1, Ordering::Relaxed); + } + } + + let response = next.run(request).await; + if let Some(telemetry) = telemetry.as_ref() { + let status = response.status(); + if status.is_client_error() { + telemetry.client_errors.fetch_add(1, Ordering::Relaxed); + } + if status.is_server_error() { + telemetry.server_errors.fetch_add(1, Ordering::Relaxed); + telemetry.record_event(SecurityEventKind::ServerError, source_ip, &path); + } + if status == StatusCode::TOO_MANY_REQUESTS { + telemetry.record_event(SecurityEventKind::RateLimited, source_ip, &path); + tracing::warn!(source_ip = %source_ip, path = %path, "request rate limited"); + } + if status == StatusCode::FORBIDDEN { + telemetry.record_event(SecurityEventKind::Forbidden, source_ip, &path); + } + if is_login && matches!(status, StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN) { + telemetry.record_event(SecurityEventKind::LoginRejected, source_ip, &path); + tracing::warn!(source_ip = %source_ip, status = %status, "login rejected"); + } + } + response +} + +fn constant_time_secret_eq(expected: &str, supplied: &str) -> bool { + let expected = Sha256::digest(expected.as_bytes()); + let supplied = Sha256::digest(supplied.as_bytes()); + expected + .iter() + .zip(supplied.iter()) + .fold(0u8, |diff, (left, right)| diff | (left ^ right)) + == 0 +} + +fn bearer_token(headers: &HeaderMap) -> Option<&str> { + let authorization = headers + .get(header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .and_then(|value| { + let mut parts = value.split_whitespace(); + let scheme = parts.next()?; + let credentials = parts.next()?; + if parts.next().is_some() { + return None; + } + scheme.eq_ignore_ascii_case("bearer").then_some(credentials) + }); + authorization.or_else(|| { + headers + .get("x-rustbox-diagnostics-token") + .and_then(|value| value.to_str().ok()) + .map(str::trim) + }) +} + +fn basic_credentials(headers: &HeaderMap) -> Option<(String, String)> { + let raw = headers.get(header::AUTHORIZATION)?.to_str().ok()?; + let mut parts = raw.split_whitespace(); + let scheme = parts.next()?; + let encoded = parts.next()?; + if parts.next().is_some() || !scheme.eq_ignore_ascii_case("basic") { + return None; + } + let decoded = STANDARD.decode(encoded).ok()?; + let decoded = String::from_utf8(decoded).ok()?; + let (username, password) = decoded.split_once(':')?; + Some((username.to_string(), password.to_string())) +} + +fn harden_diagnostics_response(mut response: Response) -> Response { + let headers = response.headers_mut(); + headers.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); + headers.insert("pragma", HeaderValue::from_static("no-cache")); + headers.insert( + "x-content-type-options", + HeaderValue::from_static("nosniff"), + ); + headers.insert( + "cross-origin-resource-policy", + HeaderValue::from_static("same-origin"), + ); + response +} + +fn unauthorized(auth_mode: &str) -> Response { + let challenge = if auth_mode.eq_ignore_ascii_case("basic") { + "Basic realm=\"RustBox diagnostics\", charset=\"UTF-8\"" + } else { + "Bearer realm=\"RustBox diagnostics\"" + }; + let mut response = ( + StatusCode::UNAUTHORIZED, + Json(json!({"error": "diagnostics authentication required"})), + ) + .into_response(); + response.headers_mut().insert( + header::WWW_AUTHENTICATE, + HeaderValue::from_str(challenge).unwrap_or_else(|_| HeaderValue::from_static("Bearer")), + ); + harden_diagnostics_response(response) +} + +fn forbidden() -> Response { + harden_diagnostics_response( + ( + StatusCode::FORBIDDEN, + Json(json!({"error": "diagnostics endpoint is not available from this network"})), + ) + .into_response(), + ) +} + +fn sanitize_url(raw: Option<&str>) -> Option { + let raw = raw?.trim(); + if raw.is_empty() { + return None; + } + if let Ok(url) = url::Url::parse(raw) { + if url.scheme() == "sqlite" { + return Some(format!("sqlite:{}", url.path())); + } + let host = url.host_str().unwrap_or(""); + let port = url + .port() + .map(|value| format!(":{value}")) + .unwrap_or_default(); + let path = if url.path() == "/" { "" } else { url.path() }; + return Some(format!("{}://{}{}{}", url.scheme(), host, port, path)); + } + Some("".to_string()) +} + +fn process_snapshot() -> Value { + // Keep this intentionally limited to portable, process-local information. + // Rust std does not expose cross-platform RSS/thread/FD metrics, and the + // diagnostics endpoint must not depend on host-specific /proc probes. + json!({ + "pid": std::process::id(), + }) +} + +fn sanitized_config(config: &AppConfig) -> Value { + json!({ + "environment": config.environment, + "pathsAndLogging": { + "dataDir": config.data_dir, + "logDir": config.log_dir, + "logLevel": config.log_level, + }, + "ui": { + "defaultTheme": config.default_theme, + "defaultInterfaceSkin": config.default_interface_skin, + }, + "listen": {"host": config.host, "port": config.port, "basePath": config.base_path}, + "mail": { + "imapHost": config.imap_host, + "imapPort": config.imap_port, + "smtpHost": config.smtp_host, + "smtpPort": config.smtp_port, + "tlsEnabled": config.tls_enabled, + "tlsVerifyOverride": config.tls_verify, + "sieveEnabled": config.sieve_enabled, + "sieveHost": config.sieve_host, + "sievePort": config.sieve_port, + "sieveTlsMode": config.sieve_tls_mode, + "sieveAllowInsecureAuth": config.sieve_allow_insecure_auth, + "sieveTimeoutSecs": config.sieve_timeout_secs, + }, + "limits": { + "maxAttachmentSizeMiB": config.max_attachment_size_mb, + "maxMessageSizeMiB": config.max_message_size_mb, + "sessionTimeoutHours": config.session_timeout_hours, + "rememberMeEnabled": config.remember_me_enabled, + "rememberMeDays": config.remember_me_days, + }, + "database": { + "backend": config.database_backend, + "url": sanitize_url(config.database_url.as_deref()), + }, + "sessions": { + "backend": config.session_backend, + "redisUrl": sanitize_url(config.redis_url.as_deref()), + "redisSentinelNodes": config.redis_sentinel_nodes.iter().map(|value| sanitize_url(Some(value)).unwrap_or_else(|| value.clone())).collect::>(), + "redisSentinelMaster": config.redis_sentinel_master, + "redisPasswordConfigured": config.redis_password.as_deref().is_some_and(|value| !value.is_empty()), + "redisSentinelPasswordConfigured": config.redis_sentinel_password.as_deref().is_some_and(|value| !value.is_empty()), + "sessionEncryptionKeyConfigured": config.session_encryption_key.as_deref().is_some_and(|value| !value.is_empty()), + }, + "clusterMode": config.cluster_mode, + "storage": { + "backend": config.storage.backend, + "path": config.storage.path, + "shared": config.storage.shared, + "bucket": config.storage.bucket, + "region": config.storage.region, + "endpoint": sanitize_url(config.storage.endpoint.as_deref()), + "prefix": config.storage.prefix, + "pathStyle": config.storage.path_style, + "allowHttp": config.storage.allow_http, + "accessKeyConfigured": config.storage.access_key_id.as_deref().is_some_and(|value| !value.is_empty()), + "secretKeyConfigured": config.storage.secret_access_key.as_deref().is_some_and(|value| !value.is_empty()), + "sessionTokenConfigured": config.storage.session_token.as_deref().is_some_and(|value| !value.is_empty()), + "requestTimeoutSecs": config.storage.request_timeout_secs, + "connectTimeoutSecs": config.storage.connect_timeout_secs, + "maxRetries": config.storage.max_retries, + "retryTimeoutSecs": config.storage.retry_timeout_secs, + "serverSideEncryption": config.storage.server_side_encryption, + "kmsKeyConfigured": config.storage.kms_key_id.as_deref().is_some_and(|value| !value.is_empty()), + }, + "search": { + "backend": config.search.backend, + "path": config.search.path, + "urls": config.search.urls.iter().map(|value| sanitize_url(Some(value)).unwrap_or_else(|| "".to_string())).collect::>(), + "indexPrefix": config.search.index_prefix, + "usernameConfigured": config.search.username.as_deref().is_some_and(|value| !value.is_empty()), + "passwordConfigured": config.search.password.as_deref().is_some_and(|value| !value.is_empty()), + "required": config.search.required, + "allowInsecureHttp": config.search.allow_insecure_http, + "requestTimeoutSecs": config.search.request_timeout_secs, + "deepIndex": config.search.deep_index, + "workerIntervalSecs": config.search.worker_interval_secs, + "workerBatchSize": config.search.worker_batch_size, + "workerMaxAttempts": config.search.worker_max_attempts, + }, + "sync": { + "intervalSecs": config.sync.interval_secs, + "deepIndexBatchSize": config.sync.deep_index_batch_size, + }, + "features": { + "userDiagnostics": config.diagnostics_enabled, + "about": config.about_enabled, + "pgp": config.pgp_enabled, + "securityHeaders": config.security_headers_enabled, + }, + "pgp": { + "enabled": config.pgp_enabled, + "wkdEnabled": config.pgp_wkd_enabled, + "hkpEnabled": config.pgp_hkp_enabled, + "hkpUrl": sanitize_url(Some(&config.pgp_hkp_url)), + "networkTimeoutSecs": config.pgp_network_timeout_secs, + "maxKeyBytes": config.pgp_max_key_bytes, + "allowPrivateExport": config.pgp_allow_private_export, + }, + "specialFolders": { + "sent": config.default_sent_folder, + "drafts": config.default_drafts_folder, + "trash": config.default_trash_folder, + "junk": config.default_junk_folder, + "archive": config.default_archive_folder, + }, + "outgoingHeaders": { + "count": config.outgoing_headers.len(), + "names": config.outgoing_headers.iter().filter_map(|entry| entry.split_once(':').map(|(name, _)| name.trim().to_string())).collect::>(), + }, + "trustedProxies": config.trusted_proxies, + "diagnosticsEndpoint": { + "enabled": config.diagnostics_endpoint.enabled, + "path": config.diagnostics_endpoint.path, + "allowedNetworks": config.diagnostics_endpoint.allowed_networks, + "auth": config.diagnostics_endpoint.auth, + "credentialConfigured": if config.diagnostics_endpoint.auth.eq_ignore_ascii_case("basic") { + config.diagnostics_endpoint.basic_username.as_deref().is_some_and(|value| !value.is_empty()) + && config.diagnostics_endpoint.basic_password.as_deref().is_some_and(|value| !value.is_empty()) + } else { + config.diagnostics_endpoint.token.as_deref().is_some_and(|value| !value.is_empty()) + }, + "includeUserIdentifiers": config.diagnostics_endpoint.include_user_identifiers, + "probeTimeoutSecs": config.diagnostics_endpoint.probe_timeout_secs, + "securityWindowSecs": config.diagnostics_endpoint.security_window_secs, + } + }) +} + +fn configuration_checks(config: &AppConfig) -> Vec { + let mut checks = Vec::new(); + if config.environment.eq_ignore_ascii_case("production") && !config.security_headers_enabled { + checks.push(json!({"severity":"warning","code":"security_headers_disabled","message":"Browser security headers are disabled in production."})); + } + if config.search.allow_insecure_http { + checks.push(json!({"severity":"warning","code":"opensearch_http_allowed","message":"OpenSearch permits plain HTTP."})); + } + if config.storage.allow_http { + checks.push(json!({"severity":"warning","code":"storage_http_allowed","message":"S3-compatible storage permits plain HTTP."})); + } + if config + .diagnostics_endpoint + .allowed_networks + .iter() + .any(|network| matches!(network.trim(), "0.0.0.0/0" | "::/0")) + { + checks.push(json!({"severity":"warning","code":"diagnostics_allowlist_global","message":"Diagnostics allowlist contains a global network; restrict it at the application and firewall/reverse-proxy layers."})); + } + if config + .diagnostics_endpoint + .auth + .eq_ignore_ascii_case("basic") + { + checks.push(json!({"severity":"info","code":"diagnostics_basic_auth","message":"Diagnostics uses Basic Auth; transport must be protected by HTTPS/TLS at RustBox or its trusted reverse proxy."})); + } + if config.diagnostics_endpoint.include_user_identifiers { + checks.push(json!({"severity":"info","code":"diagnostics_user_identifiers","message":"Diagnostics may expose authenticated user e-mail addresses and stable user hashes to authorized operators."})); + } + if config.search.deep_index { + checks.push(json!({"severity":"info","code":"deep_index_enabled","message":"Server-wide deep index is enabled; full message bodies may be fetched and stored/indexed automatically."})); + } + if config.tls_verify == Some(false) { + checks.push(json!({"severity":"warning","code":"tls_verification_disabled","message":"TLS certificate verification is globally disabled for configured mail connections."})); + } + checks +} + +async fn timed_probe(timeout: Duration, future: F) -> (Result, u128) +where + F: std::future::Future>, +{ + let started = Instant::now(); + let result = match tokio::time::timeout(timeout, future).await { + Ok(result) => result, + Err(_) => Err(format!("probe timed out after {} ms", timeout.as_millis())), + }; + (result, started.elapsed().as_millis()) +} + +pub async fn system_diagnostics( + ConnectInfo(peer): ConnectInfo, + headers: HeaderMap, + Extension(dependencies): Extension>, +) -> Response { + let config = dependencies.config.as_ref(); + let store = dependencies.store.as_ref(); + let search = dependencies.search.as_ref(); + let storage = dependencies.storage.as_ref(); + let idle_manager = dependencies.idle_manager.as_ref(); + let telemetry = dependencies.telemetry.as_ref(); + let source_ip = match diagnostics_client_ip(config, &headers, peer.ip()) { + Ok(source_ip) => source_ip, + Err(_) => { + telemetry.record_event( + SecurityEventKind::DiagnosticsDenied, + peer.ip(), + &config.diagnostics_endpoint.path, + ); + tracing::warn!(peer_ip = %peer.ip(), "administrator diagnostics proxy/source validation denied"); + return forbidden(); + } + }; + let allowed = config + .parsed_diagnostics_allowed_networks() + .map(|networks| networks.iter().any(|network| network.contains(&source_ip))) + .unwrap_or(false); + if !allowed { + telemetry.record_event( + SecurityEventKind::DiagnosticsDenied, + source_ip, + &config.diagnostics_endpoint.path, + ); + tracing::warn!(source_ip = %source_ip, "administrator diagnostics denied by network allowlist"); + return forbidden(); + } + + let authenticated = if config + .diagnostics_endpoint + .auth + .eq_ignore_ascii_case("basic") + { + match ( + config.diagnostics_endpoint.basic_username.as_deref(), + config.diagnostics_endpoint.basic_password.as_deref(), + basic_credentials(&headers), + ) { + (Some(expected_user), Some(expected_password), Some((user, password))) => { + constant_time_secret_eq(expected_user, &user) + && constant_time_secret_eq(expected_password, &password) + } + _ => false, + } + } else { + match ( + config.diagnostics_endpoint.token.as_deref(), + bearer_token(&headers), + ) { + (Some(expected), Some(supplied)) => constant_time_secret_eq(expected, supplied), + _ => false, + } + }; + if !authenticated { + telemetry.record_event( + SecurityEventKind::DiagnosticsDenied, + source_ip, + &config.diagnostics_endpoint.path, + ); + tracing::warn!(source_ip = %source_ip, auth_mode = %config.diagnostics_endpoint.auth, "administrator diagnostics authentication rejected"); + return unauthorized(&config.diagnostics_endpoint.auth); + } + + let timeout = Duration::from_secs(config.diagnostics_endpoint.probe_timeout_secs); + let db_future = async { Ok::<_, String>(db::pool::diagnostics().await) }; + let storage_future = async { storage.health().await }; + let search_future = async { search.diagnostics().await }; + let sessions_future = store.persistence_diagnostics(); + let (db_probe, storage_probe, search_probe, session_probe) = tokio::join!( + timed_probe(timeout, db_future), + timed_probe(timeout, storage_future), + timed_probe(timeout, search_future), + timed_probe(timeout, sessions_future), + ); + + let db_diag = db_probe + .0 + .unwrap_or_else(|error| db::pool::DatabaseDiagnostics { + backend: config.database_backend.clone(), + status: "error".to_string(), + latency_ms: db_probe.1, + pool_size: None, + idle_connections: None, + database_bytes: None, + engine_version: None, + account_count: None, + schema_version_min: None, + schema_version_max: None, + schema_aligned: None, + expected_schema_version: db::pool::latest_schema_version(), + error: Some(error), + }); + let storage_status = match storage_probe.0 { + Ok(()) => json!({"status":"ok","latencyMs":storage_probe.1}), + Err(error) => json!({"status":"error","latencyMs":storage_probe.1,"error":error}), + }; + let search_status = match search_probe.0 { + Ok(details) => { + let detail_status = details + .get("status") + .and_then(Value::as_str) + .unwrap_or("ok"); + let status = if detail_status == "degraded" { + "degraded" + } else { + "ok" + }; + json!({"status":status,"latencyMs":search_probe.1,"backend":search.backend_name(),"required":config.search.required,"details":details}) + } + Err(error) => { + json!({"status": if config.search.required {"error"} else {"degraded"},"latencyMs":search_probe.1,"backend":search.backend_name(),"required":config.search.required,"error":error}) + } + }; + let session_status = match session_probe.0 { + Ok(value) => json!({"status":"ok","latencyMs":session_probe.1,"persistence":value}), + Err(error) => { + json!({"status":"error","latencyMs":session_probe.1,"backend":config.session_backend,"error":error}) + } + }; + + let local_sessions = + store.diagnostics_snapshot(config.diagnostics_endpoint.include_user_identifiers); + let security = telemetry.security_snapshot(config.diagnostics_endpoint.security_window_secs); + let security_alert = security + .get("securityAlert") + .and_then(Value::as_bool) + .unwrap_or(false); + let config_checks = configuration_checks(config); + let critical = db_diag.status != "ok" + || storage_status.get("status").and_then(Value::as_str) != Some("ok") + || session_status.get("status").and_then(Value::as_str) != Some("ok") + || (config.search.required + && search_status.get("status").and_then(Value::as_str) != Some("ok")); + let degraded = !critical + && (security_alert + || db_diag.schema_aligned == Some(false) + || search_status.get("status").and_then(Value::as_str) == Some("degraded") + || config_checks + .iter() + .any(|check| check.get("severity").and_then(Value::as_str) == Some("warning"))); + + let response = json!({ + "status": if critical { "error" } else if degraded { "degraded" } else { "ok" }, + "timestampUnix": now_unix(), + "request": {"sourceIp": source_ip.to_string()}, + "application": { + "name": "RustBox", + "version": env!("CARGO_PKG_VERSION"), + "uptimeSeconds": telemetry.started.elapsed().as_secs(), + "startedAtUnix": telemetry.started_at_unix, + "process": process_snapshot(), + "clusterMode": config.cluster_mode, + }, + "runtimeCounters": { + "totalRequests": telemetry.total_requests.load(Ordering::Relaxed), + "clientErrors": telemetry.client_errors.load(Ordering::Relaxed), + "serverErrors": telemetry.server_errors.load(Ordering::Relaxed), + "loginAttempts": telemetry.login_attempts.load(Ordering::Relaxed), + "idleTasks": idle_manager.task_count(), + "idleSubscribers": idle_manager.subscriber_count(), + "idleConnections": idle_manager.diagnostics_snapshot(config.diagnostics_endpoint.include_user_identifiers), + }, + "configuration": sanitized_config(config), + "checks": config_checks, + "database": db_diag, + "storage": { + "backend": config.effective_storage_backend(), + "health": storage_status, + }, + "search": search_status, + "sessions": { + "backend": config.session_backend, + "health": session_status, + "localNode": local_sessions, + }, + "security": security, + }); + + harden_diagnostics_response( + ( + if critical { + StatusCode::SERVICE_UNAVAILABLE + } else { + StatusCode::OK + }, + Json(response), + ) + .into_response(), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn diagnostics_secret_compare_matches_only_exact_value() { + assert!(constant_time_secret_eq( + "abcdefghijklmnopqrstuvwx", + "abcdefghijklmnopqrstuvwx" + )); + assert!(!constant_time_secret_eq( + "abcdefghijklmnopqrstuvwx", + "abcdefghijklmnopqrstuvwy" + )); + } + + #[test] + fn diagnostics_url_redaction_removes_credentials_and_query() { + assert_eq!( + sanitize_url(Some( + "postgres://user:secret@db.example.test:5432/rustbox?sslmode=require" + )), + Some("postgres://db.example.test:5432/rustbox".to_string()) + ); + assert_eq!( + sanitize_url(Some("redis://:secret@redis.example.test:6379/2")), + Some("redis://redis.example.test:6379/2".to_string()) + ); + } + + #[test] + fn diagnostics_client_ip_ignores_untrusted_forwarded_header() { + let config = AppConfig { + trusted_proxies: Some("127.0.0.1/32".to_string()), + ..AppConfig::default() + }; + let mut headers = HeaderMap::new(); + headers.insert("x-forwarded-for", HeaderValue::from_static("203.0.113.42")); + let peer: IpAddr = "198.51.100.10".parse().unwrap(); + assert_eq!(effective_client_ip(&config, &headers, peer), peer); + } + + #[test] + fn diagnostics_client_ip_uses_forwarded_header_only_from_trusted_proxy() { + let config = AppConfig { + trusted_proxies: Some("127.0.0.1/32".to_string()), + ..AppConfig::default() + }; + let mut headers = HeaderMap::new(); + headers.insert("x-forwarded-for", HeaderValue::from_static("203.0.113.42")); + assert_eq!( + effective_client_ip(&config, &headers, "127.0.0.1".parse().unwrap()), + "203.0.113.42".parse::().unwrap() + ); + } + + #[test] + fn diagnostics_endpoint_rejects_proxy_headers_from_untrusted_peer() { + let config = AppConfig::default(); + let mut headers = HeaderMap::new(); + headers.insert("x-forwarded-for", HeaderValue::from_static("203.0.113.42")); + assert!(diagnostics_client_ip(&config, &headers, "127.0.0.1".parse().unwrap(),).is_err()); + } + + #[test] + fn diagnostics_endpoint_rejects_malformed_xff_from_trusted_proxy() { + let config = AppConfig { + trusted_proxies: Some("127.0.0.1/32".to_string()), + ..AppConfig::default() + }; + let mut headers = HeaderMap::new(); + headers.insert("x-forwarded-for", HeaderValue::from_static("not-an-ip")); + assert!(diagnostics_client_ip(&config, &headers, "127.0.0.1".parse().unwrap(),).is_err()); + } + + #[test] + fn diagnostics_endpoint_accepts_xff_from_configured_trusted_proxy() { + let config = AppConfig { + trusted_proxies: Some("127.0.0.1/32".to_string()), + ..AppConfig::default() + }; + let mut headers = HeaderMap::new(); + headers.insert("x-forwarded-for", HeaderValue::from_static("203.0.113.42")); + assert_eq!( + diagnostics_client_ip(&config, &headers, "127.0.0.1".parse().unwrap()).unwrap(), + "203.0.113.42".parse::().unwrap() + ); + } +} diff --git a/src/routes/tests.rs b/src/routes/tests.rs index d2a2196..2a0949d 100644 --- a/src/routes/tests.rs +++ b/src/routes/tests.rs @@ -12,7 +12,8 @@ use tower_governor::key_extractor::KeyExtractor; use crate::imap::client::mock::MockImapClient; use crate::imap::client::{ - EmailAddress, ImapAttachment, ImapError, ImapFolder, ImapMessageBody, ImapMessageHeader, + EmailAddress, FolderStatusExtended, ImapAttachment, ImapError, ImapFolder, ImapMessageBody, + ImapMessageHeader, }; use crate::smtp::client::SmtpError; use crate::smtp::client::mock::MockSmtpClient; @@ -304,15 +305,18 @@ async fn disabled_optional_features_are_not_registered() { } #[tokio::test] -async fn auth_policy_exposes_optional_feature_switches() { +async fn auth_policy_exposes_only_login_requirements_before_authentication() { let dir = setup_static_dir(); let mut config = (*test_config(dir.path().to_str().unwrap())).clone(); + config.remember_me_enabled = true; config.sieve_enabled = false; config.diagnostics_enabled = false; config.about_enabled = false; + let store = test_store(); + let (browser_id, account_id, token) = setup_test_account(&store, "alice@example.com"); let app = create_router( Arc::new(config), - test_store(), + store, test_imap_client(), test_smtp_client(), test_search_engine("/tmp/oxi-test"), @@ -321,6 +325,7 @@ async fn auth_policy_exposes_optional_feature_switches() { ); let response = app + .clone() .oneshot( Request::builder() .uri("/api/auth/policy") @@ -332,13 +337,52 @@ async fn auth_policy_exposes_optional_feature_switches() { assert_eq!(response.status(), StatusCode::OK); let body = response.into_body().collect().await.unwrap().to_bytes(); let json: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(json["rememberMeEnabled"], true); + for key in [ + "rememberMeDays", + "sessionTimeoutHours", + "pgpEnabled", + "sieveEnabled", + "diagnosticsEnabled", + "aboutEnabled", + ] { + assert!(json.get(key).is_none(), "public policy leaked {key}"); + } + + let unauth = app + .clone() + .oneshot( + Request::builder() + .uri("/api/auth/session-policy") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(unauth.status(), StatusCode::UNAUTHORIZED); + + let mut request = Request::builder() + .uri("/api/auth/session-policy") + .header("x-requested-with", "XMLHttpRequest"); + for (name, value) in auth_headers(&browser_id, &account_id, &token) { + request = request.header(name, value); + } + let response = app + .oneshot(request.body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = response.into_body().collect().await.unwrap().to_bytes(); + let json: serde_json::Value = serde_json::from_slice(&body).unwrap(); assert_eq!(json["sieveEnabled"], false); assert_eq!(json["diagnosticsEnabled"], false); assert_eq!(json["aboutEnabled"], false); + assert!(json.get("rememberMeDays").is_none()); + assert!(json.get("sessionTimeoutHours").is_none()); } #[tokio::test] -async fn security_headers_are_opt_in() { +async fn security_headers_are_opt_in_and_content_type_aware() { let dir = setup_static_dir(); let app = create_router( test_config(dir.path().to_str().unwrap()), @@ -350,43 +394,134 @@ async fn security_headers_are_opt_in() { test_idle_manager(), ); let response = app - .oneshot( - Request::builder() - .uri("/api/health") - .body(Body::empty()) - .unwrap(), - ) + .oneshot(Request::builder().uri("/").body(Body::empty()).unwrap()) .await .unwrap(); assert!(response.headers().get("content-security-policy").is_none()); - - let mut config = (*test_config(dir.path().to_str().unwrap())).clone(); - config.security_headers_enabled = true; - let app = create_router( - Arc::new(config), + assert!(response.headers().get("x-content-type-options").is_none()); + let disabled_api = create_router( + test_config(dir.path().to_str().unwrap()), test_store(), test_imap_client(), test_smtp_client(), test_search_engine("/tmp/oxi-test"), test_event_bus(), test_idle_manager(), - ); - let response = app - .oneshot( - Request::builder() - .uri("/api/health") - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(response.headers()["x-frame-options"], "DENY"); + ) + .oneshot( + Request::builder() + .uri("/api/health") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); assert!( - response.headers()["content-security-policy"] - .to_str() - .unwrap() - .contains("default-src 'self'") + disabled_api + .headers() + .get("x-content-type-options") + .is_none() ); + assert!( + disabled_api + .headers() + .get("cross-origin-resource-policy") + .is_none() + ); + assert!( + disabled_api + .headers() + .get("content-security-policy") + .is_none() + ); + assert!(disabled_api.headers().get("x-frame-options").is_none()); + + let mut config = (*test_config(dir.path().to_str().unwrap())).clone(); + config.security_headers_enabled = true; + let config = Arc::new(config); + + let html = create_router( + config.clone(), + test_store(), + test_imap_client(), + test_smtp_client(), + test_search_engine("/tmp/oxi-test"), + test_event_bus(), + test_idle_manager(), + ) + .oneshot(Request::builder().uri("/").body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(html.headers()["x-content-type-options"], "nosniff"); + assert_eq!(html.headers()["x-frame-options"], "DENY"); + assert_eq!(html.headers()["referrer-policy"], "no-referrer"); + assert_eq!( + html.headers()["permissions-policy"], + "camera=(), microphone=(), geolocation=()" + ); + assert_eq!(html.headers()["cross-origin-opener-policy"], "same-origin"); + let html_csp = html.headers()["content-security-policy"].to_str().unwrap(); + assert!(html_csp.contains("default-src 'self'")); + assert!(html_csp.contains("script-src 'self';")); + assert!(html_csp.contains("script-src-attr 'none'")); + assert!(html_csp.contains("style-src 'self'; style-src-attr 'unsafe-inline'")); + assert!(!html_csp.contains("style-src 'self' 'unsafe-inline'")); + assert!(!html_csp.contains("unsafe-inline'; script-src")); + assert!(!html_csp.contains("script-src 'self' 'unsafe-inline'")); + + let css = create_router( + config.clone(), + test_store(), + test_imap_client(), + test_smtp_client(), + test_search_engine("/tmp/oxi-test"), + test_event_bus(), + test_idle_manager(), + ) + .oneshot( + Request::builder() + .uri("/assets/app.css") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(css.headers()["x-content-type-options"], "nosniff"); + assert_eq!(css.headers()["referrer-policy"], "no-referrer"); + assert_eq!(css.headers()["cross-origin-resource-policy"], "same-origin"); + assert!(css.headers().get("x-frame-options").is_none()); + assert!(css.headers().get("permissions-policy").is_none()); + assert!(css.headers().get("cross-origin-opener-policy").is_none()); + assert!(css.headers().get("content-security-policy").is_none()); + + let api = create_router( + config, + test_store(), + test_imap_client(), + test_smtp_client(), + test_search_engine("/tmp/oxi-test"), + test_event_bus(), + test_idle_manager(), + ) + .oneshot( + Request::builder() + .uri("/api/health") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(api.headers()["x-content-type-options"], "nosniff"); + assert_eq!(api.headers()["cache-control"], "no-store"); + assert_eq!(api.headers()["cross-origin-resource-policy"], "same-origin"); + assert_eq!( + api.headers()["content-security-policy"], + "frame-ancestors 'none'" + ); + assert_eq!(api.headers()["x-frame-options"], "DENY"); + assert!(api.headers().get("referrer-policy").is_none()); + assert!(api.headers().get("permissions-policy").is_none()); + assert!(api.headers().get("cross-origin-opener-policy").is_none()); } #[tokio::test] @@ -945,6 +1080,77 @@ async fn get_folders_returns_200_with_folder_list() { assert_eq!(folders[1]["name"], "Sent"); } +#[tokio::test] +async fn folder_refresh_uses_server_counts_without_relisting_fresh_folder_tree() { + let static_dir = setup_static_dir(); + let data_dir = TempDir::new().unwrap(); + let config = test_config_with_imap( + static_dir.path().to_str().unwrap(), + data_dir.path().to_str().unwrap(), + ); + let store = test_store(); + let (browser_id, account_id, token) = setup_test_account(&store, "alice@example.com"); + let user_hash = crate::auth::user_data::hash_email("alice@example.com"); + provision_user_db(data_dir.path().to_str().unwrap(), &user_hash); + + let conn = crate::db::pool::open_user_db(data_dir.path().to_str().unwrap(), &user_hash) + .expect("open user DB"); + crate::db::folders::upsert_folder( + &conn, + "INBOX", + Some("/"), + None, + "\\HasNoChildren", + true, + 3444, + 1, + 1, + 10, + ) + .unwrap(); + crate::db::folders::touch_all_folders(&conn).unwrap(); + drop(conn); + + let mock = MockImapClient::new().with_folder_status_extended(FolderStatusExtended { + uid_validity: 1, + exists: 3444, + uid_next: 3445, + unseen: 12, + highest_modseq: 11, + }); + let app = create_router( + config, + store, + Arc::new(mock), + test_smtp_client(), + test_search_engine(data_dir.path().to_str().unwrap()), + test_event_bus(), + test_idle_manager(), + ); + + let mut request = Request::builder() + .uri("/api/folders?refresh=true") + .header("x-requested-with", "XMLHttpRequest"); + for (name, value) in auth_headers(&browser_id, &account_id, &token) { + request = request.header(name, value); + } + let response = app + .oneshot(request.body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = response.into_body().collect().await.unwrap().to_bytes(); + let json: serde_json::Value = serde_json::from_slice(&body).unwrap(); + let inbox = json["folders"] + .as_array() + .unwrap() + .iter() + .find(|folder| folder["name"] == "INBOX") + .expect("INBOX remains in the cached folder tree"); + assert_eq!(inbox["total_count"], 3444); + assert_eq!(inbox["unread_count"], 12); +} + #[tokio::test] async fn get_folders_returns_503_when_imap_fails() { let static_dir = setup_static_dir(); diff --git a/src/search/engine.rs b/src/search/engine.rs index 7bf572c..06533a2 100644 --- a/src/search/engine.rs +++ b/src/search/engine.rs @@ -219,6 +219,25 @@ impl SearchEngine { } } + pub async fn diagnostics(&self) -> Result { + match &self.backend { + SearchBackend::Disabled => Ok(json!({ + "backend": "disabled", + "status": "disabled" + })), + SearchBackend::LocalTantivy { base_dir } => { + std::fs::create_dir_all(base_dir) + .map_err(|e| format!("failed to prepare local search directory: {e}"))?; + Ok(json!({ + "backend": "local_tantivy", + "status": "ok", + "path": base_dir, + })) + } + SearchBackend::OpenSearch { backend, .. } => backend.diagnostics().await, + } + } + pub fn open_user_index(&self, user_hash: &str) -> Result { match &self.backend { SearchBackend::Disabled => Err("search index is disabled by configuration".to_string()), @@ -695,9 +714,6 @@ impl LocalUserIndex { impl OpenSearchBackend { fn event_lease_secs(&self) -> u64 { - // A mutation can probe/create the index and then write the document. - // Account for failover across all configured URLs and keep a safety margin so - // another worker cannot reclaim the same document while this request is alive. self.request_timeout_secs .saturating_mul(self.urls.len().max(1) as u64) .saturating_mul(4) @@ -796,6 +812,61 @@ impl OpenSearchBackend { } } + async fn diagnostics(&self) -> Result { + let (status, body) = self + .request(Method::GET, "_cluster/health?local=true", None) + .await?; + if !status.is_success() { + return Err(format!("OpenSearch health check failed ({status}): {body}")); + } + self.ensure_index().await?; + let health: JsonValue = serde_json::from_str(&body) + .map_err(|e| format!("invalid OpenSearch health response: {e}"))?; + let (count_status, count_body) = self + .request(Method::GET, &format!("{}/_count", self.index_name), None) + .await?; + if !count_status.is_success() { + return Err(format!( + "OpenSearch index count failed ({count_status}): {count_body}" + )); + } + let document_count = serde_json::from_str::(&count_body) + .map_err(|e| format!("invalid OpenSearch count response: {e}"))? + .get("count") + .and_then(JsonValue::as_u64) + .ok_or_else(|| { + "OpenSearch count response does not contain a numeric count".to_string() + })?; + let cluster_status = health + .get("status") + .and_then(JsonValue::as_str) + .unwrap_or("unknown"); + if cluster_status.eq_ignore_ascii_case("red") { + return Err("OpenSearch cluster health is red".to_string()); + } + let diagnostics_status = if cluster_status.eq_ignore_ascii_case("green") { + "ok" + } else { + "degraded" + }; + Ok(json!({ + "backend": "opensearch", + "status": diagnostics_status, + "clusterStatus": cluster_status, + "clusterName": health.get("cluster_name").and_then(JsonValue::as_str), + "numberOfNodes": health.get("number_of_nodes").and_then(JsonValue::as_u64), + "numberOfDataNodes": health.get("number_of_data_nodes").and_then(JsonValue::as_u64), + "activePrimaryShards": health.get("active_primary_shards").and_then(JsonValue::as_u64), + "activeShards": health.get("active_shards").and_then(JsonValue::as_u64), + "relocatingShards": health.get("relocating_shards").and_then(JsonValue::as_u64), + "initializingShards": health.get("initializing_shards").and_then(JsonValue::as_u64), + "unassignedShards": health.get("unassigned_shards").and_then(JsonValue::as_u64), + "index": self.index_name, + "documentCount": document_count, + "configuredUrls": self.urls.len(), + })) + } + fn document_id(user_hash: &str, uid: u32, folder: &str) -> String { format!( "{:x}", diff --git a/src/session_store/database.rs b/src/session_store/database.rs index 880d8cf..025fd1a 100644 --- a/src/session_store/database.rs +++ b/src/session_store/database.rs @@ -4,7 +4,7 @@ use async_trait::async_trait; use rusqlite::{Connection, params}; use sqlx::{FromRow, MySqlPool, PgPool}; -use super::{PersistedSession, SessionPersistence}; +use super::{PersistedSession, SessionPersistence, SessionPersistenceDiagnostics}; use crate::config::AppConfig; use crate::queries; @@ -148,11 +148,6 @@ pub struct DatabaseSessionStore { impl DatabaseSessionStore { pub async fn connect(config: &AppConfig) -> Result { - // Schema ownership is centralized in db::pool::configure(). Calling it here - // keeps direct/test construction safe, while the normal application startup - // hits the OnceLock fast path. Do not run schema.sql again from this store: - // that used to duplicate DDL for SQLite/MySQL/PostgreSQL and produced repeated - // PostgreSQL "relation already exists, skipping" NOTICE messages. crate::db::pool::configure(config).await?; let pool = match config.database_backend.as_str() { @@ -425,6 +420,61 @@ impl SessionPersistence for DatabaseSessionStore { } } + async fn diagnostics(&self, now_unix: u64) -> Result { + let (backend, active_sessions, unique_users, unique_browsers, latest_accessed_unix) = + match &self.pool { + Pool::Sqlite(path) => { + let path = path.clone(); + let row = sqlite_task(path, move |conn| { + conn.query_row( + "SELECT COUNT(*), COUNT(DISTINCT user_hash), COUNT(DISTINCT browser_id), MAX(last_accessed_unix) FROM sessions WHERE (last_accessed_unix + timeout_secs) >= ?1", + params![now_unix as i64], + |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?, row.get::<_, i64>(2)?, row.get::<_, Option>(3)?)), + ) + .map_err(|e| format!("failed to query SQLite session diagnostics: {e}")) + }) + .await?; + ("database/sqlite", row.0, row.1, row.2, row.3) + } + Pool::Mysql(pool) => { + let row: (i64, i64, i64, Option) = sqlx::query_as( + "SELECT COUNT(*), COUNT(DISTINCT user_hash), COUNT(DISTINCT browser_id), MAX(last_accessed_unix) FROM sessions WHERE (last_accessed_unix + timeout_secs) >= ?" + ) + .bind(now_unix as i64) + .fetch_one(pool) + .await + .map_err(|e| format!("failed to query MySQL session diagnostics: {e}"))?; + ("database/mysql", row.0, row.1, row.2, row.3) + } + Pool::Pgsql(pool) => { + let row: (i64, i64, i64, Option) = sqlx::query_as( + "SELECT COUNT(*), COUNT(DISTINCT user_hash), COUNT(DISTINCT browser_id), MAX(last_accessed_unix) FROM sessions WHERE (last_accessed_unix + timeout_secs) >= $1" + ) + .bind(now_unix as i64) + .fetch_one(pool) + .await + .map_err(|e| format!("failed to query PostgreSQL session diagnostics: {e}"))?; + ("database/pgsql", row.0, row.1, row.2, row.3) + } + }; + Ok(SessionPersistenceDiagnostics { + backend: backend.to_string(), + status: "ok".to_string(), + active_sessions: active_sessions.max(0) as usize, + unique_users: unique_users.max(0) as usize, + unique_browsers: unique_browsers.max(0) as usize, + latest_accessed_unix: latest_accessed_unix.and_then(|value| u64::try_from(value).ok()), + truncated: false, + engine_version: None, + connected_clients: None, + blocked_clients: None, + used_memory_bytes: None, + server_uptime_secs: None, + role: None, + master_link_status: None, + }) + } + async fn purge_expired(&self, now_unix: u64) -> Result { match &self.pool { Pool::Sqlite(path) => { diff --git a/src/session_store/mod.rs b/src/session_store/mod.rs index 06bab86..6315ed8 100644 --- a/src/session_store/mod.rs +++ b/src/session_store/mod.rs @@ -7,6 +7,24 @@ use serde::{Deserialize, Serialize}; use crate::config::AppConfig; +#[derive(Debug, Clone, Serialize)] +pub struct SessionPersistenceDiagnostics { + pub backend: String, + pub status: String, + pub active_sessions: usize, + pub unique_users: usize, + pub unique_browsers: usize, + pub latest_accessed_unix: Option, + pub truncated: bool, + pub engine_version: Option, + pub connected_clients: Option, + pub blocked_clients: Option, + pub used_memory_bytes: Option, + pub server_uptime_secs: Option, + pub role: Option, + pub master_link_status: Option, +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PersistedSession { pub token: String, @@ -45,6 +63,7 @@ pub trait SessionPersistence: Send + Sync { async fn delete_account(&self, account_id: &str) -> Result; async fn delete_browser(&self, browser_id: &str) -> Result<(), String>; async fn purge_expired(&self, now_unix: u64) -> Result; + async fn diagnostics(&self, now_unix: u64) -> Result; } pub async fn from_config( diff --git a/src/session_store/redis_store.rs b/src/session_store/redis_store.rs index 313a4fa..c65fc86 100644 --- a/src/session_store/redis_store.rs +++ b/src/session_store/redis_store.rs @@ -1,7 +1,7 @@ use async_trait::async_trait; use redis::AsyncCommands; -use super::{PersistedSession, SessionPersistence}; +use super::{PersistedSession, SessionPersistence, SessionPersistenceDiagnostics}; use crate::config::AppConfig; #[derive(Clone)] @@ -23,6 +23,14 @@ pub struct RedisSessionStore { mode: RedisMode, } +fn redis_info_value<'a>(info: &'a str, key: &str) -> Option<&'a str> { + info.lines().find_map(|line| { + let line = line.trim_end_matches('\r'); + let (name, value) = line.split_once(':')?; + (name == key).then_some(value.trim()) + }) +} + impl RedisSessionStore { pub async fn connect(config: &AppConfig) -> Result { let mode = if let Some(url) = config @@ -274,6 +282,107 @@ impl SessionPersistence for RedisSessionStore { Ok(()) } + async fn diagnostics(&self, now_unix: u64) -> Result { + use std::collections::HashSet; + + const MAX_SCAN_SESSIONS: usize = 5000; + let mut con = self.connection().await?; + let pong: String = redis::cmd("PING") + .query_async(&mut con) + .await + .map_err(|e| format!("Redis PING failed: {e}"))?; + if !pong.eq_ignore_ascii_case("PONG") { + return Err(format!("unexpected Redis PING response: {pong}")); + } + let info: String = redis::cmd("INFO") + .query_async(&mut con) + .await + .map_err(|e| format!("Redis INFO failed: {e}"))?; + + let mut cursor = 0u64; + let mut seen = 0usize; + let mut active = 0usize; + let mut users = HashSet::new(); + let mut browsers = HashSet::new(); + let mut latest = None; + let mut truncated = false; + + loop { + let (next_cursor, keys): (u64, Vec) = redis::cmd("SCAN") + .arg(cursor) + .arg("MATCH") + .arg("rustbox:session:*") + .arg("COUNT") + .arg(250) + .query_async(&mut con) + .await + .map_err(|e| format!("Redis SCAN failed: {e}"))?; + + if !keys.is_empty() { + let remaining = MAX_SCAN_SESSIONS.saturating_sub(seen); + if remaining == 0 { + truncated = true; + break; + } + let keys = if keys.len() > remaining { + truncated = true; + &keys[..remaining] + } else { + &keys[..] + }; + let mut command = redis::cmd("MGET"); + for key in keys { + command.arg(key); + } + let values: Vec> = command + .query_async(&mut con) + .await + .map_err(|e| format!("Redis MGET diagnostics failed: {e}"))?; + seen += keys.len(); + for raw in values.into_iter().flatten() { + let Ok(session) = serde_json::from_str::(&raw) else { + continue; + }; + if session.expired_at(now_unix) { + continue; + } + active += 1; + users.insert(session.user_hash); + browsers.insert(session.browser_id); + latest = Some(latest.map_or(session.last_accessed_unix, |value: u64| { + value.max(session.last_accessed_unix) + })); + } + } + + if truncated || next_cursor == 0 { + break; + } + cursor = next_cursor; + } + + Ok(SessionPersistenceDiagnostics { + backend: "redis".to_string(), + status: "ok".to_string(), + active_sessions: active, + unique_users: users.len(), + unique_browsers: browsers.len(), + latest_accessed_unix: latest, + truncated, + engine_version: redis_info_value(&info, "redis_version").map(str::to_string), + connected_clients: redis_info_value(&info, "connected_clients") + .and_then(|value| value.parse().ok()), + blocked_clients: redis_info_value(&info, "blocked_clients") + .and_then(|value| value.parse().ok()), + used_memory_bytes: redis_info_value(&info, "used_memory") + .and_then(|value| value.parse().ok()), + server_uptime_secs: redis_info_value(&info, "uptime_in_seconds") + .and_then(|value| value.parse().ok()), + role: redis_info_value(&info, "role").map(str::to_string), + master_link_status: redis_info_value(&info, "master_link_status").map(str::to_string), + }) + } + async fn purge_expired(&self, _now_unix: u64) -> Result { Ok(0) } diff --git a/src/ui.rs b/src/ui.rs index 5c728ca..430030a 100644 --- a/src/ui.rs +++ b/src/ui.rs @@ -9,6 +9,7 @@ use crate::config::AppConfig; const INDEX_HTML: &str = include_str!("web/index.html"); const APP_CSS: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/rustbox-ui.css")); const APP_JS: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/rustbox-ui.js")); +const THEME_BOOTSTRAP_JS: &[u8] = include_bytes!("web/theme-bootstrap.js"); const ICONS: &[u8] = include_bytes!("web/icons.svg"); const I18N_EN: &[u8] = include_bytes!("web/i18n/en.json"); const I18N_PL: &[u8] = include_bytes!("web/i18n/pl.json"); @@ -27,7 +28,6 @@ fn bytes_response( [ (header::CONTENT_TYPE, content_type), (header::CACHE_CONTROL, cache), - (header::X_CONTENT_TYPE_OPTIONS, "nosniff"), ], body, ) @@ -48,6 +48,7 @@ pub async fn serve_ui(uri: Uri, Extension(config): Extension>) -> let versioned_css = format!("assets/app.{ASSET_ID}.css"); let versioned_js = format!("assets/app.{ASSET_ID}.js"); + let versioned_theme_bootstrap = format!("assets/theme-bootstrap.{ASSET_ID}.js"); if relative == versioned_css.as_str() { return bytes_response( "text/css; charset=utf-8", @@ -62,10 +63,22 @@ pub async fn serve_ui(uri: Uri, Extension(config): Extension>) -> APP_JS, ); } + if relative == versioned_theme_bootstrap.as_str() { + return bytes_response( + "text/javascript; charset=utf-8", + "public, max-age=31536000, immutable", + THEME_BOOTSTRAP_JS, + ); + } match relative { "assets/app.css" => bytes_response("text/css; charset=utf-8", "no-store", APP_CSS), "assets/app.js" => bytes_response("text/javascript; charset=utf-8", "no-store", APP_JS), + "assets/theme-bootstrap.js" => bytes_response( + "text/javascript; charset=utf-8", + "no-store", + THEME_BOOTSTRAP_JS, + ), "assets/icons.svg" => bytes_response("image/svg+xml", "no-store", ICONS), "assets/i18n/en.json" => { bytes_response("application/json; charset=utf-8", "no-store", I18N_EN) @@ -78,7 +91,6 @@ pub async fn serve_ui(uri: Uri, Extension(config): Extension>) -> [ (header::CONTENT_TYPE, "image/svg+xml"), (header::CACHE_CONTROL, "no-store"), - (header::X_CONTENT_TYPE_OPTIONS, "nosniff"), ], FAVICON, ) @@ -121,7 +133,6 @@ pub async fn serve_ui(uri: Uri, Extension(config): Extension>) -> [ (header::CONTENT_TYPE, "text/html; charset=utf-8"), (header::CACHE_CONTROL, "no-store"), - (header::X_CONTENT_TYPE_OPTIONS, "nosniff"), ], html, ) diff --git a/src/web/css/20-shell.css b/src/web/css/20-shell.css index a20fd04..ae8337e 100644 --- a/src/web/css/20-shell.css +++ b/src/web/css/20-shell.css @@ -158,15 +158,22 @@ .indexing-indicator svg { width: 16px; height: 16px; +} + +.indexing-indicator.active svg { animation: rustbox-indexing-spin .9s linear infinite; } @keyframes rustbox-indexing-spin { - to { transform: rotate(360deg); } + to { + transform: rotate(360deg); + } } @media (prefers-reduced-motion: reduce) { - .indexing-indicator svg { animation: none; } + .indexing-indicator svg { + animation: none; + } } .account-summary { @@ -216,6 +223,8 @@ } .folder-pane { + container-name: folder-pane; + container-type: inline-size; display: flex; flex-direction: column; padding: 10px; diff --git a/src/web/css/30-mail.css b/src/web/css/30-mail.css index 3d35d98..2ca9640 100644 --- a/src/web/css/30-mail.css +++ b/src/web/css/30-mail.css @@ -49,10 +49,47 @@ } .folder-count.unread { + color: var(--subtle); +} + +.folder-count .folder-unread, +.folder-count.unread .folder-count-compact { color: var(--accent); font-weight: 700; } +.folder-count-wide { + display: inline-flex; + align-items: baseline; + gap: 0; + white-space: nowrap; +} + +.folder-count-compact { + display: none; +} + +.folder-count .folder-total, +.folder-count .folder-unread, +.folder-count-compact { + font-variant-numeric: tabular-nums; +} + +.folder-count .folder-separator { + color: var(--subtle); + font-weight: 400; +} + +@container folder-pane (max-width: 205px) { + .folder-count-wide { + display: none; + } + + .folder-count-compact { + display: inline; + } +} + .message-pane { display: grid; grid-template-rows: auto minmax(0, 1fr) auto; diff --git a/src/web/index.html b/src/web/index.html index dbc5007..e229697 100644 --- a/src/web/index.html +++ b/src/web/index.html @@ -12,16 +12,7 @@ RustBox - + @@ -77,13 +68,13 @@ /
+ - diff --git a/src/web/js/00-bootstrap.js b/src/web/js/00-bootstrap.js index 525092e..4a71fa6 100644 --- a/src/web/js/00-bootstrap.js +++ b/src/web/js/00-bootstrap.js @@ -33,7 +33,7 @@ pgpKeys: [], pgpPrefs: { wkdEnabled: true, hkpEnabled: false, requireTrustedEncryption: false }, pgpRecipientStatus: [], - authPolicy: { rememberMeEnabled: true, rememberMeDays: 30, sessionTimeoutHours: 24, pgpEnabled: true, sieveEnabled: true, diagnosticsEnabled: true, aboutEnabled: true }, + authPolicy: { rememberMeEnabled: true, pgpEnabled: false, sieveEnabled: false, diagnosticsEnabled: false, aboutEnabled: false }, contacts: [], trustedSenders: new Set(), contactGroups: [], @@ -63,6 +63,11 @@ realtimeNotifiedMessages: new Map(), realtimeWatchedFolder: null, indexingActive: false, + indexingPending: 0, + imapSyncActive: false, + imapSyncFolder: null, + imapSyncPending: 0, + imapSyncInventoryPending: 0, calendarReminderTimer: null, composeDraftId: null, composeAttachments: [], diff --git a/src/web/js/20-preferences-auth.js b/src/web/js/20-preferences-auth.js index 2c50a53..4567b59 100644 --- a/src/web/js/20-preferences-auth.js +++ b/src/web/js/20-preferences-auth.js @@ -481,17 +481,40 @@ function applyFeatureAvailability() { async function loadAuthPolicy() { try { - state.authPolicy = await request("/auth/policy", { method: "GET" }); + const policy = await request("/auth/policy", { method: "GET" }); + state.authPolicy = { ...state.authPolicy, ...policy }; } catch (_) { - state.authPolicy = { rememberMeEnabled: true, rememberMeDays: 30, sessionTimeoutHours: 24, pgpEnabled: true, sieveEnabled: true, diagnosticsEnabled: true, aboutEnabled: true }; + state.authPolicy = { ...state.authPolicy, rememberMeEnabled: true }; + } + const enabled = state.authPolicy.rememberMeEnabled !== false; + ["#login-remember-row", "#add-account-remember-row"].forEach((selector) => $(selector)?.classList.toggle("hidden", !enabled)); + ["#login-remember", "#add-account-remember"].forEach((selector) => { + const input = $(selector); + if (!input) return; + input.disabled = !enabled; + if (!enabled) input.checked = false; + }); + ["#login-remember-label", "#add-account-remember-label"].forEach((selector) => { + const label = $(selector); + if (label) label.textContent = "Keep me signed in"; + }); +} + +async function loadSessionPolicy() { + try { + const policy = await request("/auth/session-policy", { method: "GET" }); + state.authPolicy = { ...state.authPolicy, ...policy }; + } catch (_) { + state.authPolicy = { + ...state.authPolicy, + pgpEnabled: false, + sieveEnabled: false, + diagnosticsEnabled: false, + aboutEnabled: false, + }; } setOpenPgpAvailability(state.authPolicy.pgpEnabled !== false); applyFeatureAvailability(); - const enabled = state.authPolicy.rememberMeEnabled !== false; - ["#login-remember-row", "#add-account-remember-row"].forEach((selector) => $(selector)?.classList.toggle("hidden", !enabled)); - ["#login-remember", "#add-account-remember"].forEach((selector) => { const input = $(selector); if (input && !enabled) { input.checked = false; input.disabled = true; } }); - const days = Number(state.authPolicy.rememberMeDays || 30); - ["#login-remember-label", "#add-account-remember-label"].forEach((selector) => { const label = $(selector); if (label) label.textContent = `Keep me signed in (${days} days)`; }); } async function loadAccounts() { @@ -607,9 +630,11 @@ async function reloadActiveAccount() { state.selectedMessages.clear(); state.selectionMode = false; closeReader(); + await loadSessionPolicy(); await loadPreferences(); await loadMailPreferences(); - await loadOpenPgp(); + if (state.authPolicy.pgpEnabled !== false) await loadOpenPgp(); + else setOpenPgpAvailability(false); await loadIdentities(); await loadFolders(true); await loadSpecialFolders(); @@ -684,9 +709,11 @@ async function login(event) { state.accountId = data.account.id; localStorage.setItem("rustbox.activeAccount", state.accountId); await loadAccounts(); + await loadSessionPolicy(); await loadPreferences(); await loadMailPreferences(); - await loadOpenPgp(); + if (state.authPolicy.pgpEnabled !== false) await loadOpenPgp(); + else setOpenPgpAvailability(false); await loadIdentities(); await loadFolders(true); await loadSpecialFolders(); diff --git a/src/web/js/30-folders-tags.js b/src/web/js/30-folders-tags.js index de68ec6..0334ee2 100644 --- a/src/web/js/30-folders-tags.js +++ b/src/web/js/30-folders-tags.js @@ -86,6 +86,43 @@ function toggleFolderDrawer() { setFolderDrawerOpen(!$("#folder-pane")?.classList.contains("open")); } +function folderCounterParts(folder) { + const total = Math.max(0, Number(folder?.total_count) || 0); + const unread = Math.max(0, Number(folder?.unread_count) || 0); + const title = unread > 0 ? `${total} messages · ${unread} unread` : `${total} messages · no unread`; + const wide = unread > 0 + ? `${total} / ${unread}` + : `${total}`; + const compactValue = unread > 0 ? unread : total; + const compact = `${compactValue}`; + return { total, unread, title, html: `${wide}${compact}` }; +} + +function updateFolderCounter(folder) { + if (!folder) return; + const parts = folderCounterParts(folder); + const button = $$(".folder-button").find((item) => item.dataset.folder === folder.name); + const counter = button?.querySelector(".folder-count"); + if (!counter) return; + counter.classList.toggle("unread", parts.unread > 0); + counter.title = parts.title; + counter.innerHTML = parts.html; +} + +function setFolderCounts(folderName, totalCount, unreadCount) { + const folder = state.folders.find((item) => item.name === folderName); + if (!folder) return; + if (Number.isFinite(Number(totalCount))) folder.total_count = Math.max(0, Number(totalCount)); + if (Number.isFinite(Number(unreadCount))) folder.unread_count = Math.max(0, Number(unreadCount)); + updateFolderCounter(folder); + if (folderName === state.folder && !state.searchContext) { + const total = Math.max(0, Number(folder.total_count) || 0); + const count = $("#message-count"); + if (count) count.textContent = `${total} ${total === 1 ? "message" : "messages"}`; + } + updateUnreadAppBadge(); +} + function renderFolders() { const list = $("#folder-list"); list.replaceChildren(); @@ -109,7 +146,9 @@ function renderFolders() { const button = el("button", `folder-button ${folder.name === state.folder ? "active" : ""}`); button.type = "button"; - button.innerHTML = `${svgIcon(folderIconName(folder))}${escapeHtml(folderLeafName(folder))}${folder.unread_count || folder.total_count || ""}`; + button.dataset.folder = folder.name; + const counter = folderCounterParts(folder); + button.innerHTML = `${svgIcon(folderIconName(folder))}${escapeHtml(folderLeafName(folder))}${counter.html}`; button.addEventListener("contextmenu", (event) => { event.preventDefault(); openFolderContextMenu(event.clientX, event.clientY, folder); }); button.addEventListener("dragover", (event) => { if (!event.dataTransfer?.types?.includes("application/x-rustbox-message")) return; @@ -160,6 +199,8 @@ async function loadFolders(restoreView = false) { try { const data = await request("/folders", { method: "GET" }); state.folders = data.folders || []; + setImapSyncInventoryPending(data.pending_sync_count); + setIndexingIndicator(state.indexingActive, data.pending_index_count); updateUnreadAppBadge(); loadCollapsedFolders(); const inbox = state.folders.find((folder) => folder.name.toUpperCase() === "INBOX"); @@ -172,6 +213,7 @@ async function loadFolders(restoreView = false) { } renderFolders(); $("#connection-label").textContent = "Connected"; + setTimeout(() => { void refreshFolderListOnly({ forceServer: true }); }, 0); if (restoreView) { const restoredSearch = readActiveSearchContext(); if (restoredSearch) { diff --git a/src/web/js/40-messages-reader.js b/src/web/js/40-messages-reader.js index 892eaa8..a358044 100644 --- a/src/web/js/40-messages-reader.js +++ b/src/web/js/40-messages-reader.js @@ -6,10 +6,12 @@ async function loadMessages(reset = false, options = {}) { const previousScrollTop = reset && !restoreState && list ? list.scrollTop : null; if (reset) { state.page = 0; - state.messages = []; - state.selectedMessages.clear(); - clearActiveSearchContext(); - list.innerHTML = `
Loading messages…
`; + if (!options.background) { + state.messages = []; + state.selectedMessages.clear(); + clearActiveSearchContext(); + list.innerHTML = `
Loading messages…
`; + } } $("#folder-title").textContent = state.folder; try { @@ -22,6 +24,7 @@ async function loadMessages(reset = false, options = {}) { let flaggedOnTop = state.currentFolderFlaggedOnTop; let folderTotalCount = null; let folderUnreadCount = null; + let syncing = false; let lastLoadedPage = state.page; for (const page of pages) { const forceRefresh = options.force ? "&refresh=true" : ""; @@ -30,6 +33,7 @@ async function loadMessages(reset = false, options = {}) { flaggedOnTop = Boolean(data.flagged_on_top); folderTotalCount = Number(data.folder_total_count); folderUnreadCount = Number(data.unread_count); + syncing = syncing || Boolean(data.syncing); loaded.push(...(data.messages || [])); lastLoadedPage = page; if (loaded.length >= totalCount) break; @@ -38,12 +42,14 @@ async function loadMessages(reset = false, options = {}) { state.total = totalCount; state.currentFolderFlaggedOnTop = flaggedOnTop; state.messages = reset ? loaded : [...state.messages, ...loaded]; - const folderState = state.folders.find((item) => item.name === state.folder); - if (folderState) { - if (Number.isFinite(folderTotalCount)) folderState.total_count = folderTotalCount; - if (Number.isFinite(folderUnreadCount)) folderState.unread_count = folderUnreadCount; - updateUnreadAppBadge(); - renderFolders(); + if (state.folders.some((item) => item.name === state.folder)) { + setFolderCounts(state.folder, folderTotalCount, folderUnreadCount); + } + if (syncing) { + const pending = Number.isFinite(folderTotalCount) ? Math.max(0, folderTotalCount - state.messages.length) : 0; + setImapSyncIndicator(true, state.folder, pending); + } else if (state.imapSyncFolder === state.folder) { + setImapSyncIndicator(false, null, 0); } $("#connection-label").textContent = "Connected"; renderMessages(); @@ -59,28 +65,122 @@ async function loadMessages(reset = false, options = {}) { } } +function normalizedMessageFlags(value) { + const source = Array.isArray(value) ? value : String(value || "").split(/[,\s]+/).filter(Boolean); + return [...new Set(source.map((flag) => String(flag).trim()).filter(Boolean))].sort(); +} + +function findMessageRow(folder, uid) { + return $$(".message-row").find((row) => row.dataset.folder === String(folder) && Number(row.dataset.uid) === Number(uid)) || null; +} + +function updateMessageRowState(folder, uid, flags) { + const row = findMessageRow(folder, uid); + if (!row) return; + const normalized = normalizedMessageFlags(flags); + const unread = !normalized.includes("\\Seen"); + const flagged = normalized.includes("\\Flagged"); + row.classList.toggle("unread", unread); + row.classList.toggle("flagged", flagged); + const readButton = row.querySelector(".message-quick-action:not(.message-flag):not(.message-fullscreen-action)"); + if (readButton) { + readButton.title = unread ? "Mark read" : "Mark unread"; + readButton.setAttribute("aria-label", readButton.title); + } + const flagButton = row.querySelector(".message-flag"); + if (flagButton) { + flagButton.classList.toggle("active", flagged); + flagButton.title = flagged ? "Unflag" : "Flag"; + flagButton.setAttribute("aria-label", flagButton.title); + } +} + +function applyLocalMessageFlags(folder, uid, flags) { + const normalized = normalizedMessageFlags(flags); + const item = state.messages.find((message) => Number(message.uid) === Number(uid) && (message.folder || state.folder) === folder); + if (item) item.flags = normalized.join(" "); + if (state.currentDetail?.folder === folder && Number(state.currentDetail?.uid) === Number(uid)) { + state.currentDetail.flags = [...normalized]; + $("#flag-btn")?.classList.toggle("active", normalized.includes("\\Flagged")); + } + updateMessageRowState(folder, uid, normalized); +} + +function applyOptimisticMessageFlag(folder, uid, flag, add) { + const message = state.messages.find((item) => Number(item.uid) === Number(uid) && (item.folder || state.folder) === folder); + const detailMatches = state.currentDetail?.folder === folder && Number(state.currentDetail?.uid) === Number(uid); + const originalFlags = normalizedMessageFlags(message?.flags ?? (detailMatches ? state.currentDetail.flags : [])); + const next = new Set(originalFlags); + add ? next.add(flag) : next.delete(flag); + const optimisticFlags = normalizedMessageFlags([...next]); + if (originalFlags.join("\u0000") === optimisticFlags.join("\u0000")) return () => {}; + + const folderState = state.folders.find((item) => item.name === folder); + const originalTotal = Number(folderState?.total_count); + const originalUnread = Number(folderState?.unread_count); + let optimisticUnread = originalUnread; + if (flag === "\\Seen" && Number.isFinite(originalUnread)) { + const wasSeen = originalFlags.includes("\\Seen"); + const isSeen = optimisticFlags.includes("\\Seen"); + if (wasSeen !== isSeen) optimisticUnread = Math.max(0, originalUnread + (isSeen ? -1 : 1)); + } + + applyLocalMessageFlags(folder, uid, optimisticFlags); + if (Number.isFinite(originalTotal) && Number.isFinite(optimisticUnread) && optimisticUnread !== originalUnread) { + setFolderCounts(folder, originalTotal, optimisticUnread); + } + + return () => { + const current = state.messages.find((item) => Number(item.uid) === Number(uid) && (item.folder || state.folder) === folder); + const currentFlags = normalizedMessageFlags(current?.flags ?? (detailMatches ? state.currentDetail?.flags : [])); + if (currentFlags.join("\u0000") === optimisticFlags.join("\u0000")) applyLocalMessageFlags(folder, uid, originalFlags); + const latestFolder = state.folders.find((item) => item.name === folder); + if (Number.isFinite(originalTotal) && Number.isFinite(originalUnread) && Number(latestFolder?.unread_count) === optimisticUnread) { + setFolderCounts(folder, originalTotal, originalUnread); + } + }; +} + +function updateActiveMessageRow() { + for (const row of $$(".message-row")) { + row.classList.toggle( + "active", + Boolean(state.selected) && row.dataset.folder === state.selected.folder && Number(row.dataset.uid) === Number(state.selected.uid), + ); + } +} + async function quickUpdateMessageFlag(message, flag, add, successMessage) { const folder = message.folder || state.folder; + const rollback = applyOptimisticMessageFlag(folder, message.uid, flag, add); try { - await request(`/messages/${encodeURIComponent(folder)}/${message.uid}/flags`, { method: "PATCH", body: JSON.stringify({ flags: [flag], add }) }); - const flags = new Set(Array.isArray(message.flags) ? message.flags : String(message.flags || "").split(/[,\s]+/).filter(Boolean)); - add ? flags.add(flag) : flags.delete(flag); - message.flags = [...flags].join(" "); - if (state.currentDetail?.uid === message.uid && state.currentDetail?.folder === folder) { - const detailFlags = new Set(state.currentDetail.flags || []); - add ? detailFlags.add(flag) : detailFlags.delete(flag); - state.currentDetail.flags = [...detailFlags]; - renderMessageDetail(state.currentDetail); + const result = await request(`/messages/${encodeURIComponent(folder)}/${message.uid}/flags`, { method: "PATCH", body: JSON.stringify({ flags: [flag], add }) }); + const flags = Array.isArray(result.flags) ? result.flags : normalizedMessageFlags(message.flags); + if (!Array.isArray(result.flags)) { + const next = new Set(flags); + add ? next.add(flag) : next.delete(flag); + applyLocalMessageFlags(folder, message.uid, [...next]); + } else { + applyLocalMessageFlags(folder, message.uid, flags); + } + if (Number.isFinite(Number(result.total_count)) && Number.isFinite(Number(result.unread_count))) { + setFolderCounts(folder, Number(result.total_count), Number(result.unread_count)); + } + if (flag === "\\Flagged" && state.currentFolderFlaggedOnTop && !state.searchContext && folder === state.folder) { + await loadMessages(true, { background: true }); } - if (flag === "\\Flagged" && state.currentFolderFlaggedOnTop && !state.searchContext && folder === state.folder) await loadMessages(true); - else renderMessages(); toast(successMessage, "success"); - } catch (error) { toast(error.message, "error"); } + } catch (error) { + rollback(); + toast(error.message, "error"); + } } function renderMessages() { $("#folder-title").textContent = state.folder; - $("#message-count").textContent = `${state.total} ${state.total === 1 ? "message" : "messages"}`; + const folderTotal = Number(state.folders.find((folder) => folder.name === state.folder)?.total_count); + const displayTotal = !state.searchContext && Number.isFinite(folderTotal) ? folderTotal : state.total; + $("#message-count").textContent = `${displayTotal} ${displayTotal === 1 ? "message" : "messages"}`; const list = $("#message-list"); const highlightTerms = activeSearchHighlightTerms(); list.classList.toggle("selection-mode", state.selectionMode); @@ -94,6 +194,8 @@ function renderMessages() { const checked = state.selectedMessages.has(messageKey(message)); const active = state.selected?.uid === message.uid && state.selected?.folder === folder; const row = el("div", `message-row ${unread ? "unread" : ""} ${flagged ? "flagged" : ""} ${active ? "active" : ""} ${checked ? "selected" : ""}`); + row.dataset.folder = folder; + row.dataset.uid = String(message.uid); row.tabIndex = 0; row.setAttribute("role", "button"); row.setAttribute("aria-label", `${message.from_name || message.from_address || "Unknown"}: ${message.subject || "(no subject)"}`); @@ -122,13 +224,21 @@ function renderMessages() { readButton.title = unread ? "Mark read" : "Mark unread"; readButton.setAttribute("aria-label", readButton.title); readButton.innerHTML = svgIcon("mail"); - readButton.addEventListener("click", (event) => { event.stopPropagation(); quickUpdateMessageFlag(message, "\\Seen", unread, unread ? "Marked read" : "Marked unread"); }); + readButton.addEventListener("click", (event) => { + event.stopPropagation(); + const currentUnread = !normalizedMessageFlags(message.flags).includes("\\Seen"); + quickUpdateMessageFlag(message, "\\Seen", currentUnread, currentUnread ? "Marked read" : "Marked unread"); + }); const flagButton = el("button", `message-quick-action message-flag ${flagged ? "active" : ""}`); flagButton.type = "button"; flagButton.title = flagged ? "Unflag" : "Flag"; flagButton.setAttribute("aria-label", flagButton.title); flagButton.innerHTML = svgIcon("star"); - flagButton.addEventListener("click", (event) => { event.stopPropagation(); quickUpdateMessageFlag(message, "\\Flagged", !flagged, flagged ? "Flag removed" : "Message flagged"); }); + flagButton.addEventListener("click", (event) => { + event.stopPropagation(); + const currentFlagged = normalizedMessageFlags(message.flags).includes("\\Flagged"); + quickUpdateMessageFlag(message, "\\Flagged", !currentFlagged, currentFlagged ? "Flag removed" : "Message flagged"); + }); const fullscreenButton = el("button", "message-quick-action message-fullscreen-action"); fullscreenButton.type = "button"; fullscreenButton.title = "Open full screen"; @@ -281,7 +391,7 @@ async function openMessage(folder, uid) { clearTimeout(state.markReadTimer); state.markReadTimer = null; state.selected = { folder, uid }; - renderMessages(); + updateActiveMessageRow(); updateMessageNavigation(); $("#reader-empty").classList.add("hidden"); $("#reader-content").classList.remove("hidden"); @@ -311,16 +421,15 @@ function scheduleMarkRead(detail) { const delay = Math.max(0, Number(state.mailPrefs.mark_read_delay_ms || 0)); const run = async () => { if (!state.currentDetail || state.currentDetail.uid !== detail.uid || state.currentDetail.folder !== detail.folder) return; + const rollback = applyOptimisticMessageFlag(detail.folder, detail.uid, "\\Seen", true); try { - await request(`/messages/${encodeURIComponent(detail.folder)}/${detail.uid}/flags`, { method: "PATCH", body: JSON.stringify({ flags: ["\\Seen"], add: true }) }); - detail.flags = [...new Set([...(detail.flags || []), "\\Seen"])]; - const item = state.messages.find((msg) => msg.uid === detail.uid && (msg.folder || state.folder) === detail.folder); - if (item) { - const flags = Array.isArray(item.flags) ? item.flags : String(item.flags || "").split(/[,\s]+/).filter(Boolean); - item.flags = [...new Set([...flags, "\\Seen"])].join(" "); + const result = await request(`/messages/${encodeURIComponent(detail.folder)}/${detail.uid}/flags`, { method: "PATCH", body: JSON.stringify({ flags: ["\\Seen"], add: true }) }); + const flags = Array.isArray(result.flags) ? result.flags : [...new Set([...(detail.flags || []), "\\Seen"])]; + applyLocalMessageFlags(detail.folder, detail.uid, flags); + if (Number.isFinite(Number(result.total_count)) && Number.isFinite(Number(result.unread_count))) { + setFolderCounts(detail.folder, Number(result.total_count), Number(result.unread_count)); } - renderMessages(); - } catch (_) { } + } catch (_) { rollback(); } }; if (delay === 0) run(); else state.markReadTimer = setTimeout(run, delay); } diff --git a/src/web/js/60-mail-actions-search.js b/src/web/js/60-mail-actions-search.js index b9cdcc5..c5db35b 100644 --- a/src/web/js/60-mail-actions-search.js +++ b/src/web/js/60-mail-actions-search.js @@ -2,17 +2,30 @@ async function updateFlag(flag, add) { if (!state.currentDetail) return; const detail = state.currentDetail; - await request(`/messages/${encodeURIComponent(detail.folder)}/${detail.uid}/flags`, { - method: "PATCH", - body: JSON.stringify({ flags: [flag], add }), - }); - const flags = new Set(detail.flags || []); - add ? flags.add(flag) : flags.delete(flag); - detail.flags = [...flags]; - const item = state.messages.find((msg) => msg.uid === detail.uid && (msg.folder || state.folder) === detail.folder); - if (item) item.flags = detail.flags.join(" "); - renderMessages(); - renderMessageDetail(detail); + const rollback = applyOptimisticMessageFlag(detail.folder, detail.uid, flag, add); + try { + const result = await request(`/messages/${encodeURIComponent(detail.folder)}/${detail.uid}/flags`, { + method: "PATCH", + body: JSON.stringify({ flags: [flag], add }), + }); + const flags = Array.isArray(result.flags) ? result.flags : normalizedMessageFlags(detail.flags); + if (!Array.isArray(result.flags)) { + const next = new Set(flags); + add ? next.add(flag) : next.delete(flag); + applyLocalMessageFlags(detail.folder, detail.uid, [...next]); + } else { + applyLocalMessageFlags(detail.folder, detail.uid, flags); + } + if (Number.isFinite(Number(result.total_count)) && Number.isFinite(Number(result.unread_count))) { + setFolderCounts(detail.folder, Number(result.total_count), Number(result.unread_count)); + } + if (flag === "\\Flagged" && state.currentFolderFlaggedOnTop && !state.searchContext && detail.folder === state.folder) { + await loadMessages(true, { background: true }); + } + } catch (error) { + rollback(); + throw error; + } } async function toggleUnread() { @@ -231,7 +244,23 @@ async function performBulk(action, targetFolder = null) { if (action === "read" || action === "unread" || action === "flag" || action === "unflag") { const flag = action === "read" || action === "unread" ? "\\Seen" : "\\Flagged"; const add = action === "read" || action === "flag"; - await request(`/messages/${encodeURIComponent(folder)}/${message.uid}/flags`, { method: "PATCH", body: JSON.stringify({ flags: [flag], add }) }); + const rollback = applyOptimisticMessageFlag(folder, message.uid, flag, add); + let result; + try { + result = await request(`/messages/${encodeURIComponent(folder)}/${message.uid}/flags`, { method: "PATCH", body: JSON.stringify({ flags: [flag], add }) }); + } catch (error) { + rollback(); + throw error; + } + const nextFlags = Array.isArray(result?.flags) ? result.flags : (() => { + const flags = new Set(normalizedMessageFlags(message.flags)); + add ? flags.add(flag) : flags.delete(flag); + return [...flags]; + })(); + applyLocalMessageFlags(folder, message.uid, nextFlags); + if (Number.isFinite(Number(result?.total_count)) && Number.isFinite(Number(result?.unread_count))) { + setFolderCounts(folder, Number(result.total_count), Number(result.unread_count)); + } } else if (action === "move" || action === "archive" || action === "junk" || action === "delete") { let destination = targetFolder; if (action === "archive") destination = detectSpecialFolder("archive"); @@ -252,8 +281,17 @@ async function performBulk(action, targetFolder = null) { } state.selectedMessages.clear(); state.searchAllResults = null; - await refreshFolderListOnly(); - await refreshCurrentMessageView(); + const flagsOnly = ["read", "unread", "flag", "unflag"].includes(action); + const orderingChanged = ["flag", "unflag"].includes(action) && state.currentFolderFlaggedOnTop && !state.searchContext; + if (!flagsOnly) { + await refreshFolderListOnly({ forceServer: true }); + await refreshCurrentMessageView(); + } else if (state.searchContext || orderingChanged) { + await refreshCurrentMessageView(); + } else { + renderBulkToolbar(); + updateUnreadAppBadge(); + } const bulkMessages = { junk: bulkJunkToInbox ? tr("marked_not_spam", "Marked as not spam") : tr("marked_spam", "Marked as spam"), archive: "Archived", read: "Marked read", unread: "Marked unread", flag: "Flagged", unflag: "Unflagged", delete: "Deleted", move: "Moved", copy: "Copied" }; const undoAction = movedOperations > 0 && undoMoves.length === movedOperations ? bulkMoveUndoAction(undoMoves) : null; toast(`${bulkMessages[action] || "Updated"}: ${completed} message${completed === 1 ? "" : "s"}`, "success", undoAction ? 10000 : null, undoAction); @@ -264,12 +302,28 @@ async function performBulk(action, targetFolder = null) { } } -async function refreshFolderListOnly() { +async function refreshFolderListOnly(options = {}) { try { - const data = await request("/folders", { method: "GET" }); - state.folders = data.folders || []; + const suffix = options.forceServer ? "?refresh=true" : ""; + const data = await request(`/folders${suffix}`, { method: "GET" }); + const nextFolders = data.folders || []; + setImapSyncInventoryPending(data.pending_sync_count); + setIndexingIndicator(state.indexingActive, data.pending_index_count); + const previousNames = state.folders.map((folder) => folder.name).join("\u0000"); + const nextNames = nextFolders.map((folder) => folder.name).join("\u0000"); + if (previousNames !== nextNames || state.folders.length !== nextFolders.length) { + state.folders = nextFolders; + renderFolders(); + } else { + const byName = new Map(nextFolders.map((folder) => [folder.name, folder])); + for (const folder of state.folders) { + const fresh = byName.get(folder.name); + if (!fresh) continue; + Object.assign(folder, fresh); + updateFolderCounter(folder); + } + } updateUnreadAppBadge(); - renderFolders(); } catch (_) { } } @@ -621,9 +675,22 @@ function showFolderProperties(folder) { async function markFolderAllRead(name) { try { - await request(`/folders/${encodeURIComponent(name)}/mark-all-read`, { method: "POST", body: "{}" }); - if (state.folder === name) await loadMessages(true); - await refreshFolderListOnly(); + const result = await request(`/folders/${encodeURIComponent(name)}/mark-all-read`, { method: "POST", body: "{}" }); + if (state.folder === name) { + const throughUid = Number(result?.through_uid); + for (const message of state.messages) { + if (Number.isFinite(throughUid) && Number(message.uid) > throughUid) continue; + const flags = normalizedMessageFlags(message.flags); + if (!flags.includes("\\Seen")) flags.push("\\Seen"); + applyLocalMessageFlags(name, message.uid, flags); + } + } + const folder = state.folders.find((item) => item.name === name); + setFolderCounts( + name, + Number.isFinite(Number(result?.total_count)) ? Number(result.total_count) : Number(folder?.total_count || 0), + Number.isFinite(Number(result?.unread_count)) ? Number(result.unread_count) : 0, + ); toast("All messages marked read"); } catch (error) { toast(error.message, "error"); } } diff --git a/src/web/js/80-settings-realtime.js b/src/web/js/80-settings-realtime.js index e56490f..75eaedc 100644 --- a/src/web/js/80-settings-realtime.js +++ b/src/web/js/80-settings-realtime.js @@ -160,17 +160,48 @@ function playNotificationSound() { } catch (_) { } } -function setIndexingIndicator(active, pending = 0) { - state.indexingActive = Boolean(active); +function renderSyncActivityIndicator() { const indicator = $("#indexing-indicator"); if (!indicator) return; - indicator.classList.toggle("hidden", !state.indexingActive); - const count = Math.max(0, Number(pending) || 0); - const label = count > 0 ? `Indexing messages (${count} pending)` : "Indexing messages"; + const busy = Boolean(state.indexingActive || state.imapSyncActive); + indicator.classList.toggle("hidden", !busy); + indicator.classList.toggle("active", busy); + const parts = []; + if (state.imapSyncActive || state.imapSyncPending > 0) { + const folder = state.imapSyncFolder ? ` · ${state.imapSyncFolder}` : ""; + const pending = state.imapSyncPending > 0 ? ` · ${state.imapSyncPending} headers pending in task` : ""; + parts.push(`IMAP synchronization${folder}${pending}`); + } + if (state.imapSyncInventoryPending > 0) { + parts.push(`IMAP cache incomplete · ${state.imapSyncInventoryPending} message headers not cached yet`); + } + if (state.indexingActive || state.indexingPending > 0) { + const pending = state.indexingPending > 0 ? ` · ${state.indexingPending} messages pending` : ""; + parts.push(`Search indexing${pending}`); + } + const label = parts.join(" | ") || "Mail synchronization"; indicator.title = label; indicator.setAttribute("aria-label", label); } +function setIndexingIndicator(active, pending = 0) { + state.indexingActive = Boolean(active); + state.indexingPending = Math.max(0, Number(pending) || 0); + renderSyncActivityIndicator(); +} + +function setImapSyncIndicator(active, folder = null, pending = 0) { + state.imapSyncActive = Boolean(active); + state.imapSyncFolder = folder || null; + state.imapSyncPending = Math.max(0, Number(pending) || 0); + renderSyncActivityIndicator(); +} + +function setImapSyncInventoryPending(pending = 0) { + state.imapSyncInventoryPending = Math.max(0, Number(pending) || 0); + renderSyncActivityIndicator(); +} + function syncRealtimeWatchedFolder(force = false) { const socket = state.ws; if (!socket || socket.readyState !== WebSocket.OPEN || !state.accountId || !state.folder) return; @@ -296,23 +327,62 @@ function realtimeEventKey(payload) { function isDuplicateRealtimeEvent(payload) { const now = Date.now(); for (const [key, seenAt] of state.realtimeSeenEvents) if (now - seenAt > 10000) state.realtimeSeenEvents.delete(key); + + // Server websocket envelopes carry a unique eventId. When present it is the + // only safe deduplication key: state can legitimately cycle A -> B -> A in + // under a second (for example unread 1 -> 0 -> 1). Fingerprint-based + // deduplication would incorrectly drop the second A and leave stale UI. + if (payload.eventId) { + const idKey = `id:${payload.eventId}`; + const duplicate = state.realtimeSeenEvents.has(idKey); + state.realtimeSeenEvents.set(idKey, now); + return duplicate; + } + + // Fallback for legacy/nonstandard envelopes without eventId. const event = payload.event || {}; const fingerprint = `${payload.accountId || ""}|${event.type || ""}|${JSON.stringify(event.data || {})}`; - const idKey = payload.eventId ? `id:${payload.eventId}` : null; const fingerprintKey = `event:${fingerprint}`; - const duplicate = Boolean((idKey && state.realtimeSeenEvents.has(idKey)) || (state.realtimeSeenEvents.get(fingerprintKey) && now - state.realtimeSeenEvents.get(fingerprintKey) < 1500)); - if (idKey) state.realtimeSeenEvents.set(idKey, now); + const seenAt = state.realtimeSeenEvents.get(fingerprintKey); + const duplicate = Boolean(seenAt && now - seenAt < 1500); state.realtimeSeenEvents.set(fingerprintKey, now); return duplicate; } +function applyRealtimeFolderStatus(data) { + if (!data?.folder) return; + setFolderCounts(data.folder, Number(data.total_count), Number(data.unread_count)); +} + +function applyRealtimeFlagChanges(data) { + if (!data?.folder) return false; + const changes = Array.isArray(data.changes) ? data.changes : []; + for (const change of changes) { + if (!Number.isFinite(Number(change?.uid))) continue; + applyLocalMessageFlags(data.folder, Number(change.uid), change.flags || []); + } + return changes.length > 0; +} + +function applyRealtimeFolderAllRead(data) { + const folder = data?.folder; + if (!folder || folder !== state.folder) return; + const throughUid = Number(data?.through_uid); + for (const message of state.messages) { + if (Number.isFinite(throughUid) && Number(message.uid) > throughUid) continue; + const flags = normalizedMessageFlags(message.flags); + if (!flags.includes("\\Seen")) flags.push("\\Seen"); + applyLocalMessageFlags(folder, message.uid, flags); + } +} + async function resyncRealtimeState() { if (!state.accountId) return; if (state.realtimeSyncPromise) return state.realtimeSyncPromise; state.realtimeSyncPromise = (async () => { - await refreshFolderListOnly(); + await refreshFolderListOnly({ forceServer: true }); if (state.searchContext) await runSearch(new URLSearchParams(state.searchContext.params), state.searchContext.label, { restoreScrollTop: state.searchContext.scrollTop }); - else await loadMessages(true); + else await loadMessages(true, { background: true }); })().catch(() => { }).finally(() => { state.realtimeSyncPromise = null; }); return state.realtimeSyncPromise; } @@ -323,13 +393,16 @@ function scheduleRealtimeRefresh(event, accountId) { clearTimeout(state.realtimeRefreshTimer); state.realtimeRefreshTimer = setTimeout(async () => { const events = state.realtimeRefreshEvents.splice(0); - const refreshCurrent = events.some((item) => item.type === "FolderUpdated" || (["NewMessages", "FlagsChanged"].includes(item.type) && item.data?.folder === state.folder)); - const tasks = [refreshFolderListOnly()]; - if (refreshCurrent) { + const folderTreeChanged = events.some((item) => item.type === "FolderUpdated"); + const newInCurrent = events.some((item) => item.type === "NewMessages" && item.data?.folder === state.folder); + const structuralFlagsInCurrent = events.some((item) => item.type === "FlagsChanged" && item.data?.folder === state.folder && !(item.data?.changes || []).length); + const tasks = []; + if (folderTreeChanged) tasks.push(refreshFolderListOnly()); + if (folderTreeChanged || newInCurrent || structuralFlagsInCurrent) { if (state.searchContext) tasks.push(runSearch(new URLSearchParams(state.searchContext.params), state.searchContext.label, { restoreScrollTop: state.searchContext.scrollTop })); - else tasks.push(loadMessages(true)); + else tasks.push(loadMessages(true, { background: true })); } - await Promise.all(tasks); + if (tasks.length) await Promise.all(tasks); }, 80); } @@ -402,10 +475,29 @@ function connectRealtime() { notifyNewMail(data, payload.accountId); } } - if (event.type === "IndexingStatus" && payload.accountId === state.accountId) { - setIndexingIndicator(Boolean(event.data?.active), event.data?.pending); + if (payload.accountId === state.accountId) { + if (event.type === "FolderStatus") applyRealtimeFolderStatus(event.data || {}); + if (event.type === "FlagsChanged") { + const patched = applyRealtimeFlagChanges(event.data || {}); + if (!patched) scheduleRealtimeRefresh(event, payload.accountId); + } + if (event.type === "FolderAllRead") applyRealtimeFolderAllRead(event.data || {}); + if (event.type === "IndexingStatus") setIndexingIndicator(Boolean(event.data?.active), event.data?.pending); + if (event.type === "SyncStatus") { + const wasActive = state.imapSyncActive; + const syncFolder = event.data?.folder || null; + const active = Boolean(event.data?.active); + setImapSyncIndicator(active, syncFolder, event.data?.pending); + const folderState = state.folders.find((folder) => folder.name === syncFolder); + const serverTotal = Number(folderState?.total_count); + const syncCompletedForCurrentFolder = wasActive && !active && syncFolder === state.folder; + if (syncCompletedForCurrentFolder && !state.searchContext && Number.isFinite(serverTotal) && state.total < serverTotal) { + void loadMessages(true, { background: true }); + } + } + if (event.type === "SyncInventory") setImapSyncInventoryPending(event.data?.pending); } - if (["NewMessages", "FlagsChanged", "FolderUpdated"].includes(event.type)) scheduleRealtimeRefresh(event, payload.accountId); + if (["NewMessages", "FolderUpdated"].includes(event.type)) scheduleRealtimeRefresh(event, payload.accountId); } catch (error) { console.warn("Invalid realtime event", error); } }; socket.onclose = () => { @@ -413,6 +505,7 @@ function connectRealtime() { state.ws = null; state.realtimeWatchedFolder = null; setIndexingIndicator(false); + setImapSyncIndicator(false, null, 0); clearInterval(state.wsWatchdogTimer); state.wsWatchdogTimer = null; scheduleRealtimeReconnect(); @@ -435,6 +528,7 @@ function disconnectRealtime() { state.wsLastHeartbeatAt = 0; state.realtimeWatchedFolder = null; setIndexingIndicator(false); + setImapSyncIndicator(false, null, 0); const socket = state.ws; state.ws = null; if (socket && socket.readyState < WebSocket.CLOSING) socket.close(1000, "Client disconnect"); @@ -516,7 +610,8 @@ async function loadDiagnostics() { try { const info = await request("/diagnostics/search", { method: "GET" }); searchStatus.textContent = "Ready"; - $("#diagnostic-search-summary").textContent = `${info.indexedDocuments} indexed · ${info.cachedMessages} cached messages · ${info.cachedBodies} cached bodies${info.excludedMessages ? ` · ${info.excludedMessages} excluded` : ""}`; + const bodyCacheState = info.deepIndexEnabled ? "body cache enabled" : "body cache disabled"; + $("#diagnostic-search-summary").textContent = `${info.indexedDocuments} indexed · ${info.cachedMessages} cached messages · ${info.cachedBodies} cached bodies · ${bodyCacheState}${info.excludedMessages ? ` · ${info.excludedMessages} excluded` : ""}`; } catch (error) { searchStatus.textContent = "Unavailable"; $("#diagnostic-search-summary").textContent = error.message; @@ -542,6 +637,7 @@ async function rebuildSearchIndex() { const button = $("#diagnostic-search-rebuild"); if (!await appConfirm("Rebuild the search index from the local message cache?", { title: "Rebuild search index", confirmLabel: "Rebuild" })) return; button.disabled = true; + setIndexingIndicator(true, 0); try { const result = await request("/diagnostics/search/rebuild", { method: "POST", body: "{}" }); toast(`Search index rebuilt: ${result.indexedDocuments} messages`); @@ -549,6 +645,7 @@ async function rebuildSearchIndex() { } catch (error) { toast(error.message, "error"); } finally { + setIndexingIndicator(false, 0); button.disabled = false; } } diff --git a/src/web/js/95-bindings-boot.js b/src/web/js/95-bindings-boot.js index 636a349..101fa6e 100644 --- a/src/web/js/95-bindings-boot.js +++ b/src/web/js/95-bindings-boot.js @@ -237,7 +237,7 @@ function bindEvents() { const button = $("#refresh-btn"); setBusy(button, true); try { - const folderRefresh = refreshFolderListOnly(); + const folderRefresh = refreshFolderListOnly({ forceServer: true }); if (state.searchContext) { await Promise.all([folderRefresh, runSearch(new URLSearchParams(state.searchContext.params), state.searchContext.label, { restoreScrollTop: state.searchContext.scrollTop })]); } else { @@ -266,7 +266,7 @@ function bindEvents() { disconnectRealtime(); connectRealtime(); if (state.accounts.length) { - await refreshFolderListOnly(); + await refreshFolderListOnly({ forceServer: true }); if (state.searchContext) await runSearch(new URLSearchParams(state.searchContext.params), state.searchContext.label, { restoreScrollTop: state.searchContext.scrollTop }); else await loadMessages(true); } @@ -559,7 +559,7 @@ async function boot() { await loadAuthPolicy(); const authenticated = await loadAccounts(); if (authenticated) { - await loadPreferences(); await loadMailPreferences(); if (state.authPolicy.pgpEnabled !== false) await loadOpenPgp(); else setOpenPgpAvailability(false); await loadIdentities(); await loadResponseTemplates(); await loadFolders(true); await loadSpecialFolders(); await loadQuota(); await loadTags(); await loadNotificationPreferences(); await loadTrustedSenders(); startCalendarReminderWatcher(); connectRealtime(); + await loadSessionPolicy(); await loadPreferences(); await loadMailPreferences(); if (state.authPolicy.pgpEnabled !== false) await loadOpenPgp(); else setOpenPgpAvailability(false); await loadIdentities(); await loadResponseTemplates(); await loadFolders(true); await loadSpecialFolders(); await loadQuota(); await loadTags(); await loadNotificationPreferences(); await loadTrustedSenders(); startCalendarReminderWatcher(); connectRealtime(); const params = new URLSearchParams(location.search); const folder = params.get("mail_folder"); const uid = Number(params.get("mail_uid")); diff --git a/src/web/theme-bootstrap.js b/src/web/theme-bootstrap.js new file mode 100644 index 0000000..84cd3e7 --- /dev/null +++ b/src/web/theme-bootstrap.js @@ -0,0 +1,8 @@ +(() => { + const root = document.documentElement; + const configured = root.dataset.defaultTheme || "system"; + root.dataset.theme = configured === "system" + ? (matchMedia("(prefers-color-scheme: light)").matches ? "light" : "dark") + : configured; + root.dataset.interfaceSkin = root.dataset.defaultInterfaceSkin || "rustbox"; +})();