v0.2.0
This commit is contained in:
1 parent
fc81ecafd6
commit
dc57aa7089
89 files changed
+5290
-1551
No files matched your search
+11
-3
@@ -5,10 +5,10 @@ ENVIRONMENT=development
|
||||
DATA_DIR=.dev/data
|
||||
SESSION_TIMEOUT_HOURS=24
|
||||
|
||||
# Optional fixed mail hosts. Leave empty to provide them at login.
|
||||
IMAP_HOST=
|
||||
# Optional fixed mail hosts. Omit IMAP_HOST to use localhost; SMTP falls back to IMAP.
|
||||
# IMAP_HOST=localhost
|
||||
IMAP_PORT=993
|
||||
SMTP_HOST=
|
||||
# SMTP_HOST=localhost
|
||||
SMTP_PORT=587
|
||||
TLS_ENABLED=true
|
||||
|
||||
@@ -22,3 +22,11 @@ SIEVE_TIMEOUT_SECS=10
|
||||
TRUSTED_PROXIES=127.0.0.1,::1
|
||||
RUST_LOG=info
|
||||
# RUSTBOX_CONFIG=/etc/rustbox/rustbox.toml
|
||||
|
||||
# RustBox 0.2 storage/search nested overrides use double underscores.
|
||||
# STORAGE__BACKEND=s3
|
||||
# STORAGE__BUCKET=rustbox
|
||||
# STORAGE__REGION=us-east-1
|
||||
# STORAGE__ENDPOINT=https://s3.example.com
|
||||
# SEARCH__BACKEND=opensearch
|
||||
# For arrays such as search.urls, TOML config is recommended.
|
||||
@@ -9,7 +9,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: dtolnay/rust-toolchain@1.88.0
|
||||
with:
|
||||
components: rustfmt, clippy
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
@@ -17,6 +17,12 @@ jobs:
|
||||
workspaces: .
|
||||
- name: Reject TypeScript sources
|
||||
run: '! find . -type f \( -name "*.ts" -o -name "*.tsx" \) -print -quit | grep -q .'
|
||||
- name: Install static-validator dependency
|
||||
run: python3 -m pip install --disable-pip-version-check 'PyYAML>=6,<7'
|
||||
- name: Static release validation
|
||||
run: python3 scripts/validate-release-static.py
|
||||
- name: Generate dependency lockfile
|
||||
run: cargo generate-lockfile
|
||||
- name: Format
|
||||
run: cargo fmt --all -- --check
|
||||
- name: Test
|
||||
|
||||
+16
-2
@@ -1,12 +1,26 @@
|
||||
# RustBox changelog
|
||||
|
||||
### 0.2.0
|
||||
- Fixed nested database transactions during message move by reusing the caller-owned transaction for cache removal, header-sync invalidation, folder invalidation, and search outbox writes.
|
||||
- Kept `search_outbox` in sync between tenant and legacy-import table sets so schema/import parity tests pass.
|
||||
- Decoupled SQL, blob storage and search: PostgreSQL/MySQL may still use local `data_dir` backends in single-node mode, while cluster mode no longer requires persistent `data_dir` when cluster-safe backends are configured.
|
||||
- Added `BlobStorage` with filesystem/shared-POSIX and S3-compatible backends, streaming attachment I/O, SHA-256 metadata, S3 retry/timeouts/checksums/SSE options and storage readiness probes.
|
||||
- Added idempotent attachment migration from legacy filesystem paths to logical object keys (`rustbox storage migrate`).
|
||||
- Added OpenSearch as the external full-text backend with an async HTTP client, per-tenant filtering/routing, deterministic document IDs, health checks and rebuild support.
|
||||
- Added a durable SQL search outbox with caller-transaction participation, claim/lease concurrency, retry/backoff, dead-letter state and lag/dead-letter diagnostics.
|
||||
- Hardened clustered indexing against re-enqueue/lease races, sized worker leases from the OpenSearch timeout/failover budget, and made IMAP IDLE deletions enqueue index removals in the same SQL transaction.
|
||||
- Added runtime schema migration V047 for attachment checksums and the search outbox across SQLite, MySQL and PostgreSQL.
|
||||
- Added Docker variants for standalone SQLite, PostgreSQL + SILO (S3-compatible) + OpenSearch, and MySQL + NFS + OpenSearch.
|
||||
- Added tests for cluster capability validation, filesystem blob streaming, OpenSearch tenant scoping and search-outbox transaction rollback/claim/idempotency behavior.
|
||||
- Bumped the application/source release to 0.2.0.
|
||||
|
||||
### 0.1.2
|
||||
- Added `log_level` to TOML configuration (`trace`, `debug`, `info`, `warn`, `error`, `off`); `RUST_LOG` remains an explicit runtime override.
|
||||
- Audited the MySQL and PostgreSQL query catalogs, schema baselines and backend-specific server SQL without adding application tests.
|
||||
- Fixed a MySQL DDL incompatibility by changing indexed `pgp_keys.email` from `TEXT` to `VARCHAR(320)`; MySQL requires a prefix length when indexing `TEXT`/`BLOB` columns.
|
||||
- Fixed destructive MySQL cache upserts: folder refresh and message-header refresh now use `INSERT ... ON DUPLICATE KEY UPDATE` instead of `REPLACE INTO`, preserving child rows, message tags, cached bodies and other columns not part of the refresh.
|
||||
- Documented and enforced cluster storage behavior: mailbox metadata/body cache remains in the shared SQL backend, while draft attachments require shared `data_dir` mounted at the same path on all nodes.
|
||||
- Added `search_index_backend = "tantivy" | "disabled"`; cluster mode requires `disabled` because the built-in Tantivy index has no multi-node writer coordination.
|
||||
- Documented and enforced cluster storage behavior: mailbox metadata/body cache remains in shared SQL, while draft attachments use either S3-compatible storage or an explicitly shared POSIX filesystem.
|
||||
- Replaced the pre-0.2 storage/search compatibility switches with the final `[storage]` and `[search]` configuration model; cluster mode validates backend capabilities directly and rejects local Tantivy.
|
||||
- Added `pgp_enabled`; cluster mode requires OpenPGP disabled because private-key material is filesystem-backed.
|
||||
- Avoided provisioning per-user Tantivy directories when the search index backend is disabled and made diagnostics/deep indexing honor that setting.
|
||||
- Added a release audit document with SQL compatibility and distributed-storage findings.
|
||||
|
||||
Generated
+1146
-721
File diff suppressed because it is too large.
Load diff
+5
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "rustbox"
|
||||
version = "0.1.2"
|
||||
version = "0.2.0"
|
||||
edition = "2024"
|
||||
rust-version = "1.88"
|
||||
default-run = "rustbox"
|
||||
@@ -44,6 +44,10 @@ sqlx = { version = "0.8", default-features = false, features = ["runtime-tokio-r
|
||||
redis = { version = "0.32", default-features = false, features = ["tokio-comp"] }
|
||||
aes-gcm = "0.10"
|
||||
|
||||
object_store = { version = "0.14.2", features = ["aws"] }
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
|
||||
bytes = "1"
|
||||
|
||||
[dev-dependencies]
|
||||
http-body-util = "0.1"
|
||||
tempfile = "3"
|
||||
@@ -1,6 +1,6 @@
|
||||
# RustBox attribution / notice
|
||||
|
||||
RustBox 0.1.2 is derived from the Oxi webmail project:
|
||||
RustBox 0.2.0 is derived from the Oxi webmail project:
|
||||
|
||||
- Upstream: `c0h1b4/oxi`
|
||||
- Base snapshot: `7f1507c486b78cf33e54beb701dd460d3bd2bd5d`
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# RustBox 0.1.2
|
||||
# RustBox 0.2.0
|
||||
|
||||
RustBox is a webmail derived from `c0h1b4/oxi`. The consumed Oxi revision is recorded in `UPSTREAM_BASE`.
|
||||
|
||||
@@ -34,6 +34,23 @@ rustbox/
|
||||
|
||||
`build.rs` combines the CSS modules from `src/web/css/manifest.txt` and JavaScript modules from `src/web/js/manifest.txt`. Cargo embeds the generated CSS/JS bundles, HTML and SVG sprite in the RustBox executable. There is no standalone source `src/web/app.js`; `src/web/` is the single source of truth and Cargo generates the embedded bundles in `OUT_DIR`.
|
||||
|
||||
## Storage and search in 0.2.0
|
||||
|
||||
RustBox separates three concerns: SQL metadata, blob storage and search. The SQL engine does **not** decide whether `data_dir` may be used. A single-node PostgreSQL/MySQL deployment may still use filesystem blobs and local Tantivy. In cluster mode, local node state is rejected unless the configured backend is explicitly shared.
|
||||
|
||||
Blob backends:
|
||||
|
||||
- `storage.backend="filesystem"` — local disk in single-node mode; set `storage.shared=true` only for a real shared POSIX mount such as NFS, CephFS or GlusterFS.
|
||||
- `storage.backend="s3"` — AWS S3 or compatible APIs such as MinIO/Ceph RGW.
|
||||
|
||||
Search backends:
|
||||
|
||||
- `search.backend="local_tantivy"` — single-node.
|
||||
- `search.backend="opensearch"` — cluster-capable. Index mutations are written first to a durable per-tenant SQL outbox and processed with retry/backoff/dead-letter handling.
|
||||
- `search.backend="disabled"` — no full-text index.
|
||||
|
||||
In a cluster using shared SQL + S3/shared POSIX + OpenSearch/disabled search + `pgp_enabled=false`, `data_dir` is not required as persistent node state. See `docker/README.md` for ready Compose variants.
|
||||
|
||||
## Configure mail servers
|
||||
|
||||
Mail endpoints are configured by the administrator, not on the login screen. Users enter only email and password.
|
||||
@@ -71,64 +88,67 @@ If the mail host settings are omitted, RustBox uses `localhost` with IMAP `993`
|
||||
|
||||
Mail-size limits are intentionally separate. `max_attachment_size_mb` applies to each raw uploaded file, while `max_message_size_mb` is checked against the final RFC822/MIME byte stream immediately before SMTP submission. The latter therefore includes message bodies, headers, MIME boundaries, base64 expansion, inline CID parts and OpenPGP transformations. The composer displays only an estimate; the server-side final-MIME check is authoritative.
|
||||
|
||||
## Database, cache, sessions and cluster traffic
|
||||
## Database, storage, search and cluster traffic
|
||||
|
||||
RustBox 0.1.2 uses one SQL backend for the whole application. `database_backend = "sqlite" | "mysql" | "pgsql"` selects where account data is stored and, by default, where persistent login sessions are stored. SQLite defaults to `data_dir/rustbox.sqlite`; MySQL and PostgreSQL require `database_url`.
|
||||
`database_backend = "sqlite" | "mysql" | "pgsql"` selects only the authoritative SQL database. It does not force or forbid local blob/search storage. In particular, PostgreSQL/MySQL are valid in a single-node installation together with `storage.backend="filesystem"` and `search.backend="local_tantivy"`.
|
||||
|
||||
Mailbox metadata and cached message bodies are stored in that SQL backend. With a shared MySQL/PostgreSQL database the persistent mailbox cache is therefore shared by all application nodes; Redis is not used for mailbox cache data.
|
||||
`session_backend = "database"` stores sessions in the selected SQL database. `session_backend = "redis"` moves only session persistence to Redis/Redis Sentinel. Mailbox metadata and cached message bodies remain in SQL.
|
||||
|
||||
The per-user directory `data_dir/<user_hash>/` is not the active account database when MySQL/PostgreSQL is configured. It can contain filesystem-backed data:
|
||||
`data_dir` is persistent node state only when a selected local feature actually needs it: default SQLite, implicit filesystem storage, implicit local Tantivy, or local OpenPGP keys. In cluster mode it is not required when those features are replaced with cluster-safe backends. Cluster validation is capability-based:
|
||||
|
||||
- `attachments/` — draft attachments; in a cluster `data_dir` must be mounted from shared storage at the same path on every node;
|
||||
- `tantivy/` — built-in full-text index; it is single-node only and must be disabled in cluster mode;
|
||||
- `pgp/` — OpenPGP private-key material; OpenPGP must be disabled in cluster mode;
|
||||
- legacy `db.sqlite` — migration input only. After a successful 0.1.0 import it is retained as `db.sqlite.pre-0.1.1.bak`.
|
||||
- SQL: shared MySQL/PostgreSQL; SQLite is rejected;
|
||||
- blobs: S3-compatible storage, or `filesystem` with `storage.shared=true` on a real NFS/CephFS/GlusterFS mount;
|
||||
- search: OpenSearch or `disabled`; local Tantivy is rejected;
|
||||
- OpenPGP: remains disabled until a cluster-safe key backend exists;
|
||||
- all nodes use the same `session_encryption_key`.
|
||||
|
||||
For a multi-node deployment use a shared MySQL/PostgreSQL database, the same `session_encryption_key` on every node, and shared `data_dir` for draft attachments. RustBox 0.1.2 validates this explicitly with `cluster_shared_data_dir = true`. The built-in Tantivy backend has no cross-node writer coordination, so cluster mode requires `search_index_backend = "disabled"`. SQL-backed metadata/header search remains available; deep full-body Tantivy search is unavailable in cluster mode. An external search/index backend is not implemented in 0.1.2.
|
||||
|
||||
`session_backend = "database"` stores sessions in the selected application database. `session_backend = "redis"` uses Redis instead; Redis Sentinel is supported through the Sentinel settings. Redis only replaces session persistence, not the application database or mailbox cache.
|
||||
Example stateless cluster:
|
||||
|
||||
```toml
|
||||
# Single node defaults.
|
||||
database_backend = "sqlite"
|
||||
database_backend = "pgsql"
|
||||
database_url = "postgres://rustbox:secret@db/rustbox"
|
||||
session_backend = "database"
|
||||
search_index_backend = "tantivy"
|
||||
pgp_enabled = true
|
||||
session_encryption_key = "<shared-key>"
|
||||
cluster_mode = true
|
||||
pgp_enabled = false
|
||||
|
||||
# Multi-node PostgreSQL example:
|
||||
# database_backend = "pgsql"
|
||||
# database_url = "postgres://rustbox:secret@db/rustbox"
|
||||
# session_backend = "database" # or redis
|
||||
# session_encryption_key = "<base64-32-byte-key>"
|
||||
# cluster_mode = true
|
||||
# cluster_shared_data_dir = true
|
||||
# data_dir = "/srv/rustbox-shared" # same mounted path on every node
|
||||
# search_index_backend = "disabled"
|
||||
# pgp_enabled = false
|
||||
[storage]
|
||||
backend = "s3"
|
||||
bucket = "rustbox"
|
||||
region = "eu-central-1"
|
||||
# endpoint = "https://s3.example.com" # MinIO/Ceph RGW/etc.
|
||||
|
||||
# Multi-node MySQL example (MySQL 8.0.13+):
|
||||
# database_backend = "mysql"
|
||||
# database_url = "mysql://rustbox:secret@db/rustbox"
|
||||
# session_backend = "database" # or redis
|
||||
# session_encryption_key = "<base64-32-byte-key>"
|
||||
# cluster_mode = true
|
||||
# cluster_shared_data_dir = true
|
||||
# data_dir = "/srv/rustbox-shared"
|
||||
# search_index_backend = "disabled"
|
||||
# pgp_enabled = false
|
||||
|
||||
# Optional Redis session persistence; application data still uses database_backend.
|
||||
# session_backend = "redis"
|
||||
# redis_url = "redis://redis:6379/"
|
||||
|
||||
# Sentinel alternative to redis_url:
|
||||
# redis_sentinel_nodes = ["sentinel-a:26379", "sentinel-b:26379", "sentinel-c:26379"]
|
||||
# redis_sentinel_master = "mymaster"
|
||||
# redis_sentinel_password = "sentinel-secret"
|
||||
# redis_password = "secret"
|
||||
[search]
|
||||
backend = "opensearch"
|
||||
urls = ["https://opensearch.example.com:9200"]
|
||||
required = true
|
||||
```
|
||||
|
||||
All executable application SQL is centralized by dialect under `src/queries/{sqlite,mysql,pgsql}/queries.rs`. Each engine has one complete baseline in `migrations/{sqlite,mysql,pgsql}/schema.sql`, including the session table. The squashed baseline is schema version 46 and the next schema migration number is V047 for all engines. Redis requires no SQL migration. See `migrations/README.md` and `docs/DB-AND-CLUSTER-AUDIT-0.1.2.md`.
|
||||
Shared POSIX is also supported without S3:
|
||||
|
||||
```toml
|
||||
[storage]
|
||||
backend = "filesystem"
|
||||
path = "/mnt/rustbox-shared"
|
||||
shared = true
|
||||
```
|
||||
|
||||
Existing attachment files can be migrated to the configured storage backend without deleting the source files:
|
||||
|
||||
```bash
|
||||
rustbox storage migrate --from filesystem:/old/data/root --to configured --dry-run
|
||||
rustbox storage migrate --from filesystem:/old/data/root --to configured --verify
|
||||
```
|
||||
|
||||
The SQL row is switched to the logical object key only after upload and size/checksum verification. Search state is rebuildable from SQL:
|
||||
|
||||
```bash
|
||||
rustbox search reindex
|
||||
```
|
||||
|
||||
Ready Docker examples are in `docker/`: standalone SQLite, PostgreSQL + SILO (S3-compatible) + OpenSearch, and MySQL + NFS + OpenSearch.
|
||||
|
||||
All executable application SQL is centralized by dialect under `src/queries/{sqlite,mysql,pgsql}/queries.rs`. The squashed compatibility baseline remains V046; RustBox 0.2.0 adds runtime migration V047 for attachment checksums/logical blob keys and the durable search outbox. The next migration number is V048. See `migrations/README.md`.
|
||||
|
||||
|
||||
## Development
|
||||
@@ -197,7 +217,7 @@ Initialize Git after extracting the release archive:
|
||||
```bash
|
||||
git init
|
||||
git add .
|
||||
git commit -m "RustBox 0.1.2 release"
|
||||
git commit -m "RustBox 0.2.0 release"
|
||||
./scripts/setup-upstream.sh
|
||||
```
|
||||
|
||||
|
||||
+39
-32
@@ -1,38 +1,45 @@
|
||||
RustBox 0.1.2 release manifest
|
||||
RustBox 0.2.0 release manifest
|
||||
|
||||
Storage model:
|
||||
- database_backend = sqlite | mysql | pgsql selects the full application database
|
||||
- session_backend = database stores sessions in the same database
|
||||
- session_backend = redis uses Redis or Redis Sentinel for sessions
|
||||
Architecture:
|
||||
- database_backend = sqlite | mysql | pgsql selects authoritative SQL only
|
||||
- blob storage and search are independent from the SQL engine
|
||||
- PostgreSQL/MySQL single-node deployments may continue to use data_dir-backed filesystem/Tantivy
|
||||
- cluster mode clears an implicit/default data_dir; explicit data_dir is not forbidden, but local-only capabilities are rejected
|
||||
|
||||
Database baselines:
|
||||
- migrations/sqlite/schema.sql: complete SQLite schema, baseline version 46
|
||||
- migrations/mysql/schema.sql: complete MySQL schema, baseline version 46
|
||||
- migrations/pgsql/schema.sql: complete PostgreSQL schema, baseline version 46
|
||||
- each schema includes global sessions and isolated account-data tables
|
||||
- historical SQLite V001-V046 files are intentionally squashed out of this release
|
||||
- next migration number for every SQL engine: V047
|
||||
Blob storage:
|
||||
- filesystem: local single-node or shared POSIX with storage.shared=true
|
||||
- shared POSIX examples: NFS, CephFS, GlusterFS
|
||||
- s3: AWS S3 and S3-compatible endpoints such as MinIO/Ceph RGW
|
||||
- attachment rows use logical storage keys and SHA-256 metadata
|
||||
- storage migration command: rustbox storage migrate --from filesystem:/path --to configured [--dry-run] [--verify]
|
||||
|
||||
Cluster storage rules (0.1.2):
|
||||
- SQL mailbox metadata/body cache is shared by the configured MySQL/PostgreSQL backend
|
||||
- data_dir must be shared between nodes for filesystem-backed draft attachments
|
||||
- cluster_shared_data_dir=true is required as an explicit deployment acknowledgement
|
||||
- built-in Tantivy must be disabled in cluster mode (search_index_backend=disabled)
|
||||
- OpenPGP must be disabled in cluster mode (pgp_enabled=false)
|
||||
- legacy db.sqlite files remain migration-only; they are not the active database on MySQL/PostgreSQL
|
||||
Search:
|
||||
- local_tantivy: single-node only
|
||||
- opensearch: cluster-capable external index
|
||||
- disabled: no full-text index
|
||||
- OpenSearch operations are tenant-scoped by user_hash
|
||||
- durable SQL outbox provides claim/lease, retry/backoff and dead-letter handling
|
||||
- rebuild command: rustbox search reindex
|
||||
|
||||
Compatibility:
|
||||
- legacy RustBox 0.1.0 per-user SQLite databases at user_version 46 are imported on first access
|
||||
- imported source DBs are retained as db.sqlite.pre-0.1.1.bak
|
||||
- older legacy schemas must first be upgraded with RustBox 0.1.0
|
||||
Database/schema:
|
||||
- migrations/{sqlite,mysql,pgsql}/schema.sql contain the current fresh schema
|
||||
- compatibility baseline remains V046
|
||||
- runtime migration V047 adds attachment checksum metadata and search_outbox
|
||||
- next migration number: V048
|
||||
|
||||
Tests/source cleanup:
|
||||
- tests/fixtures/account-session-expired.json is retained because Rust tests include it
|
||||
- stale tests/*.py and tests/theme_fixture.html are not part of the release tree
|
||||
Docker examples:
|
||||
- docker/compose-single-sqlite.yml
|
||||
- docker/compose-postgres-s3-opensearch.yml (PostgreSQL + SILO/S3 + OpenSearch)
|
||||
- docker/compose-mysql-nfs-opensearch.yml (MySQL + real NFS volume + OpenSearch)
|
||||
|
||||
Oxi tracking:
|
||||
- UPSTREAM_BASE stores the consumed Oxi revision
|
||||
- backend source changes are mapped into the flat RustBox tree
|
||||
- upstream backend/migrations and backend/tests are review-only and never auto-imported
|
||||
- frontend changes are reviewed and manually ported to src/web/
|
||||
- scripts/finish-upstream.sh requires explicit acknowledgement for changed RustBox-owned upstream areas before advancing UPSTREAM_BASE
|
||||
Operations/docs:
|
||||
- README.md documents 0.2.0 storage/search configuration
|
||||
- docker/README.md documents Compose variants
|
||||
- docs/STORAGE_SEARCH_OPERATIONS.md documents failure modes, shared-storage requirements and migration behavior
|
||||
- TODO.md leaves live SILO/MinIO/AWS/Ceph and real multi-node acceptance validation explicitly pending
|
||||
|
||||
Build/release metadata:
|
||||
- Cargo.toml version = 0.2.0
|
||||
- rust-toolchain.toml pins Rust 1.88.0
|
||||
- Cargo.lock is intentionally not shipped because this environment could not regenerate it after dependency changes
|
||||
- CI and scripts/build-release.sh generate Cargo.lock, then run locked tests/Clippy/release build
|
||||
+29
-16
@@ -1,20 +1,33 @@
|
||||
RustBox 0.1.2 - release validation
|
||||
RustBox 0.2.0 - final source validation
|
||||
|
||||
Completed in this environment:
|
||||
- release metadata: Cargo.toml and the RustBox package entry in Cargo.lock = 0.1.2
|
||||
- TOML parse: Cargo.toml, config.example.toml, config.dev.toml and packaging/rustbox.toml
|
||||
- shell syntax: dev.sh and scripts/*.sh pass bash -n
|
||||
- query catalog parity: SQLite/MySQL/PostgreSQL expose the same 27 modules / 197 public SQL constants, including 153 Q_* executable query constants
|
||||
- logical schema parity: MySQL and PostgreSQL define the same 28 account/global tables
|
||||
- static SQL dialect scan: no PostgreSQL-only constructs found in the MySQL catalog/baseline and no MySQL-only constructs found in the PostgreSQL catalog/baseline
|
||||
- MySQL index audit: 49 indexes/constraints checked; no unprefixed TEXT/BLOB key remains and the largest utf8mb4 key estimate is 2568 bytes (<3072); pgp_keys.email is VARCHAR(320)
|
||||
- MySQL upsert audit: no REPLACE INTO remains in the MySQL catalog; folder/message refresh uses non-destructive ON DUPLICATE KEY UPDATE
|
||||
- SQLite baseline execution: schema executes successfully under Python sqlite3 with foreign keys enabled
|
||||
- cluster storage review: SQL mailbox cache is shared; draft attachments are filesystem-backed; built-in Tantivy and OpenPGP are disabled by cluster-mode validation
|
||||
- no application tests were added for the 0.1.2 audit
|
||||
- offline release validator: scripts/validate-release-static.py (139/139 checks passed)
|
||||
- all maintained TOML examples parse successfully
|
||||
- Docker Compose topology/config references are validated by the offline release validator
|
||||
- JSON files parse successfully
|
||||
- browser JavaScript assembled in src/web/js/manifest.txt order passes node --check
|
||||
- shell scripts pass bash -n
|
||||
- Python helper scripts compile successfully
|
||||
- Rust source delimiter sanity check passes after removing partial-patch leftovers
|
||||
- git diff --check passes on the final source tree
|
||||
- no Cargo.lock, *.orig, *.rej, temporary FIX/APPLY scripts, or patch leftovers are shipped
|
||||
- regression source checks cover every Clippy failure reported during the 0.2.0 release attempt
|
||||
|
||||
Not executable in this environment:
|
||||
- cargo fmt/test/clippy/build: Rust/Cargo toolchain is not installed
|
||||
- live MySQL/PostgreSQL execution: database servers/clients are not installed
|
||||
0.2.0 fixes included:
|
||||
- cluster_mode controls the implicit data_dir default; PostgreSQL/MySQL remain valid with local data_dir backends in single-node deployments
|
||||
- storage/search configuration comes only from [storage] and [search]
|
||||
- removed pre-0.2 top-level storage/search switches and the tantivy backend alias
|
||||
- removed runtime legacy attachment read-through; attachment access uses logical BlobStorage keys
|
||||
- fixed search_outbox tenant/import table parity
|
||||
- fixed nested SQLite transactions in message move paths
|
||||
- fixed the Clippy -D warnings failures reported for examples, config, storage, search, routes, realtime and formatting
|
||||
- scripts/build-release.sh removes stale Cargo.lock, runs the static gate, generates a fresh lockfile for locked Cargo commands, and cleans it on exit
|
||||
|
||||
Before publishing a production binary, run scripts/build-release.sh on a Rust 1.88+ build host and perform a live schema/query smoke test against the exact MySQL/PostgreSQL versions used in production.
|
||||
Not executed in this packaging environment:
|
||||
- cargo fmt/test/clippy/build with Rust 1.88, because Rust/Cargo is not installed here and outbound DNS is unavailable
|
||||
- live PostgreSQL/MySQL/S3/NFS/OpenSearch integration tests
|
||||
|
||||
The final release gate on a Rust host is:
|
||||
bash scripts/build-release.sh
|
||||
|
||||
This file intentionally does not claim blocked runtime/integration checks as passed.
|
||||
@@ -12,7 +12,7 @@ The target architecture should allow fully stateless RustBox application nodes u
|
||||
- an optional external full-text indexer,
|
||||
- no writable node-local state required for normal cluster operation.
|
||||
|
||||
This is a follow-up to the 0.1.2 database/cluster audit. It is not implemented in 0.1.2.
|
||||
This is a follow-up to the 0.1.2 database/cluster audit. RustBox 0.2.0 implements the storage abstraction, S3/shared-POSIX support, OpenSearch backend and durable indexing pipeline. Live interoperability tests against SILO/MinIO/AWS S3/Ceph RGW and a real multi-node cluster remain release-validation work.
|
||||
|
||||
---
|
||||
|
||||
@@ -25,23 +25,22 @@ Shared through SQL:
|
||||
- application data stored in MySQL/PostgreSQL,
|
||||
- persistent sessions when SQL is selected.
|
||||
|
||||
Filesystem-backed under `data_dir/<user_hash>/`:
|
||||
Node-local data may still exist under `data_dir/<user_hash>/` for single-node features such as local Tantivy and OpenPGP private-key material. Draft attachments use the configured `[storage]` backend.
|
||||
|
||||
- `attachments/<draft_id>/<attachment_id>` — draft attachments,
|
||||
- `tantivy/` — local full-text index,
|
||||
- `pgp/private/` — OpenPGP private-key material,
|
||||
- legacy `db.sqlite` — migration source only.
|
||||
|
||||
Current cluster restrictions:
|
||||
Cluster configuration is explicit and capability-based:
|
||||
|
||||
```toml
|
||||
cluster_mode = true
|
||||
cluster_shared_data_dir = true
|
||||
search_index_backend = "disabled"
|
||||
pgp_enabled = false
|
||||
|
||||
[storage]
|
||||
backend = "s3"
|
||||
|
||||
[search]
|
||||
backend = "opensearch"
|
||||
```
|
||||
|
||||
`cluster_shared_data_dir = true` is currently required because draft attachments are path-based and local Tantivy/PGP state is not cluster-safe.
|
||||
Shared POSIX storage is also supported with `storage.backend="filesystem"` and `storage.shared=true`. Local Tantivy remains single-node only.
|
||||
|
||||
---
|
||||
|
||||
@@ -181,14 +180,7 @@ Priority should be:
|
||||
|
||||
# 2. Cluster behavior after storage abstraction
|
||||
|
||||
When `storage.backend = "s3"`, RustBox should allow:
|
||||
|
||||
```toml
|
||||
cluster_mode = true
|
||||
cluster_shared_data_dir = false
|
||||
```
|
||||
|
||||
The cluster validation should become capability-based instead of checking only `cluster_shared_data_dir`.
|
||||
When `storage.backend = "s3"`, RustBox allows cluster mode without a writable shared `data_dir`. Cluster validation is capability-based and depends on the selected storage/search backends.
|
||||
|
||||
Expected rules:
|
||||
|
||||
@@ -567,7 +559,7 @@ Requirements:
|
||||
- rotation/recovery procedure,
|
||||
- auditability.
|
||||
|
||||
Do not make `cluster_shared_data_dir` the solution for private-key consistency.
|
||||
Do not treat shared blob storage as a solution for private-key consistency.
|
||||
|
||||
---
|
||||
|
||||
@@ -587,34 +579,21 @@ This is not required for SQL cache consistency, but is required for reliable rea
|
||||
|
||||
---
|
||||
|
||||
# 8. Configuration compatibility
|
||||
# 8. Configuration model
|
||||
|
||||
Keep backward compatibility where possible.
|
||||
|
||||
Legacy:
|
||||
|
||||
```toml
|
||||
data_dir = "/var/lib/rustbox/data"
|
||||
search_index_backend = "tantivy"
|
||||
```
|
||||
|
||||
Migration target:
|
||||
RustBox 0.2.0 uses one configuration model only. There are no compatibility aliases for pre-0.2 storage/search keys.
|
||||
|
||||
```toml
|
||||
[storage]
|
||||
backend = "filesystem"
|
||||
path = "/var/lib/rustbox/data"
|
||||
shared = false
|
||||
|
||||
[search]
|
||||
backend = "local_tantivy"
|
||||
```
|
||||
|
||||
Suggested transition:
|
||||
|
||||
- accept legacy keys for at least one release,
|
||||
- emit deprecation warnings,
|
||||
- map legacy values internally to the new backend configuration,
|
||||
- document exact removal version before deleting old settings.
|
||||
Cluster deployments must select cluster-safe backends explicitly (`storage.shared=true` for shared POSIX or `storage.backend="s3"`; `search.backend="opensearch"` or `disabled`).
|
||||
|
||||
---
|
||||
|
||||
@@ -704,54 +683,57 @@ For external search:
|
||||
|
||||
## Phase 1 — storage abstraction
|
||||
|
||||
- [ ] introduce `BlobStorage` interface,
|
||||
- [ ] convert draft attachments from absolute paths to logical object keys,
|
||||
- [ ] implement filesystem backend,
|
||||
- [ ] retain existing behavior for single-node installations,
|
||||
- [ ] add storage health checks.
|
||||
- [x] introduce `BlobStorage` interface,
|
||||
- [x] convert draft attachments from absolute paths to logical object keys,
|
||||
- [x] implement filesystem backend,
|
||||
- [x] retain existing behavior for single-node installations,
|
||||
- [x] add storage health checks.
|
||||
|
||||
## Phase 2 — S3 backend
|
||||
|
||||
- [ ] implement S3-compatible backend,
|
||||
- [ ] streaming upload/download,
|
||||
- [ ] retries/timeouts/checksums,
|
||||
- [x] implement S3-compatible backend,
|
||||
- [x] streaming upload/download,
|
||||
- [x] retries/timeouts/checksums,
|
||||
- [ ] SILO Compose integration validation,
|
||||
- [ ] MinIO integration validation,
|
||||
- [ ] AWS S3 integration validation,
|
||||
- [ ] Ceph RGW compatibility validation,
|
||||
- [ ] migration command from filesystem to S3,
|
||||
- [ ] remove hard `cluster_shared_data_dir` requirement when storage backend is cluster-safe.
|
||||
- [x] migration command from filesystem to S3,
|
||||
- [x] validate cluster safety directly from `storage.backend` and `storage.shared`.
|
||||
|
||||
## Phase 3 — search abstraction
|
||||
|
||||
- [ ] introduce `SearchBackend` interface,
|
||||
- [ ] move current Tantivy code behind `local_tantivy`,
|
||||
- [ ] preserve `disabled` backend,
|
||||
- [ ] reject `local_tantivy` in cluster mode,
|
||||
- [ ] define canonical indexed message document/schema.
|
||||
- [x] introduce `SearchBackend` interface,
|
||||
- [x] move current Tantivy code behind `local_tantivy`,
|
||||
- [x] preserve `disabled` backend,
|
||||
- [x] reject `local_tantivy` in cluster mode,
|
||||
- [x] define canonical indexed message document/schema.
|
||||
|
||||
## Phase 4 — durable index events
|
||||
|
||||
- [ ] add SQL search outbox/queue,
|
||||
- [ ] add index worker locking/concurrency,
|
||||
- [ ] retries/backoff,
|
||||
- [ ] delete events,
|
||||
- [ ] reindex command,
|
||||
- [ ] index lag/error metrics.
|
||||
- [x] add SQL search outbox/queue,
|
||||
- [x] add index worker locking/concurrency,
|
||||
- [x] retries/backoff,
|
||||
- [x] delete events,
|
||||
- [x] reindex command,
|
||||
- [x] index lag/error metrics.
|
||||
|
||||
## Phase 5 — external index adapters
|
||||
|
||||
Recommended evaluation order:
|
||||
|
||||
RustBox 0.2.0 selects **OpenSearch** as the supported external adapter because its update/delete/reindex semantics map directly to mutable mailbox documents. Quickwit and Solr remain optional future adapters.
|
||||
|
||||
- [ ] Quickwit proof of concept,
|
||||
- [ ] OpenSearch adapter,
|
||||
- [ ] compare update/delete/reindex behavior and operational footprint,
|
||||
- [ ] choose documented recommended backend,
|
||||
- [x] OpenSearch adapter,
|
||||
- [x] compare update/delete/reindex behavior and operational footprint,
|
||||
- [x] choose documented recommended backend,
|
||||
- [ ] Solr adapter only if there is real deployment demand.
|
||||
|
||||
## Phase 6 — cluster PGP
|
||||
|
||||
- [ ] design dedicated encrypted key backend,
|
||||
- [ ] keep PGP disabled in cluster mode until completed,
|
||||
- [x] keep PGP disabled in cluster mode until completed,
|
||||
- [ ] evaluate Vault/KMS/SQL-encrypted/S3-encrypted variants,
|
||||
- [ ] add migration procedure for existing local keys.
|
||||
|
||||
@@ -770,16 +752,16 @@ The work is complete when all of the following are true:
|
||||
|
||||
- [ ] three or more RustBox nodes can run with no writable shared POSIX filesystem,
|
||||
- [ ] nodes can be destroyed/recreated without losing attachments or search state,
|
||||
- [ ] S3-compatible storage works for draft attachments,
|
||||
- [ ] local filesystem remains supported for single-node mode,
|
||||
- [ ] `cluster_mode = true` accepts S3-backed storage without `cluster_shared_data_dir = true`,
|
||||
- [ ] local Tantivy is rejected in cluster mode,
|
||||
- [ ] external index failure does not corrupt SQL/cache state,
|
||||
- [ ] full index rebuild is possible,
|
||||
- [ ] index events are durable and retryable,
|
||||
- [ ] SQL remains the authoritative cache/data source,
|
||||
- [ ] no PGP private key depends on node-local filesystem in supported cluster mode,
|
||||
- [ ] configuration and migration paths are documented for every supported deployment variant.
|
||||
- [ ] S3-compatible storage works for draft attachments (live SILO/MinIO/AWS validation pending),
|
||||
- [x] local filesystem remains supported for single-node mode,
|
||||
- [x] `cluster_mode = true` accepts S3-backed storage without any legacy shared-data-dir switch,
|
||||
- [x] local Tantivy is rejected in cluster mode,
|
||||
- [x] external index failure does not corrupt SQL/cache state,
|
||||
- [x] full index rebuild is possible,
|
||||
- [x] index events are durable and retryable,
|
||||
- [x] SQL remains the authoritative cache/data source,
|
||||
- [x] no PGP private key depends on node-local filesystem in supported cluster mode,
|
||||
- [x] configuration and migration paths are documented for every supported deployment variant.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -19,8 +19,6 @@ database_backend = "sqlite" # sqlite | mysql | pgsql
|
||||
# database_url = "postgres://rustbox:secret@db.example.com/rustbox"
|
||||
session_backend = "database" # database | redis
|
||||
cluster_mode = false
|
||||
cluster_shared_data_dir = false
|
||||
search_index_backend = "tantivy"
|
||||
pgp_enabled = true
|
||||
# For shared database sessions, Redis sessions, or cluster mode provide the same 32-byte key (base64) on all nodes.
|
||||
# session_encryption_key = "..."
|
||||
@@ -48,5 +46,3 @@ sieve_allow_insecure_auth = false
|
||||
sieve_timeout_secs = 10
|
||||
|
||||
trusted_proxies = "127.0.0.1,::1"
|
||||
imap_host = ""
|
||||
smtp_host = ""
|
||||
+41
-6
@@ -1,4 +1,4 @@
|
||||
# RustBox 0.1.2
|
||||
# RustBox 0.2.0
|
||||
host = "127.0.0.1"
|
||||
port = 3001
|
||||
environment = "production"
|
||||
@@ -21,11 +21,8 @@ database_backend = "sqlite" # sqlite | mysql | pgsql
|
||||
# database_url = "postgres://rustbox:secret@db.example.com/rustbox"
|
||||
session_backend = "database" # database | redis
|
||||
cluster_mode = false
|
||||
# Required acknowledgement in cluster mode: data_dir must be shared by every node
|
||||
# because draft attachments are stored there.
|
||||
cluster_shared_data_dir = false
|
||||
# Built-in Tantivy is single-node. Cluster mode requires disabled.
|
||||
search_index_backend = "tantivy" # tantivy | disabled
|
||||
|
||||
|
||||
# For shared database sessions, Redis sessions, or cluster mode provide the same 32-byte key (base64) on all nodes.
|
||||
# session_encryption_key = "..."
|
||||
# Direct Redis session backend:
|
||||
@@ -92,3 +89,41 @@ pgp_max_key_bytes = 1048576
|
||||
# Private-key export is deliberately disabled unless the administrator opts in.
|
||||
pgp_allow_private_export = false
|
||||
|
||||
# RustBox 0.2 blob/search backends
|
||||
# Blob storage is independent from the SQL database. PostgreSQL/MySQL can still use
|
||||
# local filesystem storage in single-node mode. In cluster mode use S3-compatible
|
||||
# storage or a genuinely shared POSIX filesystem (NFS/CephFS/GlusterFS).
|
||||
[storage]
|
||||
backend = "filesystem" # filesystem | s3
|
||||
# path = "/var/lib/rustbox/blobs" # defaults to data_dir for filesystem
|
||||
shared = false # true only when every RustBox node sees the same filesystem
|
||||
prefix = ""
|
||||
# S3-compatible settings:
|
||||
# bucket = "rustbox"
|
||||
# region = "us-east-1"
|
||||
# endpoint = "https://s3.example.com" # MinIO/Ceph RGW/etc. optional for AWS S3
|
||||
# path_style = false
|
||||
# allow_http = false
|
||||
request_timeout_secs = 30
|
||||
connect_timeout_secs = 5
|
||||
max_retries = 3
|
||||
retry_timeout_secs = 60
|
||||
# access_key_id = "..." # AWS_* environment variables are also supported
|
||||
# secret_access_key = "..."
|
||||
# server_side_encryption = "AES256" # or "aws:kms"
|
||||
# kms_key_id = "..."
|
||||
|
||||
# Search is independent from blob storage. Local Tantivy is single-node only.
|
||||
[search]
|
||||
backend = "local_tantivy" # disabled | local_tantivy | opensearch
|
||||
# path = "/var/lib/rustbox/search" # defaults to data_dir for local_tantivy
|
||||
# urls = ["https://opensearch.example.com:9200"]
|
||||
index_prefix = "rustbox"
|
||||
required = false
|
||||
allow_insecure_http = false
|
||||
request_timeout_secs = 10
|
||||
worker_interval_secs = 2
|
||||
worker_batch_size = 100
|
||||
worker_max_attempts = 8
|
||||
# username = "rustbox"
|
||||
# password = "..."
|
||||
@@ -0,0 +1,4 @@
|
||||
target
|
||||
.git
|
||||
RustBox*.zip
|
||||
.dev
|
||||
@@ -0,0 +1,13 @@
|
||||
FROM rust:1.88-bookworm AS builder
|
||||
WORKDIR /build
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends pkg-config libssl-dev ca-certificates && rm -rf /var/lib/apt/lists/*
|
||||
COPY . .
|
||||
RUN cargo generate-lockfile && cargo build --release --locked
|
||||
|
||||
FROM debian:bookworm-slim
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates libssl3 && rm -rf /var/lib/apt/lists/*
|
||||
RUN useradd --system --uid 10001 --create-home --home-dir /var/lib/rustbox rustbox
|
||||
COPY --from=builder /build/target/release/rustbox /usr/local/bin/rustbox
|
||||
USER rustbox
|
||||
EXPOSE 3001
|
||||
ENTRYPOINT ["/usr/local/bin/rustbox"]
|
||||
@@ -0,0 +1,15 @@
|
||||
# RustBox 0.2.0 Docker variants
|
||||
|
||||
The example images are version-pinned where practical so a release does not silently change its integration stack. The bundled SILO topology is for local/integration validation of the S3 API; production deployments should use a security-reviewed S3-compatible service and their own lifecycle/HA policy.
|
||||
|
||||
Run commands from `docker/`.
|
||||
|
||||
- `docker compose -f compose-single-sqlite.yml up --build` — single node, SQLite, local filesystem blobs, local Tantivy.
|
||||
- `docker compose -f compose-postgres-s3-opensearch.yml up --build` — cluster-ready application shape with PostgreSQL, SILO (S3-compatible blobs) and OpenSearch. The RustBox container has no persistent `data_dir` volume.
|
||||
- `NFS_SERVER=10.0.0.10 NFS_EXPORT=/exports/rustbox docker compose -f compose-mysql-nfs-opensearch.yml up --build` — MySQL, real shared POSIX/NFS blob storage and OpenSearch.
|
||||
|
||||
Set `IMAP_HOST` and `SMTP_HOST` for the mail system. Replace all example passwords and `SESSION_ENCRYPTION_KEY` before production use. The OpenSearch examples disable its security plugin only to keep the local Compose topology self-contained; production should use TLS/authentication and `search.urls` with HTTPS.
|
||||
|
||||
The NFS variant deliberately uses an NFS volume driver instead of a local Docker named volume: a local named volume is not shared across Docker hosts and therefore is not a valid multi-node RustBox blob backend.
|
||||
|
||||
The two cluster-ready Compose examples intentionally run one RustBox application container so they remain directly runnable on a single Docker host. To run multiple RustBox application nodes, place them behind an external load balancer (or adapt the Compose networking) and do not publish the same host port from every node; the shared SQL/blob/OpenSearch services are the cluster-safe backing topology.
|
||||
@@ -0,0 +1,62 @@
|
||||
services:
|
||||
mysql:
|
||||
image: mysql:8.4
|
||||
environment:
|
||||
MYSQL_DATABASE: rustbox
|
||||
MYSQL_USER: rustbox
|
||||
MYSQL_PASSWORD: ${MYSQL_PASSWORD:-rustbox-dev-password}
|
||||
MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD:-rustbox-root-password}
|
||||
healthcheck:
|
||||
test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-uroot", "-p${MYSQL_ROOT_PASSWORD:-rustbox-root-password}"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
volumes:
|
||||
- mysql-data:/var/lib/mysql
|
||||
|
||||
opensearch:
|
||||
image: opensearchproject/opensearch:3.9.0
|
||||
environment:
|
||||
discovery.type: single-node
|
||||
DISABLE_INSTALL_DEMO_CONFIG: "true"
|
||||
DISABLE_SECURITY_PLUGIN: "true"
|
||||
OPENSEARCH_JAVA_OPTS: -Xms512m -Xmx512m
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -fsS http://localhost:9200/_cluster/health >/dev/null || exit 1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
volumes:
|
||||
- opensearch-data:/usr/share/opensearch/data
|
||||
|
||||
rustbox:
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: docker/Dockerfile
|
||||
depends_on:
|
||||
mysql:
|
||||
condition: service_healthy
|
||||
opensearch:
|
||||
condition: service_healthy
|
||||
ports:
|
||||
- "3001:3001"
|
||||
environment:
|
||||
RUSTBOX_CONFIG: /etc/rustbox/rustbox.toml
|
||||
DATABASE_URL: mysql://rustbox:${MYSQL_PASSWORD:-rustbox-dev-password}@mysql:3306/rustbox
|
||||
SESSION_ENCRYPTION_KEY: ${SESSION_ENCRYPTION_KEY:-MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=}
|
||||
IMAP_HOST: ${IMAP_HOST:-mail.example.com}
|
||||
SMTP_HOST: ${SMTP_HOST:-mail.example.com}
|
||||
volumes:
|
||||
- ./configs/mysql-nfs-opensearch.toml:/etc/rustbox/rustbox.toml:ro
|
||||
- rustbox-nfs:/mnt/rustbox-shared
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
mysql-data:
|
||||
opensearch-data:
|
||||
rustbox-nfs:
|
||||
driver: local
|
||||
driver_opts:
|
||||
type: nfs
|
||||
o: addr=${NFS_SERVER:?set NFS_SERVER},rw,nfsvers=4
|
||||
device: :${NFS_EXPORT:-/rustbox}
|
||||
@@ -0,0 +1,82 @@
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17-bookworm
|
||||
environment:
|
||||
POSTGRES_DB: rustbox
|
||||
POSTGRES_USER: rustbox
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-rustbox-dev-password}
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U rustbox -d rustbox"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
volumes:
|
||||
- postgres-data:/var/lib/postgresql/data
|
||||
|
||||
s3:
|
||||
image: docker.io/pgsty/silo:RELEASE.2026-10-26T00-00-00Z
|
||||
command: server /data --console-address :9001
|
||||
environment:
|
||||
MINIO_ROOT_USER: ${S3_ACCESS_KEY:-rustbox}
|
||||
MINIO_ROOT_PASSWORD: ${S3_SECRET_KEY:-rustbox-s3-dev-password}
|
||||
healthcheck:
|
||||
test: ["CMD", "silo", "healthcheck", "ready"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
volumes:
|
||||
- s3-data:/data
|
||||
|
||||
s3-init:
|
||||
image: docker.io/pgsty/mc:RELEASE.2026-10-26T00-00-00Z
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
MC_HOST_local: http://${S3_ACCESS_KEY:-rustbox}:${S3_SECRET_KEY:-rustbox-s3-dev-password}@s3:9000
|
||||
command: ["mb", "--ignore-existing", "local/rustbox"]
|
||||
|
||||
opensearch:
|
||||
image: opensearchproject/opensearch:3.9.0
|
||||
environment:
|
||||
discovery.type: single-node
|
||||
DISABLE_INSTALL_DEMO_CONFIG: "true"
|
||||
DISABLE_SECURITY_PLUGIN: "true"
|
||||
OPENSEARCH_JAVA_OPTS: -Xms512m -Xmx512m
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -fsS http://localhost:9200/_cluster/health >/dev/null || exit 1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
volumes:
|
||||
- opensearch-data:/usr/share/opensearch/data
|
||||
|
||||
rustbox:
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: docker/Dockerfile
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
opensearch:
|
||||
condition: service_healthy
|
||||
ports:
|
||||
- "3001:3001"
|
||||
environment:
|
||||
RUSTBOX_CONFIG: /etc/rustbox/rustbox.toml
|
||||
DATABASE_URL: postgres://rustbox:${POSTGRES_PASSWORD:-rustbox-dev-password}@postgres:5432/rustbox
|
||||
SESSION_ENCRYPTION_KEY: ${SESSION_ENCRYPTION_KEY:-MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=}
|
||||
AWS_ACCESS_KEY_ID: ${S3_ACCESS_KEY:-rustbox}
|
||||
AWS_SECRET_ACCESS_KEY: ${S3_SECRET_KEY:-rustbox-s3-dev-password}
|
||||
IMAP_HOST: ${IMAP_HOST:-mail.example.com}
|
||||
SMTP_HOST: ${SMTP_HOST:-mail.example.com}
|
||||
volumes:
|
||||
- ./configs/postgres-s3-opensearch.toml:/etc/rustbox/rustbox.toml:ro
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
postgres-data:
|
||||
s3-data:
|
||||
opensearch-data:
|
||||
@@ -0,0 +1,18 @@
|
||||
services:
|
||||
rustbox:
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: docker/Dockerfile
|
||||
ports:
|
||||
- "3001:3001"
|
||||
environment:
|
||||
RUSTBOX_CONFIG: /etc/rustbox/rustbox.toml
|
||||
IMAP_HOST: ${IMAP_HOST:-mail.example.com}
|
||||
SMTP_HOST: ${SMTP_HOST:-mail.example.com}
|
||||
volumes:
|
||||
- ./configs/single-sqlite.toml:/etc/rustbox/rustbox.toml:ro
|
||||
- rustbox-data:/var/lib/rustbox
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
rustbox-data:
|
||||
@@ -0,0 +1,22 @@
|
||||
host = "0.0.0.0"
|
||||
port = 3001
|
||||
environment = "production"
|
||||
database_backend = "mysql"
|
||||
session_backend = "database"
|
||||
cluster_mode = true
|
||||
pgp_enabled = false
|
||||
|
||||
[storage]
|
||||
backend = "filesystem"
|
||||
path = "/mnt/rustbox-shared"
|
||||
shared = true
|
||||
|
||||
[search]
|
||||
backend = "opensearch"
|
||||
urls = ["http://opensearch:9200"]
|
||||
index_prefix = "rustbox"
|
||||
allow_insecure_http = true
|
||||
required = true
|
||||
worker_interval_secs = 2
|
||||
worker_batch_size = 100
|
||||
worker_max_attempts = 8
|
||||
@@ -0,0 +1,26 @@
|
||||
host = "0.0.0.0"
|
||||
port = 3001
|
||||
environment = "production"
|
||||
database_backend = "pgsql"
|
||||
session_backend = "database"
|
||||
cluster_mode = true
|
||||
pgp_enabled = false
|
||||
|
||||
[storage]
|
||||
backend = "s3"
|
||||
bucket = "rustbox"
|
||||
region = "us-east-1"
|
||||
endpoint = "http://s3:9000"
|
||||
path_style = true
|
||||
allow_http = true
|
||||
prefix = "mail"
|
||||
|
||||
[search]
|
||||
backend = "opensearch"
|
||||
urls = ["http://opensearch:9200"]
|
||||
index_prefix = "rustbox"
|
||||
allow_insecure_http = true
|
||||
required = true
|
||||
worker_interval_secs = 2
|
||||
worker_batch_size = 100
|
||||
worker_max_attempts = 8
|
||||
@@ -0,0 +1,17 @@
|
||||
host = "0.0.0.0"
|
||||
port = 3001
|
||||
environment = "production"
|
||||
data_dir = "/var/lib/rustbox"
|
||||
database_backend = "sqlite"
|
||||
session_backend = "database"
|
||||
cluster_mode = false
|
||||
pgp_enabled = true
|
||||
|
||||
[storage]
|
||||
backend = "filesystem"
|
||||
path = "/var/lib/rustbox/blobs"
|
||||
shared = false
|
||||
|
||||
[search]
|
||||
backend = "local_tantivy"
|
||||
path = "/var/lib/rustbox/search"
|
||||
@@ -1,6 +1,18 @@
|
||||
# RustBox 0.0.4 - implementation status
|
||||
# RustBox 0.2.0 - implementation status
|
||||
|
||||
This file tracks the internal `0.0.4` baseline. The user-approved `docs/RUSTBOX_PLAN.md` remains authoritative. A feature is only listed as implemented when working code exists in this source tree; partial items are called out explicitly.
|
||||
This file tracks the `0.2.0` source snapshot. Storage/search scope and outstanding acceptance work are tracked in `TODO.md`. A feature is only listed as implemented when working code exists in this source tree; partial items are called out explicitly.
|
||||
|
||||
## 0.2.0 storage/search architecture
|
||||
|
||||
- SQL backend selection (`sqlite`/`mysql`/`pgsql`) is independent from blob storage and search backend selection.
|
||||
- Cluster mode clears an implicit/default `data_dir`; PostgreSQL/MySQL single-node deployments may still intentionally use `data_dir`-backed filesystem storage and local Tantivy.
|
||||
- Blob storage supports local/shared POSIX filesystems and S3-compatible object storage. Shared POSIX is an explicit capability (`storage.shared=true`) suitable for operator-provided NFS/CephFS/GlusterFS mounts.
|
||||
- Draft attachment upload/read/delete/snapshot/send paths use the `BlobStorage` abstraction and logical object keys instead of assuming node-local paths.
|
||||
- OpenSearch is available as the cluster-capable indexer. Local Tantivy remains single-node only; search may also be disabled.
|
||||
- Search mutations use a durable SQL outbox with claim/lease, retry/backoff and dead-letter state; the index remains rebuildable derived state.
|
||||
- V047 adds attachment checksum metadata and the search outbox across SQLite, MySQL and PostgreSQL.
|
||||
- Docker Compose examples cover standalone SQLite, PostgreSQL + SILO (S3-compatible) + OpenSearch, and MySQL + NFS + OpenSearch.
|
||||
- Live SILO/MinIO/AWS S3/Ceph RGW, live NFS multi-host, live OpenSearch and full multi-node failover acceptance remain release-environment validation items and are not represented as completed here.
|
||||
|
||||
## Implemented / wired in this working snapshot
|
||||
|
||||
@@ -39,7 +51,7 @@ This file tracks the internal `0.0.4` baseline. The user-approved `docs/RUSTBOX_
|
||||
- HTML, plain-text and Markdown compose modes. Markdown keeps its source text for the plain alternative and generates escaped/sanitized HTML for preview and delivery without accepting raw Markdown HTML.
|
||||
- Expanded HTML toolbar: bold, italic, underline, strike, lists, quote, alignment, links, font, size, color, clear formatting, undo/redo.
|
||||
- Draft autosave, explicit recovery of saved/browser-emergency drafts, and attachment upload/delete. Deliberate IMAP Drafts snapshots carry a stable RustBox draft identifier; the previous remote snapshot is removed only after the new UID is resolved unambiguously and UIDVALIDITY still matches.
|
||||
- Drag-and-drop attachments with visible upload progress. Attachment uploads stream multipart chunks into `.part` files with an enforced size limit, atomic final rename and stale/incomplete temporary-file cleanup instead of buffering the whole attachment in memory.
|
||||
- Drag-and-drop attachments with visible upload progress. Attachment uploads stream multipart chunks through `BlobStorage` with an enforced size limit instead of buffering the whole attachment in memory; filesystem and S3-compatible backends share the same logical-object flow.
|
||||
- Inline image upload using CID attachments, including paste and drag/drop of PNG/JPEG/GIF/WebP images into the composer. During editing, CID images use authenticated same-origin draft-attachment preview URLs and are serialized back to `cid:` references for saved/sent MIME content. Inline images can be resized, aligned left/center/right/inline and cropped in-app (GIF cropping is intentionally disabled to preserve animation); display-only resizing does not rewrite the source file. Unused CID attachments are cleaned when their image is removed from the editor.
|
||||
- Attachment reminder for common English/Polish wording.
|
||||
- Multiple identities.
|
||||
@@ -104,7 +116,7 @@ This file tracks the internal `0.0.4` baseline. The user-approved `docs/RUSTBOX_
|
||||
|
||||
- Fixed the `E0499` mutable-borrow failure in `mark_all_read`: the `uid_store()` stream is dropped before `session.logout()`.
|
||||
- Fixed a duplicate `else` introduced in the calendar delete handler during the calendar expansion.
|
||||
- All executable Rust SQL is centralized under `src/queries/{sqlite,mysql,pgsql}/queries.rs`, including session queries.
|
||||
- All executable Rust SQL is centralized under `src/queries/{sqlite,mysql,pgsql}/queries.rs`, including session queries.
|
||||
## Partially implemented - keep visible as incomplete
|
||||
|
||||
- Sieve visual builder now supports a three-condition nested AND/OR group, negation, two actions, capability-aware disabling, managed-rule ordering and scheduled vacation dates. An arbitrary-depth condition tree, arbitrary-length action stack and full capability-aware condition/action matrix remain unfinished.
|
||||
@@ -125,7 +137,7 @@ This file tracks the internal `0.0.4` baseline. The user-approved `docs/RUSTBOX_
|
||||
- Full CardDAV synchronization.
|
||||
- Password-change adapters remain unfinished. Session inventory/device metadata is now persistent across restarts and shared-session backends support cross-node listing/revocation.
|
||||
- Diagnostics now includes IMAP reconnect, SMTP connectivity, index rebuild and last local folder/message synchronization metadata; deeper background task/queue metrics and repair workflows remain.
|
||||
- Full security, acceptance and performance matrix from `docs/RUSTBOX_PLAN.md`.
|
||||
- Full live security, acceptance and performance matrix remains release-environment work tracked in `TODO.md`.
|
||||
|
||||
## UI reliability / workflow fixes in the current snapshot
|
||||
|
||||
|
||||
@@ -26,3 +26,9 @@ RUSTBOX_CONFIG=./config.dev.toml ./target/release/rustbox --check-config
|
||||
## Browser asset build
|
||||
|
||||
Browser JavaScript source lives in functional modules under `src/web/js/`, ordered by `src/web/js/manifest.txt`. `build.rs` concatenates those modules into the embedded `rustbox-ui.js` asset during every Cargo build. The repository has no duplicate `ui/` snapshot; `src/web/` is the only browser UI source.
|
||||
|
||||
## Updating an existing checkout to 0.2.0
|
||||
|
||||
Do not overlay the release with `cp *` or another command that skips dotfiles. That can retain stale `.github`, `.env.example`, or `Cargo.lock` files.
|
||||
|
||||
Use a clean checkout/extraction, or a dotfile-aware copy such as `rsync -a` / `cp -a source/. destination/`. The source release intentionally does not ship `Cargo.lock`; CI and `scripts/build-release.sh` generate it before running `--locked` Cargo commands.
|
||||
@@ -0,0 +1,86 @@
|
||||
# RustBox 0.2.0 storage/search operations
|
||||
|
||||
RustBox 0.2.0 treats SQL, blob storage and search as independent capabilities. The database engine does not decide whether `data_dir` is allowed.
|
||||
|
||||
## Deployment matrix
|
||||
|
||||
| Mode | SQL | Blob storage | Search | Persistent `data_dir` |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Single node | SQLite/MySQL/PostgreSQL | local filesystem | local Tantivy | normally yes |
|
||||
| Single node | MySQL/PostgreSQL | S3 | OpenSearch/disabled | no, if PGP is disabled and no local path is selected |
|
||||
| Cluster | MySQL/PostgreSQL | S3-compatible | OpenSearch/disabled | disabled by default |
|
||||
| Cluster | MySQL/PostgreSQL | shared POSIX (NFS/CephFS/GlusterFS) | OpenSearch/disabled | disabled by default |
|
||||
|
||||
In cluster mode an implicit/default `data_dir` is cleared. An explicitly configured `data_dir` is still accepted for deliberate local/migration use, but cluster validation rejects local-only features. PostgreSQL/MySQL remain valid with `data_dir` in single-node deployments.
|
||||
|
||||
## Blob storage
|
||||
|
||||
`storage.backend="filesystem"` supports both local and shared POSIX filesystems. `storage.shared=true` is an operator assertion that every RustBox node sees the same path and the same files; RustBox cannot turn a local disk or a host-local Docker volume into shared storage.
|
||||
|
||||
For NFS/CephFS/GlusterFS:
|
||||
|
||||
- mount the same logical path on every node;
|
||||
- keep ownership/permissions consistent;
|
||||
- validate rename/read/write/delete behavior before production;
|
||||
- monitor filesystem availability and capacity outside RustBox.
|
||||
|
||||
`storage.backend="s3"` supports S3-compatible object storage. The backend streams uploads, applies request/retry timeouts, records SHA-256 attachment metadata and supports S3 server-side encryption options. For multipart uploads interrupted by process/node loss, configure an object-store lifecycle rule to expire abandoned multipart uploads.
|
||||
|
||||
Attachment SQL rows store logical object keys. This keeps SQL independent from a node-specific filesystem path.
|
||||
|
||||
## Search and outbox
|
||||
|
||||
OpenSearch is the cluster-capable indexer. Every indexed document is scoped by `user_hash`; queries add a server-side tenant filter and deterministic document IDs are routed by tenant.
|
||||
|
||||
Search changes use a durable SQL outbox:
|
||||
|
||||
1. message mutations that already use a SQL transaction record the indexing event in that same transaction; other derived-state updates enqueue through the same tenant connection;
|
||||
2. workers claim events with a lease;
|
||||
3. OpenSearch operations are idempotent;
|
||||
4. failures are retried with backoff;
|
||||
5. exhausted events remain visible as dead-letter state.
|
||||
|
||||
Search is derived state. A full rebuild is available with:
|
||||
|
||||
```bash
|
||||
rustbox search reindex
|
||||
```
|
||||
|
||||
The rebuild removes superseded outbox events older than the rebuild start and keeps events created or refreshed during the rebuild. In a multi-node deployment run a full reindex in a controlled maintenance window and quiesce search workers on the other RustBox nodes; otherwise a direct rebuild can race an already claimed event from another node. Normal outbox processing does not require this maintenance step.
|
||||
|
||||
If `search.required=false`, an unavailable OpenSearch backend may leave the application usable while the outbox grows; readiness reports degraded search and workers retry later. With `search.required=true`, search health is a startup/readiness requirement.
|
||||
|
||||
## Failure behavior
|
||||
|
||||
- SQL unavailable: authoritative mailbox/session/account operations cannot proceed.
|
||||
- S3/shared filesystem unavailable: attachment upload/download/delete and storage readiness fail; SQL data is not silently rewritten to another backend.
|
||||
- OpenSearch unavailable: outbox events remain durable and retry; SQL remains the source for rebuilding the index.
|
||||
- One RustBox node lost: S3/shared POSIX attachments and shared SQL remain available to other nodes; local-only Tantivy/SQLite are rejected in cluster mode.
|
||||
- Shared POSIX mounted only on some nodes: configuration may look valid but runtime attachment operations will fail or diverge. `storage.shared=true` must only be used after infrastructure-level verification.
|
||||
|
||||
## Attachment migration
|
||||
|
||||
Migrate old filesystem attachments to the configured backend with a dry run first:
|
||||
|
||||
```bash
|
||||
rustbox storage migrate --from filesystem:/old/data/root --to configured --dry-run
|
||||
rustbox storage migrate --from filesystem:/old/data/root --to configured --verify
|
||||
```
|
||||
|
||||
The migration streams each file, verifies size/SHA-256 when requested and changes the SQL row only after a successful destination write. Source files are not deleted automatically. Run the migration during a controlled maintenance window or otherwise prevent concurrent attachment mutation for the affected drafts.
|
||||
|
||||
## Cluster prerequisites
|
||||
|
||||
A supported cluster requires:
|
||||
|
||||
- MySQL or PostgreSQL as shared SQL;
|
||||
- S3-compatible storage or a genuinely shared POSIX filesystem;
|
||||
- OpenSearch or disabled full-text search;
|
||||
- `pgp_enabled=false` until cluster-safe private-key storage exists;
|
||||
- the same explicit `session_encryption_key` on every node.
|
||||
|
||||
Cross-node realtime event fan-out remains separate from blob/search storage and is not implemented by the 0.2.0 storage work.
|
||||
|
||||
## Validation status
|
||||
|
||||
The source package includes `scripts/validate-release-static.py` for offline structural checks. Full release validation still requires a Rust 1.88 build host and live service tests against the exact PostgreSQL/MySQL, S3/NFS and OpenSearch infrastructure used in production. Live SILO/MinIO/AWS/Ceph RGW and multi-node acceptance items remain intentionally unchecked in `TODO.md` until executed.
|
||||
@@ -66,13 +66,13 @@ fn main() {
|
||||
}
|
||||
|
||||
println!("{}", "-".repeat(60));
|
||||
println!("Passed: {}, Failed: {}", passed, failed);
|
||||
println!("Passed: {passed}, Failed: {failed}");
|
||||
}
|
||||
|
||||
fn test_file(path: &Path, expected: &str) -> (bool, String) {
|
||||
let content = match fs::read(path) {
|
||||
Ok(c) => c,
|
||||
Err(e) => return (false, format!("Error reading: {}", e)),
|
||||
Err(e) => return (false, format!("Error reading: {e}")),
|
||||
};
|
||||
|
||||
let msg = match MessageParser::default().parse(&content) {
|
||||
|
||||
+17
-20
@@ -1,45 +1,40 @@
|
||||
# Database schemas and migrations
|
||||
|
||||
RustBox 0.1.2 has one database backend for the whole application:
|
||||
RustBox 0.2.0 keeps SQL, blob storage and search as independent concerns:
|
||||
|
||||
```text
|
||||
database_backend = sqlite | mysql | pgsql
|
||||
session_backend = database | redis
|
||||
storage.backend = filesystem | s3
|
||||
search.backend = disabled | local_tantivy | opensearch
|
||||
```
|
||||
|
||||
When `session_backend = "database"`, sessions live in the same configured SQL database as the application data. `session_backend = "redis"` moves only session persistence to Redis; Redis Sentinel is a Redis discovery option and has no SQL schema.
|
||||
When `session_backend = "database"`, sessions live in the selected SQL database. `session_backend = "redis"` moves only session persistence to Redis/Redis Sentinel.
|
||||
|
||||
## Layout
|
||||
|
||||
```text
|
||||
migrations/
|
||||
├── sqlite/
|
||||
│ └── schema.sql
|
||||
│ ├── schema.sql
|
||||
│ └── V047__blob_keys_and_search_outbox.sql
|
||||
├── mysql/
|
||||
│ └── schema.sql
|
||||
│ ├── schema.sql
|
||||
│ └── V047__blob_keys_and_search_outbox.sql
|
||||
└── pgsql/
|
||||
└── schema.sql
|
||||
├── schema.sql
|
||||
└── V047__blob_keys_and_search_outbox.sql
|
||||
```
|
||||
|
||||
Every `schema.sql` is the complete schema for that engine. It contains:
|
||||
Each `schema.sql` is the current complete schema for fresh installations. The historical compatibility baseline remains **V046**. RustBox 0.2.0 adds **V047**, which adds attachment SHA-256 metadata and the durable search outbox used by external indexing.
|
||||
|
||||
- global tables (`sessions`, `rustbox_accounts`), and
|
||||
- the full account-data baseline at schema version 46.
|
||||
Account data is isolated inside the selected physical database with a deterministic, collision-checked table prefix derived from the account hash. The `{{p}}` marker in tenant SQL is replaced by RustBox at provisioning/migration time. Sessions are global and therefore are not prefixed.
|
||||
|
||||
Account data is isolated inside the selected physical database with a deterministic, collision-checked table prefix derived from the account hash. The `{{p}}` marker in the tenant part of each schema is replaced by RustBox at provisioning time. Sessions are global and therefore are not prefixed.
|
||||
## Runtime migrations
|
||||
|
||||
Every schema-changing release must add the same logical migration for all supported SQL engines and register it in `src/db/pool.rs`. Do not update only `schema.sql`: fresh installs use the baseline while existing installations use the numbered migration path.
|
||||
|
||||
## Future migrations
|
||||
|
||||
The next application schema change is **V047**. A schema-changing release must add the same logical migration for every supported SQL engine, for example:
|
||||
|
||||
```text
|
||||
migrations/sqlite/V047__description.sql
|
||||
migrations/mysql/V047__description.sql
|
||||
migrations/pgsql/V047__description.sql
|
||||
```
|
||||
|
||||
Tenant migration SQL uses `{{p}}` for account tables. Register each new migration in the corresponding migration array in `src/db/pool.rs` and keep the version numbers aligned across all three engines. Do not edit `schema.sql` alone for an upgrade that must work on existing installations; update both the baseline and the versioned migration.
|
||||
The next migration number after RustBox 0.2.0 is **V048**.
|
||||
|
||||
Redis and Redis Sentinel do not use SQL migrations.
|
||||
|
||||
@@ -49,3 +44,5 @@ Redis and Redis Sentinel do not use SQL migrations.
|
||||
- MySQL requires `database_url` and targets MySQL 8.0.13 or newer.
|
||||
- PostgreSQL requires `database_url`.
|
||||
- MySQL/PostgreSQL are full application backends, not session-only databases.
|
||||
- The SQL engine does not choose blob/search storage. PostgreSQL/MySQL may still use local filesystem/Tantivy in single-node mode.
|
||||
- Cluster mode requires a shared MySQL/PostgreSQL database plus cluster-safe blob/search choices; local `data_dir` is not inherently required.
|
||||
@@ -0,0 +1,18 @@
|
||||
ALTER TABLE {{p}}draft_attachments ADD COLUMN checksum_sha256 VARCHAR(64) NULL;
|
||||
|
||||
CREATE TABLE {{p}}search_outbox (
|
||||
id VARCHAR(64) PRIMARY KEY,
|
||||
generation VARCHAR(64) NOT NULL,
|
||||
event_type VARCHAR(16) NOT NULL,
|
||||
message_uid BIGINT NOT NULL,
|
||||
folder TEXT NOT NULL,
|
||||
payload LONGTEXT NULL,
|
||||
attempts BIGINT NOT NULL DEFAULT 0,
|
||||
available_at BIGINT NOT NULL DEFAULT 0,
|
||||
claimed_until BIGINT NOT NULL DEFAULT 0,
|
||||
dead_letter BIGINT NOT NULL DEFAULT 0,
|
||||
last_error TEXT NULL,
|
||||
created_at BIGINT NOT NULL,
|
||||
updated_at BIGINT NOT NULL,
|
||||
INDEX idx_search_outbox_ready (dead_letter, available_at, claimed_until, updated_at)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||
@@ -1,7 +1,7 @@
|
||||
-- RustBox 0.1.2 MySQL schema.
|
||||
-- RustBox 0.2.0 MySQL schema.
|
||||
-- One physical database contains global sessions plus isolated per-account tables.
|
||||
-- Application tables are materialized by replacing {{p}} with a collision-checked prefix.
|
||||
-- Baseline application schema version: 46. Future migrations start at V047.
|
||||
-- Squashed compatibility baseline: V046. Current runtime schema: V047.
|
||||
|
||||
-- RUSTBOX:GLOBAL-BEGIN
|
||||
CREATE TABLE IF NOT EXISTS rustbox_accounts (
|
||||
@@ -186,9 +186,27 @@ CREATE TABLE {{p}}draft_attachments (
|
||||
content_type TEXT NOT NULL,
|
||||
size BIGINT NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
checksum_sha256 VARCHAR(64) NULL,
|
||||
created_at VARCHAR(64) NOT NULL DEFAULT (CAST(CURRENT_TIMESTAMP AS CHAR))
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||
|
||||
CREATE TABLE {{p}}search_outbox (
|
||||
id VARCHAR(64) PRIMARY KEY,
|
||||
generation VARCHAR(64) NOT NULL,
|
||||
event_type VARCHAR(16) NOT NULL,
|
||||
message_uid BIGINT NOT NULL,
|
||||
folder TEXT NOT NULL,
|
||||
payload LONGTEXT NULL,
|
||||
attempts BIGINT NOT NULL DEFAULT 0,
|
||||
available_at BIGINT NOT NULL DEFAULT 0,
|
||||
claimed_until BIGINT NOT NULL DEFAULT 0,
|
||||
dead_letter BIGINT NOT NULL DEFAULT 0,
|
||||
last_error TEXT NULL,
|
||||
created_at BIGINT NOT NULL,
|
||||
updated_at BIGINT NOT NULL,
|
||||
INDEX idx_search_outbox_ready (dead_letter, available_at, claimed_until, updated_at)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||
|
||||
CREATE TABLE {{p}}drafts (
|
||||
id VARCHAR(255) PRIMARY KEY,
|
||||
to_addresses LONGTEXT NOT NULL DEFAULT (''),
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
ALTER TABLE {{p}}draft_attachments ADD COLUMN checksum_sha256 TEXT;
|
||||
|
||||
CREATE TABLE {{p}}search_outbox (
|
||||
id TEXT PRIMARY KEY,
|
||||
generation TEXT NOT NULL,
|
||||
event_type TEXT NOT NULL,
|
||||
message_uid BIGINT NOT NULL,
|
||||
folder TEXT NOT NULL,
|
||||
payload TEXT,
|
||||
attempts BIGINT NOT NULL DEFAULT 0,
|
||||
available_at BIGINT NOT NULL DEFAULT 0,
|
||||
claimed_until BIGINT NOT NULL DEFAULT 0,
|
||||
dead_letter BIGINT NOT NULL DEFAULT 0,
|
||||
last_error TEXT,
|
||||
created_at BIGINT NOT NULL,
|
||||
updated_at BIGINT NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX {{p}}idx_search_outbox_ready
|
||||
ON {{p}}search_outbox(dead_letter, available_at, claimed_until, updated_at);
|
||||
@@ -1,7 +1,7 @@
|
||||
-- RustBox 0.1.2 PostgreSQL schema.
|
||||
-- RustBox 0.2.0 PostgreSQL schema.
|
||||
-- One physical database contains global sessions plus isolated per-account tables.
|
||||
-- Application tables are materialized by replacing {{p}} with a collision-checked prefix.
|
||||
-- Baseline application schema version: 46. Future migrations start at V047.
|
||||
-- Squashed compatibility baseline: V046. Current runtime schema: V047.
|
||||
|
||||
-- RUSTBOX:GLOBAL-BEGIN
|
||||
CREATE TABLE IF NOT EXISTS rustbox_accounts (
|
||||
@@ -186,9 +186,28 @@ CREATE TABLE {{p}}draft_attachments (
|
||||
content_type TEXT NOT NULL,
|
||||
size BIGINT NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
checksum_sha256 TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (CURRENT_TIMESTAMP::text)
|
||||
);
|
||||
|
||||
CREATE TABLE {{p}}search_outbox (
|
||||
id TEXT PRIMARY KEY,
|
||||
generation TEXT NOT NULL,
|
||||
event_type TEXT NOT NULL,
|
||||
message_uid BIGINT NOT NULL,
|
||||
folder TEXT NOT NULL,
|
||||
payload TEXT,
|
||||
attempts BIGINT NOT NULL DEFAULT 0,
|
||||
available_at BIGINT NOT NULL DEFAULT 0,
|
||||
claimed_until BIGINT NOT NULL DEFAULT 0,
|
||||
dead_letter BIGINT NOT NULL DEFAULT 0,
|
||||
last_error TEXT,
|
||||
created_at BIGINT NOT NULL,
|
||||
updated_at BIGINT NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX {{p}}idx_search_outbox_ready ON {{p}}search_outbox(dead_letter, available_at, claimed_until, updated_at);
|
||||
|
||||
CREATE TABLE {{p}}drafts (
|
||||
id TEXT PRIMARY KEY,
|
||||
to_addresses TEXT NOT NULL DEFAULT '',
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
ALTER TABLE {{p}}draft_attachments ADD COLUMN checksum_sha256 TEXT;
|
||||
|
||||
CREATE TABLE {{p}}search_outbox (
|
||||
id TEXT PRIMARY KEY,
|
||||
generation TEXT NOT NULL,
|
||||
event_type TEXT NOT NULL,
|
||||
message_uid INTEGER NOT NULL,
|
||||
folder TEXT NOT NULL,
|
||||
payload TEXT,
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
available_at INTEGER NOT NULL DEFAULT 0,
|
||||
claimed_until INTEGER NOT NULL DEFAULT 0,
|
||||
dead_letter INTEGER NOT NULL DEFAULT 0,
|
||||
last_error TEXT,
|
||||
created_at INTEGER NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX {{p}}idx_search_outbox_ready
|
||||
ON {{p}}search_outbox(dead_letter, available_at, claimed_until, updated_at);
|
||||
@@ -1,7 +1,7 @@
|
||||
-- RustBox 0.1.2 SQLite schema.
|
||||
-- RustBox 0.2.0 SQLite schema.
|
||||
-- One physical database contains global sessions plus isolated per-account tables.
|
||||
-- Application tables are materialized from the tenant section by replacing {{p}}.
|
||||
-- Baseline application schema version: 46. Future migrations start at V047.
|
||||
-- Squashed compatibility baseline: V046. Current runtime schema: V047.
|
||||
|
||||
-- RUSTBOX:GLOBAL-BEGIN
|
||||
CREATE TABLE IF NOT EXISTS rustbox_accounts (
|
||||
@@ -131,10 +131,29 @@ CREATE TABLE {{p}}draft_attachments (
|
||||
content_type TEXT NOT NULL,
|
||||
size INTEGER NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
checksum_sha256 TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
FOREIGN KEY (draft_id) REFERENCES {{p}}drafts(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE {{p}}search_outbox (
|
||||
id TEXT PRIMARY KEY,
|
||||
generation TEXT NOT NULL,
|
||||
event_type TEXT NOT NULL,
|
||||
message_uid INTEGER NOT NULL,
|
||||
folder TEXT NOT NULL,
|
||||
payload TEXT,
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
available_at INTEGER NOT NULL DEFAULT 0,
|
||||
claimed_until INTEGER NOT NULL DEFAULT 0,
|
||||
dead_letter INTEGER NOT NULL DEFAULT 0,
|
||||
last_error TEXT,
|
||||
created_at INTEGER NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX {{p}}idx_search_outbox_ready ON {{p}}search_outbox(dead_letter, available_at, claimed_until, updated_at);
|
||||
|
||||
CREATE TABLE {{p}}drafts (
|
||||
id TEXT PRIMARY KEY,
|
||||
to_addresses TEXT NOT NULL DEFAULT '',
|
||||
|
||||
+64
-117
@@ -1,68 +1,39 @@
|
||||
# RustBox 0.1.2 - packaged production configuration (DEB/RPM)
|
||||
#
|
||||
# The active values below intentionally keep the existing package profile.
|
||||
# Alternative storage/cluster examples are provided at the end of this file
|
||||
# and are fully commented out, so they do not change the default package setup.
|
||||
# RustBox 0.2.0 - packaged production configuration (DEB/RPM)
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Server
|
||||
# -----------------------------------------------------------------------------
|
||||
host = "0.0.0.0"
|
||||
port = 3001
|
||||
environment = "production"
|
||||
data_dir = "/var/lib/rustbox"
|
||||
log_dir = "/var/log/rustbox"
|
||||
log_level = "info"
|
||||
log_level = "warn"
|
||||
|
||||
# Database, sessions and "remember me".
|
||||
# With no database/session backend keys configured, the whole application uses
|
||||
# SQLite at data_dir/rustbox.sqlite; sessions are stored in that database.
|
||||
cluster_shared_data_dir = false
|
||||
search_index_backend = "tantivy"
|
||||
database_backend = "sqlite"
|
||||
session_backend = "database"
|
||||
cluster_mode = false
|
||||
session_timeout_hours = 72
|
||||
remember_me_enabled = true
|
||||
remember_me_days = 30
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Mail endpoints
|
||||
# -----------------------------------------------------------------------------
|
||||
imap_host = "localhost"
|
||||
imap_port = 993
|
||||
smtp_host = "localhost"
|
||||
smtp_port = 587
|
||||
tls_enabled = true
|
||||
# This packaged profile intentionally keeps certificate verification disabled.
|
||||
# For remote production mail servers, tls_verify = true is recommended.
|
||||
tls_verify = false
|
||||
|
||||
# Size limits in MiB.
|
||||
max_attachment_size_mb = 50
|
||||
max_message_size_mb = 100
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# ManageSieve
|
||||
# -----------------------------------------------------------------------------
|
||||
sieve_port = 4190
|
||||
sieve_tls_mode = "none"
|
||||
sieve_allow_insecure_auth = true
|
||||
sieve_timeout_secs = 10
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Mailbox/UI defaults
|
||||
# -----------------------------------------------------------------------------
|
||||
default_junk_folder = "Spam"
|
||||
default_theme = "system"
|
||||
default_interface_skin = "apple-mail"
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Reverse proxy and outgoing mail
|
||||
# -----------------------------------------------------------------------------
|
||||
trusted_proxies = "127.0.0.1,::1"
|
||||
outgoing_headers = []
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# OpenPGP
|
||||
# -----------------------------------------------------------------------------
|
||||
pgp_enabled = true
|
||||
pgp_wkd_enabled = true
|
||||
pgp_hkp_enabled = false
|
||||
@@ -71,95 +42,71 @@ pgp_network_timeout_secs = 10
|
||||
pgp_max_key_bytes = 1048576
|
||||
pgp_allow_private_export = true
|
||||
|
||||
# =============================================================================
|
||||
# OPTIONAL CONFIGURATION EXAMPLES
|
||||
# =============================================================================
|
||||
# Do not enable more than one conflicting example at the same time.
|
||||
# Copy/uncomment only the keys required by the selected profile.
|
||||
#
|
||||
# session_encryption_key must represent the same 32-byte key on every node that
|
||||
# shares the configured SQL database or Redis session backend. Keep it secret and outside source control.
|
||||
# Blob storage is independent from database_backend. This default preserves the
|
||||
# single-node package behavior. PostgreSQL/MySQL may use this local filesystem
|
||||
# backend too when cluster_mode=false.
|
||||
[storage]
|
||||
backend = "filesystem"
|
||||
path = "/var/lib/rustbox/blobs"
|
||||
shared = false
|
||||
request_timeout_secs = 30
|
||||
connect_timeout_secs = 5
|
||||
max_retries = 3
|
||||
retry_timeout_secs = 60
|
||||
|
||||
# Local Tantivy is single-node only. Cluster deployments must use opensearch or disabled.
|
||||
[search]
|
||||
backend = "local_tantivy"
|
||||
path = "/var/lib/rustbox/search"
|
||||
index_prefix = "rustbox"
|
||||
required = false
|
||||
request_timeout_secs = 10
|
||||
worker_interval_secs = 2
|
||||
worker_batch_size = 100
|
||||
worker_max_attempts = 12
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Profile A: standalone SQLite application database (default package behavior)
|
||||
# -----------------------------------------------------------------------------
|
||||
# No extra keys are required. The equivalent explicit configuration is:
|
||||
# database_backend = "sqlite"
|
||||
# session_backend = "database"
|
||||
# cluster_mode = false
|
||||
#
|
||||
# RustBox stores account data and sessions in data_dir/rustbox.sqlite and
|
||||
# creates/uses the local session encryption key for this single-node setup.
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Profile B: PostgreSQL application database (data + database sessions)
|
||||
# -----------------------------------------------------------------------------
|
||||
# database_backend = "pgsql"
|
||||
# database_url = "postgres://rustbox:change-me@127.0.0.1/rustbox"
|
||||
# session_backend = "database"
|
||||
# cluster_mode = false
|
||||
# session_encryption_key = "BASE64_32_BYTE_KEY"
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Profile C: MySQL application database (data + database sessions; MySQL 8.0.13+)
|
||||
# -----------------------------------------------------------------------------
|
||||
# database_backend = "mysql"
|
||||
# database_url = "mysql://rustbox:change-me@127.0.0.1/rustbox"
|
||||
# session_backend = "database"
|
||||
# cluster_mode = false
|
||||
# session_encryption_key = "BASE64_32_BYTE_KEY"
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Profile D: Redis sessions, single application node (account data stays in database_backend)
|
||||
# -----------------------------------------------------------------------------
|
||||
# session_backend = "redis"
|
||||
# redis_url = "redis://127.0.0.1:6379/"
|
||||
# cluster_mode = false
|
||||
# session_encryption_key = "BASE64_32_BYTE_KEY"
|
||||
#
|
||||
# If Redis requires authentication, include credentials in redis_url or use the
|
||||
# supported Redis password configuration for your deployment.
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Profile E: Redis Sentinel sessions for multiple RustBox nodes
|
||||
# -----------------------------------------------------------------------------
|
||||
# Cluster nodes must also share the main application database.
|
||||
# data_dir must be a shared filesystem mounted at the same absolute path on every node.
|
||||
# database_backend = "pgsql"
|
||||
# database_url = "postgres://rustbox:change-me@db.internal/rustbox"
|
||||
# session_backend = "redis"
|
||||
# cluster_mode = true
|
||||
# cluster_shared_data_dir = true
|
||||
# search_index_backend = "disabled"
|
||||
# pgp_enabled = false
|
||||
# redis_sentinel_nodes = ["sentinel-a:26379", "sentinel-b:26379", "sentinel-c:26379"]
|
||||
# redis_sentinel_master = "mymaster"
|
||||
# redis_sentinel_password = "sentinel-secret"
|
||||
# redis_password = "redis-secret"
|
||||
# session_encryption_key = "BASE64_32_BYTE_KEY"
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Profile F: PostgreSQL application database shared by a RustBox cluster
|
||||
# data_dir must be shared and mounted at the same absolute path on every node.
|
||||
# Cluster example: PostgreSQL + S3-compatible storage + OpenSearch
|
||||
# -----------------------------------------------------------------------------
|
||||
# database_backend = "pgsql"
|
||||
# database_url = "postgres://rustbox:change-me@db.internal/rustbox"
|
||||
# session_backend = "database"
|
||||
# cluster_mode = true
|
||||
# cluster_shared_data_dir = true
|
||||
# search_index_backend = "disabled"
|
||||
# pgp_enabled = false
|
||||
# session_backend = "database" # or redis
|
||||
# session_encryption_key = "BASE64_32_BYTE_KEY"
|
||||
# cluster_mode = true
|
||||
# pgp_enabled = false
|
||||
#
|
||||
# [storage]
|
||||
# backend = "s3"
|
||||
# bucket = "rustbox"
|
||||
# region = "eu-central-1"
|
||||
# endpoint = "https://s3.example.net" # optional for AWS S3
|
||||
# path_style = false
|
||||
# allow_http = false
|
||||
# request_timeout_secs = 30
|
||||
# connect_timeout_secs = 5
|
||||
# max_retries = 3
|
||||
# retry_timeout_secs = 60
|
||||
#
|
||||
# [search]
|
||||
# backend = "opensearch"
|
||||
# urls = ["https://opensearch.example.net:9200"]
|
||||
# index_prefix = "rustbox"
|
||||
# required = true
|
||||
# username = "rustbox"
|
||||
# password = "change-me"
|
||||
# worker_interval_secs = 2
|
||||
# worker_batch_size = 100
|
||||
# worker_max_attempts = 12
|
||||
#
|
||||
# In this shape data_dir is not required as persistent node state.
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Profile G: MySQL application database shared by a RustBox cluster
|
||||
# data_dir must be shared and mounted at the same absolute path on every node.
|
||||
# Cluster blob alternative: shared POSIX/NFS/CephFS/GlusterFS
|
||||
# -----------------------------------------------------------------------------
|
||||
# database_backend = "mysql"
|
||||
# database_url = "mysql://rustbox:change-me@db.internal/rustbox"
|
||||
# session_backend = "database"
|
||||
# cluster_mode = true
|
||||
# cluster_shared_data_dir = true
|
||||
# search_index_backend = "disabled"
|
||||
# pgp_enabled = false
|
||||
# session_encryption_key = "BASE64_32_BYTE_KEY"
|
||||
# [storage]
|
||||
# backend = "filesystem"
|
||||
# path = "/mnt/rustbox-shared"
|
||||
# shared = true
|
||||
#
|
||||
# `shared=true` is an operator assertion that every RustBox node sees the same
|
||||
# filesystem. It is not suitable for local Tantivy; use OpenSearch/disabled search.
|
||||
@@ -8,7 +8,7 @@ Type=simple
|
||||
User=rustbox
|
||||
Group=rustbox
|
||||
Environment=RUSTBOX_CONFIG=/etc/rustbox/rustbox.toml
|
||||
Environment=RUST_LOG=info
|
||||
#Environment=RUST_LOG=info
|
||||
WorkingDirectory=/var/lib/rustbox
|
||||
ExecStart=/usr/local/bin/rustbox
|
||||
Restart=on-failure
|
||||
|
||||
+2
-1
@@ -1,2 +1,3 @@
|
||||
[toolchain]
|
||||
channel = "stable"
|
||||
channel = "1.88.0"
|
||||
components = ["rustfmt", "clippy"]
|
||||
@@ -4,6 +4,14 @@ set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
# Cargo.lock is generated only for the locked build/test phase. Keep source releases
|
||||
# and the working tree clean so the offline/static gate is repeatable.
|
||||
cleanup_lockfile() {
|
||||
rm -f "$ROOT/Cargo.lock"
|
||||
}
|
||||
trap cleanup_lockfile EXIT
|
||||
rm -f "$ROOT/Cargo.lock"
|
||||
|
||||
if ! command -v cargo >/dev/null 2>&1; then
|
||||
echo "error: Rust/Cargo is required" >&2
|
||||
exit 1
|
||||
@@ -19,6 +27,17 @@ if [[ -e "$ROOT/src/web/app.js" || ! -f "$ROOT/src/web/js/manifest.txt" ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if command -v python3 >/dev/null 2>&1; then
|
||||
python3 scripts/validate-release-static.py
|
||||
else
|
||||
echo "error: python3 is required for static release validation" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f Cargo.lock ]]; then
|
||||
cargo generate-lockfile
|
||||
fi
|
||||
|
||||
cargo fmt --all
|
||||
cargo fmt --all -- --check
|
||||
cargo test --locked
|
||||
|
||||
Executable
+390
@@ -0,0 +1,390 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Static/offline release checks for RustBox 0.2.0.
|
||||
|
||||
These checks intentionally avoid pretending to replace `cargo test`: they validate
|
||||
configuration, SQL migrations, Docker examples, query-catalog parity and release
|
||||
metadata in environments where the Rust toolchain or external services are absent.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import pathlib
|
||||
import re
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import tomllib
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError: # pragma: no cover - validation host dependency
|
||||
yaml = None
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||
errors: list[str] = []
|
||||
checks: list[str] = []
|
||||
|
||||
|
||||
def ok(message: str) -> None:
|
||||
checks.append(message)
|
||||
|
||||
|
||||
def require(condition: bool, message: str) -> None:
|
||||
if condition:
|
||||
ok(message)
|
||||
else:
|
||||
errors.append(message)
|
||||
|
||||
|
||||
def read(rel: str) -> str:
|
||||
return (ROOT / rel).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def extract_section(text: str, begin: str, end: str) -> str:
|
||||
require(begin in text and end in text, f"schema markers present: {begin} / {end}")
|
||||
if begin not in text or end not in text:
|
||||
return ""
|
||||
return text.split(begin, 1)[1].split(end, 1)[0]
|
||||
|
||||
|
||||
# Release/toolchain metadata.
|
||||
cargo = tomllib.loads(read("Cargo.toml"))
|
||||
require(cargo["package"]["version"] == "0.2.0", "Cargo package version is 0.2.0")
|
||||
require(cargo["package"]["rust-version"] == "1.88", "Cargo rust-version is 1.88")
|
||||
toolchain = tomllib.loads(read("rust-toolchain.toml"))
|
||||
require(toolchain["toolchain"]["channel"] == "1.88.0", "rust-toolchain pins Rust 1.88.0")
|
||||
require(not (ROOT / "Cargo.lock").exists(), "stale Cargo.lock is not shipped")
|
||||
require("cargo generate-lockfile" in read("scripts/build-release.sh"), "release script regenerates Cargo.lock")
|
||||
require("cargo generate-lockfile" in read(".github/workflows/ci.yml"), "CI regenerates Cargo.lock before --locked checks")
|
||||
require("validate-release-static.py" in read("scripts/build-release.sh"), "release script runs offline/static release gate")
|
||||
require("validate-release-static.py" in read(".github/workflows/ci.yml"), "CI runs offline/static release gate")
|
||||
require(read("README.md").startswith("# RustBox 0.2.0"), "README identifies RustBox 0.2.0")
|
||||
require(read("RELEASE-MANIFEST.txt").startswith("RustBox 0.2.0"), "release manifest identifies RustBox 0.2.0")
|
||||
require(read("RELEASE-VALIDATION.txt").startswith("RustBox 0.2.0"), "release validation identifies RustBox 0.2.0")
|
||||
require(read("docs/IMPLEMENTATION_STATUS.md").startswith("# RustBox 0.2.0"), "implementation status identifies RustBox 0.2.0")
|
||||
require((ROOT / "docs/STORAGE_SEARCH_OPERATIONS.md").is_file(), "storage/search operations guide exists")
|
||||
require('imap_host = ""' not in read("config.dev.toml"), "development config does not override IMAP host with an invalid empty value")
|
||||
require(not re.search(r"(?m)^IMAP_HOST=\s*$", read(".env.example")), "environment example does not set an invalid empty IMAP host")
|
||||
|
||||
# Parse all maintained TOML examples.
|
||||
toml_files = [
|
||||
"Cargo.toml",
|
||||
"rust-toolchain.toml",
|
||||
"config.example.toml",
|
||||
"config.dev.toml",
|
||||
"packaging/rustbox.toml",
|
||||
"docker/configs/single-sqlite.toml",
|
||||
"docker/configs/postgres-s3-opensearch.toml",
|
||||
"docker/configs/mysql-nfs-opensearch.toml",
|
||||
]
|
||||
for rel in toml_files:
|
||||
try:
|
||||
tomllib.loads(read(rel))
|
||||
ok(f"TOML parses: {rel}")
|
||||
except Exception as exc:
|
||||
errors.append(f"TOML parse failed: {rel}: {exc}")
|
||||
|
||||
# Parse Docker Compose YAML and validate referenced config paths.
|
||||
compose_files = sorted((ROOT / "docker").glob("compose-*.yml"))
|
||||
require(len(compose_files) >= 3, "three Docker Compose deployment variants exist")
|
||||
if yaml is None:
|
||||
errors.append("PyYAML is required for Docker Compose static validation")
|
||||
else:
|
||||
for path in compose_files:
|
||||
try:
|
||||
data = yaml.safe_load(path.read_text(encoding="utf-8"))
|
||||
require(isinstance(data, dict) and "services" in data, f"Compose parses with services: {path.name}")
|
||||
rustbox = data.get("services", {}).get("rustbox", {})
|
||||
for volume in rustbox.get("volumes", []) or []:
|
||||
if isinstance(volume, str) and volume.startswith("./configs/"):
|
||||
config_rel = volume.split(":", 1)[0].removeprefix("./")
|
||||
require((ROOT / "docker" / config_rel).is_file(), f"Compose config mount exists: {path.name} -> {config_rel}")
|
||||
except Exception as exc:
|
||||
errors.append(f"Compose parse failed: {path.name}: {exc}")
|
||||
|
||||
single = tomllib.loads(read("docker/configs/single-sqlite.toml"))
|
||||
pg_s3 = tomllib.loads(read("docker/configs/postgres-s3-opensearch.toml"))
|
||||
my_nfs = tomllib.loads(read("docker/configs/mysql-nfs-opensearch.toml"))
|
||||
require(single["cluster_mode"] is False and single["storage"]["backend"] == "filesystem", "single-node Compose uses local filesystem storage")
|
||||
require(pg_s3["cluster_mode"] is True and "data_dir" not in pg_s3, "PostgreSQL/S3 cluster config omits data_dir")
|
||||
require(pg_s3["storage"]["backend"] == "s3" and pg_s3["search"]["backend"] == "opensearch", "PostgreSQL cluster uses S3 + OpenSearch")
|
||||
require(my_nfs["cluster_mode"] is True and "data_dir" not in my_nfs, "MySQL/NFS cluster config omits data_dir")
|
||||
require(my_nfs["storage"]["backend"] == "filesystem" and my_nfs["storage"]["shared"] is True, "MySQL cluster marks POSIX/NFS storage shared")
|
||||
require(my_nfs["search"]["backend"] == "opensearch", "MySQL/NFS cluster uses OpenSearch")
|
||||
require("type: nfs" in read("docker/compose-mysql-nfs-opensearch.yml"), "NFS Compose uses a real NFS volume driver")
|
||||
require("docker.io/pgsty/silo:RELEASE.2026-10-26T00-00-00Z" in read("docker/compose-postgres-s3-opensearch.yml"), "S3 Compose pins the maintained SILO integration image")
|
||||
require("docker.io/pgsty/mc:RELEASE.2026-10-26T00-00-00Z" in read("docker/compose-postgres-s3-opensearch.yml"), "S3 Compose pins the matching SILO client image")
|
||||
require('test: ["CMD", "silo", "healthcheck", "ready"]' in read("docker/compose-postgres-s3-opensearch.yml"), "S3 Compose waits on SILO readiness before bucket initialization")
|
||||
require("endpoint = \"http://s3:9000\"" in read("docker/configs/postgres-s3-opensearch.toml"), "S3 config targets the generic Compose S3 service")
|
||||
for rel in ["docker/compose-postgres-s3-opensearch.yml", "docker/compose-mysql-nfs-opensearch.yml"]:
|
||||
require("opensearchproject/opensearch:3.9.0" in read(rel), f"Compose pins OpenSearch 3.9.0: {rel}")
|
||||
require("opensearchproject/opensearch:latest" not in read(rel), f"Compose avoids moving OpenSearch latest tag: {rel}")
|
||||
require('DISABLE_INSTALL_DEMO_CONFIG: "true"' in read(rel), f"Compose disables OpenSearch demo security bootstrap: {rel}")
|
||||
require('DISABLE_SECURITY_PLUGIN: "true"' in read(rel), f"Compose explicitly disables OpenSearch security plugin for local integration: {rel}")
|
||||
|
||||
for rel in ["docker/compose-postgres-s3-opensearch.yml", "docker/compose-mysql-nfs-opensearch.yml"]:
|
||||
match = re.search(r"SESSION_ENCRYPTION_KEY:\s*\$\{SESSION_ENCRYPTION_KEY:-([^}]+)\}", read(rel))
|
||||
require(match is not None, f"Compose has SESSION_ENCRYPTION_KEY fallback: {rel}")
|
||||
if match:
|
||||
try:
|
||||
decoded = base64.b64decode(match.group(1), validate=True)
|
||||
require(len(decoded) == 32, f"Compose fallback session key decodes to 32 bytes: {rel}")
|
||||
except Exception as exc:
|
||||
errors.append(f"invalid Compose fallback session key in {rel}: {exc}")
|
||||
|
||||
# SQLite fresh-schema execution.
|
||||
schema = read("migrations/sqlite/schema.sql")
|
||||
global_sql = extract_section(schema, "-- RUSTBOX:GLOBAL-BEGIN", "-- RUSTBOX:GLOBAL-END")
|
||||
tenant_sql = extract_section(schema, "-- RUSTBOX:TENANT-BEGIN", "-- RUSTBOX:TENANT-END").replace("{{p}}", "u_test__")
|
||||
try:
|
||||
conn = sqlite3.connect(":memory:")
|
||||
conn.execute("PRAGMA foreign_keys=ON")
|
||||
conn.executescript(global_sql)
|
||||
conn.executescript(tenant_sql)
|
||||
columns = {row[1] for row in conn.execute("PRAGMA table_info(u_test__draft_attachments)")}
|
||||
require("checksum_sha256" in columns, "fresh SQLite schema has attachment checksum metadata")
|
||||
outbox = conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='u_test__search_outbox'").fetchone()
|
||||
require(outbox is not None, "fresh SQLite schema has search_outbox")
|
||||
outbox_columns = {row[1] for row in conn.execute("PRAGMA table_info(u_test__search_outbox)")}
|
||||
require("claimed_until" in outbox_columns, "fresh SQLite search_outbox separates worker lease from retry availability")
|
||||
account_default = conn.execute("PRAGMA table_info(rustbox_accounts)").fetchall()
|
||||
version_default = next((row[4] for row in account_default if row[1] == "schema_version"), None)
|
||||
require(str(version_default).strip("'\"") == "46", "fresh global compatibility baseline remains V046 before tenant registration")
|
||||
conn.close()
|
||||
except Exception as exc:
|
||||
errors.append(f"fresh SQLite schema execution failed: {exc}")
|
||||
|
||||
# V047 upgrade execution from a minimal V046 attachment table, including data preservation.
|
||||
try:
|
||||
conn = sqlite3.connect(":memory:")
|
||||
conn.executescript(
|
||||
"CREATE TABLE u_test__draft_attachments ("
|
||||
"id TEXT PRIMARY KEY, draft_id TEXT NOT NULL, filename TEXT NOT NULL, "
|
||||
"content_type TEXT NOT NULL, size INTEGER NOT NULL, file_path TEXT NOT NULL, "
|
||||
"created_at TEXT NOT NULL);"
|
||||
"INSERT INTO u_test__draft_attachments "
|
||||
"(id, draft_id, filename, content_type, size, file_path, created_at) VALUES "
|
||||
"('att-1', 'draft-1', 'report.pdf', 'application/pdf', 1234, '/legacy/report.pdf', '2026-01-01T00:00:00Z');"
|
||||
)
|
||||
migration = read("migrations/sqlite/V047__blob_keys_and_search_outbox.sql").replace("{{p}}", "u_test__")
|
||||
conn.executescript(migration)
|
||||
columns = {row[1] for row in conn.execute("PRAGMA table_info(u_test__draft_attachments)")}
|
||||
require("checksum_sha256" in columns, "V047 adds checksum_sha256 to SQLite attachments")
|
||||
row = conn.execute(
|
||||
"SELECT id, draft_id, filename, content_type, size, file_path, created_at, checksum_sha256 "
|
||||
"FROM u_test__draft_attachments WHERE id='att-1'"
|
||||
).fetchone()
|
||||
require(
|
||||
row == ('att-1', 'draft-1', 'report.pdf', 'application/pdf', 1234, '/legacy/report.pdf', '2026-01-01T00:00:00Z', None),
|
||||
"V047 preserves existing attachment rows and initializes checksum as NULL",
|
||||
)
|
||||
require(conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='u_test__search_outbox'").fetchone() is not None, "V047 creates SQLite search_outbox")
|
||||
outbox_columns = {row[1] for row in conn.execute("PRAGMA table_info(u_test__search_outbox)")}
|
||||
require("claimed_until" in outbox_columns, "V047 creates an independent SQLite outbox lease column")
|
||||
conn.close()
|
||||
except Exception as exc:
|
||||
errors.append(f"SQLite V047 migration execution failed: {exc}")
|
||||
|
||||
# Migration/query parity across SQL engines.
|
||||
for dialect in ("sqlite", "mysql", "pgsql"):
|
||||
require((ROOT / f"migrations/{dialect}/V047__blob_keys_and_search_outbox.sql").is_file(), f"V047 migration exists for {dialect}")
|
||||
queries = read(f"src/queries/{dialect}/queries.rs")
|
||||
block_match = re.search(r"pub mod db_search_outbox\s*\{(.*?)\n\}", queries, re.S)
|
||||
require(block_match is not None, f"search outbox query module exists for {dialect}")
|
||||
if block_match:
|
||||
qnames = set(re.findall(r"pub const (Q_\d{3})", block_match.group(1)))
|
||||
require(qnames == {f"Q_{i:03d}" for i in range(1, 8)}, f"search outbox Q001-Q007 parity for {dialect}")
|
||||
|
||||
pool = read("src/db/pool.rs")
|
||||
for dialect in ("SQLITE", "MYSQL", "PGSQL"):
|
||||
require(f"{dialect}_MIGRATIONS" in pool and "V047__blob_keys_and_search_outbox.sql" in pool, f"runtime migration catalog includes V047 ({dialect.lower()})")
|
||||
require("const BASE_SCHEMA_VERSION: u32 = 46" in pool, "legacy squashed baseline stays V046")
|
||||
require("latest_schema_version" in pool, "runtime schema version is computed from migrations")
|
||||
require(
|
||||
"latest_supported_sqlite = SQLITE_MIGRATIONS" in pool
|
||||
and "if version > latest_supported_sqlite" in pool,
|
||||
"legacy SQLite import accepts runtime migrations up to this build instead of stopping at V046",
|
||||
)
|
||||
|
||||
# Tenant/import table parity and transaction-bound mutation helpers.
|
||||
def const_string_array(text: str, name: str) -> list[str]:
|
||||
match = re.search(rf"const {name}: &\[&str\] = &\[(.*?)\];", text, re.S)
|
||||
return re.findall(r'"([^"\n]+)"', match.group(1)) if match else []
|
||||
|
||||
tenant_tables = const_string_array(pool, "TENANT_TABLES")
|
||||
import_tables = const_string_array(pool, "IMPORT_TABLES")
|
||||
require(
|
||||
tenant_tables and sorted(tenant_tables) == sorted(import_tables),
|
||||
"tenant and legacy-import table sets stay in sync",
|
||||
)
|
||||
messages_rs = read("src/db/messages.rs")
|
||||
routes_messages_rs = read("src/routes/messages/mod.rs")
|
||||
require(
|
||||
"pub fn remove_after_mutation_in_transaction(" in messages_rs
|
||||
and "remove_after_mutation_in_transaction(&transaction, folder, uid)?;" in messages_rs
|
||||
and "db::messages::remove_after_mutation_in_transaction(" in routes_messages_rs
|
||||
and "db::messages::remove_after_mutation(&transaction" not in routes_messages_rs,
|
||||
"move mutation reuses its caller transaction instead of nesting a second transaction",
|
||||
)
|
||||
|
||||
# Exercise the actual SQLite search-outbox query constants against the current tenant schema.
|
||||
try:
|
||||
sqlite_queries = read("src/queries/sqlite/queries.rs")
|
||||
outbox_block = re.search(r"pub mod db_search_outbox\s*\{(.*?)\n\}", sqlite_queries, re.S)
|
||||
if outbox_block is None:
|
||||
raise RuntimeError("db_search_outbox module not found")
|
||||
queries = {
|
||||
name: sql.replace("search_outbox", "u_test__search_outbox")
|
||||
for name, sql in re.findall(r'pub const (Q_\d{3}): &str = "([^"]+)";', outbox_block.group(1))
|
||||
}
|
||||
conn = sqlite3.connect(":memory:")
|
||||
conn.execute("PRAGMA foreign_keys=ON")
|
||||
conn.executescript(tenant_sql)
|
||||
conn.execute(queries["Q_001"], ("doc-1", "gen-1", "upsert", 7, "INBOX", "{}", 1000))
|
||||
ready = conn.execute(queries["Q_002"], (1000, 10)).fetchall()
|
||||
require(len(ready) == 1 and ready[0][0:3] == ("doc-1", "gen-1", "upsert"), "SQLite search outbox enqueue/list query roundtrip works")
|
||||
first_claim = conn.execute(queries["Q_006"], ("doc-1", "gen-1", 1060, 1000)).rowcount
|
||||
second_claim = conn.execute(queries["Q_006"], ("doc-1", "gen-1", 1060, 1000)).rowcount
|
||||
require(first_claim == 1 and second_claim == 0, "SQLite search outbox lease claim is exclusive")
|
||||
conn.execute(queries["Q_001"], ("doc-1", "gen-2", "delete", 7, "INBOX", None, 1001))
|
||||
requeued = conn.execute(
|
||||
"SELECT generation, event_type, attempts, available_at, claimed_until, dead_letter, created_at, updated_at "
|
||||
"FROM u_test__search_outbox WHERE id='doc-1'"
|
||||
).fetchone()
|
||||
require(
|
||||
requeued == ("gen-2", "delete", 0, 0, 1060, 0, 1001, 1001),
|
||||
"SQLite search outbox re-enqueue resets retry state while preserving an active worker lease",
|
||||
)
|
||||
require(
|
||||
conn.execute(queries["Q_002"], (1001, 10)).fetchall() == [],
|
||||
"SQLite replacement event stays hidden until the active lease expires",
|
||||
)
|
||||
stale_delete = conn.execute(queries["Q_003"], ("doc-1", "gen-1")).rowcount
|
||||
current_delete = conn.execute(queries["Q_003"], ("doc-1", "gen-2")).rowcount
|
||||
require(stale_delete == 0 and current_delete == 1, "SQLite search outbox generation guard rejects stale acknowledgements")
|
||||
conn.execute(queries["Q_001"], ("doc-2", "gen-3", "delete", 9, "Trash", None, 1002))
|
||||
failure_update = conn.execute(queries["Q_004"], ("doc-2", "gen-3", 1, 2000, 1, "boom", 1003)).rowcount
|
||||
stats = conn.execute(queries["Q_005"]).fetchone()
|
||||
claimed_after_failure = conn.execute("SELECT claimed_until FROM u_test__search_outbox WHERE id='doc-2'").fetchone()[0]
|
||||
require(failure_update == 1 and stats[0] == 1 and stats[1] == 1 and claimed_after_failure == 0, "SQLite search outbox retry/dead-letter resets worker lease and updates stats")
|
||||
conn.execute(queries["Q_001"], ("doc-old", "gen-old", "delete", 10, "Archive", None, 900))
|
||||
cleared = conn.execute(queries["Q_007"], (1000,)).rowcount
|
||||
require(cleared == 1, "SQLite search outbox can clear pre-reindex superseded events")
|
||||
conn.close()
|
||||
except Exception as exc:
|
||||
errors.append(f"SQLite search-outbox query execution failed: {exc}")
|
||||
|
||||
# Storage/search architecture invariants.
|
||||
config_rs = read("src/config.rs")
|
||||
storage_rs = read("src/storage/mod.rs")
|
||||
search_rs = read("src/search/engine.rs")
|
||||
require("if self.cluster_mode && !data_dir_explicit" in config_rs and "self.data_dir.clear()" in config_rs, "implicit data_dir is disabled in cluster mode")
|
||||
require("cluster_shared_data_dir" not in config_rs and "search_index_backend" not in config_rs, "legacy top-level storage/search switches are removed")
|
||||
require('self.storage.backend.trim().to_ascii_lowercase()' in config_rs and 'self.search.backend.trim().to_ascii_lowercase()' in config_rs, "storage/search backends come only from their dedicated config sections")
|
||||
require('"tantivy" => "local_tantivy"' not in config_rs, "legacy Tantivy backend alias is removed")
|
||||
require('"postgres"' not in config_rs.split("#[cfg(test)]", 1)[0] and '"postgresql"' not in config_rs.split("#[cfg(test)]", 1)[0] and '"postgres"' not in read("src/db/pool.rs").split("#[cfg(test)]", 1)[0] and '"postgresql"' not in read("src/db/pool.rs").split("#[cfg(test)]", 1)[0], "database backend uses canonical sqlite/mysql/pgsql names only")
|
||||
require("cluster_shared_data_dir" not in read("config.example.toml") and "search_index_backend" not in read("config.example.toml"), "example config contains no pre-0.2 compatibility switches")
|
||||
require("legacy_filesystem_root" not in storage_rs and "legacy_path(" not in storage_rs, "blob storage has no legacy attachment read-through path")
|
||||
require('"filesystem" | "s3"' in config_rs or '"filesystem", "s3"' in config_rs, "configuration accepts filesystem and S3 storage")
|
||||
require("storage.shared" in config_rs, "cluster-safe shared POSIX storage is capability-gated")
|
||||
require("AmazonS3Builder" in storage_rs and "BufWriter" in storage_rs, "S3 backend uses streaming object-store writer")
|
||||
require("with_retry" in storage_rs and "with_checksum_algorithm" in storage_rs, "S3 backend configures retries and checksums")
|
||||
require("attachment_key" in storage_rs and "users/{user_hash}/drafts/{draft_id}/attachments/{attachment_id}" in storage_rs, "attachments use logical storage keys")
|
||||
require("destination.delete(&target_key).await" in storage_rs, "attachment migration removes a failed destination blob after verification failure")
|
||||
require("let _ = upload.abort().await" in storage_rs and "failed reading source" in storage_rs, "attachment migration aborts a destination upload if source streaming fails")
|
||||
require("if let Err(error) = crate::db::drafts::update_attachment_storage" in storage_rs, "attachment migration removes destination object when SQL pointer update fails")
|
||||
require("OpenSearchBackend" in search_rs, "OpenSearch backend is implemented")
|
||||
require("reqwest::blocking" not in search_rs and 'features = ["json", "rustls-tls"]' in read("Cargo.toml"), "OpenSearch uses the async reqwest client without blocking feature")
|
||||
require("async fn request(" in search_rs and "request.send().await" in search_rs, "OpenSearch HTTP requests are asynchronous")
|
||||
require('json!({"term": {"user_hash": user_hash}})' in search_rs, "OpenSearch queries enforce server-side tenant filter")
|
||||
require("process_search_outbox" in search_rs and "claim(&conn" in search_rs, "OpenSearch worker uses durable claimed outbox events")
|
||||
require("fn event_lease_secs(&self) -> u64" in search_rs and "let lease_secs = backend.event_lease_secs();" in search_rs and "claim(&conn, &event, 60)" not in search_rs, "OpenSearch outbox lease is derived from HTTP timeout/failover budget instead of a fixed 60 seconds")
|
||||
require("opensearch_event_lease_covers_timeout_and_failover_budget" in search_rs, "OpenSearch lease budget has a source-level unit regression test")
|
||||
require("claimed_until" in read("migrations/sqlite/V047__blob_keys_and_search_outbox.sql") and "reenqueue_preserves_active_claim_lease" in read("src/db/search_outbox.rs"), "outbox re-enqueue cannot steal an active worker lease")
|
||||
require("index_messages_batch_with_connection" in search_rs and "enqueue_upsert_on" in search_rs, "OpenSearch outbox supports caller-owned SQL transactions")
|
||||
require("enqueue_participates_in_caller_transaction" in read("src/db/search_outbox.rs"), "transactional outbox rollback has a unit test")
|
||||
require("search reindex" in read("README.md") and "run_search_reindex" in read("src/main.rs"), "full search reindex command is documented and wired")
|
||||
require("clear_before(&conn, reindex_started_at)" in read("src/main.rs") and "pub fn clear_before(" in read("src/db/search_outbox.rs"), "full reindex removes only superseded pre-reindex outbox events")
|
||||
require("quiesce search workers" in read("docs/STORAGE_SEARCH_OPERATIONS.md"), "cluster reindex documentation requires a controlled maintenance window")
|
||||
require("self.backend.refresh().await?" in search_rs, "OpenSearch full reindex forces refresh before reporting completion")
|
||||
require("pub async fn document_count(&self) -> Result<usize, String>" in search_rs, "search document-count diagnostics propagate backend errors")
|
||||
require(".document_count()\n .await\n .map_err(AppError::InternalError)?" in read("src/routes/diagnostics.rs"), "diagnostics route does not convert OpenSearch count failures into zero")
|
||||
idle_rs = read("src/realtime/idle.rs")
|
||||
require("fn delete_cached_message(" in idle_rs and "delete_message_with_connection(&transaction, uid, folder)" in idle_rs, "IMAP IDLE cache deletion writes search outbox in the same DB transaction")
|
||||
require("search_engine.clone()," in read("src/realtime/ws.rs"), "websocket IDLE wiring passes the shared search engine")
|
||||
require("cargo generate-lockfile && cargo build --release --locked" in read("docker/Dockerfile"), "Docker release build generates and honors a dependency lockfile")
|
||||
require("PostgreSQL + SILO (S3-compatible) + OpenSearch" in read("CHANGELOG.md"), "0.2.0 changelog matches the pinned S3-compatible Compose service")
|
||||
require("docs/RUSTBOX_PLAN.md" not in read("docs/IMPLEMENTATION_STATUS.md"), "implementation status does not reference a missing release-plan file")
|
||||
require("intentionally run one RustBox application container" in read("docker/README.md"), "Docker docs distinguish shared backing topology from multi-app load balancing")
|
||||
|
||||
attachments_rs = read("src/routes/attachments.rs")
|
||||
require(".begin_upload(&key)" in attachments_rs and ".get(&attachment.storage_key)" in attachments_rs and (".delete(&attachment.storage_key)" in attachments_rs or ".delete(&storage_key)" in attachments_rs), "attachment HTTP routes use BlobStorage")
|
||||
require("tokio::fs::" not in attachments_rs, "attachment HTTP routes no longer access local filesystem directly")
|
||||
attachment_db_delete = attachments_rs.find("db::drafts::delete_draft_attachment")
|
||||
attachment_blob_delete = attachments_rs.find("storage.delete(&storage_key)", attachment_db_delete)
|
||||
require(attachment_db_delete >= 0 and attachment_blob_delete > attachment_db_delete, "attachment delete commits DB removal before best-effort blob cleanup")
|
||||
drafts_rs = read("src/routes/drafts.rs")
|
||||
draft_db_delete = drafts_rs.find("db::drafts::delete_draft(&conn, &id)")
|
||||
draft_blob_delete = drafts_rs.find("storage.delete(&attachment.storage_key)", draft_db_delete)
|
||||
require(draft_db_delete >= 0 and draft_blob_delete > draft_db_delete, "draft delete commits DB removal before best-effort blob cleanup")
|
||||
send_rs = read("src/routes/send.rs")
|
||||
send_db_delete = send_rs.find("db::drafts::delete_draft(&conn, draft_id)")
|
||||
send_blob_delete = send_rs.find("storage.delete(&attachment.storage_key)", send_db_delete)
|
||||
require(send_db_delete >= 0 and send_blob_delete > send_db_delete, "post-send cleanup commits DB removal before best-effort blob cleanup")
|
||||
|
||||
# Repository/source-package hygiene independent of Cargo.
|
||||
source_files = [path for path in ROOT.rglob("*") if path.is_file() and ".git" not in path.parts and "target" not in path.parts]
|
||||
conflict_marker = re.compile(r"(?m)^(<<<<<<<|=======|>>>>>>>)")
|
||||
conflicted = []
|
||||
for path in source_files:
|
||||
if path.suffix.lower() in {".png", ".jpg", ".jpeg", ".gif", ".webp", ".ico", ".woff", ".woff2", ".ttf", ".zip", ".gz"}:
|
||||
continue
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
if conflict_marker.search(text):
|
||||
conflicted.append(str(path.relative_to(ROOT)))
|
||||
require(not conflicted, f"no unresolved merge-conflict markers ({', '.join(conflicted) if conflicted else 'clean'})")
|
||||
require(not any(path.suffix in {".ts", ".tsx"} for path in source_files), "source package contains no TypeScript files")
|
||||
|
||||
manifest = [line.strip() for line in read("src/web/js/manifest.txt").splitlines() if line.strip() and not line.lstrip().startswith("#")]
|
||||
missing_manifest = [entry for entry in manifest if not (ROOT / "src/web/js" / entry).is_file()]
|
||||
require(not missing_manifest, f"browser JS manifest entries exist ({', '.join(missing_manifest) if missing_manifest else 'complete'})")
|
||||
|
||||
include_paths = []
|
||||
for path in ROOT.glob("src/**/*.rs"):
|
||||
text = path.read_text(encoding="utf-8")
|
||||
for match in re.finditer(r'include_(?:str|bytes)!\(\s*"([^"]+)"\s*\)', text):
|
||||
include_paths.append((path, match.group(1)))
|
||||
missing_includes = []
|
||||
for source, relative in include_paths:
|
||||
candidate = (source.parent / relative).resolve()
|
||||
if not candidate.is_file():
|
||||
missing_includes.append(f"{source.relative_to(ROOT)} -> {relative}")
|
||||
require(not missing_includes, f"Rust include_str/include_bytes targets exist ({'; '.join(missing_includes) if missing_includes else 'complete'})")
|
||||
|
||||
# Shell syntax checks are useful offline and deterministic.
|
||||
try:
|
||||
shell_files = [ROOT / "dev.sh", *sorted((ROOT / "scripts").glob("*.sh"))]
|
||||
subprocess.run(["bash", "-n", *map(str, shell_files)], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
||||
ok("shell scripts pass bash -n")
|
||||
except Exception as exc:
|
||||
errors.append(f"shell syntax validation failed: {exc}")
|
||||
|
||||
# TODO/release honesty: live integrations remain explicitly pending.
|
||||
todo = read("TODO.md")
|
||||
require("[ ] SILO Compose integration validation" in todo, "TODO keeps live SILO validation pending")
|
||||
require("[ ] MinIO integration validation" in todo, "TODO keeps live MinIO validation pending")
|
||||
require("[ ] AWS S3 integration validation" in todo, "TODO keeps live AWS S3 validation pending")
|
||||
require("[ ] Ceph RGW compatibility validation" in todo, "TODO keeps live Ceph RGW validation pending")
|
||||
require("[ ] S3-compatible storage works for draft attachments (live SILO/MinIO/AWS validation pending)" in todo, "acceptance does not overclaim live S3 validation")
|
||||
|
||||
print(f"static checks passed: {len(checks)}")
|
||||
for message in checks:
|
||||
print(f"PASS {message}")
|
||||
if errors:
|
||||
print(f"static checks failed: {len(errors)}", file=sys.stderr)
|
||||
for message in errors:
|
||||
print(f"FAIL {message}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
@@ -47,7 +47,7 @@ fn extract_cookie_value(req: &Request, cookie_name: &str) -> Option<String> {
|
||||
};
|
||||
for segment in header_str.split(';') {
|
||||
let trimmed = segment.trim();
|
||||
let prefix = format!("{}=", cookie_name);
|
||||
let prefix = format!("{cookie_name}=");
|
||||
if let Some(cookie_value) = trimmed.strip_prefix(&prefix) {
|
||||
let cookie_value = cookie_value.trim();
|
||||
if !cookie_value.is_empty() {
|
||||
@@ -60,7 +60,7 @@ fn extract_cookie_value(req: &Request, cookie_name: &str) -> Option<String> {
|
||||
}
|
||||
|
||||
fn extract_session_cookie(req: &Request, account_id: &str) -> Option<String> {
|
||||
let cookie_name = format!("rustbox_session_{}", account_id);
|
||||
let cookie_name = format!("rustbox_session_{account_id}");
|
||||
extract_cookie_value(req, &cookie_name)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
#[cfg(test)]
|
||||
use std::fs;
|
||||
#[cfg(test)]
|
||||
use std::path::PathBuf;
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
@@ -11,10 +13,19 @@ pub fn hash_email(email: &str) -> String {
|
||||
result.iter().map(|b| format!("{b:02x}")).collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn provision_user_data(data_dir: &str, user_hash: &str) -> Result<PathBuf, String> {
|
||||
provision_user_data_with_search(data_dir, user_hash, true)
|
||||
}
|
||||
|
||||
pub fn provision_user_database(data_dir: &str, user_hash: &str) -> Result<(), String> {
|
||||
let conn = crate::db::pool::open_user_db(data_dir, user_hash)
|
||||
.map_err(|e| format!("failed to initialize user database: {e}"))?;
|
||||
drop(conn);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn provision_user_data_with_search(
|
||||
data_dir: &str,
|
||||
user_hash: &str,
|
||||
|
||||
+425
-18
@@ -7,6 +7,118 @@ use figment::{
|
||||
use ipnet::IpNet;
|
||||
use serde::Deserialize;
|
||||
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
pub struct StorageConfig {
|
||||
#[serde(default = "default_storage_backend")]
|
||||
pub backend: String,
|
||||
#[serde(default)]
|
||||
pub path: Option<String>,
|
||||
#[serde(default)]
|
||||
pub shared: bool,
|
||||
#[serde(default)]
|
||||
pub bucket: Option<String>,
|
||||
#[serde(default = "default_s3_region")]
|
||||
pub region: String,
|
||||
#[serde(default)]
|
||||
pub endpoint: Option<String>,
|
||||
#[serde(default)]
|
||||
pub prefix: String,
|
||||
#[serde(default)]
|
||||
pub path_style: bool,
|
||||
#[serde(default)]
|
||||
pub allow_http: bool,
|
||||
#[serde(default)]
|
||||
pub access_key_id: Option<String>,
|
||||
#[serde(default)]
|
||||
pub secret_access_key: Option<String>,
|
||||
#[serde(default)]
|
||||
pub session_token: Option<String>,
|
||||
#[serde(default = "default_storage_request_timeout_secs")]
|
||||
pub request_timeout_secs: u64,
|
||||
#[serde(default = "default_storage_connect_timeout_secs")]
|
||||
pub connect_timeout_secs: u64,
|
||||
#[serde(default = "default_storage_max_retries")]
|
||||
pub max_retries: usize,
|
||||
#[serde(default = "default_storage_retry_timeout_secs")]
|
||||
pub retry_timeout_secs: u64,
|
||||
#[serde(default)]
|
||||
pub server_side_encryption: Option<String>,
|
||||
#[serde(default)]
|
||||
pub kms_key_id: Option<String>,
|
||||
}
|
||||
|
||||
impl Default for StorageConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
backend: default_storage_backend(),
|
||||
path: None,
|
||||
shared: false,
|
||||
bucket: None,
|
||||
region: default_s3_region(),
|
||||
endpoint: None,
|
||||
prefix: String::new(),
|
||||
path_style: false,
|
||||
allow_http: false,
|
||||
access_key_id: None,
|
||||
secret_access_key: None,
|
||||
session_token: None,
|
||||
request_timeout_secs: default_storage_request_timeout_secs(),
|
||||
connect_timeout_secs: default_storage_connect_timeout_secs(),
|
||||
max_retries: default_storage_max_retries(),
|
||||
retry_timeout_secs: default_storage_retry_timeout_secs(),
|
||||
server_side_encryption: None,
|
||||
kms_key_id: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
pub struct SearchConfig {
|
||||
#[serde(default = "default_search_backend")]
|
||||
pub backend: String,
|
||||
#[serde(default)]
|
||||
pub path: Option<String>,
|
||||
#[serde(default)]
|
||||
pub urls: Vec<String>,
|
||||
#[serde(default = "default_search_index_prefix")]
|
||||
pub index_prefix: String,
|
||||
#[serde(default)]
|
||||
pub username: Option<String>,
|
||||
#[serde(default)]
|
||||
pub password: Option<String>,
|
||||
#[serde(default)]
|
||||
pub required: bool,
|
||||
#[serde(default)]
|
||||
pub allow_insecure_http: bool,
|
||||
#[serde(default = "default_search_request_timeout_secs")]
|
||||
pub request_timeout_secs: u64,
|
||||
#[serde(default = "default_search_worker_interval_secs")]
|
||||
pub worker_interval_secs: u64,
|
||||
#[serde(default = "default_search_worker_batch_size")]
|
||||
pub worker_batch_size: usize,
|
||||
#[serde(default = "default_search_worker_max_attempts")]
|
||||
pub worker_max_attempts: u32,
|
||||
}
|
||||
|
||||
impl Default for SearchConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
backend: default_search_backend(),
|
||||
path: None,
|
||||
urls: Vec::new(),
|
||||
index_prefix: default_search_index_prefix(),
|
||||
username: None,
|
||||
password: None,
|
||||
required: false,
|
||||
allow_insecure_http: false,
|
||||
request_timeout_secs: default_search_request_timeout_secs(),
|
||||
worker_interval_secs: default_search_worker_interval_secs(),
|
||||
worker_batch_size: default_search_worker_batch_size(),
|
||||
worker_max_attempts: default_search_worker_max_attempts(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
#[allow(dead_code)]
|
||||
pub struct AppConfig {
|
||||
@@ -80,10 +192,10 @@ pub struct AppConfig {
|
||||
pub cluster_mode: bool,
|
||||
|
||||
#[serde(default)]
|
||||
pub cluster_shared_data_dir: bool,
|
||||
pub storage: StorageConfig,
|
||||
|
||||
#[serde(default = "default_search_index_backend")]
|
||||
pub search_index_backend: String,
|
||||
#[serde(default)]
|
||||
pub search: SearchConfig,
|
||||
|
||||
#[serde(default = "default_session_timeout_hours")]
|
||||
pub session_timeout_hours: u64,
|
||||
@@ -211,6 +323,46 @@ fn default_data_dir() -> String {
|
||||
"/data".to_string()
|
||||
}
|
||||
|
||||
fn default_s3_region() -> String {
|
||||
"us-east-1".to_string()
|
||||
}
|
||||
|
||||
fn default_storage_request_timeout_secs() -> u64 {
|
||||
30
|
||||
}
|
||||
|
||||
fn default_storage_connect_timeout_secs() -> u64 {
|
||||
5
|
||||
}
|
||||
|
||||
fn default_storage_max_retries() -> usize {
|
||||
3
|
||||
}
|
||||
|
||||
fn default_storage_retry_timeout_secs() -> u64 {
|
||||
60
|
||||
}
|
||||
|
||||
fn default_search_index_prefix() -> String {
|
||||
"rustbox".to_string()
|
||||
}
|
||||
|
||||
fn default_search_request_timeout_secs() -> u64 {
|
||||
10
|
||||
}
|
||||
|
||||
fn default_search_worker_interval_secs() -> u64 {
|
||||
2
|
||||
}
|
||||
|
||||
fn default_search_worker_batch_size() -> usize {
|
||||
100
|
||||
}
|
||||
|
||||
fn default_search_worker_max_attempts() -> u32 {
|
||||
12
|
||||
}
|
||||
|
||||
fn default_log_level() -> String {
|
||||
"info".to_string()
|
||||
}
|
||||
@@ -223,8 +375,12 @@ fn default_session_backend() -> String {
|
||||
"database".to_string()
|
||||
}
|
||||
|
||||
fn default_search_index_backend() -> String {
|
||||
"tantivy".to_string()
|
||||
fn default_storage_backend() -> String {
|
||||
"filesystem".to_string()
|
||||
}
|
||||
|
||||
fn default_search_backend() -> String {
|
||||
"local_tantivy".to_string()
|
||||
}
|
||||
|
||||
fn default_session_timeout_hours() -> u64 {
|
||||
@@ -305,8 +461,8 @@ impl Default for AppConfig {
|
||||
redis_password: None,
|
||||
redis_sentinel_password: None,
|
||||
cluster_mode: false,
|
||||
cluster_shared_data_dir: false,
|
||||
search_index_backend: default_search_index_backend(),
|
||||
storage: StorageConfig::default(),
|
||||
search: SearchConfig::default(),
|
||||
session_timeout_hours: default_session_timeout_hours(),
|
||||
remember_me_enabled: default_remember_me_enabled(),
|
||||
remember_me_days: default_remember_me_days(),
|
||||
@@ -450,7 +606,43 @@ impl AppConfig {
|
||||
figment = figment.merge(Toml::file(path));
|
||||
}
|
||||
|
||||
figment.merge(Env::raw()).extract()
|
||||
let figment = figment.merge(Env::raw().split("__"));
|
||||
let data_dir_explicit = figment.find_value("data_dir").is_ok();
|
||||
let mut config: Self = figment.extract()?;
|
||||
config.apply_cluster_data_dir_default(data_dir_explicit);
|
||||
Ok(config)
|
||||
}
|
||||
|
||||
fn apply_cluster_data_dir_default(&mut self, data_dir_explicit: bool) {
|
||||
if self.cluster_mode && !data_dir_explicit {
|
||||
self.data_dir.clear();
|
||||
}
|
||||
}
|
||||
|
||||
pub fn effective_storage_backend(&self) -> String {
|
||||
self.storage.backend.trim().to_ascii_lowercase()
|
||||
}
|
||||
|
||||
pub fn effective_search_backend(&self) -> String {
|
||||
self.search.backend.trim().to_ascii_lowercase()
|
||||
}
|
||||
|
||||
pub fn effective_storage_path(&self) -> &str {
|
||||
self.storage
|
||||
.path
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.unwrap_or(&self.data_dir)
|
||||
}
|
||||
|
||||
pub fn effective_search_path(&self) -> &str {
|
||||
self.search
|
||||
.path
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.unwrap_or(&self.data_dir)
|
||||
}
|
||||
|
||||
pub fn validate(&self) -> Result<(), String> {
|
||||
@@ -493,10 +685,7 @@ impl AppConfig {
|
||||
if self.max_attachment_size_mb > self.max_message_size_mb {
|
||||
return Err("MAX_ATTACHMENT_SIZE_MB cannot exceed MAX_MESSAGE_SIZE_MB".to_string());
|
||||
}
|
||||
if !matches!(
|
||||
self.database_backend.as_str(),
|
||||
"sqlite" | "mysql" | "pgsql" | "postgres" | "postgresql"
|
||||
) {
|
||||
if !matches!(self.database_backend.as_str(), "sqlite" | "mysql" | "pgsql") {
|
||||
return Err("DATABASE_BACKEND must be one of: sqlite, mysql, pgsql".to_string());
|
||||
}
|
||||
if self.database_backend != "sqlite"
|
||||
@@ -530,8 +719,133 @@ impl AppConfig {
|
||||
return Err("Redis sessions require REDIS_URL or REDIS_SENTINEL_NODES + REDIS_SENTINEL_MASTER".to_string());
|
||||
}
|
||||
}
|
||||
if !matches!(self.search_index_backend.as_str(), "tantivy" | "disabled") {
|
||||
return Err("SEARCH_INDEX_BACKEND must be one of: tantivy, disabled".to_string());
|
||||
let storage_backend = self.effective_storage_backend();
|
||||
if !matches!(storage_backend.as_str(), "filesystem" | "s3") {
|
||||
return Err("storage.backend must be one of: filesystem, s3".to_string());
|
||||
}
|
||||
if storage_backend == "filesystem" {
|
||||
let path = self.effective_storage_path();
|
||||
if path.trim().is_empty() {
|
||||
return Err(
|
||||
"storage.path/data_dir cannot be empty for filesystem storage".to_string(),
|
||||
);
|
||||
}
|
||||
if self.cluster_mode && !std::path::Path::new(path).is_absolute() {
|
||||
return Err("cluster filesystem storage.path must be an absolute path".to_string());
|
||||
}
|
||||
} else {
|
||||
if self
|
||||
.storage
|
||||
.bucket
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|v| !v.is_empty())
|
||||
.is_none()
|
||||
{
|
||||
return Err("storage.bucket is required when storage.backend=s3".to_string());
|
||||
}
|
||||
if self.storage.region.trim().is_empty() {
|
||||
return Err("storage.region cannot be empty".to_string());
|
||||
}
|
||||
if let Some(endpoint) = self
|
||||
.storage
|
||||
.endpoint
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|v| !v.is_empty())
|
||||
{
|
||||
let url = url::Url::parse(endpoint)
|
||||
.map_err(|e| format!("invalid storage.endpoint: {e}"))?;
|
||||
if url.scheme() != "https" && !(self.storage.allow_http && url.scheme() == "http") {
|
||||
return Err(
|
||||
"storage.endpoint must use HTTPS unless storage.allow_http=true"
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
if self.storage.request_timeout_secs == 0
|
||||
|| self.storage.connect_timeout_secs == 0
|
||||
|| self.storage.retry_timeout_secs == 0
|
||||
{
|
||||
return Err("storage timeouts must be greater than zero".to_string());
|
||||
}
|
||||
if let Some(mode) = self
|
||||
.storage
|
||||
.server_side_encryption
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|v| !v.is_empty())
|
||||
{
|
||||
if !matches!(mode, "AES256" | "aws:kms") {
|
||||
return Err("storage.server_side_encryption must be AES256 or aws:kms".to_string());
|
||||
}
|
||||
if mode == "aws:kms"
|
||||
&& self
|
||||
.storage
|
||||
.kms_key_id
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|v| !v.is_empty())
|
||||
.is_none()
|
||||
{
|
||||
return Err("storage.kms_key_id is required for aws:kms encryption".to_string());
|
||||
}
|
||||
}
|
||||
|
||||
let search_backend = self.effective_search_backend();
|
||||
if !matches!(
|
||||
search_backend.as_str(),
|
||||
"local_tantivy" | "disabled" | "opensearch"
|
||||
) {
|
||||
return Err(
|
||||
"search.backend must be one of: disabled, local_tantivy, opensearch".to_string(),
|
||||
);
|
||||
}
|
||||
if search_backend == "local_tantivy" {
|
||||
let path = self.effective_search_path();
|
||||
if path.trim().is_empty() {
|
||||
return Err("search.path/data_dir cannot be empty for local_tantivy".to_string());
|
||||
}
|
||||
}
|
||||
if search_backend == "opensearch" {
|
||||
if self.search.urls.is_empty() {
|
||||
return Err("search.urls must contain at least one URL for opensearch".to_string());
|
||||
}
|
||||
for raw in &self.search.urls {
|
||||
let parsed = url::Url::parse(raw.trim())
|
||||
.map_err(|e| format!("invalid OpenSearch URL '{raw}': {e}"))?;
|
||||
if parsed.scheme() != "https"
|
||||
&& !(self.search.allow_insecure_http && parsed.scheme() == "http")
|
||||
{
|
||||
return Err(
|
||||
"OpenSearch URLs must use HTTPS unless search.allow_insecure_http=true"
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
if !parsed.username().is_empty() || parsed.password().is_some() {
|
||||
return Err("OpenSearch credentials must not be embedded in URLs".to_string());
|
||||
}
|
||||
}
|
||||
let prefix = self.search.index_prefix.as_str();
|
||||
if prefix.is_empty()
|
||||
|| !prefix
|
||||
.chars()
|
||||
.next()
|
||||
.is_some_and(|c| c.is_ascii_lowercase() || c.is_ascii_digit())
|
||||
|| !prefix
|
||||
.chars()
|
||||
.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || matches!(c, '-' | '_'))
|
||||
{
|
||||
return Err("search.index_prefix must start with a lowercase ASCII letter/digit and contain only lowercase ASCII letters, digits, '-' or '_'".to_string());
|
||||
}
|
||||
}
|
||||
if self.search.request_timeout_secs == 0
|
||||
|| self.search.worker_interval_secs == 0
|
||||
|| self.search.worker_batch_size == 0
|
||||
|| self.search.worker_max_attempts == 0
|
||||
{
|
||||
return Err("search timeout/worker settings must be greater than zero".to_string());
|
||||
}
|
||||
if self.cluster_mode && self.database_backend == "sqlite" {
|
||||
return Err(
|
||||
@@ -539,15 +853,17 @@ impl AppConfig {
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
if self.cluster_mode && !self.cluster_shared_data_dir {
|
||||
let storage_cluster_safe =
|
||||
storage_backend == "s3" || (storage_backend == "filesystem" && self.storage.shared);
|
||||
if self.cluster_mode && !storage_cluster_safe {
|
||||
return Err(
|
||||
"CLUSTER_MODE requires CLUSTER_SHARED_DATA_DIR=true after data_dir is mounted as shared storage on every node; draft attachments are stored below data_dir"
|
||||
"CLUSTER_MODE requires cluster-safe blob storage: storage.backend=s3 or storage.backend=filesystem with storage.shared=true"
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
if self.cluster_mode && self.search_index_backend != "disabled" {
|
||||
if self.cluster_mode && search_backend == "local_tantivy" {
|
||||
return Err(
|
||||
"CLUSTER_MODE requires SEARCH_INDEX_BACKEND=disabled; the built-in Tantivy index is local and does not coordinate writers across nodes"
|
||||
"CLUSTER_MODE forbids search.backend=local_tantivy; use disabled or an external indexer"
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
@@ -669,6 +985,8 @@ mod tests {
|
||||
assert!(config.redis_sentinel_master.is_none());
|
||||
assert!(config.redis_sentinel_password.is_none());
|
||||
assert!(!config.cluster_mode);
|
||||
assert_eq!(config.storage.backend, "filesystem");
|
||||
assert_eq!(config.search.backend, "local_tantivy");
|
||||
assert_eq!(config.session_timeout_hours, 24);
|
||||
assert!(config.remember_me_enabled);
|
||||
assert_eq!(config.remember_me_days, 30);
|
||||
@@ -869,6 +1187,95 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_node_pgsql_can_still_use_data_dir_backends() {
|
||||
let config = AppConfig {
|
||||
imap_host: Some("imap.example.com".to_string()),
|
||||
database_backend: "pgsql".to_string(),
|
||||
database_url: Some("postgres://rustbox:secret@db/rustbox".to_string()),
|
||||
session_encryption_key: Some("shared-key".to_string()),
|
||||
data_dir: "/srv/rustbox".to_string(),
|
||||
..AppConfig::default()
|
||||
};
|
||||
assert!(config.validate().is_ok());
|
||||
assert_eq!(config.effective_storage_backend(), "filesystem");
|
||||
assert_eq!(config.effective_search_backend(), "local_tantivy");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cluster_mode_disables_implicit_data_dir_but_preserves_explicit_value() {
|
||||
let mut implicit = AppConfig {
|
||||
cluster_mode: true,
|
||||
data_dir: "/data".to_string(),
|
||||
..AppConfig::default()
|
||||
};
|
||||
implicit.apply_cluster_data_dir_default(false);
|
||||
assert!(implicit.data_dir.is_empty());
|
||||
|
||||
let mut explicit = AppConfig {
|
||||
cluster_mode: true,
|
||||
data_dir: "/mnt/shared-rustbox".to_string(),
|
||||
..AppConfig::default()
|
||||
};
|
||||
explicit.apply_cluster_data_dir_default(true);
|
||||
assert_eq!(explicit.data_dir, "/mnt/shared-rustbox");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cluster_with_s3_and_opensearch_does_not_require_data_dir() {
|
||||
let mut config = AppConfig {
|
||||
imap_host: Some("imap.example.com".to_string()),
|
||||
database_backend: "pgsql".to_string(),
|
||||
database_url: Some("postgres://rustbox:secret@db/rustbox".to_string()),
|
||||
session_encryption_key: Some("shared-key".to_string()),
|
||||
cluster_mode: true,
|
||||
pgp_enabled: false,
|
||||
data_dir: String::new(),
|
||||
..AppConfig::default()
|
||||
};
|
||||
config.storage.backend = "s3".to_string();
|
||||
config.storage.bucket = Some("rustbox".to_string());
|
||||
config.storage.endpoint = Some("https://s3.example.com".to_string());
|
||||
config.search.backend = "opensearch".to_string();
|
||||
config.search.urls = vec!["https://search.example.com".to_string()];
|
||||
assert!(config.validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cluster_accepts_shared_posix_storage_without_s3() {
|
||||
let mut config = AppConfig {
|
||||
imap_host: Some("imap.example.com".to_string()),
|
||||
database_backend: "mysql".to_string(),
|
||||
database_url: Some("mysql://rustbox:secret@db/rustbox".to_string()),
|
||||
session_encryption_key: Some("shared-key".to_string()),
|
||||
cluster_mode: true,
|
||||
pgp_enabled: false,
|
||||
..AppConfig::default()
|
||||
};
|
||||
config.storage.backend = "filesystem".to_string();
|
||||
config.storage.path = Some("/mnt/rustbox-shared".to_string());
|
||||
config.storage.shared = true;
|
||||
config.search.backend = "disabled".to_string();
|
||||
assert!(config.validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_0_2_backend_aliases_are_rejected() {
|
||||
let mut config = AppConfig {
|
||||
imap_host: Some("imap.example.com".to_string()),
|
||||
..AppConfig::default()
|
||||
};
|
||||
|
||||
config.search.backend = "tantivy".to_string();
|
||||
assert!(config.validate().unwrap_err().contains("search.backend"));
|
||||
|
||||
config.search.backend = "local_tantivy".to_string();
|
||||
for backend in ["postgres", "postgresql"] {
|
||||
config.database_backend = backend.to_string();
|
||||
assert!(config.validate().unwrap_err().contains("DATABASE_BACKEND"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cluster_mode_rejects_local_sqlite_even_with_redis_sessions() {
|
||||
let config = AppConfig {
|
||||
|
||||
@@ -491,7 +491,7 @@ pub fn update_preferences(
|
||||
set_bool!(attachment_reminder, "attachment_reminder");
|
||||
set_bool!(deep_index, "deep_index");
|
||||
if let Some(ref value) = data.animation_mode {
|
||||
sets.push(format!("animation_mode = ?{}", idx));
|
||||
sets.push(format!("animation_mode = ?{idx}"));
|
||||
values.push(value.to_db_value());
|
||||
idx += 1;
|
||||
}
|
||||
|
||||
+85
-40
@@ -33,7 +33,8 @@ pub struct DraftAttachment {
|
||||
pub filename: String,
|
||||
pub content_type: String,
|
||||
pub size: i64,
|
||||
pub file_path: String,
|
||||
pub storage_key: String,
|
||||
pub checksum_sha256: Option<String>,
|
||||
pub created_at: String,
|
||||
}
|
||||
|
||||
@@ -196,18 +197,31 @@ pub fn delete_draft(conn: &Connection, id: &str) -> Result<bool, String> {
|
||||
Ok(deleted > 0)
|
||||
}
|
||||
|
||||
pub struct NewDraftAttachment<'a> {
|
||||
pub id: &'a str,
|
||||
pub draft_id: &'a str,
|
||||
pub filename: &'a str,
|
||||
pub content_type: &'a str,
|
||||
pub size: i64,
|
||||
pub storage_key: &'a str,
|
||||
pub checksum_sha256: Option<&'a str>,
|
||||
}
|
||||
|
||||
pub fn add_draft_attachment(
|
||||
conn: &Connection,
|
||||
id: &str,
|
||||
draft_id: &str,
|
||||
filename: &str,
|
||||
content_type: &str,
|
||||
size: i64,
|
||||
file_path: &str,
|
||||
attachment: &NewDraftAttachment<'_>,
|
||||
) -> Result<(), String> {
|
||||
conn.execute(
|
||||
crate::dbq!(db_drafts::Q_006),
|
||||
params![id, draft_id, filename, content_type, size, file_path],
|
||||
params![
|
||||
attachment.id,
|
||||
attachment.draft_id,
|
||||
attachment.filename,
|
||||
attachment.content_type,
|
||||
attachment.size,
|
||||
attachment.storage_key,
|
||||
attachment.checksum_sha256
|
||||
],
|
||||
)
|
||||
.map_err(|e| format!("Failed to add draft attachment: {e}"))?;
|
||||
Ok(())
|
||||
@@ -229,8 +243,9 @@ pub fn get_draft_attachments(
|
||||
filename: row.get(2)?,
|
||||
content_type: row.get(3)?,
|
||||
size: row.get(4)?,
|
||||
file_path: row.get(5)?,
|
||||
created_at: row.get(6)?,
|
||||
storage_key: row.get(5)?,
|
||||
checksum_sha256: row.get(6)?,
|
||||
created_at: row.get(7)?,
|
||||
})
|
||||
})
|
||||
.map_err(|e| format!("Failed to query draft attachments: {e}"))?;
|
||||
@@ -249,6 +264,21 @@ pub fn delete_draft_attachment(conn: &Connection, id: &str) -> Result<bool, Stri
|
||||
Ok(deleted > 0)
|
||||
}
|
||||
|
||||
pub fn update_attachment_storage(
|
||||
conn: &Connection,
|
||||
id: &str,
|
||||
storage_key: &str,
|
||||
checksum_sha256: Option<&str>,
|
||||
) -> Result<bool, String> {
|
||||
let updated = conn
|
||||
.execute(
|
||||
crate::dbq!(db_drafts::Q_009),
|
||||
params![storage_key, checksum_sha256, id],
|
||||
)
|
||||
.map_err(|e| format!("Failed to update draft attachment storage: {e}"))?;
|
||||
Ok(updated > 0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -431,23 +461,29 @@ mod tests {
|
||||
|
||||
add_draft_attachment(
|
||||
&conn,
|
||||
"att-1",
|
||||
"draft-1",
|
||||
"file.pdf",
|
||||
"application/pdf",
|
||||
1024,
|
||||
"/data/abc/attachments/draft-1/att-1",
|
||||
&NewDraftAttachment {
|
||||
id: "att-1",
|
||||
draft_id: "draft-1",
|
||||
filename: "file.pdf",
|
||||
content_type: "application/pdf",
|
||||
size: 1024,
|
||||
storage_key: "/data/abc/attachments/draft-1/att-1",
|
||||
checksum_sha256: None,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
add_draft_attachment(
|
||||
&conn,
|
||||
"att-2",
|
||||
"draft-1",
|
||||
"photo.jpg",
|
||||
"image/jpeg",
|
||||
2048,
|
||||
"/data/abc/attachments/draft-1/att-2",
|
||||
&NewDraftAttachment {
|
||||
id: "att-2",
|
||||
draft_id: "draft-1",
|
||||
filename: "photo.jpg",
|
||||
content_type: "image/jpeg",
|
||||
size: 2048,
|
||||
storage_key: "/data/abc/attachments/draft-1/att-2",
|
||||
checksum_sha256: None,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
@@ -469,12 +505,15 @@ mod tests {
|
||||
.unwrap();
|
||||
add_draft_attachment(
|
||||
&conn,
|
||||
"att-1",
|
||||
"draft-1",
|
||||
"file.pdf",
|
||||
"application/pdf",
|
||||
100,
|
||||
"/path",
|
||||
&NewDraftAttachment {
|
||||
id: "att-1",
|
||||
draft_id: "draft-1",
|
||||
filename: "file.pdf",
|
||||
content_type: "application/pdf",
|
||||
size: 100,
|
||||
storage_key: "/path",
|
||||
checksum_sha256: None,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
@@ -496,22 +535,28 @@ mod tests {
|
||||
.unwrap();
|
||||
add_draft_attachment(
|
||||
&conn,
|
||||
"att-1",
|
||||
"draft-1",
|
||||
"file.pdf",
|
||||
"application/pdf",
|
||||
100,
|
||||
"/path",
|
||||
&NewDraftAttachment {
|
||||
id: "att-1",
|
||||
draft_id: "draft-1",
|
||||
filename: "file.pdf",
|
||||
content_type: "application/pdf",
|
||||
size: 100,
|
||||
storage_key: "/path",
|
||||
checksum_sha256: None,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
add_draft_attachment(
|
||||
&conn,
|
||||
"att-2",
|
||||
"draft-1",
|
||||
"pic.jpg",
|
||||
"image/jpeg",
|
||||
200,
|
||||
"/path2",
|
||||
&NewDraftAttachment {
|
||||
id: "att-2",
|
||||
draft_id: "draft-1",
|
||||
filename: "pic.jpg",
|
||||
content_type: "image/jpeg",
|
||||
size: 200,
|
||||
storage_key: "/path2",
|
||||
checksum_sha256: None,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
|
||||
@@ -232,7 +232,7 @@ pub fn update_identity(
|
||||
boolean!(signature_forward, "signature_forward");
|
||||
text!(signature_position, "signature_position");
|
||||
if let Some(ref value) = data.pgp_fingerprint {
|
||||
sets.push(format!("pgp_fingerprint = ?{}", idx));
|
||||
sets.push(format!("pgp_fingerprint = ?{idx}"));
|
||||
values.push(value.to_db_value());
|
||||
idx += 1;
|
||||
}
|
||||
|
||||
+15
-11
@@ -1,5 +1,5 @@
|
||||
use crate::db::pool::{
|
||||
Connection, DatabaseBackend, DbValue, ToDbValue, configured_backend, params,
|
||||
Connection, DatabaseBackend, DbValue, ToDbValue, Transaction, configured_backend, params,
|
||||
};
|
||||
use crate::queries::sqlite::queries::db_messages::MSG_SELECT_COLS;
|
||||
use serde::Serialize;
|
||||
@@ -108,11 +108,20 @@ pub fn upsert_message(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn remove_after_mutation_in_transaction(
|
||||
transaction: &Transaction,
|
||||
folder: &str,
|
||||
uid: u32,
|
||||
) -> Result<(), String> {
|
||||
delete_message(transaction, folder, uid)?;
|
||||
super::header_sync::invalidate(transaction)?;
|
||||
super::folders::invalidate_folder_freshness(transaction, folder)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn remove_after_mutation(conn: &Connection, folder: &str, uid: u32) -> Result<(), String> {
|
||||
let transaction = conn.unchecked_transaction().map_err(|e| e.to_string())?;
|
||||
delete_message(&transaction, folder, uid)?;
|
||||
super::header_sync::invalidate(&transaction)?;
|
||||
super::folders::invalidate_folder_freshness(&transaction, folder)?;
|
||||
remove_after_mutation_in_transaction(&transaction, folder, uid)?;
|
||||
transaction.commit().map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
@@ -495,7 +504,7 @@ pub fn get_thread_messages(
|
||||
.replace('!', "!!")
|
||||
.replace("%", "!%")
|
||||
.replace("_", "!_");
|
||||
let like_pattern = format!("%{}%", escaped_target);
|
||||
let like_pattern = format!("%{escaped_target}%");
|
||||
let sql = crate::dbq!(db_messages::FORMAT_004).replace("{MSG_SELECT_COLS}", MSG_SELECT_COLS);
|
||||
|
||||
let mut stmt = conn
|
||||
@@ -815,12 +824,7 @@ mod tests {
|
||||
ensure_folder(&conn, "INBOX");
|
||||
|
||||
for uid in 1..=5 {
|
||||
insert_sample(
|
||||
&conn,
|
||||
"INBOX",
|
||||
uid,
|
||||
&format!("2024-01-{:02}T10:00:00Z", uid),
|
||||
);
|
||||
insert_sample(&conn, "INBOX", uid, &format!("2024-01-{uid:02}T10:00:00Z"));
|
||||
}
|
||||
|
||||
let page0 = get_messages(&conn, "INBOX", 0, 2).unwrap();
|
||||
|
||||
@@ -13,6 +13,7 @@ pub mod notification_preferences;
|
||||
pub mod pgp;
|
||||
pub mod pool;
|
||||
pub mod response_templates;
|
||||
pub mod search_outbox;
|
||||
pub mod special_folders;
|
||||
pub mod tags;
|
||||
pub mod trusted_senders;
|
||||
|
||||
+78
-19
@@ -20,10 +20,20 @@ const SQLITE_SCHEMA: &str = include_str!("../../migrations/sqlite/schema.sql");
|
||||
const MYSQL_SCHEMA: &str = include_str!("../../migrations/mysql/schema.sql");
|
||||
const PGSQL_SCHEMA: &str = include_str!("../../migrations/pgsql/schema.sql");
|
||||
|
||||
// Future migrations start at V047. Each entry must exist for every supported engine.
|
||||
const SQLITE_MIGRATIONS: &[(u32, &str)] = &[];
|
||||
const MYSQL_MIGRATIONS: &[(u32, &str)] = &[];
|
||||
const PGSQL_MIGRATIONS: &[(u32, &str)] = &[];
|
||||
// The squashed legacy baseline is V046. Runtime migrations start at V047 and
|
||||
// are stored as real files for every supported SQL engine.
|
||||
const SQLITE_MIGRATIONS: &[(u32, &str)] = &[(
|
||||
47,
|
||||
include_str!("../../migrations/sqlite/V047__blob_keys_and_search_outbox.sql"),
|
||||
)];
|
||||
const MYSQL_MIGRATIONS: &[(u32, &str)] = &[(
|
||||
47,
|
||||
include_str!("../../migrations/mysql/V047__blob_keys_and_search_outbox.sql"),
|
||||
)];
|
||||
const PGSQL_MIGRATIONS: &[(u32, &str)] = &[(
|
||||
47,
|
||||
include_str!("../../migrations/pgsql/V047__blob_keys_and_search_outbox.sql"),
|
||||
)];
|
||||
|
||||
const TENANT_TABLES: &[&str] = &[
|
||||
"calendar_events",
|
||||
@@ -48,6 +58,7 @@ const TENANT_TABLES: &[&str] = &[
|
||||
"pgp_preferences",
|
||||
"response_templates",
|
||||
"saved_searches",
|
||||
"search_outbox",
|
||||
"special_folders",
|
||||
"tags",
|
||||
"trusted_senders",
|
||||
@@ -79,6 +90,7 @@ const IMPORT_TABLES: &[&str] = &[
|
||||
"pgp_preferences",
|
||||
"response_templates",
|
||||
"saved_searches",
|
||||
"search_outbox",
|
||||
"special_folders",
|
||||
"trusted_senders",
|
||||
"user_meta",
|
||||
@@ -96,7 +108,7 @@ impl DatabaseBackend {
|
||||
match value.trim().to_ascii_lowercase().as_str() {
|
||||
"sqlite" => Ok(Self::Sqlite),
|
||||
"mysql" => Ok(Self::Mysql),
|
||||
"pgsql" | "postgres" | "postgresql" => Ok(Self::Pgsql),
|
||||
"pgsql" => Ok(Self::Pgsql),
|
||||
other => Err(format!("unsupported database backend: {other}")),
|
||||
}
|
||||
}
|
||||
@@ -291,6 +303,46 @@ pub fn configured_backend() -> DatabaseBackend {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn list_account_user_hashes() -> Result<Vec<String>, String> {
|
||||
match DATABASE.get() {
|
||||
Some(ConfiguredDatabase::Sqlite { path }) => {
|
||||
let conn = rusqlite::Connection::open(path)
|
||||
.map_err(|e| format!("failed to open SQLite account registry: {e}"))?;
|
||||
let mut stmt = conn
|
||||
.prepare("SELECT user_hash FROM rustbox_accounts ORDER BY user_hash")
|
||||
.map_err(|e| format!("failed to prepare account registry query: {e}"))?;
|
||||
let rows = stmt
|
||||
.query_map([], |row| row.get::<_, String>(0))
|
||||
.map_err(|e| format!("failed to query account registry: {e}"))?;
|
||||
rows.map(|row| row.map_err(|e| format!("failed to read account registry row: {e}")))
|
||||
.collect()
|
||||
}
|
||||
Some(ConfiguredDatabase::Mysql { pool, runtime }) => {
|
||||
let pool = pool.clone();
|
||||
runtime_wait(runtime, async move {
|
||||
sqlx::query_scalar::<_, String>(
|
||||
"SELECT user_hash FROM rustbox_accounts ORDER BY user_hash",
|
||||
)
|
||||
.fetch_all(&pool)
|
||||
.await
|
||||
.map_err(|e| format!("failed to query MySQL account registry: {e}"))
|
||||
})
|
||||
}
|
||||
Some(ConfiguredDatabase::Pgsql { pool, runtime }) => {
|
||||
let pool = pool.clone();
|
||||
runtime_wait(runtime, async move {
|
||||
sqlx::query_scalar::<_, String>(
|
||||
"SELECT user_hash FROM rustbox_accounts ORDER BY user_hash",
|
||||
)
|
||||
.fetch_all(&pool)
|
||||
.await
|
||||
.map_err(|e| format!("failed to query PostgreSQL account registry: {e}"))
|
||||
})
|
||||
}
|
||||
None => Err("database is not configured".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
#[macro_export]
|
||||
macro_rules! dbq {
|
||||
($module:ident::$name:ident) => {{
|
||||
@@ -1562,12 +1614,16 @@ fn read_legacy_sqlite(path: &Path) -> Result<Vec<LegacyTableData>, String> {
|
||||
.map_err(|e| format!("failed to read legacy SQLite schema version: {e}"))?;
|
||||
if version < BASE_SCHEMA_VERSION {
|
||||
return Err(format!(
|
||||
"legacy SQLite schema version {version} is older than the RustBox 0.1.2 baseline {BASE_SCHEMA_VERSION}; upgrade with RustBox 0.1.0 first"
|
||||
"legacy SQLite schema version {version} is older than the legacy RustBox baseline {BASE_SCHEMA_VERSION}; upgrade with RustBox 0.1.0 first"
|
||||
));
|
||||
}
|
||||
if version > BASE_SCHEMA_VERSION {
|
||||
let latest_supported_sqlite = SQLITE_MIGRATIONS
|
||||
.last()
|
||||
.map(|(migration_version, _)| *migration_version)
|
||||
.unwrap_or(BASE_SCHEMA_VERSION);
|
||||
if version > latest_supported_sqlite {
|
||||
return Err(format!(
|
||||
"legacy SQLite schema version {version} is newer than the RustBox 0.1.2 baseline {BASE_SCHEMA_VERSION}"
|
||||
"legacy SQLite schema version {version} is newer than the latest SQLite schema supported by this build ({latest_supported_sqlite})"
|
||||
));
|
||||
}
|
||||
let mut output = Vec::new();
|
||||
@@ -1904,7 +1960,7 @@ fn ensure_sqlite_tenant(
|
||||
.map_err(|e| format!("failed to initialize SQLite tenant defaults: {e}"))?;
|
||||
tx.execute(
|
||||
"INSERT INTO rustbox_accounts(user_hash, table_prefix, schema_version) VALUES(?1, ?2, ?3)",
|
||||
rusqlite::params![user_hash, prefix, BASE_SCHEMA_VERSION],
|
||||
rusqlite::params![user_hash, prefix, latest_schema_version()],
|
||||
)
|
||||
.map_err(|e| format!("failed to register SQLite account schema: {e}"))?;
|
||||
tx.commit().map_err(|e| e.to_string())?;
|
||||
@@ -1922,7 +1978,7 @@ fn apply_sqlite_migrations(
|
||||
) -> Result<(), String> {
|
||||
if current < BASE_SCHEMA_VERSION {
|
||||
return Err(format!(
|
||||
"database schema version {current} is older than the RustBox 0.1.2 baseline {BASE_SCHEMA_VERSION}; upgrade with RustBox 0.1.0 first"
|
||||
"database schema version {current} is older than the legacy RustBox baseline {BASE_SCHEMA_VERSION}; upgrade with RustBox 0.1.0 first"
|
||||
));
|
||||
}
|
||||
let latest = latest_schema_version();
|
||||
@@ -1967,7 +2023,8 @@ fn ensure_server_tenant(
|
||||
user_hash: &str,
|
||||
prefix: &str,
|
||||
) -> Result<(), String> {
|
||||
let legacy_path = Path::new(data_dir).join(user_hash).join("db.sqlite");
|
||||
let legacy_path = (!data_dir.trim().is_empty())
|
||||
.then(|| Path::new(data_dir).join(user_hash).join("db.sqlite"));
|
||||
|
||||
let imported_legacy = match configured {
|
||||
ConfiguredDatabase::Mysql { pool, runtime } => {
|
||||
@@ -2015,8 +2072,8 @@ fn ensure_server_tenant(
|
||||
.bind(&prefix).fetch_optional(&pool).await.map_err(|e| e.to_string())?;
|
||||
if collision.is_some() { return Err("database account-prefix collision detected".into()); }
|
||||
|
||||
let legacy_data = if legacy_path.is_file() {
|
||||
read_legacy_sqlite(&legacy_path)?
|
||||
let legacy_data = if let Some(path) = legacy_path.as_ref().filter(|path| path.is_file()) {
|
||||
read_legacy_sqlite(path)?
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
@@ -2035,7 +2092,7 @@ fn ensure_server_tenant(
|
||||
return Err(format!("failed to initialize MySQL tenant defaults: {error}"));
|
||||
}
|
||||
if let Err(error) = sqlx::query("INSERT INTO rustbox_accounts(user_hash, table_prefix, schema_version) VALUES(?, ?, ?)")
|
||||
.bind(&user_hash).bind(&prefix).bind(i64::from(BASE_SCHEMA_VERSION)).execute(&pool).await
|
||||
.bind(&user_hash).bind(&prefix).bind(i64::from(latest_schema_version())).execute(&pool).await
|
||||
{
|
||||
drop_mysql_tenant(&pool, &prefix).await;
|
||||
return Err(format!("failed to register MySQL account schema: {error}"));
|
||||
@@ -2090,8 +2147,8 @@ fn ensure_server_tenant(
|
||||
.bind(&prefix).fetch_optional(&pool).await.map_err(|e| e.to_string())?;
|
||||
if collision.is_some() { return Err("database account-prefix collision detected".into()); }
|
||||
|
||||
let legacy_data = if legacy_path.is_file() {
|
||||
read_legacy_sqlite(&legacy_path)?
|
||||
let legacy_data = if let Some(path) = legacy_path.as_ref().filter(|path| path.is_file()) {
|
||||
read_legacy_sqlite(path)?
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
@@ -2110,7 +2167,7 @@ fn ensure_server_tenant(
|
||||
return Err(format!("failed to initialize PostgreSQL tenant defaults: {error}"));
|
||||
}
|
||||
if let Err(error) = sqlx::query("INSERT INTO rustbox_accounts(user_hash, table_prefix, schema_version) VALUES($1, $2, $3)")
|
||||
.bind(&user_hash).bind(&prefix).bind(i64::from(BASE_SCHEMA_VERSION)).execute(&pool).await
|
||||
.bind(&user_hash).bind(&prefix).bind(i64::from(latest_schema_version())).execute(&pool).await
|
||||
{
|
||||
drop_pgsql_tenant(&pool, &prefix).await;
|
||||
return Err(format!("failed to register PostgreSQL account schema: {error}"));
|
||||
@@ -2128,7 +2185,9 @@ fn ensure_server_tenant(
|
||||
};
|
||||
|
||||
if imported_legacy {
|
||||
backup_legacy_sqlite(&legacy_path);
|
||||
if let Some(path) = legacy_path.as_ref() {
|
||||
backup_legacy_sqlite(path);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -2215,7 +2274,7 @@ pub fn open_test_db() -> Connection {
|
||||
.unwrap();
|
||||
raw.execute(
|
||||
"INSERT INTO rustbox_accounts(user_hash, table_prefix, schema_version) VALUES(?1, ?2, ?3)",
|
||||
rusqlite::params![user_hash, prefix, BASE_SCHEMA_VERSION],
|
||||
rusqlite::params![user_hash, prefix, latest_schema_version()],
|
||||
)
|
||||
.unwrap();
|
||||
Connection {
|
||||
|
||||
@@ -0,0 +1,258 @@
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use crate::db::pool::{Connection, params};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct OutboxEvent {
|
||||
pub id: String,
|
||||
pub generation: String,
|
||||
pub event_type: String,
|
||||
pub message_uid: u32,
|
||||
pub folder: String,
|
||||
pub payload: Option<String>,
|
||||
pub attempts: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct OutboxStats {
|
||||
pub total: usize,
|
||||
pub dead_letters: usize,
|
||||
pub lag_seconds: u64,
|
||||
}
|
||||
|
||||
pub fn now_epoch() -> i64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
.min(i64::MAX as u64) as i64
|
||||
}
|
||||
|
||||
pub fn enqueue(
|
||||
conn: &Connection,
|
||||
id: &str,
|
||||
event_type: &str,
|
||||
message_uid: u32,
|
||||
folder: &str,
|
||||
payload: Option<&str>,
|
||||
) -> Result<String, String> {
|
||||
let generation = uuid::Uuid::new_v4().to_string();
|
||||
let now = now_epoch();
|
||||
let result = match crate::db::pool::configured_backend() {
|
||||
crate::db::pool::DatabaseBackend::Mysql => conn.execute(
|
||||
crate::dbq!(db_search_outbox::Q_001),
|
||||
params![
|
||||
id,
|
||||
&generation,
|
||||
event_type,
|
||||
message_uid,
|
||||
folder,
|
||||
payload,
|
||||
now,
|
||||
now
|
||||
],
|
||||
),
|
||||
_ => conn.execute(
|
||||
crate::dbq!(db_search_outbox::Q_001),
|
||||
params![
|
||||
id,
|
||||
&generation,
|
||||
event_type,
|
||||
message_uid,
|
||||
folder,
|
||||
payload,
|
||||
now
|
||||
],
|
||||
),
|
||||
};
|
||||
result.map_err(|e| format!("failed to enqueue search event: {e}"))?;
|
||||
Ok(generation)
|
||||
}
|
||||
|
||||
pub fn list_ready(conn: &Connection, limit: usize) -> Result<Vec<OutboxEvent>, String> {
|
||||
let now = now_epoch();
|
||||
let mut stmt = conn
|
||||
.prepare(crate::dbq!(db_search_outbox::Q_002))
|
||||
.map_err(|e| format!("failed to prepare search outbox query: {e}"))?;
|
||||
let ready_params = match crate::db::pool::configured_backend() {
|
||||
crate::db::pool::DatabaseBackend::Mysql => params![now, now, limit],
|
||||
_ => params![now, limit],
|
||||
};
|
||||
let rows = stmt
|
||||
.query_map(ready_params, |row| {
|
||||
Ok(OutboxEvent {
|
||||
id: row.get(0)?,
|
||||
generation: row.get(1)?,
|
||||
event_type: row.get(2)?,
|
||||
message_uid: row.get::<_, i64>(3)?.max(0) as u32,
|
||||
folder: row.get(4)?,
|
||||
payload: row.get(5)?,
|
||||
attempts: row.get::<_, i64>(6)?.max(0) as u32,
|
||||
})
|
||||
})
|
||||
.map_err(|e| format!("failed to query search outbox: {e}"))?;
|
||||
rows.map(|row| row.map_err(|e| format!("failed to read search outbox row: {e}")))
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn claim(conn: &Connection, event: &OutboxEvent, lease_secs: u64) -> Result<bool, String> {
|
||||
let now = now_epoch();
|
||||
let lease_until = now.saturating_add(lease_secs.max(1) as i64);
|
||||
let changed = match crate::db::pool::configured_backend() {
|
||||
crate::db::pool::DatabaseBackend::Mysql => conn.execute(
|
||||
crate::dbq!(db_search_outbox::Q_006),
|
||||
params![lease_until, &event.id, &event.generation, now, now],
|
||||
),
|
||||
_ => conn.execute(
|
||||
crate::dbq!(db_search_outbox::Q_006),
|
||||
params![&event.id, &event.generation, lease_until, now],
|
||||
),
|
||||
}
|
||||
.map_err(|e| format!("failed to claim search event: {e}"))?;
|
||||
Ok(changed > 0)
|
||||
}
|
||||
|
||||
pub fn mark_success(conn: &Connection, event: &OutboxEvent) -> Result<bool, String> {
|
||||
let changed = conn
|
||||
.execute(
|
||||
crate::dbq!(db_search_outbox::Q_003),
|
||||
params![&event.id, &event.generation],
|
||||
)
|
||||
.map_err(|e| format!("failed to acknowledge search event: {e}"))?;
|
||||
Ok(changed > 0)
|
||||
}
|
||||
|
||||
pub fn mark_failure(
|
||||
conn: &Connection,
|
||||
event: &OutboxEvent,
|
||||
error: &str,
|
||||
max_attempts: u32,
|
||||
) -> Result<bool, String> {
|
||||
let attempts = event.attempts.saturating_add(1);
|
||||
let dead = attempts >= max_attempts;
|
||||
let backoff = 2u64.saturating_pow(attempts.min(10)).min(900);
|
||||
let available_at = now_epoch().saturating_add(backoff as i64);
|
||||
let now = now_epoch();
|
||||
let truncated: String = error.chars().take(2048).collect();
|
||||
let changed = match crate::db::pool::configured_backend() {
|
||||
crate::db::pool::DatabaseBackend::Mysql => conn.execute(
|
||||
crate::dbq!(db_search_outbox::Q_004),
|
||||
params![
|
||||
attempts,
|
||||
available_at,
|
||||
dead,
|
||||
&truncated,
|
||||
now,
|
||||
&event.id,
|
||||
&event.generation
|
||||
],
|
||||
),
|
||||
_ => conn.execute(
|
||||
crate::dbq!(db_search_outbox::Q_004),
|
||||
params![
|
||||
&event.id,
|
||||
&event.generation,
|
||||
attempts,
|
||||
available_at,
|
||||
dead,
|
||||
&truncated,
|
||||
now
|
||||
],
|
||||
),
|
||||
}
|
||||
.map_err(|e| format!("failed to update search retry state: {e}"))?;
|
||||
Ok(changed > 0)
|
||||
}
|
||||
|
||||
pub fn clear_before(conn: &Connection, updated_before: i64) -> Result<usize, String> {
|
||||
conn.execute(
|
||||
crate::dbq!(db_search_outbox::Q_007),
|
||||
params![updated_before],
|
||||
)
|
||||
.map_err(|e| format!("failed to clear superseded search events: {e}"))
|
||||
}
|
||||
|
||||
pub fn stats(conn: &Connection) -> Result<OutboxStats, String> {
|
||||
let (total, dead, oldest): (i64, i64, i64) = conn
|
||||
.query_row(crate::dbq!(db_search_outbox::Q_005), [], |row| {
|
||||
Ok((row.get(0)?, row.get(1)?, row.get(2)?))
|
||||
})
|
||||
.map_err(|e| format!("failed to read search outbox stats: {e}"))?;
|
||||
let now = now_epoch();
|
||||
Ok(OutboxStats {
|
||||
total: total.max(0) as usize,
|
||||
dead_letters: dead.max(0) as usize,
|
||||
lag_seconds: if oldest > 0 {
|
||||
now.saturating_sub(oldest) as u64
|
||||
} else {
|
||||
0
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::db::pool::open_test_db;
|
||||
|
||||
#[test]
|
||||
fn enqueue_is_idempotent_per_document_and_generation_safe() {
|
||||
let conn = open_test_db();
|
||||
let first =
|
||||
enqueue(&conn, "doc-1", "upsert", 7, "INBOX", Some("{}")).expect("first enqueue");
|
||||
let second = enqueue(&conn, "doc-1", "delete", 7, "INBOX", None).expect("second enqueue");
|
||||
assert_ne!(first, second);
|
||||
let events = list_ready(&conn, 10).unwrap();
|
||||
assert_eq!(events.len(), 1);
|
||||
assert_eq!(events[0].event_type, "delete");
|
||||
let stale = OutboxEvent {
|
||||
generation: first,
|
||||
..events[0].clone()
|
||||
};
|
||||
assert!(!mark_success(&conn, &stale).unwrap());
|
||||
assert!(mark_success(&conn, &events[0]).unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enqueue_participates_in_caller_transaction() {
|
||||
let conn = open_test_db();
|
||||
{
|
||||
let transaction = conn.unchecked_transaction().unwrap();
|
||||
enqueue(&transaction, "doc-tx", "delete", 11, "INBOX", None).unwrap();
|
||||
// Drop without commit: the message mutation and its search event must roll back together.
|
||||
}
|
||||
assert!(list_ready(&conn, 10).unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn event_claim_is_exclusive_until_lease_expires() {
|
||||
let conn = open_test_db();
|
||||
enqueue(&conn, "doc-claim", "delete", 3, "INBOX", None).unwrap();
|
||||
let event = list_ready(&conn, 1).unwrap().remove(0);
|
||||
assert!(claim(&conn, &event, 60).unwrap());
|
||||
assert!(!claim(&conn, &event, 60).unwrap());
|
||||
assert!(list_ready(&conn, 10).unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reenqueue_preserves_active_claim_lease() {
|
||||
let conn = open_test_db();
|
||||
enqueue(&conn, "doc-lease", "upsert", 4, "INBOX", Some("{}")).expect("initial enqueue");
|
||||
let event = list_ready(&conn, 1).unwrap().remove(0);
|
||||
assert!(claim(&conn, &event, 60).unwrap());
|
||||
|
||||
enqueue(&conn, "doc-lease", "delete", 4, "INBOX", None).expect("replacement enqueue");
|
||||
assert!(list_ready(&conn, 10).unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failures_backoff_and_dead_letter() {
|
||||
let conn = open_test_db();
|
||||
enqueue(&conn, "doc-2", "delete", 9, "Trash", None).unwrap();
|
||||
let event = list_ready(&conn, 1).unwrap().remove(0);
|
||||
assert!(mark_failure(&conn, &event, "boom", 1).unwrap());
|
||||
let stats = stats(&conn).unwrap();
|
||||
assert_eq!(stats.total, 1);
|
||||
assert_eq!(stats.dead_letters, 1);
|
||||
}
|
||||
}
|
||||
+2
-4
@@ -751,8 +751,7 @@ mod tests {
|
||||
let colors = extract_background_colors(html);
|
||||
assert!(
|
||||
colors.iter().any(|c| c.color == "whitesmoke"),
|
||||
"expected whitesmoke, got: {:?}",
|
||||
colors
|
||||
"expected whitesmoke, got: {colors:?}"
|
||||
);
|
||||
assert!(
|
||||
!colors.iter().any(|c| c.color == "white"),
|
||||
@@ -789,8 +788,7 @@ mod tests {
|
||||
let colors = extract_background_colors(html);
|
||||
assert!(
|
||||
colors.is_empty(),
|
||||
"Fully transparent rgba should produce no colors, got {:?}",
|
||||
colors
|
||||
"Fully transparent rgba should produce no colors, got {colors:?}"
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
+1
-4
@@ -1111,10 +1111,7 @@ impl ImapClient for RealImapClient {
|
||||
|
||||
let items = {
|
||||
let mut fetch_stream = session
|
||||
.uid_fetch(
|
||||
"1:*",
|
||||
format!("(UID FLAGS) (CHANGEDSINCE {})", since_modseq),
|
||||
)
|
||||
.uid_fetch("1:*", format!("(UID FLAGS) (CHANGEDSINCE {since_modseq})"))
|
||||
.await
|
||||
.map_err(map_imap_error)?;
|
||||
|
||||
|
||||
+73
-13
@@ -13,6 +13,7 @@ mod search;
|
||||
mod session_store;
|
||||
mod sieve;
|
||||
mod smtp;
|
||||
mod storage;
|
||||
mod ui;
|
||||
|
||||
use std::fs::OpenOptions;
|
||||
@@ -58,6 +59,33 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
let config = Arc::new(loaded_config);
|
||||
let storage = storage::from_config(&config).map_err(std::io::Error::other)?;
|
||||
|
||||
if let Some(options) = storage::parse_migration_args(&args).map_err(std::io::Error::other)? {
|
||||
let report = storage::migrate_attachments(&config, storage.clone(), &options)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
println!(
|
||||
"storage migration complete: scanned={} migrated={} skipped={} failed={}",
|
||||
report.scanned, report.migrated, report.skipped, report.failed
|
||||
);
|
||||
if report.failed > 0 {
|
||||
return Err(std::io::Error::other("storage migration completed with failures").into());
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let search_engine = Arc::new(
|
||||
search::engine::SearchEngine::from_config(&config).map_err(std::io::Error::other)?,
|
||||
);
|
||||
if args.get(1).map(String::as_str) == Some("search")
|
||||
&& args.get(2).map(String::as_str) == Some("reindex")
|
||||
{
|
||||
run_search_reindex(&config, &search_engine)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let persistence = session_store::from_config(&config)
|
||||
.await
|
||||
@@ -86,24 +114,28 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let imap_client: Arc<dyn imap::client::ImapClient> = Arc::new(RealImapClient::default());
|
||||
let smtp_client: Arc<dyn smtp::client::SmtpClient> = Arc::new(RealSmtpClient);
|
||||
|
||||
let search_base = std::path::PathBuf::from(&config.data_dir);
|
||||
let search_engine = Arc::new(if config.search_index_backend == "disabled" {
|
||||
search::engine::SearchEngine::disabled(search_base)
|
||||
} else {
|
||||
search::engine::SearchEngine::new(search_base)
|
||||
});
|
||||
if config.search.required {
|
||||
search_engine
|
||||
.health()
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
}
|
||||
let _search_worker = search_engine.start_worker();
|
||||
|
||||
let event_bus = Arc::new(realtime::events::EventBus::new());
|
||||
let idle_manager = Arc::new(realtime::idle::IdleManager::new());
|
||||
|
||||
let app = routes::create_router(
|
||||
let app = routes::create_router_with_storage(
|
||||
config.clone(),
|
||||
store,
|
||||
imap_client,
|
||||
smtp_client,
|
||||
search_engine,
|
||||
event_bus,
|
||||
idle_manager.clone(),
|
||||
routes::RouterServices {
|
||||
store,
|
||||
imap_client,
|
||||
smtp_client,
|
||||
search_engine,
|
||||
storage,
|
||||
event_bus,
|
||||
idle_manager: idle_manager.clone(),
|
||||
},
|
||||
);
|
||||
|
||||
let bind_addr = format!("{}:{}", config.host, config.port);
|
||||
@@ -126,6 +158,34 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn run_search_reindex(
|
||||
config: &AppConfig,
|
||||
search_engine: &search::engine::SearchEngine,
|
||||
) -> Result<(), String> {
|
||||
if !search_engine.is_enabled() {
|
||||
return Err("search reindex requires an enabled search backend".to_string());
|
||||
}
|
||||
search_engine.health().await?;
|
||||
let reindex_started_at = db::search_outbox::now_epoch();
|
||||
let mut users = 0usize;
|
||||
let mut source_messages = 0usize;
|
||||
let mut indexed_messages = 0usize;
|
||||
for user_hash in db::pool::list_account_user_hashes()? {
|
||||
let conn = db::pool::open_user_db(&config.data_dir, &user_hash)?;
|
||||
let sources = db::messages::list_index_source_messages(&conn)?;
|
||||
source_messages += sources.len();
|
||||
let messages = routes::diagnostics::index_sources_to_messages(sources);
|
||||
let index = search_engine.open_user_index(&user_hash)?;
|
||||
indexed_messages += index.rebuild_from_messages(&messages).await?;
|
||||
db::search_outbox::clear_before(&conn, reindex_started_at)?;
|
||||
users += 1;
|
||||
}
|
||||
println!(
|
||||
"search reindex complete: users={users} source_messages={source_messages} indexed_messages={indexed_messages}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn init_tracing(log_dir: Option<&str>, log_level: &str) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let env_filter = EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| EnvFilter::new(log_level.trim().to_ascii_lowercase()));
|
||||
|
||||
@@ -126,7 +126,7 @@ async fn https_get_address(
|
||||
format!("{host}:{port}")
|
||||
};
|
||||
let request = format!(
|
||||
"GET {target} HTTP/1.1\r\nHost: {host_header}\r\nUser-Agent: RustBox/0.1.2\r\nAccept: application/octet-stream, application/pgp-keys, text/plain\r\nConnection: close\r\n\r\n"
|
||||
"GET {target} HTTP/1.1\r\nHost: {host_header}\r\nUser-Agent: RustBox/0.2.0\r\nAccept: application/octet-stream, application/pgp-keys, text/plain\r\nConnection: close\r\n\r\n"
|
||||
);
|
||||
timeout(duration, stream.write_all(request.as_bytes()))
|
||||
.await
|
||||
|
||||
+1
-2
@@ -1090,8 +1090,7 @@ mod tests {
|
||||
let encrypted =
|
||||
encrypt_bytes(clear, std::slice::from_ref(&key.public_armor)).expect("encrypt mime");
|
||||
let encrypted_mime = format!(
|
||||
"Content-Type: multipart/encrypted; protocol=\"application/pgp-encrypted\"; boundary=\"rbe\"\r\n\r\n--rbe\r\nContent-Type: application/pgp-encrypted\r\n\r\nVersion: 1\r\n--rbe\r\nContent-Type: application/octet-stream\r\n\r\n{}\r\n--rbe--\r\n",
|
||||
encrypted
|
||||
"Content-Type: multipart/encrypted; protocol=\"application/pgp-encrypted\"; boundary=\"rbe\"\r\n\r\n--rbe\r\nContent-Type: application/pgp-encrypted\r\n\r\nVersion: 1\r\n--rbe\r\nContent-Type: application/octet-stream\r\n\r\n{encrypted}\r\n--rbe--\r\n"
|
||||
);
|
||||
let secrets = vec![secret.to_string()];
|
||||
let processed = process_message(encrypted_mime.as_bytes(), &secrets, "", &public)
|
||||
|
||||
@@ -159,12 +159,14 @@ pub mod db_drafts {
|
||||
pub const Q_004: &str = "UPDATE drafts SET remote_folder = ?2, remote_uid = ?3, remote_uid_validity = ?4 WHERE id = ?1";
|
||||
pub const Q_005: &str = "DELETE FROM drafts WHERE id = ?";
|
||||
pub const Q_006: &str =
|
||||
"INSERT INTO draft_attachments (id, draft_id, filename, content_type, size, file_path)
|
||||
VALUES (?, ?, ?, ?, ?, ?)";
|
||||
"INSERT INTO draft_attachments (id, draft_id, filename, content_type, size, file_path, checksum_sha256)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)";
|
||||
pub const Q_007: &str =
|
||||
"SELECT id, draft_id, filename, content_type, size, file_path, created_at
|
||||
"SELECT id, draft_id, filename, content_type, size, file_path, checksum_sha256, created_at
|
||||
FROM draft_attachments WHERE draft_id = ? ORDER BY created_at ASC";
|
||||
pub const Q_008: &str = "DELETE FROM draft_attachments WHERE id = ?";
|
||||
pub const Q_009: &str =
|
||||
"UPDATE draft_attachments SET file_path=?, checksum_sha256=? WHERE id=?";
|
||||
}
|
||||
|
||||
pub mod db_folder_preferences {
|
||||
@@ -496,3 +498,13 @@ pub mod sessions {
|
||||
pub const DELETE_EXPIRED: &str =
|
||||
"DELETE FROM sessions WHERE last_accessed_unix + timeout_secs < ?";
|
||||
}
|
||||
|
||||
pub mod db_search_outbox {
|
||||
pub const Q_001: &str = "INSERT INTO search_outbox (id, generation, event_type, message_uid, folder, payload, attempts, available_at, dead_letter, last_error, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, 0, 0, 0, NULL, ?, ?) ON DUPLICATE KEY UPDATE generation=VALUES(generation), event_type=VALUES(event_type), message_uid=VALUES(message_uid), folder=VALUES(folder), payload=VALUES(payload), attempts=0, available_at=0, dead_letter=0, last_error=NULL, created_at=VALUES(created_at), updated_at=VALUES(updated_at)";
|
||||
pub const Q_002: &str = "SELECT id, generation, event_type, message_uid, folder, payload, attempts, created_at, updated_at FROM search_outbox WHERE dead_letter=0 AND available_at <= ? AND claimed_until <= ? ORDER BY updated_at ASC LIMIT ?";
|
||||
pub const Q_003: &str = "DELETE FROM search_outbox WHERE id=? AND generation=?";
|
||||
pub const Q_004: &str = "UPDATE search_outbox SET attempts=?, available_at=?, claimed_until=0, dead_letter=?, last_error=?, updated_at=? WHERE id=? AND generation=?";
|
||||
pub const Q_005: &str = "SELECT COUNT(*), COALESCE(SUM(CASE WHEN dead_letter <> 0 THEN 1 ELSE 0 END), 0), COALESCE(MIN(CASE WHEN dead_letter = 0 THEN created_at ELSE NULL END), 0) FROM search_outbox";
|
||||
pub const Q_006: &str = "UPDATE search_outbox SET claimed_until=? WHERE id=? AND generation=? AND dead_letter=0 AND available_at <= ? AND claimed_until <= ?";
|
||||
pub const Q_007: &str = "DELETE FROM search_outbox WHERE updated_at < ?";
|
||||
}
|
||||
@@ -160,12 +160,14 @@ pub mod db_drafts {
|
||||
pub const Q_004: &str = "UPDATE drafts SET remote_folder = $2, remote_uid = $3, remote_uid_validity = $4 WHERE id = $1";
|
||||
pub const Q_005: &str = "DELETE FROM drafts WHERE id = $1";
|
||||
pub const Q_006: &str =
|
||||
"INSERT INTO draft_attachments (id, draft_id, filename, content_type, size, file_path)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)";
|
||||
"INSERT INTO draft_attachments (id, draft_id, filename, content_type, size, file_path, checksum_sha256)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)";
|
||||
pub const Q_007: &str =
|
||||
"SELECT id, draft_id, filename, content_type, size, file_path, created_at
|
||||
"SELECT id, draft_id, filename, content_type, size, file_path, checksum_sha256, created_at
|
||||
FROM draft_attachments WHERE draft_id = $1 ORDER BY created_at ASC";
|
||||
pub const Q_008: &str = "DELETE FROM draft_attachments WHERE id = $1";
|
||||
pub const Q_009: &str =
|
||||
"UPDATE draft_attachments SET file_path=$1, checksum_sha256=$2 WHERE id=$3";
|
||||
}
|
||||
|
||||
pub mod db_folder_preferences {
|
||||
@@ -499,3 +501,13 @@ pub mod sessions {
|
||||
pub const DELETE_EXPIRED: &str =
|
||||
"DELETE FROM sessions WHERE last_accessed_unix + timeout_secs < $1";
|
||||
}
|
||||
|
||||
pub mod db_search_outbox {
|
||||
pub const Q_001: &str = "INSERT INTO search_outbox (id, generation, event_type, message_uid, folder, payload, attempts, available_at, dead_letter, last_error, created_at, updated_at) VALUES ($1, $2, $3, $4, $5, $6, 0, 0, 0, NULL, $7, $7) ON CONFLICT(id) DO UPDATE SET generation=excluded.generation, event_type=excluded.event_type, message_uid=excluded.message_uid, folder=excluded.folder, payload=excluded.payload, attempts=0, available_at=0, dead_letter=0, last_error=NULL, created_at=excluded.created_at, updated_at=excluded.updated_at";
|
||||
pub const Q_002: &str = "SELECT id, generation, event_type, message_uid, folder, payload, attempts, created_at, updated_at FROM search_outbox WHERE dead_letter=0 AND available_at <= $1 AND claimed_until <= $1 ORDER BY updated_at ASC LIMIT $2";
|
||||
pub const Q_003: &str = "DELETE FROM search_outbox WHERE id=$1 AND generation=$2";
|
||||
pub const Q_004: &str = "UPDATE search_outbox SET attempts=$3, available_at=$4, claimed_until=0, dead_letter=$5, last_error=$6, updated_at=$7 WHERE id=$1 AND generation=$2";
|
||||
pub const Q_005: &str = "SELECT COUNT(*), COALESCE(SUM(CASE WHEN dead_letter <> 0 THEN 1 ELSE 0 END), 0), COALESCE(MIN(CASE WHEN dead_letter = 0 THEN created_at ELSE NULL END), 0) FROM search_outbox";
|
||||
pub const Q_006: &str = "UPDATE search_outbox SET claimed_until=$3 WHERE id=$1 AND generation=$2 AND dead_letter=0 AND available_at <= $4 AND claimed_until <= $4";
|
||||
pub const Q_007: &str = "DELETE FROM search_outbox WHERE updated_at < $1";
|
||||
}
|
||||
@@ -157,12 +157,14 @@ pub mod db_drafts {
|
||||
pub const Q_004: &str = "UPDATE drafts SET remote_folder = ?2, remote_uid = ?3, remote_uid_validity = ?4 WHERE id = ?1";
|
||||
pub const Q_005: &str = "DELETE FROM drafts WHERE id = ?1";
|
||||
pub const Q_006: &str =
|
||||
"INSERT INTO draft_attachments (id, draft_id, filename, content_type, size, file_path)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6)";
|
||||
"INSERT INTO draft_attachments (id, draft_id, filename, content_type, size, file_path, checksum_sha256)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)";
|
||||
pub const Q_007: &str =
|
||||
"SELECT id, draft_id, filename, content_type, size, file_path, created_at
|
||||
"SELECT id, draft_id, filename, content_type, size, file_path, checksum_sha256, created_at
|
||||
FROM draft_attachments WHERE draft_id = ?1 ORDER BY created_at ASC";
|
||||
pub const Q_008: &str = "DELETE FROM draft_attachments WHERE id = ?1";
|
||||
pub const Q_009: &str =
|
||||
"UPDATE draft_attachments SET file_path=?1, checksum_sha256=?2 WHERE id=?3";
|
||||
}
|
||||
|
||||
pub mod db_folder_preferences {
|
||||
@@ -747,3 +749,13 @@ pub mod sessions {
|
||||
pub const DELETE_EXPIRED: &str =
|
||||
"DELETE FROM sessions WHERE last_accessed_unix + timeout_secs < ?";
|
||||
}
|
||||
|
||||
pub mod db_search_outbox {
|
||||
pub const Q_001: &str = "INSERT INTO search_outbox (id, generation, event_type, message_uid, folder, payload, attempts, available_at, dead_letter, last_error, created_at, updated_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, 0, 0, 0, NULL, ?7, ?7) ON CONFLICT(id) DO UPDATE SET generation=excluded.generation, event_type=excluded.event_type, message_uid=excluded.message_uid, folder=excluded.folder, payload=excluded.payload, attempts=0, available_at=0, dead_letter=0, last_error=NULL, created_at=excluded.created_at, updated_at=excluded.updated_at";
|
||||
pub const Q_002: &str = "SELECT id, generation, event_type, message_uid, folder, payload, attempts, created_at, updated_at FROM search_outbox WHERE dead_letter=0 AND available_at <= ?1 AND claimed_until <= ?1 ORDER BY updated_at ASC LIMIT ?2";
|
||||
pub const Q_003: &str = "DELETE FROM search_outbox WHERE id=?1 AND generation=?2";
|
||||
pub const Q_004: &str = "UPDATE search_outbox SET attempts=?3, available_at=?4, claimed_until=0, dead_letter=?5, last_error=?6, updated_at=?7 WHERE id=?1 AND generation=?2";
|
||||
pub const Q_005: &str = "SELECT COUNT(*), COALESCE(SUM(CASE WHEN dead_letter <> 0 THEN 1 ELSE 0 END), 0), COALESCE(MIN(CASE WHEN dead_letter = 0 THEN created_at ELSE NULL END), 0) FROM search_outbox";
|
||||
pub const Q_006: &str = "UPDATE search_outbox SET claimed_until=?3 WHERE id=?1 AND generation=?2 AND dead_letter=0 AND available_at <= ?4 AND claimed_until <= ?4";
|
||||
pub const Q_007: &str = "DELETE FROM search_outbox WHERE updated_at < ?1";
|
||||
}
|
||||
+86
-9
@@ -9,6 +9,7 @@ use crate::imap::client::ImapCredentials;
|
||||
use crate::imap::connection::ImapStream;
|
||||
use crate::imap::parse::{decode_rfc2047, flag_to_string, has_attachments, imap_address_to_email};
|
||||
use crate::realtime::events::{EventBus, MailEvent};
|
||||
use crate::search::engine::SearchEngine;
|
||||
use crate::{db, imap};
|
||||
|
||||
pub struct IdleManager {
|
||||
@@ -34,6 +35,7 @@ impl IdleManager {
|
||||
creds: ImapCredentials,
|
||||
event_bus: Arc<EventBus>,
|
||||
config: Arc<AppConfig>,
|
||||
search_engine: Arc<SearchEngine>,
|
||||
) {
|
||||
use dashmap::mapref::entry::Entry;
|
||||
|
||||
@@ -43,7 +45,15 @@ impl IdleManager {
|
||||
}
|
||||
Entry::Vacant(entry) => {
|
||||
let handle = tokio::spawn(async move {
|
||||
idle_loop(&user_hash, &folder, &creds, &event_bus, &config).await;
|
||||
idle_loop(
|
||||
&user_hash,
|
||||
&folder,
|
||||
&creds,
|
||||
&event_bus,
|
||||
&config,
|
||||
&search_engine,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
entry.insert(IdleTask {
|
||||
handle,
|
||||
@@ -94,12 +104,13 @@ async fn idle_loop(
|
||||
creds: &ImapCredentials,
|
||||
event_bus: &EventBus,
|
||||
config: &AppConfig,
|
||||
search_engine: &SearchEngine,
|
||||
) {
|
||||
let mut backoff = std::time::Duration::from_secs(1);
|
||||
let max_backoff = std::time::Duration::from_secs(60);
|
||||
|
||||
loop {
|
||||
match run_idle_session(user_hash, folder, creds, event_bus, config).await {
|
||||
match run_idle_session(user_hash, folder, creds, event_bus, config, search_engine).await {
|
||||
Ok(()) => {
|
||||
tracing::info!(user_hash = %user_hash, folder = %folder, "IDLE session ended normally");
|
||||
break;
|
||||
@@ -125,6 +136,7 @@ async fn run_idle_session(
|
||||
creds: &ImapCredentials,
|
||||
event_bus: &EventBus,
|
||||
config: &AppConfig,
|
||||
search_engine: &SearchEngine,
|
||||
) -> Result<(), String> {
|
||||
use imap::client::connect;
|
||||
|
||||
@@ -173,7 +185,8 @@ async fn run_idle_session(
|
||||
|
||||
let result;
|
||||
(session, result) =
|
||||
fetch_new_after_idle(session, user_hash, folder, config).await;
|
||||
fetch_new_after_idle(session, user_hash, folder, config, search_engine)
|
||||
.await;
|
||||
|
||||
if result.new_count > 0 {
|
||||
event_bus
|
||||
@@ -256,6 +269,7 @@ async fn fetch_new_after_idle(
|
||||
user_hash: &str,
|
||||
folder: &str,
|
||||
config: &AppConfig,
|
||||
search_engine: &SearchEngine,
|
||||
) -> (async_imap::Session<ImapStream>, ReconcileResult) {
|
||||
let (cached_modseq, max_cached_uid, cached_count, cached_uid_validity, header_sync) =
|
||||
match db::pool::open_user_db(&config.data_dir, user_hash) {
|
||||
@@ -310,7 +324,7 @@ async fn fetch_new_after_idle(
|
||||
}
|
||||
|
||||
if cached_modseq > 0 && server_modseq > 0 {
|
||||
let fetch_query = format!("(UID FLAGS) (CHANGEDSINCE {})", cached_modseq);
|
||||
let fetch_query = format!("(UID FLAGS) (CHANGEDSINCE {cached_modseq})");
|
||||
let changed_items: Option<Vec<(u32, Vec<String>)>> =
|
||||
match session.uid_fetch("1:*", &fetch_query).await {
|
||||
Ok(stream) => {
|
||||
@@ -396,8 +410,14 @@ async fn fetch_new_after_idle(
|
||||
|
||||
let mut deleted_count = 0u32;
|
||||
if server_exists < cached_count {
|
||||
deleted_count =
|
||||
fetch_uids_and_remove_deleted(&mut session, user_hash, folder, config).await;
|
||||
deleted_count = fetch_uids_and_remove_deleted(
|
||||
&mut session,
|
||||
user_hash,
|
||||
folder,
|
||||
config,
|
||||
search_engine,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
let (latest_uid, latest_sender, latest_subject) = if new_count > 0 {
|
||||
@@ -514,7 +534,7 @@ async fn fetch_new_after_idle(
|
||||
&& let Ok(conn) = db::pool::open_user_db(&config.data_dir, user_hash)
|
||||
{
|
||||
for uid in &deleted_uids {
|
||||
if db::messages::delete_message(&conn, folder, *uid).is_ok() {
|
||||
if delete_cached_message(&conn, search_engine, user_hash, folder, *uid) {
|
||||
deleted_count += 1;
|
||||
}
|
||||
}
|
||||
@@ -794,11 +814,55 @@ async fn fetch_and_store_headers(
|
||||
new_count
|
||||
}
|
||||
|
||||
fn delete_cached_message(
|
||||
conn: &db::pool::Connection,
|
||||
search_engine: &SearchEngine,
|
||||
user_hash: &str,
|
||||
folder: &str,
|
||||
uid: u32,
|
||||
) -> bool {
|
||||
let transaction = match conn.unchecked_transaction() {
|
||||
Ok(transaction) => transaction,
|
||||
Err(error) => {
|
||||
tracing::warn!(uid, folder = %folder, error = %error, "IDLE delete: failed to start transaction");
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
if let Err(error) = db::messages::delete_message(&transaction, folder, uid) {
|
||||
tracing::warn!(uid, folder = %folder, error = %error, "IDLE delete: failed to remove cached message");
|
||||
return false;
|
||||
}
|
||||
|
||||
if search_engine.is_enabled() {
|
||||
let index = match search_engine.open_user_index(user_hash) {
|
||||
Ok(index) => index,
|
||||
Err(error) => {
|
||||
tracing::warn!(uid, folder = %folder, error = %error, "IDLE delete: failed to open search index");
|
||||
return false;
|
||||
}
|
||||
};
|
||||
if let Err(error) = index.delete_message_with_connection(&transaction, uid, folder) {
|
||||
tracing::warn!(uid, folder = %folder, error = %error, "IDLE delete: failed to enqueue search deletion");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
match transaction.commit() {
|
||||
Ok(()) => true,
|
||||
Err(error) => {
|
||||
tracing::warn!(uid, folder = %folder, error = %error, "IDLE delete: failed to commit transaction");
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn fetch_uids_and_remove_deleted(
|
||||
session: &mut async_imap::Session<ImapStream>,
|
||||
user_hash: &str,
|
||||
folder: &str,
|
||||
config: &AppConfig,
|
||||
search_engine: &SearchEngine,
|
||||
) -> u32 {
|
||||
let imap_uids: Option<std::collections::HashSet<u32>> =
|
||||
match session.uid_fetch("1:*", "UID").await {
|
||||
@@ -840,7 +904,8 @@ async fn fetch_uids_and_remove_deleted(
|
||||
if let Ok(conn) = db::pool::open_user_db(&config.data_dir, user_hash) {
|
||||
let cached = db::messages::get_all_uids_and_flags(&conn, folder).unwrap_or_default();
|
||||
for (uid, _) in &cached {
|
||||
if !imap_uids.contains(uid) && db::messages::delete_message(&conn, folder, *uid).is_ok()
|
||||
if !imap_uids.contains(uid)
|
||||
&& delete_cached_message(&conn, search_engine, user_hash, folder, *uid)
|
||||
{
|
||||
deleted_count += 1;
|
||||
}
|
||||
@@ -859,6 +924,9 @@ mod tests {
|
||||
let manager = IdleManager::new();
|
||||
let config = Arc::new(serde_json::from_str::<AppConfig>("{}").unwrap());
|
||||
let event_bus = Arc::new(EventBus::new());
|
||||
let search_engine = Arc::new(SearchEngine::disabled(std::path::PathBuf::from(
|
||||
"/tmp/rustbox-idle-test-search",
|
||||
)));
|
||||
let creds = ImapCredentials {
|
||||
host: "localhost".into(),
|
||||
port: 143,
|
||||
@@ -876,6 +944,7 @@ mod tests {
|
||||
creds.clone(),
|
||||
event_bus.clone(),
|
||||
config.clone(),
|
||||
search_engine.clone(),
|
||||
)
|
||||
.await;
|
||||
let original_task = manager.tasks.get(&key).unwrap().handle.id();
|
||||
@@ -887,6 +956,7 @@ mod tests {
|
||||
creds.clone(),
|
||||
event_bus.clone(),
|
||||
config.clone(),
|
||||
search_engine.clone(),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(manager.tasks.len(), 1);
|
||||
@@ -900,7 +970,14 @@ mod tests {
|
||||
assert!(manager.tasks.is_empty());
|
||||
|
||||
manager
|
||||
.start_idle(key.0.clone(), key.1.clone(), creds, event_bus, config)
|
||||
.start_idle(
|
||||
key.0.clone(),
|
||||
key.1.clone(),
|
||||
creds,
|
||||
event_bus,
|
||||
config,
|
||||
search_engine,
|
||||
)
|
||||
.await;
|
||||
assert_ne!(manager.tasks.get(&key).unwrap().handle.id(), original_task);
|
||||
manager.stop_idle(&key.0, &key.1).await;
|
||||
|
||||
+38
-13
@@ -127,6 +127,7 @@ async fn run_sync(
|
||||
cached_modseq,
|
||||
&status,
|
||||
event_bus,
|
||||
&search_engine,
|
||||
)
|
||||
.await?;
|
||||
if folder_changed {
|
||||
@@ -143,6 +144,7 @@ async fn run_sync(
|
||||
folder_name,
|
||||
&status,
|
||||
event_bus,
|
||||
&search_engine,
|
||||
)
|
||||
.await?;
|
||||
if folder_changed {
|
||||
@@ -208,6 +210,7 @@ async fn index_message_bodies(
|
||||
let hash = user_hash.to_owned();
|
||||
database
|
||||
.run(move |conn| {
|
||||
let transaction = conn.unchecked_transaction().map_err(|e| e.to_string())?;
|
||||
let att_json = serde_json::to_string(&body.attachments).ok();
|
||||
let detected_theme = body
|
||||
.text_html
|
||||
@@ -215,7 +218,7 @@ async fn index_message_bodies(
|
||||
.and_then(|h| email_theme::detect_email_theme(h))
|
||||
.map(|t| t.as_i32());
|
||||
db::messages::cache_message_body(
|
||||
conn,
|
||||
&transaction,
|
||||
&folder,
|
||||
uid,
|
||||
body.text_html.as_deref(),
|
||||
@@ -225,22 +228,26 @@ async fn index_message_bodies(
|
||||
detected_theme,
|
||||
)?;
|
||||
if let Some(text) = body.text_plain
|
||||
&& let Some(msg) = db::messages::get_single_message(conn, &folder, uid)?
|
||||
&& let Some(msg) = db::messages::get_single_message(&transaction, &folder, uid)?
|
||||
{
|
||||
engine
|
||||
.open_user_index(&hash)?
|
||||
.index_message(&IndexableMessage {
|
||||
uid: msg.uid,
|
||||
folder: msg.folder,
|
||||
subject: msg.subject,
|
||||
from_address: msg.from_address,
|
||||
from_name: msg.from_name,
|
||||
to_addresses: msg.to_addresses,
|
||||
body_text: text,
|
||||
date_epoch: db::messages::parse_date_to_epoch_public(&msg.date),
|
||||
has_attachments: msg.has_attachments,
|
||||
})?;
|
||||
.index_message_with_connection(
|
||||
&transaction,
|
||||
&IndexableMessage {
|
||||
uid: msg.uid,
|
||||
folder: msg.folder,
|
||||
subject: msg.subject,
|
||||
from_address: msg.from_address,
|
||||
from_name: msg.from_name,
|
||||
to_addresses: msg.to_addresses,
|
||||
body_text: text,
|
||||
date_epoch: db::messages::parse_date_to_epoch_public(&msg.date),
|
||||
has_attachments: msg.has_attachments,
|
||||
},
|
||||
)?;
|
||||
}
|
||||
transaction.commit().map_err(|e| e.to_string())?;
|
||||
Ok(())
|
||||
})
|
||||
.await?;
|
||||
@@ -266,6 +273,7 @@ async fn sync_condstore(
|
||||
cached_modseq: u64,
|
||||
status: &crate::imap::types::FolderStatusExtended,
|
||||
event_bus: &EventBus,
|
||||
search_engine: &Arc<SearchEngine>,
|
||||
) -> Result<bool, String> {
|
||||
let (changed, new_modseq) = match imap_client
|
||||
.fetch_changed_flags(creds, folder_name, cached_modseq)
|
||||
@@ -286,6 +294,7 @@ async fn sync_condstore(
|
||||
folder_name,
|
||||
status,
|
||||
event_bus,
|
||||
search_engine,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -311,6 +320,8 @@ async fn sync_condstore(
|
||||
};
|
||||
let name = folder_name.to_owned();
|
||||
let status = status.clone();
|
||||
let search_engine = Arc::clone(search_engine);
|
||||
let search_user_hash = user_hash.to_owned();
|
||||
let (folder_changed, new_messages) = database
|
||||
.run(move |conn| {
|
||||
let sync_keywords = db::mail_preferences::get_preferences(conn)
|
||||
@@ -329,6 +340,9 @@ async fn sync_condstore(
|
||||
for (uid, _) in db::messages::get_all_uids_and_flags(&transaction, &name)? {
|
||||
if !imap_uids.contains(&uid) {
|
||||
db::messages::delete_message(&transaction, &name, uid)?;
|
||||
if let Ok(index) = search_engine.open_user_index(&search_user_hash) {
|
||||
index.delete_message_with_connection(&transaction, uid, &name)?;
|
||||
}
|
||||
folder_changed = true;
|
||||
}
|
||||
}
|
||||
@@ -384,6 +398,7 @@ async fn sync_condstore(
|
||||
Ok(folder_changed)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn sync_full(
|
||||
config: &AppConfig,
|
||||
user_hash: &str,
|
||||
@@ -392,6 +407,7 @@ async fn sync_full(
|
||||
folder_name: &str,
|
||||
status: &crate::imap::types::FolderStatusExtended,
|
||||
event_bus: &EventBus,
|
||||
search_engine: &Arc<SearchEngine>,
|
||||
) -> Result<bool, String> {
|
||||
let database = db::pool::UserDb::new(&config.data_dir, user_hash);
|
||||
let name = folder_name.to_owned();
|
||||
@@ -429,6 +445,8 @@ async fn sync_full(
|
||||
|
||||
let name = folder_name.to_owned();
|
||||
let status = status.clone();
|
||||
let search_engine = Arc::clone(search_engine);
|
||||
let search_user_hash = user_hash.to_owned();
|
||||
let folder_changed = database
|
||||
.run(move |conn| {
|
||||
let folder_name = &name;
|
||||
@@ -451,6 +469,13 @@ async fn sync_full(
|
||||
match imap_map.get(uid) {
|
||||
None => {
|
||||
db::messages::delete_message(&transaction, folder_name, *uid)?;
|
||||
if let Ok(index) = search_engine.open_user_index(&search_user_hash) {
|
||||
index.delete_message_with_connection(
|
||||
&transaction,
|
||||
*uid,
|
||||
folder_name,
|
||||
)?;
|
||||
}
|
||||
folder_changed = true;
|
||||
tracing::debug!(
|
||||
folder = %folder_name,
|
||||
|
||||
@@ -135,6 +135,7 @@ async fn handle_socket_multi_account(
|
||||
creds.clone(),
|
||||
event_bus.clone(),
|
||||
config.clone(),
|
||||
search_engine.clone(),
|
||||
)
|
||||
.await;
|
||||
|
||||
|
||||
+54
-87
@@ -1,46 +1,17 @@
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use axum::extract::Multipart;
|
||||
use axum::extract::Path as AxumPath;
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use axum::{Extension, Json};
|
||||
use serde::Serialize;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use crate::auth::session::SessionState;
|
||||
use crate::config::AppConfig;
|
||||
use crate::db;
|
||||
use crate::error::AppError;
|
||||
|
||||
const STALE_PART_MAX_AGE: Duration = Duration::from_secs(24 * 60 * 60);
|
||||
|
||||
async fn cleanup_stale_upload_parts(dir: &Path) {
|
||||
let Ok(mut entries) = tokio::fs::read_dir(dir).await else {
|
||||
return;
|
||||
};
|
||||
while let Ok(Some(entry)) = entries.next_entry().await {
|
||||
let path = entry.path();
|
||||
let is_part = path
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.is_some_and(|name| name.ends_with(".part"));
|
||||
if !is_part {
|
||||
continue;
|
||||
}
|
||||
let stale = entry
|
||||
.metadata()
|
||||
.await
|
||||
.ok()
|
||||
.and_then(|meta| meta.modified().ok())
|
||||
.and_then(|modified| SystemTime::now().duration_since(modified).ok())
|
||||
.is_some_and(|age| age >= STALE_PART_MAX_AGE);
|
||||
if stale {
|
||||
let _ = tokio::fs::remove_file(path).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
use crate::storage::{BlobStorage, attachment_key};
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct UploadResponse {
|
||||
@@ -58,6 +29,7 @@ struct DeleteResponse {
|
||||
pub async fn upload_attachment(
|
||||
Extension(session): Extension<SessionState>,
|
||||
Extension(config): Extension<Arc<AppConfig>>,
|
||||
Extension(storage): Extension<Arc<dyn BlobStorage>>,
|
||||
AxumPath(draft_id): AxumPath<String>,
|
||||
mut multipart: Multipart,
|
||||
) -> Result<Response, AppError> {
|
||||
@@ -74,15 +46,6 @@ pub async fn upload_attachment(
|
||||
.map_err(AppError::InternalError)?;
|
||||
}
|
||||
|
||||
let att_dir = Path::new(&config.data_dir)
|
||||
.join(&session.user_hash)
|
||||
.join("attachments")
|
||||
.join(&draft_id);
|
||||
tokio::fs::create_dir_all(&att_dir)
|
||||
.await
|
||||
.map_err(|e| AppError::InternalError(format!("Failed to create attachment dir: {e}")))?;
|
||||
cleanup_stale_upload_parts(&att_dir).await;
|
||||
|
||||
let mut uploaded = Vec::new();
|
||||
let max_attachment_size = config.max_attachment_size_bytes();
|
||||
|
||||
@@ -97,19 +60,20 @@ pub async fn upload_attachment(
|
||||
.unwrap_or("application/octet-stream")
|
||||
.to_string();
|
||||
let att_id = uuid::Uuid::new_v4().to_string();
|
||||
let file_path = att_dir.join(&att_id);
|
||||
let part_path = att_dir.join(format!(".{att_id}.part"));
|
||||
let mut file = tokio::fs::File::create(&part_path).await.map_err(|e| {
|
||||
AppError::InternalError(format!("Failed to create attachment temp file: {e}"))
|
||||
})?;
|
||||
let key = attachment_key(&session.user_hash, &draft_id, &att_id)
|
||||
.map_err(AppError::InternalError)?;
|
||||
let mut writer = storage
|
||||
.begin_upload(&key)
|
||||
.await
|
||||
.map_err(|e| AppError::ServiceUnavailable(format!("Storage unavailable: {e}")))?;
|
||||
let mut size: usize = 0;
|
||||
let mut checksum = Sha256::new();
|
||||
|
||||
loop {
|
||||
let chunk = match field.chunk().await {
|
||||
Ok(chunk) => chunk,
|
||||
Err(e) => {
|
||||
drop(file);
|
||||
let _ = tokio::fs::remove_file(&part_path).await;
|
||||
let _ = writer.abort().await;
|
||||
return Err(AppError::BadRequest(format!(
|
||||
"Failed to read field data: {e}"
|
||||
)));
|
||||
@@ -118,47 +82,43 @@ pub async fn upload_attachment(
|
||||
let Some(chunk) = chunk else { break };
|
||||
size = size.saturating_add(chunk.len());
|
||||
if size > max_attachment_size {
|
||||
drop(file);
|
||||
let _ = tokio::fs::remove_file(&part_path).await;
|
||||
let _ = writer.abort().await;
|
||||
return Err(AppError::BadRequest(format!(
|
||||
"File '{}' exceeds maximum size of {} MiB",
|
||||
filename, config.max_attachment_size_mb
|
||||
)));
|
||||
}
|
||||
if let Err(e) = file.write_all(&chunk).await {
|
||||
drop(file);
|
||||
let _ = tokio::fs::remove_file(&part_path).await;
|
||||
return Err(AppError::InternalError(format!(
|
||||
"Failed to write attachment: {e}"
|
||||
checksum.update(&chunk);
|
||||
if let Err(e) = writer.write(chunk).await {
|
||||
let _ = writer.abort().await;
|
||||
return Err(AppError::ServiceUnavailable(format!(
|
||||
"Failed to upload attachment: {e}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
if let Err(e) = file.flush().await {
|
||||
drop(file);
|
||||
let _ = tokio::fs::remove_file(&part_path).await;
|
||||
return Err(AppError::InternalError(format!(
|
||||
"Failed to flush attachment: {e}"
|
||||
)));
|
||||
}
|
||||
drop(file);
|
||||
if let Err(e) = tokio::fs::rename(&part_path, &file_path).await {
|
||||
let _ = tokio::fs::remove_file(&part_path).await;
|
||||
return Err(AppError::InternalError(format!(
|
||||
"Failed to finalize attachment: {e}"
|
||||
)));
|
||||
}
|
||||
|
||||
let stat = writer.finish().await.map_err(|e| {
|
||||
AppError::ServiceUnavailable(format!("Failed to finalize attachment upload: {e}"))
|
||||
})?;
|
||||
if stat.size != size as u64 {
|
||||
let _ = storage.delete(&key).await;
|
||||
return Err(AppError::InternalError(format!(
|
||||
"Attachment size verification failed for '{filename}'"
|
||||
)));
|
||||
}
|
||||
let checksum = format!("{:x}", checksum.finalize());
|
||||
let size = size as i64;
|
||||
if let Err(e) = db::drafts::add_draft_attachment(
|
||||
&conn,
|
||||
&att_id,
|
||||
&draft_id,
|
||||
&filename,
|
||||
&content_type,
|
||||
let attachment = db::drafts::NewDraftAttachment {
|
||||
id: &att_id,
|
||||
draft_id: &draft_id,
|
||||
filename: &filename,
|
||||
content_type: &content_type,
|
||||
size,
|
||||
file_path.to_str().unwrap_or(""),
|
||||
) {
|
||||
let _ = tokio::fs::remove_file(&file_path).await;
|
||||
storage_key: &key,
|
||||
checksum_sha256: Some(&checksum),
|
||||
};
|
||||
if let Err(e) = db::drafts::add_draft_attachment(&conn, &attachment) {
|
||||
let _ = storage.delete(&key).await;
|
||||
return Err(AppError::InternalError(e));
|
||||
}
|
||||
|
||||
@@ -183,6 +143,7 @@ pub async fn upload_attachment(
|
||||
pub async fn get_attachment_content(
|
||||
Extension(session): Extension<SessionState>,
|
||||
Extension(config): Extension<Arc<AppConfig>>,
|
||||
Extension(storage): Extension<Arc<dyn BlobStorage>>,
|
||||
AxumPath((draft_id, attachment_id)): AxumPath<(String, String)>,
|
||||
) -> Result<Response, AppError> {
|
||||
db::drafts::validate_draft_id(&draft_id).map_err(AppError::BadRequest)?;
|
||||
@@ -191,26 +152,28 @@ pub async fn get_attachment_content(
|
||||
|
||||
let attachments =
|
||||
db::drafts::get_draft_attachments(&conn, &draft_id).map_err(AppError::InternalError)?;
|
||||
|
||||
let attachment = attachments
|
||||
.iter()
|
||||
.find(|a| a.id == attachment_id)
|
||||
.ok_or_else(|| AppError::NotFound("Attachment not found".to_string()))?;
|
||||
|
||||
let data = tokio::fs::read(&attachment.file_path)
|
||||
let stream = storage
|
||||
.get(&attachment.storage_key)
|
||||
.await
|
||||
.map_err(|e| AppError::InternalError(format!("Failed to read attachment file: {e}")))?;
|
||||
.map_err(|e| AppError::ServiceUnavailable(format!("Storage read failed: {e}")))?;
|
||||
|
||||
Ok(Response::builder()
|
||||
.header("content-type", &attachment.content_type)
|
||||
.header("content-length", attachment.size.max(0).to_string())
|
||||
.header("cache-control", "private, max-age=3600")
|
||||
.body(axum::body::Body::from(data))
|
||||
.body(axum::body::Body::from_stream(stream))
|
||||
.unwrap())
|
||||
}
|
||||
|
||||
pub async fn delete_attachment(
|
||||
Extension(session): Extension<SessionState>,
|
||||
Extension(config): Extension<Arc<AppConfig>>,
|
||||
Extension(storage): Extension<Arc<dyn BlobStorage>>,
|
||||
AxumPath((draft_id, attachment_id)): AxumPath<(String, String)>,
|
||||
) -> Result<Response, AppError> {
|
||||
db::drafts::validate_draft_id(&draft_id).map_err(AppError::BadRequest)?;
|
||||
@@ -219,18 +182,22 @@ pub async fn delete_attachment(
|
||||
|
||||
let attachments =
|
||||
db::drafts::get_draft_attachments(&conn, &draft_id).map_err(AppError::InternalError)?;
|
||||
|
||||
let attachment = attachments
|
||||
.iter()
|
||||
.find(|a| a.id == attachment_id)
|
||||
.ok_or_else(|| AppError::NotFound("Attachment not found".to_string()))?;
|
||||
|
||||
let file_path = attachment.file_path.clone();
|
||||
if let Err(e) = tokio::fs::remove_file(&file_path).await {
|
||||
tracing::warn!(error = %e, path = %file_path, "Failed to delete attachment file from disk");
|
||||
let storage_key = attachment.storage_key.clone();
|
||||
let deleted = db::drafts::delete_draft_attachment(&conn, &attachment_id)
|
||||
.map_err(AppError::InternalError)?;
|
||||
if !deleted {
|
||||
return Err(AppError::NotFound("Attachment not found".to_string()));
|
||||
}
|
||||
if let Err(error) = storage.delete(&storage_key).await {
|
||||
// SQL is authoritative. A failed physical delete leaves an orphaned blob,
|
||||
// which is safer than leaving a DB row pointing at a missing attachment.
|
||||
tracing::warn!(error = %error, storage_key = %storage_key, "Failed to remove deleted attachment blob");
|
||||
}
|
||||
|
||||
db::drafts::delete_draft_attachment(&conn, &attachment_id).map_err(AppError::InternalError)?;
|
||||
|
||||
Ok(Json(DeleteResponse {
|
||||
status: "deleted".to_string(),
|
||||
|
||||
+6
-14
@@ -56,8 +56,7 @@ fn browser_cookie(
|
||||
) -> String {
|
||||
let secure_flag = if secure { " Secure;" } else { "" };
|
||||
format!(
|
||||
"{}={};{} HttpOnly; SameSite={}; Path={}; Max-Age={}",
|
||||
BROWSER_COOKIE, browser_id, secure_flag, same_site, path, max_age_secs
|
||||
"{BROWSER_COOKIE}={browser_id};{secure_flag} HttpOnly; SameSite={same_site}; Path={path}; Max-Age={max_age_secs}"
|
||||
)
|
||||
}
|
||||
|
||||
@@ -71,24 +70,21 @@ fn account_session_cookie(
|
||||
) -> String {
|
||||
let secure_flag = if secure { " Secure;" } else { "" };
|
||||
format!(
|
||||
"rustbox_session_{}={};{} HttpOnly; SameSite={}; Path={}; Max-Age={}",
|
||||
account_id, token, secure_flag, same_site, path, max_age_secs
|
||||
"rustbox_session_{account_id}={token};{secure_flag} HttpOnly; SameSite={same_site}; Path={path}; Max-Age={max_age_secs}"
|
||||
)
|
||||
}
|
||||
|
||||
fn clearing_browser_cookie(secure: bool, same_site: &str, path: &str) -> String {
|
||||
let secure_flag = if secure { " Secure;" } else { "" };
|
||||
format!(
|
||||
"{}=;{} HttpOnly; SameSite={}; Path={}; Max-Age=0",
|
||||
BROWSER_COOKIE, secure_flag, same_site, path
|
||||
"{BROWSER_COOKIE}=;{secure_flag} HttpOnly; SameSite={same_site}; Path={path}; Max-Age=0"
|
||||
)
|
||||
}
|
||||
|
||||
fn clearing_account_cookie(account_id: &str, secure: bool, same_site: &str, path: &str) -> String {
|
||||
let secure_flag = if secure { " Secure;" } else { "" };
|
||||
format!(
|
||||
"rustbox_session_{}=;{} HttpOnly; SameSite={}; Path={}; Max-Age=0",
|
||||
account_id, secure_flag, same_site, path
|
||||
"rustbox_session_{account_id}=;{secure_flag} HttpOnly; SameSite={same_site}; Path={path}; Max-Age=0"
|
||||
)
|
||||
}
|
||||
|
||||
@@ -237,13 +233,9 @@ pub async fn login(
|
||||
}
|
||||
|
||||
let user_hash = user_data::hash_email(&body.email);
|
||||
let provision_result = if config.search_index_backend == "tantivy" {
|
||||
user_data::provision_user_data(&config.data_dir, &user_hash)
|
||||
} else {
|
||||
user_data::provision_user_data_with_search(&config.data_dir, &user_hash, false)
|
||||
};
|
||||
let provision_result = user_data::provision_user_database(&config.data_dir, &user_hash);
|
||||
if let Err(e) = provision_result {
|
||||
tracing::error!(error = %e, "failed to provision user data directory");
|
||||
tracing::error!(error = %e, "failed to provision user database");
|
||||
return (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
[("content-type", "application/json")],
|
||||
|
||||
@@ -144,7 +144,7 @@ fn append_ics_event(out: &mut String, event: &db::calendar::CalendarEvent) {
|
||||
}
|
||||
if let Some(minutes) = event.reminder_minutes.filter(|value| *value >= 0) {
|
||||
out.push_str("BEGIN:VALARM\r\nACTION:DISPLAY\r\n");
|
||||
out.push_str(&format!("TRIGGER:-PT{}M\r\n", minutes));
|
||||
out.push_str(&format!("TRIGGER:-PT{minutes}M\r\n"));
|
||||
out.push_str("DESCRIPTION:Reminder\r\nEND:VALARM\r\n");
|
||||
}
|
||||
out.push_str("END:VEVENT\r\n");
|
||||
@@ -174,7 +174,7 @@ pub async fn export_ics(
|
||||
.map_err(|e| AppError::InternalError(format!("Database error: {e}")))?;
|
||||
|
||||
let mut body = String::from(
|
||||
"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//RustBox//Calendar 0.1.2//EN\r\nCALSCALE:GREGORIAN\r\n",
|
||||
"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//RustBox//Calendar 0.2.0//EN\r\nCALSCALE:GREGORIAN\r\n",
|
||||
);
|
||||
for event in &events {
|
||||
append_ics_event(&mut body, event);
|
||||
|
||||
+51
-30
@@ -44,10 +44,14 @@ pub struct SmtpDiagnosticsResponse {
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct SearchDiagnosticsResponse {
|
||||
backend: String,
|
||||
indexed_documents: usize,
|
||||
cached_messages: usize,
|
||||
cached_bodies: usize,
|
||||
excluded_messages: usize,
|
||||
queue_depth: usize,
|
||||
dead_letters: usize,
|
||||
queue_lag_seconds: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
@@ -181,7 +185,7 @@ pub async fn reconnect_imap(
|
||||
.into_response())
|
||||
}
|
||||
|
||||
fn index_sources_to_messages(
|
||||
pub(crate) fn index_sources_to_messages(
|
||||
sources: Vec<db::messages::IndexSourceMessage>,
|
||||
) -> Vec<IndexableMessage> {
|
||||
sources
|
||||
@@ -254,11 +258,11 @@ pub async fn search_diagnostics(
|
||||
) -> Result<Response, AppError> {
|
||||
let data_dir = config.data_dir.clone();
|
||||
let user_hash = session.user_hash.clone();
|
||||
let engine = search_engine.clone();
|
||||
let db_user_hash = user_hash.clone();
|
||||
|
||||
let stats =
|
||||
tokio::task::spawn_blocking(move || -> Result<SearchDiagnosticsResponse, String> {
|
||||
let conn = db::pool::open_user_db(&data_dir, &user_hash)
|
||||
let (cached_messages, cached_bodies, excluded_messages, outbox) =
|
||||
tokio::task::spawn_blocking(move || -> Result<_, String> {
|
||||
let conn = db::pool::open_user_db(&data_dir, &db_user_hash)
|
||||
.map_err(|e| format!("Database error: {e}"))?;
|
||||
let sources = db::messages::list_index_source_messages(&conn)?;
|
||||
let cached_messages = sources.len();
|
||||
@@ -269,23 +273,35 @@ pub async fn search_diagnostics(
|
||||
let cached_bodies: usize = conn
|
||||
.query_row(crate::dbq!(routes_diagnostics::Q_003), [], |row| row.get(0))
|
||||
.map_err(|e| format!("Failed to count cached bodies: {e}"))?;
|
||||
let indexed_documents = if engine.is_enabled() {
|
||||
engine.open_user_index(&user_hash)?.document_count()
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
Ok(SearchDiagnosticsResponse {
|
||||
indexed_documents,
|
||||
cached_messages,
|
||||
cached_bodies,
|
||||
excluded_messages,
|
||||
})
|
||||
let outbox = db::search_outbox::stats(&conn).unwrap_or_default();
|
||||
Ok((cached_messages, cached_bodies, excluded_messages, outbox))
|
||||
})
|
||||
.await
|
||||
.map_err(|e| AppError::InternalError(format!("Search diagnostics task failed: {e}")))?
|
||||
.map_err(AppError::InternalError)?;
|
||||
|
||||
let indexed_documents = if search_engine.is_enabled() {
|
||||
search_engine
|
||||
.open_user_index(&user_hash)
|
||||
.map_err(AppError::InternalError)?
|
||||
.document_count()
|
||||
.await
|
||||
.map_err(AppError::InternalError)?
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
let stats = SearchDiagnosticsResponse {
|
||||
backend: search_engine.backend_name().to_string(),
|
||||
indexed_documents,
|
||||
cached_messages,
|
||||
cached_bodies,
|
||||
excluded_messages,
|
||||
queue_depth: outbox.total.saturating_sub(outbox.dead_letters),
|
||||
dead_letters: outbox.dead_letters,
|
||||
queue_lag_seconds: outbox.lag_seconds,
|
||||
};
|
||||
|
||||
Ok(Json(stats).into_response())
|
||||
}
|
||||
|
||||
@@ -301,26 +317,31 @@ pub async fn rebuild_search_index(
|
||||
}
|
||||
let data_dir = config.data_dir.clone();
|
||||
let user_hash = session.user_hash.clone();
|
||||
let engine = search_engine.clone();
|
||||
let db_user_hash = user_hash.clone();
|
||||
|
||||
let result = tokio::task::spawn_blocking(move || -> Result<SearchRebuildResponse, String> {
|
||||
let conn = db::pool::open_user_db(&data_dir, &user_hash)
|
||||
let (source_messages, messages) = tokio::task::spawn_blocking(move || -> Result<_, String> {
|
||||
let conn = db::pool::open_user_db(&data_dir, &db_user_hash)
|
||||
.map_err(|e| format!("Database error: {e}"))?;
|
||||
let sources = db::messages::list_index_source_messages(&conn)?;
|
||||
let source_messages = sources.len();
|
||||
let messages = index_sources_to_messages(sources);
|
||||
let index = engine.open_user_index(&user_hash)?;
|
||||
let indexed_documents = index.rebuild_from_messages(&messages)?;
|
||||
|
||||
Ok(SearchRebuildResponse {
|
||||
status: "rebuilt",
|
||||
indexed_documents,
|
||||
source_messages,
|
||||
})
|
||||
Ok((source_messages, index_sources_to_messages(sources)))
|
||||
})
|
||||
.await
|
||||
.map_err(|e| AppError::InternalError(format!("Search rebuild task failed: {e}")))?
|
||||
.map_err(AppError::InternalError)?;
|
||||
|
||||
Ok(Json(result).into_response())
|
||||
let index = search_engine
|
||||
.open_user_index(&user_hash)
|
||||
.map_err(AppError::InternalError)?;
|
||||
let indexed_documents = index
|
||||
.rebuild_from_messages(&messages)
|
||||
.await
|
||||
.map_err(AppError::InternalError)?;
|
||||
|
||||
Ok(Json(SearchRebuildResponse {
|
||||
status: "rebuilt",
|
||||
indexed_documents,
|
||||
source_messages,
|
||||
})
|
||||
.into_response())
|
||||
}
|
||||
+30
-25
@@ -1,4 +1,3 @@
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::extract::Path as AxumPath;
|
||||
@@ -12,6 +11,7 @@ use crate::db;
|
||||
use crate::error::AppError;
|
||||
use crate::imap::client::{ImapClient, ImapCredentials};
|
||||
use crate::smtp::client::AttachmentData;
|
||||
use crate::storage::{self, BlobStorage};
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct UpsertDraftRequest {
|
||||
@@ -115,6 +115,7 @@ pub async fn upsert_draft_handler(
|
||||
Extension(session): Extension<SessionState>,
|
||||
Extension(config): Extension<Arc<AppConfig>>,
|
||||
Extension(imap_client): Extension<Arc<dyn ImapClient>>,
|
||||
Extension(storage): Extension<Arc<dyn BlobStorage>>,
|
||||
Json(req): Json<UpsertDraftRequest>,
|
||||
) -> Result<Response, AppError> {
|
||||
db::drafts::validate_draft_id(&req.id).map_err(AppError::BadRequest)?;
|
||||
@@ -172,7 +173,8 @@ pub async fn upsert_draft_handler(
|
||||
.unwrap_or_else(|| "Drafts".to_string());
|
||||
let previous = db::drafts::get_draft(&conn, &req.id).map_err(AppError::InternalError)?;
|
||||
|
||||
let snapshot_attachments = load_snapshot_attachments(&conn, &req.id);
|
||||
let snapshot_attachments =
|
||||
load_snapshot_attachments(&conn, &req.id, storage.as_ref()).await;
|
||||
match snapshot_attachments
|
||||
.and_then(|attachments| build_draft_rfc822(&req, &session.email, &attachments))
|
||||
{
|
||||
@@ -315,28 +317,30 @@ pub(crate) async fn remove_remote_draft_snapshot(
|
||||
}
|
||||
}
|
||||
|
||||
fn load_snapshot_attachments(
|
||||
async fn load_snapshot_attachments(
|
||||
conn: &crate::db::pool::Connection,
|
||||
draft_id: &str,
|
||||
storage: &dyn BlobStorage,
|
||||
) -> Result<Vec<AttachmentData>, String> {
|
||||
let records = db::drafts::get_draft_attachments(conn, draft_id)?;
|
||||
records
|
||||
.into_iter()
|
||||
.map(|record| {
|
||||
let data = std::fs::read(&record.file_path).map_err(|e| {
|
||||
let mut attachments = Vec::with_capacity(records.len());
|
||||
for record in records {
|
||||
let data = storage::read_all(storage, &record.storage_key)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
format!(
|
||||
"Failed to read draft attachment '{}' for IMAP snapshot: {e}",
|
||||
record.filename
|
||||
)
|
||||
})?;
|
||||
Ok(AttachmentData {
|
||||
filename: record.filename,
|
||||
content_type: record.content_type,
|
||||
data,
|
||||
content_id: Some(record.id),
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
attachments.push(AttachmentData {
|
||||
filename: record.filename,
|
||||
content_type: record.content_type,
|
||||
data,
|
||||
content_id: Some(record.id),
|
||||
});
|
||||
}
|
||||
Ok(attachments)
|
||||
}
|
||||
|
||||
fn build_draft_rfc822(
|
||||
@@ -535,6 +539,7 @@ pub async fn delete_draft_handler(
|
||||
Extension(session): Extension<SessionState>,
|
||||
Extension(config): Extension<Arc<AppConfig>>,
|
||||
Extension(imap_client): Extension<Arc<dyn ImapClient>>,
|
||||
Extension(storage): Extension<Arc<dyn BlobStorage>>,
|
||||
AxumPath(id): AxumPath<String>,
|
||||
) -> Result<Response, AppError> {
|
||||
db::drafts::validate_draft_id(&id).map_err(AppError::BadRequest)?;
|
||||
@@ -550,20 +555,20 @@ pub async fn delete_draft_handler(
|
||||
remove_remote_draft_snapshot(imap_client.as_ref(), &creds, &existing).await;
|
||||
}
|
||||
|
||||
let attachments =
|
||||
db::drafts::get_draft_attachments(&conn, &id).map_err(AppError::InternalError)?;
|
||||
let deleted = db::drafts::delete_draft(&conn, &id).map_err(AppError::InternalError)?;
|
||||
|
||||
if !deleted {
|
||||
return Err(AppError::NotFound("Draft not found".to_string()));
|
||||
}
|
||||
|
||||
let att_dir = Path::new(&config.data_dir)
|
||||
.join(&session.user_hash)
|
||||
.join("attachments")
|
||||
.join(&id);
|
||||
if att_dir.exists()
|
||||
&& let Err(e) = tokio::fs::remove_dir_all(&att_dir).await
|
||||
{
|
||||
tracing::warn!(error = %e, path = %att_dir.display(), "Failed to clean up attachment directory");
|
||||
for attachment in &attachments {
|
||||
if let Err(error) = storage.delete(&attachment.storage_key).await {
|
||||
tracing::warn!(
|
||||
error = %error,
|
||||
storage_key = %attachment.storage_key,
|
||||
"Failed to remove deleted draft attachment blob"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Json(DeleteResponse {
|
||||
|
||||
@@ -224,8 +224,11 @@ pub async fn empty_folder(
|
||||
|
||||
let conn = db::pool::open_user_db(&config.data_dir, &session.user_hash)
|
||||
.map_err(|e| AppError::InternalError(format!("Database error: {e}")))?;
|
||||
let transaction = conn
|
||||
.unchecked_transaction()
|
||||
.map_err(|e| AppError::InternalError(format!("Database transaction error: {e}")))?;
|
||||
let uids: Vec<u32> = {
|
||||
let mut statement = conn
|
||||
let mut statement = transaction
|
||||
.prepare(crate::dbq!(routes_folder_mgmt::Q_003))
|
||||
.map_err(|e| AppError::InternalError(format!("Database error: {e}")))?;
|
||||
let rows = statement
|
||||
@@ -233,18 +236,22 @@ pub async fn empty_folder(
|
||||
.map_err(|e| AppError::InternalError(format!("Database error: {e}")))?;
|
||||
rows.filter_map(Result::ok).collect()
|
||||
};
|
||||
db::messages::delete_folder_messages(&conn, &name)
|
||||
db::messages::delete_folder_messages(&transaction, &name)
|
||||
.map_err(|e| AppError::InternalError(format!("Database error: {e}")))?;
|
||||
conn.execute(crate::dbq!(routes_folder_mgmt::Q_004), [&name])
|
||||
transaction
|
||||
.execute(crate::dbq!(routes_folder_mgmt::Q_004), [&name])
|
||||
.map_err(|e| AppError::InternalError(format!("Database error: {e}")))?;
|
||||
|
||||
if let Ok(index) = search_engine.open_user_index(&session.user_hash) {
|
||||
for uid in uids {
|
||||
if let Err(error) = index.delete_message(uid, &name) {
|
||||
tracing::warn!(uid, folder = %name, %error, "Failed to remove emptied message from search index");
|
||||
}
|
||||
index
|
||||
.delete_message_with_connection(&transaction, uid, &name)
|
||||
.map_err(|e| AppError::InternalError(format!("Search outbox error: {e}")))?;
|
||||
}
|
||||
}
|
||||
transaction
|
||||
.commit()
|
||||
.map_err(|e| AppError::InternalError(format!("Database commit error: {e}")))?;
|
||||
|
||||
Ok(Json(MessageResponse { status: "emptied" }).into_response())
|
||||
}
|
||||
|
||||
+28
-7
@@ -1,4 +1,3 @@
|
||||
use std::fs;
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::http::StatusCode;
|
||||
@@ -7,22 +6,41 @@ use axum::{Extension, Json};
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::config::AppConfig;
|
||||
use crate::search::engine::SearchEngine;
|
||||
use crate::storage::BlobStorage;
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct HealthResponse {
|
||||
pub status: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub storage: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub search: Option<String>,
|
||||
}
|
||||
|
||||
pub async fn health_check() -> Json<HealthResponse> {
|
||||
Json(HealthResponse {
|
||||
status: "ok".to_string(),
|
||||
storage: None,
|
||||
search: None,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn readiness_check(Extension(config): Extension<Arc<AppConfig>>) -> Response {
|
||||
let ready = fs::create_dir_all(&config.data_dir).is_ok()
|
||||
&& fs::metadata(&config.data_dir).is_ok_and(|metadata| metadata.is_dir());
|
||||
let status = if ready { "ready" } else { "not_ready" };
|
||||
pub async fn readiness_check(
|
||||
Extension(config): Extension<Arc<AppConfig>>,
|
||||
Extension(storage): Extension<Arc<dyn BlobStorage>>,
|
||||
Extension(search): Extension<Arc<SearchEngine>>,
|
||||
) -> Response {
|
||||
let storage_status = match storage.health().await {
|
||||
Ok(()) => "ok".to_string(),
|
||||
Err(error) => format!("error: {error}"),
|
||||
};
|
||||
let search_status = match search.health().await {
|
||||
Ok(()) => "ok".to_string(),
|
||||
Err(error) if !config.search.required => format!("degraded: {error}"),
|
||||
Err(error) => format!("error: {error}"),
|
||||
};
|
||||
let ready = storage_status == "ok" && (!config.search.required || search_status == "ok");
|
||||
(
|
||||
if ready {
|
||||
StatusCode::OK
|
||||
@@ -30,7 +48,9 @@ pub async fn readiness_check(Extension(config): Extension<Arc<AppConfig>>) -> Re
|
||||
StatusCode::SERVICE_UNAVAILABLE
|
||||
},
|
||||
Json(HealthResponse {
|
||||
status: status.to_string(),
|
||||
status: if ready { "ready" } else { "not_ready" }.to_string(),
|
||||
storage: Some(storage_status),
|
||||
search: Some(search_status),
|
||||
}),
|
||||
)
|
||||
.into_response()
|
||||
@@ -118,7 +138,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn readiness_creates_data_directory_and_returns_ready() {
|
||||
async fn readiness_checks_storage_without_requiring_node_local_data_dir_in_cluster_shape() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let data_dir = temp.path().join("nested").join("rustbox");
|
||||
let app = test_router(test_config(data_dir.to_string_lossy().into_owned()));
|
||||
@@ -136,6 +156,7 @@ mod tests {
|
||||
let body = response.into_body().collect().await.unwrap().to_bytes();
|
||||
let json: serde_json::Value = serde_json::from_slice(&body).unwrap();
|
||||
assert_eq!(json["status"], "ready");
|
||||
assert_eq!(json["storage"], "ok");
|
||||
assert!(data_dir.is_dir());
|
||||
}
|
||||
}
|
||||
+78
-58
@@ -402,7 +402,9 @@ fn upsert_and_index_headers(
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let _ = user_index.index_messages_batch(&indexable);
|
||||
user_index
|
||||
.index_messages_batch_with_connection(conn, &indexable)
|
||||
.map_err(|e| AppError::InternalError(format!("Search outbox error: {e}")))?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -545,7 +547,6 @@ pub async fn get_message(
|
||||
.map_err(|e| AppError::InternalError(format!("Database error: {e}")))?;
|
||||
|
||||
let usable_cache = cached_body.filter(|c| c.attachments_json.is_some());
|
||||
let fetched_body = usable_cache.is_none();
|
||||
|
||||
let (body_html, body_text, attachments, raw_headers, email_theme) = if let Some(cached) =
|
||||
usable_cache
|
||||
@@ -585,18 +586,32 @@ pub async fn get_message(
|
||||
.await
|
||||
.map_err(|e| match e {
|
||||
crate::imap::client::ImapError::MessageNotFound { .. } => {
|
||||
let _ = db::messages::delete_message(&conn, &folder, uid);
|
||||
let _ = db::folders::invalidate_folder_freshness(&conn, &folder);
|
||||
if let Ok(user_index) = search_engine.open_user_index(&session.user_hash) {
|
||||
let _ = user_index.delete_message(uid, &folder);
|
||||
if let Ok(transaction) = conn.unchecked_transaction() {
|
||||
let _ = db::messages::delete_message(&transaction, &folder, uid);
|
||||
let _ = db::folders::invalidate_folder_freshness(&transaction, &folder);
|
||||
if let Ok(user_index) = search_engine.open_user_index(&session.user_hash) {
|
||||
let _ = user_index.delete_message_with_connection(
|
||||
&transaction,
|
||||
uid,
|
||||
&folder,
|
||||
);
|
||||
}
|
||||
let _ = transaction.commit();
|
||||
}
|
||||
AppError::NotFound(format!("Message UID {uid} not found in folder {folder}"))
|
||||
}
|
||||
other => {
|
||||
let _ = db::messages::delete_message(&conn, &folder, uid);
|
||||
let _ = db::folders::invalidate_folder_freshness(&conn, &folder);
|
||||
if let Ok(user_index) = search_engine.open_user_index(&session.user_hash) {
|
||||
let _ = user_index.delete_message(uid, &folder);
|
||||
if let Ok(transaction) = conn.unchecked_transaction() {
|
||||
let _ = db::messages::delete_message(&transaction, &folder, uid);
|
||||
let _ = db::folders::invalidate_folder_freshness(&transaction, &folder);
|
||||
if let Ok(user_index) = search_engine.open_user_index(&session.user_hash) {
|
||||
let _ = user_index.delete_message_with_connection(
|
||||
&transaction,
|
||||
uid,
|
||||
&folder,
|
||||
);
|
||||
}
|
||||
let _ = transaction.commit();
|
||||
}
|
||||
AppError::ServiceUnavailable(format!("IMAP error: {other}"))
|
||||
}
|
||||
@@ -639,8 +654,11 @@ pub async fn get_message(
|
||||
.and_then(|h| email_theme::detect_email_theme(h))
|
||||
.map(|t| t.as_i32());
|
||||
|
||||
let transaction = conn
|
||||
.unchecked_transaction()
|
||||
.map_err(|e| AppError::InternalError(format!("Database transaction error: {e}")))?;
|
||||
db::messages::cache_message_body(
|
||||
&conn,
|
||||
&transaction,
|
||||
&folder,
|
||||
uid,
|
||||
resolved_html.as_deref(),
|
||||
@@ -651,6 +669,31 @@ pub async fn get_message(
|
||||
)
|
||||
.map_err(|e| AppError::InternalError(format!("Database error: {e}")))?;
|
||||
|
||||
if let Some(text) = body.text_plain.as_ref()
|
||||
&& !UserIndex::is_excluded_folder(&folder)
|
||||
&& let Some(msg) = db::messages::get_single_message(&transaction, &folder, uid)
|
||||
.map_err(|e| AppError::InternalError(format!("Database error: {e}")))?
|
||||
&& let Ok(user_index) = search_engine.open_user_index(&session.user_hash)
|
||||
{
|
||||
let indexable = IndexableMessage {
|
||||
uid: msg.uid,
|
||||
folder: msg.folder,
|
||||
subject: msg.subject,
|
||||
from_address: msg.from_address,
|
||||
from_name: msg.from_name,
|
||||
to_addresses: msg.to_addresses,
|
||||
body_text: text.clone(),
|
||||
date_epoch: crate::db::messages::parse_date_to_epoch_public(&msg.date),
|
||||
has_attachments: msg.has_attachments,
|
||||
};
|
||||
user_index
|
||||
.index_message_with_connection(&transaction, &indexable)
|
||||
.map_err(|e| AppError::InternalError(format!("Search outbox error: {e}")))?;
|
||||
}
|
||||
transaction
|
||||
.commit()
|
||||
.map_err(|e| AppError::InternalError(format!("Database commit error: {e}")))?;
|
||||
|
||||
(
|
||||
resolved_html,
|
||||
body.text_plain,
|
||||
@@ -742,30 +785,6 @@ pub async fn get_message(
|
||||
}
|
||||
};
|
||||
|
||||
if fetched_body
|
||||
&& let Some(ref text) = body_text
|
||||
&& !UserIndex::is_excluded_folder(&folder)
|
||||
{
|
||||
let indexable = IndexableMessage {
|
||||
uid: msg.uid,
|
||||
folder: msg.folder.clone(),
|
||||
subject: msg.subject.clone(),
|
||||
from_address: msg.from_address.clone(),
|
||||
from_name: msg.from_name.clone(),
|
||||
to_addresses: msg.to_addresses.clone(),
|
||||
body_text: text.clone(),
|
||||
date_epoch: crate::db::messages::parse_date_to_epoch_public(&msg.date),
|
||||
has_attachments: msg.has_attachments,
|
||||
};
|
||||
let search_engine = Arc::clone(&search_engine);
|
||||
let user_hash = session.user_hash.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
if let Ok(user_index) = search_engine.open_user_index(&user_hash) {
|
||||
let _ = user_index.index_message(&indexable);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let thread_messages = if let Some(ref message_id) = msg.message_id {
|
||||
db::messages::get_full_thread(&conn, message_id, msg.references_header.as_deref())
|
||||
.unwrap_or_default()
|
||||
@@ -973,34 +992,31 @@ pub async fn move_message_handler(
|
||||
let conn = db::pool::open_user_db(&config.data_dir, &session.user_hash)
|
||||
.map_err(|e| AppError::InternalError(format!("Database error: {e}")))?;
|
||||
|
||||
db::messages::remove_after_mutation(&conn, &body.from_folder, body.uid)
|
||||
let transaction = conn
|
||||
.unchecked_transaction()
|
||||
.map_err(|e| AppError::InternalError(format!("Database transaction error: {e}")))?;
|
||||
db::messages::remove_after_mutation_in_transaction(&transaction, &body.from_folder, body.uid)
|
||||
.map_err(|e| AppError::InternalError(format!("Database error: {e}")))?;
|
||||
|
||||
let user_hash = session.user_hash.clone();
|
||||
let from_folder = body.from_folder.clone();
|
||||
let uid = body.uid;
|
||||
let search_result = tokio::task::spawn_blocking(move || {
|
||||
let index = search_engine.open_user_index(&user_hash)?;
|
||||
index.delete_message(uid, &from_folder)
|
||||
})
|
||||
.await;
|
||||
if !matches!(search_result, Ok(Ok(()))) {
|
||||
tracing::warn!(
|
||||
?search_result,
|
||||
"Failed to remove moved message from search index"
|
||||
);
|
||||
if let Ok(index) = search_engine.open_user_index(&session.user_hash) {
|
||||
index
|
||||
.delete_message_with_connection(&transaction, body.uid, &body.from_folder)
|
||||
.map_err(|e| AppError::InternalError(format!("Search outbox error: {e}")))?;
|
||||
}
|
||||
|
||||
db::folders::refresh_unread_count(&conn, &body.from_folder)
|
||||
db::folders::refresh_unread_count(&transaction, &body.from_folder)
|
||||
.map_err(|e| AppError::InternalError(format!("Database error: {e}")))?;
|
||||
|
||||
if was_unread {
|
||||
db::folders::adjust_unread_count(&conn, &body.to_folder, 1)
|
||||
db::folders::adjust_unread_count(&transaction, &body.to_folder, 1)
|
||||
.map_err(|e| AppError::InternalError(format!("Database error: {e}")))?;
|
||||
}
|
||||
|
||||
db::folders::invalidate_folder_freshness(&conn, &body.to_folder)
|
||||
db::folders::invalidate_folder_freshness(&transaction, &body.to_folder)
|
||||
.map_err(|e| AppError::InternalError(format!("Database error: {e}")))?;
|
||||
transaction
|
||||
.commit()
|
||||
.map_err(|e| AppError::InternalError(format!("Database commit error: {e}")))?;
|
||||
|
||||
let undo_move = undo_destination_uid.map(|destination_uid| {
|
||||
serde_json::json!({
|
||||
@@ -1454,6 +1470,7 @@ mod sync_tests {
|
||||
limit: 10,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.unwrap()
|
||||
.0
|
||||
.is_empty()
|
||||
@@ -1646,12 +1663,14 @@ mod sync_tests {
|
||||
.is_none()
|
||||
);
|
||||
assert!(db::folders::is_folder_fresh(&conn, "INBOX", 30).unwrap());
|
||||
let requests = fresh_client.header_requests.lock().unwrap();
|
||||
assert_eq!(requests.len(), 4);
|
||||
assert!(requests.iter().all(|(_, uids)| {
|
||||
uids.split(',')
|
||||
.all(|uid| uid.parse::<u32>().unwrap() <= 900)
|
||||
}));
|
||||
{
|
||||
let requests = fresh_client.header_requests.lock().unwrap();
|
||||
assert_eq!(requests.len(), 4);
|
||||
assert!(requests.iter().all(|(_, uids)| {
|
||||
uids.split(',')
|
||||
.all(|uid| uid.parse::<u32>().unwrap() <= 900)
|
||||
}));
|
||||
}
|
||||
let results = engine
|
||||
.open_user_index(user_hash)
|
||||
.unwrap()
|
||||
@@ -1661,6 +1680,7 @@ mod sync_tests {
|
||||
limit: 1100,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.unwrap()
|
||||
.0;
|
||||
assert!(results.iter().any(|message| message.uid == 1));
|
||||
|
||||
@@ -101,6 +101,7 @@ impl KeyExtractor for ProxyAwareIpExtractor {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn create_router(
|
||||
config: Arc<AppConfig>,
|
||||
store: Arc<SessionStore>,
|
||||
@@ -110,6 +111,42 @@ pub fn create_router(
|
||||
event_bus: Arc<EventBus>,
|
||||
idle_manager: Arc<IdleManager>,
|
||||
) -> Router {
|
||||
let storage =
|
||||
crate::storage::from_config(&config).expect("validated RustBox storage configuration");
|
||||
create_router_with_storage(
|
||||
config,
|
||||
RouterServices {
|
||||
store,
|
||||
imap_client,
|
||||
smtp_client,
|
||||
search_engine,
|
||||
storage,
|
||||
event_bus,
|
||||
idle_manager,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
pub struct RouterServices {
|
||||
pub store: Arc<SessionStore>,
|
||||
pub imap_client: Arc<dyn ImapClient>,
|
||||
pub smtp_client: Arc<dyn SmtpClient>,
|
||||
pub search_engine: Arc<crate::search::engine::SearchEngine>,
|
||||
pub storage: Arc<dyn crate::storage::BlobStorage>,
|
||||
pub event_bus: Arc<EventBus>,
|
||||
pub idle_manager: Arc<IdleManager>,
|
||||
}
|
||||
|
||||
pub fn create_router_with_storage(config: Arc<AppConfig>, services: RouterServices) -> Router {
|
||||
let RouterServices {
|
||||
store,
|
||||
imap_client,
|
||||
smtp_client,
|
||||
search_engine,
|
||||
storage,
|
||||
event_bus,
|
||||
idle_manager,
|
||||
} = services;
|
||||
let governor_conf = GovernorConfigBuilder::default()
|
||||
.key_extractor(ProxyAwareIpExtractor {
|
||||
trusted_proxies: config.parsed_trusted_proxies(),
|
||||
@@ -477,6 +514,7 @@ pub fn create_router(
|
||||
.layer(Extension(event_bus))
|
||||
.layer(Extension(smtp_client))
|
||||
.layer(Extension(search_engine))
|
||||
.layer(Extension(storage))
|
||||
.layer(Extension(imap_client))
|
||||
.layer(Extension(store))
|
||||
.layer(Extension(config.clone()))
|
||||
|
||||
+14
-12
@@ -362,18 +362,20 @@ pub async fn search_messages(
|
||||
if !parsed.text.is_empty()
|
||||
&& results.len() < limit
|
||||
&& let Ok(user_index) = search_engine.open_user_index(&session.user_hash)
|
||||
&& let Ok((tantivy_results, _)) = user_index.search(&SearchQuery {
|
||||
text: parsed.text,
|
||||
subject_only: subject,
|
||||
folder,
|
||||
from,
|
||||
to,
|
||||
date_from,
|
||||
date_to,
|
||||
has_attachment,
|
||||
limit,
|
||||
offset: if select_all { 0 } else { search_offset },
|
||||
})
|
||||
&& let Ok((tantivy_results, _)) = user_index
|
||||
.search(&SearchQuery {
|
||||
text: parsed.text,
|
||||
subject_only: subject,
|
||||
folder,
|
||||
from,
|
||||
to,
|
||||
date_from,
|
||||
date_to,
|
||||
has_attachment,
|
||||
limit,
|
||||
offset: if select_all { 0 } else { search_offset },
|
||||
})
|
||||
.await
|
||||
{
|
||||
let data_dir2 = config.data_dir.clone();
|
||||
let user_hash2 = session.user_hash.clone();
|
||||
|
||||
+49
-25
@@ -1,4 +1,3 @@
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::response::{IntoResponse, Response};
|
||||
@@ -12,6 +11,7 @@ use crate::error::AppError;
|
||||
use crate::imap::client::{ImapClient, ImapCredentials};
|
||||
use crate::smtp::client::{AttachmentData, SendableMessage, SmtpClient, SmtpCredentials};
|
||||
use crate::smtp::types::{PgpMode, PgpSendConfig};
|
||||
use crate::storage::{self, BlobStorage};
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct SendRequest {
|
||||
@@ -72,6 +72,7 @@ pub async fn send_message_handler(
|
||||
Extension(config): Extension<Arc<AppConfig>>,
|
||||
Extension(smtp_client): Extension<Arc<dyn SmtpClient>>,
|
||||
Extension(imap_client): Extension<Arc<dyn ImapClient>>,
|
||||
Extension(storage): Extension<Arc<dyn BlobStorage>>,
|
||||
Json(req): Json<SendRequest>,
|
||||
) -> Result<Response, AppError> {
|
||||
if let Some(ref draft_id) = req.draft_id {
|
||||
@@ -242,7 +243,13 @@ pub async fn send_message_handler(
|
||||
}
|
||||
|
||||
let mut attachments = if let Some(ref draft_id) = req.draft_id {
|
||||
load_draft_attachments(&config.data_dir, &session.user_hash, draft_id)?
|
||||
load_draft_attachments(
|
||||
&config.data_dir,
|
||||
&session.user_hash,
|
||||
draft_id,
|
||||
storage.as_ref(),
|
||||
)
|
||||
.await?
|
||||
} else {
|
||||
vec![]
|
||||
};
|
||||
@@ -360,7 +367,13 @@ pub async fn send_message_handler(
|
||||
)
|
||||
.await;
|
||||
}
|
||||
cleanup_draft(&config.data_dir, &session.user_hash, draft_id).await;
|
||||
cleanup_draft(
|
||||
&config.data_dir,
|
||||
&session.user_hash,
|
||||
draft_id,
|
||||
storage.as_ref(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
Ok(Json(SendResponse {
|
||||
@@ -370,10 +383,11 @@ pub async fn send_message_handler(
|
||||
.into_response())
|
||||
}
|
||||
|
||||
fn load_draft_attachments(
|
||||
async fn load_draft_attachments(
|
||||
data_dir: &str,
|
||||
user_hash: &str,
|
||||
draft_id: &str,
|
||||
storage: &dyn BlobStorage,
|
||||
) -> Result<Vec<AttachmentData>, AppError> {
|
||||
let conn = db::pool::open_user_db(data_dir, user_hash)
|
||||
.map_err(|e| AppError::InternalError(format!("Failed to open database: {e}")))?;
|
||||
@@ -381,14 +395,16 @@ fn load_draft_attachments(
|
||||
let db_attachments =
|
||||
db::drafts::get_draft_attachments(&conn, draft_id).map_err(AppError::InternalError)?;
|
||||
|
||||
let mut attachments = Vec::new();
|
||||
let mut attachments = Vec::with_capacity(db_attachments.len());
|
||||
for att in db_attachments {
|
||||
let data = std::fs::read(&att.file_path).map_err(|e| {
|
||||
AppError::InternalError(format!(
|
||||
"Failed to read attachment file '{}': {e}",
|
||||
att.filename
|
||||
))
|
||||
})?;
|
||||
let data = storage::read_all(storage, &att.storage_key)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
AppError::InternalError(format!(
|
||||
"Failed to read attachment '{}': {e}",
|
||||
att.filename
|
||||
))
|
||||
})?;
|
||||
attachments.push(AttachmentData {
|
||||
filename: att.filename,
|
||||
content_type: att.content_type,
|
||||
@@ -399,21 +415,29 @@ fn load_draft_attachments(
|
||||
Ok(attachments)
|
||||
}
|
||||
|
||||
async fn cleanup_draft(data_dir: &str, user_hash: &str, draft_id: &str) {
|
||||
if let Ok(conn) = db::pool::open_user_db(data_dir, user_hash)
|
||||
&& let Err(e) = db::drafts::delete_draft(&conn, draft_id)
|
||||
{
|
||||
tracing::warn!(error = %e, "Failed to delete draft after send");
|
||||
async fn cleanup_draft(data_dir: &str, user_hash: &str, draft_id: &str, storage: &dyn BlobStorage) {
|
||||
let Ok(conn) = db::pool::open_user_db(data_dir, user_hash) else {
|
||||
return;
|
||||
};
|
||||
let attachments = db::drafts::get_draft_attachments(&conn, draft_id).unwrap_or_default();
|
||||
let deleted = match db::drafts::delete_draft(&conn, draft_id) {
|
||||
Ok(deleted) => deleted,
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, "Failed to delete draft after send");
|
||||
return;
|
||||
}
|
||||
};
|
||||
if !deleted {
|
||||
return;
|
||||
}
|
||||
|
||||
let att_dir = Path::new(data_dir)
|
||||
.join(user_hash)
|
||||
.join("attachments")
|
||||
.join(draft_id);
|
||||
if att_dir.exists()
|
||||
&& let Err(e) = tokio::fs::remove_dir_all(&att_dir).await
|
||||
{
|
||||
tracing::warn!(error = %e, path = %att_dir.display(), "Failed to clean up attachment directory");
|
||||
for attachment in attachments {
|
||||
if let Err(e) = storage.delete(&attachment.storage_key).await {
|
||||
tracing::warn!(
|
||||
error = %e,
|
||||
storage_key = %attachment.storage_key,
|
||||
"Failed to delete draft attachment after send"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -771,7 +771,7 @@ async fn logout_clears_cookie() {
|
||||
|
||||
let set_cookie = response.headers().get_all("set-cookie");
|
||||
|
||||
let cookie_str = format!("{:?}", set_cookie);
|
||||
let cookie_str = format!("{set_cookie:?}");
|
||||
assert!(cookie_str.contains("rustbox_browser=;") || cookie_str.contains("Max-Age=0"));
|
||||
}
|
||||
|
||||
|
||||
@@ -228,7 +228,7 @@ async fn post_one_click_to_address(
|
||||
format!("{host}:{port}")
|
||||
};
|
||||
let request = format!(
|
||||
"POST {path} HTTP/1.1\r\nHost: {host_header}\r\nUser-Agent: RustBox/0.1.2\r\nContent-Type: application/x-www-form-urlencoded\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{ONE_CLICK_BODY}",
|
||||
"POST {path} HTTP/1.1\r\nHost: {host_header}\r\nUser-Agent: RustBox/0.2.0\r\nContent-Type: application/x-www-form-urlencoded\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{ONE_CLICK_BODY}",
|
||||
ONE_CLICK_BODY.len()
|
||||
);
|
||||
|
||||
|
||||
+763
-59
@@ -1,7 +1,12 @@
|
||||
use std::ops::Bound;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use serde::Serialize;
|
||||
use reqwest::Method;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Value as JsonValue, json};
|
||||
use sha2::{Digest, Sha256};
|
||||
use tantivy::collector::{Count, TopDocs};
|
||||
use tantivy::query::{BooleanQuery, Occur, QueryParser, RangeQuery, TermQuery};
|
||||
use tantivy::schema::Value;
|
||||
@@ -11,6 +16,7 @@ use tantivy::schema::{
|
||||
use tantivy::snippet::SnippetGenerator;
|
||||
use tantivy::{Index, IndexReader, IndexWriter, ReloadPolicy, TantivyDocument, Term, doc};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct IndexableMessage {
|
||||
pub uid: u32,
|
||||
pub folder: String,
|
||||
@@ -46,8 +52,36 @@ pub struct SearchQuery {
|
||||
}
|
||||
|
||||
pub struct SearchEngine {
|
||||
base_dir: PathBuf,
|
||||
enabled: bool,
|
||||
backend: SearchBackend,
|
||||
}
|
||||
|
||||
enum SearchBackend {
|
||||
Disabled,
|
||||
LocalTantivy {
|
||||
base_dir: PathBuf,
|
||||
},
|
||||
OpenSearch {
|
||||
backend: Arc<OpenSearchBackend>,
|
||||
data_dir: String,
|
||||
worker_interval_secs: u64,
|
||||
worker_batch_size: usize,
|
||||
worker_max_attempts: u32,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct OpenSearchBackend {
|
||||
client: reqwest::Client,
|
||||
urls: Vec<String>,
|
||||
index_name: String,
|
||||
username: Option<String>,
|
||||
password: Option<String>,
|
||||
request_timeout_secs: u64,
|
||||
}
|
||||
|
||||
struct OpenSearchUserIndex {
|
||||
backend: Arc<OpenSearchBackend>,
|
||||
user_hash: String,
|
||||
}
|
||||
|
||||
struct SchemaFields {
|
||||
@@ -62,7 +96,7 @@ struct SchemaFields {
|
||||
has_attachments: Field,
|
||||
}
|
||||
|
||||
pub struct UserIndex {
|
||||
struct LocalUserIndex {
|
||||
index: Index,
|
||||
reader: IndexReader,
|
||||
#[allow(dead_code)]
|
||||
@@ -110,56 +144,261 @@ fn build_schema() -> (Schema, SchemaFields) {
|
||||
impl SearchEngine {
|
||||
pub fn new(base_dir: PathBuf) -> Self {
|
||||
Self {
|
||||
base_dir,
|
||||
enabled: true,
|
||||
backend: SearchBackend::LocalTantivy { base_dir },
|
||||
}
|
||||
}
|
||||
|
||||
pub fn disabled(base_dir: PathBuf) -> Self {
|
||||
pub fn disabled(_base_dir: PathBuf) -> Self {
|
||||
Self {
|
||||
base_dir,
|
||||
enabled: false,
|
||||
backend: SearchBackend::Disabled,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_config(config: &crate::config::AppConfig) -> Result<Self, String> {
|
||||
match config.effective_search_backend().as_str() {
|
||||
"disabled" => Ok(Self::disabled(PathBuf::from(
|
||||
config.effective_search_path(),
|
||||
))),
|
||||
"local_tantivy" => Ok(Self::new(PathBuf::from(config.effective_search_path()))),
|
||||
"opensearch" => {
|
||||
let client = reqwest::Client::builder()
|
||||
.connect_timeout(Duration::from_secs(
|
||||
config.search.request_timeout_secs.min(30),
|
||||
))
|
||||
.timeout(Duration::from_secs(config.search.request_timeout_secs))
|
||||
.build()
|
||||
.map_err(|e| format!("failed to build OpenSearch client: {e}"))?;
|
||||
let backend = OpenSearchBackend {
|
||||
client,
|
||||
urls: config
|
||||
.search
|
||||
.urls
|
||||
.iter()
|
||||
.map(|url| url.trim_end_matches('/').to_string())
|
||||
.collect(),
|
||||
index_name: format!("{}-messages-v1", config.search.index_prefix),
|
||||
username: config.search.username.clone(),
|
||||
password: config.search.password.clone(),
|
||||
request_timeout_secs: config.search.request_timeout_secs,
|
||||
};
|
||||
Ok(Self {
|
||||
backend: SearchBackend::OpenSearch {
|
||||
backend: Arc::new(backend),
|
||||
data_dir: config.data_dir.clone(),
|
||||
worker_interval_secs: config.search.worker_interval_secs,
|
||||
worker_batch_size: config.search.worker_batch_size,
|
||||
worker_max_attempts: config.search.worker_max_attempts,
|
||||
},
|
||||
})
|
||||
}
|
||||
other => Err(format!("unsupported search backend: {other}")),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_enabled(&self) -> bool {
|
||||
self.enabled
|
||||
!matches!(&self.backend, SearchBackend::Disabled)
|
||||
}
|
||||
|
||||
pub fn backend_name(&self) -> &'static str {
|
||||
match &self.backend {
|
||||
SearchBackend::Disabled => "disabled",
|
||||
SearchBackend::LocalTantivy { .. } => "local_tantivy",
|
||||
SearchBackend::OpenSearch { .. } => "opensearch",
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn health(&self) -> Result<(), String> {
|
||||
match &self.backend {
|
||||
SearchBackend::Disabled => Ok(()),
|
||||
SearchBackend::LocalTantivy { base_dir } => {
|
||||
std::fs::create_dir_all(base_dir)
|
||||
.map_err(|e| format!("failed to prepare local search directory: {e}"))?;
|
||||
Ok(())
|
||||
}
|
||||
SearchBackend::OpenSearch { backend, .. } => backend.health().await,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn open_user_index(&self, user_hash: &str) -> Result<UserIndex, String> {
|
||||
if !self.enabled {
|
||||
return Err("Tantivy search index is disabled by configuration".to_string());
|
||||
match &self.backend {
|
||||
SearchBackend::Disabled => Err("search index is disabled by configuration".to_string()),
|
||||
SearchBackend::LocalTantivy { base_dir } => {
|
||||
let index_dir = base_dir.join(user_hash).join("tantivy");
|
||||
std::fs::create_dir_all(&index_dir)
|
||||
.map_err(|e| format!("Failed to create index directory: {e}"))?;
|
||||
|
||||
let (schema, fields) = build_schema();
|
||||
let index = if Index::open_in_dir(&index_dir).is_ok() {
|
||||
Index::open_in_dir(&index_dir)
|
||||
.map_err(|e| format!("Failed to open existing index: {e}"))?
|
||||
} else {
|
||||
Index::create_in_dir(&index_dir, schema.clone())
|
||||
.map_err(|e| format!("Failed to create index: {e}"))?
|
||||
};
|
||||
let reader = index
|
||||
.reader_builder()
|
||||
.reload_policy(ReloadPolicy::OnCommitWithDelay)
|
||||
.try_into()
|
||||
.map_err(|e| format!("Failed to create index reader: {e}"))?;
|
||||
Ok(UserIndex {
|
||||
inner: UserIndexBackend::Local(LocalUserIndex {
|
||||
index,
|
||||
reader,
|
||||
schema,
|
||||
fields,
|
||||
}),
|
||||
})
|
||||
}
|
||||
SearchBackend::OpenSearch { backend, .. } => Ok(UserIndex {
|
||||
inner: UserIndexBackend::OpenSearch(OpenSearchUserIndex {
|
||||
backend: backend.clone(),
|
||||
user_hash: user_hash.to_string(),
|
||||
}),
|
||||
}),
|
||||
}
|
||||
let index_dir = self.base_dir.join(user_hash).join("tantivy");
|
||||
std::fs::create_dir_all(&index_dir)
|
||||
.map_err(|e| format!("Failed to create index directory: {e}"))?;
|
||||
}
|
||||
|
||||
let (schema, fields) = build_schema();
|
||||
|
||||
let index = if Index::open_in_dir(&index_dir).is_ok() {
|
||||
Index::open_in_dir(&index_dir)
|
||||
.map_err(|e| format!("Failed to open existing index: {e}"))?
|
||||
} else {
|
||||
Index::create_in_dir(&index_dir, schema.clone())
|
||||
.map_err(|e| format!("Failed to create index: {e}"))?
|
||||
pub fn start_worker(&self) -> Option<tokio::task::JoinHandle<()>> {
|
||||
let SearchBackend::OpenSearch {
|
||||
backend,
|
||||
data_dir,
|
||||
worker_interval_secs,
|
||||
worker_batch_size,
|
||||
worker_max_attempts,
|
||||
} = &self.backend
|
||||
else {
|
||||
return None;
|
||||
};
|
||||
|
||||
let reader = index
|
||||
.reader_builder()
|
||||
.reload_policy(ReloadPolicy::OnCommitWithDelay)
|
||||
.try_into()
|
||||
.map_err(|e| format!("Failed to create index reader: {e}"))?;
|
||||
|
||||
Ok(UserIndex {
|
||||
index,
|
||||
reader,
|
||||
schema,
|
||||
fields,
|
||||
})
|
||||
let backend = backend.clone();
|
||||
let data_dir = data_dir.clone();
|
||||
let interval_secs = *worker_interval_secs;
|
||||
let batch_size = *worker_batch_size;
|
||||
let max_attempts = *worker_max_attempts;
|
||||
Some(tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(Duration::from_secs(interval_secs));
|
||||
loop {
|
||||
interval.tick().await;
|
||||
match process_search_outbox(&backend, &data_dir, batch_size, max_attempts).await {
|
||||
Ok(processed) if processed > 0 => {
|
||||
tracing::debug!(processed, "processed OpenSearch outbox events");
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(error) => tracing::warn!(%error, "OpenSearch outbox worker pass failed"),
|
||||
}
|
||||
}
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
pub struct UserIndex {
|
||||
inner: UserIndexBackend,
|
||||
}
|
||||
|
||||
enum UserIndexBackend {
|
||||
Local(LocalUserIndex),
|
||||
OpenSearch(OpenSearchUserIndex),
|
||||
}
|
||||
|
||||
impl UserIndex {
|
||||
pub fn is_excluded_folder(folder: &str) -> bool {
|
||||
LocalUserIndex::is_excluded_folder(folder)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn index_message(&self, msg: &IndexableMessage) -> Result<(), String> {
|
||||
match &self.inner {
|
||||
UserIndexBackend::Local(index) => index.index_message(msg),
|
||||
UserIndexBackend::OpenSearch(_) => {
|
||||
Err("test-only direct indexing is unavailable for OpenSearch".to_string())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn index_message_with_connection(
|
||||
&self,
|
||||
conn: &crate::db::pool::Connection,
|
||||
msg: &IndexableMessage,
|
||||
) -> Result<(), String> {
|
||||
match &self.inner {
|
||||
UserIndexBackend::Local(index) => index.index_message(msg),
|
||||
UserIndexBackend::OpenSearch(index) => index.enqueue_upsert_on(conn, msg),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn document_count(&self) -> Result<usize, String> {
|
||||
match &self.inner {
|
||||
UserIndexBackend::Local(index) => Ok(index.document_count()),
|
||||
UserIndexBackend::OpenSearch(index) => index.document_count().await,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn rebuild_from_messages(
|
||||
&self,
|
||||
messages: &[IndexableMessage],
|
||||
) -> Result<usize, String> {
|
||||
match &self.inner {
|
||||
UserIndexBackend::Local(index) => index.rebuild_from_messages(messages),
|
||||
UserIndexBackend::OpenSearch(index) => index.rebuild_from_messages(messages).await,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn index_messages_batch(&self, messages: &[IndexableMessage]) -> Result<(), String> {
|
||||
match &self.inner {
|
||||
UserIndexBackend::Local(index) => index.index_messages_batch(messages),
|
||||
UserIndexBackend::OpenSearch(_) => {
|
||||
Err("test-only direct batch indexing is unavailable for OpenSearch".to_string())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn index_messages_batch_with_connection(
|
||||
&self,
|
||||
conn: &crate::db::pool::Connection,
|
||||
messages: &[IndexableMessage],
|
||||
) -> Result<(), String> {
|
||||
match &self.inner {
|
||||
UserIndexBackend::Local(index) => index.index_messages_batch(messages),
|
||||
UserIndexBackend::OpenSearch(index) => {
|
||||
for message in messages {
|
||||
index.enqueue_upsert_on(conn, message)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn delete_message(&self, uid: u32, folder: &str) -> Result<(), String> {
|
||||
match &self.inner {
|
||||
UserIndexBackend::Local(index) => index.delete_message(uid, folder),
|
||||
UserIndexBackend::OpenSearch(_) => {
|
||||
Err("test-only direct deletion is unavailable for OpenSearch".to_string())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn delete_message_with_connection(
|
||||
&self,
|
||||
conn: &crate::db::pool::Connection,
|
||||
uid: u32,
|
||||
folder: &str,
|
||||
) -> Result<(), String> {
|
||||
match &self.inner {
|
||||
UserIndexBackend::Local(index) => index.delete_message(uid, folder),
|
||||
UserIndexBackend::OpenSearch(index) => index.enqueue_delete_on(conn, uid, folder),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn search(&self, query: &SearchQuery) -> Result<(Vec<SearchResult>, usize), String> {
|
||||
match &self.inner {
|
||||
UserIndexBackend::Local(index) => index.search(query),
|
||||
UserIndexBackend::OpenSearch(index) => index.search(query).await,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl LocalUserIndex {
|
||||
pub fn is_excluded_folder(folder: &str) -> bool {
|
||||
let lower = folder.to_lowercase();
|
||||
lower == "trash"
|
||||
@@ -432,6 +671,426 @@ impl UserIndex {
|
||||
}
|
||||
}
|
||||
|
||||
impl OpenSearchBackend {
|
||||
fn event_lease_secs(&self) -> u64 {
|
||||
// A mutation can probe/create the index and then write the document.
|
||||
// Account for failover across all configured URLs and keep a safety margin so
|
||||
// another worker cannot reclaim the same document while this request is alive.
|
||||
self.request_timeout_secs
|
||||
.saturating_mul(self.urls.len().max(1) as u64)
|
||||
.saturating_mul(4)
|
||||
.saturating_add(30)
|
||||
.max(60)
|
||||
}
|
||||
|
||||
async fn request(
|
||||
&self,
|
||||
method: Method,
|
||||
path: &str,
|
||||
body: Option<&JsonValue>,
|
||||
) -> Result<(reqwest::StatusCode, String), String> {
|
||||
let mut last_error = None;
|
||||
for base in &self.urls {
|
||||
let url = format!(
|
||||
"{}/{}",
|
||||
base.trim_end_matches('/'),
|
||||
path.trim_start_matches('/')
|
||||
);
|
||||
let mut request = self.client.request(method.clone(), &url);
|
||||
if let Some(username) = self.username.as_deref() {
|
||||
request = request.basic_auth(username, self.password.as_deref());
|
||||
}
|
||||
if let Some(body) = body {
|
||||
request = request.json(body);
|
||||
}
|
||||
match request.send().await {
|
||||
Ok(response) => {
|
||||
let status = response.status();
|
||||
let text = response.text().await.unwrap_or_default();
|
||||
if status.is_server_error() {
|
||||
last_error = Some(format!("OpenSearch {url} returned {status}: {text}"));
|
||||
continue;
|
||||
}
|
||||
return Ok((status, text));
|
||||
}
|
||||
Err(error) => {
|
||||
last_error = Some(format!("OpenSearch request to {url} failed: {error}"))
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(last_error.unwrap_or_else(|| "no OpenSearch URLs configured".to_string()))
|
||||
}
|
||||
|
||||
async fn ensure_index(&self) -> Result<(), String> {
|
||||
let (status, body) = self.request(Method::HEAD, &self.index_name, None).await?;
|
||||
if status.is_success() {
|
||||
return Ok(());
|
||||
}
|
||||
if status != reqwest::StatusCode::NOT_FOUND {
|
||||
return Err(format!(
|
||||
"OpenSearch index probe failed with {status}: {body}"
|
||||
));
|
||||
}
|
||||
let mapping = json!({
|
||||
"mappings": {
|
||||
"dynamic": "strict",
|
||||
"properties": {
|
||||
"user_hash": {"type": "keyword"},
|
||||
"uid": {"type": "long"},
|
||||
"folder": {"type": "keyword"},
|
||||
"subject": {"type": "text"},
|
||||
"from_address": {"type": "keyword"},
|
||||
"from_name": {"type": "text"},
|
||||
"to_addresses": {"type": "text"},
|
||||
"body_text": {"type": "text"},
|
||||
"date_epoch": {"type": "long"},
|
||||
"has_attachments": {"type": "boolean"}
|
||||
}
|
||||
}
|
||||
});
|
||||
let (status, body) = self
|
||||
.request(Method::PUT, &self.index_name, Some(&mapping))
|
||||
.await?;
|
||||
if status.is_success()
|
||||
|| (status == reqwest::StatusCode::BAD_REQUEST
|
||||
&& body.contains("resource_already_exists_exception"))
|
||||
{
|
||||
Ok(())
|
||||
} else {
|
||||
Err(format!(
|
||||
"failed to create OpenSearch index ({status}): {body}"
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
async fn health(&self) -> Result<(), String> {
|
||||
let (status, body) = self
|
||||
.request(Method::GET, "_cluster/health?local=true", None)
|
||||
.await?;
|
||||
if status.is_success() {
|
||||
self.ensure_index().await
|
||||
} else {
|
||||
Err(format!("OpenSearch health check failed ({status}): {body}"))
|
||||
}
|
||||
}
|
||||
|
||||
fn document_id(user_hash: &str, uid: u32, folder: &str) -> String {
|
||||
format!(
|
||||
"{:x}",
|
||||
Sha256::digest(format!("{user_hash}\0{folder}\0{uid}").as_bytes())
|
||||
)
|
||||
}
|
||||
|
||||
async fn index_direct(&self, user_hash: &str, msg: &IndexableMessage) -> Result<(), String> {
|
||||
if LocalUserIndex::is_excluded_folder(&msg.folder) {
|
||||
return self.delete_direct(user_hash, msg.uid, &msg.folder).await;
|
||||
}
|
||||
self.ensure_index().await?;
|
||||
let id = Self::document_id(user_hash, msg.uid, &msg.folder);
|
||||
let path = format!("{}/_doc/{}?routing={}", self.index_name, id, user_hash);
|
||||
let body = json!({
|
||||
"user_hash": user_hash,
|
||||
"uid": msg.uid,
|
||||
"folder": &msg.folder,
|
||||
"subject": &msg.subject,
|
||||
"from_address": &msg.from_address,
|
||||
"from_name": &msg.from_name,
|
||||
"to_addresses": &msg.to_addresses,
|
||||
"body_text": &msg.body_text,
|
||||
"date_epoch": msg.date_epoch,
|
||||
"has_attachments": msg.has_attachments
|
||||
});
|
||||
let (status, response) = self.request(Method::PUT, &path, Some(&body)).await?;
|
||||
if status.is_success() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(format!(
|
||||
"OpenSearch document upsert failed ({status}): {response}"
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
async fn delete_direct(&self, user_hash: &str, uid: u32, folder: &str) -> Result<(), String> {
|
||||
self.ensure_index().await?;
|
||||
let id = Self::document_id(user_hash, uid, folder);
|
||||
let path = format!("{}/_doc/{}?routing={}", self.index_name, id, user_hash);
|
||||
let (status, response) = self.request(Method::DELETE, &path, None).await?;
|
||||
if status.is_success() || status == reqwest::StatusCode::NOT_FOUND {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(format!(
|
||||
"OpenSearch document delete failed ({status}): {response}"
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
async fn delete_user_documents(&self, user_hash: &str) -> Result<(), String> {
|
||||
self.ensure_index().await?;
|
||||
let path = format!(
|
||||
"{}/_delete_by_query?refresh=true&routing={}",
|
||||
self.index_name, user_hash
|
||||
);
|
||||
let body = json!({"query": {"term": {"user_hash": user_hash}}});
|
||||
let (status, response) = self.request(Method::POST, &path, Some(&body)).await?;
|
||||
if status.is_success() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(format!(
|
||||
"OpenSearch user reindex cleanup failed ({status}): {response}"
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
async fn refresh(&self) -> Result<(), String> {
|
||||
self.ensure_index().await?;
|
||||
let path = format!("{}/_refresh", self.index_name);
|
||||
let (status, response) = self.request(Method::POST, &path, None).await?;
|
||||
if status.is_success() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(format!("OpenSearch refresh failed ({status}): {response}"))
|
||||
}
|
||||
}
|
||||
|
||||
async fn count(&self, user_hash: &str) -> Result<usize, String> {
|
||||
self.ensure_index().await?;
|
||||
let path = format!("{}/_count?routing={}", self.index_name, user_hash);
|
||||
let body = json!({"query": {"term": {"user_hash": user_hash}}});
|
||||
let (status, response) = self.request(Method::POST, &path, Some(&body)).await?;
|
||||
if !status.is_success() {
|
||||
return Err(format!("OpenSearch count failed ({status}): {response}"));
|
||||
}
|
||||
let parsed: JsonValue = serde_json::from_str(&response)
|
||||
.map_err(|e| format!("invalid OpenSearch count response: {e}"))?;
|
||||
Ok(parsed.get("count").and_then(JsonValue::as_u64).unwrap_or(0) as usize)
|
||||
}
|
||||
|
||||
async fn search(
|
||||
&self,
|
||||
user_hash: &str,
|
||||
query: &SearchQuery,
|
||||
) -> Result<(Vec<SearchResult>, usize), String> {
|
||||
self.ensure_index().await?;
|
||||
let body = build_opensearch_query(user_hash, query);
|
||||
let path = format!("{}/_search?routing={}", self.index_name, user_hash);
|
||||
let (status, response) = self.request(Method::POST, &path, Some(&body)).await?;
|
||||
if !status.is_success() {
|
||||
return Err(format!("OpenSearch search failed ({status}): {response}"));
|
||||
}
|
||||
let parsed: JsonValue = serde_json::from_str(&response)
|
||||
.map_err(|e| format!("invalid OpenSearch search response: {e}"))?;
|
||||
let total = parsed
|
||||
.pointer("/hits/total/value")
|
||||
.and_then(JsonValue::as_u64)
|
||||
.unwrap_or(0) as usize;
|
||||
let mut results = Vec::new();
|
||||
if let Some(hits) = parsed.pointer("/hits/hits").and_then(JsonValue::as_array) {
|
||||
for hit in hits {
|
||||
let source = hit.get("_source").unwrap_or(&JsonValue::Null);
|
||||
let uid = source.get("uid").and_then(JsonValue::as_u64).unwrap_or(0) as u32;
|
||||
let folder = source
|
||||
.get("folder")
|
||||
.and_then(JsonValue::as_str)
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let score = hit.get("_score").and_then(JsonValue::as_f64).unwrap_or(0.0) as f32;
|
||||
let snippet = hit
|
||||
.pointer("/highlight/body_text/0")
|
||||
.or_else(|| hit.pointer("/highlight/subject/0"))
|
||||
.and_then(JsonValue::as_str)
|
||||
.or_else(|| source.get("subject").and_then(JsonValue::as_str))
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
results.push(SearchResult {
|
||||
uid,
|
||||
folder,
|
||||
score,
|
||||
snippet,
|
||||
});
|
||||
}
|
||||
}
|
||||
Ok((results, total))
|
||||
}
|
||||
}
|
||||
|
||||
fn build_opensearch_query(user_hash: &str, query: &SearchQuery) -> JsonValue {
|
||||
let mut must = Vec::new();
|
||||
let mut filter = vec![json!({"term": {"user_hash": user_hash}})];
|
||||
if !query.text.trim().is_empty() {
|
||||
must.push(json!({
|
||||
"multi_match": {
|
||||
"query": query.text.as_str(),
|
||||
"fields": ["subject^2", "body_text", "from_name", "to_addresses"]
|
||||
}
|
||||
}));
|
||||
}
|
||||
if let Some(subject) = query.subject_only.as_deref() {
|
||||
must.push(json!({"match": {"subject": subject}}));
|
||||
}
|
||||
if let Some(folder) = query.folder.as_deref() {
|
||||
filter.push(json!({"term": {"folder": folder}}));
|
||||
}
|
||||
if let Some(from) = query.from.as_deref() {
|
||||
if from.contains('@') {
|
||||
filter.push(json!({"term": {"from_address": from}}));
|
||||
} else {
|
||||
must.push(json!({"match": {"from_name": from}}));
|
||||
}
|
||||
}
|
||||
if let Some(to) = query.to.as_deref() {
|
||||
must.push(json!({"match": {"to_addresses": to}}));
|
||||
}
|
||||
if query.date_from.is_some() || query.date_to.is_some() {
|
||||
let mut range = serde_json::Map::new();
|
||||
if let Some(value) = query.date_from {
|
||||
range.insert("gte".to_string(), json!(value));
|
||||
}
|
||||
if let Some(value) = query.date_to {
|
||||
range.insert("lte".to_string(), json!(value));
|
||||
}
|
||||
filter.push(json!({"range": {"date_epoch": JsonValue::Object(range)}}));
|
||||
}
|
||||
if let Some(value) = query.has_attachment {
|
||||
filter.push(json!({"term": {"has_attachments": value}}));
|
||||
}
|
||||
let query_body = if must.is_empty() {
|
||||
json!({"bool": {"filter": filter}})
|
||||
} else {
|
||||
json!({"bool": {"must": must, "filter": filter}})
|
||||
};
|
||||
json!({
|
||||
"from": query.offset,
|
||||
"size": query.limit.max(1),
|
||||
"track_total_hits": true,
|
||||
"query": query_body,
|
||||
"highlight": {
|
||||
"pre_tags": ["<b>"],
|
||||
"post_tags": ["</b>"],
|
||||
"fields": {"subject": {}, "body_text": {"fragment_size": 200, "number_of_fragments": 1}}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
impl OpenSearchUserIndex {
|
||||
fn event_id(&self, uid: u32, folder: &str) -> String {
|
||||
OpenSearchBackend::document_id(&self.user_hash, uid, folder)
|
||||
}
|
||||
|
||||
fn enqueue_upsert_on(
|
||||
&self,
|
||||
conn: &crate::db::pool::Connection,
|
||||
msg: &IndexableMessage,
|
||||
) -> Result<(), String> {
|
||||
let id = self.event_id(msg.uid, &msg.folder);
|
||||
if LocalUserIndex::is_excluded_folder(&msg.folder) {
|
||||
crate::db::search_outbox::enqueue(conn, &id, "delete", msg.uid, &msg.folder, None)?;
|
||||
} else {
|
||||
let payload = serde_json::to_string(msg)
|
||||
.map_err(|e| format!("failed to serialize search document: {e}"))?;
|
||||
crate::db::search_outbox::enqueue(
|
||||
conn,
|
||||
&id,
|
||||
"upsert",
|
||||
msg.uid,
|
||||
&msg.folder,
|
||||
Some(&payload),
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn enqueue_delete_on(
|
||||
&self,
|
||||
conn: &crate::db::pool::Connection,
|
||||
uid: u32,
|
||||
folder: &str,
|
||||
) -> Result<(), String> {
|
||||
let id = self.event_id(uid, folder);
|
||||
crate::db::search_outbox::enqueue(conn, &id, "delete", uid, folder, None)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn document_count(&self) -> Result<usize, String> {
|
||||
self.backend.count(&self.user_hash).await
|
||||
}
|
||||
|
||||
async fn search(&self, query: &SearchQuery) -> Result<(Vec<SearchResult>, usize), String> {
|
||||
self.backend.search(&self.user_hash, query).await
|
||||
}
|
||||
|
||||
async fn rebuild_from_messages(&self, messages: &[IndexableMessage]) -> Result<usize, String> {
|
||||
self.backend.delete_user_documents(&self.user_hash).await?;
|
||||
let mut indexed = 0;
|
||||
for message in messages {
|
||||
if LocalUserIndex::is_excluded_folder(&message.folder) {
|
||||
continue;
|
||||
}
|
||||
self.backend.index_direct(&self.user_hash, message).await?;
|
||||
indexed += 1;
|
||||
}
|
||||
self.backend.refresh().await?;
|
||||
Ok(indexed)
|
||||
}
|
||||
}
|
||||
|
||||
async fn process_search_outbox(
|
||||
backend: &OpenSearchBackend,
|
||||
data_dir: &str,
|
||||
batch_size: usize,
|
||||
max_attempts: u32,
|
||||
) -> Result<usize, String> {
|
||||
let mut processed = 0;
|
||||
for user_hash in crate::db::pool::list_account_user_hashes()? {
|
||||
let conn = match crate::db::pool::open_user_db(data_dir, &user_hash) {
|
||||
Ok(conn) => conn,
|
||||
Err(error) => {
|
||||
tracing::warn!(%error, %user_hash, "failed to open tenant DB for search outbox");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let events = crate::db::search_outbox::list_ready(&conn, batch_size)?;
|
||||
let lease_secs = backend.event_lease_secs();
|
||||
for event in events {
|
||||
if !crate::db::search_outbox::claim(&conn, &event, lease_secs)? {
|
||||
continue;
|
||||
}
|
||||
let result = match event.event_type.as_str() {
|
||||
"upsert" => {
|
||||
let message = event
|
||||
.payload
|
||||
.as_deref()
|
||||
.ok_or_else(|| "search upsert event is missing payload".to_string())
|
||||
.and_then(|payload| {
|
||||
serde_json::from_str::<IndexableMessage>(payload)
|
||||
.map_err(|e| format!("invalid search outbox payload: {e}"))
|
||||
});
|
||||
match message {
|
||||
Ok(message) => backend.index_direct(&user_hash, &message).await,
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
"delete" => {
|
||||
backend
|
||||
.delete_direct(&user_hash, event.message_uid, &event.folder)
|
||||
.await
|
||||
}
|
||||
other => Err(format!("unsupported search outbox event type: {other}")),
|
||||
};
|
||||
match result {
|
||||
Ok(()) => {
|
||||
crate::db::search_outbox::mark_success(&conn, &event)?;
|
||||
processed += 1;
|
||||
}
|
||||
Err(error) => {
|
||||
crate::db::search_outbox::mark_failure(&conn, &event, &error, max_attempts)?;
|
||||
tracing::warn!(%error, event_id = %event.id, %user_hash, "OpenSearch outbox event failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(processed)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -446,6 +1105,26 @@ mod tests {
|
||||
(tmp, user_index)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn opensearch_event_lease_covers_timeout_and_failover_budget() {
|
||||
let backend = OpenSearchBackend {
|
||||
client: reqwest::Client::new(),
|
||||
urls: vec!["http://os-a".to_string(), "http://os-b".to_string()],
|
||||
index_name: "rustbox-test".to_string(),
|
||||
username: None,
|
||||
password: None,
|
||||
request_timeout_secs: 15,
|
||||
};
|
||||
assert_eq!(backend.event_lease_secs(), 150);
|
||||
|
||||
let single = OpenSearchBackend {
|
||||
urls: vec!["http://os-a".to_string()],
|
||||
request_timeout_secs: 5,
|
||||
..backend
|
||||
};
|
||||
assert_eq!(single.event_lease_secs(), 60);
|
||||
}
|
||||
|
||||
fn make_message(uid: u32, folder: &str, subject: &str, body: &str) -> IndexableMessage {
|
||||
IndexableMessage {
|
||||
uid,
|
||||
@@ -460,8 +1139,8 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn index_and_search_by_subject() {
|
||||
#[tokio::test]
|
||||
async fn index_and_search_by_subject() {
|
||||
let (_tmp, idx) = setup();
|
||||
|
||||
let msg1 = make_message(1, "INBOX", "Meeting tomorrow morning", "Let us meet.");
|
||||
@@ -475,7 +1154,7 @@ mod tests {
|
||||
limit: 10,
|
||||
..Default::default()
|
||||
};
|
||||
let (results, total) = idx.search(&query).unwrap();
|
||||
let (results, total) = idx.search(&query).await.unwrap();
|
||||
|
||||
assert_eq!(total, 1);
|
||||
assert_eq!(results.len(), 1);
|
||||
@@ -483,8 +1162,8 @@ mod tests {
|
||||
assert_eq!(results[0].folder, "INBOX");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn index_and_search_by_body() {
|
||||
#[tokio::test]
|
||||
async fn index_and_search_by_body() {
|
||||
let (_tmp, idx) = setup();
|
||||
|
||||
let msg1 = make_message(1, "INBOX", "Hello", "The quarterly report is attached.");
|
||||
@@ -502,15 +1181,15 @@ mod tests {
|
||||
limit: 10,
|
||||
..Default::default()
|
||||
};
|
||||
let (results, total) = idx.search(&query).unwrap();
|
||||
let (results, total) = idx.search(&query).await.unwrap();
|
||||
|
||||
assert_eq!(total, 1);
|
||||
assert_eq!(results.len(), 1);
|
||||
assert_eq!(results[0].uid, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_with_folder_filter() {
|
||||
#[tokio::test]
|
||||
async fn search_with_folder_filter() {
|
||||
let (_tmp, idx) = setup();
|
||||
|
||||
let msg1 = make_message(1, "INBOX", "Project update", "Status of the project.");
|
||||
@@ -524,7 +1203,7 @@ mod tests {
|
||||
limit: 10,
|
||||
..Default::default()
|
||||
};
|
||||
let (results, total) = idx.search(&query).unwrap();
|
||||
let (results, total) = idx.search(&query).await.unwrap();
|
||||
|
||||
assert_eq!(total, 1);
|
||||
assert_eq!(results.len(), 1);
|
||||
@@ -532,8 +1211,8 @@ mod tests {
|
||||
assert_eq!(results[0].folder, "Sent");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_with_date_range() {
|
||||
#[tokio::test]
|
||||
async fn search_with_date_range() {
|
||||
let (_tmp, idx) = setup();
|
||||
|
||||
let mut msg1 = make_message(1, "INBOX", "January news", "News from January.");
|
||||
@@ -554,15 +1233,15 @@ mod tests {
|
||||
limit: 10,
|
||||
..Default::default()
|
||||
};
|
||||
let (results, total) = idx.search(&query).unwrap();
|
||||
let (results, total) = idx.search(&query).await.unwrap();
|
||||
|
||||
assert_eq!(total, 1);
|
||||
assert_eq!(results.len(), 1);
|
||||
assert_eq!(results[0].uid, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_removes_from_index() {
|
||||
#[tokio::test]
|
||||
async fn delete_removes_from_index() {
|
||||
let (_tmp, idx) = setup();
|
||||
|
||||
let msg1 = make_message(1, "INBOX", "Important email", "Very important content.");
|
||||
@@ -575,19 +1254,19 @@ mod tests {
|
||||
limit: 10,
|
||||
..Default::default()
|
||||
};
|
||||
let (_, total) = idx.search(&query).unwrap();
|
||||
let (_, total) = idx.search(&query).await.unwrap();
|
||||
assert_eq!(total, 2);
|
||||
|
||||
idx.delete_message(1, "INBOX").unwrap();
|
||||
|
||||
let (results, total) = idx.search(&query).unwrap();
|
||||
let (results, total) = idx.search(&query).await.unwrap();
|
||||
assert_eq!(total, 1);
|
||||
assert_eq!(results.len(), 1);
|
||||
assert_eq!(results[0].uid, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_empty_text_returns_all() {
|
||||
#[tokio::test]
|
||||
async fn search_empty_text_returns_all() {
|
||||
let (_tmp, idx) = setup();
|
||||
|
||||
let msg = make_message(1, "INBOX", "Test subject", "Test body text.");
|
||||
@@ -598,14 +1277,39 @@ mod tests {
|
||||
limit: 10,
|
||||
..Default::default()
|
||||
};
|
||||
let (results, total) = idx.search(&query).unwrap();
|
||||
let (results, total) = idx.search(&query).await.unwrap();
|
||||
|
||||
assert_eq!(total, 1);
|
||||
assert_eq!(results.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_pagination() {
|
||||
#[tokio::test]
|
||||
async fn opensearch_query_always_scopes_to_tenant() {
|
||||
let query = SearchQuery {
|
||||
text: "invoice".to_string(),
|
||||
folder: Some("INBOX".to_string()),
|
||||
limit: 25,
|
||||
..Default::default()
|
||||
};
|
||||
let body = build_opensearch_query("tenant-a", &query);
|
||||
let filters = body
|
||||
.pointer("/query/bool/filter")
|
||||
.and_then(JsonValue::as_array)
|
||||
.expect("OpenSearch filter array");
|
||||
|
||||
assert!(filters.iter().any(|filter| {
|
||||
filter
|
||||
.pointer("/term/user_hash")
|
||||
.and_then(JsonValue::as_str)
|
||||
== Some("tenant-a")
|
||||
}));
|
||||
assert!(filters.iter().any(|filter| {
|
||||
filter.pointer("/term/folder").and_then(JsonValue::as_str) == Some("INBOX")
|
||||
}));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn search_pagination() {
|
||||
let (_tmp, idx) = setup();
|
||||
|
||||
let messages: Vec<IndexableMessage> = (1..=5)
|
||||
@@ -627,7 +1331,7 @@ mod tests {
|
||||
offset: 0,
|
||||
..Default::default()
|
||||
};
|
||||
let (results1, total1) = idx.search(&query_page1).unwrap();
|
||||
let (results1, total1) = idx.search(&query_page1).await.unwrap();
|
||||
assert_eq!(total1, 5);
|
||||
assert_eq!(results1.len(), 2);
|
||||
|
||||
@@ -637,7 +1341,7 @@ mod tests {
|
||||
offset: 2,
|
||||
..Default::default()
|
||||
};
|
||||
let (results2, total2) = idx.search(&query_page2).unwrap();
|
||||
let (results2, total2) = idx.search(&query_page2).await.unwrap();
|
||||
assert_eq!(total2, 5);
|
||||
assert_eq!(results2.len(), 2);
|
||||
|
||||
|
||||
@@ -178,7 +178,7 @@ impl DatabaseSessionStore {
|
||||
.map_err(|e| format!("MySQL database schema initialization failed: {e}"))?;
|
||||
Pool::Mysql(pool)
|
||||
}
|
||||
"pgsql" | "postgres" | "postgresql" => {
|
||||
"pgsql" => {
|
||||
let url = config
|
||||
.database_url
|
||||
.as_deref()
|
||||
|
||||
+2
-3
@@ -122,7 +122,7 @@ impl ManageSieveClient {
|
||||
|
||||
let plain = format!("\0{}\0{}", creds.username, creds.password);
|
||||
let encoded = base64::engine::general_purpose::STANDARD.encode(plain.as_bytes());
|
||||
let command = format!("AUTHENTICATE \"PLAIN\" \"{}\"\r\n", encoded);
|
||||
let command = format!("AUTHENTICATE \"PLAIN\" \"{encoded}\"\r\n");
|
||||
write_all_timeout(&mut stream, command.as_bytes(), creds.timeout).await?;
|
||||
let auth = read_response(&mut stream, creds.timeout).await?;
|
||||
if auth.status == ResponseStatus::No || auth.status == ResponseStatus::Bye {
|
||||
@@ -421,8 +421,7 @@ async fn read_wire_line<S: AsyncRead + Unpin>(stream: &mut S) -> Result<WireLine
|
||||
|
||||
if size > MAX_LITERAL_BYTES {
|
||||
return Err(SieveError::Protocol(format!(
|
||||
"server literal exceeds {} bytes",
|
||||
MAX_LITERAL_BYTES
|
||||
"server literal exceeds {MAX_LITERAL_BYTES} bytes"
|
||||
)));
|
||||
}
|
||||
let mut literal = vec![0u8; size];
|
||||
|
||||
+1
-1
@@ -29,7 +29,7 @@ impl fmt::Display for SieveError {
|
||||
write!(f, "ManageSieve server rejected the command: {msg}")
|
||||
}
|
||||
Self::Timeout => write!(f, "ManageSieve operation timed out"),
|
||||
Self::ScriptTooLarge => write!(f, "Sieve script exceeds the RustBox 0.1.2 size limit"),
|
||||
Self::ScriptTooLarge => write!(f, "Sieve script exceeds the RustBox 0.2.0 size limit"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,633 @@
|
||||
use std::path::{Path as FsPath, PathBuf};
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use bytes::Bytes;
|
||||
use futures::{Stream, StreamExt};
|
||||
use object_store::aws::{AmazonS3Builder, Checksum};
|
||||
use object_store::buffered::BufWriter;
|
||||
use object_store::client::ClientOptions;
|
||||
use object_store::local::LocalFileSystem;
|
||||
use object_store::path::Path;
|
||||
use object_store::{ObjectStore, ObjectStoreExt, RetryConfig};
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
|
||||
use crate::config::{AppConfig, StorageConfig};
|
||||
|
||||
pub type BlobStream = Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct BlobStat {
|
||||
pub size: u64,
|
||||
pub e_tag: Option<String>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub trait BlobUpload: Send {
|
||||
async fn write(&mut self, chunk: Bytes) -> Result<(), String>;
|
||||
async fn finish(self: Box<Self>) -> Result<BlobStat, String>;
|
||||
async fn abort(self: Box<Self>) -> Result<(), String>;
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub trait BlobStorage: Send + Sync {
|
||||
async fn begin_upload(&self, key: &str) -> Result<Box<dyn BlobUpload>, String>;
|
||||
async fn get(&self, key: &str) -> Result<BlobStream, String>;
|
||||
async fn delete(&self, key: &str) -> Result<(), String>;
|
||||
async fn exists(&self, key: &str) -> Result<bool, String>;
|
||||
async fn health(&self) -> Result<(), String>;
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ObjectStorage {
|
||||
store: Arc<dyn ObjectStore>,
|
||||
prefix: String,
|
||||
}
|
||||
|
||||
impl ObjectStorage {
|
||||
fn validate_logical_key(key: &str) -> Result<(), String> {
|
||||
if key.is_empty() || key.starts_with('/') || key.contains('\\') {
|
||||
return Err("invalid blob key".to_string());
|
||||
}
|
||||
if key
|
||||
.split('/')
|
||||
.any(|part| part.is_empty() || part == "." || part == "..")
|
||||
{
|
||||
return Err("invalid blob key".to_string());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn object_path(&self, key: &str) -> Result<Path, String> {
|
||||
Self::validate_logical_key(key)?;
|
||||
let full = if self.prefix.is_empty() {
|
||||
key.to_string()
|
||||
} else {
|
||||
format!("{}/{}", self.prefix.trim_matches('/'), key)
|
||||
};
|
||||
Path::parse(full).map_err(|e| format!("invalid object key: {e}"))
|
||||
}
|
||||
|
||||
fn stat_from_meta(meta: object_store::ObjectMeta) -> BlobStat {
|
||||
BlobStat {
|
||||
size: meta.size,
|
||||
e_tag: meta.e_tag,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct ObjectUpload {
|
||||
writer: BufWriter,
|
||||
store: Arc<dyn ObjectStore>,
|
||||
path: Path,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl BlobUpload for ObjectUpload {
|
||||
async fn write(&mut self, chunk: Bytes) -> Result<(), String> {
|
||||
self.writer
|
||||
.put(chunk)
|
||||
.await
|
||||
.map_err(|e| format!("blob upload failed: {e}"))
|
||||
}
|
||||
|
||||
async fn finish(mut self: Box<Self>) -> Result<BlobStat, String> {
|
||||
self.writer
|
||||
.shutdown()
|
||||
.await
|
||||
.map_err(|e| format!("blob upload finalization failed: {e}"))?;
|
||||
let meta = self
|
||||
.store
|
||||
.head(&self.path)
|
||||
.await
|
||||
.map_err(|e| format!("blob stat after upload failed: {e}"))?;
|
||||
Ok(ObjectStorage::stat_from_meta(meta))
|
||||
}
|
||||
|
||||
async fn abort(mut self: Box<Self>) -> Result<(), String> {
|
||||
self.writer
|
||||
.abort()
|
||||
.await
|
||||
.map_err(|e| format!("blob upload abort failed: {e}"))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl BlobStorage for ObjectStorage {
|
||||
async fn begin_upload(&self, key: &str) -> Result<Box<dyn BlobUpload>, String> {
|
||||
let path = self.object_path(key)?;
|
||||
let writer = BufWriter::with_capacity(self.store.clone(), path.clone(), 8 * 1024 * 1024)
|
||||
.with_max_concurrency(4);
|
||||
Ok(Box::new(ObjectUpload {
|
||||
writer,
|
||||
store: self.store.clone(),
|
||||
path,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn get(&self, key: &str) -> Result<BlobStream, String> {
|
||||
let path = self.object_path(key)?;
|
||||
let result = self
|
||||
.store
|
||||
.get(&path)
|
||||
.await
|
||||
.map_err(|e| format!("blob read failed: {e}"))?;
|
||||
Ok(Box::pin(
|
||||
result
|
||||
.into_stream()
|
||||
.map(|item| item.map_err(std::io::Error::other)),
|
||||
))
|
||||
}
|
||||
|
||||
async fn delete(&self, key: &str) -> Result<(), String> {
|
||||
let path = self.object_path(key)?;
|
||||
match self.store.delete(&path).await {
|
||||
Ok(()) => Ok(()),
|
||||
Err(object_store::Error::NotFound { .. }) => Ok(()),
|
||||
Err(e) => Err(format!("blob delete failed: {e}")),
|
||||
}
|
||||
}
|
||||
|
||||
async fn exists(&self, key: &str) -> Result<bool, String> {
|
||||
let path = self.object_path(key)?;
|
||||
match self.store.head(&path).await {
|
||||
Ok(_) => Ok(true),
|
||||
Err(object_store::Error::NotFound { .. }) => Ok(false),
|
||||
Err(e) => Err(format!("blob exists check failed: {e}")),
|
||||
}
|
||||
}
|
||||
|
||||
async fn health(&self) -> Result<(), String> {
|
||||
let key = format!("_health/{}", uuid::Uuid::new_v4());
|
||||
let mut upload = self.begin_upload(&key).await?;
|
||||
upload.write(Bytes::from_static(b"ok")).await?;
|
||||
let stat = upload.finish().await?;
|
||||
if stat.size != 2 {
|
||||
let _ = self.delete(&key).await;
|
||||
return Err("storage health probe returned unexpected size".to_string());
|
||||
}
|
||||
self.delete(&key).await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_prefix(prefix: &str) -> Result<String, String> {
|
||||
let trimmed = prefix.trim_matches('/');
|
||||
if trimmed.is_empty() {
|
||||
return Ok(String::new());
|
||||
}
|
||||
ObjectStorage::validate_logical_key(trimmed)?;
|
||||
Ok(trimmed.to_string())
|
||||
}
|
||||
|
||||
fn filesystem_storage(
|
||||
config: &AppConfig,
|
||||
storage: &StorageConfig,
|
||||
) -> Result<Arc<dyn BlobStorage>, String> {
|
||||
let path = storage
|
||||
.path
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.unwrap_or(&config.data_dir);
|
||||
let configured_root = PathBuf::from(path);
|
||||
let root = if configured_root.is_absolute() {
|
||||
configured_root
|
||||
} else {
|
||||
std::env::current_dir()
|
||||
.map_err(|e| format!("failed to resolve relative storage path: {e}"))?
|
||||
.join(configured_root)
|
||||
};
|
||||
std::fs::create_dir_all(&root).map_err(|e| {
|
||||
format!(
|
||||
"failed to create storage directory '{}': {e}",
|
||||
root.display()
|
||||
)
|
||||
})?;
|
||||
let store = LocalFileSystem::new_with_prefix(&root)
|
||||
.map_err(|e| format!("failed to initialize filesystem storage: {e}"))?;
|
||||
let prefix = normalize_prefix(&storage.prefix)?;
|
||||
Ok(Arc::new(ObjectStorage {
|
||||
store: Arc::new(store),
|
||||
prefix,
|
||||
}))
|
||||
}
|
||||
|
||||
fn s3_storage(storage: &StorageConfig) -> Result<Arc<dyn BlobStorage>, String> {
|
||||
let bucket = storage
|
||||
.bucket
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.ok_or_else(|| "storage.bucket is required for S3".to_string())?;
|
||||
let client_options = ClientOptions::new()
|
||||
.with_connect_timeout(Duration::from_secs(storage.connect_timeout_secs))
|
||||
.with_timeout(Duration::from_secs(storage.request_timeout_secs));
|
||||
let retry = RetryConfig {
|
||||
max_retries: storage.max_retries,
|
||||
retry_timeout: Duration::from_secs(storage.retry_timeout_secs),
|
||||
..RetryConfig::default()
|
||||
};
|
||||
let mut builder = AmazonS3Builder::from_env()
|
||||
.with_bucket_name(bucket)
|
||||
.with_region(storage.region.trim())
|
||||
.with_virtual_hosted_style_request(!storage.path_style)
|
||||
.with_allow_http(storage.allow_http)
|
||||
.with_client_options(client_options)
|
||||
.with_retry(retry)
|
||||
.with_checksum_algorithm(Checksum::SHA256);
|
||||
if let Some(endpoint) = storage
|
||||
.endpoint
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|v| !v.is_empty())
|
||||
{
|
||||
builder = builder.with_endpoint(endpoint);
|
||||
}
|
||||
if let Some(value) = storage
|
||||
.access_key_id
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|v| !v.is_empty())
|
||||
{
|
||||
builder = builder.with_access_key_id(value);
|
||||
}
|
||||
if let Some(value) = storage
|
||||
.secret_access_key
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|v| !v.is_empty())
|
||||
{
|
||||
builder = builder.with_secret_access_key(value);
|
||||
}
|
||||
if let Some(value) = storage
|
||||
.session_token
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|v| !v.is_empty())
|
||||
{
|
||||
builder = builder.with_token(value);
|
||||
}
|
||||
match storage
|
||||
.server_side_encryption
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|v| !v.is_empty())
|
||||
{
|
||||
Some("AES256") => {
|
||||
let key: object_store::aws::AmazonS3ConfigKey = "aws_server_side_encryption"
|
||||
.parse()
|
||||
.map_err(|e| format!("invalid S3 encryption config key: {e}"))?;
|
||||
builder = builder.with_config(key, "AES256");
|
||||
}
|
||||
Some("aws:kms") => {
|
||||
builder =
|
||||
builder.with_sse_kms_encryption(storage.kms_key_id.clone().unwrap_or_default());
|
||||
}
|
||||
Some(other) => {
|
||||
return Err(format!(
|
||||
"unsupported S3 server-side encryption mode: {other}"
|
||||
));
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
let store = builder
|
||||
.build()
|
||||
.map_err(|e| format!("failed to initialize S3 storage: {e}"))?;
|
||||
let prefix = normalize_prefix(&storage.prefix)?;
|
||||
Ok(Arc::new(ObjectStorage {
|
||||
store: Arc::new(store),
|
||||
prefix,
|
||||
}))
|
||||
}
|
||||
|
||||
pub fn from_config(config: &AppConfig) -> Result<Arc<dyn BlobStorage>, String> {
|
||||
match config.effective_storage_backend().as_str() {
|
||||
"filesystem" => filesystem_storage(config, &config.storage),
|
||||
"s3" => s3_storage(&config.storage),
|
||||
other => Err(format!("unsupported storage backend: {other}")),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn read_all(storage: &dyn BlobStorage, key: &str) -> Result<Vec<u8>, String> {
|
||||
let mut stream = storage.get(key).await?;
|
||||
let mut data = Vec::new();
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk.map_err(|e| format!("blob stream read failed: {e}"))?;
|
||||
data.extend_from_slice(&chunk);
|
||||
}
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
pub fn attachment_key(
|
||||
user_hash: &str,
|
||||
draft_id: &str,
|
||||
attachment_id: &str,
|
||||
) -> Result<String, String> {
|
||||
for value in [user_hash, draft_id, attachment_id] {
|
||||
if value.is_empty()
|
||||
|| !value
|
||||
.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_'))
|
||||
{
|
||||
return Err("invalid attachment key component".to_string());
|
||||
}
|
||||
}
|
||||
Ok(format!(
|
||||
"users/{user_hash}/drafts/{draft_id}/attachments/{attachment_id}"
|
||||
))
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct MigrationOptions {
|
||||
pub from_filesystem: PathBuf,
|
||||
pub dry_run: bool,
|
||||
pub verify: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct MigrationReport {
|
||||
pub scanned: usize,
|
||||
pub migrated: usize,
|
||||
pub skipped: usize,
|
||||
pub failed: usize,
|
||||
}
|
||||
|
||||
pub fn parse_migration_args(args: &[String]) -> Result<Option<MigrationOptions>, String> {
|
||||
if args.get(1).map(String::as_str) != Some("storage")
|
||||
|| args.get(2).map(String::as_str) != Some("migrate")
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
let mut from = None;
|
||||
let mut dry_run = false;
|
||||
let mut verify = false;
|
||||
let mut index = 3;
|
||||
while index < args.len() {
|
||||
match args[index].as_str() {
|
||||
"--from" => {
|
||||
index += 1;
|
||||
let value = args
|
||||
.get(index)
|
||||
.ok_or_else(|| "--from requires filesystem:/path".to_string())?;
|
||||
let path = value.strip_prefix("filesystem:").ok_or_else(|| {
|
||||
"--from currently supports filesystem:/absolute/path".to_string()
|
||||
})?;
|
||||
let path = PathBuf::from(path);
|
||||
if !path.is_absolute() {
|
||||
return Err(
|
||||
"storage migration source must be an absolute filesystem path".to_string(),
|
||||
);
|
||||
}
|
||||
from = Some(path);
|
||||
}
|
||||
"--to" => {
|
||||
index += 1;
|
||||
let value = args
|
||||
.get(index)
|
||||
.ok_or_else(|| "--to requires configured".to_string())?;
|
||||
if value != "configured" {
|
||||
return Err("--to currently accepts only 'configured'; configure [storage] for the destination backend".to_string());
|
||||
}
|
||||
}
|
||||
"--dry-run" => dry_run = true,
|
||||
"--verify" => verify = true,
|
||||
other => return Err(format!("unknown storage migrate argument: {other}")),
|
||||
}
|
||||
index += 1;
|
||||
}
|
||||
Ok(Some(MigrationOptions {
|
||||
from_filesystem: from.ok_or_else(|| {
|
||||
"storage migrate requires --from filesystem:/absolute/path".to_string()
|
||||
})?,
|
||||
dry_run,
|
||||
verify,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn checksum_stream(storage: &dyn BlobStorage, key: &str) -> Result<String, String> {
|
||||
use sha2::{Digest, Sha256};
|
||||
let mut stream = storage.get(key).await?;
|
||||
let mut hasher = Sha256::new();
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk.map_err(|e| format!("failed to verify migrated blob: {e}"))?;
|
||||
hasher.update(&chunk);
|
||||
}
|
||||
Ok(format!("{:x}", hasher.finalize()))
|
||||
}
|
||||
|
||||
fn migration_source_path(root: &FsPath, stored_key: &str) -> Result<PathBuf, String> {
|
||||
let candidate = if FsPath::new(stored_key).is_absolute() {
|
||||
PathBuf::from(stored_key)
|
||||
} else {
|
||||
ObjectStorage::validate_logical_key(stored_key)?;
|
||||
root.join(stored_key)
|
||||
};
|
||||
let canonical_root = root.canonicalize().map_err(|e| {
|
||||
format!(
|
||||
"failed to resolve migration source '{}': {e}",
|
||||
root.display()
|
||||
)
|
||||
})?;
|
||||
let canonical = candidate.canonicalize().map_err(|e| {
|
||||
format!(
|
||||
"failed to resolve source blob '{}': {e}",
|
||||
candidate.display()
|
||||
)
|
||||
})?;
|
||||
if !canonical.starts_with(&canonical_root) {
|
||||
return Err(format!(
|
||||
"source blob '{}' escapes migration root",
|
||||
candidate.display()
|
||||
));
|
||||
}
|
||||
Ok(canonical)
|
||||
}
|
||||
|
||||
pub async fn migrate_attachments(
|
||||
config: &AppConfig,
|
||||
destination: Arc<dyn BlobStorage>,
|
||||
options: &MigrationOptions,
|
||||
) -> Result<MigrationReport, String> {
|
||||
if !options.from_filesystem.is_dir() {
|
||||
return Err(format!(
|
||||
"storage migration source '{}' is not a directory",
|
||||
options.from_filesystem.display()
|
||||
));
|
||||
}
|
||||
let mut report = MigrationReport::default();
|
||||
for user_hash in crate::db::pool::list_account_user_hashes()? {
|
||||
let conn = crate::db::pool::open_user_db(&config.data_dir, &user_hash)?;
|
||||
for draft in crate::db::drafts::list_drafts(&conn)? {
|
||||
for attachment in crate::db::drafts::get_draft_attachments(&conn, &draft.id)? {
|
||||
report.scanned += 1;
|
||||
let target_key = attachment_key(&user_hash, &draft.id, &attachment.id)?;
|
||||
let source_path = match migration_source_path(
|
||||
&options.from_filesystem,
|
||||
&attachment.storage_key,
|
||||
) {
|
||||
Ok(path) => path,
|
||||
Err(error) => {
|
||||
// A logical DB key may already point at the configured destination after a previous run.
|
||||
if attachment.storage_key == target_key
|
||||
&& destination.exists(&target_key).await?
|
||||
{
|
||||
report.skipped += 1;
|
||||
println!("skip {target_key}: already migrated");
|
||||
continue;
|
||||
}
|
||||
report.failed += 1;
|
||||
eprintln!("fail {target_key}: {error}");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let source_meta = match tokio::fs::metadata(&source_path).await {
|
||||
Ok(meta) if meta.is_file() => meta,
|
||||
Ok(_) => {
|
||||
report.failed += 1;
|
||||
eprintln!("fail {target_key}: source is not a regular file");
|
||||
continue;
|
||||
}
|
||||
Err(error) => {
|
||||
report.failed += 1;
|
||||
eprintln!("fail {target_key}: source stat failed: {error}");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if options.dry_run {
|
||||
report.migrated += 1;
|
||||
println!(
|
||||
"dry-run {} -> {} ({} bytes)",
|
||||
source_path.display(),
|
||||
target_key,
|
||||
source_meta.len()
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut source = tokio::fs::File::open(&source_path).await.map_err(|e| {
|
||||
format!("failed to open source '{}': {e}", source_path.display())
|
||||
})?;
|
||||
let mut upload = destination.begin_upload(&target_key).await?;
|
||||
use sha2::Digest as _;
|
||||
let mut hasher = sha2::Sha256::new();
|
||||
let mut buffer = vec![0u8; 1024 * 1024];
|
||||
loop {
|
||||
let read = match source.read(&mut buffer).await {
|
||||
Ok(read) => read,
|
||||
Err(error) => {
|
||||
let _ = upload.abort().await;
|
||||
return Err(format!(
|
||||
"failed reading source '{}': {error}",
|
||||
source_path.display()
|
||||
));
|
||||
}
|
||||
};
|
||||
if read == 0 {
|
||||
break;
|
||||
}
|
||||
hasher.update(&buffer[..read]);
|
||||
if let Err(error) = upload.write(Bytes::copy_from_slice(&buffer[..read])).await
|
||||
{
|
||||
let _ = upload.abort().await;
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
let source_checksum = format!("{:x}", hasher.finalize());
|
||||
let target_stat = upload.finish().await?;
|
||||
if target_stat.size != source_meta.len() {
|
||||
let _ = destination.delete(&target_key).await;
|
||||
report.failed += 1;
|
||||
eprintln!("fail {target_key}: size verification failed");
|
||||
continue;
|
||||
}
|
||||
if options.verify {
|
||||
let target_checksum = match checksum_stream(destination.as_ref(), &target_key)
|
||||
.await
|
||||
{
|
||||
Ok(checksum) => checksum,
|
||||
Err(error) => {
|
||||
let _ = destination.delete(&target_key).await;
|
||||
report.failed += 1;
|
||||
eprintln!("fail {target_key}: checksum verification failed: {error}");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if target_checksum != source_checksum {
|
||||
let _ = destination.delete(&target_key).await;
|
||||
report.failed += 1;
|
||||
eprintln!("fail {target_key}: checksum verification failed");
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if let Err(error) = crate::db::drafts::update_attachment_storage(
|
||||
&conn,
|
||||
&attachment.id,
|
||||
&target_key,
|
||||
Some(&source_checksum),
|
||||
) {
|
||||
let _ = destination.delete(&target_key).await;
|
||||
return Err(error);
|
||||
}
|
||||
report.migrated += 1;
|
||||
println!("migrated {} -> {}", source_path.display(), target_key);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(report)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::AppConfig;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[tokio::test]
|
||||
async fn filesystem_backend_streams_round_trip() {
|
||||
let temp = TempDir::new().expect("temp dir");
|
||||
let config = AppConfig {
|
||||
data_dir: temp.path().to_string_lossy().into_owned(),
|
||||
storage: StorageConfig {
|
||||
backend: "filesystem".to_string(),
|
||||
path: Some(temp.path().join("blobs").to_string_lossy().into_owned()),
|
||||
..StorageConfig::default()
|
||||
},
|
||||
..AppConfig::default()
|
||||
};
|
||||
|
||||
let storage = from_config(&config).expect("filesystem storage");
|
||||
let key = attachment_key("user_1", "draft-1", "attachment-1").expect("key");
|
||||
let mut upload = storage.begin_upload(&key).await.expect("upload");
|
||||
upload
|
||||
.write(Bytes::from_static(b"hello "))
|
||||
.await
|
||||
.expect("chunk 1");
|
||||
upload
|
||||
.write(Bytes::from_static(b"world"))
|
||||
.await
|
||||
.expect("chunk 2");
|
||||
let stat = upload.finish().await.expect("finish");
|
||||
|
||||
assert_eq!(stat.size, 11);
|
||||
assert!(storage.exists(&key).await.expect("exists"));
|
||||
assert_eq!(
|
||||
read_all(storage.as_ref(), &key).await.expect("read"),
|
||||
b"hello world"
|
||||
);
|
||||
storage.delete(&key).await.expect("delete");
|
||||
assert!(!storage.exists(&key).await.expect("missing"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attachment_keys_are_logical_and_path_safe() {
|
||||
assert_eq!(
|
||||
attachment_key("abc123", "draft_1", "att-1").unwrap(),
|
||||
"users/abc123/drafts/draft_1/attachments/att-1"
|
||||
);
|
||||
assert!(attachment_key("../user", "draft", "att").is_err());
|
||||
assert!(attachment_key("user", "draft/other", "att").is_err());
|
||||
}
|
||||
}
|
||||
@@ -1094,4 +1094,3 @@ html[data-interface-skin="notion"][data-theme="dark"] {
|
||||
--field: #191919; --selected: #353535; --hover: #2b2b2b; --shadow: rgba(0, 0, 0, .38);
|
||||
--folder-bg: #202020; --message-bg: #202020; --reader-bg: #202020;
|
||||
}
|
||||
|
||||
@@ -162,4 +162,3 @@
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
@@ -344,4 +344,3 @@ function setBusy(button, busy) {
|
||||
button.disabled = busy;
|
||||
button.setAttribute("aria-busy", String(busy));
|
||||
}
|
||||
|
||||
@@ -690,4 +690,3 @@ async function logout() {
|
||||
localStorage.removeItem("rustbox.activeAccount");
|
||||
showLogin();
|
||||
}
|
||||
|
||||
@@ -450,4 +450,3 @@ async function bulkAddTag(tagId) {
|
||||
toast(`Label “${tag.name}” added`);
|
||||
} catch (error) { toast(error.message, "error"); }
|
||||
}
|
||||
|
||||
@@ -849,4 +849,3 @@ function closeReader() {
|
||||
$("#reader-empty").classList.remove("hidden");
|
||||
$("#reader-content").classList.add("hidden");
|
||||
}
|
||||
|
||||
@@ -1227,4 +1227,3 @@ async function sendMessage(event) {
|
||||
errorBox.classList.remove("hidden");
|
||||
} finally { setBusy(send, false); }
|
||||
}
|
||||
|
||||
@@ -1145,4 +1145,3 @@ async function createFilterFromCurrentMessage(kind = "sender") {
|
||||
dialog.showModal();
|
||||
setTimeout(() => value.focus(), 20);
|
||||
}
|
||||
|
||||
@@ -291,4 +291,3 @@ function composeToContact() {
|
||||
openCompose("new");
|
||||
$("#compose-to").value = email;
|
||||
}
|
||||
|
||||
@@ -266,4 +266,3 @@ async function deleteCalendarEvent() {
|
||||
try { await request(`/calendar/events/${encodeURIComponent(id)}`, { method: "DELETE" }); $("#calendar-form").classList.add("hidden"); await loadCalendar(); toast("Event deleted"); }
|
||||
catch (error) { toast(error.message, "error"); }
|
||||
}
|
||||
|
||||
@@ -554,4 +554,3 @@ async function autoDetectSpecialFolders() {
|
||||
}
|
||||
await saveSpecialFolders();
|
||||
}
|
||||
|
||||
@@ -465,4 +465,3 @@ async function rebuildSearchIndex() {
|
||||
button.disabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1022,4 +1022,3 @@ async function deleteSieve() {
|
||||
toast(`Sieve script “${name}” deleted`, "success");
|
||||
} catch (error) { toast(error.message, "error"); }
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user