544 lines
39 KiB
Python
Executable File
544 lines
39 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Static/offline release checks for RustBox 0.2.6.
|
|
|
|
These checks intentionally avoid pretending to replace `cargo test`: they validate
|
|
configuration, SQL migrations, Docker examples, query-catalog parity and release
|
|
metadata in environments where the Rust toolchain or external services are absent.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import pathlib
|
|
import re
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
import tomllib
|
|
|
|
try:
|
|
import yaml
|
|
except ImportError: # pragma: no cover - validation host dependency
|
|
yaml = None
|
|
|
|
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
|
errors: list[str] = []
|
|
checks: list[str] = []
|
|
|
|
|
|
def ok(message: str) -> None:
|
|
checks.append(message)
|
|
|
|
|
|
def require(condition: bool, message: str) -> None:
|
|
if condition:
|
|
ok(message)
|
|
else:
|
|
errors.append(message)
|
|
|
|
|
|
def read(rel: str) -> str:
|
|
return (ROOT / rel).read_text(encoding="utf-8")
|
|
|
|
|
|
def extract_section(text: str, begin: str, end: str) -> str:
|
|
require(begin in text and end in text, f"schema markers present: {begin} / {end}")
|
|
if begin not in text or end not in text:
|
|
return ""
|
|
return text.split(begin, 1)[1].split(end, 1)[0]
|
|
|
|
|
|
# Release/toolchain metadata.
|
|
cargo = tomllib.loads(read("Cargo.toml"))
|
|
require(cargo["package"]["version"] == "0.2.6", "Cargo package version is 0.2.6")
|
|
require(cargo["package"]["rust-version"] == "1.99.1", "Cargo rust-version is 1.99.1")
|
|
toolchain = tomllib.loads(read("rust-toolchain.toml"))
|
|
require(toolchain["toolchain"]["channel"] == "1.99.1", "rust-toolchain pins Rust 1.99.1")
|
|
require((ROOT / "Cargo.lock").is_file(), "Cargo.lock is retained for reproducible application builds")
|
|
lock = tomllib.loads(read("Cargo.lock"))
|
|
lock_rustbox = next((pkg for pkg in lock.get("package", []) if pkg.get("name") == "rustbox"), {})
|
|
require(lock_rustbox.get("version") == "0.2.6", "Cargo.lock RustBox package version is 0.2.6")
|
|
require("cargo generate-lockfile" in read("scripts/build-release.sh"), "release script generates Cargo.lock when missing")
|
|
release_script = read("scripts/build-release.sh")
|
|
release_script_flat = " ".join(release_script.replace("\\", " ").split())
|
|
require("cargo test --locked" in release_script_flat, "release script requires the full Rust test suite")
|
|
require("cargo clippy --locked --all-targets -- -D warnings" in release_script_flat, "release script requires warning-free Clippy")
|
|
require("cargo build --release --locked" in release_script_flat, "release script requires a locked release build")
|
|
package_script = read("scripts/package-release.sh")
|
|
require("./scripts/build-release.sh" in package_script, "release packaging runs the full build gate before creating artifacts")
|
|
require("make_zip.py" in package_script and "sha256sum" in package_script, "release packaging creates ZIP and SHA256 only after the build gate")
|
|
require("cargo generate-lockfile" in read(".github/workflows/ci.yml"), "CI regenerates Cargo.lock before --locked checks")
|
|
require("validate-release-static.py" in read("scripts/build-release.sh"), "release script runs offline/static release gate")
|
|
require("validate-release-static.py" in read(".github/workflows/ci.yml"), "CI runs offline/static release gate")
|
|
require(read("README.md").startswith("# RustBox 0.2.6"), "README identifies RustBox 0.2.6")
|
|
require(read("RELEASE-MANIFEST.txt").startswith("RustBox 0.2.6"), "release manifest identifies RustBox 0.2.6")
|
|
require(read("RELEASE-VALIDATION.txt").startswith("RustBox 0.2.6"), "release validation identifies RustBox 0.2.6")
|
|
require(read("docs/IMPLEMENTATION_STATUS.md").startswith("# RustBox 0.2.6"), "implementation status identifies RustBox 0.2.6")
|
|
require((ROOT / "docs/STORAGE_SEARCH_OPERATIONS.md").is_file(), "storage/search operations guide exists")
|
|
require('imap_host = ""' not in read("config.dev.toml"), "development config does not override IMAP host with an invalid empty value")
|
|
require(not re.search(r"(?m)^IMAP_HOST=\s*$", read(".env.example")), "environment example does not set an invalid empty IMAP host")
|
|
require("pgp_enabled: bool" in read("src/routes/auth.rs"), "auth policy exposes OpenPGP availability")
|
|
require("setOpenPgpAvailability" in read("src/web/js/72-openpgp-identities.js"), "UI hides administratively disabled OpenPGP")
|
|
storage_rs = read("src/storage/mod.rs")
|
|
require(
|
|
".with_allow_http(storage.allow_http)" in storage_rs.split("let client_options = ClientOptions::new()", 1)[1].split("let retry =", 1)[0],
|
|
"S3 allow_http is applied to the final ClientOptions",
|
|
)
|
|
require(
|
|
".with_allow_http(storage.allow_http)" not in storage_rs.split("let mut builder = AmazonS3Builder::from_env()", 1)[1].split("if let Some(endpoint)", 1)[0],
|
|
"S3 builder does not overwrite allow_http with replacement client options",
|
|
)
|
|
require(
|
|
"s3_http_endpoint_works_when_explicitly_allowed" in storage_rs,
|
|
"S3 HTTP regression test is present",
|
|
)
|
|
config_rs = read("src/config.rs")
|
|
routes_mod = read("src/routes/mod.rs")
|
|
require("pub sieve_enabled: bool" in config_rs and "pub diagnostics_enabled: bool" in config_rs and "pub about_enabled: bool" in config_rs, "configuration exposes Sieve, diagnostics and About feature gates")
|
|
require("pub security_headers_enabled: bool" in config_rs and "security_headers_enabled: false" in config_rs, "security headers are configurable and disabled by default")
|
|
require("if config.sieve_enabled" in routes_mod and "if config.diagnostics_enabled" in routes_mod, "disabled Sieve and diagnostics routes are not registered")
|
|
require(".fallback(|| async { axum::http::StatusCode::NOT_FOUND })" in routes_mod, "API namespace has an explicit 404 fallback before the SPA fallback")
|
|
require("if config.security_headers_enabled" in routes_mod and "add_security_headers" in routes_mod, "security-header middleware is opt-in")
|
|
require('.route("/settings/reset", post(settings::reset_all_settings))' in routes_mod, "authenticated settings reset route is registered")
|
|
require("db::settings::reset_all" in read("src/routes/settings.rs"), "settings reset route delegates preference reset to the DB layer")
|
|
require('$("#quick-filter-btn")?.classList.toggle("hidden", state.authPolicy.sieveEnabled === false)' in read("src/web/js/20-preferences-auth.js"), "Sieve quick-filter UI disappears when Sieve is disabled")
|
|
require("state.authPolicy.sieveEnabled === false ? []" in read("src/web/js/60-mail-actions-search.js"), "Sieve context-menu actions disappear when Sieve is disabled")
|
|
mail_actions_js = read("src/web/js/60-mail-actions-search.js")
|
|
require("function nextMessageTargetAfterRemoval(folder, uid)" in mail_actions_js, "reader computes the adjacent message before removing the current item")
|
|
require("state.messages[index + 1] || state.messages[index - 1]" in mail_actions_js, "reader prefers the next message and falls back to the previous item")
|
|
require(mail_actions_js.count("await removeCurrentMessageFromView(") >= 2, "move and permanent-delete flows share automatic reader advancement")
|
|
require("if (nextTarget) await openMessage(nextTarget.folder, nextTarget.uid);" in mail_actions_js, "reader opens the adjacent message after current-message removal")
|
|
require("is_opensearch_backpressure_error" in read("src/search/engine.rs"), "OpenSearch worker classifies backpressure failures")
|
|
require("worker pass failed; backing off" in read("src/search/engine.rs"), "OpenSearch worker backs off after failures")
|
|
require("ENVIRONMENT: ${ENVIRONMENT:-development}" in read("docker/compose-postgres-s3-opensearch.yml"), "PostgreSQL Compose defaults direct HTTP to development cookie mode")
|
|
session_db = read("src/session_store/database.rs")
|
|
db_pool = read("src/db/pool.rs")
|
|
require("crate::db::pool::configure(config).await?" in session_db, "database session store delegates schema initialization to db::pool::configure")
|
|
require("global_schema_for_backend" not in session_db, "database session store does not run a second backend schema pass")
|
|
require("sqlx::raw_sql(&schema)" not in session_db and "execute_batch(&schema)" not in session_db, "session store avoids duplicate schema DDL for SQLite/MySQL/PostgreSQL")
|
|
require("SET LOCAL client_min_messages TO WARNING" in db_pool, "PostgreSQL idempotent startup DDL suppresses expected NOTICE noise locally")
|
|
require("Executor as _" in db_pool, "PostgreSQL startup imports SQLx Executor for direct raw DDL execution")
|
|
require(".execute(schema.as_str())" in db_pool, "PostgreSQL startup executes schema DDL through Executor instead of raw_sql")
|
|
require(not re.search(r"raw_sql\(&schema\)\s*\.execute\(&mut \*tx\)", db_pool), "PostgreSQL startup avoids raw_sql mutable-connection lifetime regression")
|
|
require("sqlite_test_configuration_does_not_pin_deleted_tempdir" in db_pool, "SQLite test isolation has a regression test for deleted TempDir reuse")
|
|
require("let config = AppConfig {" in db_pool and "let mut config = AppConfig::default();" not in db_pool, "SQLite isolation regression test is Clippy-clean with struct initialization")
|
|
sqlite_configure = db_pool.split("DatabaseBackend::Sqlite => {", 1)[1].split("DatabaseBackend::Mysql => {", 1)[0]
|
|
require("#[cfg(test)]" in sqlite_configure and "return Ok(());" in sqlite_configure and "#[cfg(not(test))]" in sqlite_configure, "SQLite test configuration does not publish a temporary database path globally")
|
|
require("RUST_LOG: ${RUST_LOG:-info,sqlx::postgres::notice=warn}" not in read("docker/compose-postgres-s3-opensearch.yml"), "PostgreSQL Compose no longer depends on a notice-filter workaround")
|
|
pgsql_queries = read("src/queries/pgsql/queries.rs")
|
|
require("ON CONFLICT(key) DO UPDATE SET value = CAST(user_meta.value AS INTEGER) + 1" in pgsql_queries, "PostgreSQL header-sync UPSERT qualifies the existing target value")
|
|
require("tenant_rewriter_prefixes_header_sync_target_qualifier" in db_pool, "tenant SQL rewriter has a regression test for qualified header-sync value")
|
|
mysql_queries = read("src/queries/mysql/queries.rs")
|
|
require("ON DUPLICATE KEY UPDATE value = CAST(value AS SIGNED) + 1" in mysql_queries, "MySQL header-sync uses its non-EXCLUDED duplicate-key form")
|
|
|
|
|
|
# PostgreSQL UPSERT audit: any read of an INSERT target column inside
|
|
# ON CONFLICT ... DO UPDATE must be qualified as either excluded.<column>
|
|
# or <target_table>.<column>. PostgreSQL otherwise treats names such as
|
|
# `value` as ambiguous between the existing target row and EXCLUDED.
|
|
def postgres_upsert_ambiguities(source: str) -> list[str]:
|
|
const_re = re.compile(
|
|
r'pub const\s+(\w+)\s*:\s*&str\s*=\s*(r#".*?"#|"(?:\\.|[^"\\])*")\s*;',
|
|
re.S,
|
|
)
|
|
problems: list[str] = []
|
|
for const_name, literal in const_re.findall(source):
|
|
if literal.startswith('r#"'):
|
|
sql = literal[3:-2]
|
|
else:
|
|
sql = bytes(literal[1:-1], 'utf-8').decode('unicode_escape')
|
|
if not re.search(r'ON\s+CONFLICT.*DO\s+UPDATE\s+SET', sql, re.I | re.S):
|
|
continue
|
|
insert = re.search(
|
|
r'INSERT\s+INTO\s+([A-Za-z_][\w]*)\s*\((.*?)\)\s*VALUES',
|
|
sql,
|
|
re.I | re.S,
|
|
)
|
|
if insert is None:
|
|
problems.append(f'{const_name}: unable to parse INSERT target')
|
|
continue
|
|
table = insert.group(1)
|
|
columns = [column.strip().strip('"') for column in insert.group(2).split(',')]
|
|
update = re.split(r'DO\s+UPDATE\s+SET', sql, maxsplit=1, flags=re.I)[1]
|
|
scan = re.sub(r"'(?:''|[^'])*'", '', update)
|
|
for column in columns:
|
|
candidate = re.sub(
|
|
rf'\b(?:excluded|{re.escape(table)})\s*\.\s*{re.escape(column)}\b',
|
|
'',
|
|
scan,
|
|
flags=re.I,
|
|
)
|
|
candidate = re.sub(
|
|
rf'(?:(?<=^)|(?<=,))\s*{re.escape(column)}\s*=',
|
|
'=',
|
|
candidate,
|
|
flags=re.I,
|
|
)
|
|
if re.search(rf'(?<!\.)\b{re.escape(column)}\b', candidate, re.I):
|
|
problems.append(f'{const_name}: unqualified target read `{column}` in {table} UPSERT')
|
|
return problems
|
|
|
|
pgsql_upsert_ambiguities = postgres_upsert_ambiguities(pgsql_queries)
|
|
require(
|
|
not pgsql_upsert_ambiguities,
|
|
'PostgreSQL ON CONFLICT target-column reads are fully qualified'
|
|
+ (f" ({'; '.join(pgsql_upsert_ambiguities)})" if pgsql_upsert_ambiguities else ''),
|
|
)
|
|
|
|
# Parse all maintained TOML examples.
|
|
toml_files = [
|
|
"Cargo.toml",
|
|
"rust-toolchain.toml",
|
|
"config.example.toml",
|
|
"config.dev.toml",
|
|
"packaging/rustbox.toml",
|
|
"docker/configs/single-sqlite.toml",
|
|
"docker/configs/postgres-s3-opensearch.toml",
|
|
"docker/configs/mysql-nfs-opensearch.toml",
|
|
"docker/configs/ha-garage.toml",
|
|
]
|
|
for rel in toml_files:
|
|
try:
|
|
tomllib.loads(read(rel))
|
|
ok(f"TOML parses: {rel}")
|
|
except Exception as exc:
|
|
errors.append(f"TOML parse failed: {rel}: {exc}")
|
|
|
|
# Parse Docker Compose YAML and validate referenced config paths.
|
|
compose_files = sorted((ROOT / "docker").glob("compose-*.yml"))
|
|
require(len(compose_files) >= 4, "four Docker Compose deployment variants exist")
|
|
if yaml is None:
|
|
errors.append("PyYAML is required for Docker Compose static validation")
|
|
else:
|
|
for path in compose_files:
|
|
try:
|
|
data = yaml.safe_load(path.read_text(encoding="utf-8"))
|
|
require(isinstance(data, dict) and "services" in data, f"Compose parses with services: {path.name}")
|
|
services = data.get("services", {})
|
|
for service_name, service in services.items():
|
|
if service_name != "rustbox" and not service_name.startswith("rustbox-"):
|
|
continue
|
|
for volume in service.get("volumes", []) or []:
|
|
if isinstance(volume, str) and volume.startswith("./configs/"):
|
|
config_rel = volume.split(":", 1)[0].removeprefix("./")
|
|
require((ROOT / "docker" / config_rel).is_file(), f"Compose config mount exists: {path.name}/{service_name} -> {config_rel}")
|
|
except Exception as exc:
|
|
errors.append(f"Compose parse failed: {path.name}: {exc}")
|
|
|
|
single = tomllib.loads(read("docker/configs/single-sqlite.toml"))
|
|
pg_s3 = tomllib.loads(read("docker/configs/postgres-s3-opensearch.toml"))
|
|
my_nfs = tomllib.loads(read("docker/configs/mysql-nfs-opensearch.toml"))
|
|
require(single["cluster_mode"] is False and single["storage"]["backend"] == "filesystem", "single-node Compose uses local filesystem storage")
|
|
require(pg_s3["cluster_mode"] is True and "data_dir" not in pg_s3, "PostgreSQL/S3 cluster config omits data_dir")
|
|
require(pg_s3["storage"]["backend"] == "s3" and pg_s3["search"]["backend"] == "opensearch", "PostgreSQL cluster uses S3 + OpenSearch")
|
|
require(my_nfs["cluster_mode"] is True and "data_dir" not in my_nfs, "MySQL/NFS cluster config omits data_dir")
|
|
require(my_nfs["storage"]["backend"] == "filesystem" and my_nfs["storage"]["shared"] is True, "MySQL cluster marks POSIX/NFS storage shared")
|
|
require(my_nfs["search"]["backend"] == "opensearch", "MySQL/NFS cluster uses OpenSearch")
|
|
ha = tomllib.loads(read("docker/configs/ha-garage.toml"))
|
|
require(ha["cluster_mode"] is True and ha["session_backend"] == "redis", "HA/Garage config enables cluster mode with Redis-compatible sessions")
|
|
require(ha["redis_url"] == "redis://valkey:6379/", "HA/Garage config targets Valkey for shared sessions")
|
|
require(ha["storage"]["backend"] == "s3" and ha["storage"]["endpoint"] == "http://garage:3900", "HA/Garage config stores blobs in Garage S3")
|
|
require(ha["search"]["backend"] == "opensearch", "HA/Garage config uses OpenSearch")
|
|
dockerfile = read("docker/Dockerfile")
|
|
require("FROM rust:1.99.1-trixie AS builder" in dockerfile, "Docker builder pins Rust 1.99.1 on Debian Trixie")
|
|
require("FROM debian:trixie-slim" in dockerfile, "Docker runtime uses Debian Trixie slim")
|
|
require("libssl3t64" in dockerfile and "libssl3 " not in dockerfile, "Docker runtime uses the Debian Trixie OpenSSL runtime package")
|
|
require("bookworm" not in dockerfile.lower(), "RustBox Dockerfile no longer uses Debian Bookworm")
|
|
require("postgres:17-trixie" in read("docker/compose-postgres-s3-opensearch.yml"), "PostgreSQL/S3 Compose uses PostgreSQL 17 Trixie")
|
|
require("bookworm" not in read("docker/compose-postgres-s3-opensearch.yml").lower(), "PostgreSQL/S3 Compose no longer uses Bookworm")
|
|
ha_compose = read("docker/compose-ha-garage.yml")
|
|
require("postgres:17-trixie" in ha_compose, "HA Compose uses PostgreSQL 17 Trixie")
|
|
require("bookworm" not in ha_compose.lower(), "HA Compose no longer uses Bookworm")
|
|
require("rustbox-1:" in ha_compose and "rustbox-2:" in ha_compose, "HA Compose defines two RustBox application nodes")
|
|
for service in ["postgres:", "valkey:", "opensearch:", "garage:", "envoy:"]:
|
|
require(service in ha_compose, f"HA Compose includes {service.removesuffix(':')}")
|
|
require("valkey/valkey:9.1.3-alpine" in ha_compose, "HA Compose pins Valkey 9.1.3")
|
|
require("opensearchproject/opensearch:3.9.0" in ha_compose, "HA Compose pins OpenSearch 3.9.0")
|
|
require("dxflrs/garage:v2.5.0" in ha_compose, "HA Compose pins Garage 2.5.0")
|
|
require("envoyproxy/envoy:v1.40.2" in ha_compose, "HA Compose pins Envoy 1.40.2")
|
|
require("X-RustBox-Backend" in read("docker/envoy/envoy.yaml"), "Envoy exposes the selected RustBox backend response header")
|
|
require("address: rustbox-1" in read("docker/envoy/envoy.yaml") and "address: rustbox-2" in read("docker/envoy/envoy.yaml"), "Envoy balances both RustBox nodes")
|
|
require('sieve_enabled = false' in read("docker/configs/ha-garage.toml"), "HA example disables optional Sieve by default")
|
|
require('security_headers_enabled = false' in read("docker/configs/ha-garage.toml"), "HA example keeps optional security headers disabled by default")
|
|
require("type: nfs" in read("docker/compose-mysql-nfs-opensearch.yml"), "NFS Compose uses a real NFS volume driver")
|
|
require("docker.io/pgsty/silo:RELEASE.2026-10-26T00-00-00Z" in read("docker/compose-postgres-s3-opensearch.yml"), "S3 Compose pins the maintained SILO integration image")
|
|
require("docker.io/pgsty/mc:RELEASE.2026-10-26T00-00-00Z" in read("docker/compose-postgres-s3-opensearch.yml"), "S3 Compose pins the matching SILO client image")
|
|
require('test: ["CMD", "silo", "healthcheck", "ready"]' in read("docker/compose-postgres-s3-opensearch.yml"), "S3 Compose waits on SILO readiness before bucket initialization")
|
|
require("endpoint = \"http://s3:9000\"" in read("docker/configs/postgres-s3-opensearch.toml"), "S3 config targets the generic Compose S3 service")
|
|
for rel in ["docker/compose-postgres-s3-opensearch.yml", "docker/compose-mysql-nfs-opensearch.yml"]:
|
|
require("opensearchproject/opensearch:3.9.0" in read(rel), f"Compose pins OpenSearch 3.9.0: {rel}")
|
|
require("opensearchproject/opensearch:latest" not in read(rel), f"Compose avoids moving OpenSearch latest tag: {rel}")
|
|
require('DISABLE_INSTALL_DEMO_CONFIG: "true"' in read(rel), f"Compose disables OpenSearch demo security bootstrap: {rel}")
|
|
require('DISABLE_SECURITY_PLUGIN: "true"' in read(rel), f"Compose explicitly disables OpenSearch security plugin for local integration: {rel}")
|
|
|
|
for rel in ["docker/compose-postgres-s3-opensearch.yml", "docker/compose-mysql-nfs-opensearch.yml"]:
|
|
match = re.search(r"SESSION_ENCRYPTION_KEY:\s*\$\{SESSION_ENCRYPTION_KEY:-([^}]+)\}", read(rel))
|
|
require(match is not None, f"Compose has SESSION_ENCRYPTION_KEY fallback: {rel}")
|
|
if match:
|
|
try:
|
|
decoded = base64.b64decode(match.group(1), validate=True)
|
|
require(len(decoded) == 32, f"Compose fallback session key decodes to 32 bytes: {rel}")
|
|
except Exception as exc:
|
|
errors.append(f"invalid Compose fallback session key in {rel}: {exc}")
|
|
|
|
# SQLite fresh-schema execution.
|
|
schema = read("migrations/sqlite/schema.sql")
|
|
global_sql = extract_section(schema, "-- RUSTBOX:GLOBAL-BEGIN", "-- RUSTBOX:GLOBAL-END")
|
|
tenant_sql = extract_section(schema, "-- RUSTBOX:TENANT-BEGIN", "-- RUSTBOX:TENANT-END").replace("{{p}}", "u_test__")
|
|
try:
|
|
conn = sqlite3.connect(":memory:")
|
|
conn.execute("PRAGMA foreign_keys=ON")
|
|
conn.executescript(global_sql)
|
|
conn.executescript(tenant_sql)
|
|
columns = {row[1] for row in conn.execute("PRAGMA table_info(u_test__draft_attachments)")}
|
|
require("checksum_sha256" in columns, "fresh SQLite schema has attachment checksum metadata")
|
|
outbox = conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='u_test__search_outbox'").fetchone()
|
|
require(outbox is not None, "fresh SQLite schema has search_outbox")
|
|
outbox_columns = {row[1] for row in conn.execute("PRAGMA table_info(u_test__search_outbox)")}
|
|
require("claimed_until" in outbox_columns, "fresh SQLite search_outbox separates worker lease from retry availability")
|
|
sqlite_header_sync = "INSERT INTO u_test__user_meta (key, value) VALUES ('header_sync_generation', '1') ON CONFLICT(key) DO UPDATE SET value = CAST(value AS INTEGER) + 1"
|
|
conn.execute(sqlite_header_sync)
|
|
conn.execute(sqlite_header_sync)
|
|
sqlite_generation = conn.execute("SELECT value FROM u_test__user_meta WHERE key='header_sync_generation'").fetchone()
|
|
require(sqlite_generation == ("2",), "SQLite header-sync UPSERT remains unambiguous and increments generation")
|
|
account_default = conn.execute("PRAGMA table_info(rustbox_accounts)").fetchall()
|
|
version_default = next((row[4] for row in account_default if row[1] == "schema_version"), None)
|
|
require(str(version_default).strip("'\"") == "46", "fresh global compatibility baseline remains V046 before tenant registration")
|
|
conn.close()
|
|
except Exception as exc:
|
|
errors.append(f"fresh SQLite schema execution failed: {exc}")
|
|
|
|
# V047 upgrade execution from a minimal V046 attachment table, including data preservation.
|
|
try:
|
|
conn = sqlite3.connect(":memory:")
|
|
conn.executescript(
|
|
"CREATE TABLE u_test__draft_attachments ("
|
|
"id TEXT PRIMARY KEY, draft_id TEXT NOT NULL, filename TEXT NOT NULL, "
|
|
"content_type TEXT NOT NULL, size INTEGER NOT NULL, file_path TEXT NOT NULL, "
|
|
"created_at TEXT NOT NULL);"
|
|
"INSERT INTO u_test__draft_attachments "
|
|
"(id, draft_id, filename, content_type, size, file_path, created_at) VALUES "
|
|
"('att-1', 'draft-1', 'report.pdf', 'application/pdf', 1234, '/legacy/report.pdf', '2026-01-01T00:00:00Z');"
|
|
)
|
|
migration = read("migrations/sqlite/V047__blob_keys_and_search_outbox.sql").replace("{{p}}", "u_test__")
|
|
conn.executescript(migration)
|
|
columns = {row[1] for row in conn.execute("PRAGMA table_info(u_test__draft_attachments)")}
|
|
require("checksum_sha256" in columns, "V047 adds checksum_sha256 to SQLite attachments")
|
|
row = conn.execute(
|
|
"SELECT id, draft_id, filename, content_type, size, file_path, created_at, checksum_sha256 "
|
|
"FROM u_test__draft_attachments WHERE id='att-1'"
|
|
).fetchone()
|
|
require(
|
|
row == ('att-1', 'draft-1', 'report.pdf', 'application/pdf', 1234, '/legacy/report.pdf', '2026-01-01T00:00:00Z', None),
|
|
"V047 preserves existing attachment rows and initializes checksum as NULL",
|
|
)
|
|
require(conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='u_test__search_outbox'").fetchone() is not None, "V047 creates SQLite search_outbox")
|
|
outbox_columns = {row[1] for row in conn.execute("PRAGMA table_info(u_test__search_outbox)")}
|
|
require("claimed_until" in outbox_columns, "V047 creates an independent SQLite outbox lease column")
|
|
conn.close()
|
|
except Exception as exc:
|
|
errors.append(f"SQLite V047 migration execution failed: {exc}")
|
|
|
|
# Migration/query parity across SQL engines.
|
|
for dialect in ("sqlite", "mysql", "pgsql"):
|
|
require((ROOT / f"migrations/{dialect}/V047__blob_keys_and_search_outbox.sql").is_file(), f"V047 migration exists for {dialect}")
|
|
queries = read(f"src/queries/{dialect}/queries.rs")
|
|
block_match = re.search(r"pub mod db_search_outbox\s*\{(.*?)\n\}", queries, re.S)
|
|
require(block_match is not None, f"search outbox query module exists for {dialect}")
|
|
if block_match:
|
|
qnames = set(re.findall(r"pub const (Q_\d{3})", block_match.group(1)))
|
|
require(qnames == {f"Q_{i:03d}" for i in range(1, 8)}, f"search outbox Q001-Q007 parity for {dialect}")
|
|
|
|
pool = read("src/db/pool.rs")
|
|
for dialect in ("SQLITE", "MYSQL", "PGSQL"):
|
|
require(f"{dialect}_MIGRATIONS" in pool and "V047__blob_keys_and_search_outbox.sql" in pool, f"runtime migration catalog includes V047 ({dialect.lower()})")
|
|
require("const BASE_SCHEMA_VERSION: u32 = 46" in pool, "legacy squashed baseline stays V046")
|
|
require("latest_schema_version" in pool, "runtime schema version is computed from migrations")
|
|
require(
|
|
"latest_supported_sqlite = SQLITE_MIGRATIONS" in pool
|
|
and "if version > latest_supported_sqlite" in pool,
|
|
"legacy SQLite import accepts runtime migrations up to this build instead of stopping at V046",
|
|
)
|
|
|
|
# Tenant/import table parity and transaction-bound mutation helpers.
|
|
def const_string_array(text: str, name: str) -> list[str]:
|
|
match = re.search(rf"const {name}: &\[&str\] = &\[(.*?)\];", text, re.S)
|
|
return re.findall(r'"([^"\n]+)"', match.group(1)) if match else []
|
|
|
|
tenant_tables = const_string_array(pool, "TENANT_TABLES")
|
|
import_tables = const_string_array(pool, "IMPORT_TABLES")
|
|
require(
|
|
tenant_tables and sorted(tenant_tables) == sorted(import_tables),
|
|
"tenant and legacy-import table sets stay in sync",
|
|
)
|
|
messages_rs = read("src/db/messages.rs")
|
|
routes_messages_rs = read("src/routes/messages/mod.rs")
|
|
require(
|
|
"pub fn remove_after_mutation_in_transaction(" in messages_rs
|
|
and "remove_after_mutation_in_transaction(&transaction, folder, uid)?;" in messages_rs
|
|
and "db::messages::remove_after_mutation_in_transaction(" in routes_messages_rs
|
|
and "db::messages::remove_after_mutation(&transaction" not in routes_messages_rs,
|
|
"move mutation reuses its caller transaction instead of nesting a second transaction",
|
|
)
|
|
|
|
# Exercise the actual SQLite search-outbox query constants against the current tenant schema.
|
|
try:
|
|
sqlite_queries = read("src/queries/sqlite/queries.rs")
|
|
outbox_block = re.search(r"pub mod db_search_outbox\s*\{(.*?)\n\}", sqlite_queries, re.S)
|
|
if outbox_block is None:
|
|
raise RuntimeError("db_search_outbox module not found")
|
|
queries = {
|
|
name: sql.replace("search_outbox", "u_test__search_outbox")
|
|
for name, sql in re.findall(r'pub const (Q_\d{3}): &str = "([^"]+)";', outbox_block.group(1))
|
|
}
|
|
conn = sqlite3.connect(":memory:")
|
|
conn.execute("PRAGMA foreign_keys=ON")
|
|
conn.executescript(tenant_sql)
|
|
conn.execute(queries["Q_001"], ("doc-1", "gen-1", "upsert", 7, "INBOX", "{}", 1000))
|
|
ready = conn.execute(queries["Q_002"], (1000, 10)).fetchall()
|
|
require(len(ready) == 1 and ready[0][0:3] == ("doc-1", "gen-1", "upsert"), "SQLite search outbox enqueue/list query roundtrip works")
|
|
first_claim = conn.execute(queries["Q_006"], ("doc-1", "gen-1", 1060, 1000)).rowcount
|
|
second_claim = conn.execute(queries["Q_006"], ("doc-1", "gen-1", 1060, 1000)).rowcount
|
|
require(first_claim == 1 and second_claim == 0, "SQLite search outbox lease claim is exclusive")
|
|
conn.execute(queries["Q_001"], ("doc-1", "gen-2", "delete", 7, "INBOX", None, 1001))
|
|
requeued = conn.execute(
|
|
"SELECT generation, event_type, attempts, available_at, claimed_until, dead_letter, created_at, updated_at "
|
|
"FROM u_test__search_outbox WHERE id='doc-1'"
|
|
).fetchone()
|
|
require(
|
|
requeued == ("gen-2", "delete", 0, 0, 1060, 0, 1001, 1001),
|
|
"SQLite search outbox re-enqueue resets retry state while preserving an active worker lease",
|
|
)
|
|
require(
|
|
conn.execute(queries["Q_002"], (1001, 10)).fetchall() == [],
|
|
"SQLite replacement event stays hidden until the active lease expires",
|
|
)
|
|
stale_delete = conn.execute(queries["Q_003"], ("doc-1", "gen-1")).rowcount
|
|
current_delete = conn.execute(queries["Q_003"], ("doc-1", "gen-2")).rowcount
|
|
require(stale_delete == 0 and current_delete == 1, "SQLite search outbox generation guard rejects stale acknowledgements")
|
|
conn.execute(queries["Q_001"], ("doc-2", "gen-3", "delete", 9, "Trash", None, 1002))
|
|
failure_update = conn.execute(queries["Q_004"], ("doc-2", "gen-3", 1, 2000, 1, "boom", 1003)).rowcount
|
|
stats = conn.execute(queries["Q_005"]).fetchone()
|
|
claimed_after_failure = conn.execute("SELECT claimed_until FROM u_test__search_outbox WHERE id='doc-2'").fetchone()[0]
|
|
require(failure_update == 1 and stats[0] == 1 and stats[1] == 1 and claimed_after_failure == 0, "SQLite search outbox retry/dead-letter resets worker lease and updates stats")
|
|
conn.execute(queries["Q_001"], ("doc-old", "gen-old", "delete", 10, "Archive", None, 900))
|
|
cleared = conn.execute(queries["Q_007"], (1000,)).rowcount
|
|
require(cleared == 1, "SQLite search outbox can clear pre-reindex superseded events")
|
|
conn.close()
|
|
except Exception as exc:
|
|
errors.append(f"SQLite search-outbox query execution failed: {exc}")
|
|
|
|
# Storage/search architecture invariants.
|
|
config_rs = read("src/config.rs")
|
|
storage_rs = read("src/storage/mod.rs")
|
|
search_rs = read("src/search/engine.rs")
|
|
require("if self.cluster_mode && !data_dir_explicit" in config_rs and "self.data_dir.clear()" in config_rs, "implicit data_dir is disabled in cluster mode")
|
|
require("cluster_shared_data_dir" not in config_rs and "search_index_backend" not in config_rs, "legacy top-level storage/search switches are removed")
|
|
require('self.storage.backend.trim().to_ascii_lowercase()' in config_rs and 'self.search.backend.trim().to_ascii_lowercase()' in config_rs, "storage/search backends come only from their dedicated config sections")
|
|
require('"tantivy" => "local_tantivy"' not in config_rs, "legacy Tantivy backend alias is removed")
|
|
require('"postgres"' not in config_rs.split("#[cfg(test)]", 1)[0] and '"postgresql"' not in config_rs.split("#[cfg(test)]", 1)[0] and '"postgres"' not in read("src/db/pool.rs").split("#[cfg(test)]", 1)[0] and '"postgresql"' not in read("src/db/pool.rs").split("#[cfg(test)]", 1)[0], "database backend uses canonical sqlite/mysql/pgsql names only")
|
|
require("cluster_shared_data_dir" not in read("config.example.toml") and "search_index_backend" not in read("config.example.toml"), "example config contains no pre-0.2 compatibility switches")
|
|
require("legacy_filesystem_root" not in storage_rs and "legacy_path(" not in storage_rs, "blob storage has no legacy attachment read-through path")
|
|
require('"filesystem" | "s3"' in config_rs or '"filesystem", "s3"' in config_rs, "configuration accepts filesystem and S3 storage")
|
|
require("storage.shared" in config_rs, "cluster-safe shared POSIX storage is capability-gated")
|
|
require("AmazonS3Builder" in storage_rs and "BufWriter" in storage_rs, "S3 backend uses streaming object-store writer")
|
|
require("with_retry" in storage_rs and "with_checksum_algorithm" in storage_rs, "S3 backend configures retries and checksums")
|
|
require("attachment_key" in storage_rs and "users/{user_hash}/drafts/{draft_id}/attachments/{attachment_id}" in storage_rs, "attachments use logical storage keys")
|
|
require("destination.delete(&target_key).await" in storage_rs, "attachment migration removes a failed destination blob after verification failure")
|
|
require("let _ = upload.abort().await" in storage_rs and "failed reading source" in storage_rs, "attachment migration aborts a destination upload if source streaming fails")
|
|
require("if let Err(error) = crate::db::drafts::update_attachment_storage" in storage_rs, "attachment migration removes destination object when SQL pointer update fails")
|
|
require("OpenSearchBackend" in search_rs, "OpenSearch backend is implemented")
|
|
require("reqwest::blocking" not in search_rs and 'features = ["json", "rustls-tls"]' in read("Cargo.toml"), "OpenSearch uses the async reqwest client without blocking feature")
|
|
require("async fn request(" in search_rs and "request.send().await" in search_rs, "OpenSearch HTTP requests are asynchronous")
|
|
require('json!({"term": {"user_hash": user_hash}})' in search_rs, "OpenSearch queries enforce server-side tenant filter")
|
|
require("process_search_outbox" in search_rs and "claim(&conn" in search_rs, "OpenSearch worker uses durable claimed outbox events")
|
|
require("fn event_lease_secs(&self) -> u64" in search_rs and "let lease_secs = backend.event_lease_secs();" in search_rs and "claim(&conn, &event, 60)" not in search_rs, "OpenSearch outbox lease is derived from HTTP timeout/failover budget instead of a fixed 60 seconds")
|
|
require("opensearch_event_lease_covers_timeout_and_failover_budget" in search_rs, "OpenSearch lease budget has a source-level unit regression test")
|
|
require("claimed_until" in read("migrations/sqlite/V047__blob_keys_and_search_outbox.sql") and "reenqueue_preserves_active_claim_lease" in read("src/db/search_outbox.rs"), "outbox re-enqueue cannot steal an active worker lease")
|
|
require("index_messages_batch_with_connection" in search_rs and "enqueue_upsert_on" in search_rs, "OpenSearch outbox supports caller-owned SQL transactions")
|
|
require("enqueue_participates_in_caller_transaction" in read("src/db/search_outbox.rs"), "transactional outbox rollback has a unit test")
|
|
require("search reindex" in read("README.md") and "run_search_reindex" in read("src/main.rs"), "full search reindex command is documented and wired")
|
|
require("clear_before(&conn, reindex_started_at)" in read("src/main.rs") and "pub fn clear_before(" in read("src/db/search_outbox.rs"), "full reindex removes only superseded pre-reindex outbox events")
|
|
require("quiesce search workers" in read("docs/STORAGE_SEARCH_OPERATIONS.md"), "cluster reindex documentation requires a controlled maintenance window")
|
|
require("self.backend.refresh().await?" in search_rs, "OpenSearch full reindex forces refresh before reporting completion")
|
|
require("pub async fn document_count(&self) -> Result<usize, String>" in search_rs, "search document-count diagnostics propagate backend errors")
|
|
require(".document_count()\n .await\n .map_err(AppError::InternalError)?" in read("src/routes/diagnostics.rs"), "diagnostics route does not convert OpenSearch count failures into zero")
|
|
idle_rs = read("src/realtime/idle.rs")
|
|
require("fn delete_cached_message(" in idle_rs and "delete_message_with_connection(&transaction, uid, folder)" in idle_rs, "IMAP IDLE cache deletion writes search outbox in the same DB transaction")
|
|
require("search_engine.clone()," in read("src/realtime/ws.rs"), "websocket IDLE wiring passes the shared search engine")
|
|
require("cargo generate-lockfile && cargo build --release --locked" in read("docker/Dockerfile"), "Docker release build generates and honors a dependency lockfile")
|
|
require("PostgreSQL + SILO (S3-compatible) + OpenSearch" in read("CHANGELOG.md"), "0.2.0 changelog matches the pinned S3-compatible Compose service")
|
|
require("docs/RUSTBOX_PLAN.md" not in read("docs/IMPLEMENTATION_STATUS.md"), "implementation status does not reference a missing release-plan file")
|
|
require("intentionally run one RustBox application container" in read("docker/README.md"), "Docker docs distinguish shared backing topology from multi-app load balancing")
|
|
|
|
attachments_rs = read("src/routes/attachments.rs")
|
|
require(".begin_upload(&key)" in attachments_rs and ".get(&attachment.storage_key)" in attachments_rs and (".delete(&attachment.storage_key)" in attachments_rs or ".delete(&storage_key)" in attachments_rs), "attachment HTTP routes use BlobStorage")
|
|
require("tokio::fs::" not in attachments_rs, "attachment HTTP routes no longer access local filesystem directly")
|
|
attachment_db_delete = attachments_rs.find("db::drafts::delete_draft_attachment")
|
|
attachment_blob_delete = attachments_rs.find("storage.delete(&storage_key)", attachment_db_delete)
|
|
require(attachment_db_delete >= 0 and attachment_blob_delete > attachment_db_delete, "attachment delete commits DB removal before best-effort blob cleanup")
|
|
drafts_rs = read("src/routes/drafts.rs")
|
|
draft_db_delete = drafts_rs.find("db::drafts::delete_draft(&conn, &id)")
|
|
draft_blob_delete = drafts_rs.find("storage.delete(&attachment.storage_key)", draft_db_delete)
|
|
require(draft_db_delete >= 0 and draft_blob_delete > draft_db_delete, "draft delete commits DB removal before best-effort blob cleanup")
|
|
send_rs = read("src/routes/send.rs")
|
|
send_db_delete = send_rs.find("db::drafts::delete_draft(&conn, draft_id)")
|
|
send_blob_delete = send_rs.find("storage.delete(&attachment.storage_key)", send_db_delete)
|
|
require(send_db_delete >= 0 and send_blob_delete > send_db_delete, "post-send cleanup commits DB removal before best-effort blob cleanup")
|
|
|
|
# Repository/source-package hygiene independent of Cargo.
|
|
source_files = [path for path in ROOT.rglob("*") if path.is_file() and ".git" not in path.parts and "target" not in path.parts]
|
|
conflict_marker = re.compile(r"(?m)^(<<<<<<<|=======|>>>>>>>)")
|
|
conflicted = []
|
|
for path in source_files:
|
|
if path.suffix.lower() in {".png", ".jpg", ".jpeg", ".gif", ".webp", ".ico", ".woff", ".woff2", ".ttf", ".zip", ".gz"}:
|
|
continue
|
|
try:
|
|
text = path.read_text(encoding="utf-8")
|
|
except UnicodeDecodeError:
|
|
continue
|
|
if conflict_marker.search(text):
|
|
conflicted.append(str(path.relative_to(ROOT)))
|
|
require(not conflicted, f"no unresolved merge-conflict markers ({', '.join(conflicted) if conflicted else 'clean'})")
|
|
require(not any(path.suffix in {".ts", ".tsx"} for path in source_files), "source package contains no TypeScript files")
|
|
|
|
manifest = [line.strip() for line in read("src/web/js/manifest.txt").splitlines() if line.strip() and not line.lstrip().startswith("#")]
|
|
missing_manifest = [entry for entry in manifest if not (ROOT / "src/web/js" / entry).is_file()]
|
|
require(not missing_manifest, f"browser JS manifest entries exist ({', '.join(missing_manifest) if missing_manifest else 'complete'})")
|
|
|
|
include_paths = []
|
|
for path in ROOT.glob("src/**/*.rs"):
|
|
text = path.read_text(encoding="utf-8")
|
|
for match in re.finditer(r'include_(?:str|bytes)!\(\s*"([^"]+)"\s*\)', text):
|
|
include_paths.append((path, match.group(1)))
|
|
missing_includes = []
|
|
for source, relative in include_paths:
|
|
candidate = (source.parent / relative).resolve()
|
|
if not candidate.is_file():
|
|
missing_includes.append(f"{source.relative_to(ROOT)} -> {relative}")
|
|
require(not missing_includes, f"Rust include_str/include_bytes targets exist ({'; '.join(missing_includes) if missing_includes else 'complete'})")
|
|
|
|
# Shell syntax checks are useful offline and deterministic.
|
|
try:
|
|
shell_files = [ROOT / "dev.sh", *sorted((ROOT / "scripts").glob("*.sh")), ROOT / "docker/garage/init/init.sh"]
|
|
subprocess.run(["bash", "-n", *map(str, shell_files)], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
|
ok("shell scripts pass bash -n")
|
|
except Exception as exc:
|
|
errors.append(f"shell syntax validation failed: {exc}")
|
|
|
|
# TODO/release honesty: live integrations remain explicitly pending.
|
|
todo = read("TODO.md")
|
|
require("[ ] SILO Compose integration validation" in todo, "TODO keeps live SILO validation pending")
|
|
require("[ ] MinIO integration validation" in todo, "TODO keeps live MinIO validation pending")
|
|
require("[ ] AWS S3 integration validation" in todo, "TODO keeps live AWS S3 validation pending")
|
|
require("[ ] Ceph RGW compatibility validation" in todo, "TODO keeps live Ceph RGW validation pending")
|
|
require("[ ] S3-compatible storage works for draft attachments (live SILO/MinIO/AWS validation pending)" in todo, "acceptance does not overclaim live S3 validation")
|
|
|
|
print(f"static checks passed: {len(checks)}")
|
|
for message in checks:
|
|
print(f"PASS {message}")
|
|
if errors:
|
|
print(f"static checks failed: {len(errors)}", file=sys.stderr)
|
|
for message in errors:
|
|
print(f"FAIL {message}", file=sys.stderr)
|
|
raise SystemExit(1)
|