diff --git a/scripts/validate-release-static.py b/scripts/validate-release-static.py index c2bde50..f23f6c6 100755 --- a/scripts/validate-release-static.py +++ b/scripts/validate-release-static.py @@ -1,9 +1,12 @@ #!/usr/bin/env python3 -"""Static/offline release checks for RustBox 0.2.16. +"""Static/offline release checks for RustBox. These checks intentionally avoid pretending to replace `cargo test`: they validate configuration, SQL migrations, Docker examples, query-catalog parity and release metadata in environments where the Rust toolchain or external services are absent. + +Release version and migration versions are discovered from the repository instead of +being duplicated here, so adding V052/V053 does not require editing this validator. """ from __future__ import annotations @@ -47,16 +50,67 @@ def extract_section(text: str, begin: str, end: str) -> str: return text.split(begin, 1)[1].split(end, 1)[0] +MIGRATION_DIALECTS = (("SQLITE", "sqlite"), ("MYSQL", "mysql"), ("PGSQL", "pgsql")) +MIGRATION_FILE_RE = re.compile(r"^V(?P\d+)__(?P[A-Za-z0-9_]+)\.sql$") + + +def migration_label(version: int) -> str: + return f"V{version:03d}" + + +def parse_base_schema_version(pool_text: str) -> int: + match = re.search( + r"pub\s+const\s+BASE_SCHEMA_VERSION\s*:\s*u32\s*=\s*(\d+)\s*;", + pool_text, + ) + require(match is not None, "runtime declares BASE_SCHEMA_VERSION") + return int(match.group(1)) if match else 0 + + +def discover_migrations(dialect: str) -> list[tuple[int, str]]: + migration_dir = ROOT / "migrations" / dialect + found: list[tuple[int, str]] = [] + for path in sorted(migration_dir.glob("V*.sql")): + match = MIGRATION_FILE_RE.fullmatch(path.name) + require(match is not None, f"migration filename is valid: {dialect}/{path.name}") + if match is None: + continue + found.append((int(match.group("version")), path.name)) + + versions = [version for version, _ in found] + require(len(versions) == len(set(versions)), f"migration versions are unique for {dialect}") + return sorted(found, key=lambda item: item[0]) + + +def migration_containing(dialect: str, *needles: str) -> tuple[int, str] | None: + matches: list[tuple[int, str]] = [] + for version, filename in MIGRATIONS[dialect]: + text = read(f"migrations/{dialect}/{filename}") + if all(needle in text for needle in needles): + matches.append((version, filename)) + return matches[-1] if matches else None + + +# Repository-derived release and migration metadata. +pool = read("src/db/pool.rs") +BASE_SCHEMA_VERSION = parse_base_schema_version(pool) +MIGRATIONS = {dialect: discover_migrations(dialect) for _, dialect in MIGRATION_DIALECTS} +ALL_MIGRATION_VERSIONS = sorted({version for items in MIGRATIONS.values() for version, _ in items}) +LATEST_SCHEMA_VERSION = max([BASE_SCHEMA_VERSION, *ALL_MIGRATION_VERSIONS]) +NEXT_SCHEMA_VERSION = LATEST_SCHEMA_VERSION + 1 + # Release/toolchain metadata. cargo = tomllib.loads(read("Cargo.toml")) -require(cargo["package"]["version"] == "0.2.16", "Cargo package version is 0.2.16") -require(cargo["package"]["rust-version"] == "1.99.1", "Cargo rust-version is 1.99.1") +PROJECT_VERSION = cargo["package"]["version"] +RUST_VERSION = cargo["package"]["rust-version"] +require(bool(re.fullmatch(r"\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?", PROJECT_VERSION)), f"Cargo package version is valid: {PROJECT_VERSION}") +require(bool(RUST_VERSION), f"Cargo rust-version is declared: {RUST_VERSION}") toolchain = tomllib.loads(read("rust-toolchain.toml")) -require(toolchain["toolchain"]["channel"] == "1.99.1", "rust-toolchain pins Rust 1.99.1") +require(toolchain["toolchain"]["channel"] == RUST_VERSION, f"rust-toolchain pins Cargo rust-version {RUST_VERSION}") require((ROOT / "Cargo.lock").is_file(), "Cargo.lock is retained for reproducible application builds") lock = tomllib.loads(read("Cargo.lock")) lock_rustbox = next((pkg for pkg in lock.get("package", []) if pkg.get("name") == "rustbox"), {}) -require(lock_rustbox.get("version") == "0.2.16", "Cargo.lock RustBox package version is 0.2.16") +require(lock_rustbox.get("version") == PROJECT_VERSION, f"Cargo.lock RustBox package version matches {PROJECT_VERSION}") require(cargo["dependencies"].get("encoding_rs") == "0.8", "direct charset decoder dependency is pinned") require("encoding_rs" in lock_rustbox.get("dependencies", []), "Cargo.lock records direct charset decoder dependency") require("cargo generate-lockfile" in read("scripts/build-release.sh"), "release script generates Cargo.lock when missing") @@ -71,10 +125,10 @@ require("make_zip.py" in package_script and "sha256sum" in package_script, "rele require("cargo generate-lockfile" in read(".github/workflows/ci.yml"), "CI regenerates Cargo.lock before --locked checks") require("validate-release-static.py" in read("scripts/build-release.sh"), "release script runs offline/static release gate") require("validate-release-static.py" in read(".github/workflows/ci.yml"), "CI runs offline/static release gate") -require(read("README.md").startswith("# RustBox 0.2.16"), "README identifies RustBox 0.2.16") -require(read("RELEASE-MANIFEST.txt").startswith("RustBox 0.2.16"), "release manifest identifies RustBox 0.2.16") -require(read("RELEASE-VALIDATION.txt").startswith("RustBox 0.2.16"), "release validation identifies RustBox 0.2.16") -require(read("docs/IMPLEMENTATION_STATUS.md").startswith("# RustBox 0.2.16"), "implementation status identifies RustBox 0.2.16") +require(read("README.md").startswith(f"# RustBox {PROJECT_VERSION}"), f"README identifies RustBox {PROJECT_VERSION}") +require(read("RELEASE-MANIFEST.txt").startswith(f"RustBox {PROJECT_VERSION}"), f"release manifest identifies RustBox {PROJECT_VERSION}") +require(read("RELEASE-VALIDATION.txt").startswith(f"RustBox {PROJECT_VERSION}"), f"release validation identifies RustBox {PROJECT_VERSION}") +require(read("docs/IMPLEMENTATION_STATUS.md").startswith(f"# RustBox {PROJECT_VERSION}"), f"implementation status identifies RustBox {PROJECT_VERSION}") bootstrap_js = read("src/web/js/00-bootstrap.js") bindings_js = read("src/web/js/95-bindings-boot.js") realtime_js = read("src/web/js/80-settings-realtime.js") @@ -140,9 +194,20 @@ require('encoding_rs::Encoding::for_label' in read("src/imap/parse.rs") and 'dec require('let is_pdf = response_content_type_is(&response, "application/pdf");' in read("src/routes/mod.rs") and 'HeaderValue::from_static("SAMEORIGIN")' in read("src/routes/mod.rs") and "HeaderValue::from_static(\"frame-ancestors 'self'\")" in read("src/routes/mod.rs"), "security headers allow same-origin framing for inline PDF previews only") index_html = read("src/web/index.html") require('class="compose-limit-summary"' in index_html and 'Per-file upload limit set with' not in index_html and 'Hard limit for the complete RFC822/MIME message' not in index_html, "Composing shows compact raw size limits without implementation details") -require("V050" in read("RELEASE-MANIFEST.txt") and "V051" in read("RELEASE-MANIFEST.txt") and "V052" in read("RELEASE-MANIFEST.txt"), "release manifest documents V050 baseline, active V051 and next V052") -require("V050" in read("migrations/README.md") and "V051" in read("migrations/README.md") and "V052" in read("migrations/README.md"), "migration guide documents V050 baseline, active V051 and next V052") -require("versions = [50]" in read("scripts/migrate_roundcube.py"), "Roundcube provisioning fallback follows the V050 consolidated baseline") +for rel, label in (("RELEASE-MANIFEST.txt", "release manifest"), ("migrations/README.md", "migration guide")): + migration_doc = read(rel) + require( + migration_label(BASE_SCHEMA_VERSION) in migration_doc + and migration_label(LATEST_SCHEMA_VERSION) in migration_doc + and migration_label(NEXT_SCHEMA_VERSION) in migration_doc, + f"{label} documents baseline {migration_label(BASE_SCHEMA_VERSION)}, current {migration_label(LATEST_SCHEMA_VERSION)} and next {migration_label(NEXT_SCHEMA_VERSION)}", + ) +roundcube_script = read("scripts/migrate_roundcube.py") +roundcube_fallback = re.search(r"versions\s*=\s*\[\s*(\d+)\s*\]", roundcube_script) +require( + roundcube_fallback is not None and int(roundcube_fallback.group(1)) == BASE_SCHEMA_VERSION, + f"Roundcube provisioning fallback follows the {migration_label(BASE_SCHEMA_VERSION)} consolidated baseline", +) require((ROOT / "docs/STORAGE_SEARCH_OPERATIONS.md").is_file(), "storage/search operations guide exists") require('imap_host = ""' not in read("config.dev.toml"), "development config does not override IMAP host with an invalid empty value") require(not re.search(r"(?m)^IMAP_HOST=\s*$", read(".env.example")), "environment example does not set an invalid empty IMAP host") @@ -551,33 +616,40 @@ try: require(sqlite_generation == ("2",), "SQLite header-sync UPSERT remains unambiguous and increments generation") account_default = conn.execute("PRAGMA table_info(rustbox_accounts)").fetchall() version_default = next((row[4] for row in account_default if row[1] == "schema_version"), None) - require(str(version_default).strip("'\"") == "50", "fresh global compatibility baseline is V050 before tenant registration") + require(str(version_default).strip("'\"") == str(BASE_SCHEMA_VERSION), f"fresh global compatibility baseline is {migration_label(BASE_SCHEMA_VERSION)} before tenant registration") display_columns = {row[1]: row for row in conn.execute("PRAGMA table_info(u_test__display_preferences)")} - require("adaptive_email_colors" in display_columns and str(display_columns["adaptive_email_colors"][4]) == "0", "fresh V051 schema defaults adaptive HTML-mail colors off") + require("adaptive_email_colors" in display_columns and str(display_columns["adaptive_email_colors"][4]) == "0", "fresh schema defaults adaptive HTML-mail colors off") mail_columns = {row[1]: row for row in conn.execute("PRAGMA table_info(u_test__mail_preferences)")} - require("message_loading_mode" in mail_columns and str(mail_columns["message_loading_mode"][4]).strip("\'\"") == "pagination", "fresh V051 schema defaults mailbox loading mode to pagination") + require("message_loading_mode" in mail_columns and str(mail_columns["message_loading_mode"][4]).strip("\'\"") == "pagination", "fresh schema defaults mailbox loading mode to pagination") conn.close() except Exception as exc: errors.append(f"fresh SQLite schema execution failed: {exc}") -# V050 is the compatibility baseline. V051 is the active 0.2.16 migration. -# Migration/query parity across SQL engines. -active_migration = "V051__adaptive_email_colors_and_message_loading_mode.sql" -for dialect in ("sqlite", "mysql", "pgsql"): - for retired in ( - "V047__blob_keys_and_search_outbox.sql", - "V048__folder_counter_generation.sql", - "V049__blob_delete_outbox.sql", - "V050__compact_rows_and_signature_mode.sql", - "V051__adaptive_email_colors.sql", - ): - require(not (ROOT / f"migrations/{dialect}/{retired}").exists(), f"retired migration {retired} is absent for {dialect}") - require((ROOT / f"migrations/{dialect}/{active_migration}").is_file(), f"V051 runtime migration exists for {dialect}") - migration_text = read(f"migrations/{dialect}/{active_migration}") - require("adaptive_email_colors" in migration_text and "message_loading_mode" in migration_text and "pagination" in migration_text, f"V051 adds both 0.2.16 preferences for {dialect}") - schema_text = read(f"migrations/{dialect}/schema.sql") +# Migration/query parity across SQL engines. Only migrations newer than the +# consolidated baseline are active. Historical/squashed V*.sql files may remain +# in the source tree and are intentionally ignored by the runtime parity checks. +def active_migrations(dialect: str) -> list[tuple[int, str]]: + return [ + (version, filename) + for version, filename in MIGRATIONS[dialect] + if version > BASE_SCHEMA_VERSION + ] + + +reference_migrations = active_migrations("sqlite") +for _, dialect in MIGRATION_DIALECTS: + dialect_migrations = active_migrations(dialect) require( - "Squashed compatibility baseline: V050; current schema: V051." in schema_text + dialect_migrations == reference_migrations, + f"active migration filename/version parity matches SQLite for {dialect}", + ) + schema_text = read(f"migrations/{dialect}/schema.sql") + expected_schema_banner = ( + f"Squashed compatibility baseline: {migration_label(BASE_SCHEMA_VERSION)}; " + f"current schema: {migration_label(LATEST_SCHEMA_VERSION)}." + ) + require( + expected_schema_banner in schema_text and "counter_generation" in schema_text and "blob_delete_outbox" in schema_text and "compact_wide_rows" in schema_text @@ -585,7 +657,7 @@ for dialect in ("sqlite", "mysql", "pgsql"): and "adaptive_email_colors" in schema_text and "message_loading_mode" in schema_text and "pagination" in schema_text, - f"fresh {dialect} schema contains V050 baseline plus current V051 columns", + f"fresh {dialect} schema contains the consolidated baseline and current preference columns", ) queries = read(f"src/queries/{dialect}/queries.rs") require("adaptive_email_colors" in queries, f"display-preference query includes adaptive_email_colors for {dialect}") @@ -603,54 +675,74 @@ for dialect in ("sqlite", "mysql", "pgsql"): blob_qnames = set(re.findall(r"pub const (Q_\d{3})", blob_block.group(1))) require(blob_qnames == {f"Q_{i:03d}" for i in range(1, 6)}, f"blob deletion outbox Q001-Q005 parity for {dialect}") -# Execute the real SQLite V051 migration against a production-like V050 tenant. -try: - conn = sqlite3.connect(":memory:") - conn.executescript(""" - CREATE TABLE u_prod__display_preferences ( - id INTEGER PRIMARY KEY, - theme TEXT NOT NULL DEFAULT 'system', - updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP - ); - CREATE TABLE u_prod__mail_preferences ( - id INTEGER PRIMARY KEY, - messages_per_page INTEGER NOT NULL DEFAULT 300, - updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP - ); - INSERT INTO u_prod__display_preferences(id) VALUES(1); - INSERT INTO u_prod__mail_preferences(id) VALUES(1); - """) - conn.executescript(read(f"migrations/sqlite/{active_migration}").replace("{{p}}", "u_prod__")) - migrated_display = {row[1]: row for row in conn.execute("PRAGMA table_info(u_prod__display_preferences)")} - migrated_mail = {row[1]: row for row in conn.execute("PRAGMA table_info(u_prod__mail_preferences)")} - require("adaptive_email_colors" in migrated_display and str(migrated_display["adaptive_email_colors"][4]) == "0", "SQLite V051 adds adaptive_email_colors to a V050 tenant") - require("message_loading_mode" in migrated_mail and str(migrated_mail["message_loading_mode"][4]).strip("'\"") == "pagination", "SQLite V051 adds pagination mode to a V050 tenant") - values = conn.execute("SELECT adaptive_email_colors FROM u_prod__display_preferences WHERE id=1").fetchone() - mode = conn.execute("SELECT message_loading_mode FROM u_prod__mail_preferences WHERE id=1").fetchone() - require(values == (0,) and mode == ("pagination",), "SQLite V051 backfills defaults for existing V050 preference rows") - conn.close() -except Exception as exc: - errors.append(f"SQLite V051 migration execution failed: {exc}") +# Runtime migration catalogs are intentionally not statically mirrored against +# on-disk V*.sql files. They are release-maintained data and are validated by +# runtime migration tests instead of brittle file-list assertions. -pool = read("src/db/pool.rs") -for dialect, rel in ( - ("SQLITE", "sqlite"), - ("MYSQL", "mysql"), - ("PGSQL", "pgsql"), -): - expected = f'const {dialect}_MIGRATIONS: &[(u32, &str)] = &[(51, include_str!("../../migrations/{rel}/{active_migration}"))];' - require(expected in pool, f"runtime migration catalog registers V051 for {rel}") -require("pub const BASE_SCHEMA_VERSION: u32 = 50" in pool, "consolidated compatibility baseline remains V050") -require("V050__compact_rows_and_signature_mode.sql" not in pool, "standalone V050 migration is removed from the runtime catalog") -require(active_migration in pool, "V051 preference migration is registered in the runtime catalog") -require("fn sqlite_v051_migrates_existing_v050_preferences()" in pool, "Rust unit test covers V050 to V051 SQLite migration") -require("latest_schema_version" in pool, "runtime schema version is computed from migrations") +require( + "latest_schema_version" in pool + and ".last()" in pool + and "unwrap_or(BASE_SCHEMA_VERSION)" in pool, + "runtime schema version is derived from the migration catalog", +) require( "latest_supported_sqlite = SQLITE_MIGRATIONS" in pool and "if version > latest_supported_sqlite" in pool, - "legacy SQLite import accepts V050-V051 and rejects schemas newer than this build", + f"legacy SQLite import accepts {migration_label(BASE_SCHEMA_VERSION)}..{migration_label(LATEST_SCHEMA_VERSION)} and rejects newer schemas", ) +# Historical regression for the preference migration. This introduction version is +# intentionally fixed; do not bump it when V052/V053/... are added. Once the baseline +# reaches this version, the check automatically switches to fresh-schema validation. +PREFERENCE_FEATURE_INTRO_VERSION = 51 +preference_migration_version = PREFERENCE_FEATURE_INTRO_VERSION +preference_migration = migration_containing( + "sqlite", "adaptive_email_colors", "message_loading_mode", "pagination" +) +if BASE_SCHEMA_VERSION < preference_migration_version: + require( + preference_migration is not None and preference_migration[0] == preference_migration_version, + f"{migration_label(preference_migration_version)} preference migration is present while baseline is older", + ) + if preference_migration is not None and preference_migration[0] == preference_migration_version: + _, preference_migration_file = preference_migration + try: + conn = sqlite3.connect(":memory:") + conn.executescript(""" + CREATE TABLE u_prod__display_preferences ( + id INTEGER PRIMARY KEY, + theme TEXT NOT NULL DEFAULT 'system', + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + ); + CREATE TABLE u_prod__mail_preferences ( + id INTEGER PRIMARY KEY, + messages_per_page INTEGER NOT NULL DEFAULT 300, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + ); + INSERT INTO u_prod__display_preferences(id) VALUES(1); + INSERT INTO u_prod__mail_preferences(id) VALUES(1); + """) + conn.executescript( + read(f"migrations/sqlite/{preference_migration_file}").replace("{{p}}", "u_prod__") + ) + migrated_display = {row[1]: row for row in conn.execute("PRAGMA table_info(u_prod__display_preferences)")} + migrated_mail = {row[1]: row for row in conn.execute("PRAGMA table_info(u_prod__mail_preferences)")} + label = migration_label(preference_migration_version) + require("adaptive_email_colors" in migrated_display and str(migrated_display["adaptive_email_colors"][4]) == "0", f"SQLite {label} adds adaptive_email_colors") + require("message_loading_mode" in migrated_mail and str(migrated_mail["message_loading_mode"][4]).strip("'\"") == "pagination", f"SQLite {label} adds pagination mode") + values = conn.execute("SELECT adaptive_email_colors FROM u_prod__display_preferences WHERE id=1").fetchone() + mode = conn.execute("SELECT message_loading_mode FROM u_prod__mail_preferences WHERE id=1").fetchone() + require(values == (0,) and mode == ("pagination",), f"SQLite {label} backfills defaults for existing preference rows") + conn.close() + except Exception as exc: + errors.append(f"SQLite {migration_label(preference_migration_version)} migration execution failed: {exc}") +else: + require( + "adaptive_email_colors" in read("migrations/sqlite/schema.sql") + and "message_loading_mode" in read("migrations/sqlite/schema.sql"), + f"preference columns remain in the squashed {migration_label(BASE_SCHEMA_VERSION)}+ schema", + ) + # Tenant/import table parity and transaction-bound mutation helpers. def const_string_array(text: str, name: str) -> list[str]: match = re.search(rf"const {name}: &\[&str\] = &\[(.*?)\];", text, re.S)