v0.33.8

scripts/performance/regression.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+"""Run black-box acceptance tests against one or more real RustProxy binaries.
+
+Example:
+  python3 scripts/performance/regression.py --binary before=/abs/before \
+    --binary after=/abs/after --out /abs/results/regression
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from pathlib import Path
+import platform
+import sys
+import time
+import traceback
+
+from common import Fixture, seal_results, write_json
+from peer import Peer
+import checks_cache
+import checks_policy
+import checks_proxy
+import checks_static
+
+
+GROUPS = {"static": checks_static.run, "proxy": checks_proxy.run,
+          "policy": checks_policy.run, "cache": checks_cache.run,
+          "transform": checks_cache.transforms}
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--binary", action="append", required=True, metavar="LABEL=/ABS/PATH")
+    parser.add_argument("--out", required=True, type=Path)
+    parser.add_argument("--workers", type=int, default=4)
+    parser.add_argument("--groups", default=",".join(GROUPS))
+    args = parser.parse_args()
+    groups = args.groups.split(",")
+    if any(group not in GROUPS for group in groups):
+        parser.error(f"groups must be selected from {','.join(GROUPS)}")
+    args.out = args.out.resolve()
+    args.out.mkdir(parents=True, exist_ok=True)
+    metadata = {"kind": "live RustProxy black-box HTTP acceptance", "simulation": False,
+                "python": sys.version, "platform": platform.platform(),
+                "cpu_affinity": sorted(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else None,
+                "started_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+                "groups": groups,
+                "not_covered": ["TLS/ALPN", "HTTP/2", "HTTP/3", "PHP-FPM",
+                                "Tinyauth verifier", "real DNS rebinding", "24-hour soak",
+                                "concurrent filesystem rename-race proof"]}
+    write_json(args.out / "environment.json", metadata)
+    reports = {}
+    for binary_arg in args.binary:
+        label, separator, name = binary_arg.partition("=")
+        if not separator or not label.replace("-", "").replace("_", "").isalnum():
+            parser.error("--binary must be LABEL=/ABS/PATH with a simple unique label")
+        if label in reports:
+            parser.error("binary labels must be unique")
+        cases = []
+        started = time.monotonic()
+        output = args.out / label
+
+        def check(case, passed, **details):
+            row = {"case": case, "passed": passed, **details}
+            cases.append(row)
+            print(f"{label} {'SKIP' if passed is None else 'PASS' if passed else 'FAIL'} {case}", flush=True)
+            write_json(output / "checks.json", cases)
+
+        try:
+            with Fixture(output, workers=args.workers) as fixture:
+                peer = Peer(fixture.origin_port, fixture.site)
+                try:
+                    rust = fixture.rust(Path(name), label)
+                    check("runtime.started", True)
+                    for group in groups:
+                        try:
+                            rust.reset()
+                            GROUPS[group](check, rust, peer)
+                        except Exception as exc:
+                            check(group + ".runner", False, error=str(exc), traceback=traceback.format_exc())
+                        finally:
+                            rust.reset()
+                    check("runtime.still_alive", rust.process.poll() is None)
+                finally:
+                    peer.close()
+        except Exception as exc:
+            check("runtime.runner", False, error=str(exc), traceback=traceback.format_exc())
+        report = {"label": label, "kind": metadata["kind"], "simulation": False,
+                  "seconds": time.monotonic() - started, "passed": sum(c["passed"] is True for c in cases),
+                  "failed": sum(c["passed"] is False for c in cases),
+                  "skipped": sum(c["passed"] is None for c in cases), "cases": cases}
+        reports[label] = report
+        write_json(output / "report.json", report)
+        print(f"{label}: {report['passed']} passed, {report['failed']} failed, {report['skipped']} skipped", flush=True)
+    comparison = {"variants": {name: {key: report[key] for key in ["passed", "failed", "skipped", "seconds"]}
+                               for name, report in reports.items()},
+                  "cases": [{"case": case, **{name: next((entry["passed"] for entry in report["cases"]
+                              if entry["case"] == case), "not_run") for name, report in reports.items()}}
+                            for case in sorted({entry["case"] for report in reports.values() for entry in report["cases"]})]}
+    write_json(args.out / "comparison.json", comparison)
+    seal_results(args.out)
+    return 1 if any(report["failed"] for report in reports.values()) else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())