#!/usr/bin/env python3
"""Run black-box acceptance tests against one or more real RustProxy binaries.

Example:
  python3 scripts/performance/regression.py --binary before=/abs/before \
    --binary after=/abs/after --out /abs/results/regression
"""
from __future__ import annotations

import argparse
import json
import os
from pathlib import Path
import platform
import sys
import time
import traceback

from common import Fixture, seal_results, write_json
from peer import Peer
import checks_cache
import checks_policy
import checks_proxy
import checks_static


GROUPS = {"static": checks_static.run, "proxy": checks_proxy.run,
          "policy": checks_policy.run, "cache": checks_cache.run,
          "transform": checks_cache.transforms}


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--binary", action="append", required=True, metavar="LABEL=/ABS/PATH")
    parser.add_argument("--out", required=True, type=Path)
    parser.add_argument("--workers", type=int, default=4)
    parser.add_argument("--groups", default=",".join(GROUPS))
    args = parser.parse_args()
    groups = args.groups.split(",")
    if any(group not in GROUPS for group in groups):
        parser.error(f"groups must be selected from {','.join(GROUPS)}")
    args.out = args.out.resolve()
    args.out.mkdir(parents=True, exist_ok=True)
    metadata = {"kind": "live RustProxy black-box HTTP acceptance", "simulation": False,
                "python": sys.version, "platform": platform.platform(),
                "cpu_affinity": sorted(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else None,
                "started_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
                "groups": groups,
                "not_covered": ["TLS/ALPN", "HTTP/2", "HTTP/3", "PHP-FPM",
                                "Tinyauth verifier", "real DNS rebinding", "24-hour soak",
                                "concurrent filesystem rename-race proof"]}
    write_json(args.out / "environment.json", metadata)
    reports = {}
    for binary_arg in args.binary:
        label, separator, name = binary_arg.partition("=")
        if not separator or not label.replace("-", "").replace("_", "").isalnum():
            parser.error("--binary must be LABEL=/ABS/PATH with a simple unique label")
        if label in reports:
            parser.error("binary labels must be unique")
        cases = []
        started = time.monotonic()
        output = args.out / label

        def check(case, passed, **details):
            row = {"case": case, "passed": passed, **details}
            cases.append(row)
            print(f"{label} {'SKIP' if passed is None else 'PASS' if passed else 'FAIL'} {case}", flush=True)
            write_json(output / "checks.json", cases)

        try:
            with Fixture(output, workers=args.workers) as fixture:
                peer = Peer(fixture.origin_port, fixture.site)
                try:
                    rust = fixture.rust(Path(name), label)
                    check("runtime.started", True)
                    for group in groups:
                        try:
                            rust.reset()
                            GROUPS[group](check, rust, peer)
                        except Exception as exc:
                            check(group + ".runner", False, error=str(exc), traceback=traceback.format_exc())
                        finally:
                            rust.reset()
                    check("runtime.still_alive", rust.process.poll() is None)
                finally:
                    peer.close()
        except Exception as exc:
            check("runtime.runner", False, error=str(exc), traceback=traceback.format_exc())
        report = {"label": label, "kind": metadata["kind"], "simulation": False,
                  "seconds": time.monotonic() - started, "passed": sum(c["passed"] is True for c in cases),
                  "failed": sum(c["passed"] is False for c in cases),
                  "skipped": sum(c["passed"] is None for c in cases), "cases": cases}
        reports[label] = report
        write_json(output / "report.json", report)
        print(f"{label}: {report['passed']} passed, {report['failed']} failed, {report['skipped']} skipped", flush=True)
    comparison = {"variants": {name: {key: report[key] for key in ["passed", "failed", "skipped", "seconds"]}
                               for name, report in reports.items()},
                  "cases": [{"case": case, **{name: next((entry["passed"] for entry in report["cases"]
                              if entry["case"] == case), "not_run") for name, report in reports.items()}}
                            for case in sorted({entry["case"] for report in reports.values() for entry in report["cases"]})]}
    write_json(args.out / "comparison.json", comparison)
    seal_results(args.out)
    return 1 if any(report["failed"] for report in reports.values()) else 0


if __name__ == "__main__":
    sys.exit(main())