108 lines
4.9 KiB
Python
Executable File
108 lines
4.9 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Run black-box acceptance tests against one or more real RustProxy binaries.
|
|
|
|
Example:
|
|
python3 scripts/performance/regression.py --binary before=/abs/before \
|
|
--binary after=/abs/after --out /abs/results/regression
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import platform
|
|
import sys
|
|
import time
|
|
import traceback
|
|
|
|
from common import Fixture, seal_results, write_json
|
|
from peer import Peer
|
|
import checks_cache
|
|
import checks_policy
|
|
import checks_proxy
|
|
import checks_static
|
|
|
|
|
|
GROUPS = {"static": checks_static.run, "proxy": checks_proxy.run,
|
|
"policy": checks_policy.run, "cache": checks_cache.run,
|
|
"transform": checks_cache.transforms}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--binary", action="append", required=True, metavar="LABEL=/ABS/PATH")
|
|
parser.add_argument("--out", required=True, type=Path)
|
|
parser.add_argument("--workers", type=int, default=4)
|
|
parser.add_argument("--groups", default=",".join(GROUPS))
|
|
args = parser.parse_args()
|
|
groups = args.groups.split(",")
|
|
if any(group not in GROUPS for group in groups):
|
|
parser.error(f"groups must be selected from {','.join(GROUPS)}")
|
|
args.out = args.out.resolve()
|
|
args.out.mkdir(parents=True, exist_ok=True)
|
|
metadata = {"kind": "live RustProxy black-box HTTP acceptance", "simulation": False,
|
|
"python": sys.version, "platform": platform.platform(),
|
|
"cpu_affinity": sorted(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else None,
|
|
"started_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
"groups": groups,
|
|
"not_covered": ["TLS/ALPN", "HTTP/2", "HTTP/3", "PHP-FPM",
|
|
"Tinyauth verifier", "real DNS rebinding", "24-hour soak",
|
|
"concurrent filesystem rename-race proof"]}
|
|
write_json(args.out / "environment.json", metadata)
|
|
reports = {}
|
|
for binary_arg in args.binary:
|
|
label, separator, name = binary_arg.partition("=")
|
|
if not separator or not label.replace("-", "").replace("_", "").isalnum():
|
|
parser.error("--binary must be LABEL=/ABS/PATH with a simple unique label")
|
|
if label in reports:
|
|
parser.error("binary labels must be unique")
|
|
cases = []
|
|
started = time.monotonic()
|
|
output = args.out / label
|
|
|
|
def check(case, passed, **details):
|
|
row = {"case": case, "passed": passed, **details}
|
|
cases.append(row)
|
|
print(f"{label} {'SKIP' if passed is None else 'PASS' if passed else 'FAIL'} {case}", flush=True)
|
|
write_json(output / "checks.json", cases)
|
|
|
|
try:
|
|
with Fixture(output, workers=args.workers) as fixture:
|
|
peer = Peer(fixture.origin_port, fixture.site)
|
|
try:
|
|
rust = fixture.rust(Path(name), label)
|
|
check("runtime.started", True)
|
|
for group in groups:
|
|
try:
|
|
rust.reset()
|
|
GROUPS[group](check, rust, peer)
|
|
except Exception as exc:
|
|
check(group + ".runner", False, error=str(exc), traceback=traceback.format_exc())
|
|
finally:
|
|
rust.reset()
|
|
check("runtime.still_alive", rust.process.poll() is None)
|
|
finally:
|
|
peer.close()
|
|
except Exception as exc:
|
|
check("runtime.runner", False, error=str(exc), traceback=traceback.format_exc())
|
|
report = {"label": label, "kind": metadata["kind"], "simulation": False,
|
|
"seconds": time.monotonic() - started, "passed": sum(c["passed"] is True for c in cases),
|
|
"failed": sum(c["passed"] is False for c in cases),
|
|
"skipped": sum(c["passed"] is None for c in cases), "cases": cases}
|
|
reports[label] = report
|
|
write_json(output / "report.json", report)
|
|
print(f"{label}: {report['passed']} passed, {report['failed']} failed, {report['skipped']} skipped", flush=True)
|
|
comparison = {"variants": {name: {key: report[key] for key in ["passed", "failed", "skipped", "seconds"]}
|
|
for name, report in reports.items()},
|
|
"cases": [{"case": case, **{name: next((entry["passed"] for entry in report["cases"]
|
|
if entry["case"] == case), "not_run") for name, report in reports.items()}}
|
|
for case in sorted({entry["case"] for report in reports.values() for entry in report["cases"]})]}
|
|
write_json(args.out / "comparison.json", comparison)
|
|
seal_results(args.out)
|
|
return 1 if any(report["failed"] for report in reports.values()) else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|