"""CI quality gauge — run chimera solve on the pytest-graded local tasks and report the pass-rate.

Linux/CI-friendly (unlike run_ab.py, which is Windows-oriented): uses `sys.executable`, the `chimera`
CLI on PATH, and a temp dir per task. For each task it writes the starter files + the strict pytest
file, runs `chimera solve --verify` with the M14 scaffolding, then re-runs pytest INDEPENDENTLY for
the verdict — the pass-rate is the tests' word, never solve's self-report.

This is a *quality gauge* over time (does a change move the number?), not a pass/fail build gate: it
always exits 0 and just prints the pass-rate (and writes it to the GitHub step summary). Configure
via env: BENCH_MODEL (a cheap/free model), BENCH_TIMEOUT (per-solve seconds), BENCH_TASKS (comma ids).
"""

from __future__ import annotations

import json
import os
import shutil
import subprocess
import sys
import tempfile
import time
from pathlib import Path

HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
from tasks import TASKS  # noqa: E402

_MODEL = os.environ.get("BENCH_MODEL", "openrouter/deepseek/deepseek-chat-v3.1:free")
_TIMEOUT = int(os.environ.get("BENCH_TIMEOUT", "360"))
_ONLY = {t.strip() for t in os.environ.get("BENCH_TASKS", "").split(",") if t.strip()}
_ARM = os.environ.get("BENCH_ARM", "chimera")
_HYGIENE = ["--no-remember", "--no-collect", "--no-evolve-skills"]
# Two arms for an honest A/B on the SAME model+tasks — the only variable is Chimera's scaffolding:
#   baseline = the raw model, one shot, no plan/manager/scaffolding (still --verify-graded)
#   chimera  = the full solve loop: plan + verify-or-revert + the M14 scaffolding
_ARMS = {
    "baseline": ["--no-plan", "--no-manager", "--max-attempts", "1", *_HYGIENE],
    "chimera": ["--repo-map", "--progress-ledger", "--checklist", "--replan", "--max-attempts", "3", *_HYGIENE],
}
_FLAGS = _ARMS.get(_ARM, _ARMS["chimera"])


def _setup(task: dict, root: Path) -> Path:
    ws = root / task["id"]
    ws.mkdir(parents=True, exist_ok=True)
    for rel, content in task.get("files", {}).items():
        target = ws / rel
        target.parent.mkdir(parents=True, exist_ok=True)
        target.write_text(content, encoding="utf-8")
    (ws / task["test"]).write_text(task["test_src"], encoding="utf-8")
    return ws


def _pytest_passes(task: dict, ws: Path) -> bool:
    proc = subprocess.run(
        [sys.executable, "-m", "pytest", "-q", task["test"]],
        cwd=str(ws), capture_output=True, text=True, encoding="utf-8", errors="replace", check=False,
    )
    return proc.returncode == 0


def _solve(task: dict, ws: Path) -> tuple[int, str]:
    verify = f'"{sys.executable}" -m pytest -q {task["test"]}'
    argv = ["chimera", "solve", task["prompt"], "--workspace", str(ws), "--model", _MODEL,
            "--verify", verify, *_FLAGS]
    try:
        proc = subprocess.run(
            argv, capture_output=True, text=True, encoding="utf-8", errors="replace",
            timeout=_TIMEOUT, check=False,
        )
        tail = ((proc.stdout or "") + (proc.stderr or "")).strip().replace("\n", " ")[-160:]
        return proc.returncode, tail
    except subprocess.TimeoutExpired:
        return 124, "solve timed out"


def main() -> None:
    tasks = [t for t in TASKS if not _ONLY or t["id"] in _ONLY]
    print(f"arm={_ARM} model={_MODEL} tasks={len(tasks)} timeout={_TIMEOUT}s", flush=True)
    rows: list[dict] = []
    root = Path(tempfile.mkdtemp(prefix="chimbench-"))
    try:
        for task in tasks:
            ws = _setup(task, root)
            t0 = time.time()
            exit_code, tail = _solve(task, ws)
            passed = _pytest_passes(task, ws)
            dt = round(time.time() - t0, 1)
            rows.append({"task": task["id"], "passed": passed, "solve_exit": exit_code, "seconds": dt})
            # On a fast failure, surface the solve tail — it's usually a model/provider error (e.g. a
            # 429 rate-limit on a free model), not a real quality miss. Diagnostic, not decorative.
            note = "" if passed else f"  «{tail}»"
            print(f"  {task['id']:<18} {'PASS' if passed else 'fail'}  ({dt:.0f}s, exit={exit_code}){note}", flush=True)
    finally:
        shutil.rmtree(root, ignore_errors=True)

    n = len(rows)
    npass = sum(1 for r in rows if r["passed"])
    rate = npass / n if n else 0.0
    print(f"\n=== [{_ARM}] pass-rate: {npass}/{n} = {rate:.0%} (model {_MODEL}) ===", flush=True)

    out = HERE / "results"
    out.mkdir(exist_ok=True)
    # Per-arm file (ci-baseline.json / ci-chimera.json) so a two-arm A/B doesn't overwrite itself;
    # also a flat pass/fail list per arm, ready for `chimera bench-compare`.
    payload = {"arm": _ARM, "model": _MODEL, "pass": npass, "total": n, "rows": rows}
    (out / f"ci-{_ARM}.json").write_text(json.dumps(payload, indent=2), encoding="utf-8")
    (out / f"trials-{_ARM}.json").write_text(
        json.dumps([bool(r["passed"]) for r in rows]), encoding="utf-8"
    )

    summary = os.environ.get("GITHUB_STEP_SUMMARY")
    if summary:
        lines = [f"### Chimera quality gauge — `{_ARM}` on `{_MODEL}`", "", f"**Pass-rate: {npass}/{n} = {rate:.0%}**", "",
                 "| task | result | time | solve exit |", "|---|---|---|---|"]
        lines += [f"| {r['task']} | {'✅' if r['passed'] else '❌'} | {r['seconds']}s | {r['solve_exit']} |" for r in rows]
        Path(summary).write_text("\n".join(lines) + "\n", encoding="utf-8")


if __name__ == "__main__":
    main()
