from __future__ import annotations

import json

from grantflow.eval import harness


def test_eval_harness_cli_supports_suite_label_and_runtime_override_flags(tmp_path, monkeypatch):
    json_out = tmp_path / "llm-eval-report.json"
    text_out = tmp_path / "llm-eval-report.txt"

    monkeypatch.setattr(
        harness,
        "load_eval_cases",
        lambda fixtures_dir=None, case_files=None: [
            {"case_id": "stub", "donor_id": "usaid", "llm_mode": False, "architect_rag_enabled": False}
        ],
    )

    captured: dict[str, object] = {}

    def fake_run_eval_suite(cases, *, suite_label=None, skip_expectations=False):
        captured["cases"] = cases
        captured["suite_label"] = suite_label
        captured["skip_expectations"] = skip_expectations
        return {
            "suite_label": suite_label or "baseline",
            "expectations_skipped": bool(skip_expectations),
            "case_count": 1,
            "passed_count": 1,
            "failed_count": 0,
            "all_passed": True,
            "cases": [],
        }

    monkeypatch.setattr(harness, "run_eval_suite", fake_run_eval_suite)

    exit_code = harness.main(
        [
            "--suite-label",
            "llm-eval",
            "--force-llm",
            "--skip-expectations",
            "--donor-id",
            "usaid",
            "--case-id",
            "stub",
            "--json-out",
            str(json_out),
            "--text-out",
            str(text_out),
        ]
    )
    assert exit_code == 0
    payload = json.loads(json_out.read_text(encoding="utf-8"))
    assert payload["suite_label"] == "llm-eval"
    assert payload["runtime_overrides"]["force_llm"] is True
    assert payload["runtime_overrides"]["force_architect_rag"] is False
    assert payload["runtime_overrides"]["force_no_architect_rag"] is False
    assert payload["runtime_overrides"]["skip_expectations"] is True
    assert payload["runtime_overrides"]["donor_filters"] == ["usaid"]
    assert payload["runtime_overrides"]["case_filters"] == ["stub"]
    assert payload["runtime_overrides"]["cases_files"] == []
    assert payload["runtime_overrides"]["max_cases"] is None
    assert payload["runtime_overrides"]["sample_seed"] is None
    assert payload["expectations_skipped"] is True
    assert captured["suite_label"] == "llm-eval"
    assert captured["skip_expectations"] is True
    captured_cases = captured["cases"]
    assert isinstance(captured_cases, list) and captured_cases
    assert captured_cases[0]["llm_mode"] is True
    assert captured_cases[0]["architect_rag_enabled"] is False
    text = text_out.read_text(encoding="utf-8")
    assert "Suite: llm-eval" in text
    assert "Expectations: skipped" in text


def test_eval_harness_cli_supports_force_no_architect_rag(tmp_path, monkeypatch):
    json_out = tmp_path / "ab-report.json"
    text_out = tmp_path / "ab-report.txt"

    monkeypatch.setattr(
        harness,
        "load_eval_cases",
        lambda fixtures_dir=None, case_files=None: [
            {"case_id": "stub", "donor_id": "usaid", "llm_mode": False, "architect_rag_enabled": True}
        ],
    )

    captured: dict[str, object] = {}

    def fake_run_eval_suite(cases, *, suite_label=None, skip_expectations=False):
        captured["cases"] = cases
        return {
            "suite_label": suite_label or "baseline",
            "expectations_skipped": bool(skip_expectations),
            "case_count": 1,
            "passed_count": 1,
            "failed_count": 0,
            "all_passed": True,
            "cases": [],
        }

    monkeypatch.setattr(harness, "run_eval_suite", fake_run_eval_suite)

    exit_code = harness.main(
        [
            "--suite-label",
            "grounded-ab-b",
            "--force-no-architect-rag",
            "--skip-expectations",
            "--json-out",
            str(json_out),
            "--text-out",
            str(text_out),
        ]
    )
    assert exit_code == 0
    payload = json.loads(json_out.read_text(encoding="utf-8"))
    assert payload["runtime_overrides"]["force_architect_rag"] is False
    assert payload["runtime_overrides"]["force_no_architect_rag"] is True
    captured_cases = captured["cases"]
    assert isinstance(captured_cases, list) and captured_cases
    assert captured_cases[0]["architect_rag_enabled"] is False
