from __future__ import annotations

import json

from grantflow.eval import harness


def test_eval_harness_cli_supports_max_cases_and_sample_seed(tmp_path, monkeypatch):
    json_out = tmp_path / "sampled-report.json"
    text_out = tmp_path / "sampled-report.txt"

    monkeypatch.setattr(
        harness,
        "load_eval_cases",
        lambda fixtures_dir=None, case_files=None: [
            {"case_id": "c1", "donor_id": "usaid"},
            {"case_id": "c2", "donor_id": "eu"},
            {"case_id": "c3", "donor_id": "worldbank"},
            {"case_id": "c4", "donor_id": "state_department"},
        ],
    )

    captured: dict[str, object] = {}

    def fake_run_eval_suite(cases, *, suite_label=None, skip_expectations=False):
        captured["cases"] = cases
        return {
            "suite_label": suite_label or "baseline",
            "expectations_skipped": bool(skip_expectations),
            "case_count": len(cases),
            "passed_count": len(cases),
            "failed_count": 0,
            "all_passed": True,
            "cases": [],
        }

    monkeypatch.setattr(harness, "run_eval_suite", fake_run_eval_suite)

    exit_code = harness.main(
        [
            "--suite-label",
            "llm-eval-sampled",
            "--max-cases",
            "2",
            "--sample-seed",
            "42",
            "--json-out",
            str(json_out),
            "--text-out",
            str(text_out),
        ]
    )
    assert exit_code == 0
    payload = json.loads(json_out.read_text(encoding="utf-8"))
    assert payload["runtime_overrides"]["max_cases"] == 2
    assert payload["runtime_overrides"]["sample_seed"] == 42
    captured_cases = captured["cases"]
    assert isinstance(captured_cases, list)
    assert [str(case.get("case_id")) for case in captured_cases] == ["c1", "c4"]


def test_eval_harness_cli_supports_cases_file_argument(tmp_path):
    json_out = tmp_path / "eval-report.json"
    text_out = tmp_path / "eval-report.txt"
    cases_file = tmp_path / "explicit-cases.json"
    cases_file.write_text(
        json.dumps(
            [
                {
                    "case_id": "explicit_only",
                    "donor_id": "usaid",
                    "input_context": {"project": "Water", "country": "Kenya"},
                    "llm_mode": False,
                    "architect_rag_enabled": False,
                    "expectations": {"toc_schema_valid": True},
                }
            ]
        ),
        encoding="utf-8",
    )

    exit_code = harness.main(
        [
            "--suite-label",
            "explicit-file-suite",
            "--cases-file",
            str(cases_file),
            "--json-out",
            str(json_out),
            "--text-out",
            str(text_out),
        ]
    )
    assert exit_code == 0
    payload = json.loads(json_out.read_text(encoding="utf-8"))
    assert payload["case_count"] == 1
    assert payload["runtime_overrides"]["cases_files"] == [str(cases_file)]
    assert payload["cases"][0]["case_id"] == "explicit_only"


def test_eval_harness_cli_supports_sample_id_argument(tmp_path, monkeypatch):
    json_out = tmp_path / "sample-report.json"
    text_out = tmp_path / "sample-report.txt"

    monkeypatch.setattr(
        harness,
        "load_sample_eval_cases",
        lambda sample_ids: [
            {
                "case_id": "sample_case",
                "donor_id": "eu",
                "input_context": {"project": "P", "country": "C"},
                "_sample_id": sample_ids[0],
            }
        ],
    )

    captured: dict[str, object] = {}

    def fake_run_eval_suite(cases, *, suite_label=None, skip_expectations=False):
        captured["cases"] = cases
        return {
            "suite_label": suite_label or "baseline",
            "expectations_skipped": bool(skip_expectations),
            "case_count": len(cases),
            "passed_count": len(cases),
            "failed_count": 0,
            "all_passed": True,
            "cases": [],
        }

    monkeypatch.setattr(harness, "run_eval_suite", fake_run_eval_suite)

    exit_code = harness.main(
        [
            "--suite-label",
            "sample-suite",
            "--sample-id",
            "rbm-eu-youth-employment-jordan",
            "--json-out",
            str(json_out),
            "--text-out",
            str(text_out),
        ]
    )
    assert exit_code == 0
    payload = json.loads(json_out.read_text(encoding="utf-8"))
    assert payload["runtime_overrides"]["sample_ids"] == ["rbm-eu-youth-employment-jordan"]
    captured_cases = captured["cases"]
    assert isinstance(captured_cases, list) and len(captured_cases) == 1
    assert captured_cases[0]["case_id"] == "sample_case"


def test_eval_harness_cli_returns_nonzero_when_filters_match_no_cases(tmp_path):
    json_out = tmp_path / "eval-report.json"
    text_out = tmp_path / "eval-report.txt"
    exit_code = harness.main(
        [
            "--donor-id",
            "nonexistent_donor",
            "--json-out",
            str(json_out),
            "--text-out",
            str(text_out),
        ]
    )
    assert exit_code == 2
    assert not json_out.exists()
    assert not text_out.exists()
