from __future__ import annotations

import json
from pathlib import Path

from grantflow.eval.harness import (
    available_sample_ids,
    evaluate_seed_readiness,
    expected_doc_families_by_donor,
    filter_eval_cases,
    limit_eval_cases,
    load_eval_cases,
    load_sample_eval_cases,
)


def test_load_eval_cases_supports_explicit_case_files(tmp_path):
    payload = [
        {
            "case_id": "explicit_case",
            "donor_id": "usaid",
            "input_context": {"project": "P", "country": "C"},
            "expectations": {"toc_schema_valid": True},
        }
    ]
    path = tmp_path / "cases.json"
    path.write_text(json.dumps(payload), encoding="utf-8")

    rows = load_eval_cases(case_files=[path])
    assert len(rows) == 1
    assert rows[0]["case_id"] == "explicit_case"
    assert rows[0]["_fixture_file"] == "cases.json"


def test_load_sample_eval_cases_supports_named_presets():
    cases = load_sample_eval_cases(["rbm-usaid-ai-civil-service-kazakhstan"])
    assert len(cases) == 1
    case = cases[0]
    assert case["donor_id"] == "usaid"
    assert "input_context" in case and isinstance(case["input_context"], dict)
    assert str(case["input_context"].get("project") or "").strip() != ""
    assert case["_sample_id"] == "rbm-usaid-ai-civil-service-kazakhstan"
    assert case["_fixture_file"] == "rbm-sample-usaid-ai-civil-service-kazakhstan.json"


def test_load_sample_eval_cases_supports_all_alias():
    sample_ids = available_sample_ids()
    assert sample_ids
    cases = load_sample_eval_cases(["all"])
    case_ids = {str(item.get("_sample_id") or "") for item in cases}
    assert case_ids == set(sample_ids)


def test_grounded_cases_expectations_are_strict_quality_gate():
    rows = load_eval_cases(case_files=[Path("grantflow/eval/cases/grounded_cases.json")])
    assert rows, "Expected grounded eval fixtures"
    for case in rows:
        expectations = case.get("expectations") if isinstance(case.get("expectations"), dict) else {}
        assert expectations.get("min_quality_score") >= 8.0
        assert expectations.get("min_critic_score") >= 8.0
        assert expectations.get("max_fatal_flaws") <= 1
        assert expectations.get("max_high_severity_fatal_flaws") == 0
        assert expectations.get("max_rag_low_confidence_citations") == 0
        assert expectations.get("max_fallback_namespace_citations") == 0
        assert expectations.get("max_architect_fallback_claim_ratio") == 0.0
        assert expectations.get("max_non_retrieval_citation_rate") <= 0.1
        assert expectations.get("max_traceability_gap_citation_rate") <= 0.1


def test_grounded_regression_snapshot_covers_grounded_cases():
    cases = load_eval_cases(case_files=[Path("grantflow/eval/cases/grounded_cases.json")])
    expected_case_ids = {str(case.get("case_id") or "").strip() for case in cases if case.get("case_id")}
    assert expected_case_ids, "Expected non-empty grounded case ids"
    snapshot = json.loads(Path("grantflow/eval/fixtures/grounded_regression_snapshot.json").read_text(encoding="utf-8"))
    snapshot_cases = snapshot.get("cases") if isinstance(snapshot.get("cases"), dict) else {}
    snapshot_case_ids = {str(case_id).strip() for case_id in snapshot_cases}
    assert snapshot_case_ids == expected_case_ids


def test_grounded_tail_regression_snapshot_covers_grounded_tail_cases():
    cases = load_eval_cases(case_files=[Path("grantflow/eval/cases/grounded_tail_cases.json")])
    expected_case_ids = {str(case.get("case_id") or "").strip() for case in cases if case.get("case_id")}
    assert expected_case_ids, "Expected non-empty grounded tail case ids"
    snapshot = json.loads(
        Path("grantflow/eval/fixtures/grounded_tail_regression_snapshot.json").read_text(encoding="utf-8")
    )
    snapshot_cases = snapshot.get("cases") if isinstance(snapshot.get("cases"), dict) else {}
    snapshot_case_ids = {str(case_id).strip() for case_id in snapshot_cases}
    assert snapshot_case_ids == expected_case_ids


def test_llm_grounded_strict_cases_have_grounding_expectations():
    rows = load_eval_cases(case_files=[Path("grantflow/eval/cases/llm_grounded_strict_cases.json")])
    assert rows, "Expected llm grounded strict fixtures"
    for case in rows:
        expectations = case.get("expectations") if isinstance(case.get("expectations"), dict) else {}
        assert expectations.get("min_quality_score") >= 6.5
        assert expectations.get("min_critic_score") >= 6.5
        assert expectations.get("min_retrieval_grounded_citation_rate") >= 0.65
        assert expectations.get("max_non_retrieval_citation_rate") <= 0.35
        assert expectations.get("max_traceability_gap_citation_rate") <= 0.2
        assert expectations.get("max_errors") == 0
        expected_doc_families = case.get("expected_doc_families")
        assert isinstance(expected_doc_families, list) and expected_doc_families


def test_agent_safety_cases_are_manual_contract_checks():
    rows = load_eval_cases(case_files=[Path("grantflow/eval/cases/agent_safety_cases.json")])
    assert rows, "Expected agent safety cases"
    case_ids = [str(case.get("case_id") or "").strip() for case in rows]
    assert len(case_ids) == len(set(case_ids))

    for case in rows:
        assert case.get("manual_only") is True
        assert not case.get("expectations"), "Manual safety cases should not masquerade as automated gates"
        safety_focus = case.get("safety_focus")
        assert isinstance(safety_focus, list) and safety_focus
        expected = case.get("expected_agent_behaviors")
        forbidden = case.get("forbidden_agent_behaviors")
        assert isinstance(expected, list) and len(expected) >= 3
        assert isinstance(forbidden, list) and len(forbidden) >= 2


def test_llm_grounded_strict_regression_snapshot_covers_strict_cases():
    cases = load_eval_cases(case_files=[Path("grantflow/eval/cases/llm_grounded_strict_cases.json")])
    expected_case_ids = {str(case.get("case_id") or "").strip() for case in cases if case.get("case_id")}
    assert expected_case_ids, "Expected non-empty llm strict case ids"
    snapshot = json.loads(
        Path("grantflow/eval/fixtures/llm_grounded_strict_regression_snapshot.json").read_text(encoding="utf-8")
    )
    snapshot_cases = snapshot.get("cases") if isinstance(snapshot.get("cases"), dict) else {}
    snapshot_case_ids = {str(case_id).strip() for case_id in snapshot_cases}
    assert snapshot_case_ids == expected_case_ids


def test_filter_eval_cases_supports_donor_and_case_filters():
    source_cases = [
        {"case_id": "usaid_a", "donor_id": "usaid"},
        {"case_id": "eu_a", "donor_id": "eu"},
        {"case_id": "wb_a", "donor_id": "worldbank"},
    ]
    filtered = filter_eval_cases(source_cases, donor_ids=["usaid", "eu"], case_ids=["eu_a", "usaid_a"])
    assert [case["case_id"] for case in filtered] == ["usaid_a", "eu_a"]


def test_limit_eval_cases_supports_head_and_seeded_sampling():
    source_cases = [
        {"case_id": "c1", "donor_id": "usaid"},
        {"case_id": "c2", "donor_id": "eu"},
        {"case_id": "c3", "donor_id": "worldbank"},
        {"case_id": "c4", "donor_id": "state_department"},
    ]
    head = limit_eval_cases(source_cases, max_cases=2, sample_seed=None)
    seeded = limit_eval_cases(source_cases, max_cases=2, sample_seed=42)
    assert [case["case_id"] for case in head] == ["c1", "c2"]
    assert [case["case_id"] for case in seeded] == ["c1", "c4"]


def test_expected_doc_families_by_donor_merges_and_deduplicates():
    rows = expected_doc_families_by_donor(
        [
            {"case_id": "c1", "donor_id": "usaid", "expected_doc_families": ["donor_policy", "country_context"]},
            {"case_id": "c2", "donor_id": "usaid", "expected_doc_families": ["country_context", "risk_context"]},
            {"case_id": "c3", "donor_id": "worldbank"},
            {"case_id": "c4", "donor_id": "worldbank", "expected_doc_families": ["donor_results_guidance"]},
        ]
    )
    assert rows == {
        "usaid": ["donor_policy", "country_context", "risk_context"],
        "worldbank": ["donor_results_guidance"],
    }


def test_evaluate_seed_readiness_reports_missing_doc_families():
    readiness = evaluate_seed_readiness(
        seeded_summary={
            "donor_doc_family_counts": {
                "usaid": {"donor_policy": 1, "country_context": 0},
                "worldbank": {"donor_results_guidance": 2, "country_context": 1},
            }
        },
        expected_doc_families={
            "usaid": ["donor_policy", "country_context"],
            "worldbank": ["donor_results_guidance", "country_context"],
        },
        min_uploads_per_family=1,
    )
    assert readiness["enabled"] is True
    assert readiness["all_ready"] is False
    assert readiness["expected_donor_count"] == 2
    assert readiness["ready_donor_count"] == 1
    usaid = readiness["donors"]["usaid"]
    assert usaid["ready"] is False
    assert usaid["missing_doc_families"] == ["country_context"]
    assert usaid["ready_doc_families"] == ["donor_policy"]
    assert usaid["coverage_rate"] == 0.5
