from __future__ import annotations

import json

from grantflow.eval import harness
from grantflow.eval.harness import (
    build_regression_baseline_snapshot,
    compare_suite_to_baseline,
    format_eval_comparison_report,
    load_eval_cases,
    run_eval_suite,
)


def test_eval_harness_regression_comparison_flags_only_degradations():
    suite = {
        "cases": [
            {
                "case_id": "case_a",
                "donor_id": "usaid",
                "metrics": {
                    "quality_score": 9.0,
                    "critic_score": 8.5,
                    "citations_total": 4,
                    "architect_citation_count": 3,
                    "mel_citation_count": 1,
                    "high_confidence_citation_count": 1,
                    "architect_threshold_hit_rate": 0.5,
                    "citation_confidence_avg": 0.4,
                    "low_confidence_citation_count": 1,
                    "rag_low_confidence_citation_count": 0,
                    "fallback_namespace_citation_count": 0,
                    "draft_version_count": 2,
                    "fatal_flaw_count": 0,
                    "high_severity_fatal_flaw_count": 0,
                    "error_count": 0,
                    "toc_schema_valid": True,
                    "has_toc_draft": True,
                    "has_logframe_draft": True,
                    "needs_revision": False,
                },
            },
            {
                "case_id": "new_case",
                "donor_id": "eu",
                "metrics": {
                    "quality_score": 9.5,
                    "critic_score": 9.0,
                    "citations_total": 5,
                    "architect_citation_count": 4,
                    "mel_citation_count": 1,
                    "high_confidence_citation_count": 2,
                    "architect_threshold_hit_rate": 0.8,
                    "citation_confidence_avg": 0.6,
                    "low_confidence_citation_count": 0,
                    "rag_low_confidence_citation_count": 0,
                    "fallback_namespace_citation_count": 0,
                    "draft_version_count": 2,
                    "fatal_flaw_count": 0,
                    "high_severity_fatal_flaw_count": 0,
                    "error_count": 0,
                    "toc_schema_valid": True,
                    "has_toc_draft": True,
                    "has_logframe_draft": True,
                    "needs_revision": False,
                },
            },
        ]
    }
    baseline = {
        "cases": {
            "case_a": {
                "donor_id": "usaid",
                "metrics": {
                    "quality_score": 9.25,  # higher-is-better regression
                    "critic_score": 8.0,  # improvement, should not fail
                    "citations_total": 4,
                    "architect_citation_count": 2,
                    "mel_citation_count": 1,
                    "high_confidence_citation_count": 0,
                    "architect_threshold_hit_rate": 0.6,
                    "citation_confidence_avg": 0.3,
                    "low_confidence_citation_count": 1,
                    "rag_low_confidence_citation_count": 0,
                    "fallback_namespace_citation_count": 0,
                    "draft_version_count": 2,
                    "fatal_flaw_count": 0,
                    "high_severity_fatal_flaw_count": 0,
                    "error_count": 0,
                    "toc_schema_valid": True,
                    "has_toc_draft": True,
                    "has_logframe_draft": True,
                    "needs_revision": False,
                },
            },
            "missing_now": {
                "donor_id": "giz",
                "metrics": {},
            },
        }
    }

    comparison = compare_suite_to_baseline(suite, baseline)
    assert comparison["has_regressions"] is True
    assert comparison["regression_count"] == 2
    assert {item["metric"] for item in comparison["regressions"]} == {"quality_score", "architect_threshold_hit_rate"}
    assert comparison["warning_count"] >= 2  # new case + missing baseline case
    assert comparison["donor_breakdown"]["usaid"]["regression_count"] == 2
    assert comparison["donor_breakdown"]["eu"]["warning_count"] >= 1
    assert comparison["donor_breakdown"]["giz"]["warning_count"] >= 1
    assert comparison["severity_weighted_regression_score"] >= 6
    assert comparison["high_priority_regression_count"] >= 1
    assert comparison["priority_metric_breakdown"]["architect_threshold_hit_rate"]["weight"] >= 4
    assert comparison["donor_priority_breakdown"]["usaid"]["weighted_score"] >= 1

    text = format_eval_comparison_report(comparison)
    assert "Donor regression breakdown" in text
    assert "usaid: regressions=2" in text
    assert "Severity-weighted regression summary" in text
    assert "Top donor weighted risk" in text


def test_eval_harness_regression_comparison_can_ignore_missing_baseline_cases():
    suite = {
        "cases": [
            {
                "case_id": "case_a",
                "donor_id": "usaid",
                "metrics": {
                    "quality_score": 9.0,
                    "critic_score": 8.5,
                    "citations_total": 4,
                    "architect_citation_count": 3,
                    "mel_citation_count": 1,
                    "high_confidence_citation_count": 1,
                    "architect_threshold_hit_rate": 0.5,
                    "citation_confidence_avg": 0.4,
                    "low_confidence_citation_count": 1,
                    "rag_low_confidence_citation_count": 0,
                    "fallback_namespace_citation_count": 0,
                    "draft_version_count": 2,
                    "fatal_flaw_count": 0,
                    "high_severity_fatal_flaw_count": 0,
                    "error_count": 0,
                    "toc_schema_valid": True,
                    "has_toc_draft": True,
                    "has_logframe_draft": True,
                    "needs_revision": False,
                },
            }
        ]
    }
    baseline = {
        "cases": {
            "case_a": {"donor_id": "usaid", "metrics": {"quality_score": 8.0}},
            "missing_now": {"donor_id": "eu", "metrics": {"quality_score": 8.0}},
        }
    }

    with_missing_warning = compare_suite_to_baseline(suite, baseline)
    without_missing_warning = compare_suite_to_baseline(
        suite,
        baseline,
        ignore_missing_current_cases=True,
    )
    assert with_missing_warning["warning_count"] >= 1
    assert without_missing_warning["warning_count"] == 0
    assert without_missing_warning["has_regressions"] is False


def test_eval_harness_cli_can_write_baseline_and_comparison_reports(tmp_path):
    suite_cases = load_eval_cases()
    suite = run_eval_suite(suite_cases)
    baseline = build_regression_baseline_snapshot(suite)
    baseline_path = tmp_path / "baseline.json"
    baseline_path.write_text(json.dumps(baseline, indent=2), encoding="utf-8")

    json_out = tmp_path / "eval-report.json"
    text_out = tmp_path / "eval-report.txt"
    cmp_json_out = tmp_path / "eval-compare.json"
    cmp_text_out = tmp_path / "eval-compare.txt"
    regenerated_baseline_out = tmp_path / "baseline-regenerated.json"

    exit_code = harness.main(
        [
            "--json-out",
            str(json_out),
            "--text-out",
            str(text_out),
            "--compare-to-baseline",
            str(baseline_path),
            "--comparison-json-out",
            str(cmp_json_out),
            "--comparison-text-out",
            str(cmp_text_out),
            "--baseline-snapshot-out",
            str(regenerated_baseline_out),
        ]
    )
    assert exit_code == 0
    comparison = json.loads(cmp_json_out.read_text(encoding="utf-8"))
    assert comparison["has_regressions"] is False
    assert comparison["regression_count"] == 0
    assert comparison["severity_weighted_regression_score"] == 0
    assert regenerated_baseline_out.exists()
    cmp_text = cmp_text_out.read_text(encoding="utf-8")
    assert "No regressions detected" in cmp_text
