"""Runtime evaluation harnesses for live Sibyl search surfaces."""

from __future__ import annotations

import json
import re
import time
from dataclasses import dataclass, field, replace
from pathlib import Path
from typing import Any, Literal

import httpx
import structlog

from sibyl_core.evals.context import (
    ContextPackCaseResult,
    ContextPackEvalCase,
    ContextPackEvalReport,
    ContextPackEvalResult,
    context_pack_from_dict,
    evaluate_context_pack,
    get_sample_context_pack_cases,
    load_context_pack_cases,
)
from sibyl_core.evals.metrics import (
    EvalMetrics,
    EvalQuery,
    RetrievalResult,
    aggregate_metrics,
    compute_metrics,
)

log = structlog.get_logger()
SearchType = Literal["unified", "rag", "hybrid", "code-examples"]


@dataclass
class EvalConfig:
    """Configuration for an evaluation run."""

    api_base_url: str = "http://localhost:3334/api"
    headers: dict[str, str] = field(default_factory=lambda: {"Content-Type": "application/json"})
    k_values: list[int] = field(default_factory=lambda: [1, 3, 5, 10])
    timeout_seconds: float = 30.0
    output_dir: Path = field(default_factory=lambda: Path("eval_results"))
    save_results: bool = True
    label: str | None = None
    metadata: dict[str, Any] = field(default_factory=dict)


@dataclass
class EvalResult:
    """Result from evaluating a single query."""

    query: EvalQuery
    results: list[RetrievalResult]
    metrics: EvalMetrics
    error: str | None = None


@dataclass
class EvalReport:
    """Complete evaluation report."""

    config: EvalConfig
    queries: list[EvalResult]
    aggregated: EvalMetrics
    search_type: SearchType
    timestamp: str = field(default_factory=lambda: time.strftime("%Y-%m-%d %H:%M:%S"))

    def to_dict(self) -> dict[str, Any]:
        """Convert the report to a JSON-safe dictionary."""
        return {
            "timestamp": self.timestamp,
            "label": self.config.label,
            "api_base_url": self.config.api_base_url,
            "search_type": self.search_type,
            "num_queries": len(self.queries),
            "metadata": dict(self.config.metadata),
            "metrics": self.aggregated.to_dict(),
            "per_query": [
                {
                    "query": result.query.query,
                    "metrics": result.metrics.to_dict(),
                    "error": result.error,
                }
                for result in self.queries
            ],
        }

    def save(self, path: Path | None = None) -> Path:
        """Save the report to disk."""
        if path is None:
            self.config.output_dir.mkdir(parents=True, exist_ok=True)
            label = ""
            if self.config.label:
                slug = re.sub(r"[^a-z0-9]+", "_", self.config.label.lower()).strip("_")
                if slug:
                    label = f"_{slug}"
            filename = f"eval_{self.search_type}{label}_{time.strftime('%Y%m%d_%H%M%S')}.json"
            path = self.config.output_dir / filename

        path.write_text(json.dumps(self.to_dict(), indent=2), encoding="utf-8")
        log.info("Saved evaluation report", path=str(path))
        return path

    def print_summary(self) -> None:
        """Print a compact summary for interactive runs."""
        metrics = self.aggregated.to_dict()
        print(f"\nSibyl evaluation summary ({self.search_type})")
        print(f"  queries: {len(self.queries)}")
        print(f"  ndcg@10: {metrics['ndcg@10']:.3f}")
        print(f"  success@5: {metrics['success@5']:.3f}")
        print(f"  mrr: {metrics['mrr']:.3f}")
        print(f"  latency_ms: {metrics['latency_ms']:.1f}")


def load_queries(path: Path) -> list[EvalQuery]:
    """Load evaluation queries from a JSON file."""
    data = json.loads(path.read_text(encoding="utf-8"))
    return [
        EvalQuery(
            query=item["query"],
            expected_ids=item.get("expected_ids", []),
            relevance_grades=item.get("relevance_grades", {}),
            metadata=item.get("metadata", {}),
        )
        for item in data["queries"]
    ]


def get_sample_queries() -> list[EvalQuery]:
    """Return sample queries for smoke-testing the harness itself."""
    return [
        EvalQuery(
            query="How to install FastAPI",
            expected_ids=["fastapi-install-1", "fastapi-quickstart-1"],
            relevance_grades={"fastapi-install-1": 3, "fastapi-quickstart-1": 2},
        ),
        EvalQuery(
            query="authentication best practices",
            expected_ids=["auth-patterns-1", "security-guide-1", "jwt-setup-1"],
            relevance_grades={
                "auth-patterns-1": 3,
                "security-guide-1": 2,
                "jwt-setup-1": 2,
            },
        ),
        EvalQuery(
            query="database connection pooling Python",
            expected_ids=["sqlalchemy-pool-1", "async-db-1"],
            relevance_grades={"sqlalchemy-pool-1": 3, "async-db-1": 2},
        ),
        EvalQuery(
            query="error handling patterns async await",
            expected_ids=["async-errors-1", "exception-patterns-1"],
            relevance_grades={"async-errors-1": 3, "exception-patterns-1": 2},
        ),
        EvalQuery(
            query="GraphQL subscription implementation",
            expected_ids=["graphql-subscriptions-1"],
            relevance_grades={"graphql-subscriptions-1": 3},
        ),
    ]


class EvalRunner:
    """Runner for live Sibyl evaluation queries."""

    def __init__(self, config: EvalConfig | None = None):
        self.config = config or EvalConfig()
        self._http_client: httpx.AsyncClient | Any | None = None

    async def _get_client(self) -> httpx.AsyncClient | Any:
        if self._http_client is None:
            self._http_client = httpx.AsyncClient(
                base_url=self.config.api_base_url,
                headers=self.config.headers,
                timeout=self.config.timeout_seconds,
            )
        return self._http_client

    async def close(self) -> None:
        """Close the underlying HTTP client if needed."""
        if self._http_client is not None and hasattr(self._http_client, "aclose"):
            await self._http_client.aclose()
        self._http_client = None

    def _build_request(
        self, query: EvalQuery, search_type: SearchType
    ) -> tuple[str, dict[str, Any]]:
        if search_type == "unified":
            return "/search", {"query": query.query, "limit": 20, "include_content": True}
        if search_type == "code-examples":
            return "/rag/code-examples", {"query": query.query, "match_count": 20}
        if search_type == "hybrid":
            return "/rag/hybrid-search", {"query": query.query, "match_count": 20}
        return "/rag/search", {"query": query.query, "match_count": 20}

    def _parse_results(
        self, data: dict[str, Any], search_type: SearchType
    ) -> list[RetrievalResult]:
        if search_type == "code-examples":
            return [
                RetrievalResult(
                    id=item.get("chunk_id", ""),
                    content=item.get("code", ""),
                    score=item.get("similarity", 0.0),
                    metadata={"language": item.get("language"), "url": item.get("url")},
                )
                for item in data.get("examples", [])
            ]

        if search_type == "unified":
            return [
                RetrievalResult(
                    id=item.get("id", ""),
                    content=item.get("content", ""),
                    score=item.get("score", 0.0),
                    metadata={
                        "type": item.get("type"),
                        "result_origin": item.get("result_origin"),
                        "source": item.get("source"),
                        "url": item.get("url"),
                    },
                )
                for item in data.get("results", [])
            ]

        return [
            RetrievalResult(
                id=item.get("chunk_id", item.get("document_id", "")),
                content=item.get("content", ""),
                score=item.get("similarity", 0.0),
                metadata={
                    "document_id": item.get("document_id"),
                    "source_id": item.get("source_id"),
                    "source_name": item.get("source_name"),
                    "url": item.get("url"),
                },
            )
            for item in data.get("results", [])
        ]

    async def run_query(self, query: EvalQuery, search_type: SearchType = "unified") -> EvalResult:
        """Run a single evaluation query against a live Sibyl endpoint."""
        client = await self._get_client()
        endpoint, payload = self._build_request(query, search_type)

        start_time = time.perf_counter()
        results: list[RetrievalResult] = []
        error = None

        try:
            response = await client.post(endpoint, json=payload)
            response.raise_for_status()
            results = self._parse_results(response.json(), search_type)
        except Exception as exc:
            error = str(exc)
            log.exception("evaluation_query_failed", query=query.query, error=error)

        latency_ms = (time.perf_counter() - start_time) * 1000
        metrics = compute_metrics(
            results=results,
            query=query,
            latency_ms=latency_ms,
            k_values=self.config.k_values,
        )
        return EvalResult(query=query, results=results, metrics=metrics, error=error)

    async def run_evaluation(
        self,
        queries: list[EvalQuery],
        search_type: SearchType = "unified",
    ) -> EvalReport:
        """Run a full evaluation and aggregate the results."""
        log.info("Starting evaluation", num_queries=len(queries), search_type=search_type)
        results = []
        for index, query in enumerate(queries, start=1):
            log.debug("Running query", index=index, total=len(queries), query=query.query[:50])
            results.append(await self.run_query(query, search_type))

        report = EvalReport(
            config=self.config,
            queries=results,
            aggregated=aggregate_metrics([result.metrics for result in results]),
            search_type=search_type,
        )
        if self.config.save_results:
            report.save()

        log.info(
            "Evaluation complete",
            num_queries=len(queries),
            search_type=search_type,
            ndcg_at_10=report.aggregated.ndcg_at_k.get(10, 0.0),
            success_at_5=report.aggregated.success_at_k.get(5, 0.0),
        )
        return report

    @staticmethod
    def _build_context_pack_request(case: ContextPackEvalCase) -> dict[str, Any]:
        payload: dict[str, Any] = {
            "goal": case.goal,
            "intent": case.intent.value,
            "layer": case.layer.value,
            "limit": case.limit,
            "include_related": case.include_related,
            "related_limit": case.related_limit,
        }
        if case.domain:
            payload["domain"] = case.domain
        if case.project:
            payload["project"] = case.project
        if case.agent_id:
            payload["agent_id"] = case.agent_id
        return payload

    @staticmethod
    def _apply_context_pack_latency_guard(
        result: ContextPackEvalResult,
        case: ContextPackEvalCase,
        latency_ms: float,
    ) -> ContextPackEvalResult:
        metrics = {**result.metrics, "latency_ms": latency_ms}
        if case.fixture.max_latency_ms is None or latency_ms <= case.fixture.max_latency_ms:
            return replace(result, metrics=metrics)
        failures = [
            *result.failures,
            f"latency too high: {latency_ms:.1f} ms > {case.fixture.max_latency_ms:.1f} ms",
        ]
        return replace(result, passed=False, failures=failures, metrics=metrics)

    async def run_context_pack_case(self, case: ContextPackEvalCase) -> ContextPackCaseResult:
        """Run one context-pack fixture against the live API."""

        client = await self._get_client()
        start_time = time.perf_counter()
        error = None
        try:
            response = await client.post(
                "/context/pack",
                json=self._build_context_pack_request(case),
            )
            response.raise_for_status()
            pack = context_pack_from_dict(response.json())
            result = evaluate_context_pack(pack, case.fixture)
        except Exception as exc:
            error = str(exc) or exc.__class__.__name__
            log.warning("context_pack_evaluation_failed", case=case.name, error=error)
            result = ContextPackEvalResult(
                fixture=case.fixture.name,
                passed=False,
                failures=[error],
            )

        latency_ms = (time.perf_counter() - start_time) * 1000
        result = self._apply_context_pack_latency_guard(result, case, latency_ms)
        return ContextPackCaseResult(
            case=case,
            result=result,
            latency_ms=latency_ms,
            error=error,
        )

    async def run_context_pack_evaluation(
        self,
        cases: list[ContextPackEvalCase],
        *,
        repeat_count: int = 1,
    ) -> ContextPackEvalReport:
        """Run context-pack evaluation cases against a live Sibyl endpoint."""

        if repeat_count < 1:
            msg = "repeat_count must be at least 1"
            raise ValueError(msg)

        results: list[ContextPackCaseResult] = []
        for repeat_index in range(1, repeat_count + 1):
            for case in cases:
                result = await self.run_context_pack_case(case)
                results.append(replace(result, repeat_index=repeat_index))
        metadata = {
            **dict(self.config.metadata),
            "repeat_count": str(repeat_count),
            "case_count_per_repeat": str(len(cases)),
        }
        report = ContextPackEvalReport(
            cases=results,
            label=self.config.label,
            metadata=metadata,
        )
        if self.config.save_results:
            report.save(self.config.output_dir)
        return report

    async def __aenter__(self) -> EvalRunner:
        return self

    async def __aexit__(self, *args: object) -> None:
        await self.close()


async def run_evaluation_cli(
    queries_file: Path | None = None,
    search_type: SearchType = "unified",
    config: EvalConfig | None = None,
) -> EvalReport:
    """Run an evaluation from a query file or the built-in sample set."""
    queries = load_queries(queries_file) if queries_file else get_sample_queries()
    async with EvalRunner(config) as runner:
        report = await runner.run_evaluation(queries, search_type)
        report.print_summary()
        return report


async def run_context_pack_evaluation_cli(
    cases_file: Path | None = None,
    config: EvalConfig | None = None,
    *,
    repeat_count: int = 1,
) -> ContextPackEvalReport:
    """Run context-pack evals from a case file or the smoke-test sample."""

    cases = load_context_pack_cases(cases_file) if cases_file else get_sample_context_pack_cases()
    async with EvalRunner(config) as runner:
        report = await runner.run_context_pack_evaluation(cases, repeat_count=repeat_count)
        report.print_summary()
        return report
