"""Test execution service: run_test, run_tests."""

from collections.abc import Awaitable
from collections.abc import Callable
from datetime import datetime
import logging
import uuid

from voicetest.engine.session import ConversationRunner
from voicetest.judges.flow import FlowJudge
from voicetest.judges.flow import FlowResult
from voicetest.judges.metric import MetricJudge
from voicetest.judges.rule import RuleJudge
from voicetest.models.agent import AgentGraph
from voicetest.models.agent import MetricsConfig
from voicetest.models.results import Message
from voicetest.models.results import MetricResult
from voicetest.models.results import ModelOverride
from voicetest.models.results import ModelsUsed
from voicetest.models.results import TestResult
from voicetest.models.results import TestRun
from voicetest.models.test_case import RunOptions
from voicetest.models.test_case import TestCase
from voicetest.services.settings import SettingsService
from voicetest.settings import resolve_model
from voicetest.simulator.user_sim import SimulatorResponse
from voicetest.simulator.user_sim import UserSimulator
from voicetest.util.audio import AudioRoundTrip
from voicetest.util.retry import OnErrorCallback
from voicetest.util.templating import substitute_variables


# Callback type for turn updates
OnTurnCallback = Callable[[list[Message]], Awaitable[None] | None]

# Callback type for token updates: receives token string and source ("agent" or "user")
OnTokenCallback = Callable[[str, str], Awaitable[None] | None]


def resolve_run_options(
    options: RunOptions | None,
    settings_service: SettingsService,
) -> RunOptions:
    """Fill missing model fields from settings.

    If options is None, all run params come from settings too. If options
    is provided, only the None model fields are filled — caller's run
    params (max_turns, timeout, etc.) are respected.
    """
    settings = settings_service.get_settings()
    if options is None:
        return RunOptions(
            agent_model=settings.models.agent,
            simulator_model=settings.models.simulator,
            judge_model=settings.models.judge,
            max_turns=settings.run.max_turns,
            turn_timeout_seconds=settings.run.turn_timeout_seconds,
            verbose=settings.run.verbose,
            flow_judge=settings.run.flow_judge,
            streaming=settings.run.streaming,
            test_model_precedence=settings.run.test_model_precedence,
            audio_eval=settings.run.audio_eval,
            pattern_engine=settings.run.pattern_engine,
        )
    return options.model_copy(
        update={
            "agent_model": options.agent_model or settings.models.agent,
            "simulator_model": options.simulator_model or settings.models.simulator,
            "judge_model": options.judge_model or settings.models.judge,
        }
    )


class TestExecutionService:
    """Runs tests against agent graphs. Stateless."""

    def __init__(self, settings_service: SettingsService):
        self._settings = settings_service

    async def evaluate_global_metrics(
        self,
        transcript: list[Message],
        metrics_config: MetricsConfig,
        judge_model: str | None = None,
        on_error: OnErrorCallback | None = None,
        use_heard: bool = False,
    ) -> list[MetricResult]:
        """Evaluate a transcript against an agent's global metrics.

        Args:
            transcript: Conversation transcript to evaluate.
            metrics_config: Agent metrics configuration with threshold and global metrics.
            judge_model: LLM model to use for evaluation. Reads from settings if None.
            on_error: Optional callback for retry notifications.
            use_heard: If True, use metadata["heard"] for assistant messages.

        Returns:
            List of MetricResult objects for each enabled global metric.
        """
        if judge_model is None:
            judge_model = resolve_model(self._settings.get_settings().models.judge)
        metric_judge = MetricJudge(judge_model)
        threshold = metrics_config.threshold
        results: list[MetricResult] = []

        for gm in metrics_config.global_metrics:
            if gm.enabled:
                gm_threshold = gm.threshold if gm.threshold is not None else threshold
                result = await metric_judge.evaluate(
                    transcript,
                    gm.criteria,
                    threshold=gm_threshold,
                    on_error=on_error,
                    use_heard=use_heard,
                )
                result = result.model_copy(update={"metric": f"[{gm.name}]"})
                results.append(result)

        return results

    async def run_test(
        self,
        graph: AgentGraph,
        test_case: TestCase,
        options: RunOptions | None = None,
        metrics_config: MetricsConfig | None = None,
        _mock_mode: bool = False,
        on_turn: OnTurnCallback | None = None,
        on_token: OnTokenCallback | None = None,
        on_error: OnErrorCallback | None = None,
    ) -> TestResult:
        """Run a single test case against an agent.

        Args:
            graph: The agent graph to test.
            test_case: Test case definition.
            options: Optional run options.
            metrics_config: Optional metrics configuration with threshold and global metrics.
            _mock_mode: If True, use mock responses (for testing).
            on_turn: Optional callback invoked after each turn with current transcript.
            on_token: Optional callback invoked for each token during streaming.
            on_error: Optional callback invoked on retryable errors (e.g., rate limits).

        Returns:
            TestResult with pass/fail status, transcript, and metrics.
        """
        options = resolve_run_options(options, self._settings)
        overrides: list[ModelOverride] = []
        tmp = options.test_model_precedence

        # Resolve models via central precedence chain
        resolved_agent = resolve_model(options.agent_model, graph.default_model, tmp)
        resolved_sim = resolve_model(options.simulator_model, test_case.llm_model, tmp)
        resolved_judge = resolve_model(options.judge_model)

        # Track overrides for diagnostics
        overrides.extend(
            _model_overrides("agent", options.agent_model, graph.default_model, resolved_agent, tmp)
        )
        overrides.extend(
            _model_overrides(
                "simulator", options.simulator_model, test_case.llm_model, resolved_sim, tmp
            )
        )

        options = options.model_copy(
            update={
                "agent_model": resolved_agent,
                "simulator_model": resolved_sim,
                "judge_model": resolved_judge,
            }
        )

        models_used = ModelsUsed(
            agent=options.agent_model,
            simulator=options.simulator_model,
            judge=options.judge_model,
        )

        start_time = datetime.now()

        # Track transcript for error recovery
        error_transcript: list[Message] = []

        # Wrap on_turn to capture transcript
        original_on_turn = on_turn

        async def tracking_on_turn(transcript: list[Message]) -> None:
            error_transcript.clear()
            error_transcript.extend(transcript)
            if original_on_turn:
                result = original_on_turn(transcript)
                if result is not None and hasattr(result, "__await__"):
                    await result

        try:
            # Substitute dynamic variables
            dynamic_vars = test_case.dynamic_variables
            user_prompt = substitute_variables(test_case.user_prompt, dynamic_vars)

            # Setup components
            runner = ConversationRunner(
                graph,
                options,
                mock_mode=_mock_mode,
                dynamic_variables=dynamic_vars,
            )
            simulator = UserSimulator(user_prompt, options.simulator_model)
            metric_judge = MetricJudge(options.judge_model)
            rule_judge = RuleJudge(pattern_engine=options.pattern_engine)
            flow_judge = FlowJudge(options.judge_model)

            # Get threshold from metrics_config or use default
            threshold = metrics_config.threshold if metrics_config else 0.7

            # Enable mock mode for testing
            if _mock_mode:
                simulator._mock_mode = True
                simulator._mock_responses = [
                    SimulatorResponse(message="Hello, I need help."),
                    SimulatorResponse(message="Thanks, that's helpful."),
                ]
                metric_judge._mock_mode = True

                # Build mock results for test metrics + enabled global metrics
                mock_results = [
                    MetricResult(
                        metric=m,
                        score=0.9,
                        passed=True,
                        reasoning="Mock evaluation",
                        threshold=threshold,
                        confidence=0.9,
                    )
                    for m in test_case.metrics
                ]
                if metrics_config:
                    for gm in metrics_config.global_metrics:
                        if gm.enabled:
                            gm_threshold = gm.threshold if gm.threshold is not None else threshold
                            mock_results.append(
                                MetricResult(
                                    metric=f"[{gm.name}]",
                                    score=0.9,
                                    passed=True,
                                    reasoning="Mock global metric evaluation",
                                    threshold=gm_threshold,
                                    confidence=0.9,
                                )
                            )
                metric_judge._mock_results = mock_results

                flow_judge._mock_mode = True
                flow_judge._mock_result = FlowResult(
                    valid=True, issues=[], reasoning="Mock flow validation"
                )

            # Run conversation
            state = await runner.run(
                test_case, simulator, on_turn=tracking_on_turn, on_token=on_token, on_error=on_error
            )

            # Evaluate based on test type
            test_type = test_case.effective_type
            if test_type == "rule":
                # Rule-based evaluation (deterministic pattern matching)
                metric_results = await rule_judge.evaluate(
                    state.transcript,
                    test_case.includes,
                    test_case.excludes,
                    test_case.patterns,
                )
            else:
                # LLM-based evaluation (semantic metrics)
                metric_results = await metric_judge.evaluate_all(
                    state.transcript, test_case.metrics, threshold=threshold, on_error=on_error
                )

            # Evaluate global metrics if configured
            if metrics_config:
                global_results = await self.evaluate_global_metrics(
                    state.transcript,
                    metrics_config,
                    judge_model=options.judge_model,
                    on_error=on_error,
                )
                metric_results.extend(global_results)

            # Audio evaluation (when enabled)
            audio_metric_results: list[MetricResult] = []
            if options.audio_eval:
                try:
                    audio_rt = AudioRoundTrip.from_settings()
                    state.transcript = await audio_rt.transform_transcript(state.transcript)

                    if test_type == "rule":
                        audio_metric_results = await rule_judge.evaluate(
                            state.transcript,
                            test_case.includes,
                            test_case.excludes,
                            test_case.patterns,
                            use_heard=True,
                        )
                    else:
                        audio_metric_results = await metric_judge.evaluate_all(
                            state.transcript,
                            test_case.metrics,
                            threshold=threshold,
                            on_error=on_error,
                            use_heard=True,
                        )

                    if metrics_config:
                        audio_global_results = await self.evaluate_global_metrics(
                            state.transcript,
                            metrics_config,
                            judge_model=options.judge_model,
                            on_error=on_error,
                            use_heard=True,
                        )
                        audio_metric_results.extend(audio_global_results)

                    await audio_rt.close()
                except Exception:
                    logging.getLogger(__name__).exception("Audio evaluation failed")

            # Check flow constraints (optional, informational only)
            flow_issues: list[str] = []
            if options.flow_judge:
                flow_result = await flow_judge.evaluate(
                    graph, state.transcript, state.nodes_visited, on_error=on_error
                )
                flow_issues = flow_result.issues

            # Determine overall status (based on metrics only)
            metrics_passed = all(r.passed for r in metric_results)
            status = "pass" if metrics_passed else "fail"

            duration_ms = int((datetime.now() - start_time).total_seconds() * 1000)

            return TestResult(
                test_id=test_case.name,
                test_name=test_case.name,
                status=status,
                transcript=state.transcript,
                metric_results=metric_results,
                audio_metric_results=audio_metric_results,
                nodes_visited=state.nodes_visited,
                tools_called=state.tools_called,
                constraint_violations=flow_issues,
                turn_count=state.turn_count,
                duration_ms=duration_ms,
                end_reason=state.end_reason,
                models_used=models_used,
                model_overrides=overrides,
            )

        except BaseExceptionGroup as eg:
            # DSPy streamify wraps exceptions in ExceptionGroup - unwrap to surface real error
            duration_ms = int((datetime.now() - start_time).total_seconds() * 1000)
            real_error = eg.exceptions[0] if eg.exceptions else eg
            return TestResult(
                test_id=test_case.name,
                test_name=test_case.name,
                status="error",
                transcript=error_transcript,
                duration_ms=duration_ms,
                error_message=str(real_error),
                models_used=models_used,
                model_overrides=overrides,
            )
        except Exception as e:
            duration_ms = int((datetime.now() - start_time).total_seconds() * 1000)
            return TestResult(
                test_id=test_case.name,
                test_name=test_case.name,
                status="error",
                transcript=error_transcript,
                duration_ms=duration_ms,
                error_message=str(e),
                models_used=models_used,
                model_overrides=overrides,
            )

    async def run_tests(
        self,
        graph: AgentGraph,
        test_cases: list[TestCase],
        options: RunOptions | None = None,
        _mock_mode: bool = False,
    ) -> TestRun:
        """Run multiple test cases, return aggregated results.

        Args:
            graph: The agent graph to test.
            test_cases: List of test case definitions.
            options: Optional run options.
            _mock_mode: If True, use mock responses (for testing).

        Returns:
            TestRun with aggregated results.
        """
        run_id = str(uuid.uuid4())
        started_at = datetime.now()

        options = resolve_run_options(options, self._settings)
        results = []
        for test_case in test_cases:
            result = await self.run_test(graph, test_case, options, _mock_mode=_mock_mode)
            results.append(result)

        return TestRun(
            run_id=run_id,
            started_at=started_at,
            completed_at=datetime.now(),
            results=results,
        )


def _model_overrides(
    role: str,
    settings_value: str | None,
    role_default: str | None,
    resolved: str,
    test_model_precedence: bool,
) -> list[ModelOverride]:
    """Generate diagnostic override records when resolved model differs from inputs."""
    if not role_default or not settings_value:
        if role_default and not settings_value:
            return [
                ModelOverride(
                    role=role,
                    requested="(not configured)",
                    actual=role_default,
                    reason=f"role default used (global {role} not configured)",
                )
            ]
        return []
    if role_default == settings_value:
        return []
    if resolved == role_default:
        return [
            ModelOverride(
                role=role,
                requested=settings_value,
                actual=role_default,
                reason="role default used (test_model_precedence enabled)",
            )
        ]
    return [
        ModelOverride(
            role=role,
            requested=role_default,
            actual=settings_value,
            reason=f"global settings override {role} default",
        )
    ]
