"""
API functions for benchmarking.

This module provides functions for running benchmarks programmatically.
"""

from pathlib import Path
from typing import Any, Dict, List, Optional

from loguru import logger

from ..config.thread_settings import get_setting_from_snapshot
from ..llm.providers.base import normalize_provider

from ..benchmarks import (
    calculate_metrics,
    generate_report,
    run_benchmark,
    run_browsecomp_benchmark,
    run_simpleqa_benchmark,
    run_xbench_deepsearch_benchmark,
)


def evaluate_simpleqa(
    num_examples: int = 100,
    search_iterations: int = 3,
    questions_per_iteration: int = 3,
    search_tool: str = "searxng",
    human_evaluation: bool = False,
    evaluation_model: Optional[str] = None,
    evaluation_provider: Optional[str] = None,
    output_dir: str = "benchmark_results",
    search_model: Optional[str] = None,
    search_provider: Optional[str] = None,
    endpoint_url: Optional[str] = None,
    search_strategy: str = "source_based",
) -> Dict[str, Any]:
    """
    Run SimpleQA benchmark evaluation.

    Args:
        num_examples: Number of examples to evaluate
        search_iterations: Number of search iterations per query
        questions_per_iteration: Number of questions per iteration
        search_tool: Search engine to use (e.g., 'searxng', 'wikipedia')
        human_evaluation: Whether to use human evaluation
        evaluation_model: Optional custom model for evaluation
        evaluation_provider: Optional custom provider for evaluation
        output_dir: Directory to save results
        search_model: Optional model to use for the search system
        search_provider: Optional provider to use for the search system
        endpoint_url: Optional endpoint URL for OpenRouter or other API services
        search_strategy: Search strategy to use (default: 'source_based')

    Returns:
        Dictionary with benchmark results
    """
    logger.info(f"Starting SimpleQA benchmark with {num_examples} examples")

    # Set up search configuration
    search_config = {
        "iterations": search_iterations,
        "questions_per_iteration": questions_per_iteration,
        "search_tool": search_tool,
        "search_strategy": search_strategy,
    }

    # Add model configurations if provided
    if search_model:
        search_config["model_name"] = search_model
    if search_provider:
        search_config["provider"] = search_provider
    if endpoint_url:
        search_config["openai_endpoint_url"] = endpoint_url

    # Check settings for additional configuration
    if env_model := get_setting_from_snapshot("llm.model"):
        search_config["model_name"] = env_model
    if env_provider := get_setting_from_snapshot("llm.provider"):
        search_config["provider"] = env_provider
    if env_url := get_setting_from_snapshot("llm.openai_endpoint.url"):
        search_config["openai_endpoint_url"] = env_url

    # Set up evaluation configuration if needed
    evaluation_config = None
    if evaluation_model or evaluation_provider:
        evaluation_config = {
            "temperature": 0  # Always use zero temperature for evaluation
        }
        if evaluation_model:
            evaluation_config["model_name"] = evaluation_model
        if evaluation_provider:
            evaluation_provider = normalize_provider(evaluation_provider)
            evaluation_config["provider"] = evaluation_provider
            # Add endpoint URL if using openai_endpoint
            if evaluation_provider == "openai_endpoint" and endpoint_url:
                evaluation_config["openai_endpoint_url"] = endpoint_url
            elif evaluation_provider == "openai_endpoint" and env_url:
                evaluation_config["openai_endpoint_url"] = env_url

    # Run the benchmark
    return run_simpleqa_benchmark(
        num_examples=num_examples,
        output_dir=output_dir,
        search_config=search_config,
        evaluation_config=evaluation_config,
        human_evaluation=human_evaluation,
    )


def evaluate_browsecomp(
    num_examples: int = 100,
    search_iterations: int = 3,
    questions_per_iteration: int = 3,
    search_tool: str = "searxng",
    human_evaluation: bool = False,
    evaluation_model: Optional[str] = None,
    evaluation_provider: Optional[str] = None,
    output_dir: str = "benchmark_results",
    search_model: Optional[str] = None,
    search_provider: Optional[str] = None,
    endpoint_url: Optional[str] = None,
    search_strategy: str = "source_based",
) -> Dict[str, Any]:
    """
    Run BrowseComp benchmark evaluation.

    Args:
        num_examples: Number of examples to evaluate
        search_iterations: Number of search iterations per query
        questions_per_iteration: Number of questions per iteration
        search_tool: Search engine to use (e.g., 'searxng', 'wikipedia')
        human_evaluation: Whether to use human evaluation
        evaluation_model: Optional custom model for evaluation
        evaluation_provider: Optional custom provider for evaluation
        output_dir: Directory to save results
        search_model: Optional model to use for the search system
        search_provider: Optional provider to use for the search system
        endpoint_url: Optional endpoint URL for OpenRouter or other API services
        search_strategy: Search strategy to use (default: 'source_based')

    Returns:
        Dictionary with benchmark results
    """
    logger.info(f"Starting BrowseComp benchmark with {num_examples} examples")

    # Set up search configuration
    search_config = {
        "iterations": search_iterations,
        "questions_per_iteration": questions_per_iteration,
        "search_tool": search_tool,
        "search_strategy": search_strategy,
    }

    # Add model configurations if provided
    if search_model:
        search_config["model_name"] = search_model
    if search_provider:
        search_config["provider"] = search_provider
    if endpoint_url:
        search_config["openai_endpoint_url"] = endpoint_url

    # Check settings for additional configuration
    if env_model := get_setting_from_snapshot("llm.model"):
        search_config["model_name"] = env_model
    if env_provider := get_setting_from_snapshot("llm.provider"):
        search_config["provider"] = env_provider
    if env_url := get_setting_from_snapshot("llm.openai_endpoint.url"):
        search_config["openai_endpoint_url"] = env_url

    # Set up evaluation configuration if needed
    evaluation_config = None
    if evaluation_model or evaluation_provider:
        evaluation_config = {
            "temperature": 0  # Always use zero temperature for evaluation
        }
        if evaluation_model:
            evaluation_config["model_name"] = evaluation_model
        if evaluation_provider:
            evaluation_provider = normalize_provider(evaluation_provider)
            evaluation_config["provider"] = evaluation_provider
            # Add endpoint URL if using openai_endpoint
            if evaluation_provider == "openai_endpoint" and endpoint_url:
                evaluation_config["openai_endpoint_url"] = endpoint_url
            elif evaluation_provider == "openai_endpoint" and env_url:
                evaluation_config["openai_endpoint_url"] = env_url

    # Run the benchmark
    return run_browsecomp_benchmark(
        num_examples=num_examples,
        output_dir=output_dir,
        search_config=search_config,
        evaluation_config=evaluation_config,
        human_evaluation=human_evaluation,
    )


def evaluate_xbench_deepsearch(
    num_examples: int = 100,
    search_iterations: int = 4,
    questions_per_iteration: int = 3,
    search_tool: str = "searxng",
    human_evaluation: bool = False,
    evaluation_model: Optional[str] = None,
    evaluation_provider: Optional[str] = None,
    output_dir: str = "benchmark_results",
    search_model: Optional[str] = None,
    search_provider: Optional[str] = None,
    endpoint_url: Optional[str] = None,
    search_strategy: str = "source_based",
) -> Dict[str, Any]:
    """
    Run xbench-DeepSearch benchmark evaluation.

    Args:
        num_examples: Number of examples to evaluate (default 100 - full dataset)
        search_iterations: Number of search iterations per query
        questions_per_iteration: Number of questions per iteration
        search_tool: Search engine to use
        human_evaluation: Whether to use human evaluation
        evaluation_model: Optional custom model for evaluation
        evaluation_provider: Optional custom provider for evaluation
        output_dir: Directory to save results
        search_model: Optional model to use for the search system
        search_provider: Optional provider to use for the search system
        endpoint_url: Optional endpoint URL for API services
        search_strategy: Search strategy to use

    Returns:
        Dictionary with benchmark results
    """
    logger.info(
        f"Starting xbench-DeepSearch benchmark with {num_examples} examples"
    )

    # Set up search configuration
    search_config = {
        "iterations": search_iterations,
        "questions_per_iteration": questions_per_iteration,
        "search_tool": search_tool,
        "search_strategy": search_strategy,
    }

    # Add model configurations if provided
    if search_model:
        search_config["model_name"] = search_model
    if search_provider:
        search_config["provider"] = search_provider
    if endpoint_url:
        search_config["openai_endpoint_url"] = endpoint_url

    # Check settings for additional configuration
    if env_model := get_setting_from_snapshot("llm.model"):
        search_config["model_name"] = env_model
    if env_provider := get_setting_from_snapshot("llm.provider"):
        search_config["provider"] = env_provider
    if env_url := get_setting_from_snapshot("llm.openai_endpoint.url"):
        search_config["openai_endpoint_url"] = env_url

    # Set up evaluation configuration if needed
    evaluation_config = None
    if evaluation_model or evaluation_provider:
        evaluation_config = {
            "temperature": 0  # Always use zero temperature for evaluation
        }
        if evaluation_model:
            evaluation_config["model_name"] = evaluation_model
        if evaluation_provider:
            evaluation_provider = normalize_provider(evaluation_provider)
            evaluation_config["provider"] = evaluation_provider
            # Add endpoint URL if using openai_endpoint
            if evaluation_provider == "openai_endpoint" and endpoint_url:
                evaluation_config["openai_endpoint_url"] = endpoint_url
            elif evaluation_provider == "openai_endpoint" and env_url:
                evaluation_config["openai_endpoint_url"] = env_url

    # Run the benchmark
    return run_xbench_deepsearch_benchmark(
        num_examples=num_examples,
        output_dir=output_dir,
        search_config=search_config,
        evaluation_config=evaluation_config,
        human_evaluation=human_evaluation,
    )


def get_available_benchmarks() -> List[Dict[str, str]]:
    """
    Get information about available benchmarks.

    Returns:
        List of dictionaries with benchmark information
    """
    return [
        {
            "id": "simpleqa",
            "name": "SimpleQA",
            "description": "Benchmark for factual question answering",
            "recommended_examples": 100,
        },
        {
            "id": "browsecomp",
            "name": "BrowseComp",
            "description": "Benchmark for web browsing comprehension",
            "recommended_examples": 100,
        },
        {
            "id": "xbench_deepsearch",
            "name": "xbench-DeepSearch",
            "description": "Deep research and search capability evaluation",
            "recommended_examples": 100,
        },
    ]


def compare_configurations(
    dataset_type: str = "simpleqa",
    num_examples: int = 20,
    configurations: List[Dict[str, Any]] = None,
    output_dir: str = "benchmark_comparisons",
) -> Dict[str, Any]:
    """
    Compare multiple search configurations on the same benchmark.

    Args:
        dataset_type: Type of dataset to use
        num_examples: Number of examples to evaluate
        configurations: List of search configurations to compare
        output_dir: Directory to save results

    Returns:
        Dictionary with comparison results
    """
    if not configurations:
        # Default configurations to compare
        configurations = [
            {
                "name": "Base Config",
                "search_tool": "searxng",
                "iterations": 1,
                "questions_per_iteration": 3,
            },
            {
                "name": "More Iterations",
                "search_tool": "searxng",
                "iterations": 3,
                "questions_per_iteration": 3,
            },
            {
                "name": "More Questions",
                "search_tool": "searxng",
                "iterations": 1,
                "questions_per_iteration": 5,
            },
        ]

    # Create output directory

    Path(output_dir).mkdir(parents=True, exist_ok=True)

    # Run benchmarks for each configuration
    results = []
    for config in configurations:
        config_name = config.pop("name", f"Config-{len(results)}")

        logger.info(f"Running benchmark with configuration: {config_name}")

        search_config = {
            "iterations": config.pop("iterations", 1),
            "questions_per_iteration": config.pop("questions_per_iteration", 3),
            "search_tool": config.pop("search_tool", "searxng"),
        }

        # Add any remaining config items
        search_config.update(config)

        # Run benchmark with this configuration
        benchmark_result = run_benchmark(
            dataset_type=dataset_type,
            num_examples=num_examples,
            output_dir=str(Path(output_dir) / config_name.replace(" ", "_")),
            search_config=search_config,
            run_evaluation=True,
        )

        # Add configuration name to results
        benchmark_result["configuration_name"] = config_name
        benchmark_result["search_config"] = search_config

        results.append(benchmark_result)

    # Generate comparison report
    import time

    from ..security.file_write_verifier import write_file_verified

    timestamp = time.strftime("%Y%m%d_%H%M%S")
    report_file = str(
        Path(output_dir) / f"comparison_{dataset_type}_{timestamp}.md"
    )

    # Build report content
    content = f"# Configuration Comparison - {dataset_type.capitalize()}\n\n"

    # Write summary table
    content += "## Summary\n\n"
    content += "| Configuration | Accuracy | Avg. Time | Examples |\n"
    content += "|---------------|----------|-----------|----------|\n"

    for result in results:
        accuracy = result.get("metrics", {}).get("accuracy", 0)
        avg_time = result.get("metrics", {}).get("average_processing_time", 0)
        examples = result.get("total_examples", 0)

        content += f"| {result['configuration_name']} | {accuracy:.3f} | {avg_time:.2f}s | {examples} |\n"

    content += "\n## Configuration Details\n\n"

    for result in results:
        content += f"### {result['configuration_name']}\n\n"

        config = result.get("search_config", {})
        content += "```\n"
        for key, value in config.items():
            content += f"{key}: {value}\n"
        content += "```\n\n"

    write_file_verified(
        report_file,
        content,
        "benchmark.allow_file_output",
        context="benchmark comparison report",
    )

    logger.info(f"Comparison report saved to {report_file}")

    return {
        "status": "complete",
        "dataset_type": dataset_type,
        "configurations_tested": len(configurations),
        "report_path": report_file,
        "results": results,
    }


# Export the API functions
__all__ = [
    "calculate_metrics",
    "compare_configurations",
    "evaluate_browsecomp",
    "evaluate_simpleqa",
    "generate_report",
    "get_available_benchmarks",
    "run_benchmark",  # For advanced users
]
