import typer
import asyncio
import subprocess
import time
from typing import Optional
from rich.console import Console
from rich.table import Table
from rich import box

from evalmonkey.simulator.load_gen import LoadGenerator
from evalmonkey.evals.local_assets import load_local_evals
from evalmonkey.evals.runner import LLMJudgeProvider
from evalmonkey.evals.asset_generator import EvalAssetGenerator, FailingTrace, build_output_dir
from evalmonkey.reporting.markdown import (
    print_banner, 
    print_benchmark_score, 
    print_chaos_result,
    print_history_trends,
    print_regression_warning,
    print_recommend_suite,
)
from evalmonkey.scenarios.standard_benchmarks import load_standard_benchmark, get_supported_benchmarks, get_benchmarks_by_category, get_benchmark_categories
from evalmonkey.reporting.history import record_run, get_history, calculate_production_reliability, detect_regression
from evalmonkey.config.agent_config import load_config, generate_config_yaml, FRAMEWORK_PRESETS, AGENT_TYPE_BENCHMARKS

app = typer.Typer(help="EvalMonkey: Open-source Agent Benchmarking and Chaos Framework")
console = Console()

@app.command()
def init(
    framework: str = typer.Option("custom", help=f"Agent framework preset. Choices: {', '.join(FRAMEWORK_PRESETS.keys())}"),
    name: str = typer.Option("My Agent", help="Display name for your agent"),
    port: int = typer.Option(8000, help="Port your agent server is running on"),
    output: str = typer.Option("evalmonkey.yaml", help="Output config file path")
):
    """
    Generate a pre-filled evalmonkey.yaml config file for your agent.
    Run this once in your agent's project directory.
    """
    if framework not in FRAMEWORK_PRESETS:
        console.print(f"[bold red]Unknown framework '{framework}'. Choose from: {', '.join(FRAMEWORK_PRESETS.keys())}[/bold red]")
        raise typer.Exit(1)
    
    yaml_content = generate_config_yaml(framework=framework, name=name, port=port)
    with open(output, "w") as f:
        f.write(yaml_content)
    
    preset = FRAMEWORK_PRESETS[framework]
    console.print(f"\n[bold green]✅ Generated {output} for {preset['description']}![/bold green]")
    console.print(f"[bold cyan]=> request_key: '{preset['request_key']}' | response_path: '{preset['response_path']}'[/bold cyan]")
    console.print("\nNext steps:")
    console.print("  1. Edit evalmonkey.yaml to set your agent URL and eval_model")
    console.print("  2. Run: [bold]evalmonkey run-benchmark --scenario mmlu[/bold]")
    console.print("  3. Run: [bold]evalmonkey run-chaos --scenario mmlu --chaos-profile client_prompt_injection[/bold]\n")

@app.command(name="generate-ci")
def generate_ci(
    output: str = typer.Option(".github/workflows/evalmonkey.yml", help="Path to write the GitHub Actions workflow")
):
    """
    Generate a GitHub Actions workflow to run EvalMonkey on every pull request.
    This creates an automated feedback loop for agent reliability!
    """
    import os
    from pathlib import Path
    
    CI_YAML_TEMPLATE = """name: Agent Reliability Benchmark

on:
  pull_request:
    branches: [ main, master ]

jobs:
  evalmonkey-benchmark:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.11'
          
      - name: Install EvalMonkey
        run: pip install git+https://github.com/Corbell-AI/evalmonkey.git
        
      # TODO: Add steps here to install dependencies and start your agent in the background
      # - name: Install Agent dependencies
      #   run: pip install -r requirements.txt
      # - name: Start Agent
      #   run: python src/agent.py &
      #   env:
      #     OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          
      - name: Wait for Agent
        run: sleep 5
        
      - name: Run EvalMonkey Benchmark
        run: evalmonkey run-benchmark --scenario gsm8k --limit 10
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          EVAL_MODEL: "gpt-4o"
"""
    os.makedirs(os.path.dirname(output), exist_ok=True)
    with open(output, "w") as f:
        f.write(CI_YAML_TEMPLATE)
        
    console.print(f"[bold green]✅ Generated CI/CD workflow at {output}[/bold green]")
    console.print("This will run EvalMonkey benchmarks on every PR to ensure your agent's reliability.")
    console.print("Make sure to update the workflow to start your agent in the background!")


@app.command()
def list_benchmarks(
    category: str = typer.Option(None, help="Filter by agent category (e.g. Coding, Reasoning, Q&A, Research, Safety, Tool Use, Instruction Following, Voice)")
):
    """Lists the off-the-shelf benchmark datasets natively supported, optionally filtered by agent category."""
    print_banner()
    label = f"🐵 EvalMonkey Natively Supported Benchmarks"
    if category:
        label += f" — Category: {category}"
    console.print(f"\n[bold cyan]{label}[/bold cyan]")
    table = Table(box=box.SIMPLE, show_header=True, header_style="bold magenta")
    table.add_column("Scenario ID", style="bold white")
    table.add_column("Category", style="cyan")
    table.add_column("Description")
    
    if category:
        benchmarks = get_benchmarks_by_category(category)
        from evalmonkey.scenarios.standard_benchmarks import get_benchmark_categories
        cats = get_benchmark_categories()
    else:
        benchmarks = get_supported_benchmarks()
        from evalmonkey.scenarios.standard_benchmarks import get_benchmark_categories
        cats = get_benchmark_categories()

    if not benchmarks:
        console.print(f"[bold yellow]No benchmarks found for category '{category}'. "
                      f"Available: Coding, Reasoning, Q&A, Research, Safety, Tool Use, Instruction Following, Voice[/bold yellow]")
        return

    for b_id, desc in benchmarks.items():
        table.add_row(b_id, cats.get(b_id, ""), desc)
        
    console.print(table)
    console.print("\n[dim]Run them via: evalmonkey run-benchmark --scenario <id> --target-url <url>[/dim]")
    if not category:
        console.print("[dim]Filter by category: evalmonkey list-benchmarks --category Coding[/dim]\n")
    else:
        console.print("[dim]Remove --category to see all benchmarks[/dim]\n")


def _spawn_sample_agent(sample_agent: str):
    console.print(f"[bold yellow]=> Spawning sample agent '{sample_agent}' in the background...[/bold yellow]")
    if sample_agent == "rag_app":
        target_url = "http://127.0.0.1:8001/solve"
        proc = subprocess.Popen(["python", "apps/rag_app/app.py"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
        time.sleep(3)
        return proc, target_url
    elif sample_agent == "research_agent":
        target_url = "http://127.0.0.1:8002/solve"
        proc = subprocess.Popen(["python", "apps/research_agent/app.py"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
        time.sleep(3)
        return proc, target_url
    elif sample_agent == "coding_agent":
        target_url = "http://127.0.0.1:8003/solve"
        proc = subprocess.Popen(["python", "apps/coding_agent/app.py"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
        time.sleep(3)
        return proc, target_url
    return None, None

@app.command()
def run_benchmark(
    scenario: str = typer.Option(..., help="Scenario ID, standard benchmark (e.g. gsm8k), or custom_eval ID"),
    dataset: str = typer.Option(None, help="Path to a local dataset file (.jsonl, .json, .csv) to use as the benchmark source."),
    target_url: str = typer.Option(None, help="Address of the BYO agent API (e.g. http://localhost:8000). Required unless using --sample-agent."),
    sample_agent: str = typer.Option(None, help="Automatically spawn a sample agent in the background (rag_app or research_agent)"),
    eval_file: str = typer.Option("custom_evals.yaml", help="Path to evaluation assets"),
    limit: int = typer.Option(5, help="Number of samples from the dataset to load and evaluate."),
    request_key: str = typer.Option("question", help="JSON key to use for the question in the request body. E.g. 'message', 'prompt', 'input'."),
    response_path: str = typer.Option("data", help="Dot-notation path to extract the answer from the response. E.g. 'data', 'output.text', 'choices.0.message.content'.")
):
    """
    Run the agent against a standard dataset or custom evaluation and record the score.
    """
    print_banner()
    
    # Auto-load evalmonkey.yaml from CWD — flags override config if both provided
    cfg = load_config()
    if cfg and not sample_agent:
        effective_url = target_url or cfg.url
        effective_req_key = request_key if request_key != "question" else cfg.request_key
        effective_resp_path = response_path if response_path != "data" else cfg.response_path
        if not target_url:
            console.print(f"[bold green]=> Using agent config from evalmonkey.yaml: {cfg.name} @ {effective_url}[/bold green]")
    else:
        effective_url = target_url
        effective_req_key = request_key
        effective_resp_path = response_path

    agent_process = None
    if sample_agent:
        agent_process, effective_url = _spawn_sample_agent(sample_agent)
        if not agent_process:
            console.print(f"[bold red]Unknown sample agent: {sample_agent}[/bold red]")
            return
    elif cfg and cfg.agent_command:
        # Auto-start the user's own agent via the command in evalmonkey.yaml
        console.print(f"[bold cyan]=> Spawning your agent: {cfg.agent_command}[/bold cyan]")
        agent_process = subprocess.Popen(
            cfg.agent_command, shell=True,
        )
        wait = cfg.agent_startup_wait
        console.print(f"[bold cyan]=> Waiting {wait}s for agent to be ready...[/bold cyan]")
        time.sleep(wait)

    if not effective_url:
        console.print("[bold red]❌ No agent URL found. Run 'evalmonkey init' or pass --target-url[/bold red]")
        return
    
    standard_evals = load_standard_benchmark(scenario, limit=limit)
    
    if standard_evals:
        console.print(f"[bold cyan]=> Loaded {len(standard_evals)} samples from standard benchmark subset: {scenario}[/bold cyan]")
        evals_to_run = standard_evals
    elif dataset:
        # --dataset flag: load from a local file directly
        from evalmonkey.scenarios.private_benchmarks import LocalFileLoader
        console.print(f"[bold cyan]=> Loading dataset from local file: {dataset}[/bold cyan]")
        loader = LocalFileLoader(dataset)
        evals_to_run = loader.load(limit=limit)
        if not evals_to_run:
            console.print(f"[bold red]No eval rows found in {dataset}. Check the file format (JSONL/JSON/CSV).[/bold red]")
            if agent_process: agent_process.terminate()
            return
    else:
        console.print(f"[bold cyan]=> Loading local BYO eval assets from {eval_file}[/bold cyan]")
        evals = load_local_evals(eval_file)
        target_eval = next((e for e in evals if e.id == scenario), None)
        if not target_eval:
            console.print(f"[bold red]Scenario {scenario} not found in standard sets or {eval_file}[/bold red]")
            if agent_process: agent_process.terminate()
            return
        evals_to_run = [target_eval]

    try:
        generator = LoadGenerator(effective_url, request_key=effective_req_key, response_path=effective_resp_path)
        judge = LLMJudgeProvider()
        asset_gen = EvalAssetGenerator()
        scores = []
        overall_reasoning = ""
        
        console.print(f"[bold cyan]=> Executing {len(evals_to_run)} payload(s) sequentially against {effective_url}...[/bold cyan]")
        
        for idx, eval_task in enumerate(evals_to_run):
            resp = asyncio.run(generator.run_scenario(scenario, eval_task.input_payload))
            if resp["status"] == "error":
                console.print(f"[bold red]❌ Request failed on sample {idx}: {resp.get('error_message')}[/bold red]")
                continue
                
            agent_output_text = str(resp.get("data", "No output returned"))
            evaluation = judge.score_run(eval_task.expected_behavior_rubric, agent_output_text)
            score = evaluation.get("score", 0)
            reasoning = evaluation.get("reasoning", "No reasoning provided.")
            scores.append(score)
            if idx == 0:
                overall_reasoning = reasoning

            # Record as a failure candidate for eval asset generation
            asset_gen.record_failure(FailingTrace(
                scenario=scenario,
                eval_id=eval_task.id,
                input_payload=eval_task.input_payload,
                agent_output=agent_output_text,
                expected_rubric=eval_task.expected_behavior_rubric,
                score=score,
                reasoning=reasoning,
            ))
        
        if not scores:
            console.print("[bold red]❌ All requests failed to execute![/bold red]")
            return
            
        final_score = int(sum(scores) / len(scores))
        
        # Check baseline from history
        hist = get_history(scenario)
        baseline = hist[-1]["score"] if hist else None
        
        record_run(scenario, "baseline", final_score, details={"reasoning": overall_reasoning, "sample_size": len(scores)})
        print_benchmark_score(scenario, final_score, overall_reasoning, baseline)

        # ── Regression detection (informational — use `evalmonkey guard` for CI gating) ──
        import os as _os
        _threshold = int(_os.getenv("EVAL_REGRESSION_THRESHOLD", "5"))
        regression = detect_regression(scenario, final_score, threshold=_threshold)
        if regression:
            print_regression_warning(
                scenario=scenario,
                prev_score=regression["prev_score"],
                curr_score=final_score,
                drop=regression["drop"],
            )

        # ── Eval Asset Generation on failure ──────────────────────────────
        if asset_gen.has_failures:
            output_dir = build_output_dir(scenario)
            saved_path = asset_gen.save(output_dir)
            console.print(f"\n[bold yellow]⚠️  {asset_gen.failure_count} sample(s) scored below threshold — eval assets saved.[/bold yellow]")
            console.print(f"[dim]   Output → {saved_path}[/dim]")
            console.print("\n[bold cyan]🛠  Next steps to improve your agent:[/bold cyan]")
            console.print(f"[bold white]  1. Regenerate evals anytime:[/bold white]")
            console.print(f"     [green]evalmonkey generate-evals --traces-file {saved_path}/traces.json[/green]")
            console.print(f"[bold white]  2. Pass improvement brief to your coding agent:[/bold white]")
            console.print(f"     [green]cat {saved_path}/improvement_prompt.md | pbcopy  # then paste into Claude/Cursor[/green]")
            console.print(f"[bold white]  3. Re-run this benchmark after fixing:[/bold white]")
            console.print(f"     [green]evalmonkey run-benchmark --scenario {scenario}[/green]\n")
    finally:
        if agent_process:
            agent_process.terminate()


@app.command()
def run_chaos(
    scenario: str = typer.Option(..., help="Scenario ID"),
    target_url: str = typer.Option(None, help="Address of the BYO agent API"),
    sample_agent: str = typer.Option(None, help="Automatically spawn a sample agent in the background (rag_app or research_agent)"),
    chaos_profile: str = typer.Option(..., help="The X-Chaos-Profile value to inject (e.g. latency_spike, schema_error)"),
    eval_file: str = typer.Option("custom_evals.yaml", help="Path to evaluation assets"),
    limit: int = typer.Option(5, help="Number of benchmark samples to evaluate in Chaos Mode"),
    request_key: str = typer.Option("question", help="JSON key for the question in the request body."),
    response_path: str = typer.Option("data", help="Dot-notation path to extract the answer from the response.")
):
    """
    Injects a chaos header against an endpoint, recording the degradation of the capability.
    """
    print_banner()
    
    # Auto-load evalmonkey.yaml from CWD — flags override config if both provided
    cfg = load_config()
    if cfg and not sample_agent:
        effective_url = target_url or cfg.url
        effective_req_key = request_key if request_key != "question" else cfg.request_key
        effective_resp_path = response_path if response_path != "data" else cfg.response_path
        if not target_url:
            console.print(f"[bold green]=> Using agent config from evalmonkey.yaml: {cfg.name} @ {effective_url}[/bold green]")
    else:
        effective_url = target_url
        effective_req_key = request_key
        effective_resp_path = response_path

    agent_process = None
    if sample_agent:
        agent_process, effective_url = _spawn_sample_agent(sample_agent)
        if not agent_process:
            console.print(f"[bold red]Unknown sample agent: {sample_agent}[/bold red]")
            return
    elif cfg and cfg.agent_command:
        # Auto-start the user's own agent via the command in evalmonkey.yaml
        console.print(f"[bold cyan]=> Spawning your agent: {cfg.agent_command}[/bold cyan]")
        agent_process = subprocess.Popen(
            cfg.agent_command, shell=True,
        )
        wait = cfg.agent_startup_wait
        console.print(f"[bold cyan]=> Waiting {wait}s for agent to be ready...[/bold cyan]")
        time.sleep(wait)

    if not effective_url:
        console.print("[bold red]❌ No agent URL found. Run 'evalmonkey init' or pass --target-url[/bold red]")
        return

    console.print(f"[bold red]=> 🔥 INJECTING CHAOS PROFILE: {chaos_profile} 🔥[/bold red]")
    if chaos_profile.startswith("client_"):
        console.print("[bold yellow]⚠️ WARNING: Client-side prompt mutations append adversarial instructions which slightly increases your prompt token payload size![/bold yellow]")
    
    standard_evals = load_standard_benchmark(scenario, limit=limit)
    if standard_evals:
        evals_to_run = standard_evals
    else:
        target_eval = next((e for e in load_local_evals(eval_file) if e.id == scenario), None)
        if not target_eval:
            console.print(f"[bold red]Scenario {scenario} not found.[/bold red]")
            if agent_process: agent_process.terminate()
            return
        evals_to_run = [target_eval]

    try:
        generator = LoadGenerator(effective_url, request_key=effective_req_key, response_path=effective_resp_path)
        judge = LLMJudgeProvider()
        asset_gen = EvalAssetGenerator()
        scores = []
        
        for eval_task in evals_to_run:
            resp = asyncio.run(generator.run_scenario(scenario, eval_task.input_payload, chaos_profile=chaos_profile))
            agent_output_text = str(resp.get("data", resp.get("error_message", "No output")))
            evaluation = judge.score_run(eval_task.expected_behavior_rubric, agent_output_text)
            score = evaluation.get("score", 0)
            reasoning = evaluation.get("reasoning", "No reasoning provided.")
            scores.append(score)

            asset_gen.record_failure(FailingTrace(
                scenario=scenario,
                eval_id=eval_task.id,
                input_payload=eval_task.input_payload,
                agent_output=agent_output_text,
                expected_rubric=eval_task.expected_behavior_rubric,
                score=score,
                reasoning=reasoning,
                chaos_profile=chaos_profile,
            ))
            
        if not scores:
            return
            
        final_score = int(sum(scores) / len(scores))
        
        hist = get_history(scenario)
        baselines = [h for h in hist if h["run_type"] == "baseline"]
        original_baseline = baselines[-1]["score"] if baselines else 0
        
        record_run(scenario, "chaos", final_score, details={"chaos_profile": chaos_profile, "sample_size": len(scores)})
        print_chaos_result(scenario, chaos_profile, final_score, original_baseline)

        # ── Eval Asset Generation on chaos failure ─────────────────────────
        if asset_gen.has_failures:
            output_dir = build_output_dir(f"{scenario}_{chaos_profile}")
            saved_path = asset_gen.save(output_dir)
            console.print(f"\n[bold yellow]⚠️  Chaos caused {asset_gen.failure_count} failure(s) — eval assets saved.[/bold yellow]")
            console.print(f"[dim]   Output → {saved_path}[/dim]")
            console.print(f"[bold cyan]🛠  Run this to get your improvement brief:[/bold cyan]")
            console.print(f"     [green]cat {saved_path}/improvement_prompt.md[/green]\n")
    finally:
        if agent_process:
            agent_process.terminate()


@app.command()
def history(scenario: str = typer.Option(None, help="Specific scenario ID to view history for. Views all if blank.")):
    """
    Prints the beautiful historical trend of agent scores and the Production Reliability metric.
    """
    print_banner()
    
    hist = get_history(scenario)
    if not hist:
        console.print("[bold yellow]No local history found. Run a benchmark first![/bold yellow]")
        return
    
    scenarios = {h["scenario"] for h in hist} if not scenario else {scenario}
    for s in scenarios:
        s_hist = [h for h in hist if h["scenario"] == s]
        reliability = calculate_production_reliability(scenario=s)
        print_history_trends(s, s_hist, reliability)


@app.command()
def recommend():
    """
    Show the recommended benchmark suite for your agent type.
    Reads agent_type from evalmonkey.yaml (default: general).
    Set agent_type in your config to get a curated list instead of all 22 benchmarks.
    """
    print_banner()
    cfg = load_config()
    agent_type = getattr(cfg, "agent_type", "general") if cfg else "general"

    benchmark_ids = AGENT_TYPE_BENCHMARKS.get(agent_type, AGENT_TYPE_BENCHMARKS["general"])
    all_benchmarks = get_supported_benchmarks()
    categories = get_benchmark_categories()

    # Keep only IDs that exist in the catalogue (guard against stale config values)
    relevant = {bid: all_benchmarks[bid] for bid in benchmark_ids if bid in all_benchmarks}

    if not relevant:
        console.print(
            f"[bold yellow]No benchmarks found for agent_type '{agent_type}'. "
            f"Available types: {', '.join(AGENT_TYPE_BENCHMARKS.keys())}[/bold yellow]"
        )
        return

    print_recommend_suite(agent_type, relevant, categories)


@app.command()
def guard(
    scenario: str = typer.Option(..., help="Benchmark scenario to check for regression"),
    fail_threshold: int = typer.Option(
        None,
        help="Score drop (pts) that triggers failure. Defaults to EVAL_REGRESSION_THRESHOLD env var (default: 5).",
    ),
):
    """
    Check for a score regression vs the last baseline and exit with code 1 if detected.
    Use this in CI/CD pipelines to block deploys when your agent regresses.

    Example (GitHub Actions):
      - run: evalmonkey guard --scenario gsm8k
    """
    import os as _os
    threshold = fail_threshold if fail_threshold is not None else int(_os.getenv("EVAL_REGRESSION_THRESHOLD", "5"))

    hist = get_history(scenario)
    baselines = sorted(
        [r for r in hist if r.get("run_type") == "baseline"],
        key=lambda r: r.get("timestamp", ""),
    )

    if len(baselines) < 2:
        console.print(
            f"[dim]Not enough baseline history for '{scenario}' to detect regression "
            f"(need ≥ 2 runs). Run evalmonkey run-benchmark --scenario {scenario} at least twice.[/dim]"
        )
        raise SystemExit(0)

    curr_score = baselines[-1].get("score", 0)
    prev_score = baselines[-2].get("score", 0)
    drop = prev_score - curr_score

    if drop >= threshold:
        print_regression_warning(scenario, prev_score, curr_score, drop)
        console.print(
            f"\n[bold red]❌ Guard failed: {scenario} regressed by {drop} pts "
            f"(threshold: {threshold}). Exiting with code 1.[/bold red]\n"
        )
        raise SystemExit(1)
    else:
        trend = f"+{abs(drop)}" if drop < 0 else f"±0" if drop == 0 else f"-{drop}"
        color = "green" if drop <= 0 else "yellow"
        console.print(
            f"\n[bold {color}]✅ Guard passed: {scenario} — "
            f"score {curr_score}/100 (vs prev {prev_score}/100, Δ {trend}). "
            f"No regression detected.[/bold {color}]\n"
        )
        raise SystemExit(0)


@app.command()
def report(
    output: str = typer.Option("evalmonkey_report.md", help="Output Markdown file path"),
    agent_name: str = typer.Option(
        None,
        help="Agent display name for the report title. Defaults to name in evalmonkey.yaml.",
    ),
):
    """
    Generate a shareable Markdown Agent Card from your local benchmark history.
    Includes a shields.io badge, per-scenario score table, and production reliability.

    Paste the badge into your README to show your agent's benchmark scores!
    """
    from evalmonkey.reporting.report_generator import generate_report

    cfg = load_config()
    name = agent_name or (cfg.name if cfg else "My Agent")

    hist = get_history()
    if not hist:
        console.print("[bold yellow]No benchmark history found. Run evalmonkey run-benchmark first![/bold yellow]")
        return

    generate_report(output_path=output, agent_name=name)
    console.print(f"\n[bold green]✅ Agent card generated: {output}[/bold green]")
    console.print(f"[dim]Embed the badge in your README, share the file, or commit it to your repo.[/dim]")
    console.print(f"[bold cyan]   cat {output}[/bold cyan]\n")


@app.command()
def run_chaos_suite(
    scenario: str = typer.Option(..., help="Scenario ID to test all chaos profiles against"),
    target_url: str = typer.Option(None, help="Address of the BYO agent API"),
    sample_agent: str = typer.Option(None, help="Automatically spawn a sample agent in the background"),
    eval_file: str = typer.Option("custom_evals.yaml", help="Path to evaluation assets"),
    limit: int = typer.Option(5, help="Number of benchmark samples to evaluate per chaos step"),
    request_key: str = typer.Option("question", help="JSON key for the question in the request body."),
    response_path: str = typer.Option("data", help="Dot-notation path to extract the answer from the response.")
):
    """
    Barrage an endpoint with EVERY available client-side chaos profile sequentially.
    """
    PROFILES = [
        # Client-side general (12)
        "client_prompt_injection", "client_typo_injection", "client_schema_mutation",
        "client_language_shift", "client_payload_bloat", "client_empty_payload",
        "client_context_truncation", "client_unicode_flood", "client_role_impersonation",
        "client_repetition_loop", "client_negative_sentiment", "client_length_constraint_violation",
        # Coding-agent-specific (6)
        "code_context_strip", "code_wrong_language", "code_syntax_break",
        "code_test_poison", "code_incomplete_signature", "code_conflicting_constraints",
        # Voice-agent-specific (5)
        "voice_asr_noise", "voice_filler_words", "voice_background_noise_sim",
        "voice_truncated_speech", "voice_dialect_shift",
    ]
    console.print("[bold cyan]=> 🌪️ STARTING FULL CHAOS BARRAGE SUITE 🌪️[/bold cyan]")
    
    cfg = load_config()
    effective_url = target_url or (cfg.url if cfg else None)
    effective_req_key = request_key if request_key != "question" else (cfg.request_key if cfg else request_key)
    effective_resp_path = response_path if response_path != "data" else (cfg.response_path if cfg else response_path)

    agent_process = None
    if sample_agent:
        agent_process, effective_url = _spawn_sample_agent(sample_agent)
    elif cfg and cfg.agent_command:
        console.print(f"[bold cyan]=> Spawning your agent: {cfg.agent_command}[/bold cyan]")
        agent_process = subprocess.Popen(cfg.agent_command, shell=True)
        time.sleep(cfg.agent_startup_wait)

    if not effective_url:
        console.print("[bold red]❌ No agent URL found. Run 'evalmonkey init' or pass --target-url[/bold red]")
        if agent_process: agent_process.terminate()
        return

    standard_evals = load_standard_benchmark(scenario, limit=limit)
    evals_to_run = standard_evals if standard_evals else [e for e in load_local_evals(eval_file) if e.id == scenario]
    
    if not evals_to_run:
        console.print(f"[bold red]Scenario {scenario} not found.[/bold red]")
        if agent_process: agent_process.terminate()
        return

    try:
        generator = LoadGenerator(effective_url, request_key=effective_req_key, response_path=effective_resp_path)
        judge = LLMJudgeProvider()
        
        for profile in PROFILES:
            scores = []
            console.print(f"[bold yellow]\n=> Injecting: {profile}[/bold yellow]")
            for eval_task in evals_to_run:
                resp = asyncio.run(generator.run_scenario(scenario, eval_task.input_payload, chaos_profile=profile))
                agent_output_text = str(resp.get("data", resp.get("error_message", "No output")))
                evaluation = judge.score_run(eval_task.expected_behavior_rubric, agent_output_text)
                scores.append(evaluation.get("score", 0))    
            if scores:
                final_score = int(sum(scores) / len(scores))
                console.print(f"[bold red]   Score: {final_score}/100[/bold red]")
                record_run(scenario, "chaos", final_score, details={"chaos_profile": profile, "sample_size": len(scores)})
    finally:
        if agent_process:
            agent_process.terminate()
    console.print("\n[bold green]=> Barrage Complete! Run `evalmonkey history --scenario <id>` to see all entries.[/bold green]")


@app.command()
def generate_evals(
    traces_file: str = typer.Option(..., help="Path to a traces.json file produced by a previous benchmark run"),
    output_dir: str = typer.Option(None, help="Directory to write evals.json and improvement_prompt.md (defaults to same folder as traces_file)"),
    langfuse_dataset: str = typer.Option(None, help="If set, also push the evals to this Langfuse dataset name (requires LANGFUSE_PUBLIC_KEY + LANGFUSE_SECRET_KEY)"),
    n: int = typer.Option(5, help="Number of improvement eval scenarios to generate"),
):
    """
    (Re)generate improvement eval scenarios from a saved traces.json file.
    Pass the output improvement_prompt.md to Claude Code, Cursor, or any coding agent
    to automatically improve your agent based on its failures.
    """
    import json as _json
    from pathlib import Path as _Path
    from evalmonkey.evals.asset_generator import EvalAssetGenerator, FailingTrace

    traces_path = _Path(traces_file)
    if not traces_path.exists():
        console.print(f"[bold red]❌ traces.json not found: {traces_file}[/bold red]")
        raise typer.Exit(1)

    raw = _json.loads(traces_path.read_text(encoding="utf-8"))
    asset_gen = EvalAssetGenerator(failure_threshold=101)  # treat all loaded traces as failures
    for item in raw:
        asset_gen.record_failure(FailingTrace(
            scenario=item.get("scenario", ""),
            eval_id=item.get("eval_id", ""),
            input_payload=item.get("input_payload", {}),
            agent_output=item.get("agent_output", ""),
            expected_rubric=item.get("expected_rubric", ""),
            score=item.get("score", 0),
            reasoning=item.get("reasoning", ""),
            chaos_profile=item.get("chaos_profile"),
        ))

    out = output_dir or str(traces_path.parent)
    saved = asset_gen.save(out)
    console.print(f"[bold green]✅ Improvement evals saved to {saved}[/bold green]")
    console.print(f"[bold cyan]=> Coding agent prompt:[/bold cyan]")
    console.print(f"     [green]cat {saved}/improvement_prompt.md | pbcopy[/green]")
    console.print(f"   Then paste into Claude Code or Cursor to auto-fix your agent.\n")

    if langfuse_dataset:
        ok = asset_gen.export_to_langfuse(langfuse_dataset)
        if ok:
            console.print(f"[bold green]✅ Evals pushed to Langfuse dataset: '{langfuse_dataset}'[/bold green]")
        else:
            console.print("[bold yellow]⚠️  Langfuse export skipped (LANGFUSE_PUBLIC_KEY / LANGFUSE_SECRET_KEY not set)[/bold yellow]")

@app.command()
def serve_mcp():
    """
    Start the EvalMonkey MCP Server via stdio.
    This exposes EvalMonkey's commands as native tools for external tools/agents (e.g., Claude Desktop, Cursor).
    """
    from evalmonkey.mcp_server import mcp
    mcp.run(transport="stdio")

if __name__ == "__main__":
    app()
