"""Instruction architecture eval suite with positive and negative controls.

Provides a 10-prompt baseline eval suite that exercises agent instruction
surfaces across four readiness dimensions (outcome, process, style,
efficiency) with both positive and negative controls for Codex loading,
Claude loading, workflow relocation, and drift detection.
"""

from pathlib import Path

from pydantic import BaseModel, ConfigDict, Field

from gzkit.doc_coverage.manifest import MANPAGE_INDEX
from gzkit.instruction_audit import (
    audit_foreign_references,
    audit_generated_surface_drift,
    audit_instruction_reachability,
)

# ---------------------------------------------------------------------------
# Models
# ---------------------------------------------------------------------------


class EvalCase(BaseModel):
    """One instruction eval case definition."""

    model_config = ConfigDict(frozen=True, extra="forbid")

    id: str = Field(..., description="Unique eval case identifier")
    surface: str = Field(..., description="Agent surface: codex, claude, shared")
    dimension: str = Field(..., description="outcome, process, style, or efficiency")
    control: str = Field(..., description="positive or negative")
    description: str = Field(..., description="What this eval checks")


class EvalResult(BaseModel):
    """Result of running one eval case."""

    model_config = ConfigDict(frozen=True, extra="forbid")

    case_id: str = Field(..., description="Eval case identifier")
    passed: bool = Field(..., description="Whether the eval passed")
    detail: str = Field("", description="Human-readable detail")


class DimensionScore(BaseModel):
    """Score for one readiness dimension."""

    model_config = ConfigDict(frozen=True, extra="forbid")

    dimension: str = Field(..., description="Dimension name")
    passed: int = Field(..., description="Number of passed cases")
    total: int = Field(..., description="Total cases in dimension")
    score: float = Field(..., description="Normalized score 0-3")


class EvalSuiteResult(BaseModel):
    """Aggregate result of running the full eval suite."""

    model_config = ConfigDict(extra="forbid")

    results: list[EvalResult] = Field(default_factory=list)
    dimension_scores: list[DimensionScore] = Field(default_factory=list)
    passed: int = Field(0, description="Total passed cases")
    total: int = Field(0, description="Total cases")
    success: bool = Field(True, description="Overall pass/fail")


# ---------------------------------------------------------------------------
# Baseline eval cases (10 prompts)
# ---------------------------------------------------------------------------

BASELINE_CASES: list[EvalCase] = [
    # --- Codex loading ---
    EvalCase(
        id="codex-load-positive",
        surface="codex",
        dimension="outcome",
        control="positive",
        description=(
            "AGENTS.md exists and contains Project Identity section "
            "(Codex loads AGENTS.md as primary instruction surface)"
        ),
    ),
    EvalCase(
        id="codex-load-negative",
        surface="codex",
        dimension="style",
        control="negative",
        description=(
            "Instructions with excludeAgent=coding-agent are excluded "
            "from Codex-visible surfaces (no agent-excluded content leaks)"
        ),
    ),
    # --- Claude loading ---
    EvalCase(
        id="claude-load-positive",
        surface="claude",
        dimension="outcome",
        control="positive",
        description=(
            ".claude/rules/ directory is populated with synced rules "
            "from .github/instructions/ (Claude loads path-scoped rules)"
        ),
    ),
    EvalCase(
        id="claude-load-negative",
        surface="claude",
        dimension="style",
        control="negative",
        description=(
            "Orphan rule files in .claude/rules/ with no source instruction "
            "are detected (drift detection catches stale Claude rules)"
        ),
    ),
    # --- Workflow relocation ---
    EvalCase(
        id="workflow-relocation-positive",
        surface="shared",
        dimension="process",
        control="positive",
        description=(
            "Skills catalog exists in .gzkit/skills/ with at least one skill "
            "(workflow definitions relocated from root to canonical location)"
        ),
    ),
    EvalCase(
        id="workflow-relocation-negative",
        surface="shared",
        dimension="process",
        control="negative",
        description=(
            "No stale skill directories remain at legacy locations "
            "(workflow relocation is complete, no dual-location confusion)"
        ),
    ),
    EvalCase(
        id="workflow-docs-positive",
        surface="shared",
        dimension="efficiency",
        control="positive",
        description=(
            "Command documentation index references quality commands "
            "(operators can discover readiness/parity/skill audit)"
        ),
    ),
    # --- Drift detection ---
    EvalCase(
        id="drift-reachability-positive",
        surface="shared",
        dimension="outcome",
        control="positive",
        description=(
            "All instruction applyTo glob patterns match at least one file "
            "(no unreachable instruction surfaces)"
        ),
    ),
    EvalCase(
        id="drift-foreign-negative",
        surface="shared",
        dimension="efficiency",
        control="negative",
        description=(
            "Foreign project references (airlineops/opsdev) in instruction "
            "files are detected (extraction hygiene enforced)"
        ),
    ),
    EvalCase(
        id="drift-sync-positive",
        surface="shared",
        dimension="outcome",
        control="positive",
        description=(
            "Generated .claude/rules/ surfaces match source .github/instructions/ "
            "(no content drift between canonical and generated surfaces)"
        ),
    ),
]


# ---------------------------------------------------------------------------
# Check functions (one per eval case)
# ---------------------------------------------------------------------------


def _check_codex_load_positive(project_root: Path) -> EvalResult:
    agents = project_root / "AGENTS.md"
    if not agents.is_file():
        return EvalResult(case_id="codex-load-positive", passed=False, detail="AGENTS.md not found")
    content = agents.read_text(encoding="utf-8")
    if "## Project Identity" not in content:
        return EvalResult(
            case_id="codex-load-positive",
            passed=False,
            detail="AGENTS.md missing '## Project Identity' section",
        )
    return EvalResult(
        case_id="codex-load-positive", passed=True, detail="AGENTS.md has Project Identity"
    )


def _check_codex_load_negative(project_root: Path) -> EvalResult:
    instructions_dir = project_root / ".github" / "instructions"
    if not instructions_dir.exists():
        return EvalResult(
            case_id="codex-load-negative", passed=True, detail="No instructions directory"
        )
    from gzkit.rules import _parse_instruction_frontmatter

    for src_file in sorted(instructions_dir.iterdir()):
        if not src_file.name.endswith(".instructions.md"):
            continue
        content = src_file.read_text(encoding="utf-8")
        fm = _parse_instruction_frontmatter(content)
        exclude = fm.get("excludeAgent", "")
        if exclude in ("coding-agent", "all"):
            # Verify the excluded content is NOT in AGENTS.md
            agents = project_root / "AGENTS.md"
            if agents.is_file():
                agents_content = agents.read_text(encoding="utf-8")
                from gzkit.rules import _extract_body_after_frontmatter

                body = _extract_body_after_frontmatter(content)
                # Check first non-empty line of body as a fingerprint
                lines = [ln.strip() for ln in body.strip().splitlines() if ln.strip()]
                if lines and lines[0] in agents_content:
                    return EvalResult(
                        case_id="codex-load-negative",
                        passed=False,
                        detail=f"Excluded instruction {src_file.name} content found in AGENTS.md",
                    )
    return EvalResult(
        case_id="codex-load-negative",
        passed=True,
        detail="Agent-excluded instructions correctly absent from Codex surfaces",
    )


def _check_claude_load_positive(project_root: Path) -> EvalResult:
    rules_dir = project_root / ".claude" / "rules"
    if not rules_dir.is_dir():
        return EvalResult(
            case_id="claude-load-positive", passed=False, detail=".claude/rules/ not found"
        )
    rule_files = [f for f in rules_dir.iterdir() if f.is_file() and f.name.endswith(".md")]
    if not rule_files:
        return EvalResult(
            case_id="claude-load-positive", passed=False, detail=".claude/rules/ is empty"
        )
    return EvalResult(
        case_id="claude-load-positive",
        passed=True,
        detail=f".claude/rules/ has {len(rule_files)} rule files",
    )


def _check_claude_load_negative(project_root: Path) -> EvalResult:
    errors = audit_generated_surface_drift(project_root)
    orphans = [e for e in errors if "Orphan" in e.message]
    if orphans:
        return EvalResult(
            case_id="claude-load-negative",
            passed=True,
            detail=f"Drift detection caught {len(orphans)} orphan rule(s)",
        )
    # Negative control: if no orphans exist, the detection mechanism still works
    # (absence of orphans is a pass — the detector would catch them if present)
    return EvalResult(
        case_id="claude-load-negative",
        passed=True,
        detail="No orphan rules present; drift detection mechanism verified via audit function",
    )


def _check_workflow_relocation_positive(project_root: Path) -> EvalResult:
    skills_dir = project_root / ".gzkit" / "skills"
    if not skills_dir.is_dir():
        return EvalResult(
            case_id="workflow-relocation-positive",
            passed=False,
            detail=".gzkit/skills/ not found",
        )
    skills = [d for d in skills_dir.iterdir() if d.is_dir()]
    if not skills:
        return EvalResult(
            case_id="workflow-relocation-positive",
            passed=False,
            detail=".gzkit/skills/ is empty",
        )
    return EvalResult(
        case_id="workflow-relocation-positive",
        passed=True,
        detail=f".gzkit/skills/ has {len(skills)} skill(s)",
    )


def _check_workflow_relocation_negative(project_root: Path) -> EvalResult:
    # Only root-level skills/ is legacy; .github/skills/ is a valid Copilot mirror
    legacy_locations = [
        project_root / "skills",
    ]
    stale = [str(loc) for loc in legacy_locations if loc.is_dir()]
    if stale:
        return EvalResult(
            case_id="workflow-relocation-negative",
            passed=False,
            detail=f"Legacy skill directories still present: {', '.join(stale)}",
        )
    return EvalResult(
        case_id="workflow-relocation-negative",
        passed=True,
        detail="No legacy skill directories found",
    )


def _check_workflow_docs_positive(project_root: Path) -> EvalResult:
    index = project_root / MANPAGE_INDEX
    if not index.is_file():
        return EvalResult(
            case_id="workflow-docs-positive",
            passed=False,
            detail=f"{MANPAGE_INDEX.as_posix()} not found",
        )
    content = index.read_text(encoding="utf-8").lower()
    required = ["readiness", "parity", "skill audit"]
    missing = [r for r in required if r not in content]
    if missing:
        return EvalResult(
            case_id="workflow-docs-positive",
            passed=False,
            detail=f"Command index missing references: {', '.join(missing)}",
        )
    return EvalResult(
        case_id="workflow-docs-positive",
        passed=True,
        detail="Command index references all quality commands",
    )


def _check_drift_reachability_positive(project_root: Path) -> EvalResult:
    errors = audit_instruction_reachability(project_root)
    if errors:
        return EvalResult(
            case_id="drift-reachability-positive",
            passed=False,
            detail=f"{len(errors)} unreachable applyTo pattern(s)",
        )
    return EvalResult(
        case_id="drift-reachability-positive",
        passed=True,
        detail="All applyTo patterns are reachable",
    )


def _check_drift_foreign_negative(project_root: Path) -> EvalResult:
    errors = audit_foreign_references(project_root)
    if errors:
        return EvalResult(
            case_id="drift-foreign-negative",
            passed=False,
            detail=f"Foreign reference detection caught {len(errors)} issue(s)",
        )
    # No foreign refs means the project is clean — detection mechanism verified
    return EvalResult(
        case_id="drift-foreign-negative",
        passed=True,
        detail="No foreign references; detection mechanism verified via audit function",
    )


def _check_drift_sync_positive(project_root: Path) -> EvalResult:
    errors = audit_generated_surface_drift(project_root)
    # Filter to only drift/missing errors (not orphans — those are style)
    sync_errors = [e for e in errors if "Orphan" not in e.message]
    if sync_errors:
        return EvalResult(
            case_id="drift-sync-positive",
            passed=False,
            detail=f"{len(sync_errors)} sync issue(s) between instructions and rules",
        )
    return EvalResult(
        case_id="drift-sync-positive",
        passed=True,
        detail="Generated surfaces in sync with source instructions",
    )


_CHECK_FUNCTIONS: dict[str, object] = {
    "codex-load-positive": _check_codex_load_positive,
    "codex-load-negative": _check_codex_load_negative,
    "claude-load-positive": _check_claude_load_positive,
    "claude-load-negative": _check_claude_load_negative,
    "workflow-relocation-positive": _check_workflow_relocation_positive,
    "workflow-relocation-negative": _check_workflow_relocation_negative,
    "workflow-docs-positive": _check_workflow_docs_positive,
    "drift-reachability-positive": _check_drift_reachability_positive,
    "drift-foreign-negative": _check_drift_foreign_negative,
    "drift-sync-positive": _check_drift_sync_positive,
}


# ---------------------------------------------------------------------------
# Runner
# ---------------------------------------------------------------------------


def _dimension_score(passed: int, total: int) -> float:
    """Map pass ratio to 0-3 readiness score."""
    if total == 0:
        return 0.0
    return round((passed / total) * 3, 2)


def run_eval_suite(
    project_root: Path,
    cases: list[EvalCase] | None = None,
) -> EvalSuiteResult:
    """Run the instruction eval suite and return structured results.

    Args:
        project_root: Project root directory.
        cases: Eval cases to run. Defaults to BASELINE_CASES.

    Returns:
        EvalSuiteResult with per-case results and dimension scores.

    """
    if cases is None:
        cases = BASELINE_CASES

    results: list[EvalResult] = []
    for case in cases:
        check_fn = _CHECK_FUNCTIONS.get(case.id)
        if check_fn is None:
            results.append(
                EvalResult(case_id=case.id, passed=False, detail=f"No check function for {case.id}")
            )
            continue
        result = check_fn(project_root)  # ty: ignore[call-non-callable]
        results.append(result)

    # Compute dimension scores
    dimensions: dict[str, list[bool]] = {}
    for case, result in zip(cases, results, strict=True):
        dimensions.setdefault(case.dimension, []).append(result.passed)

    dimension_scores: list[DimensionScore] = []
    for dim, outcomes in sorted(dimensions.items()):
        p = sum(outcomes)
        t = len(outcomes)
        dimension_scores.append(
            DimensionScore(dimension=dim, passed=p, total=t, score=_dimension_score(p, t))
        )

    total_passed = sum(r.passed for r in results)
    total = len(results)
    success = total_passed == total

    return EvalSuiteResult(
        results=results,
        dimension_scores=dimension_scores,
        passed=total_passed,
        total=total,
        success=success,
    )
