import { test, expect } from "bun:test"
import type {
  BenchmarkTask,
  QualityDimension,
  DimensionScore,
  RunScore,
  TaskVariantReport,
  AblationResult,
  SessionReport,
  HarnessVariant,
  InternalVariant,
  CompetitorVariant,
  HarnessConfig,
  BenchmarkSession,
  ModelVariant,
  DimensionRubric,
  TaskFixture,
  SuccessCriteria,
  DriftReport,
  TaskRunResult,
} from "../src/types.js"
import type { CompetitorRunner } from "../src/competitors/types.js"
import { REAL_WORLD_TASKS } from "../src/tasks/real-world.js"
import { ABLATION_VARIANTS, resolveTasks, mergeConfigs } from "../src/session.js"
import { computeDrift, exceedsThreshold } from "../src/ci.js"
import { aggregateRuns, computeAllAblation, summarizeDimensions } from "../src/runner.js"
import { regressionGateSession } from "../src/sessions/regression-gate.js"
import { realWorldFullSession } from "../src/sessions/real-world-full.js"
import { competitorComparisonSession } from "../src/sessions/competitor-comparison.js"
import { localModelsSession } from "../src/sessions/local-models.js"
import { parseArgs } from "../src/run.js"
import { runSession } from "../src/index.js"
import { BENCHMARK_TASKS } from "../src/task-registry.js"
import { matchSuccessCriteria, parsePartialCreditScore, computeReliability } from "../src/judge.js"
import { COMPETITOR_RUNNERS } from "../src/competitors/index.js"

test("QualityDimension covers all 10 dimensions", () => {
  const dims: QualityDimension[] = [
    "accuracy", "reasoning", "tool-mastery", "memory-fidelity",
    "loop-intelligence", "resilience", "efficiency", "reliability",
    "scope-discipline", "honest-uncertainty",
  ]
  expect(dims).toHaveLength(10)
})

test("SuccessCriteria discriminated union has 4 variants", () => {
  const a: SuccessCriteria = { type: "regex", pattern: "foo" }
  const b: SuccessCriteria = { type: "verifiable", command: "bun check output.ts" }
  const c: SuccessCriteria = { type: "llm-judge", rubric: "Is it correct?" }
  const d: SuccessCriteria = { type: "schema", schema: {} }
  expect([a, b, c, d]).toHaveLength(4)
})

test("HarnessVariant discriminated union works", () => {
  const internal: InternalVariant = {
    type: "internal", id: "bare-llm", label: "Bare LLM", config: {},
  }
  const competitor: CompetitorVariant = {
    type: "competitor", id: "langchain-react", label: "LangChain JS", framework: "langchain",
  }
  const variants: HarnessVariant[] = [internal, competitor]
  expect(variants).toHaveLength(2)
})

test("BenchmarkTask accepts real-world tier and v2 optional fields", () => {
  const task: BenchmarkTask = {
    id: "rw-1", tier: "real-world", name: "Test", prompt: "Do X",
    successCriteria: { type: "regex", pattern: "ok" },
    primaryDimensions: ["accuracy"],
    fixtures: [{ path: "data.csv", content: "a,b\n1,2\n" }],
  }
  expect(task.tier).toBe("real-world")
})

test("task-registry exports BENCHMARK_TASKS with 25 tasks", async () => {
  const { BENCHMARK_TASKS } = await import("../src/task-registry.js")
  expect(BENCHMARK_TASKS.length).toBe(25)
})

test("getTasksByTier returns only tasks of that tier", async () => {
  const { getTasksByTier } = await import("../src/task-registry.js")
  const trivial = getTasksByTier("trivial")
  expect(trivial.every(t => t.tier === "trivial")).toBe(true)
  expect(trivial.length).toBeGreaterThan(0)
})

test("REAL_WORLD_TASKS exports exactly 10 tasks", () => {
  expect(REAL_WORLD_TASKS).toHaveLength(10)
})

test("all real-world tasks have required v2 fields", () => {
  for (const task of REAL_WORLD_TASKS) {
    expect(task.tier).toBe("real-world")
    expect(task.successCriteria).toBeDefined()
    expect(task.primaryDimensions?.length).toBeGreaterThanOrEqual(2)
    expect(task.dimensionRubrics?.length).toBeGreaterThanOrEqual(2)
  }
})

test("rw-2 fixture includes sales-data.csv with 3 day headers", () => {
  const task = REAL_WORLD_TASKS.find(t => t.id === "rw-2")!
  const csv = task.fixtures?.find(f => f.path === "sales-data.csv")
  expect(csv).toBeDefined()
  expect(csv!.content).toContain("2025-03-10")
  expect(csv!.content).toContain("2025-03-11")
  expect(csv!.content).toContain("ELEC-4K-TV-001")
})

test("rw-7 fixture includes three buggy TypeScript files", () => {
  const task = REAL_WORLD_TASKS.find(t => t.id === "rw-7")!
  const paths = task.fixtures?.map(f => f.path) ?? []
  expect(paths).toContain("src/validator.ts")
  expect(paths).toContain("src/processor.ts")
  expect(paths).toContain("src/pipeline.ts")
})

test("ABLATION_VARIANTS has exactly 10 variants", () => {
  // 9 production variants + 1 ablation variant (ra-full-noop-verifier for M3).
  // (Sprint-1 A2, 2026-06-02): ra-full-assembly-off removed alongside the
  // RA_ASSEMBLY flag deletion — canonical project() is the sole assembler.
  expect(ABLATION_VARIANTS).toHaveLength(10)
})

test("ABLATION_VARIANTS has 5 internal variants and 5 competitor variants", () => {
  // Internal: bare-llm, manual-react, ra-reasoning, ra-full, ra-full-noop-verifier.
  const internal = ABLATION_VARIANTS.filter(v => v.type === "internal")
  const competitor = ABLATION_VARIANTS.filter(v => v.type === "competitor")
  expect(internal).toHaveLength(5)
  expect(competitor).toHaveLength(5)
})

test("ABLATION_VARIANTS contains all 5 competitor frameworks", () => {
  const frameworks = ABLATION_VARIANTS
    .filter(v => v.type === "competitor")
    .map(v => (v as CompetitorVariant).framework)
  expect(frameworks).toContain("langchain")
  expect(frameworks).toContain("vercel-ai")
  expect(frameworks).toContain("openai-agents")
  expect(frameworks).toContain("mastra")
  expect(frameworks).toContain("llamaindex")
})

test("resolveTasks by taskIds returns matching tasks", () => {
  const session = {
    id: "test", name: "test", version: "1.0.0",
    taskIds: ["rw-1", "rw-2"],
    models: [], harnessVariants: [],
  }
  const tasks = resolveTasks(session, [...BENCHMARK_TASKS, ...REAL_WORLD_TASKS])
  expect(tasks).toHaveLength(2)
  expect(tasks.map(t => t.id)).toContain("rw-1")
})

test("resolveTasks by tier returns tasks of that tier", () => {
  const session = {
    id: "test", name: "test", version: "1.0.0",
    tiers: ["real-world" as const],
    models: [], harnessVariants: [],
  }
  const tasks = resolveTasks(session, [...BENCHMARK_TASKS, ...REAL_WORLD_TASKS])
  expect(tasks.length).toBe(10)
  expect(tasks.every(t => t.tier === "real-world")).toBe(true)
})

test("mergeConfigs combines base and override", () => {
  const base = { tools: true, reasoning: true }
  const override = { reactiveIntelligence: true, strategy: "react" as const }
  const merged = mergeConfigs(base, override)
  expect(merged.tools).toBe(true)
  expect(merged.reasoning).toBe(true)
  expect(merged.reactiveIntelligence).toBe(true)
  expect(merged.strategy).toBe("react")
})

test("computeReliability returns 1.0 for single run", () => {
  const runs = [{ dimensions: [{ dimension: "accuracy" as const, score: 0.8 }], tokensUsed: 100, durationMs: 1000, status: "pass" as const, output: "", runIndex: 0 }]
  expect(computeReliability(runs)).toBe(1)
})

test("computeReliability returns 1.0 for perfectly consistent runs", () => {
  const makeRun = (score: number, i: number) => ({
    runIndex: i, dimensions: [{ dimension: "accuracy" as const, score }],
    tokensUsed: 100, durationMs: 1000, status: "pass" as const, output: "",
  })
  const runs = [makeRun(0.9, 0), makeRun(0.9, 1), makeRun(0.9, 2)]
  expect(computeReliability(runs)).toBeCloseTo(1.0, 3)
})

test("computeReliability returns < 0.5 for highly inconsistent runs", () => {
  const makeRun = (score: number, i: number) => ({
    runIndex: i, dimensions: [{ dimension: "accuracy" as const, score }],
    tokensUsed: 100, durationMs: 1000, status: "pass" as const, output: "",
  })
  const runs = [makeRun(0.0, 0), makeRun(1.0, 1), makeRun(0.0, 2)]
  expect(computeReliability(runs)).toBeLessThan(0.5)
})

test("matchSuccessCriteria regex: matches correctly", () => {
  expect(matchSuccessCriteria("the answer is 42", { type: "regex", pattern: "42" })).toBe(1.0)
  expect(matchSuccessCriteria("the answer is 7", { type: "regex", pattern: "42" })).toBe(0.0)
})

test("matchSuccessCriteria regex: case-insensitive", () => {
  expect(matchSuccessCriteria("Hello World", { type: "regex", pattern: "hello" })).toBe(1.0)
})

test("parsePartialCreditScore: extracts pass ratio from bun test output", () => {
  const output = "3 pass\n1 fail\n"
  expect(parsePartialCreditScore(output)).toBeCloseTo(0.75, 2)
})

test("parsePartialCreditScore: returns 1.0 when all pass", () => {
  const output = "5 pass\n0 fail\n"
  expect(parsePartialCreditScore(output)).toBe(1.0)
})

test("parsePartialCreditScore: returns 0.0 when all fail", () => {
  const output = "0 pass\n3 fail\n"
  expect(parsePartialCreditScore(output)).toBe(0.0)
})

test("parsePartialCreditScore: returns 0.0 when output is unparseable", () => {
  expect(parsePartialCreditScore("some random output")).toBe(0.0)
})

test("CompetitorRunner interface is importable", () => {
  // Type-level check — if this compiles, the interface is correct
  const mockRunner: CompetitorRunner = {
    id: "test-runner",
    label: "Test Runner",
    framework: "langchain",
    pinnedVersion: "0.3.0",
    async run(_task, _model, _tmpDir, _timeoutMs): Promise<TaskRunResult> {
      return { output: "test", tokensUsed: 0, durationMs: 0, iterations: 0, status: "pass" }
    },
  }
  expect(mockRunner.id).toBe("test-runner")
})

test("COMPETITOR_RUNNERS registry has all 5 frameworks", () => {
  expect(Object.keys(COMPETITOR_RUNNERS)).toContain("langchain")
  expect(Object.keys(COMPETITOR_RUNNERS)).toContain("vercel-ai")
  expect(Object.keys(COMPETITOR_RUNNERS)).toContain("openai-agents")
  expect(Object.keys(COMPETITOR_RUNNERS)).toContain("mastra")
  expect(Object.keys(COMPETITOR_RUNNERS)).toContain("llamaindex")
})

test("each competitor runner has required fields", () => {
  for (const runner of Object.values(COMPETITOR_RUNNERS)) {
    expect(typeof runner.id).toBe("string")
    expect(typeof runner.label).toBe("string")
    expect(typeof runner.framework).toBe("string")
    expect(typeof runner.pinnedVersion).toBe("string")
    expect(typeof runner.run).toBe("function")
  }
})

test("openai-agents runner skips for non-OpenAI provider", async () => {
  const runner = COMPETITOR_RUNNERS["openai-agents"]!
  const fakeTask = {
    id: "test", tier: "trivial" as const, name: "Test", prompt: "Say hi",
  }
  const fakeModel = {
    id: "haiku", provider: "anthropic" as const, model: "claude-haiku-4-5",
  }
  const result = await runner.run(fakeTask, fakeModel, "/tmp", 5000)
  expect(result.status).toBe("error")
  expect(result.error).toContain("OpenAI")
})


function makeReport(taskId: string, variantId: string, accuracyScore: number): TaskVariantReport {
  const dim: DimensionScore = { dimension: "accuracy", score: accuracyScore }
  const run: RunScore = { runIndex: 0, dimensions: [dim], tokensUsed: 100, durationMs: 500, status: accuracyScore >= 0.6 ? "pass" : "fail", output: "" }
  return {
    taskId, modelVariantId: "haiku", variantId, variantLabel: variantId,
    runs: [run], meanScores: [dim], variance: 0,
    meanTokens: 100, meanDurationMs: 500, passRate: accuracyScore >= 0.6 ? 1 : 0,
  }
}

test("computeDrift detects regression when score drops > threshold", () => {
  const baseline = [makeReport("rw-1", "ra-full", 0.9)]
  const current  = [makeReport("rw-1", "ra-full", 0.6)]
  const drift = computeDrift(baseline, current, "abc123", 0.15)
  expect(drift.hasRegressions).toBe(true)
  expect(drift.regressions).toHaveLength(1)
  expect(drift.regressions[0]!.delta).toBeCloseTo(-0.3, 2)
})

test("computeDrift detects improvement when score rises", () => {
  const baseline = [makeReport("rw-1", "ra-full", 0.5)]
  const current  = [makeReport("rw-1", "ra-full", 0.8)]
  const drift = computeDrift(baseline, current, "abc123", 0.15)
  expect(drift.hasRegressions).toBe(false)
  expect(drift.improvements).toHaveLength(1)
})

test("computeDrift returns no changes when scores are within threshold", () => {
  const baseline = [makeReport("rw-1", "ra-full", 0.8)]
  const current  = [makeReport("rw-1", "ra-full", 0.75)]
  const drift = computeDrift(baseline, current, "abc123", 0.15)
  expect(drift.hasRegressions).toBe(false)
  expect(drift.regressions).toHaveLength(0)
  expect(drift.improvements).toHaveLength(0)
})

test("exceedsThreshold returns true when maxRegressionDelta > threshold", () => {
  const baseline = [makeReport("rw-1", "ra-full", 0.9)]
  const current  = [makeReport("rw-1", "ra-full", 0.5)]
  const drift = computeDrift(baseline, current, "abc123", 0.15)
  expect(exceedsThreshold(drift, 0.2)).toBe(true)
})


test("aggregateRuns computes correct passRate and meanTokens", () => {
  const runs = [
    { runIndex: 0, dimensions: [{ dimension: "accuracy" as const, score: 1.0 }], tokensUsed: 100, durationMs: 500, status: "pass" as const, output: "" },
    { runIndex: 1, dimensions: [{ dimension: "accuracy" as const, score: 0.0 }], tokensUsed: 200, durationMs: 600, status: "fail" as const, output: "" },
  ]
  const report = aggregateRuns("rw-1", "haiku", { type: "internal", id: "ra-full", label: "RA Full", config: {} }, runs)
  expect(report.passRate).toBeCloseTo(0.5, 2)
  expect(report.meanTokens).toBe(150)
  expect(report.meanScores[0]!.score).toBeCloseTo(0.5, 2)
})

test("computeAllAblation computes harnessLift correctly", () => {
  const baseReport = {
    taskId: "rw-1", modelVariantId: "haiku",
    variantId: "bare-llm", variantLabel: "Bare LLM",
    runs: [], meanScores: [{ dimension: "accuracy" as const, score: 0.3 }],
    variance: 0, meanTokens: 100, meanDurationMs: 500, passRate: 0.3,
  }
  const fullReport = {
    ...baseReport, variantId: "ra-full", variantLabel: "RA Full",
    meanScores: [{ dimension: "accuracy" as const, score: 0.9 }],
    passRate: 0.9,
  }
  const ablation = computeAllAblation([baseReport, fullReport])
  expect(ablation).toHaveLength(1)
  expect(ablation[0]!.harnessLift).toBeCloseTo(0.6, 2)
})


test("regressionGateSession has ra-full only and 1 run", () => {
  expect(regressionGateSession.harnessVariants).toHaveLength(1)
  expect(regressionGateSession.harnessVariants[0]!.id).toBe("ra-full")
  expect(regressionGateSession.runs).toBe(1)
})

test("realWorldFullSession has all 10 variants and 3 runs", () => {
  expect(realWorldFullSession.harnessVariants).toHaveLength(10)
  expect(realWorldFullSession.runs).toBe(3)
  expect(realWorldFullSession.tiers).toContain("real-world")
})

test("competitorComparisonSession has both Anthropic and OpenAI models", () => {
  const providers = competitorComparisonSession.models.map(m => m.provider)
  expect(providers).toContain("anthropic")
  expect(providers).toContain("openai")
  expect(competitorComparisonSession.harnessVariants).toHaveLength(10)
})

test("localModelsSession has only bare-llm and ra-full variants", () => {
  const ids = localModelsSession.harnessVariants.map(v => v.id)
  expect(ids).toContain("bare-llm")
  expect(ids).toContain("ra-full")
  expect(ids).not.toContain("langchain-react")
})


test("parseArgs: existing --provider flag still works", () => {
  const args = parseArgs(["--provider", "anthropic", "--model", "claude-haiku-4-5"])
  expect(args.provider).toBe("anthropic")
  expect(args.model).toBe("claude-haiku-4-5")
})

test("parseArgs: --session flag parsed", () => {
  const args = parseArgs(["--session", "local-models"])
  expect(args.session).toBe("local-models")
})

test("parseArgs: --runs flag parsed as number", () => {
  const args = parseArgs(["--session", "local-models", "--runs", "3"])
  expect(args.runs).toBe(3)
})

test("parseArgs: --save-baseline and --ci flags parsed", () => {
  const args = parseArgs(["--session", "regression-gate", "--save-baseline"])
  expect(args.saveBaseline).toBe(true)
  const args2 = parseArgs(["--session", "regression-gate", "--ci"])
  expect(args2.ci).toBe(true)
})


test("index.ts exports all v2 public APIs", () => {
  expect(typeof runSession).toBe("function")
  expect(ABLATION_VARIANTS).toHaveLength(10)
  expect(REAL_WORLD_TASKS).toHaveLength(10)
  expect(regressionGateSession.id).toBe("regression-gate")
  expect(realWorldFullSession.id).toBe("real-world-full")
  expect(competitorComparisonSession.id).toBe("competitor-comparison")
  expect(localModelsSession.id).toBe("local-models")
  expect(typeof computeReliability).toBe("function")
  expect(typeof computeDrift).toBe("function")
})
