/**
 * eval.js - Eval harness for measuring skill/agent/workflow effectiveness
 *
 * Orchestrates eval runs: loads scenarios, spawns trial sessions,
 * tracks results as JSONL, and computes pass@k metrics.
 *
 * Zero external dependencies — Node.js standard library only.
 */

const fs = require('node:fs');
const path = require('node:path');
const { execCommand, ensureDir, getTimestamp, CLAUDE_MAX_BUFFER } = require('./utils');
const stats = require('./eval-stats');
const graders = require('./eval-graders');
const {
  SCENARIOS_DIR,
  normalizeClaimType,
  inferClaimType,
  compactDate,
  parseEvalName,
  resolveMaxTurns,
  parseScenario,
  listScenarios,
  findScenario,
} = require('./eval-scenario');
const {
  createTrialDir,
  cleanupTrialDir,
  runSetup,
  writeIsolationSettings,
  buildIsolationSettings,
} = require('./eval-trial-env');
const { parseStreamJsonOutput, parseActionsFromTranscript } = require('./eval-transcript');

/**
 * Eval scenario parsed from a markdown file
 * @typedef {Object} EvalScenario
 * @property {string} name - Eval name
 * @property {string} scope - 'skill' | 'agent' | 'workflow'
 * @property {string} scenario - The prompt/task to run
 * @property {string} context - Setup context
 * @property {string[]} assertions - List of assertions to verify
 * @property {string} grader - 'code' | 'model' | 'human'
 * @property {string} graderConfig - Grader-specific configuration
 * @property {string} setup - Shell command to prepare trial directory (empty = use projectRoot)
 * @property {string} [preflight] - Optional preflight policy ('skip' to bypass A/B preflight gate)
 * @property {string} [verdictPolicy] - Optional A/B verdict policy ('non-regression' to judge treatment pass/fail instead of delta)
 * @property {string} [claimType] - Evidence claim type ('non-regression' | 'discriminative-lift' | 'self-improvement-smoke' | 'infra')
 */

/**
 * Single trial result
 * @typedef {Object} TrialResult
 * @property {string} eval - Eval scenario name
 * @property {number} trial - Trial number
 * @property {number} k - Total trials planned
 * @property {boolean} passed - Whether this trial passed
 * @property {string} grader - Grader type used
 * @property {number} score - Score from 0.0 to 1.0
 * @property {string} timestamp - ISO timestamp
 * @property {number|null} duration_ms - Wall-clock duration of the trial in ms (null if unavailable)
 * @property {number|null} input_tokens - Input tokens used by the trial agent (null if unavailable)
 * @property {number|null} output_tokens - Output tokens used by the trial agent (null if unavailable)
 * @property {string} [transcript] - Path to transcript file
 * @property {string} [trialDir] - Isolated temp directory used for this trial
 * @property {string} [error] - Error message if failed
 * @property {string} [errorType] - Machine-readable error category (e.g., 'model_grader_failed')
 * @property {boolean} [gradeError] - True if the grader failed to produce a score
 * @property {boolean} [infraError] - True if the trial runner failed to capture output
 * @property {number[]} [assertionScores] - Per-assertion scores (0.0-1.0)
 * @property {string[]} [evidence] - Per-assertion evidence notes from grader
 * @property {number[][]} [blockRefs] - Per-assertion transcript block references (1-indexed)
 * @property {Array<{type: string, name?: string, args?: string, content?: string, index: number}>} [actions] - Parsed actions from transcript
 */

const EVALS_DIR = 'evals';
const RESULTS_DIR = path.join(EVALS_DIR, 'results');
const BENCHMARKS_DIR = path.join(EVALS_DIR, 'benchmarks');

/**
 * Run a single eval trial by spawning a Claude session.
 * Runs in a temp directory for workspace safety. When isolated (default),
 * plugins are disabled and MCP servers stripped. When not isolated,
 * the agent has access to the full toolkit (plugins, MCP, skills, hooks).
 * @param {EvalScenario} scenario - The eval scenario
 * @param {number} trialNumber - Trial number (1-indexed)
 * @param {number} totalTrials - Total number of trials (k)
 * @param {Object} options - Run options
 * @param {string} [options.projectRoot] - Project root (for transcript storage + code grading)
 * @param {string} [options.isolationSettings] - Cached isolation settings JSON
 * @param {boolean} [options.isolated=true] - Whether to disable plugins and MCP
 * @param {string} [options.pluginDir] - Plugin directory for semi-isolated mode
 * @param {number} [options.maxTurns] - Max turns for Claude CLI
 * @returns {TrialResult} Trial result
 */
function runTrial(scenario, trialNumber, totalTrials, options = {}) {
  const {
    projectRoot = process.cwd(),
    label,
    isolationSettings,
    isolated = true,
    model,
    runId,
    pluginDir: rawPluginDir,
    maxTurns: rawMaxTurns,
  } = options;
  const timestamp = getTimestamp();

  const buildInfraError = (error, errorType, extra = {}) => {
    const evalName = label ? `${scenario.name}-${label}` : scenario.name;
    return {
      eval: evalName,
      trial: trialNumber,
      k: totalTrials,
      passed: false,
      grader: scenario.grader,
      score: 0,
      timestamp,
      duration_ms: null,
      input_tokens: null,
      output_tokens: null,
      error,
      errorType,
      infraError: true,
      ...(model ? { model } : {}),
      ...(runId ? { runId } : {}),
      ...extra,
    };
  };

  // Merge CLI overrides with scenario defaults (only when not fully isolated)
  const pluginDir = rawPluginDir || (!isolated ? scenario.pluginDir : undefined) || undefined;

  // Validate pluginDir exists before running trial
  if (pluginDir && !fs.existsSync(path.resolve(pluginDir))) {
    return buildInfraError(`Plugin dir does not exist: ${pluginDir}`, 'plugin_dir_missing');
  }

  // Always run in trial dir for workspace safety
  const trialDir = createTrialDir(scenario.name, trialNumber, projectRoot);
  try {
    if (scenario.setup) runSetup(scenario.setup, trialDir, projectRoot);
  } catch (error) {
    return buildInfraError(error.message || String(error), 'setup_failed', { trialDir });
  }

  // Isolation mode: full isolation uses writeIsolationSettings,
  // pluginDir uses semi-isolation (no claudeMdExcludes)
  if (pluginDir) {
    const semiSettings = isolationSettings || buildIsolationSettings({ excludeClaudeMd: false });
    writeIsolationSettings(trialDir, semiSettings);
  } else if (isolated) {
    writeIsolationSettings(trialDir, isolationSettings);
  }

  const prompt = buildTrialPrompt(scenario);

  const claudeArgs = [
    '-p',
    '--output-format',
    'stream-json',
    '--verbose',
    '--no-session-persistence',
    '--disable-slash-commands',
  ];
  if (isolated && !pluginDir) {
    claudeArgs.push('--strict-mcp-config');
    claudeArgs.push(
      '--append-system-prompt',
      `IMPORTANT: You are running in an isolated eval trial. Your working directory is ${trialDir}. Do NOT read, search, or access any files outside this directory. All files you need are already in the working directory.`,
    );
  }
  if (pluginDir) {
    claudeArgs.push('--plugin-dir', path.resolve(pluginDir));
    // Eval trials run unattended in ephemeral dirs — no human to approve permission prompts
    claudeArgs.push('--dangerously-skip-permissions');
  }

  // Resolve max-turns: CLI > scenario > pluginDir default (10)
  const resolvedMaxTurns = resolveMaxTurns({
    maxTurns: rawMaxTurns,
    scenarioMaxTurns: scenario.maxTurns,
    pluginDir,
  });
  if (resolvedMaxTurns != null) claudeArgs.push('--max-turns', String(resolvedMaxTurns));

  if (model) claudeArgs.push('--model', model);

  // Debug: log command for troubleshooting
  if (process.env.EVAL_DEBUG) {
    console.error(`[eval-debug] cwd: ${trialDir}`);
    console.error(`[eval-debug] cmd: claude ${claudeArgs.join(' ')}`);
    console.error(`[eval-debug] prompt: ${prompt.slice(0, 100)}...`);
  }
  const t0 = Date.now();
  const result = execCommand('claude', claudeArgs, {
    input: prompt,
    cwd: trialDir,
    timeout: 300000,
    maxBuffer: CLAUDE_MAX_BUFFER,
  });
  const wallDuration = Date.now() - t0;
  if (process.env.EVAL_DEBUG) {
    console.error(`[eval-debug] exitCode: ${result.exitCode}`);
    console.error(`[eval-debug] stdout length: ${(result.stdout || '').length}`);
    console.error(`[eval-debug] stderr: ${(result.stderr || '').slice(0, 300)}`);
  }

  const evalName = label ? `${scenario.name}-${label}` : scenario.name;
  // With stream-json, stdout may contain valid tool-use data even on non-zero exit
  // (e.g., max-turns reached). Try stdout first, fall back to stderr.
  const rawOutput = result.stdout || result.stderr || '';
  const { textResult, richTranscript, usage } = parseStreamJsonOutput(rawOutput);
  // Prefer the stream-json duration_ms if available; fall back to wall-clock.
  const duration_ms = usage.duration_ms ?? wallDuration;
  const parsedOutput = richTranscript || textResult;
  const transcriptOutput = parsedOutput || rawOutput;
  const transcript = saveTranscript(evalName, trialNumber, transcriptOutput, projectRoot, runId);
  const actions = parseActionsFromTranscript(richTranscript);

  const base = {
    eval: evalName,
    trial: trialNumber,
    k: totalTrials,
    passed: false, // Will be set by grader
    grader: scenario.grader,
    score: 0,
    timestamp,
    duration_ms,
    input_tokens: usage.input_tokens,
    output_tokens: usage.output_tokens,
    transcript,
    trialDir,
    ...(actions.length > 0 ? { actions } : {}),
    ...(model ? { model } : {}),
    ...(runId ? { runId } : {}),
  };
  if (result.exitCode !== 0 && !parsedOutput) {
    // Only treat as error if no usable output was captured
    return { ...base, error: result.stderr };
  }

  if (parsedOutput) {
    return { ...base, output: parsedOutput };
  }

  return {
    ...base,
    output: '',
    error: 'No assistant output captured from stream-json output',
    errorType: 'trial_output_missing',
    infraError: true,
  };
}

/**
 * Save full trial output to a transcript file
 * @param {string} evalName - Eval name (may include label suffix)
 * @param {number} trialNumber - Trial number
 * @param {string} output - Full output text
 * @param {string} projectRoot - Project root directory
 * @returns {string} Path to transcript file
 */
function saveTranscript(evalName, trialNumber, output, projectRoot, runId) {
  const { scenarioName, condition } = parseEvalName(evalName);
  const prefix = runId || compactDate();
  const transcriptsPath = path.join(projectRoot, RESULTS_DIR, scenarioName, prefix, 'transcripts');
  ensureDir(transcriptsPath);
  const fileName =
    condition === 'results' ? `trial-${trialNumber}.txt` : `${condition}-trial-${trialNumber}.txt`;
  const filePath = path.join(transcriptsPath, fileName);
  fs.writeFileSync(filePath, output);
  return filePath;
}

/**
 * Execute a single trial: run → grade → append → callback → cleanup.
 * Shared helper for both sequential and interleaved A/B modes.
 * @param {EvalScenario} trialScenario - Scenario to run (may have skill instruction in context)
 * @param {EvalScenario} gradeScenario - Original scenario for grading (without skill instruction)
 * @param {number} trialNumber - Trial number (1-indexed)
 * @param {number} k - Total trials per condition
 * @param {Object} opts - { projectRoot, label, onTrialComplete }
 * @returns {TrialResult} Graded result
 */
function executeAndGradeTrial(trialScenario, gradeScenario, trialNumber, k, opts) {
  const {
    projectRoot,
    label,
    onTrialComplete,
    isolationSettings,
    isolated,
    model,
    runId,
    pluginDir,
    maxTurns,
  } = opts;
  const result = runTrial(trialScenario, trialNumber, k, {
    projectRoot,
    label,
    isolationSettings,
    isolated,
    model,
    runId,
    pluginDir,
    maxTurns,
  });
  if (result.infraError) {
    try {
      appendResult(result, projectRoot);
      if (onTrialComplete) onTrialComplete(label, trialNumber, result);
      return result;
    } finally {
      cleanupTrialDir(result.trialDir);
    }
  }
  try {
    const graded = graders.gradeTrialResult(result, gradeScenario, projectRoot, result.actions);
    const versioned = gradeScenario.version
      ? { ...graded, version: gradeScenario.version }
      : graded;
    appendResult(versioned, projectRoot);
    if (onTrialComplete) onTrialComplete(label, trialNumber, versioned);
    return versioned;
  } finally {
    cleanupTrialDir(result.trialDir);
  }
}

/**
 * Run an A/B eval: execute k trials for each condition, grade, compute delta.
 * Shared by both skill and workflow evals — only the options differ.
 * @param {EvalScenario} baseScenario - Scenario for baseline trials
 * @param {EvalScenario} treatScenario - Scenario for treatment trials (may differ from base)
 * @param {EvalScenario} gradeScenario - Original scenario for grading
 * @param {number} k - Trials per condition
 * @param {Object} bOpts - Baseline trial options
 * @param {Object} tOpts - Treatment trial options
 * @param {boolean} interleave - Alternate baseline/treatment trials
 * @returns {{ baseline: TrialResult[], treatment: TrialResult[], delta: number }}
 */
function runAbTrials(baseScenario, treatScenario, gradeScenario, k, bOpts, tOpts, interleave) {
  const baseline = [];
  const treatment = [];

  if (interleave) {
    for (let t = 1; t <= k; t++) {
      baseline.push(executeAndGradeTrial(baseScenario, gradeScenario, t, k, bOpts));
      treatment.push(executeAndGradeTrial(treatScenario, gradeScenario, t, k, tOpts));
    }
  } else {
    for (let t = 1; t <= k; t++) {
      baseline.push(executeAndGradeTrial(baseScenario, gradeScenario, t, k, bOpts));
    }
    for (let t = 1; t <= k; t++) {
      treatment.push(executeAndGradeTrial(treatScenario, gradeScenario, t, k, tOpts));
    }
  }

  return { baseline, treatment, delta: stats.computeDelta(baseline, treatment) };
}

/**
 * Run a skill eval as A/B comparison: baseline (without skill) vs treatment (with skill).
 * Both conditions run in isolated environments; the treatment prepends skill instruction.
 * @param {EvalScenario} scenario - Scenario with scope='skill'
 * @param {number} k - Number of trials per condition
 * @param {Object} options - Run options
 * @param {string} [options.projectRoot] - Project root
 * @param {string} [options.skillInstruction] - Instruction to prepend for treatment trials
 * @param {boolean} [options.interleave=false] - Alternate baseline/treatment trials
 * @returns {{ baseline: TrialResult[], treatment: TrialResult[], delta: number }}
 */
function runSkillEval(scenario, k, options = {}) {
  const {
    projectRoot = process.cwd(),
    skillInstruction,
    onTrialComplete,
    interleave = false,
    model,
    runId,
    pluginDir,
    maxTurns,
  } = options;
  const isolationSettings = buildIsolationSettings();

  const treatmentScenario = {
    ...scenario,
    context: skillInstruction ? `${skillInstruction}\n\n${scenario.context}` : scenario.context,
  };

  const bOpts = {
    projectRoot,
    label: 'baseline',
    onTrialComplete,
    isolationSettings,
    model,
    runId,
  };
  const tOpts = {
    projectRoot,
    label: 'treatment',
    onTrialComplete,
    isolationSettings,
    model,
    runId,
    ...(pluginDir ? { pluginDir, isolated: false } : {}),
    ...(maxTurns != null ? { maxTurns } : {}),
  };
  return runAbTrials(scenario, treatmentScenario, scenario, k, bOpts, tOpts, interleave);
}

/**
 * Run a workflow eval as A/B comparison: isolated baseline vs semi-isolated/non-isolated treatment.
 * Baseline always runs fully isolated. Treatment uses semi-isolated mode with --plugin-dir
 * when the scenario has a pluginDir field; otherwise falls back to non-isolated mode.
 * @param {EvalScenario} scenario - Scenario with scope='workflow'
 * @param {number} k - Number of trials per condition
 * @param {Object} options - Run options
 * @param {string} [options.projectRoot] - Project root
 * @param {boolean} [options.interleave=false] - Alternate baseline/treatment trials
 * @param {Function} [options.onTrialComplete] - Callback per trial
 * @returns {{ baseline: TrialResult[], treatment: TrialResult[], delta: number }}
 */
function runWorkflowEval(scenario, k, options = {}) {
  const {
    projectRoot = process.cwd(),
    onTrialComplete,
    interleave = false,
    model,
    runId,
    pluginDir,
    maxTurns,
  } = options;
  const isolationSettings = buildIsolationSettings();
  const resolvedPluginDir = pluginDir || scenario.pluginDir;
  // Cache semi-isolation settings once (avoids spawning `claude plugin list` per trial)
  const semiSettings = resolvedPluginDir
    ? buildIsolationSettings({ excludeClaudeMd: false })
    : undefined;

  const bOpts = {
    projectRoot,
    label: 'baseline',
    onTrialComplete,
    isolationSettings,
    isolated: true,
    model,
    runId,
  };

  const tOpts = {
    projectRoot,
    label: 'treatment',
    onTrialComplete,
    isolated: false,
    model,
    runId,
    ...(resolvedPluginDir ? { pluginDir: resolvedPluginDir, isolationSettings: semiSettings } : {}),
    ...(maxTurns != null ? { maxTurns } : {}),
  };
  return runAbTrials(scenario, scenario, scenario, k, bOpts, tOpts, interleave);
}

/**
 * Build a prompt for a trial run
 * @param {EvalScenario} scenario - The eval scenario
 * @returns {string} Prompt text
 */
function buildTrialPrompt(scenario) {
  const parts = [];

  if (scenario.context) {
    parts.push(`## Context\n${scenario.context}`);
  }

  parts.push(`## Task\n${scenario.scenario}`);

  // Assertions are NOT included in the prompt — they are grading criteria
  // for the grader (Step 4), not requirements for the agent (Step 3).

  return parts.join('\n\n');
}

/**
 * Append a trial result to the results JSONL file
 * @param {TrialResult} result - Trial result
 * @param {string} projectRoot - Project root directory
 */
function appendResult(result, projectRoot) {
  const resultsPath = path.join(projectRoot, RESULTS_DIR);

  // Truncate output for storage only (grading already used full output)
  const maxStorageLen = 50000;
  const storable =
    result.output && result.output.length > maxStorageLen
      ? { ...result, output: `${result.output.slice(0, maxStorageLen)}\n[truncated for storage]` }
      : result;

  const { scenarioName, condition } = parseEvalName(storable.eval);
  const runId = storable.runId || compactDate(storable.timestamp);
  const runDir = path.join(resultsPath, scenarioName, runId);
  ensureDir(runDir);
  const filePath = path.join(runDir, `${condition}.jsonl`);

  fs.appendFileSync(filePath, `${JSON.stringify(storable)}\n`);
}

/**
 * Load results for a specific eval with optional filtering.
 * Uses exact segment match to avoid cross-eval contamination.
 * @param {string} evalName - Eval scenario name
 * @param {string} projectRoot - Project root directory
 * @param {Object} [options] - Filter options
 * @param {string} [options.version] - Only include results matching this version
 * @param {string} [options.since] - Only include results with timestamp >= this ISO date string
 * @returns {TrialResult[]} Filtered results for this eval
 */
function loadResults(evalName, projectRoot, options = {}) {
  const resultsPath = path.join(projectRoot, RESULTS_DIR);
  if (!fs.existsSync(resultsPath)) {
    return [];
  }

  const results = [];
  const { scenarioName, condition } = parseEvalName(evalName);
  const sinceCompact = options.since ? options.since.slice(0, 10).replace(/-/g, '') : undefined;

  // ── 1. Hierarchical: results/{scenarioName}/*/{condition}.jsonl ──
  const scenarioDir = path.join(resultsPath, scenarioName);
  let foundHierarchical = false;
  if (fs.existsSync(scenarioDir) && fs.statSync(scenarioDir).isDirectory()) {
    foundHierarchical = true;
    const entries = fs.readdirSync(scenarioDir, { withFileTypes: true });
    entries.sort((a, b) => a.name.localeCompare(b.name));
    for (const entry of entries) {
      if (!entry.isDirectory() || entry.name === 'transcripts') continue;
      // Date filter on runId prefix (first 8 chars = YYYYMMDD)
      if (sinceCompact && entry.name.slice(0, 8) < sinceCompact) continue;

      const jsonlPath = path.join(scenarioDir, entry.name, `${condition}.jsonl`);
      if (!fs.existsSync(jsonlPath)) continue;

      const content = fs.readFileSync(jsonlPath, 'utf8');
      for (const line of content.split('\n').filter((l) => l.trim())) {
        try {
          results.push(JSON.parse(line));
        } catch {
          /* skip malformed lines */
        }
      }
    }
  }

  // ── 2. Legacy flat: results/{date}-{evalName}.jsonl ──
  if (!foundHierarchical) {
    const suffix = `-${evalName}.jsonl`;
    const sinceDate = options.since ? options.since.slice(0, 10) : undefined;
    const files = fs.readdirSync(resultsPath).filter((f) => {
      if (!f.endsWith(suffix)) return false;
      const prefix = f.slice(0, f.length - suffix.length);
      if (!/^\d{4}-\d{2}-\d{2}$/.test(prefix)) return false;
      if (sinceDate && prefix < sinceDate) return false;
      return true;
    });

    for (const file of files) {
      const content = fs.readFileSync(path.join(resultsPath, file), 'utf8');
      for (const line of content.split('\n').filter((l) => l.trim())) {
        try {
          results.push(JSON.parse(line));
        } catch {
          /* skip malformed lines */
        }
      }
    }
  }

  // ── 3. Apply filters ──
  if (!options.version && !options.since && !options.model) return results;
  return results.filter((r) => {
    if (options.version && (r.version || '1') !== options.version) return false;
    if (options.since && r.timestamp < options.since) return false;
    if (options.model && r.model !== options.model) return false;
    return true;
  });
}

/**
 * Ensure evals directory structure exists
 * @param {string} projectRoot - Project root directory
 */
function ensureEvalsDir(projectRoot) {
  ensureDir(path.join(projectRoot, SCENARIOS_DIR));
  ensureDir(path.join(projectRoot, RESULTS_DIR));
  ensureDir(path.join(projectRoot, BENCHMARKS_DIR));
}

module.exports = {
  // Orchestration
  parseEvalName,
  parseScenario,
  normalizeClaimType,
  inferClaimType,
  compactDate,
  listScenarios,
  findScenario,
  createTrialDir,
  cleanupTrialDir,
  runSetup,
  writeIsolationSettings,
  buildIsolationSettings,
  buildPluginDirSettings: () => buildIsolationSettings({ excludeClaudeMd: false }),
  parseStreamJsonOutput,
  parseActionsFromTranscript,
  resolveMaxTurns,
  runTrial,
  buildTrialPrompt,
  executeAndGradeTrial,
  runSkillEval,
  runWorkflowEval,
  saveTranscript,
  appendResult,
  loadResults,
  ensureEvalsDir,
  // Re-export graders for backward compatibility
  ...graders,
  // Re-export stats for backward compatibility
  ...stats,
  // Constants
  EVALS_DIR,
  SCENARIOS_DIR,
  RESULTS_DIR,
  BENCHMARKS_DIR,
};
