#!/usr/bin/env npx tsx
/**
 * HackAPrompt Benchmark Runner
 *
 * Loads a sample from the HackAPrompt 600K+ adversarial prompt dataset
 * (produced by scripts/hackaprompt-to-corpus.py) and runs it through the
 * ATR evaluation harness to measure recall against real prompt-hacking
 * attempts collected at competition scale.
 *
 * Usage:
 *   npx tsx src/eval/run-hackaprompt-benchmark.ts
 *
 * HackAPrompt is an all-adversarial corpus, so we measure recall, latency,
 * and tier breakdown. Precision/FP rate require a benign companion source.
 *
 * @module agent-threat-rules/eval/run-hackaprompt-benchmark
 */

import { resolve, join } from 'node:path';
import { loadHackaPromptCorpus, getHackaPromptCorpusStats } from './hackaprompt-corpus.js';
import { runEval } from './eval-harness.js';
import { writeMeasurementFromEvalReport } from '../measurement/from-eval-harness.js';

function formatPercent(n: number): string {
  return `${(n * 100).toFixed(1)}%`;
}

function formatMs(n: number): string {
  return `${n.toFixed(2)}ms`;
}

async function main(): Promise<void> {
  const base = resolve(join(import.meta.dirname ?? '.', '..', '..'));
  const rulesDir = join(base, 'rules');
  const dataPath = join(base, 'data', 'hackaprompt', 'hackaprompt-corpus.json');
  const outputPath = join(base, 'data', 'hackaprompt', 'hackaprompt-eval-report.json');

  console.log('\n=== HackAPrompt Benchmark -- ATR Evaluation ===\n');
  console.log(`Corpus: ${dataPath}`);
  console.log(`Rules:  ${rulesDir}\n`);

  const corpus = loadHackaPromptCorpus(dataPath);
  const stats = getHackaPromptCorpusStats(corpus);

  console.log(`Loaded ${stats.total} samples (${stats.attacks} attacks, ${stats.benign} benign)`);
  console.log(`Categories: ${Object.entries(stats.byCategory).map(([k, v]) => `${k}=${v}`).join(', ')}`);
  console.log(`Difficulty: ${Object.entries(stats.byDifficulty).map(([k, v]) => `${k}=${v}`).join(', ')}`);

  // HackAPrompt is an all-adversarial corpus from a public global competition.
  // It contains heavy paraphrasing, role-play, multilingual, and creative
  // attacks. Recall is the headline number. FP rate is meaningless here
  // because there are no benign samples in the corpus.
  const hackaPromptThresholds = {
    minRecall: 0.10,
    maxFpRate: 1.0,
    minF1: 0.0,
    maxP95LatencyMs: 200,
  };

  const { report, tiersUsed, ruleQuality } = await runEval({
    rulesDir,
    corpus,
    thresholds: hackaPromptThresholds,
    outputPath,
  });

  console.log(`\nTiers: ${tiersUsed.join(' + ')}`);
  console.log(`\n--- Overall ---`);
  console.log(`  Recall:     ${formatPercent(report.overall.recall)}`);
  console.log(`  Precision:  ${formatPercent(report.overall.precision)} (N/A meaning - no benign samples)`);
  console.log(`  Confusion:  TP=${report.overall.confusion.tp} FN=${report.overall.confusion.fn}`);

  console.log(`\n--- Latency ---`);
  console.log(`  P50:  ${formatMs(report.latency.p50)}`);
  console.log(`  P95:  ${formatMs(report.latency.p95)}`);
  console.log(`  P99:  ${formatMs(report.latency.p99)}`);
  console.log(`  Mean: ${formatMs(report.latency.mean)}`);

  console.log(`\n--- By Category ---`);
  for (const cat of report.byCategory) {
    const m = cat.metrics;
    const tp = m.confusion.tp;
    const fn = m.confusion.fn;
    const recall = tp + fn === 0 ? 0 : tp / (tp + fn);
    console.log(`  ${cat.category}: recall=${formatPercent(recall)} (TP=${tp} FN=${fn})`);
  }

  console.log(`\n--- Top Firing Rules (top 15) ---`);
  const fired = ruleQuality?.topRules ?? [];
  for (const r of fired.slice(0, 15)) {
    console.log(`  ${r.ruleId}: matches=${r.matchCount} TP=${r.tpCount} FP=${r.fpCount}`);
  }

  console.log(`\nReport saved to: ${outputPath}`);

  // Standardized Measurement file (version-pinned, immutable).
  const { measurementPath } = writeMeasurementFromEvalReport(report, {
    source: 'hackaprompt',
    source_version: 'v1',
    source_url: 'https://huggingface.co/datasets/hackaprompt/hackaprompt-dataset',
    notes: 'HackAPrompt competition dataset. 100% adversarial — fp_rate is undefined on this corpus.',
  });
  console.log(`Measurement: ${measurementPath}`);
  console.log('Done.');
}

main().catch((err) => {
  console.error('Error:', err);
  process.exit(1);
});
