#!/usr/bin/env node
// Benchmark runner for claude-mem-lite search quality
// Uses the exact same BM25 scoring formula from server.mjs

import { readFileSync } from 'fs';
import { join, dirname } from 'path';
import { fileURLToPath } from 'url';
import { sanitizeFtsQuery, estimateTokens } from '../utils.mjs';
import { createTestDb } from '../tests/test-helpers.mjs';

const __dirname = dirname(fileURLToPath(import.meta.url));

// ─── Seed Database ──────────────────────────────────────────────────────────

export function seedDatabase(db, data) {
  const now = Date.now();

  // Create sessions first (referenced by observations via memory_session_id)
  const sessionIds = new Set();
  const insertSession = db.prepare(`
    INSERT OR IGNORE INTO sdk_sessions (content_session_id, memory_session_id, project, started_at, started_at_epoch, status)
    VALUES (?, ?, ?, ?, ?, 'completed')
  `);

  // Collect all unique session_ids from observations
  for (const obs of data.observations) {
    if (!sessionIds.has(obs.session_id)) {
      sessionIds.add(obs.session_id);
      const epoch = now + (obs.epoch_offset_days * 86400000);
      insertSession.run(obs.session_id, obs.session_id, obs.project, new Date(epoch).toISOString(), epoch);
    }
  }

  // Insert observations
  const insertObs = db.prepare(`
    INSERT INTO observations (id, memory_session_id, project, text, type, title, narrative, facts, concepts, files_modified,
      created_at, created_at_epoch, importance)
    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
  `);

  const obsTransaction = db.transaction(() => {
    for (const obs of data.observations) {
      const epoch = now + (obs.epoch_offset_days * 86400000);
      insertObs.run(
        obs.id, obs.session_id, obs.project, obs.text, obs.type,
        obs.title, obs.narrative, obs.facts, obs.concepts, obs.files_modified,
        new Date(epoch).toISOString(), epoch, obs.importance
      );
    }
  });
  obsTransaction();

  // Insert session summaries
  if (data.sessions && data.sessions.length > 0) {
    const insertSummary = db.prepare(`
      INSERT INTO session_summaries (id, memory_session_id, project, request, investigated, learned, completed, next_steps,
        created_at, created_at_epoch)
      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
    `);

    const sessTransaction = db.transaction(() => {
      for (const sess of data.sessions) {
        const epoch = now + (sess.epoch_offset_days * 86400000);
        // Ensure the session exists
        if (!sessionIds.has(sess.session_id)) {
          sessionIds.add(sess.session_id);
          insertSession.run(sess.session_id, sess.session_id, sess.project, new Date(epoch).toISOString(), epoch);
        }
        insertSummary.run(
          sess.id, sess.session_id, sess.project, sess.request,
          sess.investigated, sess.learned, sess.completed, sess.next_steps,
          new Date(epoch).toISOString(), epoch
        );
      }
    });
    sessTransaction();
  }

  return { observations: data.observations.length, sessions: (data.sessions || []).length };
}

// ─── Search (mirrors server.mjs BM25 query exactly) ────────────────────────
//
// Modes:
//   'hybrid'        — production scoring: BM25 × time-decay × project-boost ×
//                     importance × access-bonus. Mirrors server.mjs.
//   'bm25_only'     — strips all multipliers, pure BM25 ranking. Tests whether
//                     production multipliers add lift over raw FTS5.
//   'recency'       — no FTS, ORDER BY created_at_epoch DESC. Tests whether FTS
//                     retrieval adds anything over "newest-first" baseline.
//   'random'        — deterministic shuffle (seeded by query). Sanity floor —
//                     anything that doesn't beat random is broken.
//   'no_decay'      — drop time-decay; keep project/importance/access. Per-term
//                     ablation: how much does the recency multiplier earn?
//   'no_project'    — drop project boost; isolates whether current-project bias
//                     is doing real work on the eval set.
//   'no_importance' — drop the (0.5+0.5*importance) multiplier.
//   'no_access'     — drop the (1+0.1*ln(1+access)) multiplier.
//
// Per-term ablation modes were added to answer "do these multipliers earn their
// keep?" — the previous matrix only compared full-hybrid vs bm25_only, leaving
// per-multiplier contribution invisible. Each MODE_TERMS entry lists which
// multipliers stay in the formula; placeholder threading stays correct because
// each multiplier's parameters are appended in MULT_PARAMS order.

const MULT_EXPR = {
  decay:      '(1.0 + EXP(-0.693 * (? - o.created_at_epoch) / 1209600000.0))',
  project:    '(CASE WHEN ? IS NOT NULL AND o.project = ? THEN 2.0 ELSE 1.0 END)',
  importance: '(0.5 + 0.5 * COALESCE(o.importance, 1))',
  access:     '(1.0 + 0.1 * LN(1 + COALESCE(o.access_count, 0)))',
};
const MULT_PARAMS = {
  decay:      (now)         => [now],
  project:    (_now, project)=> [project, project],
  importance: ()            => [],
  access:     ()            => [],
};
const MODE_TERMS = {
  hybrid:        ['decay', 'project', 'importance', 'access'],
  bm25_only:     [],
  no_decay:      ['project', 'importance', 'access'],
  no_project:    ['decay', 'importance', 'access'],
  no_importance: ['decay', 'project', 'access'],
  no_access:     ['decay', 'project', 'importance'],
};

function searchObservations(db, query, options = {}) {
  const mode = options.mode ?? 'hybrid';
  const limit = options.limit ?? 20;
  const project = options.project ?? null;
  const obsType = options.type ?? null;

  if (mode === 'random') return searchRandom(db, query, { limit, project, obsType });
  if (mode === 'recency') return searchRecency(db, { limit, project, obsType });

  const terms = MODE_TERMS[mode];
  if (!terms) throw new Error(`Unknown benchmark mode: ${mode}`);

  const ftsQuery = sanitizeFtsQuery(query);
  if (!ftsQuery) return [];

  const now = Date.now();
  const baseBm25 = 'bm25(observations_fts, 10, 5, 5, 3, 3, 2, 8)';
  const scoreExpr = terms.length === 0
    ? `${baseBm25} as score`
    : `${baseBm25} * ${terms.map((t) => MULT_EXPR[t]).join(' * ')} as score`;

  const scoreParams = terms.flatMap((t) => MULT_PARAMS[t](now, project));

  const sql = `
    SELECT o.id, o.type, o.title, o.subtitle, o.project, o.created_at, o.importance,
           o.files_modified, o.narrative, o.text,
           ${scoreExpr}
    FROM observations_fts
    JOIN observations o ON observations_fts.rowid = o.id
    WHERE observations_fts MATCH ?
      AND COALESCE(o.compressed_into, 0) = 0
      AND (? IS NULL OR o.project = ?)
      AND (? IS NULL OR o.type = ?)
    ORDER BY score
    LIMIT ?
  `;
  const params = [...scoreParams, ftsQuery, project, project, obsType, obsType, limit];
  const rows = db.prepare(sql).all(...params);

  return rows.map(r => ({
    id: r.id, type: r.type, title: r.title, project: r.project,
    score: r.score, importance: r.importance,
    tokens: estimateTokens((r.title || '') + ' ' + (r.narrative || ''))
  }));
}

function searchRecency(db, { limit, project, obsType }) {
  const rows = db.prepare(`
    SELECT o.id, o.type, o.title, o.subtitle, o.project, o.importance, o.narrative
    FROM observations o
    WHERE COALESCE(o.compressed_into, 0) = 0
      AND (? IS NULL OR o.project = ?)
      AND (? IS NULL OR o.type = ?)
    ORDER BY o.created_at_epoch DESC
    LIMIT ?
  `).all(project, project, obsType, obsType, limit);
  return rows.map(r => ({
    id: r.id, type: r.type, title: r.title, project: r.project,
    score: 0, importance: r.importance,
    tokens: estimateTokens((r.title || '') + ' ' + (r.narrative || ''))
  }));
}

function searchRandom(db, query, { limit, project, obsType }) {
  const rows = db.prepare(`
    SELECT o.id, o.type, o.title, o.project, o.importance, o.narrative
    FROM observations o
    WHERE COALESCE(o.compressed_into, 0) = 0
      AND (? IS NULL OR o.project = ?)
      AND (? IS NULL OR o.type = ?)
  `).all(project, project, obsType, obsType);
  // Deterministic shuffle: seed = hash(query) so repeated runs reproduce.
  let seed = 0;
  for (let i = 0; i < query.length; i++) seed = (seed * 31 + query.charCodeAt(i)) | 0;
  const shuffled = rows.slice();
  for (let i = shuffled.length - 1; i > 0; i--) {
    seed = (seed * 1103515245 + 12345) & 0x7fffffff;
    const j = seed % (i + 1);
    [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
  }
  return shuffled.slice(0, limit).map(r => ({
    id: r.id, type: r.type, title: r.title, project: r.project,
    score: 0, importance: r.importance,
    tokens: estimateTokens((r.title || '') + ' ' + (r.narrative || ''))
  }));
}

// ─── Metrics ────────────────────────────────────────────────────────────────

export function computeRecallAtK(results, relevantIds, k = 10) {
  if (!relevantIds || relevantIds.length === 0) return 0;
  const topK = results.slice(0, k).map(r => r.id);
  const hits = topK.filter(id => relevantIds.includes(id)).length;
  return hits / relevantIds.length;
}

export function computePrecisionAtK(results, relevantIds, k = 10) {
  const topK = results.slice(0, k);
  if (topK.length === 0) return 0;
  const hits = topK.filter(r => relevantIds.includes(r.id)).length;
  return hits / topK.length;
}

export function computeNDCG(results, relevantIds, k = 10) {
  if (!relevantIds || relevantIds.length === 0) return 0;
  const topK = results.slice(0, k);

  // Compute DCG
  let dcg = 0;
  for (let i = 0; i < topK.length; i++) {
    const rel = relevantIds.includes(topK[i].id) ? 1 : 0;
    dcg += rel / Math.log2(i + 2); // i+2 because log2(1) = 0
  }

  // Compute ideal DCG (all relevant docs at top)
  const idealHits = Math.min(relevantIds.length, k);
  let idcg = 0;
  for (let i = 0; i < idealHits; i++) {
    idcg += 1 / Math.log2(i + 2);
  }

  return idcg === 0 ? 0 : dcg / idcg;
}

export function computeMRR(results, relevantIds) {
  if (!relevantIds || relevantIds.length === 0) return 0;
  for (let i = 0; i < results.length; i++) {
    if (relevantIds.includes(results[i].id)) {
      return 1 / (i + 1);
    }
  }
  return 0;
}

export function measureLatencyP95(db, queries, mode = 'hybrid') {
  const latencies = [];
  // Run each query 3 times to get stable measurements
  for (let run = 0; run < 3; run++) {
    for (const q of queries) {
      const start = performance.now();
      searchObservations(db, q.query, { project: q.project, type: q.type, mode });
      const elapsed = performance.now() - start;
      latencies.push(elapsed);
    }
  }
  latencies.sort((a, b) => a - b);
  const p95Index = Math.floor(latencies.length * 0.95);
  return latencies[p95Index] || latencies[latencies.length - 1] || 0;
}

// ─── Run Benchmark ──────────────────────────────────────────────────────────

export function runBenchmark(db, queries, mode = 'hybrid') {
  const results = {
    timestamp: new Date().toISOString(),
    mode,
    queryCount: queries.length,
    metrics: {
      recall_at_10: 0,
      precision_at_10: 0,
      ndcg_at_10: 0,
      mrr_at_10: 0,
      avg_tokens_injected: 0,
      p95_search_latency_ms: 0,
    },
    perQuery: [],
    byCategory: {},
  };

  let totalRecall = 0;
  let totalPrecision = 0;
  let totalNDCG = 0;
  let totalMRR = 0;
  let totalTokens = 0;

  // Category accumulators
  const catAccum = {};

  for (const q of queries) {
    const searchResults = searchObservations(db, q.query, {
      project: q.project,
      type: q.type,
      limit: 10,
      mode,
    });

    const recall = computeRecallAtK(searchResults, q.relevant_ids, 10);
    const precision = computePrecisionAtK(searchResults, q.relevant_ids, 10);
    const ndcg = computeNDCG(searchResults, q.relevant_ids, 10);
    const mrr = computeMRR(searchResults, q.relevant_ids);
    const tokens = searchResults.reduce((sum, r) => sum + r.tokens, 0);

    totalRecall += recall;
    totalPrecision += precision;
    totalNDCG += ndcg;
    totalMRR += mrr;
    totalTokens += tokens;

    // Track per-category
    const cat = q.category || 'standard';
    if (!catAccum[cat]) catAccum[cat] = { recall: 0, precision: 0, ndcg: 0, mrr: 0, count: 0 };
    catAccum[cat].recall += recall;
    catAccum[cat].precision += precision;
    catAccum[cat].ndcg += ndcg;
    catAccum[cat].mrr += mrr;
    catAccum[cat].count++;

    results.perQuery.push({
      id: q.id,
      query: q.query,
      category: cat,
      recall_at_10: round(recall),
      precision_at_10: round(precision),
      ndcg_at_10: round(ndcg),
      mrr: round(mrr),
      result_ids: searchResults.map(r => r.id),
      relevant_ids: q.relevant_ids,
      tokens,
    });
  }

  const n = queries.length;
  results.metrics.recall_at_10 = round(totalRecall / n);
  results.metrics.precision_at_10 = round(totalPrecision / n);
  results.metrics.ndcg_at_10 = round(totalNDCG / n);
  results.metrics.mrr_at_10 = round(totalMRR / n);
  results.metrics.avg_tokens_injected = Math.round(totalTokens / n);
  results.metrics.p95_search_latency_ms = round(measureLatencyP95(db, queries, mode));

  // Per-category averages
  for (const [cat, acc] of Object.entries(catAccum)) {
    results.byCategory[cat] = {
      count: acc.count,
      recall_at_10: round(acc.recall / acc.count),
      precision_at_10: round(acc.precision / acc.count),
      ndcg_at_10: round(acc.ndcg / acc.count),
      mrr_at_10: round(acc.mrr / acc.count),
    };
  }

  return results;
}

function round(v) {
  return Math.round(v * 10000) / 10000;
}

// ─── Baseline Matrix ────────────────────────────────────────────────────────
//
// Runs the same query set across all four scoring modes. Surfaces the lift
// (or absence of lift) each layer adds: hybrid vs bm25_only isolates the
// contribution of production multipliers; bm25_only vs recency isolates
// FTS5 retrieval value; recency vs random isolates ordering value.
//
// Without this, the audit conclusion "no baseline" stands — every retrieval
// metric is unfalsifiable. With this, future scoring tuning has an evidence
// anchor: a tuning that doesn't beat bm25_only is overfitting; one that
// doesn't beat recency-only is broken.

const BASELINE_MODES = ['hybrid', 'bm25_only', 'recency', 'random'];
const ABLATION_MODES = ['no_decay', 'no_project', 'no_importance', 'no_access'];

/**
 * Deterministically partition a query fixture into train + eval splits so the
 * vocabulary built from `train` can be evaluated on truly held-out queries.
 * Without this, the benchmark trains and evaluates on the same set, inflating
 * metrics. Mulberry32 PRNG seeded by `seed` so partitions are reproducible.
 *
 * @param {Array}  queries      Full query fixture (each carries id + relevant_ids).
 * @param {number} [ratio=0.3]  Fraction reserved for eval split.
 * @param {number} [seed=42]    PRNG seed; same input → same split.
 * @returns {{ train: Array, eval: Array }}
 */
export function splitFixture(queries, ratio = 0.3, seed = 42) {
  const arr = queries.slice();
  let s = seed >>> 0;
  for (let i = arr.length - 1; i > 0; i--) {
    s = (s + 0x6d2b79f5) >>> 0;
    let t = s;
    t = Math.imul(t ^ (t >>> 15), t | 1);
    t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
    const r = ((t ^ (t >>> 14)) >>> 0) / 4294967296;
    const j = Math.floor(r * (i + 1));
    [arr[i], arr[j]] = [arr[j], arr[i]];
  }
  const evalCount = Math.max(1, Math.round(arr.length * ratio));
  return { eval: arr.slice(0, evalCount), train: arr.slice(evalCount) };
}

export function runBenchmarkMatrix(db, queries, options = {}) {
  const includeAblations = options.ablations !== false;
  const matrix = {
    timestamp: new Date().toISOString(),
    queryCount: queries.length,
    modes: {},
    deltas: {},
    perQueryDeltas: {},
  };
  // Lock the DB to read-only for the matrix run. searchObservations in this
  // file is currently SELECT-only, but server.mjs's production search bumps
  // access_count on read — if a future contributor wires that path here,
  // mode results would silently become order-dependent. `query_only=ON`
  // converts any accidental write into an error so the divergence surfaces.
  // Restored after the loop so this function plays nice with shared db.
  const wasQueryOnly = db.pragma('query_only', { simple: true });
  db.pragma('query_only = ON');
  try {
    const modes = includeAblations
      ? [...BASELINE_MODES, ...ABLATION_MODES]
      : BASELINE_MODES;
    for (const mode of modes) {
      matrix.modes[mode] = runBenchmark(db, queries, mode);
    }
  } finally {
    db.pragma(`query_only = ${wasQueryOnly ? 'ON' : 'OFF'}`);
  }
  // Lift over each lower tier (each delta tells you what that layer is buying).
  matrix.deltas.hybrid_over_bm25 = diffMetrics(matrix.modes.hybrid, matrix.modes.bm25_only);
  matrix.deltas.bm25_over_recency = diffMetrics(matrix.modes.bm25_only, matrix.modes.recency);
  matrix.deltas.recency_over_random = diffMetrics(matrix.modes.recency, matrix.modes.random);
  // Per-term ablation deltas: hybrid vs the single-multiplier-removed mode.
  // Positive Δ on a metric means dropping that multiplier hurts; near-zero
  // means it earns no measurable lift on this fixture and is a candidate for
  // removal (smaller surface, fewer SQL ops, identical quality).
  if (includeAblations) {
    for (const ab of ABLATION_MODES) {
      if (matrix.modes[ab]) {
        matrix.deltas[`hybrid_over_${ab}`] = diffMetrics(matrix.modes.hybrid, matrix.modes[ab]);
      }
    }
  }
  // Per-query Δ — surfaces WHICH queries gain from multipliers vs which are
  // multiplier-neutral. If hybrid_over_bm25 aggregate is 0, this answers
  // whether (a) all queries are individually 0-lift (multipliers are dead
  // weight on this corpus) or (b) some queries gain and some lose,
  // averaging to 0 (multipliers are noisy, possibly directionally wrong).
  // The delete-or-defend decision needs this granularity.
  matrix.perQueryDeltas.hybrid_over_bm25 = perQueryDelta(matrix.modes.hybrid, matrix.modes.bm25_only);
  matrix.perQueryDeltas.bm25_over_recency = perQueryDelta(matrix.modes.bm25_only, matrix.modes.recency);
  return matrix;
}

function diffMetrics(a, b) {
  const out = {};
  for (const k of ['recall_at_10', 'precision_at_10', 'ndcg_at_10', 'mrr_at_10']) {
    out[k] = round(a.metrics[k] - b.metrics[k]);
  }
  return out;
}

function perQueryDelta(higher, lower) {
  // Pair up by query id (perQuery arrays are ordered identically — same
  // input queries iterated in the same order across modes).
  const byId = new Map();
  for (const q of lower.perQuery) byId.set(q.id, q);
  const out = [];
  for (const hi of higher.perQuery) {
    const lo = byId.get(hi.id);
    if (!lo) continue;
    out.push({
      id: hi.id,
      query: hi.query,
      ndcg_delta: round(hi.ndcg_at_10 - lo.ndcg_at_10),
      mrr_delta: round(hi.mrr - lo.mrr),
      recall_delta: round(hi.recall_at_10 - lo.recall_at_10),
    });
  }
  // Sort by ndcg_delta DESC — biggest lifts first. Ties broken by mrr_delta.
  out.sort((a, b) => (b.ndcg_delta - a.ndcg_delta) || (b.mrr_delta - a.mrr_delta));
  return out;
}

// Summarize per-query delta into bins: how many queries gained, lost, or stayed flat.
// Default threshold 0.001 sits one decimal above round()'s 0.0001 precision —
// any 4-decimal-rounded delta of 0.0001/0.0002 falls in `neutral`, which is
// the right behavior (sub-rounding deltas are noise, not signal). Threshold
// 0.001 represents "≥0.1% nDCG move" — the smallest difference reliably
// reproducible across runs on this fixture.
export function summarizePerQueryDelta(deltas, threshold = 0.001) {
  const bins = { gained: 0, neutral: 0, lost: 0 };
  for (const d of deltas) {
    if (d.ndcg_delta > threshold) bins.gained++;
    else if (d.ndcg_delta < -threshold) bins.lost++;
    else bins.neutral++;
  }
  return bins;
}

// ─── CLI Entry Point ────────────────────────────────────────────────────────

function main() {
  const seedPath = join(__dirname, 'fixtures', 'seed-data.json');
  const queriesPath = join(__dirname, 'fixtures', 'test-queries.json');

  const seedData = JSON.parse(readFileSync(seedPath, 'utf-8'));
  const queryData = JSON.parse(readFileSync(queriesPath, 'utf-8'));

  console.error('Creating benchmark database...');
  const db = createTestDb();

  console.error('Seeding database...');
  const counts = seedDatabase(db, seedData);
  console.error(`  Seeded ${counts.observations} observations, ${counts.sessions} sessions`);

  const args = new Set(process.argv.slice(2));
  const matrixMode = args.has('--matrix') || args.has('--baselines');
  const holdoutMode = args.has('--holdout');
  const ablations = !args.has('--no-ablations');

  // --holdout splits the query fixture into train/eval (default 70/30,
  // deterministic seed) and runs the matrix on the eval split only. Closes the
  // overfitting hole where vocabulary built from seed-data was being scored on
  // the same set used to build it.
  let evalQueries = queryData.queries;
  let split = null;
  if (holdoutMode) {
    split = splitFixture(queryData.queries, 0.3, 42);
    evalQueries = split.eval;
    console.error(`Holdout mode: ${split.train.length} train (vocab build) / ${split.eval.length} eval queries`);
  }

  if (matrixMode) {
    console.error(`Running benchmark matrix (hybrid + 3 baselines${ablations ? ' + 4 ablations' : ''})...`);
    const matrix = runBenchmarkMatrix(db, evalQueries, { ablations });
    if (split) matrix.holdout = { train: split.train.length, eval: split.eval.length, seed: 42, ratio: 0.3 };
    console.log(JSON.stringify(matrix, null, 2));

    console.error('\n─── Benchmark Matrix ───');
    const head = `  ${'mode'.padEnd(15)} ${'R@10'.padStart(7)} ${'P@10'.padStart(7)} ${'nDCG'.padStart(7)} ${'MRR'.padStart(7)} ${'p95ms'.padStart(7)}`;
    console.error(head);
    console.error(`  ${'-'.repeat(15)} ${'-'.repeat(7)} ${'-'.repeat(7)} ${'-'.repeat(7)} ${'-'.repeat(7)} ${'-'.repeat(7)}`);
    const allModes = ablations ? [...BASELINE_MODES, ...ABLATION_MODES] : BASELINE_MODES;
    for (const mode of allModes) {
      const m = matrix.modes[mode].metrics;
      console.error(`  ${mode.padEnd(15)} ${String(m.recall_at_10).padStart(7)} ${String(m.precision_at_10).padStart(7)} ${String(m.ndcg_at_10).padStart(7)} ${String(m.mrr_at_10).padStart(7)} ${String(m.p95_search_latency_ms).padStart(7)}`);
    }
    console.error('\n─── Lift (Δ recall / Δ precision / Δ nDCG / Δ MRR) ───');
    const dh = matrix.deltas.hybrid_over_bm25;
    const db1 = matrix.deltas.bm25_over_recency;
    const dr = matrix.deltas.recency_over_random;
    console.error(`  hybrid    over bm25_only: R=${dh.recall_at_10} P=${dh.precision_at_10} nDCG=${dh.ndcg_at_10} MRR=${dh.mrr_at_10}`);
    console.error(`  bm25_only over recency:   R=${db1.recall_at_10} P=${db1.precision_at_10} nDCG=${db1.ndcg_at_10} MRR=${db1.mrr_at_10}`);
    console.error(`  recency   over random:    R=${dr.recall_at_10} P=${dr.precision_at_10} nDCG=${dr.ndcg_at_10} MRR=${dr.mrr_at_10}`);
    console.error('\n  Read: positive Δ on each row = that layer is doing work. ≈0 = layer is dead weight.');

    // Per-query bin summary — answers "is the aggregate 0 because all queries
    // are 0, or because gains and losses cancel?". If `lost` > 0 anywhere,
    // multipliers are at least directionally noisy on those queries.
    const hbBins = summarizePerQueryDelta(matrix.perQueryDeltas.hybrid_over_bm25);
    const brBins = summarizePerQueryDelta(matrix.perQueryDeltas.bm25_over_recency);
    console.error('\n─── Per-query Δ bins (n=' + matrix.queryCount + ', threshold=±0.001 nDCG) ───');
    console.error(`  hybrid    over bm25_only: gained=${hbBins.gained}  neutral=${hbBins.neutral}  lost=${hbBins.lost}`);
    console.error(`  bm25_only over recency:   gained=${brBins.gained}  neutral=${brBins.neutral}  lost=${brBins.lost}`);
    if (hbBins.gained === 0 && hbBins.lost === 0) {
      console.error('  → multipliers (decay/project/importance/access) flat on EVERY query — strong dead-weight signal');
    } else if (hbBins.lost > hbBins.gained) {
      console.error(`  → multipliers HURT more queries (${hbBins.lost}) than they helped (${hbBins.gained}) — directionally wrong`);
    } else if (hbBins.gained > 0) {
      console.error(`  → multipliers help ${hbBins.gained} query/ies — top lift: "${matrix.perQueryDeltas.hybrid_over_bm25[0].query}" (Δ=${matrix.perQueryDeltas.hybrid_over_bm25[0].ndcg_delta})`);
    }

    // Per-multiplier ablation summary — answers "which of the 4 multipliers
    // earn their keep on this fixture?". Read positive Δ as "dropping this
    // multiplier hurt", ≈0 as "this multiplier was idle, candidate to remove".
    if (ablations) {
      console.error('\n─── Per-multiplier ablation (Δ hybrid over single-multiplier-removed) ───');
      for (const ab of ABLATION_MODES) {
        const d = matrix.deltas[`hybrid_over_${ab}`];
        if (!d) continue;
        const dropped = ab.replace(/^no_/, '');
        console.error(`  drop ${dropped.padEnd(11)} → ΔR=${d.recall_at_10}  ΔP=${d.precision_at_10}  ΔnDCG=${d.ndcg_at_10}  ΔMRR=${d.mrr_at_10}`);
      }
      console.error('  Read: large positive Δ = this multiplier earns its keep. ≈0 = drop candidate.');
    }

    db.close();
    return;
  }

  console.error('Running benchmark...');
  const results = runBenchmark(db, queryData.queries);

  // Output JSON to stdout
  console.log(JSON.stringify(results, null, 2));

  // Summary to stderr
  console.error('\n─── Benchmark Results ───');
  console.error(`  Recall@10:       ${results.metrics.recall_at_10}`);
  console.error(`  Precision@10:    ${results.metrics.precision_at_10}`);
  console.error(`  nDCG@10:         ${results.metrics.ndcg_at_10}`);
  console.error(`  MRR@10:          ${results.metrics.mrr_at_10}`);
  console.error(`  Avg tokens:      ${results.metrics.avg_tokens_injected}`);
  console.error(`  P95 latency:     ${results.metrics.p95_search_latency_ms}ms`);
  console.error('  (run with --matrix for hybrid vs bm25_only / recency / random comparison)');

  // Per-category breakdown
  if (Object.keys(results.byCategory).length > 1) {
    console.error('\n─── By Category ───');
    for (const [cat, m] of Object.entries(results.byCategory)) {
      console.error(`  ${cat} (${m.count}q): R@10=${m.recall_at_10} P@10=${m.precision_at_10} nDCG=${m.ndcg_at_10} MRR=${m.mrr_at_10}`);
    }
  }

  // Queries with zero recall
  const zeroRecall = results.perQuery.filter(q => q.recall_at_10 === 0);
  if (zeroRecall.length > 0) {
    console.error(`\n  ⚠ ${zeroRecall.length} queries with zero recall:`);
    for (const q of zeroRecall) {
      console.error(`    - ${q.id}: "${q.query}" (expected: [${q.relevant_ids.join(',')}], got: [${q.result_ids.join(',')}])`);
    }
  }

  db.close();
}

// Run if executed directly
const isMain = process.argv[1] && fileURLToPath(import.meta.url).includes(process.argv[1].replace(/\.mjs$/, ''));
if (isMain) {
  main();
}
