// claude-mem-lite: LLM-powered database optimization
// Background worker for intelligent maintenance: re-enrich, normalize, cluster-merge, smart-compress
// Triggered from auto-maintain (24h) or manually via mem_optimize MCP tool / CLI

import { readFileSync, writeFileSync } from 'fs';
import { join } from 'path';
import {
  truncate, debugLog, debugCatch, COMPRESSED_AUTO,
  computeMinHash, estimateJaccardFromMinHash, jaccardSimilarity, clampImportance, cjkBigrams,
  notLowSignalTitleClause,
} from './utils.mjs';
import { callModelJSON } from './haiku-client.mjs';
import { acquireLLMSlot, releaseLLMSlot } from './hook-semaphore.mjs';
import { scrubRecord } from './lib/scrub-record.mjs';
import { getVocabulary, computeVector, cosineSimilarity } from './tfidf.mjs';
import { DB_DIR } from './schema.mjs';

const RUNTIME_DIR = join(DB_DIR, 'runtime');

// ─── Budget ─────────────────────────────────────────────────────────────────

export function distributeBudget(total = 15) {
  const normalize = 1;
  const reenrich = Math.max(1, Math.floor(total * 0.4));
  const clusterMerge = Math.max(1, Math.floor(total * 0.3));
  const smartCompress = Math.max(1, total - reenrich - normalize - clusterMerge);
  // Clamp: if total is too small for 4 tasks, cap each so sum ≤ total
  if (reenrich + normalize + clusterMerge + smartCompress > total) {
    return { reenrich: Math.max(1, total - 3), normalize: 1, clusterMerge: 1, smartCompress: 1 };
  }
  return { reenrich, normalize, clusterMerge, smartCompress };
}

// ─── Shared Helpers ─────────────────────────────────────────────────────────

/**
 * Rebuild TF-IDF vector for an observation. Non-critical — swallows errors.
 * Exported for testing; also kept as the single source of vector-rebuild logic
 * for the optimize / re-enrich path to avoid drift with the hook-llm write path.
 */
export function rebuildVector(db, obsId, textParts) {
  try {
    const vocab = getVocabulary(db);
    if (!vocab) return;
    const vec = computeVector(textParts.filter(Boolean).join(' '), vocab);
    if (vec) {
      // Bug #1 fix: column is `created_at_epoch`, not `computed_at`. Every other
      // INSERT callsite (server.mjs, hook-llm.mjs, mem-cli.mjs) uses the correct
      // name; this was the only drift, silently caught by the catch below until
      // the R-7 experiment surfaced it.
      db.prepare(`
        INSERT OR REPLACE INTO observation_vectors (observation_id, vector, vocab_version, created_at_epoch)
        VALUES (?, ?, ?, ?)
      `).run(obsId, Buffer.from(vec.buffer), vocab.version, Date.now());
    }
  } catch (e) { debugCatch(e, 'optimize-vector'); }
}

// ─── Task 1: Re-enrich ─────────────────────────────────────────────────────

/**
 * Find observations eligible for LLM re-enrichment.
 *
 * Two scopes:
 * - 'narrow' (default): fully-degraded observations — Haiku failed to extract
 *   concepts / facts / lesson / aliases. Conservative; preserves pre-R-7 behavior.
 * - 'wide' (R-7): substantive bugfix / refactor / feature / decision observations
 *   that have concepts + facts populated but are missing lesson_learned.
 *   Targets the "Haiku ran but judged 'none'" cases that dominate the library.
 *   Excludes LOW_SIGNAL titles (no source material to extract from) and
 *   thin narratives (<100 chars → nothing to rewrite into a lesson).
 *
 * Both scopes respect optimized_at (idempotent) and skip compressed/superseded rows.
 *
 * @param {object} db better-sqlite3 database handle
 * @param {number} limit max candidates to return
 * @param {{ scope?: 'narrow' | 'wide', project?: string }} [opts] Optional project filter (e.g. inferProject()-resolved name) narrows candidates to a single project — opt-in to preserve prior cross-project default.
 */
export function findReenrichCandidates(db, limit = 10, { scope = 'narrow', project } = {}) {
  const projectClause = project ? 'AND project = ?' : '';
  if (scope === 'wide') {
    const stmt = db.prepare(`
      SELECT id, title, narrative, type, subtitle, concepts, facts, project
      FROM observations
      WHERE COALESCE(compressed_into, 0) = 0
        AND superseded_at IS NULL
        AND optimized_at IS NULL
        AND type IN ('bugfix','refactor','feature','decision')
        AND (lesson_learned IS NULL OR lesson_learned = '')
        AND LENGTH(COALESCE(narrative, '')) > 100
        AND ${notLowSignalTitleClause('')}
        ${projectClause}
      ORDER BY
        CASE type WHEN 'decision' THEN 0 WHEN 'bugfix' THEN 1 WHEN 'refactor' THEN 2 ELSE 3 END,
        created_at_epoch DESC
      LIMIT ?
    `);
    return project ? stmt.all(project, limit) : stmt.all(limit);
  }
  const stmt = db.prepare(`
    SELECT id, title, narrative, type, subtitle, project
    FROM observations
    WHERE COALESCE(compressed_into, 0) = 0
      AND (concepts IS NULL OR concepts = '')
      AND (facts IS NULL OR facts = '')
      AND lesson_learned IS NULL
      AND search_aliases IS NULL
      AND optimized_at IS NULL
      ${projectClause}
    ORDER BY created_at_epoch DESC
    LIMIT ?
  `);
  return project ? stmt.all(project, limit) : stmt.all(limit);
}

export async function executeReenrich(db, limit = 10, { scope = 'narrow', project } = {}) {
  const candidates = findReenrichCandidates(db, limit, { scope, project });
  if (candidates.length === 0) return { processed: 0, skipped: 0 };

  let processed = 0, skipped = 0;
  const validTypes = new Set(['decision', 'bugfix', 'feature', 'refactor', 'discovery', 'change']);

  for (const cand of candidates) {
    const gotSlot = await acquireLLMSlot();
    if (!gotSlot) { skipped++; continue; }

    try {
      const prompt = `Re-enrich this observation with structured metadata. Return ONLY valid JSON, no markdown fences.

Title: ${truncate(cand.title || '(untitled)', 200)}
Narrative: ${truncate(cand.narrative || '(no narrative)', 500)}
Type: ${cand.type || 'change'}

JSON: {"type":"decision|bugfix|feature|refactor|discovery|change","title":"improved ≤120 char title","narrative":"improved 2-3 sentence narrative","concepts":["kw1","kw2"],"facts":["specific fact 1","specific fact 2"],"importance":1,"lesson_learned":"non-obvious insight or 'none' if routine","search_aliases":["alt query 1","alt query 2"]}
importance: 0=no value, 1=routine, 2=notable non-obvious insight, 3=critical. Default 1.
lesson_learned: State what was learned. If routine, write "none".
search_aliases: 2-6 alternative search terms (include CJK if applicable).`;

      const parsed = await callModelJSON(prompt, 'haiku', { timeout: 15000, maxTokens: 500 });
      if (!parsed || !parsed.title) { skipped++; continue; }

      if (parsed.importance === 0 || parsed.importance === '0') {
        db.prepare(`UPDATE observations SET compressed_into = ${COMPRESSED_AUTO}, optimized_at = ? WHERE id = ?`)
          .run(Date.now(), cand.id);
        processed++;
        continue;
      }

      const type = validTypes.has(parsed.type) ? parsed.type : cand.type || 'change';
      const concepts = Array.isArray(parsed.concepts) ? parsed.concepts.slice(0, 10) : [];
      const facts = Array.isArray(parsed.facts) ? parsed.facts.slice(0, 10) : [];
      const conceptsText = concepts.join(' ');
      const factsText = facts.join(' ');
      const lessonLearned = typeof parsed.lesson_learned === 'string'
        && parsed.lesson_learned.toLowerCase() !== 'none'
        && parsed.lesson_learned.trim().length > 0
        ? parsed.lesson_learned.slice(0, 500) : null;
      const searchAliases = Array.isArray(parsed.search_aliases)
        ? parsed.search_aliases.slice(0, 6).join(' ') : null;
      const title = truncate(parsed.title, 120);
      const narrative = truncate(parsed.narrative || cand.narrative || '', 500);
      const importance = clampImportance(parsed.importance);

      const bigramText = cjkBigrams((title || '') + ' ' + (narrative || ''));
      const textField = [conceptsText, factsText, searchAliases || '', bigramText].filter(Boolean).join(' ');
      const minhashSig = computeMinHash((title || '') + ' ' + (narrative || ''));

      // Scrub LLM-output text fields at the UPDATE boundary. type is an
      // enum, importance is numeric, minhash_sig is hash bytes.
      const safe = scrubRecord('observations', {
        title, narrative,
        concepts: conceptsText,
        facts: factsText,
        text: textField,
        lesson_learned: lessonLearned,
        search_aliases: searchAliases,
      });
      db.prepare(`
        UPDATE observations SET type=?, title=?, narrative=?, concepts=?, facts=?,
          text=?, importance=?, lesson_learned=?, search_aliases=?, minhash_sig=?, optimized_at=?
        WHERE id = ?
      `).run(type, safe.title, safe.narrative, safe.concepts, safe.facts, safe.text,
        importance, safe.lesson_learned, safe.search_aliases, minhashSig, Date.now(), cand.id);

      rebuildVector(db, cand.id, [title, narrative, conceptsText]);

      processed++;
    } catch (e) {
      debugCatch(e, 'reenrich');
      skipped++;
    } finally {
      releaseLLMSlot();
    }
  }

  if (processed > 0) debugLog('DEBUG', 'llm-optimize', `re-enriched ${processed} degraded observations`);
  return { processed, skipped };
}

// ─── Task 2: Normalize ─────────────────────────────────────────────────────

const NORMALIZE_GATE_FILE = join(RUNTIME_DIR, 'last-normalize.json');
const NORMALIZE_INTERVAL_MS = 7 * 86400000; // 7 days

export function shouldRunNormalize() {
  try {
    const last = JSON.parse(readFileSync(NORMALIZE_GATE_FILE, 'utf8'));
    return Date.now() - last.epoch >= NORMALIZE_INTERVAL_MS;
  } catch {
    return true;
  }
}

export function extractUniqueConcepts(db, limit = 500, { project } = {}) {
  const projectClause = project ? 'AND project = ?' : '';
  const stmt = db.prepare(`
    SELECT concepts FROM observations
    WHERE COALESCE(compressed_into, 0) = 0
      AND concepts IS NOT NULL AND concepts != ''
      ${projectClause}
    ORDER BY created_at_epoch DESC
    LIMIT 2000
  `);
  const rows = project ? stmt.all(project) : stmt.all();

  const conceptSet = new Set();
  for (const row of rows) {
    for (const c of row.concepts.split(/\s+/)) {
      const trimmed = c.trim();
      if (trimmed.length >= 2) conceptSet.add(trimmed);
    }
  }
  return [...conceptSet].slice(0, limit);
}

export async function identifySynonymGroups(concepts) {
  const gotSlot = await acquireLLMSlot();
  if (!gotSlot) return [];

  try {
    const prompt = `Analyze these concept terms from a code memory database and identify synonym groups (terms that refer to the same concept). Include cross-language synonyms (English/Chinese). Return ONLY valid JSON.

Concepts: ${concepts.join(', ')}

JSON: {"groups":[{"canonical":"preferred term","aliases":["synonym1","synonym2"]}, ...]}

Rules:
- Only include groups where you are confident the terms are true synonyms
- canonical should be the most specific/technical term
- Include CJK ↔ English equivalents if present
- Skip terms that have no synonyms in the list`;

    const parsed = await callModelJSON(prompt, 'sonnet', { timeout: 20000, maxTokens: 1000 });
    if (!parsed?.groups || !Array.isArray(parsed.groups)) return [];
    return parsed.groups.filter(g => g.canonical && Array.isArray(g.aliases) && g.aliases.length > 0);
  } catch (e) {
    debugCatch(e, 'normalize-identify');
    return [];
  } finally {
    releaseLLMSlot();
  }
}

export function applyNormalization(db, groups) {
  if (!groups || groups.length === 0) return { updated: 0 };

  const aliasMap = new Map();
  for (const g of groups) {
    for (const alias of g.aliases) {
      aliasMap.set(alias.toLowerCase(), g.canonical);
    }
  }

  const rows = db.prepare(`
    SELECT id, concepts, search_aliases FROM observations
    WHERE COALESCE(compressed_into, 0) = 0
      AND concepts IS NOT NULL AND concepts != ''
  `).all();

  let updated = 0;
  const updateStmt = db.prepare(`
    UPDATE observations SET concepts = ?, search_aliases = ?, optimized_at = ? WHERE id = ?
  `);

  for (const row of rows) {
    const terms = row.concepts.split(/\s+/);
    let changed = false;
    const newTerms = terms.map(t => {
      const canonical = aliasMap.get(t.toLowerCase());
      if (canonical && canonical !== t) { changed = true; return canonical; }
      return t;
    });

    if (changed) {
      const uniqueConcepts = [...new Set(newTerms)].join(' ');
      const existingAliases = row.search_aliases || '';
      const originalTerms = terms.filter(t => aliasMap.has(t.toLowerCase()) && aliasMap.get(t.toLowerCase()) !== t);
      const newAliases = [existingAliases, ...originalTerms].filter(Boolean).join(' ');
      // Defense-in-depth scrub. Canonical concept names come from LLM output
      // (identifySynonymGroups via Sonnet); existing values are already
      // scrubbed but free LLM tokens can re-introduce secret-shaped strings.
      const safe = scrubRecord('observations', {
        concepts: uniqueConcepts,
        search_aliases: newAliases,
      });
      updateStmt.run(safe.concepts, safe.search_aliases, Date.now(), row.id);
      updated++;
    }
  }

  if (updated > 0) debugLog('DEBUG', 'llm-optimize', `normalized concepts in ${updated} observations`);
  return { updated };
}

export async function executeNormalize(db, force = false, { project } = {}) {
  if (!force && !shouldRunNormalize()) return { skipped: true, reason: 'gate' };

  const concepts = extractUniqueConcepts(db, 500, { project });
  if (concepts.length < 5) return { skipped: true, reason: 'too few concepts' };

  const groups = await identifySynonymGroups(concepts);
  if (groups.length === 0) return { processed: 0, groups: 0 };

  const result = applyNormalization(db, groups);

  try { writeFileSync(NORMALIZE_GATE_FILE, JSON.stringify({ epoch: Date.now() })); } catch {}

  return { processed: result.updated, groups: groups.length };
}

// ─── Task 3: Cluster-merge ─────────────────────────────────────────────────

const MERGE_TIME_WINDOW_MS = 30 * 86400000;
const MERGE_JACCARD_LOW = 0.4;
const MERGE_JACCARD_HIGH = 0.85;

export function findMergeCandidates(db, maxClusters = 5, { project } = {}) {
  const cutoff = Date.now() - MERGE_TIME_WINDOW_MS;
  const projectClause = project ? 'AND project = ?' : '';
  const stmt = db.prepare(`
    SELECT id, title, narrative, project, type, access_count, created_at_epoch, minhash_sig
    FROM observations
    WHERE COALESCE(compressed_into, 0) = 0
      AND optimized_at IS NULL
      AND title IS NOT NULL AND title != ''
      AND created_at_epoch > ?
      ${projectClause}
    ORDER BY created_at_epoch DESC
    LIMIT 200
  `);
  const rows = project ? stmt.all(cutoff, project) : stmt.all(cutoff);

  const used = new Set();
  const clusters = [];

  for (let i = 0; i < rows.length && clusters.length < maxClusters; i++) {
    if (used.has(rows[i].id)) continue;
    const cluster = [rows[i]];

    for (let j = i + 1; j < rows.length && cluster.length < 5; j++) {
      if (used.has(rows[j].id)) continue;
      if (rows[i].project !== rows[j].project) continue;
      if (Math.abs(rows[i].created_at_epoch - rows[j].created_at_epoch) > MERGE_TIME_WINDOW_MS) continue;

      if (rows[i].minhash_sig && rows[j].minhash_sig) {
        const est = estimateJaccardFromMinHash(rows[i].minhash_sig, rows[j].minhash_sig);
        if (est < MERGE_JACCARD_LOW * 0.8) continue;
      }

      const titleSim = jaccardSimilarity(rows[i].title, rows[j].title);
      if (titleSim >= MERGE_JACCARD_LOW && titleSim < MERGE_JACCARD_HIGH) {
        cluster.push(rows[j]);
        used.add(rows[j].id);
      }
    }

    if (cluster.length >= 2) {
      used.add(rows[i].id);
      clusters.push(cluster);
    }
  }

  return clusters;
}

export async function executeMergeCluster(db, cluster) {
  if (cluster.length < 2) return { merged: false };

  const gotSlot = await acquireLLMSlot();
  if (!gotSlot) return { merged: false };

  try {
    const obsDescriptions = cluster.map((o, i) =>
      `${i + 1}. [${o.type || 'change'}] "${truncate(o.title, 200)}" — ${truncate(o.narrative || '(no narrative)', 500)}`
    ).join('\n');

    const prompt = `These observations from a code memory database may be about the same topic. Should they be merged into a single observation?

Observations:
${obsDescriptions}

Return ONLY valid JSON:
- If they should NOT be merged: {"should_merge":false}
- If they SHOULD be merged: {"should_merge":true,"merged_title":"≤120 char comprehensive title","merged_narrative":"comprehensive ≤800 char summary preserving all key details","merged_concepts":["kw1","kw2"],"merged_facts":["specific fact 1"],"merged_lesson":"synthesized non-obvious lesson or null","importance":2}`;

    const parsed = await callModelJSON(prompt, 'sonnet', { timeout: 20000, maxTokens: 1000 });
    if (!parsed || !parsed.should_merge) return { merged: false };

    const keeper = cluster.reduce((best, o) =>
      (o.access_count || 0) > (best.access_count || 0) ? o : best
    , cluster[0]);
    const others = cluster.filter(o => o.id !== keeper.id);

    const concepts = Array.isArray(parsed.merged_concepts) ? parsed.merged_concepts.slice(0, 10) : [];
    const facts = Array.isArray(parsed.merged_facts) ? parsed.merged_facts.slice(0, 10) : [];
    const conceptsText = concepts.join(' ');
    const factsText = facts.join(' ');
    const title = truncate(parsed.merged_title, 120);
    const narrative = truncate(parsed.merged_narrative || '', 800);
    const lessonLearned = typeof parsed.merged_lesson === 'string'
      && parsed.merged_lesson.trim().length > 0
      ? parsed.merged_lesson.slice(0, 500) : null;

    const bigramText = cjkBigrams((title || '') + ' ' + (narrative || ''));
    const textField = [conceptsText, factsText, bigramText].filter(Boolean).join(' ');
    const minhashSig = computeMinHash((title || '') + ' ' + (narrative || ''));
    const importance = clampImportance(parsed.importance || 2);

    // Scrub LLM-output cluster-merge text fields at the UPDATE boundary.
    // importance is numeric; minhash_sig is hash bytes.
    const safe = scrubRecord('observations', {
      title, narrative,
      concepts: conceptsText,
      facts: factsText,
      text: textField,
      lesson_learned: lessonLearned,
    });
    db.transaction(() => {
      db.prepare(`
        UPDATE observations SET title=?, narrative=?, concepts=?, facts=?, text=?,
          importance=?, lesson_learned=?, minhash_sig=?, optimized_at=?
        WHERE id = ?
      `).run(safe.title, safe.narrative, safe.concepts, safe.facts, safe.text,
        importance, safe.lesson_learned, minhashSig, Date.now(), keeper.id);

      const otherIds = others.map(o => o.id);
      const ph = otherIds.map(() => '?').join(',');
      db.prepare(`UPDATE observations SET compressed_into = ? WHERE id IN (${ph})`)
        .run(keeper.id, ...otherIds);
    })();

    rebuildVector(db, keeper.id, [title, narrative, conceptsText]);

    debugLog('DEBUG', 'llm-optimize', `merged ${cluster.length} observations into #${keeper.id}`);
    return { merged: true, keeperId: keeper.id, mergedCount: others.length };
  } catch (e) {
    debugCatch(e, 'cluster-merge');
    return { merged: false };
  } finally {
    releaseLLMSlot();
  }
}

export async function executeClusterMerge(db, maxClusters = 5, { project } = {}) {
  const clusters = findMergeCandidates(db, maxClusters, { project });
  if (clusters.length === 0) return { processed: 0, merged: 0 };

  let merged = 0;
  for (const cluster of clusters) {
    const result = await executeMergeCluster(db, cluster);
    if (result.merged) merged++;
  }

  return { processed: clusters.length, merged };
}

// ─── Task 4: Smart-compress ────────────────────────────────────────────────

const COMPRESS_TIME_SPLIT_MS = 14 * 86400000;
const COMPRESS_COSINE_THRESHOLD = 0.3;

export function findSmartCompressCandidates(db, ageDays = 30, { project } = {}) {
  const cutoff = Date.now() - ageDays * 86400000;
  const projectClause = project ? 'AND project = ?' : '';
  const stmt = db.prepare(`
    SELECT id, title, narrative, lesson_learned, project, type, created_at_epoch
    FROM observations
    WHERE COALESCE(compressed_into, 0) = 0
      AND COALESCE(importance, 1) = 1
      AND COALESCE(access_count, 0) = 0
      AND created_at_epoch < ?
      ${projectClause}
    ORDER BY project, created_at_epoch
  `);
  return project ? stmt.all(cutoff, project) : stmt.all(cutoff);
}

export function clusterForCompression(candidates, db) {
  if (candidates.length < 3) return [];

  const byProject = new Map();
  for (const c of candidates) {
    if (!byProject.has(c.project)) byProject.set(c.project, []);
    byProject.get(c.project).push(c);
  }

  const clusters = [];

  for (const [project, obs] of byProject) {
    if (obs.length < 3) continue;

    let vocab;
    try { vocab = getVocabulary(db); } catch {}

    if (vocab) {
      const vectors = obs.map(o => {
        const text = [o.title || '', o.narrative || ''].join(' ');
        return computeVector(text, vocab);
      });

      const used = new Set();
      for (let i = 0; i < obs.length; i++) {
        if (used.has(i) || !vectors[i]) continue;
        const cluster = [{ obs: obs[i], idx: i }];
        used.add(i);

        for (let j = i + 1; j < obs.length; j++) {
          if (used.has(j) || !vectors[j]) continue;
          const sim = cosineSimilarity(vectors[i], vectors[j]);
          if (sim >= COMPRESS_COSINE_THRESHOLD) {
            cluster.push({ obs: obs[j], idx: j });
            used.add(j);
          }
        }

        if (cluster.length >= 3) {
          const sorted = cluster.map(c => c.obs).sort((a, b) => a.created_at_epoch - b.created_at_epoch);
          let subCluster = [sorted[0]];
          for (let k = 1; k < sorted.length; k++) {
            if (sorted[k].created_at_epoch - subCluster[0].created_at_epoch > COMPRESS_TIME_SPLIT_MS) {
              if (subCluster.length >= 3) clusters.push({ project, observations: subCluster });
              subCluster = [sorted[k]];
            } else {
              subCluster.push(sorted[k]);
            }
          }
          if (subCluster.length >= 3) clusters.push({ project, observations: subCluster });
        }
      }
    } else {
      // Fallback: group by time window only
      const sorted = obs.sort((a, b) => a.created_at_epoch - b.created_at_epoch);
      let subCluster = [sorted[0]];
      for (let k = 1; k < sorted.length; k++) {
        if (sorted[k].created_at_epoch - subCluster[0].created_at_epoch > COMPRESS_TIME_SPLIT_MS) {
          if (subCluster.length >= 3) clusters.push({ project, observations: subCluster });
          subCluster = [sorted[k]];
        } else {
          subCluster.push(sorted[k]);
        }
      }
      if (subCluster.length >= 3) clusters.push({ project, observations: subCluster });
    }
  }

  return clusters;
}

export async function executeSmartCompressCluster(db, observations, project) {
  if (observations.length < 3) return { compressed: false };

  const gotSlot = await acquireLLMSlot();
  if (!gotSlot) return { compressed: false };

  try {
    const obsDescriptions = observations.map((o, i) =>
      `${i + 1}. [${o.type || 'change'}] "${truncate(o.title || '(untitled)', 200)}" — ${truncate(o.narrative || '(no narrative)', 500)}${o.lesson_learned ? ` | Lesson: ${truncate(o.lesson_learned, 200)}` : ''}`
    ).join('\n');

    const prompt = `Summarize these related code memory observations into ONE comprehensive summary. Preserve all important decisions, lessons, and specific facts. Return ONLY valid JSON.

Observations:
${obsDescriptions}

JSON: {"title":"descriptive summary ≤120 chars","narrative":"comprehensive summary ≤800 chars preserving key decisions and lessons","concepts":["kw1","kw2"],"facts":["all specific facts preserved"],"lesson_learned":"most important synthesized lesson or 'none'","search_aliases":["alt search 1","alt search 2"]}`;

    const parsed = await callModelJSON(prompt, 'sonnet', { timeout: 20000, maxTokens: 1000 });
    if (!parsed || !parsed.title) return { compressed: false };

    const title = truncate(parsed.title, 120);
    const narrative = truncate(parsed.narrative || '', 800);
    const concepts = Array.isArray(parsed.concepts) ? parsed.concepts.slice(0, 10) : [];
    const facts = Array.isArray(parsed.facts) ? parsed.facts.slice(0, 10) : [];
    const conceptsText = concepts.join(' ');
    const factsText = facts.join(' ');
    const lessonLearned = typeof parsed.lesson_learned === 'string'
      && parsed.lesson_learned.toLowerCase() !== 'none'
      && parsed.lesson_learned.trim().length > 0
      ? parsed.lesson_learned.slice(0, 500) : null;
    const searchAliases = Array.isArray(parsed.search_aliases)
      ? parsed.search_aliases.slice(0, 6).join(' ') : null;

    const bigramText = cjkBigrams((title || '') + ' ' + (narrative || ''));
    const textField = [conceptsText, factsText, searchAliases || '', bigramText].filter(Boolean).join(' ');

    const epochs = observations.map(o => o.created_at_epoch).sort((a, b) => a - b);
    const medianEpoch = epochs[Math.floor(epochs.length / 2)];

    const summaryId = db.transaction(() => {
      const sessionId = `compress-${project}`;
      const now = new Date();
      db.prepare(`INSERT OR IGNORE INTO sdk_sessions
        (content_session_id, memory_session_id, project, started_at, started_at_epoch, status)
        VALUES (?,?,?,?,?,'active')`
      ).run(sessionId, sessionId, project, now.toISOString(), now.getTime());

      // Defense-in-depth: title/narrative/etc. are LLM-generated compression
      // output; scrub at the persistence boundary regardless of upstream trust.
      const safe = scrubRecord('observations', {
        text: textField,
        title,
        narrative,
        concepts: conceptsText,
        facts: factsText,
        lesson_learned: lessonLearned,
        search_aliases: searchAliases,
      });
      const result = db.prepare(`INSERT INTO observations
        (memory_session_id, project, text, type, title, subtitle, narrative, concepts, facts,
         files_read, files_modified, importance, lesson_learned, search_aliases, optimized_at,
         created_at, created_at_epoch)
        VALUES (?,?,?,?,?,'',?,?,?,'[]','[]',2,?,?,?,?,?)`
      ).run(sessionId, project, safe.text, 'discovery', safe.title, safe.narrative,
        safe.concepts, safe.facts, safe.lesson_learned, safe.search_aliases, Date.now(),
        new Date(medianEpoch).toISOString(), medianEpoch);

      const sId = Number(result.lastInsertRowid);

      const obsIds = observations.map(o => o.id);
      const ph = obsIds.map(() => '?').join(',');
      db.prepare(`UPDATE observations SET compressed_into = ? WHERE id IN (${ph})`)
        .run(sId, ...obsIds);

      return sId;
    })();

    rebuildVector(db, summaryId, [title, narrative, conceptsText]);

    debugLog('DEBUG', 'llm-optimize', `smart-compressed ${observations.length} observations into #${summaryId}`);
    return { compressed: true, summaryId, count: observations.length };
  } catch (e) {
    debugCatch(e, 'smart-compress');
    return { compressed: false };
  } finally {
    releaseLLMSlot();
  }
}

export async function executeSmartCompress(db, maxClusters = 5, { project } = {}) {
  const candidates = findSmartCompressCandidates(db, 30, { project });
  if (candidates.length < 3) return { processed: 0, compressed: 0 };

  const clusters = clusterForCompression(candidates, db);
  if (clusters.length === 0) return { processed: 0, compressed: 0 };

  let compressed = 0;
  const toProcess = clusters.slice(0, maxClusters);
  for (const cluster of toProcess) {
    const result = await executeSmartCompressCluster(db, cluster.observations, cluster.project);
    if (result.compressed) compressed++;
  }

  return { processed: toProcess.length, compressed };
}

// ─── Pipeline Orchestrator ──────────────────────────────────────────────────

/**
 * @param {object} db better-sqlite3 database handle
 * @param {{ project?: string, detail?: boolean }} [opts]
 *   project: scope all candidate finders to a single project (opt-in; default scans all).
 *   detail: when true, also return `mergeClusters` / `reenrichSamples` / `compressSamples`
 *     arrays alongside the aggregate counts so callers (CLI --verbose, MCP detail mode)
 *     can render auditable previews without re-running the finders. The candidate-count
 *     arms still call the finders with high limits — detail mode does NOT widen scope,
 *     it surfaces the same rows the counts already crossed.
 */
export function optimizePreview(db, { project, detail = false } = {}) {
  const reenrichCandidates = findReenrichCandidates(db, 1000, { project });
  const reenrich = reenrichCandidates.length;
  // R-7: also report the widened-scope candidate count so users can see how many
  // bugfix/refactor/feature/decision observations are eligible for lesson backfill.
  const reenrichWide = findReenrichCandidates(db, 5000, { scope: 'wide', project }).length;

  const concepts = extractUniqueConcepts(db, 500, { project });
  const normalizeReady = shouldRunNormalize() && concepts.length >= 5;

  const mergeClusters = findMergeCandidates(db, 50, { project });
  const clusterMerge = mergeClusters.length;

  const compressCandidates = findSmartCompressCandidates(db, 30, { project });
  const compressClusters = clusterForCompression(compressCandidates, db);
  const smartCompress = compressClusters.length;

  const result = {
    reenrich,
    reenrichWide,
    normalize: normalizeReady ? concepts.length : 0,
    normalizeGateOpen: shouldRunNormalize(),
    clusterMerge,
    smartCompress,
    total: reenrich + (normalizeReady ? 1 : 0) + clusterMerge + smartCompress,
  };
  if (detail) {
    // Caps avoid dumping arbitrarily large arrays into CLI/MCP output — 20 picks
    // a sample size big enough to be auditable but small enough to fit a terminal
    // page. Callers that need more can drop --verbose and run the finders directly.
    result.mergeClusters = mergeClusters;
    result.reenrichSamples = reenrichCandidates.slice(0, 20);
    result.compressSamples = compressClusters.slice(0, 5);
  }
  return result;
}

/**
 * Run optimization tasks against the memory DB.
 *
 * @param {object} db better-sqlite3 handle
 * @param {object} [opts]
 * @param {string[]} [opts.tasks] Subset of tasks to run (default: all). When a single
 *   task is selected, it receives the FULL maxItems budget instead of the proportional
 *   slice from distributeBudget() — otherwise explicit `--max N --task re-enrich`
 *   would silently waste 60% of the requested budget.
 * @param {number} [opts.maxItems=15] Total item budget across all selected tasks.
 * @param {boolean} [opts.force=false] Bypass time-based gates (e.g. normalize interval).
 * @param {'narrow'|'wide'} [opts.reenrichScope='narrow'] Scope for the re-enrich task.
 *   'wide' targets bugfix/refactor/feature/decision with narrative but no lesson (R-7).
 * @param {string} [opts.project] Filter all tasks to a single project. Opt-in;
 *   absence preserves the prior all-projects default.
 */
export async function optimizeRun(db, { tasks, maxItems = 15, force = false, reenrichScope = 'narrow', project } = {}) {
  const allTasks = ['re-enrich', 'normalize', 'cluster-merge', 'smart-compress'];
  const selectedTasks = tasks && tasks.length > 0 ? tasks : allTasks;
  // Single-task mode: give that task the full budget. Distribution only makes sense
  // when multiple tasks compete for the same pool.
  const budget = selectedTasks.length === 1
    ? { reenrich: maxItems, normalize: maxItems, clusterMerge: maxItems, smartCompress: maxItems }
    : distributeBudget(maxItems);
  const results = {};

  for (const task of selectedTasks) {
    try {
      switch (task) {
        case 're-enrich':
          results.reenrich = await executeReenrich(db, budget.reenrich, { scope: reenrichScope, project });
          break;
        case 'normalize':
          results.normalize = await executeNormalize(db, force, { project });
          break;
        case 'cluster-merge':
          results.clusterMerge = await executeClusterMerge(db, budget.clusterMerge, { project });
          break;
        case 'smart-compress':
          results.smartCompress = await executeSmartCompress(db, budget.smartCompress, { project });
          break;
      }
    } catch (e) {
      debugCatch(e, `optimize:${task}`);
      results[task] = { error: e.message };
    }
  }

  return results;
}

export async function handleLLMOptimize() {
  const { ensureDb } = await import('./schema.mjs');
  let db;
  try {
    db = ensureDb();
  } catch {
    return;
  }

  try {
    // v2.54.0: auto-maintain default scope is 'wide'. Narrow scope (the prior
    // default) only matches fully-degraded rows (no concepts AND no facts AND
    // no lesson AND no aliases) — production diagnostic 2026-04-30 found only
    // 56 obs ever optimized after months of daily auto-maintain runs. Wide
    // targets bugfix/refactor/feature/decision rows with substantive narrative
    // but missing lesson_learned, which is exactly the audit's 11.2% coverage
    // gap. CLI `mem optimize` keeps narrow as default for explicit invocations.
    const results = await optimizeRun(db, { reenrichScope: 'wide' });
    const parts = [];
    if (results.reenrich?.processed) parts.push(`re-enriched: ${results.reenrich.processed}`);
    if (results.normalize?.processed) parts.push(`normalized: ${results.normalize.processed}`);
    if (results.clusterMerge?.merged) parts.push(`merged: ${results.clusterMerge.merged}`);
    if (results.smartCompress?.compressed) parts.push(`compressed: ${results.smartCompress.compressed}`);
    if (parts.length > 0) debugLog('DEBUG', 'llm-optimize', parts.join(', '));
  } catch (e) {
    debugCatch(e, 'llm-optimize');
  } finally {
    db.close();
  }
}
