/**
 * 清洗步骤 2：删除模板化重复内容
 * 前 100 字符相同的记忆，每组保留 accessCount 最高的 1 条
 * （步骤 1 已清精确重复，这里抓"近似重复"——同一段内容被切成不同长度存了多次）
 *
 * 用法：
 *   bun run scripts/cleanup-step2-template-dups.ts [--execute]
 */

import lancedb from "@lancedb/lancedb";

const DB_PATH = "./data/lancedb";
const TABLE_NAME = "memories";
const EXECUTE = process.argv.includes("--execute");

interface MemoryRow {
  id: string;
  text: string;
  category: string;
  scope: string;
  importance: number;
  timestamp: number;
  metadata?: string;
}

async function main() {
  console.log(`\n🧹 清洗步骤 2：模板化重复删除 ${EXECUTE ? "[执行模式]" : "[预览模式]"}`);
  console.log("=".repeat(60));

  const db = await lancedb.connect(DB_PATH);
  const table = await db.openTable(TABLE_NAME);

  console.log("⏳ 读取全量数据...");
  const allRows: MemoryRow[] = await table
    .query()
    .select(["id", "text", "category", "scope", "importance", "timestamp", "metadata"])
    .toArray() as any;

  const total = allRows.length;
  console.log(`✅ ${total.toLocaleString()} 条记忆\n`);

  const parsedRows = allRows.map(row => ({
    ...row,
    meta: row.metadata ? JSON.parse(row.metadata) : {},
  }));

  // Group by first 100 chars (normalized)
  const fpGroups = new Map<string, typeof parsedRows>();
  for (const row of parsedRows) {
    const fp = row.text.trim().slice(0, 100).toLowerCase();
    if (!fpGroups.has(fp)) fpGroups.set(fp, []);
    fpGroups.get(fp)!.push(row);
  }

  // Find groups with duplicates (after step 1, exact dups are gone, so these are near-dups)
  const dupGroups = [...fpGroups.entries()].filter(([, rows]) => rows.length > 1);
  const idsToDelete: string[] = [];

  for (const [, rows] of dupGroups) {
    // Keep the one with highest accessCount, then longest text, then oldest
    rows.sort((a, b) => {
      const acA = a.meta.accessCount ?? a.meta.access_count ?? 0;
      const acB = b.meta.accessCount ?? b.meta.access_count ?? 0;
      if (acB !== acA) return acB - acA;
      if (b.text.length !== a.text.length) return b.text.length - a.text.length; // longer is more complete
      return a.timestamp - b.timestamp;
    });
    for (let i = 1; i < rows.length; i++) {
      idsToDelete.push(rows[i].id);
    }
  }

  console.log(`📊 统计结果:`);
  console.log(`  近似重复组数:  ${dupGroups.length.toLocaleString()}`);
  console.log(`  待删除条数:    ${idsToDelete.length.toLocaleString()}`);
  console.log(`  保留条数:      ${(total - idsToDelete.length).toLocaleString()}`);
  console.log(`  瘦身比例:      ${(idsToDelete.length / total * 100).toFixed(1)}%`);

  // Show top 15 largest groups
  const topGroups = dupGroups.sort((a, b) => b[1].length - a[1].length).slice(0, 15);
  console.log(`\n  Top 15 近似重复组:`);
  for (const [fp, rows] of topGroups) {
    console.log(`    [×${rows.length.toString().padStart(3)}] ${fp.slice(0, 65)}...`);
  }

  if (!EXECUTE) {
    console.log(`\n⚠️  预览模式，未执行删除。加 --execute 参数执行。`);
    return;
  }

  console.log(`\n🔥 开始删除 ${idsToDelete.length.toLocaleString()} 条...`);
  const batchSize = 100;
  let deleted = 0;

  for (let i = 0; i < idsToDelete.length; i += batchSize) {
    const batch = idsToDelete.slice(i, i + batchSize);
    const escaped = batch.map(id => id.replace(/'/g, "''"));
    const whereClause = `id IN (${escaped.map(id => `'${id}'`).join(",")})`;
    await table.delete(whereClause);
    deleted += batch.length;
    if (deleted % 500 === 0 || deleted === idsToDelete.length) {
      console.log(`  ✅ ${deleted.toLocaleString()} / ${idsToDelete.length.toLocaleString()}`);
    }
  }

  const remaining = await table.countRows();
  console.log(`\n✅ 删除完成！`);
  console.log(`  删除前: ${total.toLocaleString()} 条`);
  console.log(`  删除后: ${remaining.toLocaleString()} 条`);
  console.log(`  实际删除: ${(total - remaining).toLocaleString()} 条`);
}

main().catch(console.error);
