#!/usr/bin/env python3
"""Targeted-refetch tool for Linked Art person name variants.

Canonical population of `person_names` is done by `harvest-vocabulary-db.py`
(LA-shape `identified_by[]` for 21xxx persons + Schema.org `alternateName`
for 31xxx persons, both via `INSERT OR IGNORE`). This script is a post-hoc
refetch over the LA endpoint for the 21xxx cohort only — useful when you
want to pick up upstream corrections without rerunning the full harvest.

Writes are always `INSERT OR IGNORE` and never delete existing rows; persons
already present in `person_names` are skipped by default. Run as often as
you like — it is idempotent and cannot lose data on partial failure.

Usage:
    python3 scripts/harvest-person-names.py                    # Default: data/vocabulary.db
    python3 scripts/harvest-person-names.py --db path/to.db    # Custom DB path
    python3 scripts/harvest-person-names.py --refetch-all      # Re-fetch every person (still INSERT OR IGNORE)
"""

import argparse
import json
import sqlite3
import time
import urllib.request
from pathlib import Path

# ─── Constants ────────────────────────────────────────────────────────

SCRIPT_DIR = Path(__file__).parent
PROJECT_DIR = SCRIPT_DIR.parent
DB_PATH = PROJECT_DIR / "data" / "vocabulary.db"

LINKED_ART_BASE = "https://data.rijksmuseum.nl"
USER_AGENT = "rijksmuseum-mcp-harvest/1.0"
BATCH_SIZE = 500

LANG_EN = "http://vocab.getty.edu/aat/300388277"
LANG_NL = "http://vocab.getty.edu/aat/300388256"

# AAT name classification URIs → short labels
AAT_CLASSIFICATION = {
    "300404670": "display",
    "300404671": "preferred",
    "300404672": "inverted",
}

# ─── Schema ───────────────────────────────────────────────────────────

PERSON_NAMES_SCHEMA = """
CREATE TABLE IF NOT EXISTS person_names (
    person_id       TEXT NOT NULL REFERENCES vocabulary(id),
    name            TEXT NOT NULL,
    lang            TEXT,
    classification  TEXT,
    UNIQUE(person_id, name, lang)
);
CREATE INDEX IF NOT EXISTS idx_person_names_id ON person_names(person_id);
"""


def fetch_person_names(person_id: str) -> list[dict] | None:
    """Fetch a person's Linked Art entity and extract all name variants."""
    url = f"{LINKED_ART_BASE}/{person_id}"
    req = urllib.request.Request(url, headers={
        "Accept": "application/ld+json",
        "Profile": "https://linked.art/ns/v1/linked-art.json",
        "User-Agent": USER_AGENT,
    })
    try:
        with urllib.request.urlopen(req, timeout=15) as resp:
            data = json.loads(resp.read())
    except Exception:
        return None

    seen = set()  # deduplicate (name, lang) pairs
    variants = []

    for entry in data.get("identified_by", []):
        # Only harvest Name entries, not Identifier entries (ULAN IDs, registry numbers)
        if entry.get("type") != "Name":
            continue
        content = entry.get("content", "")
        if not content or not isinstance(content, str):
            continue

        # Determine language
        lang = None
        for l in entry.get("language", []):
            lid = l.get("id", "")
            if lid == LANG_EN:
                lang = "en"
                break
            elif lid == LANG_NL:
                lang = "nl"
                break

        # Determine classification from classified_as AAT URIs.
        # Linked Art occasionally returns bare-string classified_as entries instead of dicts (~7%
        # rate per #231); skip those defensively rather than crashing on .get().
        classification = None
        for c in entry.get("classified_as", []):
            if not isinstance(c, dict):
                continue
            cid = c.get("id", "")
            for suffix, label in AAT_CLASSIFICATION.items():
                if cid.endswith(suffix):
                    classification = label
                    break
            if classification:
                break

        key = (content, lang)
        if key not in seen:
            seen.add(key)
            variants.append({
                "person_id": person_id,
                "name": content,
                "lang": lang,
                "classification": classification,
            })

    return variants


def main():
    parser = argparse.ArgumentParser(description="Targeted refetch of Linked Art person name variants")
    parser.add_argument("--db", type=str, default=str(DB_PATH), help="Path to vocabulary.db")
    parser.add_argument(
        "--refetch-all",
        action="store_true",
        help="Re-fetch every person (default skips persons already in person_names). Writes use INSERT OR IGNORE.",
    )
    args = parser.parse_args()

    conn = sqlite3.connect(args.db)
    conn.execute("PRAGMA journal_mode=WAL")

    # Create schema
    conn.executescript(PERSON_NAMES_SCHEMA)
    conn.commit()

    # Get person IDs from vocabulary
    all_persons = [r[0] for r in conn.execute(
        "SELECT id FROM vocabulary WHERE type = 'person'"
    ).fetchall()]
    print(f"Total persons in vocabulary: {len(all_persons):,}")

    if args.refetch_all:
        persons = all_persons
        print(f"Refetch-all: {len(persons):,} persons (INSERT OR IGNORE — no rows will be deleted)")
    else:
        done = {r[0] for r in conn.execute(
            "SELECT DISTINCT person_id FROM person_names"
        ).fetchall()}
        persons = [p for p in all_persons if p not in done]
        print(f"Skipping persons already in person_names: {len(done):,} done, {len(persons):,} remaining")

    if not persons:
        print("Nothing to do.")
        return

    t0 = time.time()
    fetched = 0
    failed = 0
    total_names = 0
    batch = []

    for i, person_id in enumerate(persons, 1):
        variants = fetch_person_names(person_id)
        if variants is not None:
            batch.extend(variants)
            total_names += len(variants)
            fetched += 1
        else:
            failed += 1

        if i % BATCH_SIZE == 0:
            if batch:
                conn.executemany(
                    "INSERT OR IGNORE INTO person_names (person_id, name, lang, classification) "
                    "VALUES (:person_id, :name, :lang, :classification)",
                    batch,
                )
                conn.commit()
                batch = []

        if i % 1000 == 0:
            elapsed = time.time() - t0
            rate = i / elapsed
            remaining = (len(persons) - i) / rate
            print(
                f"  {i:,}/{len(persons):,} ({fetched:,} ok, {failed:,} failed, "
                f"{total_names:,} names, {rate:.0f}/s, ~{remaining:.0f}s left)",
                flush=True,
            )

    # Flush remaining batch
    if batch:
        conn.executemany(
            "INSERT OR IGNORE INTO person_names (person_id, name, lang, classification) "
            "VALUES (:person_id, :name, :lang, :classification)",
            batch,
        )
        conn.commit()

    elapsed = time.time() - t0
    print(f"\nHarvest complete: {fetched:,} persons, {failed:,} failed, {total_names:,} names, {elapsed:.0f}s")

    # Build FTS5 index
    print("Building person_names_fts index...")
    conn.execute("DROP TABLE IF EXISTS person_names_fts")
    conn.execute("""
        CREATE VIRTUAL TABLE person_names_fts USING fts5(
            name,
            content='person_names', content_rowid='rowid',
            tokenize='unicode61 remove_diacritics 2'
        )
    """)
    conn.execute("INSERT INTO person_names_fts(person_names_fts) VALUES('rebuild')")
    fts_count = conn.execute("SELECT COUNT(*) FROM person_names_fts").fetchone()[0]
    print(f"  person_names_fts: {fts_count:,} rows")
    conn.commit()

    # Final stats
    distinct_persons = conn.execute("SELECT COUNT(DISTINCT person_id) FROM person_names").fetchone()[0]
    total_rows = conn.execute("SELECT COUNT(*) FROM person_names").fetchone()[0]
    avg_names = total_rows / distinct_persons if distinct_persons else 0
    print(f"\nFinal: {total_rows:,} name rows for {distinct_persons:,} persons ({avg_names:.1f} names/person avg)")

    conn.close()


if __name__ == "__main__":
    main()
