# SPDX-License-Identifier: AGPL-3.0-or-later
# Copyright (C) 2026 Paul Chen / axoviq.com
from __future__ import annotations

import asyncio
import hashlib
import json
import logging
import re
import unicodedata
from dataclasses import dataclass, field
from datetime import date, datetime, timezone
from pathlib import Path
from typing import Optional

from synthadoc.agents.search_decompose_agent import SearchDecomposeAgent
from synthadoc.agents.skill_agent import SkillAgent
from synthadoc.core.cache import CACHE_VERSION, CacheManager, make_cache_key
from synthadoc.providers.base import LLMProvider, Message
from synthadoc.storage.log import AuditDB, LogWriter
from synthadoc.storage.search import HybridSearch
from synthadoc.storage.wiki import SourceRef, WikiPage, WikiStorage, LifecycleState, is_url, TriggerSource
from synthadoc.skills.web_search.scripts.main import _INTENT_RE as _WEB_INTENT_RE
from synthadoc.agents.lint_agent import LINT_SKIP_SLUGS

logger = logging.getLogger(__name__)


@dataclass
class IngestResult:
    source: str
    pages_created: list[str] = field(default_factory=list)
    pages_updated: list[str] = field(default_factory=list)
    pages_flagged: list[str] = field(default_factory=list)
    child_sources: list[str] = field(default_factory=list)
    tokens_used: int = 0
    input_tokens: int = 0
    output_tokens: int = 0
    cost_usd: float = 0.0
    cache_hits: int = 0
    skipped: bool = False
    skip_reason: str = ""


_ANALYSIS_PROMPT = (
    "Analyse the source text below. Return ONLY valid JSON with no markdown fences:\n"
    '{"entities": [...], "tags": [...], "summary": "One to three sentences describing '
    'the main topic, key claims, and relevance.", "relevant": true}\n\n'
    "Keep entities and tags under 10 items each.\n\n"
)

_ENTITY_PROMPT = (
    "Extract key entities, concepts, and tags from the text below.\n"
    "Return ONLY valid JSON: {\"entities\": [...], \"concepts\": [...], \"tags\": [...]}\n"
    "Keep each list under 10 items.\n\n"
)

_DECISION_PROMPT = (
    "You maintain a knowledge wiki. Decide how to handle a new source document.\n"
    "Return ONLY valid JSON - no markdown fences, no explanation.\n\n"
    "First write a 'reasoning' field explaining your decision, then set 'action'.\n\n"
    "WIKILINKS: Whenever you write page content (update_content or page_content), cross-reference\n"
    "related topics using [[slug]] notation where slug matches a page listed below.\n"
    "Example: 'Turing worked at [[bletchley-park]] on the [[enigma]] cipher.'\n"
    "Only link to pages that actually exist in the wiki (slugs shown below).\n\n"
    "Decision rules (apply in this order):\n\n"
    "RULE 1 — FLAG: If the new source DISPUTES or ARGUES AGAINST a factual claim in an existing page,\n"
    "use action='flag'. This includes academic debates, alternative historical interpretations,\n"
    "or sources that explicitly say an existing claim is wrong or a myth.\n"
    "Example: page says 'A-0 was the first compiler' + source says 'A-0 was a loader, not a compiler'\n"
    "-> action='flag', target=the slug of the page whose claim is disputed\n\n"
    "RULE 2 — UPDATE: If the source adds new information about a subject ALREADY covered by an existing page,\n"
    "and there is no factual dispute, use action='update'.\n"
    "-> action='update', target=slug of page to extend,\n"
    "   update_content=new ## section(s) to append (use [[slug]] links to related pages)\n\n"
    "RULE 3 — CREATE: ONLY if the source covers a subject not in any existing page.\n"
    "-> action='create', new_slug=meaningful_topic_slug (e.g. 'history-of-computing', NOT 'watch' or URL path segments),\n"
    "   page_content=full synthesized Markdown body (# Title + paragraphs with [[slug]] links)\n\n"
    'Return: {{"reasoning":"...","action":"...","target":"","new_slug":"","update_content":"","page_content":""}}\n\n'
    "Existing wiki pages (top matches):\n{pages}\n\n"
    "New source:\n{summary}\n\n"
    "Detected entities: {entities}"
)

_OVERVIEW_PROMPT = (
    "Write a 2-paragraph overview of a knowledge wiki based on the page titles and "
    "excerpts below.\n"
    "First paragraph: what topics this wiki covers.\n"
    "Second paragraph: key themes and concepts found.\n"
    "Keep it under 200 words. Plain text only — no markdown headings.\n\n"
    "Pages:\n{pages}"
)

CITATION_PASS4_CACHE_VERSION = "v1"
_CITATION_EXCERPT_LEN = 100
_CITATION_RE = re.compile(r'\^\[([^\]:]+):(\d+)-(\d+)\]')

_CITATION_PROMPT = (
    "You are a citation annotator. Given a wiki page section and the source text it was "
    "compiled from, insert ^[FILENAME:L-L] at the END of each paragraph that makes a "
    "substantive claim traceable to the source. L-L is the 1-based line range in the "
    "numbered source text where the supporting passage appears. Do not annotate headings, "
    "transition sentences, or [[wikilinks]].\n"
    "Return ONLY the annotated section — identical to the input except for added ^[...] markers.\n\n"
    "Source filename: {filename}\n\n"
    "Source text (lines numbered):\n{numbered_source}\n\n"
    "Page section to annotate:\n{section}"
)

_CONFIDENCE_RANK = {"high": 3, "medium": 2, "low": 1}


def _confidence_passes_threshold(confidence: str, min_confidence: str) -> bool:
    return _CONFIDENCE_RANK.get(confidence, 0) >= _CONFIDENCE_RANK.get(min_confidence, 0)


_SLUG_BLACKLIST = frozenset({
    "wikilinks", "wikilink", "wiki", "obsidian", "dataview",
    # URL path segments that are never meaningful topic names
    "watch", "embed", "video", "index", "page", "post", "article", "content",
})

# Matches any YouTube URL form and captures the 11-char video ID.
_YOUTUBE_ID_RE = re.compile(
    r"(?:youtube\.com/(?:watch\?v=|shorts/|live/|embed/|v/)|youtu\.be/)([A-Za-z0-9_-]{11})"
)


def _canonical_source(source: str) -> str:
    """Normalise YouTube URL variants to a single canonical form.

    youtu.be/<id>, /shorts/, /live/, /embed/ all become
    https://www.youtube.com/watch?v=<id> so dedup hashes are consistent.
    """
    m = _YOUTUBE_ID_RE.search(source)
    if m:
        return f"https://www.youtube.com/watch?v={m.group(1)}"
    return source


def _coerce_str_list(lst: object) -> list[str]:
    """Ensure every item in an LLM-returned list is a plain string.

    Some models return entities as dicts ({"name": "Canada", "type": "location"})
    instead of strings.  Extract the most useful field or fall back to str().
    """
    if not isinstance(lst, list):
        return []
    result = []
    for item in lst:
        if isinstance(item, str):
            result.append(item)
        elif isinstance(item, dict):
            value = item.get("name") or item.get("value") or item.get("label") or item.get("text") or ""
            if value:
                result.append(str(value))
        else:
            result.append(str(item))
    return [s for s in result if s.strip()]


def _parse_json_response(text: str) -> dict:
    """Parse a JSON object from an LLM response, handling markdown code fences."""
    text = text.strip()
    # Direct parse
    try:
        result = json.loads(text)
        if isinstance(result, dict):
            return result
        # LLM sometimes wraps the dict in a top-level array; unwrap first element
        if isinstance(result, list) and result and isinstance(result[0], dict):
            return result[0]
    except json.JSONDecodeError:
        pass
    # Strip markdown code block: ```json ... ``` or ``` ... ```
    m = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", text, re.DOTALL)
    if m:
        try:
            return json.loads(m.group(1))
        except json.JSONDecodeError:
            pass
    # Find first {...} in the response
    m = re.search(r"\{.*\}", text, re.DOTALL)
    if m:
        try:
            return json.loads(m.group(0))
        except json.JSONDecodeError:
            pass
    return {}


def _slugify(title: str) -> str:
    # Decompose accented characters (é → e + combining accent) so they map to ASCII
    normalized = unicodedata.normalize("NFKD", title)
    # Keep ASCII alphanumeric and CJK character blocks (valid Obsidian filename chars)
    slug = re.sub(
        r"[^a-z0-9\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af]+",
        "-",
        normalized.lower(),
    ).strip("-")
    # Fallback: if title was entirely symbols with no slug-able chars, use a content hash
    return slug or "page-" + hashlib.md5(title.encode()).hexdigest()[:8]


class IngestAgent:
    def __init__(self, provider: LLMProvider, store: WikiStorage, search: HybridSearch,
                 log_writer: LogWriter, audit_db: AuditDB, cache: CacheManager,
                 max_pages: int = 15, wiki_root: Optional[Path] = None,
                 cache_version: str = CACHE_VERSION,
                 fetch_timeout: int = 30,
                 routing_path: Optional[Path] = None,
                 cfg=None) -> None:
        self._provider = provider
        self._store = store
        self._search = search
        self._log = log_writer
        self._audit = audit_db
        self._cache = cache
        self._max_pages = max_pages
        self._wiki_root = Path(wiki_root) if wiki_root is not None else None
        self._routing_path = Path(routing_path) if routing_path is not None else None
        self._cfg = cfg
        self._cache_version = cache_version
        self._skill_agent = SkillAgent(skill_kwargs={
            "url": {"fetch_timeout": fetch_timeout},
            "youtube": {"provider": self._provider},
            "image": {"provider": self._provider},
        })
        self._purpose = self._load_purpose()

    def _write_sidecar(self, source_path: str, text: str, pagemap: dict) -> None:
        """Write .synthadoc/extracted/<name>.txt and (for PDFs) <name>.pdf.pagemap."""
        if not self._wiki_root:
            return
        extracted_dir = self._wiki_root / ".synthadoc" / "extracted"
        extracted_dir.mkdir(parents=True, exist_ok=True)
        name = Path(source_path).stem
        (extracted_dir / f"{name}.txt").write_text(text, encoding="utf-8")
        if pagemap:
            (extracted_dir / f"{name}.pdf.pagemap").write_text(
                json.dumps(pagemap, indent=2), encoding="utf-8"
            )

    async def _annotate_citations(
        self, section: str, source_text: str, filename: str
    ) -> tuple[str, list[dict]]:
        """Pass 4: annotate section with ^[filename:L-L] markers.

        Returns (annotated_section, citation_list). On any failure returns
        (original_section, []) so ingest always succeeds.
        """
        if not section.strip() or not source_text.strip():
            return section, []

        numbered = "\n".join(
            f"{i+1}: {line}" for i, line in enumerate(source_text.splitlines())
        )
        body_hash = hashlib.sha256(section.encode()).hexdigest()
        ck = make_cache_key(
            "citation-pass4",
            {"body_hash": body_hash, "filename": filename},
            version=CITATION_PASS4_CACHE_VERSION,
        )
        cached = await self._cache.get(ck)
        if cached:
            annotated = cached
        else:
            try:
                resp = await self._provider.complete(
                    messages=[Message(
                        role="user",
                        content=_CITATION_PROMPT.format(
                            filename=filename,
                            numbered_source=numbered[:6000],
                            section=section,
                        ),
                    )],
                    temperature=0.0,
                )
                raw = resp.text.strip() or section
                # Sanity check: annotated text must preserve the original content.
                # If the LLM returned something structurally different (e.g. JSON),
                # fall back to original to avoid corrupting the page body.
                _first_line = section.split("\n")[0][:40].strip()
                if _first_line and _first_line not in raw:
                    logger.warning(
                        "Pass 4 response for %s failed sanity check — using original section",
                        filename,
                    )
                    await self._audit.write_event(
                        "citation_pass4_skipped",
                        metadata={"source": filename, "error": "sanity_check_failed"},
                    )
                    return section, []
                annotated = raw
                await self._cache.set(ck, annotated)
            except Exception as exc:
                logger.warning("Pass 4 citation annotation failed for %s: %s", filename, exc)
                await self._audit.write_event(
                    "citation_pass4_skipped",
                    metadata={"source": filename, "error": type(exc).__name__},
                )
                return section, []

        citations = [
            {
                "source_file": filename,
                "line_start": int(m.group(2)),
                "line_end": int(m.group(3)),
                "claim_excerpt": section.split("\n")[0][:_CITATION_EXCERPT_LEN],
            }
            for m in _CITATION_RE.finditer(annotated)
            if m.group(1) == filename
        ]
        return annotated, citations

    async def _pick_routing_branch(self, slug: str, page: WikiPage, ri) -> str:
        """Ask LLM to select the best ROUTING.md branch for a newly created page."""
        from synthadoc.agents._routing import pick_routing_branches
        context = f"New page slug: {slug}\nTitle: {page.title}\nTags: {', '.join(page.tags)}"
        result = await pick_routing_branches(
            self._provider, ri.branches, context, multi=False
        )
        return result[0] if result else next(iter(ri.branches))

    async def _analyse(self, text: str, bust_cache: bool = False) -> dict:
        """Step 1 — analysis pass: entity extraction + summary. Cached by content hash."""
        text_hash = hashlib.sha256(text.encode()).hexdigest()
        ck = make_cache_key("analyse-v1", {"text_hash": text_hash}, version=self._cache_version)
        if not bust_cache:
            cached = await self._cache.get(ck)
            if cached:
                return cached
        resp = await self._provider.complete(
            messages=[Message(role="user", content=f"{_ANALYSIS_PROMPT}{text[:3000]}")],
            temperature=0.0,
        )
        data = _parse_json_response(resp.text)
        if not isinstance(data, dict):
            data = {}
        data["entities"] = _coerce_str_list(data.get("entities", []))
        data["tags"] = _coerce_str_list(data.get("tags", []))
        data.setdefault("summary", text[:200])
        data.setdefault("relevant", True)
        data["_tokens"] = resp.total_tokens
        await self._cache.set(ck, data)
        return data

    async def _update_overview(self) -> None:
        """Regenerate wiki/overview.md from the 10 most-recently-modified pages."""
        if self._wiki_root is None:
            return
        wiki_dir = self._wiki_root / "wiki"
        pages = sorted(
            [p for p in wiki_dir.glob("*.md")
             if p.stem not in {"overview", "index", "dashboard", "log"}],
            key=lambda p: p.stat().st_mtime,
            reverse=True,
        )[:10]
        if not pages:
            return
        page_ctx = []
        for p in pages:
            snippet = p.read_text(encoding="utf-8")[:200].replace("\n", " ")
            page_ctx.append(f"- {p.stem}: {snippet}")
        pages_str = "\n".join(page_ctx)
        resp = await self._provider.complete(
            messages=[Message(role="user",
                              content=_OVERVIEW_PROMPT.format(pages=pages_str))],
            temperature=0.3,
            max_tokens=512,
        )
        content = (
            f"---\ntitle: Wiki Overview\nstatus: auto\n"
            f"updated: {date.today().isoformat()}\n---\n\n"
            f"# Wiki Overview\n\n{resp.text.strip()}\n"
        )
        (wiki_dir / "overview.md").write_text(content, encoding="utf-8", newline="\n")

    def _staging_policy(self) -> str:
        return self._cfg.ingest.staging_policy if self._cfg else "off"

    def _write_or_stage(self, slug: str, page: "WikiPage", policy: str) -> bool:
        """Write page directly to wiki/ or to candidates/ when policy requires it.
        Returns True if staged, False if written directly."""
        if policy == "all" and self._wiki_root:
            from synthadoc.storage.wiki import WikiStorage as _WS
            cand_dir = self._wiki_root / "wiki" / "candidates"
            cand_dir.mkdir(exist_ok=True)
            _WS(cand_dir).write_page(slug, page)
            return True
        self._store.write_page(slug, page)
        self._search.invalidate_index()
        return False

    def _load_purpose(self) -> str:
        """Load wiki/purpose.md for scope filtering. Returns '' if absent."""
        if self._wiki_root is None:
            return ""
        p = self._wiki_root / "wiki" / "purpose.md"
        if not p.exists():
            return ""
        return p.read_text(encoding="utf-8")[:12000]

    def _hash(self, path: str) -> tuple[str, int]:
        data = Path(path).read_bytes()
        return hashlib.sha256(data).hexdigest(), len(data)

    def _needs_file_check(self, source: str) -> bool:
        """Return True when source must exist as a local file before ingestion."""
        return self._skill_agent.needs_path_resolution(source)

    async def _already_ingested(self, src_hash: str, src_size: int) -> bool:
        """Return True only if this source was ingested AND its wiki page still exists."""
        existing = await self._audit.find_by_hash(src_hash, src_size)
        if not existing:
            return False
        wiki_page = existing.get("wiki_page", "")
        return not wiki_page or self._store.page_exists(wiki_page)

    async def ingest(self, source: str, force: bool = False, bust_cache: bool = False) -> IngestResult:
        source = _canonical_source(source)
        result = IngestResult(source=source)

        if self._needs_file_check(source):
            p = Path(source).resolve()

            # Security: reject sources outside wiki_root
            if self._wiki_root is not None:
                root_resolved = self._wiki_root.resolve()
                try:
                    p.relative_to(root_resolved)
                except ValueError:
                    raise PermissionError(
                        f"Source {p} is outside wiki root {root_resolved}"
                    )

            if not p.exists():
                raise FileNotFoundError(f"Source not found: {source}")
            if p.stat().st_size == 0:
                raise ValueError(f"Source file is empty: {source}")

            # Dedup: hash + size (file sources only)
            src_hash, src_size = self._hash(str(p))

            # Check for hash collision (same hash, different size)
            if not force:
                existing = await self._audit.find_by_hash_only(src_hash)
                if existing and existing["size"] != src_size:
                    logger.warning(
                        "Hash collision detected: hash=%s matches existing record but size differs "
                        "(existing=%d, current=%d). Treating as new source.",
                        src_hash, existing["size"], src_size
                    )
                elif await self._already_ingested(src_hash, src_size):
                    result.skipped = True
                    result.skip_reason = "already ingested"
                    return result

            # Normalise to absolute path so the skill always receives an OS path,
            # not whatever vault-relative or CWD-relative string the caller passed in.
            source = str(p)

        # For URL / non-file sources p, src_hash, src_size are not set above.
        # Provide safe defaults so the audit call at the end always succeeds.
        if not self._needs_file_check(source):
            p = Path(source.split("?")[0].rstrip("/").split("/")[-1] or "url-source")
            _canonical = _canonical_source(source)
            src_hash = hashlib.sha256(_canonical.encode()).hexdigest()
            src_size = len(_canonical.encode())
            if not force and await self._already_ingested(src_hash, src_size):
                result.skipped = True
                result.skip_reason = "already ingested"
                return result

        # Web search decomposition: detect intent, decompose into keyword sub-queries,
        # fire N parallel Tavily searches, deduplicate URLs across results.
        try:
            _skill_meta = self._skill_agent.detect_skill(source)
            _is_web_search = _skill_meta.name == "web_search"
        except Exception:
            _is_web_search = False

        if _is_web_search:
            _bare_query = _WEB_INTENT_RE.sub("", source).strip() or source
            _sub_queries = await SearchDecomposeAgent(self._provider).decompose(_bare_query)
            _sub_results = await asyncio.gather(*[
                self._skill_agent.extract(f"search for: {q}") for q in _sub_queries
            ])
            _seen: set[str] = set()
            _merged_urls: list[str] = []
            for _r in _sub_results:
                for _url in _r.metadata.get("child_sources", []):
                    if _url not in _seen:
                        _seen.add(_url)
                        _merged_urls.append(_url)
            from synthadoc.skills.base import ExtractedContent as _EC
            extracted = _EC(
                text="", source_path=source,
                metadata={"child_sources": _merged_urls, "query": _bare_query,
                          "results_count": len(_merged_urls)},
            )
        else:
            extracted = await self._skill_agent.extract(source)

        # Web search fan-out: return child sources; orchestrator enqueues them as jobs
        if extracted.metadata.get("child_sources"):
            result.child_sources = extracted.metadata["child_sources"]
            return result

        # Write text sidecar so the Obsidian Source Viewer can display extracted content.
        #   local file  → not is_url, not _is_web_search → sidecar from source path
        #   http/s URL  → is_url → sidecar keyed by URL slug (str(p))
        #   intent phrase ("search for: …") → _is_web_search=True → skip: the source string
        #     contains ":" which macOS treats as a path separator; content arrives via child_sources.
        #     (If Tavily returns no results the child_sources early-return was skipped, but we still
        #     must not attempt to write a sidecar with an intent phrase as the filename.)
        page_boundaries = extracted.metadata.get("page_boundaries", {})
        if not is_url(source) and not _is_web_search:
            self._write_sidecar(source, extracted.text, page_boundaries)
        elif is_url(source) and extracted.text:
            self._write_sidecar(str(p), extracted.text, page_boundaries)

        # Skill-level token costs (e.g. vision pre-pass in ImageSkill)
        if extracted.metadata.get("tokens_input"):
            result.tokens_used += extracted.metadata["tokens_input"] + extracted.metadata.get("tokens_output", 0)
            result.input_tokens += extracted.metadata["tokens_input"]
            result.output_tokens += extracted.metadata.get("tokens_output", 0)

        text = extracted.text[:8000]

        # Step 1: analysis pass (cached separately from decision)
        analysis = await self._analyse(text, bust_cache=bust_cache)
        result.tokens_used += analysis.pop("_tokens", 0)
        # input/output split not available for the analyse call (cached via _analyse)

        entities = _coerce_str_list(analysis.get("entities", []))
        tags = _coerce_str_list(analysis.get("tags", []))
        summary = analysis.get("summary", text[:1500])

        # Fallback: if LLM entity extraction returned nothing, extract key phrases
        # directly from the source text so BM25 always has meaningful search terms.
        if not entities:
            # English: capitalized noun phrases
            english = re.findall(r'\b[A-Z][a-zA-Z]+(?:\s+[A-Z][a-zA-Z]+)*\b', text[:2000])
            # CJK: 2–6 consecutive chars — shorter is too granular, longer risks full sentences
            cjk = re.findall(
                r'[\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af]{2,6}',
                text[:2000],
            )
            entities = list(dict.fromkeys(english + cjk))[:12]
            logger.debug("Entity extraction returned empty; using text-extracted phrases: %s", entities)

        # Pass 2: hybrid search
        candidates = self._search.bm25_search(entities + tags, top_n=self._max_pages)

        # Build page context: top 5 candidates with content snippets
        pages_ctx = []
        for r in candidates[:5]:
            page = self._store.read_page(r.slug)
            if page:
                snippet = page.content[:600].replace("\n", " ")
                pages_ctx.append(f"[{r.slug}]: {snippet}")
        pages_str = "\n".join(pages_ctx) or "none"

        # Pass 3: decision (cached by summary hash + candidate slugs + prompt hash)
        # prompt_hash is included so any change to purpose.md or the purpose_block
        # instructions automatically busts the cache.
        slugs = [r.slug for r in candidates]
        summary_hash = hashlib.sha256(summary.encode()).hexdigest()
        decision_prompt = _DECISION_PROMPT
        if self._purpose:
            purpose_block = (
                f"Wiki scope (from purpose.md):\n{self._purpose}\n\n"
                "action=\"skip\" means the source is completely OUTSIDE the wiki's domain "
                "(e.g. spam, medical receipts, unrelated e-commerce). "
                "action=\"skip\" must NEVER be used because a topic is already covered by an "
                "existing page — that is what action=\"update\" is for. "
                "A source that covers the same topic as an existing page should ALWAYS be "
                "action=\"update\" (to add its unique perspective, examples, or depth) or "
                "action=\"create\" (if it introduces a distinct sub-topic). "
                "IMPORTANT: never skip based on source format — a YouTube video, podcast "
                "transcript, or lecture by a researcher in this domain must be ingested.\n\n"
            )
            decision_prompt = purpose_block + _DECISION_PROMPT
        prompt_hash = hashlib.sha256(decision_prompt.encode()).hexdigest()[:16]
        ck2 = make_cache_key(
            "make-decision",
            {"text_hash": summary_hash, "slugs": slugs, "prompt_hash": prompt_hash},
            version=self._cache_version,
        )
        cached2 = None if bust_cache else await self._cache.get(ck2)
        if cached2:
            result.cache_hits += 1
            decisions = cached2
        else:
            resp2 = await self._provider.complete(
                messages=[Message(role="user", content=decision_prompt.format(
                    pages=pages_str,
                    summary=summary,
                    entities=entities,
                ))],
                temperature=0.0,
            )
            result.tokens_used += resp2.total_tokens
            result.input_tokens += resp2.input_tokens
            result.output_tokens += resp2.output_tokens
            decisions = _parse_json_response(resp2.text)
            await self._cache.set(ck2, decisions)

        # Pass 4: writes based on action
        action = decisions.get("action", "create")

        # Sources with a structured summary (e.g. YouTube) always get their own
        # page so the executive summary and transcript are not appended to an
        # existing page.  Override update → create and let the LLM-supplied
        # new_slug stand; fall back to a slug derived from the source path.
        if extracted.metadata.get("has_summary") and action == "update":
            action = "create"
            if not decisions.get("new_slug"):
                decisions["new_slug"] = p.stem.lower().replace(" ", "-")
            logger.info(
                "ingest: has_summary source forced to create (was update) — slug=%s",
                decisions["new_slug"],
            )

        logger.info(
            "ingest decision: source=%s action=%s target=%s new_slug=%s | %s",
            source[:80], action,
            decisions.get("target", "") or "-",
            decisions.get("new_slug", "") or "-",
            (decisions.get("reasoning", "") or "")[:200],
        )

        if action == "skip":
            logger.warning(
                "ingest skip: source=%s — LLM deemed out of scope. reasoning=%s",
                source[:80], (decisions.get("reasoning", "") or "")[:300],
            )
            result.skipped = True
            result.skip_reason = "out of scope (purpose.md)"
            return result
        target = decisions.get("target", "")
        new_slug = decisions.get("new_slug") or ""
        update_content = decisions.get("update_content", "")
        page_content = decisions.get("page_content", "")
        title = p.stem.replace("-", " ").replace("_", " ").title()

        citations: list[dict] = []
        final_slug: str = ""

        if action == "flag" and target and target not in LINT_SKIP_SLUGS and self._store.page_exists(target):
            with self._store.page_lock(target):
                page = self._store.read_page(target)
                if page:
                    page.status = "contradicted"
                    page.unresolved_note = None  # clear any previous auto-resolve failure
                    reasoning = decisions.get("reasoning", "")
                    page.contradiction_note = (
                        f"Flagged while ingesting '{p.name}': {reasoning}" if reasoning
                        else f"Flagged while ingesting '{p.name}'"
                    )
                    self._store.write_page(target, page)
                    self._search.invalidate_index()
            result.pages_flagged.append(target)

        elif action == "update" and target and self._store.page_exists(target):
            if not text and not update_content:
                logger.warning(
                    "ingest: update skipped for slug=%s — no extractable content from %s",
                    target, source[:80]
                )
                result.skipped = True
                result.skip_reason = "no extractable text"
            else:
                policy = self._staging_policy()
                staged = False
                with self._store.page_lock(target):
                    page = self._store.read_page(target)
                    if page:
                        # Reset stale pages to draft on re-ingest
                        if page.status == LifecycleState.STALE:
                            page.status = LifecycleState.DRAFT
                            self._stale_to_draft_slug = target
                        if extracted.metadata.get("has_summary"):
                            section = extracted.text
                        elif update_content:
                            section = update_content
                        else:
                            section = f"## From {p.name}\n\n{text[:1000]}"
                        # Pass 4: annotate only the new update section
                        section, citations = await self._annotate_citations(
                            section, extracted.text, p.name
                        )
                        page.content = page.content.rstrip() + f"\n\n{section}"
                        staged = self._write_or_stage(target, page, policy)
                if staged:
                    logger.info("ingest: staged update to candidates slug=%s source=%s", target, source[:80])
                else:
                    logger.info("ingest: updated page slug=%s source=%s", target, source[:80])
                result.pages_updated.append(target)
                final_slug = target

        else:  # "create" or fallback
            # Don't create a page if there's no content to put in it
            if not text or not text.strip():
                logger.warning("Skipping page creation for %s — no text extracted", source)
                result.skip_reason = "no extractable text"
                result.skipped = True
            else:
                # Reject slugs that look like wiki syntax artifacts rather than real topics
                raw_slug = _slugify(new_slug or title)
                slug = raw_slug if raw_slug not in _SLUG_BLACKLIST else _slugify(title)
                # If title-based fallback is also blacklisted (e.g. URL path "watch"),
                # use the skill's suggested_slug or a hash-based ID rather than writing
                # a page with a generic, meaningless slug.
                if slug in _SLUG_BLACKLIST:
                    suggested = _slugify(extracted.metadata.get("suggested_slug", ""))
                    slug = suggested if suggested and suggested not in _SLUG_BLACKLIST \
                        else f"source-{src_hash[:12]}"

                if self._store.page_exists(slug):
                    # Slug already exists — never overwrite; append as update instead
                    policy = self._staging_policy()
                    staged = False
                    with self._store.page_lock(slug):
                        page = self._store.read_page(slug)
                        if page:
                            # Reset stale pages to draft on re-ingest
                            if page.status == LifecycleState.STALE:
                                page.status = LifecycleState.DRAFT
                                self._stale_to_draft_slug = slug
                            if extracted.metadata.get("has_summary"):
                                section = extracted.text
                            else:
                                section = f"## From {p.name}\n\n{text[:1500]}"
                            # Pass 4: annotate only the new section
                            section, citations = await self._annotate_citations(
                                section, extracted.text, p.name
                            )
                            page.content = page.content.rstrip() + f"\n\n{section}"
                            staged = self._write_or_stage(slug, page, policy)
                    if staged:
                        logger.info("ingest: staged update to candidates slug=%s source=%s", slug, source[:80])
                    else:
                        logger.info("ingest: updated existing page slug=%s source=%s", slug, source[:80])
                    result.pages_updated.append(slug)
                    final_slug = slug
                else:
                    if extracted.metadata.get("has_summary"):
                        body = extracted.text
                    elif page_content.strip():
                        body = page_content.strip()
                    else:
                        body = f"# {title}\n\n{text[:4000]}"
                    # Pass 4: annotate the full new page body
                    body, citations = await self._annotate_citations(
                        body, extracted.text, p.name
                    )
                    today = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S")
                    new_page = WikiPage(
                        title=title, tags=tags,
                        content=body,
                        status="draft", confidence="medium",
                        sources=[SourceRef(
                            file=source,
                            hash=src_hash or "",
                            size=src_size or 0,
                            ingested=today,
                        )],
                        created=today,
                    )

                    # Staging fork: route to candidates/ based on policy
                    policy = self._staging_policy()
                    go_to_candidates = (policy == "all") or (
                        policy == "threshold"
                        and self._cfg is not None
                        and not _confidence_passes_threshold(
                            new_page.confidence,
                            self._cfg.ingest.staging_confidence_min,
                        )
                    )

                    if go_to_candidates and self._wiki_root:
                        from synthadoc.storage.wiki import WikiStorage as _WS
                        cand_dir = self._wiki_root / "wiki" / "candidates"
                        cand_dir.mkdir(exist_ok=True)
                        _WS(cand_dir).write_page(slug, new_page)
                        logger.info("ingest: staged to candidates slug=%s source=%s", slug, source[:80])
                        result.pages_created.append(slug)
                        final_slug = slug
                    else:
                        with self._store.page_lock(slug):
                            self._store.write_page(slug, new_page)
                            self._search.invalidate_index()
                        logger.info("ingest: created page slug=%s source=%s", slug, source[:80])
                        result.pages_created.append(slug)
                        final_slug = slug
                        self._store.append_to_index(slug, new_page.title)
                        if self._routing_path:
                            from synthadoc.core.routing import RoutingIndex
                            ri = RoutingIndex.parse(self._routing_path)
                            if ri.branches:
                                branch = await self._pick_routing_branch(slug, new_page, ri)
                                ri.add_slug(slug, branch)
                                ri.save(self._routing_path)

        if result.pages_created or result.pages_updated:
            await self._update_overview()

        self._log.log_ingest(source=p.name,
                             pages_created=result.pages_created,
                             pages_updated=result.pages_updated,
                             pages_flagged=result.pages_flagged,
                             tokens=result.tokens_used,
                             cost_usd=result.cost_usd,
                             cache_hits=result.cache_hits)
        _wiki_page = (result.pages_created + result.pages_updated
                      + result.pages_flagged or [title])[0]
        await asyncio.gather(
            self._audit.record_ingest(src_hash, src_size, source,
                                      _wiki_page, result.tokens_used, result.cost_usd),
            self._audit.record_claim_citations(final_slug or _wiki_page, citations)
            if citations else asyncio.sleep(0),
        )
        if self._audit:
            if result.pages_created:
                await self._audit.set_page_state(final_slug or _wiki_page, LifecycleState.DRAFT, TriggerSource.INGEST)
                await self._audit.record_lifecycle_event(
                    final_slug or _wiki_page, None, LifecycleState.DRAFT,
                    "new page created by ingest", TriggerSource.INGEST
                )
            elif result.pages_updated and getattr(self, "_stale_to_draft_slug", None):
                await self._audit.set_page_state(self._stale_to_draft_slug, LifecycleState.DRAFT, TriggerSource.INGEST)
                await self._audit.record_lifecycle_event(
                    self._stale_to_draft_slug, LifecycleState.STALE, LifecycleState.DRAFT,
                    "re-ingest of stale page", TriggerSource.INGEST
                )
        return result
