from __future__ import annotations

import fnmatch
import os
import re
from dataclasses import dataclass
from pathlib import Path
from typing import Any

from lightrag.constants import (
    DEFAULT_CHUNK_P_SIZE,
    DEFAULT_R_SEPARATORS,
    DEFAULT_SENTENCE_SPLIT_REGEX,
    FULL_DOCS_FORMAT_LIGHTRAG,
    FULL_DOCS_FORMAT_PENDING_PARSE,
    FULL_DOCS_FORMAT_RAW,
    PARSER_ENGINE_LEGACY,
    PARSER_ENGINE_NATIVE,
    PROCESS_OPTION_CHUNK_CHARS,
    PROCESS_OPTION_CHUNK_FIXED,
    PROCESS_OPTION_CHUNK_VECTOR,
    PROCESS_OPTION_CHUNK_PARAGRAH,
    PROCESS_OPTION_CHUNK_RECURSIVE,
    PROCESS_OPTION_EQUATIONS,
    PROCESS_OPTION_IMAGES,
    PROCESS_OPTION_SKIP_KG,
    PROCESS_OPTION_TABLES,
    ProcessChunkingOption,
    SUPPORTED_PROCESS_OPTIONS,
)
from lightrag.parser.registry import (
    PARSER_ENGINE_PASSTHROUGH,
    PARSER_ENGINE_REUSE,
    engine_endpoint_configured,
    engine_endpoint_requirement,
    supported_parser_engines,
    suffix_capabilities,
)
from lightrag.parser.param_schema import (
    parse_chunk_params,
    parse_engine_params,
    render_engine_params,
    split_top_level,
    take_paren_block,
)
from lightrag.utils import logger, parse_optional_float

import json
from collections.abc import Mapping
from copy import deepcopy

# Trailing parser-hint pattern: matches ``.[engine].ext`` at end of basename.
# Group 1 captures the raw engine token (still needs normalize_parser_engine
# and SUPPORTED_PARSER_ENGINES validation); group 2 captures ``.ext`` so it
# can be reattached when stripping the hint.
_PARSER_HINT_RE = re.compile(r"\.\[([^\]]*)\](\.[^.]+)$")

# Per-suffix default engine override, consulted before the global ``legacy``
# fallback. ``.textpack`` is handled only by the native engine, so it routes
# there automatically (no filename hint / LIGHTRAG_PARSER rule needed). ``.md``
# is deliberately absent — it keeps the legacy default and opts into native the
# same way ``.docx`` does (hint or rule).
_DEFAULT_ENGINE_BY_SUFFIX: dict[str, str] = {"textpack": PARSER_ENGINE_NATIVE}


class ParserRoutingConfigError(ValueError):
    """Raised when LIGHTRAG_PARSER contains an invalid routing rule."""


class FilenameParserHintError(ValueError):
    """Raised when a filename parser hint is invalid for ingestion."""


def normalize_parser_engine(engine: Any) -> str:
    """Normalize engine hints such as mineru-iet to mineru.

    Also strips an engine-level parameter block, so a stored/encoded
    ``parse_engine`` like ``mineru(page_range=1-3)`` resolves to the bare
    engine ``mineru`` (the single chokepoint that keeps parser selection and
    engine comparisons working once params are encoded into ``parse_engine``).
    """
    text = str(engine or "").strip()
    paren = text.find("(")
    if paren != -1:
        text = text[:paren]
    return text.split("-", 1)[0].strip().lower()


def encode_parse_engine(engine: str, engine_params: Mapping[str, Any] | None) -> str:
    """Encode ``(engine, engine_params)`` into the stored ``parse_engine`` field.

    Returns the bare ``engine`` when there are no params, else
    ``engine(key=value,...)`` in hint syntax (list params as repeated keys,
    bools as ``true``/``false``).  Defensively normalises the params and raises
    ``ValueError`` on invalid input, so it can never emit a string that
    :func:`decode_parse_engine` would later reject.
    """
    if not engine_params:
        return engine
    inner, errors = render_engine_params(engine, engine_params)
    if errors:
        raise ValueError(
            f"cannot encode engine parameters for {engine!r}: " + "; ".join(errors)
        )
    return f"{engine}({inner})" if inner else engine


def decode_parse_engine(
    value: Any,
) -> tuple[str, dict[str, Any], list[str]]:
    """Decode a stored ``parse_engine`` field into ``(engine, params, errors)``.

    ``value`` may be a bare engine (``mineru``) or an encoded directive
    (``mineru(page_range=1-3,language=en)``).  ``engine`` is the bare,
    normalised engine name; ``params`` is the canonical coerced dict; ``errors``
    is non-empty for a malformed/unbalanced block or invalid params (callers on
    the parse path raise so the doc fails visibly instead of dropping params).
    """
    raw = str(value or "").strip()
    if not raw:
        return "", {}, []
    idx = raw.find("(")
    if idx == -1:
        return normalize_parser_engine(raw), {}, []
    engine = normalize_parser_engine(raw[:idx])
    inner, after = take_paren_block(raw, idx)
    if inner is None:
        return engine, {}, [f"unbalanced '(' in parse_engine {raw!r}"]
    if raw[after:].strip():
        return engine, {}, [f"unexpected text after ')' in parse_engine {raw!r}"]
    params, errors = parse_engine_params(
        inner, engine=engine, label=f"parse_engine {raw!r}"
    )
    return engine, params, errors


# ---------------------------------------------------------------------------
# Per-file processing options (i/t/e/!/F/R/V/P)
# ---------------------------------------------------------------------------


@dataclass(frozen=True)
class ProcessOptions:
    """Decoded view of a ``process_options`` string.

    The ``raw`` string is preserved verbatim (with duplicates and ordering)
    for storage / audit purposes; boolean flags reflect the deduped logical
    state used by the pipeline.
    """

    raw: str = ""
    images: bool = False
    tables: bool = False
    equations: bool = False
    skip_kg: bool = False
    chunking: ProcessChunkingOption = PROCESS_OPTION_CHUNK_FIXED

    @property
    def chunking_explicit(self) -> bool:
        """True iff ``raw`` actually contains a chunking selector char.

        Distinguishes "user explicitly opted into a chunking strategy"
        from "no chunking selector supplied — pipeline used the default".
        ``chunking`` itself is unreliable for this question because it
        falls back to :data:`PROCESS_OPTION_CHUNK_FIXED` in both cases.
        Used by ``process_single_document`` to decide whether to
        dispatch via the new file-chunker contract or to honor the
        legacy externally-supplied :attr:`LightRAG.chunking_func`.
        """
        return any(c in PROCESS_OPTION_CHUNK_CHARS for c in self.raw)


_PROCESS_OPTION_DEFAULT = ProcessOptions()


def sanitize_process_options(options: Any) -> str:
    """Strip non-supported characters / hyphen / whitespace from an options string.

    Returns the raw token sequence as-is (no dedup, no reorder) so the
    canonical user intent is preserved on disk.  Invalid characters are
    silently dropped — the caller is expected to have already validated.
    """
    if not options:
        return ""
    return "".join(ch for ch in str(options) if ch in SUPPORTED_PROCESS_OPTIONS)


def validate_process_options(
    options: str, *, label: str = "process options"
) -> list[str]:
    """Return a list of error messages for an options string; empty if valid."""
    errors: list[str] = []
    if not options:
        return errors
    seen_chunkers: list[str] = []
    for ch in options:
        if ch in (" ", "-"):
            continue
        if ch not in SUPPORTED_PROCESS_OPTIONS:
            errors.append(f"{label} contains unsupported character {ch!r}")
            continue
        if ch in PROCESS_OPTION_CHUNK_CHARS and ch not in seen_chunkers:
            seen_chunkers.append(ch)
    if len(seen_chunkers) > 1:
        errors.append(
            f"{label} specifies multiple chunking modes "
            f"({'/'.join(seen_chunkers)}); pick one of "
            f"{PROCESS_OPTION_CHUNK_FIXED}/{PROCESS_OPTION_CHUNK_RECURSIVE}/{PROCESS_OPTION_CHUNK_VECTOR}/{PROCESS_OPTION_CHUNK_PARAGRAH}"
        )
    return errors


def parse_process_options(options: Any) -> ProcessOptions:
    """Decode a process-options string into a :class:`ProcessOptions` view."""
    raw = sanitize_process_options(options)
    if not raw:
        return _PROCESS_OPTION_DEFAULT
    chars = set(raw)
    chunking: ProcessChunkingOption = PROCESS_OPTION_CHUNK_FIXED
    # Pick the first chunking selector encountered; validate_process_options
    # already filters duplicates upstream.
    for ch in raw:
        if ch in PROCESS_OPTION_CHUNK_CHARS:
            chunking = ch  # type: ignore[assignment]
            break
    return ProcessOptions(
        raw=raw,
        images=PROCESS_OPTION_IMAGES in chars,
        tables=PROCESS_OPTION_TABLES in chars,
        equations=PROCESS_OPTION_EQUATIONS in chars,
        skip_kg=PROCESS_OPTION_SKIP_KG in chars,
        chunking=chunking,
    )


# ---------------------------------------------------------------------------
# Per-chunker parameter snapshot (chunk_options) — counterpart to the
# F/R/V/P selector in ``ProcessOptions``.  ``process_options`` chooses
# the strategy; ``chunk_options`` carries the parameters the chosen
# strategy reads.
#
# Storage shape: the per-document snapshot persisted to
# ``full_docs[doc_id]['chunk_options']`` carries ONLY the sub-dict of
# the chunking strategy selected by ``process_options`` — the other
# strategies' parameters are dropped because they are never consumed
# during processing.  Reparsing a document overwrites both
# ``process_options`` and ``chunk_options`` together.
# ---------------------------------------------------------------------------


# Strategy selector (F/R/V/P) → snapshot sub-dict key.  Single source
# of truth for the slim ``chunk_options`` shape — used by
# :func:`resolve_chunk_options` to pick which strategy block to keep
# and by :func:`slim_chunk_options` to project caller-supplied dicts
# down to the selected strategy.
_CHUNK_STRATEGY_KEYS: dict[str, str] = {
    PROCESS_OPTION_CHUNK_FIXED: "fixed_token",
    PROCESS_OPTION_CHUNK_RECURSIVE: "recursive_character",
    PROCESS_OPTION_CHUNK_VECTOR: "semantic_vector",
    PROCESS_OPTION_CHUNK_PARAGRAH: "paragraph_semantic",
}


def chunk_strategy_key(process_options: Any) -> str:
    """Return the ``chunk_options`` sub-dict key for ``process_options``.

    Accepts a raw options string or a :class:`ProcessOptions` value.
    Falls back to ``"fixed_token"`` when no chunking selector is
    present — F is the default strategy used both by the file-chunker
    dispatcher (when ``chunking_explicit`` is False the legacy
    ``chunking_func`` runs, which defaults to fixed-token chunking
    that reads from the same sub-dict).
    """
    if isinstance(process_options, ProcessOptions):
        strategy = process_options.chunking
    else:
        strategy = parse_process_options(process_options).chunking
    return _CHUNK_STRATEGY_KEYS.get(strategy, "fixed_token")


def slim_chunk_options(
    snapshot: Mapping[str, Any] | None,
    process_options: Any = "",
) -> dict[str, Any]:
    """Project a (possibly full) chunker snapshot down to the active strategy.

    Keeps the top-level ``chunk_token_size`` and the one strategy
    sub-dict picked by :func:`chunk_strategy_key`; everything else is
    discarded.  Idempotent: a slim snapshot whose key already matches
    ``process_options`` passes through unchanged (deep-copied for
    isolation).  When the matching strategy block is absent from the
    input, an empty dict is used so downstream consumers always see a
    dict-shaped slot.

    Strategy-specific default backfill: for ``paragraph_semantic`` we
    guarantee a populated ``chunk_token_size`` slot before returning
    (caller-supplied value > ``CHUNK_P_SIZE`` env >
    ``DEFAULT_CHUNK_P_SIZE``).  This is the single chokepoint that
    every enqueue path runs through — both the
    ``resolve_chunk_options`` path (built from addon_params) AND the
    direct ``chunk_options=`` kwarg path (caller supplies the dict)
    flow through here, so the backfill cannot be bypassed by runtime
    addon_params mutation or by passing an explicit ``chunk_options``
    that omits the P slot.  P must NOT inherit the top-level
    ``chunk_token_size`` (global ``CHUNK_SIZE`` / legacy ctor) —
    paragraph-semantic merging needs more headroom than the global
    default.
    """
    key = chunk_strategy_key(process_options)
    src: Mapping[str, Any] = snapshot or {}
    result: dict[str, Any] = {}
    if "chunk_token_size" in src:
        result["chunk_token_size"] = deepcopy(src["chunk_token_size"])
    result[key] = deepcopy(dict(src.get(key) or {}))
    if key == "paragraph_semantic":
        if "chunk_token_size" not in result[key]:
            p_size_raw = os.getenv("CHUNK_P_SIZE")
            result[key]["chunk_token_size"] = (
                int(p_size_raw) if p_size_raw is not None else DEFAULT_CHUNK_P_SIZE
            )
        # Mirror the CHUNK_P_DROP_REFERENCES env default for the drop-references
        # switch here — this is the single chokepoint every enqueue path runs
        # through, so a runtime ``addon_params`` mutation or an explicit
        # ``chunk_options=`` that omits the slot still picks up the env default.
        # ``setdefault`` keeps any caller-supplied value (hint/addon/kwarg).  The
        # detection-tuning knobs (tail window / heading prefixes) are NOT
        # snapshotted — the chunker reads them live from env at run time.
        if os.getenv("CHUNK_P_DROP_REFERENCES") is not None:
            result[key].setdefault(
                "drop_references", _env_bool("CHUNK_P_DROP_REFERENCES")
            )
    return result


def _env_optional_str(key: str) -> str | None:
    """Return the env value as a string, collapsing empty / 'None' to None."""
    raw = os.getenv(key)
    if raw is None:
        return None
    stripped = raw.strip()
    if not stripped or stripped.lower() == "none":
        return None
    return raw


def _env_bool(key: str, default: bool = False) -> bool:
    raw = os.getenv(key)
    if raw is None:
        return default
    return raw.strip().lower() in ("1", "true", "yes", "on", "t", "y")


def default_chunker_config() -> dict[str, Any]:
    """Snapshot the **strategy-specific** env-driven defaults for every shipped chunker.

    Builds a per-strategy sub-dict whose keys mirror each strategy's
    keyword-only signature (so :func:`resolve_chunk_options` can splat
    them straight into the chunker call).

    Provenance / precedence note: this function reads only
    *strategy-specific* env vars (``CHUNK_F_SIZE``,
    ``CHUNK_F_OVERLAP_SIZE``, ``CHUNK_R_SIZE``, ``CHUNK_R_OVERLAP_SIZE``,
    ``CHUNK_R_SEPARATORS``, ``CHUNK_V_SIZE``, ``CHUNK_V_*``,
    ``CHUNK_P_SIZE``, ``CHUNK_P_OVERLAP_SIZE``,
    ``CHUNK_F_SPLIT_BY_CHARACTER``…).  It does **not** read the legacy
    top-level envs ``CHUNK_SIZE`` / ``CHUNK_OVERLAP_SIZE``, and it
    deliberately **omits** ``chunk_overlap_token_size`` from a strategy
    sub-dict when its own env var is unset — leaving the slot empty is
    the signal that lets
    :meth:`LightRAG._apply_chunk_size_overlay` apply the legacy
    constructor field (``LightRAG(chunk_overlap_token_size=…)``) and
    finally the legacy ``CHUNK_OVERLAP_SIZE`` env in that order.  Same
    rationale for top-level ``chunk_token_size`` — overlay fills it from
    ``LightRAG(chunk_token_size=…)`` then ``CHUNK_SIZE`` env.  Net
    precedence (high → low): ``addon_params`` explicit > strategy env
    > legacy ctor field > legacy env.

    Read at instance-creation time via
    :func:`lightrag.addon_params.default_addon_params`; users can mutate
    ``addon_params['chunker']`` at runtime to change the defaults applied
    to subsequently enqueued documents (already-enqueued docs hold a
    frozen ``full_docs[doc_id]['chunk_options']`` snapshot).
    """
    config: dict[str, Any] = {
        "fixed_token": {
            "split_by_character": _env_optional_str("CHUNK_F_SPLIT_BY_CHARACTER"),
            "split_by_character_only": _env_bool(
                "CHUNK_F_SPLIT_BY_CHARACTER_ONLY", False
            ),
        },
        "recursive_character": {
            # Default separators include CJK sentence-ending punctuation
            # so Chinese / mixed-language documents split at semantic
            # boundaries instead of falling through to character-level
            # splitting.  See ``constants.DEFAULT_R_SEPARATORS`` for
            # cascade order rationale.
            "separators": json.loads(
                os.getenv("CHUNK_R_SEPARATORS", json.dumps(list(DEFAULT_R_SEPARATORS)))
            ),
        },
        "semantic_vector": {
            "breakpoint_threshold_type": os.getenv(
                "CHUNK_V_BREAKPOINT_THRESHOLD_TYPE", "percentile"
            ),
            "breakpoint_threshold_amount": parse_optional_float(
                os.getenv("CHUNK_V_BREAKPOINT_THRESHOLD_AMOUNT")
            ),
            "buffer_size": int(os.getenv("CHUNK_V_BUFFER_SIZE", "1")),
            # Default extends LangChain's English-only sentence splitter
            # with CJK terminators so SemanticChunker can actually find
            # sentence boundaries on Chinese input.  Override per
            # deployment if you need a different language mix.
            "sentence_split_regex": os.getenv(
                "CHUNK_V_SENTENCE_SPLIT_REGEX", DEFAULT_SENTENCE_SPLIT_REGEX
            ),
        },
        "paragraph_semantic": {},
    }

    # Strategy-specific overlap envs only — leave the slot absent when
    # unset so overlay can detect provenance and fill from the legacy
    # tier (constructor field → CHUNK_OVERLAP_SIZE env).
    f_overlap_raw = os.getenv("CHUNK_F_OVERLAP_SIZE")
    if f_overlap_raw is not None:
        config["fixed_token"]["chunk_overlap_token_size"] = int(f_overlap_raw)
    r_overlap_raw = os.getenv("CHUNK_R_OVERLAP_SIZE")
    if r_overlap_raw is not None:
        config["recursive_character"]["chunk_overlap_token_size"] = int(r_overlap_raw)
    p_overlap_raw = os.getenv("CHUNK_P_OVERLAP_SIZE")
    if p_overlap_raw is not None:
        config["paragraph_semantic"]["chunk_overlap_token_size"] = int(p_overlap_raw)

    # P strategy carries its own ``chunk_token_size`` override so the
    # paragraph-semantic merge target can diverge from the global
    # ``CHUNK_SIZE`` (e.g. heading-aligned chunks may want a larger
    # ceiling).  Unlike R/V, the slot is ALWAYS populated — when
    # ``CHUNK_P_SIZE`` is unset we use ``DEFAULT_CHUNK_P_SIZE`` (2000)
    # rather than letting the dispatcher fall back to the global
    # ``CHUNK_SIZE`` (1200): paragraph-semantic merging needs more
    # headroom than the global default to keep related paragraphs
    # together, and silently inheriting the smaller global ceiling
    # defeats the strategy's purpose.
    p_size_raw = os.getenv("CHUNK_P_SIZE")
    config["paragraph_semantic"]["chunk_token_size"] = (
        int(p_size_raw) if p_size_raw is not None else DEFAULT_CHUNK_P_SIZE
    )

    # F/R/V strategies likewise carry their own optional ``chunk_token_size``
    # overrides (fixed-token may want a deployment-specific window, recursive
    # character splitting a smaller target, semantic-vector clustering a larger
    # advisory ceiling).  Same slot-absent convention as P: leave the slot
    # absent when the env is unset so the strategy inherits the top-level
    # ``chunk_token_size`` fallback at consumption time.
    f_size_raw = os.getenv("CHUNK_F_SIZE")
    if f_size_raw is not None:
        config["fixed_token"]["chunk_token_size"] = int(f_size_raw)
    r_size_raw = os.getenv("CHUNK_R_SIZE")
    if r_size_raw is not None:
        config["recursive_character"]["chunk_token_size"] = int(r_size_raw)
    v_size_raw = os.getenv("CHUNK_V_SIZE")
    if v_size_raw is not None:
        config["semantic_vector"]["chunk_token_size"] = int(v_size_raw)

    return config


def resolve_chunk_options(
    addon_params: Mapping[str, Any] | None,
    *,
    process_options: Any = "",
    split_by_character: str | None = None,
    split_by_character_only: bool = False,
) -> dict[str, Any]:
    """Build a per-document slim ``chunk_options`` snapshot.

    Reads the chunker config from ``addon_params['chunker']``, falling
    back to a freshly built :func:`default_chunker_config` when the
    addon-params mapping is missing or hasn't been populated, then
    keeps only the parameters of the strategy selected by
    ``process_options`` (the other strategies' sub-dicts are dropped —
    they would never be consumed during processing).  See
    :func:`slim_chunk_options` for the projection rules and
    :func:`chunk_strategy_key` for the strategy → sub-dict mapping
    (default F → ``fixed_token``).

    The F runtime args from ``LightRAG.ainsert`` overlay the
    ``fixed_token`` sub-dict when (and only when) the active strategy
    is F — for R/V/P these args have no slot to land in and are
    silently dropped:

      - ``split_by_character`` overrides the env when **non-None**.
        ``None`` (signature default) means "use the env / addon_params
        default".
      - ``split_by_character_only`` overrides the env when **True**.
        ``False`` (signature default) means "use the env / addon_params
        default" — there's no clean way to distinguish "unset" from
        "explicit False" with a positional default, so the env wins
        unless the caller actively opts in.

    The returned snapshot is an independent deep copy: mutating it has
    no effect on subsequent resolutions.
    """
    src: Mapping[str, Any] | None = None
    if isinstance(addon_params, Mapping):
        candidate = addon_params.get("chunker")
        if isinstance(candidate, Mapping):
            src = candidate
    if src is None:
        src = default_chunker_config()

    snapshot = slim_chunk_options(src, process_options)
    if chunk_strategy_key(process_options) == "fixed_token":
        fixed = snapshot["fixed_token"]
        if split_by_character is not None:
            fixed["split_by_character"] = split_by_character
        if split_by_character_only:
            fixed["split_by_character_only"] = True
    # P-strategy ``chunk_token_size`` backfill lives in
    # ``slim_chunk_options`` — that's the single chokepoint shared by
    # every enqueue path (this function AND the direct
    # ``chunk_options=`` kwarg path in ``_chunk_options_at``).
    return snapshot


def _extract_param_blocks(
    inner: str,
) -> tuple[str, str | None, dict[str, str], list[str]]:
    """Strip ``(...)`` parameter blocks from a hint / rule inner string.

    Returns ``(stripped, engine_param_text, chunk_param_texts, errors)``:

    * ``stripped`` is ``inner`` with every parameter block removed, so the
      existing engine / selector parsing (:func:`split_engine_and_options`,
      :func:`_rule_engine_and_options`, :func:`validate_process_options`) runs
      on a parameter-free string and legacy behaviour is preserved verbatim.
    * ``engine_param_text`` is the text inside an engine-level ``(...)`` block
      (before the engine/options ``-``) when present, else ``None``.  Engine
      parameters are not accepted in Phase 1; callers reject them.
    * ``chunk_param_texts`` maps each chunk selector char (F/R/V/P) to the raw
      text of the block that immediately follows it.
    * ``errors`` collects structural problems (unbalanced parens, a block not
      following a chunk strategy, duplicate blocks on one char).

    A parameter-free ``inner`` returns ``(inner, None, {}, [])`` unchanged.
    """
    out: list[str] = []
    engine_param: str | None = None
    chunk_params: dict[str, str] = {}
    errors: list[str] = []
    i = 0
    n = len(inner)
    seen_dash = False
    prev_meaningful: str | None = None
    while i < n:
        ch = inner[i]
        if ch == "(":
            block, nxt = take_paren_block(inner, i)
            if block is None:
                errors.append(f"unbalanced '(' in {inner!r}")
                out.append(inner[i:])
                break
            if seen_dash and prev_meaningful in PROCESS_OPTION_CHUNK_CHARS:
                if prev_meaningful in chunk_params:
                    errors.append(
                        f"chunk strategy {prev_meaningful!r} has more than one "
                        "parameter block"
                    )
                else:
                    chunk_params[prev_meaningful] = block
            elif not seen_dash and prev_meaningful is not None:
                # Engine-level block, e.g. ``mineru(page_range=1-3)``.
                if engine_param is not None:
                    errors.append("parser engine has more than one parameter block")
                else:
                    engine_param = block
            else:
                errors.append(
                    f"parameters '({block})' must follow a chunk strategy (F/R/V/P)"
                )
            i = nxt
            prev_meaningful = None
            continue
        out.append(ch)
        if ch == "-":
            seen_dash = True
        if ch != " ":
            prev_meaningful = ch
        i += 1
    return "".join(out), engine_param, chunk_params, errors


def _parse_chunk_param_texts(
    chunk_param_texts: dict[str, str], *, label: str
) -> tuple[dict[str, dict[str, Any]], list[str]]:
    """Parse raw chunk-param block texts into canonical per-selector dicts.

    Returns ``(chunk_params, errors)``; ``chunk_params`` only contains the
    selectors whose block parsed to a non-empty dict.
    """
    chunk_params: dict[str, dict[str, Any]] = {}
    errors: list[str] = []
    for selector, text in chunk_param_texts.items():
        parsed, perrors = parse_chunk_params(
            text, selector=selector, label=f"{label} chunk strategy {selector!r}"
        )
        errors.extend(perrors)
        if parsed:
            chunk_params[selector] = parsed
    return chunk_params, errors


def split_engine_and_options(bracket_inner: str) -> tuple[str | None, str]:
    """Decompose a bracket-hint inner string into ``(engine, options)``.

    Format rules (see docs/FileProcessingPipeline-zh.md):
        - ``ENGINE-OPTIONS``: first ``-``-separated segment is the engine
          candidate; the remainder is the options string.
        - ``ENGINE``: matches a supported engine name as a whole.
        - ``-OPTIONS``: leading ``-`` marks an options-only hint.
    """
    inner = (bracket_inner or "").strip()
    if not inner:
        return None, ""

    if inner.startswith("-"):
        return None, inner[1:].strip()

    if "-" in inner:
        head, _, tail = inner.partition("-")
        engine_candidate = normalize_parser_engine(head)
        if engine_candidate in supported_parser_engines():
            return engine_candidate, tail.strip()
        return None, ""

    engine_candidate = normalize_parser_engine(inner)
    if engine_candidate in supported_parser_engines():
        return engine_candidate, ""
    return None, ""


def parser_suffix(file_path: str | Path) -> str:
    return Path(file_path).suffix.lower().lstrip(".")


def parser_engine_supports_suffix(engine: str, suffix: str) -> bool:
    return suffix.lower().lstrip(".") in suffix_capabilities(engine)


def parser_engine_endpoint_configured(engine: str) -> bool:
    # Endpoint capability lives on the registry ParserSpec (single source).
    return engine_endpoint_configured(engine)


def parser_engine_endpoint_requirement(engine: str) -> str | None:
    return engine_endpoint_requirement(engine)


def _engine_is_usable(
    engine: str,
    suffix: str,
    *,
    require_external_endpoint: bool,
) -> bool:
    if engine not in supported_parser_engines():
        return False
    if not parser_engine_supports_suffix(engine, suffix):
        return False
    if require_external_endpoint and not parser_engine_endpoint_configured(engine):
        return False
    return True


def _filename_hint_match(
    file_path: str | Path,
) -> tuple[re.Match[str], str, str, dict[str, dict[str, Any]], dict[str, Any]] | None:
    """Locate a supported ``[hint]`` segment in a basename.

    Returns ``(match, engine_or_empty, options, chunk_params, engine_params)``
    when the bracket inner is a recognised hint per the spec; otherwise
    ``None``.  ``chunk_params`` maps a chunk selector char to its canonical
    parameter dict; ``engine_params`` is the engine-token block's canonical
    parameters (empty when the hint carries none).  This low-level helper stays
    non-throwing because scan grouping and basename canonicalization need a
    best-effort classifier.  Ingestion entrypoints must call
    :func:`resolve_parser_directives`, which validates malformed hints and
    raises instead of falling back.
    """
    basename = Path(file_path).name
    m = _PARSER_HINT_RE.search(basename)
    if not m:
        return None
    raw_inner = m.group(1).strip()
    if raw_inner.startswith("-") and not raw_inner[1:].strip():
        return None

    inner, engine_param, chunk_param_texts, struct_errors = _extract_param_blocks(
        raw_inner
    )
    label = f"filename hint {m.group(0)!r}"
    if struct_errors:
        logger.warning(
            f"[parser_routing] ignoring {label} in {basename!r}: "
            f"{'; '.join(struct_errors)}"
        )
        return None
    inner = inner.strip()

    if (
        "-" in inner
        and not inner.startswith("-")
        and not inner.partition("-")[2].strip()
    ):
        return None
    engine, options = split_engine_and_options(inner)
    if options:
        option_errors = validate_process_options(options)
        if option_errors:
            logger.warning(
                f"[parser_routing] ignoring {label} in {basename!r}: "
                f"{'; '.join(option_errors)}"
            )
            return None
    chunk_params, param_errors = _parse_chunk_param_texts(
        chunk_param_texts, label=label
    )
    engine_params: dict[str, Any] = {}
    if engine_param is not None:
        engine_params, eparam_errors = parse_engine_params(
            engine_param, engine=engine or "", label=label
        )
        param_errors = [*param_errors, *eparam_errors]
    if param_errors:
        logger.warning(
            f"[parser_routing] ignoring {label} in {basename!r}: "
            f"{'; '.join(param_errors)}"
        )
        return None
    if engine in supported_parser_engines():
        return m, engine, options, chunk_params, engine_params
    if engine is None and (options or chunk_params):
        return m, "", options, chunk_params, engine_params
    return None


def _validate_filename_hint_for_resolution(
    file_path: str | Path,
    *,
    require_external_endpoint: bool,
) -> None:
    """Fail fast for malformed filename hints on ingestion entrypoints."""
    basename = Path(file_path).name
    m = _PARSER_HINT_RE.search(basename)
    if not m:
        return

    inner = m.group(1)
    errors: list[str] = []

    if not inner.strip():
        errors.append(f"filename hint {m.group(0)!r} is empty")
        raise FilenameParserHintError(
            f"Invalid filename parser hint in {basename!r}: " + "; ".join(errors)
        )

    # Strip and validate parameter blocks first; the engine / selector
    # branches below then run on a parameter-free string exactly as before.
    hint_label = f"filename hint {m.group(0)!r}"
    inner, engine_param, chunk_param_texts, struct_errors = _extract_param_blocks(inner)
    errors.extend(f"{hint_label}: {msg}" for msg in struct_errors)
    _, param_errors = _parse_chunk_param_texts(chunk_param_texts, label=hint_label)
    errors.extend(param_errors)
    # Engine-param validation is deferred until ``engine`` is resolved below.

    engine: str | None = None
    options = ""

    if inner.startswith("-"):
        options = inner[1:].strip()
        if not options:
            errors.append(f"filename hint {m.group(0)!r} has empty process options")
        else:
            errors.extend(
                validate_process_options(
                    options,
                    label=f"filename hint {m.group(0)!r} options",
                )
            )
    elif "-" in inner:
        engine_name, _, options = inner.partition("-")
        engine = normalize_parser_engine(engine_name)
        if engine not in supported_parser_engines():
            supported = ", ".join(sorted(supported_parser_engines()))
            errors.append(
                f"filename hint {m.group(0)!r} uses unsupported parser engine "
                f"{engine_name.strip()!r}; supported engines: {supported}"
            )
        elif not options.strip():
            errors.append(f"filename hint {m.group(0)!r} has empty process options")
        else:
            errors.extend(
                validate_process_options(
                    options,
                    label=f"filename hint {m.group(0)!r} options",
                )
            )
    else:
        engine = normalize_parser_engine(inner)
        if engine not in supported_parser_engines():
            supported = ", ".join(sorted(supported_parser_engines()))
            message = (
                f"filename hint {m.group(0)!r} uses unsupported parser engine "
                f"{inner.strip()!r}; supported engines: {supported}"
            )
            if all(ch in SUPPORTED_PROCESS_OPTIONS or ch == " " for ch in inner):
                message += (
                    "; options-only filename hints must start with '-' "
                    f"(use '[-{inner.strip()}]' instead)"
                )
            errors.append(message)

    if engine_param is not None:
        _, e_perrors = parse_engine_params(
            engine_param, engine=engine or "", label=hint_label
        )
        errors.extend(e_perrors)

    if engine in supported_parser_engines():
        suffix = parser_suffix(file_path)
        if not parser_engine_supports_suffix(engine, suffix):
            supported_suffixes = ", ".join(sorted(suffix_capabilities(engine)))
            errors.append(
                f"filename hint {m.group(0)!r} uses parser engine {engine!r} "
                f"for unsupported suffix {suffix!r}; supported suffixes: "
                f"{supported_suffixes}"
            )
        endpoint_req = parser_engine_endpoint_requirement(engine)
        if (
            require_external_endpoint
            and endpoint_req
            and not parser_engine_endpoint_configured(engine)
        ):
            errors.append(
                f"filename hint {m.group(0)!r} requires {endpoint_req} to be configured"
            )

    if errors:
        raise FilenameParserHintError(
            f"Invalid filename parser hint in {basename!r}: " + "; ".join(errors)
        )


def filename_parser_hint(file_path: str | Path) -> str | None:
    """Return the engine inferred from a filename hint, or ``None``."""
    found = _filename_hint_match(file_path)
    if not found:
        return None
    return found[1] or None


def filename_process_options(file_path: str | Path) -> str:
    """Return the raw process-options string from a filename hint."""
    found = _filename_hint_match(file_path)
    if not found:
        return ""
    return found[2]


def filename_parser_directives(file_path: str | Path) -> tuple[str | None, str]:
    """Return ``(engine, options)`` decoded from a filename hint."""
    found = _filename_hint_match(file_path)
    if not found:
        return None, ""
    return (found[1] or None), found[2]


def filename_chunk_params(file_path: str | Path) -> dict[str, dict[str, Any]]:
    """Return the per-selector chunk parameters decoded from a filename hint.

    Maps a chunk selector char (F/R/V/P) to its canonical parameter dict;
    empty when the hint carries no parameters or is not a usable hint.
    """
    found = _filename_hint_match(file_path)
    if not found:
        return {}
    return found[3]


def filename_engine_params(file_path: str | Path) -> dict[str, Any]:
    """Return the canonical engine parameters decoded from a filename hint.

    These belong to the hint's engine token (e.g. ``mineru(page_range=1-3)``);
    empty when the hint carries no engine parameters or is not a usable hint.
    """
    found = _filename_hint_match(file_path)
    if not found:
        return {}
    return found[4]


def canonicalize_parser_hinted_basename(file_path: str | Path) -> str:
    """Return basename with a supported parser hint removed.

    Only the final ``.[engine].ext`` (or ``.[engine-options].ext`` /
    ``.[-options].ext``) segment is stripped, exactly once, and only when the
    bracket content is a recognised hint.  Nested hints such as
    ``name.[native].[mineru].pdf`` therefore become ``name.[native].pdf`` —
    additional outer hints are not unwrapped.
    """
    basename = Path(file_path).name
    found = _filename_hint_match(file_path)
    if not found:
        return basename
    m = found[0]
    return f"{basename[: m.start()]}{m.group(2)}"


def parser_rules_from_env() -> str:
    return os.getenv("LIGHTRAG_PARSER", "").strip()


def _iter_parser_rule_items(rules: str) -> list[tuple[int, str]]:
    # Parenthesis-aware split: ';' (preferred) or ',' (legacy) separate rules
    # at paren depth 0, so commas inside a chunk-parameter block such as
    # ``R(chunk_ts=800,chunk_ol=80)`` never split the surrounding rule.
    return [
        (index, item.strip())
        for index, item in enumerate(split_top_level(rules, ";,"), start=1)
        if item.strip()
    ]


def _rule_pattern_matches_engine_capability(pattern: str, engine: str) -> bool:
    supported_suffixes = suffix_capabilities(engine)
    return any(fnmatch.fnmatch(suffix, pattern) for suffix in supported_suffixes)


def _rule_engine_and_options(engine_hint: str) -> tuple[str, str]:
    """Split a ``LIGHTRAG_PARSER`` rule's RHS (``engine[-options]``).

    Returns ``(normalized_engine, options_str)``.  Unlike the filename hint
    splitter this always treats the first ``-`` as the engine/options
    boundary, since ``LIGHTRAG_PARSER`` rules cannot be options-only.
    """
    head, _, tail = engine_hint.partition("-")
    return normalize_parser_engine(head), tail.strip()


def validate_parser_routing_config(parser_rules: str | None = None) -> None:
    """Validate LIGHTRAG_PARSER syntax and required external parser endpoints."""
    rules = parser_rules_from_env() if parser_rules is None else parser_rules.strip()
    if not rules:
        return

    errors: list[str] = []
    for index, item in _iter_parser_rule_items(rules):
        label = f"rule {index} ({item!r})"
        if ":" not in item:
            errors.append(f"{label} must use '<suffix-pattern>:<engine>'")
            continue

        pattern, engine_hint = item.split(":", 1)
        pattern = pattern.strip().lower()
        engine_hint = engine_hint.strip()
        stripped_hint, engine_param, chunk_param_texts, struct_errors = (
            _extract_param_blocks(engine_hint)
        )
        engine, options_str = _rule_engine_and_options(stripped_hint)

        if not pattern:
            errors.append(f"{label} has an empty suffix pattern")
            continue
        if "." in pattern:
            errors.append(
                f"{label} matches suffixes without dots; use 'pdf', not '*.pdf'"
            )
            continue
        if not engine_hint:
            errors.append(f"{label} has an empty parser engine")
            continue
        if engine not in supported_parser_engines():
            supported = ", ".join(sorted(supported_parser_engines()))
            errors.append(
                f"{label} uses unsupported parser engine {engine_hint!r}; "
                f"supported engines: {supported}"
            )
            continue
        if not _rule_pattern_matches_engine_capability(pattern, engine):
            supported_suffixes = ", ".join(sorted(suffix_capabilities(engine)))
            errors.append(
                f"{label} does not match any suffix supported by {engine}; "
                f"supported suffixes: {supported_suffixes}"
            )
        endpoint_req = parser_engine_endpoint_requirement(engine)
        if endpoint_req and not parser_engine_endpoint_configured(engine):
            errors.append(f"{label} requires {endpoint_req} to be configured")
        errors.extend(f"{label}: {msg}" for msg in struct_errors)
        if engine_param is not None:
            _, e_perrors = parse_engine_params(engine_param, engine=engine, label=label)
            errors.extend(e_perrors)
        if options_str:
            errors.extend(
                f"{label}: {msg}"
                for msg in validate_process_options(
                    options_str, label="process options"
                )
            )
        _, param_errors = _parse_chunk_param_texts(chunk_param_texts, label=label)
        errors.extend(param_errors)

    if errors:
        raise ParserRoutingConfigError(
            "Invalid LIGHTRAG_PARSER configuration: " + "; ".join(errors)
        )


def _matching_rule_directives(
    file_path: str | Path,
    *,
    parser_rules: str | None,
    require_external_endpoint: bool,
) -> tuple[str | None, str, dict[str, dict[str, Any]], dict[str, Any]]:
    """Find the first matching ``LIGHTRAG_PARSER`` rule for ``file_path``.

    Returns ``(engine, options_str, chunk_params, engine_params)`` where
    ``engine`` is ``None`` when no usable rule is found.  ``options_str`` is
    empty when a rule matched but has no ``-options`` suffix; ``chunk_params``
    maps a chunk selector char to its canonical parameter dict; ``engine_params``
    is the matched rule's engine-token parameters.  Rule syntax is validated at
    startup (:func:`validate_parser_routing_config`), so a malformed parameter
    block here is skipped best-effort rather than raised.
    """
    suffix = parser_suffix(file_path)
    rules = parser_rules_from_env() if parser_rules is None else parser_rules.strip()
    if not rules:
        return None, "", {}, {}
    for _, item in _iter_parser_rule_items(rules):
        if ":" not in item:
            continue
        pattern, engine_hint = item.split(":", 1)
        pattern = pattern.strip().lower()
        stripped_hint, engine_param, chunk_param_texts, _errs = _extract_param_blocks(
            engine_hint.strip()
        )
        engine, options_str = _rule_engine_and_options(stripped_hint)
        if not fnmatch.fnmatch(suffix, pattern):
            continue
        if _engine_is_usable(
            engine,
            suffix,
            require_external_endpoint=require_external_endpoint,
        ):
            chunk_params, _param_errors = _parse_chunk_param_texts(
                chunk_param_texts, label=f"rule {item!r}"
            )
            engine_params: dict[str, Any] = {}
            if engine_param is not None:
                engine_params, _e_perrors = parse_engine_params(
                    engine_param, engine=engine, label=f"rule {item!r}"
                )
            return engine, options_str, chunk_params, engine_params
    return None, "", {}, {}


@dataclass(frozen=True)
class ParserDirectives:
    """Fully resolved per-file parser directives.

    ``process_options`` stays a pure selector string (``i/t/e/!/F/R/V/P``);
    parameters live in separate fields.  ``chunk_params`` maps a chunk
    selector char to its canonical parameter dict and feeds the existing
    ``chunk_options`` channel.  ``engine_params`` is the flat, canonical
    parameter dict for the resolved engine (e.g. MinerU ``page_range`` /
    ``language`` / ``local_parse_method``, Docling ``force_ocr``); it is
    encoded into the persisted ``parse_engine`` field and consumed by the
    external parser at parse time.
    """

    engine: str
    process_options: str
    chunk_params: dict[str, dict[str, Any]]
    engine_params: dict[str, Any]


def resolve_parser_directives(
    file_path: str | Path,
    *,
    parser_rules: str | None = None,
    require_external_endpoint: bool = True,
) -> ParserDirectives:
    """Resolve engine, process options and per-file parameters for a file.

    Resolution order (mirrors :func:`resolve_file_parser_engine`):
        1. Filename ``[hint]`` — engine and / or options take precedence.
        2. ``LIGHTRAG_PARSER`` rules — first matching rule provides defaults
           for whichever of engine / options the filename hint did not
           specify.
        3. Default engine ``legacy`` with empty options.

    Selector (``i/t/e/!/FRVP``) keeps the legacy "filename options wholesale
    override rule options" behaviour.  Chunk parameters overlay per selector
    char: rule parameters first, then filename-hint parameters (filename wins
    on a shared key).
    """
    suffix = parser_suffix(file_path)
    _validate_filename_hint_for_resolution(
        file_path,
        require_external_endpoint=require_external_endpoint,
    )

    hinted_engine, hinted_options = filename_parser_directives(file_path)
    hinted_chunk_params = filename_chunk_params(file_path)
    hinted_engine_params = filename_engine_params(file_path)
    # The engine a filename hint's params belong to (captured before the
    # usability null-out below); engine params are dropped if this engine does
    # not win resolution.  Note: under strict validation the null-out is
    # effectively unreachable (the validator raises first), so this normally
    # equals the resolved engine when a hint engine is present.
    hinted_engine_for_params = hinted_engine
    if hinted_engine and not _engine_is_usable(
        hinted_engine, suffix, require_external_endpoint=require_external_endpoint
    ):
        # Hinted engine cannot handle this file (e.g. wrong suffix or missing
        # endpoint); fall back to rule-based resolution but keep the hinted
        # options if any.
        hinted_engine = None

    rule_engine, rule_options, rule_chunk_params, rule_engine_params = (
        _matching_rule_directives(
            file_path,
            parser_rules=parser_rules,
            require_external_endpoint=require_external_endpoint,
        )
    )

    default_engine = _DEFAULT_ENGINE_BY_SUFFIX.get(suffix)
    if default_engine and not _engine_is_usable(
        default_engine, suffix, require_external_endpoint=require_external_endpoint
    ):
        default_engine = None

    engine = hinted_engine or rule_engine or default_engine or PARSER_ENGINE_LEGACY
    options_str = hinted_options or rule_options

    # Overlay chunk params per selector char: rule first, filename-hint wins.
    chunk_params: dict[str, dict[str, Any]] = {}
    for selector in set(rule_chunk_params) | set(hinted_chunk_params):
        merged = {
            **rule_chunk_params.get(selector, {}),
            **hinted_chunk_params.get(selector, {}),
        }
        if merged:
            chunk_params[selector] = merged

    # Engine params are flat (one resolved engine per file). Keep only the
    # params whose attached engine == the resolved engine: rule params first,
    # then filename-hint params (filename wins on a shared key). Params attached
    # to an engine that lost resolution are dropped.
    engine_params: dict[str, Any] = {}
    if rule_engine == engine:
        engine_params.update(rule_engine_params)
    if hinted_engine_for_params == engine:
        engine_params.update(hinted_engine_params)

    return ParserDirectives(
        engine=engine,
        process_options=sanitize_process_options(options_str),
        chunk_params=chunk_params,
        engine_params=engine_params,
    )


def resolve_file_parser_engine(
    file_path: str | Path,
    *,
    parser_rules: str | None = None,
    require_external_endpoint: bool = True,
) -> str:
    """Resolve the extraction engine for a source file before content extraction."""
    engine, _ = resolve_file_parser_directives(
        file_path,
        parser_rules=parser_rules,
        require_external_endpoint=require_external_endpoint,
    )
    return engine


def resolve_file_parser_directives(
    file_path: str | Path,
    *,
    parser_rules: str | None = None,
    require_external_endpoint: bool = True,
) -> tuple[str, str]:
    """Resolve ``(engine, process_options)`` for a source file before extraction.

    Backward-compatible thin wrapper over :func:`resolve_parser_directives`;
    callers that also need the per-file chunk / engine parameters should use
    :func:`resolve_parser_directives` directly.
    """
    directives = resolve_parser_directives(
        file_path,
        parser_rules=parser_rules,
        require_external_endpoint=require_external_endpoint,
    )
    return directives.engine, directives.process_options


def resolve_stored_document_parser_engine(
    file_path: str | Path,
    content_data: dict[str, Any] | None,
) -> str:
    """Resolve the parser engine key for a full_docs row during processing.

    Returns a registry key: the internal ``reuse``/``passthrough`` handlers for
    the no-op formats, or a real engine name for ``pending_parse``.  Never
    raises and never filters an unknown stored engine — the parse worker
    decides fallback/validation (so its warning path actually fires).
    """
    if content_data:
        doc_format = content_data.get("parse_format", FULL_DOCS_FORMAT_RAW)
        # All lightrag rows reuse the already-parsed sidecar (sidecar optional;
        # ReuseParser tolerates a missing one). Reached on resume/retry.
        if doc_format == FULL_DOCS_FORMAT_LIGHTRAG:
            return PARSER_ENGINE_REUSE
        # Anything not pending is already-extracted content (raw direct-insert
        # or legacy-extracted RAW) -> pass through verbatim.
        if doc_format != FULL_DOCS_FORMAT_PENDING_PARSE:
            return PARSER_ENGINE_PASSTHROUGH
        # PENDING_PARSE: honour the stored engine verbatim; fall back to
        # filename rules only when it is absent.
        pending_engine = normalize_parser_engine(content_data.get("parse_engine"))
        if pending_engine:
            return pending_engine

    return resolve_file_parser_engine(file_path)
