# SPDX-License-Identifier: Apache-2.0
"""
Tests for reasoning content extraction parsers.

Tests cover:
- Parser registry (registration, lookup, listing)
- Qwen3 parser (non-streaming and streaming)
- DeepSeek-R1 parser (non-streaming and streaming)
- Gemma 4 parser (channel protocol, streaming, channel name stripping)
- Edge cases (no tags, partial tags, etc.)
"""

import pytest

from vllm_mlx.reasoning import (
    DeltaMessage,
    ReasoningParser,
    get_parser,
    list_parsers,
    register_parser,
)


class TestParserRegistry:
    """Tests for the parser registry functions."""

    def test_list_parsers_includes_builtin(self):
        """Built-in parsers should be registered."""
        parsers = list_parsers()
        assert "qwen3" in parsers
        assert "deepseek_r1" in parsers
        assert "gemma4" in parsers
        assert "glm4" in parsers

    def test_get_parser_qwen3(self):
        """Should be able to get Qwen3 parser."""
        parser_cls = get_parser("qwen3")
        parser = parser_cls()
        assert isinstance(parser, ReasoningParser)

    def test_get_parser_deepseek(self):
        """Should be able to get DeepSeek-R1 parser."""
        parser_cls = get_parser("deepseek_r1")
        parser = parser_cls()
        assert isinstance(parser, ReasoningParser)

    def test_get_parser_glm4(self):
        """Should be able to get GLM4 parser."""
        parser_cls = get_parser("glm4")
        parser = parser_cls()
        assert isinstance(parser, ReasoningParser)

    def test_get_unknown_parser_raises(self):
        """Unknown parser name should raise KeyError."""
        with pytest.raises(KeyError) as exc_info:
            get_parser("unknown_parser")
        assert "unknown_parser" in str(exc_info.value)
        assert "Available parsers" in str(exc_info.value)

    def test_register_custom_parser(self):
        """Should be able to register custom parsers."""

        class CustomParser(ReasoningParser):
            def extract_reasoning(self, model_output):
                return None, model_output

            def extract_reasoning_streaming(self, prev, curr, delta):
                return DeltaMessage(content=delta)

        register_parser("custom_test", CustomParser)
        assert "custom_test" in list_parsers()

        parser = get_parser("custom_test")()
        assert isinstance(parser, CustomParser)


class TestQwen3Parser:
    """Tests for the Qwen3 reasoning parser."""

    @pytest.fixture
    def parser(self):
        """Create a fresh Qwen3 parser for each test."""
        return get_parser("qwen3")()

    # Non-streaming tests

    def test_extract_with_both_tags(self, parser):
        """Should extract reasoning when both tags present."""
        output = "<think>Let me analyze this problem</think>The answer is 42."
        reasoning, content = parser.extract_reasoning(output)
        assert reasoning == "Let me analyze this problem"
        assert content == "The answer is 42."

    def test_extract_only_reasoning(self, parser):
        """Should handle case where only reasoning is present."""
        output = "<think>Just thinking out loud</think>"
        reasoning, content = parser.extract_reasoning(output)
        assert reasoning == "Just thinking out loud"
        assert content is None

    def test_extract_multiline_reasoning(self, parser):
        """Should preserve newlines in reasoning content."""
        output = (
            "<think>Step 1: Analyze\nStep 2: Solve\nStep 3: Verify</think>Result: 42"
        )
        reasoning, content = parser.extract_reasoning(output)
        assert "Step 1" in reasoning
        assert "Step 2" in reasoning
        assert "Step 3" in reasoning
        assert content == "Result: 42"

    def test_no_tags_returns_content_only(self, parser):
        """Qwen3 requires both tags - no tags means pure content."""
        output = "Just a regular response without thinking."
        reasoning, content = parser.extract_reasoning(output)
        assert reasoning is None
        assert content == output

    def test_only_start_tag_no_reasoning(self, parser):
        """Qwen3 requires both tags - missing end tag means no reasoning."""
        output = "<think>Started thinking but never finished"
        reasoning, content = parser.extract_reasoning(output)
        assert reasoning is None
        assert content == output

    def test_only_end_tag_implicit_mode(self, parser):
        """Qwen3 supports implicit mode - when <think> is in prompt, only </think> in output."""
        output = "Some text</think>more text"
        reasoning, content = parser.extract_reasoning(output)
        # Implicit mode: everything before </think> is reasoning
        assert reasoning == "Some text"
        assert content == "more text"

    # Streaming tests

    def test_streaming_simple_flow(self, parser):
        """Test basic streaming with reasoning then content."""
        parser.reset_state()

        # Simulate streaming tokens
        deltas = ["<think>", "think", "ing", "</think>", "answer"]
        accumulated = ""
        results = []

        for delta in deltas:
            prev = accumulated
            accumulated += delta
            result = parser.extract_reasoning_streaming(prev, accumulated, delta)
            if result:
                results.append(result)

        # Collect reasoning and content
        reasoning_parts = [r.reasoning for r in results if r.reasoning]
        content_parts = [r.content for r in results if r.content]

        assert "".join(reasoning_parts) == "thinking"
        assert "".join(content_parts) == "answer"

    def test_streaming_skip_tags(self, parser):
        """Special tokens themselves should be skipped."""
        parser.reset_state()

        # Just the start tag
        result = parser.extract_reasoning_streaming("", "<think>", "<think>")
        assert result is None

        # Just the end tag
        result = parser.extract_reasoning_streaming(
            "<think>reasoning", "<think>reasoning</think>", "</think>"
        )
        assert result is None

    def test_streaming_transition_chunk(self, parser):
        """Chunk containing end tag should split reasoning and content."""
        parser.reset_state()

        # Previous has start, delta contains end and content
        prev = "<think>reasoning"
        delta = " more</think>content here"
        curr = prev + delta

        result = parser.extract_reasoning_streaming(prev, curr, delta)

        assert result is not None
        assert result.reasoning == " more"
        assert result.content == "content here"


class TestDeepSeekR1Parser:
    """Tests for the DeepSeek-R1 reasoning parser."""

    @pytest.fixture
    def parser(self):
        """Create a fresh DeepSeek-R1 parser for each test."""
        return get_parser("deepseek_r1")()

    # Non-streaming tests

    def test_extract_with_both_tags(self, parser):
        """Should extract reasoning when both tags present."""
        output = "<think>Step by step analysis</think>Final answer: 42"
        reasoning, content = parser.extract_reasoning(output)
        assert reasoning == "Step by step analysis"
        assert content == "Final answer: 42"

    def test_extract_implicit_start_tag(self, parser):
        """DeepSeek-R1 handles implicit start tag (missing <think>)."""
        output = "Implicit reasoning content</think>The answer"
        reasoning, content = parser.extract_reasoning(output)
        assert reasoning == "Implicit reasoning content"
        assert content == "The answer"

    def test_extract_no_tags_pure_content(self, parser):
        """No tags should return pure content."""
        output = "Just a regular response."
        reasoning, content = parser.extract_reasoning(output)
        assert reasoning is None
        assert content == output

    def test_extract_multiline_reasoning(self, parser):
        """Should preserve newlines in reasoning content."""
        output = "<think>Line 1\nLine 2\nLine 3</think>Result"
        reasoning, content = parser.extract_reasoning(output)
        assert "Line 1" in reasoning
        assert "Line 2" in reasoning
        assert "Line 3" in reasoning
        assert content == "Result"

    # Streaming tests

    def test_streaming_simple_flow(self, parser):
        """Test basic streaming with reasoning then content."""
        parser.reset_state()

        deltas = ["<think>", "think", "ing", "</think>", "answer"]
        accumulated = ""
        results = []

        for delta in deltas:
            prev = accumulated
            accumulated += delta
            result = parser.extract_reasoning_streaming(prev, accumulated, delta)
            if result:
                results.append(result)

        reasoning_parts = [r.reasoning for r in results if r.reasoning]
        content_parts = [r.content for r in results if r.content]

        assert "".join(reasoning_parts) == "thinking"
        assert "".join(content_parts) == "answer"


class TestDeltaMessage:
    """Tests for the DeltaMessage dataclass."""

    def test_reasoning_content_alias(self):
        """reasoning_content should alias reasoning."""
        msg = DeltaMessage(reasoning="test reasoning")
        assert msg.reasoning == "test reasoning"
        assert msg.reasoning_content == "test reasoning"

    def test_content_only(self):
        """Should handle content-only messages."""
        msg = DeltaMessage(content="just content")
        assert msg.content == "just content"
        assert msg.reasoning is None
        assert msg.reasoning_content is None

    def test_both_fields(self):
        """Should handle transition messages with both."""
        msg = DeltaMessage(reasoning="ending", content="starting")
        assert msg.reasoning == "ending"
        assert msg.content == "starting"


class TestEdgeCases:
    """Test edge cases across parsers."""

    @pytest.fixture(params=["qwen3", "deepseek_r1"])
    def parser(self, request):
        """Parametrized fixture for both parsers."""
        return get_parser(request.param)()

    def test_empty_output(self, parser):
        """Empty output should return (None, '')."""
        reasoning, content = parser.extract_reasoning("")
        # Either both None or content is empty string
        assert reasoning is None or reasoning == ""

    def test_whitespace_only_reasoning(self, parser):
        """Whitespace-only reasoning should be treated as None."""
        output = "<think>   </think>content"
        reasoning, content = parser.extract_reasoning(output)
        # Whitespace-only should be stripped to None
        if reasoning is not None:
            assert reasoning.strip() == "" or reasoning is None

    def test_nested_tags_not_supported(self, parser):
        """Nested tags are not officially supported - behavior may vary."""
        output = "<think>outer<think>inner</think>still outer</think>content"
        # Just ensure it doesn't crash
        reasoning, content = parser.extract_reasoning(output)
        # Result may vary by parser implementation

    def test_repeated_leading_think_blocks_do_not_leak_to_content(self, parser):
        """Repeated leading think blocks are reasoning, not final content."""
        output = "reasoning</think>\n\n<think></think>\n\nanswer"
        reasoning, content = parser.extract_reasoning(output)
        assert reasoning == "reasoning"
        assert content == "answer"

    def test_streaming_reset_state(self, parser):
        """reset_state should allow reuse of parser."""
        # First stream
        parser.reset_state()
        parser.extract_reasoning_streaming("", "<think>", "<think>")

        # Reset for new stream
        parser.reset_state()

        # Should work fresh
        result = parser.extract_reasoning_streaming("", "content", "content")
        assert result is not None


class TestRealisticStreaming:
    """Tests for realistic streaming scenarios simulating actual model output."""

    @pytest.fixture(params=["qwen3", "deepseek_r1"])
    def parser(self, request):
        """Parametrized fixture for both parsers."""
        return get_parser(request.param)()

    def test_token_by_token_streaming(self, parser):
        """Simulate realistic token-by-token streaming."""
        # Typical model output broken into tokens
        tokens = [
            "<",
            "think",
            ">",  # Start tag split across tokens
            "Let",
            " me",
            " analyze",
            " this",
            ".",
            "\n",
            "Step",
            " 1",
            ":",
            " check",
            " input",
            "\n",
            "Step",
            " 2",
            ":",
            " compute",
            "</",
            "think",
            ">",  # End tag split across tokens
            "The",
            " answer",
            " is",
            " 42",
            ".",
        ]

        parser.reset_state()
        accumulated = ""
        reasoning_parts = []
        content_parts = []

        for token in tokens:
            prev = accumulated
            accumulated += token
            result = parser.extract_reasoning_streaming(prev, accumulated, token)
            if result:
                if result.reasoning:
                    reasoning_parts.append(result.reasoning)
                if result.content:
                    content_parts.append(result.content)

        full_reasoning = "".join(reasoning_parts)
        full_content = "".join(content_parts)

        # Verify reasoning was captured
        assert "Let me analyze" in full_reasoning
        assert "Step 1" in full_reasoning
        assert "Step 2" in full_reasoning

        # Verify content was captured
        assert "The answer is 42" in full_content

    def test_long_reasoning_streaming(self, parser):
        """Test streaming with extended reasoning."""
        # Long reasoning content
        reasoning_text = """
        First, I need to understand the problem.
        The user is asking about quantum computing.

        Let me break this down:
        1. Quantum bits (qubits) can be in superposition
        2. Entanglement allows correlated states
        3. Quantum gates perform operations

        After careful analysis, I can provide an answer.
        """

        output = f"<think>{reasoning_text}</think>Quantum computing uses qubits."

        # Simulate character-by-character streaming
        parser.reset_state()
        accumulated = ""
        reasoning_parts = []
        content_parts = []

        for char in output:
            prev = accumulated
            accumulated += char
            result = parser.extract_reasoning_streaming(prev, accumulated, char)
            if result:
                if result.reasoning:
                    reasoning_parts.append(result.reasoning)
                if result.content:
                    content_parts.append(result.content)

        full_reasoning = "".join(reasoning_parts)
        full_content = "".join(content_parts)

        assert "quantum computing" in full_reasoning.lower()
        assert "qubits" in full_reasoning.lower()
        assert "Quantum computing uses qubits" in full_content

    def test_streaming_no_content_after_reasoning(self, parser):
        """Test streaming when there's only reasoning, no content."""
        tokens = ["<think>", "just", " thinking", "</think>"]

        parser.reset_state()
        accumulated = ""
        reasoning_parts = []
        content_parts = []

        for token in tokens:
            prev = accumulated
            accumulated += token
            result = parser.extract_reasoning_streaming(prev, accumulated, token)
            if result:
                if result.reasoning:
                    reasoning_parts.append(result.reasoning)
                if result.content:
                    content_parts.append(result.content)

        assert "just thinking" in "".join(reasoning_parts)
        assert len(content_parts) == 0 or "".join(content_parts).strip() == ""


class TestUnicodeAndSpecialCharacters:
    """Tests for Unicode and special characters in reasoning."""

    @pytest.fixture(params=["qwen3", "deepseek_r1"])
    def parser(self, request):
        """Parametrized fixture for both parsers."""
        return get_parser(request.param)()

    def test_unicode_reasoning(self, parser):
        """Test reasoning with Unicode characters."""
        output = "<think>分析这个问题：日本語テスト émojis: 🤔💭</think>答案是42"
        reasoning, content = parser.extract_reasoning(output)
        assert "分析" in reasoning
        assert "日本語" in reasoning
        assert "🤔" in reasoning
        assert "42" in content

    def test_code_in_reasoning(self, parser):
        """Test reasoning containing code snippets."""
        output = """<think>
Let me analyze the code:
```python
def factorial(n):
    if n <= 1:
        return 1
    return n * factorial(n-1)
```
This is a recursive implementation.
</think>The factorial function uses recursion."""

        reasoning, content = parser.extract_reasoning(output)
        assert "def factorial" in reasoning
        assert "recursive" in reasoning
        assert "uses recursion" in content

    def test_html_like_content(self, parser):
        """Test that HTML-like content doesn't confuse the parser."""
        output = "<think>The user mentioned <div> and <span> tags</think>Use CSS for styling."
        reasoning, content = parser.extract_reasoning(output)
        assert "<div>" in reasoning
        assert "<span>" in reasoning
        assert "CSS" in content

    def test_math_expressions(self, parser):
        """Test reasoning with mathematical expressions."""
        output = "<think>Given: x² + 2x + 1 = 0, so (x+1)² = 0, x = -1</think>x = -1"
        reasoning, content = parser.extract_reasoning(output)
        assert "x²" in reasoning
        assert "(x+1)²" in reasoning
        assert "-1" in content


class TestAPIModelsIntegration:
    """Tests for integration with API models."""

    def test_assistant_message_with_reasoning(self):
        """Test that AssistantMessage can hold reasoning content."""
        from vllm_mlx.api.models import AssistantMessage

        msg = AssistantMessage(
            content="The answer is 42.", reasoning="Let me think step by step..."
        )
        assert msg.content == "The answer is 42."
        assert msg.reasoning == "Let me think step by step..."
        assert msg.role == "assistant"

    def test_assistant_message_reasoning_none(self):
        """Test AssistantMessage with no reasoning."""
        from vllm_mlx.api.models import AssistantMessage

        msg = AssistantMessage(content="Simple response without reasoning.")
        assert msg.content == "Simple response without reasoning."
        assert msg.reasoning is None

    def test_chat_completion_chunk_delta_with_reasoning(self):
        """Test that ChatCompletionChunkDelta can hold reasoning."""
        from vllm_mlx.api.models import ChatCompletionChunkDelta

        delta = ChatCompletionChunkDelta(reasoning="thinking...")
        assert delta.reasoning == "thinking..."
        assert delta.content is None

        delta2 = ChatCompletionChunkDelta(content="response text")
        assert delta2.content == "response text"
        assert delta2.reasoning is None

    def test_delta_transition(self):
        """Test delta during transition from reasoning to content."""
        from vllm_mlx.api.models import ChatCompletionChunkDelta

        # During transition, both might have values
        delta = ChatCompletionChunkDelta(
            reasoning="final thought", content="starting answer"
        )
        assert delta.reasoning == "final thought"
        assert delta.content == "starting answer"


class TestParserPerformance:
    """Basic performance tests for parsers."""

    @pytest.fixture(params=["qwen3", "deepseek_r1"])
    def parser(self, request):
        """Parametrized fixture for both parsers."""
        return get_parser(request.param)()

    def test_large_output_extraction(self, parser):
        """Test extraction from large output."""
        # Generate large reasoning content
        reasoning_lines = [f"Step {i}: processing data chunk {i}" for i in range(100)]
        reasoning_text = "\n".join(reasoning_lines)
        output = f"<think>{reasoning_text}</think>Processing complete."

        reasoning, content = parser.extract_reasoning(output)

        assert reasoning is not None
        assert "Step 0" in reasoning
        assert "Step 99" in reasoning
        assert content == "Processing complete."

    def test_streaming_many_chunks(self, parser):
        """Test streaming with many small chunks."""
        parser.reset_state()

        # Generate many small chunks
        base_output = "<think>A" * 100 + "</think>" + "B" * 50
        accumulated = ""
        chunk_count = 0

        for char in base_output:
            prev = accumulated
            accumulated += char
            result = parser.extract_reasoning_streaming(prev, accumulated, char)
            if result:
                chunk_count += 1

        # Should have processed all characters
        assert chunk_count > 0

    def test_repeated_parsing(self, parser):
        """Test parsing same output multiple times."""
        output = "<think>Quick thought</think>Quick answer"

        for _ in range(100):
            reasoning, content = parser.extract_reasoning(output)
            assert reasoning == "Quick thought"
            assert content == "Quick answer"


class TestDeepSeekSpecificCases:
    """Tests specific to DeepSeek-R1 parser behavior."""

    @pytest.fixture
    def parser(self):
        """Create DeepSeek-R1 parser."""
        return get_parser("deepseek_r1")()

    def test_implicit_reasoning_streaming(self, parser):
        """Test streaming when start tag is implicit (DeepSeek-R1 specific)."""
        # DeepSeek-R1 sometimes omits <think> but includes </think>
        tokens = ["reasoning", " text", " here", "</think>", "answer"]

        parser.reset_state()
        accumulated = ""
        reasoning_parts = []
        content_parts = []

        for token in tokens:
            prev = accumulated
            accumulated += token
            result = parser.extract_reasoning_streaming(prev, accumulated, token)
            if result:
                if result.reasoning:
                    reasoning_parts.append(result.reasoning)
                if result.content:
                    content_parts.append(result.content)

        # For DeepSeek-R1, content before </think> without <think> is treated as content
        # until </think> appears in the delta
        all_parts = reasoning_parts + content_parts
        assert len(all_parts) > 0

    def test_deepseek_long_implicit_reasoning(self, parser):
        """Test long implicit reasoning without start tag."""
        output = """Let me think about this problem carefully.

First, I need to consider the constraints.
Then, I'll apply the algorithm.
Finally, I'll verify the result.</think>The answer is 42."""

        reasoning, content = parser.extract_reasoning(output)
        assert reasoning is not None
        assert "think about this problem" in reasoning
        assert "42" in content


class TestQwen3SpecificCases:
    """Tests specific to Qwen3 parser behavior."""

    @pytest.fixture
    def parser(self):
        """Create Qwen3 parser."""
        return get_parser("qwen3")()

    def test_qwen3_implicit_mode_support(self, parser):
        """Qwen3 supports implicit mode for OpenCode compatibility."""
        # Only end tag - implicit mode (think injected in prompt)
        output1 = "some text</think>more text"
        reasoning, content = parser.extract_reasoning(output1)
        # Implicit mode: everything before </think> is reasoning
        assert reasoning == "some text"
        assert content == "more text"

        # Only start tag - no </think> means model is still generating
        # Qwen3 requires </think> to extract reasoning (treats as pure content until then)
        output2 = "<think>incomplete reasoning"
        reasoning, content = parser.extract_reasoning(output2)
        # No </think> = no reasoning extraction, entire output is content
        assert reasoning is None
        assert content == output2

    def test_qwen3_empty_think_tags(self, parser):
        """Test empty think tags."""
        output = "<think></think>Just the answer."
        reasoning, content = parser.extract_reasoning(output)
        # Empty reasoning should be None
        assert reasoning is None or reasoning.strip() == ""
        assert content == "Just the answer."

    def test_qwen3_empty_think_tags_after_implicit_transition(self, parser):
        """Empty blocks after an implicit end tag must not leak to content."""
        output = (
            "brief reasoning</think>\n\n</think>\n\n<think></think>\n\nACK_THINK_READY"
        )
        reasoning, content = parser.extract_reasoning(output)
        assert reasoning == "brief reasoning"
        assert content == "ACK_THINK_READY"

    def test_qwen3_whitespace_between_tags(self, parser):
        """Test various whitespace patterns."""
        test_cases = [
            ("<think> </think>answer", None, "answer"),
            ("<think>\n\n</think>answer", None, "answer"),
            ("<think>\t\t</think>answer", None, "answer"),
        ]

        for output, expected_reasoning, expected_content in test_cases:
            reasoning, content = parser.extract_reasoning(output)
            if expected_reasoning is None:
                assert reasoning is None or reasoning.strip() == ""
            assert expected_content in (content or "")

    def test_qwen3_streaming_empty_think_tags_after_transition(self, parser):
        """Streaming parser suppresses repeated leading think blocks in content."""
        parser.reset_state()
        deltas = [
            "brief reasoning",
            "</think>",
            "</think>",
            "\n\n<thi",
            "nk></think>\n\n",
            "ACK_THINK_READY",
        ]
        previous = ""
        current = ""
        reasoning_parts = []
        content_parts = []

        for delta in deltas:
            previous = current
            current += delta
            result = parser.extract_reasoning_streaming(previous, current, delta)
            if result is None:
                continue
            if result.reasoning:
                reasoning_parts.append(result.reasoning)
            if result.content:
                content_parts.append(result.content)

        assert "".join(reasoning_parts) == "brief reasoning"
        assert "".join(content_parts) == "ACK_THINK_READY"


class TestGptOssParser:
    """Tests for the GPT-OSS reasoning parser (channel-based format)."""

    @pytest.fixture
    def parser(self):
        """Create a fresh GPT-OSS parser for each test."""
        return get_parser("gpt_oss")()

    # Non-streaming tests

    def test_extract_both_channels(self, parser):
        """Should extract reasoning from analysis and content from final."""
        output = (
            "<|channel|>analysis<|message|>Let me think step by step"
            "<|start|>assistant<|channel|>final<|message|>The answer is 42<|return|>"
        )
        reasoning, content = parser.extract_reasoning(output)
        assert reasoning == "Let me think step by step"
        assert content == "The answer is 42"

    def test_extract_only_final(self, parser):
        """Should handle output with only final channel."""
        output = "<|channel|>final<|message|>Just the answer<|return|>"
        reasoning, content = parser.extract_reasoning(output)
        assert reasoning is None
        assert content == "Just the answer"

    def test_extract_only_analysis(self, parser):
        """Should handle output with only analysis channel."""
        output = "<|channel|>analysis<|message|>Just thinking out loud"
        reasoning, content = parser.extract_reasoning(output)
        assert reasoning == "Just thinking out loud"
        assert content is None

    def test_no_channel_tokens_fallback(self, parser):
        """No channel tokens should return pure content."""
        output = "Just a regular response."
        reasoning, content = parser.extract_reasoning(output)
        assert reasoning is None
        assert content == output

    def test_empty_analysis_channel(self, parser):
        """Empty analysis channel should return None reasoning."""
        output = (
            "<|channel|>analysis<|message|>"
            "<|start|>assistant<|channel|>final<|message|>Content here<|return|>"
        )
        reasoning, content = parser.extract_reasoning(output)
        assert reasoning is None
        assert content == "Content here"

    def test_multiline_analysis(self, parser):
        """Should preserve multiline reasoning content."""
        output = (
            "<|channel|>analysis<|message|>Step 1: Analyze\nStep 2: Solve\nStep 3: Verify"
            "<|start|>assistant<|channel|>final<|message|>Result: 42<|return|>"
        )
        reasoning, content = parser.extract_reasoning(output)
        assert "Step 1" in reasoning
        assert "Step 2" in reasoning
        assert "Step 3" in reasoning
        assert content == "Result: 42"

    def test_no_return_token(self, parser):
        """Should handle missing <|return|> at end."""
        output = (
            "<|channel|>analysis<|message|>Thinking"
            "<|start|>assistant<|channel|>final<|message|>Answer"
        )
        reasoning, content = parser.extract_reasoning(output)
        assert reasoning == "Thinking"
        assert content == "Answer"

    # Streaming tests

    def test_streaming_full_flow(self, parser):
        """Test streaming through analysis -> transition -> final phases."""
        parser.reset_state()

        # Simulate token-by-token streaming
        tokens = [
            "<|channel|>",
            "analysis",
            "<|message|>",
            "Let me ",
            "think",
            "<|start|>",
            "assistant",
            "<|channel|>",
            "final",
            "<|message|>",
            "The answer",
            " is 42",
            "<|return|>",
        ]

        accumulated = ""
        reasoning_parts = []
        content_parts = []

        for token in tokens:
            prev = accumulated
            accumulated += token
            result = parser.extract_reasoning_streaming(prev, accumulated, token)
            if result:
                if result.reasoning:
                    reasoning_parts.append(result.reasoning)
                if result.content:
                    content_parts.append(result.content)

        full_reasoning = "".join(reasoning_parts)
        full_content = "".join(content_parts)

        assert "Let me think" in full_reasoning
        assert "The answer is 42" in full_content

    def test_streaming_only_final(self, parser):
        """Test streaming with only final channel."""
        parser.reset_state()

        tokens = [
            "<|channel|>",
            "final",
            "<|message|>",
            "Direct ",
            "answer",
            "<|return|>",
        ]

        accumulated = ""
        content_parts = []

        for token in tokens:
            prev = accumulated
            accumulated += token
            result = parser.extract_reasoning_streaming(prev, accumulated, token)
            if result and result.content:
                content_parts.append(result.content)

        assert "Direct answer" in "".join(content_parts)

    def test_streaming_suppresses_structural_tokens(self, parser):
        """Structural tokens should not leak into reasoning or content."""
        parser.reset_state()

        tokens = [
            "<|channel|>analysis<|message|>",
            "thinking",
            "<|start|>",
            "assistant",
            "<|channel|>final<|message|>",
            "answer",
            "<|return|>",
        ]

        accumulated = ""
        all_output = []

        for token in tokens:
            prev = accumulated
            accumulated += token
            result = parser.extract_reasoning_streaming(prev, accumulated, token)
            if result:
                if result.reasoning:
                    all_output.append(result.reasoning)
                if result.content:
                    all_output.append(result.content)

        combined = "".join(all_output)
        assert "<|" not in combined

    def test_registry_includes_gpt_oss(self):
        """gpt_oss should be in the parser registry."""
        assert "gpt_oss" in list_parsers()

    def test_extract_constrain_format(self, parser):
        """Should handle extended format with <|constrain|> token."""
        output = (
            "<|channel|>analysis<|message|>We need to output JSON"
            "<|end|><|channel|>final <|constrain|>JSON<|message|>"
            '{"hello":"world"}<|return|>'
        )
        reasoning, content = parser.extract_reasoning(output)
        assert reasoning == "We need to output JSON"
        assert content == '{"hello":"world"}'

    def test_extract_constrain_no_analysis(self, parser):
        """Should handle constrain format with only final channel."""
        output = (
            '<|channel|>final <|constrain|>JSON<|message|>{"key":"value"}<|return|>'
        )
        reasoning, content = parser.extract_reasoning(output)
        assert reasoning is None
        assert content == '{"key":"value"}'

    def test_streaming_constrain_format(self, parser):
        """Streaming should handle <|constrain|> in channel marker."""
        parser.reset_state()

        tokens = [
            "<|channel|>analysis<|message|>",
            "Thinking...",
            "<|end|>",
            "<|channel|>final <|constrain|>JSON<|message|>",
            '{"result":',
            '"ok"}',
            "<|return|>",
        ]

        accumulated = ""
        reasoning_parts = []
        content_parts = []

        for token in tokens:
            prev = accumulated
            accumulated += token
            result = parser.extract_reasoning_streaming(prev, accumulated, token)
            if result:
                if result.reasoning:
                    reasoning_parts.append(result.reasoning)
                if result.content:
                    content_parts.append(result.content)

        full_reasoning = "".join(reasoning_parts)
        full_content = "".join(content_parts)

        assert "Thinking" in full_reasoning
        assert '{"result":"ok"}' in full_content
        assert "<|constrain|>" not in full_content

    def test_constrain_tokens_stripped(self, parser):
        """<|constrain|> should not leak into output."""
        output = (
            '<|channel|>final <|constrain|>JSON<|message|>{"hello":"world"}<|return|>'
        )
        reasoning, content = parser.extract_reasoning(output)
        assert "<|constrain|>" not in (content or "")
        assert "<|channel|>" not in (content or "")


class TestGemma4Parser:
    """Tests for the Gemma 4 reasoning parser (channel-based protocol)."""

    @pytest.fixture
    def parser(self):
        """Create a fresh Gemma 4 parser for each test."""
        return get_parser("gemma4")()

    # --- Non-streaming tests ---

    def test_extract_standard_format(self, parser):
        """Standard format: <|channel>thought...<channel|>response."""
        output = (
            "<|channel>thought\nLet me think step by step.\n<channel|>The answer is 42."
        )
        reasoning, content = parser.extract_reasoning(output)
        assert reasoning == "Let me think step by step."
        assert content == "The answer is 42."

    def test_extract_alternative_format(self, parser):
        """Alternative format: <|channel>thought...<|channel>response..."""
        output = "<|channel>thought\nAnalyzing the problem.\n<|channel>response\nThe result is 7."
        reasoning, content = parser.extract_reasoning(output)
        assert reasoning == "Analyzing the problem."
        assert content == "The result is 7."

    def test_extract_strips_thought_prefix(self, parser):
        """Channel name 'thought' should be stripped from reasoning."""
        output = "<|channel>thought\nActual reasoning here<channel|>Content"
        reasoning, content = parser.extract_reasoning(output)
        assert reasoning == "Actual reasoning here"
        assert "thought" not in reasoning

    def test_extract_no_tags_pure_content(self, parser):
        """No channel tags at all should return pure content."""
        output = "Just a regular response without thinking."
        reasoning, content = parser.extract_reasoning(output)
        assert reasoning is None
        assert content == output

    def test_extract_only_start_tag(self, parser):
        """Only start tag means incomplete reasoning (no content yet)."""
        output = "<|channel>thought\nStill thinking..."
        reasoning, content = parser.extract_reasoning(output)
        assert reasoning == "Still thinking..."
        assert content is None

    def test_extract_only_end_tag(self, parser):
        """Only end tag (think injected in prompt)."""
        output = "thought\nImplicit reasoning<channel|>The answer"
        reasoning, content = parser.extract_reasoning(output)
        assert reasoning == "Implicit reasoning"
        assert content == "The answer"

    def test_extract_empty_reasoning(self, parser):
        """Empty reasoning should return None."""
        output = "<|channel>thought\n<channel|>Only content here."
        reasoning, content = parser.extract_reasoning(output)
        assert reasoning is None
        assert content == "Only content here."

    def test_extract_multiline_reasoning(self, parser):
        """Should preserve multiline reasoning content."""
        output = (
            "<|channel>thought\n"
            "Step 1: Understand the question.\n"
            "Step 2: Analyze the data.\n"
            "Step 3: Form conclusion.\n"
            "<channel|>The conclusion is clear."
        )
        reasoning, content = parser.extract_reasoning(output)
        assert "Step 1" in reasoning
        assert "Step 2" in reasoning
        assert "Step 3" in reasoning
        assert content == "The conclusion is clear."

    def test_extract_unicode_reasoning(self, parser):
        """Should handle Unicode in reasoning."""
        output = "<|channel>thought\n日本語テスト 🤔\n<channel|>答えは42"
        reasoning, content = parser.extract_reasoning(output)
        assert "日本語テスト" in reasoning
        assert "🤔" in reasoning
        assert "42" in content

    def test_registry_includes_gemma4(self):
        """gemma4 should be in the parser registry."""
        assert "gemma4" in list_parsers()

    # --- Streaming tests ---

    def test_streaming_no_tags_plain_content(self, parser):
        """Streaming without any channel tags should return content."""
        parser.reset_state()
        result = parser.extract_reasoning_streaming("", "Hello", "Hello")
        assert result is not None
        assert result.content == "Hello"
        assert result.reasoning is None

    def test_streaming_standard_format(self, parser):
        """Test streaming through <|channel>thought...<channel|>content flow."""
        parser.reset_state()

        tokens = [
            "<|channel>",
            "thought",
            "\n",
            "Let me ",
            "think.",
            "<channel|>",
            "The ",
            "answer.",
        ]

        accumulated = ""
        reasoning_parts = []
        content_parts = []

        for token in tokens:
            prev = accumulated
            accumulated += token
            result = parser.extract_reasoning_streaming(prev, accumulated, token)
            if result:
                if result.reasoning:
                    reasoning_parts.append(result.reasoning)
                if result.content:
                    content_parts.append(result.content)

        full_reasoning = "".join(reasoning_parts)
        full_content = "".join(content_parts)

        # "thought\n" prefix should be stripped
        assert "thought" not in full_reasoning or "thought" in "Let me think."
        assert "Let me think." in full_reasoning
        assert "The answer." in full_content

    def test_streaming_alternative_format(self, parser):
        """Test streaming with <|channel>response transition."""
        parser.reset_state()

        tokens = [
            "<|channel>",
            "thought",
            "\n",
            "Analyzing.",
            "<|channel>response",
            "\n",
            "Result: ",
            "42",
        ]

        accumulated = ""
        reasoning_parts = []
        content_parts = []

        for token in tokens:
            prev = accumulated
            accumulated += token
            result = parser.extract_reasoning_streaming(prev, accumulated, token)
            if result:
                if result.reasoning:
                    reasoning_parts.append(result.reasoning)
                if result.content:
                    content_parts.append(result.content)

        full_content = "".join(content_parts)
        assert "Result: 42" in full_content

    def test_streaming_suppresses_channel_names(self, parser):
        """Channel names 'thought' and 'response' should not appear in output."""
        parser.reset_state()

        # Simulate realistic Gemma 4 output
        tokens = [
            "<|channel>",
            "thought",
            "\n",
            "Real ",
            "reasoning.",
            "<channel|>",
            "Real ",
            "content.",
        ]

        accumulated = ""
        all_output = []

        for token in tokens:
            prev = accumulated
            accumulated += token
            result = parser.extract_reasoning_streaming(prev, accumulated, token)
            if result:
                if result.reasoning:
                    all_output.append(("r", result.reasoning))
                if result.content:
                    all_output.append(("c", result.content))

        # Verify no raw "thought" token leaked as reasoning
        reasoning_text = "".join(t for tag, t in all_output if tag == "r")
        content_text = "".join(t for tag, t in all_output if tag == "c")

        assert "Real reasoning." in reasoning_text
        assert "Real content." in content_text

    def test_streaming_token_by_token(self, parser):
        """Test character-by-character streaming (worst case)."""
        parser.reset_state()

        output = "<|channel>thought\nStep 1: Think\nStep 2: Analyze\n<channel|>Final answer: 42."

        accumulated = ""
        reasoning_parts = []
        content_parts = []

        for char in output:
            prev = accumulated
            accumulated += char
            result = parser.extract_reasoning_streaming(prev, accumulated, char)
            if result:
                if result.reasoning:
                    reasoning_parts.append(result.reasoning)
                if result.content:
                    content_parts.append(result.content)

        full_reasoning = "".join(reasoning_parts)
        full_content = "".join(content_parts)

        assert "Step 1: Think" in full_reasoning
        assert "Step 2: Analyze" in full_reasoning
        assert "Final answer: 42." in full_content

    def test_streaming_long_thinking_no_end_tag(self, parser):
        """When model generates long thinking without end tag, all goes to reasoning."""
        parser.reset_state()

        # Simulate model that hits max_tokens before <channel|>
        tokens = [
            "<|channel>",
            "thought",
            "\n",
            "This is a very long ",
            "reasoning process ",
            "that continues ",
            "without ending.",
        ]

        accumulated = ""
        reasoning_parts = []
        content_parts = []

        for token in tokens:
            prev = accumulated
            accumulated += token
            result = parser.extract_reasoning_streaming(prev, accumulated, token)
            if result:
                if result.reasoning:
                    reasoning_parts.append(result.reasoning)
                if result.content:
                    content_parts.append(result.content)

        full_reasoning = "".join(reasoning_parts)
        assert "very long reasoning process" in full_reasoning
        assert len(content_parts) == 0


class TestGlm4Parser:
    """Tests for the GLM-4 reasoning parser."""

    @pytest.fixture
    def parser(self):
        """Create a fresh GLM-4 parser for each test."""
        return get_parser("glm4")()

    def test_registry_includes_glm4(self):
        """glm4 should be in the parser registry."""
        assert "glm4" in list_parsers()

    # Non-streaming tests

    def test_extract_with_both_tags(self, parser):
        """Should extract reasoning when both tags present."""
        output = "<think>Let me analyze this</think>The answer is 42."
        reasoning, content = parser.extract_reasoning(output)
        assert reasoning == "Let me analyze this"
        assert content == "The answer is 42."

    def test_no_tags_returns_content(self, parser):
        """GLM-4 without tags = normal content (not reasoning)."""
        output = "Just a regular response."
        reasoning, content = parser.extract_reasoning(output)
        assert reasoning is None
        assert content == output

    def test_implicit_mode(self, parser):
        """Only closing tag (think injected by agent)."""
        output = "reasoning text</think>content text"
        reasoning, content = parser.extract_reasoning(output)
        assert reasoning == "reasoning text"
        assert content == "content text"

    def test_strips_box_tags(self, parser):
        """GLM-4.6V box container tags should be stripped."""
        output = "<|begin_of_box|>Paris<|end_of_box|>"
        reasoning, content = parser.extract_reasoning(output)
        assert reasoning is None
        assert content == "Paris"

    def test_strips_box_tags_with_thinking(self, parser):
        """Box tags should be stripped from reasoning output too."""
        output = "<think><|begin_of_box|>analysis</think><|begin_of_box|>answer<|end_of_box|>"
        reasoning, content = parser.extract_reasoning(output)
        assert reasoning == "analysis"
        assert content == "answer"

    # Streaming tests

    def test_streaming_no_tags_emits_content(self, parser):
        """GLM-4 streaming without tags should emit content (not reasoning)."""
        parser.reset_state()
        result = parser.extract_reasoning_streaming("", "Hello", "Hello")
        assert result is not None
        assert result.content == "Hello"
        assert result.reasoning is None

    def test_streaming_with_thinking(self, parser):
        """Test streaming with think tags."""
        parser.reset_state()

        tokens = ["<think>", "analyze", "</think>", "answer"]
        accumulated = ""
        reasoning_parts = []
        content_parts = []

        for token in tokens:
            prev = accumulated
            accumulated += token
            result = parser.extract_reasoning_streaming(prev, accumulated, token)
            if result:
                if result.reasoning:
                    reasoning_parts.append(result.reasoning)
                if result.content:
                    content_parts.append(result.content)

        assert "analyze" in "".join(reasoning_parts)
        assert "answer" in "".join(content_parts)

    def test_streaming_strips_box_tags(self, parser):
        """Box tags should be stripped from streaming deltas."""
        parser.reset_state()

        tokens = ["<|begin_of_box|>", "Paris", "<|end_of_box|>"]
        accumulated = ""
        content_parts = []

        for token in tokens:
            prev = accumulated
            accumulated += token
            result = parser.extract_reasoning_streaming(prev, accumulated, token)
            if result and result.content:
                content_parts.append(result.content)

        full = "".join(content_parts)
        assert "Paris" in full
        assert "<|begin_of_box|>" not in full
        assert "<|end_of_box|>" not in full

    def test_streaming_box_tag_returns_none(self, parser):
        """A delta that is purely a box tag should return None."""
        parser.reset_state()
        result = parser.extract_reasoning_streaming(
            "", "<|begin_of_box|>", "<|begin_of_box|>"
        )
        assert result is None


class TestDuplicateThinkEndTag:
    """Test duplicate </think> stripping in BaseThinkingReasoningParser."""

    @pytest.fixture(params=["qwen3", "deepseek_r1", "glm4"])
    def parser(self, request):
        return get_parser(request.param)()

    def test_duplicate_end_tag_stripped(self, parser):
        """Extra </think> after reasoning should be stripped from content."""
        output = "<think>reasoning</think></think>content"
        reasoning, content = parser.extract_reasoning(output)
        assert reasoning == "reasoning"
        assert content == "content"

    def test_triple_end_tag_stripped(self, parser):
        """Multiple extra </think> tags should all be stripped."""
        output = "<think>reasoning</think></think></think>content"
        reasoning, content = parser.extract_reasoning(output)
        assert reasoning == "reasoning"
        assert content == "content"


class TestReasoningStrippedFromToolCallContent:
    """
    Verify that extract_reasoning strips reasoning markers from text that
    already had tool call markup removed (simulating server.py tool_calls path).

    When tool calls are present, server.py first runs the tool parser (which
    strips tool markup but leaves reasoning markers), then runs
    extract_reasoning on the cleaned_text to strip reasoning markers.
    """

    def test_gemma4_channel_tokens_stripped_after_tool_parse(self):
        """Gemma 4 channel tokens should be stripped from post-tool-parse text."""
        parser = get_parser("gemma4")()
        # Simulate what tool parser leaves behind: reasoning markers + residual text
        # (tool markup like ```json...``` already removed by tool parser)
        post_tool_text = "<|channel>thought\nLet me find the weather.\n<channel|>"
        reasoning, content = parser.extract_reasoning(post_tool_text)
        assert reasoning == "Let me find the weather."
        # content should be None or empty (tool calls carry the actual response)
        assert content is None or content == ""

    def test_gemma4_alternative_format_stripped_after_tool_parse(self):
        """Alternative <|channel>response format should also be stripped."""
        parser = get_parser("gemma4")()
        post_tool_text = "<|channel>thought\nChecking parameters.\n<|channel>response\n"
        reasoning, content = parser.extract_reasoning(post_tool_text)
        assert reasoning == "Checking parameters."
        assert content is None or content.strip() == ""

    def test_gemma4_empty_input_after_tool_parse(self):
        """Empty string (all content was tool markup) should not crash."""
        parser = get_parser("gemma4")()
        reasoning, content = parser.extract_reasoning("")
        assert reasoning is None

    def test_qwen3_think_tags_stripped_after_tool_parse(self):
        """Qwen3 <think> tags should be stripped from post-tool-parse text."""
        parser = get_parser("qwen3")()
        post_tool_text = "<think>Let me call the function.</think>"
        reasoning, content = parser.extract_reasoning(post_tool_text)
        assert reasoning == "Let me call the function."
        assert content is None or content == ""

    def test_deepseek_think_tags_stripped_after_tool_parse(self):
        """DeepSeek-R1 <think> tags should be stripped from post-tool-parse text."""
        parser = get_parser("deepseek_r1")()
        post_tool_text = "<think>I need to call this API.</think>"
        reasoning, content = parser.extract_reasoning(post_tool_text)
        assert reasoning == "I need to call this API."
        assert content is None or content == ""

    def test_gemma4_residual_text_preserved(self):
        """Non-reasoning text after channel markers should be preserved as content."""
        parser = get_parser("gemma4")()
        post_tool_text = "<|channel>thought\nThinking...\n<channel|>Some residual text"
        reasoning, content = parser.extract_reasoning(post_tool_text)
        assert reasoning == "Thinking..."
        assert content == "Some residual text"


class TestGemma4DegenerateCycling:
    """
    Test handling of degenerate thought/response cycling in Gemma 4.

    On long prompts with tools, Gemma 4 may oscillate between thought and
    response channels many times before producing valid output. The parser
    must split at the LAST <channel|> so all cycles go into reasoning and
    only the final response goes into content.
    """

    def test_multiple_channel_end_tokens_in_content(self):
        """Multiple <channel|> tokens: rpartition splits at the LAST one."""
        parser = get_parser("gemma4")()
        output = (
            "<|channel>thought\nGarbage loop\n<channel|>thought\n<channel|>The answer"
        )
        reasoning, content = parser.extract_reasoning(output)
        assert content == "The answer"
        assert "Garbage loop" in reasoning

    def test_residual_thought_channel_stripped_from_content(self):
        """thought\\n<channel|> residuals must not leak into content."""
        parser = get_parser("gemma4")()
        output = (
            "<|channel>thought\nLoop1\n<channel|>"
            "<|channel>thought\nLoop2\n<channel|>"
            "Final response here"
        )
        reasoning, content = parser.extract_reasoning(output)
        assert content == "Final response here"
        assert "<channel|>" not in (content or "")
        assert "<|channel>" not in (content or "")

    def test_many_cycles_all_go_to_reasoning(self):
        """All intermediate thought/response cycles end up in reasoning."""
        parser = get_parser("gemma4")()
        output = (
            "<|channel>thought\nCycle1\n<channel|>"
            "<|channel>thought\nCycle2\n<channel|>"
            "<|channel>thought\nCycle3\n<channel|>"
            "Final content"
        )
        reasoning, content = parser.extract_reasoning(output)
        assert content == "Final content"
        assert "Cycle1" in reasoning
        assert "Cycle2" in reasoning
        assert "Cycle3" in reasoning

    def test_channel_tokens_stripped_from_reasoning(self):
        """Channel special tokens should not appear in reasoning output."""
        parser = get_parser("gemma4")()
        output = (
            "<|channel>thought\nThink1\n<channel|>"
            "<|channel>thought\nThink2\n<channel|>"
            "Result"
        )
        reasoning, content = parser.extract_reasoning(output)
        assert "<channel|>" not in (reasoning or "")
        assert "<|channel>" not in (reasoning or "")

    def test_streaming_degenerate_cycling(self):
        """Streaming: re-entry into thinking after content transition."""
        parser = get_parser("gemma4")()
        deltas = [
            "<|channel>",
            "thought\n",
            "Thinking...",
            "\n<channel|>",
            "\nSome content",
            "\n<|channel>",
            "thought\n",
            "More thinking",
            "\n<channel|>",
            "\nFinal answer",
        ]
        reasoning_parts = []
        content_parts = []
        accumulated = ""
        for delta in deltas:
            prev = accumulated
            accumulated += delta
            result = parser.extract_reasoning_streaming(prev, accumulated, delta)
            if result is not None:
                if result.reasoning:
                    reasoning_parts.append(result.reasoning)
                if result.content:
                    content_parts.append(result.content)

        final = parser.finalize_stream()
        if final:
            if final.reasoning:
                reasoning_parts.append(final.reasoning)
            if final.content:
                content_parts.append(final.content)

        full_content = "".join(content_parts)
        # Final answer should be in content (after last <channel|>)
        assert "Final answer" in full_content
        # Channel tokens must not leak
        assert "<channel|>" not in full_content
        assert "<|channel>" not in full_content
