"""Tests for individual extractor classes.

Covers: TrafilaturaExtractor, ReadabilityExtractor, JustextExtractor,
NewspaperExtractor — focusing on the untested paths (ImportError,
extraction failure, empty results, language fallback).

All external libraries are mocked.
"""

from unittest.mock import MagicMock, Mock


# ---------------------------------------------------------------------------
# TrafilaturaExtractor
# ---------------------------------------------------------------------------


class TestTrafilaturaExtractor:
    def test_empty_html(self):
        from local_deep_research.research_library.downloaders.extraction.trafilatura_extractor import (
            TrafilaturaExtractor,
        )

        ext = TrafilaturaExtractor()
        assert ext.extract("") is None
        assert ext.extract("   ") is None
        assert ext.extract(None) is None

    def test_successful_extraction(self, monkeypatch):
        from local_deep_research.research_library.downloaders.extraction.trafilatura_extractor import (
            TrafilaturaExtractor,
        )

        mock_traf = MagicMock()
        mock_traf.extract.return_value = "Extracted markdown content"
        monkeypatch.setitem(__import__("sys").modules, "trafilatura", mock_traf)
        ext = TrafilaturaExtractor()
        result = ext.extract("<html><body><p>Hello world</p></body></html>")
        assert result == "Extracted markdown content"

    def test_import_error(self, monkeypatch):
        from local_deep_research.research_library.downloaders.extraction.trafilatura_extractor import (
            TrafilaturaExtractor,
        )

        # Temporarily hide trafilatura
        import sys

        monkeypatch.delitem(sys.modules, "trafilatura", raising=False)

        # Patch the import to fail
        original_import = (
            __builtins__.__import__
            if hasattr(__builtins__, "__import__")
            else __import__
        )

        def fake_import(name, *args, **kwargs):
            if name == "trafilatura":
                raise ImportError("no trafilatura")
            return original_import(name, *args, **kwargs)

        monkeypatch.setattr("builtins.__import__", fake_import)
        ext = TrafilaturaExtractor()
        result = ext.extract("<html><body>content</body></html>")
        assert result is None

    def test_extraction_exception(self, monkeypatch):
        from local_deep_research.research_library.downloaders.extraction.trafilatura_extractor import (
            TrafilaturaExtractor,
        )

        mock_traf = MagicMock()
        mock_traf.extract.side_effect = RuntimeError("parse error")
        monkeypatch.setitem(__import__("sys").modules, "trafilatura", mock_traf)
        ext = TrafilaturaExtractor()
        result = ext.extract("<html><body>content</body></html>")
        assert result is None

    def test_whitespace_only_result(self, monkeypatch):
        from local_deep_research.research_library.downloaders.extraction.trafilatura_extractor import (
            TrafilaturaExtractor,
        )

        mock_traf = MagicMock()
        mock_traf.extract.return_value = "   \n\t  "
        monkeypatch.setitem(__import__("sys").modules, "trafilatura", mock_traf)
        ext = TrafilaturaExtractor()
        result = ext.extract("<html><body>content</body></html>")
        assert result is None

    def test_custom_options(self):
        from local_deep_research.research_library.downloaders.extraction.trafilatura_extractor import (
            TrafilaturaExtractor,
        )

        ext = TrafilaturaExtractor(
            output_format="text",
            include_tables=False,
            include_links=True,
            include_comments=True,
            include_formatting=False,
        )
        assert ext.output_format == "text"
        assert ext.include_tables is False
        assert ext.include_links is True


# ---------------------------------------------------------------------------
# ReadabilityExtractor
# ---------------------------------------------------------------------------


class TestReadabilityExtractor:
    def test_empty_html(self):
        from local_deep_research.research_library.downloaders.extraction.readability_extractor import (
            ReadabilityExtractor,
        )

        ext = ReadabilityExtractor()
        assert ext.extract("") is None
        assert ext.extract("  ") is None
        assert ext.extract(None) is None

    def test_successful_extraction(self, monkeypatch):
        from local_deep_research.research_library.downloaders.extraction.readability_extractor import (
            ReadabilityExtractor,
        )

        mock_module = MagicMock()
        mock_module.simple_json_from_html_string.return_value = {
            "content": "<div><p>Main article content</p></div>"
        }
        monkeypatch.setitem(
            __import__("sys").modules, "readabilipy", mock_module
        )
        ext = ReadabilityExtractor()
        result = ext.extract("<html><body>content</body></html>")
        assert result == "<div><p>Main article content</p></div>"

    def test_import_error(self, monkeypatch):
        from local_deep_research.research_library.downloaders.extraction.readability_extractor import (
            ReadabilityExtractor,
        )
        import sys

        monkeypatch.delitem(sys.modules, "readabilipy", raising=False)

        original_import = __import__

        def fake_import(name, *args, **kwargs):
            if name == "readabilipy":
                raise ImportError("no readabilipy")
            return original_import(name, *args, **kwargs)

        monkeypatch.setattr("builtins.__import__", fake_import)
        ext = ReadabilityExtractor()
        result = ext.extract("<html><body>content</body></html>")
        assert result is None

    def test_extraction_exception(self, monkeypatch):
        from local_deep_research.research_library.downloaders.extraction.readability_extractor import (
            ReadabilityExtractor,
        )

        mock_module = MagicMock()
        mock_module.simple_json_from_html_string.side_effect = RuntimeError(
            "parse error"
        )
        monkeypatch.setitem(
            __import__("sys").modules, "readabilipy", mock_module
        )
        ext = ReadabilityExtractor()
        result = ext.extract("<html><body>content</body></html>")
        assert result is None

    def test_empty_article_result(self, monkeypatch):
        from local_deep_research.research_library.downloaders.extraction.readability_extractor import (
            ReadabilityExtractor,
        )

        mock_module = MagicMock()
        mock_module.simple_json_from_html_string.return_value = None
        monkeypatch.setitem(
            __import__("sys").modules, "readabilipy", mock_module
        )
        ext = ReadabilityExtractor()
        assert ext.extract("<html><body>content</body></html>") is None

    def test_content_key_missing(self, monkeypatch):
        from local_deep_research.research_library.downloaders.extraction.readability_extractor import (
            ReadabilityExtractor,
        )

        mock_module = MagicMock()
        mock_module.simple_json_from_html_string.return_value = {
            "title": "Test"
        }
        monkeypatch.setitem(
            __import__("sys").modules, "readabilipy", mock_module
        )
        ext = ReadabilityExtractor()
        assert ext.extract("<html><body>content</body></html>") is None

    def test_whitespace_content(self, monkeypatch):
        from local_deep_research.research_library.downloaders.extraction.readability_extractor import (
            ReadabilityExtractor,
        )

        mock_module = MagicMock()
        mock_module.simple_json_from_html_string.return_value = {
            "content": "   "
        }
        monkeypatch.setitem(
            __import__("sys").modules, "readabilipy", mock_module
        )
        ext = ReadabilityExtractor()
        assert ext.extract("<html><body>content</body></html>") is None


# ---------------------------------------------------------------------------
# JustextExtractor
# ---------------------------------------------------------------------------


class TestJustextExtractor:
    def test_empty_html(self):
        from local_deep_research.research_library.downloaders.extraction.justext_extractor import (
            JustextExtractor,
        )

        ext = JustextExtractor()
        assert ext.extract("") is None
        assert ext.extract("  ") is None
        assert ext.extract(None) is None

    def test_successful_extraction(self, monkeypatch):
        from local_deep_research.research_library.downloaders.extraction.justext_extractor import (
            JustextExtractor,
        )

        mock_para1 = Mock(
            is_boilerplate=False,
            text="Main content paragraph",
            is_heading=False,
        )
        mock_para2 = Mock(
            is_boilerplate=True, text="Cookie notice", is_heading=False
        )
        mock_para3 = Mock(
            is_boilerplate=False, text="More content", is_heading=False
        )

        mock_jt = MagicMock()
        mock_jt.get_stoplist.return_value = ["the", "a"]
        mock_jt.justext.return_value = [mock_para1, mock_para2, mock_para3]
        monkeypatch.setitem(__import__("sys").modules, "justext", mock_jt)

        ext = JustextExtractor()
        result = ext.extract("<html><body>content</body></html>")
        assert "Main content paragraph" in result
        assert "More content" in result
        assert "Cookie notice" not in result

    def test_heading_formatting(self, monkeypatch):
        from local_deep_research.research_library.downloaders.extraction.justext_extractor import (
            JustextExtractor,
        )

        mock_heading = Mock(
            is_boilerplate=False, text="Section Title", is_heading=True
        )
        mock_body = Mock(
            is_boilerplate=False, text="Body text", is_heading=False
        )

        mock_jt = MagicMock()
        mock_jt.get_stoplist.return_value = []
        mock_jt.justext.return_value = [mock_heading, mock_body]
        monkeypatch.setitem(__import__("sys").modules, "justext", mock_jt)

        ext = JustextExtractor()
        result = ext.extract("<html><body>content</body></html>")
        assert "## Section Title" in result

    def test_language_fallback(self, monkeypatch):
        from local_deep_research.research_library.downloaders.extraction.justext_extractor import (
            JustextExtractor,
        )

        mock_jt = MagicMock()
        call_args = []

        def mock_get_stoplist(lang):
            call_args.append(lang)
            if lang == "Klingon":
                raise ValueError(f"Unknown language: {lang}")
            return ["the", "a"]

        mock_jt.get_stoplist = mock_get_stoplist
        mock_jt.justext.return_value = []
        monkeypatch.setitem(__import__("sys").modules, "justext", mock_jt)

        ext = JustextExtractor(language="Klingon")
        ext.extract("<html><body>content</body></html>")
        assert "Klingon" in call_args
        assert "English" in call_args

    def test_import_error(self, monkeypatch):
        from local_deep_research.research_library.downloaders.extraction.justext_extractor import (
            JustextExtractor,
        )
        import sys

        monkeypatch.delitem(sys.modules, "justext", raising=False)

        original_import = __import__

        def fake_import(name, *args, **kwargs):
            if name == "justext":
                raise ImportError("no justext")
            return original_import(name, *args, **kwargs)

        monkeypatch.setattr("builtins.__import__", fake_import)
        ext = JustextExtractor()
        result = ext.extract("<html><body>content</body></html>")
        assert result is None

    def test_justext_exception(self, monkeypatch):
        from local_deep_research.research_library.downloaders.extraction.justext_extractor import (
            JustextExtractor,
        )

        mock_jt = MagicMock()
        mock_jt.get_stoplist.return_value = []
        mock_jt.justext.side_effect = RuntimeError("parse error")
        monkeypatch.setitem(__import__("sys").modules, "justext", mock_jt)

        ext = JustextExtractor()
        result = ext.extract("<html><body>content</body></html>")
        assert result is None

    def test_all_boilerplate(self, monkeypatch):
        from local_deep_research.research_library.downloaders.extraction.justext_extractor import (
            JustextExtractor,
        )

        mock_para = Mock(
            is_boilerplate=True, text="All boilerplate", is_heading=False
        )

        mock_jt = MagicMock()
        mock_jt.get_stoplist.return_value = []
        mock_jt.justext.return_value = [mock_para]
        monkeypatch.setitem(__import__("sys").modules, "justext", mock_jt)

        ext = JustextExtractor()
        result = ext.extract("<html><body>content</body></html>")
        assert result is None

    def test_empty_paragraph_text_skipped(self, monkeypatch):
        from local_deep_research.research_library.downloaders.extraction.justext_extractor import (
            JustextExtractor,
        )

        mock_para = Mock(is_boilerplate=False, text="   ", is_heading=False)

        mock_jt = MagicMock()
        mock_jt.get_stoplist.return_value = []
        mock_jt.justext.return_value = [mock_para]
        monkeypatch.setitem(__import__("sys").modules, "justext", mock_jt)

        ext = JustextExtractor()
        result = ext.extract("<html><body>content</body></html>")
        assert result is None


# ---------------------------------------------------------------------------
# NewspaperExtractor
# ---------------------------------------------------------------------------


class TestNewspaperExtractor:
    def test_empty_html(self):
        from local_deep_research.research_library.downloaders.extraction.newspaper_extractor import (
            NewspaperExtractor,
        )

        ext = NewspaperExtractor()
        assert ext.extract("") is None
        assert ext.extract("  ") is None
        assert ext.extract(None) is None

    def test_successful_extraction(self, monkeypatch):
        from local_deep_research.research_library.downloaders.extraction.newspaper_extractor import (
            NewspaperExtractor,
        )

        mock_article = MagicMock()
        mock_article.text = "Extracted newspaper article content"
        mock_config = MagicMock()

        mock_newspaper = MagicMock()
        mock_newspaper.Article.return_value = mock_article
        mock_newspaper.Config.return_value = mock_config
        monkeypatch.setitem(
            __import__("sys").modules, "newspaper", mock_newspaper
        )

        ext = NewspaperExtractor()
        result = ext.extract(
            "<html><body><article>news</article></body></html>",
            url="https://news.com/story",
        )
        assert result == "Extracted newspaper article content"
        mock_article.download.assert_called_once()
        mock_article.parse.assert_called_once()

    def test_import_error(self, monkeypatch):
        from local_deep_research.research_library.downloaders.extraction.newspaper_extractor import (
            NewspaperExtractor,
        )
        import sys

        monkeypatch.delitem(sys.modules, "newspaper", raising=False)

        original_import = __import__

        def fake_import(name, *args, **kwargs):
            if name == "newspaper":
                raise ImportError("no newspaper4k")
            return original_import(name, *args, **kwargs)

        monkeypatch.setattr("builtins.__import__", fake_import)
        ext = NewspaperExtractor()
        result = ext.extract("<html><body>content</body></html>")
        assert result is None

    def test_extraction_exception(self, monkeypatch):
        from local_deep_research.research_library.downloaders.extraction.newspaper_extractor import (
            NewspaperExtractor,
        )

        mock_newspaper = MagicMock()
        mock_newspaper.Article.side_effect = RuntimeError("parse error")
        mock_newspaper.Config.return_value = MagicMock()
        monkeypatch.setitem(
            __import__("sys").modules, "newspaper", mock_newspaper
        )

        ext = NewspaperExtractor()
        result = ext.extract("<html><body>content</body></html>")
        assert result is None

    def test_empty_text_result(self, monkeypatch):
        from local_deep_research.research_library.downloaders.extraction.newspaper_extractor import (
            NewspaperExtractor,
        )

        mock_article = MagicMock()
        mock_article.text = ""
        mock_newspaper = MagicMock()
        mock_newspaper.Article.return_value = mock_article
        mock_newspaper.Config.return_value = MagicMock()
        monkeypatch.setitem(
            __import__("sys").modules, "newspaper", mock_newspaper
        )

        ext = NewspaperExtractor()
        result = ext.extract("<html><body>content</body></html>")
        assert result is None

    def test_whitespace_text_result(self, monkeypatch):
        from local_deep_research.research_library.downloaders.extraction.newspaper_extractor import (
            NewspaperExtractor,
        )

        mock_article = MagicMock()
        mock_article.text = "   \n  "
        mock_newspaper = MagicMock()
        mock_newspaper.Article.return_value = mock_article
        mock_newspaper.Config.return_value = MagicMock()
        monkeypatch.setitem(
            __import__("sys").modules, "newspaper", mock_newspaper
        )

        ext = NewspaperExtractor()
        result = ext.extract("<html><body>content</body></html>")
        assert result is None

    def test_default_url_when_none_provided(self, monkeypatch):
        from local_deep_research.research_library.downloaders.extraction.newspaper_extractor import (
            NewspaperExtractor,
        )

        mock_article = MagicMock()
        mock_article.text = "Content"
        mock_newspaper = MagicMock()
        mock_newspaper.Article.return_value = mock_article
        mock_newspaper.Config.return_value = MagicMock()
        monkeypatch.setitem(
            __import__("sys").modules, "newspaper", mock_newspaper
        )

        ext = NewspaperExtractor()
        ext.extract("<html><body>content</body></html>")
        # Should use "https://example.com" as default URL
        mock_newspaper.Article.assert_called_once()
        call_args = mock_newspaper.Article.call_args
        assert call_args[0][0] == "https://example.com"

    def test_ssrf_protection_fetch_images_disabled(self, monkeypatch):
        from local_deep_research.research_library.downloaders.extraction.newspaper_extractor import (
            NewspaperExtractor,
        )

        mock_article = MagicMock()
        mock_article.text = "Content"
        mock_config = MagicMock()
        mock_newspaper = MagicMock()
        mock_newspaper.Article.return_value = mock_article
        mock_newspaper.Config.return_value = mock_config
        monkeypatch.setitem(
            __import__("sys").modules, "newspaper", mock_newspaper
        )

        ext = NewspaperExtractor()
        ext.extract("<html><body>content</body></html>", url="https://test.com")
        assert mock_config.fetch_images is False
