"""Tests for HTMLDownloader pure-logic methods.

Covers: can_handle, _extract_content, _format_extracted_content.
Network/session methods are NOT tested here — only HTML parsing logic.
"""

import pytest

from local_deep_research.research_library.downloaders.html import HTMLDownloader


@pytest.fixture
def dl():
    """Create an HTMLDownloader for testing."""
    downloader = HTMLDownloader(timeout=30)
    yield downloader
    downloader.close()


def make_html(body, head=""):
    """Build a minimal HTML document from head and body fragments."""
    return f"<html><head>{head}</head><body>{body}</body></html>"


# Long paragraph that exceeds both the 10-char fragment threshold
# and contributes enough to pass the 50-char minimum content threshold.
LONG_PARA = "This is a sufficiently long paragraph for content extraction testing purposes."
ANOTHER_PARA = "Another paragraph with enough characters to pass all the length thresholds."


# ===================================================================
# can_handle
# ===================================================================


class TestHTMLDownloaderCanHandle:
    """Tests for HTMLDownloader.can_handle()."""

    def test_http_url(self, dl):
        """Accepts http:// URLs."""
        assert dl.can_handle("http://example.com") is True

    def test_https_url(self, dl):
        """Accepts https:// URLs."""
        assert dl.can_handle("https://example.com/page") is True

    def test_ftp_url(self, dl):
        """Rejects ftp:// URLs."""
        assert dl.can_handle("ftp://files.example.com/data") is False

    def test_empty_string(self, dl):
        """Rejects empty string."""
        assert dl.can_handle("") is False

    def test_plain_text(self, dl):
        """Rejects plain text that is not a URL."""
        assert dl.can_handle("not-a-url") is False


# ===================================================================
# _extract_content
# ===================================================================


class TestExtractContent:
    """Tests for HTMLDownloader._extract_content()."""

    # --- Title extraction ---

    def test_extracts_title_from_title_tag(self, dl):
        """Extracts title from <title> tag."""
        html = make_html(
            f"<p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p>",
            head="<title>My Page Title</title>",
        )
        result = dl._extract_content(html, "http://example.com")
        assert result["title"] == "My Page Title"

    def test_og_title_overrides_title_tag(self, dl):
        """Prefers og:title over <title> tag."""
        html = make_html(
            f"<p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p>",
            head='<title>HTML Title</title><meta property="og:title" content="OG Title">',
        )
        result = dl._extract_content(html, "http://example.com")
        assert result["title"] == "OG Title"

    def test_no_title_returns_none_title(self, dl):
        """Returns None title when no title tag present."""
        html = make_html(f"<p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p>")
        result = dl._extract_content(html, "http://example.com")
        assert result["title"] is None

    # --- Description extraction ---

    def test_extracts_meta_description(self, dl):
        """Extracts description from meta tag."""
        html = make_html(
            f"<p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p>",
            head='<meta name="description" content="A page description">',
        )
        result = dl._extract_content(html, "http://example.com")
        assert result["description"] == "A page description"

    def test_og_description_overrides_meta(self, dl):
        """Prefers og:description over meta description."""
        html = make_html(
            f"<p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p>",
            head=(
                '<meta name="description" content="Meta desc">'
                '<meta property="og:description" content="OG desc">'
            ),
        )
        result = dl._extract_content(html, "http://example.com")
        assert result["description"] == "OG desc"

    # --- Element removal ---

    def test_removes_script_and_style(self, dl):
        """Strips <script> and <style> elements."""
        html = make_html(
            f"<script>alert('x')</script><style>.x{{}}</style>"
            f"<p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p>"
        )
        result = dl._extract_content(html, "http://example.com")
        assert "alert" not in result["content"]

    def test_removes_nav_footer_aside(self, dl):
        """Readability/justext filters out <nav>, <footer>, and <aside> content."""
        html = make_html(
            f"<nav><a href='/'>Home</a><a href='/about'>About</a></nav>"
            f"<footer><p>Copyright 2024 terms privacy contact us</p></footer>"
            f"<aside><p>Sidebar widget with related links and ads</p></aside>"
            f"<article><p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p></article>"
        )
        result = dl._extract_content(html, "http://example.com")
        assert result is not None
        assert LONG_PARA in result["content"]
        # Verify boilerplate is actually removed, not just that content is present
        assert "Copyright 2024" not in result["content"]
        assert "Sidebar widget" not in result["content"]

    def test_removes_elements_matching_class_patterns(self, dl):
        """Extraction pipeline filters sidebar/cookie boilerplate."""
        html = make_html(
            f'<nav><a href="/">Home</a><a href="/about">About</a></nav>'
            f'<div class="sidebar-widget"><p>Sidebar stuff that appears in the sidebar of the page with some navigation links</p></div>'
            f'<div class="cookie-banner"><p>Accept cookies to continue browsing this website and agree to our terms</p></div>'
            f"<article><p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p></article>"
            f"<footer><p>Copyright 2024 All rights reserved terms privacy contact</p></footer>"
        )
        result = dl._extract_content(html, "http://example.com")
        assert result is not None
        assert LONG_PARA in result["content"]
        # Verify boilerplate is actually removed
        assert "Sidebar stuff" not in result["content"]
        assert "Accept cookies" not in result["content"]
        assert "Copyright 2024" not in result["content"]

    def test_removes_elements_matching_id_patterns(self, dl):
        """Extraction pipeline filters sidebar boilerplate matched by element IDs."""
        html = make_html(
            f'<nav><a href="/">Home</a><a href="/about">About</a></nav>'
            f'<div id="sidebar-widget"><p>Sidebar by ID that appears in the sidebar of the page</p></div>'
            f"<article><p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p></article>"
            f"<footer><p>Copyright 2024 All rights reserved terms privacy contact</p></footer>"
        )
        result = dl._extract_content(html, "http://example.com")
        assert result is not None
        assert LONG_PARA in result["content"]
        # Verify boilerplate is actually removed
        assert "Sidebar by ID" not in result["content"]
        assert "Copyright 2024" not in result["content"]

    # --- Content area detection ---

    def test_extracts_from_article_tag(self, dl):
        """Extracts content from <article> via justext."""
        html = make_html(
            f"<article><p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p></article>"
            f"<p>Outside article content that should not appear here ideally</p>"
        )
        result = dl._extract_content(html, "http://example.com")
        assert LONG_PARA in result["content"]

    def test_extracts_from_main_tag(self, dl):
        """Extracts content from <main> via justext."""
        html = make_html(
            f"<main><p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p></main>"
        )
        result = dl._extract_content(html, "http://example.com")
        assert LONG_PARA in result["content"]

    def test_extracts_from_content_div(self, dl):
        """Extracts content from div.content via justext."""
        html = make_html(
            f'<div class="content"><p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p></div>'
        )
        result = dl._extract_content(html, "http://example.com")
        assert LONG_PARA in result["content"]

    def test_extracts_from_body(self, dl):
        """Extracts content from <body> via justext."""
        html = make_html(f"<p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p>")
        result = dl._extract_content(html, "http://example.com")
        assert LONG_PARA in result["content"]

    # --- Text structure ---

    def test_headings_preserved(self, dl):
        """Headings are preserved in extraction output."""
        # Use enough content for trafilatura to treat this as a real page
        html = make_html(
            f"<article>"
            f"<h2>Important Section Heading</h2>"
            f"<p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p>"
            f"<p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p>"
            f"</article>"
        )
        result = dl._extract_content(html, "http://example.com")
        assert "Important Section Heading" in result["content"]

    def test_list_items_extracted_as_text(self, dl):
        """List items are extracted as plain text."""
        html = make_html(
            f"<article>"
            f"<p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p>"
            f"<ul><li>First list item text here that is long enough to be kept</li></ul>"
            f"<p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p>"
            f"</article>"
        )
        result = dl._extract_content(html, "http://example.com")
        assert "First list item" in result["content"]

    def test_blockquotes_extracted_as_text(self, dl):
        """Blockquotes are extracted as plain text (no angle bracket prefix)."""
        html = make_html(
            f"<blockquote>A notable quoted passage here</blockquote>"
            f"<p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p>"
        )
        result = dl._extract_content(html, "http://example.com")
        assert "A notable quoted passage here" in result["content"]

    def test_pre_blocks_extracted_as_text(self, dl):
        """Pre blocks are extracted as plain text (no backtick wrapping)."""
        html = make_html(
            f"<pre>def hello_world_function(): pass</pre>"
            f"<p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p>"
        )
        result = dl._extract_content(html, "http://example.com")
        assert "def hello_world_function(): pass" in result["content"]

    # --- Edge cases ---

    def test_returns_none_when_content_too_short(self, dl):
        """Returns None when extracted content is below minimum threshold."""
        html = make_html("<p>Tiny</p>")
        result = dl._extract_content(html, "http://example.com")
        assert result is None

    def test_returns_none_when_no_body(self, dl):
        """Returns None when HTML has no content."""
        result = dl._extract_content(
            "<html><head></head></html>", "http://example.com"
        )
        assert result is None

    def test_extracts_text_in_span_via_justext(self, dl):
        """justext can extract text from any container, including spans in divs."""
        long_text = "A" * 200
        html = make_html(f"<div><span>{long_text}</span></div>")
        result = dl._extract_content(html, "http://example.com")
        assert result is not None
        assert long_text in result["content"]

    def test_realistic_page_boilerplate_removal(self, dl):
        """Pipeline removes boilerplate from a realistic page with multiple noise sections."""
        article_text = (
            "Scientists at CERN have discovered a new particle that could "
            "revolutionize our understanding of quantum physics. The finding, "
            "published in Nature, describes a subatomic interaction that was "
            "previously thought to be impossible under the Standard Model."
        )
        html = make_html(
            '<header><div class="top-bar">'
            "<span>Subscribe to our newsletter for daily updates</span>"
            "</div></header>"
            '<nav class="main-nav">'
            '<a href="/">Home</a><a href="/science">Science</a>'
            '<a href="/tech">Tech</a><a href="/about">About Us</a>'
            "</nav>"
            f"<article><h1>New Particle Discovery</h1><p>{article_text}</p></article>"
            '<div class="cookie-consent">'
            "<p>We use cookies to improve your experience. Accept all cookies.</p>"
            "</div>"
            '<aside class="sidebar">'
            "<h3>Trending Articles</h3>"
            "<ul><li>Top 10 Physics Breakthroughs</li>"
            "<li>What is Dark Matter?</li></ul>"
            "</aside>"
            "<footer>"
            "<p>Copyright 2025 Science Daily. Privacy Policy. Terms of Service.</p>"
            "<p>Contact us at editor@sciencedaily.com</p>"
            "</footer>"
        )
        result = dl._extract_content(html, "http://example.com")
        assert result is not None
        content = result["content"]

        # Main article content must be present
        assert "CERN" in content
        assert "quantum physics" in content

        # Boilerplate must be absent
        assert "Subscribe to our newsletter" not in content
        assert "Accept all cookies" not in content
        assert "Privacy Policy" not in content
        assert "editor@sciencedaily.com" not in content
        assert "Trending Articles" not in content


# ===================================================================
# Extractor configuration
# ===================================================================


class TestExtractorConfigurations:
    """Tests for different extractor configurations."""

    def test_justext_only_extracts_content(self, dl):
        """justext-only mode extracts article content."""
        html = make_html(
            f"<article><p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p></article>"
        )
        result = dl._extract_content(html, "http://example.com")
        assert result is not None
        assert LONG_PARA in result["content"]

    def test_language_param_stored(self):
        """language parameter is stored on the downloader."""
        downloader = HTMLDownloader(timeout=30, language="German")
        assert downloader.language == "German"
        downloader.close()

    def test_extraction_with_article(self, dl):
        """Pipeline extracts content from article HTML."""
        html = make_html(
            f"<article><p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p></article>"
        )
        result = dl._extract_content(html, "http://example.com")
        assert result is not None
        assert LONG_PARA in result["content"]

    def test_extraction_with_minimal_html(self, dl):
        """Pipeline handles minimal HTML with just paragraphs."""
        html = make_html(f"<p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p>")
        result = dl._extract_content(html, "http://example.com")
        assert result is not None
        assert LONG_PARA in result["content"]


# ===================================================================
# _format_extracted_content
# ===================================================================


class TestFormatExtractedContent:
    """Tests for HTMLDownloader._format_extracted_content()."""

    def test_title_as_heading(self, dl):
        """Renders title as a markdown heading."""
        result = dl._format_extracted_content(
            {
                "title": "My Title",
                "description": None,
                "url": None,
                "content": "body",
            }
        )
        assert result.startswith("# My Title")

    def test_description_in_italics(self, dl):
        """Renders description in italics."""
        result = dl._format_extracted_content(
            {
                "title": None,
                "description": "A desc",
                "url": None,
                "content": "body",
            }
        )
        assert "*A desc*" in result

    def test_source_url_line(self, dl):
        """Includes source URL line when URL is present."""
        result = dl._format_extracted_content(
            {
                "title": None,
                "description": None,
                "url": "http://example.com",
                "content": "body",
            }
        )
        assert "Source: http://example.com" in result

    def test_content_included(self, dl):
        """Includes body content in output."""
        result = dl._format_extracted_content(
            {
                "title": None,
                "description": None,
                "url": None,
                "content": "Main body text",
            }
        )
        assert "Main body text" in result

    def test_missing_fields_omitted(self, dl):
        """Omits title, description, and source when they are None."""
        result = dl._format_extracted_content(
            {
                "title": None,
                "description": None,
                "url": None,
                "content": "Only content",
            }
        )
        assert "#" not in result
        assert "*" not in result
        assert "Source:" not in result
        assert "Only content" in result
