"""
Tests for uncovered code paths in HTMLDownloader.

Targets:
- can_handle: non-http schemes, exception handling
- download: PDF content type rejection, extraction failure
- download_with_result: all branches (PDF, empty content, extraction failure, exception)
- _fetch_html: non-200, non-HTML content type, exception
- _extract_content: article/main/div/body fallback chain, tag formatting (h/li/blockquote/pre)
- _format_extracted_content: all field formatting
- get_metadata: success, failure, missing fields
"""

from unittest.mock import Mock, patch

import pytest

from local_deep_research.research_library.downloaders.html import (
    HTMLDownloader,
)
from local_deep_research.research_library.downloaders.base import (
    ContentType,
)


@pytest.fixture
def downloader():
    dl = HTMLDownloader(timeout=30)
    dl.rate_tracker = Mock()
    dl.rate_tracker.apply_rate_limit.return_value = 0
    dl.rate_tracker.record_outcome = Mock()
    return dl


# ---------------------------------------------------------------------------
# can_handle
# ---------------------------------------------------------------------------


class TestCanHandle:
    def test_http_url(self, downloader):
        assert downloader.can_handle("http://example.com") is True

    def test_https_url(self, downloader):
        assert downloader.can_handle("https://example.com/page") is True

    def test_ftp_url(self, downloader):
        assert downloader.can_handle("ftp://files.example.com/data") is False

    def test_invalid_url(self, downloader):
        # urlparse doesn't usually raise, but we test the except branch
        assert downloader.can_handle("") is False


# ---------------------------------------------------------------------------
# download
# ---------------------------------------------------------------------------


class TestDownload:
    def test_pdf_content_type_rejected(self, downloader):
        """PDF content type returns None."""
        result = downloader.download(
            "https://example.com/paper", ContentType.PDF
        )
        assert result is None

    def test_success_returns_bytes(self, downloader):
        """Successful download returns UTF-8 encoded text."""
        html = """
        <html><head><title>Test</title></head>
        <body><article><p>This is a substantial paragraph with enough text to pass the length filter.</p></article></body>
        </html>
        """
        with patch.object(downloader, "_fetch_html", return_value=html):
            result = downloader.download("https://example.com/page")

        assert result is not None
        assert isinstance(result, bytes)
        assert b"substantial paragraph" in result

    def test_fetch_failure_returns_none(self, downloader):
        """Returns None when fetch fails."""
        with patch.object(downloader, "_fetch_html", return_value=None):
            result = downloader.download("https://example.com/page")
        assert result is None

    def test_extraction_failure_returns_none(self, downloader):
        """Returns None when content extraction fails."""
        with patch.object(
            downloader, "_fetch_html", return_value="<html></html>"
        ):
            with patch.object(
                downloader, "_extract_content", return_value=None
            ):
                result = downloader.download("https://example.com/page")
        assert result is None

    def test_exception_returns_none(self, downloader):
        """Returns None on unexpected exception."""
        with patch.object(
            downloader, "_fetch_html", side_effect=RuntimeError("fail")
        ):
            result = downloader.download("https://example.com/page")
        assert result is None


# ---------------------------------------------------------------------------
# download_with_result
# ---------------------------------------------------------------------------


class TestDownloadWithResult:
    def test_pdf_content_type(self, downloader):
        """PDF content type returns skip reason."""
        result = downloader.download_with_result(
            "https://example.com", ContentType.PDF
        )
        assert result.is_success is False
        assert "PDF" in result.skip_reason

    def test_fetch_failure(self, downloader):
        """Fetch failure returns skip reason."""
        with patch.object(downloader, "_fetch_html", return_value=None):
            result = downloader.download_with_result("https://example.com")
        assert result.is_success is False
        assert "Failed to fetch" in result.skip_reason

    def test_extraction_failure(self, downloader):
        """Extraction failure returns skip reason."""
        with patch.object(
            downloader, "_fetch_html", return_value="<html></html>"
        ):
            with patch.object(
                downloader, "_extract_content", return_value=None
            ):
                result = downloader.download_with_result("https://example.com")
        assert result.is_success is False
        assert "extract" in result.skip_reason.lower()

    def test_empty_content(self, downloader):
        """Empty extracted content returns skip reason."""
        with patch.object(
            downloader, "_fetch_html", return_value="<html></html>"
        ):
            with patch.object(
                downloader,
                "_extract_content",
                return_value={"title": "T", "content": "ok", "url": "u"},
            ):
                with patch.object(
                    downloader, "_format_extracted_content", return_value="   "
                ):
                    result = downloader.download_with_result(
                        "https://example.com"
                    )
        assert result.is_success is False
        assert "empty" in result.skip_reason.lower()

    def test_success(self, downloader):
        """Successful extraction returns content."""
        with patch.object(
            downloader, "_fetch_html", return_value="<html></html>"
        ):
            with patch.object(
                downloader,
                "_extract_content",
                return_value={
                    "title": "T",
                    "content": "Real content",
                    "url": "https://example.com",
                },
            ):
                with patch.object(
                    downloader,
                    "_format_extracted_content",
                    return_value="# T\n\nReal content",
                ):
                    result = downloader.download_with_result(
                        "https://example.com"
                    )
        assert result.is_success is True
        assert b"Real content" in result.content

    def test_exception(self, downloader):
        """Exception returns error skip reason."""
        with patch.object(
            downloader, "_fetch_html", side_effect=ValueError("bad url")
        ):
            result = downloader.download_with_result("https://example.com")
        assert result.is_success is False
        assert "Error" in result.skip_reason

    def test_exception_skip_reason_scrubs_credentials(self, downloader):
        """The exception path's skip_reason propagates to the browser via the
        download SSE stream, so a credential echoed in the error (e.g. a fetch
        URL with an api_key) must be redacted, not leaked. Regression guard
        for the download-service credential-leak fix."""
        secret = "supersecret1234567890"
        boom = ValueError(
            f"fetch failed for https://api.example.com/doc?api_key={secret}"
        )
        with patch.object(downloader, "_fetch_html", side_effect=boom):
            result = downloader.download_with_result(
                "https://api.example.com/doc"
            )
        assert result.is_success is False
        assert secret not in result.skip_reason
        # Still an error message, just with the credential masked.
        assert result.skip_reason.startswith("Error:")


# ---------------------------------------------------------------------------
# _fetch_html
# ---------------------------------------------------------------------------


class TestFetchHtml:
    def test_non_200_returns_none(self, downloader):
        """Non-200 status returns None and records failure."""
        mock_resp = Mock()
        mock_resp.status_code = 403
        downloader.session = Mock()
        downloader.session.get.return_value = mock_resp

        result = downloader._fetch_html("https://example.com")

        assert result is None
        downloader.rate_tracker.record_outcome.assert_called()

    def test_non_html_content_type(self, downloader):
        """Non-HTML content type returns None."""
        mock_resp = Mock()
        mock_resp.status_code = 200
        mock_resp.headers = {"content-type": "application/pdf"}
        downloader.session = Mock()
        downloader.session.get.return_value = mock_resp

        result = downloader._fetch_html("https://example.com")
        assert result is None

    def test_success_html(self, downloader):
        """200 with HTML content type returns text."""
        mock_resp = Mock()
        mock_resp.status_code = 200
        mock_resp.headers = {"content-type": "text/html; charset=utf-8"}
        mock_resp.text = "<html><body>Hello</body></html>"
        downloader.session = Mock()
        downloader.session.get.return_value = mock_resp

        result = downloader._fetch_html("https://example.com")
        assert result == "<html><body>Hello</body></html>"

    def test_xhtml_content_type(self, downloader):
        """application/xhtml content type is accepted."""
        mock_resp = Mock()
        mock_resp.status_code = 200
        mock_resp.headers = {"content-type": "application/xhtml+xml"}
        mock_resp.text = "<html>XHTML</html>"
        downloader.session = Mock()
        downloader.session.get.return_value = mock_resp

        result = downloader._fetch_html("https://example.com")
        assert result == "<html>XHTML</html>"

    def test_exception_returns_none(self, downloader):
        """Network exception returns None and records failure."""
        downloader.session = Mock()
        downloader.session.get.side_effect = ConnectionError("fail")

        result = downloader._fetch_html("https://example.com")

        assert result is None
        downloader.rate_tracker.record_outcome.assert_called()


# ---------------------------------------------------------------------------
# _extract_content - tag formatting
# ---------------------------------------------------------------------------


class TestExtractContent:
    def test_article_tag_preferred(self, downloader):
        """Article tag content is extracted via justext."""
        html = """
        <html><body>
        <div class="sidebar">Sidebar content here is long enough</div>
        <article><p>Article content that is substantial and long enough to pass filter easily.</p></article>
        </body></html>
        """
        result = downloader._extract_content(html, "https://example.com")
        assert result is not None
        assert "Article content" in result["content"]

    def test_main_tag_fallback(self, downloader):
        """Falls back to main tag when no article."""
        html = """
        <html><body>
        <main><p>Main content that has enough text to be considered real content in the extraction.</p></main>
        </body></html>
        """
        result = downloader._extract_content(html, "https://example.com")
        assert result is not None
        assert "Main content" in result["content"]

    def test_body_fallback(self, downloader):
        """Falls back to body when no article/main/content divs."""
        html = """
        <html><body>
        <p>Body paragraph that has enough text content to pass the minimum length filters.</p>
        </body></html>
        """
        result = downloader._extract_content(html, "https://example.com")
        assert result is not None
        assert "Body paragraph" in result["content"]

    def test_heading_formatting(self, downloader):
        """Headings are preserved in extracted content."""
        html = """
        <html><body><article>
        <h2>Important Section Title Here</h2>
        <p>Paragraph content that is substantial and long enough to pass the minimum length filter.</p>
        </article></body></html>
        """
        result = downloader._extract_content(html, "https://example.com")
        assert result is not None
        assert "Important Section Title Here" in result["content"]

    def test_list_item_formatting(self, downloader):
        """List items are extracted as plain text by justext."""
        html = """
        <html><body><article>
        <p>Introduction paragraph with enough text content to pass the filter requirements.</p>
        <li>First list item that is long enough to not be filtered out by length</li>
        </article></body></html>
        """
        result = downloader._extract_content(html, "https://example.com")
        if result:
            assert "First list item" in result["content"]

    def test_og_title_preferred(self, downloader):
        """Open Graph title takes precedence over regular title."""
        html = """
        <html><head>
        <title>Regular Title</title>
        <meta property="og:title" content="OG Title">
        </head><body><article>
        <p>Content that is substantial enough to pass the minimum length requirements for extraction.</p>
        </article></body></html>
        """
        result = downloader._extract_content(html, "https://example.com")
        assert result is not None
        assert result["title"] == "OG Title"

    def test_meta_description_extracted(self, downloader):
        """Meta description is extracted."""
        html = """
        <html><head>
        <meta name="description" content="Page description here">
        </head><body><article>
        <p>Content that is substantial enough to pass the minimum length requirements for extraction.</p>
        </article></body></html>
        """
        result = downloader._extract_content(html, "https://example.com")
        assert result is not None
        assert result["description"] == "Page description here"

    def test_short_content_returns_none(self, downloader):
        """Content shorter than 50 chars returns None."""
        html = """
        <html><body><p>Short</p></body></html>
        """
        result = downloader._extract_content(html, "https://example.com")
        assert result is None


# ---------------------------------------------------------------------------
# _format_extracted_content
# ---------------------------------------------------------------------------


class TestFormatExtractedContent:
    def test_all_fields(self, downloader):
        """Formats all available fields."""
        extracted = {
            "title": "Test Title",
            "description": "A description",
            "url": "https://example.com",
            "content": "Main content here",
        }
        result = downloader._format_extracted_content(extracted)

        assert "# Test Title" in result
        assert "*A description*" in result
        assert "Source: https://example.com" in result
        assert "Main content here" in result

    def test_minimal_fields(self, downloader):
        """Formats with only content field."""
        extracted = {"content": "Just content"}
        result = downloader._format_extracted_content(extracted)

        assert result == "Just content"

    def test_no_content(self, downloader):
        """Handles missing content field."""
        extracted = {"title": "Only Title"}
        result = downloader._format_extracted_content(extracted)

        assert "# Only Title" in result


# ---------------------------------------------------------------------------
# get_metadata
# ---------------------------------------------------------------------------


class TestGetMetadata:
    def test_success(self, downloader):
        """Extracts metadata from HTML."""
        html = """
        <html><head>
        <title>Page Title</title>
        <meta name="description" content="Page desc">
        <meta name="author" content="John Doe">
        <meta property="article:published_time" content="2024-01-15">
        </head><body></body></html>
        """
        with patch.object(downloader, "_fetch_html", return_value=html):
            meta = downloader.get_metadata("https://example.com")

        assert meta["title"] == "Page Title"
        assert meta["description"] == "Page desc"
        assert meta["author"] == "John Doe"
        assert meta["published_date"] == "2024-01-15"

    def test_fetch_failure(self, downloader):
        """Returns empty dict when fetch fails."""
        with patch.object(downloader, "_fetch_html", return_value=None):
            meta = downloader.get_metadata("https://example.com")
        assert meta == {}

    def test_minimal_metadata(self, downloader):
        """Returns what's available when some meta tags are missing."""
        html = (
            "<html><head><title>Just Title</title></head><body></body></html>"
        )
        with patch.object(downloader, "_fetch_html", return_value=html):
            meta = downloader.get_metadata("https://example.com")

        assert meta["title"] == "Just Title"
        assert "author" not in meta
        assert "published_date" not in meta
