"""
Tests for CrawlerService content extraction and configuration.
"""

import asyncio
import sys
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch

import pytest

from app.services.crawler_service import CrawlerService

pytestmark = pytest.mark.asyncio


def _make_crawl_result(
    url: str = "https://example.com",
    title: str = "Test Page",
    raw_markdown: str = "raw content",
    fit_markdown: str | None = "fit content",
    html: str = "<html></html>",
    success: bool = True,
    error_message: str | None = None,
    status_code: int | None = 200,
):
    """Build a fake crawl4ai CrawlResult."""
    md = SimpleNamespace(raw_markdown=raw_markdown, fit_markdown=fit_markdown)
    metadata = {"title": title}
    return SimpleNamespace(
        url=url,
        markdown=md,
        metadata=metadata,
        html=html,
        success=success,
        error_message=error_message,
        status_code=status_code,
    )


async def _run_crawl(results):
    """Helper: run crawl_urls with mocked arun_many returning given results."""
    service = CrawlerService()
    service.initialized = True
    service._crawl_count = 0

    async def fake_arun_many(urls, config):
        async def gen():
            for r in results:
                yield r

        return gen()

    service._crawler = MagicMock()
    service._crawler.arun_many = fake_arun_many

    with patch(
        "app.services.crawler_service.CrawlerService._extract_structured_data_from_html",
        return_value={},
    ):
        return await service.crawl_urls(["https://example.com"])


class TestCrawlUrlsMarkdownSelection:
    """Verify that crawl_urls prefers fit_markdown and falls back to raw_markdown."""

    async def test_uses_fit_markdown_when_available(self):
        result = _make_crawl_result(fit_markdown="clean content", raw_markdown="noisy content")
        pages = await _run_crawl([result])

        assert len(pages) == 1
        assert pages[0]["content"] == "clean content"

    async def test_falls_back_to_raw_markdown_when_fit_is_none(self):
        result = _make_crawl_result(fit_markdown=None, raw_markdown="raw content")
        pages = await _run_crawl([result])

        assert len(pages) == 1
        assert pages[0]["content"] == "raw content"

    async def test_falls_back_to_raw_markdown_when_fit_is_empty(self):
        result = _make_crawl_result(fit_markdown="", raw_markdown="raw content")
        pages = await _run_crawl([result])

        assert len(pages) == 1
        assert pages[0]["content"] == "raw content"

    async def test_skips_failed_results(self):
        result = _make_crawl_result(success=False, error_message="404")
        pages = await _run_crawl([result])

        assert len(pages) == 0

    async def test_word_count_uses_selected_markdown(self):
        result = _make_crawl_result(fit_markdown="one two three")
        pages = await _run_crawl([result])

        assert pages[0]["word_count"] == 3

    async def test_successful_result_includes_status_code(self):
        result = _make_crawl_result(status_code=200)
        pages = await _run_crawl([result])

        assert pages[0]["status_code"] == 200


class TestCrawlUrlsStatusCodeFiltering:
    """Verify that 4xx/5xx status codes are returned with content=None
    and that 3xx redirects preserve content (Crawl4AI reports the first
    status in a redirect chain, but content comes from the final page)."""

    async def test_502_returns_with_none_content(self):
        result = _make_crawl_result(title="502 Bad Gateway", status_code=502)
        pages = await _run_crawl([result])

        assert len(pages) == 1
        assert pages[0]["content"] is None
        assert pages[0]["status_code"] == 502

    async def test_404_returns_with_none_content(self):
        result = _make_crawl_result(title="Not Found", status_code=404)
        pages = await _run_crawl([result])

        assert len(pages) == 1
        assert pages[0]["content"] is None
        assert pages[0]["status_code"] == 404

    async def test_500_returns_with_none_content(self):
        result = _make_crawl_result(status_code=500)
        pages = await _run_crawl([result])

        assert len(pages) == 1
        assert pages[0]["content"] is None
        assert pages[0]["status_code"] == 500

    async def test_allows_200_ok(self):
        result = _make_crawl_result(status_code=200)
        pages = await _run_crawl([result])

        assert len(pages) == 1
        assert pages[0]["content"] is not None
        assert pages[0]["status_code"] == 200

    async def test_allows_none_status_code(self):
        result = _make_crawl_result(status_code=None)
        pages = await _run_crawl([result])

        assert len(pages) == 1
        assert pages[0]["content"] is not None

    async def test_301_redirect_preserves_content(self):
        result = _make_crawl_result(status_code=301)
        pages = await _run_crawl([result])

        assert len(pages) == 1
        assert pages[0]["content"] is not None
        assert pages[0]["status_code"] == 301

    async def test_302_redirect_preserves_content(self):
        result = _make_crawl_result(status_code=302)
        pages = await _run_crawl([result])

        assert len(pages) == 1
        assert pages[0]["content"] is not None
        assert pages[0]["status_code"] == 302

    async def test_mixed_batch_preserves_both(self):
        ok = _make_crawl_result(url="https://example.com/ok", status_code=200)
        bad = _make_crawl_result(url="https://example.com/bad", status_code=503)
        pages = await _run_crawl([ok, bad])

        assert len(pages) == 2
        by_url = {p["url"]: p for p in pages}
        assert by_url["https://example.com/ok"]["content"] is not None
        assert by_url["https://example.com/bad"]["content"] is None
        assert by_url["https://example.com/bad"]["status_code"] == 503

    async def test_all_error_batch(self):
        r1 = _make_crawl_result(url="https://example.com/a", status_code=502)
        r2 = _make_crawl_result(url="https://example.com/b", status_code=404)
        pages = await _run_crawl([r1, r2])

        assert len(pages) == 2
        assert all(p["content"] is None for p in pages)


class TestCrawlerRunConfigSetup:
    """Verify that CrawlerRunConfig is created with the correct filtering options."""

    async def test_config_has_excluded_tags(self):
        """Ensure excluded_tags and content filter are set in the config."""
        captured_config = {}

        async def fake_arun_many(urls, config):
            captured_config["config"] = config

            async def gen():
                return
                yield

            return gen()

        service = CrawlerService()
        service.initialized = True
        service._crawl_count = 0
        service._crawler = MagicMock()
        service._crawler.arun_many = fake_arun_many

        await service.crawl_urls(["https://example.com"])

        config = captured_config["config"]
        assert "nav" in config.excluded_tags
        assert "footer" in config.excluded_tags
        assert "header" in config.excluded_tags
        assert "aside" in config.excluded_tags
        assert "select" in config.excluded_tags
        assert "option" in config.excluded_tags
        assert config.exclude_external_links is True
        assert config.exclude_social_media_links is True


class TestBrowserConfig:
    """Verify that BrowserConfig is created with container-optimized Chrome flags."""

    async def test_browser_config_has_container_flags(self):
        """Chrome should launch with flags that prevent zombie processes in Docker."""
        captured_config = {}

        with (
            patch("crawl4ai.BrowserConfig") as mock_bc,
            patch("crawl4ai.AsyncWebCrawler") as mock_crawler,
        ):
            mock_bc.side_effect = lambda **kwargs: captured_config.update(kwargs) or MagicMock()
            mock_crawler.return_value.__aenter__ = AsyncMock()

            service = CrawlerService()
            await service.initialize()

        extra_args = captured_config["extra_args"]
        assert "--disable-dev-shm-usage" in extra_args
        assert "--disable-gpu" in extra_args
        assert "--no-zygote" in extra_args
        assert "--disable-breakpad" in extra_args
        assert "--disable-crash-reporter" in extra_args


class TestExtractStructuredData:
    """Verify structured data extraction from HTML."""

    def test_extracts_json_ld(self):
        service = CrawlerService()
        html = """
        <html>
        <head>
            <script type="application/ld+json">{"@type": "Product", "name": "Test"}</script>
        </head>
        <body></body>
        </html>
        """
        data = service._extract_structured_data_from_html(html)
        assert "json_ld" in data
        assert data["json_ld"][0]["@type"] == "Product"

    def test_extracts_opengraph(self):
        service = CrawlerService()
        html = """
        <html>
        <head>
            <meta property="og:title" content="Test Product" />
            <meta property="og:price:amount" content="49.00" />
        </head>
        <body></body>
        </html>
        """
        data = service._extract_structured_data_from_html(html)
        assert "opengraph" in data
        assert data["opengraph"]["title"] == "Test Product"

    def test_extracts_meta_tags(self):
        service = CrawlerService()
        html = """
        <html>
        <head>
            <meta name="description" content="A test page" />
            <meta name="keywords" content="test, page" />
        </head>
        <body></body>
        </html>
        """
        data = service._extract_structured_data_from_html(html)
        assert "meta" in data
        assert data["meta"]["description"] == "A test page"
        assert data["meta"]["keywords"] == "test, page"

    def test_returns_empty_dict_for_no_structured_data(self):
        service = CrawlerService()
        html = "<html><head></head><body><p>Hello</p></body></html>"
        data = service._extract_structured_data_from_html(html)
        assert data == {}

    def test_handles_malformed_json_ld(self):
        service = CrawlerService()
        html = """
        <html>
        <head>
            <script type="application/ld+json">not valid json</script>
        </head>
        <body></body>
        </html>
        """
        data = service._extract_structured_data_from_html(html)
        assert "json_ld" not in data


def _make_bfs_result(url: str, success: bool = True):
    """Build a fake CrawlResult for BFS discovery tests."""
    return SimpleNamespace(url=url, success=success)


def _make_sitemap_page(url: str):
    """Build a fake sitemap page object."""
    return SimpleNamespace(url=url)


def _mock_sitemap(mock_tree):
    """Mock the usp.tree module since it's not installed in the test environment."""
    mock_usp_tree = MagicMock()
    mock_usp_tree.sitemap_tree_for_homepage = MagicMock(return_value=mock_tree)
    return patch.dict(sys.modules, {"usp": MagicMock(), "usp.tree": mock_usp_tree})


class TestDiscoverUrlsBfsFallback:
    """Verify BFS fallback triggers when sitemap finds fewer than 10 URLs."""

    async def test_no_bfs_when_sitemap_returns_enough_urls(self):
        """Sitemap returns ≥10 URLs → BFS fallback should not be called."""
        service = CrawlerService()
        service.initialized = True

        pages = [_make_sitemap_page(f"https://example.com/page{i}") for i in range(10)]
        mock_tree = MagicMock()
        mock_tree.all_pages.return_value = pages

        with patch(
            "app.services.crawler_service.CrawlerService._discover_urls_bfs",
            new_callable=AsyncMock,
        ) as mock_bfs:
            with _mock_sitemap(mock_tree):
                urls = await service.discover_urls(domain="example.com")

            mock_bfs.assert_not_called()
            assert len(urls) == 10

    async def test_bfs_triggered_when_sitemap_returns_zero(self):
        """Sitemap returns 0 URLs → BFS fallback should run."""
        service = CrawlerService()
        service.initialized = True
        service._crawl_count = 0
        service._crawler = MagicMock()

        bfs_results = [
            _make_bfs_result("https://example.com/"),
            _make_bfs_result("https://example.com/about"),
            _make_bfs_result("https://example.com/contact"),
        ]
        service._crawler.arun = AsyncMock(return_value=bfs_results)

        mock_tree = MagicMock()
        mock_tree.all_pages.return_value = []

        with _mock_sitemap(mock_tree):
            urls = await service.discover_urls(domain="example.com")

        assert len(urls) == 3
        assert {u["url"] for u in urls} == {
            "https://example.com/",
            "https://example.com/about",
            "https://example.com/contact",
        }

    async def test_bfs_results_merged_with_sitemap(self):
        """BFS results are merged with sitemap results, deduplicated."""
        service = CrawlerService()
        service.initialized = True
        service._crawl_count = 0
        service._crawler = MagicMock()

        # Sitemap finds 2 URLs (below threshold)
        pages = [
            _make_sitemap_page("https://example.com/"),
            _make_sitemap_page("https://example.com/about"),
        ]
        mock_tree = MagicMock()
        mock_tree.all_pages.return_value = pages

        # BFS finds 3 URLs, one overlapping with sitemap
        bfs_results = [
            _make_bfs_result("https://example.com/"),
            _make_bfs_result("https://example.com/about"),
            _make_bfs_result("https://example.com/products"),
        ]
        service._crawler.arun = AsyncMock(return_value=bfs_results)

        with _mock_sitemap(mock_tree):
            urls = await service.discover_urls(domain="example.com")

        assert len(urls) == 3
        assert {u["url"] for u in urls} == {
            "https://example.com/",
            "https://example.com/about",
            "https://example.com/products",
        }

    async def test_bfs_applies_pattern_filter(self):
        """BFS fallback respects the fnmatch pattern filter."""
        service = CrawlerService()
        service.initialized = True
        service._crawl_count = 0
        service._crawler = MagicMock()

        bfs_results = [
            _make_bfs_result("https://example.com/docs/intro"),
            _make_bfs_result("https://example.com/blog/post1"),
            _make_bfs_result("https://example.com/docs/guide"),
        ]
        service._crawler.arun = AsyncMock(return_value=bfs_results)

        mock_tree = MagicMock()
        mock_tree.all_pages.return_value = []

        with _mock_sitemap(mock_tree):
            urls = await service.discover_urls(domain="example.com", pattern="*/docs/*")

        assert len(urls) == 2
        assert all("/docs/" in u["url"] for u in urls)

    async def test_bfs_handles_timeout(self):
        """BFS fallback returns empty list on timeout."""
        service = CrawlerService()
        service.initialized = True
        service._crawl_count = 0
        service._crawler = MagicMock()

        async def slow_arun(**kwargs):
            await asyncio.sleep(10)

        service._crawler.arun = slow_arun

        mock_tree = MagicMock()
        mock_tree.all_pages.return_value = []

        with _mock_sitemap(mock_tree):
            urls = await service.discover_urls(domain="example.com", timeout=0.1)

        assert urls == []

    async def test_bfs_skips_failed_results(self):
        """BFS fallback skips CrawlResults with success=False."""
        service = CrawlerService()
        service.initialized = True
        service._crawl_count = 0
        service._crawler = MagicMock()

        bfs_results = [
            _make_bfs_result("https://example.com/", success=True),
            _make_bfs_result("https://example.com/broken", success=False),
            _make_bfs_result("https://example.com/ok", success=True),
        ]
        service._crawler.arun = AsyncMock(return_value=bfs_results)

        mock_tree = MagicMock()
        mock_tree.all_pages.return_value = []

        with _mock_sitemap(mock_tree):
            urls = await service.discover_urls(domain="example.com")

        assert len(urls) == 2
        assert "https://example.com/broken" not in {u["url"] for u in urls}

    async def test_bfs_exception_preserves_sitemap_urls(self):
        """BFS failure should not discard already-collected sitemap URLs."""
        service = CrawlerService()
        service.initialized = True
        service._crawl_count = 0
        service._crawler = MagicMock()

        # BFS will raise a non-timeout exception
        service._crawler.arun = AsyncMock(side_effect=RuntimeError("browser crashed"))

        # Sitemap returns 3 URLs (below threshold, triggers BFS)
        pages = [
            _make_sitemap_page("https://example.com/"),
            _make_sitemap_page("https://example.com/about"),
            _make_sitemap_page("https://example.com/contact"),
        ]
        mock_tree = MagicMock()
        mock_tree.all_pages.return_value = pages

        with _mock_sitemap(mock_tree):
            urls = await service.discover_urls(domain="example.com")

        # Sitemap URLs must be preserved despite BFS failure
        assert len(urls) == 3
        assert {u["url"] for u in urls} == {
            "https://example.com/",
            "https://example.com/about",
            "https://example.com/contact",
        }
