"""
HTML Downloader with JavaScript rendering support.

Uses Crawl4AI (default) or plain Playwright for JS-rendered pages.
Crawl4AI adds: robots.txt checking, shadow DOM flattening, iframe
inlining, smart scrolling for lazy-loaded content, and caching.
Falls back to plain Playwright if Crawl4AI is not installed.

No stealth/anti-detection features are used — the browser identifies
honestly via BROWSER_USER_AGENT and respects robots.txt.
"""

import asyncio
from typing import Optional
from urllib.parse import urlparse

from loguru import logger

from .html import HTMLDownloader
from ...constants import BROWSER_USER_AGENT


# Signals that a page is a JS-rendered SPA and needs browser rendering
SPA_SIGNALS = [
    'id="root"',
    'id="app"',
    'id="__next"',
    "__NEXT_DATA__",
    "data-reactroot",
    'ng-version="',
    "<noscript>You need to enable JavaScript",
    "<noscript>Please enable JavaScript",
    "window.__INITIAL_STATE__",
]


def _run_async(coro, timeout: float = None):
    """Run an async coroutine from synchronous code.

    Handles the case where an event loop is already running
    (e.g. inside Jupyter or an async framework) by creating
    a new thread with its own loop.

    Args:
        coro: The coroutine to run.
        timeout: Max seconds to wait for the result. Prevents
            indefinite hangs if the coroutine's internal timeout fails.
    """
    try:
        loop = asyncio.get_running_loop()
    except RuntimeError:
        loop = None

    if loop is None:
        return asyncio.run(coro)

    # Already inside an event loop — run in a new thread
    import concurrent.futures

    with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
        future = pool.submit(asyncio.run, coro)
        return future.result(timeout=timeout)


class PlaywrightHTMLDownloader(HTMLDownloader):
    """HTML downloader with JS rendering via Crawl4AI or Playwright.

    Default: Crawl4AI (robots.txt, shadow DOM, iframes, caching).
    Fallback: plain Playwright if Crawl4AI is not installed.

    No stealth or anti-detection features are used.
    """

    def __init__(
        self,
        timeout: int = 30,
        language: str = "English",
        wait_until: str = "networkidle",
        block_resources: bool = True,
        **kwargs,
    ):
        super().__init__(timeout=timeout, language=language)
        self.wait_until = wait_until
        self.block_resources = block_resources
        # Plain Playwright fallback state
        self._playwright = None
        self._browser = None

    def _fetch_html(self, url: str) -> Optional[str]:
        """Fetch HTML with JS rendering.

        Tries Crawl4AI first (with robots.txt, shadow DOM, iframes),
        falls back to plain Playwright.
        """
        # Try Crawl4AI first (richer features, robots.txt)
        html = self._fetch_with_crawl4ai(url)
        if html is not None:
            # Crawl4AI succeeded (non-empty) or intentionally blocked
            # by robots.txt (empty string). Either way, don't fall
            # through to Playwright.
            return html or None

        # Crawl4AI not installed or failed — fall back to Playwright
        return self._fetch_with_playwright(url)

    def _fetch_with_crawl4ai(self, url: str) -> Optional[str]:
        """Fetch HTML using Crawl4AI with ethical defaults."""
        domain = urlparse(url).netloc
        engine_type = f"crawl4ai_download_{domain}"

        try:
            from crawl4ai import (
                AsyncWebCrawler,
                BrowserConfig,
                CrawlerRunConfig,
            )
        except ImportError:
            logger.debug("crawl4ai not installed — using Playwright")
            return None

        logger.debug(f"Crawl4AI fetch: {url}")
        wait_time = self.rate_tracker.apply_rate_limit(engine_type)

        browser_cfg = BrowserConfig(
            headless=True,
            verbose=False,
            user_agent=BROWSER_USER_AGENT,
        )
        run_cfg = CrawlerRunConfig(
            # Ethical: respect robots.txt
            check_robots_txt=True,
            # Better extraction: flatten modern web features
            flatten_shadow_dom=True,
            process_iframes=True,
            # Trigger lazy-loaded content
            scan_full_page=True,
            # Performance
            wait_until=self.wait_until,
            page_timeout=self.timeout * 1000,
            exclude_all_images=self.block_resources,
            # No stealth
            override_navigator=False,
            magic=False,
            simulate_user=False,
            verbose=False,
        )

        try:

            async def _crawl():
                async with AsyncWebCrawler(config=browser_cfg) as crawler:
                    return await crawler.arun(url=url, config=run_cfg)

            result = _run_async(_crawl(), timeout=self.timeout + 30)

            if result.success and result.html:
                html = result.html
                logger.debug(f"Crawl4AI: got {len(html)} bytes from {url}")
                self.rate_tracker.record_outcome(
                    engine_type=engine_type,
                    wait_time=wait_time,
                    success=True,
                    retry_count=1,
                    search_result_count=1,
                )
                return html

            # Check if blocked by robots.txt
            error_msg = getattr(result, "error_message", "") or ""
            if "robots.txt" in error_msg.lower():
                logger.info(f"Crawl4AI: blocked by robots.txt for {url}")
                # Don't fall back to Playwright — respect the block
                self.rate_tracker.record_outcome(
                    engine_type=engine_type,
                    wait_time=wait_time,
                    success=False,
                    retry_count=1,
                    error_type="robots_txt_blocked",
                )
                return ""  # Empty string signals intentional skip

            status = getattr(result, "status_code", "unknown")
            logger.debug(
                f"Crawl4AI: failed for {url} — "
                f"success={result.success}, status={status}"
            )
            self.rate_tracker.record_outcome(
                engine_type=engine_type,
                wait_time=wait_time,
                success=False,
                retry_count=1,
                error_type=f"crawl4ai_status_{status}",
            )
            return None

        except Exception as e:
            logger.debug(f"Crawl4AI error for {url}: {e}")
            self.rate_tracker.record_outcome(
                engine_type=engine_type,
                wait_time=wait_time,
                success=False,
                retry_count=1,
                error_type=type(e).__name__,
            )
            return None

    def _fetch_with_playwright(self, url: str) -> Optional[str]:
        """Fetch HTML using plain Playwright (fallback)."""
        logger.debug(f"Playwright fetch: {url}")
        domain = urlparse(url).netloc
        engine_type = f"playwright_download_{domain}"

        wait_time = self.rate_tracker.apply_rate_limit(engine_type)

        try:
            from playwright.sync_api import sync_playwright

            # Lazy-init browser (reuse across multiple fetches)
            if self._browser is None:
                logger.debug("Playwright: launching Chromium browser")
                pw = sync_playwright().start()
                try:
                    self._browser = pw.chromium.launch(headless=True)
                except Exception:
                    pw.stop()
                    raise
                self._playwright = pw

            page = self._browser.new_page(
                user_agent=BROWSER_USER_AGENT,
            )
            try:
                # Block heavy resources to speed up rendering
                if self.block_resources:
                    page.route(
                        "**/*.{png,jpg,jpeg,gif,webp,svg,ico,woff,woff2,"
                        "ttf,eot,mp4,webm,mp3,ogg,css}",
                        lambda route: route.abort(),
                    )

                page.goto(
                    url,
                    wait_until=self.wait_until,
                    timeout=self.timeout * 1000,
                )
                html = page.content()
            finally:
                try:
                    page.close()
                except Exception:
                    logger.debug("Failed to close Playwright page")

            if html:
                logger.debug(f"Playwright: got {len(html)} bytes from {url}")
                self.rate_tracker.record_outcome(
                    engine_type=engine_type,
                    wait_time=wait_time,
                    success=True,
                    retry_count=1,
                    search_result_count=1,
                )
                return html

            logger.debug(f"Playwright: empty response from {url}")
            return None

        except ImportError:
            logger.warning("playwright not installed — cannot use JS rendering")
            return None
        except Exception as e:
            logger.exception(f"Playwright error fetching {url}")
            self.rate_tracker.record_outcome(
                engine_type=engine_type,
                wait_time=wait_time,
                success=False,
                retry_count=1,
                error_type=type(e).__name__,
            )
            return None

    def close(self):
        """Clean up Playwright browser and resources."""
        if self._browser:
            try:
                self._browser.close()
            except Exception:
                logger.debug(
                    "Failed to close Playwright browser", exc_info=True
                )
            self._browser = None
        if self._playwright:
            try:
                self._playwright.stop()
            except Exception:
                logger.debug("Failed to stop Playwright", exc_info=True)
            self._playwright = None
        super().close()


class AutoHTMLDownloader(HTMLDownloader):
    """HTML downloader that tries static fetch first, falls back to
    Crawl4AI/Playwright when the page needs JavaScript rendering.

    Detection heuristics:
    - Extracted content is too short (<200 chars)
    - Raw HTML contains SPA framework signals (React, Vue, Angular, Next.js)
    """

    def __init__(
        self,
        timeout: int = 30,
        language: str = "English",
        min_content_length: int = 200,
        **kwargs,
    ):
        super().__init__(timeout=timeout, language=language)
        self.min_content_length = min_content_length
        self._playwright_downloader = None

    def _get_playwright_downloader(self) -> PlaywrightHTMLDownloader:
        """Lazy-init JS rendering downloader for fallback."""
        if self._playwright_downloader is None:
            self._playwright_downloader = PlaywrightHTMLDownloader(
                timeout=self.timeout,
                language=self.language,
            )
        return self._playwright_downloader

    @staticmethod
    def _has_spa_signals(html: str) -> bool:
        """Check if HTML contains signals of a JS-rendered SPA."""
        html_lower = html[:5000].lower()  # Only check head/early body
        return any(signal.lower() in html_lower for signal in SPA_SIGNALS)

    def _fetch_html(self, url: str) -> Optional[str]:
        """Fetch HTML statically, storing raw response for SPA detection.

        Note: _last_raw_html is instance state read by download()/download_with_result().
        This is safe because AutoHTMLDownloader instances are created per-request
        in fetch_and_extract/batch_fetch_and_extract — not shared across threads.
        """
        self._last_raw_html = None
        # Try the normal static fetch
        html = super()._fetch_html(url)
        if html:
            self._last_raw_html = html
            return html

        # Static fetch failed (403, etc.) — try raw GET to check for
        # challenge pages / SPA signals even on non-200 responses
        try:
            response = self.session.get(
                url,
                timeout=self.timeout,
                allow_redirects=True,
            )
            self._last_raw_html = response.text
        except Exception:
            logger.debug("Failed to fetch raw HTML for SPA detection")
        return None

    def download(self, url, content_type=None):
        """Try static fetch, fall back to JS rendering if needed."""
        from .base import ContentType

        if content_type is None:
            content_type = ContentType.TEXT

        # First: try static fetch (fast)
        logger.debug(f"Auto: trying static fetch for {url}")
        result = super().download(url, content_type)

        if result and len(result) >= self.min_content_length:
            logger.debug(
                f"Auto: static fetch succeeded ({len(result)} bytes) for {url}"
            )
            return result

        # Check if we should retry with JS rendering
        raw_html = getattr(self, "_last_raw_html", None)
        needs_js = raw_html and self._has_spa_signals(raw_html)
        no_content = result is None or len(result) < self.min_content_length

        if needs_js or no_content:
            reason = "SPA signals" if needs_js else "no/short content"
            logger.info(
                f"Auto: {reason} for {url}, falling back to JS rendering"
            )
            pw_dl = self._get_playwright_downloader()
            pw_result = pw_dl.download(url, content_type)
            if pw_result and len(pw_result) > len(result or b""):
                logger.info(
                    f"Auto: JS rendering succeeded ({len(pw_result)} bytes) for {url}"
                )
                return pw_result
            logger.debug(f"Auto: JS rendering did not improve result for {url}")

        return result

    def download_with_result(self, url, content_type=None):
        """Try static fetch, fall back to JS rendering if needed."""
        from .base import ContentType

        if content_type is None:
            content_type = ContentType.TEXT

        # First: try static fetch (fast)
        logger.debug(f"Auto: trying static fetch for {url}")
        result = super().download_with_result(url, content_type)

        if (
            result.is_success
            and result.content
            and len(result.content) >= self.min_content_length
        ):
            logger.debug(
                f"Auto: static fetch succeeded ({len(result.content)} bytes) for {url}"
            )
            return result

        # Check if we should retry with JS rendering
        raw_html = getattr(self, "_last_raw_html", None)
        needs_js = raw_html and self._has_spa_signals(raw_html)
        no_content = (
            not result.is_success
            or not result.content
            or len(result.content) < self.min_content_length
        )

        if needs_js or no_content:
            reason = "SPA signals" if needs_js else "no/short content"
            logger.info(
                f"Auto: {reason} for {url}, falling back to JS rendering"
            )
            pw_dl = self._get_playwright_downloader()
            pw_result = pw_dl.download_with_result(url, content_type)
            if (
                pw_result.is_success
                and pw_result.content
                and len(pw_result.content) > len(result.content or b"")
            ):
                logger.info(
                    f"Auto: JS rendering succeeded "
                    f"({len(pw_result.content)} bytes) for {url}"
                )
                return pw_result
            logger.debug(f"Auto: JS rendering did not improve result for {url}")

        return result

    def close(self):
        """Clean up both static and JS rendering resources."""
        if self._playwright_downloader:
            self._playwright_downloader.close()
            self._playwright_downloader = None
        super().close()
