"""
Unified Content Fetcher.

Provides a single interface to fetch content from various sources:
- Academic papers (arXiv, PubMed, Semantic Scholar)
- Web pages (HTML)
- Direct PDF links
"""

from typing import Any, Dict, List, Optional
from loguru import logger

from .url_classifier import URLClassifier, URLType
from ..research_library.downloaders.base import ContentType
from ..security.egress.fetch import policy_aware_validate_url
from ..utilities.resource_utils import safe_close

# Default maximum content length (500KB of text)
DEFAULT_MAX_CONTENT_LENGTH = 500_000

# URL types where HTML fallback is pointless when the specialized downloader fails
_NO_HTML_FALLBACK = {URLType.HTML, URLType.DOI, URLType.INVALID, URLType.PDF}


class ContentFetcher:
    """
    Unified content fetcher that routes to appropriate downloaders.

    Automatically detects URL type and uses the best downloader.
    """

    def __init__(
        self,
        timeout: int = 30,
        language: str = "English",
        enable_js_rendering: bool = False,
        egress_context: Any = None,
    ):
        """
        Initialize the content fetcher.

        Args:
            timeout: Request timeout in seconds
            language: Language for justext stoplist (passed to HTML downloader)
            enable_js_rendering: When True, the HTML/DOI downloader falls back
                to a headless browser (Crawl4AI/Playwright) for pages that need
                JavaScript to render. Defaults to False because the default
                Docker production image ships without Chromium and the fallback
                otherwise wastes work on every fetch. In limited (mostly
                accidental) internal benchmark comparisons between dev
                instances that happened to have Chromium installed and routine
                Docker runs that did not, JS rendering did not measurably
                improve research quality, and most regular benchmark runs are
                on Docker without Chromium anyway — so disabling by default
                does not regress observed quality. The user-facing toggle is
                the ``web.enable_javascript_rendering`` setting.
            egress_context: Optional ``EgressContext`` from the active
                policy. When present and the run's scope is
                ``PRIVATE_ONLY``, SSRF validation permits private IPs so
                local lab deployments (Ollama at 127.0.0.1, SearXNG on
                192.168.x) can actually be reached without forcing the
                operator to set SSRF_ALLOW_PRIVATE_IPS=1 globally.
        """
        self.timeout = timeout
        self.language = language
        self.enable_js_rendering = enable_js_rendering
        self.egress_context = egress_context
        self._downloaders: Dict[URLType, Any] = {}

    def _get_downloader(self, url_type: URLType):
        """Get or create the appropriate downloader for a URL type."""
        if url_type in self._downloaders:
            return self._downloaders[url_type]

        downloader: Any = None

        if url_type == URLType.ARXIV:
            try:
                from ..research_library.downloaders.arxiv import ArxivDownloader

                downloader = ArxivDownloader(timeout=self.timeout)
            except ImportError:
                logger.warning("ArxivDownloader not available")

        elif url_type in (URLType.PUBMED, URLType.PMC):
            try:
                from ..research_library.downloaders.pubmed import (
                    PubMedDownloader,
                )

                downloader = PubMedDownloader(timeout=self.timeout)
            except ImportError:
                logger.warning("PubMedDownloader not available")

        elif url_type == URLType.SEMANTIC_SCHOLAR:
            try:
                from ..research_library.downloaders.semantic_scholar import (
                    SemanticScholarDownloader,
                )

                downloader = SemanticScholarDownloader(timeout=self.timeout)
            except ImportError:
                logger.warning("SemanticScholarDownloader not available")

        elif url_type in (URLType.BIORXIV, URLType.MEDRXIV):
            try:
                from ..research_library.downloaders.biorxiv import (
                    BioRxivDownloader,
                )

                downloader = BioRxivDownloader(timeout=self.timeout)
            except ImportError:
                logger.warning("BioRxivDownloader not available")

        elif url_type == URLType.PDF:
            try:
                from ..research_library.downloaders.direct_pdf import (
                    DirectPDFDownloader,
                )

                downloader = DirectPDFDownloader(timeout=self.timeout)
            except ImportError:
                logger.warning("DirectPDFDownloader not available")

        elif url_type == URLType.HTML:
            try:
                from ..research_library.downloaders.playwright_html import (
                    AutoHTMLDownloader as HTMLDownloader,
                )

                downloader = HTMLDownloader(
                    timeout=self.timeout,
                    language=self.language,
                    enable_js_rendering=self.enable_js_rendering,
                )
            except ImportError:
                logger.warning("HTMLDownloader not available")

        elif url_type == URLType.DOI:
            # DOI URLs typically redirect to publisher pages
            # Use HTML downloader as fallback
            try:
                from ..research_library.downloaders.playwright_html import (
                    AutoHTMLDownloader as HTMLDownloader,
                )

                downloader = HTMLDownloader(
                    timeout=self.timeout,
                    language=self.language,
                    enable_js_rendering=self.enable_js_rendering,
                )
            except ImportError:
                logger.warning("HTMLDownloader not available")

        # Cache the downloader
        if downloader:
            self._apply_egress_policy_to_downloader(downloader)
            self._downloaders[url_type] = downloader

        return downloader

    def _apply_egress_policy_to_downloader(self, downloader: Any) -> None:
        """Relax a downloader's SafeSession to allow private IPs when the
        active scope is PRIVATE_ONLY, mirroring ``policy_aware_validate_url``.

        Without this, a private/lab URL that ContentFetcher already approved
        (policy_aware_validate_url + evaluate_url both allow private hosts
        under PRIVATE_ONLY) is then rejected by the downloader's OWN strict
        SafeSession SSRF re-validation (allow_private_ips defaults to False),
        breaking PRIVATE_ONLY's documented "reach your local services" use
        case. ``SafeSession.request`` reads ``allow_private_ips`` per-request,
        so setting it post-construction takes effect. Cloud-metadata IPs stay
        blocked regardless (``is_ip_blocked`` always rejects them).
        """
        if downloader is None or self.egress_context is None:
            return
        # Narrow except (not bare Exception): the only things that can throw
        # here are the policy import and the ctx.scope attribute access. The
        # failure direction is fail-SAFE — if we don't set allow_private_ips it
        # stays False (strict SSRF), i.e. over-restrictive, never a bypass — but
        # catching narrowly avoids masking an unrelated bug (e.g. a refactor
        # that breaks the import) while still never breaking a fetch on the
        # expected misconfiguration cases.
        try:
            from ..security.egress.policy import EgressScope

            scope = self.egress_context.scope
        except (ImportError, AttributeError):  # pragma: no cover - defensive
            logger.debug(
                "could not resolve egress scope for downloader session",
                exc_info=True,
            )
            return
        if scope == EgressScope.PRIVATE_ONLY:
            session = getattr(downloader, "session", None)
            if session is not None and hasattr(session, "allow_private_ips"):
                session.allow_private_ips = True

    def fetch(
        self,
        url: str,
        max_length: Optional[int] = None,
        prefer_text: bool = True,
    ) -> Dict[str, Any]:
        """
        Fetch content from a URL.

        Automatically detects the URL type and uses the appropriate downloader.

        Args:
            url: The URL to fetch content from
            max_length: Maximum content length to return (chars). Defaults to 500KB.
            prefer_text: If True, prefer text extraction over PDF download

        Returns:
            Dict with:
                - status: "success" or "error"
                - content: Extracted text content
                - url: Original URL
                - source_type: Type of source (arxiv, pubmed, html, etc.)
                - title: Title if available
                - error: Error message if failed
        """
        # Apply default max_length if not specified
        if max_length is None:
            max_length = DEFAULT_MAX_CONTENT_LENGTH

        # Classify the URL
        url_type = URLClassifier.classify(url)
        source_name = URLClassifier.get_source_name(url_type)

        # Reject invalid/dangerous URLs
        if url_type == URLType.INVALID:
            return {
                "status": "error",
                "url": url,
                "source_type": source_name,
                "error": "Invalid or unsupported URL scheme (only http/https allowed)",
            }

        # SSRF validation: reject private/internal IPs before reaching downloaders.
        # Policy-aware so PRIVATE_ONLY egress scope can actually reach
        # private hosts (lab deployments) without disabling SSRF globally.
        if not policy_aware_validate_url(url, self.egress_context):
            logger.warning(f"URL failed SSRF validation: {url}")
            return {
                "status": "error",
                "url": url,
                "source_type": source_name,
                "error": "URL failed security validation (blocked by SSRF protection)",
            }

        # Egress policy: reject URLs that are SSRF-OK but scope-incompatible.
        # SSRF only checks the IP class; scope enforcement is a separate axis
        # (PRIVATE_ONLY blocks public hosts, PUBLIC_ONLY blocks private hosts).
        # Centralized here so all callers (LangGraph fetch tool, MCP
        # download_content, future tools) get uniform enforcement instead of
        # each remembering to wrap the call site.
        if self.egress_context is not None:
            from ..security.egress.policy import evaluate_url
            from ..security.ssrf_validator import redact_url_for_log

            url_decision = evaluate_url(url, self.egress_context)
            if not url_decision.allowed:
                logger.bind(policy_audit=True).warning(
                    "fetch URL denied by egress policy",
                    # Redact: a denied URL may carry userinfo creds / API-key
                    # query params; log only scheme://host:port.
                    url=redact_url_for_log(url),
                    scope=self.egress_context.scope.value,
                    reason=url_decision.reason,
                )
                # Return a structured error rather than raise — callers in
                # the fetch path already handle dict-shaped errors.
                return {
                    "status": "error",
                    "url": url,
                    "source_type": source_name,
                    "error": (
                        f"URL refused by egress policy ({url_decision.reason})"
                    ),
                }

        logger.info(f"Fetching content from {url} (detected: {source_name})")

        # Get the appropriate downloader
        downloader = self._get_downloader(url_type)

        if not downloader:
            # Fall back to generic HTML downloader.  This triggers when a
            # specialized downloader (ArXiv, SemanticScholar, etc.) failed
            # to import — playwright_html may still be available.
            # Use _get_downloader so the instance is cached and cleaned up
            # by close().
            downloader = self._get_downloader(URLType.HTML)
            if not downloader:
                return {
                    "status": "error",
                    "url": url,
                    "source_type": source_name,
                    "error": "No suitable downloader available",
                }

        # Determine content type
        content_type = ContentType.TEXT if prefer_text else ContentType.PDF

        # Download content
        try:
            result = downloader.download_with_result(url, content_type)

            # HTML fallback: when a specialized downloader fails (e.g.
            # arXiv PDF unavailable, PubMed paywalled), try generic HTML
            # extraction — the abstract/landing page often has useful content.
            if not result.is_success and url_type not in _NO_HTML_FALLBACK:
                logger.debug(
                    f"Specialized downloader failed for {url}, "
                    "trying HTML fallback"
                )
                html_downloader = self._get_downloader(URLType.HTML)
                if html_downloader:
                    result = html_downloader.download_with_result(
                        url, content_type
                    )
                    # Use the HTML downloader for metadata too, so we
                    # don't call the failed specialized downloader's
                    # get_metadata (which would re-fetch or return wrong data).
                    downloader = html_downloader

            if result.is_success and result.content:
                # Decode content — check PDF magic bytes first, then try
                # UTF-8, and reject anything that is neither.
                if result.content[:4] == b"%PDF":
                    from ..research_library.downloaders.base import (
                        BaseDownloader,
                    )

                    content = BaseDownloader.extract_text_from_pdf(
                        result.content
                    )
                    if not content:
                        return {
                            "status": "error",
                            "url": url,
                            "source_type": source_name,
                            "error": "Could not extract text from PDF",
                        }
                else:
                    try:
                        content = result.content.decode("utf-8")
                    except UnicodeDecodeError:
                        return {
                            "status": "error",
                            "url": url,
                            "source_type": source_name,
                            "error": "Content is not valid UTF-8 and not a PDF",
                        }

                # Truncate if needed
                if max_length and len(content) > max_length:
                    content = (
                        content[:max_length] + "\n\n[... content truncated ...]"
                    )

                # Try to get metadata
                metadata = {}
                if hasattr(downloader, "get_metadata"):
                    try:
                        metadata = downloader.get_metadata(url)
                    except Exception:
                        logger.debug(
                            "Failed to fetch metadata for {}",
                            url,
                            exc_info=True,
                        )

                return {
                    "status": "success",
                    "content": content,
                    "url": url,
                    "source_type": source_name,
                    "title": metadata.get("title"),
                    "author": metadata.get("author"),
                    "published_date": metadata.get("published_date"),
                }

            return {
                "status": "error",
                "url": url,
                "source_type": source_name,
                "error": result.skip_reason or "Download failed",
            }

        except Exception as e:
            logger.exception(f"Error fetching content from {url}")
            return {
                "status": "error",
                "url": url,
                "source_type": source_name,
                "error": str(e),
            }

    def fetch_text(
        self, url: str, max_length: Optional[int] = None
    ) -> Optional[str]:
        """
        Convenience method to fetch just the text content.

        Args:
            url: The URL to fetch
            max_length: Maximum content length

        Returns:
            Text content or None if failed
        """
        result = self.fetch(url, max_length=max_length, prefer_text=True)
        if result.get("status") == "success":
            return result.get("content")
        return None

    def fetch_batch(self, urls: List[str]) -> Dict[str, Optional[str]]:
        """Fetch multiple URLs, routing each to the best downloader.

        Specialized downloaders (arXiv, PubMed, etc.) are tried first;
        generic HTML extraction is used as fallback.  Downloaders are
        cached by URL type, so a single Playwright browser is shared
        across all HTML URLs.

        Returns:
            Dict mapping URL → extracted text (or None if failed).
        """
        return {url: self.fetch_text(url) for url in urls}

    def get_url_info(self, url: str) -> Dict[str, Any]:
        """
        Get information about a URL without downloading.

        Args:
            url: The URL to analyze

        Returns:
            Dict with url_type, source_name, and extracted_id
        """
        url_type = URLClassifier.classify(url)
        return {
            "url": url,
            "url_type": url_type.value,
            "source_name": URLClassifier.get_source_name(url_type),
            "extracted_id": URLClassifier.extract_id(url, url_type),
        }

    def close(self):
        """Close all cached downloaders and their HTTP sessions."""
        for url_type, downloader in self._downloaders.items():
            safe_close(downloader, f"downloader-{url_type.value}")
        self._downloaders.clear()

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.close()
        return False
