"""
justext-based boilerplate removal extractor.

Uses statistical NLP (paragraph length, stopword density, link density)
to classify paragraphs as content vs. boilerplate.
"""

from typing import Optional

from loguru import logger

from .base import BaseExtractor


class JustextExtractor(BaseExtractor):
    """Extract content using justext boilerplate removal."""

    def __init__(self, language: str = "English"):
        self.language = language

    def extract(self, html: str) -> Optional[str]:
        if not html or not html.strip():
            return None

        try:
            import justext
        except ImportError:
            logger.warning(
                "justext not installed — skipping justext extraction"
            )
            return None

        try:
            stoplist = justext.get_stoplist(self.language)
        except ValueError:
            logger.warning(
                f"justext stoplist not found for '{self.language}', "
                "falling back to English"
            )
            stoplist = justext.get_stoplist("English")

        try:
            paragraphs = justext.justext(html, stoplist)
        except Exception:
            logger.exception("justext extraction failed")
            return None

        text_parts = []
        for p in paragraphs:
            if not p.is_boilerplate and p.text.strip():
                if p.is_heading:
                    text_parts.append(f"\n## {p.text.strip()}\n")
                else:
                    text_parts.append(p.text.strip())

        content = "\n\n".join(text_parts)
        return content if content.strip() else None
