use super::RustScraper;
use readability::extractor;
use regex::Regex;
use scraper::{Html, Selector};
use select::{
    document::Document as SelectDoc,
    predicate::{Attr as SelAttr, Class as SelClass, Name as SelName, Predicate},
};
use tracing::{debug, info, warn};
use url::Url;

// ─────────────────────────────────────────────────────────────────────────────
// 🧬 Content Router — NeuroSiphon DNA Transfer
// Detect page type once, dispatch to the sharpest extractor.
// ─────────────────────────────────────────────────────────────────────────────

/// Coarse page-type categories used to select the optimal extraction strategy.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ContentType {
    /// GitHub, GitLab, Bitbucket, StackOverflow, Pastebin — code / docs repos.
    CodeRepository,
    /// Next.js / Nuxt / Remix SPAs that embed all data in `__NEXT_DATA__` / similar JSON blobs.
    SinglePageApp,
    /// Traditional article / blog / news pages.
    NewsArticle,
    /// E-commerce product pages (price, SKU, availability).
    ECommerce,
    /// API endpoint — raw JSON or XML response (not an HTML page).
    ApiResponse,
    /// Fallback default.
    Unknown,
}

/// Fast heuristic for SPA state markers.
///
/// Some frameworks (notably Nuxt) may place the embedded state JSON very late
/// in the HTML, so sampling the first N bytes can miss it.
pub fn looks_like_spa(html: &str) -> bool {
    html.contains("__NEXT_DATA__")
        || html.contains("__NUXT_DATA__")
        || html.contains("data-nuxt-data")
        || html.contains("__REMIX_CONTEXT__")
        || html.contains("__APP_DATA__")
        || html.contains("react-app.embeddedData") // GitHub React SPA payload
}

/// Classify a URL + a small HTML peek (first 8 KB is enough) into a `ContentType`.
pub fn determine_content_type(url: &Url, html_peek: &str) -> ContentType {
    let host = url.host_str().unwrap_or("").to_ascii_lowercase();
    let path = url.path().to_ascii_lowercase();

    // ── Code / Repository sites ──────────────────────────────────────────────
    let code_hosts = [
        "github.com",
        "gitlab.com",
        "bitbucket.org",
        "raw.githubusercontent.com",
        "gist.github.com",
        "sourcehub.io",
        "codeberg.org",
    ];
    let code_paths = [
        "/blob/",
        "/tree/",
        "/commit/",
        "/pulls",
        "/issues",
        "/releases",
    ];
    let qa_hosts = [
        "stackoverflow.com",
        "superuser.com",
        "serverfault.com",
        "askubuntu.com",
    ];
    let paste_hosts = [
        "pastebin.com",
        "paste.ee",
        "hastebin.com",
        "dpaste.com",
        "nowsecure.io",
    ];

    if code_hosts.iter().any(|h| host.contains(h))
        || code_paths.iter().any(|p| path.contains(p))
        || qa_hosts.iter().any(|h| host.contains(h))
        || paste_hosts.iter().any(|h| host.contains(h))
    {
        return ContentType::CodeRepository;
    }

    // ── Raw API / JSON response ───────────────────────────────────────────────
    let trimmed = html_peek.trim_start();
    if trimmed.starts_with('{') || trimmed.starts_with('[') {
        return ContentType::ApiResponse;
    }

    // ── SPA / Next.js — look for embedded JSON state blobs ───────────────────
    let spa_signals = [
        "__NEXT_DATA__",
        "__NUXT_DATA__",
        "__REMIX_CONTEXT__",
        "__APP_DATA__",
        "window.__PRELOADED_STATE__",
        "window.__INITIAL_STATE__",
    ];
    if spa_signals.iter().any(|s| html_peek.contains(s)) {
        return ContentType::SinglePageApp;
    }

    // ── E-Commerce — structured product signals ───────────────────────────────
    let ecom_hosts = [
        "amazon.",
        "ebay.",
        "shopify.",
        "etsy.",
        "aliexpress.",
        "walmart.",
        "bestbuy.",
        "target.",
        "lazada.",
        "tokopedia.",
        "shopee.",
    ];
    let ecom_signals = [
        "\"@type\":\"Product\"",
        "\"@type\": \"Product\"",
        "application/ld+json",
        "og:price",
        "product:price",
        "itemprop=\"price\"",
        "class=\"price\"",
        "class=\"product",
    ];
    if ecom_hosts.iter().any(|h| host.contains(h))
        || ecom_signals.iter().any(|s| html_peek.contains(s))
    {
        return ContentType::ECommerce;
    }

    // ── News / Article ────────────────────────────────────────────────────────
    let article_signals = [
        "<article",
        "\"@type\":\"Article\"",
        "\"@type\":\"NewsArticle\"",
        "\"@type\": \"Article\"",
        "\"@type\": \"NewsArticle\"",
        "itemprop=\"articleBody\"",
        "class=\"article",
        "class=\"post-content",
        "class=\"entry-content",
    ];
    if article_signals.iter().any(|s| html_peek.contains(s)) {
        return ContentType::NewsArticle;
    }

    ContentType::Unknown
}

impl RustScraper {
    pub(super) fn extract_clean_content_with_metrics(
        &self,
        html: &str,
        base_url: &Url,
    ) -> (String, f64) {
        let clean = self.extract_clean_content(html, base_url);
        // Approximate "noise reduction" as percentage removed by our cleaning pipeline.
        // This is computed relative to a whitespace-normalized raw text snapshot.
        let raw = self.clean_text(&html2md::parse_html(html));
        let ratio = if raw.is_empty() {
            0.0
        } else {
            (raw.len().saturating_sub(clean.len())) as f64 / raw.len() as f64
        };
        (clean, ratio.clamp(0.0, 1.0))
    }

    /// Fallback to og:description when main content is missing or too small
    pub(super) fn apply_og_description_fallback(
        &self,
        clean_content: String,
        og_description: &Option<String>,
    ) -> String {
        let current_words = self.count_words(&clean_content);
        if current_words >= 50 && !clean_content.trim().is_empty() {
            return clean_content;
        }

        if let Some(desc) = og_description {
            let trimmed = desc.trim();
            if !trimmed.is_empty() {
                return trimmed.to_string();
            }
        }

        clean_content
    }

    /// Extract clean, readable content using readability, preceded by AGGRESSIVE HTML preprocessing
    /// Includes TEXT-ONLY fallback mode for high-noise sites (e.g., Reddit, social media)
    pub(super) fn extract_clean_content(&self, html: &str, base_url: &Url) -> String {
        // ── 🧬 Content Router: dispatch to the optimal strategy per page type ──
        // Use a larger peek window to reduce false negatives for SPA markers like
        // __NEXT_DATA__ which may appear later in the document.
        let peek = if html.len() > 50_000 {
            safe_utf8_prefix_bytes(html, 50_000)
        } else {
            html
        };
        let content_type = if !crate::core::config::neurosiphon_enabled() {
            ContentType::Unknown
        } else if looks_like_spa(html) {
            ContentType::SinglePageApp
        } else {
            determine_content_type(base_url, peek)
        };
        info!(
            "🧭 Content Router → {:?} for {}",
            content_type,
            base_url.host_str().unwrap_or("")
        );

        match content_type {
            ContentType::ApiResponse => {
                // Raw JSON/XML — serve as-is (truncate if massive)
                info!("⚡ API response: serving raw JSON/XML directly");
                let text = if html.len() > 50_000 {
                    safe_utf8_prefix_bytes(html, 50_000)
                } else {
                    html
                };
                return text.to_string();
            }

            ContentType::SinglePageApp => {
                // 🧬 Rule C: Extract __NEXT_DATA__ / other embedded JSON state blobs.
                // Only use the SPA JSON as the final answer when it contains enough
                // human-readable words (≥ 100) or `extract_app_state` is explicitly set.
                // Otherwise fall through so readability gets a chance to produce
                // better output (e.g. visible landing-page text on Nuxt sites).
                if let Some(extracted) = self.extract_spa_json_state(html) {
                    let wc = self.count_words(&extracted);
                    if !extracted.trim().is_empty() && (self.extract_app_state || wc >= 100) {
                        info!(
                            "⚡ SPA JSON state extracted ({} chars, {} words)",
                            extracted.len(),
                            wc
                        );
                        return extracted;
                    } else {
                        warn!(
                            "SPA JSON state too sparse ({} words); falling through to readability for richer output",
                            wc
                        );
                    }
                }
                // Fall through to normal flow if extraction fails or is sparse
                warn!("SPA fast-path skipped — using standard readability pipeline");
            }

            ContentType::CodeRepository => {
                // NeuroSiphon-style: prefer <pre>/<code> blocks, markdown-body, then skeletal text
                if let Some(pre_content) = self.extract_pre_formatted(html) {
                    if !pre_content.trim().is_empty() {
                        info!(
                            "⚡ Code repo: {} chars from <pre>/<code>",
                            pre_content.len()
                        );
                        return pre_content;
                    }
                }
                // Also try .markdown-body (GitHub README)
                if let Some(md) = self.extract_mdbook_like(html) {
                    if !md.trim().is_empty() {
                        info!("⚡ Code repo: {} chars from markdown-body", md.len());
                        return md;
                    }
                }
                // Fall through — let normal pipeline handle it
            }

            ContentType::NewsArticle => {
                // Fast-path: readability is excellent here, skip heuristics
                let pre = self.preprocess_html(html);
                if let Ok(product) = extractor::extract(&mut pre.as_bytes(), base_url) {
                    let text = html2md::parse_html(&product.content);
                    let cleaned = self.post_clean_text(&text);
                    if self.count_words(&cleaned) >= 50 {
                        info!("⚡ Article fast-path: {} words", self.count_words(&cleaned));
                        return cleaned;
                    }
                }
                // Fall through
            }

            ContentType::ECommerce | ContentType::Unknown => {
                // Use the full multi-pass pipeline (default behaviour)
            }
        }

        // ── Legacy multi-pass path (used as fallback for all types) ──────────

        // 🧬 Special handling for code/pre-heavy sites (NowSecure, Pastebin, etc.)
        let is_pre_priority = base_url
            .host_str()
            .map(|h| h.contains("nowsecure") || h.contains("pastebin"))
            .unwrap_or(false);

        if is_pre_priority {
            warn!(
                "🧬 Pre-formatted content priority mode for {}",
                base_url.host_str().unwrap_or("")
            );
            if let Some(pre_content) = self.extract_pre_formatted(html) {
                if !pre_content.trim().is_empty() {
                    info!("✓ Extracted {} chars from <pre> tags", pre_content.len());
                    return pre_content;
                }
            }
        }

        // 1) AGGRESSIVE Pre-clean HTML to strip nav, header, footer, forms, buttons, hidden elements
        let pre = self.preprocess_html(html);

        // 1a) mdBook-style extractor (e.g., Rust Book) — try focused body first
        if let Some(md_text) = self.extract_mdbook_like(&pre) {
            if md_text.len() > 30 {
                return self.post_clean_text(&md_text);
            }
        }

        // 2) Readability pass
        let readability_text = match extractor::extract(&mut pre.as_bytes(), base_url) {
            Ok(product) => {
                let text = html2md::parse_html(&product.content);
                self.post_clean_text(&text)
            }
            Err(e) => {
                warn!("Readability extraction failed: {}, will try heuristics", e);
                String::new()
            }
        };

        // 3) Heuristic main-content extraction (article/main/role=main/etc.)
        let heuristic_text = self.heuristic_main_extraction(&pre);

        // 4) Choose the better result by word count
        let rt_words = self.count_words(&readability_text);
        let ht_words = self.count_words(&heuristic_text);
        info!(
            "Extraction passes - Readability: {} words, Heuristic: {} words",
            rt_words, ht_words
        );

        let chosen = if rt_words == 0 && ht_words > 0 {
            heuristic_text
        } else if ht_words == 0 && rt_words > 0 {
            readability_text
        } else if ht_words > rt_words.saturating_add(20) {
            heuristic_text
        } else if rt_words > 0 {
            readability_text
        } else {
            // 5) Fallback to simple whole-document text extraction
            self.fallback_text_extraction(&pre)
        };

        let final_text = self.post_clean_text(&chosen);
        if self.is_high_noise_content(&final_text) {
            warn!(
                "High-noise content detected (likely JS/UI heavy site like Reddit), using TEXT-ONLY extraction"
            );
            return self.text_only_extraction(&pre);
        }

        if final_text.len() < 80 {
            let whole = html2md::parse_html(&pre);
            return self.post_clean_text(&whole);
        }
        final_text
    }

    /// Extract content from mdBook-like structures (#content, main, article) using select crate
    fn extract_mdbook_like(&self, html: &str) -> Option<String> {
        let doc = SelectDoc::from(html);

        // Try GitHub markdown body first (for GitHub README views)
        if let Some(node) = doc.find(SelClass("markdown-body")).next() {
            let inner = node.inner_html();
            let text = html2md::parse_html(&inner);
            let cleaned = self.clean_text(&text);
            let word_count = self.count_words(&cleaned);
            info!("mdBook extractor (.markdown-body): {} words", word_count);
            if word_count > 50 {
                return Some(cleaned);
            }
        }

        // Try #content first - this is mdBook's main content container
        if let Some(node) = doc
            .find(SelName("div").and(SelAttr("id", "content")))
            .next()
        {
            let inner = node.inner_html();
            let text = html2md::parse_html(&inner);
            let cleaned = self.clean_text(&text);
            let word_count = self.count_words(&cleaned);
            info!("mdBook extractor (#content): {} words", word_count);
            if word_count > 50 {
                return Some(cleaned);
            }
        }
        // Try main
        if let Some(node) = doc.find(SelName("main")).next() {
            let inner = node.inner_html();
            let text = html2md::parse_html(&inner);
            let cleaned = self.clean_text(&text);
            let word_count = self.count_words(&cleaned);
            info!("mdBook extractor (main): {} words", word_count);
            if word_count > 50 {
                return Some(cleaned);
            }
        }
        // Try article
        if let Some(node) = doc.find(SelName("article")).next() {
            let inner = node.inner_html();
            let text = html2md::parse_html(&inner);
            let cleaned = self.clean_text(&text);
            let word_count = self.count_words(&cleaned);
            info!("mdBook extractor (article): {} words", word_count);
            if word_count > 50 {
                return Some(cleaned);
            }
        }
        info!("mdBook extractor found no suitable content");
        None
    }

    /// 🧬 Extract SPA JSON state (Next.js __NEXT_DATA__, Nuxt, Remix, etc.)
    /// Returns a compact, human-readable markdown representation of the embedded JSON.
    pub(super) fn extract_spa_json_state(&self, html: &str) -> Option<String> {
        let document = Html::parse_document(html);

        // ── GitHub React embedded payload (highest priority for GitHub pages) ────────
        // GitHub serves blob/tree/issue pages as a React SPA with all content in:
        // <script type="application/json" data-target="react-app.embeddedData">{"payload":{...}}</script>
        if html.contains("react-app.embeddedData") {
            if let Ok(sel) = Selector::parse("script[data-target='react-app.embeddedData']") {
                if let Some(el) = document.select(&sel).next() {
                    let raw = el.text().collect::<String>();
                    if !raw.trim().is_empty() {
                        if let Ok(json) = serde_json::from_str::<serde_json::Value>(&raw) {
                            if let Some(content) = extract_github_embedded_content(&json) {
                                info!(
                                    "⚡ GitHub embedded payload extracted ({} chars)",
                                    content.len()
                                );
                                return Some(content);
                            }
                            // Fall back to generic flatten when specific fields not found
                            let flat = flatten_json_to_text(&json, "GitHub");
                            if !flat.trim().is_empty() {
                                return Some(flat);
                            }
                        }
                    }
                }
            }
        }

        // Priority-ordered list of embedded JSON state containers
        let candidates = [
            ("__NEXT_DATA__", "Next.js"),
            ("__NUXT_DATA__", "Nuxt"),
            ("__REMIX_CONTEXT__", "Remix"),
            ("__APP_DATA__", "SPA"),
        ];

        for (id, label) in &candidates {
            // Try <script id="__NEXT_DATA__" ...>
            if let Ok(sel) = Selector::parse(&format!("script[id='{}']", id)) {
                if let Some(el) = document.select(&sel).next() {
                    let raw = el.text().collect::<String>();
                    if !raw.trim().is_empty() {
                        if let Ok(json) = serde_json::from_str::<serde_json::Value>(&raw) {
                            let flat = flatten_json_to_text(&json, label);
                            if !flat.trim().is_empty() {
                                return Some(flat);
                            }
                        }
                        // Return raw JSON if it can't be parsed (still better than noisy HTML)
                        return Some(format!(
                            "```json\n{}\n```",
                            safe_utf8_prefix_bytes(&raw, 20_000)
                        ));
                    }
                }
            }
        }

        // Fallback: inline window.__PRELOADED_STATE__ / window.__INITIAL_STATE__ patterns
        let re_window_state = Regex::new(
            r#"(?s)window\.__(?:PRELOADED|INITIAL|STORE)_STATE__\s*=\s*(\{.*?\});\s*(?:</script>|window\.)"#,
        ).ok()?;

        if let Ok(sel) = Selector::parse("script:not([src])") {
            for el in document.select(&sel) {
                let text = el.text().collect::<String>();
                if let Some(caps) = re_window_state.captures(&text) {
                    if let Some(m) = caps.get(1) {
                        let json_str = m.as_str();
                        if let Ok(json) = serde_json::from_str::<serde_json::Value>(json_str) {
                            let flat = flatten_json_to_text(&json, "WindowState");
                            if !flat.trim().is_empty() {
                                return Some(flat);
                            }
                        }
                    }
                }
            }
        }

        None
    }

    /// 🧬 Extract pre-formatted content (for NowSecure, code sites)
    fn extract_pre_formatted(&self, html: &str) -> Option<String> {
        let document = Html::parse_document(html);
        let mut content = String::new();

        // Extract all <pre> tags
        if let Ok(pre_selector) = Selector::parse("pre") {
            for element in document.select(&pre_selector) {
                let text = element.text().collect::<Vec<_>>().join("");
                if !text.trim().is_empty() {
                    content.push_str(&text);
                    content.push_str("\n\n");
                }
            }
        }

        // Also check for <code> blocks outside <pre>
        if content.is_empty() {
            if let Ok(code_selector) = Selector::parse("code") {
                for element in document.select(&code_selector) {
                    let text = element.text().collect::<Vec<_>>().join("");
                    if text.len() > 20 {
                        content.push_str(&text);
                        content.push('\n');
                    }
                }
            }
        }

        if content.trim().is_empty() {
            None
        } else {
            Some(self.post_clean_text(&content))
        }
    }

    /// Fallback text extraction when readability fails
    fn fallback_text_extraction(&self, html: &str) -> String {
        let document = Html::parse_document(html);

        let mut text_parts = Vec::new();

        if let Ok(body_selector) = Selector::parse("body") {
            if let Some(body) = document.select(&body_selector).next() {
                self.extract_text_recursive(&body, &mut text_parts);
            }
        } else {
            for node in document.tree.nodes() {
                if let Some(text) = node.value().as_text() {
                    text_parts.push(text.text.to_string());
                }
            }
        }

        let text = text_parts.join(" ");
        self.clean_text(&text)
    }

    /// Recursively extract text from elements.
    ///
    /// Includes universal anti-skeleton behavior:
    /// - If more than 3 consecutive child nodes have identical short text, treat as placeholders and drop them.
    /// - Skip very low text-density containers to avoid deep navigation/boilerplate.
    fn extract_text_recursive(&self, element: &scraper::ElementRef, text_parts: &mut Vec<String>) {
        fn flush_run(
            scraper: &RustScraper,
            run_text: &mut Option<String>,
            run_nodes: &mut Vec<scraper::ElementRef>,
            text_parts: &mut Vec<String>,
        ) {
            if run_nodes.is_empty() {
                *run_text = None;
                return;
            }

            let text = run_text.as_deref().unwrap_or("");
            let is_short = text.len() <= 120 && text.split_whitespace().count() <= 12;
            let is_placeholder_run = is_short && run_nodes.len() > 3;

            if !is_placeholder_run {
                for el in run_nodes.drain(..) {
                    scraper.extract_text_recursive(&el, text_parts);
                }
            } else {
                run_nodes.clear();
            }

            *run_text = None;
        }

        let mut run_text: Option<String> = None;
        let mut run_nodes: Vec<scraper::ElementRef> = Vec::new();

        for child in element.children() {
            if let Some(child_element) = scraper::ElementRef::wrap(child) {
                let tag_name = child_element.value().name();
                if matches!(
                    tag_name,
                    "script"
                        | "style"
                        | "noscript"
                        | "svg"
                        | "canvas"
                        | "iframe"
                        | "form"
                        | "header"
                        | "footer"
                        | "nav"
                        | "aside"
                ) {
                    flush_run(self, &mut run_text, &mut run_nodes, text_parts);
                    continue;
                }

                let attrs = child_element.value();
                let mut skip = false;
                if let Some(id) = attrs.id() {
                    skip |= self.is_noise_identifier(id);
                }
                for class in attrs.classes() {
                    if self.is_noise_identifier(class) {
                        skip = true;
                        break;
                    }
                }
                if skip {
                    flush_run(self, &mut run_text, &mut run_nodes, text_parts);
                    continue;
                }

                if self.is_low_text_density_container(&child_element) {
                    flush_run(self, &mut run_text, &mut run_nodes, text_parts);
                    continue;
                }

                let signature = {
                    let raw = child_element.text().collect::<Vec<_>>().join(" ");
                    let cleaned = self.clean_text(&raw);
                    let trimmed = cleaned.trim();
                    if trimmed.len() > 200 {
                        truncate_utf8_chars(trimmed, 200)
                    } else {
                        trimmed.to_string()
                    }
                };

                if let Some(current) = &run_text {
                    if *current == signature && !signature.is_empty() {
                        run_nodes.push(child_element);
                        continue;
                    }

                    flush_run(self, &mut run_text, &mut run_nodes, text_parts);
                }

                run_text = Some(signature);
                run_nodes.push(child_element);
            } else if let Some(text_node) = child.value().as_text() {
                flush_run(self, &mut run_text, &mut run_nodes, text_parts);
                text_parts.push(text_node.text.to_string());
            }
        }

        flush_run(self, &mut run_text, &mut run_nodes, text_parts);
    }

    fn is_low_text_density_container(&self, element: &scraper::ElementRef) -> bool {
        let tag = element.value().name();
        // Never drop typical content-bearing tags.
        if matches!(tag, "p" | "li" | "h1" | "h2" | "h3" | "h4" | "h5" | "h6") {
            return false;
        }

        // Only apply to common UI scaffolding containers.
        if !matches!(tag, "div" | "section" | "span") {
            return false;
        }

        // This heuristic targets UI scaffolding: many nodes / controls, little text.
        // Semantic protection: keep likely "core data" containers (currency/price/location/job title).
        let text = element.text().collect::<Vec<_>>().join(" ");
        let text_lower = text.to_ascii_lowercase();
        let has_currency = text.contains('$')
            || text.contains('€')
            || text.contains('£')
            || text.contains('¥')
            || text.contains('฿');
        let has_high_value_tokens = has_currency
            || text_lower.contains("price")
            || text_lower.contains("location")
            || text_lower.contains("job title")
            || text_lower.contains("salary")
            || text_lower.contains("per night");
        let text_words = text.split_whitespace().count();
        if text_words >= 25 {
            return false;
        }

        let mut child_elements = 0usize;
        let mut control_like = 0usize;
        for child in element.children() {
            let Some(el) = scraper::ElementRef::wrap(child) else {
                continue;
            };
            child_elements += 1;
            let n = el.value().name();
            if matches!(n, "a" | "button" | "input" | "select" | "option" | "svg") {
                control_like += 1;
            }
            if child_elements >= 60 {
                break;
            }
        }

        let mut cond_a = text_words <= 4 && control_like >= 8;
        let mut cond_b = text_words <= 6 && child_elements >= 40;

        if has_high_value_tokens {
            // Lower rejection threshold by ~50% => require "more clearly boilerplate" to drop.
            cond_a = text_words <= 2 && control_like >= 10;
            cond_b = text_words <= 3 && child_elements >= 55;
        }

        cond_a || cond_b
    }

    /// Clean extracted text (whitespace normalization)
    pub(super) fn clean_text(&self, text: &str) -> String {
        let re_whitespace = Regex::new(r"\s+").unwrap();
        let re_newlines = Regex::new(r"\n\s*\n").unwrap();

        let cleaned = re_whitespace.replace_all(text, " ");
        let cleaned = re_newlines.replace_all(&cleaned, "\n\n");

        cleaned.trim().to_string()
    }

    /// Final post-processing to strip boilerplate lines
    fn post_clean_text(&self, text: &str) -> String {
        let out = self.clean_text(text);
        let input_words = self.count_words(&out);

        let garbage = [
            r"(?i)^subscribe$",
            r"(?i)^sign up$",
            r"(?i)^cookie",
            r"(?i)^accept all$",
            r"(?i)^advert",
            r"(?i)^sponsor",
            r"(?i)^newsletter$",
            r"(?i)^related articles",
            r"(?i)^comments?$",
            r"(?i)^read more$",
            r"(?i)^continue reading$",
            r"(?i)^select categories$",
            r"(?i)^close$",
            r"(?i)^open$",
            r"(?i)^menu$",
            r"(?i)^skip to content$",
            r"(?i)^filters?$",
            r"(?i)^sort$",
        ];
        let re_garbage = Regex::new(&garbage.join("|")).unwrap();

        let mut kept = Vec::new();
        for line in out.split('\n') {
            let line_trim = line.trim();
            if line_trim.is_empty() {
                continue;
            }
            let min_len = if self.is_aggressive_mode() { 4 } else { 2 };
            if line_trim.len() < min_len {
                continue;
            }
            if re_garbage.is_match(line_trim) {
                continue;
            }
            if self.is_aggressive_mode() && self.count_words(line_trim) <= 2 && line_trim.len() < 24
            {
                continue;
            }
            // Drop lines that are leaked JSON fragments (SPA boilerplate leaking through the pipeline).
            // A line is treated as JSON noise when ≥55% of its characters are structural JSON tokens.
            if is_json_noise_line(line_trim) {
                continue;
            }
            kept.push(line_trim.to_string());
        }

        kept.dedup();
        let result = kept.join("\n");

        // Extra de-noising for modern SPA pages (collapse placeholder spam, consecutive dup lines, etc.)
        let result = self.clean_noise(&result);

        let output_words = self.count_words(&result);
        if output_words < input_words / 2 {
            debug!(
                "post_clean_text stripped >50% content ({} → {} words)",
                input_words, output_words
            );
        }

        let re_multi_nl = Regex::new(r"\n{3,}").unwrap();
        re_multi_nl.replace_all(&result, "\n\n").to_string()
    }

    /// Utility filter: removes consecutive identical lines and collapses repeated boilerplate phrases.
    ///
    /// Designed to handle SPA placeholder spam such as repeated short phrases.
    pub(super) fn clean_noise(&self, text: &str) -> String {
        let mut s = text.replace("==========", "\n");

        // Collapse repeated short token sequences (common in skeleton/loading UIs)
        // without relying on site-specific phrase lists.
        s = collapse_repeated_ngram_runs(&s, 4, 5);
        s = collapse_repeated_ngram_runs(&s, 3, 6);

        let mut out_lines: Vec<String> = Vec::new();
        let mut last_line: Option<String> = None;
        let mut run_len: usize = 0;

        for raw_line in s.lines() {
            let line = raw_line.trim();
            if line.is_empty() {
                continue;
            }

            if let Some(last) = &last_line {
                if last == line {
                    run_len += 1;
                    // If the same short line repeats many times, collapse it.
                    if run_len >= 4 && line.len() <= 120 {
                        continue;
                    }
                    continue;
                }
            }

            last_line = Some(line.to_string());
            run_len = 1;
            out_lines.push(line.to_string());
        }

        out_lines.join("\n")
    }

    /// Preprocess HTML before readability
    fn preprocess_html(&self, html: &str) -> String {
        let mut s = html.to_string();

        s = self.normalize_details_summary_html(&s);

        let re_block = Regex::new(
            r"(?is)<(?:script|style|noscript|svg|canvas|iframe)[^>]*?>.*?</(?:script|style|noscript|svg|canvas|iframe)>",
        )
        .unwrap();
        s = re_block.replace_all(&s, " ").to_string();

        let re_structural = Regex::new(
            r"(?is)<(?:nav|header|footer|aside)[^>]*?>.*?</(?:nav|header|footer|aside)>",
        )
        .unwrap();
        s = re_structural.replace_all(&s, " ").to_string();

        let re_interactive =
            Regex::new(r"(?is)<(?:form|button)[^>]*?>.*?</(?:form|button)>").unwrap();
        s = re_interactive.replace_all(&s, " ").to_string();

        let re_hidden = Regex::new(
            r#"(?is)<[^>]*?(?:display:\s*none|visibility:\s*hidden|aria-hidden=\"true\")[^>]*?>.*?</[^>]+>"#,
        )
        .unwrap();
        s = re_hidden.replace_all(&s, " ").to_string();

        let re_ad_blocks = Regex::new(
            r#"(?is)<(?:div|section|article)[^>]*?(?:id|class)=(?:'|\")[^'\">]*(?:ads|advert|sponsor|promo|related|cookie|banner|modal|subscribe|newsletter|share|social|sidebar|comments|breadcrumb|pagination)[^'\">]*(?:'|\")[^>]*?>.*?</(?:div|section|article)>"#,
        )
        .unwrap();
        s = re_ad_blocks.replace_all(&s, " ").to_string();

        s = self.strip_non_semantic_attributes(&s);

        s
    }

    fn normalize_details_summary_html(&self, html: &str) -> String {
        let mut s = html.to_string();

        let re_summary = Regex::new(r"(?is)<summary[^>]*>(.*?)</summary>").unwrap();
        s = re_summary
            .replace_all(&s, |caps: &regex::Captures| {
                let title = caps
                    .get(1)
                    .map(|m| m.as_str())
                    .unwrap_or("")
                    .replace('\n', " ")
                    .trim()
                    .to_string();
                if title.is_empty() {
                    "<h3>Summary</h3>".to_string()
                } else {
                    format!("<h3>{}</h3>", title)
                }
            })
            .to_string();

        let re_details_open = Regex::new(r"(?is)<details[^>]*>").unwrap();
        s = re_details_open.replace_all(&s, "<section>").to_string();

        let re_details_close = Regex::new(r"(?is)</details>").unwrap();
        s = re_details_close.replace_all(&s, "</section>").to_string();

        s
    }

    fn strip_non_semantic_attributes(&self, html: &str) -> String {
        let mut s = html.to_string();

        let re_attrs = if self.is_aggressive_mode() {
            Regex::new(
                r#"(?i)\s(?:class|id|style|role|tabindex|aria-[\w:-]+|data-[\w:-]+|on[\w:-]+)\s*=\s*(?:"[^"]*"|'[^']*'|[^\s>]+)"#,
            )
            .unwrap()
        } else {
            Regex::new(
                r#"(?i)\s(?:class|id|style|aria-[\w:-]+|on[\w:-]+)\s*=\s*(?:"[^"]*"|'[^']*'|[^\s>]+)"#,
            )
            .unwrap()
        };
        s = re_attrs.replace_all(&s, "").to_string();

        let re_empty_attrs = Regex::new(r#"\s{2,}"#).unwrap();
        s = re_empty_attrs.replace_all(&s, " ").to_string();

        s
    }

    fn is_noise_identifier(&self, ident: &str) -> bool {
        let ident = ident.to_ascii_lowercase();
        let needles = [
            "ads",
            "advert",
            "adsense",
            "adunit",
            "ad-slot",
            "ad_container",
            "adbox",
            "sponsor",
            "promo",
            "cookie",
            "consent",
            "banner",
            "modal",
            "subscribe",
            "newsletter",
            "share",
            "social",
            "sidebar",
            "comments",
            "related",
            "breadcrumb",
            "pagination",
            "nav",
            "footer",
            "header",
            "hero",
            "toolbar",
        ];
        if needles.iter().any(|n| ident.contains(n)) {
            return true;
        }
        if ident.contains("-ad")
            || ident.contains("ad-")
            || ident.contains("_ad")
            || ident.contains("ad_")
        {
            return true;
        }
        false
    }

    fn heuristic_main_extraction(&self, html: &str) -> String {
        let document = Html::parse_document(html);

        let selectors = [
            "article",
            "main",
            "[role=main]",
            "[itemprop=articleBody]",
            ".entry-content",
            ".post-content",
            ".article-content",
            "#content",
            "#main",
            ".content",
            ".post",
            ".article",
        ];

        let mut best_text = String::new();
        let mut best_words = 0usize;

        for sel_str in selectors.iter() {
            if let Ok(sel) = Selector::parse(sel_str) {
                for el in document.select(&sel) {
                    let mut parts = Vec::new();
                    self.extract_text_recursive(&el, &mut parts);
                    let text = parts.join(" ");
                    let cleaned = self.clean_text(&text);
                    let wc = self.count_words(&cleaned);
                    if wc > best_words {
                        best_words = wc;
                        best_text = cleaned;
                    }
                }
            }
        }

        best_text
    }

    pub(super) fn count_words(&self, text: &str) -> usize {
        text.split_whitespace().count()
    }

    fn is_high_noise_content(&self, text: &str) -> bool {
        let lines: Vec<&str> = text.lines().collect();
        if lines.len() < 10 {
            return false;
        }

        let noise_keywords = [
            "share",
            "upvote",
            "downvote",
            "comment",
            "reply",
            "login",
            "sign up",
            "subscribe",
            "follow",
            "like",
            "tweet",
            "retweet",
            "menu",
            "navigation",
        ];
        let mut noise_lines = 0;
        let mut total_chars = 0;

        for line in &lines {
            let trimmed = line.trim();
            if trimmed.len() < 10 {
                noise_lines += 1;
                continue;
            }
            total_chars += trimmed.len();

            let lower = trimmed.to_lowercase();
            if noise_keywords.iter().any(|kw| lower.contains(kw)) && trimmed.len() < 40 {
                noise_lines += 1;
            }
        }

        let noise_ratio = noise_lines as f64 / lines.len() as f64;
        let avg_line_length = if !lines.is_empty() {
            total_chars / lines.len()
        } else {
            0
        };

        noise_ratio > 0.6 || avg_line_length < 20
    }

    fn text_only_extraction(&self, html: &str) -> String {
        let document = Html::parse_document(html);

        let content_selectors = [
            "article",
            "main",
            ".content",
            ".post-body",
            "#main",
            ".entry-content",
        ];

        for sel_str in &content_selectors {
            if let Ok(selector) = Selector::parse(sel_str) {
                for element in document.select(&selector) {
                    let mut parts = Vec::new();
                    self.extract_text_recursive(&element, &mut parts);
                    let text = parts.join(" ");
                    let cleaned = self.clean_text(&text);
                    if self.count_words(&cleaned) > 50 {
                        return cleaned;
                    }
                }
            }
        }

        let mut paragraphs = Vec::new();
        if let Ok(p_selector) = Selector::parse("p") {
            for element in document.select(&p_selector) {
                let text = element.text().collect::<String>().trim().to_string();
                if text.len() > 30 {
                    paragraphs.push(text);
                }
            }
        }

        let combined = paragraphs.join("\n\n");
        self.clean_text(&combined)
    }

    pub(super) fn normalize_markdown_fragments(&self, text: &str) -> String {
        let mut out = text.to_string();

        let re_summary = Regex::new(r"(?is)<summary[^>]*>(.*?)</summary>").unwrap();
        out = re_summary
            .replace_all(&out, |caps: &regex::Captures| {
                let title = caps
                    .get(1)
                    .map(|m| m.as_str())
                    .unwrap_or("")
                    .replace('\n', " ")
                    .trim()
                    .to_string();
                if title.is_empty() {
                    "### Summary".to_string()
                } else {
                    format!("### {}", title)
                }
            })
            .to_string();

        let re_details = Regex::new(r"(?is)</?details[^>]*>").unwrap();
        out = re_details.replace_all(&out, "").to_string();

        normalize_markdown(out)
    }
}

fn collapse_repeated_ngram_runs(input: &str, n: usize, min_repeats_exclusive: usize) -> String {
    if n == 0 {
        return input.to_string();
    }

    let tokens: Vec<&str> = input.split_whitespace().collect();
    if tokens.len() < n * (min_repeats_exclusive + 1) {
        return input.to_string();
    }

    let mut out: Vec<&str> = Vec::with_capacity(tokens.len());
    let mut i = 0usize;
    while i < tokens.len() {
        if i + n <= tokens.len() {
            let base = &tokens[i..i + n];
            let mut repeats = 1usize;
            while i + (repeats + 1) * n <= tokens.len() {
                let next = &tokens[i + repeats * n..i + (repeats + 1) * n];
                if next == base {
                    repeats += 1;
                } else {
                    break;
                }
            }

            if repeats > min_repeats_exclusive {
                out.extend_from_slice(base);
                i += repeats * n;
                continue;
            }
        }

        out.push(tokens[i]);
        i += 1;
    }

    out.join(" ")
}
// ─────────────────────────────────────────────────────────────────────────────
// 🧬 JSON flattener — NeuroSiphon DNA Transfer
// Converts deeply-nested SPA JSON blobs into clean, readable key: value text
// that LLMs can consume at a fraction of the token cost of raw JSON.
// ─────────────────────────────────────────────────────────────────────────────

/// Detect lines that are leaked JSON fragments (SPA state leaking through the extraction pipeline).
///
/// A line is classified as JSON noise when:
/// - It starts with a JSON structural character `{`, `[`, `}`, `]` and is long, OR
/// - ≥55% of its characters are JSON structural tokens (`{}[]":,`)
fn is_json_noise_line(line: &str) -> bool {
    if line.len() < 20 {
        return false; // Short lines are harmless
    }
    let first = line.chars().next().unwrap_or(' ');
    if matches!(first, '{' | '[') && line.len() > 40 {
        return true;
    }
    let structural: usize = line
        .chars()
        .filter(|c| matches!(c, '{' | '}' | '[' | ']' | '"' | ':' | ','))
        .count();
    let ratio = structural as f32 / line.len() as f32;
    ratio >= 0.55
}

// ─────────────────────────────────────────────────────────────────────────────
// 🧬 Markdown Normalizer Helpers — Feature 1 (Post-Processor)
// Applied after html2md conversion to strip token-wasting escapes and noise.
// ─────────────────────────────────────────────────────────────────────────────

/// Unescape over-escaped Markdown characters emitted by html2md.
///
/// html2md sometimes emits `\#`, `\_`, `\*`, `\[`, `\]` to escape characters
/// that are already safe in prose context, wasting tokens and confusing LLMs.
/// This function reverses those escapes while leaving code fence contents intact.
fn unescape_markdown_outside_fences(text: &str) -> String {
    let mut out = String::with_capacity(text.len());
    let mut inside_fence = false;
    let lines: Vec<&str> = text.split('\n').collect();
    let total = lines.len();

    for (idx, line) in lines.iter().enumerate() {
        if !inside_fence {
            let trimmed = line.trim_start();
            if trimmed.starts_with("```") || trimmed.starts_with("~~~") {
                // Opening fence — enter verbatim mode
                inside_fence = true;
                out.push_str(line);
            } else {
                // Normal prose line — unescape common over-escaped characters
                let unescaped = line
                    .replace(r"\#", "#")
                    .replace(r"\_", "_")
                    .replace(r"\*", "*")
                    .replace(r"\[", "[")
                    .replace(r"\]", "]")
                    .replace(r"\`", "`")
                    .replace(r"\.", ".")
                    .replace(r"\!", "!")
                    .replace(r"\-", "-")
                    .replace(r"\+", "+")
                    .replace(r"\|", "|");
                out.push_str(&unescaped);
            }
        } else {
            out.push_str(line);
            // Detect closing fence (line that is only the fence marker)
            let trimmed = line.trim();
            if trimmed == "```"
                || trimmed == "~~~"
                || (trimmed.starts_with("```") && trimmed.chars().all(|c| c == '`'))
                || (trimmed.starts_with("~~~") && trimmed.chars().all(|c| c == '~'))
            {
                inside_fence = false;
            }
        }
        if idx < total - 1 {
            out.push('\n');
        }
    }

    if text.ends_with('\n') && !out.ends_with('\n') {
        out.push('\n');
    }
    out
}

/// Remove duplicate navigation-menu link lines.
///
/// When html→Markdown conversion emits an entire navigation sidebar as a
/// sequence of `[Label](url)` lines, the same URL often appears multiple times.
/// We keep only the **first** occurrence of each URL within contiguous link runs.
/// Non-trivial non-link lines reset the seen-URL set so the same link can
/// legitimately reappear in a different meaningful section.
fn dedupe_nav_link_lines(text: &str) -> String {
    let re_pure_link = Regex::new(r"^\s*\[([^\]]*)\]\(([^)]+)\)\s*$").unwrap();
    let mut seen_urls = std::collections::HashSet::<String>::new();
    let mut out_lines: Vec<&str> = Vec::new();

    for line in text.split('\n') {
        if let Some(caps) = re_pure_link.captures(line) {
            let url = caps.get(2).map(|m| m.as_str()).unwrap_or("").to_string();
            if !seen_urls.insert(url) {
                // Duplicate URL — drop this line
                continue;
            }
        } else {
            // Non-link content resets tracking so the same link can appear again
            // in a meaningfully different section further in the document.
            let trimmed = line.trim();
            if !trimmed.is_empty() && !trimmed.starts_with('#') {
                seen_urls.clear();
            }
        }
        out_lines.push(line);
    }

    out_lines.join("\n")
}

/// Markdown post-processor for token efficiency.
///
/// This is the standalone entrypoint requested by the Enterprise-Grade pipeline prompt.
/// It is pure (no `RustScraper` state) and safe to run as the final step after html2md.
///
/// Actions:
/// - Unescape common over-escaped characters (outside code fences)
/// - Collapse excessive blank lines to max 2
/// - Dedupe repeated navigation link lines
pub fn normalize_markdown(text: String) -> String {
    let mut out = text;
    out = unescape_markdown_outside_fences(&out);
    let re_excess_nl = Regex::new(r"\n{3,}").unwrap();
    out = re_excess_nl.replace_all(&out, "\n\n").to_string();
    out = dedupe_nav_link_lines(&out);
    out.trim().to_string()
}

/// Extract clean readable content from GitHub's React embedded payload JSON.
///
/// GitHub embeds `{"payload": {...}}` in `<script data-target="react-app.embeddedData">`.
/// Field priority (highest first):
/// 1. `payload.blob.text` — raw file content (blob viewer pages)
/// 2. `payload.blob.richText` — rendered HTML of file (converted to markdown)
/// 3. `payload.readme.richText` / `.text` — README for repo home pages
/// 4. `payload.issue.body` / `payload.pullRequest.body` — issue/PR description
/// 5. `payload.discussion.body` — discussion body
fn extract_github_embedded_content(json: &serde_json::Value) -> Option<String> {
    let payload = json.get("payload")?;

    // 1. File blob content (github.com/*/blob/* pages)
    if let Some(blob) = payload.get("blob") {
        if let Some(text) = blob.get("text").and_then(|v| v.as_str()) {
            if !text.trim().is_empty() {
                return Some(text.to_string());
            }
        }
        // richText is rendered HTML of the file
        if let Some(rich) = blob.get("richText").and_then(|v| v.as_str()) {
            if !rich.trim().is_empty() {
                let md = html2md::parse_html(rich);
                if !md.trim().is_empty() {
                    return Some(md);
                }
            }
        }
        // Surface raw URL as a hint of last resort
        if let Some(raw_url) = blob.get("rawBlobUrl").and_then(|v| v.as_str()) {
            return Some(format!("(Raw content available at: {})", raw_url));
        }
    }

    // 2. Repository README (repo home pages / tree pages)
    if let Some(readme) = payload.get("readme") {
        if let Some(rich) = readme.get("richText").and_then(|v| v.as_str()) {
            if !rich.trim().is_empty() {
                let md = html2md::parse_html(rich);
                if !md.trim().is_empty() {
                    return Some(md);
                }
            }
        }
        if let Some(text) = readme.get("text").and_then(|v| v.as_str()) {
            if !text.trim().is_empty() {
                return Some(text.to_string());
            }
        }
    }

    // 3. Issue body
    if let Some(issue) = payload.get("issue") {
        if let Some(body) = issue.get("body").and_then(|v| v.as_str()) {
            if !body.trim().is_empty() {
                return Some(body.to_string());
            }
        }
    }

    // 4. Pull request description
    if let Some(pr) = payload.get("pullRequest") {
        if let Some(body) = pr.get("body").and_then(|v| v.as_str()) {
            if !body.trim().is_empty() {
                return Some(body.to_string());
            }
        }
    }

    // 5. Discussion body
    if let Some(disc) = payload.get("discussion") {
        if let Some(body) = disc.get("body").and_then(|v| v.as_str()) {
            if !body.trim().is_empty() {
                return Some(body.to_string());
            }
        }
    }

    None
}

/// Recursively walk a `serde_json::Value` and emit human-readable `key: value` lines.
/// Short string values (≤ 500 chars) are kept; arrays of strings are joined with ", ".
/// Large objects / arrays are summarised by count. Binary / base64 blobs are omitted.
pub fn flatten_json_to_text(value: &serde_json::Value, label: &str) -> String {
    let mut lines: Vec<String> = Vec::new();
    lines.push(format!("# {} Data", label));
    flatten_json_recursive(value, "", &mut lines, 0);
    lines.join("\n")
}

fn is_blob_like(s: &str) -> bool {
    // Detect base64 / binary blobs: long strings with very high non-alphanum density
    if s.len() < 200 {
        return false;
    }
    let non_alnum = s
        .chars()
        .filter(|c| {
            !c.is_alphanumeric() && *c != '=' && *c != '+' && *c != '/' && *c != '-' && *c != '_'
        })
        .count();
    non_alnum as f64 / (s.len() as f64) < 0.05
}

fn flatten_json_recursive(
    value: &serde_json::Value,
    prefix: &str,
    lines: &mut Vec<String>,
    depth: usize,
) {
    // Hard cap depth to avoid exponential blowup on deeply-nested structures
    if depth > 6 {
        return;
    }
    // Hard cap total output
    if lines.len() > 400 {
        return;
    }

    match value {
        serde_json::Value::Object(map) => {
            for (k, v) in map {
                // Skip internal Next.js / build metadata keys
                if k.starts_with("__") && k.ends_with("__") && k != "__NEXT_DATA__" {
                    continue;
                }
                if matches!(
                    k.as_str(),
                    "buildId"
                        | "runtimeConfig"
                        | "isFallback"
                        | "customServer"
                        | "gsp"
                        | "gssp"
                        | "locale"
                        | "locales"
                        | "defaultLocale"
                        | "scriptLoader"
                        | "nextExport"
                        | "autoExport"
                        | "isPreview"
                ) {
                    continue;
                }

                let new_prefix = if prefix.is_empty() {
                    k.clone()
                } else {
                    format!("{}.{}", prefix, k)
                };
                flatten_json_recursive(v, &new_prefix, lines, depth + 1);
            }
        }

        serde_json::Value::Array(arr) => {
            if arr.is_empty() {
                return;
            }

            // Array of short strings → join them
            if arr
                .iter()
                .all(|v| matches!(v, serde_json::Value::String(_)))
            {
                let joined: Vec<&str> = arr
                    .iter()
                    .filter_map(|v| v.as_str())
                    .filter(|s| !s.is_empty())
                    .take(20)
                    .collect();
                if !joined.is_empty() {
                    lines.push(format!("{}: {}", prefix, joined.join(", ")));
                }
                return;
            }

            // Array of objects → expand first 5
            let limit = arr.len().min(5);
            for (i, item) in arr.iter().enumerate().take(limit) {
                let new_prefix = format!("{}[{}]", prefix, i);
                flatten_json_recursive(item, &new_prefix, lines, depth + 1);
            }
            if arr.len() > limit {
                lines.push(format!(
                    "{}: [{} items total, showing first {}]",
                    prefix,
                    arr.len(),
                    limit
                ));
            }
        }

        serde_json::Value::String(s) => {
            if s.is_empty() {
                return;
            }
            if is_blob_like(s) {
                return;
            }
            // Skip internal URL paths and hashes
            if s.starts_with("/_next/")
                || (s.len() == 32 && s.chars().all(|c| c.is_ascii_hexdigit()))
            {
                return;
            }
            let display = if s.len() > 500 {
                truncate_utf8_chars(s, 500)
            } else {
                s.to_string()
            };
            if !prefix.is_empty() {
                lines.push(format!("{}: {}", prefix, display));
            }
        }

        serde_json::Value::Number(n) => {
            if !prefix.is_empty() {
                lines.push(format!("{}: {}", prefix, n));
            }
        }

        serde_json::Value::Bool(b) => {
            if !prefix.is_empty() {
                lines.push(format!("{}: {}", prefix, b));
            }
        }

        serde_json::Value::Null => {}
    }
}

fn safe_utf8_prefix_bytes(s: &str, max_bytes: usize) -> &str {
    if s.len() <= max_bytes {
        return s;
    }

    let mut end = max_bytes;
    while end > 0 && !s.is_char_boundary(end) {
        end -= 1;
    }
    &s[..end]
}

fn truncate_utf8_chars(s: &str, max_chars: usize) -> String {
    s.chars().take(max_chars).collect()
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_is_json_noise_line_detects_json_fragments() {
        // Long JSON object lines should be flagged
        assert!(is_json_noise_line(
            r#"{"payload":{"codeLineWrapEnabled":false,"refInfo":{}}}"#
        ));
        assert!(is_json_noise_line(
            r#"{"allShortcutsEnabled":false,"linkifiedBlobPath":""}"#
        ));
        assert!(is_json_noise_line(
            r#"{"a":"b","c":"d","e":"f","g":"h","i":"j"}"#
        ));
    }

    #[test]
    fn test_is_json_noise_line_keeps_real_content() {
        // Normal prose should never be dropped
        assert!(!is_json_noise_line(
            "This is a regular sentence about code."
        ));
        assert!(!is_json_noise_line("## Installation Requirements"));
        assert!(!is_json_noise_line("pip install unsloth"));
        // Short JSON lines are kept (< 20 chars threshold)
        assert!(!is_json_noise_line(r#"{"a":1}"#));
    }

    #[test]
    fn test_looks_like_spa_detects_github() {
        let with_gh = concat!(
            r#"<html><script type="application/json" "#,
            r#"data-target="react-app.embeddedData">{"payload":{}}</script></html>"#,
        );
        assert!(looks_like_spa(with_gh));
        assert!(!looks_like_spa("<html><body>Hello world</body></html>"));
    }

    #[test]
    fn test_extract_github_embedded_content_blob_text() {
        let json: serde_json::Value = serde_json::json!({
            "payload": {
                "blob": { "text": "# Hello\nThis is the file content." }
            }
        });
        assert_eq!(
            extract_github_embedded_content(&json).as_deref(),
            Some("# Hello\nThis is the file content."),
        );
    }

    #[test]
    fn test_extract_github_embedded_content_readme() {
        let json: serde_json::Value = serde_json::json!({
            "payload": {
                "readme": { "text": "## Readme\nThis is the README." }
            }
        });
        assert_eq!(
            extract_github_embedded_content(&json).as_deref(),
            Some("## Readme\nThis is the README."),
        );
    }

    // ── Feature 1: Markdown Normalizer ────────────────────────────────────────

    #[test]
    fn test_unescape_markdown_outside_fences_basic() {
        let input = r"This is \#heading and \_italic\_ and \*bold\*";
        let result = unescape_markdown_outside_fences(input);
        assert_eq!(result, "This is #heading and _italic_ and *bold*");
    }

    #[test]
    fn test_unescape_markdown_preserves_code_fences() {
        let input = "Prose \\#header\n```rust\nlet x = \\_hello;\n```\nMore \\#prose";
        let result = unescape_markdown_outside_fences(input);
        // Inside is unchanged, outside is unescaped
        assert!(result.contains("# header") || result.contains("#header"));
        assert!(result.contains(r"\_hello;")); // preserved inside fence
        assert!(result.contains("# prose") || result.contains("#prose"));
    }

    #[test]
    fn test_unescape_markdown_brackets() {
        // html2md escapes [ and ] but leaves ( and ) alone
        let input = r"\[link text\]";
        let result = unescape_markdown_outside_fences(input);
        assert_eq!(result, "[link text]");
    }

    #[test]
    fn test_dedupe_nav_link_lines_removes_duplicates() {
        let input = "[Home](https://example.com)\n[About](https://example.com/about)\n[Home](https://example.com)\n[Contact](https://example.com/contact)";
        let result = dedupe_nav_link_lines(input);
        // Second occurrence of /home should be removed
        let home_count = result.matches("https://example.com)").count();
        assert_eq!(home_count, 1, "Duplicate nav link should be removed");
        // About and Contact should still be present
        assert!(result.contains("https://example.com/about"));
        assert!(result.contains("https://example.com/contact"));
    }

    #[test]
    fn test_dedupe_nav_link_resets_on_prose() {
        let input =
            "[Link](https://a.com)\nSome real paragraph content here.\n[Link](https://a.com)";
        let result = dedupe_nav_link_lines(input);
        // The prose line resets the set, so the second [Link] should be kept
        assert_eq!(result.matches("https://a.com").count(), 2);
    }
}
