#!/usr/bin/env python3
"""
Jupyter Notebook (.ipynb) to Skill Converter

Converts Jupyter Notebooks into skills.
Uses nbformat for notebook parsing, extracts markdown prose, code cells with
outputs, kernel metadata, and cell-level tags.

Supports both single .ipynb files and directories containing multiple notebooks.

Usage:
    skill-seekers jupyter --notebook notebook.ipynb --name myskill
    skill-seekers jupyter --notebook ./notebooks/ --name myskill
    skill-seekers jupyter --from-json notebook_extracted.json
"""

import json
import logging
import os
import re
from pathlib import Path

# Optional dependency guard
try:
    import nbformat

    JUPYTER_AVAILABLE = True
except ImportError:
    JUPYTER_AVAILABLE = False

from .document_skill_builder import DocumentSkillBuilder
from skill_seekers.cli.scraper_utils import score_code_quality

logger = logging.getLogger(__name__)

# Import pattern categories for code analysis
_IMPORT_PATTERNS: dict[str, list[re.Pattern]] = {
    "python": [
        re.compile(r"^\s*import\s+([\w.]+)", re.MULTILINE),
        re.compile(r"^\s*from\s+([\w.]+)\s+import", re.MULTILINE),
    ],
    "r": [
        re.compile(r"^\s*library\(([\w.]+)\)", re.MULTILINE),
        re.compile(r"^\s*require\(([\w.]+)\)", re.MULTILINE),
    ],
    "julia": [
        re.compile(r"^\s*using\s+([\w.]+)", re.MULTILINE),
        re.compile(r"^\s*import\s+([\w.]+)", re.MULTILINE),
    ],
    "javascript": [
        re.compile(
            r"""^\s*(?:const|let|var)\s+\w+\s*=\s*require\(['"]([\w./@-]+)['"]\)""", re.MULTILINE
        ),
        re.compile(r"""^\s*import\s+.*?\s+from\s+['"]([\w./@-]+)['"]""", re.MULTILINE),
    ],
    "scala": [re.compile(r"^\s*import\s+([\w.]+)", re.MULTILINE)],
}

# Topic keywords used for content categorization
_TOPIC_KEYWORDS: dict[str, list[str]] = {
    "data_loading": [
        "read_csv",
        "read_json",
        "read_excel",
        "read_sql",
        "load_data",
        "open(",
        "pd.read",
        "fetch",
        "download",
        "dataset",
    ],
    "data_cleaning": [
        "dropna",
        "fillna",
        "replace",
        "strip",
        "clean",
        "preprocess",
        "missing",
        "null",
        "nan",
        "duplicate",
        "rename",
    ],
    "visualization": [
        "plot",
        "plt.",
        "figure",
        "ax.",
        "chart",
        "graph",
        "histogram",
        "scatter",
        "seaborn",
        "sns.",
        "bokeh",
        "plotly",
        "matplotlib",
    ],
    "modeling": [
        "fit",
        "predict",
        "train",
        "model",
        "classifier",
        "regressor",
        "sklearn",
        "tensorflow",
        "torch",
        "keras",
        "xgboost",
    ],
    "evaluation": [
        "accuracy",
        "precision",
        "recall",
        "f1",
        "score",
        "metric",
        "confusion_matrix",
        "roc",
        "auc",
        "loss",
        "evaluate",
    ],
    "feature_engineering": [
        "feature",
        "transform",
        "encode",
        "scale",
        "normalize",
        "one_hot",
        "label_encode",
        "polynomial",
        "pca",
    ],
    "setup": [
        "install",
        "pip",
        "conda",
        "import",
        "config",
        "setup",
        "environment",
        "version",
        "requirements",
    ],
    "analysis": [
        "describe",
        "info",
        "shape",
        "head",
        "tail",
        "summary",
        "statistics",
        "correlation",
        "groupby",
        "aggregate",
    ],
}


def _check_jupyter_deps():
    """Raise RuntimeError if nbformat is not installed."""
    if not JUPYTER_AVAILABLE:
        raise RuntimeError(
            "nbformat is required for Jupyter Notebook support.\n"
            'Install with: pip install "skill-seekers[jupyter]"\n'
            "Or: pip install nbformat"
        )


def infer_description_from_notebook(metadata: dict | None = None, name: str = "") -> str:
    """Infer skill description from notebook metadata.

    Args:
        metadata: Notebook-level metadata dict (kernelspec, language_info, etc.)
        name: Skill name for fallback

    Returns:
        Description string suitable for "Use when..." format
    """
    if metadata:
        lang_info = metadata.get("language_info", {})
        lang_name = lang_info.get("name", "") if isinstance(lang_info, dict) else ""
        title = metadata.get("title", "")
        if title and len(title) > 10:
            return f"Use when working with {title.lower()}"
        kernelspec = metadata.get("kernelspec", {})
        display_name = kernelspec.get("display_name", "") if isinstance(kernelspec, dict) else ""
        if display_name and len(display_name) > 3 and lang_name:
            return f"Use when working with {lang_name} notebooks ({display_name} kernel)"
        if lang_name:
            return f"Use when working with {lang_name} notebook content"
    return (
        f"Use when referencing {name} notebook documentation"
        if name
        else "Use when referencing this notebook documentation"
    )


class JupyterToSkillConverter(DocumentSkillBuilder):
    """Convert Jupyter Notebook (.ipynb) to skill.

    Shared build machinery (load/build orchestration, reference-file wrapper,
    filename resolution) comes from DocumentSkillBuilder; notebook cells
    aren't plain sections, so categorization, the per-section reference body,
    index.md, and SKILL.md stay jupyter-specific overrides.
    """

    SOURCE_TYPE = "jupyter"
    SOURCE_PATH_ATTR = "notebook_path"
    SOURCE_LABEL = "notebook"
    FOOTER_LABEL = "Jupyter Notebook Scraper"
    # Notebook analysis stages on top of the shared documentation keywords.
    PATTERN_KEYWORDS = DocumentSkillBuilder.PATTERN_KEYWORDS + (
        "data loading",
        "preprocessing",
        "modeling",
        "evaluation",
        "results",
        "conclusion",
        "summary",
    )

    @property
    def source_stem(self) -> str:
        """Notebook stem only when notebook_path is a single FILE.

        Directory sources keep the generic ``section_*`` reference naming
        (the base property would use the directory name as the stem).
        """
        path = self.source_path
        return Path(path).stem if path and Path(path).is_file() else ""

    def __init__(self, config: dict):
        super().__init__(config)
        self.config = config
        self.name = config["name"]
        self.notebook_path = config.get("notebook_path", "")
        self.description = (
            config.get("description") or f"Use when referencing {self.name} notebook documentation"
        )
        # skill_dir is resolved once in SkillConverter.__init__
        self.data_file = self.data_file_for()
        self.categories = config.get("categories", {})
        self.extracted_data: dict | None = None

    def extract(self):
        """SkillConverter interface — delegates to extract_notebook()."""
        return self.extract_notebook()

    # ------------------------------------------------------------------
    # Extraction
    # ------------------------------------------------------------------

    def extract_notebook(self) -> bool:
        """Extract content from Jupyter Notebook file(s).

        Reads .ipynb via nbformat v4, extracts markdown/code/raw cells,
        detects language from kernel metadata, extracts imports, scores quality.
        Saves intermediate JSON to {name}_extracted.json. Returns True on success.
        """
        _check_jupyter_deps()
        print(f"\n🔍 Extracting from Jupyter Notebook: {self.notebook_path}")

        path = Path(self.notebook_path)
        if not path.exists():
            raise FileNotFoundError(f"Notebook path not found: {self.notebook_path}")

        notebook_files = self._collect_notebook_files(path)
        if not notebook_files:
            raise ValueError(
                f"No .ipynb files found at: {self.notebook_path}\n"
                "Provide a path to a .ipynb file or directory containing notebooks."
            )
        print(f"   Found {len(notebook_files)} notebook(s)")

        all_sections: list[dict] = []
        combined_metadata: dict = {}
        total_code_blocks = total_markdown_cells = total_raw_cells = 0
        languages_detected: dict[str, int] = {}
        all_imports: list[str] = []
        section_number = 0

        for nb_file in notebook_files:
            try:
                nb_data = self._parse_single_notebook(nb_file)
            except Exception as e:
                logger.warning("Failed to parse notebook %s: %s", nb_file, e)
                print(f"   ⚠️  Skipping {nb_file.name}: {e}")
                continue

            if not combined_metadata:
                combined_metadata = nb_data["metadata"]
            nb_lang = nb_data["language"]
            if nb_lang:
                languages_detected[nb_lang] = (
                    languages_detected.get(nb_lang, 0) + nb_data["code_cell_count"]
                )
            for section in nb_data["sections"]:
                section_number += 1
                section["section_number"] = section_number
                section["source_notebook"] = nb_file.name
            all_sections.extend(nb_data["sections"])
            total_code_blocks += nb_data["code_cell_count"]
            total_markdown_cells += nb_data["markdown_cell_count"]
            total_raw_cells += nb_data["raw_cell_count"]
            all_imports.extend(nb_data["imports"])
            print(
                f"   📓 {nb_file.name}: {nb_data['code_cell_count']} code, "
                f"{nb_data['markdown_cell_count']} markdown, {nb_data['raw_cell_count']} raw cells"
            )

        if not self.config.get("description"):
            self.description = infer_description_from_notebook(combined_metadata, self.name)

        # Detect languages via LanguageDetector for unlabelled code cells
        try:
            from skill_seekers.cli.language_detector import LanguageDetector

            detector = LanguageDetector(min_confidence=0.15)
        except ImportError:
            detector = None
        if detector:
            for section in all_sections:
                for cs in section.get("code_samples", []):
                    if not cs.get("language") and cs.get("code"):
                        lang, conf = detector.detect_from_code(cs["code"])
                        if lang and conf >= 0.3:
                            cs["language"] = lang
                            languages_detected[lang] = languages_detected.get(lang, 0) + 1

        result_data = {
            "source_file": str(self.notebook_path),
            "metadata": combined_metadata,
            "total_sections": len(all_sections),
            "total_code_blocks": total_code_blocks,
            "total_markdown_cells": total_markdown_cells,
            "total_raw_cells": total_raw_cells,
            "total_notebooks": len(notebook_files),
            "languages_detected": languages_detected,
            "imports": sorted(set(all_imports)),
            "pages": all_sections,
        }
        os.makedirs(os.path.dirname(self.data_file) or ".", exist_ok=True)
        with open(self.data_file, "w", encoding="utf-8") as f:
            json.dump(result_data, f, indent=2, ensure_ascii=False, default=str)
        print(f"\n💾 Saved extracted data to: {self.data_file}")
        self.extracted_data = result_data
        print(
            f"✅ Extracted {len(all_sections)} sections, "
            f"{total_code_blocks} code blocks, {total_markdown_cells} markdown cells"
        )
        return True

    def _collect_notebook_files(self, path: Path) -> list[Path]:
        """Collect .ipynb files from a path (single file or directory)."""
        if path.is_file():
            if not path.name.endswith(".ipynb"):
                raise ValueError(f"Not a Jupyter Notebook (expected .ipynb): {path}")
            return [path]
        if path.is_dir():
            notebooks = sorted(path.glob("**/*.ipynb"))
            return [nb for nb in notebooks if ".ipynb_checkpoints" not in str(nb)]
        raise ValueError(f"Path is not a file or directory: {path}")

    def _parse_single_notebook(self, nb_path: Path) -> dict:
        """Parse a single .ipynb file and return structured data."""
        with open(nb_path, encoding="utf-8") as f:
            nb = nbformat.read(f, as_version=4)
        metadata = dict(nb.metadata) if nb.metadata else {}
        language = self._detect_language(metadata)
        sections: list[dict] = []
        code_cell_count = markdown_cell_count = raw_cell_count = 0
        imports: list[str] = []

        for cell_index, cell in enumerate(nb.cells):
            cell_type = cell.get("cell_type", "")
            source = cell.get("source", "")
            tags = dict(cell.get("metadata", {})).get("tags", [])

            if cell_type == "markdown":
                markdown_cell_count += 1
                sections.extend(self._parse_markdown_cell(source, cell_index, tags, nb_path.name))
            elif cell_type == "code":
                code_cell_count += 1
                sections.append(
                    self._parse_code_cell(cell, cell_index, language, tags, nb_path.name)
                )
                imports.extend(self._extract_imports(source, language))
            elif cell_type == "raw":
                raw_cell_count += 1
                sections.append(self._parse_raw_cell(source, cell_index, tags, nb_path.name))

        return {
            "metadata": metadata,
            "language": language,
            "sections": sections,
            "code_cell_count": code_cell_count,
            "markdown_cell_count": markdown_cell_count,
            "raw_cell_count": raw_cell_count,
            "imports": imports,
        }

    def _parse_markdown_cell(
        self, source: str, cell_index: int, tags: list[str], notebook_name: str
    ) -> list[dict]:
        """Parse a markdown cell, splitting by heading boundaries."""
        if not source.strip():
            return []
        lines = source.split("\n")
        sections: list[dict] = []
        current_heading = current_heading_level = ""
        current_lines: list[str] = []

        for line in lines:
            heading_match = re.match(r"^(#{1,6})\s+(.+)", line)
            if heading_match:
                if current_heading or current_lines:
                    sections.append(
                        self._build_markdown_section(
                            current_heading,
                            current_heading_level,
                            current_lines,
                            cell_index,
                            tags,
                            notebook_name,
                        )
                    )
                current_heading = heading_match.group(2).strip()
                current_heading_level = f"h{len(heading_match.group(1))}"
                current_lines = []
            else:
                current_lines.append(line)

        if current_heading or current_lines:
            sections.append(
                self._build_markdown_section(
                    current_heading,
                    current_heading_level,
                    current_lines,
                    cell_index,
                    tags,
                    notebook_name,
                )
            )
        return sections

    def _build_markdown_section(
        self,
        heading: str,
        heading_level: str,
        lines: list[str],
        cell_index: int,
        tags: list[str],
        notebook_name: str,
    ) -> dict:
        """Build a section dict from parsed markdown content."""
        text = "\n".join(lines).strip()
        code_samples = []
        code_block_pattern = re.compile(r"```(\w*)\n(.*?)```", re.DOTALL)
        for match in code_block_pattern.finditer(text):
            lang, code = match.group(1) or "", match.group(2).strip()
            if code:
                code_samples.append(
                    {
                        "code": code,
                        "language": lang,
                        "quality_score": score_code_quality(code, notebook_mode=True),
                    }
                )
        prose_text = code_block_pattern.sub("", text).strip()
        sub_headings = []
        for line in lines:
            sub_match = re.match(r"^(#{3,6})\s+(.+)", line)
            if sub_match:
                sub_text = sub_match.group(2).strip()
                if sub_text:
                    sub_headings.append({"level": f"h{len(sub_match.group(1))}", "text": sub_text})
        return {
            "section_number": 0,
            "heading": heading,
            "heading_level": heading_level or "h1",
            "text": prose_text,
            "headings": sub_headings,
            "code_samples": code_samples,
            "tables": [],
            "images": [],
            "cell_type": "markdown",
            "cell_index": cell_index,
            "tags": tags,
            "source_notebook": notebook_name,
        }

    def _parse_code_cell(
        self, cell: dict, cell_index: int, language: str, tags: list[str], notebook_name: str
    ) -> dict:
        """Parse a code cell including source and outputs."""
        source = cell.get("source", "")
        execution_count = cell.get("execution_count")
        code_samples = []
        if source.strip():
            code_samples.append(
                {
                    "code": source.strip(),
                    "language": language,
                    "quality_score": score_code_quality(source, notebook_mode=True),
                    "execution_count": execution_count,
                }
            )
        output_texts: list[str] = []
        output_errors: list[str] = []
        output_display: list[dict] = []
        for output in cell.get("outputs", []):
            output_type = output.get("output_type", "")
            if output_type == "stream":
                stream_text = output.get("text", "")
                if isinstance(stream_text, list):
                    stream_text = "".join(stream_text)
                if output.get("name", "stdout") == "stderr":
                    output_errors.append(stream_text)
                else:
                    output_texts.append(stream_text)
            elif output_type in ("execute_result", "display_data"):
                data = output.get("data", {})
                text_plain = data.get("text/plain", "")
                if isinstance(text_plain, list):
                    text_plain = "".join(text_plain)
                if text_plain:
                    output_texts.append(text_plain)
                for mime in ("text/html", "image/png", "image/svg+xml"):
                    if mime in data:
                        output_display.append({"mime_type": mime, "has_data": True})
            elif output_type == "error":
                ename, evalue = output.get("ename", "Error"), output.get("evalue", "")
                error_msg = f"{ename}: {evalue}"
                tb = output.get("traceback", [])
                if tb:
                    clean_tb = [re.sub(r"\x1b\[[0-9;]*m", "", line) for line in tb]
                    error_msg += "\n" + "\n".join(clean_tb)
                output_errors.append(error_msg)

        return {
            "section_number": 0,
            "heading": self._infer_code_heading(source, execution_count),
            "heading_level": "h3",
            "text": "\n".join(output_texts).strip() if output_texts else "",
            "headings": [],
            "code_samples": code_samples,
            "tables": [],
            "images": [],
            "cell_type": "code",
            "cell_index": cell_index,
            "tags": tags,
            "source_notebook": notebook_name,
            "execution_count": execution_count,
            "output_text": "\n".join(output_texts).strip(),
            "output_errors": output_errors,
            "output_display": output_display,
        }

    def _parse_raw_cell(
        self, source: str, cell_index: int, tags: list[str], notebook_name: str
    ) -> dict:
        """Parse a raw cell (unrendered text)."""
        return {
            "section_number": 0,
            "heading": "",
            "heading_level": "h3",
            "text": source.strip(),
            "headings": [],
            "code_samples": [],
            "tables": [],
            "images": [],
            "cell_type": "raw",
            "cell_index": cell_index,
            "tags": tags,
            "source_notebook": notebook_name,
        }

    def _infer_code_heading(self, source: str, execution_count: int | None) -> str:
        """Infer a descriptive heading for a code cell from first meaningful line."""
        if not source.strip():
            return f"Code Cell [{execution_count or '?'}]"
        first_line = source.strip().split("\n")[0].strip()
        comment_match = re.match(r"^#\s+(.+)", first_line)
        if comment_match:
            heading = comment_match.group(1).strip()
            return heading[:77] + "..." if len(heading) > 80 else heading
        def_match = re.match(r"^(?:def|class|async\s+def)\s+(\w+)", first_line)
        if def_match:
            return f"Define: {def_match.group(1)}"
        assign_match = re.match(r"^(\w+)\s*=", first_line)
        if assign_match and len(assign_match.group(1)) > 1:
            return f"Assign: {assign_match.group(1)}"
        magic_match = re.match(r"^(%+\w+)", first_line)
        if magic_match:
            return f"Magic: {magic_match.group(1)}"
        if first_line.startswith("!"):
            cmd = first_line[1:].strip().split()[0] if first_line[1:].strip() else "shell"
            return f"Shell: {cmd}"
        prefix = f"[{execution_count}]" if execution_count else ""
        return f"Code Cell {prefix}".strip()

    def _detect_language(self, metadata: dict) -> str:
        """Detect programming language from notebook kernel metadata."""
        kernelspec = metadata.get("kernelspec", {})
        if isinstance(kernelspec, dict):
            kernel_lang = kernelspec.get("language", "")
            if kernel_lang:
                return kernel_lang.lower()
            kernel_name = kernelspec.get("name", "")
            if kernel_name:
                name_lower = kernel_name.lower()
                for keyword, lang in [
                    ("python", "python"),
                    ("julia", "julia"),
                    ("scala", "scala"),
                    ("rust", "rust"),
                ]:
                    if keyword in name_lower:
                        return lang
                if name_lower in ("ir", "r"):
                    return "r"
                if "javascript" in name_lower or "node" in name_lower:
                    return "javascript"
                if "csharp" in name_lower or "dotnet" in name_lower:
                    return "csharp"
        lang_info = metadata.get("language_info", {})
        if isinstance(lang_info, dict):
            lang_name = lang_info.get("name", "")
            if lang_name:
                return lang_name.lower()
        return ""

    def _extract_imports(self, source: str, language: str) -> list[str]:
        """Extract import/library statements from code source."""
        if not source.strip():
            return []
        imports: list[str] = []
        lang_key = language.lower() if language else "python"
        patterns = _IMPORT_PATTERNS.get(lang_key, _IMPORT_PATTERNS.get("python", []))
        for pattern in patterns:
            for match in pattern.finditer(source):
                module_name = match.group(1).strip()
                if module_name and module_name not in imports:
                    imports.append(module_name)
        return imports

    # ------------------------------------------------------------------
    # Categorize (load/build orchestration inherited from DocumentSkillBuilder)
    # ------------------------------------------------------------------

    def categorize_content(self) -> dict[str, dict]:
        """Categorize sections based on cell type and topic keywords."""
        print("\n📋 Categorizing content...")
        categorized: dict[str, dict] = {}
        sections = self.extracted_data.get("pages", [])

        # Single notebook — use basename as category
        if self.notebook_path and Path(self.notebook_path).is_file():
            nb_basename = Path(self.notebook_path).stem
            categorized[self._sanitize_filename(nb_basename)] = {
                "title": nb_basename,
                "pages": sections,
            }
            print(f"✅ Created 1 category (single notebook source)")
            print(f"   - {nb_basename}: {len(sections)} sections")
            return categorized

        # Custom keyword-based categories
        if self.categories:
            first_value = next(iter(self.categories.values()), None)
            if isinstance(first_value, list) and first_value and isinstance(first_value[0], dict):
                for cat_key, pages in self.categories.items():
                    categorized[cat_key] = {
                        "title": cat_key.replace("_", " ").title(),
                        "pages": pages,
                    }
            else:
                for cat_key in self.categories:
                    categorized[cat_key] = {
                        "title": cat_key.replace("_", " ").title(),
                        "pages": [],
                    }
                for section in sections:
                    combined = self._section_text(section)
                    scores = {}
                    for cat_key, keywords in self.categories.items():
                        if isinstance(keywords, list):
                            score = sum(
                                1
                                for kw in keywords
                                if isinstance(kw, str) and kw.lower() in combined
                            )
                        else:
                            score = 0
                        if score > 0:
                            scores[cat_key] = score
                    if scores:
                        categorized[max(scores, key=scores.get)]["pages"].append(section)
                    else:
                        categorized.setdefault("other", {"title": "Other", "pages": []})
                        categorized["other"]["pages"].append(section)
            self._print_categories(categorized)
            return categorized

        # Auto-categorize by topic keywords
        topic_buckets: dict[str, list[dict]] = {}
        uncategorized: list[dict] = []
        for section in sections:
            combined = self._section_text(section)
            matched_topic, best_score = "", 0
            for topic, keywords in _TOPIC_KEYWORDS.items():
                score = sum(1 for kw in keywords if kw.lower() in combined)
                if score > best_score:
                    best_score, matched_topic = score, topic
            if matched_topic and best_score >= 2:
                topic_buckets.setdefault(matched_topic, []).append(section)
            else:
                uncategorized.append(section)
        for topic, pages in sorted(topic_buckets.items()):
            categorized[topic] = {"title": topic.replace("_", " ").title(), "pages": pages}
        if uncategorized:
            categorized["other"] = {"title": "Other", "pages": uncategorized}
        if not categorized:
            categorized["content"] = {"title": "Content", "pages": sections}
        self._print_categories(categorized)
        return categorized

    def _section_text(self, section: dict) -> str:
        """Combine section text, heading, and code into a single lowercase string."""
        text = section.get("text", "").lower()
        heading = section.get("heading", "").lower()
        code = " ".join(cs.get("code", "").lower() for cs in section.get("code_samples", []))
        return f"{text} {heading} {code}"

    def _print_categories(self, categorized: dict[str, dict]) -> None:
        print(f"✅ Created {len(categorized)} categories")
        for cat_data in categorized.values():
            print(f"   - {cat_data['title']}: {len(cat_data['pages'])} sections")

    # ------------------------------------------------------------------
    # Private generation methods
    # ------------------------------------------------------------------

    def _write_reference_section(self, f, section) -> None:
        """Write one notebook section (markdown/code/raw cell) into a
        reference file — cells carry execution counts, outputs, and tags
        the base heading+text+code shape doesn't know about."""
        sec_num = section.get("section_number", "?")
        heading = section.get("heading", "")
        heading_level = section.get("heading_level", "h1")
        cell_type = section.get("cell_type", "markdown")

        f.write(f"---\n\n**📄 Source: Section {sec_num}**")
        if cell_type == "code":
            ec = section.get("execution_count")
            f.write(f" (Code Cell{f' [In {ec}]' if ec else ''})")
        elif cell_type == "raw":
            f.write(" (Raw Cell)")
        f.write("\n\n")

        if heading:
            md_lvl = "#" * (int(heading_level[1]) + 1) if heading_level.startswith("h") else "##"
            f.write(f"{md_lvl} {heading}\n\n")
        for sub in section.get("headings", []):
            sl, st = sub.get("level", "h3"), sub.get("text", "")
            if st:
                smd = "#" * (int(sl[1]) + 1) if sl.startswith("h") else "###"
                f.write(f"{smd} {st}\n\n")
        if section.get("text"):
            f.write(f"{section['text']}\n\n")
        for code in section.get("code_samples", []):
            ec = code.get("execution_count")
            if ec:
                f.write(f"**In [{ec}]:**\n\n")
            f.write(f"```{code.get('language', '')}\n{code['code']}\n```\n\n")
        if section.get("output_text"):
            f.write(f"**Output:**\n\n```\n{section['output_text']}\n```\n\n")
        for err in section.get("output_errors", []):
            f.write(f"**Errors:**\n\n```\n{err}\n```\n\n")
        disp = section.get("output_display", [])
        if disp:
            mimes = [d.get("mime_type", "") for d in disp]
            f.write(f"*Rich output: {', '.join(mimes)}*\n\n")
        for table in section.get("tables", []):
            self._write_markdown_table(f, table)
        tags = section.get("tags", [])
        if tags:
            f.write(f"*Tags: {', '.join(str(t) for t in tags)}*\n\n")
        f.write("---\n\n")

    def _generate_index(self, categorized: dict[str, dict]) -> None:
        """Generate reference index file."""
        filename = f"{self.skill_dir}/references/index.md"
        total_cats = len(categorized)

        with open(filename, "w", encoding="utf-8") as f:
            f.write(f"# {self.name.title()} Notebook Reference\n\n## Categories\n\n")
            for section_num, (_ck, cd) in enumerate(categorized.items(), 1):
                pages = cd["pages"]
                count = len(pages)
                if pages:
                    snums = [s.get("section_number", i + 1) for i, s in enumerate(pages)]
                    rng = f"Sections {min(snums)}-{max(snums)}"
                else:
                    rng = "N/A"
                # Route through the shared helper so the index link always
                # matches the reference file the writer creates (no drift).
                link = self._reference_filename(cd, section_num, total_cats)
                f.write(f"- [{cd['title']}]({link}) ({count} sections, {rng})\n")

            f.write("\n## Statistics\n\n")
            ed = self.extracted_data
            f.write(f"- Total sections: {ed.get('total_sections', 0)}\n")
            f.write(f"- Code cells: {ed.get('total_code_blocks', 0)}\n")
            f.write(f"- Markdown cells: {ed.get('total_markdown_cells', 0)}\n")
            f.write(f"- Raw cells: {ed.get('total_raw_cells', 0)}\n")
            f.write(f"- Notebooks: {ed.get('total_notebooks', 1)}\n")

            meta = ed.get("metadata", {})
            ks = meta.get("kernelspec", {})
            if isinstance(ks, dict) and ks.get("display_name"):
                f.write(f"- Kernel: {ks['display_name']}\n")
            li = meta.get("language_info", {})
            if isinstance(li, dict) and li.get("version"):
                f.write(f"- Language version: {li.get('name', '')} {li['version']}\n")

            imports = ed.get("imports", [])
            if imports:
                f.write(f"\n## Imported Packages ({len(imports)})\n\n")
                for imp in imports[:30]:
                    f.write(f"- `{imp}`\n")
                if len(imports) > 30:
                    f.write(f"- ... and {len(imports) - 30} more\n")
        print(f"   Generated: {filename}")

    def _generate_skill_md(self, categorized: dict[str, dict]) -> None:
        """Generate main SKILL.md file."""
        filename = f"{self.skill_dir}/SKILL.md"
        skill_name = self.name.lower().replace("_", "-").replace(" ", "-")[:64]
        desc = self.description[:1024]
        ed = self.extracted_data

        with open(filename, "w", encoding="utf-8") as f:
            f.write(f"---\nname: {skill_name}\ndescription: {desc}\n---\n\n")
            f.write(f"# {self.name.title()} Notebook Skill\n\n{self.description}\n\n")

            # Notebook metadata
            meta = ed.get("metadata", {})
            ks = meta.get("kernelspec", {})
            li = meta.get("language_info", {})
            has_ks = isinstance(ks, dict) and ks.get("display_name")
            has_li = isinstance(li, dict) and li.get("name")
            if has_ks or has_li:
                f.write("## 📋 Notebook Information\n\n")
                if has_ks:
                    f.write(f"**Kernel:** {ks['display_name']}\n\n")
                if has_li:
                    ver = li.get("version", "")
                    f.write(f"**Language:** {li['name']}{' ' + ver if ver else ''}\n\n")

            f.write("## 💡 When to Use This Skill\n\nUse this skill when you need to:\n")
            f.write(f"- Understand {self.name} concepts and analysis workflow\n")
            f.write("- Reference code examples and their outputs\n")
            f.write("- Reproduce data analysis or computation steps\n")
            f.write("- Review methodology, visualizations, and results\n")
            f.write("- Find library usage patterns and best practices\n\n")

            total_sections = ed.get("total_sections", 0)
            f.write(f"## 📖 Section Overview\n\n**Total Sections:** {total_sections}\n\n")
            f.write("**Content Breakdown:**\n\n")
            for cd in categorized.values():
                f.write(f"- **{cd['title']}**: {len(cd['pages'])} sections\n")
            f.write("\n")

            f.write(self._format_key_concepts())

            imports = ed.get("imports", [])
            if imports:
                f.write(f"## 📦 Dependencies\n\n*{len(imports)} package(s) imported*\n\n")
                for imp in imports[:20]:
                    f.write(f"- `{imp}`\n")
                if len(imports) > 20:
                    f.write(f"- ... and {len(imports) - 20} more\n")
                f.write("\n")

            f.write("## ⚡ Quick Reference\n\n")
            f.write(self._format_patterns_from_content())

            # Top code examples
            all_code: list[dict] = []
            for section in ed.get("pages", []):
                all_code.extend(section.get("code_samples", []))
            all_code.sort(key=lambda x: x.get("quality_score", 0), reverse=True)
            top_code = all_code[:15]
            if top_code:
                f.write("## 📝 Code Examples\n\n*High-quality code cells from notebook*\n\n")
                by_lang: dict[str, list] = {}
                for c in top_code:
                    by_lang.setdefault(c.get("language", "unknown"), []).append(c)
                for lang in sorted(by_lang):
                    examples = by_lang[lang]
                    f.write(f"### {lang.title()} Examples ({len(examples)})\n\n")
                    for i, c in enumerate(examples[:5], 1):
                        quality = c.get("quality_score", 0)
                        ec = c.get("execution_count")
                        label = f"In [{ec}]" if ec else f"Example {i}"
                        code_text = c.get("code", "")
                        f.write(f"**{label}** (Quality: {quality:.1f}/10):\n\n```{lang}\n")
                        f.write(code_text[:500] + ("\n..." if len(code_text) > 500 else ""))
                        f.write("\n```\n\n")

            f.write("## 📊 Notebook Statistics\n\n")
            f.write(f"- **Total Sections**: {total_sections}\n")
            f.write(f"- **Code Cells**: {ed.get('total_code_blocks', 0)}\n")
            f.write(f"- **Markdown Cells**: {ed.get('total_markdown_cells', 0)}\n")
            f.write(f"- **Raw Cells**: {ed.get('total_raw_cells', 0)}\n")
            f.write(f"- **Notebooks**: {ed.get('total_notebooks', 1)}\n")
            langs = ed.get("languages_detected", {})
            if langs:
                f.write(f"- **Programming Languages**: {len(langs)}\n\n")
                f.write("**Language Breakdown:**\n\n")
                for lang, count in sorted(langs.items(), key=lambda x: x[1], reverse=True):
                    f.write(f"- {lang}: {count} code cells\n")
                f.write("\n")

            f.write("## 🗺️ Navigation\n\n**Reference Files:**\n\n")
            # Derive nav links from the SAME helper that names the reference
            # files so the links match real filenames (DOC-07).
            _total = len(categorized)
            for _sn, cd in enumerate(categorized.values(), 1):
                cat_file = self._reference_filename(cd, _sn, _total)
                f.write(f"- `references/{cat_file}` - {cd['title']}\n")
            f.write("\nSee `references/index.md` for complete notebook structure.\n\n")
            f.write("---\n\n**Generated by Skill Seeker** | Jupyter Notebook Scraper\n")

        with open(filename, encoding="utf-8") as f:
            line_count = len(f.read().split("\n"))
        print(f"   Generated: {filename} ({line_count} lines)")

    # ------------------------------------------------------------------
    # Formatting helpers
    # ------------------------------------------------------------------

    def _format_key_concepts(self) -> str:
        """Extract key concepts from markdown headings across all sections."""
        all_headings: list[tuple[str, str]] = []
        for section in self.extracted_data.get("pages", []):
            heading = section.get("heading", "").strip()
            level = section.get("heading_level", "h1")
            if heading and len(heading) > 3 and section.get("cell_type") == "markdown":
                all_headings.append((level, heading))
            for sub in section.get("headings", []):
                st = sub.get("text", "").strip()
                if st and len(st) > 3:
                    all_headings.append((sub.get("level", "h3"), st))
        if not all_headings:
            return ""
        content = "## 🔑 Key Concepts\n\n*Main topics covered in this notebook*\n\n"
        h1s = [text for lvl, text in all_headings if lvl == "h1"]
        h2s = [text for lvl, text in all_headings if lvl == "h2"]
        if h1s:
            content += "**Major Topics:**\n\n" + "".join(f"- {h}\n" for h in h1s[:10]) + "\n"
        if h2s:
            content += "**Subtopics:**\n\n" + "".join(f"- {h}\n" for h in h2s[:15]) + "\n"
        return content

    # _format_patterns_from_content is inherited from DocumentSkillBuilder —
    # PATTERN_KEYWORDS above supplies the notebook keyword list.


# ---------------------------------------------------------------------------
# Module-level helpers
# ---------------------------------------------------------------------------
