#!/usr/bin/env python3
"""
GitHub Repository to AI Skill Converter (Tasks C1.1-C1.12)

Converts GitHub repositories into AI skills by extracting:
- README and documentation
- Code structure and signatures
- GitHub Issues, Changelog, and Releases
- Usage examples from tests

Usage:
    skill-seekers github --repo facebook/react
    skill-seekers github --config configs/react_github.json
    skill-seekers github --repo owner/repo --token $GITHUB_TOKEN
"""

import fnmatch
import itertools
import json
import logging
import os
import re
import sys
from pathlib import Path
from typing import Any, Optional

try:
    from github import Github, GithubException, Repository
    from github.GithubException import RateLimitExceededException
except ImportError:
    print("Error: PyGithub not installed. Run: pip install PyGithub")
    sys.exit(1)

from skill_seekers.cli.skill_converter import SkillConverter

# Try to import pathspec for .gitignore support
try:
    import pathspec

    PATHSPEC_AVAILABLE = True
except ImportError:
    PATHSPEC_AVAILABLE = False

logger = logging.getLogger(__name__)

# Import code analyzer for deep code analysis
try:
    from .code_analyzer import CodeAnalyzer

    CODE_ANALYZER_AVAILABLE = True
except ImportError:
    CODE_ANALYZER_AVAILABLE = False
    logger.warning("Code analyzer not available - deep analysis disabled")

# Directories to exclude from local repository analysis
EXCLUDED_DIRS = {
    # Virtual environments
    "venv",
    "env",
    ".venv",
    ".env",
    # Dependencies and caches
    "node_modules",
    "__pycache__",
    ".pytest_cache",
    # Version control
    ".git",
    ".svn",
    ".hg",
    # Build artifacts
    "build",
    "dist",
    "*.egg-info",
    # Coverage reports
    "htmlcov",
    ".coverage",
    # Testing environments
    ".tox",
    ".nox",
    # Linter caches
    ".mypy_cache",
    ".ruff_cache",
    # Unity (critical - contains massive build cache)
    "Library",
    "Temp",
    "Logs",
    "UserSettings",
    "MemoryCaptures",
    "Recordings",
    # Unreal Engine
    "Intermediate",
    "Saved",
    "DerivedDataCache",
    # Godot
    ".godot",
    ".import",
    # Misc
    "tmp",
    ".tmp",
}


def extract_description_from_readme(readme_content: str, repo_name: str) -> str:
    """
    Extract a meaningful description from README content for skill description.

    Parses README to find the first meaningful paragraph that describes
    what the project does, suitable for "Use when..." format.

    Args:
        readme_content: README.md content
        repo_name: Repository name (e.g., 'facebook/react')

    Returns:
        Description string, or improved fallback if extraction fails
    """
    if not readme_content:
        return f"Use when working with {repo_name.split('/')[-1]}"

    try:
        lines = readme_content.split("\n")

        # Skip badges, images, title - find first meaningful text paragraph
        meaningful_paragraph = None
        in_code_block = False

        for _i, line in enumerate(lines):
            stripped = line.strip()

            # Track code blocks
            if stripped.startswith("```"):
                in_code_block = not in_code_block
                continue

            # Skip if in code block
            if in_code_block:
                continue

            # Skip empty lines, badges, images, HTML
            if not stripped or stripped.startswith(("#", "!", "<", "[![", "[![")):
                continue

            # Skip lines that are just links or badges
            if stripped.startswith("[") and "](" in stripped and len(stripped) < 100:
                continue

            # Found a meaningful paragraph - take up to 200 chars
            if len(stripped) > 20:  # Meaningful length
                meaningful_paragraph = stripped
                break

        if meaningful_paragraph:
            # Clean up and extract purpose
            # Remove markdown formatting
            clean = re.sub(r"\[([^\]]+)\]\([^\)]+\)", r"\1", meaningful_paragraph)  # Links
            clean = re.sub(r"[*_`]", "", clean)  # Bold, italic, code
            clean = re.sub(r"<[^>]+>", "", clean)  # HTML tags

            # Truncate if too long (keep first sentence or ~150 chars)
            if ". " in clean:
                first_sentence = clean.split(". ")[0] + "."
                if len(first_sentence) < 200:
                    clean = first_sentence

            if len(clean) > 150:
                clean = clean[:147] + "..."

            # Format as "Use when..." description
            # If it already starts with action words, use as-is
            action_words = ["build", "create", "develop", "work", "use", "implement", "manage"]
            if any(clean.lower().startswith(word) for word in action_words):
                return f"Use when {clean.lower()}"
            else:
                return f"Use when working with {clean.lower()}"

    except Exception as e:
        logger.debug(f"Could not extract description from README: {e}")

    # Improved fallback
    project_name = repo_name.split("/")[-1]
    return f"Use when working with {project_name}"


class GitHubScraper(SkillConverter):
    """
    GitHub Repository Scraper (C1.1-C1.9)

    Extracts repository information for skill generation:
    - Repository structure
    - README files
    - Code comments and docstrings
    - Programming language detection
    - Function/class signatures
    - Test examples
    - GitHub Issues
    - CHANGELOG
    - Releases
    """

    SOURCE_TYPE = "github"

    def __init__(self, config: dict[str, Any], local_repo_path: str | None = None):
        """Initialize GitHub scraper with configuration."""
        super().__init__(config)
        self.config = config
        self.repo_name = config["repo"]
        self.name = config.get("name", self.repo_name.split("/")[-1])
        # Set initial description (will be improved after README extraction if not in config)
        self.description = config.get(
            "description", f"Use when working with {self.repo_name.split('/')[-1]}"
        )

        # Local repository path (optional - enables unlimited analysis)
        self.local_repo_path = local_repo_path or config.get("local_repo_path")
        if self.local_repo_path:
            self.local_repo_path = os.path.expanduser(self.local_repo_path)
            if not os.path.isdir(self.local_repo_path):
                logger.warning(
                    f"local_repo_path does not exist or is not a directory: {self.local_repo_path}"
                )
                logger.warning("Falling back to GitHub API mode (local_repo_path ignored)")
                self.local_repo_path = None
            else:
                logger.info(f"Local repository mode enabled: {self.local_repo_path}")

        # Configure directory exclusions (smart defaults + optional customization)
        self.excluded_dirs = set(EXCLUDED_DIRS)  # Start with smart defaults

        # Option 1: Replace mode - Use only specified exclusions
        if "exclude_dirs" in config:
            self.excluded_dirs = set(config["exclude_dirs"])
            logger.warning(
                f"Using custom directory exclusions ({len(self.excluded_dirs)} dirs) - defaults overridden"
            )
            logger.debug(f"Custom exclusions: {sorted(self.excluded_dirs)}")

        # Option 2: Extend mode - Add to default exclusions
        elif "exclude_dirs_additional" in config:
            additional = set(config["exclude_dirs_additional"])
            self.excluded_dirs = self.excluded_dirs.union(additional)
            logger.info(
                f"Added {len(additional)} custom directory exclusions (total: {len(self.excluded_dirs)})"
            )
            logger.debug(f"Additional exclusions: {sorted(additional)}")

        # Load .gitignore for additional exclusions (C2.1)
        self.gitignore_spec = None
        if self.local_repo_path:
            self.gitignore_spec = self._load_gitignore()

        # GitHub client setup (C1.1)
        token = self._get_token()
        self.github = Github(token) if token else Github()
        self.repo: Repository.Repository | None = None

        # Options
        self.include_issues = config.get("include_issues", True)
        self.max_issues = config.get("max_issues", 100)
        self.max_comments = config.get("max_comments", 0)
        self.issue_since = config.get("issue_since")
        self.issue_labels = config.get("issue_labels", [])
        self.issue_state = config.get("issue_state", "all")
        self.per_issue_files = config.get("per_issue_files", False)
        self.include_changelog = config.get("include_changelog", True)
        self.include_releases = config.get("include_releases", True)
        self.include_code = config.get("include_code", True)
        self.code_analysis_depth = config.get(
            "code_analysis_depth", "deep"
        )  # 'surface', 'deep', 'full'
        self.file_patterns = config.get("file_patterns", [])

        # Initialize code analyzer if deep analysis requested
        self.code_analyzer = None
        if self.code_analysis_depth != "surface" and CODE_ANALYZER_AVAILABLE:
            self.code_analyzer = CodeAnalyzer(depth=self.code_analysis_depth)
            logger.info(f"Code analysis depth: {self.code_analysis_depth}")

        # Output paths
        # skill_dir is resolved once in SkillConverter.__init__
        self.data_file = self.data_file_for("_github_data.json")

        # Extracted data storage
        self.extracted_data = {
            "repo_info": {},
            "readme": "",
            "file_tree": [],
            "languages": {},
            "signatures": [],
            "test_examples": [],
            "issues": [],
            "changelog": "",
            "releases": [],
        }

    def _get_token(self) -> str | None:
        """
        Get GitHub token from env var or config (both options supported).
        Priority: GITHUB_TOKEN env var > config file > None
        """
        # Try environment variable first (recommended)
        token = os.getenv("GITHUB_TOKEN")
        if token:
            logger.info("Using GitHub token from GITHUB_TOKEN environment variable")
            return token

        # Fall back to config file
        token = self.config.get("github_token")
        if token:
            logger.warning("Using GitHub token from config file (less secure)")
            return token

        logger.warning(
            "No GitHub token provided - using unauthenticated access (lower rate limits)"
        )
        return None

    def scrape(self) -> dict[str, Any]:
        """
        Main scraping entry point.
        Executes all C1 tasks in sequence.
        """
        try:
            logger.info(f"Starting GitHub scrape for: {self.repo_name}")

            # C1.1: Fetch repository
            self._fetch_repository()

            # C1.2: Extract README
            self._extract_readme()

            # C1.3-C1.6: Extract code structure
            self._extract_code_structure()

            # C1.7: Extract Issues
            if self.include_issues:
                self._extract_issues()

            # C1.8: Extract CHANGELOG
            if self.include_changelog:
                self._extract_changelog()

            # C1.9: Extract Releases
            if self.include_releases:
                self._extract_releases()

            # Save extracted data
            self._save_data()

            logger.info(f"✅ Scraping complete! Data saved to: {self.data_file}")
            return self.extracted_data

        except RateLimitExceededException:
            logger.error("GitHub API rate limit exceeded. Please wait or use authentication token.")
            raise
        except GithubException as e:
            logger.error(f"GitHub API error: {e}")
            raise
        except Exception as e:
            logger.error(f"Unexpected error during scraping: {e}")
            raise

    def extract(self):
        """SkillConverter interface — delegates to scrape()."""
        self.scrape()

    def build_skill(self):
        """SkillConverter interface — delegates to GitHubToSkillConverter."""
        converter = GitHubToSkillConverter(self.config)
        converter.build_skill()

    def _fetch_repository(self):
        """C1.1: Fetch repository structure using GitHub API."""
        logger.info(f"Fetching repository: {self.repo_name}")

        try:
            self.repo = self.github.get_repo(self.repo_name)

            # Extract basic repo info
            self.extracted_data["repo_info"] = {
                "name": self.repo.name,
                "full_name": self.repo.full_name,
                "description": self.repo.description,
                "url": self.repo.html_url,
                "homepage": self.repo.homepage,
                "stars": self.repo.stargazers_count,
                "forks": self.repo.forks_count,
                "open_issues": self.repo.open_issues_count,
                "default_branch": self.repo.default_branch,
                "created_at": self.repo.created_at.isoformat() if self.repo.created_at else None,
                "updated_at": self.repo.updated_at.isoformat() if self.repo.updated_at else None,
                "language": self.repo.language,
                "license": self.repo.license.name if self.repo.license else None,
                "topics": self.repo.get_topics(),
            }

            logger.info(
                f"Repository fetched: {self.repo.full_name} ({self.repo.stargazers_count} stars)"
            )

        except GithubException as e:
            if e.status == 404:
                raise ValueError(f"Repository not found: {self.repo_name}") from e
            raise

    def _get_file_content(self, file_path: str) -> str | None:
        """
        Safely get file content, handling symlinks and encoding issues.

        Args:
            file_path: Path to file in repository

        Returns:
            File content as string, or None if file not found/error
        """
        try:
            content = self.repo.get_contents(file_path)
            if not content:
                return None

            # Handle symlinks - follow the target to get actual file
            if hasattr(content, "type") and content.type == "symlink":
                target = getattr(content, "target", None)
                if target:
                    target = target.strip()
                    logger.debug(f"File {file_path} is a symlink to {target}, following...")
                    try:
                        content = self.repo.get_contents(target)
                    except GithubException as e:
                        logger.warning(f"Failed to follow symlink {file_path} -> {target}: {e}")
                        return None
                else:
                    logger.warning(f"Symlink {file_path} has no target")
                    return None

            # Handle large files (encoding="none") - download via URL
            # GitHub API doesn't base64-encode files >1MB
            if hasattr(content, "encoding") and content.encoding in [None, "none"]:
                download_url = getattr(content, "download_url", None)
                file_size = getattr(content, "size", 0)

                if download_url:
                    logger.info(
                        f"File {file_path} is large ({file_size:,} bytes), downloading via URL..."
                    )
                    try:
                        import requests

                        response = requests.get(download_url, timeout=30)
                        response.raise_for_status()
                        return response.text
                    except Exception as e:
                        logger.warning(f"Failed to download {file_path} from {download_url}: {e}")
                        return None
                else:
                    logger.warning(
                        f"File {file_path} has no download URL (encoding={content.encoding})"
                    )
                    return None

            # Handle regular files - decode content
            try:
                if isinstance(content.decoded_content, bytes):
                    return content.decoded_content.decode("utf-8")
                else:
                    return str(content.decoded_content)
            except (UnicodeDecodeError, AttributeError, LookupError, AssertionError) as e:
                logger.warning(f"Encoding issue with {file_path}: {e}")
                # Try alternative encoding
                try:
                    if isinstance(content.decoded_content, bytes):
                        return content.decoded_content.decode("latin-1")
                except Exception:
                    return None
                return None

        except GithubException:
            return None
        except Exception as e:
            logger.warning(f"Error reading {file_path}: {e}")
            return None

    def _extract_readme(self):
        """C1.2: Extract README.md files."""
        logger.info("Extracting README...")

        # Try common README locations
        readme_files = [
            "README.md",
            "README.rst",
            "README.txt",
            "README",
            "docs/README.md",
            ".github/README.md",
        ]

        for readme_path in readme_files:
            readme_content = self._get_file_content(readme_path)
            if readme_content:
                self.extracted_data["readme"] = readme_content
                logger.info(f"README found: {readme_path}")

                # Update description if not explicitly set in config
                if "description" not in self.config:
                    smart_description = extract_description_from_readme(
                        self.extracted_data["readme"], self.repo_name
                    )
                    self.description = smart_description
                    logger.debug(f"Generated description: {self.description}")

                return

        logger.warning("No README found in repository")

    def _extract_code_structure(self):
        """
        C1.3-C1.6: Extract code structure, languages, signatures, and test examples.
        Surface layer only - no full implementation code.
        """
        logger.info("Extracting code structure...")

        # C1.4: Get language breakdown
        self._extract_languages()

        # Get file tree
        self._extract_file_tree()

        # Extract signatures and test examples
        if self.include_code:
            self._extract_signatures_and_tests()

    def _extract_languages(self):
        """C1.4: Detect programming languages in repository."""
        logger.info("Detecting programming languages...")

        try:
            languages = self.repo.get_languages()
            # Filter out non-integer metadata (e.g., "url" key from some API configurations)
            non_lang_keys = {k for k, v in languages.items() if not isinstance(v, int)}
            if non_lang_keys:
                logger.debug(f"Filtered non-language keys from API response: {non_lang_keys}")
                languages = {k: v for k, v in languages.items() if isinstance(v, int)}
            total_bytes = sum(languages.values())

            self.extracted_data["languages"] = {
                lang: {
                    "bytes": bytes_count,
                    "percentage": round((bytes_count / total_bytes) * 100, 2)
                    if total_bytes > 0
                    else 0,
                }
                for lang, bytes_count in languages.items()
            }

            logger.info(f"Languages detected: {', '.join(languages.keys())}")

        except GithubException as e:
            logger.warning(f"Could not fetch languages: {e}")

    def should_exclude_dir(self, dir_name: str, dir_path: str = None) -> bool:
        """
        Check if directory should be excluded from analysis.

        Args:
            dir_name: Directory name (e.g., "Examples & Extras")
            dir_path: Full relative path (e.g., "TextMesh Pro/Examples & Extras")

        Returns:
            True if directory should be excluded
        """
        # Check directory name
        if dir_name in self.excluded_dirs or dir_name.startswith("."):
            return True

        # Check full path if provided (for nested exclusions like "TextMesh Pro/Examples & Extras")
        if dir_path:
            for excluded in self.excluded_dirs:
                # Match if path contains the exclusion pattern
                if excluded in dir_path or dir_path.startswith(excluded):
                    return True

        # Check .gitignore rules if available (C2.1)
        if self.gitignore_spec and dir_path:
            # For directories, we need to check both with and without trailing slash
            # as .gitignore patterns can match either way
            dir_path_with_slash = dir_path if dir_path.endswith("/") else dir_path + "/"
            if self.gitignore_spec.match_file(dir_path) or self.gitignore_spec.match_file(
                dir_path_with_slash
            ):
                logger.debug(f"Directory excluded by .gitignore: {dir_path}")
                return True

        return False

    def _load_gitignore(self) -> Optional["pathspec.PathSpec"]:
        """
        Load .gitignore file and create pathspec matcher (C2.1).

        Returns:
            PathSpec object if .gitignore found, None otherwise
        """
        if not PATHSPEC_AVAILABLE:
            logger.warning("pathspec not installed - .gitignore support disabled")
            logger.warning("Install with: pip install pathspec")
            return None

        if not self.local_repo_path:
            return None

        gitignore_path = Path(self.local_repo_path) / ".gitignore"
        if not gitignore_path.exists():
            logger.debug(f"No .gitignore found in {self.local_repo_path}")
            return None

        try:
            with open(gitignore_path, encoding="utf-8") as f:
                spec = pathspec.PathSpec.from_lines("gitwildmatch", f)
            logger.info(f"Loaded .gitignore from {gitignore_path}")
            return spec
        except Exception as e:
            logger.warning(f"Failed to load .gitignore: {e}")
            return None

    def _extract_file_tree(self):
        """Extract repository file tree structure (dual-mode: GitHub API or local filesystem)."""
        logger.info("Building file tree...")

        if self.local_repo_path:
            # Local filesystem mode - unlimited files
            self._extract_file_tree_local()
        else:
            # GitHub API mode - limited by API rate limits
            self._extract_file_tree_github()

    def _extract_file_tree_local(self):
        """Extract file tree from local filesystem (unlimited files)."""
        if not os.path.exists(self.local_repo_path):
            logger.error(f"Local repository path not found: {self.local_repo_path}")
            return

        # Log exclusions for debugging
        logger.info(
            f"Directory exclusions ({len(self.excluded_dirs)} total): {sorted(list(self.excluded_dirs)[:10])}"
        )

        file_tree = []
        excluded_count = 0
        for root, dirs, files in os.walk(self.local_repo_path):
            # Calculate relative path from repo root first (needed for exclusion checks)
            rel_root = os.path.relpath(root, self.local_repo_path)
            if rel_root == ".":
                rel_root = ""

            # Exclude directories in-place to prevent os.walk from descending into them
            # Pass both dir name and full path for path-based exclusions
            filtered_dirs = []
            for d in dirs:
                dir_path = os.path.join(rel_root, d) if rel_root else d
                if self.should_exclude_dir(d, dir_path):
                    excluded_count += 1
                    logger.debug(f"Excluding directory: {dir_path}")
                else:
                    filtered_dirs.append(d)
            dirs[:] = filtered_dirs

            # Add directories
            for dir_name in dirs:
                dir_path = os.path.join(rel_root, dir_name) if rel_root else dir_name
                file_tree.append({"path": dir_path, "type": "dir", "size": None})

            # Add files
            for file_name in files:
                file_path = os.path.join(rel_root, file_name) if rel_root else file_name
                full_path = os.path.join(root, file_name)
                try:
                    file_size = os.path.getsize(full_path)
                except OSError:
                    file_size = None

                file_tree.append({"path": file_path, "type": "file", "size": file_size})

        self.extracted_data["file_tree"] = file_tree
        logger.info(
            f"File tree built (local mode): {len(file_tree)} items ({excluded_count} directories excluded)"
        )

    def _extract_file_tree_github(self):
        """Extract file tree from GitHub API (rate-limited)."""
        try:
            from collections import deque

            # Cap the walk: each directory is a separate API call, so an unbounded
            # tree on a huge repo could exhaust the rate-limit quota.
            max_tree_items = 5000
            contents = deque(self.repo.get_contents(""))
            file_tree = []
            truncated = False

            while contents:
                if len(file_tree) >= max_tree_items:
                    truncated = True
                    break
                file_content = contents.popleft()

                file_info = {
                    "path": file_content.path,
                    "type": file_content.type,
                    "size": file_content.size if file_content.type == "file" else None,
                }
                file_tree.append(file_info)

                if file_content.type == "dir":
                    contents.extend(self.repo.get_contents(file_content.path))

            self.extracted_data["file_tree"] = file_tree
            if truncated:
                logger.warning(
                    "File tree truncated at %d items to preserve API quota", max_tree_items
                )
            logger.info(f"File tree built (GitHub API mode): {len(file_tree)} items")

        except GithubException as e:
            logger.warning(f"Could not build file tree: {e}")

    def _extract_signatures_and_tests(self):
        """
        C1.3, C1.5, C1.6: Extract signatures, docstrings, and test examples.

        Extraction depth depends on code_analysis_depth setting:
        - surface: File tree only (minimal)
        - deep: Parse files for signatures, parameters, types
        - full: Complete AST analysis (future enhancement)
        """
        if self.code_analysis_depth == "surface":
            logger.info("Code extraction: Surface level (file tree only)")
            return

        if not self.code_analyzer:
            logger.warning("Code analyzer not available - skipping deep analysis")
            return

        logger.info(f"Extracting code signatures ({self.code_analysis_depth} analysis)...")

        # Build reverse extension → language map for per-file detection
        extension_map = {
            "Python": [".py"],
            "JavaScript": [".js", ".jsx"],
            "TypeScript": [".ts", ".tsx"],
            "Kotlin": [".kt", ".kts"],
            "Java": [".java"],
            "C": [".c", ".h"],
            "C++": [".cpp", ".hpp", ".cc", ".hh", ".cxx"],
            "C#": [".cs"],
            "Go": [".go"],
            "Rust": [".rs"],
            "Swift": [".swift"],
            "Ruby": [".rb"],
            "PHP": [".php"],
            "GDScript": [".gd"],
        }
        ext_to_lang = {}
        for lang, exts in extension_map.items():
            for ext in exts:
                ext_to_lang[ext] = lang

        # Optional: filter to specific languages from config
        target_languages = None
        config_language = self.config.get("language", "")
        if config_language:
            target_languages = {lang.strip() for lang in config_language.split(",")}
            logger.info(f"Language filter from config: {', '.join(sorted(target_languages))}")

        # Analyze ALL files, detecting language per-file from extension
        analyzed_files = []
        file_tree = self.extracted_data.get("file_tree", [])
        languages_found = set()

        for file_info in file_tree:
            file_path = file_info["path"]

            # Detect language from file extension
            ext = os.path.splitext(file_path)[1].lower()
            language = ext_to_lang.get(ext)
            if not language:
                continue

            # Apply language filter if config specifies target languages
            if target_languages and language not in target_languages:
                continue

            # Check if file matches patterns (if specified in config)
            if self.file_patterns and not any(
                fnmatch.fnmatch(file_path, pattern) for pattern in self.file_patterns
            ):
                continue

            # Analyze this file with the correct language
            try:
                if self.local_repo_path:
                    full_path = os.path.join(self.local_repo_path, file_path)
                    with open(full_path, encoding="utf-8") as f:
                        content = f.read()
                else:
                    file_content = self.repo.get_contents(file_path)
                    content = file_content.decoded_content.decode("utf-8")

                analysis_result = self.code_analyzer.analyze_file(file_path, content, language)

                if analysis_result and (
                    analysis_result.get("classes") or analysis_result.get("functions")
                ):
                    analyzed_files.append(
                        {"file": file_path, "language": language, **analysis_result}
                    )
                    languages_found.add(language)

            except Exception as e:
                logger.debug(f"Could not analyze {file_path}: {e}")
                continue

        # Determine primary language for backward compat in output
        repo_languages = self.extracted_data.get("languages", {})
        primary_language = (
            max(repo_languages.items(), key=lambda x: x[1]["bytes"])[0]
            if repo_languages
            else "Unknown"
        )

        self.extracted_data["code_analysis"] = {
            "depth": self.code_analysis_depth,
            "language": primary_language,
            "languages_analyzed": sorted(languages_found),
            "files_analyzed": len(analyzed_files),
            "files": analyzed_files,
        }

        # Calculate totals
        total_classes = sum(len(f.get("classes", [])) for f in analyzed_files)
        total_functions = sum(len(f.get("functions", [])) for f in analyzed_files)

        lang_summary = ", ".join(sorted(languages_found)) if languages_found else "none"
        logger.info(
            f"Code analysis complete: {len(analyzed_files)} files, {total_classes} classes, {total_functions} functions ({lang_summary})"
        )

    def _extract_issues(self):
        """C1.7: Extract GitHub Issues (open/closed, labels, milestones)."""
        logger.info(f"Extracting GitHub Issues (max {self.max_issues})...")

        try:
            # Build kwargs for get_issues
            kwargs: dict[str, Any] = {
                "state": self.issue_state or "all",
                "sort": "updated",
                "direction": "desc",
            }
            if self.issue_labels:
                kwargs["labels"] = self.issue_labels
            if self.issue_since:
                from datetime import datetime

                since_str = (
                    self.issue_since[:-1] + "+00:00"
                    if self.issue_since.endswith("Z")
                    else self.issue_since
                )
                kwargs["since"] = datetime.fromisoformat(since_str)

            # Fetch issues with filters
            issues = self.repo.get_issues(**kwargs)

            issue_list = []
            for issue in itertools.islice(issues, self.max_issues):
                # Skip pull requests (they appear in issues)
                if issue.pull_request:
                    continue

                # Fetch comments only when explicitly requested
                comments_list = []
                if self.max_comments > 0:
                    try:
                        for comment in issue.get_comments()[: self.max_comments]:
                            comments_list.append(
                                {
                                    "author": comment.user.login if comment.user else "unknown",
                                    "created_at": (
                                        comment.created_at.isoformat()
                                        if comment.created_at
                                        else None
                                    ),
                                    "body": comment.body,
                                }
                            )
                    except Exception as e:
                        logger.debug(f"Could not fetch comments for issue #{issue.number}: {e}")

                # data.json body field: with per_issue_files=True the full body is
                # mirrored here AND in references/issues/*.md. With per_issue_files=False
                # we keep only a 500-char preview to keep data.json small. Downstream
                # packagers needing the full text should fetch the live issue.
                issue_data = {
                    "number": issue.number,
                    "title": issue.title,
                    "state": issue.state,
                    "labels": [label.name for label in issue.labels],
                    "milestone": issue.milestone.title if issue.milestone else None,
                    "created_at": issue.created_at.isoformat() if issue.created_at else None,
                    "updated_at": issue.updated_at.isoformat() if issue.updated_at else None,
                    "closed_at": issue.closed_at.isoformat() if issue.closed_at else None,
                    "url": issue.html_url,
                    "body": (
                        issue.body or None
                        if self.per_issue_files
                        else (issue.body[:500] if issue.body else None)
                    ),
                    "comments": comments_list,
                }
                issue_list.append(issue_data)

            self.extracted_data["issues"] = issue_list
            logger.info(f"Extracted {len(issue_list)} issues")

        except GithubException as e:
            logger.warning(f"Could not fetch issues: {e}")

    def _extract_changelog(self):
        """C1.8: Extract CHANGELOG.md and release notes."""
        logger.info("Extracting CHANGELOG...")

        # Try common changelog locations
        changelog_files = [
            "CHANGELOG.md",
            "CHANGES.md",
            "HISTORY.md",
            "CHANGELOG.rst",
            "CHANGELOG.txt",
            "CHANGELOG",
            "docs/CHANGELOG.md",
            ".github/CHANGELOG.md",
        ]

        for changelog_path in changelog_files:
            changelog_content = self._get_file_content(changelog_path)
            if changelog_content:
                self.extracted_data["changelog"] = changelog_content
                logger.info(f"CHANGELOG found: {changelog_path}")
                return

        logger.warning("No CHANGELOG found in repository")

    def _extract_releases(self):
        """C1.9: Extract GitHub Releases with version history."""
        logger.info("Extracting GitHub Releases...")

        try:
            releases = self.repo.get_releases()

            release_list = []
            for release in releases:
                release_data = {
                    "tag_name": release.tag_name,
                    "name": release.title,
                    "body": release.body,
                    "draft": release.draft,
                    "prerelease": release.prerelease,
                    "created_at": release.created_at.isoformat() if release.created_at else None,
                    "published_at": release.published_at.isoformat()
                    if release.published_at
                    else None,
                    "url": release.html_url,
                    "tarball_url": release.tarball_url,
                    "zipball_url": release.zipball_url,
                }
                release_list.append(release_data)

            self.extracted_data["releases"] = release_list
            logger.info(f"Extracted {len(release_list)} releases")

        except GithubException as e:
            logger.warning(f"Could not fetch releases: {e}")

    def _save_data(self):
        """Save extracted data to JSON file."""
        # Create the actual parent of data_file. self.data_file derives from
        # self.skill_dir (config["output_dir"]), so with a nested --output the
        # parent isn't "output/" — hardcoding it raised FileNotFoundError here
        # (this runs during extract, before build_skill makes skill_dir).
        os.makedirs(os.path.dirname(self.data_file) or ".", exist_ok=True)

        with open(self.data_file, "w", encoding="utf-8") as f:
            json.dump(self.extracted_data, f, indent=2, ensure_ascii=False)

        logger.info(f"Data saved to: {self.data_file}")


class GitHubToSkillConverter:
    """
    Convert extracted GitHub data to AI skill format (C1.10).

    Builder strategy invoked by GitHubScraper (the registered SkillConverter
    for "github") and by UnifiedScraper._scrape_github() to build the
    standalone sub-skill. Not a SkillConverter itself: it has no extract()
    phase — it only consumes the already-written {skill_dir}_github_data.json
    produced by GitHubScraper.scrape(), so it sits outside the run() template
    (extract → build_skill) by design.
    """

    def __init__(self, config: dict[str, Any]):
        """Initialize converter with configuration."""
        self.config = config
        self.name = config.get("name", config["repo"].split("/")[-1])
        self.repo_owner = config["repo"].split("/")[0]
        self.repo_short_name = config["repo"].split("/")[-1]
        self.per_issue_files = config.get("per_issue_files", False)

        # Paths
        self.skill_dir = SkillConverter.resolve_skill_dir(config, self.name)
        self.data_file = f"{self.skill_dir}_github_data.json"

        # Load extracted data
        self.data = self._load_data()

        # Set description (smart extraction from README if available)
        if "description" in config:
            self.description = config["description"]
        else:
            # Try to extract from README in loaded data
            readme_content = self.data.get("readme", "")
            repo_name = config["repo"]
            if readme_content:
                self.description = extract_description_from_readme(readme_content, repo_name)
            else:
                self.description = f"Use when working with {repo_name.split('/')[-1]}"

    def _load_data(self) -> dict[str, Any]:
        """Load extracted GitHub data from JSON."""
        if not os.path.exists(self.data_file):
            raise FileNotFoundError(f"Data file not found: {self.data_file}")

        with open(self.data_file, encoding="utf-8") as f:
            return json.load(f)

    def build_skill(self):
        """Build complete skill structure."""
        logger.info(f"Building skill for: {self.name}")

        # Create directories
        os.makedirs(self.skill_dir, exist_ok=True)
        os.makedirs(f"{self.skill_dir}/references", exist_ok=True)
        os.makedirs(f"{self.skill_dir}/scripts", exist_ok=True)
        os.makedirs(f"{self.skill_dir}/assets", exist_ok=True)

        # Generate SKILL.md
        self._generate_skill_md()

        # Generate reference files
        self._generate_references()

        logger.info(f"✅ Skill built successfully: {self.skill_dir}/")

    def _generate_skill_md(self):
        """Generate main SKILL.md file (rich version with C3.x data if available)."""
        repo_info = self.data.get("repo_info", {})
        c3_data = self.data.get("c3_analysis", {})
        has_c3_data = bool(c3_data)

        # Generate skill name (lowercase, hyphens only, max 64 chars)
        skill_name = self.name.lower().replace("_", "-").replace(" ", "-")[:64]

        # Truncate description to 1024 chars if needed
        desc = self.description[:1024] if len(self.description) > 1024 else self.description

        doc_version = self.config.get("doc_version", "")

        # Build skill content
        skill_content = f"""---
name: {skill_name}
description: {desc}
doc_version: {doc_version}
---

# {repo_info.get("name", self.name)}

{self.description}

## Description

{repo_info.get("description", "GitHub repository skill")}

**Repository:** [{repo_info.get("full_name", "N/A")}]({repo_info.get("url", "#")})
**Language:** {repo_info.get("language", "N/A")}
**Stars:** {repo_info.get("stars", 0):,}
**License:** {repo_info.get("license", "N/A")}

## When to Use This Skill

Use this skill when you need to:
- Understand how to use {repo_info.get("name", self.name)}
- Look up API documentation and implementation details
- Find real-world usage examples from the codebase
- Review design patterns and architecture
- Check for known issues or recent changes
- Explore release history and changelogs
"""

        # Add Quick Reference section (enhanced with C3.x if available)
        skill_content += "\n## ⚡ Quick Reference\n\n"

        # Repository info
        skill_content += "### Repository Info\n"
        skill_content += f"- **Homepage:** {repo_info.get('homepage') or 'N/A'}\n"
        skill_content += f"- **Topics:** {', '.join(repo_info.get('topics', []))}\n"
        skill_content += f"- **Open Issues:** {repo_info.get('open_issues', 0)}\n"
        updated_at = repo_info.get("updated_at") or "N/A"
        skill_content += f"- **Last Updated:** {updated_at[:10]}\n\n"

        # Languages
        skill_content += "### Languages\n"
        skill_content += self._format_languages() + "\n\n"

        # Add C3.x pattern summary if available
        if has_c3_data and c3_data.get("patterns"):
            skill_content += self._format_pattern_summary(c3_data)

        # Add code examples if available (C3.2 test examples)
        if has_c3_data and c3_data.get("test_examples"):
            skill_content += self._format_code_examples(c3_data)

        # Add API Reference if available (C2.5)
        if has_c3_data and c3_data.get("api_reference"):
            skill_content += self._format_api_reference(c3_data)

        # Add Architecture Overview if available (C3.7)
        if has_c3_data and c3_data.get("architecture"):
            skill_content += self._format_architecture(c3_data)

        # Add Known Issues section
        skill_content += self._format_known_issues()

        # Add Recent Releases
        skill_content += "### Recent Releases\n"
        skill_content += self._format_recent_releases() + "\n\n"

        # Available References
        skill_content += "## 📖 Available References\n\n"
        skill_content += "- `references/README.md` - Complete README documentation\n"
        skill_content += "- `references/CHANGELOG.md` - Version history and changes\n"
        skill_content += "- `references/issues.md` - Recent GitHub issues\n"
        if self.per_issue_files:
            skill_content += (
                "- `references/issues/` - Per-issue markdown files with YAML frontmatter\n"
            )
        skill_content += "- `references/releases.md` - Release notes\n"
        skill_content += "- `references/file_structure.md` - Repository structure\n"

        if has_c3_data:
            skill_content += "\n### Codebase Analysis References\n\n"
            if c3_data.get("patterns"):
                skill_content += (
                    "- `references/codebase_analysis/patterns/` - Design patterns detected\n"
                )
            if c3_data.get("test_examples"):
                skill_content += (
                    "- `references/codebase_analysis/examples/` - Test examples extracted\n"
                )
            if c3_data.get("config_patterns"):
                skill_content += (
                    "- `references/codebase_analysis/configuration/` - Configuration analysis\n"
                )
            if c3_data.get("architecture"):
                skill_content += (
                    "- `references/codebase_analysis/ARCHITECTURE.md` - Architecture overview\n"
                )

        # Usage
        skill_content += "\n## 💻 Usage\n\n"
        skill_content += "See README.md for complete usage instructions and examples.\n\n"

        # Footer
        skill_content += "---\n\n"
        if has_c3_data:
            skill_content += "**Generated by Skill Seeker** | GitHub Repository Scraper with C3.x Codebase Analysis\n"
        else:
            skill_content += "**Generated by Skill Seeker** | GitHub Repository Scraper\n"

        # Write to file
        skill_path = f"{self.skill_dir}/SKILL.md"
        with open(skill_path, "w", encoding="utf-8") as f:
            f.write(skill_content)

        line_count = len(skill_content.split("\n"))
        logger.info(f"Generated: {skill_path} ({line_count} lines)")

    def _format_languages(self) -> str:
        """Format language breakdown."""
        languages = self.data.get("languages", {})
        if not languages:
            return "No language data available"

        lines = []
        for lang, info in sorted(languages.items(), key=lambda x: x[1]["bytes"], reverse=True):
            lines.append(f"- **{lang}:** {info['percentage']:.1f}%")

        return "\n".join(lines)

    def _format_recent_releases(self) -> str:
        """Format recent releases (top 3)."""
        releases = self.data.get("releases", [])
        if not releases:
            return "No releases available"

        lines = []
        for release in releases[:3]:
            published_at = release.get("published_at") or "N/A"
            release_name = release.get("name") or release["tag_name"]
            lines.append(f"- **{release['tag_name']}** ({published_at[:10]}): {release_name}")

        return "\n".join(lines)

    def _format_pattern_summary(self, c3_data: dict[str, Any]) -> str:
        """Format design patterns summary (C3.1)."""
        patterns_data = c3_data.get("patterns", [])
        if not patterns_data:
            return ""

        # Count patterns by type (deduplicate by class, keep highest confidence)
        pattern_counts = {}
        by_class = {}

        for pattern_file in patterns_data:
            for pattern in pattern_file.get("patterns", []):
                ptype = pattern.get("pattern_type", "Unknown")
                cls = pattern.get("class_name", "")
                confidence = pattern.get("confidence", 0)

                # Skip low confidence
                if confidence < 0.7:
                    continue

                # Deduplicate by class
                key = f"{cls}:{ptype}"
                if key not in by_class or by_class[key]["confidence"] < confidence:
                    by_class[key] = pattern

                # Count by type
                pattern_counts[ptype] = pattern_counts.get(ptype, 0) + 1

        if not pattern_counts:
            return ""

        content = "### Design Patterns Detected\n\n"
        content += "*From C3.1 codebase analysis (confidence > 0.7)*\n\n"

        # Top 5 pattern types
        for ptype, count in sorted(pattern_counts.items(), key=lambda x: x[1], reverse=True)[:5]:
            content += f"- **{ptype}**: {count} instances\n"

        content += f"\n*Total: {len(by_class)} high-confidence patterns*\n\n"
        return content

    def _format_code_examples(self, c3_data: dict[str, Any]) -> str:
        """Format code examples (C3.2)."""
        examples_data = c3_data.get("test_examples", {})
        examples = examples_data.get("examples", [])

        if not examples:
            return ""

        # Filter high-value examples (complexity > 0.7)
        high_value = [ex for ex in examples if ex.get("complexity_score", 0) > 0.7]

        if not high_value:
            return ""

        content = "## 📝 Code Examples\n\n"
        content += "*High-quality examples from codebase (C3.2)*\n\n"

        # Top 10 examples
        for ex in sorted(high_value, key=lambda x: x.get("complexity_score", 0), reverse=True)[:10]:
            desc = ex.get("description", "Example")
            lang = ex.get("language", "python")
            code = ex.get("code", "")
            complexity = ex.get("complexity_score", 0)

            content += f"**{desc}** (complexity: {complexity:.2f})\n\n"
            content += f"```{lang}\n{code}\n```\n\n"

        return content

    def _format_api_reference(self, c3_data: dict[str, Any]) -> str:
        """Format API reference (C2.5)."""
        api_ref = c3_data.get("api_reference", {})

        if not api_ref:
            return ""

        content = "## 🔧 API Reference\n\n"
        content += "*Extracted from codebase analysis (C2.5)*\n\n"

        # Top 5 modules
        for module_name, module_md in list(api_ref.items())[:5]:
            content += f"### {module_name}\n\n"
            # First 500 chars of module documentation
            content += module_md[:500]
            if len(module_md) > 500:
                content += "...\n\n"
            else:
                content += "\n\n"

        content += "*See `references/codebase_analysis/api_reference/` for complete API docs*\n\n"
        return content

    def _format_architecture(self, c3_data: dict[str, Any]) -> str:
        """Format architecture overview (C3.7)."""
        arch_data = c3_data.get("architecture", {})

        if not arch_data:
            return ""

        content = "## 🏗️ Architecture Overview\n\n"
        content += "*From C3.7 codebase analysis*\n\n"

        # Architecture patterns
        patterns = arch_data.get("patterns", [])
        if patterns:
            content += "**Architectural Patterns:**\n"
            for pattern in patterns[:5]:
                content += (
                    f"- {pattern.get('name', 'Unknown')}: {pattern.get('description', 'N/A')}\n"
                )
            content += "\n"

        # Dependencies (C2.6)
        dep_data = c3_data.get("dependency_graph", {})
        if dep_data:
            total_deps = dep_data.get("total_dependencies", 0)
            circular = len(dep_data.get("circular_dependencies", []))
            if total_deps > 0:
                content += f"**Dependencies:** {total_deps} total"
                if circular > 0:
                    content += f" (⚠️  {circular} circular dependencies detected)"
                content += "\n\n"

        content += "*See `references/codebase_analysis/ARCHITECTURE.md` for complete overview*\n\n"
        return content

    def _format_known_issues(self) -> str:
        """Format known issues from GitHub."""
        issues = self.data.get("issues", [])

        if not issues:
            return ""

        content = "## ⚠️ Known Issues\n\n"
        content += "*Recent issues from GitHub*\n\n"

        # Top 5 issues
        for issue in issues[:5]:
            title = issue.get("title", "Untitled")
            number = issue.get("number", 0)
            labels = ", ".join(issue.get("labels", []))
            content += f"- **#{number}**: {title}"
            if labels:
                content += f" [`{labels}`]"
            content += "\n"

        content += "\n*See `references/issues.md` for complete list*\n\n"
        return content

    def _generate_references(self):
        """Generate all reference files."""
        # README
        if self.data.get("readme"):
            readme_path = f"{self.skill_dir}/references/README.md"
            with open(readme_path, "w", encoding="utf-8") as f:
                f.write(self.data["readme"])
            logger.info(f"Generated: {readme_path}")

        # CHANGELOG
        if self.data.get("changelog"):
            changelog_path = f"{self.skill_dir}/references/CHANGELOG.md"
            with open(changelog_path, "w", encoding="utf-8") as f:
                f.write(self.data["changelog"])
            logger.info(f"Generated: {changelog_path}")

        # Issues
        if self.data.get("issues"):
            self._generate_issues_reference()

        # Releases
        if self.data.get("releases"):
            self._generate_releases_reference()

        # File structure
        if self.data.get("file_tree"):
            self._generate_file_structure_reference()

    def _generate_issues_reference(self):
        """Generate issues.md summary and per-issue reference files."""
        issues = self.data["issues"]

        # --- Summary file (issues.md) ---
        content = f"# GitHub Issues\n\nRecent issues from the repository ({len(issues)} total).\n\n"

        # Group by state
        open_issues = [i for i in issues if i["state"] == "open"]
        closed_issues = [i for i in issues if i["state"] == "closed"]

        content += f"## Open Issues ({len(open_issues)})\n\n"
        for issue in open_issues:
            labels = ", ".join(issue["labels"]) if issue["labels"] else "No labels"
            created_at = issue.get("created_at") or "N/A"
            content += f"### #{issue['number']}: {issue['title']}\n"
            content += f"**Labels:** {labels} | **Created:** {created_at[:10]}\n"
            content += f"[View on GitHub]({issue['url']})\n\n"

        content += f"\n## Recently Closed Issues ({len(closed_issues)})\n\n"
        for issue in closed_issues:
            labels = ", ".join(issue["labels"]) if issue["labels"] else "No labels"
            closed_at = issue.get("closed_at") or "N/A"
            content += f"### #{issue['number']}: {issue['title']}\n"
            content += f"**Labels:** {labels} | **Closed:** {closed_at[:10]}\n"
            content += f"[View on GitHub]({issue['url']})\n\n"

        issues_path = f"{self.skill_dir}/references/issues.md"
        with open(issues_path, "w", encoding="utf-8") as f:
            f.write(content)
        logger.info(f"Generated: {issues_path}")

        # --- Per-issue files (opt-in) ---
        if self.per_issue_files:
            os.makedirs(f"{self.skill_dir}/references/issues", exist_ok=True)
            for issue in issues:
                self._write_per_issue_file(issue)
            logger.info(f"Generated {len(issues)} per-issue files")

    def _write_per_issue_file(self, issue: dict):
        """Write a single per-issue markdown file with YAML frontmatter.

        Args:
            issue: Issue data dict with number, title, body, comments, etc.
        """
        number = issue["number"]
        title = issue["title"]
        state = issue["state"]
        labels = issue.get("labels", [])
        created_at = issue.get("created_at") or "N/A"
        updated_at = issue.get("updated_at") or "N/A"
        url = issue.get("url", "")
        body = issue.get("body") or ""
        comments = issue.get("comments", [])

        # Escape title for YAML (wrap in quotes)
        yaml_title = title.replace('"', '\\"')
        labels_yaml = json.dumps(labels)

        file_content = f"""---
type: github_issue
issue_number: {number}
title: "{yaml_title}"
state: {state}
labels: {labels_yaml}
created_at: "{created_at}"
updated_at: "{updated_at}"
url: "{url}"
---

# Issue #{number}: {title}

**State:** {state} | **Labels:** {", ".join(labels) if labels else "None"} | **Created:** {created_at[:10]}
**URL:** {url}

## Description

{body}
"""

        if comments:
            file_content += f"\n## Comments ({len(comments)})\n"
            for comment in comments:
                author = comment.get("author", "unknown")
                comment_date = comment.get("created_at", "N/A")
                comment_body = comment.get("body", "")
                file_content += f"\n### {author} — {comment_date}\n{comment_body}\n\n---\n"

        filename = f"{self.repo_owner}-{self.repo_short_name}-{number}.md"
        filepath = f"{self.skill_dir}/references/issues/{filename}"
        with open(filepath, "w", encoding="utf-8") as f:
            f.write(file_content)
        logger.debug(f"Generated per-issue file: {filepath}")

    def _generate_releases_reference(self):
        """Generate releases.md reference file."""
        releases = self.data["releases"]

        content = (
            f"# Releases\n\nVersion history for this repository ({len(releases)} releases).\n\n"
        )

        for release in releases:
            published_at = release.get("published_at") or "N/A"
            release_name = release.get("name") or release["tag_name"]
            release_body = release.get("body") or ""
            content += f"## {release['tag_name']}: {release_name}\n"
            content += f"**Published:** {published_at[:10]}\n"
            if release["prerelease"]:
                content += "**Pre-release**\n"
            content += f"\n{release_body}\n\n"
            content += f"[View on GitHub]({release['url']})\n\n---\n\n"

        releases_path = f"{self.skill_dir}/references/releases.md"
        with open(releases_path, "w", encoding="utf-8") as f:
            f.write(content)
        logger.info(f"Generated: {releases_path}")

    def _generate_file_structure_reference(self):
        """Generate file_structure.md reference file."""
        file_tree = self.data["file_tree"]

        content = "# Repository File Structure\n\n"
        content += f"Total items: {len(file_tree)}\n\n"
        content += "```\n"

        # Build tree structure
        for item in file_tree:
            indent = "  " * item["path"].count("/")
            icon = "📁" if item["type"] == "dir" else "📄"
            content += f"{indent}{icon} {os.path.basename(item['path'])}\n"

        content += "```\n"

        structure_path = f"{self.skill_dir}/references/file_structure.md"
        with open(structure_path, "w", encoding="utf-8") as f:
            f.write(content)
        logger.info(f"Generated: {structure_path}")
