#!/usr/bin/env python3
# prompt_foo.py

# Pipulate: A hand-cranked, local-first AI SEO software framework.
# Copyright (C) 2026 Michael Jay Levin
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.

# > We've got content. It's groovy context!  
# > Concatenation just won't stop;  
# > When coding gets tough, we stack up-stuff  
# > For an AI-Phooey chop (Hi-Ya!)  

import os
import re
import sys
import pydot
import argparse
import tiktoken
import subprocess
import tempfile
import shutil
import json
import urllib.request
from typing import Dict, List, Optional, Tuple
from pathlib import Path

try:
    import jupytext
    JUPYTEXT_AVAILABLE = True
except ImportError:
    JUPYTEXT_AVAILABLE = False


CONFIG_DIR = Path.home() / ".config" / "articleizer"
TARGETS_FILE = CONFIG_DIR / "targets.json"

DEFAULT_TARGETS = {
    "1": {
        "name": "Local Project (Default)",
        "path": "/home/mike/repos/trimnoir/_posts" # Updated default fallback
    }
}

# ============================================================================
# --- Logging & Capture ---
# ============================================================================
class Logger:
    """Captures stdout for inclusion in the generated prompt."""
    def __init__(self):
        self.logs = []

    def print(self, *args, **kwargs):
        # Construct the string exactly as print would
        sep = kwargs.get('sep', ' ')
        end = kwargs.get('end', '\n')
        text = sep.join(map(str, args)) + end
        
        # Capture it
        self.logs.append(text)
        
        # Actually print it to stdout
        print(*args, **kwargs)

    def get_captured_text(self):
        return "".join(self.logs)

# Global logger instance
logger = Logger()

def load_targets():
    """Loads publishing targets from external config."""
    if TARGETS_FILE.exists():
        try:
            with open(TARGETS_FILE, 'r') as f:
                return json.load(f)
        except json.JSONDecodeError:
            logger.print(f"Warning: {TARGETS_FILE} is corrupt. Using defaults.")
    return DEFAULT_TARGETS

# Initialize with defaults, but allow override
CONFIG = {
    "PROJECT_NAME": "pipulate",
    "POSTS_DIRECTORY": DEFAULT_TARGETS["1"]["path"]
}

# ============================================================================
# --- Configuration ---
# ============================================================================
def find_repo_root(start_path: str) -> str:
    """Find the git repository root from a starting path."""
    path = os.path.abspath(start_path)
    while path != os.path.dirname(path):
        if os.path.isdir(os.path.join(path, '.git')):
            return path
        path = os.path.dirname(path)
    raise FileNotFoundError("Could not find the repository root (.git directory).")

REPO_ROOT = find_repo_root(os.path.dirname(__file__))

CONFIG = {
    "PROJECT_NAME": "pipulate",
    "POSTS_DIRECTORY": "/home/mike/repos/trimnoir/_posts"
}

# ============================================================================
# --- Static Analysis Configuration ---
# ============================================================================
# Set to False to skip Ruff during prompt compilation.
# Useful when transitioning linters or to reduce context noise.
ENABLE_STATIC_ANALYSIS = True

# ============================================================================
# --- Literary Size Scale & Token/Word Counting ---
# ============================================================================
LITERARY_SIZE_SCALE = [
    (3000, "Short Essay"), (7500, "Short Story"), (20000, "Novelette"),
    (50000, "Novella or a Master's Dissertation"),
    (80000, "Average Paperback Novel or a Ph.D. Dissertation"),
    (120000, "Long Novel"), (200000, "Epic Fantasy Novel"),
    (500000, "Seriously Long Epic (like 'Infinite Jest')"),
]

def get_literary_perspective(word_count: int, token_word_ratio: float) -> str:
    description = f"Longer than {LITERARY_SIZE_SCALE[-1][1]}"
    for words, desc in LITERARY_SIZE_SCALE:
        if word_count <= words:
            description = desc
            break
    density_warning = ""
    if token_word_ratio > 1.8:
        density_warning = (
            f" (Note: With a token/word ratio of {token_word_ratio:.2f}, "
            f"this content is far denser and more complex than typical prose of this length)."
        )
    return f"📚 Equivalent in length to a **{description}**{density_warning}"

def count_tokens(text: str, model: str = "gpt-4o") -> int:
    try:
        encoding = tiktoken.encoding_for_model(model)
        return len(encoding.encode(text))
    except Exception:
        return len(text.split())

def count_words(text: str) -> int:
    return len(text.split())

# ============================================================================
# --- Auto-Context Generation (UML, Tree, Narrative) ---
# ============================================================================
def add_holographic_shards(builder, articles: List[Dict]):
    """Finds and injects JSON context shards for a specific list of articles."""
    shards = {}
    found_count = 0
    
    for article in articles:
        # Resolve path: _posts/filename.md -> _posts/_context/filename.json
        article_path = article['path']
        parent_dir = os.path.dirname(article_path)
        stem = os.path.splitext(os.path.basename(article_path))[0]
        json_path = os.path.join(parent_dir, "_context", f"{stem}.json")
        
        if os.path.exists(json_path):
            try:
                with open(json_path, 'r', encoding='utf-8') as f:
                    # Load as object to consolidate
                    shard_data = json.load(f)
                    shards[stem] = shard_data
                    found_count += 1
            except Exception as e:
                logger.print(f"Warning: Could not read context shard {json_path}: {e}")

    if shards:
        title = "Holographic Context Shards"
        # Dense serialization to save tokens
        consolidated_json = json.dumps(shards, separators=(',', ':'))
        content = f"--- START: Holographic Context Shards (Consolidated) ---\n{consolidated_json}\n--- END: Holographic Context Shards ---"
        
        builder.add_auto_context(title, content)
        cdata = builder.auto_context.get(title, {})
        logger.print(f"Matched context shards: ({found_count} files | {cdata.get('tokens',0):,} tokens)")


def generate_uml_and_dot(target_file: str, project_name: str) -> Dict:
    pyreverse_exec = shutil.which("pyreverse")
    plantuml_exec = shutil.which("plantuml")
    if not pyreverse_exec or not plantuml_exec:
        msg = []
        if not pyreverse_exec: msg.append("`pyreverse` (from pylint)")
        if not plantuml_exec: msg.append("`plantuml`")
        return {"ascii_uml": f"Skipping: Required command(s) not found: {', '.join(msg)}."}

    target_path = target_file if os.path.isabs(target_file) else os.path.join(REPO_ROOT, target_file)
    if not os.path.exists(target_path):
        return {"ascii_uml": f"Skipping: Target file for UML generation not found: {target_path}"}

    with tempfile.TemporaryDirectory() as temp_dir:
        dot_file_path = os.path.join(temp_dir, "classes.dot")
        puml_file_path = os.path.join(temp_dir, "diagram.puml")
        try:
            pyreverse_cmd = [pyreverse_exec, "-f", "dot", "-o", "dot", "-p", project_name, target_path]
            subprocess.run(pyreverse_cmd, check=True, capture_output=True, text=True, cwd=temp_dir)
            generated_dot_name = f"classes_{project_name}.dot"
            os.rename(os.path.join(temp_dir, generated_dot_name), dot_file_path)
        except (subprocess.CalledProcessError, FileNotFoundError) as e:
            error_msg = e.stderr if hasattr(e, 'stderr') else str(e)
            return {"ascii_uml": f"Error: pyreverse failed. {error_msg}", "dot_graph": None}

        try:
            graphs = pydot.graph_from_dot_file(dot_file_path)
            if not graphs:
                return {"ascii_uml": f"Note: No classes found in {target_file} to generate a diagram.", "dot_graph": None}
            graph = graphs[0]
            dot_content = graph.to_string()
            puml_lines = ["@startuml", "skinparam linetype ortho", ""]
            def sanitize_line(line):
                clean = re.sub(r'<br[^>]*>', '', line)
                clean = re.sub(r'<[^>]+>', '', clean)
                return clean.strip()
            for node in graph.get_nodes():
                label = node.get_label()
                if not label: continue
                parts = label.strip('<>{} ').split('|')
                class_name = sanitize_line(parts[0])
                puml_lines.append(f"class {class_name} {{")
                if len(parts) > 1:
                    for attr in re.split(r'<br[^>]*>', parts[1]):
                        clean_attr = sanitize_line(attr).split(':')[0].strip()
                        if clean_attr: puml_lines.append(f"  - {clean_attr}")
                if len(parts) > 2:
                    method_block = parts[2].strip()
                    for method_line in re.split(r'<br[^>]*>', method_block):
                        clean_method = sanitize_line(method_line)
                        if clean_method: puml_lines.append(f"  + {clean_method}")
                puml_lines.append("}\n")
            for edge in graph.get_edges():
                source_name = edge.get_source().strip('"').split('.')[-1]
                dest_name = edge.get_destination().strip('"').split('.')[-1]
                puml_lines.append(f"{source_name} ..> {dest_name}")
            puml_lines.append("@enduml")
            with open(puml_file_path, 'w') as f: f.write('\n'.join(puml_lines))
        except Exception as e:
            with open(dot_file_path, 'r') as f: dot_content_on_error = f.read()
            return {"ascii_uml": f"Error: DOT to PUML conversion failed. {str(e)}", "dot_graph": dot_content_on_error}

        try:
            plantuml_cmd = ["plantuml", "-tutxt", puml_file_path]
            subprocess.run(plantuml_cmd, check=True, capture_output=True, text=True, cwd=temp_dir)
            utxt_file_path = puml_file_path.replace(".puml", ".utxt")
            with open(utxt_file_path, 'r') as f: ascii_uml = f.read()
            lines = ascii_uml.splitlines()
            non_empty_lines = [line for line in lines if line.strip()]
            if non_empty_lines:
                min_indent = min(len(line) - len(line.lstrip(' ')) for line in non_empty_lines)
                dedented_lines = [line[min_indent:] for line in lines]
                stripped_lines = [line.rstrip() for line in dedented_lines]
                ascii_uml = '\n'.join(stripped_lines)
                if ascii_uml: ascii_uml = '\n' + ascii_uml
        except (subprocess.CalledProcessError, FileNotFoundError) as e:
            error_msg = e.stderr if hasattr(e, 'stderr') else str(e)
            return {"ascii_uml": f"Error: plantuml failed. {error_msg}", "dot_graph": dot_content}

    return {"ascii_uml": ascii_uml, "dot_graph": dot_content}


def _get_article_list_data(posts_dir: str = CONFIG["POSTS_DIRECTORY"], url_config: dict = None) -> List[Dict]:
    posts_data = []
    if not os.path.isdir(posts_dir):
        logger.print(f"Warning: Article directory not found at {posts_dir}", file=sys.stderr)
        return []

    # Dynamically import lsa.py to avoid sys.path issues regardless of where prompt_foo is run
    sys.path.insert(0, os.path.join(REPO_ROOT, 'scripts', 'articles'))
    try:
        import lsa
    except ImportError as e:
        logger.print(f"Error importing lsa.py: {e}", file=sys.stderr)
        sys.path.pop(0)
        return []
    sys.path.pop(0)

    # 1. Delegate the metadata and JSON shard extraction to the universal parser
    raw_metadata = lsa.get_holographic_article_data(posts_dir)

    # 2. Append the heavy lifting (tokens/bytes) and URL mapping specific to prompt_foo
    for item in raw_metadata:
        filepath = item['path']
        filename = item['filename']
        
        full_url = ""
        if url_config:
            slug = item['permalink'].strip('/')
            if not slug:
                raw_slug = os.path.splitext(filename)[0]
                if re.match(r'\d{4}-\d{2}-\d{2}-', raw_slug):
                     raw_slug = raw_slug[11:]
                style = url_config.get('permalink_style', '/:slug/')
                slug_path = style.replace(':slug', raw_slug)
            else:
                  slug_path = "/" + slug.lstrip('/')

            base = url_config.get('base_url', '')
            full_url = f"{base}{slug_path}"

        try:
            # We still need the full content here for token counting
            with open(filepath, 'r', encoding='utf-8') as f: 
                content = f.read()
                
            article_tokens = count_tokens(content)
            article_bytes = len(content.encode('utf-8'))
            
            posts_data.append({
                'path': filepath,
                'date': item['date'],
                'sort_order': item['sort_order'],
                'title': item['title'],
                'summary': item['summary'],
                'url': full_url,
                'tokens': article_tokens,
                'bytes': article_bytes,
                # Pass along the newly extracted JSON Shards!
                'shard_kw': item['shard_kw'],
                'shard_sub': item['shard_sub'],
                'shard_sum': item['shard_sum']
            })
        except Exception:
            continue

    # Reverse to match prompt_foo's original oldest-to-newest expectation
    posts_data.reverse()
    return posts_data

def parse_slice_arg(arg_str: str):
    if not arg_str or not arg_str.startswith('[') or not arg_str.endswith(']'): return None
    content = arg_str[1:-1].strip()
    if ':' in content:
        parts = content.split(':', 1)
        start = int(parts[0].strip()) if parts[0].strip() else None
        end = int(parts[1].strip()) if parts[1].strip() else None
        return slice(start, end)
    elif content: return int(content)
    return slice(None, None)

def run_tree_command() -> str:
    eza_exec = shutil.which("eza")
    if not eza_exec: return "Skipping: `eza` command not found."
    try:
        # Added --level 3 to keep the tree from exploding if the repo grows deeper
        result = subprocess.run(
            [eza_exec, '--tree', '--level', '3', '--git-ignore', '--color=never'],
            capture_output=True, text=True, cwd=REPO_ROOT, check=True
        )
        return result.stdout
    except Exception as e: return f"Error running eza command: {e}"

def run_static_analysis(python_files: List[str]) -> str:
    """Runs Ruff on the target files with high terminal transparency."""
    if not python_files:
        return ""

    if not ENABLE_STATIC_ANALYSIS:
        logger.print("\n⏭️  Static Analysis skipped (ENABLE_STATIC_ANALYSIS = False).")
        return ""
        
    logger.print("\n🔍 Running Static Analysis Telemetry...")
    diagnostics = []
    
    # Ruff (replaces both Vulture and Pylint)
    ruff_exec = shutil.which("ruff")
    if ruff_exec:
        logger.print("   -> Checking for errors and dead code (Ruff)...")
        cmd = [ruff_exec, "check"] + python_files
        try:
            result = subprocess.run(cmd, capture_output=True, text=True)
            if result.stdout:
                diagnostics.append("### Ruff\n```text\n" + result.stdout.strip() + "\n```")
                logger.print(result.stdout.strip())  # Transparent terminal output
        except Exception as e:
            logger.print(f"      [Error running Ruff: {e}]")
             
    logger.print("✅ Static Analysis Complete.\n")
    return "\n\n".join(diagnostics)

# ============================================================================
# --- Helper Functions (File Parsing, Clipboard) ---
# ============================================================================
def annotate_tree_with_tokens(tree_output: str, processed_files: List[Dict], repo_root: str) -> str:
    """Injects (X,XXX tokens) next to every included file in the eza tree."""
    token_map = {}
    for f in processed_files:
        if f.get('path', '').endswith(('.py', '.ipynb', '.md', '.nix', '.sh')):  # expand as needed
            rel_path = os.path.relpath(f['path'], repo_root)
            token_map[rel_path] = f['tokens']
    
    lines = tree_output.splitlines()
    annotated = []
    for line in lines:
        for rel_path, tokens in token_map.items():
            filename = os.path.basename(rel_path)
            if filename in line and line.strip().endswith(filename):
                # Preserve the beautiful tree art, just tack on the size
                line = f"{line}  ({tokens:,} tokens)"
                break
        annotated.append(line)
    return '\n'.join(annotated)

def parse_file_list_from_config(chop_var: str = "AI_PHOOEY_CHOP", format_kwargs: dict = None) -> List[Tuple[str, str]]:
    try:
        # Explicitly bind the repo root to the path for static analysis parity
        if REPO_ROOT not in sys.path:
            sys.path.insert(0, REPO_ROOT)
            
        import foo_files
        files_raw = getattr(foo_files, chop_var)
    except (ImportError, AttributeError):
        logger.print(f"ERROR: foo_files.py not found or doesn't contain '{chop_var}'.")
        sys.exit(1)
    
    # 💥 SAFE REPLACEMENT: Prevents crashing on bash/awk curly braces {}
    if format_kwargs:
        for key, val in format_kwargs.items():
            placeholder = f"{{{key}}}"
            files_raw = files_raw.replace(placeholder, str(val))
            
    lines = files_raw.strip().splitlines()
    seen_files, parsed_files = set(), []
    for line in lines:
        line = line.strip()
        if not line or line.startswith('#'): continue
        parts = re.split(r'\s*<--\s*|\s*#\s*', line, 1)
        file_path = parts[0].strip()
        comment = parts[1].strip() if len(parts) > 1 else ""
        if file_path and file_path not in seen_files:
            seen_files.add(file_path)
            parsed_files.append((file_path, comment))
    return parsed_files

def copy_to_clipboard(text: str):
    """Copies text to the system clipboard gracefully across macOS and Linux."""
    import platform
    
    # === THE 80/20 SSH BYPASS ===
    # If logged in via SSH, dump to the bridge file instead of fighting X11
    if os.getenv("SSH_CLIENT"):
        try:
            with open("/tmp/clipboard_bridge.txt", "w", encoding="utf-8") as f:
                f.write(text)
            logger.print("✨ Markdown output routed to SSH Bridge (/tmp/clipboard_bridge.txt)")
        except Exception as e:
            logger.print(f"\nWarning: Could not write to SSH Bridge: {e}")
        return
    # ============================

    system = platform.system().lower()
    
    if system == "darwin":
        cmd = ['pbcopy']
    elif system == "linux":
        cmd = ['xclip', '-selection', 'clipboard']
    else:
        logger.print(f"\nWarning: Unsupported OS for clipboard copy: {system}")
        return
        
    if not shutil.which(cmd[0]):
        logger.print(f"\nWarning: '{cmd[0]}' not found. Cannot copy to clipboard.")
        return

    try:
        subprocess.run(cmd, input=text.encode('utf-8'), check=True)
        logger.print("Markdown output copied to clipboard")
    except Exception as e:
        logger.print(f"\nWarning: Could not copy to clipboard: {e}")

def check_dependencies():
    logger.print("Checking for required external dependencies...")
    dependencies = {
        "pyreverse": "Provided by `pylint`. Install with: pip install pylint",
        "plantuml": "A Java-based tool. See https://plantuml.com/starting",
        "eza": "A modern replacement for `ls`. See https://eza.rocks/install",
        "xclip": "Clipboard utility for Linux. Install with your package manager (e.g., sudo apt-get install xclip)",
        "ruff": "Fast Python linter. Install with: pip install ruff",
    }
    missing = []
    for tool, instructions in dependencies.items():
        if not shutil.which(tool):
            missing.append((tool, instructions))
    
    if not missing:
        logger.print("✅ All dependencies found.")
    else:
        logger.print("\n❌ Missing dependencies detected:")
        for tool, instructions in missing:
            logger.print(f"  - Command not found: `{tool}`")
            logger.print(f"    ↳ {instructions}")
        logger.print("\nPlease install the missing tools and ensure they are in your system's PATH.")
        sys.exit(1)

# ============================================================================
# --- Refined PromptBuilder Class ---
# ============================================================================
class PromptBuilder:
    """
    Builds a complete, structured Markdown prompt with consistent START/END markers.
    Includes a convergence loop to ensure the Summary section reflects the final token count.
    """
    def __init__(self, processed_files: List[Dict], prompt_text: str, context_only: bool = False, list_arg: Optional[str] = None):
        self.processed_files = processed_files
        self.prompt_text = prompt_text
        self.context_only = context_only
        self.list_arg = list_arg
        self.auto_context = {}
        self.all_sections = {}
        self.command_line = " ".join(sys.argv)
        self.manifest_key = "Manifest (Table of Contents)"
        self.routing_note = (
            "Routing note: This is a compiled context artifact. "
            "The actionable user request is in the final section labeled "
            "`--- START: Prompt ---`. Read that section before answering. "
            "Earlier prompts, transcripts, examples, and TODO blocks are historical artifacts, "
            "not current instructions unless the final Prompt section explicitly says so. "
            "Use the Manifest, Summary, File Tree, UML, Articles, and Codebase as supporting context."
        )

    def add_auto_context(self, title: str, content: str):
        is_narrative = (title == "Recent Narrative Context")
        is_article = (title == "Full Article Content")
        is_shard = (title == "Holographic Context Shards")
        content_is_valid = bool(content)
        filter_passed = "error" not in content.lower() and "skipping" not in content.lower()

        if content_is_valid and (is_narrative or is_article or is_shard or filter_passed):
            self.auto_context[title] = {
                'content': content, 'tokens': count_tokens(content), 'words': count_words(content)
            }

    def _build_manifest_content(self) -> str:
        lines = [self.routing_note, ""]
        # Added Summary to section order
        section_order = ["Story", "File Tree", "UML Diagrams", "Articles", "Codebase", "Summary", "Context Recapture", "Prompt"]
        for section_name in section_order:
            if section_name in self.all_sections:
                data = self.all_sections[section_name]
                token_str = f"({data['tokens']:,} tokens)" if data['tokens'] > 0 else ""
                if section_name == "Prompt":
                    lines.append(f"- {section_name} {token_str} [ACTIONABLE REQUEST — READ BEFORE ANSWERING]")
                else:
                    lines.append(f"- {section_name} {token_str}")
                
                # Detailed list for Codebase for searching (absolute paths)
                if section_name == "Codebase" and not self.context_only and self.processed_files:
                     for f in self.processed_files:
                          byte_len = len(f['content'].encode('utf-8'))
                          lines.append(f"  - {f['path']} ({f['tokens']:,} tokens | {byte_len:,} bytes)")
                          
        return "\n".join(lines)

    def _build_story_content(self) -> str:
        title = "Recent Narrative Context"
        return self.auto_context.get(title, {}).get('content', '').strip()

    def _build_tree_content(self) -> str:
        title = "Codebase Structure (eza --tree + token sizes)"  # ← sync with main()
        if title in self.auto_context:
            content = self.auto_context[title]['content'].strip()
            return f"```text\n{content}\n```"
        return ""

    def _build_uml_content(self) -> str:
        uml_parts = []
        for title, data in self.auto_context.items():
            if "UML Class Diagram" in title:
                uml_parts.append(f"## {title}\n```text\n{data['content']}\n```")
        return "\n\n".join(uml_parts)

    def _build_articles_content(self) -> str:
        parts = []

        # 1. Grab Full Article Content if it exists
        if "Full Article Content" in self.auto_context:
            parts.append(self.auto_context["Full Article Content"]['content'].strip())
            
        # 2. Grab Holographic Shards if they exist
        if "Holographic Context Shards" in self.auto_context:
            parts.append(self.auto_context["Holographic Context Shards"]['content'].strip())
            
        return "\n\n".join(parts).strip()


    def _build_codebase_content(self) -> str:
        if self.context_only: return ""
        if not self.processed_files: return ""
        
        lines = []
        for f in self.processed_files:
            # Using Absolute Paths in markers
            lines.append(f"--- START: {f['path']} ({f['tokens']:,} tokens) ---")
            lines.append(f"```{f['lang']}:{f['path']}")
            lines.append(f['content'])
            lines.append("```")
            lines.append(f"--- END: {f['path']} ---\n")
        return "\n".join(lines).strip()

    def _build_recapture_content(self) -> str:
        """Generates the commented-out variable block for reconstructing this context."""
        lines = ["```python", "# # PURPOSE OF CONTEXT: ", '# AI_PHOOEY_CHOP = """\\']
        for f in self.processed_files:
            path_str = f['path']
            # Keep relative if inside repo, absolute if outside
            if path_str.startswith(REPO_ROOT):
                path_str = os.path.relpath(path_str, REPO_ROOT)
            lines.append(f"# {path_str}")
        lines.append('# """')
        lines.append("```")
        return "\n".join(lines)

    def _build_prompt_content(self) -> str:
        checklist = self._generate_ai_checklist()
        return f"{checklist}\n\n{self.prompt_text}"

    def _generate_ai_checklist(self) -> str:
        return '''# AI Self-Correction Checklist

Before addressing the user's prompt, perform the following verification steps:

1.  **Review the Manifest vs. Codebase Structure:** Compare the file list in the manifest against the `eza --tree` output. Are there any obviously related, critical files missing from the manifest that would be necessary to understand the user's request? (e.g., if `core.py` is present, is `__init__.py` also present?).
2.  **Analyze Import Statements:** Briefly scan the `import` statements in the provided Python files. Do they suggest dependencies on local files that were *not* included in the manifest?
3.  **Check for Incompleteness:** If you determine that critical files are missing, do not proceed. Instead, your **primary task** is to inform me of the missing files and recommend adding them to `foo_files.py` to create a more complete context.
4.  **Confirm Understanding:** If the context appears complete, state "Context verified." and then proceed with the user's request.
5.  **THE SEARCH/REPLACE PROTOCOL:** When executing a code edit, you MUST respond exclusively with one or more SEARCH/REPLACE blocks. You MUST NOT use unified diffs, `@@` hunks, or line numbers. Reproduce the SEARCH block EXACTLY as it appears in the original file, including all whitespace and indentation. You MUST use `[[[SEARCH]]]`, `[[[DIVIDER]]]`, and `[[[REPLACE]]]` markers. Make the minimal change necessary. If multiple similar blocks exist, make the SEARCH section long enough to be uniquely identifiable.
6.  **INDENTATION IS SACRED:** Every SEARCH block must be copy-pasted character-for-character from the source file shown in this prompt. Count the leading spaces on the first line of the target block in the source. Your SEARCH block must start with exactly that many spaces — not more, not fewer. If the file uses 4-space indentation and the target line has 4 spaces, your SEARCH block must also have exactly 4 spaces. LLMs that add extra padding to align with conversational context are a known failure mode. Check your leading spaces before submitting. The REPLACE block must have the same base indentation as the SEARCH block, with any nested code indented relatively from that base. Note: the source files in this prompt contain raw bytes with no line-number prefixes unless `--line-numbers` was passed. If you see `1: `, `2: ` prefixes, the context was generated in review mode and you must strip those prefixes before constructing a SEARCH block.
7.  **THE FENCED OUTFLOW INVARIANT:** You MUST enclose the `Target: filename` line and the entire SEARCH/DIVIDER/REPLACE block inside a single ` ```text ` markdown code block. This prevents web chat UIs from stripping leading whitespace, ensuring `apply.py` receives the exact indentation.
8.  **THE TARGET ADJACENCY RULE:** Every `[[[SEARCH]]]` marker must be immediately preceded by `Target: filename` on the line directly above it — no blank lines, no fences, no prose between the Target line and the marker. The filename in the Target line is what `apply.py` uses to find the file; omitting it or separating it with blank lines causes a fatal "Missing target filename" error. Example: `Target: scripts/articles/lsa.py` or `Target: /home/mike/repos/pipulate/scripts/articles/lsa.py`. Both relative and absolute paths work.
'''

    def _generate_summary_content(self, verified_token_count: int) -> str:
        """
        Generates a consolidated terminal report and prompt summary.
        Stacks: Command -> Execution Logs -> File Manifest -> Metrics.
        """
        lines = []
        
        # 1. The Command line
        lines.append(f"**Command:** `{self.command_line}`\n")

        # 2. RESTORED: Execution Log (Haptic Feedback)
        # Captures "Skipping tree", notebook conversion status, etc.
        logs = logger.get_captured_text().strip()
        if logs:
            lines.append("--- Processing Log ---")
            lines.append(f"```\n{logs}\n```\n")

        # 3. NEW: File Manifest (The "Knapped Arrowhead" List)
        if self.processed_files:
            lines.append("--- Codebase Files Included ---")
            for f in self.processed_files:
                # Show relative path for terminal scannability
                display_path = os.path.relpath(f['path'], REPO_ROOT) if os.path.isabs(f['path']) else f['path']
                lines.append(f"• {display_path} ({f['tokens']:,} tokens)")
            lines.append("")

        # 4. Article Narrative Echo
        narrative = self.auto_context.get("Recent Narrative Context")
        if narrative:
            lines.append("--- Articles Included ---")
            titles = re.findall(r"### (.*?) \(", narrative['content'])
            for title in titles:
                lines.append(f"• {title}")
            lines.append("")

        # 5. Auto-Context Metadata
        if self.auto_context:
            lines.append("--- Auto-Context Metadata ---")
            for title, data in self.auto_context.items():
                byte_len = len(data['content'].encode('utf-8'))
                lines.append(f"• {title} ({data['tokens']:,} tokens | {byte_len:,} bytes)")

        # 6. Final Metrics (The Convergence Proof)
        total_tokens = sum(v.get('tokens', 0) for k, v in self.all_sections.items() if k != self.manifest_key)
        
        total_words = 0
        final_content_for_metrics = ""
        for section, data in self.all_sections.items():
            content_part = data.get('content', '')
            final_content_for_metrics += content_part
            if section != "Prompt":
                total_words += count_words(content_part)

        char_count = len(final_content_for_metrics)
        byte_count = len(final_content_for_metrics.encode('utf-8'))

        lines.append("\n--- Prompt Summary ---")
        lines.append(f"Summed Tokens:    {total_tokens:,} (from section parts)")
        lines.append(f"Verified Tokens: {verified_token_count:,} (from final output)")
        
        if total_tokens != verified_token_count:
            diff = verified_token_count - total_tokens
            lines.append(f"  (Difference: {diff:+,})")
            
        lines.append(f"Total Words:      {total_words:,} (content only)")
        lines.append(f"Total Chars:      {char_count:,}")
        lines.append(f"Total Bytes:      {byte_count:,} (UTF-8)")

        ratio = verified_token_count / total_words if total_words > 0 else 0
        perspective = get_literary_perspective(total_words, ratio)
        lines.append("\n--- Size Perspective ---")
        lines.append(perspective)

        return "\n".join(lines)

    def build_final_prompt(self) -> str:
        """Assembles all parts into the final Markdown string with convergence loop for accuracy."""
        
        # 1. Build static sections
        story_content = self._build_story_content()
        tree_content = self._build_tree_content()
        uml_content = self._build_uml_content()
        articles_content = self._build_articles_content()
        codebase_content = self._build_codebase_content()
        recapture_content = self._build_recapture_content()
        prompt_content = self._build_prompt_content()

        # Placeholders
        placeholders = {
            "Story": f"# Narrative context not requested. Use the -l or --list flag to include recent articles.",
            "File Tree": "# File tree generation failed or was skipped.",
            "UML Diagrams": "# No Python files with classes were included, or UML generation failed.",
            "Articles": "# No full articles requested. Use the -a or --article flag to include full article content.",
            "Codebase": ("# No files were specified for inclusion in the codebase." if not self.processed_files 
                         else "# Running in --context-only mode. File contents are omitted."),
        }

        # Store basic sections
        self.all_sections["Story"] = {'content': story_content, 'tokens': count_tokens(story_content)}
        self.all_sections["File Tree"] = {'content': tree_content, 'tokens': count_tokens(tree_content)}
        self.all_sections["UML Diagrams"] = {'content': uml_content, 'tokens': count_tokens(uml_content)}
        self.all_sections["Articles"] = {'content': articles_content, 'tokens': count_tokens(articles_content)}
        self.all_sections["Codebase"] = {'content': codebase_content, 'tokens': sum(f['tokens'] for f in self.processed_files) if not self.context_only else 0}
        self.all_sections["Context Recapture"] = {'content': recapture_content, 'tokens': count_tokens(recapture_content)}
        self.all_sections["Prompt"] = {'content': prompt_content, 'tokens': count_tokens(prompt_content)}

        # Helper to assemble text
        def assemble_text(manifest_txt, summary_txt):
            parts = [f"# KUNG FU PROMPT CONTEXT\n\nWhat you will find below is:\n\n- {self.manifest_key}\n- Story\n- File Tree\n- UML Diagrams\n- Articles\n- Codebase\n- Summary\n- Context Recapture\n- Prompt"]
            
            def add(name, content, placeholder):
                final = content.strip() if content and content.strip() else placeholder
                parts.append(f"--- START: {name} ---\n{final}\n--- END: {name} ---")

            add(self.manifest_key, manifest_txt, "# Manifest generation failed.")
            add("Story", story_content, placeholders["Story"] if self.list_arg is None else "# No articles found for the specified slice.")
            add("File Tree", tree_content, placeholders["File Tree"])
            add("UML Diagrams", uml_content, placeholders["UML Diagrams"])
            add("Articles", articles_content, placeholders["Articles"])
            add("Codebase", codebase_content, placeholders["Codebase"])
            add("Summary", summary_txt, "# Summary generation failed.")
            add("Context Recapture", recapture_content, "# Context Recapture failed.")
            add("Prompt", prompt_content, "# No prompt was provided.")
            
            return "\n\n".join(parts)

        # 2. Convergence Loop
        # We need the Summary to contain the final token count, but the Summary is part of the text.
        # We iterate to allow the numbers to stabilize.
        
        current_token_count = 0
        final_output_text = ""
        
        # Initial estimate (sum of static parts)
        current_token_count = sum(v['tokens'] for v in self.all_sections.values())
        
        for _ in range(3): # Max 3 iterations, usually converges in 2
            # Generate Summary with current count
            summary_content = self._generate_summary_content(current_token_count)
            self.all_sections["Summary"] = {'content': summary_content, 'tokens': count_tokens(summary_content)}
            
            # Generate Manifest (might change if Summary token count changes length like 999->1000)
            manifest_content = self._build_manifest_content()
            self.all_sections[self.manifest_key] = {'content': manifest_content, 'tokens': count_tokens(manifest_content)}
            
            # Assemble full text
            final_output_text = assemble_text(manifest_content, summary_content)
            
            # Verify count
            new_token_count = count_tokens(final_output_text)
            
            if new_token_count == current_token_count:
                break # Converged
            
            current_token_count = new_token_count

        return final_output_text


def annotate_foo_files_in_place(chop_var: str = "AI_PHOOEY_CHOP"):
    """Reads foo_files.py, annotates file paths with token/byte sizes, and writes it back."""
    foo_path = os.path.join(REPO_ROOT, "foo_files.py")
    if not os.path.exists(foo_path):
        return

    try:
        with open(foo_path, 'r', encoding='utf-8') as f:
            content = f.read()

        # Isolate the requested chop block dynamically
        match = re.search(rf'({chop_var}\s*=\s*"""\\?\n)(.*?)("""|\\n""")', content, re.DOTALL)
        if not match:
            return

        prefix, body, suffix = match.groups()
        new_lines = []
        modified = False

        for line in body.split('\n'):
            stripped = line.strip()
            
            # Skip empties, headers, URLs, and chisel strikes
            if (not stripped or stripped.startswith('# =') or 
                stripped.startswith('# CHAPTER') or 'http' in stripped or 
                stripped.startswith('!') or stripped.startswith('# !')):
                new_lines.append(line)
                continue

            # Match potential file paths: optional comment hash -> path -> remainder
            m = re.match(r'^(\s*(?:#\s*)?)([^#\s]+)(.*)$', line)
            if m:
                # indent_and_hash = m.group(1)
                filepath = m.group(2)
                remainder = m.group(3)

                # 1. Idempotency: If it already has an inline comment, skip it
                if '#' in remainder:
                    new_lines.append(line)
                    continue
                
                # 2. Skip obvious prose/structural comments (words without dots or slashes)
                if not ('/' in filepath or '.' in filepath):
                    new_lines.append(line)
                    continue

                # 3. Resolve absolute path and measure
                full_path = os.path.join(REPO_ROOT, filepath) if not os.path.isabs(filepath) else filepath
                if os.path.exists(full_path) and os.path.isfile(full_path):
                    try:
                        with open(full_path, 'r', encoding='utf-8') as f:
                            file_content = f.read()
                        
                        t_count = count_tokens(file_content)
                        b_count = len(file_content.encode('utf-8'))
                        
                        # Reconstruct line with annotation at the end
                        new_line = f"{line}  # [{t_count:,} tokens | {b_count:,} bytes]"
                        new_lines.append(new_line)
                        modified = True
                    except Exception:
                        new_lines.append(line) # Error reading, leave untouched
                else:
                    new_lines.append(line) # File not found, leave untouched
            else:
                new_lines.append(line) # Didn't match pattern

        # Write back ONLY if we added new annotations
        if modified:
            new_body = '\n'.join(new_lines)
            new_content = content[:match.start()] + prefix + new_body + suffix + content[match.end():]
            with open(foo_path, 'w', encoding='utf-8') as f:
                f.write(new_content)
            logger.print("✨ Auto-annotated foo_files.py with token/byte sizes.")

    except Exception as e:
        logger.print(f"Warning: Failed to auto-annotate foo_files.py: {e}")

# ============================================================================
# --- Paintbox & Repository Profiling ---
# ============================================================================
STORY_EXTENSIONS = {
    '.py', '.js', '.css', '.html', '.md', '.markdown', '.txt',
    '.json', '.nix', '.sh', '.ipynb', '.toml', '.in', '.cfg',
    '.svg', '.xsd', '.sql', '.lua', '.yaml', '.yml',
}

def collect_repo_files(repo_root: str) -> set:
    """Use `git ls-files` to get only tracked, non-ignored files."""
    try:
        result = subprocess.run(
            ['git', 'ls-files'],
            capture_output=True, text=True, cwd=repo_root, check=True
        )
        repo_files = set()
        for line in result.stdout.strip().splitlines():
            line = line.strip()
            if not line:
                continue
            ext = os.path.splitext(line)[1].lower()
            if ext in STORY_EXTENSIONS:
                repo_files.add(line)
        return repo_files
    except (subprocess.CalledProcessError, FileNotFoundError):
        logger.print("⚠️  `git ls-files` failed. Cannot run Paintbox check.\n")
        return set()


def update_paintbox_in_place():
    """Finds unclaimed files in the repo and injects them into the Paintbox section of foo_files.py."""
    foo_path = os.path.join(REPO_ROOT, "foo_files.py")
    if not os.path.exists(foo_path):
        return

    try:
        with open(foo_path, "r", encoding="utf-8") as f:
            lines = f.readlines()

        # Phase 1: Parse the current map to see what is already "claimed"
        in_story_section = False
        all_claimed_files = set()

        for line in lines:
            line = line.strip()
            if "AI_PHOOEY_CHOP =" in line:
                in_story_section = True
                continue
            if not in_story_section:
                continue
            if "VIII. THE PAINTBOX" in line:
                break # Stop before parsing the unused tubes themselves

            clean_line = line.lstrip("#").strip()
            if (not clean_line or clean_line.startswith("=") or 
                clean_line.startswith("CHAPTER") or clean_line.startswith("THE 404") or
                clean_line.startswith("!") or clean_line.startswith("http")):
                continue

            file_path = clean_line.split()[0]
            ext = os.path.splitext(file_path)[1].lower()
            if ext in STORY_EXTENSIONS or ('/' in file_path and '.' in file_path):
                if os.path.isabs(file_path):
                    if file_path.startswith(REPO_ROOT):
                        rel_path = os.path.relpath(file_path, REPO_ROOT)
                        all_claimed_files.add(os.path.normpath(rel_path))
                else:
                    all_claimed_files.add(os.path.normpath(file_path))

        # Phase 2: Diff the map against the territory
        repo_files = collect_repo_files(REPO_ROOT)
        if not repo_files:
            return # Bail if git failed
            
        unused_tubes = sorted(repo_files - all_claimed_files)

        # Phase 3: Inject the unused tubes idempotently
        with open(foo_path, "r", encoding="utf-8") as f:
            foo_content = f.read()

        PAINTBOX_MARKER = "# ============================================================================\n# VIII. THE PAINTBOX (Unused Colors)\n# ============================================================================"
        marker_index = foo_content.find(PAINTBOX_MARKER)
        
        if marker_index != -1:
            base_content = foo_content[:marker_index].rstrip() + "\n\n"
        else:
            end_quote_idx = foo_content.rfind('"""')
            base_content = foo_content[:end_quote_idx].rstrip() + "\n\n"

        if not unused_tubes:
            with open(foo_path, "w", encoding="utf-8") as f:
                f.write(base_content + '\n')
            return # Clean exit, no unused paint

        paintbox_lines = [
            PAINTBOX_MARKER,
            "# Files tracked by git but not yet mixed into the palette above.",
            "# Move these into the active chapters to paint them onto the context canvas.\n"
        ]

        total_files = len(repo_files)
        mapped_files = total_files - len(unused_tubes)
        coverage = (mapped_files / total_files) * 100 if total_files > 0 else 100
        
        logger.print(f"🗺️  Codex Mapping Coverage: {coverage:.1f}% ({mapped_files}/{total_files} tracked files).")
        logger.print(f"📦 Appending {len(unused_tubes)} uncategorized files to the Paintbox ledger for future documentation...")
        for tube_path in unused_tubes:
            full_path = os.path.join(REPO_ROOT, tube_path)
            try:
                with open(full_path, "r", encoding="utf-8") as f:
                    content = f.read()
                tokens = count_tokens(content)
                b_size = len(content.encode('utf-8'))
                paintbox_lines.append(f"# {tube_path}  # [{tokens:,} tokens | {b_size:,} bytes]")
            except Exception:
                paintbox_lines.append(f"# {tube_path}  # [Error reading file]")

        # THIS IS THE CRITICAL FIX: No string closure appended!
        final_content = base_content + "\n".join(paintbox_lines) + '\n'
        
        with open(foo_path, "w", encoding="utf-8") as f:
            f.write(final_content)

    except Exception as e:
        logger.print(f"Warning: Failed to update the Paintbox: {e}")


def check_topological_integrity(chop_var: str = "AI_PHOOEY_CHOP", format_kwargs: dict = None):
    """Reports references in foo_files.py that no longer exist on disk."""
    import foo_files
    raw_content = getattr(foo_files, chop_var, "")
    
    # Inject dynamic arguments before parsing paths
    if format_kwargs:
        for key, val in format_kwargs.items():
            raw_content = raw_content.replace(f"{{{key}}}", str(val))
    
    # 1. Identify all potential file-like strings in the CHOP
    extensions = '|'.join([ext.lstrip('.') for ext in STORY_EXTENSIONS])
    potential_refs = set(re.findall(rf'([\w\d\./\\%-]+\.(?:{extensions}))(?!\w)', raw_content))
    
    # 2. Get the reality of the disk
    repo_files = collect_repo_files(REPO_ROOT)
    
    # 3. Find the Ghosts
    broken_refs = []
    for ref in potential_refs:
        # Ignore HTTP, Commands, double-slashes, and server-side absolute paths
        if ref.startswith(('http', '!', '//', '/www/')) or '://' in ref: 
            continue
        
        full_path = os.path.join(REPO_ROOT, ref) if not os.path.isabs(ref) else ref
        if not os.path.exists(full_path):
            broken_refs.append(ref)
            
    if broken_refs:
        logger.print("\n⚠️  TOPOLOGICAL INTEGRITY ALERT (Broken References):")
        for ghost in sorted(broken_refs):
            logger.print(f"  • {ghost}")
    else:
        logger.print("\n✅ Topological Integrity Verified: All references exist.")


# ============================================================================
# --- Main Execution Logic ---
# ============================================================================
def main():
    """Main function to parse args, process files, and generate output."""
    parser = argparse.ArgumentParser(description='Generate a Markdown context file for AI code assistance.')
    parser.add_argument('prompt', nargs='?', default=None, help='A prompt string or path to a prompt file (e.g., prompt.md).')
    parser.add_argument('-o', '--output', type=str, help='Optional: Output filename.')
    parser.add_argument('--no-clipboard', action='store_true', help='Disable copying output to clipboard.')
    parser.add_argument('--check-dependencies', action='store_true', help='Verify that all required external tools are installed.')
    parser.add_argument('--context-only', action='store_true', help='Generate a context-only prompt without file contents.')
    parser.add_argument('-n', '--no-tree', action='store_true', help='Suppress file tree and UML generation.')
    parser.add_argument('--chop', type=str, default='AI_PHOOEY_CHOP', help='Specify an alternative payload variable from foo_files.py')
    parser.add_argument('--line-numbers', action='store_true', help='Prefix source lines with line numbers (review mode). Incompatible with SEARCH/REPLACE patching.')
    
    # 💥 NEW: Dynamic argument injection
    parser.add_argument('--arg', action='append', help='Pass dynamic arguments to CHOP templates (format: key=value)')
    
    parser.add_argument(
        '-l', '--list',
        nargs='?', const='[-5:]', default=None,
        help='Include a list of recent articles. Optionally provide a slice, e.g., "[:]". Defaults to "[-5:]".'
    )
    parser.add_argument(
        '-a', '--article',
        nargs='?', const='[-1:]', default=None,
        help='Include FULL CONTENT of recent articles. Provide a slice, e.g., "[-5:]". Defaults to "[-1:]".'
    )
    parser.add_argument(
        '-c', '--context',
        action='store_true',
        help='Include matching Holographic Context JSONs for any articles listed/included.'
    )
    parser.add_argument('-t', '--target', type=str, help="Target ID from targets.json (e.g., '1')")
    parser.add_argument('-k', '--key', type=str, help="API key alias from keys.json (ignored by prompt_foo, here for compatibility)")
    parser.add_argument(
        '--decanter', action='append',
        help='Inject full raw Markdown for specific article paths (can be used multiple times).'
    )
    parser.add_argument(
        '--slugs', nargs='+', metavar='SLUG',
        help='Specify one or more article slugs to automatically decant full raw content.'
    )
    parser.add_argument(
        '--decanter-from', type=str, metavar='FILE',
        help='Read article paths from a file or "-" for stdin, one path per line. Equivalent to multiple --decanter args.'
    )
    args = parser.parse_args()

    # 💥 NEW: Parse --arg into a dictionary
    format_kwargs = {}
    if args.arg:
        for a in args.arg:
            if '=' in a:
                k, v = a.split('=', 1)
                format_kwargs[k.strip()] = v.strip()
            else:
                logger.print(f"Warning: Invalid argument format '{a}'. Expected key=value.")

    # Handle Target Selection (unchanged)
    targets = load_targets()
    active_target_config = None  
    if args.target:
        if args.target in targets:
            selected = targets[args.target]
            CONFIG["POSTS_DIRECTORY"] = selected["path"]
            active_target_config = selected  
            logger.print(f"🎯 Target set to: {selected['name']} ({selected['path']})")
        else:
            logger.print(f"❌ Invalid target key: {args.target}. Using default.")
    else:
        if "1" in targets:
            active_target_config = targets["1"]

    if args.check_dependencies:
        check_dependencies()
        sys.exit(0)

    # 1. Handle user prompt
    prompt_content = "Please review the provided context and assist with the codebase."
    if args.prompt:
        if args.prompt.startswith("@"):
            prompt_var = args.prompt[1:]
            import foo_files
            prompt_content = getattr(foo_files, prompt_var, "")
            if not prompt_content:
                logger.print(f"Warning: Prompt variable '{prompt_var}' not found in foo_files.py. Using default prompt.")
                prompt_content = "Please review the provided context and assist with the codebase."
        elif os.path.exists(args.prompt):
            with open(args.prompt, 'r', encoding='utf-8') as f: prompt_content = f.read()
        else:
            prompt_content = args.prompt
    elif os.path.exists("prompt.md"):
        with open("prompt.md", 'r', encoding='utf-8') as f: prompt_content = f.read()

    # 2. Process all specified files (💥 UPDATED WITH KWARGS)
    annotate_foo_files_in_place(args.chop)
    update_paintbox_in_place()
    check_topological_integrity(args.chop, format_kwargs)
    files_to_process = parse_file_list_from_config(args.chop, format_kwargs)

    # Inject --slugs as direct file paths into the processing queue
    if args.slugs:
        all_articles = _get_article_list_data(CONFIG["POSTS_DIRECTORY"], url_config=active_target_config)
        target_slugs = set(args.slugs)
        for article in all_articles:
            stem = os.path.splitext(os.path.basename(article['path']))[0]
            clean_slug = re.sub(r'^\d{4}-\d{2}-\d{2}-', '', stem)
            if clean_slug in target_slugs:
                files_to_process.append((article['path'], f"slug:{clean_slug}"))
                logger.print(f"🎯 Resolved slug '{clean_slug}' to: {article['path']}")

    processed_files_data = []

    logger.print("--- Processing Files ---")
    for path, comment in files_to_process:
        # HANDLE DYNAMIC COMMANDS (The ! Chisel-Strike)
        if path.startswith('! '):
            raw_command = path[2:].strip()
            
            # Pragmatic Templating: Replace {{filename.ext}} with local file contents
            def inject_file(match):
                filepath = match.group(1).strip()
                full_filepath = os.path.join(REPO_ROOT, filepath) if not os.path.isabs(filepath) else filepath
                if os.path.exists(full_filepath):
                    with open(full_filepath, 'r', encoding='utf-8') as f:
                        return f.read()
                return f"ERROR_FILE_NOT_FOUND:{filepath}"
            
            command_str = re.sub(r'\{\{(.+?)\}\}', inject_file, raw_command)
            
            logger.print(f"   -> Executing: {raw_command}")
            try:
                result = subprocess.run(command_str, shell=True, capture_output=True, text=True, check=True)
                content = result.stdout.strip() or "(Executed successfully, no output)"
                
                processed_files_data.append({
                    "path": f"COMMAND: {raw_command}", "comment": comment, "content": content,
                    "tokens": count_tokens(content), "words": count_words(content), "lang": "text"
                })
            except subprocess.CalledProcessError as e:
                logger.print(f"      [Error] Exit {e.returncode}: {e.stderr.strip()}")
            continue

        # HANDLE REMOTE URLS (And JIT Optical Distillation)
        if path.startswith(('http://', 'https://', '!http://', '!https://')):
            target_url = path[1:].strip() if path.startswith('!') else path.strip()
            
            if path.startswith('!'):
                # JIT OPTICAL DISTILLATION (The MST3K Balcony)
                logger.print(f"   -> 👁️‍🗨️ Engaging LLM Optics for: {target_url}")
                from tools.scraper_tools import selenium_automation
                from urllib.parse import urlparse, quote
                
                parsed = urlparse(target_url)
                domain = parsed.netloc
                path_slug = quote(parsed.path or '/', safe='').replace('/', '_')[:100]

                scrape_params = {
                    "url": target_url, "domain": domain, "url_path_slug": path_slug,
                    "take_screenshot": False, "headless": False, "is_notebook_context": True, "verbose": False,
                    "override_cache": True  # 💥 Bust the cached Cloudflare 403 page!
                }
                
                import asyncio
                result = asyncio.run(selenium_automation(scrape_params))
                
                if result.get("success"):
                    artifacts = result.get("looking_at_files", {})
                    lenses = [('seo_md', 'SEO Metadata'), ('accessibility_tree_summary', 'Semantic Outline'), ('hierarchy_txt', 'DOM Hierarchy')]
                    
                    for key, title in lenses:
                        file_path = artifacts.get(key)
                        if file_path and os.path.exists(file_path):
                            with open(file_path, 'r', encoding='utf-8') as f: content = f.read()
                            processed_files_data.append({
                                "path": f"OPTICS [{title}]: {target_url}", "comment": comment, "content": content,
                                "tokens": count_tokens(content), "words": count_words(content), "lang": "markdown" if key == 'seo_md' else "text"
                            })
                else:
                    logger.print(f"      [Error] Scrape failed: {result.get('error')}")
            else:
                # STANDARD NAIVE FETCH (For raw text/code)
                try:
                    logger.print(f"   -> Fetching URL: {target_url}")
                    with urllib.request.urlopen(target_url) as response:
                        content = response.read().decode('utf-8')
                    ext = os.path.splitext(target_url.split('?')[0])[1].lower()
                    lang_map = {'.py': 'python', '.js': 'javascript', '.html': 'html', '.css': 'css', '.md': 'markdown', '.json': 'json', '.nix': 'nix', '.sh': 'bash'}
                    lang = lang_map.get(ext, 'text')
                except UnicodeDecodeError:
                    content = f"# [Binary file or incompatible encoding omitted from text context: {os.path.basename(target_url)}]"
                    lang = "text"
                    processed_files_data.append({
                        "path": target_url, "comment": comment, "content": content,
                        "tokens": count_tokens(content), "words": count_words(content), "lang": lang
                    })
                except Exception as e:
                    logger.print(f"Error fetching URL {target_url}: {e}")
            continue

        # ABSOLUTE PATH CERTAINTY: Resolve to absolute path immediately
        full_path = os.path.join(REPO_ROOT, path) if not os.path.isabs(path) else path
        
        if not os.path.exists(full_path):
            logger.print(f"Warning: FILE NOT FOUND AND WILL BE SKIPPED: {full_path} <--------------------------- !!!")
            continue
        content, lang = "", "text"
        ext = os.path.splitext(full_path)[1].lower()
        if ext == '.ipynb':
            if JUPYTEXT_AVAILABLE:
                logger.print(f"   -> Converting notebook: {full_path}")
                try:
                    notebook = jupytext.read(full_path)
                    content = jupytext.writes(notebook, fmt='py:percent')
                    lang = 'python'
                except Exception as e:
                    content = f"# FAILED TO CONVERT NOTEBOOK: {full_path}\n# ERROR: {e}"
                    logger.print(f"Warning: Failed to convert {full_path}: {e}")
            else:
                content = f"# SKIPPING NOTEBOOK CONVERSION: jupytext not installed for {full_path}"
                logger.print(f"Warning: `jupytext` library not found. Skipping conversion for {full_path}.")
        else:
            try:
                with open(full_path, 'r', encoding='utf-8') as f: content = f.read()
                lang_map = {'.py': 'python', '.js': 'javascript', '.html': 'html', '.css': 'css', '.md': 'markdown', '.json': 'json', '.nix': 'nix', '.sh': 'bash'}
                lang = lang_map.get(ext, 'text')
            except UnicodeDecodeError:
                content = f"# [Binary file or incompatible encoding omitted from text context: {os.path.basename(full_path)}]"
                lang = "text"
            except Exception as e:
                logger.print(f"ERROR: Could not read or process {full_path}: {e}")
                sys.exit(1)

        # The Tokenizer Physics: Raw bytes for the actuator. Use --line-numbers for review mode.
        if args.line_numbers and ext in {'.py', '.js', '.sh', '.nix', '.lua', '.json', '.toml', '.yaml', '.yml', '.sql', '.css', '.html', '.ipynb'}:
            content = '\n'.join(f"{i}: {line}" for i, line in enumerate(content.split('\n'), start=1))
        
        # Store using full_path for the key to ensure uniqueness and absolute reference
        processed_files_data.append({
            "path": full_path, "comment": comment, "content": content,
            "tokens": count_tokens(content), "words": count_words(content), "lang": lang
        })

    # 3. Build the prompt and add auto-generated context
    builder = PromptBuilder(processed_files_data, prompt_content, context_only=args.context_only, list_arg=args.list)

    # Only generate the codebase tree if .py files are explicitly included AND --no-tree is not set.
    # This avoids clutter when only .md, .nix, or .ipynb files are present, or when explicitly disabled.
    include_tree = any(f['path'].endswith('.py') for f in processed_files_data) and not args.no_tree

    if include_tree:
        logger.print("Python file(s) detected. Generating codebase tree diagram...", end='', flush=True)
        tree_output = run_tree_command()
        
        # === ONE-LINE MAGIC: now with token sizes ===
        annotated_tree = annotate_tree_with_tokens(tree_output, processed_files_data, REPO_ROOT)
        
        title = "Codebase Structure (eza --tree + token sizes)"
        builder.add_auto_context(title, annotated_tree)
        
        # Live feedback
        tree_data = builder.auto_context.get(title, {})
        t_count = tree_data.get('tokens', 0)
        b_count = len(tree_data.get('content', '').encode('utf-8'))
        logger.print(f" ({t_count:,} tokens | {b_count:,} bytes)")
    elif args.no_tree:
        logger.print("Skipping codebase tree (--no-tree flag detected).")
    else:
        logger.print("Skipping codebase tree (no .py files included).")

    if args.list is not None:
        logger.print("Adding narrative context from articles...", end='', flush=True)
        all_articles = _get_article_list_data(CONFIG["POSTS_DIRECTORY"], url_config=active_target_config)
        sliced_articles = []
        try:
            slice_or_index = parse_slice_arg(args.list)
            if isinstance(slice_or_index, int): sliced_articles = [all_articles[slice_or_index]]
            elif isinstance(slice_or_index, slice): sliced_articles = all_articles[slice_or_index]
        except (ValueError, IndexError):
            logger.print(f" (invalid slice '{args.list}')")
            sliced_articles = []
        
        if sliced_articles:
            # ULTRA-COMPRESSED FORMAT: Date, Slug, Title, and Semantic Keywords only
            narrative_content = ""
            for article in sliced_articles:
                # We normalize to filename/slug for topological awareness
                filename = os.path.basename(article['path'])
                slug = os.path.splitext(filename)[0]
                
                # We strip the date from the slug to make it cleaner
                slug = re.sub(r'^\d{4}-\d{2}-\d{2}-', '', slug)

                # Combine Keywords and Subtopics into a single, dense semantic vector
                raw_kws = article.get('shard_kw', [])
                raw_subs = article.get('shard_sub', [])
                
                # Handle cases where they might be strings instead of lists
                if isinstance(raw_kws, str): raw_kws = [k.strip() for k in raw_kws.split(',') if k.strip()]
                if isinstance(raw_subs, str): raw_subs = [s.strip() for s in raw_subs.split(',') if s.strip()]
                
                combined_semantics = raw_kws + raw_subs
                semantic_string = ", ".join(combined_semantics) if combined_semantics else "No semantic data"

                # OPTIMIZATION: Ultra-dense semantic hashing. Drop summaries, URLs, and token counts.
                # Format: [Date] slug-name | Title | (sem_1, sem_2, sem_3)
                dense_line = f"[{article['date']}] {slug} | {article['title']} | ({semantic_string})"
                
                narrative_content += f"{dense_line}\n"
            
            title = "Recent Narrative Context"
            builder.add_auto_context(title, narrative_content.strip())
        else:
            logger.print(" (no articles found or invalid slice)")
    
    if args.article is not None:
        logger.print("Adding full article content...")
        all_articles = _get_article_list_data(CONFIG["POSTS_DIRECTORY"], url_config=active_target_config)
        sliced_articles = []
        try:
            slice_or_index = parse_slice_arg(args.article)
            if isinstance(slice_or_index, int):
                sliced_articles = [all_articles[slice_or_index]]
            elif isinstance(slice_or_index, slice):
                sliced_articles = all_articles[slice_or_index]
        except (ValueError, IndexError):
            logger.print(f" (invalid slice '{args.article}')")

        full_content_parts = []
        
        if sliced_articles:
            for idx, article in enumerate(sliced_articles, start=1):
                try:
                    with open(article['path'], 'r', encoding='utf-8') as f:
                        content = f.read()
                    full_content_parts.append(f"--- START: Article: {os.path.basename(article['path'])} ---\n{content.strip()}\n--- END: Article ---\n")
                    t_count = article.get('tokens', count_tokens(content))
                    b_count = article.get('bytes', len(content.encode('utf-8')))
                    order = article.get('sort_order', 0)
                    abs_path = os.path.abspath(article['path'])
                    logger.print(f"{abs_path}  # [Idx: {idx} | Order: {order} | Tokens: {t_count:,} | Bytes: {b_count:,}]", flush=True)
                except Exception as e:
                    logger.print(f"\nWarning: Could not read article {article['path']}: {e}")

        # NEW: Process paths from --decanter-from (file or stdin)
        if args.decanter_from:
            import sys as _sys
            if args.decanter_from == '-':
                extra_paths = [line.strip() for line in _sys.stdin if line.strip()]
            else:
                with open(args.decanter_from, 'r', encoding='utf-8') as _df:
                    extra_paths = [line.strip() for line in _df if line.strip()]
            if args.decanter is None:
                args.decanter = []
            args.decanter.extend(extra_paths)

        # NEW: Map raw slugs directly to absolute paths and stage them for decanting
        if args.slugs:
            if args.decanter is None:
                args.decanter = []
            all_articles = _get_article_list_data(CONFIG["POSTS_DIRECTORY"], url_config=active_target_config)
            target_slugs = set(args.slugs)
            for article in all_articles:
                filename = os.path.basename(article['path'])
                stem = os.path.splitext(filename)[0]
                clean_slug = re.sub(r'^\d{4}-\d{2}-\d{2}-', '', stem)
                if clean_slug in target_slugs:
                    args.decanter.append(article['path'])
                    logger.print(f"🎯 Resolved slug '{clean_slug}' to: {article['path']}")
            # Ensure article processing runs when only --slugs is passed
            if args.article is None and args.decanter:
                args.article = "[-0:]"

        # NEW: Process explicitly targeted Decanter files
        if args.decanter:
            for idx, decanter_path in enumerate(args.decanter, start=1):
                try:
                    # Resolve absolute path to be safe
                    full_path = os.path.abspath(decanter_path)
                    if os.path.exists(full_path):
                        with open(full_path, 'r', encoding='utf-8') as f:
                            content = f.read()
                        full_content_parts.append(f"--- START: Decanter Article: {os.path.basename(full_path)} ---\n{content.strip()}\n--- END: Decanter Article ---\n")
                        t_count = count_tokens(content)
                        b_count = len(content.encode('utf-8'))
                        logger.print(f"{full_path}  # [Decanter Idx: {idx} | Tokens: {t_count:,} | Bytes: {b_count:,}]", flush=True)
                    else:
                        logger.print(f"\nWarning: Decanter target not found: {full_path}")
                except Exception as e:
                    logger.print(f"\nWarning: Could not read decanter target {decanter_path}: {e}")
        if full_content_parts:
            full_article_content = "\n".join(full_content_parts)
            title = "Full Article Content"
            builder.add_auto_context(title, full_article_content)
            
            # Calculate sizes for live display
            article_data = builder.auto_context.get(title, {})
            t_count = article_data.get('tokens', 0)
            b_count = len(article_data.get('content', '').encode('utf-8'))
            
            # Adjust log message to account for mixed sources
            total_articles = len(sliced_articles) + (len(args.decanter) if args.decanter else 0)
            logger.print(f"  Total: {total_articles} full articles | {t_count:,} tokens | {b_count:,} bytes")
        elif not args.article and not args.decanter:
            logger.print("  (no articles found or invalid slice)")

    # After slicing articles for -l or -a...
    if args.context and sliced_articles:
        logger.print("Pairing holographic context shards...", end='', flush=True)
        add_holographic_shards(builder, sliced_articles)

    python_files_to_diagram = [f['path'] for f in processed_files_data if f['path'].endswith('.py')]
    if python_files_to_diagram and not args.no_tree:
        logger.print("Python file(s) detected. Generating UML diagrams...")
        for py_file_path in python_files_to_diagram:
            logger.print(f"   -> Generating for {py_file_path}...", end='', flush=True)
            uml_context = generate_uml_and_dot(py_file_path, CONFIG["PROJECT_NAME"])
            uml_content = uml_context.get("ascii_uml")
            title = f"UML Class Diagram (ASCII for {py_file_path})"
            builder.add_auto_context(title, uml_content)
            
            if title in builder.auto_context:
                uml_data = builder.auto_context[title]
                b_count = len(uml_data['content'].encode('utf-8'))
                logger.print(f" ({uml_data['tokens']:,} tokens | {b_count:,} bytes)")
            elif uml_content and "note: no classes" in uml_content.lower():
                logger.print(" (skipped, no classes)")
            else:
                logger.print(" (skipped)")
        logger.print("...UML generation complete.\n")
 
    python_files_to_analyze = [f['path'] for f in processed_files_data if f['path'].endswith('.py') and os.path.isfile(f['path'])]
    if python_files_to_analyze:
        analysis_output = run_static_analysis(python_files_to_analyze)
        if analysis_output:
            builder.add_auto_context("Static Analysis Diagnostics", analysis_output)   
    # 4. Generate final output with convergence loop
    final_output = builder.build_final_prompt()

    # 5. Print the Summary section to console for immediate feedback
    if "Summary" in builder.all_sections:
        console_summary = builder.all_sections["Summary"]["content"]
        
        # FIX: Strip the redundant Processing Log specifically for terminal display
        # This uses DOTALL to catch everything between the header and the closing fence
        console_summary = re.sub(r'--- Processing Log ---.*?```\n\n', '', console_summary, flags=re.DOTALL)
        
        # Clean up the remaining fences for terminal readability
        console_summary = console_summary.replace("```\n", "").replace("```", "")
        print(console_summary.strip())

    # 6. Handle output
    if args.output:
        with open(args.output, 'w', encoding='utf-8') as f: f.write(final_output)
        print(f"\nOutput written to '{args.output}'")
    if not args.no_clipboard:
        copy_to_clipboard(final_output)

if __name__ == "__main__":
    main()
