"""
The Serena Model Context Protocol (MCP) Server
"""

import dataclasses
import os
import re
import shutil
from collections.abc import Iterator, Sequence
from copy import deepcopy
from dataclasses import dataclass, field
from datetime import UTC, datetime
from enum import Enum
from functools import cached_property
from pathlib import Path
from typing import TYPE_CHECKING, Any, Optional, Self, TypeVar

import yaml
from ruamel.yaml.comments import CommentedMap
from sensai.util import logging
from sensai.util.logging import LogTime, datetime_tag
from sensai.util.string import ToStringMixin

from serena.constants import (
    DEFAULT_SOURCE_FILE_ENCODING,
    PROJECT_LOCAL_TEMPLATE_FILE,
    PROJECT_TEMPLATE_FILE,
    REPO_ROOT,
    RESOURCES_DIR,
    SERENA_CONFIG_TEMPLATE_FILE,
    SERENA_FILE_ENCODING,
    SERENA_MANAGED_DIR_NAME,
)
from serena.util.inspection import determine_programming_language_composition
from serena.util.text_utils import glob_match
from serena.util.yaml import YamlCommentNormalisation, load_yaml, normalise_yaml_comments, save_yaml, transfer_yaml_comments
from solidlsp.ls_config import Language

from ..analytics import RegisteredTokenCountEstimator
from ..util.class_decorators import singleton
from ..util.cli_util import ask_yes_no
from ..util.dataclass import get_dataclass_default

if TYPE_CHECKING:
    from ..project import Project
    from ..tools.tools_base import Tool

log = logging.getLogger(__name__)
T = TypeVar("T")
DEFAULT_TOOL_TIMEOUT: float = 240
DictType = dict | CommentedMap
TDict = TypeVar("TDict", bound=DictType)


@singleton
class SerenaPaths:
    """
    Provides paths to various Serena-related directories and files.
    """

    def __init__(self) -> None:
        home_dir = os.getenv("SERENA_HOME")
        if home_dir is None or home_dir.strip() == "":
            home_dir = str(Path.home() / SERENA_MANAGED_DIR_NAME)
        else:
            home_dir = home_dir.strip()
        self.resources_dir: str = RESOURCES_DIR
        """
        the resources directory (within the `serena` package) 
        """
        self.serena_user_home_dir: str = home_dir
        """
        the path to the Serena home directory, where the user's configuration/data is stored.
        This is ~/.serena by default, but it can be overridden via the SERENA_HOME environment variable.
        """
        self.user_prompt_templates_dir: str = os.path.join(self.serena_user_home_dir, "prompt_templates")
        """
        directory containing prompt templates defined by the user.
        Prompts defined by the user take precedence over Serena's built-in prompt templates.
        """
        self.user_contexts_dir: str = os.path.join(self.serena_user_home_dir, "contexts")
        """
        directory containing contexts defined by the user. 
        If a name of a context matches a name of a context in SERENAS_OWN_CONTEXT_YAMLS_DIR, 
        the user context will override the default context definition.
        """
        self.user_modes_dir: str = os.path.join(self.serena_user_home_dir, "modes")
        """
        directory containing modes defined by the user.
        If a name of a mode matches a name of a mode in SERENAS_OWN_MODES_YAML_DIR,
        the user mode will override the default mode definition.
        """
        self.news_legacy_last_read_id_file: str = os.path.join(self.serena_user_home_dir, "last_read_news_snippet_id.txt")
        """
        file containing the ID of the last read news snippet
        """
        self.news_read_items_file: str = os.path.join(self.serena_user_home_dir, "news_read.pkl")
        """
        file containing the ID of the last read news snippet
        """
        self.news_etag_file: str = os.path.join(self.serena_user_home_dir, "news_etag.txt")
        """
        file containing the ETag of the last fetched remote news JSON
        """
        self.news_file: str = os.path.join(self.serena_user_home_dir, "news.json")
        """
        local cache of the remote news JSON file
        """
        self.news_dir: str = os.path.join(REPO_ROOT, "news")
        """
        repository news directory containing the source HTML snippets and generated news.json
        """
        global_memories_path = Path(os.path.join(self.serena_user_home_dir, "memories", "global"))
        global_memories_path.mkdir(parents=True, exist_ok=True)
        self.global_memories_path = global_memories_path
        """
        directory where global memories are stored, i.e. memories that are available across all projects
        """
        self.last_returned_log_file_path: str | None = None
        """
        the path to the last log file returned by `get_next_log_file_path`. If this is not None, the logs
        are currently being written to this file
        """

    def get_next_log_file_path(self, prefix: str) -> str:
        """
        :param prefix: the filename prefix indicating the type of the log file
        :return: the full path to the log file to use
        """
        log_dir = os.path.join(self.serena_user_home_dir, "logs", datetime.now().strftime("%Y-%m-%d"))
        os.makedirs(log_dir, exist_ok=True)
        self.last_returned_log_file_path = os.path.join(log_dir, prefix + "_" + datetime_tag() + f"_{os.getpid()}" + ".txt")
        return self.last_returned_log_file_path

    def get_resource_path(self, *path_elems: str) -> Path:
        return Path(os.path.join(self.resources_dir, *path_elems))

    # TODO: Paths from constants.py should be moved here


@dataclass
class ToolInclusionDefinition:
    """
    Defines which tools to include/exclude in Serena's operation.
    This can mean either
      * defining exclusions/inclusions to apply to an existing set of tools [incremental mode], or
      * defining a fixed set of tools to use [fixed mode].
    """

    excluded_tools: Sequence[str] = ()
    """
    the names of tools to exclude from use [incremental mode]
    """
    included_optional_tools: Sequence[str] = ()
    """
    the names of optional tools to include [incremental mode]
    """
    fixed_tools: Sequence[str] = ()
    """
    the names of tools to use as a fixed set of tools [fixed mode]
    """

    def is_fixed_tool_set(self) -> bool:
        num_fixed = len(self.fixed_tools)
        num_incremental = len(self.excluded_tools) + len(self.included_optional_tools)
        if num_fixed > 0 and num_incremental > 0:
            raise ValueError("Cannot use both fixed_tools and excluded_tools/included_optional_tools at the same time.")
        return num_fixed > 0


@dataclass
class NamedToolInclusionDefinition(ToolInclusionDefinition):
    name: str | None = None

    def __str__(self) -> str:
        return f"ToolInclusionDefinition[{self.name}]"


@dataclass
class ModeSelectionDefinition:
    default_modes: Sequence[str] | None = None


@dataclass
class ModeSelectionDefinitionWithBaseModes(ModeSelectionDefinition):
    base_modes: Sequence[str] | None = ("interactive", "editing")
    """
    the base modes to use, which are always guaranteed to be included
    """


@dataclass
class ModeSelectionDefinitionWithAddedModes(ModeSelectionDefinition):
    added_modes: Sequence[str] | None = None


class LanguageBackend(Enum):
    LSP = "LSP"
    """
    Use the language server protocol (LSP), spawning freely available language servers
    via the SolidLSP library that is part of Serena
    """
    JETBRAINS = "JetBrains"
    """
    Use the Serena plugin in your JetBrains IDE.
    (requires the plugin to be installed and the project being worked on to be open in your IDE)
    """

    @staticmethod
    def from_str(backend_str: str) -> "LanguageBackend":
        for backend in LanguageBackend:
            if backend.value.lower() == backend_str.lower():
                return backend
        raise ValueError(f"Unknown language backend '{backend_str}': valid values are {[b.value for b in LanguageBackend]}")

    def is_lsp(self) -> bool:
        return self == LanguageBackend.LSP

    def is_jetbrains(self) -> bool:
        return self == LanguageBackend.JETBRAINS

    def get_lsp_tool_class_replacements(self) -> "dict[type[Tool], type[Tool]]":
        """
        :return: mapping from LSP tool classes to replacement tool classes (functional replacements)
        """
        match self:
            case LanguageBackend.LSP:
                return {}
            case LanguageBackend.JETBRAINS:
                from ..tools import jetbrains_tools, symbol_tools

                return {
                    symbol_tools.FindSymbolTool: jetbrains_tools.JetBrainsFindSymbolTool,
                    symbol_tools.GetSymbolsOverviewTool: jetbrains_tools.JetBrainsGetSymbolsOverviewTool,
                    symbol_tools.FindReferencingSymbolsTool: jetbrains_tools.JetBrainsFindReferencingSymbolsTool,
                    symbol_tools.FindImplementationsTool: jetbrains_tools.JetBrainsFindImplementationsTool,
                    symbol_tools.FindDeclarationTool: jetbrains_tools.JetBrainsFindDeclarationTool,
                    symbol_tools.RenameSymbolTool: jetbrains_tools.JetBrainsRenameTool,
                    symbol_tools.SafeDeleteSymbol: jetbrains_tools.JetBrainsSafeDeleteTool,
                }
            case _:
                raise NotImplementedError()


class LineEnding(Enum):
    """Line ending convention for file writes."""

    LF = "lf"
    CRLF = "crlf"
    NATIVE = "native"

    @property
    def newline_str(self) -> str | None:
        """The newline parameter value for :func:`open` and :meth:`Path.write_text`.

        Returns ``None`` for native mode (platform default).
        """
        if self is LineEnding.LF:
            return "\n"
        elif self is LineEnding.CRLF:
            return "\r\n"
        return None

    @classmethod
    def from_str(cls, value: str) -> "LineEnding":
        """Parse a string value into a :class:`LineEnding`."""
        try:
            return cls(value.lower())
        except ValueError as e:
            valid = [le.value for le in cls]
            raise ValueError(f"Invalid line_ending: {value!r}. Valid values are: {valid}") from e


@dataclass
class SharedConfig(ToolInclusionDefinition, ToStringMixin):
    """Shared between SerenaConfig and ProjectConfig, the latter used to override values in the form
    (same as in ModeSelectionDefinition).
    The defaults here shall be none and should be set to the global default values in SerenaConfig.
    """

    symbol_info_budget: float | None = None
    language_backend: LanguageBackend | None = None
    line_ending: LineEnding | None = None
    read_only_memory_patterns: list[str] = field(default_factory=list)
    ignored_memory_patterns: list[str] = field(default_factory=list)
    ls_specific_settings: dict = field(default_factory=dict)
    """Advanced configuration option allowing to configure language server implementation specific options, see SolidLSPSettings for more info."""


class SerenaConfigError(Exception):
    pass


DEFAULT_PROJECT_SERENA_FOLDER_LOCATION = "$projectDir/" + SERENA_MANAGED_DIR_NAME
"""
The default template for the project Serena folder location.
Uses $projectDir and $projectFolderName as placeholders.
"""


@dataclass(kw_only=True)
class ProjectConfig(SharedConfig, ModeSelectionDefinitionWithAddedModes):
    project_name: str
    languages: list[Language]
    ignored_paths: list[str] = field(default_factory=list)
    additional_workspace_folders: list[str] = field(default_factory=list)
    read_only: bool = False
    ignore_all_files_in_gitignore: bool = True
    initial_prompt: str = ""
    encoding: str = DEFAULT_SOURCE_FILE_ENCODING

    # internal fields which are not mapped to/from the configuration file (must start with "_")
    _local_override_keys: list[str] = field(default_factory=list)

    # class-level constants
    SERENA_PROJECT_FILE = "project.yml"
    SERENA_LOCAL_PROJECT_FILE = "project.local.yml"
    FIELDS_WITHOUT_DEFAULTS = {"project_name", "languages"}
    YAML_COMMENT_NORMALISATION = YamlCommentNormalisation.LEADING
    """
    the comment normalisation strategy to use when loading/saving project configuration files.
    The template file must match this configuration (i.e. it must use leading comments if this is set to LEADING).
    """

    def _tostring_includes(self) -> list[str]:
        return ["project_name"]

    @classmethod
    def autogenerate(
        cls,
        project_root: str | Path,
        serena_config: "SerenaConfig",
        project_name: str | None = None,
        languages: list[Language] | None = None,
        save_to_disk: bool = True,
        interactive: bool = False,
    ) -> Self:
        """
        Autogenerate a project configuration for a given project root.

        :param project_root: the path to the project root
        :param serena_config: the global Serena configuration
        :param project_name: the name of the project; if None, the name of the project will be the name of the directory
            containing the project
        :param languages: the languages of the project; if None, they will be determined automatically
        :param save_to_disk: whether to save the project configuration to disk
        :param interactive: whether to run in interactive CLI mode, asking the user for input where appropriate
        :return: the project configuration
        """
        project_root = Path(project_root).resolve()
        if not project_root.exists():
            raise FileNotFoundError(f"Project root not found: {project_root}")
        with LogTime("Project configuration auto-generation", logger=log):
            log.info("Project root: %s", project_root)
            project_folder_name = project_root.name
            project_name = project_name or project_folder_name
            if languages is None:
                # determine languages automatically
                log.info("Determining programming languages used in the project")
                language_composition = determine_programming_language_composition(str(project_root))
                log.info("Language composition: %s", language_composition)
                if len(language_composition) == 0:
                    log.warning(
                        "No source files for supported language servers were found in %s. "
                        "Creating project with no configured languages. "
                        "Symbol-related tools (e.g. find_symbol, get_symbols_overview) will not work "
                        "when using the LSP backend. You can add languages later via the Serena dashboard "
                        "or by manually editing the project configuration.",
                        project_root,
                    )
                    languages_to_use: list[str] = []
                else:
                    # sort languages by number of files found
                    languages_and_percentages = sorted(
                        language_composition.items(), key=lambda item: (item[1], item[0].get_priority()), reverse=True
                    )
                    # find the language with the highest percentage and enable it
                    top_language_pair = languages_and_percentages[0]
                    other_language_pairs = languages_and_percentages[1:]
                    languages_to_use = [top_language_pair[0].value]
                    # if in interactive mode, ask the user which other languages to enable
                    if len(other_language_pairs) > 0 and interactive:
                        print(
                            "Detected and enabled main language '%s' (%.2f%% of source files)."
                            % (top_language_pair[0].value, top_language_pair[1])
                        )
                        print(f"Additionally detected {len(other_language_pairs)} other language(s).\n")
                        print("Note: Enable only languages you need symbolic retrieval/editing capabilities for.")
                        print("      Additional language servers use resources and some languages may require additional")
                        print("      system-level installations/configuration (see Serena documentation).")
                        print("\nWhich additional languages do you want to enable?")
                        for lang, perc in other_language_pairs:
                            enable = ask_yes_no("Enable %s (%.2f%% of source files)?" % (lang.value, perc), default=False)
                            if enable:
                                languages_to_use.append(lang.value)
                        print()
                log.info("Using languages: %s", languages_to_use)
            else:
                languages_to_use = [lang.value for lang in languages]
            config_with_comments, _ = cls._load_yaml_dict(PROJECT_TEMPLATE_FILE)
            config_with_comments["project_name"] = project_name
            config_with_comments["languages"] = languages_to_use

            if save_to_disk:
                project_yml_path = serena_config.get_project_yml_location(str(project_root))
                log.info("Saving project configuration to %s", project_yml_path)
                save_yaml(project_yml_path, config_with_comments)
                project_local_yml_path = os.path.join(os.path.dirname(project_yml_path), cls.SERENA_LOCAL_PROJECT_FILE)
                shutil.copy(PROJECT_LOCAL_TEMPLATE_FILE, project_local_yml_path)

            return cls._from_dict(config_with_comments, local_override_keys=[])

    @classmethod
    def default_project_yml_path(cls, project_root: str | Path) -> str:
        """
        :return: the default path to the project.yml file (inside ``$projectDir/.serena/``).
            This is suitable as a fallback when no ``SerenaConfig`` is available to resolve
            a potentially customised location.
        """
        return os.path.join(str(project_root), SERENA_MANAGED_DIR_NAME, cls.SERENA_PROJECT_FILE)

    @classmethod
    def _load_yaml_dict(
        cls,
        yml_path: str,
        comment_normalisation: YamlCommentNormalisation = YamlCommentNormalisation.NONE,
        apply_defaults: bool = True,
    ) -> tuple[CommentedMap, bool]:
        """
        Load the project configuration as a CommentedMap, preserving comments and ensuring
        completeness of the configuration by applying default values for missing fields
        and backward compatibility adjustments.

        :param yml_path: the path to the project.yml file
        :param comment_normalisation: the strategy to use for normalising comments in the loaded YAML
        :param apply_defaults: whether to apply default values for missing fields
        :return: a tuple `(dict, was_complete)` where dict is a CommentedMap representing a
          full project configuration and `was_complete` indicates whether the loaded configuration
          was complete (i.e., did not require any default values to be applied) for the case where
          `apply_defaults` is True; If `apply_defaults` is False, the returned dict may be incomplete
          and `was_complete` will always be True.
        """
        data = load_yaml(yml_path, comment_normalisation=comment_normalisation)

        # apply defaults
        was_complete = True
        if apply_defaults:
            for field_info in dataclasses.fields(cls):
                key = field_info.name
                if key.startswith("_"):
                    continue
                if key in cls.FIELDS_WITHOUT_DEFAULTS:
                    continue
                if key not in data:
                    was_complete = False
                    default_value = get_dataclass_default(cls, key)
                    data.setdefault(key, default_value)

        # backward compatibility
        # NOTE: This must also work for project.local.yml files, which may be highly incomplete
        # * handle single "language" field
        if "languages" not in data and "language" in data:
            data["languages"] = [data["language"]]
            del data["language"]

        # Note: Checks for validity of fields must not happen here but in _from_dict.
        # Here, the data may be incomplete, because this function is also used for
        # loading project.local.yml files.

        return data, was_complete

    @classmethod
    def _from_dict(cls, data: dict[str, Any], local_override_keys: list[str]) -> Self:
        """
        Create a ProjectConfig instance from a (full) configuration dictionary

        :param data: the configuration dictionary; must contain all required fields and use the same field names as
            the ProjectConfig dataclass
        :param local_override_keys: the list of keys that have been overridden from project.local.yml
        """
        lang_name_mapping = {"javascript": "typescript"}
        languages: list[Language] = []
        for language_str in data["languages"]:
            orig_language_str = language_str
            try:
                language_str = language_str.lower()
                if language_str in lang_name_mapping:
                    language_str = lang_name_mapping[language_str]
                language = Language(language_str)
                languages.append(language)
            except ValueError as e:
                raise ValueError(
                    f"Invalid language: {orig_language_str}.\nValid language_strings are: {[l.value for l in Language]}"
                ) from e

        # Validate symbol_info_budget
        symbol_info_budget_raw = data["symbol_info_budget"]
        symbol_info_budget = symbol_info_budget_raw
        if symbol_info_budget is not None:
            try:
                symbol_info_budget = float(symbol_info_budget_raw)
            except (TypeError, ValueError) as e:
                raise ValueError(f"symbol_info_budget must be a number or null, got: {symbol_info_budget_raw}") from e
            if symbol_info_budget < 0:
                raise ValueError(f"symbol_info_budget cannot be negative, got: {symbol_info_budget}")

        language_backend_value = data.get("language_backend")
        language_backend = LanguageBackend.from_str(language_backend_value) if language_backend_value else None

        line_ending_value = data.get("line_ending")
        line_ending = LineEnding.from_str(line_ending_value) if line_ending_value else None

        # gracefully handle user errors: incorrect use of None/empty where a list is required
        ignored_paths = data["ignored_paths"] or []
        fixed_tools = data["fixed_tools"] or []
        excluded_tools = data["excluded_tools"] or []
        included_optional_tools = data["included_optional_tools"] or []
        additional_workspace_folders = data.get("additional_workspace_folders") or []

        if "base_modes" in data and data["base_modes"] is not None:
            log.warning("The base_modes setting in project.yml is deprecated and will be ignored.")

        return cls(
            project_name=data["project_name"],
            languages=languages,
            ignored_paths=ignored_paths,
            additional_workspace_folders=additional_workspace_folders,
            excluded_tools=excluded_tools,
            fixed_tools=fixed_tools,
            included_optional_tools=included_optional_tools,
            read_only=data["read_only"],
            read_only_memory_patterns=data.get("read_only_memory_patterns", []),
            ignored_memory_patterns=data.get("ignored_memory_patterns", []),
            ignore_all_files_in_gitignore=data["ignore_all_files_in_gitignore"],
            initial_prompt=data["initial_prompt"],
            encoding=data["encoding"],
            line_ending=line_ending,
            language_backend=language_backend,
            added_modes=data["added_modes"],
            default_modes=data["default_modes"],
            symbol_info_budget=symbol_info_budget,
            ls_specific_settings=data.get("ls_specific_settings", {}),
            _local_override_keys=local_override_keys,
        )

    def _to_yaml_dict(self) -> dict:
        """
        :return: a yaml-serializable dictionary representation of this configuration
        """
        d = dataclasses.asdict(self)

        # drop internal fields starting with underscore
        keys = list(d.keys())
        for k in keys:
            if k.startswith("_"):
                del d[k]

        # map fields using non-primitive types to a YAML-compatible representation
        d["languages"] = [lang.value for lang in self.languages]
        d["language_backend"] = self.language_backend.value if self.language_backend is not None else None
        d["line_ending"] = self.line_ending.value if self.line_ending is not None else None

        return d

    @classmethod
    def _project_local_yml_path(cls, project_yml_path: str) -> str:
        return os.path.join(os.path.dirname(project_yml_path), cls.SERENA_LOCAL_PROJECT_FILE)

    @classmethod
    def load(cls, project_root: Path | str, serena_config: "SerenaConfig", autogenerate: bool = False) -> Self:
        """
        Load a ProjectConfig instance from the path to the project root.

        :param project_root: the path to the project root
        :param serena_config: the global Serena configuration
        :param autogenerate: whether to auto-generate the configuration if it does not exist
        """
        project_root = Path(project_root)
        project_folder_name = project_root.name
        yaml_path = serena_config.get_project_yml_location(project_root)
        log.debug("Loading project configuration from %s", yaml_path)

        # auto-generate if necessary
        if not os.path.exists(yaml_path):
            if autogenerate:
                return cls.autogenerate(project_root, serena_config)
            else:
                raise FileNotFoundError(f"Project configuration file not found: {yaml_path}")

        # load the configuration dictionary
        yaml_data, was_complete = cls._load_yaml_dict(str(yaml_path))
        if "project_name" not in yaml_data:
            yaml_data["project_name"] = project_folder_name

        # apply overrides from project.local.yml, if present
        local_yaml_path = cls._project_local_yml_path(str(yaml_path))
        local_override_keys = []
        if os.path.exists(local_yaml_path):
            local_yaml_data, _ = cls._load_yaml_dict(local_yaml_path, apply_defaults=False)
            if local_yaml_data:
                local_override_keys = list(local_yaml_data.keys())
                log.debug(
                    "Applying project configuration overrides from %s with keys %s",
                    local_yaml_path,
                    local_override_keys,
                )
                yaml_data.update(local_yaml_data)

        # instantiate the ProjectConfig
        project_config = cls._from_dict(yaml_data, local_override_keys=local_override_keys)

        # if the configuration was incomplete, re-save it to disk
        if not was_complete:
            log.info("Project configuration in %s was incomplete, re-saving with default values for missing fields", yaml_path)
            project_config.save(str(yaml_path), save_project_local_yml=False)

        return project_config

    def save(self, project_yml_path: str, save_project_local_yml: bool = True) -> None:
        """
        Saves the project configuration to disk, updating both the project.yml file and, optionally,
        the project.local.yml file to reflect overridden keys.

        Keys that are overridden by project.local.yml are not updated in project.yml.
        Only keys that are overridden are updated in project.local.yml.

        :param project_yml_path: the path to the project.yml file
        :param save_project_local_yml: whether to also update the project.local.yml file to reflect overridden keys
        """
        config_path = project_yml_path
        log.info("Saving updated project configuration to %s", config_path)

        # get the current configuration as a dictionary
        cur_dict = self._to_yaml_dict()

        # load commented map from the original file and update all non-overridden keys
        config_with_comments, _ = self._load_yaml_dict(config_path, self.YAML_COMMENT_NORMALISATION)
        for key in cur_dict:
            if key not in self._local_override_keys:
                config_with_comments[key] = cur_dict[key]

        # transfer missing comments from the template file
        template_config, _ = self._load_yaml_dict(PROJECT_TEMPLATE_FILE, self.YAML_COMMENT_NORMALISATION)
        transfer_yaml_comments(template_config, config_with_comments, self.YAML_COMMENT_NORMALISATION, force_update_all=True)

        # save project.yml
        save_yaml(config_path, config_with_comments)

        # update project.local.yml to reflect overridden keys if necessary
        if save_project_local_yml:
            project_local_yml_path = self._project_local_yml_path(project_yml_path)
            if self._local_override_keys and os.path.exists(project_local_yml_path):
                log.info("Saving updated local project configuration to %s", project_local_yml_path)
                local_config_with_comments, _ = self._load_yaml_dict(
                    project_local_yml_path, comment_normalisation=YamlCommentNormalisation.NONE, apply_defaults=False
                )
                for key in self._local_override_keys:
                    if key in cur_dict:
                        local_config_with_comments[key] = cur_dict[key]
                save_yaml(project_local_yml_path, local_config_with_comments)


class RegisteredProject(ToStringMixin):
    def __init__(
        self,
        project_root: str,
        project_config: "ProjectConfig",
        project_instance: Optional["Project"] = None,
    ) -> None:
        """
        Represents a registered project in the Serena configuration.

        :param project_root: the root directory of the project
        :param project_config: the configuration of the project
        :param project_instance: an existing project instance (if already loaded)
        """
        self.project_root = Path(project_root).resolve()
        self.project_config = project_config
        self._project_instance = project_instance

    def _tostring_exclude_private(self) -> bool:
        return True

    @property
    def project_name(self) -> str:
        return self.project_config.project_name

    @classmethod
    def from_project_instance(cls, project_instance: "Project") -> "RegisteredProject":
        return RegisteredProject(
            project_root=project_instance.project_root,
            project_config=project_instance.project_config,
            project_instance=project_instance,
        )

    @classmethod
    def from_project_root(cls, project_root: str | Path, serena_config: "SerenaConfig") -> "RegisteredProject":
        project_config = ProjectConfig.load(project_root, serena_config=serena_config)
        return RegisteredProject(
            project_root=str(project_root),
            project_config=project_config,
        )

    def matches_root_path(self, path: str | Path) -> bool:
        """
        Check if the given path matches the project root path.

        :param path: the path to check
        :return: True if the path matches the project root, False otherwise (including the case
            where this project's root directory no longer exists, e.g. a removed git worktree)
        """
        try:
            return self.project_root.samefile(Path(path).resolve())
        except OSError:
            # typically raised if the path does not exist (e.g., a removed git worktree)
            return False

    def get_project_instance(self, serena_config: "SerenaConfig") -> "Project":
        """
        Returns the project instance for this registered project, loading it if necessary.
        """
        if self._project_instance is None:
            from ..project import Project

            with LogTime(f"Loading project instance for {self}", logger=log):
                self._project_instance = Project(
                    project_root=str(self.project_root),
                    project_config=self.project_config,
                    serena_config=serena_config,
                )
        return self._project_instance


@dataclass(kw_only=True)
class SerenaConfig(SharedConfig, ModeSelectionDefinitionWithBaseModes):
    """
    Holds the Serena agent configuration, which is typically loaded from a YAML configuration file
    (when instantiated via :method:`from_config_file`), which is updated when projects are added or removed.
    For testing purposes, it can also be instantiated directly with the desired parameters.
    """

    # *** fields that are mapped directly to/from the configuration file (DO NOT RENAME) ***

    projects: list[RegisteredProject] = field(default_factory=list)
    gui_log_window: bool = False
    log_level: int = logging.INFO
    trace_lsp_communication: bool = False
    web_dashboard: bool = True
    web_dashboard_open_on_launch: bool = True
    web_dashboard_interface: str | None = None
    web_dashboard_listen_address: str = "127.0.0.1"
    web_dashboard_trusted_hosts: list[str] = field(default_factory=lambda: ["127.0.0.1", "localhost"])
    jetbrains_plugin_server_address: str = "127.0.0.1"
    jetbrains_launch_command: str | None = None
    """
    JetBrains IDE launch command, which can be used to auto-start an IDE instance on demand.
    """
    tool_timeout: float = DEFAULT_TOOL_TIMEOUT

    token_count_estimator: str = RegisteredTokenCountEstimator.CHAR_COUNT.name
    """Only relevant if `record_tool_usage` is True; the name of the token count estimator to use for tool usage statistics.
    See the `RegisteredTokenCountEstimator` enum for available options.
    
    Note: some token estimators (like tiktoken) may require downloading data files
    on the first run, which can take some time and require internet access. Others, like the Anthropic ones, may require an API key
    and rate limits may apply.
    """
    default_max_tool_answer_chars: int = 150_000
    """Used as default for tools where the apply method has a default maximal answer length.
    Even though the value of the max_answer_chars can be changed when calling the tool, it may make sense to adjust this default 
    through the global configuration.
    """

    ignored_paths: list[str] = field(default_factory=list)
    """List of paths to ignore across all projects. Same syntax as gitignore, so you can use * and **.
    These patterns are merged additively with each project's own ignored_paths."""

    project_serena_folder_location: str = DEFAULT_PROJECT_SERENA_FOLDER_LOCATION
    """
    Template for the location of the per-project .serena data folder (memories, caches, etc.).
    Supports the following placeholders:
      - $projectDir: the absolute path to the project root directory
      - $projectFolderName: the name of the project folder
    Examples:
      - "$projectDir/.serena" (default, stores data inside the project)
      - "/projects-metadata/$projectFolderName/.serena" (stores data in a central location)
    """

    trusted_project_path_patterns: list[str] = field(default_factory=lambda: ["**"])
    """
    list of glob patterns for project root directories that are considered trusted.
    The default "**" considers all project roots as trusted, which is necessary for backward compatibility.
    The default will apply if a user does not yet have the setting, while new users will get the value
    defined in the configuration template file. 
    """

    # settings with overridden defaults

    language_backend: LanguageBackend = LanguageBackend.LSP
    """
    the language backend to use for code understanding features
    """
    line_ending: LineEnding = LineEnding.NATIVE
    symbol_info_budget: float = 10.0
    """
    Time budget (seconds) for requests when tools request include_info (currently
    only supported for LSP-based tools).

    If the budget is exceeded, Serena stops issuing further requests and returns partial info results.
    0 disables the budget (no early stopping). Negative values are invalid.
    """

    # *** fields that are NOT mapped to/from the configuration file ***

    _loaded_commented_yaml: CommentedMap | None = None
    _config_file_path: str | None = None
    """
    the path to the configuration file to which updates of the configuration shall be saved;
    if None, the configuration is not saved to disk
    """

    # *** static members ***

    CONFIG_FILE = "serena_config.yml"
    CONFIG_FIELDS_WITH_TYPE_CONVERSION = {"projects", "language_backend", "line_ending"}

    # *** methods ***
    @classmethod
    def get_config_file_creation_date(cls) -> datetime | None:
        """
        :return: the creation date of the configuration file, or None if the configuration file does not exist
        """
        config_file_path = cls._determine_config_file_path()
        if not os.path.exists(config_file_path):
            return None

        # for unix systems st_ctime is the inode change time (change of metadata),
        # which is good enough for our purposes
        creation_timestamp = os.stat(config_file_path).st_ctime
        return datetime.fromtimestamp(creation_timestamp, UTC)

    @property
    def config_file_path(self) -> str | None:
        return self._config_file_path

    def _iter_config_file_mapped_fields_without_type_conversion(self) -> Iterator[str]:
        for field_info in dataclasses.fields(self):
            field_name = field_info.name
            if field_name.startswith("_"):
                continue
            if field_name in self.CONFIG_FIELDS_WITH_TYPE_CONVERSION:
                continue
            yield field_name

    def _tostring_includes(self) -> list[str]:
        return ["config_file_path"]

    @classmethod
    def _generate_config_file(cls, config_file_path: str) -> None:
        """
        Generates a Serena configuration file at the specified path from the template file.

        :param config_file_path: the path where the configuration file should be generated
        """
        log.info(f"Auto-generating Serena configuration file in {config_file_path}")
        loaded_commented_yaml = load_yaml(SERENA_CONFIG_TEMPLATE_FILE)
        save_yaml(config_file_path, loaded_commented_yaml)

    @classmethod
    def _determine_config_file_path(cls) -> str:
        """
        :return: the location where the Serena configuration file is stored/should be stored
        """
        config_path = os.path.join(SerenaPaths().serena_user_home_dir, cls.CONFIG_FILE)

        # if the config file does not exist, check if we can migrate it from the old location
        if not os.path.exists(config_path):
            old_config_path = os.path.join(REPO_ROOT, cls.CONFIG_FILE)
            if os.path.exists(old_config_path):
                log.info(f"Moving Serena configuration file from {old_config_path} to {config_path}")
                os.makedirs(os.path.dirname(config_path), exist_ok=True)
                shutil.move(old_config_path, config_path)

        return config_path

    @classmethod
    def from_config_file(cls, generate_if_missing: bool = True) -> "SerenaConfig":
        """
        Static constructor to create SerenaConfig from the configuration file
        """
        config_file_path = cls._determine_config_file_path()

        # create the configuration file from the template if necessary
        if not os.path.exists(config_file_path):
            if not generate_if_missing:
                raise FileNotFoundError(f"Serena configuration file not found: {config_file_path}")
            log.info(f"Serena configuration file not found at {config_file_path}, autogenerating...")
            cls._generate_config_file(config_file_path)

        # load the configuration
        log.info(f"Loading Serena configuration from {config_file_path}")
        try:
            loaded_commented_yaml = load_yaml(config_file_path)
        except Exception as e:
            raise ValueError(f"Error loading Serena configuration from {config_file_path}: {e}") from e

        # create the configuration instance
        instance = cls(_loaded_commented_yaml=loaded_commented_yaml, _config_file_path=config_file_path)
        num_migrations = 0

        def get_value_or_default(field_name: str) -> Any:
            nonlocal num_migrations
            if field_name not in loaded_commented_yaml:
                num_migrations += 1
            return loaded_commented_yaml.get(field_name, get_dataclass_default(SerenaConfig, field_name))

        # transfer regular fields that do not require type conversion
        for field_name in instance._iter_config_file_mapped_fields_without_type_conversion():
            assert hasattr(instance, field_name)
            setattr(instance, field_name, get_value_or_default(field_name))

        # read projects
        if "projects" not in loaded_commented_yaml:
            raise SerenaConfigError("`projects` key not found in Serena configuration. Please update your `serena_config.yml` file.")
        instance.projects = []
        for path in loaded_commented_yaml["projects"] or []:
            path = Path(path).resolve()
            try:
                path_exists = path.exists()
            except OSError as e:
                log.warning(f"Project path {path} is not accessible ({e}), skipping.")
                continue
            if not path_exists or (path.is_dir() and not os.path.isfile(instance.get_project_yml_location(str(path)))):
                log.warning(f"Project path {path} does not exist or no associated project configuration file found, skipping.")
                continue
            if path.is_file():
                path = cls._migrate_out_of_project_config_file(path)
                if path is None:
                    continue
                num_migrations += 1
            try:
                project_config = ProjectConfig.load(path, serena_config=instance)  # instance is sufficiently populated
            except Exception as e:
                log.error(
                    "Failed to load project configuration for %s: %s. "
                    "This project will be skipped. Fix or delete its "
                    ".serena/project.yml (or remove it from "
                    "serena_config.yml) to re-enable it.",
                    path,
                    e,
                )
                continue
            project = RegisteredProject(
                project_root=str(path),
                project_config=project_config,
            )
            instance.projects.append(project)

        # determine language backend
        language_backend = get_dataclass_default(SerenaConfig, "language_backend")
        if "language_backend" in loaded_commented_yaml:
            backend_str = loaded_commented_yaml["language_backend"]
            language_backend = LanguageBackend.from_str(backend_str)
        else:
            # backward compatibility (migrate Boolean field "jetbrains")
            if "jetbrains" in loaded_commented_yaml:
                num_migrations += 1
                if loaded_commented_yaml["jetbrains"]:
                    language_backend = LanguageBackend.JETBRAINS
                del loaded_commented_yaml["jetbrains"]
        instance.language_backend = language_backend

        # determine line ending
        line_ending_value = loaded_commented_yaml.get("line_ending")
        if line_ending_value:
            instance.line_ending = LineEnding.from_str(line_ending_value)
        else:
            num_migrations += 1
            instance.line_ending = get_dataclass_default(SerenaConfig, "line_ending")

        # migrate deprecated "gui_log_level" field if necessary
        if "gui_log_level" in loaded_commented_yaml:
            num_migrations += 1
            if "log_level" not in loaded_commented_yaml:
                instance.log_level = loaded_commented_yaml["gui_log_level"]
            del loaded_commented_yaml["gui_log_level"]

        # migrate "edit_global_memories"
        if "edit_global_memories" in loaded_commented_yaml:
            num_migrations += 1
            edit_global_memories = loaded_commented_yaml["edit_global_memories"]
            if not edit_global_memories:
                instance.read_only_memory_patterns.append("global/.*")
            del loaded_commented_yaml["edit_global_memories"]

        # re-save the configuration file if any migrations were performed
        if num_migrations > 0:
            log.info("Legacy configuration was migrated; re-saving configuration file")
            instance._save()

        return instance

    @classmethod
    def _migrate_out_of_project_config_file(cls, path: Path) -> Path | None:
        """
        Migrates a legacy project configuration file (which is a YAML file containing the project root) to the
        in-project configuration file (project.yml) inside the project root directory.

        :param path: the path to the legacy project configuration file
        :return: the project root path if the migration was successful, None otherwise.
        """
        log.info(f"Found legacy project configuration file {path}, migrating to in-project configuration.")
        try:
            with open(path, encoding=SERENA_FILE_ENCODING) as f:
                project_config_data = yaml.safe_load(f)
            if "project_name" not in project_config_data:
                project_name = path.stem
                with open(path, "a", encoding=SERENA_FILE_ENCODING) as f:
                    f.write(f"\nproject_name: {project_name}")
            project_root = project_config_data["project_root"]
            shutil.move(str(path), ProjectConfig.default_project_yml_path(project_root))
            return Path(project_root).resolve()
        except Exception as e:
            log.error(f"Error migrating configuration file: {e}")
            return None

    @classmethod
    def init(cls, language_backend: LanguageBackend) -> "SerenaConfig":
        """
        Supports the config initialisation CLI command, allowing the user to configure fundamental settings before
        the first launch.

        :param language_backend: the language backend to use
        :return: the created SerenaConfig instance
        """
        config = cls.from_config_file()
        config.language_backend = language_backend
        config._save()
        return config

    def with_headless_mode_overrides(self) -> "SerenaConfig":
        """
        Modifies this instance to apply overrides for headless mode, where any GUI/user interaction-based features are disabled.
        This is intended to be applied for cases where a `SerenaConfig` instance is needed to instantiate a `SerenaAgent` instance
        while the user is not expected to interact with the system (e.g. a CLI command or a test).

        :return: the instance with overrides applied for headless mode
        """
        self.gui_log_window = False
        self.web_dashboard = False
        self.jetbrains_launch_command = None
        return self

    @cached_property
    def project_paths(self) -> list[str]:
        return sorted(str(project.project_root) for project in self.projects)

    @cached_property
    def project_names(self) -> list[str]:
        return sorted(project.project_config.project_name for project in self.projects)

    def get_registered_project(self, project_root_or_name: str, autoregister: bool = False) -> Optional[RegisteredProject]:
        """
        :param project_root_or_name: path to the project root or the name of the project
        :param autoregister: whether to register the project if it exists but is not registered yet
        :return: the registered project, or None if not found
        """
        # look for project by name
        project_candidates = []
        for project in self.projects:
            if project.project_config.project_name == project_root_or_name:
                project_candidates.append(project)
        if len(project_candidates) == 1:
            return project_candidates[0]
        elif len(project_candidates) > 1:
            raise ValueError(
                f"Multiple projects found with name '{project_root_or_name}'. Please reference it by location instead. "
                f"Locations: {[p.project_root for p in project_candidates]}"
            )
        # no project found by name; check if it's a path
        if os.path.isdir(project_root_or_name):
            for project in self.projects:
                if project.matches_root_path(project_root_or_name):
                    return project
        # no registered project found; auto-register if project configuration exists
        if autoregister:
            config_path = self.get_project_yml_location(project_root_or_name)
            if os.path.isfile(config_path):
                registered_project = RegisteredProject.from_project_root(project_root_or_name, serena_config=self)
                self.add_registered_project(registered_project)
                return registered_project
        # nothing found
        return None

    def get_project(self, project_root_or_name: str) -> Optional["Project"]:
        registered_project = self.get_registered_project(project_root_or_name)
        if registered_project is None:
            return None
        else:
            return registered_project.get_project_instance(serena_config=self)

    def add_registered_project(self, registered_project: RegisteredProject) -> None:
        """
        Adds a registered project, persisting the updated project list
        """
        self.projects.append(registered_project)
        self._persist_projects()

    def add_project_from_path(self, project_root: Path | str) -> "Project":
        """
        Adds a new project to the Serena configuration from a given path, auto-generating the project
        with defaults if it does not exist.
        Will raise a FileExistsError if a project already exists at the path.

        :param project_root: the path to the project to add
        :return: the project that was added
        """
        from ..project import Project

        project_root = Path(project_root).resolve()
        if not project_root.exists():
            raise FileNotFoundError(f"Error: Path does not exist: {project_root}")
        if not project_root.is_dir():
            raise FileNotFoundError(f"Error: Path is not a directory: {project_root}")

        for already_registered_project in self.projects:
            if str(already_registered_project.project_root) == str(project_root):
                raise FileExistsError(
                    f"Project with path {project_root} was already added with name '{already_registered_project.project_name}'."
                )

        project_config = ProjectConfig.load(project_root, serena_config=self, autogenerate=True)

        new_project = Project(
            project_root=str(project_root),
            project_config=project_config,
            is_newly_created=True,
            serena_config=self,
        )
        self.add_registered_project(RegisteredProject.from_project_instance(new_project))

        return new_project

    def remove_project(self, project_name: str) -> None:
        # find the index of the project with the desired name and remove it
        for i, project in enumerate(list(self.projects)):
            if project.project_name == project_name:
                del self.projects[i]
                break
        else:
            raise ValueError(f"Project '{project_name}' not found in Serena configuration; valid project names: {self.project_names}")
        self._persist_projects()

    def _persist_projects(self) -> None:
        """
        Persists ONLY the registered-projects list, leaving every other setting at its on-disk value.

        Project (de)registration can happen while a session is running with transient runtime overrides applied
        to this in-memory instance (e.g. ``start-mcp-server --language-backend`` / ``--log-level``). A full
        :meth:`save` would write those overrides back to the global config, silently clobbering the user's
        settings. Instead we re-load the persisted config, copy in the current project list, and save that — so
        only ``projects`` is ever mutated on disk.
        """
        if self.config_file_path is None:
            return
        persisted = SerenaConfig.from_config_file()
        persisted.projects = list(self.projects)
        persisted._save()

    def _save(self) -> None:
        """
        Saves the full configuration to the file from which it was loaded (if any)

        NOTE: This method is private, because it is not usually safe to save a configuration instance used
          at runtime, because it often contains transient overrides (e.g. specified through the CLI)
          that should never be persisted back to the configuration file.
        """
        if self.config_file_path is None:
            return

        assert self._loaded_commented_yaml is not None, "Cannot save configuration without loaded YAML"

        commented_yaml = deepcopy(self._loaded_commented_yaml)

        # update fields with current values
        for field_name in self._iter_config_file_mapped_fields_without_type_conversion():
            commented_yaml[field_name] = getattr(self, field_name)

        # convert project objects into list of paths
        commented_yaml["projects"] = sorted({str(project.project_root) for project in self.projects})

        # convert language backend to string
        commented_yaml["language_backend"] = self.language_backend.value

        # convert line ending to string
        commented_yaml["line_ending"] = self.line_ending.value

        # transfer comments from the template file
        # NOTE: The template file now uses leading comments, but we previously used trailing comments,
        #       so we apply a conversion, which detects the old style and transforms it.
        # For some keys, we force updates, because old comments are problematic/misleading.
        normalise_yaml_comments(commented_yaml, YamlCommentNormalisation.LEADING_WITH_CONVERSION_FROM_TRAILING)
        template_yaml = load_yaml(SERENA_CONFIG_TEMPLATE_FILE, comment_normalisation=YamlCommentNormalisation.LEADING)
        transfer_yaml_comments(template_yaml, commented_yaml, YamlCommentNormalisation.LEADING, force_update_all=True)

        save_yaml(self.config_file_path, commented_yaml)

    @staticmethod
    def _resolve_serena_folder_location(template: str, placeholders: dict[str, str]) -> str:
        """
        Resolves a folder location template by replacing known ``$placeholder`` tokens
        and raising on any unrecognised ones.

        :param template: the template string (e.g. ``"$projectDir/.serena"``)
        :param placeholders: mapping from placeholder name (without ``$``) to replacement value
        :return: the resolved absolute path
        :raises SerenaConfigError: if the template contains an unknown ``$placeholder``
        """

        def _replace(match: re.Match[str]) -> str:
            name = match.group(1)
            if name not in placeholders:
                raise SerenaConfigError(
                    f"Unknown placeholder '${name}' in project_serena_folder_location. "
                    f"Supported placeholders: {', '.join('$' + k for k in placeholders)}"
                )
            return placeholders[name]

        result = re.sub(r"\$([A-Za-z_]\w*)", _replace, template)
        return os.path.abspath(result)

    def get_configured_project_serena_folder(self, project_root: str | Path) -> str:
        """
        Returns the resolved absolute path to the .serena data folder for a project,
        applying placeholder substitution to ``project_serena_folder_location``
        without any fallback logic.

        :param project_root: the absolute path to the project root directory
        :return: the resolved absolute path to the project's .serena folder
        :raises SerenaConfigError: if the template contains an unknown placeholder
        """
        project_folder_name = Path(project_root).name
        placeholders = {
            "projectDir": str(project_root),
            "projectFolderName": project_folder_name,
        }
        return self._resolve_serena_folder_location(self.project_serena_folder_location, placeholders)

    def get_project_serena_folder(self, project_root: str | Path) -> str:
        """
        Resolves the location of the project's .serena data folder using fallback logic:

        1. If the folder exists at the configured path (``project_serena_folder_location``), use it.
        2. Otherwise, if it exists at the default location inside the project root, use that.
        3. If neither exists, return the configured path (for creation).

        :param project_root: the absolute path to the project root directory
        :return: the resolved absolute path to the .serena data folder
        :raises SerenaConfigError: if the configured template contains an unknown placeholder
        """
        configured_path = self.get_configured_project_serena_folder(project_root)
        if os.path.isdir(configured_path):
            return configured_path
        default_path = os.path.join(str(project_root), SERENA_MANAGED_DIR_NAME)
        if configured_path != default_path and os.path.isdir(default_path):
            return default_path
        return configured_path

    def get_project_yml_location(self, project_root: str | Path) -> str:
        """
        Returns the resolved absolute path to the project.yml configuration file,
        based on the resolved .serena data folder (with fallback logic).

        :param project_root: the absolute path to the project root directory
        :return: the resolved absolute path to the project's project.yml file
        """
        serena_folder = self.get_project_serena_folder(project_root)
        return os.path.join(serena_folder, ProjectConfig.SERENA_PROJECT_FILE)

    def propagate_settings(self) -> None:
        """
        Propagate settings from this configuration to individual components that are statically configured
        """
        from serena.tools import JetBrainsPluginClient

        JetBrainsPluginClient.set_server_address(self.jetbrains_plugin_server_address)

    def is_trusted_project_path(self, project_root: str | Path) -> bool:
        """
        Checks if the given project root path matches any of the trusted project root patterns.

        :param project_root: the path to the project root directory
        :return: True if the project root is trusted, False otherwise
        """
        project_root_str = str(project_root)
        for pattern in self.trusted_project_path_patterns:
            if glob_match(pattern, project_root_str):
                return True
        return False
