"""
Settings Routes Module

This module handles all settings-related HTTP endpoints for the application.

CHECKBOX HANDLING PATTERN:
--------------------------
This module supports TWO submission modes to handle checkboxes correctly:

**MODE 1: AJAX/JSON Submission (Primary - /save_all_settings)**
- JavaScript intercepts form submission with e.preventDefault()
- Checkbox values read directly from DOM via checkbox.checked
- Data sent as JSON: {"setting.key": true/false}
- Hidden fallback inputs are managed but NOT used in this mode
- Provides better UX with instant feedback and validation

**MODE 2: Traditional POST Submission (Fallback - /save_settings)**
- Used when JavaScript is disabled (accessibility/no-JS environments)
- Browser submits form data naturally via request.form
- Hidden fallback pattern CRITICAL here:
  * Checked checkbox: Submits checkbox value, hidden input disabled
  * Unchecked checkbox: Submits hidden input value "false"
- Ensures unchecked checkboxes are captured (HTML limitation workaround)

**Implementation Details:**
1. Each checkbox has `data-hidden-fallback` attribute → hidden input ID
2. checkbox_handler.js manages hidden input disabled state
3. AJAX mode: settings.js reads checkbox.checked directly (lines 2233-2240)
4. POST mode: Flask reads request.form including enabled hidden inputs
5. Both modes use convert_setting_value() for consistent boolean conversion

**Why Both Patterns?**
- AJAX: Better UX, immediate validation, no page reload
- Traditional POST: Accessibility, progressive enhancement, JavaScript-free operation
- Hidden inputs: Only meaningful for traditional POST, ignored in AJAX mode

This dual-mode approach ensures the app works for all users while providing
optimal experience when JavaScript is available.
"""

import platform
import time
from typing import Any, Optional, Tuple
from datetime import UTC, datetime, timedelta, timezone

import requests
from flask import (
    Blueprint,
    flash,
    jsonify,
    redirect,
    request,
    session,
    url_for,
)
from flask_wtf.csrf import generate_csrf
from loguru import logger
from sqlalchemy.orm import Session

from ...config.constants import DEFAULT_OLLAMA_URL
from ...llm.providers.base import normalize_provider
from ...config.paths import get_data_directory, get_encrypted_database_path
from ...database.models import RateLimitEstimate, Setting, SettingType
from ...database.session_context import get_user_db_session
from ...database.encrypted_db import db_manager
from ...utilities.db_utils import get_settings_manager
from ...utilities.url_utils import normalize_url
from ...security.decorators import require_json_body
from ...security.egress.policy import DEFAULT_EGRESS_SCOPE
from ...security.egress.validators import (
    validate_allowed_local_hostnames,
)
from ..auth.decorators import login_required
from ..utils.request_helpers import parse_bool_arg
from ...security.rate_limiter import settings_limit
from ...settings.manager import get_typed_setting_value, parse_boolean
from ..services.settings_service import (
    create_or_update_setting,
    invalidate_settings_caches,
    reschedule_document_jobs_if_needed,
    set_setting,
)
from ..utils.route_decorators import with_user_session
from ..utils.templates import render_template_with_defaults


from ...security import safe_get
from ..warning_checks import calculate_warnings
from ...constants import DEFAULT_SEARCH_TOOL

# Settings whose changes trigger a warning recalculation.
WARNING_AFFECTING_KEYS = frozenset(
    [
        "llm.provider",
        "search.tool",
        "search.iterations",
        "search.questions_per_iteration",
        "llm.local_context_window_size",
        "llm.context_window_unrestricted",
        "llm.context_window_size",
        "policy.egress_scope",
        "llm.require_local_endpoint",
        "embeddings.require_local",
    ]
)

# Create a Blueprint for settings
settings_bp = Blueprint("settings", __name__, url_prefix="/settings")


def _filter_editable_settings(form_data: dict, db_session: Session) -> dict:
    """Remove non-editable keys from *form_data* in place.

    Returns a dict of *all* ``{key: Setting}`` records from the database
    so callers can reuse it for further validation (e.g. egress-policy checks).
    """
    all_db_settings = {
        setting.key: setting for setting in db_session.query(Setting).all()
    }

    non_editable_keys = [
        key
        for key in form_data.keys()
        if key in all_db_settings and not all_db_settings[key].editable
    ]
    if non_editable_keys:
        logger.warning("Skipping non-editable settings: {}", non_editable_keys)
        for key in non_editable_keys:
            del form_data[key]

    return all_db_settings


# NOTE: Routes use session["username"] (not .get()) intentionally.
# @login_required guarantees the key exists; direct access fails fast
# if the decorator is ever removed.

# Settings with dynamically populated options (excluded from validation)
DYNAMIC_SETTINGS = ["llm.provider", "llm.model", "search.tool"]


# Namespace validation for new setting creation via the web API.
# Keys starting with any ALLOWED prefix may be created; any prefix in
# BLOCKED takes precedence and is rejected even if it also matches an
# allowed prefix. Existing keys (updates) bypass this check — it only
# applies to creation of new DB rows through the three write routes.
ALLOWED_SETTING_PREFIXES = frozenset(
    {
        "app.",
        "backup.",
        "benchmark.",
        "chat.",
        "database.",
        "document_scheduler.",
        "embeddings.",
        "focused_iteration.",
        "general.",
        "langgraph_agent.",
        "llm.",
        "local_search_",
        "news.",
        "notifications.",
        "rag.",
        "rate_limiting.",
        "report.",
        "research_library.",
        "search.",
        "ui.",
        "web.",
    }
)
BLOCKED_SETTING_PREFIXES = frozenset(
    {
        "auth.",
        "bootstrap.",
        "db_config.",
        "security.",
        "server.",
        "testing.",
    }
)


def _is_allowed_new_setting_key(key: str) -> bool:
    """Return True if *key* is permitted to be created via the web API."""
    if not isinstance(key, str) or not key or ".." in key:
        return False
    key = key.lower()
    for prefix in BLOCKED_SETTING_PREFIXES:
        if key.startswith(prefix):
            return False
    for prefix in ALLOWED_SETTING_PREFIXES:
        if key.startswith(prefix):
            return True
    return False


def _get_setting_from_session(key: str | None, default=None):
    """Helper to get a setting using the current session context.

    A ``None`` key returns ``default``. ``SettingsManager.get_setting``
    treats ``key=None`` as "return all settings" (a real feature used by
    ``get_all_settings`` for enumeration); this route helper is for
    fetching one named setting and must not inherit that bulk-read
    semantic. Without the guard, callers would receive a dict of every
    setting (including other providers' API keys) when a provider
    declares ``api_key_setting = None``.

    NOTE: As of the LLM construction collapse (PR follow-up to #3957),
    LM Studio and Llama.cpp now declare real ``api_key_setting`` values
    with ``api_key_optional = True``, so this None guard fires only for
    truly key-less providers (custom subclasses that don't use settings
    at all).
    """
    if key is None:
        return default
    username = session.get("username")
    with get_user_db_session(username) as db_session:
        if db_session:
            settings_manager = get_settings_manager(db_session, username)
            return settings_manager.get_setting(key, default)
    return default


def _model_list_local_only() -> bool:
    """True when the model-list endpoints must NOT call cloud provider APIs
    (doing so sends the API key off-machine).

    The effective local-only posture = the ``llm.require_local_endpoint``
    toggle OR a private egress scope (``private_only``, or ``adaptive``
    resolving to private). Checking only the raw toggle missed a PRIVATE_ONLY
    user who never ticked it — the scope forces local at run time but the
    stored toggle is still False. Best-effort: returns False (allow) on error
    so a transient settings hiccup doesn't break the model dropdown.
    """
    username = session.get("username")
    try:
        with get_user_db_session(username) as db_session:
            if not db_session:
                return False
            sm = get_settings_manager(db_session, username)
            if bool(sm.get_setting("llm.require_local_endpoint", False)):
                return True
            scope = str(
                sm.get_setting("policy.egress_scope", DEFAULT_EGRESS_SCOPE)
            ).lower()
            if scope == "private_only":
                return True
            if scope == "adaptive":
                from ...security.egress.policy import context_from_snapshot

                snap = sm.get_settings_snapshot()
                if isinstance(snap, dict):
                    primary = sm.get_setting("search.tool", DEFAULT_SEARCH_TOOL)
                    return bool(
                        context_from_snapshot(
                            snap,
                            primary or DEFAULT_SEARCH_TOOL,
                            username=username,
                        ).require_local_llm
                    )
            return False
    except Exception:
        logger.debug(
            "model-list local-only check failed; allowing", exc_info=True
        )
        return False


def validate_setting(
    setting: Setting, value: Any
) -> Tuple[bool, Optional[str]]:
    """
    Validate a setting value based on its type and constraints.

    Args:
        setting: The Setting object to validate against
        value: The value to validate

    Returns:
        Tuple of (is_valid, error_message)
    """
    # Convert value to appropriate type first using SettingsManager's logic
    value = get_typed_setting_value(
        key=str(setting.key),
        value=value,
        ui_element=str(setting.ui_element),
        default=None,
        check_env=False,
    )

    # Validate based on UI element type
    if setting.ui_element == "checkbox":
        # After conversion, should be boolean
        if not isinstance(value, bool):
            return False, "Value must be a boolean"

    elif setting.ui_element in ("number", "slider", "range"):
        # After conversion, should be numeric
        if not isinstance(value, (int, float)):
            return False, "Value must be a number"

        # Check min/max constraints if defined
        if setting.min_value is not None and value < setting.min_value:
            return False, f"Value must be at least {setting.min_value}"
        if setting.max_value is not None and value > setting.max_value:
            return False, f"Value must be at most {setting.max_value}"

    elif setting.ui_element == "select":
        # Check if value is in the allowed options
        if setting.options:
            # Skip options validation for dynamically populated dropdowns
            if setting.key not in DYNAMIC_SETTINGS:
                allowed_values = [
                    opt.get("value") if isinstance(opt, dict) else opt
                    for opt in list(setting.options)  # type: ignore[arg-type]
                ]
                if value not in allowed_values:
                    return (
                        False,
                        f"Value must be one of: {', '.join(str(v) for v in allowed_values)}",
                    )

    # All checks passed
    return True, None


def coerce_setting_for_write(key: str, value: Any, ui_element: str) -> Any:
    """Coerce an incoming value to the correct type before writing to the DB.

    All web routes that save settings should use this function to ensure
    consistent type conversion.

    No JSON pre-parsing (``json.loads``) is needed here because:
    - ``get_typed_setting_value`` already parses JSON strings internally
      via ``_parse_json_value`` (for ``ui_element="json"``) and
      ``_parse_multiselect`` (for ``ui_element="multiselect"``).
    - For JSON API endpoints, ``request.get_json()`` already delivers
      dicts/lists as native Python objects.
    - For ``ui_element="text"``, pre-parsing would corrupt data: a JSON
      string like ``'{"k": "v"}'`` would become a dict, then ``str()``
      would produce ``"{'k': 'v'}"`` (Python repr, not valid JSON).
    """
    # check_env=False: we are persisting a user-supplied value, not reading
    # from an environment variable override.  check_env=True (the default)
    # would silently replace the user's value with an env var, which is
    # incorrect on the write path.
    return get_typed_setting_value(
        key=key,
        value=value,
        ui_element=ui_element,
        default=None,
        check_env=False,
    )


@settings_bp.route("/", methods=["GET"])
@login_required
def settings_page():
    """Main settings dashboard with links to specialized config pages"""
    return render_template_with_defaults("settings_dashboard.html")


@settings_bp.route("/save_all_settings", methods=["POST"])
@login_required
@settings_limit
@require_json_body(
    error_format="status", error_message="No settings data provided"
)
@with_user_session()
def save_all_settings(
    db_session: Optional[Session] = None, settings_manager=None
):
    """Handle saving all settings at once from the unified settings page"""
    try:
        from ...security.data_sanitizer import DataSanitizer

        # Process JSON data
        form_data = request.get_json()
        if not form_data:
            return (
                jsonify(
                    {
                        "status": "error",
                        "message": "No settings data provided",
                    }
                ),
                400,
            )

        # Track validation errors
        validation_errors = []
        settings_by_type: dict[str, Any] = {}

        # Track changes for logging
        updated_settings = []
        created_settings = []

        # Store original values for better messaging
        original_values = {}

        # Fetch all settings and remove non-editable keys
        all_db_settings = _filter_editable_settings(form_data, db_session)

        # Reject public hostnames being added to the local-hosts allowlist.
        _hosts_err = validate_allowed_local_hostnames(
            form_data, all_db_settings
        )
        if _hosts_err is not None:
            validation_errors.append(_hosts_err)
            return (
                jsonify(
                    {
                        "status": "error",
                        "message": "Validation errors",
                        "errors": validation_errors,
                    }
                ),
                400,
            )

        # Update each setting
        for key, value in form_data.items():
            # Skip corrupted keys or empty strings as keys
            if not key or not isinstance(key, str) or key.strip() == "":
                continue

            # Get the setting metadata from pre-fetched dict
            current_setting = all_db_settings.get(key)

            # SAFETY NET: an empty string OR the redaction sentinel for a
            # password-typed setting is a no-op, never a "clear the value"
            # request. Reasons:
            #   1. The form templates (Jinja2 + JS-rendered) deliberately
            #      render password inputs empty so the saved value never
            #      enters the HTML source. A user who blurs the field
            #      without typing must not wipe their stored API key.
            #   2. /settings/api redacts password values to the redaction
            #      sentinel ("[REDACTED]"). A stale browser tab could submit
            #      either "" or the sentinel — both must be idempotent on
            #      the DB, or the round-trip would persist the literal
            #      sentinel over the real secret.
            #   3. Defense-in-depth against direct cURL or automation
            #      mistakes that POST an empty/sentinel value.
            # To unset a password setting, clear the source env var or
            # use settings import.
            if (
                current_setting
                and DataSanitizer.is_sensitive_setting(
                    key, current_setting.ui_element
                )
                and isinstance(value, str)
                and value in ("", DataSanitizer.REDACTION_TEXT)
            ):
                logger.debug(f"Skipping empty secret write for {key} (no-op)")
                continue

            # EARLY VALIDATION: Convert checkbox values BEFORE any other processing
            # This prevents incorrect triggering of corrupted value detection
            if current_setting and current_setting.ui_element == "checkbox":
                if not isinstance(value, bool):
                    logger.debug(
                        f"Converting checkbox {key} from {type(value).__name__} to bool: {value}"
                    )
                    value = parse_boolean(value)
                    form_data[key] = (
                        value  # Update the form_data with converted value
                    )

            # Store original value for messaging
            if current_setting:
                original_values[key] = current_setting.value

            # Determine setting type and category
            if key.startswith("llm."):
                setting_type = SettingType.LLM
                category = "llm_general"
                if (
                    "temperature" in key
                    or "max_tokens" in key
                    or "batch" in key
                    or "layers" in key
                ):
                    category = "llm_parameters"
            elif key.startswith("search."):
                setting_type = SettingType.SEARCH
                category = "search_general"
                if (
                    "iterations" in key
                    or "results" in key
                    or "region" in key
                    or "questions" in key
                    or "section" in key
                ):
                    category = "search_parameters"
            elif key.startswith("report."):
                setting_type = SettingType.REPORT
                category = "report_parameters"
            elif key.startswith("database."):
                setting_type = SettingType.DATABASE
                category = "database_parameters"
            elif key.startswith("app."):
                setting_type = SettingType.APP
                category = "app_interface"
            elif key.startswith("chat."):
                setting_type = SettingType.CHAT
                category = "chat"
            else:
                setting_type = None
                category = None

            # Special handling for corrupted or empty values
            if value == "[object Object]" or (
                isinstance(value, str)
                and value.strip() in ["{}", "[]", "{", "["]
            ):
                if key.startswith("report."):
                    value = {}
                else:
                    # Use default or null for other types
                    if key == "llm.model":
                        value = ""
                    elif key == "llm.provider":
                        value = "ollama"
                    elif key == "search.tool":
                        value = DEFAULT_SEARCH_TOOL
                    elif key in ["app.theme", "app.default_theme"]:
                        value = "dark"
                    else:
                        value = None

                logger.warning(f"Corrected corrupted value for {key}: {value}")
                # NOTE: No JSON pre-parsing is done here.  After the
                # corruption replacement above, values are Python dicts
                # (e.g. {}), hardcoded strings, or None — none are JSON
                # strings that need parsing.  Type conversion below via
                # coerce_setting_for_write() handles everything; that
                # function delegates to get_typed_setting_value() which
                # already parses JSON internally for "json" and
                # "multiselect" ui_elements.

            if current_setting:
                # Coerce to correct Python type (e.g. str "5" → int 5
                # for number settings, str "true" → bool for checkboxes).
                converted_value = coerce_setting_for_write(
                    key=current_setting.key,
                    value=value,
                    ui_element=current_setting.ui_element,
                )

                # Validate the setting
                is_valid, error_message = validate_setting(
                    current_setting, converted_value
                )

                if is_valid:
                    # Save the converted setting using the same session
                    success = set_setting(
                        key, converted_value, db_session=db_session
                    )
                    if success:
                        updated_settings.append(key)

                    # Track settings by type for exporting
                    if current_setting.type not in settings_by_type:
                        settings_by_type[current_setting.type] = []
                    settings_by_type[current_setting.type].append(
                        current_setting
                    )
                else:
                    # Add to validation errors
                    validation_errors.append(
                        {
                            "key": key,
                            "name": current_setting.name,
                            "error": error_message,
                        }
                    )
            else:
                # Namespace validation: reject new keys outside allowed prefixes.
                if not _is_allowed_new_setting_key(key):
                    logger.warning(
                        "Security: Rejected setting outside allowed namespaces: {!r} (user={!r})",
                        key,
                        session["username"],
                    )
                    validation_errors.append(
                        {
                            "key": key,
                            "name": key,
                            "error": "Creating settings under this namespace is not allowed.",
                        }
                    )
                    continue

                # Create a new setting
                new_setting = {
                    "key": key,
                    "value": value,
                    "type": setting_type.value.lower()
                    if setting_type is not None
                    else "app",
                    "name": key.split(".")[-1].replace("_", " ").title(),
                    "description": f"Setting for {key}",
                    "category": category,
                    "ui_element": "text",  # Default UI element
                }

                # Determine better UI element based on value type
                if isinstance(value, bool):
                    new_setting["ui_element"] = "checkbox"
                elif isinstance(value, (int, float)) and not isinstance(
                    value, bool
                ):
                    new_setting["ui_element"] = "number"
                elif isinstance(value, (dict, list)):
                    new_setting["ui_element"] = "textarea"

                # Create the setting
                db_setting = create_or_update_setting(
                    new_setting, db_session=db_session
                )

                if db_setting:
                    created_settings.append(key)
                    # Track settings by type for exporting
                    if db_setting.type not in settings_by_type:
                        settings_by_type[db_setting.type] = []
                    settings_by_type[db_setting.type].append(db_setting)
                else:
                    validation_errors.append(
                        {
                            "key": key,
                            "name": new_setting["name"],
                            "error": "Failed to create setting",
                        }
                    )

        # Report validation errors if any
        if validation_errors:
            return (
                jsonify(
                    {
                        "status": "error",
                        "message": "Validation errors",
                        "errors": validation_errors,
                    }
                ),
                400,
            )

        # Get all settings to return to the client for proper state update
        all_settings = {}
        for setting in db_session.query(Setting).all():
            # Convert enum to string if present
            setting_type = setting.type
            if hasattr(setting_type, "value"):
                setting_type = setting_type.value

            all_settings[setting.key] = {
                "value": setting.value,
                "name": setting.name,
                "description": setting.description,
                "type": setting_type,
                "category": setting.category,
                "ui_element": setting.ui_element,
                "editable": setting.editable,
                "options": setting.options,
                "visible": setting.visible,
                "min_value": setting.min_value,
                "max_value": setting.max_value,
                "step": setting.step,
            }

        # Customize the success message based on what changed
        success_message = ""
        if len(updated_settings) == 1:
            # For a single update, provide more specific info about what changed
            key = updated_settings[0]
            # Reuse the already-fetched setting from our pre-fetched dict
            updated_setting = all_db_settings.get(key)
            name = (
                updated_setting.name
                if updated_setting
                else key.split(".")[-1].replace("_", " ").title()
            )

            # Format the message
            if key in original_values:
                new_value = updated_setting.value if updated_setting else None

                # If it's a boolean, use "enabled/disabled" language
                if isinstance(new_value, bool):
                    state = "enabled" if new_value else "disabled"
                    success_message = f"{name} {state}"
                else:
                    # For non-boolean values
                    if isinstance(new_value, (dict, list)):
                        success_message = f"{name} updated"
                    else:
                        success_message = f"{name} updated"
            else:
                success_message = f"{name} updated"
        else:
            # Multiple settings or generic message
            success_message = f"Settings saved successfully ({len(updated_settings)} updated, {len(created_settings)} created)"

        # Check if any warning-affecting settings were changed and include warnings.
        # Redact secret values in the echoed settings so a POST response never
        # ships plaintext API keys back to the browser — matching the
        # redaction the GET /settings/api endpoint already applies.
        response_data = {
            "status": "success",
            "message": success_message,
            "updated": updated_settings,
            "created": created_settings,
            "settings": DataSanitizer.redact_settings_snapshot(all_settings),
        }

        warning_affecting_keys = WARNING_AFFECTING_KEYS

        # Check if any warning-affecting settings were changed
        if any(
            key in warning_affecting_keys
            for key in updated_settings + created_settings
        ):
            warnings = calculate_warnings()
            response_data["warnings"] = warnings
            logger.info(
                f"Bulk settings update affected warning keys, calculated {len(warnings)} warnings"
            )

        invalidate_settings_caches(session["username"])
        reschedule_document_jobs_if_needed(
            session["username"], updated_settings + created_settings
        )
        return jsonify(response_data)

    except Exception:
        logger.exception("Error saving settings")
        return (
            jsonify(
                {
                    "status": "error",
                    "message": "An internal error occurred while saving settings.",
                }
            ),
            500,
        )


@settings_bp.route("/reset_to_defaults", methods=["POST"])
@login_required
@settings_limit
@with_user_session()
def reset_to_defaults(
    db_session: Optional[Session] = None, settings_manager=None
):
    """Reset all settings to their default values"""
    try:
        settings_manager.load_from_defaults_file()

        logger.info("Successfully imported settings from default files")

    except Exception:
        logger.exception("Error importing default settings")
        return jsonify(
            {
                "status": "error",
                "message": "Failed to reset settings to defaults",
            }
        ), 500

    invalidate_settings_caches(session["username"])
    return jsonify(
        {
            "status": "success",
            "message": "All settings have been reset to default values",
        }
    )


@settings_bp.route("/save_settings", methods=["POST"])
@login_required
@settings_limit
@with_user_session()
def save_settings(db_session: Optional[Session] = None, settings_manager=None):
    """Save all settings from the form using POST method - fallback when JavaScript is disabled"""
    try:
        from ...security.data_sanitizer import DataSanitizer

        # Get form data
        form_data = request.form.to_dict()

        # Remove CSRF token from the data
        form_data.pop("csrf_token", None)

        updated_count = 0
        failed_count = 0
        rejected_count = 0

        # Fetch all settings and remove non-editable keys
        all_db_settings = _filter_editable_settings(form_data, db_session)

        # Egress-policy validators — the JSON route (save_all_settings) runs
        # these; the POST fallback must too, or a JS-disabled client could
        # whitelist a public hostname as "local", which the JSON route does
        # not permit.
        for _policy_validator in (validate_allowed_local_hostnames,):
            _policy_err = _policy_validator(form_data, all_db_settings)
            if _policy_err is not None:
                flash(
                    _policy_err.get("error", "Invalid policy setting"), "error"
                )
                return redirect(url_for("settings.settings_page"))

        # Process each setting
        for key, value in form_data.items():
            try:
                # Get the setting from pre-fetched dict
                db_setting = all_db_settings.get(key)

                # Namespace validation: reject new keys outside allowed prefixes.
                # Existing keys (updates) bypass this check — it only applies
                # to creation of brand-new rows through this form-POST route.
                if db_setting is None and not _is_allowed_new_setting_key(key):
                    logger.warning(
                        "Security: Rejected setting outside allowed namespaces: {!r} (user={!r})",
                        key,
                        session["username"],
                    )
                    rejected_count += 1
                    continue

                # SAFETY NET: empty string OR the redaction sentinel for a
                # password-typed setting is a no-op — never "clear my key".
                # The no-JS form renders password inputs empty (and GET
                # redacts them to the sentinel), so a plain form submit
                # must not wipe the stored secret. Matches the guards in
                # save_all_settings + api_update_setting.
                if (
                    db_setting
                    and DataSanitizer.is_sensitive_setting(
                        key, db_setting.ui_element
                    )
                    and isinstance(value, str)
                    and value in ("", DataSanitizer.REDACTION_TEXT)
                ):
                    logger.debug(
                        f"Skipping empty secret write for {key} via "
                        "save_settings (no-op)"
                    )
                    continue

                # Coerce form POST string to correct Python type.
                if db_setting:
                    value = coerce_setting_for_write(
                        key=db_setting.key,
                        value=value,
                        ui_element=db_setting.ui_element,
                    )

                # Save the setting
                if settings_manager.set_setting(key, value, commit=False):
                    updated_count += 1
                else:
                    failed_count += 1
                    logger.warning(f"Failed to save setting {key}")

            except Exception:
                logger.exception(f"Error saving setting {key}")
                failed_count += 1

        # Commit all changes at once
        try:
            db_session.commit()

            flash(
                f"Settings saved successfully! Updated {updated_count} settings.",
                "success",
            )
            if failed_count > 0:
                flash(
                    f"Warning: {failed_count} settings failed to save.",
                    "warning",
                )
            if rejected_count > 0:
                flash(
                    f"Rejected {rejected_count} settings (unknown namespace). "
                    "This may indicate a bug or an attempted injection.",
                    "error",
                )
            invalidate_settings_caches(session["username"])

        except Exception:
            db_session.rollback()
            logger.exception("Failed to commit settings")
            flash("Error saving settings. Please try again.", "error")

        return redirect(url_for("settings.settings_page"))

    except Exception:
        logger.exception("Error in save_settings")
        flash("An internal error occurred while saving settings.", "error")
        return redirect(url_for("settings.settings_page"))


# API Routes
@settings_bp.route("/api", methods=["GET"])
@login_required
@with_user_session()
def api_get_all_settings(
    db_session: Optional[Session] = None, settings_manager=None
):
    """Get all settings.

    The response is redacted via ``DataSanitizer.redact_settings_snapshot``
    so password-typed values (API keys, OAuth tokens, etc.) come back as
    ``"[REDACTED]"`` instead of plaintext. The previous implementation
    leaked env-overridden API keys to anyone who could authenticate as a
    user — even though the values render unmasked into the server-side
    settings form too (a separate template-level concern), the JSON API
    is its own surface area: it gets cached by clients, logged by
    proxies, and copy/pasted into bug reports. Redaction here is
    defense-in-depth.

    No JS caller round-trips this endpoint's values back into a form
    (the settings form is server-rendered from the template, which uses
    the model directly), so redacting the response does not break the UI.
    """
    try:
        from ...security.data_sanitizer import DataSanitizer

        # Get query parameters
        category = request.args.get("category")

        # Get settings (nested-with-metadata shape:
        # {key: {value, ui_element, type, ...}})
        settings = settings_manager.get_all_settings()

        # Filter by category if requested
        if category:
            # Need to get all setting details to check category
            db_settings = db_session.query(Setting).all()
            category_keys = [
                s.key for s in db_settings if s.category == category
            ]

            # Filter settings by keys
            settings = {
                key: value
                for key, value in settings.items()
                if key in category_keys
            }

        settings = DataSanitizer.redact_settings_snapshot(settings)

        return jsonify({"status": "success", "settings": settings})
    except Exception:
        logger.exception("Error getting settings")
        return jsonify({"error": "Failed to retrieve settings"}), 500


@settings_bp.route("/api/<path:key>", methods=["GET"])
@login_required
@with_user_session()
def api_get_db_setting(
    key, db_session: Optional[Session] = None, settings_manager=None
):
    """Get a specific setting by key from DB, falling back to defaults.

    Password-typed values are redacted in the response so this endpoint
    matches the redaction contract enforced on ``/settings/api`` (the bulk
    GET) and ``/settings/api/bulk``. The same threat applies: any
    authenticated caller could otherwise grab a plaintext API key with
    a single GET. Metadata fields (``ui_element``, ``type``, ...) stay
    intact so the front-end can still render the correct input control.
    """
    from ...security.data_sanitizer import DataSanitizer

    try:
        # Get setting from database using the same session
        db_setting = (
            db_session.query(Setting).filter(Setting.key == key).first()
        )

        if db_setting:
            # Return full setting details from DB
            setting_data = {
                "key": db_setting.key,
                "value": DataSanitizer.redact_value(
                    db_setting.key, db_setting.ui_element, db_setting.value
                ),
                "type": db_setting.type
                if isinstance(db_setting.type, str)
                else db_setting.type.value,
                "name": db_setting.name,
                "description": db_setting.description,
                "category": db_setting.category,
                "ui_element": db_setting.ui_element,
                "options": db_setting.options,
                "min_value": db_setting.min_value,
                "max_value": db_setting.max_value,
                "step": db_setting.step,
                "visible": db_setting.visible,
                "editable": db_setting.editable,
            }
            return jsonify(setting_data)

        # Not in DB — check defaults so this endpoint is consistent
        # with GET /settings/api which includes default settings
        default_meta = settings_manager.default_settings.get(key)
        if default_meta:
            default_ui = default_meta.get("ui_element", "text")
            setting_data = {
                "key": key,
                "value": DataSanitizer.redact_value(
                    key, default_ui, default_meta.get("value")
                ),
                "type": default_meta.get("type", "APP"),
                "name": default_meta.get("name", key),
                "description": default_meta.get("description"),
                "category": default_meta.get("category"),
                "ui_element": default_ui,
                "options": default_meta.get("options"),
                "min_value": default_meta.get("min_value"),
                "max_value": default_meta.get("max_value"),
                "step": default_meta.get("step"),
                "visible": default_meta.get("visible", True),
                "editable": default_meta.get("editable", True),
            }
            return jsonify(setting_data)

        return jsonify({"error": f"Setting not found: {key}"}), 404
    except Exception:
        logger.exception(f"Error getting setting {key}")
        return jsonify({"error": "Failed to retrieve settings"}), 500


@settings_bp.route("/api/<path:key>", methods=["PUT"])
@login_required
@settings_limit
@require_json_body(error_message="No data provided")
@with_user_session(include_settings_manager=False)
def api_update_setting(key, db_session: Optional[Session] = None):
    """Update a setting"""
    try:
        from ...security.data_sanitizer import DataSanitizer

        # Get request data
        data = request.get_json()
        value = data.get("value")
        if value is None:
            return jsonify({"error": "No value provided"}), 400

        # Check if setting exists
        db_setting = (
            db_session.query(Setting).filter(Setting.key == key).first()
        )

        if db_setting:
            # Check if setting is editable
            if not db_setting.editable:
                return jsonify({"error": f"Setting {key} is not editable"}), 403

            # SAFETY NET: an empty string OR the redaction sentinel on a
            # password-typed setting is a no-op (never "clear my key").
            # Companion guard to the same check in save_all_settings — see
            # the longer rationale there. Returning 200 with a message keeps
            # the endpoint idempotent so client-side save indicators don't
            # error.
            if (
                DataSanitizer.is_sensitive_setting(key, db_setting.ui_element)
                and isinstance(value, str)
                and value in ("", DataSanitizer.REDACTION_TEXT)
            ):
                logger.debug(
                    f"Skipping empty secret write for {key} via "
                    "api_update_setting (no-op)"
                )
                return jsonify(
                    {
                        "message": (
                            f"Setting {key} unchanged (empty password ignored)"
                        )
                    }
                ), 200

            # Coerce to correct Python type before saving.
            # Without this, values from JSON API requests are stored
            # as-is (e.g. string "5" instead of int 5 for number
            # settings, string "true" instead of bool for checkboxes).
            value = coerce_setting_for_write(
                key=db_setting.key,
                value=value,
                ui_element=db_setting.ui_element,
            )

            # Validate the setting (matches save_all_settings pattern)
            is_valid, error_message = validate_setting(db_setting, value)
            if not is_valid:
                logger.warning(
                    f"Validation failed for setting {key}: {error_message}"
                )
                return jsonify(
                    {"error": f"Invalid value for setting {key}"}
                ), 400

            # Cross-field egress-policy validation. The full-form saves
            # (save_all_settings / save_settings) run these guards, but this
            # single-key PUT endpoint did not — letting a client smuggle a
            # public hostname into llm.allowed_local_hostnames (which the host
            # classifier then trusts as local), one key at a time. Run the
            # same validators here for the keys they govern.
            if key in (
                "policy.egress_scope",
                "search.tool",
                "llm.allowed_local_hostnames",
            ):
                _all_db_settings = {
                    s.key: s for s in db_session.query(Setting).all()
                }
                _form = {key: value}
                for _validator in (validate_allowed_local_hostnames,):
                    _err = _validator(_form, _all_db_settings)
                    if _err is not None:
                        logger.bind(policy_audit=True).warning(
                            "egress-policy setting rejected at "
                            "api_update_setting",
                            key=key,
                            reason=_err.get("error"),
                        )
                        return jsonify({"error": _err["error"]}), 400

            # Update setting
            # Pass the db_session to avoid session lookup issues
            success = set_setting(key, value, db_session=db_session)
            if success:
                response_data: dict[str, Any] = {
                    "message": f"Setting {key} updated successfully"
                }

                # If this is a key that affects warnings, include warning calculations
                warning_affecting_keys = WARNING_AFFECTING_KEYS

                if key in warning_affecting_keys:
                    warnings = calculate_warnings()
                    response_data["warnings"] = warnings
                    logger.debug(
                        f"Setting {key} changed to {value}, calculated {len(warnings)} warnings"
                    )

                invalidate_settings_caches(session["username"])
                # A document_scheduler.* toggle (e.g. sweep_library_collections
                # or generate_rag) must take effect without a re-login.
                reschedule_document_jobs_if_needed(session["username"], [key])
                return jsonify(response_data)
            return jsonify({"error": f"Failed to update setting {key}"}), 500

        # Namespace validation: reject new keys outside allowed prefixes.
        if not _is_allowed_new_setting_key(key):
            logger.warning(
                "Security: Rejected setting outside allowed namespaces: {!r} (user={!r})",
                key,
                session["username"],
            )
            return jsonify(
                {
                    "error": f"Creating settings under this namespace is not allowed: {key}"
                }
            ), 400

        # Create new setting with default metadata
        setting_dict = {
            "key": key,
            "value": value,
            "name": key.split(".")[-1].replace("_", " ").title(),
            "description": f"Setting for {key}",
        }

        # Add additional metadata if provided.
        # 'visible' and 'editable' are system-controlled — not accepted from callers.
        for field in [
            "type",
            "name",
            "description",
            "category",
            "ui_element",
            "options",
            "min_value",
            "max_value",
            "step",
        ]:
            if field in data:
                setting_dict[field] = data[field]

        # Create setting
        db_setting = create_or_update_setting(
            setting_dict, db_session=db_session
        )

        if db_setting:
            invalidate_settings_caches(session["username"])
            reschedule_document_jobs_if_needed(session["username"], [key])
            return (
                jsonify(
                    {
                        "message": f"Setting {key} created successfully",
                        "setting": {
                            "key": db_setting.key,
                            # Don't echo a freshly-created password value
                            # back in plaintext — redact like every other
                            # response that ships settings to the browser.
                            "value": (
                                DataSanitizer.REDACTION_TEXT
                                if DataSanitizer.is_sensitive_setting(
                                    key, db_setting.ui_element
                                )
                                else db_setting.value
                            ),
                            "type": db_setting.type.value,
                            "name": db_setting.name,
                        },
                    }
                ),
                201,
            )
        return jsonify({"error": f"Failed to create setting {key}"}), 500
    except Exception:
        logger.exception(f"Error updating setting {key}")
        return jsonify({"error": "Failed to update setting"}), 500


@settings_bp.route("/api/<path:key>", methods=["DELETE"])
@login_required
@with_user_session()
def api_delete_setting(
    key, db_session: Optional[Session] = None, settings_manager=None
):
    """Delete a setting"""
    try:
        # Check if setting exists
        db_setting = (
            db_session.query(Setting).filter(Setting.key == key).first()
        )
        if not db_setting:
            return jsonify({"error": f"Setting not found: {key}"}), 404

        # Check if setting is editable
        if not db_setting.editable:
            return jsonify({"error": f"Setting {key} is not editable"}), 403

        # Delete setting
        success = settings_manager.delete_setting(key)
        if success:
            invalidate_settings_caches(session["username"])
            return jsonify({"message": f"Setting {key} deleted successfully"})
        return jsonify({"error": f"Failed to delete setting {key}"}), 500
    except Exception:
        logger.exception(f"Error deleting setting {key}")
        return jsonify({"error": "Failed to delete setting"}), 500


@settings_bp.route("/api/import", methods=["POST"])
@login_required
@settings_limit
@with_user_session()
def api_import_settings(
    db_session: Optional[Session] = None, settings_manager=None
):
    """Import settings from defaults file"""
    try:
        settings_manager.load_from_defaults_file()

        invalidate_settings_caches(session["username"])
        return jsonify({"message": "Settings imported successfully"})
    except Exception:
        logger.exception("Error importing settings")
        return jsonify({"error": "Failed to import settings"}), 500


@settings_bp.route("/api/categories", methods=["GET"])
@login_required
@with_user_session(include_settings_manager=False)
def api_get_categories(db_session: Optional[Session] = None):
    """Get all setting categories"""
    try:
        # Get all distinct categories
        categories = db_session.query(Setting.category).distinct().all()
        category_list = [c[0] for c in categories if c[0] is not None]

        return jsonify({"categories": category_list})
    except Exception:
        logger.exception("Error getting categories")
        return jsonify({"error": "Failed to retrieve settings"}), 500


@settings_bp.route("/api/types", methods=["GET"])
@login_required
def api_get_types():
    """Get all setting types"""
    try:
        # Get all setting types
        types = [t.value for t in SettingType]
        return jsonify({"types": types})
    except Exception:
        logger.exception("Error getting types")
        return jsonify({"error": "Failed to retrieve settings"}), 500


@settings_bp.route("/api/ui_elements", methods=["GET"])
@login_required
def api_get_ui_elements():
    """Get all UI element types"""
    try:
        # Define supported UI element types
        ui_elements = [
            "text",
            "select",
            "checkbox",
            "slider",
            "number",
            "textarea",
            "color",
            "date",
            "file",
            "password",
        ]

        return jsonify({"ui_elements": ui_elements})
    except Exception:
        logger.exception("Error getting UI elements")
        return jsonify({"error": "Failed to retrieve settings"}), 500


@settings_bp.route("/api/available-models", methods=["GET"])
@login_required
def api_get_available_models():
    """Get available LLM models from various providers"""
    endpoint_start = time.perf_counter()
    try:
        from ...database.models import ProviderModel

        # Check if force_refresh is requested
        force_refresh = parse_bool_arg("force_refresh")

        # Get all auto-discovered providers (show all so users can discover
        # and configure providers they haven't set up yet)
        from ...llm.providers import get_discovered_provider_options

        # LlamaCppProvider is auto-discovered (provider_name="llama.cpp"), so
        # it is already included here. The previous hardcoded LLAMACPP append
        # produced a duplicate entry in the dropdown.
        provider_options = get_discovered_provider_options()

        # Available models by provider
        providers: dict[str, Any] = {}

        # Check database cache first (unless force_refresh is True)
        if not force_refresh:
            try:
                # Define cache expiration (24 hours)
                cache_expiry = datetime.now(UTC) - timedelta(hours=24)

                # Get cached models from database
                username = session["username"]
                with get_user_db_session(username) as db_session:
                    cached_models = (
                        db_session.query(ProviderModel)
                        .filter(ProviderModel.last_updated > cache_expiry)
                        .all()
                    )

                if cached_models:
                    logger.info(
                        f"Found {len(cached_models)} cached models in database"
                    )

                    # Group models by provider
                    for model in cached_models:
                        provider_key = (
                            f"{normalize_provider(model.provider)}_models"
                        )
                        if provider_key not in providers:
                            providers[provider_key] = []

                        providers[provider_key].append(
                            {
                                "value": model.model_key,
                                "label": model.model_label,
                                "provider": model.provider.upper(),
                            }
                        )

                    # If we have cached data for all providers, return it
                    if providers:
                        _log_available_models_duration(
                            endpoint_start, cache_hit=True
                        )
                        logger.info("Returning cached models from database")
                        return jsonify(
                            {
                                "provider_options": provider_options,
                                "providers": providers,
                            }
                        )

            except Exception:
                logger.warning("Error reading cached models from database")
                # Continue to fetch fresh data

        # Ollama / OpenAI / Anthropic model listing is handled by the
        # auto-discovery loop below (their provider classes' list_models_for_api),
        # which is the single fetch path. The previous hand-rolled per-provider
        # blocks here were dead in normal mode (overwritten by discovery) and an
        # inferior duplicate (no SSRF validation, no auth headers); local-only mode
        # now keeps the local providers via the LOCAL_PROVIDERS filter below.

        # Fetch models from auto-discovered providers
        from ...llm.providers import discover_providers

        discovered_providers = discover_providers()

        # Egress policy: when the effective posture is local-only the user
        # has opted into local-only inference, so don't list cloud providers
        # (OpenRouter, Google, XAI, IonOS, OpenAI, Anthropic, ...). We still
        # list the known-local providers (ollama/llamacpp/lmstudio) via their
        # provider classes — which validate the URL (SSRF) and support auth
        # headers — by filtering the discovered set to LOCAL_PROVIDERS rather
        # than skipping discovery entirely. Use the scope-aware helper so a
        # PRIVATE_ONLY / adaptive-private user is covered even if the raw
        # toggle is False.
        require_local_for_discovered = _model_list_local_only()
        if require_local_for_discovered:
            from ...llm.providers._helpers import LOCAL_PROVIDERS

            local_only_providers = {
                key: info
                for key, info in discovered_providers.items()
                if normalize_provider(key) in LOCAL_PROVIDERS
            }
            logger.bind(policy_audit=True).info(
                "local-only egress posture: limiting discovered model lists "
                "to local providers",
                kept=list(local_only_providers.keys()),
                skipped=[
                    key
                    for key in discovered_providers
                    if key not in local_only_providers
                ],
            )
            discovered_providers = local_only_providers

        for provider_key, provider_info in discovered_providers.items():
            provider_models = []
            try:
                logger.info(
                    f"Fetching models from {provider_info.provider_name}"
                )

                # Get the provider class
                provider_class = provider_info.provider_class

                # Get API key if configured
                api_key = _get_setting_from_session(
                    provider_class.api_key_setting, ""
                )

                # Get base URL if provider has configurable URL
                provider_base_url: str | None = None
                if (
                    hasattr(provider_class, "url_setting")
                    and provider_class.url_setting
                ):
                    provider_base_url = _get_setting_from_session(
                        provider_class.url_setting, ""
                    )

                # Use the provider's list_models_for_api method
                models = provider_class.list_models_for_api(
                    api_key, provider_base_url
                )

                # Format models for the API response
                for model in models:
                    provider_models.append(
                        {
                            "value": model["value"],
                            "label": model[
                                "label"
                            ],  # Use provider's label as-is
                            "provider": provider_key,
                        }
                    )

                logger.info(
                    f"Successfully fetched {len(provider_models)} models from {provider_info.provider_name}"
                )

            except Exception:
                logger.exception(
                    f"Error getting {provider_info.provider_name} models"
                )

            # Set models in providers dict using lowercase key
            providers[f"{normalize_provider(provider_key)}_models"] = (
                provider_models
            )
            logger.info(
                f"Final {provider_key} models count: {len(provider_models)}"
            )

        # Save fetched models to database cache
        if force_refresh or providers:
            # We fetched fresh data, save it to database
            username = session["username"]
            with get_user_db_session(username) as db_session:
                try:
                    if force_refresh:
                        # When force refresh, clear ALL cached models to remove any stale data
                        # from old code versions or deleted providers
                        deleted_count = db_session.query(ProviderModel).delete()
                        logger.info(
                            f"Force refresh: cleared all {deleted_count} cached models"
                        )
                    else:
                        # Clear old cache entries only for providers we're updating
                        for provider_key in providers:
                            provider_name = provider_key.replace(
                                "_models", ""
                            ).upper()
                            db_session.query(ProviderModel).filter(
                                ProviderModel.provider == provider_name
                            ).delete()

                    # Insert new models
                    for provider_key, models in providers.items():
                        provider_name = provider_key.replace(
                            "_models", ""
                        ).upper()
                        for model in models:
                            if (
                                isinstance(model, dict)
                                and "value" in model
                                and "label" in model
                            ):
                                new_model = ProviderModel(
                                    provider=provider_name,
                                    model_key=model["value"],
                                    model_label=model["label"],
                                    last_updated=datetime.now(UTC),
                                )
                                db_session.add(new_model)

                    db_session.commit()
                    logger.info("Successfully cached models to database")

                except Exception:
                    logger.exception("Error saving models to database cache")
                    db_session.rollback()

        # Return all options
        _log_available_models_duration(endpoint_start, cache_hit=False)
        return jsonify(
            {"provider_options": provider_options, "providers": providers}
        )

    except Exception:
        logger.exception("Error getting available models")
        _log_available_models_duration(
            endpoint_start, cache_hit=False, error=True
        )
        return jsonify(
            {
                "status": "error",
                "message": "Failed to retrieve available models",
            }
        ), 500


def _log_available_models_duration(
    start: float, cache_hit: bool, error: bool = False
) -> None:
    """Log /api/available-models endpoint duration.

    Uses INFO when the endpoint took > 1s (indicating a real provider fetch
    latency worth flagging), DEBUG otherwise. This is the likely culprit for
    Path C (LLM provider timeout masquerading as backend hang).
    """
    elapsed_ms = (time.perf_counter() - start) * 1000
    path = (
        "error"
        if error
        else ("cache hit" if cache_hit else "full provider fetch")
    )
    if elapsed_ms > 1000:
        logger.info(f"/api/available-models ({path}) took {elapsed_ms:.0f}ms")
    else:
        logger.debug(f"/api/available-models ({path}) took {elapsed_ms:.0f}ms")


def _get_engine_icon_and_category(
    engine_data: dict, engine_class=None
) -> tuple:
    """
    Get icon emoji and category label for a search engine based on its attributes.

    Args:
        engine_data: Engine configuration dictionary
        engine_class: Optional loaded engine class to check attributes

    Returns:
        Tuple of (icon, category) strings
    """
    # Check attributes from either the class or the engine data
    if engine_class:
        is_scientific = getattr(engine_class, "is_scientific", False)
        is_generic = getattr(engine_class, "is_generic", False)
        is_local = getattr(engine_class, "is_local", False)
        is_news = getattr(engine_class, "is_news", False)
        is_code = getattr(engine_class, "is_code", False)
    else:
        is_scientific = engine_data.get("is_scientific", False)
        is_generic = engine_data.get("is_generic", False)
        is_local = engine_data.get("is_local", False)
        is_news = engine_data.get("is_news", False)
        is_code = engine_data.get("is_code", False)

    # Check books attribute
    if engine_class:
        is_books = getattr(engine_class, "is_books", False)
    else:
        is_books = engine_data.get("is_books", False)

    # Return icon and category based on engine type
    # Priority: local > scientific > news > code > books > generic > default
    if is_local:
        return "📁", "Local RAG"
    if is_scientific:
        return "🔬", "Scientific"
    if is_news:
        return "📰", "News"
    if is_code:
        return "💻", "Code"
    if is_books:
        return "📚", "Books"
    if is_generic:
        return "🌐", "Web Search"
    return "🔍", "Search"


@settings_bp.route("/api/available-search-engines", methods=["GET"])
@login_required
@with_user_session()
def api_get_available_search_engines(
    db_session: Optional[Session] = None, settings_manager=None
):
    """Get available search engines"""
    try:
        # Get search engines using the same approach as search_engines_config.py
        from ...web_search_engines.search_engines_config import search_config
        from ...web_search_engines.engine_groups import (
            classify_engine_group,
            effective_group,
            group_label,
            group_order,
        )

        username = session["username"]
        search_engines = search_config(username=username, db_session=db_session)

        # Get user's favorites using SettingsManager
        favorites = settings_manager.get_setting("search.favorites", [])
        if not isinstance(favorites, list):
            favorites = []

        # Extract search engines from config
        engines_dict = {}
        engine_options = []

        if search_engines:
            # Format engines for API response with metadata
            from ...security.module_whitelist import (
                get_safe_module_class,
                SecurityError,
            )

            for engine_id, engine_data in search_engines.items():
                # Try to load the engine class to get metadata
                engine_class = None
                try:
                    module_path = engine_data.get("module_path")
                    class_name = engine_data.get("class_name")
                    if module_path and class_name:
                        # Use secure whitelist-validated import
                        engine_class = get_safe_module_class(
                            module_path, class_name
                        )
                except SecurityError:
                    logger.warning(
                        f"Security: Blocked unsafe module for {engine_id}"
                    )
                except Exception as e:
                    logger.debug(
                        f"Could not load engine class for {engine_id}: {e}"
                    )

                # Get icon and category from engine attributes
                icon, category = _get_engine_icon_and_category(
                    engine_data, engine_class
                )

                # Check if engine requires an API key
                requires_api_key = engine_data.get("requires_api_key", False)

                # Build display name with icon, category, and API key status
                base_name = engine_data.get("display_name", engine_id)
                if requires_api_key:
                    label = f"{icon} {base_name} ({category}, API key)"
                else:
                    label = f"{icon} {base_name} ({category}, Free)"

                # Check if engine is a favorite
                is_favorite = engine_id in favorites

                # Classify the engine into a selector band. base_group ignores
                # favorite status (used by the frontend to move an engine back
                # to its category when un-starred); the effective band is the
                # Favorites overlay when starred. See engine_groups.py.
                base_group = classify_engine_group(
                    engine_id, category, requires_api_key
                )
                shown_group = effective_group(base_group, is_favorite)

                engines_dict[engine_id] = {
                    "display_name": base_name,
                    "description": engine_data.get("description", ""),
                    "strengths": engine_data.get("strengths", []),
                    "icon": icon,
                    "category": category,
                    "requires_api_key": requires_api_key,
                    "is_favorite": is_favorite,
                }

                engine_options.append(
                    {
                        "value": engine_id,
                        "label": label,
                        "icon": icon,
                        "category": category,
                        "requires_api_key": requires_api_key,
                        "is_favorite": is_favorite,
                        "group": shown_group,
                        "group_label": group_label(shown_group),
                        "group_order": group_order(shown_group),
                        "base_group": base_group,
                        "base_group_label": group_label(base_group),
                        "base_group_order": group_order(base_group),
                    }
                )

        # Sort engine_options by band order (favorites band first), then
        # alphabetically by label within each band.
        engine_options.sort(
            key=lambda x: (
                x.get("group_order", 999),
                x.get("label", "").lower(),
            )
        )

        # If no engines found, log the issue but return empty list
        if not engine_options:
            logger.warning("No search engines found in configuration")

        return jsonify(
            {
                "engines": engines_dict,
                "engine_options": engine_options,
                "favorites": favorites,
            }
        )

    except Exception:
        logger.exception("Error getting available search engines")
        return jsonify({"error": "Failed to retrieve search engines"}), 500


@settings_bp.route("/api/search-favorites", methods=["GET"])
@login_required
@with_user_session()
def api_get_search_favorites(
    db_session: Optional[Session] = None, settings_manager=None
):
    """Get the list of favorite search engines for the current user"""
    try:
        favorites = settings_manager.get_setting("search.favorites", [])
        if not isinstance(favorites, list):
            favorites = []
        return jsonify({"favorites": favorites})

    except Exception:
        logger.exception("Error getting search favorites")
        return jsonify({"error": "Failed to retrieve favorites"}), 500


@settings_bp.route("/api/search-favorites", methods=["PUT"])
@login_required
@require_json_body(error_message="No data provided")
@with_user_session()
def api_update_search_favorites(
    db_session: Optional[Session] = None, settings_manager=None
):
    """Update the list of favorite search engines for the current user"""
    try:
        data = request.get_json()
        favorites = data.get("favorites")
        if favorites is None:
            return jsonify({"error": "No favorites provided"}), 400

        if not isinstance(favorites, list):
            return jsonify({"error": "Favorites must be a list"}), 400

        if settings_manager.set_setting("search.favorites", favorites):
            invalidate_settings_caches(session["username"])
            return jsonify(
                {
                    "message": "Favorites updated successfully",
                    "favorites": favorites,
                }
            )
        return jsonify({"error": "Failed to update favorites"}), 500

    except Exception:
        logger.exception("Error updating search favorites")
        return jsonify({"error": "Failed to update favorites"}), 500


@settings_bp.route("/api/search-favorites/toggle", methods=["POST"])
@login_required
@require_json_body(error_message="No data provided")
@with_user_session()
def api_toggle_search_favorite(
    db_session: Optional[Session] = None, settings_manager=None
):
    """Toggle a search engine as favorite"""
    try:
        data = request.get_json()
        engine_id = data.get("engine_id")
        if not engine_id:
            return jsonify({"error": "No engine_id provided"}), 400

        # Get current favorites
        favorites = settings_manager.get_setting("search.favorites", [])
        if not isinstance(favorites, list):
            favorites = []
        else:
            # Make a copy to avoid modifying the original
            favorites = list(favorites)

        # Toggle the engine
        is_favorite = engine_id in favorites
        if is_favorite:
            favorites.remove(engine_id)
            is_favorite = False
        else:
            favorites.append(engine_id)
            is_favorite = True

        # Update the setting
        if settings_manager.set_setting("search.favorites", favorites):
            invalidate_settings_caches(session["username"])
            return jsonify(
                {
                    "message": "Favorite toggled successfully",
                    "engine_id": engine_id,
                    "is_favorite": is_favorite,
                    "favorites": favorites,
                }
            )
        return jsonify({"error": "Failed to toggle favorite"}), 500

    except Exception:
        logger.exception("Error toggling search favorite")
        return jsonify({"error": "Failed to toggle favorite"}), 500


# Legacy routes for backward compatibility - these will redirect to the new routes
@settings_bp.route("/main", methods=["GET"])
@login_required
def main_config_page():
    """Redirect to app settings page"""
    return redirect(url_for("settings.settings_page"))


@settings_bp.route("/collections", methods=["GET"])
@login_required
def collections_config_page():
    """Redirect to app settings page"""
    return redirect(url_for("settings.settings_page"))


@settings_bp.route("/api_keys", methods=["GET"])
@login_required
def api_keys_config_page():
    """Redirect to LLM settings page"""
    return redirect(url_for("settings.settings_page"))


@settings_bp.route("/search_engines", methods=["GET"])
@login_required
def search_engines_config_page():
    """Redirect to search settings page"""
    return redirect(url_for("settings.settings_page"))


@settings_bp.route("/llm", methods=["GET"])
@login_required
def llm_config_page():
    """Redirect to LLM settings page"""
    return redirect(url_for("settings.settings_page"))


@settings_bp.route("/open_file_location", methods=["POST"])
@login_required
def open_file_location():
    """Open the location of a configuration file.

    Security: This endpoint is disabled for server deployments.
    It only makes sense for desktop usage where the server and client are on the same machine.
    """
    return jsonify(
        {
            "status": "error",
            "message": "This feature is disabled. It is only available in desktop mode.",
        }
    ), 403


@settings_bp.context_processor
def inject_csrf_token():
    """Inject CSRF token into the template context for all settings routes."""
    return {"csrf_token": generate_csrf}


@settings_bp.route("/fix_corrupted_settings", methods=["POST"])
@login_required
@settings_limit
@with_user_session(include_settings_manager=False)
def fix_corrupted_settings(db_session: Optional[Session] = None):
    """Fix corrupted settings in the database"""
    try:
        # Track fixed and removed settings
        fixed_settings = []
        removed_duplicate_settings = []
        # First, find and remove duplicate settings with the same key
        # This happens because of errors in settings import/export
        from sqlalchemy import func as sql_func

        # Find keys with duplicates
        duplicate_keys = (
            db_session.query(Setting.key)
            .group_by(Setting.key)
            .having(sql_func.count(Setting.key) > 1)
            .all()
        )
        duplicate_keys = [key[0] for key in duplicate_keys]

        # For each duplicate key, keep the latest updated one and remove others
        for key in duplicate_keys:
            dupe_settings = (
                db_session.query(Setting)
                .filter(Setting.key == key)
                .order_by(Setting.updated_at.desc())
                .all()
            )

            # Keep the first one (most recently updated) and delete the rest
            for i, setting in enumerate(dupe_settings):
                if i > 0:  # Skip the first one (keep it)
                    db_session.delete(setting)
                    removed_duplicate_settings.append(key)

        # Check for settings with corrupted values
        all_settings = db_session.query(Setting).all()
        for setting in all_settings:
            # Check different types of corruption
            is_corrupted = False

            if (
                setting.value is None
                or (
                    isinstance(setting.value, str)
                    and setting.value
                    in [
                        "{",
                        "[",
                        "{}",
                        "[]",
                        "[object Object]",
                        "null",
                        "undefined",
                    ]
                )
                or (isinstance(setting.value, dict) and len(setting.value) == 0)
            ):
                is_corrupted = True

            # Skip if not corrupted
            if not is_corrupted:
                continue

            default_value: Any = None

            # Try to find a matching default setting based on key
            if setting.key.startswith("llm."):
                if setting.key == "llm.model":
                    default_value = ""
                elif setting.key == "llm.provider":
                    default_value = "ollama"
                elif setting.key == "llm.temperature":
                    default_value = 0.7
                elif setting.key == "llm.max_tokens":
                    default_value = 1024
            elif setting.key.startswith("search."):
                if setting.key == "search.tool":
                    default_value = DEFAULT_SEARCH_TOOL
                elif setting.key == "search.max_results":
                    default_value = 10
                elif setting.key == "search.region":
                    default_value = "us"
                elif setting.key == "search.questions_per_iteration":
                    default_value = 3
                elif setting.key == "search.searches_per_section":
                    default_value = 2
                elif setting.key == "search.skip_relevance_filter":
                    default_value = False
                elif setting.key == "search.safe_search":
                    default_value = True
                elif setting.key == "search.search_language":
                    default_value = "English"
            elif setting.key.startswith("report."):
                if setting.key == "report.searches_per_section":
                    default_value = 2
            elif setting.key.startswith("app."):
                if (
                    setting.key == "app.theme"
                    or setting.key == "app.default_theme"
                ):
                    default_value = "dark"
                elif setting.key == "app.enable_notifications" or (
                    setting.key == "app.enable_web"
                    or setting.key == "app.web_interface"
                ):
                    default_value = True
                elif setting.key == "app.host":
                    default_value = "0.0.0.0"
                elif setting.key == "app.port":
                    default_value = 5000
                elif setting.key == "app.debug":
                    default_value = True

            # Update the setting with the default value if found
            if default_value is not None:
                setting.value = default_value
                fixed_settings.append(setting.key)
            else:
                # If no default found but it's a corrupted JSON, set to empty object
                if setting.key.startswith("report."):
                    setting.value = {}
                    fixed_settings.append(setting.key)

        # Commit changes
        if fixed_settings or removed_duplicate_settings:
            db_session.commit()
            logger.info(
                f"Fixed {len(fixed_settings)} corrupted settings: {', '.join(fixed_settings)}"
            )
            if removed_duplicate_settings:
                logger.info(
                    f"Removed {len(removed_duplicate_settings)} duplicate settings"
                )
            invalidate_settings_caches(session["username"])

        # Return success
        return jsonify(
            {
                "status": "success",
                "message": f"Fixed {len(fixed_settings)} corrupted settings, removed {len(removed_duplicate_settings)} duplicates",
                "fixed_settings": fixed_settings,
                "removed_duplicates": removed_duplicate_settings,
            }
        )

    except Exception:
        logger.exception("Error fixing corrupted settings")
        db_session.rollback()
        return (
            jsonify(
                {
                    "status": "error",
                    "message": "An internal error occurred while fixing corrupted settings. Please try again later.",
                }
            ),
            500,
        )


@settings_bp.route("/api/warnings", methods=["GET"])
@login_required
def api_get_warnings():
    """Get current warnings based on settings"""
    try:
        warnings = calculate_warnings()
        return jsonify({"warnings": warnings})
    except Exception:
        logger.exception("Error getting warnings")
        return jsonify({"error": "Failed to retrieve warnings"}), 500


@settings_bp.route("/api/backup-status", methods=["GET"])
@login_required
def api_get_backup_status():
    """Get backup status for the current user."""
    try:
        from ...config.paths import get_user_backup_directory

        username = session.get("username")
        if not username:
            return jsonify({"error": "Not authenticated"}), 401

        from ...utilities.formatting import human_size
        from ...database.backup.backup_service import is_safe_glob_result

        backup_dir = get_user_backup_directory(username)

        # Sort by modification time (not filename) for robustness
        backup_list = []
        total_size = 0
        for b in backup_dir.glob("ldr_backup_*.db"):
            # Skip symlinks / entries resolving outside backup_dir so a
            # planted symlink can't have its (external) target's metadata
            # reported here — same glob-hardening applied in BackupService.
            if not is_safe_glob_result(b, backup_dir):
                continue
            try:
                stat = b.stat()
                total_size += stat.st_size
                backup_list.append(
                    {
                        "filename": b.name,
                        "size_bytes": stat.st_size,
                        "size_human": human_size(stat.st_size),
                        "created_at": datetime.fromtimestamp(
                            stat.st_mtime, tz=timezone.utc
                        ).isoformat(),
                        "_mtime": stat.st_mtime,
                    }
                )
            except FileNotFoundError:
                continue

        # Sort newest first by mtime, then remove internal field
        backup_list.sort(key=lambda x: x["_mtime"], reverse=True)
        for entry in backup_list:
            del entry["_mtime"]

        backup_enabled = _get_setting_from_session("backup.enabled", True)

        return jsonify(
            {
                "enabled": bool(backup_enabled),
                "count": len(backup_list),
                "backups": backup_list,
                "total_size_bytes": total_size,
                "total_size_human": human_size(total_size),
            }
        )

    except Exception:
        logger.exception("Error getting backup status")
        return jsonify({"error": "Failed to retrieve backup status"}), 500


@settings_bp.route("/api/ollama-status", methods=["GET"])
@login_required
def check_ollama_status():
    """Check if Ollama is running and available"""
    try:
        # Get Ollama URL from settings
        raw_base_url = _get_setting_from_session(
            "llm.ollama.url", DEFAULT_OLLAMA_URL
        )
        base_url = (
            normalize_url(raw_base_url) if raw_base_url else DEFAULT_OLLAMA_URL
        )

        response = safe_get(
            f"{base_url}/api/version",
            timeout=2,
            allow_localhost=True,
            allow_private_ips=True,
        )

        if response.status_code == 200:
            return jsonify(
                {
                    "running": True,
                    "version": response.json().get("version", "unknown"),
                }
            )
        return jsonify(
            {
                "running": False,
                "error": f"Ollama returned status code {response.status_code}",
            }
        )
    except requests.exceptions.RequestException:
        logger.exception("Ollama check failed")
        return jsonify(
            {"running": False, "error": "Failed to check search engine status"}
        )


@settings_bp.route("/api/rate-limiting/status", methods=["GET"])
@login_required
def api_get_rate_limiting_status():
    """Get current rate limiting status and statistics"""
    try:
        username = session["username"]

        # exploration_rate / learning_rate are the *configured* settings, not
        # the effective operating values: AdaptiveRateLimitTracker._apply_profile
        # scales them by the active profile (conservative ~0.5x/0.7x, aggressive
        # ~1.5x/1.3x, each capped). The profile is reported alongside so a
        # consumer can tell which transform the running tracker applies; we
        # deliberately don't duplicate that scaling math here.
        status = {
            # Default True to match the schema default (default_settings.json)
            # and the tracker's web-mode default; the prior endpoint reported
            # the tracker's effective enabled state, which was on by default.
            "enabled": _get_setting_from_session("rate_limiting.enabled", True),
            "profile": _get_setting_from_session(
                "rate_limiting.profile", "balanced"
            ),
            "exploration_rate": _get_setting_from_session(
                "rate_limiting.exploration_rate", 0.1
            ),
            "learning_rate": _get_setting_from_session(
                "rate_limiting.learning_rate", 0.45
            ),
            "memory_window": _get_setting_from_session(
                "rate_limiting.memory_window", 100
            ),
        }

        with get_user_db_session(username) as db_session:
            estimates = (
                db_session.query(RateLimitEstimate)
                .order_by(RateLimitEstimate.engine_type)
                .all()
            )

            engines = []
            for est in estimates:
                engines.append(
                    {
                        "engine_type": est.engine_type,
                        "base_wait_seconds": round(est.base_wait_seconds, 2),
                        "min_wait_seconds": round(est.min_wait_seconds, 2),
                        "max_wait_seconds": round(est.max_wait_seconds, 2),
                        "last_updated": est.last_updated,
                        "total_attempts": est.total_attempts,
                        "success_rate": round(est.success_rate * 100, 1),
                    }
                )

        return jsonify({"status": status, "engines": engines})

    except Exception:
        logger.exception("Error getting rate limiting status")
        return jsonify({"error": "An internal error occurred"}), 500


@settings_bp.route(
    "/api/rate-limiting/engines/<engine_type>/reset", methods=["POST"]
)
@login_required
def api_reset_engine_rate_limiting(engine_type):
    """Reset (forget) the learned rate-limit estimate for a specific engine.

    Deletes the engine's persisted ``RateLimitEstimate`` row from the user's
    database so the adaptive tracker re-learns it from scratch. The previous
    implementation called the per-request ``get_tracker()``, whose mutation
    path is gated on a research-session context that is absent in an analytics
    HTTP request — so it was a silent no-op that never cleared the persisted
    estimate the ``/status`` and ``/current`` endpoints display (#4721).
    """
    try:
        username = session["username"]
        with get_user_db_session(username) as db_session:
            db_session.query(RateLimitEstimate).filter_by(
                engine_type=engine_type
            ).delete(synchronize_session=False)
            db_session.commit()

        return jsonify(
            {"message": f"Rate limiting data reset for {engine_type}"}
        )

    except Exception:
        logger.exception(f"Error resetting rate limiting for {engine_type}")
        return jsonify({"error": "An internal error occurred"}), 500


@settings_bp.route("/api/rate-limiting/cleanup", methods=["POST"])
@login_required
def api_cleanup_rate_limiting():
    """Clean up old rate limiting data.

    Note: not using @require_json_body because the JSON body is optional
    here — the endpoint works with or without a payload (defaults to 30 days).
    """
    try:
        data = request.get_json() if request.is_json else None
        days = data.get("days", 30) if data is not None else 30

        try:
            days = int(days)
        except (TypeError, ValueError):
            return jsonify({"error": "'days' must be an integer"}), 400
        if days < 1 or days > 365:
            return jsonify({"error": "'days' must be between 1 and 365"}), 400

        # Delete persisted estimates not updated within the window. Mirrors the
        # read endpoints (#4721): operate on RateLimitEstimate rather than the
        # per-request get_tracker(), whose cleanup path is a no-op outside a
        # research-session context. last_updated is a unix timestamp (Float).
        username = session["username"]
        cutoff = time.time() - days * 86400
        with get_user_db_session(username) as db_session:
            db_session.query(RateLimitEstimate).filter(
                RateLimitEstimate.last_updated < cutoff
            ).delete(synchronize_session=False)
            db_session.commit()

        return jsonify(
            {"message": f"Cleaned up rate limiting data older than {days} days"}
        )

    except Exception:
        logger.exception("Error cleaning up rate limiting data")
        return jsonify({"error": "An internal error occurred"}), 500


@settings_bp.route("/api/bulk", methods=["GET"])
@login_required
def get_bulk_settings():
    """Get multiple settings at once for performance.

    The caller-supplied ``keys[]`` is unrestricted, so this endpoint is a
    candidate channel to exfiltrate password-typed settings — any
    authenticated client can request ``keys[]=llm.openai.api_key`` and
    would otherwise receive the plaintext value. Redact password fields
    by checking the leaf segment of the dotted key against
    ``DataSanitizer.DEFAULT_SENSITIVE_KEYS`` (e.g. ``api_key``,
    ``password``, ``access_token``, …). Same defense-in-depth predicate
    used in ``redact_settings_snapshot``; here we use the suffix-only
    check because the response shape is ``{value, exists}`` — no
    ``ui_element`` is fetched. ``exists`` is preserved so callers can
    still tell whether the key is configured.
    """
    try:
        from ...security.data_sanitizer import DataSanitizer

        # Get requested settings from query parameters
        requested = request.args.getlist("keys[]")
        if not requested:
            # Default to common settings if none specified
            requested = [
                "llm.provider",
                "llm.model",
                "search.tool",
                "search.iterations",
                "search.questions_per_iteration",
                "search.search_strategy",
                "benchmark.evaluation.provider",
                "benchmark.evaluation.model",
                "benchmark.evaluation.temperature",
                "benchmark.evaluation.endpoint_url",
            ]

        # Fetch all settings at once
        result = {}
        for key in requested:
            try:
                value = _get_setting_from_session(key)
                exists = value is not None
                # No ui_element metadata in the bulk path, so redact_value
                # falls back to the suffix-only arm of the shared predicate.
                value = DataSanitizer.redact_value(key, None, value)
                result[key] = {"value": value, "exists": exists}
            except Exception:
                logger.warning(f"Error getting setting {key}")
                result[key] = {
                    "value": None,
                    "exists": False,
                    "error": "Failed to retrieve setting",
                }

        return jsonify({"success": True, "settings": result})

    except Exception:
        logger.exception("Error getting bulk settings")
        return jsonify(
            {"success": False, "error": "An internal error occurred"}
        ), 500


@settings_bp.route("/api/data-location", methods=["GET"])
@login_required
def api_get_data_location():
    """Get information about data storage location and security"""
    try:
        # Get the data directory path
        data_dir = get_data_directory()
        # Get the encrypted databases path
        encrypted_db_path = get_encrypted_database_path()

        # Check if LDR_DATA_DIR environment variable is set
        from local_deep_research.settings.manager import SettingsManager

        settings_manager = SettingsManager()
        custom_data_dir = settings_manager.get_setting("bootstrap.data_dir")

        # Get platform-specific default location info
        platform_info = {
            "Windows": "C:\\Users\\Username\\AppData\\Local\\local-deep-research",
            "macOS": "~/Library/Application Support/local-deep-research",
            "Linux": "~/.local/share/local-deep-research",
        }

        # Current platform
        current_platform = platform.system()
        if current_platform == "Darwin":
            current_platform = "macOS"

        # Get SQLCipher settings from environment
        from ...database.sqlcipher_utils import get_sqlcipher_settings

        # Debug logging
        logger.info(f"db_manager type: {type(db_manager)}")
        logger.info(
            f"db_manager.has_encryption: {getattr(db_manager, 'has_encryption', 'ATTRIBUTE NOT FOUND')}"
        )

        cipher_settings = (
            get_sqlcipher_settings() if db_manager.has_encryption else {}
        )

        return jsonify(
            {
                "data_directory": str(data_dir),
                "database_path": str(encrypted_db_path),
                "encrypted_database_path": str(encrypted_db_path),
                "is_custom": custom_data_dir is not None,
                "custom_env_var": "LDR_DATA_DIR",
                "custom_env_value": custom_data_dir,
                "platform": current_platform,
                "platform_default": platform_info.get(
                    current_platform, str(data_dir)
                ),
                "platform_info": platform_info,
                "security_notice": {
                    "encrypted": db_manager.has_encryption,
                    "warning": "All data including API keys stored in the database are securely encrypted."
                    if db_manager.has_encryption
                    else "All data including API keys stored in the database are currently unencrypted. Please ensure appropriate file system permissions are set.",
                    "recommendation": "Your data is protected with database encryption."
                    if db_manager.has_encryption
                    else "Consider using environment variables for sensitive API keys instead of storing them in the database.",
                },
                "encryption_settings": cipher_settings,
            }
        )

    except Exception:
        logger.exception("Error getting data location information")
        return jsonify({"error": "Failed to retrieve data location"}), 500


@settings_bp.route("/api/notifications/test-url", methods=["POST"])
@login_required
def api_test_notification_url():
    """
    Test a notification service URL.

    This endpoint creates a temporary NotificationService instance to test
    the provided URL. No database session or password is required because:
    - The service URL is provided directly in the request body
    - Test notifications use a temporary Apprise instance
    - No user settings or database queries are performed

    Security note: Rate limiting is not applied here because users need to
    test URLs when configuring notifications. Abuse is mitigated by the
    @login_required decorator and the fact that users can only spam their
    own notification services.
    """
    try:
        from ...notifications.service import NotificationService
        from ...settings.env_registry import get_env_setting

        data = request.get_json()
        if not data or "service_url" not in data:
            return jsonify(
                {"success": False, "error": "service_url is required"}
            ), 400

        service_url = data["service_url"]

        # Create notification service instance and test the URL.
        # Gate by the env-only master switch so the test endpoint cannot
        # bypass the operator's risk-acceptance decision (see SECURITY.md
        # "Notification Webhook SSRF").
        notification_service = NotificationService(
            allow_private_ips=bool(
                get_env_setting("notifications.allow_private_ips", False)
            ),
            outbound_allowed=bool(
                get_env_setting("notifications.allow_outbound", False)
            ),
        )
        result = notification_service.test_service(service_url)

        # Only return expected fields to prevent information leakage
        safe_response = {
            "success": result.get("success", False),
            "message": result.get("message", ""),
            "error": result.get("error", ""),
        }
        return jsonify(safe_response)

    except Exception:
        logger.exception("Error testing notification URL")
        return jsonify(
            {
                "success": False,
                "error": "Failed to test notification service. Check logs for details.",
            }
        ), 500
