import io
import json
from datetime import datetime, UTC
from pathlib import Path

from flask import (
    Blueprint,
    jsonify,
    redirect,
    request,
    send_file,
    session,
    url_for,
)
from loguru import logger
from ...settings.logger import log_settings
from sqlalchemy import func
from sqlalchemy.orm import Session

# Security imports
from ...config.constants import DEFAULT_OLLAMA_URL
from ...exceptions import DuplicateResearchError, SystemAtCapacityError
from ...llm.providers.base import normalize_provider
from ...constants import HISTORY_LOGS_HARD_CAP, ResearchStatus
from ...security import (
    FileUploadValidator,
    UnsafeFilenameError,
    filter_research_metadata,
    sanitize_filename,
    strip_settings_snapshot,
)
from ...utilities.url_utils import is_safe_custom_llm_endpoint
from ...security.rate_limiter import (
    api_rate_limit,
    upload_rate_limit_ip,
    upload_rate_limit_user,
)
from ...security.decorators import require_json_body
from ...config.paths import get_config_directory

# Services imports
from ..services.pdf_extraction_service import get_pdf_extraction_service
from ..services.pdf_service import (
    MissingPDFDependencyError,
    get_weasyprint_install_instructions,
)

from ...database.models import (
    QueuedResearch,
    ResearchHistory,
    ResearchLog,
    UserActiveResearch,
)
from ...database.models.library import Document as Document
from ...database.session_context import get_g_db_session, get_user_db_session
from ..auth.decorators import login_required
from ..auth.password_utils import get_user_password, resolve_user_password
from ..models.database import calculate_duration
from ..services.research_service import (
    export_report_to_memory,
    run_research_process,
    start_research_process,
)
from ...security.rate_limiter import limiter
from ..utils.templates import render_template_with_defaults
from .globals import (
    append_research_log,
    get_active_research_ids,
    get_research_field,
    is_research_active,
    set_termination_flag,
)
from ...constants import DEFAULT_SEARCH_TOOL

# Create a Blueprint for the research application
research_bp = Blueprint("research", __name__)


# NOTE: Routes use session["username"] (not .get()) intentionally.
# @login_required guarantees the key exists; direct access fails fast
# if the decorator is ever removed.


# Add static route at the root level
@research_bp.route("/redirect-static/<path:path>")
def redirect_static(path):
    """Redirect old static URLs to new static URLs"""
    return redirect(url_for("static", filename=path))


@research_bp.route("/progress/<string:research_id>")
@login_required
def progress_page(research_id):
    """Render the research progress page"""
    return render_template_with_defaults("pages/progress.html")


@research_bp.route("/details/<string:research_id>")
@login_required
def research_details_page(research_id):
    """Render the research details page"""
    return render_template_with_defaults("pages/details.html")


@research_bp.route("/results/<string:research_id>")
@login_required
def results_page(research_id):
    """Render the research results page"""
    return render_template_with_defaults("pages/results.html")


@research_bp.route("/history")
@login_required
def history_page():
    """Render the history page"""
    return render_template_with_defaults("pages/history.html")


# Add missing settings routes
@research_bp.route("/settings", methods=["GET"])
@login_required
def settings_page():
    """Render the settings page"""
    return render_template_with_defaults("settings_dashboard.html")


def _extract_research_params(data, settings_manager):
    """Extract and resolve research parameters from request data and settings.

    Returns a dict with keys: model_provider, model, custom_endpoint,
    ollama_url, search_engine, max_results, time_period, iterations,
    questions_per_iteration, strategy.
    """
    model_provider = data.get("model_provider")
    if not model_provider:
        model_provider = settings_manager.get_setting("llm.provider", "ollama")
        logger.debug(
            f"No model_provider in request, using database setting: {model_provider}"
        )
    else:
        logger.debug(f"Using model_provider from request: {model_provider}")
    # Normalize provider to lowercase canonical form
    model_provider = normalize_provider(model_provider)

    model = data.get("model")
    if not model:
        model = settings_manager.get_setting("llm.model", None)
        logger.debug(f"No model in request, using database setting: {model}")
    else:
        logger.debug(f"Using model from request: {model}")

    custom_endpoint = data.get("custom_endpoint")
    if not custom_endpoint and model_provider == "openai_endpoint":
        custom_endpoint = settings_manager.get_setting(
            "llm.openai_endpoint.url", None
        )
        logger.debug(
            f"No custom_endpoint in request, using database setting: {custom_endpoint}"
        )

    ollama_url = data.get("ollama_url")
    if not ollama_url and model_provider == "ollama":
        ollama_url = settings_manager.get_setting(
            "llm.ollama.url", DEFAULT_OLLAMA_URL
        )
        logger.debug(
            f"No ollama_url in request, using database setting: {ollama_url}"
        )

    search_engine = data.get("search_engine") or data.get("search_tool")
    if not search_engine:
        search_engine = settings_manager.get_setting(
            "search.tool", DEFAULT_SEARCH_TOOL
        )

    max_results = data.get("max_results")
    time_period = data.get("time_period")

    iterations = data.get("iterations")
    if iterations is None:
        iterations = settings_manager.get_setting("search.iterations", 5)

    questions_per_iteration = data.get("questions_per_iteration")
    if questions_per_iteration is None:
        questions_per_iteration = settings_manager.get_setting(
            "search.questions_per_iteration", 5
        )

    strategy = data.get("strategy")
    if not strategy:
        strategy = settings_manager.get_setting(
            "search.search_strategy", "source-based"
        )

    # Egress policy per-research overrides. Mirror the
    # model/search_engine pattern: missing values fall back to saved
    # settings; supplied values override JUST FOR THIS RUN. They do
    # NOT persist to the user's settings DB.
    policy_egress_scope = data.get("policy_egress_scope")
    llm_require_local_endpoint = data.get("llm_require_local_endpoint")
    embeddings_require_local = data.get("embeddings_require_local")

    return {
        "model_provider": model_provider,
        "model": model,
        "custom_endpoint": custom_endpoint,
        "ollama_url": ollama_url,
        "search_engine": search_engine,
        "max_results": max_results,
        "time_period": time_period,
        "iterations": iterations,
        "questions_per_iteration": questions_per_iteration,
        "strategy": strategy,
        "policy_egress_scope": policy_egress_scope,
        "llm_require_local_endpoint": llm_require_local_endpoint,
        "embeddings_require_local": embeddings_require_local,
    }


def _precheck_engine_policy(settings_manager, params, search_engine, username):
    """Validate the requested search engine against the saved egress
    policy at the request boundary.

    Returns a Flask ``(response, status)`` tuple when the request should
    be rejected, or ``None`` to continue. Falls through (returns None)
    when there's no real dict snapshot to validate or the policy module
    errors — the factory PEP still enforces at engine-instantiation time.
    """
    try:
        from ...security.egress.policy import (
            PolicyDeniedError,
            context_from_snapshot,
            evaluate_engine,
            resolve_run_primary_engine,
        )

        policy_snapshot = settings_manager.get_settings_snapshot()
        # Only validate against a real dict snapshot. A test double or
        # an unavailable settings backend hands back something else;
        # skip rather than misfire.
        if not isinstance(policy_snapshot, dict):
            return None

        # Overlay per-research form overrides so the snapshot reflects
        # what the user picked for THIS run (per-research overrides, not
        # a global settings save).
        _apply_policy_overrides(policy_snapshot, params)

        # Resolve the primary the SAME way the worker does (single source of
        # truth) so the precheck and the background worker agree on accept vs.
        # refuse — including the fail-closed missing-primary case, which the
        # ValueError handler below maps to a 400. (Previously this substituted
        # searxng and accepted runs the worker then refused.)
        try:
            primary = resolve_run_primary_engine(policy_snapshot)
            policy_ctx = context_from_snapshot(
                policy_snapshot,
                primary,
                username=username,
            )
        except PolicyDeniedError as exc:
            return (
                jsonify(
                    {
                        "status": "error",
                        "message": (
                            f"Egress policy refused this run: "
                            f"{exc.decision.reason}"
                        ),
                    }
                ),
                400,
            )
        except ValueError as exc:
            # An invalid policy config is unrecoverable. Previously this
            # raised, fell through to the outer ``except Exception``
            # below, and silently returned None — so the run started
            # successfully at the precheck and only failed at a
            # downstream PEP. Surface it here as a 400 instead.
            logger.bind(policy_audit=True).warning(
                "POST /api/start_research policy precheck rejected",
                reason=str(exc),
            )
            return (
                jsonify(
                    {
                        "status": "error",
                        "message": (f"Egress policy refused this run: {exc}"),
                    }
                ),
                400,
            )

        decision = evaluate_engine(
            search_engine,
            policy_ctx,
            settings_snapshot=policy_snapshot,
        )
        if not decision.allowed:
            logger.bind(policy_audit=True).warning(
                "POST /api/start_research search_engine refused",
                engine=search_engine,
                reason=decision.reason,
            )
            # Local import: this module imports the egress policy lazily inside
            # this function to avoid a circular import at load time; keep the
            # guidance import with it.
            from ...security.egress.guidance import denial_guidance

            return (
                jsonify(
                    {
                        "status": "error",
                        # Clear, actionable message (what + why + how to allow),
                        # plus the raw reason code for support/logs.
                        "message": denial_guidance(
                            decision.reason,
                            target=f"Search engine '{search_engine}'",
                        ),
                        "reason": decision.reason,
                    }
                ),
                400,
            )
        return None
    except Exception:
        # Policy module unavailable / internal error → log and fall
        # through; the factory PEP will catch any actual violation.
        logger.exception("egress policy pre-check skipped")
        return None


def _apply_policy_overrides(settings_snapshot, params):
    """Overlay form-supplied egress policy values onto the snapshot.

    Per-research overrides: the user picked these values for this
    specific run and they do NOT persist to the settings DB. Mirrors
    how model / search_engine overrides work today.
    """
    if not isinstance(settings_snapshot, dict):
        return
    if params.get("policy_egress_scope") is not None:
        settings_snapshot["policy.egress_scope"] = params["policy_egress_scope"]
    if params.get("llm_require_local_endpoint") is not None:
        settings_snapshot["llm.require_local_endpoint"] = bool(
            params["llm_require_local_endpoint"]
        )
    if params.get("embeddings_require_local") is not None:
        settings_snapshot["embeddings.require_local"] = bool(
            params["embeddings_require_local"]
        )


def _research_not_found(research_id, message="Research not found"):
    """Return a consistent 404 JSON for a missing research.

    Emits ``status``, ``error`` and ``message`` so the body is a strict
    superset of both historical 404 shapes — every frontend reader and
    existing test keeps working without changes:
      - Shape A readers read ``data.error``
      - Shape B readers read ``data.status`` (``== "error"``) and/or
        ``data.message``

    ``research_id`` is used only for a debug log identifying which research
    was missing; it is intentionally never echoed in the response body.
    """
    logger.debug(f"404 for research {research_id}: {message}")
    return (
        jsonify({"status": "error", "error": message, "message": message}),
        404,
    )


def _queue_research(
    db_session: Session,
    username,
    research_id,
    query,
    mode,
    research_settings,
    params,
    session_id,
    reason="",
    research=None,
):
    """Add research to queue and notify processor. Returns a JSON response.

    Args:
        reason: Optional prefix explaining why the research was queued
                (e.g. "due to concurrent limit").
        research: Optional ResearchHistory object whose status should be set
                  to QUEUED atomically with the queue record insertion.
    """
    max_position = (
        db_session.query(func.max(QueuedResearch.position))
        .filter_by(username=username)
        .scalar()
        or 0
    )

    queued_record = QueuedResearch(
        username=username,
        research_id=research_id,
        query=query,
        mode=mode,
        settings_snapshot=research_settings,
        position=max_position + 1,
    )
    db_session.add(queued_record)
    if research is not None:
        research.status = ResearchStatus.QUEUED  # type: ignore[assignment]
    db_session.commit()
    logger.info(
        f"Queued research {research_id} at position {max_position + 1} for user {username}"
    )

    from ..queue.processor_v2 import queue_processor

    queue_processor.notify_research_queued(
        username,
        research_id,
        session_id=session_id,
        query=query,
        mode=mode,
        settings_snapshot=research_settings,
        model_provider=params["model_provider"],
        model=params["model"],
        custom_endpoint=params["custom_endpoint"],
        search_engine=params["search_engine"],
        max_results=params["max_results"],
        time_period=params["time_period"],
        iterations=params["iterations"],
        questions_per_iteration=params["questions_per_iteration"],
        strategy=params["strategy"],
    )

    position = max_position + 1
    reason_text = f" {reason}" if reason else ""
    message = f"Your research has been queued{reason_text}. Position in queue: {position}"
    return jsonify(
        {
            "status": ResearchStatus.QUEUED,
            "research_id": research_id,
            "queue_position": position,
            "message": message,
        }
    )


@research_bp.route("/api/start_research", methods=["POST"])
@login_required
@api_rate_limit
@require_json_body(error_format="status")
def start_research():
    data = request.json
    # Debug logging to trace model parameter
    logger.debug(f"Request data keys: {list(data.keys())}")

    # Check if this is a news search
    metadata = data.get("metadata", {})
    if metadata.get("is_news_search"):
        logger.info(
            f"News search request received: triggered_by={metadata.get('triggered_by', 'unknown')}"
        )

    query = data.get("query")
    mode = data.get("mode", "quick")

    # Replace date placeholders if they exist
    if query and "YYYY-MM-DD" in query:
        # Use local system time
        current_date = datetime.now(UTC).strftime("%Y-%m-%d")

        original_query = query
        query = query.replace("YYYY-MM-DD", current_date)
        logger.info(
            f"Replaced date placeholder in query: {original_query[:100]}... -> {query[:100]}..."
        )
        logger.info(f"Using date: {current_date}")

        # Update metadata to track the replacement
        if not metadata:
            metadata = {}
        metadata["original_query"] = original_query
        metadata["processed_query"] = query
        metadata["date_replaced"] = current_date
        data["metadata"] = metadata

    # Get parameters from request or use database settings
    from ...settings.manager import SettingsManager

    username = session["username"]

    with get_user_db_session(username) as db_session:
        settings_manager = SettingsManager(db_session=db_session)
        params = _extract_research_params(data, settings_manager)

        model_provider = params["model_provider"]
        model = params["model"]
        custom_endpoint = params["custom_endpoint"]
        search_engine = params["search_engine"]
        max_results = params["max_results"]
        time_period = params["time_period"]
        iterations = params["iterations"]
        questions_per_iteration = params["questions_per_iteration"]
        strategy = params["strategy"]

        # Egress policy: server-side check on the requested search
        # engine BEFORE we enqueue the research. Defends against an API
        # client posting an engine name that violates the saved policy
        # (e.g. ``{"search_engine": "pubmed"}`` under STRICT+primary=arxiv).
        # The factory PEP catches the same case at instantiation time,
        # but rejecting at the request boundary lets us return a clean
        # 4xx instead of an opaque background failure.
        policy_error = _precheck_engine_policy(
            settings_manager, params, search_engine, username
        )
        if policy_error is not None:
            return policy_error

    # Debug logging for model parameter specifically
    logger.debug(
        f"Extracted model value: '{model}' (type: {type(model).__name__})"
    )

    # Log the selections for troubleshooting
    logger.info(
        f"Starting research with provider: {model_provider}, model: {model}, search engine: {search_engine}"
    )
    logger.info(
        f"Additional parameters: max_results={max_results}, time_period={time_period}, iterations={iterations}, questions={questions_per_iteration}, strategy={strategy}"
    )

    if not query:
        return jsonify({"status": "error", "message": "Query is required"}), 400

    # Validate required parameters based on provider
    if model_provider == "openai_endpoint" and not custom_endpoint:
        return (
            jsonify(
                {
                    "status": "error",
                    "message": "Custom endpoint URL is required for OpenAI endpoint provider",
                }
            ),
            400,
        )

    # SSRF pre-flight on the user-supplied LLM endpoint: reject metadata /
    # link-local targets at the request boundary, before any research thread
    # is spawned. This is fail-fast defense-in-depth — the OpenAI-compatible
    # provider's assert_base_url_safe re-validates the same URL before the
    # client is built. Private IPs / localhost are permitted so local LLMs
    # (vLLM, Ollama, LM Studio) work, including scheme-less endpoints
    # (the helper normalizes exactly as the provider does).
    if not is_safe_custom_llm_endpoint(custom_endpoint):
        return (
            jsonify(
                {"status": "error", "message": "Invalid custom endpoint URL"}
            ),
            400,
        )

    if not model:
        logger.error(
            f"No model specified or configured. Provider: {model_provider}"
        )
        return jsonify(
            {
                "status": "error",
                "message": "Model is required. Please configure a model in the settings.",
            }
        ), 400

    # Check if the user has too many active researches
    username = session["username"]

    # Get max concurrent researches from settings
    from ...settings import SettingsManager

    with get_user_db_session() as db_session:
        settings_manager = SettingsManager(db_session)
        max_concurrent_researches = settings_manager.get_setting(
            "app.max_concurrent_researches", 3
        )

    # Use existing session from g to check active researches
    try:
        db_session = get_g_db_session()
        if db_session:
            # First, clean up stale entries where the research thread has died
            # (e.g. crashed with an unhandled exception before cleanup ran).
            # Without this, dead researches permanently block the queue.
            from ..routes.globals import reclaim_stale_user_active_research

            # No grace cutoff here — research_routes.start_research has
            # always reclaimed all dead-thread rows immediately. The chat
            # send-message path uses a 30s grace via grace_cutoff_dt.
            if reclaim_stale_user_active_research(
                db_session, username, logger=logger
            ):
                db_session.commit()

            # Now count truly active researches
            active_count = (
                db_session.query(UserActiveResearch)
                .filter_by(username=username, status=ResearchStatus.IN_PROGRESS)
                .count()
            )

            # Debug logging
            logger.info(
                f"Active research count for {username}: {active_count}/{max_concurrent_researches}"
            )

            should_queue = active_count >= max_concurrent_researches
            logger.info(f"Should queue new research: {should_queue}")
        else:
            logger.warning(
                "No database session available to check active researches"
            )
            should_queue = False
    except Exception:
        logger.exception("Failed to check active researches")
        # Default to not queueing if we can't check
        should_queue = False

    # For non-queued research, verify password is available BEFORE creating DB records
    # (queued research gets password later via queue processor)
    user_password = None
    if not should_queue:
        user_password, session_expired = resolve_user_password(username)
        if session_expired:
            # Use status/message keys to match the research API convention
            # (the research frontend checks data.status and data.message)
            return jsonify(
                {
                    "status": "error",
                    "message": "Your session has expired. Please log out and log back in to start research.",
                }
            ), 401

    # Create a record in the database with explicit UTC timestamp
    import uuid
    import threading

    created_at = datetime.now(UTC).isoformat()
    research_id = str(uuid.uuid4())

    # Create organized research metadata with settings snapshot
    research_settings = {
        # Direct submission parameters
        "submission": {
            "model_provider": model_provider,
            "model": model,
            "custom_endpoint": custom_endpoint,
            "search_engine": search_engine,
            "max_results": max_results,
            "time_period": time_period,
            "iterations": iterations,
            "questions_per_iteration": questions_per_iteration,
            "strategy": strategy,
        },
        # System information
        "system": {
            "timestamp": created_at,
            "user": username,
            "version": "1.0",  # Track metadata version for future migrations
            "server_url": request.host_url,  # Add server URL for link generation
        },
    }

    # Add any additional metadata from request
    additional_metadata = data.get("metadata", {})
    if additional_metadata:
        research_settings.update(additional_metadata)
    # Get complete settings snapshot for this research
    try:
        from local_deep_research.settings import SettingsManager

        # Get or lazily create a session for settings snapshot
        db_session_for_settings = get_g_db_session()
        if db_session_for_settings:
            # Create SettingsManager with the existing session
            username = session["username"]
            # Ensure any pending changes are committed
            try:
                db_session_for_settings.commit()
            except Exception:
                db_session_for_settings.rollback()
            settings_manager = SettingsManager(
                db_session_for_settings, owns_session=False
            )
            # Get all current settings as a snapshot (bypass cache to ensure fresh data)
            all_settings = settings_manager.get_all_settings(bypass_cache=True)
            # Apply per-research egress policy overrides (form-supplied
            # values override saved settings JUST FOR THIS RUN; they do
            # not persist to the user's settings DB).
            _apply_policy_overrides(all_settings, params)

            # Add settings snapshot to metadata
            research_settings["settings_snapshot"] = all_settings
            logger.info(
                f"Captured {len(all_settings)} settings for research {research_id}"
            )
        else:
            # If no session in g, create a new one temporarily to get settings
            logger.warning(
                "No database session in g, creating temporary session for settings snapshot"
            )
            from ...database.thread_local_session import get_metrics_session

            password = get_user_password(username)

            if password:
                temp_session = get_metrics_session(username, password)
                if temp_session:
                    username = session["username"]
                    settings_manager = SettingsManager(
                        temp_session, owns_session=False
                    )
                    all_settings = settings_manager.get_all_settings(
                        bypass_cache=True
                    )
                    _apply_policy_overrides(all_settings, params)
                    research_settings["settings_snapshot"] = all_settings
                    logger.info(
                        f"Captured {len(all_settings)} settings using temporary session for research {research_id}"
                    )
                else:
                    logger.error(
                        "Failed to create temporary session for settings snapshot"
                    )
                    return jsonify(
                        {
                            "status": "error",
                            "message": "Cannot create research without settings snapshot.",
                        }
                    ), 500
            else:
                logger.error(
                    "No password available to create session for settings snapshot"
                )
                return jsonify(
                    {
                        "status": "error",
                        "message": "Cannot create research without settings snapshot.",
                    }
                ), 500
    except Exception:
        logger.exception("Failed to capture settings snapshot")
        # Cannot continue without settings snapshot for thread-based research
        return jsonify(
            {
                "status": "error",
                "message": "Failed to capture settings for research. Please try again.",
            }
        ), 500

    # Use existing session from g
    username = session["username"]

    try:
        # Get or lazily create a session
        db_session = get_g_db_session()
        if db_session:
            # Determine initial status based on whether we need to queue
            initial_status = (
                ResearchStatus.QUEUED
                if should_queue
                else ResearchStatus.IN_PROGRESS
            )

            research = ResearchHistory(
                id=research_id,  # Set UUID as primary key
                query=query,
                mode=mode,
                status=initial_status,
                created_at=created_at,
                progress_log=[{"time": created_at, "progress": 0}],
                research_meta=research_settings,
            )
            db_session.add(research)
            db_session.commit()
            logger.info(
                f"Created research entry with UUID: {research_id}, status: {initial_status}"
            )

            if should_queue:
                session_id = session.get("session_id")
                return _queue_research(
                    db_session,
                    username,
                    research_id,
                    query,
                    mode,
                    research_settings,
                    params,
                    session_id,
                )
            # Start immediately
            # Create active research tracking record
            import threading

            active_record = UserActiveResearch(
                username=username,
                research_id=research_id,
                status=ResearchStatus.IN_PROGRESS,
                thread_id=str(threading.current_thread().ident),
                settings_snapshot=research_settings,
            )
            db_session.add(active_record)
            db_session.commit()
            logger.info(f"Created active research record for user {username}")

            # Double-check the count after committing to handle race conditions
            # Use the existing session for the recheck
            try:
                # Use the same session we already have
                recheck_session = db_session
                final_count = (
                    recheck_session.query(UserActiveResearch)
                    .filter_by(
                        username=username, status=ResearchStatus.IN_PROGRESS
                    )
                    .count()
                )
                logger.info(
                    f"Final active count after commit: {final_count}/{max_concurrent_researches}"
                )

                if final_count > max_concurrent_researches:
                    # We exceeded the limit due to a race condition
                    # Remove this record and queue instead
                    logger.warning(
                        f"Race condition detected: {final_count} > {max_concurrent_researches}, moving to queue"
                    )
                    db_session.delete(active_record)
                    db_session.commit()

                    session_id = session.get("session_id")
                    return _queue_research(
                        db_session,
                        username,
                        research_id,
                        query,
                        mode,
                        research_settings,
                        params,
                        session_id,
                        reason="due to concurrent limit",
                        research=research,
                    )
            except Exception:
                logger.warning("Could not recheck active count")

    except Exception:
        logger.exception("Failed to create research entry")
        return jsonify(
            {"status": "error", "message": "Failed to create research entry"}
        ), 500

    # Only start the research if not queued
    if not should_queue:
        # Save the research strategy to the database before starting the thread
        try:
            from ..services.research_service import save_research_strategy

            save_research_strategy(research_id, strategy, username=username)
        except Exception:
            logger.warning("Could not save research strategy")

        # Debug logging for settings snapshot
        snapshot_data = research_settings.get("settings_snapshot", {})
        log_settings(snapshot_data, "Settings snapshot being passed to thread")
        if "search.tool" in snapshot_data:
            logger.debug(
                f"search.tool in snapshot: {snapshot_data['search.tool']}"
            )
        else:
            logger.debug("search.tool NOT in snapshot")

        # Start the research process with the selected parameters.
        # If the spawn raises, the UserActiveResearch + IN_PROGRESS
        # ResearchHistory rows persisted above would otherwise be
        # permanently orphaned (no thread, no cleanup path). Catch any
        # exception, mark the research FAILED, delete the active row,
        # and return 500 — same contract as the queue processor's
        # terminal-failure branch introduced in #3481.
        try:
            research_thread = start_research_process(
                research_id,
                query,
                mode,
                run_research_process,
                username=username,  # Pass username to the thread
                user_password=user_password,  # Pass password for database access
                model_provider=model_provider,
                model=model,
                custom_endpoint=custom_endpoint,
                search_engine=search_engine,
                max_results=max_results,
                time_period=time_period,
                iterations=iterations,
                questions_per_iteration=questions_per_iteration,
                strategy=strategy,
                settings_snapshot=snapshot_data,  # Pass complete settings
            )
        except DuplicateResearchError:
            # A live thread already owns this research_id. Do NOT delete
            # the UserActiveResearch row or mark ResearchHistory FAILED —
            # that state belongs to the live thread, and mutating it
            # would terminate a running research from the user's
            # perspective while it keeps executing. Same contract as the
            # queue processor's dedicated dup branch (#3506).
            logger.warning(
                f"Duplicate live thread detected for {research_id} "
                "on direct submission; leaving state intact"
            )
            return jsonify(
                {
                    "status": "error",
                    "message": "Research is already running.",
                }
            ), 409
        except SystemAtCapacityError:
            # System at concurrent-research capacity. Roll back the rows
            # committed above (UserActiveResearch + IN_PROGRESS history)
            # and return 429 so the client can retry shortly.
            logger.warning(
                f"SystemAtCapacityError on direct submission for "
                f"{research_id}; rolling back orphan rows"
            )
            try:
                with get_user_db_session(username) as cleanup_session:
                    stale_active = (
                        cleanup_session.query(UserActiveResearch)
                        .filter_by(username=username, research_id=research_id)
                        .first()
                    )
                    if stale_active:
                        cleanup_session.delete(stale_active)
                    cleanup_session.query(ResearchHistory).filter_by(
                        id=research_id
                    ).delete()
                    cleanup_session.commit()
            except Exception:
                logger.exception(
                    "Cleanup after SystemAtCapacityError raised; "
                    "leaving orphan rows for the reconciler"
                )
            return jsonify(
                {
                    "status": "error",
                    "message": "Server is at research capacity. Please retry shortly.",
                }
            ), 429
        except Exception:
            logger.exception(
                f"Failed to spawn research thread for {research_id}"
            )
            try:
                with get_user_db_session(username) as cleanup_session:
                    stale_active = (
                        cleanup_session.query(UserActiveResearch)
                        .filter_by(username=username, research_id=research_id)
                        .first()
                    )
                    if stale_active:
                        cleanup_session.delete(stale_active)
                    research_row = (
                        cleanup_session.query(ResearchHistory)
                        .filter_by(id=research_id)
                        .first()
                    )
                    if research_row:
                        research_row.status = ResearchStatus.FAILED
                    cleanup_session.commit()
            except Exception:
                logger.exception(
                    "Cleanup after spawn failure raised; leaving "
                    "orphan rows for the reconciler to handle"
                )
            return jsonify(
                {
                    "status": "error",
                    "message": "Failed to start research. Please try again.",
                }
            ), 500

        # Update the active research record with the actual thread ID.
        try:
            with get_user_db_session(username) as thread_session:
                active_record = (
                    thread_session.query(UserActiveResearch)
                    .filter_by(username=username, research_id=research_id)
                    .first()
                )
                if active_record:
                    active_record.thread_id = str(research_thread.ident)
                    thread_session.commit()
        except Exception:
            logger.warning("Could not update thread ID")

    return jsonify({"status": "success", "research_id": research_id})


@research_bp.route("/api/terminate/<string:research_id>", methods=["POST"])
@login_required
def terminate_research(research_id):
    """Terminate an in-progress research process"""
    username = session["username"]

    # Check if the research exists and is in progress
    try:
        with get_user_db_session(username) as db_session:
            research = (
                db_session.query(ResearchHistory)
                .filter_by(id=research_id)
                .first()
            )

            if not research:
                return _research_not_found(research_id)

            status = research.status

            # If it's already in a terminal state, return success
            if status in (
                ResearchStatus.COMPLETED,
                ResearchStatus.SUSPENDED,
                ResearchStatus.FAILED,
                ResearchStatus.ERROR,
            ):
                return jsonify(
                    {
                        "status": "success",
                        "message": f"Research already {status}",
                    }
                )

            # Check if it's in the active_research dict
            if not is_research_active(research_id):
                # The worker may not be registered in _active_research yet: a
                # just-submitted research commits its IN_PROGRESS row before the
                # worker thread registers itself (spawn-grace window). Set the
                # termination flag anyway so a worker that starts right after
                # this still sees it and aborts at its first checkpoint —
                # otherwise the user's Stop is silently ignored and the research
                # runs to completion (overwriting this SUSPENDED status). The
                # flag is harmless if no worker ever starts.
                set_termination_flag(research_id)
                research.status = ResearchStatus.SUSPENDED
                db_session.commit()
                return jsonify(
                    {"status": "success", "message": "Research terminated"}
                )

            # Set the termination flag
            set_termination_flag(research_id)

            # Log the termination request - using UTC timestamp
            timestamp = datetime.now(UTC).isoformat()
            termination_message = "Research termination requested by user"
            current_progress = get_research_field(research_id, "progress", 0)

            # Create log entry
            log_entry = {
                "time": timestamp,
                "message": termination_message,
                "progress": current_progress,
                "metadata": {"phase": "termination"},
            }

            # Add to in-memory log
            append_research_log(research_id, log_entry)

            # Add to database log
            logger.log("MILESTONE", f"Research ended: {termination_message}")

            # Update the log in the database
            if research.progress_log:
                try:
                    if isinstance(research.progress_log, str):
                        current_log = json.loads(research.progress_log)
                    else:
                        current_log = research.progress_log
                except Exception:
                    current_log = []
            else:
                current_log = []

            current_log.append(log_entry)
            research.progress_log = current_log
            research.status = ResearchStatus.SUSPENDED
            db_session.commit()

            # Emit a socket event for the termination request
            try:
                event_data = {
                    "status": ResearchStatus.SUSPENDED,
                    "message": "Research was suspended by user request",
                }

                from ..services.socket_service import SocketIOService

                SocketIOService().emit_to_subscribers(
                    "progress", research_id, event_data
                )

            except Exception:
                logger.exception("Socket emit error (non-critical)")

            return jsonify(
                {
                    "status": "success",
                    "message": "Research termination requested",
                }
            )
    except Exception:
        logger.exception("Error terminating research")
        return jsonify(
            {"status": "error", "message": "Failed to terminate research"}
        ), 500


@research_bp.route("/api/delete/<string:research_id>", methods=["DELETE"])
@login_required
def delete_research(research_id):
    """Delete a research record"""
    username = session["username"]

    try:
        with get_user_db_session(username) as db_session:
            research = (
                db_session.query(ResearchHistory)
                .filter_by(id=research_id)
                .first()
            )

            if not research:
                return _research_not_found(research_id)

            status = research.status
            report_path = research.report_path

            # Don't allow deleting research in progress
            if status == ResearchStatus.IN_PROGRESS and is_research_active(
                research_id
            ):
                return (
                    jsonify(
                        {
                            "status": "error",
                            "message": "Cannot delete research that is in progress",
                        }
                    ),
                    400,
                )

            # Delete report file if it exists
            if report_path and Path(report_path).exists():
                try:
                    Path(report_path).unlink()
                except Exception:
                    logger.exception("Error removing report file")

            # Delete the database record
            db_session.delete(research)
            db_session.commit()

            return jsonify({"status": "success"})
    except Exception:
        logger.exception("Error deleting research")
        return jsonify(
            {"status": "error", "message": "Failed to delete research"}
        ), 500


@research_bp.route("/api/clear_history", methods=["POST"])
@login_required
def clear_history():
    """Clear all research history"""
    username = session["username"]

    try:
        with get_user_db_session(username) as db_session:
            # Get all research records first to clean up files. Select
            # only id + report_path (the columns the cleanup loop uses)
            # so clearing history doesn't load every report body into
            # memory (#4560).
            research_records = db_session.query(
                ResearchHistory.id, ResearchHistory.report_path
            ).all()

            # Get IDs of currently active research (snapshot)
            active_ids = get_active_research_ids()

            # Clean up report files
            for research in research_records:
                # Skip active research
                if research.id in active_ids:
                    continue

                # Delete report file if it exists
                if research.report_path and Path(research.report_path).exists():
                    try:
                        Path(research.report_path).unlink()
                    except Exception:
                        logger.exception("Error removing report file")

            # Query.delete() bypasses ORM cascade; child rows clean up
            # via DDL-level ondelete="CASCADE" only because PRAGMA
            # foreign_keys = ON is set on every connection.
            if active_ids:
                db_session.query(ResearchHistory).filter(
                    ~ResearchHistory.id.in_(active_ids)
                ).delete(synchronize_session=False)
            else:
                db_session.query(ResearchHistory).delete(
                    synchronize_session=False
                )

            db_session.commit()

            return jsonify({"status": "success"})
    except Exception:
        logger.exception("Error clearing history")
        return jsonify(
            {"status": "error", "message": "Failed to process request"}
        ), 500


@research_bp.route("/open_file_location", methods=["POST"])
@login_required
def open_file_location():
    """Open a file location in the system file explorer.

    Security: This endpoint is disabled for server deployments.
    It only makes sense for desktop usage where the server and client are on the same machine.
    """
    return jsonify(
        {
            "status": "error",
            "message": "This feature is disabled. It is only available in desktop mode.",
        }
    ), 403


@research_bp.route("/api/save_raw_config", methods=["POST"])
@login_required
@require_json_body(error_format="success")
def save_raw_config():
    """Save raw configuration"""
    data = request.json
    raw_config = data.get("raw_config")

    if not raw_config:
        return (
            jsonify(
                {"success": False, "error": "Raw configuration is required"}
            ),
            400,
        )

    # Security: Parse and validate the TOML to block dangerous keys
    try:
        import tomllib
    except ImportError:
        import tomli as tomllib  # type: ignore[no-redef]

    try:
        parsed_config = tomllib.loads(raw_config)
    except Exception:
        logger.warning("Invalid TOML configuration")
        # Don't expose internal exception details to users (CWE-209)
        return jsonify(
            {
                "success": False,
                "error": "Invalid TOML syntax. Please check your configuration format.",
            }
        ), 400

    # Security: Check for dangerous keys that could enable code execution
    # These patterns match keys used for dynamic module imports
    BLOCKED_KEY_PATTERNS = ["module_path", "class_name", "module", "class"]

    def find_blocked_keys(obj, path=""):
        """Recursively find any blocked keys in the config."""
        blocked = []
        if isinstance(obj, dict):
            for key, value in obj.items():
                current_path = f"{path}.{key}" if path else key
                key_lower = key.lower()
                for pattern in BLOCKED_KEY_PATTERNS:
                    if pattern in key_lower:
                        blocked.append(current_path)
                        break
                # Recurse into nested dicts
                blocked.extend(find_blocked_keys(value, current_path))
        elif isinstance(obj, list):
            for i, item in enumerate(obj):
                blocked.extend(find_blocked_keys(item, f"{path}[{i}]"))
        return blocked

    blocked_keys = find_blocked_keys(parsed_config)
    if blocked_keys:
        logger.warning(
            f"Security: Blocked attempt to write config with dangerous keys: {blocked_keys}"
        )
        return jsonify(
            {
                "success": False,
                "error": "Configuration contains protected keys that cannot be modified",
                "blocked_keys": blocked_keys,
            }
        ), 403

    try:
        from ...security.file_write_verifier import write_file_verified

        # Get the config file path (uses centralized path config, respects LDR_DATA_DIR)
        config_dir = get_config_directory()
        config_path = config_dir / "config.toml"

        # Write the configuration to file
        write_file_verified(
            config_path,
            raw_config,
            "system.allow_config_write",
            context="system configuration file",
        )

        return jsonify({"success": True})
    except Exception:
        logger.exception("Error saving configuration file")
        return jsonify(
            {"success": False, "error": "Failed to process request"}
        ), 500


@research_bp.route("/api/history", methods=["GET"])
@login_required
def get_history():
    """Get research history"""
    username = session["username"]

    # Bound the result set. Without a limit this endpoint loaded every
    # research row (and its research_meta JSON, which can hold a settings
    # snapshot) into memory at once (#4560). Mirrors the clamp used by the
    # symmetric /history/api endpoint.
    limit = request.args.get("limit", 200, type=int)
    limit = max(1, min(limit, 500))
    offset = max(0, request.args.get("offset", 0, type=int))

    try:
        with get_user_db_session(username) as db_session:
            # Query research history ordered by created_at. Project
            # only the metadata columns the loop below consumes — never
            # the large ``report_content`` Text body — so this listing
            # doesn't pull every report into memory (#4560). This mirrors
            # the projection used by the symmetric /history/api endpoint.
            research_records = (
                db_session.query(
                    ResearchHistory.id,
                    ResearchHistory.title,
                    ResearchHistory.query,
                    ResearchHistory.mode,
                    ResearchHistory.status,
                    ResearchHistory.created_at,
                    ResearchHistory.completed_at,
                    ResearchHistory.research_meta,
                    ResearchHistory.chat_session_id,
                )
                .order_by(ResearchHistory.created_at.desc())
                .limit(limit)
                .offset(offset)
                .all()
            )

            # Pre-compute Document counts in a single GROUP BY query
            # to avoid an N+1 SELECT-COUNT-per-row inside the loop. The
            # symmetric /history/api endpoint in history_routes.py
            # already uses an outerjoin + group_by — this brings
            # /api/history to parity for users with deep history.
            research_ids = [r.id for r in research_records]
            if research_ids:
                doc_count_rows = (
                    db_session.query(
                        Document.research_id, func.count(Document.id)
                    )
                    .filter(Document.research_id.in_(research_ids))
                    .group_by(Document.research_id)
                    .all()
                )
                doc_counts = dict(doc_count_rows)
            else:
                doc_counts = {}

            # Build history items while session is active to avoid
            # DetachedInstanceError on ORM attribute access
            history_items = []
            for research in research_records:
                # Calculate duration if completed
                duration_seconds = None
                if research.completed_at and research.created_at:
                    try:
                        duration_seconds = calculate_duration(
                            research.created_at, research.completed_at
                        )
                    except Exception:
                        logger.exception("Error calculating duration")

                # Look up the pre-computed document count.
                doc_count = doc_counts.get(research.id, 0)

                # Create a history item
                item = {
                    "id": research.id,
                    "query": research.query,
                    "mode": research.mode,
                    "status": research.status,
                    "created_at": research.created_at,
                    "completed_at": research.completed_at,
                    "duration_seconds": duration_seconds,
                    "metadata": filter_research_metadata(
                        research.research_meta
                    ),
                    "document_count": doc_count,
                }
                if research.chat_session_id is not None:
                    item["metadata"]["chat_session_id"] = (
                        research.chat_session_id
                    )

                # Add title if it exists
                if hasattr(research, "title") and research.title is not None:
                    item["title"] = research.title

                history_items.append(item)

        return jsonify({"status": "success", "items": history_items})
    except Exception:
        logger.exception("Error getting history")
        return jsonify(
            {"status": "error", "message": "Failed to process request"}
        ), 500


@research_bp.route("/api/research/<string:research_id>")
@login_required
def get_research_details(research_id):
    """Get full details of a research using ORM"""
    username = session["username"]

    try:
        with get_user_db_session(username) as db_session:
            research = (
                db_session.query(ResearchHistory)
                .filter(ResearchHistory.id == research_id)
                .first()
            )

            if not research:
                return _research_not_found(research_id)

            return jsonify(
                {
                    "id": research.id,
                    "query": research.query,
                    "status": research.status,
                    "progress": research.progress,
                    "progress_percentage": research.progress or 0,
                    "mode": research.mode,
                    "created_at": research.created_at,
                    "completed_at": research.completed_at,
                    "report_path": research.report_path,
                    "metadata": strip_settings_snapshot(research.research_meta),
                }
            )
    except Exception:
        logger.exception("Error getting research details")
        return jsonify({"error": "An internal error has occurred"}), 500


@research_bp.route("/api/research/<string:research_id>/logs")
@login_required
def get_research_logs(research_id):
    """Get logs for a specific research.

    Accepts an optional ``?limit=N`` that bounds the response to the newest
    ``N`` rows (returned oldest-first, matching the default ordering) and is
    clamped to ``[1, HISTORY_LOGS_HARD_CAP]`` so a client cannot force an
    unbounded load — a long langgraph run can persist thousands of rows. When
    ``?limit`` is absent or not a valid integer (Flask ``type=int`` yields
    ``None``) the historical contract is preserved: every row is returned. The
    frontend log panel always sends a valid limit; this only affects direct
    API callers that omit or malform one.
    """
    username = session["username"]

    limit = request.args.get("limit", type=int)
    if limit is not None:
        limit = max(1, min(limit, HISTORY_LOGS_HARD_CAP))

    try:
        # First check if the research exists
        with get_user_db_session(username) as db_session:
            research = (
                db_session.query(ResearchHistory)
                .filter_by(id=research_id)
                .first()
            )
            if not research:
                return _research_not_found(research_id)

            # Get logs from research_logs table
            log_query = db_session.query(ResearchLog).filter_by(
                research_id=research_id
            )
            if limit is None:
                log_results = log_query.order_by(
                    ResearchLog.timestamp, ResearchLog.id
                ).all()
            else:
                # Take the newest ``limit`` rows at the SQL layer, then flip
                # back to oldest-first so the response ordering is unchanged.
                # ``id`` is the tie-break: timestamps are not unique, so without
                # it the rows that survive ``.limit()`` at a shared-timestamp
                # boundary would be SQL-undefined.
                log_results = list(
                    reversed(
                        log_query.order_by(
                            ResearchLog.timestamp.desc(),
                            ResearchLog.id.desc(),
                        )
                        .limit(limit)
                        .all()
                    )
                )

            # Extract log attributes while session is active
            # to avoid DetachedInstanceError on ORM attribute access
            logs = []
            for row in log_results:
                logs.append(
                    {
                        "id": row.id,
                        "message": row.message,
                        "timestamp": row.timestamp,
                        "log_type": row.level,
                    }
                )

        return jsonify(logs)

    except Exception:
        logger.exception("Error getting research logs")
        return jsonify({"error": "An internal error has occurred"}), 500


@research_bp.route("/api/report/<string:research_id>")
@login_required
def get_research_report(research_id):
    """Get the research report content"""
    username = session["username"]

    try:
        with get_user_db_session(username) as db_session:
            # Query using ORM
            research = (
                db_session.query(ResearchHistory)
                .filter_by(id=research_id)
                .first()
            )

            if research is None:
                return _research_not_found(research_id)

            # Parse metadata if it exists
            metadata = research.research_meta

            # research.report_content holds the answer-only string;
            # rebuild the legacy display shape (answer + Sources from
            # research_resources + Metrics from research_meta) on demand.
            from ..services.report_assembly_service import (
                assemble_full_report,
                get_research_source_links_batch,
            )

            content = assemble_full_report(research, db_session)
            # Only None means "research not found" — guarded above.
            # Empty-but-found rows return "" and are valid responses.
            if content is None:
                return _research_not_found(
                    research_id, message="Report not found"
                )

            # Sources live in the research_resources table, not research_meta.
            # The post-refactor save path never writes the legacy
            # `all_links_of_system` metadata key, so reading it here returned
            # [] for every research created since chat-mode-v2 (#3665). Read
            # the structured table instead — the same source of truth the
            # assembled `content` and the news feed already use. limit=None
            # returns every source (this field was never top-N), matching the
            # full list the assembled `content` renders for the same research.
            sources = get_research_source_links_batch(
                [research.id], db_session, limit=None
            ).get(research.id, [])

            # Return the report data with backwards-compatible fields
            # Examples expect 'summary', 'sources', 'findings' at top level
            safe_metadata = strip_settings_snapshot(metadata)
            return jsonify(
                {
                    "content": content,
                    # Backwards-compatible fields for examples
                    "summary": content,  # The markdown report is the summary
                    "sources": sources,
                    "findings": safe_metadata.get("findings", []),
                    "metadata": {
                        "title": research.title if research.title else None,
                        "query": research.query,
                        "mode": research.mode if research.mode else None,
                        "created_at": research.created_at
                        if research.created_at
                        else None,
                        "completed_at": research.completed_at
                        if research.completed_at
                        else None,
                        "report_path": research.report_path,
                        **safe_metadata,
                    },
                }
            )

    except Exception:
        logger.exception("Error getting research report")
        return jsonify({"error": "An internal error has occurred"}), 500


@research_bp.route(
    "/api/v1/research/<research_id>/export/<format>", methods=["POST"]
)
@login_required
def export_research_report(research_id, format):
    """Export research report to different formats (LaTeX, Quarto, RIS, PDF, ODT, etc.)"""
    try:
        # Use the exporter registry to validate format
        from ...exporters import ExporterRegistry

        if not ExporterRegistry.is_format_supported(format):
            available = ExporterRegistry.get_available_formats()
            return jsonify(
                {
                    "error": f"Invalid format. Available formats: {', '.join(available)}"
                }
            ), 400

        # Get research from database
        username = session["username"]

        try:
            with get_user_db_session(username) as db_session:
                research = (
                    db_session.query(ResearchHistory)
                    .filter_by(id=research_id)
                    .first()
                )
                if not research:
                    return _research_not_found(research_id)

                # Build the full assembled report (answer + Sources +
                # Metrics) so exporters get the same shape they did
                # before the report_content refactor.
                from ..services.report_assembly_service import (
                    assemble_full_report,
                )

                report_content = assemble_full_report(research, db_session)
                if report_content is None:
                    return _research_not_found(
                        research_id, message="Report content not found"
                    )

                # Export to requested format (all in memory)
                try:
                    # Use title or query for the PDF title
                    pdf_title = research.title or research.query

                    # Generate export content in memory
                    export_content, filename, mimetype = (
                        export_report_to_memory(
                            report_content, format, title=pdf_title
                        )
                    )

                    # Send the file directly from memory
                    return send_file(
                        io.BytesIO(export_content),
                        as_attachment=True,
                        download_name=filename,
                        mimetype=mimetype,
                    )
                except MissingPDFDependencyError:
                    logger.exception(
                        "PDF export failed: WeasyPrint unavailable"
                    )
                    return jsonify(
                        {"error": get_weasyprint_install_instructions()}
                    ), 500
                except Exception:
                    logger.exception("Error exporting report")
                    return jsonify(
                        {
                            "error": f"Failed to export to {format}. Please try again later."
                        }
                    ), 500

        except Exception:
            logger.exception("Error in export endpoint")
            return jsonify({"error": "An internal error has occurred"}), 500

    except Exception:
        logger.exception("Unexpected error in export endpoint")
        return jsonify({"error": "An internal error has occurred"}), 500


@research_bp.route("/api/research/<string:research_id>/status")
@limiter.exempt
@login_required
def get_research_status(research_id):
    """Get the status of a research process"""
    username = session["username"]

    try:
        with get_user_db_session(username) as db_session:
            research = (
                db_session.query(ResearchHistory)
                .filter_by(id=research_id)
                .first()
            )

            if research is None:
                return _research_not_found(research_id)

            status = research.status
            progress = research.progress
            completed_at = research.completed_at
            report_path = research.report_path
            metadata = research.research_meta or {}

            # Extract and format error information for better UI display
            error_info = {}
            if metadata and "error" in metadata:
                error_msg = metadata["error"]
                error_type = "unknown"

                # Detect specific error types
                if "timeout" in error_msg.lower():
                    error_type = "timeout"
                    error_info = {
                        "type": "timeout",
                        "message": "LLM service timed out during synthesis. This may be due to high server load or connectivity issues.",
                        "suggestion": "Try again later or use a smaller query scope.",
                    }
                elif (
                    "token limit" in error_msg.lower()
                    or "context length" in error_msg.lower()
                ):
                    error_type = "token_limit"
                    error_info = {
                        "type": "token_limit",
                        "message": "The research query exceeded the AI model's token limit during synthesis.",
                        "suggestion": "Try using a more specific query or reduce the research scope.",
                    }
                elif (
                    "final answer synthesis fail" in error_msg.lower()
                    or "llm error" in error_msg.lower()
                ):
                    error_type = "llm_error"
                    error_info = {
                        "type": "llm_error",
                        "message": "The AI model encountered an error during final answer synthesis.",
                        "suggestion": "Check that your LLM service is running correctly or try a different model.",
                    }
                elif "ollama" in error_msg.lower():
                    error_type = "ollama_error"
                    error_info = {
                        "type": "ollama_error",
                        "message": "The Ollama service is not responding properly.",
                        "suggestion": "Make sure Ollama is running with 'ollama serve' and the model is downloaded.",
                    }
                elif "connection" in error_msg.lower():
                    error_type = "connection"
                    error_info = {
                        "type": "connection",
                        "message": "Connection error with the AI service.",
                        "suggestion": "Check your internet connection and AI service status.",
                    }
                elif metadata.get("solution"):
                    # Use the solution provided in metadata if available
                    error_info = {
                        "type": error_type,
                        "message": error_msg,
                        "suggestion": str(metadata.get("solution")),
                    }
                else:
                    # Generic error with the original message
                    error_info = {
                        "type": error_type,
                        "message": error_msg,
                        "suggestion": "Try again with a different query or check the application logs.",
                    }

            # Get the latest milestone log for this research
            latest_milestone = None
            try:
                milestone_log = (
                    db_session.query(ResearchLog)
                    .filter_by(research_id=research_id, level="MILESTONE")
                    # id tie-breaks equal timestamps so "latest" is
                    # deterministic (the most recently inserted milestone).
                    .order_by(
                        ResearchLog.timestamp.desc(), ResearchLog.id.desc()
                    )
                    .first()
                )
                if milestone_log:
                    latest_milestone = {
                        "message": milestone_log.message,
                        "time": milestone_log.timestamp.isoformat()
                        if milestone_log.timestamp
                        else None,
                        "type": "MILESTONE",
                    }
                    logger.debug(
                        f"Found latest milestone for research {research_id}: {milestone_log.message}"
                    )
                else:
                    logger.debug(
                        f"No milestone logs found for research {research_id}"
                    )
            except Exception:
                logger.warning("Error fetching latest milestone")

            filtered_metadata = strip_settings_snapshot(metadata)
            if error_info:
                filtered_metadata["error_info"] = error_info

            response_data = {
                "status": status,
                "progress": progress,
                "completed_at": completed_at,
                "report_path": report_path,
                "metadata": filtered_metadata,
            }

            # Include latest milestone as a log_entry for frontend compatibility
            if latest_milestone:
                response_data["log_entry"] = latest_milestone

            return jsonify(response_data)
    except Exception:
        logger.exception("Error getting research status")
        return jsonify({"error": "Error checking research status"}), 500


@research_bp.route("/api/queue/status", methods=["GET"])
@login_required
def get_queue_status():
    """Get the current queue status for the user"""
    username = session["username"]

    from ..queue import QueueManager

    try:
        queue_items = QueueManager.get_user_queue(username)

        return jsonify(
            {
                "status": "success",
                "queue": queue_items,
                "total": len(queue_items),
            }
        )
    except Exception:
        logger.exception("Error getting queue status")
        return jsonify(
            {"status": "error", "message": "Failed to process request"}
        ), 500


@research_bp.route("/api/queue/<string:research_id>/position", methods=["GET"])
@login_required
def get_queue_position(research_id):
    """Get the queue position for a specific research"""
    username = session["username"]

    from ..queue import QueueManager

    try:
        position = QueueManager.get_queue_position(username, research_id)

        if position is None:
            return _research_not_found(
                research_id, message="Research not found in queue"
            )

        return jsonify({"status": "success", "position": position})
    except Exception:
        logger.exception("Error getting queue position")
        return jsonify(
            {"status": "error", "message": "Failed to process request"}
        ), 500


@research_bp.route("/api/config/limits", methods=["GET"])
@login_required
def get_upload_limits():
    """
    Get file upload configuration limits.

    Returns the backend's authoritative limits for file uploads,
    allowing the frontend to stay in sync without hardcoding values.
    """
    return jsonify(
        {
            "max_file_size": FileUploadValidator.MAX_FILE_SIZE,
            "max_files": FileUploadValidator.MAX_FILES_PER_REQUEST,
            "allowed_mime_types": list(FileUploadValidator.ALLOWED_MIME_TYPES),
        }
    )


@research_bp.route("/api/upload/pdf", methods=["POST"])
@login_required
@upload_rate_limit_user
@upload_rate_limit_ip
def upload_pdf():
    """
    Upload and extract text from PDF files with comprehensive security validation.

    Security features:
    - Rate limiting (10 uploads/min, 100/hour per user)
    - File size validation (50MB max per file)
    - File count validation (100 files max)
    - PDF structure validation
    - MIME type validation

    Performance improvements:
    - Single-pass PDF processing (text + metadata)
    - Optimized extraction service
    """
    try:
        # Early request size validation (before reading any files)
        # This prevents memory exhaustion from chunked encoding attacks
        max_request_size = (
            FileUploadValidator.MAX_FILES_PER_REQUEST
            * FileUploadValidator.MAX_FILE_SIZE
        )
        if request.content_length and request.content_length > max_request_size:
            return jsonify(
                {
                    "error": f"Request too large. Maximum size is {max_request_size // (1024 * 1024)}MB"
                }
            ), 413

        # Check if files are present in the request
        if "files" not in request.files:
            return jsonify({"error": "No files provided"}), 400

        files = request.files.getlist("files")
        if not files or files[0].filename == "":
            return jsonify({"error": "No files selected"}), 400

        # Validate file count
        is_valid, error_msg = FileUploadValidator.validate_file_count(
            len(files)
        )
        if not is_valid:
            return jsonify({"error": error_msg}), 400

        # Get PDF extraction service
        pdf_service = get_pdf_extraction_service()

        extracted_texts = []
        total_files = len(files)
        processed_files = 0
        errors = []

        for file in files:
            if not file or not file.filename:
                errors.append("Unnamed file: Skipped")
                continue

            try:
                filename = sanitize_filename(
                    file.filename, allowed_extensions={".pdf"}
                )
            except UnsafeFilenameError:
                errors.append("Rejected file: invalid or disallowed filename")
                continue

            try:
                # Read file content (with disk spooling, large files are read from temp file)
                pdf_content = file.read()

                # Comprehensive validation
                is_valid, error_msg = FileUploadValidator.validate_upload(
                    filename=filename,
                    file_content=pdf_content,
                    content_length=file.content_length,
                )

                if not is_valid:
                    errors.append(f"{filename}: {error_msg}")
                    continue

                # Extract text and metadata in single pass (performance fix)
                result = pdf_service.extract_text_and_metadata(
                    pdf_content, filename
                )

                if result["success"]:
                    extracted_texts.append(
                        {
                            "filename": result["filename"],
                            "text": result["text"],
                            "size": result["size"],
                            "pages": result["pages"],
                        }
                    )
                    processed_files += 1
                else:
                    errors.append(f"{filename}: {result['error']}")

            except Exception:
                logger.exception(f"Error processing {filename}")
                errors.append(f"{filename}: Error processing file")
            finally:
                # Close the file stream to release resources
                try:
                    file.close()
                except Exception:
                    logger.debug("best-effort file stream close", exc_info=True)

        # Prepare response
        response_data = {
            "status": "success",
            "processed_files": processed_files,
            "total_files": total_files,
            "extracted_texts": extracted_texts,
            "combined_text": "\n\n".join(
                [
                    f"--- From {item['filename']} ---\n{item['text']}"
                    for item in extracted_texts
                ]
            ),
            "errors": errors,
        }

        if processed_files == 0:
            return jsonify(
                {
                    "status": "error",
                    "message": "No files were processed successfully",
                    "errors": errors,
                }
            ), 400

        return jsonify(response_data)

    except Exception:
        logger.exception("Error processing PDF upload")
        return jsonify({"error": "Failed to process PDF files"}), 500
