"""
Coverage tests for rag_routes.py targeting the largest untested blocks.

Covers:
- get_current_settings: settings retrieval, JSON parsing, error handling
- configure_rag: settings persistence, validation, collection update
- index_collection: SSE streaming, collection not found, no docs, success/fail
"""

import json
from contextlib import contextmanager
from unittest.mock import Mock, patch, MagicMock

import pytest
from flask import Flask, jsonify

from local_deep_research.constants import DEFAULT_LOCAL_SEARCH_TEXT_SEPARATORS
from local_deep_research.web.auth.routes import auth_bp
from local_deep_research.research_library.routes.rag_routes import rag_bp

# Module path shorthands
_ROUTES = "local_deep_research.research_library.routes.rag_routes"
_DB_CTX = "local_deep_research.database.session_context"

# ---------------------------------------------------------------------------
# Test infrastructure
# ---------------------------------------------------------------------------


def _create_app():
    app = Flask(__name__)
    app.config["SECRET_KEY"] = "test-secret"
    app.config["WTF_CSRF_ENABLED"] = False
    app.config["TESTING"] = True
    app.register_blueprint(auth_bp)
    app.register_blueprint(rag_bp)

    @app.errorhandler(500)
    def _handle_500(error):
        return jsonify({"error": "Internal server error"}), 500

    return app


def _mock_db_manager():
    mock_db = Mock()
    mock_db.is_user_connected.return_value = True
    mock_db.connections = {"testuser": True}
    mock_db.has_encryption = False
    return mock_db


def _build_mock_query(all_result=None, first_result=None, count_result=0):
    q = Mock()
    q.all.return_value = all_result if all_result is not None else []
    q.first.return_value = first_result
    q.count.return_value = count_result
    q.filter_by.return_value = q
    q.filter.return_value = q
    q.order_by.return_value = q
    q.outerjoin.return_value = q
    q.join.return_value = q
    q.options.return_value = q
    q.limit.return_value = q
    q.offset.return_value = q
    return q


def _make_db_session():
    db_session = Mock()
    db_session.query = Mock(return_value=_build_mock_query())
    db_session.commit = Mock()
    db_session.add = Mock()
    db_session.flush = Mock()
    db_session.expire_all = Mock()
    return db_session


def _make_settings_mock(overrides=None):
    mock_sm = Mock()
    defaults = {
        "local_search_embedding_model": "all-MiniLM-L6-v2",
        "local_search_embedding_provider": "sentence_transformers",
        "local_search_chunk_size": 1000,
        "local_search_chunk_overlap": 200,
        "local_search_splitter_type": "recursive",
        "local_search_text_separators": '["\\n\\n", "\\n", ". ", " ", ""]',
        "local_search_distance_metric": "cosine",
        "local_search_normalize_vectors": True,
        "local_search_index_type": "flat",
        "research_library.upload_pdf_storage": "none",
        "research_library.storage_path": "/tmp/test_lib",
        "rag.indexing_batch_size": 15,
        "research_library.auto_index_enabled": True,
    }
    if overrides:
        defaults.update(overrides)
    mock_sm.get_setting.side_effect = lambda k, d=None: defaults.get(k, d)
    mock_sm.get_bool_setting.side_effect = lambda k, d=None: defaults.get(k, d)
    mock_sm.get_all_settings.return_value = {}
    mock_sm.set_setting = Mock()
    mock_sm.get_settings_snapshot.return_value = {}
    return mock_sm


@contextmanager
def _auth_client(
    app,
    mock_db_session=None,
    settings_overrides=None,
    extra_patches=None,
):
    mock_db = _mock_db_manager()
    db_session = mock_db_session or _make_db_session()
    mock_sm = _make_settings_mock(settings_overrides)

    @contextmanager
    def fake_get_user_db_session(*a, **kw):
        yield db_session

    patches = [
        patch("local_deep_research.web.auth.decorators.db_manager", mock_db),
        patch(
            f"{_DB_CTX}.get_user_db_session",
            side_effect=fake_get_user_db_session,
        ),
        patch(f"{_ROUTES}.get_settings_manager", return_value=mock_sm),
        patch(
            "local_deep_research.utilities.db_utils.get_settings_manager",
            return_value=mock_sm,
        ),
        patch(f"{_ROUTES}.limiter", Mock(exempt=lambda f: f)),
        patch(f"{_ROUTES}.upload_rate_limit_user", lambda f: f),
        patch(f"{_ROUTES}.upload_rate_limit_ip", lambda f: f),
    ]
    if extra_patches:
        patches.extend(extra_patches)

    started = []
    try:
        for p in patches:
            started.append(p.start())
        with app.test_client() as client:
            with client.session_transaction() as sess:
                sess["username"] = "testuser"
                sess["session_id"] = "test-session-id"
            yield (
                client,
                {"db_session": db_session, "settings": mock_sm},
            )
    finally:
        for p in patches:
            p.stop()


@pytest.fixture
def app():
    return _create_app()


# ===========================================================================
# get_current_settings
# ===========================================================================


class TestGetCurrentSettings:
    """Tests for the GET /api/rag/settings endpoint."""

    def test_returns_all_settings(self, app):
        """Settings are returned with correct defaults."""
        with _auth_client(app) as (client, ctx):
            resp = client.get("/library/api/rag/settings")
            assert resp.status_code == 200
            data = resp.get_json()
            assert data["success"] is True
            s = data["settings"]
            assert s["embedding_model"] == "all-MiniLM-L6-v2"
            assert s["embedding_provider"] == "sentence_transformers"
            assert s["chunk_size"] == 1000
            assert s["chunk_overlap"] == 200
            assert s["splitter_type"] == "recursive"
            assert s["distance_metric"] == "cosine"
            assert s["normalize_vectors"] is True
            assert s["index_type"] == "flat"

    def test_text_separators_parsed_from_json_string(self, app):
        """text_separators stored as JSON string is parsed to list."""
        with _auth_client(app) as (client, ctx):
            resp = client.get("/library/api/rag/settings")
            data = resp.get_json()
            separators = data["settings"]["text_separators"]
            assert isinstance(separators, list)
            assert "\n\n" in separators

    def test_text_separators_invalid_json_falls_back_to_defaults(self, app):
        """Invalid JSON for text_separators falls back to the default
        separators rather than being kept as a raw string. Migration #4298
        heals existing corrupt rows."""
        with _auth_client(
            app,
            settings_overrides={
                "local_search_text_separators": "not-valid-json{"
            },
        ) as (client, ctx):
            resp = client.get("/library/api/rag/settings")
            data = resp.get_json()
            assert data["success"] is True
            separators = data["settings"]["text_separators"]
            assert separators == DEFAULT_LOCAL_SEARCH_TEXT_SEPARATORS

    def test_text_separators_python_repr_falls_back_to_defaults(self, app):
        """Issue #4230: a prior bug stored ``str(list)`` (Python repr with
        single quotes) instead of ``json.dumps(list)``. That value is not
        valid JSON and is no longer ast-recovered at read time; it now falls
        back to the default separators. Migration #4298 heals existing
        corrupt rows."""
        # Same content the original buggy write would have produced:
        # str(["\n\n", "\n", ". ", " ", ""]) == "['\\n\\n', '\\n', '. ', ' ', '']"
        corrupt = str(DEFAULT_LOCAL_SEARCH_TEXT_SEPARATORS)
        with _auth_client(
            app,
            settings_overrides={"local_search_text_separators": corrupt},
        ) as (client, ctx):
            resp = client.get("/library/api/rag/settings")
            data = resp.get_json()
            assert data["success"] is True
            separators = data["settings"]["text_separators"]
            assert separators == DEFAULT_LOCAL_SEARCH_TEXT_SEPARATORS

    def test_error_returns_500(self, app):
        """Exception in settings retrieval returns error response."""
        broken_sm = Mock()
        broken_sm.get_setting.side_effect = RuntimeError("DB down")

        with _auth_client(app) as (client, ctx):
            with patch(
                f"{_ROUTES}.get_settings_manager", return_value=broken_sm
            ):
                resp = client.get("/library/api/rag/settings")
                assert resp.status_code == 500


# ===========================================================================
# configure_rag
# ===========================================================================


class TestConfigureRag:
    """Tests for the POST /api/rag/configure endpoint."""

    def test_missing_required_fields_returns_400(self, app):
        """Omitting required fields returns 400."""
        with _auth_client(app) as (client, ctx):
            resp = client.post(
                "/library/api/rag/configure",
                json={"embedding_model": "test-model"},
                content_type="application/json",
            )
            assert resp.status_code == 400
            data = resp.get_json()
            assert data["success"] is False

    def test_saves_default_settings_without_collection(self, app):
        """When no collection_id, saves default settings."""
        with _auth_client(app) as (client, ctx):
            resp = client.post(
                "/library/api/rag/configure",
                json={
                    "embedding_model": "test-model",
                    "embedding_provider": "sentence_transformers",
                    "chunk_size": 500,
                    "chunk_overlap": 100,
                },
                content_type="application/json",
            )
            assert resp.status_code == 200
            data = resp.get_json()
            assert data["success"] is True
            assert "Default" in data["message"]

            # Verify settings were persisted
            sm = ctx["settings"]
            calls = {c[0][0]: c[0][1] for c in sm.set_setting.call_args_list}
            assert calls["local_search_embedding_model"] == "test-model"
            assert calls["local_search_chunk_size"] == 500
            assert calls["local_search_chunk_overlap"] == 100

    def test_saves_advanced_settings(self, app):
        """Advanced settings (splitter_type, distance_metric, etc.) are saved."""
        with _auth_client(app) as (client, ctx):
            resp = client.post(
                "/library/api/rag/configure",
                json={
                    "embedding_model": "m",
                    "embedding_provider": "p",
                    "chunk_size": 500,
                    "chunk_overlap": 100,
                    "splitter_type": "character",
                    "distance_metric": "l2",
                    "normalize_vectors": False,
                    "index_type": "ivf",
                    "text_separators": ["\n", " "],
                },
                content_type="application/json",
            )
            assert resp.status_code == 200
            sm = ctx["settings"]
            calls = {c[0][0]: c[0][1] for c in sm.set_setting.call_args_list}
            assert calls["local_search_splitter_type"] == "character"
            assert calls["local_search_distance_metric"] == "l2"
            assert calls["local_search_normalize_vectors"] is False
            assert calls["local_search_index_type"] == "ivf"

    def test_text_separators_list_stored_as_json(self, app):
        """text_separators list is stored directly for storage."""
        with _auth_client(app) as (client, ctx):
            resp = client.post(
                "/library/api/rag/configure",
                json={
                    "embedding_model": "m",
                    "embedding_provider": "p",
                    "chunk_size": 500,
                    "chunk_overlap": 100,
                    "text_separators": ["\n", " "],
                },
                content_type="application/json",
            )
            assert resp.status_code == 200
            sm = ctx["settings"]
            calls = {c[0][0]: c[0][1] for c in sm.set_setting.call_args_list}
            stored = calls["local_search_text_separators"]
            assert isinstance(stored, list)
            assert stored == ["\n", " "]

    def test_with_collection_id_creates_rag_service(self, app):
        """When collection_id is provided, creates RAG service for that collection."""
        mock_rag = MagicMock()
        mock_rag.__enter__ = Mock(return_value=mock_rag)
        mock_rag.__exit__ = Mock(return_value=False)
        mock_index = Mock()
        mock_index.index_hash = "abc123"
        mock_rag._get_or_create_rag_index.return_value = mock_index

        with _auth_client(
            app,
            extra_patches=[
                patch(f"{_ROUTES}.LibraryRAGService", return_value=mock_rag)
            ],
        ) as (client, ctx):
            resp = client.post(
                "/library/api/rag/configure",
                json={
                    "embedding_model": "m",
                    "embedding_provider": "p",
                    "chunk_size": 500,
                    "chunk_overlap": 100,
                    "collection_id": "coll-1",
                },
                content_type="application/json",
            )
            assert resp.status_code == 200
            data = resp.get_json()
            assert data["success"] is True
            assert data["index_hash"] == "abc123"

    def test_exception_returns_500(self, app):
        """Exception during configure returns error."""
        broken_sm = Mock()
        broken_sm.set_setting.side_effect = RuntimeError("DB error")

        with _auth_client(app) as (client, ctx):
            with patch(
                f"{_ROUTES}.get_settings_manager", return_value=broken_sm
            ):
                resp = client.post(
                    "/library/api/rag/configure",
                    json={
                        "embedding_model": "m",
                        "embedding_provider": "p",
                        "chunk_size": 500,
                        "chunk_overlap": 100,
                    },
                    content_type="application/json",
                )
                assert resp.status_code == 500


# ===========================================================================
# index_collection — SSE streaming
# ===========================================================================


class TestIndexCollection:
    """Tests for the GET /api/collections/<id>/index endpoint (SSE)."""

    def _collect_sse_events(self, response):
        """Parse SSE events from streaming response."""
        events = []
        for line in response.data.decode().split("\n"):
            if line.startswith("data: "):
                events.append(json.loads(line[6:]))
        return events

    def test_collection_not_found(self, app):
        """Returns error event when collection doesn't exist."""
        db_session = _make_db_session()
        db_session.query.return_value = _build_mock_query(first_result=None)

        mock_rag = Mock()
        with _auth_client(
            app,
            mock_db_session=db_session,
            extra_patches=[
                patch(f"{_ROUTES}.get_rag_service", return_value=mock_rag),
                patch(
                    "local_deep_research.database.session_passwords.session_password_store",
                    Mock(get_session_password=Mock(return_value=None)),
                ),
            ],
        ) as (client, ctx):
            resp = client.get("/library/api/collections/nonexistent/index")
            assert resp.status_code == 200
            assert "text/event-stream" in resp.content_type
            events = self._collect_sse_events(resp)
            assert any(e.get("type") == "error" for e in events)
            assert any("not found" in e.get("error", "") for e in events)

    def test_no_documents_to_index(self, app):
        """Returns complete event with zero counts when no docs need indexing."""
        mock_coll = Mock()
        mock_coll.id = "coll-1"
        mock_coll.name = "Test"
        mock_coll.embedding_model = "already-set"

        db_session = _make_db_session()
        call_count = [0]

        def query_side_effect(*args):
            call_count[0] += 1
            q = _build_mock_query()
            if call_count[0] == 1:
                q.first.return_value = mock_coll
            elif call_count[0] == 2:
                # join + filter chain for documents
                q.join.return_value = q
                q.filter.return_value = q
                q.all.return_value = []  # No documents
            return q

        db_session.query = Mock(side_effect=query_side_effect)

        mock_rag = Mock()
        with _auth_client(
            app,
            mock_db_session=db_session,
            extra_patches=[
                patch(f"{_ROUTES}.get_rag_service", return_value=mock_rag),
                patch(
                    "local_deep_research.database.session_passwords.session_password_store",
                    Mock(get_session_password=Mock(return_value=None)),
                ),
            ],
        ) as (client, ctx):
            resp = client.get("/library/api/collections/coll-1/index")
            events = self._collect_sse_events(resp)
            complete = [e for e in events if e.get("type") == "complete"]
            assert len(complete) == 1
            assert complete[0]["results"]["successful"] == 0
            assert complete[0]["results"]["message"] == "No documents to index"

    def test_successful_indexing(self, app):
        """Documents are indexed and progress/complete events are emitted."""
        mock_coll = Mock()
        mock_coll.id = "coll-1"
        mock_coll.name = "Test Collection"
        mock_coll.embedding_model = "model"

        mock_link = Mock()
        mock_doc = Mock()
        mock_doc.id = "doc-1"
        mock_doc.filename = "test.pdf"
        mock_doc.title = None

        db_session = _make_db_session()
        call_count = [0]

        def query_side_effect(*args):
            call_count[0] += 1
            q = _build_mock_query()
            if call_count[0] == 1:
                q.first.return_value = mock_coll
            elif call_count[0] == 2:
                q.join.return_value = q
                q.filter.return_value = q
                q.all.return_value = [(mock_link, mock_doc)]
            return q

        db_session.query = Mock(side_effect=query_side_effect)

        mock_rag = Mock()
        mock_rag.index_document.return_value = {"status": "success"}

        with _auth_client(
            app,
            mock_db_session=db_session,
            extra_patches=[
                patch(f"{_ROUTES}.get_rag_service", return_value=mock_rag),
                patch(
                    "local_deep_research.database.session_passwords.session_password_store",
                    Mock(get_session_password=Mock(return_value=None)),
                ),
            ],
        ) as (client, ctx):
            resp = client.get("/library/api/collections/coll-1/index")
            events = self._collect_sse_events(resp)

            types = [e.get("type") for e in events]
            assert "start" in types
            assert "progress" in types
            assert "complete" in types

            complete = [e for e in events if e["type"] == "complete"][0]
            assert complete["results"]["successful"] == 1
            assert complete["results"]["failed"] == 0

    def test_indexing_with_failed_document(self, app):
        """Failed document is counted and error event emitted."""
        mock_coll = Mock()
        mock_coll.id = "coll-1"
        mock_coll.name = "Test"
        mock_coll.embedding_model = "model"

        mock_link = Mock()
        mock_doc = Mock()
        mock_doc.id = "doc-1"
        mock_doc.filename = "bad.pdf"
        mock_doc.title = None

        db_session = _make_db_session()
        call_count = [0]

        def query_side_effect(*args):
            call_count[0] += 1
            q = _build_mock_query()
            if call_count[0] == 1:
                q.first.return_value = mock_coll
            elif call_count[0] == 2:
                q.join.return_value = q
                q.filter.return_value = q
                q.all.return_value = [(mock_link, mock_doc)]
            return q

        db_session.query = Mock(side_effect=query_side_effect)

        mock_rag = Mock()
        mock_rag.index_document.side_effect = RuntimeError("Parse error")

        with _auth_client(
            app,
            mock_db_session=db_session,
            extra_patches=[
                patch(f"{_ROUTES}.get_rag_service", return_value=mock_rag),
                patch(
                    "local_deep_research.database.session_passwords.session_password_store",
                    Mock(get_session_password=Mock(return_value=None)),
                ),
            ],
        ) as (client, ctx):
            resp = client.get("/library/api/collections/coll-1/index")
            events = self._collect_sse_events(resp)

            complete = [e for e in events if e["type"] == "complete"][0]
            assert complete["results"]["failed"] == 1
            assert len(complete["results"]["errors"]) == 1
            assert "bad.pdf" in complete["results"]["errors"][0]["filename"]

    def test_skipped_document(self, app):
        """Document returning 'skipped' status is counted correctly."""
        mock_coll = Mock()
        mock_coll.id = "coll-1"
        mock_coll.name = "Test"
        mock_coll.embedding_model = "model"

        mock_link = Mock()
        mock_doc = Mock()
        mock_doc.id = "doc-1"
        mock_doc.filename = "already.pdf"
        mock_doc.title = None

        db_session = _make_db_session()
        call_count = [0]

        def query_side_effect(*args):
            call_count[0] += 1
            q = _build_mock_query()
            if call_count[0] == 1:
                q.first.return_value = mock_coll
            elif call_count[0] == 2:
                q.join.return_value = q
                q.filter.return_value = q
                q.all.return_value = [(mock_link, mock_doc)]
            return q

        db_session.query = Mock(side_effect=query_side_effect)

        mock_rag = Mock()
        mock_rag.index_document.return_value = {"status": "skipped"}

        with _auth_client(
            app,
            mock_db_session=db_session,
            extra_patches=[
                patch(f"{_ROUTES}.get_rag_service", return_value=mock_rag),
                patch(
                    "local_deep_research.database.session_passwords.session_password_store",
                    Mock(get_session_password=Mock(return_value=None)),
                ),
            ],
        ) as (client, ctx):
            resp = client.get("/library/api/collections/coll-1/index")
            events = self._collect_sse_events(resp)

            complete = [e for e in events if e["type"] == "complete"][0]
            assert complete["results"]["skipped"] == 1

    def test_stores_embedding_metadata_on_first_index(self, app):
        """Embedding metadata is stored on collection when embedding_model is None."""
        mock_coll = Mock()
        mock_coll.id = "coll-1"
        mock_coll.name = "Test"
        mock_coll.embedding_model = None  # First index

        db_session = _make_db_session()
        call_count = [0]

        def query_side_effect(*args):
            call_count[0] += 1
            q = _build_mock_query()
            if call_count[0] == 1:
                q.first.return_value = mock_coll
            elif call_count[0] == 2:
                q.join.return_value = q
                q.filter.return_value = q
                q.all.return_value = []  # No docs
            return q

        db_session.query = Mock(side_effect=query_side_effect)

        mock_rag = Mock()
        mock_rag.embedding_model = "test-embed"
        mock_rag.embedding_provider = "sentence_transformers"
        mock_rag.chunk_size = 500
        mock_rag.chunk_overlap = 50
        mock_rag.splitter_type = "recursive"
        mock_rag.text_separators = '["\n"]'
        mock_rag.distance_metric = "cosine"
        mock_rag.normalize_vectors = True
        mock_rag.index_type = "flat"
        mock_rag.embedding_manager = Mock(spec=[])  # No provider attr

        with _auth_client(
            app,
            mock_db_session=db_session,
            extra_patches=[
                patch(f"{_ROUTES}.get_rag_service", return_value=mock_rag),
                patch(
                    "local_deep_research.database.session_passwords.session_password_store",
                    Mock(get_session_password=Mock(return_value=None)),
                ),
            ],
        ) as (client, ctx):
            resp = client.get("/library/api/collections/coll-1/index")
            self._collect_sse_events(resp)

            # Verify metadata was stored on collection
            assert mock_coll.embedding_model == "test-embed"
            assert mock_coll.chunk_size == 500
            assert mock_coll.chunk_overlap == 50
            db_session.commit.assert_called()

    def test_force_reindex_param(self, app):
        """force_reindex=true re-stores embedding metadata."""
        mock_coll = Mock()
        mock_coll.id = "coll-1"
        mock_coll.name = "Test"
        mock_coll.embedding_model = "old-model"  # Already set

        db_session = _make_db_session()
        call_count = [0]

        def query_side_effect(*args):
            call_count[0] += 1
            q = _build_mock_query()
            if call_count[0] == 1:
                q.first.return_value = mock_coll
            elif call_count[0] == 2:
                q.join.return_value = q
                q.all.return_value = []
            return q

        db_session.query = Mock(side_effect=query_side_effect)

        mock_rag = Mock()
        mock_rag.embedding_model = "new-model"
        mock_rag.embedding_provider = "openai"
        mock_rag.chunk_size = 800
        mock_rag.chunk_overlap = 100
        mock_rag.splitter_type = "recursive"
        mock_rag.text_separators = "[]"
        mock_rag.distance_metric = "l2"
        mock_rag.normalize_vectors = False
        mock_rag.index_type = "ivf"
        mock_rag.embedding_manager = Mock(spec=[])

        with _auth_client(
            app,
            mock_db_session=db_session,
            extra_patches=[
                patch(f"{_ROUTES}.get_rag_service", return_value=mock_rag),
                patch(
                    "local_deep_research.database.session_passwords.session_password_store",
                    Mock(get_session_password=Mock(return_value=None)),
                ),
            ],
        ) as (client, ctx):
            resp = client.get(
                "/library/api/collections/coll-1/index?force_reindex=true"
            )
            self._collect_sse_events(resp)

            # force_reindex should re-store metadata even though model was set
            assert mock_coll.embedding_model == "new-model"

    def test_sse_response_headers(self, app):
        """SSE response has correct headers for streaming."""
        db_session = _make_db_session()
        db_session.query.return_value = _build_mock_query(first_result=None)

        mock_rag = Mock()
        with _auth_client(
            app,
            mock_db_session=db_session,
            extra_patches=[
                patch(f"{_ROUTES}.get_rag_service", return_value=mock_rag),
                patch(
                    "local_deep_research.database.session_passwords.session_password_store",
                    Mock(get_session_password=Mock(return_value=None)),
                ),
            ],
        ) as (client, ctx):
            resp = client.get("/library/api/collections/coll-1/index")
            assert "text/event-stream" in resp.content_type
            assert resp.headers.get("Cache-Control") == "no-cache, no-transform"
            assert resp.headers.get("X-Accel-Buffering") == "no"


class TestIndexAll:
    """The bulk index-all SSE route must share the indexing helpers.

    Regression for H3 incompleteness: index_all previously stored no embedding
    metadata and never reset stale chunks/indices on force-reindex (the two
    drift bugs the dedup was meant to eliminate everywhere).
    """

    def test_force_reindex_stores_metadata_and_resets(self, app):
        mock_coll = Mock()
        mock_coll.id = "coll-1"
        mock_coll.embedding_model = None
        db_session = _make_db_session()
        db_session.query = Mock(
            return_value=_build_mock_query(first_result=mock_coll)
        )
        mock_rag = Mock()

        with (
            patch(
                f"{_ROUTES}._store_collection_embedding_metadata"
            ) as mock_store,
            patch(f"{_ROUTES}._reset_collection_for_reindex") as mock_reset,
            patch(f"{_ROUTES}._query_documents_to_index", return_value=[]),
        ):
            with _auth_client(
                app,
                mock_db_session=db_session,
                extra_patches=[
                    patch(f"{_ROUTES}.get_rag_service", return_value=mock_rag),
                ],
            ) as (client, ctx):
                resp = client.get(
                    "/library/api/rag/index-all"
                    "?collection_id=coll-1&force_reindex=true"
                )
                resp.get_data()  # drain the SSE generator to completion

            mock_store.assert_called_once_with(mock_coll, mock_rag)
            mock_reset.assert_called_once_with(db_session, "coll-1")

    def test_incremental_does_not_reset(self, app):
        """A non-force index-all must NOT wipe existing chunks/indices."""
        mock_coll = Mock()
        mock_coll.id = "coll-1"
        mock_coll.embedding_model = "already-set"
        db_session = _make_db_session()
        db_session.query = Mock(
            return_value=_build_mock_query(first_result=mock_coll)
        )

        with (
            patch(f"{_ROUTES}._store_collection_embedding_metadata"),
            patch(f"{_ROUTES}._reset_collection_for_reindex") as mock_reset,
            patch(f"{_ROUTES}._query_documents_to_index", return_value=[]),
        ):
            with _auth_client(
                app,
                mock_db_session=db_session,
                extra_patches=[
                    patch(f"{_ROUTES}.get_rag_service", return_value=Mock()),
                ],
            ) as (client, ctx):
                resp = client.get(
                    "/library/api/rag/index-all?collection_id=coll-1"
                )
                resp.get_data()

            mock_reset.assert_not_called()


class TestRagServiceCloseLifecycle:
    """Regression coverage for the RAG-service close-on-exit guarantee.

    Without these tests, the existing route fixtures only assert status
    codes — they accept ``Mock().close()`` silently and would not detect
    a regression that drops the ``finally: safe_close(...)`` block from
    an SSE generator, or the ``with get_rag_service(...) as ...``
    wrapper from a synchronous route. The leak the wider PR series
    closes (#3816-shaped FD ramp on the embeddings side) lives behind
    exactly these close calls — so they need explicit assertions.

    Each test asserts that ``close()`` (or its ``__exit__`` cousin)
    fires exactly once when the route handler completes — happy path
    for sync routes, stream-drained path for SSE routes.
    """

    def _collect_sse_events(self, response):
        """Parse SSE events from streaming response."""
        events = []
        for line in response.data.decode().split("\n"):
            if line.startswith("data: "):
                events.append(json.loads(line[6:]))
        return events

    def test_with_wrap_endpoint_calls_exit_on_completion(self, app):
        """Synchronous ``with get_rag_service(...) as rag_service:`` routes
        must invoke the service's ``__exit__`` — the entry-point for
        ``LibraryRAGService.close()`` which in turn closes the embedding
        manager's httpx clients.
        """
        # MagicMock so the route's `with` block works; pin __enter__ to self
        # so the body sees this mock, then assert __exit__ fires.
        mock_rag = MagicMock()
        mock_rag.__enter__.return_value = mock_rag
        mock_rag.get_current_index_info.return_value = {"total_chunks": 0}

        with _auth_client(
            app,
            extra_patches=[
                patch(f"{_ROUTES}.get_rag_service", return_value=mock_rag),
                patch(
                    "local_deep_research.database.library_init.get_default_library_id",
                    return_value="default-lib",
                ),
            ],
        ) as (client, ctx):
            resp = client.get("/library/api/rag/info")
            assert resp.status_code == 200

        mock_rag.__exit__.assert_called_once()

    def test_sse_index_collection_calls_close_at_stream_end(self, app):
        """``index_collection`` is one of three SSE routes that construct
        ``rag_service`` at request scope but use it inside a streamed
        generator. The fix moves the close into the generator's
        ``finally:`` so it fires at stream completion (or client
        disconnect via ``GeneratorExit``) — wrapping the construction
        in a ``with`` at request scope would tear the service down
        before ``stream_with_context`` iterates the generator.
        """
        db_session = _make_db_session()
        db_session.query.return_value = _build_mock_query(first_result=None)

        mock_rag = Mock()  # bare Mock — its close() is auto-attr.

        with _auth_client(
            app,
            mock_db_session=db_session,
            extra_patches=[
                patch(f"{_ROUTES}.get_rag_service", return_value=mock_rag),
                patch(
                    "local_deep_research.database.session_passwords.session_password_store",
                    Mock(get_session_password=Mock(return_value=None)),
                ),
            ],
        ) as (client, ctx):
            resp = client.get("/library/api/collections/coll-1/index")
            # Drain the stream so the generator runs to completion and
            # its ``finally:`` fires. (test_client buffers in memory; the
            # decode in _collect_sse_events forces iteration.)
            self._collect_sse_events(resp)

        # Exactly one close call — the generator's ``finally`` ran
        # without the outer route closing it prematurely.
        mock_rag.close.assert_called_once()

    def test_sse_index_collection_calls_close_even_on_generator_exception(
        self, app
    ):
        """If the SSE generator raises mid-stream, the ``finally:`` block
        must still close ``rag_service``. Mock the DB query to raise; the
        generator's outer ``except`` catches the error, yields an SSE
        error event, and the ``finally`` still runs.
        """
        db_session = _make_db_session()
        db_session.query.side_effect = RuntimeError(
            "simulated DB failure inside generator"
        )

        mock_rag = Mock()

        with _auth_client(
            app,
            mock_db_session=db_session,
            extra_patches=[
                patch(f"{_ROUTES}.get_rag_service", return_value=mock_rag),
                patch(
                    "local_deep_research.database.session_passwords.session_password_store",
                    Mock(get_session_password=Mock(return_value=None)),
                ),
            ],
        ) as (client, ctx):
            resp = client.get("/library/api/collections/coll-1/index")
            events = self._collect_sse_events(resp)

            # The generator's except branch should have surfaced an
            # error event before finally fired.
            assert any(e.get("type") == "error" for e in events)

        # And close still ran despite the exception.
        mock_rag.close.assert_called_once()


# ===========================================================================
# get_collections — agent_enabled serialization
# ===========================================================================


class TestGetCollectionsAgentEnabled:
    """GET /api/collections must serialize the agent_enabled flag default-on.

    Guards the get_collections() expression against regression: a stored NULL
    or a missing attribute must serialize to True (default-on), while an
    explicit False survives. Drives the real route via the auth client.
    """

    def _collection(self, **attrs):
        from types import SimpleNamespace

        base = dict(
            id="c1",
            name="C1",
            description="d",
            created_at=None,
            collection_type="user_uploads",
            is_default=False,
            is_public=False,
            document_links=[],
            linked_folders=[],
            embedding_model=None,
        )
        base.update(attrs)
        return SimpleNamespace(**base)

    def _list(self, app, collections):
        db_session = _make_db_session()
        db_session.query.return_value = _build_mock_query(
            all_result=collections
        )
        with _auth_client(app, mock_db_session=db_session) as (client, _ctx):
            resp = client.get("/library/api/collections")
            assert resp.status_code == 200
            body = resp.get_json()
            assert body["success"] is True
            return {c["name"]: c for c in body["collections"]}

    def test_true_false_null_and_missing(self, app):
        cols = self._list(
            app,
            [
                self._collection(name="on", agent_enabled=True),
                self._collection(name="off", agent_enabled=False),
                self._collection(name="null", agent_enabled=None),
                self._collection(name="missing"),  # attribute absent entirely
            ],
        )
        assert cols["on"]["agent_enabled"] is True
        assert cols["off"]["agent_enabled"] is False
        # NULL in the DB → default-on (matches get_collection_documents()).
        assert cols["null"]["agent_enabled"] is True
        # Pre-migration row with no column → default-on.
        assert cols["missing"]["agent_enabled"] is True


class TestUpdateCollectionAgentEnabled:
    """PUT /api/collections/<id> serializes agent_enabled consistently with GET.

    Guards the create/update response serializers (which used a bare
    ``bool(collection.agent_enabled)`` that mis-rendered a legacy NULL row as
    False) and the explicit-null input normalization (None → available).
    """

    def _collection(self, **attrs):
        from types import SimpleNamespace

        base = dict(
            id="c1",
            name="C1",
            description="d",
            created_at=None,
            collection_type="user_uploads",
            is_public=False,
            agent_enabled=None,  # legacy NULL row by default
        )
        base.update(attrs)
        return SimpleNamespace(**base)

    def _update(self, app, collection, body):
        db_session = _make_db_session()
        db_session.query.return_value = _build_mock_query(
            first_result=collection
        )
        with _auth_client(app, mock_db_session=db_session) as (client, _ctx):
            resp = client.put(
                f"/library/api/collections/{collection.id}", json=body
            )
            assert resp.status_code == 200
            return resp.get_json()["collection"], collection

    def test_legacy_null_row_untouched_serializes_true(self, app):
        # A pre-migration NULL row updated without agent_enabled in the body
        # must serialize as available (True) — the same value GET returns.
        # (Touch description, not name, to avoid the duplicate-name path the
        # shared mock query would otherwise trip.)
        coll = self._collection(agent_enabled=None)
        payload, stored = self._update(app, coll, {"description": "updated"})
        assert payload["agent_enabled"] is True
        assert stored.agent_enabled is None  # storage left untouched

    def test_explicit_null_normalizes_to_available(self, app):
        # {"agent_enabled": null} → stored True (available), serialized True.
        coll = self._collection(agent_enabled=None)
        payload, stored = self._update(app, coll, {"agent_enabled": None})
        assert stored.agent_enabled is True
        assert payload["agent_enabled"] is True

    def test_explicit_false_disables(self, app):
        coll = self._collection(agent_enabled=True)
        payload, stored = self._update(app, coll, {"agent_enabled": False})
        assert stored.agent_enabled is False
        assert payload["agent_enabled"] is False
