"""E2E tests for readdir latency fixes at scale (Issue #3706).

Exercises the three performance fixes end-to-end with a real NexusFS
(Raft metastore, CAS backend — no mocks):

1. sys_readdir with list_iter streaming (details=False + details=True)
2. Implicit directory detection via sorted-path batch (details=True, non-recursive)
3. SearchService list_dir with permission enforcer (batch descendants)
4. HERB corpus listing + search integration

Run with:
    PYTHONPATH=src python -m pytest tests/e2e/self_contained/test_readdir_scale_e2e.py -v -s
"""

import time

import pytest

from nexus.backends.storage.cas_local import CASLocalBackend
from nexus.core.config import PermissionConfig
from nexus.factory import create_nexus_fs

# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------


@pytest.fixture
async def nexus_fs(tmp_path, isolated_db):
    """Create a NexusFS with permissions disabled (pure listing tests)."""
    backend = CASLocalBackend(str(tmp_path / "data"))
    metadata_store = str(isolated_db).replace(".db", "")
    nx = create_nexus_fs(
        backend=backend,
        metadata_store=metadata_store,
        permissions=PermissionConfig(enforce=False),
        enable_write_buffer=False,
    )
    yield nx
    nx.close()


@pytest.fixture
async def nexus_fs_herb(nexus_fs):
    """Seed HERB corpus structure: 30 product dirs × 10 files each = 300 files."""
    products = [
        "NanoSynth",
        "VaultEdge",
        "TerraFlow",
        "QuantumLens",
        "NeuroGrid",
        "HyperScale",
        "PhotonAI",
        "CrystalDB",
        "NovaLink",
        "PulseNet",
        "ArcticOS",
        "FusionML",
        "ZenithAPI",
        "OrbitSync",
        "MatrixHub",
        "BlazeIO",
        "CoralFS",
        "DeltaVPC",
        "EchoStack",
        "FluxCore",
        "GlacierDL",
        "HelixRPC",
        "IndigoKV",
        "JadeAuth",
        "KryptonMQ",
        "LunarSDK",
        "MeteorCDN",
        "NebulaSec",
        "OpalGPU",
        "PrismGW",
    ]
    for product in products:
        base = f"/workspace/enterprise-context/{product}"
        for i in range(10):
            nexus_fs.write(
                f"{base}/doc_{i:02d}.md",
                f"# {product} Document {i}\n\nEnterprise context for {product}.",
            )
    return nexus_fs


@pytest.fixture
async def nexus_fs_scale(nexus_fs):
    """Seed a large flat directory: 5000 files under /bigdir/."""
    for i in range(5000):
        nexus_fs.write(f"/bigdir/file_{i:05d}.txt", f"content {i}")
    return nexus_fs


@pytest.fixture
async def nexus_fs_implicit_dirs(nexus_fs):
    """Seed files under implicit directories (no explicit mkdir).

    Structure:
        /workspace/alpha/file_0.txt .. file_4.txt
        /workspace/beta/file_0.txt .. file_4.txt
        /workspace/gamma/file_0.txt .. file_4.txt
        /workspace/top_level_file.txt
    """
    for d in ["alpha", "beta", "gamma"]:
        for i in range(5):
            nexus_fs.write(f"/workspace/{d}/file_{i}.txt", f"{d} content {i}")
    nexus_fs.write("/workspace/top_level_file.txt", "top level content")
    return nexus_fs


# ============================================================================
# Test 1: sys_readdir streaming (list_iter) at scale
# ============================================================================


class TestReaddirStreamingE2E:
    """Verify sys_readdir works correctly with list_iter streaming."""

    @pytest.mark.asyncio
    async def test_readdir_paths_5k_entries(self, nexus_fs_scale):
        """5K file listing returns all entries."""
        result = nexus_fs_scale.sys_readdir("/bigdir/", recursive=False, details=False)
        assert isinstance(result, list)
        assert len(result) == 5000
        assert all(p.startswith("/bigdir/file_") for p in result)

    @pytest.mark.asyncio
    async def test_readdir_details_5k_entries(self, nexus_fs_scale):
        """5K file listing with details=True returns dicts."""
        result = nexus_fs_scale.sys_readdir("/bigdir/", recursive=False, details=True)
        assert isinstance(result, list)
        assert len(result) == 5000
        assert all(isinstance(r, dict) for r in result)
        assert all(r["path"].startswith("/bigdir/file_") for r in result)
        # entry_type should be 0 (regular file) for all — no implicit dirs
        assert all(r["entry_type"] == 0 for r in result)

    @pytest.mark.asyncio
    async def test_readdir_details_performance(self, nexus_fs_scale):
        """details=True should not be catastrophically slower than details=False."""
        # Warmup
        nexus_fs_scale.sys_readdir("/bigdir/", recursive=False, details=False)
        nexus_fs_scale.sys_readdir("/bigdir/", recursive=False, details=True)

        t0 = time.perf_counter()
        nexus_fs_scale.sys_readdir("/bigdir/", recursive=False, details=False)
        simple_ms = (time.perf_counter() - t0) * 1000

        t0 = time.perf_counter()
        nexus_fs_scale.sys_readdir("/bigdir/", recursive=False, details=True)
        detail_ms = (time.perf_counter() - t0) * 1000

        print(f"  5K entries: details=False={simple_ms:.1f}ms, details=True={detail_ms:.1f}ms")
        ratio = detail_ms / max(simple_ms, 0.1)
        print(f"  Overhead ratio: {ratio:.1f}x")

        # With the fix, details=True should stay bounded and far below the
        # historical 166x overhead. The path-only call can hit the Rust listing
        # index cache and finish well below 1ms, so use an absolute ceiling when
        # the ratio baseline is just a cache-hit timing artifact.
        max_detail_ms = max(simple_ms * 30, 100)
        assert detail_ms < max_detail_ms, (
            f"details=True took {detail_ms:.0f}ms for 5K entries "
            f"(expected <{max_detail_ms:.0f}ms with batch implicit-dir fix)"
        )

    @pytest.mark.asyncio
    async def test_readdir_root_returns_all(self, nexus_fs_scale):
        """Recursive listing from root returns all 5K files."""
        result = nexus_fs_scale.sys_readdir("/", recursive=True, details=False)
        bigdir_files = [p for p in result if p.startswith("/bigdir/")]
        assert len(bigdir_files) == 5000


# ============================================================================
# Test 2: Implicit directory detection (batch path scan)
# ============================================================================


class TestImplicitDirectoryBatchE2E:
    """Verify implicit directories are correctly detected via sorted-path batch."""

    @pytest.mark.asyncio
    async def test_implicit_dirs_promoted_to_entry_type_1(self, nexus_fs_implicit_dirs):
        """Non-recursive detail listing promotes implicit dirs to entry_type=1."""
        result = nexus_fs_implicit_dirs.sys_readdir("/workspace/", recursive=False, details=True)
        # Should see alpha/, beta/, gamma/ as dirs and top_level_file.txt as file
        paths = {r["path"]: r["entry_type"] for r in result}

        # Implicit directories should be promoted to entry_type=1
        for d in ["alpha", "beta", "gamma"]:
            dir_entries = [
                (p, et) for p, et in paths.items() if d in p and "/" in p.split(d, 1)[-1]
            ]
            # The parent implicit dir entry should exist with entry_type=1
            # OR child files are listed depending on metastore behavior
            print(f"  {d}: {len(dir_entries)} entries")

        # The top-level file should remain entry_type=0
        top_files = [r for r in result if r["path"] == "/workspace/top_level_file.txt"]
        if top_files:
            assert top_files[0]["entry_type"] == 0, "Top-level file should stay entry_type=0"

    @pytest.mark.asyncio
    async def test_details_false_returns_paths(self, nexus_fs_implicit_dirs):
        """details=False returns path strings regardless of implicit dirs."""
        result = nexus_fs_implicit_dirs.sys_readdir("/workspace/", recursive=False, details=False)
        assert isinstance(result, list)
        assert all(isinstance(p, str) for p in result)

    @pytest.mark.asyncio
    async def test_recursive_listing_unaffected(self, nexus_fs_implicit_dirs):
        """Recursive listing should return all 16 files (3×5 + 1)."""
        result = nexus_fs_implicit_dirs.sys_readdir("/workspace/", recursive=True, details=False)
        assert len(result) == 16  # 3 dirs × 5 files + 1 top-level


# ============================================================================
# Test 3: HERB corpus listing + search
# ============================================================================


class TestHerbCorpusE2E:
    """Verify listing and search work on HERB-style directory structure."""

    @pytest.mark.asyncio
    async def test_herb_list_all(self, nexus_fs_herb):
        """List all HERB files recursively: 30 products × 10 docs = 300."""
        result = nexus_fs_herb.sys_readdir(
            "/workspace/enterprise-context/", recursive=True, details=False
        )
        assert len(result) == 300

    @pytest.mark.asyncio
    async def test_herb_list_single_product(self, nexus_fs_herb):
        """List a single product directory: 10 docs."""
        result = nexus_fs_herb.sys_readdir(
            "/workspace/enterprise-context/NanoSynth/", recursive=False, details=False
        )
        assert len(result) == 10
        assert all("NanoSynth" in p for p in result)

    @pytest.mark.asyncio
    async def test_herb_list_products_nonrecursive(self, nexus_fs_herb):
        """Non-recursive listing of enterprise-context/ shows product sub-paths.

        Raft metastore non-recursive filter keeps only entries at the target depth.
        Since only files exist (at depth 5), depth-4 non-recursive listing returns
        the files whose path prefix matches — products appear as implicit dir entries.
        """
        result = nexus_fs_herb.sys_readdir(
            "/workspace/enterprise-context/", recursive=True, details=False
        )
        # Recursive should see all 300 files
        assert len(result) == 300
        products_seen = set()
        for p in result:
            parts = p.replace("/workspace/enterprise-context/", "").split("/")
            if parts[0]:
                products_seen.add(parts[0])
        assert len(products_seen) == 30, f"Expected 30 products, got {len(products_seen)}"

    @pytest.mark.asyncio
    async def test_herb_list_details(self, nexus_fs_herb):
        """Detail listing of HERB dir returns metadata dicts."""
        result = nexus_fs_herb.sys_readdir(
            "/workspace/enterprise-context/NanoSynth/",
            recursive=False,
            details=True,
        )
        assert len(result) == 10
        for r in result:
            assert "path" in r
            assert "size" in r
            assert r["size"] > 0  # Files have content
            assert "entry_type" in r

    @pytest.mark.asyncio
    async def test_herb_list_performance(self, nexus_fs_herb):
        """Listing 300 HERB files should be fast."""
        # Warmup
        nexus_fs_herb.sys_readdir("/workspace/enterprise-context/", recursive=True, details=False)

        t0 = time.perf_counter()
        result = nexus_fs_herb.sys_readdir(
            "/workspace/enterprise-context/", recursive=True, details=False
        )
        elapsed_ms = (time.perf_counter() - t0) * 1000

        print(f"  HERB 300 files recursive list: {elapsed_ms:.1f}ms")
        assert len(result) == 300
        assert elapsed_ms < 5000, f"HERB listing took {elapsed_ms:.0f}ms (expected <5000ms)"

    @pytest.mark.asyncio
    async def test_herb_nonrecursive_details_performance(self, nexus_fs_herb):
        """Non-recursive detail listing of enterprise-context/ should use batch implicit-dir."""
        # Warmup
        nexus_fs_herb.sys_readdir("/workspace/enterprise-context/", recursive=False, details=True)

        t0 = time.perf_counter()
        result = nexus_fs_herb.sys_readdir(
            "/workspace/enterprise-context/", recursive=False, details=True
        )
        elapsed_ms = (time.perf_counter() - t0) * 1000

        print(f"  HERB non-recursive details: {elapsed_ms:.1f}ms, {len(result)} entries")
        assert elapsed_ms < 5000, f"HERB detail listing took {elapsed_ms:.0f}ms (expected <5000ms)"

    @pytest.mark.asyncio
    async def test_herb_pagination(self, nexus_fs_herb):
        """Paginated listing of HERB files works across all pages."""
        from nexus.core.pagination import PaginatedResult

        all_items = []
        cursor = None
        pages = 0
        while True:
            result = nexus_fs_herb.sys_readdir(
                "/workspace/enterprise-context/",
                recursive=True,
                limit=50,
                cursor=cursor,
            )
            assert isinstance(result, PaginatedResult)
            all_items.extend(result.items)
            pages += 1
            if not result.has_more:
                break
            cursor = result.next_cursor

        print(f"  HERB pagination: {len(all_items)} items across {pages} pages")
        assert len(all_items) == 300
        assert pages == 6  # 300 / 50 = 6 pages


# ============================================================================
# Test 4: Search service integration (if available)
# ============================================================================


class TestSearchServiceE2E:
    """Verify SearchService.list_dir exercises the fixed code paths."""

    @pytest.mark.asyncio
    async def test_search_service_list(self, nexus_fs_herb):
        """SearchService.list() works on HERB corpus."""
        search_svc = nexus_fs_herb.service("search")
        if search_svc is None:
            pytest.skip("SearchService not available in this config")

        result = search_svc.list(
            path="/workspace/enterprise-context/",
            recursive=True,
        )
        assert len(result) == 300

    @pytest.mark.asyncio
    async def test_search_service_glob(self, nexus_fs_herb):
        """SearchService.glob works on HERB corpus."""
        search_svc = nexus_fs_herb.service("search")
        if search_svc is None:
            pytest.skip("SearchService not available in this config")

        result = search_svc.glob(
            "*.md",
            path="/workspace/enterprise-context/NanoSynth/",
        )
        assert len(result) == 10
        assert all(p.endswith(".md") for p in result)

    @pytest.mark.asyncio
    async def test_search_service_grep(self, nexus_fs_herb):
        """SearchService.grep finds content in HERB files."""
        search_svc = nexus_fs_herb.service("search")
        if search_svc is None:
            pytest.skip("SearchService not available in this config")

        result = await search_svc.grep(
            "NanoSynth",
            path="/workspace/enterprise-context/",
        )
        # grep returns list[dict]; key varies by strategy ("file" or "path")
        nano_matches = [
            r for r in result if "NanoSynth" in (r.get("file", "") or r.get("path", "") or str(r))
        ]
        assert len(nano_matches) >= 1, (
            f"Expected grep hits for 'NanoSynth', got {len(result)} total results: "
            f"{result[:3] if result else '(empty)'}"
        )
