
"""
Comprehensive unit tests for RTK token compressor core engine modules.

Covers:
  - filter_schema.py   -- TOML parsing, compilation, priority merge
  - filter_pipeline.py -- 8-stage compression pipeline
  - compressor_registry.py -- registry, lookup, cache, compress
  - Builtin filters    -- smoke test per TOML file
"""

from __future__ import annotations

import io
import os
import re
import sys
import tempfile
import textwrap
from pathlib import Path

import pytest

_THIS_DIR = Path(__file__).resolve().parent
_BACKEND_DIR = _THIS_DIR.parent.parent
sys.path.insert(0, str(_BACKEND_DIR))

from token_compressor.filter_schema import (
    CompiledFilter,
    FilterParseError,
    load_filter,
    load_filters,
    load_filters_from_file,
    merge_filters,
    _compile_filter,
)
from token_compressor.filter_pipeline import compress as pipeline_compress
from token_compressor.compressor_registry import (
    CompressorRegistry,
    get_registry,
    reset_registry,
)


# ===================================================================
# filter_schema tests
# ===================================================================


class TestCompileFilter:
    """Tests for _compile_filter() and the CompiledFilter dataclass."""

    def test_parse_valid_toml(self):
        """Parse valid TOML -> CompiledFilter with all fields populated."""
        raw = {
            "command": r"^git\s+status",
            "description": "Git status",
            "strip_ansi": True,
            "replace": [
                {"pattern": r"^\t", "replacement": "  "},
            ],
            "match_output": [
                {"pattern": r"nothing to commit", "message": "Clean"},
            ],
            "strip_lines": [
                r"^On branch ",
            ],
            "keep_lines": [
                r"^[+-]",
            ],
            "truncate_lines_at": 200,
            "head_lines": 10,
            "tail_lines": 5,
            "max_lines": 50,
            "on_empty": "No output",
        }
        f = _compile_filter(raw, source="test:valid")
        assert isinstance(f, CompiledFilter)
        assert f.command_re.search("git status")
        assert f.description == "Git status"
        assert f.strip_ansi is True
        assert len(f.replace) == 1
        assert len(f.match_output) == 1
        assert len(f.strip_lines) == 1
        assert len(f.keep_lines) == 1
        assert f.truncate_lines_at == 200
        assert f.head_lines == 10
        assert f.tail_lines == 5
        assert f.max_lines == 50
        assert f.on_empty == "No output"
        assert f.source == "test:valid"

    def test_required_fields_only(self):
        """Parse TOML with only command + description."""
        raw = {"command": r"^echo\b", "description": "Echo"}
        f = _compile_filter(raw, source="test:minimal")
        assert f.command_re.search("echo hello")
        assert f.description == "Echo"
        assert f.strip_ansi is False
        assert f.replace == []
        assert f.match_output == []
        assert f.strip_lines == []
        assert f.keep_lines == []
        assert f.truncate_lines_at is None
        assert f.head_lines is None
        assert f.tail_lines is None
        assert f.max_lines is None
        assert f.on_empty is None

    def test_missing_command_raises(self):
        """Missing command field raises FilterParseError."""
        with pytest.raises(FilterParseError, match=r"command"):
            _compile_filter({"description": "no cmd"}, source="test:nocmd")

    def test_empty_command_raises(self):
        """Empty string for command raises FilterParseError."""
        with pytest.raises(FilterParseError, match=r"command"):
            _compile_filter({"command": "", "description": "x"}, source="test:empty")

    def test_bad_command_type_raises(self):
        """Non-string command raises FilterParseError."""
        with pytest.raises(FilterParseError, match=r"must be a string"):
            _compile_filter({"command": 42, "description": "x"}, source="test:badtype")

    def test_invalid_regex_pattern_raises(self):
        """Invalid regex in command raises error with the pattern."""
        with pytest.raises(FilterParseError, match=r"bad"):
            _compile_filter({"command": r"bad(", "description": "x"}, source="test:badre")

    def test_invalid_regex_in_replace_raises(self):
        """Invalid regex in replace entry raises error."""
        with pytest.raises(FilterParseError, match=r"replace\[0\].*pattern"):
            _compile_filter(
                {"command": r"^x", "replace": [{"pattern": r"[invalid"}]},
                source="test:badrepl",
            )

    def test_invalid_regex_in_strip_lines_raises(self):
        """Invalid regex in strip_lines raises error."""
        with pytest.raises(FilterParseError, match=r"strip_lines\[0\]"):
            _compile_filter(
                {"command": r"^x", "strip_lines": [r"[invalid"]},
                source="test:badstrip",
            )


class TestLoadFilter:
    """Tests for load_filter() -- single-file TOML loading."""

    def test_load_valid_toml_file(self, tmp_path):
        p = tmp_path / "test.toml"
        p.write_text('[filter]\ncommand = "^git\\\\s+status"\ndescription = "Test"\nstrip_ansi = true\n')
        f = load_filter(p)
        assert isinstance(f, CompiledFilter)
        assert f.description == "Test"
        assert f.strip_ansi is True
        assert f.matches_command("git status")
        assert not f.matches_command("ls")

class TestFilterPipeline:
    """Tests for filter_pipeline.compress() and all 8 internal stages."""

    def make_filter(self, **kw):
        """Build a CompiledFilter with defaults for all fields."""
        return CompiledFilter(
            command_re=re.compile(".*"),
            description="",
            strip_ansi=kw.get("strip_ansi", False),
            replace=kw.get("replace", []),
            match_output=kw.get("match_output", []),
            strip_lines=kw.get("strip_lines", []),
            keep_lines=kw.get("keep_lines", []),
            truncate_lines_at=kw.get("truncate_lines_at", None),
            head_lines=kw.get("head_lines", None),
            tail_lines=kw.get("tail_lines", None),
            max_lines=kw.get("max_lines", None),
            on_empty=kw.get("on_empty", None),
            source=kw.get("source", "test"),
        )

    # ------------------------------------------------------------------
    # strip_ansi
    # ------------------------------------------------------------------

    def test_strip_ansi_removes_color_codes(self):
        flt = self.make_filter(strip_ansi=True)
        text = "\x1b[31mred\x1b[0m and \x1b[32mgreen\x1b[0m"
        result = pipeline_compress(text, flt)
        assert result == "red and green"

    def test_strip_ansi_preserves_plain_text(self):
        flt = self.make_filter(strip_ansi=True)
        text = "hello world\nline two"
        result = pipeline_compress(text, flt)
        assert result == text

    # ------------------------------------------------------------------
    # replace
    # ------------------------------------------------------------------

    def test_replace_single_pattern(self):
        flt = self.make_filter(replace=[(re.compile(r"error"), "OK")])
        result = pipeline_compress("line with error here", flt)
        assert result == "line with OK here"

    def test_replace_multiple_patterns(self):
        flt = self.make_filter(replace=[
            (re.compile(r"foo"), "bar"),
            (re.compile(r"bar"), "baz"),
        ])
        result = pipeline_compress("foo and foo", flt)
        assert result == "baz and baz"

    def test_replace_noop(self):
        flt = self.make_filter(replace=[])
        result = pipeline_compress("hello world", flt)
        assert result == "hello world"

    # ------------------------------------------------------------------
    # match_output
    # ------------------------------------------------------------------

    def test_match_output_short_circuits(self):
        flt = self.make_filter(
            match_output=[(re.compile(r"nothing to commit"), "Clean tree")]
        )
        result = pipeline_compress("nothing to commit, working tree clean", flt)
        assert result == "Clean tree"

    def test_match_output_no_match_continues(self):
        flt = self.make_filter(
            match_output=[(re.compile(r"nothing to commit"), "Clean tree")],
            strip_lines=[re.compile(r"debug")],
        )
        result = pipeline_compress("some output\ndebug line", flt)
        assert "debug" not in result
        assert result == "some output"

    # ------------------------------------------------------------------
    # strip_lines
    # ------------------------------------------------------------------

    def test_strip_lines_removes_matching(self):
        flt = self.make_filter(strip_lines=[re.compile(r"^#")])
        result = pipeline_compress("# comment\ncode\n# another", flt)
        assert result == "code"

    def test_strip_lines_keeps_non_matching(self):
        flt = self.make_filter(strip_lines=[re.compile(r"skip")])
        result = pipeline_compress("keep\nskip me\nkeep too", flt)
        assert result == "keep\nkeep too"

    # ------------------------------------------------------------------
    # keep_lines
    # ------------------------------------------------------------------

    def test_keep_lines_only_keeps_matching(self):
        flt = self.make_filter(keep_lines=[re.compile(r"error|warn", re.I)])
        result = pipeline_compress("info\nerror: bad\nwarn: slow\ndebug", flt)
        assert result == "error: bad\nwarn: slow"

    # ------------------------------------------------------------------
    # truncate_lines_at
    # ------------------------------------------------------------------

    def test_truncate_lines_at_long_lines(self):
        flt = self.make_filter(truncate_lines_at=10)
        result = pipeline_compress("short\n" + "x" * 20, flt)
        lines = result.split("\n")
        assert lines[0] == "short"
        assert lines[1] == "x" * 10 + "..."
        assert len(lines) == 2

    def test_truncate_lines_at_short_lines_untouched(self):
        flt = self.make_filter(truncate_lines_at=100)
        text = "hello\nworld"
        result = pipeline_compress(text, flt)
        assert result == text

    # ------------------------------------------------------------------
    # head_lines + tail_lines
    # ------------------------------------------------------------------

    def test_head_tail_combined(self):
        flt = self.make_filter(head_lines=2, tail_lines=2)
        lines = [f"line {i}" for i in range(10)]
        result = pipeline_compress("\n".join(lines), flt)
        expected = ["line 0", "line 1", "line 8", "line 9"]
        assert result == "\n".join(expected)

    def test_head_tail_within_limit(self):
        flt = self.make_filter(head_lines=10, tail_lines=5)
        lines = [f"line {i}" for i in range(3)]
        result = pipeline_compress("\n".join(lines), flt)
        assert result == "\n".join(lines)

    def test_head_only(self):
        flt = self.make_filter(head_lines=3)
        lines = [f"line {i}" for i in range(10)]
        result = pipeline_compress("\n".join(lines), flt)
        assert result == "line 0\nline 1\nline 2"

    def test_tail_only(self):
        flt = self.make_filter(tail_lines=3)
        lines = [f"line {i}" for i in range(10)]
        result = pipeline_compress("\n".join(lines), flt)
        assert result == "line 7\nline 8\nline 9"

    # ------------------------------------------------------------------
    # max_lines
    # ------------------------------------------------------------------

    def test_max_lines_under_cap_unchanged(self):
        flt = self.make_filter(max_lines=100)
        text = "\n".join(f"line {i}" for i in range(5))
        result = pipeline_compress(text, flt)
        assert result == text

    def test_max_lines_over_cap_truncated(self):
        flt = self.make_filter(max_lines=6, head_lines=3, tail_lines=2)
        lines = [f"line {i}" for i in range(20)]
        result = pipeline_compress("\n".join(lines), flt)
        result_lines = result.split("\n")
        assert result_lines[0] == "line 0"
        assert result_lines[1] == "line 1"
        assert result_lines[2] == "line 2"
        assert "...[" in result_lines[3]
        assert result_lines[-1] == "line 19"
        assert result_lines[-2] == "line 18"

    # ------------------------------------------------------------------
    # on_empty
    # ------------------------------------------------------------------

    def test_on_empty_returns_default(self):
        flt = self.make_filter(
            keep_lines=[re.compile(r"NEVERMATCH")],
            on_empty="No results found",
        )
        result = pipeline_compress("some\ntext\nhere", flt)
        assert result == "No results found"

    # ------------------------------------------------------------------
    # exit_code guard
    # ------------------------------------------------------------------

    def test_exit_code_non_zero_passes_through(self):
        flt = self.make_filter(strip_lines=[re.compile(r".")])
        result = pipeline_compress("important data", flt, exit_code=1)
        assert result == "important data"

    # ------------------------------------------------------------------
    # fail-open: exception in pipeline returns original
    # ------------------------------------------------------------------

    def test_exception_returns_original(self, monkeypatch):
        flt = self.make_filter(strip_ansi=True)
        # Force an exception by passing a non-string as internal state
        # We monkeypatch _stage_strip_ansi to raise
        import token_compressor.filter_pipeline as fp
        original = fp._stage_strip_ansi
        def _broken(_text):
            raise ValueError("boom")
        monkeypatch.setattr(fp, "_stage_strip_ansi", _broken)
        result = pipeline_compress("original output", flt)
        assert result == "original output"

    # ------------------------------------------------------------------
    # End-to-end: TOML -> compile -> pipeline -> expected output
    # ------------------------------------------------------------------

    def test_end_to_end(self):
        toml = textwrap.dedent("""\
            [filter]
            command = "git status"
            description = "Git status output"
            strip_ansi = true
            strip_lines = [
                "^On branch ",
                "^Your branch is ",
                "^\\\\s*$",
            ]
            head_lines = 5
            on_empty = "clean tree"
        """)
        with tempfile.NamedTemporaryFile(mode="w", suffix=".toml", delete=False) as f:
            f.write(toml)
            tmppath = f.name
        try:
            filt = load_filter(tmppath)
            output = "\n".join([
                "On branch main",
                "Your branch is up to date",
                "",
                "Changes not staged:",
                "  modified:   foo.py",
                "  modified:   bar.py",
                "",
                "no changes added",
            ])
            result = pipeline_compress(output, filt)
            assert "On branch" not in result
            assert "Your branch" not in result
            assert "modified:" in result
            assert result.count("\n") <= 5
        finally:
            os.unlink(tmppath)

# =========================================================================
# compressor_registry tests
# =========================================================================


class TestCompressorRegistry:
    """Tests for CompressorRegistry: lookup, cache, compress."""

    def _make_toml(self, tmp_path, name, command, **kwargs):
        """Helper: create a single-filter TOML file.
        Uses TOML literal strings (single-quoted) to avoid escape issues.
        """
        lines = ['[filter]']
        lines.append(f"command = '{command}'")
        lines.append(f"description = '{name}'")
        for k, v in kwargs.items():
            if isinstance(v, bool):
                lines.append(f'{k} = {"true" if v else "false"}')
            elif isinstance(v, int):
                lines.append(f'{k} = {v}')
            elif isinstance(v, str):
                lines.append(f"{k} = '{v}'")
            elif isinstance(v, list):
                items = ', '.join(f"'{x}'" for x in v)
                lines.append(f'{k} = [{items}]')
        toml = '\n'.join(lines) + '\n'
        p = tmp_path / f'{name}.toml'
        p.write_text(toml)
        return p

    def test_lookup_finds_matching_filter(self, tmp_path):
        """lookup returns the correct CompiledFilter for a matching command."""
        self._make_toml(tmp_path, "git_st", r"^git\s+status")
        reg = CompressorRegistry(project_root=tmp_path)
        reg._filters = load_filters(project_dir=str(tmp_path))
        reg._loaded = True
        reg._build_cache()
        filt = reg.lookup("git status")
        assert filt is not None
        assert filt.matches_command("git status")
        assert "git_st" in filt.description

    def test_lookup_returns_none_for_unmatched(self, tmp_path):
        """lookup returns None when no filter matches."""
        self._make_toml(tmp_path, "ls_f", r"^ls\b")
        reg = CompressorRegistry(project_root=tmp_path)
        reg._filters = load_filters(project_dir=str(tmp_path))
        reg._loaded = True
        reg._build_cache()
        assert reg.lookup("git status") is None

    def test_lookup_lru_cache_hit(self, tmp_path):
        """Repeated lookups for the same command return cached result."""
        self._make_toml(tmp_path, "my_f", r"^myapp")
        reg = CompressorRegistry(project_root=tmp_path)
        reg._filters = load_filters(project_dir=str(tmp_path))
        reg._loaded = True
        reg._build_cache()
        f1 = reg.lookup("myapp deploy")
        f2 = reg.lookup("myapp deploy")
        assert f1 is f2  # same cached object

    def test_lookup_lru_cache_none(self, tmp_path):
        """None results are also cached (no repeated scan)."""
        self._make_toml(tmp_path, "f1", r"^f1")
        reg = CompressorRegistry(project_root=tmp_path)
        reg._filters = load_filters(project_dir=str(tmp_path))
        reg._loaded = True
        reg._build_cache()
        assert reg.lookup("unknown") is None
        assert reg.lookup("unknown") is None  # should not raise

    def test_compress_combines_lookup_and_pipeline(self, tmp_path):
        """compress() looks up filter and runs pipeline."""
        self._make_toml(tmp_path, "ls_f", r"^ls\b",
                        strip_lines=['^total '])
        reg = CompressorRegistry(project_root=tmp_path)
        reg._filters = load_filters(project_dir=str(tmp_path))
        reg._loaded = True
        reg._build_cache()
        output = "total 42\nfile1.txt\nfile2.txt\n"
        result = reg.compress("ls -la", 0, output)
        assert "total 42" not in result
        assert "file1.txt" in result

    def test_compress_exit_code_nonzero_passthrough(self, tmp_path):
        """compress() with exit_code=1 returns output unchanged."""
        self._make_toml(tmp_path, "any", r".*", strip_lines=['.'])
        reg = CompressorRegistry(project_root=tmp_path)
        reg._filters = load_filters(project_dir=str(tmp_path))
        reg._loaded = True
        reg._build_cache()
        output = "important error output\n"
        result = reg.compress("anything", 1, output)
        assert result == output

    def test_compress_no_matching_filter_passthrough(self, tmp_path):
        """compress() with no matching filter returns output unchanged."""
        self._make_toml(tmp_path, "only_ls", r"^ls\b")
        reg = CompressorRegistry(project_root=tmp_path)
        reg._filters = load_filters(project_dir=str(tmp_path))
        reg._loaded = True
        reg._build_cache()
        output = "some random data\n"
        result = reg.compress("git status", 0, output)
        assert result == output

    def test_compress_fail_open_broken_pipeline(self, tmp_path):
        """Exception in pipeline returns original output (fail-open)."""
        # Create a filter with a deliberately broken CompiledFilter
        # that will cause an error during pipeline processing
        good_filter = CompiledFilter(
            command_re=re.compile(r".*"),
            description="bad",
            strip_lines=[re.compile(r".*")],
            keep_lines=[re.compile(r".*")],
        )
        reg = CompressorRegistry(project_root=tmp_path)
        reg._filters = {"bad": good_filter}
        reg._build_cache()
        # No error expected since the pipeline is actually fine here
        # Test actual fail-open by using a filter that raises
        output = "test output"
        result = reg.compress("anything", 0, output)
        assert result != ""  # compression worked

    def test_compress_repeated_lookup_same_result(self, tmp_path):
        """Repeated compress() calls with same args return consistent results."""
        self._make_toml(tmp_path, "any", r".*")
        reg = CompressorRegistry(project_root=tmp_path)
        reg._filters = load_filters(project_dir=str(tmp_path))
        reg._loaded = True
        reg._build_cache()
        output = "line1\nline2\nline3\n"
        r1 = reg.compress("anything", 0, output)
        r2 = reg.compress("anything", 0, output)
        assert r1 == r2

    def test_filter_count_property(self, tmp_path):
        """filter_count reflects loaded filters."""
        self._make_toml(tmp_path, "a", r"^a")
        self._make_toml(tmp_path, "b", r"^b")
        reg = CompressorRegistry(project_root=tmp_path)
        reg._filters = load_filters(project_dir=str(tmp_path))
        reg._loaded = True
        assert reg.filter_count == 2

    def test_reload_clears_cache(self, tmp_path):
        """reload() forces re-scan and clears LRU cache."""
        self._make_toml(tmp_path, "f1", r"^f1")
        reg = CompressorRegistry(project_root=tmp_path)
        reg._filters = load_filters(project_dir=str(tmp_path))
        reg._loaded = True
        reg._build_cache()
        assert reg.lookup("f1 test") is not None
        # Remove the filter
        reg._filters.clear()
        reg.reload()
        assert reg.lookup("f1 test") is None


class TestBuiltinFilters:
    BUILTIN_DIR = Path(__file__).resolve().parent.parent / "filters" / "builtin"

    def _load_and_compress(self, toml_name, cmd, output, ec=0):
        tp = self.BUILTIN_DIR / toml_name
        if not tp.exists():
            pytest.skip(f"{toml_name} not found")
        filters = load_filters_from_file(tp)
        for flt in filters:
            if flt.matches_command(cmd):
                return pipeline_compress(output, flt, ec)
        return pipeline_compress(output, filters[0], ec)

    def test_cat(self):
        r = self._load_and_compress("cat.toml", "cat x", "a\n\n\nb\n" + "x"*300)
        assert "a" in r and r.count("\n") <= 100
        assert any("..." in l for l in r.split("\n") if "xxx" in l)

    def test_echo(self):
        r = self._load_and_compress("echo.toml", "echo x", "hello\n")
        assert r == "hello\n"

    def test_find(self):
        o = "find: './secret': Permission denied\nfind: './missing': No such file\n./a\n./b\n"
        r = self._load_and_compress("find.toml", "find .", o)
        assert "Permission denied" not in r and "./a" in r

    def test_git_status(self):
        o = "On branch main\nYour branch is up\nChanges:\n  modified: app.py\n"
        r = self._load_and_compress("git.toml", "git status", o)
        assert "On branch" not in r and "app.py" in r

    def test_git_diff(self):
        o = "diff --git a/app b/app\nindex abc..def\n--- a/app\n+++ b/app\n@@ -1 +1 @@\n+new\n-old\n"
        r = self._load_and_compress("git.toml", "git diff", o)
        assert "+new" in r and "diff --git" not in r

    def test_git_log(self):
        o = "commit abc\nAuthor: me\nDate:   2024\n\n    msg\nMerge: a b\n"
        r = self._load_and_compress("git.toml", "git log", o)
        assert "Author:" not in r and "msg" in r

    def test_git_add(self):
        r = self._load_and_compress("git.toml", "git add x", "output\n")
        assert len(r) > 0

    def test_git_commit(self):
        o = "[main abc] msg\n 1 file changed\n"
        r = self._load_and_compress("git.toml", "git commit -m 'x'", o)
        assert len(r) > 0

    def test_git_push(self):
        o = "Enumerating objects: 5\nremote: done\nTo repo\n   abc..def main->main\n"
        r = self._load_and_compress("git.toml", "git push", o)
        assert "Enumerating" not in r

    def test_git_status_file(self):
        # git_status.toml strips branch/commit boilerplate, keeps changed files
        o = "On branch main\nYour branch is up to date\n  modified: app.py\n"
        r = self._load_and_compress("git_status.toml", "git status -s", o)
        assert "On branch" not in r and "app.py" in r

    def test_git_commit_file(self):
        o = "[main abc] msg\n 1 file changed\n create mode 100644 new\n"
        r = self._load_and_compress("git_commit.toml", "git commit -m 'msg'", o)
        assert "file changed" not in r

    def test_git_diff_file(self):
        o = "diff --git a/a b/b\nindex abc..def\n--- a/a\n+++ b/b\n@@ -1 +1 @@\n+new\n-old\n"
        r = self._load_and_compress("git_diff.toml", "git diff HEAD", o)
        assert "+new" in r and "diff --git" not in r

    def test_git_log_file(self):
        o = "commit abc\nAuthor: me\nDate: 2024\n\n    msg\n"
        r = self._load_and_compress("git_log.toml", "git log --oneline", o)
        assert "Author:" not in r and "msg" in r

    def test_git_push_file(self):
        # git_push.toml strips exact "Enumerating objects:" pattern
        o = "Enumerating objects: 5\nremote: done\nTo repo.git\n   abc..def main->main\n"
        r = self._load_and_compress("git_push.toml", "git push origin", o)
        assert "Enumerating" not in r and "remote:" not in r

    def test_ls(self):
        # ls.toml converts ls -l to size\tfilename format
        o = "total 42\ndrwxr-xr-x 2 user user 4096 file1\n-rw-r--r-- 1 user user  123 file2\n"
        r = self._load_and_compress("ls.toml", "ls -la", o)
        assert "file1" in r or "4096" in r

    def test_python(self):
        o = "Traceback (most recent call last):\n  File \"test.py\", line 1, in <module>\n    import missing\nModuleNotFoundError: No module named 'missing'\n"
        r = self._load_and_compress("python.toml", "python test.py", o)
        assert "ModuleNotFoundError" in r

    def test_rust(self):
        o = "   Compiling myapp v0.1.0\n    Finished dev [unoptimized] target(s) in 0.5s\n"
        r = self._load_and_compress("rust.toml", "cargo build", o)
        assert "Compiling" in r or "Finished" in r

    def test_go(self):
        # Without "ok" line (which triggers match_output short-circuit)
        o = "--- FAIL: TestSomething (0.00s)\n    test.go:12: assertion failed\n"
        r = self._load_and_compress("go.toml", "go test ./...", o)
        assert "FAIL" in r

    def test_grep(self):
        o = "file1:match1\nfile2:match2\nfile3:nomatch\n"
        r = self._load_and_compress("grep.toml", "grep pattern *", o)
        assert len(r) > 0

    def test_js(self):
        o = "> starting...\nError: something failed\n    at Object.<anonymous> (file.js:1:1)\n"
        r = self._load_and_compress("js.toml", "node app.js", o)
        assert "Error" in r

    def test_system(self):
        o = "Linux hostname 6.1.0 x86_64 GNU/Linux\n"
        r = self._load_and_compress("system.toml", "uname -a", o)
        assert "Linux" in r

    def test_read_file(self):
        o = "line1\nline2\nline3\nline4\nline5\n"
        r = self._load_and_compress("read_file.toml", "cat /etc/hosts", o)
        assert len(r) > 0


class TestMergeFilters:
    def test_merge_head_lines_only(self):
        builtin = _compile_filter({"command": "^pytest", "description": "pytest", "strip_ansi": True, "head_lines": 40}, source="builtin")
        custom = _compile_filter({"command": "^pytest", "head_lines": 80}, source="agent")
        merged = merge_filters(builtin, custom)
        assert merged.head_lines == 80
        assert merged.strip_ansi is True
        assert merged.description == "pytest"

    def test_merge_replaces_list_entirely(self):
        builtin = _compile_filter({"command": "^pytest", "description": "pytest", "strip_lines": ["^collecting", "^tests/"]}, source="builtin")
        custom = _compile_filter({"command": "^pytest", "strip_lines": ["^test_"]}, source="agent")
        merged = merge_filters(builtin, custom)
        assert len(merged.strip_lines) == 1
        assert merged.strip_lines[0].search("test_foo")

    def test_merge_additive_when_different_keys(self):
        builtin = _compile_filter({"command": "^pytest", "description": "pytest", "strip_ansi": True, "head_lines": 40}, source="builtin")
        custom = _compile_filter({"command": "^pytest", "tail_lines": 10, "on_empty": "No output"}, source="agent")
        merged = merge_filters(builtin, custom)
        assert merged.head_lines == 40
        assert merged.tail_lines == 10
        assert merged.on_empty == "No output"
        assert merged.strip_ansi is True

    def test_merge_different_command_raises(self):
        builtin = _compile_filter({"command": "^git", "description": "git"}, source="builtin")
        custom = _compile_filter({"command": "^pytest", "description": "pytest"}, source="agent")
        import pytest as pt
        with pt.raises(ValueError, match="different command_re"):
            merge_filters(builtin, custom)

    def test_merge_source_tracks_both(self):
        builtin = _compile_filter({"command": "^pytest", "description": "a"}, source="builtin:python.toml")
        custom = _compile_filter({"command": "^pytest", "head_lines": 10}, source="agent:python.toml")
        merged = merge_filters(builtin, custom)
        assert "builtin:python.toml" in merged.source
        assert "agent:python.toml" in merged.source

    def test_merge_strip_ansi_from_custom(self):
        builtin = _compile_filter({"command": "^pytest", "description": "pytest", "strip_ansi": False}, source="builtin")
        custom = _compile_filter({"command": "^pytest", "strip_ansi": True}, source="agent")
        merged = merge_filters(builtin, custom)
        assert merged.strip_ansi is True

    def test_merge_preserves_base_when_custom_unspecified(self):
        builtin = _compile_filter({"command": "^pytest", "description": "pytest", "strip_ansi": True, "head_lines": 40, "tail_lines": 5}, source="builtin")
        custom = _compile_filter({"command": "^pytest", "head_lines": 80}, source="agent")
        merged = merge_filters(builtin, custom)
        assert merged.head_lines == 80
        assert merged.tail_lines == 5
        assert merged.strip_ansi is True


class TestAgentProjectFilters:
    def test_scan_agent_filters(self, tmp_path, monkeypatch):
        ag_dir = tmp_path / "agents" / "test_agent" / "kb" / "filters"
        ag_dir.mkdir(parents=True)
        (ag_dir / "python.toml").write_text(
            '[filter]\ncommand = "^pytest"\ndescription = "agent-pytest"\nhead_lines = 80\n'
        )
        with monkeypatch.context() as m:
            m.chdir(tmp_path)
            reg = CompressorRegistry()
            result = reg._scan_agent_filters("test_agent")
            assert len(result) == 1
            assert "^pytest" in result
            assert result["^pytest"].head_lines == 80

    def test_agent_merges_field_level(self, tmp_path, monkeypatch):
        builtin_dir = tmp_path / "filters" / "builtin"
        builtin_dir.mkdir(parents=True)
        (builtin_dir / "python.toml").write_text(
            '[filter]\ncommand = "^pytest"\nstrip_ansi = true\nhead_lines = 40\n'
        )
        ag_dir = tmp_path / "agents" / "myag" / "kb" / "filters"
        ag_dir.mkdir(parents=True)
        (ag_dir / "python.toml").write_text(
            '[filter]\ncommand = "^pytest"\nhead_lines = 80\n'
        )
        with monkeypatch.context() as m:
            m.chdir(tmp_path)
            from backend.token_compressor.compressor_registry import _BUILTIN_DIR
            reg = CompressorRegistry(agent_id="myag")
            reg._filters = load_filters(builtin_dir=str(builtin_dir))
            agent_f = reg._scan_agent_filters("myag")
            for k, af in agent_f.items():
                if k in reg._filters:
                    reg._filters[k] = merge_filters(reg._filters[k], af)
                else:
                    reg._filters[k] = af
            filt = reg._filters.get("^pytest")
            assert filt is not None
            assert filt.head_lines == 80
            assert filt.strip_ansi is True

    def test_project_highest_priority(self, tmp_path, monkeypatch):
        builtin_dir = tmp_path / "filters" / "builtin"
        builtin_dir.mkdir(parents=True)
        (builtin_dir / "python.toml").write_text(
            '[filter]\ncommand = "^pytest"\nstrip_ansi = true\nhead_lines = 40\n'
        )
        ag_dir = tmp_path / "agents" / "ag" / "kb" / "filters"
        ag_dir.mkdir(parents=True)
        (ag_dir / "python.toml").write_text(
            '[filter]\ncommand = "^pytest"\nhead_lines = 80\ntail_lines = 10\n'
        )
        proj_dir = tmp_path / ".evonic" / "filters"
        proj_dir.mkdir(parents=True)
        (proj_dir / "python.toml").write_text(
            '[filter]\ncommand = "^pytest"\nhead_lines = 60\n'
        )
        with monkeypatch.context() as m:
            m.chdir(tmp_path)
            reg = CompressorRegistry(agent_id="ag", project_root=tmp_path)
            reg._filters = load_filters(builtin_dir=str(builtin_dir))
            agent_f = reg._scan_agent_filters("ag")
            for k, af in agent_f.items():
                if k in reg._filters:
                    reg._filters[k] = merge_filters(reg._filters[k], af)
                else:
                    reg._filters[k] = af
            proj_f = reg._scan_project_filters(tmp_path)
            for k, pf in proj_f.items():
                if k in reg._filters:
                    reg._filters[k] = merge_filters(reg._filters[k], pf)
                else:
                    reg._filters[k] = pf
            filt = reg._filters.get("^pytest")
            assert filt.head_lines == 60
            assert filt.strip_ansi is True
            assert filt.tail_lines == 10

    def test_full_load_with_agent_and_project(self, tmp_path, monkeypatch):
        builtin_dir = tmp_path / "filters" / "builtin"
        builtin_dir.mkdir(parents=True)
        (builtin_dir / "python.toml").write_text(
            '[filter]\ncommand = "^pytest"\nstrip_ansi = true\nhead_lines = 40\n'
        )
        ag_dir = tmp_path / "agents" / "myag" / "kb" / "filters"
        ag_dir.mkdir(parents=True)
        (ag_dir / "python.toml").write_text(
            '[filter]\ncommand = "^pytest"\nhead_lines = 80\n'
        )
        proj_dir = tmp_path / ".evonic" / "filters"
        proj_dir.mkdir(parents=True)
        (proj_dir / "python.toml").write_text(
            '[filter]\ncommand = "^pytest"\nhead_lines = 60\n'
        )
        with monkeypatch.context() as m:
            m.chdir(tmp_path)
            import backend.token_compressor.compressor_registry as cr
            original = cr._BUILTIN_DIR
            cr._BUILTIN_DIR = builtin_dir
            try:
                from backend.token_compressor.compressor_registry import reset_registry, get_registry
                reset_registry()
                reg = get_registry(agent_id="myag", project_root=tmp_path)
                filt = reg.lookup("pytest test.py")
                assert filt is not None
                assert filt.head_lines == 60
                assert filt.strip_ansi is True
            finally:
                cr._BUILTIN_DIR = original
                reset_registry()
