from __future__ import annotations

import argparse
import json
import sys
from pathlib import Path

import pytest


REPO_ROOT = Path(__file__).resolve().parents[3]
BENCHMARKING_DIR = REPO_ROOT / "benchmarking"
if str(BENCHMARKING_DIR) not in sys.path:
    sys.path.insert(0, str(BENCHMARKING_DIR))

from run_amb_cortex import (  # noqa: E402
    CASE_ERROR_FILENAME,
    _build_matrix_run_args,
    _collect_matrix_case_result,
    _execute_matrix_case,
    _load_matrix_cases,
    run_cadence,
    run_matrix,
)

_PRACTICAL_NON_LONGMEM_DATASETS = {
    "locomo",
    "lifebench",
    "membench",
    "personamem",
    "memsim",
    "beam",
}


def _base_args() -> argparse.Namespace:
    return argparse.Namespace(
        mode="rag",
        category=None,
        query_limit=None,
        query_id=None,
        doc_limit=None,
        oracle=False,
        description=None,
        run_name_prefix="matrix",
        token_gate_mode="dynamic",
        provider_profile="auto",
        baseline_file="benchmarking/configs/token-gate-baselines.json",
        disable_baseline_gates=False,
        no_auto_tighten_baseline=False,
        min_queries_for_baseline_update=20,
        baseline_token_headroom_pct=0.08,
        baseline_accuracy_headroom=0.02,
        recall_budget=300,
        quality_token_target="custom",
        retrieval_profile="max-quality",
        min_accuracy=0.9,
        max_recall_tokens=300,
        max_avg_recall_tokens=300.0,
        allow_missing_recall_metrics=False,
        no_enforce_gate=False,
        start_index=1,
        max_cases=None,
        max_runtime_seconds=1200,
        max_case_runtime_seconds=900,
        dry_run=False,
        continue_on_error=False,
        matrix_file=None,
        summary_file=None,
    )


def _base_cadence_args() -> argparse.Namespace:
    args = _base_args()
    args.matrix_files = []
    args.max_matrices = None
    return args


def test_load_matrix_cases_accepts_cases_object(tmp_path: Path) -> None:
    matrix_path = tmp_path / "matrix.json"
    matrix_path.write_text(
        json.dumps(
            {
                "cases": [
                    {
                        "dataset": "longmemeval",
                        "split": "s",
                        "query_limit": 20,
                        "retrieval_profile": "balanced",
                    },
                    {"dataset": "membench", "split": "FirstAgentLowLevel"},
                ]
            }
        ),
        encoding="utf-8",
    )
    cases = _load_matrix_cases(matrix_path)

    assert len(cases) == 2
    assert cases[0]["dataset"] == "longmemeval"
    assert cases[0]["split"] == "s"
    assert cases[0]["query_limit"] == 20
    assert cases[0]["retrieval_profile"] == "balanced"
    assert str(cases[0]["id"]).startswith("01-")
    assert cases[1]["dataset"] == "membench"
    assert str(cases[1]["id"]).startswith("02-")


def test_load_matrix_cases_rejects_invalid_schema(tmp_path: Path) -> None:
    matrix_path = tmp_path / "invalid.json"
    matrix_path.write_text(json.dumps({"cases": [{"split": "s"}]}), encoding="utf-8")

    with pytest.raises(ValueError, match="dataset"):
        _load_matrix_cases(matrix_path)


def test_build_matrix_run_args_applies_case_overrides() -> None:
    args = _base_args()
    case = {
        "id": "locomo-locomo10",
        "dataset": "locomo",
        "split": "locomo10",
        "mode": "rag",
        "query_limit": 12,
        "oracle": True,
        "recall_budget": 420,
        "quality_token_target": "lean-detail",
        "retrieval_profile": "token-saver",
    }
    run_args = _build_matrix_run_args(args, case)

    assert run_args.dataset == "locomo"
    assert run_args.split == "locomo10"
    assert run_args.query_limit == 12
    assert run_args.oracle is True
    assert run_args.recall_budget == 420
    assert run_args.quality_token_target == "lean-detail"
    assert run_args.retrieval_profile == "token-saver"
    assert run_args.run_name == "matrix-locomo-locomo10"


def test_build_matrix_run_args_uses_base_retrieval_profile_by_default() -> None:
    args = _base_args()
    args.retrieval_profile = "balanced"
    case = {
        "id": "longmemeval-s",
        "dataset": "longmemeval",
        "split": "s",
    }

    run_args = _build_matrix_run_args(args, case)

    assert run_args.retrieval_profile == "balanced"


def test_collect_matrix_case_result_reads_summary_and_gate(tmp_path: Path) -> None:
    run_dir = tmp_path / "run"
    run_dir.mkdir(parents=True, exist_ok=True)
    (run_dir / "summary.json").write_text(
        json.dumps({"accuracy": 0.55, "correct": 11, "total_queries": 20}),
        encoding="utf-8",
    )
    (run_dir / "gate-report.json").write_text(
        json.dumps(
            {
                "quality_gate": {"passed": False, "failures": ["accuracy below floor"]},
                "recall_stats": {
                    "avg_recall_tokens": 226.7,
                    "max_recall_tokens": 326,
                    "over_budget_count": 3,
                },
                "score_per_1k_recall_tokens": 41.2844,
                "tradeoff": {
                    "quality_token_target": "balanced-detail",
                    "effective_retrieval_profile": "balanced",
                    "profile_delta": {
                        "delta_vs_token_gate": {
                            "max_recall_tokens": {"absolute": -14.0, "percent": -4.29}
                        }
                    },
                },
            }
        ),
        encoding="utf-8",
    )
    result = _collect_matrix_case_result(
        case={"id": "longmemeval-s", "dataset": "longmemeval", "split": "s"},
        run_dir=run_dir,
        exit_code=1,
        error="quality gate failed",
    )

    assert result["dataset"] == "longmemeval"
    assert result["split"] == "s"
    assert result["accuracy"] == 0.55
    assert result["correct"] == 11
    assert result["total"] == 20
    assert result["avg_tokens"] == 226.7
    assert result["max_tokens"] == 326
    assert result["over_budget"] == 3
    assert result["score_per_1k_recall_tokens"] == 41.2844
    assert result["quality_token_target"] == "balanced-detail"
    assert result["retrieval_profile_effective"] == "balanced"
    assert result["profile_delta_vs_token_gate"]["max_recall_tokens"]["absolute"] == -14.0
    assert result["quality_gate_passed"] is False
    assert result["error"] == "quality gate failed"


def test_collect_matrix_case_result_includes_worker_error_and_missing_artifacts(tmp_path: Path) -> None:
    run_dir = tmp_path / "run"
    run_dir.mkdir(parents=True, exist_ok=True)
    (run_dir / CASE_ERROR_FILENAME).write_text(
        json.dumps(
            {
                "type": "RuntimeError",
                "error": "intentional worker failure",
                "traceback": "line-1\nline-2\nline-3",
            }
        ),
        encoding="utf-8",
    )

    result = _collect_matrix_case_result(
        case={"id": "case-a", "dataset": "locomo", "split": "locomo10"},
        run_dir=run_dir,
        exit_code=1,
        error="case exited with code 1",
    )

    assert result["missing_artifacts"] == ["summary.json", "gate-report.json"]
    assert result["worker_error_type"] == "RuntimeError"
    assert result["worker_error"] == "intentional worker failure"
    assert "line-2" in str(result["worker_traceback_tail"])


def test_run_matrix_dry_run_honors_start_index_and_max_cases(tmp_path: Path) -> None:
    matrix_path = tmp_path / "matrix.json"
    summary_path = tmp_path / "summary.json"
    run_dir = tmp_path / "run"
    run_dir.mkdir(parents=True, exist_ok=True)
    matrix_path.write_text(
        json.dumps(
            {
                "cases": [
                    {"id": "case-a", "dataset": "longmemeval", "split": "s"},
                    {"id": "case-b", "dataset": "locomo", "split": "locomo10"},
                    {"id": "case-c", "dataset": "beam", "split": "100k"},
                ]
            }
        ),
        encoding="utf-8",
    )
    args = _base_args()
    args.matrix_file = str(matrix_path)
    args.summary_file = str(summary_path)
    args.dry_run = True
    args.start_index = 2
    args.max_cases = 1

    run_matrix(args, run_dir)

    preview = json.loads(summary_path.read_text(encoding="utf-8"))
    assert len(preview) == 1
    assert preview[0]["id"] == "case-b"
    assert preview[0]["dataset"] == "locomo"
    preflight = json.loads((run_dir / "fair-run-preflight.json").read_text(encoding="utf-8"))
    assert preflight["passed"] is True


def test_execute_matrix_case_returns_error_for_run_benchmark_exception(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
) -> None:
    run_dir = tmp_path / "run"
    run_dir.mkdir(parents=True, exist_ok=True)

    def _boom(_: argparse.Namespace, __: Path) -> None:
        raise RuntimeError("intentional-case-failure")

    monkeypatch.setattr("run_amb_cortex.run_benchmark", _boom)

    exit_code, error = _execute_matrix_case(
        run_args=_base_args(),
        run_dir=run_dir,
        timeout_seconds=0,
    )
    assert exit_code == 1
    assert error and "intentional-case-failure" in error
    worker_error_payload = json.loads((run_dir / CASE_ERROR_FILENAME).read_text(encoding="utf-8"))
    assert worker_error_payload["type"] == "RuntimeError"
    assert worker_error_payload["error"] == "intentional-case-failure"


def _assert_strict_non_longmem_matrix(cases: list[dict[str, object]], *, query_limit: int) -> None:
    assert cases
    datasets = {str(case["dataset"]) for case in cases}
    assert datasets == _PRACTICAL_NON_LONGMEM_DATASETS
    assert all(str(case["dataset"]) != "longmemeval" for case in cases)
    assert all(int(case.get("query_limit", -1)) == query_limit for case in cases)
    assert all(bool(case.get("oracle", False)) is False for case in cases)
    assert all("query_id" not in case for case in cases)
    assert all("doc_limit" not in case for case in cases)


def test_strict_nonlongmem_q5_matrix_is_fair_and_no_longmemeval() -> None:
    matrix_path = BENCHMARKING_DIR / "configs" / "amb-eval-matrix.nonlongmem.q5.json"
    cases = _load_matrix_cases(matrix_path)

    _assert_strict_non_longmem_matrix(cases, query_limit=5)


def test_strict_nonlongmem_q10_matrix_is_fair_and_no_longmemeval() -> None:
    matrix_path = BENCHMARKING_DIR / "configs" / "amb-eval-matrix.nonlongmem.q10.json"
    cases = _load_matrix_cases(matrix_path)

    _assert_strict_non_longmem_matrix(cases, query_limit=10)


def test_strict_nonlongmem_q5_matrix_execution_profile_is_strict() -> None:
    matrix_path = BENCHMARKING_DIR / "configs" / "amb-eval-matrix.nonlongmem.q5.json"
    payload = json.loads(matrix_path.read_text(encoding="utf-8"))
    profile = payload["execution_profile"]

    assert profile["max_runtime_seconds"] == 1200
    assert profile["max_case_runtime_seconds"] == 900
    assert profile["oracle"] is False
    assert profile["no_enforce_gate"] is False
    assert profile["allow_missing_recall_metrics"] is False


def test_run_matrix_applies_execution_profile_with_strict_shortcut_defaults(tmp_path: Path) -> None:
    matrix_path = tmp_path / "matrix-profile.json"
    summary_path = tmp_path / "summary.json"
    run_dir = tmp_path / "run"
    run_dir.mkdir(parents=True, exist_ok=True)
    matrix_path.write_text(
        json.dumps(
            {
                "execution_profile": {
                    "max_runtime_seconds": 1200,
                    "max_case_runtime_seconds": 900,
                    "oracle": False,
                    "no_enforce_gate": False,
                    "allow_missing_recall_metrics": False,
                },
                "cases": [
                    {"id": "case-a", "dataset": "locomo", "split": "locomo10", "query_limit": 5}
                ],
            }
        ),
        encoding="utf-8",
    )
    args = _base_args()
    args.matrix_file = str(matrix_path)
    args.summary_file = str(summary_path)
    args.max_runtime_seconds = 9999
    args.max_case_runtime_seconds = 9999
    args.oracle = False
    args.no_enforce_gate = False
    args.allow_missing_recall_metrics = False
    args.dry_run = True

    run_matrix(args, run_dir)

    assert args.max_runtime_seconds == 1200
    assert args.max_case_runtime_seconds == 900
    assert args.oracle is False
    assert args.no_enforce_gate is False
    assert args.allow_missing_recall_metrics is False
    preview = json.loads(summary_path.read_text(encoding="utf-8"))
    assert len(preview) == 1
    assert preview[0]["dataset"] == "locomo"
    preflight = json.loads((run_dir / "fair-run-preflight.json").read_text(encoding="utf-8"))
    assert preflight["passed"] is True
    assert preflight["requested_shortcuts"]["oracle"] is False
    assert preflight["requested_shortcuts"]["no_enforce_gate"] is False
    assert preflight["requested_shortcuts"]["allow_missing_recall_metrics"] is False


def test_run_matrix_rejects_requested_oracle_even_when_profile_resets_it(tmp_path: Path) -> None:
    matrix_path = tmp_path / "matrix-profile-oracle-reset.json"
    run_dir = tmp_path / "run"
    run_dir.mkdir(parents=True, exist_ok=True)
    matrix_path.write_text(
        json.dumps(
            {
                "execution_profile": {
                    "max_runtime_seconds": 1200,
                    "max_case_runtime_seconds": 900,
                    "oracle": False,
                },
                "cases": [
                    {"id": "case-a", "dataset": "locomo", "split": "locomo10", "query_limit": 5}
                ],
            }
        ),
        encoding="utf-8",
    )
    args = _base_args()
    args.matrix_file = str(matrix_path)
    args.oracle = True
    args.dry_run = True

    with pytest.raises(ValueError, match="requested oracle=true"):
        run_matrix(args, run_dir)

    preflight = json.loads((run_dir / "fair-run-preflight.json").read_text(encoding="utf-8"))
    assert preflight["passed"] is False
    assert preflight["requested_shortcuts"]["oracle"] is True
    assert any("requested oracle=true" in item for item in preflight["violations"])


def test_run_matrix_rejects_requested_no_enforce_gate_even_when_profile_resets_it(tmp_path: Path) -> None:
    matrix_path = tmp_path / "matrix-profile-no-enforce-reset.json"
    run_dir = tmp_path / "run"
    run_dir.mkdir(parents=True, exist_ok=True)
    matrix_path.write_text(
        json.dumps(
            {
                "execution_profile": {
                    "max_runtime_seconds": 1200,
                    "max_case_runtime_seconds": 900,
                    "no_enforce_gate": False,
                },
                "cases": [
                    {"id": "case-a", "dataset": "locomo", "split": "locomo10", "query_limit": 5}
                ],
            }
        ),
        encoding="utf-8",
    )
    args = _base_args()
    args.matrix_file = str(matrix_path)
    args.no_enforce_gate = True
    args.dry_run = True

    with pytest.raises(ValueError, match="requested no_enforce_gate=true"):
        run_matrix(args, run_dir)

    preflight = json.loads((run_dir / "fair-run-preflight.json").read_text(encoding="utf-8"))
    assert preflight["passed"] is False
    assert preflight["requested_shortcuts"]["no_enforce_gate"] is True
    assert any("requested no_enforce_gate=true" in item for item in preflight["violations"])


def test_run_matrix_rejects_requested_missing_metrics_even_when_profile_resets_it(tmp_path: Path) -> None:
    matrix_path = tmp_path / "matrix-profile-missing-metrics-reset.json"
    run_dir = tmp_path / "run"
    run_dir.mkdir(parents=True, exist_ok=True)
    matrix_path.write_text(
        json.dumps(
            {
                "execution_profile": {
                    "max_runtime_seconds": 1200,
                    "max_case_runtime_seconds": 900,
                    "allow_missing_recall_metrics": False,
                },
                "cases": [
                    {"id": "case-a", "dataset": "locomo", "split": "locomo10", "query_limit": 5}
                ],
            }
        ),
        encoding="utf-8",
    )
    args = _base_args()
    args.matrix_file = str(matrix_path)
    args.allow_missing_recall_metrics = True
    args.dry_run = True

    with pytest.raises(ValueError, match="requested allow_missing_recall_metrics=true"):
        run_matrix(args, run_dir)

    preflight = json.loads((run_dir / "fair-run-preflight.json").read_text(encoding="utf-8"))
    assert preflight["passed"] is False
    assert preflight["requested_shortcuts"]["allow_missing_recall_metrics"] is True
    assert any("requested allow_missing_recall_metrics=true" in item for item in preflight["violations"])


def test_run_matrix_rejects_oracle_case_even_in_dry_run(tmp_path: Path) -> None:
    matrix_path = tmp_path / "matrix-oracle.json"
    run_dir = tmp_path / "run"
    run_dir.mkdir(parents=True, exist_ok=True)
    matrix_path.write_text(
        json.dumps(
            {
                "cases": [
                    {"id": "case-a", "dataset": "locomo", "split": "locomo10", "query_limit": 5, "oracle": True}
                ]
            }
        ),
        encoding="utf-8",
    )
    args = _base_args()
    args.matrix_file = str(matrix_path)
    args.dry_run = True

    with pytest.raises(ValueError, match="oracle"):
        run_matrix(args, run_dir)


def test_run_matrix_rejects_default_query_id_even_in_dry_run(tmp_path: Path) -> None:
    matrix_path = tmp_path / "matrix-query-id.json"
    run_dir = tmp_path / "run"
    run_dir.mkdir(parents=True, exist_ok=True)
    matrix_path.write_text(
        json.dumps({"cases": [{"id": "case-a", "dataset": "locomo", "split": "locomo10"}]}),
        encoding="utf-8",
    )
    args = _base_args()
    args.matrix_file = str(matrix_path)
    args.query_id = "q-1"
    args.dry_run = True

    with pytest.raises(ValueError, match="query_id"):
        run_matrix(args, run_dir)


def test_run_matrix_rejects_default_doc_limit_even_in_dry_run(tmp_path: Path) -> None:
    matrix_path = tmp_path / "matrix-doc-limit.json"
    run_dir = tmp_path / "run"
    run_dir.mkdir(parents=True, exist_ok=True)
    matrix_path.write_text(
        json.dumps({"cases": [{"id": "case-a", "dataset": "locomo", "split": "locomo10"}]}),
        encoding="utf-8",
    )
    args = _base_args()
    args.matrix_file = str(matrix_path)
    args.doc_limit = 1
    args.dry_run = True

    with pytest.raises(ValueError, match="doc_limit"):
        run_matrix(args, run_dir)


def test_run_matrix_rejects_default_no_enforce_gate_even_in_dry_run(tmp_path: Path) -> None:
    matrix_path = tmp_path / "matrix-no-enforce-gate.json"
    run_dir = tmp_path / "run"
    run_dir.mkdir(parents=True, exist_ok=True)
    matrix_path.write_text(
        json.dumps({"cases": [{"id": "case-a", "dataset": "locomo", "split": "locomo10"}]}),
        encoding="utf-8",
    )
    args = _base_args()
    args.matrix_file = str(matrix_path)
    args.no_enforce_gate = True
    args.dry_run = True

    with pytest.raises(ValueError, match="no_enforce_gate"):
        run_matrix(args, run_dir)


def test_run_matrix_rejects_default_missing_metrics_flag_even_in_dry_run(tmp_path: Path) -> None:
    matrix_path = tmp_path / "matrix-missing-metrics.json"
    run_dir = tmp_path / "run"
    run_dir.mkdir(parents=True, exist_ok=True)
    matrix_path.write_text(
        json.dumps({"cases": [{"id": "case-a", "dataset": "locomo", "split": "locomo10"}]}),
        encoding="utf-8",
    )
    args = _base_args()
    args.matrix_file = str(matrix_path)
    args.allow_missing_recall_metrics = True
    args.dry_run = True

    with pytest.raises(ValueError, match="allow_missing_recall_metrics"):
        run_matrix(args, run_dir)


def test_run_matrix_rejects_execution_profile_no_enforce_gate_even_in_dry_run(tmp_path: Path) -> None:
    matrix_path = tmp_path / "matrix-profile-no-enforce-gate.json"
    run_dir = tmp_path / "run"
    run_dir.mkdir(parents=True, exist_ok=True)
    matrix_path.write_text(
        json.dumps(
            {
                "execution_profile": {
                    "max_runtime_seconds": 1200,
                    "max_case_runtime_seconds": 900,
                    "no_enforce_gate": True,
                },
                "cases": [
                    {"id": "case-a", "dataset": "locomo", "split": "locomo10", "query_limit": 5}
                ],
            }
        ),
        encoding="utf-8",
    )
    args = _base_args()
    args.matrix_file = str(matrix_path)
    args.dry_run = True

    with pytest.raises(ValueError, match="no_enforce_gate"):
        run_matrix(args, run_dir)


def test_run_matrix_rejects_execution_profile_missing_metrics_flag_even_in_dry_run(tmp_path: Path) -> None:
    matrix_path = tmp_path / "matrix-profile-missing-metrics.json"
    run_dir = tmp_path / "run"
    run_dir.mkdir(parents=True, exist_ok=True)
    matrix_path.write_text(
        json.dumps(
            {
                "execution_profile": {
                    "max_runtime_seconds": 1200,
                    "max_case_runtime_seconds": 900,
                    "allow_missing_recall_metrics": True,
                },
                "cases": [
                    {"id": "case-a", "dataset": "locomo", "split": "locomo10", "query_limit": 5}
                ],
            }
        ),
        encoding="utf-8",
    )
    args = _base_args()
    args.matrix_file = str(matrix_path)
    args.dry_run = True

    with pytest.raises(ValueError, match="allow_missing_recall_metrics"):
        run_matrix(args, run_dir)


def test_run_matrix_skips_membench_case_without_dataset_files(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    matrix_path = tmp_path / "matrix-membench.json"
    run_dir = tmp_path / "run"
    run_dir.mkdir(parents=True, exist_ok=True)
    matrix_path.write_text(
        json.dumps(
            {
                "cases": [
                    {
                        "id": "membench-case",
                        "dataset": "membench",
                        "split": "FirstAgentLowLevel",
                        "query_limit": 5,
                    }
                ]
            }
        ),
        encoding="utf-8",
    )
    missing_data_dir = tmp_path / "missing-membench-data"
    monkeypatch.setenv("MEMBENCH_DATA_PATH", str(missing_data_dir))
    args = _base_args()
    args.matrix_file = str(matrix_path)
    args.dry_run = True

    run_matrix(args, run_dir)

    preflight = json.loads((run_dir / "fair-run-preflight.json").read_text(encoding="utf-8"))
    assert preflight["passed"] is True
    assert preflight["case_count_selected"] == 1
    assert preflight["case_count_runnable"] == 0
    assert preflight["case_count_skipped_prereq"] == 1
    assert preflight["violations"] == []
    assert len(preflight["dataset_skips"]) == 1
    assert preflight["dataset_skips"][0]["id"] == "membench-case"
    assert any(
        "missing required files" in item
        for item in preflight["dataset_skips"][0]["violations"]
    )

    summary = json.loads((run_dir / "matrix-summary.json").read_text(encoding="utf-8"))
    assert len(summary) == 1
    assert summary[0]["id"] == "membench-case"
    assert summary[0]["status"] == "skipped-prereq"
    assert "missing required files" in str(summary[0]["skip_reason"])


def test_run_matrix_rejects_matrix_runtime_cap_above_ceiling(tmp_path: Path) -> None:
    matrix_path = tmp_path / "matrix-timeout.json"
    run_dir = tmp_path / "run"
    run_dir.mkdir(parents=True, exist_ok=True)
    matrix_path.write_text(
        json.dumps({"cases": [{"id": "case-a", "dataset": "locomo", "split": "locomo10"}]}),
        encoding="utf-8",
    )
    args = _base_args()
    args.matrix_file = str(matrix_path)
    args.max_runtime_seconds = 1201
    args.dry_run = True

    with pytest.raises(ValueError, match="max-runtime-seconds"):
        run_matrix(args, run_dir)


def test_run_cadence_dry_run_honors_max_matrices(tmp_path: Path) -> None:
    matrix_a = tmp_path / "matrix-a.json"
    matrix_b = tmp_path / "matrix-b.json"
    run_dir = tmp_path / "cadence-run"
    run_dir.mkdir(parents=True, exist_ok=True)
    matrix_payload = {
        "cases": [
            {"id": "case-a", "dataset": "locomo", "split": "locomo10", "query_limit": 5}
        ]
    }
    matrix_a.write_text(json.dumps(matrix_payload), encoding="utf-8")
    matrix_b.write_text(json.dumps(matrix_payload), encoding="utf-8")

    args = _base_cadence_args()
    args.matrix_files = [str(matrix_a), str(matrix_b)]
    args.max_matrices = 1
    args.dry_run = True

    run_cadence(args, run_dir)

    cadence_summary = json.loads((run_dir / "cadence-summary.json").read_text(encoding="utf-8"))
    assert cadence_summary["matrix_count_requested"] == 2
    assert cadence_summary["matrix_count_executed"] == 1
    assert cadence_summary["matrix_count_failed"] == 0
    assert cadence_summary["results"][0]["status"] == "passed"
    assert Path(cadence_summary["results"][0]["summary_file"]).exists()


def test_run_cadence_continue_on_error_runs_remaining_matrices(tmp_path: Path) -> None:
    bad_matrix = tmp_path / "matrix-bad.json"
    good_matrix = tmp_path / "matrix-good.json"
    run_dir = tmp_path / "cadence-run"
    run_dir.mkdir(parents=True, exist_ok=True)
    bad_matrix.write_text(
        json.dumps(
            {
                "cases": [
                    {
                        "id": "bad-oracle-case",
                        "dataset": "locomo",
                        "split": "locomo10",
                        "query_limit": 5,
                        "oracle": True,
                    }
                ]
            }
        ),
        encoding="utf-8",
    )
    good_matrix.write_text(
        json.dumps(
            {
                "cases": [
                    {"id": "good-case", "dataset": "locomo", "split": "locomo10", "query_limit": 5}
                ]
            }
        ),
        encoding="utf-8",
    )

    args = _base_cadence_args()
    args.matrix_files = [str(bad_matrix), str(good_matrix)]
    args.continue_on_error = True
    args.dry_run = True

    with pytest.raises(RuntimeError, match="cadence run failed"):
        run_cadence(args, run_dir)

    cadence_summary = json.loads((run_dir / "cadence-summary.json").read_text(encoding="utf-8"))
    assert cadence_summary["matrix_count_requested"] == 2
    assert cadence_summary["matrix_count_executed"] == 2
    assert cadence_summary["matrix_count_failed"] == 1
    assert cadence_summary["results"][0]["status"] == "failed"
    assert cadence_summary["results"][1]["status"] == "passed"


def test_practical_nonlongmem_matrix_is_fair_and_timeout_friendly() -> None:
    matrix_path = BENCHMARKING_DIR / "configs" / "amb-eval-matrix.nonlongmem.practical.json"
    cases = _load_matrix_cases(matrix_path)

    assert cases
    datasets = {str(case["dataset"]) for case in cases}
    assert datasets == _PRACTICAL_NON_LONGMEM_DATASETS
    assert all(str(case["dataset"]) != "longmemeval" for case in cases)
    assert all(bool(case.get("oracle", False)) is False for case in cases)
    assert all("query_id" not in case for case in cases)
    assert all("doc_limit" not in case for case in cases)
    assert all(1 <= int(case.get("query_limit", 0)) <= 6 for case in cases)


def test_practical_nonlongmem_matrix_execution_profile_is_strict() -> None:
    matrix_path = BENCHMARKING_DIR / "configs" / "amb-eval-matrix.nonlongmem.practical.json"
    payload = json.loads(matrix_path.read_text(encoding="utf-8"))
    profile = payload["execution_profile"]

    assert profile["max_runtime_seconds"] == 1200
    assert profile["max_case_runtime_seconds"] == 900
    assert profile["token_gate_mode"] == "dynamic"
    assert profile["provider_profile"] == "gemini"
    assert profile["recall_budget"] == 300
    assert profile["min_accuracy"] == 0.9
    assert profile["max_recall_tokens"] == 300
    assert profile["max_avg_recall_tokens"] == 246.0
    assert profile["allow_missing_recall_metrics"] is False
    assert profile["no_enforce_gate"] is False
    assert profile["oracle"] is False
