"""Markdown-backed negative lessons with lifecycle (PRD-06 Phase C.2 follow-up).

The original ``negative_lessons.jsonl`` store records *what failed* —
parameter sets that posted FAIL continuity, calibrations that diverged,
or non-physical parameter regions. Round 7 promotes that data to a
markdown store that mirrors the existing ``lessons_learned.md``
lifecycle (HTML-comment-fenced YAML metadata, exponential confidence
decay, active/dormant/retired buckets, archive on retire).

Why two formats?
----------------
The JSONL store stays as the v1 wire format and as the per-run
"append exactly one row" path. The markdown store is the *curatable*
surface — a human can grep ``## continuity_fail_saanich_b8`` in their
editor, see the metadata fence, and read the body without spinning
up a Python REPL. The lifecycle (decay → dormant → retired → archive)
runs against the markdown only, mirroring ``lessons_lifecycle.py``.

File layout
-----------
``memory/modeling-memory/negative_lessons.md`` has one ``## <name>``
heading per recorded failure, immediately followed by a fenced YAML
preamble. Example::

    ## continuity_fail_saanich_b8

    <!-- aiswmm-negative-lesson-metadata
    metadata:
      case: saanich-b8
      lesson_type: continuity_fail
      parameters_tried:
        manning_n_overland: 0.25
      note: postflight FAIL on runoff_continuity_pct
      first_seen_utc: '2026-05-19T10:00:00Z'
      last_seen_utc: '2026-05-19T10:00:00Z'
      evidence_count: 1
      evidence_runs:
        - 20260519-100000_saanich
      status: active
      confidence_score: 1.0
      half_life_days: 90
    /aiswmm-negative-lesson-metadata -->

    Free-form body text the curator may add later.

Atomicity
---------
Every write follows the tmp-file + ``os.replace`` rename pattern so a
torn write leaves the previous content intact. The fence sentinels are
deliberately distinct from ``lessons_learned.md``
(``aiswmm-negative-lesson-metadata`` vs ``aiswmm-metadata``) so the two
parsers cannot accidentally cross-contaminate when both files are
read by a future indexer.
"""

from __future__ import annotations

import math
import os
import re
import tempfile
from dataclasses import dataclass, field
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable

import yaml


SCHEMA_VERSION = "1.0"

DEFAULT_HALF_LIFE_DAYS = 90
DEFAULT_ACTIVE_THRESHOLD = 0.5
DEFAULT_DORMANT_THRESHOLD = 0.2

METADATA_OPEN = "<!-- aiswmm-negative-lesson-metadata"
METADATA_CLOSE = "/aiswmm-negative-lesson-metadata -->"

# ``## <name>`` headings whose name is a snake_case identifier are the
# canonical section heads. Names are generated by callers (e.g.
# ``continuity_fail_<case>``) and round-trip back to the dataclass.
_HEADING_RE = re.compile(
    r"^##\s+(?P<name>[a-z][a-z0-9_]*)\s*$", re.MULTILINE
)

_FENCE_RE = re.compile(
    re.escape(METADATA_OPEN)
    + r"\n(?P<payload>.*?)\n"
    + re.escape(METADATA_CLOSE),
    re.DOTALL,
)

_VALID_LESSON_TYPES = frozenset(
    {"continuity_fail", "calibration_diverged", "non_physical_param"}
)


class _StringTimestampLoader(yaml.SafeLoader):
    """SafeLoader that keeps ISO-8601 timestamps as strings.

    Mirrors the behaviour in :mod:`lessons_metadata` so reads and
    writes stay symmetric across both lifecycle modules.
    """


_StringTimestampLoader.yaml_implicit_resolvers = {
    key: [
        (tag, regexp)
        for tag, regexp in resolvers
        if tag != "tag:yaml.org,2002:timestamp"
    ]
    for key, resolvers in _StringTimestampLoader.yaml_implicit_resolvers.items()
}


@dataclass(frozen=True)
class NegativeLessonMd:
    """One markdown-backed negative lesson.

    Frozen so the lifecycle helpers cannot scribble on a record between
    the parser handing it back and the caller acting on it. ``name`` is
    the snake_case section heading; the writer derives it from
    ``(lesson_type, case)`` when callers do not pass one explicitly.
    """

    name: str
    case: str
    lesson_type: str
    parameters_tried: dict[str, float] = field(default_factory=dict)
    note: str = ""
    first_seen: str = ""
    last_seen: str = ""
    evidence_count: int = 0
    evidence_runs: list[str] = field(default_factory=list)
    status: str = "active"
    confidence_score: float = 1.0
    half_life_days: int = DEFAULT_HALF_LIFE_DAYS

    def to_metadata_dict(self) -> dict[str, Any]:
        """Serialise the metadata fence payload (sans ``name``)."""
        return {
            "case": self.case,
            "lesson_type": self.lesson_type,
            "parameters_tried": dict(self.parameters_tried),
            "note": self.note,
            "first_seen_utc": self.first_seen,
            "last_seen_utc": self.last_seen,
            "evidence_count": int(self.evidence_count),
            "evidence_runs": list(self.evidence_runs),
            "status": self.status,
            "confidence_score": round(float(self.confidence_score), 3),
            "half_life_days": int(self.half_life_days),
        }


# ---------------------------------------------------------------------------
# Helpers (internal)


def _now_utc_iso(now: datetime | None = None) -> str:
    current = (now or datetime.now(timezone.utc)).astimezone(timezone.utc)
    return current.isoformat(timespec="seconds").replace("+00:00", "Z")


def _parse_iso(stamp: str) -> datetime:
    text = (stamp or "").strip()
    if not text:
        raise ValueError("empty timestamp")
    if text.endswith("Z"):
        text = text[:-1] + "+00:00"
    parsed = datetime.fromisoformat(text)
    if parsed.tzinfo is None:
        parsed = parsed.replace(tzinfo=timezone.utc)
    return parsed.astimezone(timezone.utc)


def _ensure_header(store: Path) -> None:
    if store.is_file():
        return
    store.parent.mkdir(parents=True, exist_ok=True)
    store.write_text(
        f"<!-- schema_version: {SCHEMA_VERSION} -->\n"
        "# Negative Lessons\n"
        "\n"
        "Parameter regions that consistently fail. Each section carries a\n"
        "metadata fence describing when the lesson was first/last seen,\n"
        "what parameters were tried, how many runs confirmed it, and the\n"
        "lifecycle status (active / dormant / retired).\n"
        "\n",
        encoding="utf-8",
    )


def _atomic_write(path: Path, text: str) -> None:
    """Write ``text`` to ``path`` atomically via tmp + ``os.replace``."""
    path.parent.mkdir(parents=True, exist_ok=True)
    fd, tmp = tempfile.mkstemp(
        prefix=path.name + ".", suffix=".tmp", dir=str(path.parent)
    )
    try:
        with os.fdopen(fd, "w", encoding="utf-8") as handle:
            handle.write(text)
        os.replace(tmp, path)
    except Exception:
        try:
            os.unlink(tmp)
        except OSError:
            pass
        raise


def _iter_sections(text: str) -> Iterable[tuple[str, int, int]]:
    """Yield ``(name, start, end)`` spans for each ``## <section>`` block."""
    matches = list(_HEADING_RE.finditer(text))
    for i, match in enumerate(matches):
        name = match.group("name")
        start = match.start()
        end = matches[i + 1].start() if i + 1 < len(matches) else len(text)
        yield name, start, end


def _parse_metadata(block: str) -> dict[str, Any] | None:
    match = _FENCE_RE.search(block)
    if not match:
        return None
    try:
        parsed = yaml.load(match.group("payload"), Loader=_StringTimestampLoader)
    except yaml.YAMLError:
        return None
    if not isinstance(parsed, dict):
        return None
    meta = parsed.get("metadata")
    if not isinstance(meta, dict):
        return None
    return meta


def _format_metadata_yaml(meta: dict[str, Any]) -> str:
    canonical = [
        "case",
        "lesson_type",
        "parameters_tried",
        "note",
        "first_seen_utc",
        "last_seen_utc",
        "evidence_count",
        "evidence_runs",
        "status",
        "confidence_score",
        "half_life_days",
    ]
    ordered: dict[str, Any] = {}
    for key in canonical:
        if key in meta:
            ordered[key] = meta[key]
    for key in sorted(k for k in meta if k not in canonical):
        ordered[key] = meta[key]
    body = yaml.safe_dump(
        {"metadata": ordered},
        sort_keys=False,
        default_flow_style=False,
        allow_unicode=True,
    ).rstrip()
    return body


def _record_to_block(record: NegativeLessonMd) -> str:
    """Compose the ``## <name> ... <fence>`` block text."""
    payload = _format_metadata_yaml(record.to_metadata_dict())
    fence = f"{METADATA_OPEN}\n{payload}\n{METADATA_CLOSE}"
    return f"## {record.name}\n\n{fence}\n"


def _record_from_metadata(name: str, meta: dict[str, Any]) -> NegativeLessonMd:
    return NegativeLessonMd(
        name=name,
        case=str(meta.get("case") or ""),
        lesson_type=str(meta.get("lesson_type") or ""),
        parameters_tried={
            str(k): float(v)
            for k, v in (meta.get("parameters_tried") or {}).items()
            if _is_number(v)
        },
        note=str(meta.get("note") or ""),
        first_seen=str(meta.get("first_seen_utc") or ""),
        last_seen=str(meta.get("last_seen_utc") or ""),
        evidence_count=int(meta.get("evidence_count") or 0),
        evidence_runs=[str(r) for r in (meta.get("evidence_runs") or [])],
        status=str(meta.get("status") or "active"),
        confidence_score=float(meta.get("confidence_score") or 0.0),
        half_life_days=int(meta.get("half_life_days") or DEFAULT_HALF_LIFE_DAYS),
    )


def _is_number(value: Any) -> bool:
    try:
        float(value)
        return True
    except (TypeError, ValueError):
        return False


def _classify(score: float) -> str:
    if score > DEFAULT_ACTIVE_THRESHOLD:
        return "active"
    if score > DEFAULT_DORMANT_THRESHOLD:
        return "dormant"
    return "retired"


# ---------------------------------------------------------------------------
# Public API


def add_negative_lesson(store: Path, lesson: NegativeLessonMd) -> None:
    """Append ``lesson`` to ``store``; bump evidence on duplicate name.

    When ``store`` does not contain a ``## <lesson.name>`` block we
    append a new section with the metadata fence. When a section with
    the same name already exists we:

      * Increment ``evidence_count`` by 1.
      * Set ``last_seen_utc`` to the lesson's ``last_seen`` (or now
        when the lesson left it empty).
      * Append every entry of ``lesson.evidence_runs`` that is not yet
        in the existing list.
      * Re-compute ``confidence_score`` from the updated
        ``evidence_count`` and ``last_seen_utc`` so the bucket stays
        consistent with the lifecycle rules.

    The body text outside the metadata fence is preserved verbatim;
    only the fence is rewritten. Atomic — a crashed write leaves the
    previous content intact.
    """
    if lesson.lesson_type not in _VALID_LESSON_TYPES:
        raise ValueError(
            f"NegativeLessonMd.lesson_type must be one of "
            f"{sorted(_VALID_LESSON_TYPES)}; got {lesson.lesson_type!r}"
        )
    if not lesson.case.strip():
        raise ValueError("NegativeLessonMd.case must be a non-empty string")

    _ensure_header(Path(store))
    text = Path(store).read_text(encoding="utf-8")

    now_iso = lesson.last_seen or _now_utc_iso()
    first_iso = lesson.first_seen or now_iso

    spans = list(_iter_sections(text))
    existing_span = next((span for span in spans if span[0] == lesson.name), None)

    if existing_span is None:
        record = NegativeLessonMd(
            name=lesson.name,
            case=lesson.case,
            lesson_type=lesson.lesson_type,
            parameters_tried=dict(lesson.parameters_tried),
            note=lesson.note,
            first_seen=first_iso,
            last_seen=now_iso,
            evidence_count=max(1, int(lesson.evidence_count)),
            evidence_runs=list(lesson.evidence_runs),
            status=lesson.status,
            confidence_score=float(lesson.confidence_score) or float(
                max(1, int(lesson.evidence_count))
            ),
            half_life_days=int(lesson.half_life_days),
        )
        block = _record_to_block(record)
        if not text.endswith("\n"):
            text += "\n"
        if not text.endswith("\n\n"):
            text += "\n"
        new_text = text + block
        _atomic_write(Path(store), new_text)
        return

    name, start, end = existing_span
    block = text[start:end]
    meta = _parse_metadata(block) or {}
    existing = _record_from_metadata(name, meta)

    new_runs = list(existing.evidence_runs)
    for run_id in lesson.evidence_runs:
        if run_id and run_id not in new_runs:
            new_runs.append(run_id)

    new_count = existing.evidence_count + 1
    half_life = existing.half_life_days or DEFAULT_HALF_LIFE_DAYS
    new_score = float(new_count) * math.exp(0.0)  # last_seen == now → age 0
    updated = NegativeLessonMd(
        name=name,
        case=existing.case or lesson.case,
        lesson_type=existing.lesson_type or lesson.lesson_type,
        parameters_tried=existing.parameters_tried or dict(lesson.parameters_tried),
        note=existing.note or lesson.note,
        first_seen=existing.first_seen or first_iso,
        last_seen=now_iso,
        evidence_count=new_count,
        evidence_runs=new_runs,
        status=_classify(new_score),
        confidence_score=new_score,
        half_life_days=half_life,
    )

    # Preserve body text between the metadata fence and the next ``##``.
    fence_match = _FENCE_RE.search(block)
    if fence_match is None:
        # Block without a fence — rebuild from scratch.
        new_block = _record_to_block(updated)
    else:
        new_payload = _format_metadata_yaml(updated.to_metadata_dict())
        new_fence = f"{METADATA_OPEN}\n{new_payload}\n{METADATA_CLOSE}"
        new_block = (
            block[: fence_match.start()] + new_fence + block[fence_match.end():]
        )

    new_text = text[:start] + new_block + text[end:]
    _atomic_write(Path(store), new_text)


def list_negative_lessons(
    store: Path, *, status: str | None = None
) -> list[NegativeLessonMd]:
    """Parse every ``## <name>`` section into typed records.

    ``status`` filters to a single bucket (``"active"``, ``"dormant"``,
    ``"retired"``); ``None`` returns all. Missing files yield ``[]``.
    Sections without a metadata fence are skipped silently — they may
    represent free-form curator notes that were appended after the
    writer's last pass.
    """
    store = Path(store)
    if not store.is_file():
        return []
    text = store.read_text(encoding="utf-8")
    out: list[NegativeLessonMd] = []
    for name, start, end in _iter_sections(text):
        meta = _parse_metadata(text[start:end])
        if meta is None:
            continue
        record = _record_from_metadata(name, meta)
        if status is not None and record.status != status:
            continue
        out.append(record)
    return out


def is_param_set_known_bad_md(
    store: Path,
    case_name: str,
    params: dict[str, float],
    *,
    tolerance_pct: float = 5.0,
) -> NegativeLessonMd | None:
    """Mirror of ``is_param_set_known_bad`` against the markdown store.

    A match requires the candidate ``params`` cover every recorded key
    and every covered value lie within ``tolerance_pct`` percent of the
    recorded value (relative to the recorded magnitude). Retired
    lessons are skipped — they have decayed below the dormant threshold
    so we no longer use them to block new proposals.
    """
    if tolerance_pct < 0:
        raise ValueError("tolerance_pct must be non-negative")

    for lesson in list_negative_lessons(store):
        if lesson.case != case_name:
            continue
        if lesson.status == "retired":
            continue
        recorded = lesson.parameters_tried
        if not recorded:
            continue
        if not all(key in params for key in recorded):
            continue
        within = True
        for key, recorded_value in recorded.items():
            candidate = params[key]
            denom = max(abs(float(recorded_value)), 1e-9)
            pct = (
                abs(float(candidate) - float(recorded_value)) / denom * 100.0
            )
            if pct > tolerance_pct:
                within = False
                break
        if within:
            return lesson
    return None


def apply_decay(
    store: Path, *, now: str | None = None
) -> dict[str, int]:
    """Apply exponential decay across every section, returning bucket counts.

    The decay formula matches ``lessons_metadata.compute_confidence``:
    ``score = evidence_count * exp(-age_days / half_life_days)``. Age is
    computed from ``last_seen_utc`` against ``now`` (or the current UTC
    time when ``now`` is None). Status transitions:

      * score > 0.5  → ``active``
      * 0.2 < score ≤ 0.5 → ``dormant``
      * score ≤ 0.2  → ``retired``

    Returns a dict with bucket counts (``active``, ``dormant``,
    ``retired``) plus a ``moved`` count that totals the sections whose
    status changed during this pass.
    """
    store = Path(store)
    counts = {"active": 0, "dormant": 0, "retired": 0, "moved": 0}
    if not store.is_file():
        return counts

    current = _parse_iso(now) if now else datetime.now(timezone.utc)
    text = store.read_text(encoding="utf-8")
    spans = list(_iter_sections(text))
    if not spans:
        return counts

    new_text = text
    # Walk in reverse so byte offsets stay stable after substitutions
    # past the current span.
    for name, start, end in reversed(spans):
        block = new_text[start:end]
        meta = _parse_metadata(block)
        if meta is None:
            continue
        record = _record_from_metadata(name, meta)
        if record.evidence_count <= 0 or record.half_life_days <= 0:
            counts[record.status] = counts.get(record.status, 0) + 1
            continue
        try:
            last_seen = _parse_iso(record.last_seen)
        except ValueError:
            counts[record.status] = counts.get(record.status, 0) + 1
            continue
        age_days = max(
            0.0, (current - last_seen).total_seconds() / 86400.0
        )
        score = float(record.evidence_count) * math.exp(
            -age_days / float(record.half_life_days)
        )
        new_status = _classify(score)
        if new_status != record.status:
            counts["moved"] += 1
        counts[new_status] = counts.get(new_status, 0) + 1

        new_meta = dict(meta)
        new_meta["confidence_score"] = round(score, 3)
        new_meta["status"] = new_status
        new_payload = _format_metadata_yaml(new_meta)
        fence_match = _FENCE_RE.search(block)
        if fence_match is None:
            continue
        new_fence = f"{METADATA_OPEN}\n{new_payload}\n{METADATA_CLOSE}"
        new_block = (
            block[: fence_match.start()] + new_fence + block[fence_match.end():]
        )
        new_text = new_text[:start] + new_block + new_text[end:]

    if new_text != text:
        _atomic_write(store, new_text)
    return counts


def archive_retired(store: Path, archive_store: Path) -> int:
    """Move retired sections out of ``store`` into ``archive_store``.

    Returns the count of sections moved. The archive file is created on
    first use with a small header. The source store has its retired
    sections excised but its other content (header, active/dormant
    sections, curator notes) is preserved.
    """
    store = Path(store)
    archive = Path(archive_store)
    if not store.is_file():
        return 0
    text = store.read_text(encoding="utf-8")
    spans = list(_iter_sections(text))
    retired_blocks: list[str] = []
    keep_text = text

    # Walk in reverse so removal stays index-safe.
    for name, start, end in reversed(spans):
        block = keep_text[start:end]
        meta = _parse_metadata(block)
        if meta is None:
            continue
        if str(meta.get("status") or "active") != "retired":
            continue
        retired_blocks.append(block)
        # Excise; merge seam with a single blank line.
        head = keep_text[:start]
        tail = keep_text[end:]
        if head.endswith("\n\n") and tail.startswith("\n"):
            tail = tail.lstrip("\n")
            head = head.rstrip("\n") + "\n\n"
        keep_text = head + tail

    if not retired_blocks:
        return 0

    _atomic_write(store, keep_text)

    if not archive.is_file():
        archive.parent.mkdir(parents=True, exist_ok=True)
        archive.write_text(
            f"<!-- schema_version: {SCHEMA_VERSION} -->\n"
            "# Negative Lessons (Archived)\n"
            "\n"
            "Sections that decayed below the dormant threshold. Move a\n"
            "section back to ``negative_lessons.md`` to revive its\n"
            "lifecycle.\n"
            "\n",
            encoding="utf-8",
        )
    existing = archive.read_text(encoding="utf-8")
    if not existing.endswith("\n"):
        existing += "\n"
    if not existing.endswith("\n\n"):
        existing += "\n"
    # retired_blocks was built in reverse; flip back to insertion order.
    appended = existing + "".join(reversed(retired_blocks))
    _atomic_write(archive, appended)
    return len(retired_blocks)


def migrate_jsonl_to_md(jsonl_path: Path, md_path: Path) -> int:
    """Convert ``jsonl_path`` rows to ``md_path`` sections.

    Idempotent: rows whose derived section name already exists in
    ``md_path`` are bumped via the same ``add_negative_lesson`` path
    used by the audit hook. Returns the count of *new* sections
    written; updates to pre-existing sections do not increment.

    The section name is derived as ``"<lesson_type>_<case_slug>"``
    where ``case_slug`` is the case name with non-alphanumeric
    characters replaced by ``_``. This keeps every section heading a
    valid snake_case identifier that the parser can locate.
    """
    from agentic_swmm.memory.negative_lessons import recall_negative_lessons

    jsonl = Path(jsonl_path)
    md = Path(md_path)
    if not jsonl.is_file():
        return 0
    existing_names = {lesson.name for lesson in list_negative_lessons(md)}
    rows = recall_negative_lessons(jsonl, {})
    new_count = 0
    for row in rows:
        name = _section_name(row.lesson_type, row.case_name)
        recorded_at = row.recorded_at or _now_utc_iso()
        record = NegativeLessonMd(
            name=name,
            case=row.case_name,
            lesson_type=row.lesson_type,
            parameters_tried=dict(row.parameters_tried),
            note=row.note,
            first_seen=recorded_at,
            last_seen=recorded_at,
            evidence_count=1,
            evidence_runs=[row.run_id] if row.run_id else [],
            status="active",
            confidence_score=1.0,
            half_life_days=DEFAULT_HALF_LIFE_DAYS,
        )
        was_new = name not in existing_names
        add_negative_lesson(md, record)
        if was_new:
            new_count += 1
            existing_names.add(name)
    return new_count


def _section_name(lesson_type: str, case_name: str) -> str:
    case_slug = re.sub(r"[^a-z0-9]+", "_", (case_name or "").lower()).strip("_")
    lt = re.sub(r"[^a-z0-9]+", "_", (lesson_type or "").lower()).strip("_")
    name = f"{lt}_{case_slug}" if case_slug else lt
    if not re.match(r"^[a-z]", name):
        name = "lesson_" + name
    return name


__all__ = [
    "SCHEMA_VERSION",
    "DEFAULT_HALF_LIFE_DAYS",
    "DEFAULT_ACTIVE_THRESHOLD",
    "DEFAULT_DORMANT_THRESHOLD",
    "METADATA_OPEN",
    "METADATA_CLOSE",
    "NegativeLessonMd",
    "add_negative_lesson",
    "list_negative_lessons",
    "is_param_set_known_bad_md",
    "apply_decay",
    "archive_retired",
    "migrate_jsonl_to_md",
]
