"""Prompt-context renderers for the Nipux worker loop."""

from __future__ import annotations

import re
from typing import Any

from nipux_cli.memory_graph import memory_graph_for_prompt
from nipux_cli.metric_format import format_metric_value
from nipux_cli.operator_context import active_prompt_operator_entries, operator_entry_is_prompt_relevant
from nipux_cli.tui_outcomes import hourly_outcome_summary, model_update_event_parts, outcome_counts
from nipux_cli.worker_policy import (
    MAX_WORKER_PROMPT_CHARS,
    PROMPT_SECTION_BUDGETS,
    SECTION_ITEM_CHARS,
    TIMELINE_PROMPT_AGENT_TITLES,
    TIMELINE_PROMPT_EVENT_TYPES,
    TIMELINE_PROMPT_EVENTS,
    TIMELINE_PROMPT_TOOL_STATUSES,
)
from nipux_cli.worker_prompt_format import clip_text as _clip_text


NEGATIVE_EXISTENCE_MARKERS = (
    "cannot access",
    "does not exist",
    "failed to find",
    "has not been",
    "missing",
    "no ",
    "no such",
    "none",
    "not available",
    "not detected",
    "not downloaded",
    "not found",
    "not installed",
    "unavailable",
    "was not",
    "without",
)
NEGATIVE_EVIDENCE_LINE_MARKERS = (
    "cannot access",
    "denied",
    "does not exist",
    "error",
    "failed",
    "failure",
    "has not been",
    "missing",
    "no such",
    "not available",
    "not detected",
    "not downloaded",
    "not found",
    "not installed",
    "permission",
    "timeout",
    "unavailable",
    "was not",
)


def _memory_entries_for_prompt(memory_entries: list[dict[str, Any]], *, limit: int = 2) -> list[dict[str, Any]]:
    entries = [entry for entry in memory_entries if isinstance(entry, dict)]
    rolling = next((entry for entry in entries if entry.get("key") == "rolling_state"), None)
    selected: list[dict[str, Any]] = []
    if rolling:
        selected.append(rolling)
    for entry in entries:
        if len(selected) >= limit:
            break
        if rolling is not None and entry is rolling:
            continue
        selected.append(entry)
    return selected[:limit]


def _render_worker_prompt(job: dict[str, Any], *, sections: list[tuple[str, str]]) -> str:
    objective = _clip_text(job.get("objective") or "", 2_000)
    header = f"Job: {job['title']}\nKind: {job['kind']}\nObjective:\n{objective}"
    metadata = job.get("metadata") if isinstance(job.get("metadata"), dict) else {}
    stale_tokens = _stale_claim_tokens_for_prompt(
        metadata,
        reference_text=" ".join(str(job.get(key) or "") for key in ("title", "objective", "kind")),
    )
    instruction = (
        "Take exactly one bounded next action. If recent state contains search results, do not search the same query again. "
        "If recent state contains extracted page evidence, write an artifact before doing more search or browsing."
    )
    scale = 1.0
    while True:
        parts = [header]
        for title, body in sections:
            base_budget = PROMPT_SECTION_BUDGETS.get(title, SECTION_ITEM_CHARS)
            budget = max(260, int(base_budget * scale))
            safe_body = _redact_stale_tokens_for_prompt(body, stale_tokens)
            parts.append(f"{title}:\n{_clip_text(safe_body, budget)}")
        parts.append(instruction)
        content = "\n\n".join(parts)
        if len(content) <= MAX_WORKER_PROMPT_CHARS or scale <= 0.45:
            break
        scale -= 0.12
    if len(content) <= MAX_WORKER_PROMPT_CHARS:
        return content
    suffix_sections: list[str] = []
    for title, body in sections:
        if title == "Operator context":
            suffix_sections.append(f"Operator context:\n{_clip_text(_redact_stale_tokens_for_prompt(body, stale_tokens), 900)}")
        elif title == "Next-action constraint":
            suffix_sections.append(f"Next-action constraint:\n{_clip_text(_redact_stale_tokens_for_prompt(body, stale_tokens), 900)}")
    suffix = "\n\n".join(suffix_sections + [instruction])
    marker = "\n\n...[middle context clipped; operator context and next action repeated below]...\n"
    head_budget = max(0, MAX_WORKER_PROMPT_CHARS - len(suffix) - len(marker))
    return _clip_text(content, head_budget) + marker + suffix


def _redact_stale_tokens_for_prompt(text: str, stale_tokens: list[str]) -> str:
    redacted = str(text or "")
    for token in sorted((str(token) for token in stale_tokens if str(token).strip()), key=len, reverse=True):
        pattern = r"(?<![A-Za-z0-9])" + re.escape(token) + r"(?![A-Za-z0-9])"
        redacted = re.sub(
            pattern,
            lambda match: match.group(0) if _match_inside_path_like_span(match.string, match.start(), match.end()) else "[unsupported-stale-claim]",
            redacted,
            flags=re.IGNORECASE,
        )
    return redacted


def _match_inside_path_like_span(text: str, start: int, end: int) -> bool:
    left = start
    while left > 0 and not text[left - 1].isspace() and text[left - 1] not in "'\"`<>|;":
        left -= 1
    right = end
    while right < len(text) and not text[right].isspace() and text[right] not in "'\"`<>|;":
        right += 1
    span = text[left:right]
    return "/" in span or "\\" in span or span.startswith(("~", "."))


def _operator_messages_for_prompt(
    job: dict[str, Any],
    *,
    active_messages: list[dict[str, Any]] | None = None,
    include_unclaimed: bool = True,
) -> str:
    metadata = job.get("metadata") if isinstance(job.get("metadata"), dict) else {}
    messages = metadata.get("operator_messages") if isinstance(metadata.get("operator_messages"), list) else []
    lines = []
    active_messages = active_prompt_operator_entries(active_messages or [])
    active_ids = {active.get("event_id") for active in active_messages if isinstance(active, dict)}
    if active_messages:
        lines.append("Newly delivered operator messages for this turn:")
    for entry in active_messages:
        line = _operator_message_line(entry)
        if line:
            lines.append(line)
    active_context = [
        entry
        for entry in messages
        if isinstance(entry, dict)
        and operator_entry_is_prompt_relevant(entry)
        and _operator_message_visible_in_prompt(entry, include_unclaimed=include_unclaimed)
        and entry.get("event_id") not in active_ids
    ]
    if active_context:
        if lines:
            lines.append("Still-active durable operator context:")
        for entry in active_context[-6:]:
            line = _operator_message_line(entry)
            if line:
                lines.append(line)
    return "\n".join(lines) if lines else "No active operator context."


def _operator_message_line(entry: dict[str, Any]) -> str:
    if not isinstance(entry, dict):
        return ""
    at = str(entry.get("at") or "")
    source = str(entry.get("source") or "operator")
    mode = str(entry.get("mode") or "steer")
    event_id = str(entry.get("event_id") or "")
    message = " ".join(str(entry.get("message") or "").split())
    if message:
        states = []
        if entry.get("claimed_at"):
            states.append("delivered")
        if entry.get("acknowledged_at"):
            states.append("acknowledged")
        if entry.get("superseded_at"):
            states.append("superseded")
        state_text = f" ({', '.join(states)})" if states else ""
        id_text = f" id={event_id}" if event_id else ""
        return f"-{id_text} {at} {source} {mode}{state_text}: {_clip_text(message, 420)}"
    return ""


def _lessons_for_prompt(job: dict[str, Any]) -> str:
    metadata = job.get("metadata") if isinstance(job.get("metadata"), dict) else {}
    lessons = metadata.get("lessons") if isinstance(metadata.get("lessons"), list) else []
    if not lessons:
        return "No durable lessons yet."
    reference_text = " ".join(str(job.get(key) or "") for key in ("title", "objective", "kind"))
    positive_lines = _positive_durable_lines_for_lesson_conflicts(metadata)
    stale_lesson_ids = _stale_negative_record_ids(metadata, kind="lesson")
    lines = []
    for entry in lessons[-5:]:
        if not isinstance(entry, dict):
            continue
        category = str(entry.get("category") or "memory")
        raw_lesson = str(entry.get("lesson") or "")
        record_id = _record_id_for_staleness(entry)
        conflicting_tokens = _negative_lesson_conflict_tokens(raw_lesson, positive_lines)
        if conflicting_tokens:
            lesson = (
                "Potentially stale negative lesson suppressed for "
                + ", ".join(conflicting_tokens[:6])
                + ". Re-verify against fresh evidence before using this claim."
            )
        elif record_id in stale_lesson_ids:
            lesson = "Potentially stale negative lesson suppressed after fresh contradictory evidence. Re-verify before using this claim."
        else:
            lesson = _lesson_prompt_text(raw_lesson, reference_text=reference_text)
        if lesson:
            lines.append(f"- {category}: {_clip_text(lesson, SECTION_ITEM_CHARS)}")
    return "\n".join(lines) if lines else "No durable lessons yet."


def _lesson_prompt_text(lesson: str, *, reference_text: str = "") -> str:
    lesson = " ".join(str(lesson or "").split())
    if "unsupported concrete tokens" not in lesson.lower():
        return lesson
    reference_norm = _normalize_claim_text(reference_text)
    stale_tokens = []
    seen = set()
    for token in _unsupported_tokens_from_lesson(lesson):
        cleaned = " ".join(str(token or "").split())
        if not cleaned:
            continue
        key = cleaned.lower()
        if reference_norm and _normalize_claim_text(cleaned) in reference_norm:
            continue
        if key in seen or not _stale_token_is_distinctive(cleaned):
            continue
        seen.add(key)
        stale_tokens.append(cleaned)
    if stale_tokens:
        return (
            "Evidence grounding rejected unsupported durable-record claims: "
            + ", ".join(stale_tokens[:8])
            + ". Re-verify them from fresh evidence before using them."
        )
    return "Evidence grounding rejected an unsupported durable record. Re-verify from fresh evidence before using it."


def _positive_durable_lines_for_lesson_conflicts(metadata: dict[str, Any]) -> list[str]:
    lines: list[str] = []
    for key in ("finding_ledger", "experiment_ledger", "source_ledger"):
        records = metadata.get(key) if isinstance(metadata.get(key), list) else []
        for record in records[-30:]:
            if isinstance(record, dict):
                text = _dict_scalar_text(record)
                if text:
                    lines.append(text)
    graph = metadata.get("memory_graph") if isinstance(metadata.get("memory_graph"), dict) else {}
    nodes = graph.get("nodes") if isinstance(graph.get("nodes"), list) else []
    for node in nodes[-30:]:
        if isinstance(node, dict):
            text = _dict_scalar_text(node)
            if text:
                lines.append(text)
    return lines


def _negative_lesson_conflict_tokens(lesson: str, positive_lines: list[str]) -> list[str]:
    lesson = " ".join(str(lesson or "").split())
    lesson_lower = lesson.lower()
    if not positive_lines or not any(marker in lesson_lower for marker in NEGATIVE_EXISTENCE_MARKERS):
        return []
    tokens = _distinctive_claim_tokens(lesson)
    conflicts: list[str] = []
    seen: set[str] = set()
    for token in tokens:
        key = token.lower()
        if key in seen:
            continue
        seen.add(key)
        if not _token_near_negative_marker(lesson, token):
            continue
        if _positive_line_contains_token(positive_lines, token):
            conflicts.append(token)
    return conflicts


def _dict_scalar_text(record: dict[str, Any]) -> str:
    parts: list[str] = []
    for key, value in record.items():
        if key in {"created_at", "updated_at", "event_id", "id"}:
            continue
        if isinstance(value, (str, int, float, bool)):
            parts.append(str(value))
        elif isinstance(value, dict):
            parts.append(_dict_scalar_text(value))
    return " ".join(part for part in parts if part)


def _distinctive_claim_tokens(text: str) -> list[str]:
    tokens: list[str] = []
    for raw in re.findall(r"\b[A-Za-z][A-Za-z0-9_.+-]{1,}\b", text):
        token = raw.strip("._+-")
        if token and _stale_token_is_distinctive(token):
            tokens.append(token)
    return tokens


def _token_near_negative_marker(text: str, token: str, *, window: int = 140) -> bool:
    text_lower = text.lower()
    token_lower = token.lower()
    start = 0
    while True:
        index = text_lower.find(token_lower, start)
        if index < 0:
            return False
        nearby = text_lower[max(0, index - window): index + len(token_lower) + window]
        if any(marker in nearby for marker in NEGATIVE_EXISTENCE_MARKERS):
            return True
        start = index + len(token_lower)


def _positive_line_contains_token(lines: list[str], token: str) -> bool:
    token_lower = token.lower()
    for line in lines:
        line_lower = line.lower()
        if token_lower not in line_lower:
            continue
        if any(marker in line_lower for marker in NEGATIVE_EVIDENCE_LINE_MARKERS):
            continue
        return True
    return False


def _memory_graph_for_prompt(job: dict[str, Any]) -> str:
    metadata = job.get("metadata") if isinstance(job.get("metadata"), dict) else {}
    stale_tokens = _stale_claim_tokens_for_prompt(
        metadata,
        reference_text=" ".join(str(job.get(key) or "") for key in ("title", "objective", "kind")),
    )
    return memory_graph_for_prompt(job, limit=10, stale_tokens=stale_tokens)


def _roadmap_for_prompt(job: dict[str, Any]) -> str:
    metadata = job.get("metadata") if isinstance(job.get("metadata"), dict) else {}
    roadmap = metadata.get("roadmap") if isinstance(metadata.get("roadmap"), dict) else {}
    if not roadmap:
        return (
            "No roadmap yet. If the objective is broad, multi-phase, or needs validation checkpoints, "
            "use record_roadmap to define compact milestones, features, acceptance criteria, and validation evidence."
        )
    milestones = roadmap.get("milestones") if isinstance(roadmap.get("milestones"), list) else []
    status_counts: dict[str, int] = {}
    validation_counts: dict[str, int] = {}
    for milestone in milestones:
        if not isinstance(milestone, dict):
            continue
        status = str(milestone.get("status") or "planned")
        validation_status = str(milestone.get("validation_status") or "not_started")
        status_counts[status] = status_counts.get(status, 0) + 1
        validation_counts[validation_status] = validation_counts.get(validation_status, 0) + 1
    lines = [
        _clip_text(
            f"{roadmap.get('status') or 'planned'}: {roadmap.get('title') or 'Roadmap'}"
            + (f" | current={roadmap.get('current_milestone')}" if roadmap.get("current_milestone") else ""),
            520,
        ),
        "Milestone counts: " + (", ".join(f"{key}={value}" for key, value in sorted(status_counts.items())) or "none"),
        "Validation counts: " + (", ".join(f"{key}={value}" for key, value in sorted(validation_counts.items())) or "none"),
    ]
    if roadmap.get("scope"):
        lines.append("Scope: " + _clip_text(str(roadmap.get("scope") or ""), 420))
    if roadmap.get("validation_contract"):
        lines.append("Validation contract: " + _clip_text(str(roadmap.get("validation_contract") or ""), 520))
    selected = [
        milestone for milestone in milestones
        if isinstance(milestone, dict)
        and str(milestone.get("status") or "planned") in {"active", "validating", "planned", "blocked"}
    ][:6]
    if not selected:
        selected = [milestone for milestone in milestones if isinstance(milestone, dict)][-4:]
    for milestone in selected[:6]:
        features = milestone.get("features") if isinstance(milestone.get("features"), list) else []
        open_features = sum(1 for feature in features if isinstance(feature, dict) and str(feature.get("status") or "planned") in {"planned", "active"})
        detail = " | ".join(
            bit
            for bit in [
                str(milestone.get("status") or "planned"),
                f"validation={milestone.get('validation_status') or 'not_started'}",
                f"p={milestone.get('priority') or 0}",
                str(milestone.get("title") or "milestone"),
                f"features={len(features)}/{open_features} open" if features else "",
            ]
            if bit
        )
        if milestone.get("acceptance_criteria"):
            detail += f" | accept={milestone.get('acceptance_criteria')}"
        if milestone.get("evidence_needed"):
            detail += f" | evidence={milestone.get('evidence_needed')}"
        if milestone.get("validation_result"):
            detail += f" | validation_result={milestone.get('validation_result')}"
        if milestone.get("next_action"):
            detail += f" | next={milestone.get('next_action')}"
        lines.append("- " + _clip_text(detail, 620))
    return "\n".join(lines)


def _tasks_for_prompt(job: dict[str, Any]) -> str:
    tasks = _metadata_list(job, "task_queue")
    if not tasks:
        return (
            "No durable task queue yet. If the objective is broad, use record_tasks "
            "to create a few concrete open branches with output contracts and acceptance criteria before continuing."
        )
    status_rank = {"active": 0, "open": 1, "blocked": 2, "done": 3, "skipped": 4}
    ranked = sorted(
        tasks,
        key=lambda task: (status_rank.get(str(task.get("status") or "open"), 9), -_as_int(task.get("priority"))),
    )
    counts: dict[str, int] = {}
    for task in tasks:
        status = str(task.get("status") or "open")
        counts[status] = counts.get(status, 0) + 1
    lines = ["Task counts: " + ", ".join(f"{key}={value}" for key, value in sorted(counts.items()))]
    selected = [task for task in ranked if str(task.get("status") or "open") in {"active", "open"}][:6]
    if len(selected) < 6:
        selected.extend([task for task in ranked if str(task.get("status") or "open") == "blocked"][: 6 - len(selected)])
    if len(selected) < 6:
        selected.extend([task for task in ranked if task not in selected][: 6 - len(selected)])
    for task in selected[:6]:
        output_contract = _task_output_contract(task)
        bits = [
            str(task.get("status") or "open"),
            f"priority={task.get('priority') or 0}",
            str(task.get("title") or "untitled"),
        ]
        if output_contract:
            bits.append(f"contract={output_contract}")
        detail = " | ".join(bit for bit in bits if bit)
        if task.get("goal"):
            detail += f" | goal={task.get('goal')}"
        if task.get("acceptance_criteria"):
            detail += f" | accept={task.get('acceptance_criteria')}"
        if task.get("evidence_needed"):
            detail += f" | evidence={task.get('evidence_needed')}"
        if task.get("stall_behavior"):
            detail += f" | stall={task.get('stall_behavior')}"
        if task.get("source_hint"):
            detail += f" | source_hint={task.get('source_hint')}"
        if task.get("result"):
            detail += f" | result={task.get('result')}"
        lines.append("- " + _clip_text(detail, 520))
    return "\n".join(lines)


def _task_output_contract(task: dict[str, Any]) -> str:
    metadata = task.get("metadata") if isinstance(task.get("metadata"), dict) else {}
    return str(task.get("output_contract") or task.get("contract") or metadata.get("output_contract") or metadata.get("contract") or "")


def _timeline_for_prompt(events: list[dict[str, Any]]) -> str:
    if not events:
        return "No timeline events yet."
    selected: list[tuple[str, str, str]] = []
    counts: dict[str, int] = {}
    for event in events:
        rendered = _timeline_event_for_prompt(event)
        if not rendered:
            continue
        at, event_type, detail = rendered
        counts[event_type] = counts.get(event_type, 0) + 1
        selected.append((at, event_type, detail))
    if not selected:
        return "No high-signal timeline events yet. Recent state covers raw tool activity."
    summary = ", ".join(f"{key}={value}" for key, value in sorted(counts.items()))
    lines = [f"High-signal timeline counts: {summary}"]
    for at, event_type, detail in selected[-TIMELINE_PROMPT_EVENTS:]:
        prefix = f"- {at} {event_type}: " if at else f"- {event_type}: "
        lines.append(prefix + _clip_text(detail, SECTION_ITEM_CHARS))
    return "\n".join(lines)


def _outcomes_for_prompt(events: list[dict[str, Any]]) -> str:
    """Summarize durable outputs so the worker sees progress, not just activity."""

    if not events:
        return "No durable outcomes visible in recent timeline."
    counts = outcome_counts(events, include_research=False, include_failures=True)
    summary = hourly_outcome_summary(counts)
    lines = [f"Outcome counts: {summary or 'none'}."]
    seen: set[str] = set()
    for event in reversed(events):
        parsed = model_update_event_parts(event, width=240, compact=False)
        if not parsed:
            continue
        label, text, _clock = parsed
        if label in {"DONE", "PLAN", "UPDATE"}:
            continue
        key = f"{label}:{text}"
        if key in seen:
            continue
        seen.add(key)
        lines.append(f"- {label.lower()}: {_clip_text(text, 360)}")
        if len(lines) >= 8:
            break
    if len(lines) == 1:
        lines.append("No durable output/finding/measurement records are visible; prioritize creating or accounting for one.")
    return "\n".join(lines)


def _ledgers_for_prompt(job: dict[str, Any]) -> str:
    metadata = job.get("metadata") if isinstance(job.get("metadata"), dict) else {}
    findings = _metadata_list(job, "finding_ledger")
    sources = _metadata_list(job, "source_ledger")
    stale_tokens = _stale_claim_tokens_for_prompt(
        metadata,
        reference_text=" ".join(str(job.get(key) or "") for key in ("title", "objective", "kind")),
    )
    stale_record_ids = _stale_negative_record_ids(metadata, kind="finding")
    stale_findings = [
        finding
        for finding in findings
        if _record_contains_stale_token(finding, stale_tokens)
        or _record_id_for_staleness(finding) in stale_record_ids
    ]
    active_findings = [finding for finding in findings if finding not in stale_findings]
    lines = [
        f"Finding ledger: {len(findings)} unique candidates.",
        f"Source ledger: {len(sources)} scored sources.",
    ]
    if stale_tokens:
        lines.append(
            "Unsupported/stale claim tokens to avoid until re-verified: "
            + ", ".join(stale_tokens[:12])
        )
    if active_findings:
        lines.append("Recent findings:")
        for finding in active_findings[-5:]:
            bits = [
                str(finding.get("name") or "unknown"),
                str(finding.get("category") or "").strip(),
                str(finding.get("location") or "").strip(),
                f"score={finding.get('score')}" if finding.get("score") is not None else "",
            ]
            lines.append("- " + _clip_text(" | ".join(bit for bit in bits if bit), 360))
    if stale_findings:
        lines.append(
            f"Suppressed {len(stale_findings)} stale finding(s) matching unsupported tokens; "
            "do not use them as facts until observed again."
        )
    stale_negative_records = _stale_negative_records_for_prompt(metadata, kind="finding")
    if stale_negative_records:
        lines.append("Contradicted negative findings suppressed:")
        for record in stale_negative_records[-4:]:
            lines.append(
                "- "
                + _clip_text(
                    f"{record.get('title') or 'finding'} token={record.get('token') or ''} evidence={record.get('evidence') or ''}",
                    360,
                )
            )
    if sources:
        usable_sources = [
            source
            for source in sources
            if _as_float(source.get("usefulness_score")) >= 0.2
            or _as_int(source.get("yield_count")) > 0
        ]
        low_quality_sources = [
            source
            for source in sources
            if _as_float(source.get("usefulness_score")) < 0.2
            and _as_int(source.get("yield_count")) <= 0
            and (_as_int(source.get("fail_count")) > 0 or source.get("warnings"))
        ]
        ranked = sorted(
            usable_sources,
            key=lambda item: (_as_float(item.get("usefulness_score")), _as_int(item.get("yield_count"))),
            reverse=True,
        )
        if ranked:
            lines.append("High-yield/current sources:")
            for source in ranked[:4]:
                lines.append(
                    "- "
                    + _clip_text(
                        f"{source.get('source')} type={source.get('source_type') or 'unknown'} "
                        f"score={source.get('usefulness_score')} findings={source.get('yield_count') or 0} "
                        f"fails={source.get('fail_count') or 0} outcome={source.get('last_outcome') or ''}",
                        420,
                    )
                )
        if low_quality_sources:
            lines.append("Low-yield/blocked source patterns to avoid:")
            for source in low_quality_sources[-3:]:
                lines.append(
                    "- "
                    + _clip_text(
                        f"{source.get('source')} type={source.get('source_type') or 'unknown'} "
                        f"score={source.get('usefulness_score')} fails={source.get('fail_count') or 0} "
                        f"warnings={', '.join(source.get('warnings') or [])} outcome={source.get('last_outcome') or ''}",
                        420,
                    )
                )
    return "\n".join(lines)


def _stale_claim_tokens_for_prompt(metadata: dict[str, Any], *, reference_text: str = "") -> list[str]:
    raw_tokens = metadata.get("unsupported_claim_tokens")
    tokens: list[str] = []
    seen: set[str] = set()
    candidates: list[Any] = []
    reference_norm = _normalize_claim_text(reference_text)
    if isinstance(raw_tokens, list):
        candidates.extend(raw_tokens)
    stale_records = metadata.get("stale_negative_records")
    if isinstance(stale_records, list):
        for record in stale_records:
            if isinstance(record, dict):
                candidates.append(record.get("token"))
    lessons = metadata.get("lessons")
    if isinstance(lessons, list):
        for lesson in lessons[-25:]:
            if not isinstance(lesson, dict):
                continue
            candidates.extend(_unsupported_tokens_from_lesson(str(lesson.get("lesson") or "")))
    for raw in candidates:
        token = " ".join(str(raw or "").split())
        if not token:
            continue
        if reference_norm and _normalize_claim_text(token) in reference_norm:
            continue
        if not _stale_token_is_distinctive(token):
            continue
        key = token.lower()
        if key in seen:
            continue
        seen.add(key)
        tokens.append(token)
    return tokens[-100:]


def _unsupported_tokens_from_lesson(lesson: str) -> list[str]:
    marker = "unsupported concrete tokens"
    if marker not in lesson.lower():
        return []
    match = re.search(r"unsupported concrete tokens for .*?:\s*(.*?)(?:\.\s+Treat matching|$)", lesson, flags=re.IGNORECASE)
    if not match:
        match = re.search(r"unsupported concrete tokens for .*?:\s*(.*?)(?:\.|$)", lesson, flags=re.IGNORECASE)
    if not match:
        return []
    return [part.strip() for part in match.group(1).split(",") if part.strip()]


def _stale_token_is_distinctive(token: str) -> bool:
    lowered = token.lower()
    if lowered.startswith(".") and re.match(r"^\.[a-z0-9][a-z0-9_-]{1,12}$", lowered):
        return lowered not in {".app", ".co", ".com", ".dev", ".edu", ".gov", ".io", ".net", ".org", ".www"}
    if lowered in {
        "api",
        "ascii",
        "blocked",
        "broken",
        "candidate",
        "candidates",
        "cdn",
        "cli",
        "critical",
        "cpu",
        "cuda",
        "discovered",
        "discovery",
        "ggml",
        "gguf",
        "gpu",
        "hf_token",
        "html",
        "http",
        "https",
        "incomplete",
        "json",
        "lfs",
        "not_found",
        "oid",
        "onnx",
        "planned",
        "python",
        "python3",
        "ram",
        "rest",
        "severe",
        "sha",
        "sha256",
        "search",
        "usable",
        "unvalidated",
        "valid",
        "validity",
        "validate",
        "validated",
        "vram",
        "xml",
        "xet",
        "yaml",
        "yml",
    }:
        return False
    if lowered.startswith((
        "art_",
        "step_",
        "step-",
        "shell_",
        "shell-",
        "web_",
        "web-",
        "episode-",
        "fact-",
        "source-",
        "quality-",
        "constraint-",
        "baseline-",
        "question-",
        "verified_",
        "verified-",
        "timeout_",
        "timeout-",
    )):
        return False
    if lowered.endswith((".md", ".py", ".json", ".yaml", ".yml", ".gguf", ".txt", ".log")):
        return False
    if lowered.startswith(("python-", "pip", "pip3")):
        return False
    if len(token) < 4:
        return False
    return (any(ch.isalpha() for ch in token) and any(ch.isdigit() for ch in token)) or (token.isupper() and len(token) >= 4)


def _normalize_claim_text(text: str) -> str:
    return re.sub(r"[^a-z0-9]+", "", str(text or "").lower())


def _record_contains_stale_token(record: dict[str, Any], stale_tokens: list[str]) -> bool:
    if not stale_tokens:
        return False
    text = " ".join(
        str(record.get(key) or "")
        for key in ("name", "category", "location", "contact", "reason", "source_url", "url")
    )
    metadata = record.get("metadata") if isinstance(record.get("metadata"), dict) else {}
    text += " " + " ".join(str(value) for value in metadata.values() if isinstance(value, (str, int, float)))
    for token in stale_tokens:
        pattern = r"(?<![A-Za-z0-9])" + re.escape(token) + r"(?![A-Za-z0-9])"
        if re.search(pattern, text, flags=re.IGNORECASE):
            return True
    return False


def _stale_negative_records_for_prompt(metadata: dict[str, Any], *, kind: str) -> list[dict[str, Any]]:
    records = metadata.get("stale_negative_records")
    if not isinstance(records, list):
        return []
    return [
        record
        for record in records
        if isinstance(record, dict) and str(record.get("kind") or "") == kind
    ]


def _stale_negative_record_ids(metadata: dict[str, Any], *, kind: str) -> set[str]:
    ids: set[str] = set()
    for record in _stale_negative_records_for_prompt(metadata, kind=kind):
        record_id = str(record.get("record_id") or "").strip()
        if record_id:
            ids.add(record_id)
    return ids


def _record_id_for_staleness(record: dict[str, Any]) -> str:
    for key in ("key", "event_id", "id"):
        value = str(record.get(key) or "").strip()
        if value:
            return value
    return _normalize_claim_text(str(record.get("name") or record.get("title") or ""))[:120]


def _experiments_for_prompt(job: dict[str, Any]) -> str:
    experiments = _metadata_list(job, "experiment_ledger")
    if not experiments:
        return (
            "No experiments tracked yet. If this objective involves improving, "
            "comparing, benchmarking, reducing, increasing, or otherwise measuring something, "
            "turn candidate ideas into record_experiment entries with exact config, metric, result, and next action."
        )
    measured = [experiment for experiment in experiments if experiment.get("metric_value") is not None]
    best = [
        experiment
        for experiment in measured
        if bool(experiment.get("best_observed"))
    ]
    status_counts: dict[str, int] = {}
    for experiment in experiments:
        status = str(experiment.get("status") or "planned")
        status_counts[status] = status_counts.get(status, 0) + 1
    lines = [
        f"Experiment counts: {', '.join(f'{key}={value}' for key, value in sorted(status_counts.items()))}.",
        f"Measured results: {len(measured)}.",
    ]
    if best:
        lines.append("Best observed results:")
        for experiment in best[-3:]:
            metric = format_metric_value(
                experiment.get("metric_name") or "metric",
                experiment.get("metric_value"),
                experiment.get("metric_unit") or "",
            )
            lines.append(
                "- "
                + _clip_text(" | ".join(
                    bit
                    for bit in [
                        str(experiment.get("title") or "experiment"),
                        metric,
                        f"result={experiment.get('result')}" if experiment.get("result") else "",
                        f"next={experiment.get('next_action')}" if experiment.get("next_action") else "",
                    ]
                    if bit
                ), 520)
            )
    recent = experiments[-4:]
    if recent:
        lines.append("Recent experiments:")
        for experiment in recent:
            metric = ""
            if experiment.get("metric_value") is not None:
                metric = format_metric_value(
                    experiment.get("metric_name") or "metric",
                    experiment.get("metric_value"),
                    experiment.get("metric_unit") or "",
                )
            delta = ""
            if experiment.get("delta_from_previous_best") is not None:
                delta = f"delta={experiment.get('delta_from_previous_best')}"
            lines.append(
                "- "
                + _clip_text(" | ".join(
                    bit
                    for bit in [
                        str(experiment.get("status") or "planned"),
                        str(experiment.get("title") or "experiment"),
                        metric,
                        delta,
                        f"next={experiment.get('next_action')}" if experiment.get("next_action") else "",
                    ]
                    if bit
                ), 520)
            )
    return "\n".join(lines)


def _operator_message_visible_in_prompt(entry: dict[str, Any], *, include_unclaimed: bool) -> bool:
    mode = str(entry.get("mode") or "steer").strip().lower().replace("-", "_")
    if entry.get("claimed_at") or mode == "note":
        return True
    return include_unclaimed and mode == "steer"


def _metadata_list(job: dict[str, Any], key: str) -> list[dict[str, Any]]:
    metadata = job.get("metadata") if isinstance(job.get("metadata"), dict) else {}
    values = metadata.get(key)
    if not isinstance(values, list):
        return []
    return [value for value in values if isinstance(value, dict)]


def _timeline_event_for_prompt(event: dict[str, Any]) -> tuple[str, str, str] | None:
    event_type = str(event.get("event_type") or "event")
    if event_type == "operator_message":
        return None
    title = " ".join(str(event.get("title") or "").split())
    body = " ".join(str(event.get("body") or "").split())
    metadata = event.get("metadata") if isinstance(event.get("metadata"), dict) else {}
    title_lower = title.lower()
    if event_type == "tool_result":
        status = str(metadata.get("status") or "").lower()
        if status not in TIMELINE_PROMPT_TOOL_STATUSES:
            return None
    elif event_type == "agent_message":
        if title_lower not in TIMELINE_PROMPT_AGENT_TITLES:
            return None
    elif event_type not in TIMELINE_PROMPT_EVENT_TYPES:
        return None
    at = str(event.get("created_at") or "")
    detail = title if title else event_type
    if body:
        detail = f"{detail}: {body}"
    if event_type == "tool_result":
        status = str(metadata.get("status") or "").lower()
        detail = f"{status} {detail}".strip()
    return at, event_type, detail


def _as_float(value: Any, default: float = 0.0) -> float:
    try:
        return float(value)
    except (TypeError, ValueError):
        return default


def _as_int(value: Any, default: int = 0) -> int:
    try:
        return int(value)
    except (TypeError, ValueError):
        return default
