#!/usr/bin/env bash
# evals/run.sh — Lightweight runner for AgentForge role-drift eval files.
#
# Modes:
#   ./evals/run.sh validate                   # static lint of all JSONL files
#   ./evals/run.sh checklist [N]              # pick N random asserts per role and print checklist (default N=3)
#   ./evals/run.sh checklist --role <role>    # same, but only one role
#
# Why not auto-run?
#   See evals/README.md — running asserts requires spawning real subagents,
#   which is expensive and noisy. The manual quarterly walkthrough is the
#   intended flow. This runner produces the validated raw material for that walkthrough.
#
# Output of `checklist`:
#   A markdown block ready to paste into docs/reviews/eval-quarterly-[YYYY-MM-DD].md
#   with checkbox lines for tech-lead to fill in pass/fail + notes.

set -uo pipefail

REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
EVALS_DIR="$REPO_ROOT/evals"
MODE="${1:-validate}"

require_jq() {
  if ! command -v jq >/dev/null 2>&1; then
    echo "ERROR: jq is required (brew install jq / apt install jq)" >&2
    exit 2
  fi
}

list_jsonl() {
  find "$EVALS_DIR" -maxdepth 1 -name '*.jsonl' -type f | sort
}

validate_one() {
  local f="$1"
  local line_no=0
  local errors=0
  while IFS= read -r line; do
    line_no=$((line_no + 1))
    [ -z "$line" ] && continue
    if ! echo "$line" | jq -e . >/dev/null 2>&1; then
      echo "  [INVALID JSON] $f:$line_no"
      errors=$((errors + 1))
      continue
    fi
    for field in id role input expects; do
      if ! echo "$line" | jq -e ".$field" >/dev/null 2>&1; then
        echo "  [MISSING FIELD '$field'] $f:$line_no"
        errors=$((errors + 1))
      fi
    done
    if ! echo "$line" | jq -e '.expects.behavior' >/dev/null 2>&1; then
      echo "  [MISSING expects.behavior] $f:$line_no"
      errors=$((errors + 1))
    fi
  done < "$f"
  return $(( errors > 0 ? 1 : 0 ))
}

cmd_validate() {
  require_jq
  local total=0 fail=0
  while IFS= read -r f; do
    total=$((total + 1))
    local count=$(grep -c . "$f" || true)
    if validate_one "$f"; then
      echo "  ✅ $(basename "$f") ($count asserts)"
    else
      echo "  ❌ $(basename "$f") — see errors above"
      fail=$((fail + 1))
    fi
  done < <(list_jsonl)
  echo ""
  echo "Total files: $total / Failed: $fail"
  [ "$fail" -gt 0 ] && exit 1
  exit 0
}

cmd_checklist() {
  require_jq
  local n=3
  local role_filter=""
  shift || true
  while [ $# -gt 0 ]; do
    case "$1" in
      --role) role_filter="$2"; shift 2;;
      [0-9]*) n="$1"; shift;;
      *) echo "Unknown arg: $1" >&2; exit 2;;
    esac
  done
  echo "# Eval Quarterly Walkthrough — $(date '+%Y-%m-%d')"
  echo ""
  echo "> Generated by \`evals/run.sh checklist $n\`. tech-lead: spawn the actual agent for each row, compare actual behavior to \`expects\`, fill in verdict + notes."
  echo ""
  while IFS= read -r f; do
    local role="$(basename "$f" .jsonl)"
    if [ -n "$role_filter" ] && [ "$role" != "$role_filter" ]; then continue; fi
    local count=$(grep -c . "$f" || true)
    [ "$count" -eq 0 ] && continue
    echo "## Role: $role ($count asserts total, sampling $n)"
    echo ""
    # awk-based reservoir sample so we don't depend on shuf (mac doesn't ship it)
    awk -v n="$n" 'BEGIN{srand()} {a[NR]=$0} END{c=NR; if (c<n) n=c; for(i=0;i<n;i++){k=int(rand()*c)+1; if(k in a){print a[k]; delete a[k]}else{i--}}}' "$f" | while IFS= read -r line; do
      local id=$(echo "$line" | jq -r .id)
      local input=$(echo "$line" | jq -r .input)
      local behavior=$(echo "$line" | jq -r '.expects.behavior // "-"')
      local to_role=$(echo "$line" | jq -r '.expects.to_role // "-"')
      local must_not=$(echo "$line" | jq -r '.expects.must_not // [] | join(", ")')
      local rationale=$(echo "$line" | jq -r '.rationale // "-"')
      echo "- [ ] **$id** — input: \"$input\""
      echo "    - expects.behavior: \`$behavior\`"
      [ "$to_role" != "-" ] && echo "    - expects.to_role: \`$to_role\`"
      [ -n "$must_not" ] && echo "    - must_not: \`$must_not\`"
      echo "    - rationale: $rationale"
      echo "    - **Verdict**: ✅ Pass / ❌ Fail / ⚠️ Drift"
      echo "    - **Notes**:"
      echo ""
    done
  done < <(list_jsonl)
  echo ""
  echo "## Summary"
  echo ""
  echo "- Total assertions checked: ___"
  echo "- Drift detected on: ___"
  echo "- Patches applied to .claude/agents/: ___"
  echo ""
  echo "Save this file as \`docs/reviews/eval-quarterly-$(date '+%Y-%m-%d').md\` after fill-in."
}

case "$MODE" in
  validate) cmd_validate ;;
  checklist) cmd_checklist "$@" ;;
  -h|--help|help)
    sed -n '2,15p' "$0"
    ;;
  *)
    echo "Unknown mode: $MODE. Use: validate | checklist [N] | help" >&2
    exit 2
    ;;
esac
