#!/bin/bash
# update_leaderboard.sh — regenerate LEADERBOARD.md from all benched run cards.
#
# Reads runs/*/run_card.meta + runs/*/bench_result.meta, sorts by bench_pass
# desc, emits a markdown table to LEADERBOARD.md. Idempotent — overwrites
# every call. Captures the current SOTA + which arm holds it as a footer.
#
# Usage: tools/orch/update_leaderboard.sh

set -u
SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
REPO_ROOT=$(cd "$SCRIPT_DIR/../.." && pwd)
cd "$REPO_ROOT"

LB=LEADERBOARD.md
TMP=$(mktemp)

cat > "$TMP" <<'EOF'
# LEADERBOARD.md

Auto-generated by `tools/orch/update_leaderboard.sh`. One row per arm in
`runs/`. Sorted by `bench_strip` (single-best strip-graded pass count) desc.
Ensemble (per-prompt max-pass across rows) computed by `tools/orch/condense.rail`.

| Rank | Arm | Status | Best val_loss | Bench/30 | Fund | IO | Tools | Comp | Adv | Compr | Wall (min) |
|-----:|-----|:------:|--------------:|:--------:|:----:|:--:|:-----:|:----:|:---:|:-----:|-----------:|
EOF

# Collect rows
ROWS=$(mktemp)
for run_dir in runs/*/; do
  rc=$run_dir/run_card.meta
  br=$run_dir/bench_result.meta
  [ -f "$rc" ] || continue

  get_key() { grep "^$1=" "$rc" 2>/dev/null | head -1 | cut -d'=' -f2-; }
  get_br() { grep "^$1=" "$br" 2>/dev/null | head -1 | cut -d'=' -f2-; }

  arm_id=$(get_key arm_id)
  status=$(get_key status)
  bench_pass="-"
  fund="-"; io="-"; tools="-"; comp="-"; adv="-"; compr="-"
  if [ -f "$br" ]; then
    bench_pass=$(get_br bench_pass)
    fund=$(get_br fundamentals)
    io=$(get_br practical_io)
    tools=$(get_br real_tools)
    comp=$(get_br compiler)
    adv=$(get_br advanced)
    compr=$(get_br comprehend)
  fi

  # Best val_loss: try .meta of the _best ckpt
  ckpt_prefix=$(get_key ckpt_prefix)
  best_vl="-"
  if [ -n "${ckpt_prefix:-}" ] && [ -f "${ckpt_prefix}_best.meta" ]; then
    best_vl=$(grep "^best_val_loss=" "${ckpt_prefix}_best.meta" | head -1 | cut -d'=' -f2)
    # round to 3 decimals if numeric
    if [ -n "$best_vl" ]; then
      best_vl=$(printf '%.3f' "$best_vl" 2>/dev/null || echo "$best_vl")
    fi
  fi

  # Wall-clock from launched_at + completed_at if both present
  wall="-"
  launched=$(get_key launched_at)
  completed=$(get_key completed_at)
  if [ -n "${launched:-}" ] && [ -n "${completed:-}" ]; then
    # Convert to epoch (BSD date on macOS)
    s=$(date -j -f "%Y-%m-%dT%H:%M:%S%z" "$launched" +%s 2>/dev/null || echo 0)
    e=$(date -j -f "%Y-%m-%dT%H:%M:%S%z" "$completed" +%s 2>/dev/null || echo 0)
    if [ "$s" -gt 0 ] && [ "$e" -gt "$s" ]; then
      wall=$(( (e - s + 30) / 60 ))
    fi
  fi

  # Sort key: bench_pass numeric (or -1 if none) for desc sort
  if [[ "$bench_pass" =~ ^[0-9]+$ ]]; then
    key=$bench_pass
  else
    key=-1
  fi

  printf '%d\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
    "$key" "$arm_id" "$status" "$best_vl" "$bench_pass" \
    "$fund" "$io" "$tools" "$comp" "$adv" "$compr" "$wall" \
    >> "$ROWS"
done

# Sort by key desc, emit table rows
SORTED=$(sort -t$'\t' -k1,1nr "$ROWS")
RANK=1
SOTA_PASS=-1
SOTA_ARM=""
while IFS=$'\t' read -r key arm_id status best_vl bench_pass fund io tools comp adv compr wall; do
  printf '| %d | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s |\n' \
    "$RANK" "$arm_id" "$status" "$best_vl" "$bench_pass" \
    "$fund" "$io" "$tools" "$comp" "$adv" "$compr" "$wall" \
    >> "$TMP"
  # SOTA only counts BENCHED arms — failed/killed/queued/running don't qualify
  # even if a stale bench_result.meta says bench_pass=N.
  if [ "$status" = "benched" ] && [ "$key" -gt "$SOTA_PASS" ] 2>/dev/null; then
    SOTA_PASS=$key
    SOTA_ARM=$arm_id
  fi
  RANK=$((RANK + 1))
done <<< "$SORTED"

rm -f "$ROWS"

cat >> "$TMP" <<EOF

---

**Current SOTA (single-best, strip-graded):** $SOTA_ARM at $SOTA_PASS/30.
Ensemble (per-prompt max-pass) is the honest project SOTA — see \`condense.rail\` output.

Generated $(date +%Y-%m-%dT%H:%M:%S%z) by tools/orch/update_leaderboard.sh.
EOF

mv "$TMP" "$LB"
echo "wrote $LB ($((RANK - 1)) rows; SOTA=$SOTA_ARM @ $SOTA_PASS/30)"
exit 0
