#!/usr/bin/env bash
#
# Comprehensive devnet test orchestrator.
#
# Runs (in order, on top of an already-started 6-node devnet):
#   1. v10-rc-validation.sh                (15-section API smoke)
#   2. _devnet-full-sweep.sh               (baseline harnesses)
#   3. rc11 recovery tests (promote-crash + shutdown-mid-publish)
#   4. rfc38-all aggregator (lu5/lu5-pub/lu7/lu8/lu9/lu10/e2e/xcg/mm/scale/lj)
#   5. rc.12 feature probes (hub-rotation, multi-RPC failover, libp2p tunables,
#      CG-phonebook agent discovery, structured ACK rejection reasons)
#   6. node-ui smoke
#   7. soak suite (libp2p / SWM / RS)
#
# Bash 3.2 compatible (uses parallel indexed arrays instead of
# `declare -A`).
#
# IMPORTANT: do NOT edit this file while an instance is running — bash 3.2
# re-reads the script as it executes, and a mid-run byte shift will derail
# the parser (we've observed double-emitted PASS/FAIL log lines under that
# race). Wait for the run to finish, or duplicate the script before
# editing.
#
# Env knobs:
#   RESULTS_DIR         override the output directory
#                       (default: $REPO_ROOT/.devnet/comprehensive-results/<ts>)
#   SKIP_SOAK=1         skip the long soak suite
#   SOAK_ONLY=1         run only the soak suite
#   SKIP_PROBES=1       skip the rc.12-specific probes
#   SKIP_RFC38_EXTRAS=1 skip the rfc38-all suite
#   SKIP_UI=1           skip the node-ui smoke
#   FAIL_FAST=1         stop on first FAIL
#   SOAK_RS_SECONDS     length of the devnet-soak-rs run (default 1800)
#   SOAK_LIBP2P_CYCLES  libp2p-soak cycle count (default 5; each cycle ~60s)
#   SOAK_SWM_CYCLES     swm-soak cycle count (default 10)

set -u

REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
TS=$(date -u +'%Y%m%dT%H%M%SZ')
RESULTS="${RESULTS_DIR:-$REPO_ROOT/.devnet/comprehensive-results/$TS}"
mkdir -p "$RESULTS"
# `latest` symlink for convenience. Use the absolute path as the target so
# it resolves correctly even when RESULTS_DIR is overridden to a directory
# outside the default `<parent>/<ts>` layout. -sfn makes re-runs atomically
# replace any prior link without leaving a "latest/latest" trail.
ln -sfn "$RESULTS" "$(dirname "$RESULTS")/latest" 2>/dev/null || true

log() { echo "[orch $(date -u +'%H:%M:%S')] $*" | tee -a "$RESULTS/orchestrator.log"; }

# ── Pre-flight ───────────────────────────────────────────────────
log "Pre-flight: devnet status"
HARDHAT_PORT="${HARDHAT_PORT:-8545}"
API_PORT_BASE="${API_PORT_BASE:-9201}"
DEVNET_DIR="${DEVNET_DIR:-$REPO_ROOT/.devnet}"

if ! curl -sf "http://127.0.0.1:$HARDHAT_PORT" -X POST -H "Content-Type: application/json" \
   -d '{"jsonrpc":"2.0","method":"eth_chainId","params":[],"id":1}' > /dev/null 2>&1; then
  log "FATAL: Hardhat not responding on :$HARDHAT_PORT — start the devnet first ($REPO_ROOT/scripts/devnet.sh start)"
  exit 2
fi
AUTH=$(grep -v '^#' "$DEVNET_DIR/node1/auth.token" 2>/dev/null | head -1 || echo "")
if [ -z "$AUTH" ]; then
  log "FATAL: no auth token at $DEVNET_DIR/node1/auth.token"
  exit 2
fi
if ! curl -sf -H "Authorization: Bearer $AUTH" "http://127.0.0.1:$API_PORT_BASE/api/status" > /dev/null 2>&1; then
  log "FATAL: node 1 not responding on :$API_PORT_BASE"
  exit 2
fi
# Don't just trust node 1 — earlier orchestrator runs let v10-rc-validation
# and 4 sibling suites fail downstream because one node had silently died
# overnight. Check every node we'll actually exercise so triage time isn't
# wasted on "test broke" when the truth is "devnet broke".
NUM_NODES="${NUM_NODES:-6}"
DOWN_NODES=""
for n in $(seq 1 "$NUM_NODES"); do
  port=$((API_PORT_BASE + n - 1))
  code=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: Bearer $AUTH" \
    "http://127.0.0.1:$port/api/status" 2>/dev/null || echo "000")
  if [ "$code" != "200" ]; then
    DOWN_NODES="${DOWN_NODES} node${n}(port=$port,http=$code)"
  fi
done
if [ -n "$DOWN_NODES" ]; then
  log "FATAL: not all $NUM_NODES nodes are reachable. Down:${DOWN_NODES}"
  log "Hint: ./scripts/devnet.sh stop && ./scripts/devnet.sh start"
  exit 2
fi
export DKG_AUTH="$AUTH"
log "Devnet is up — all $NUM_NODES nodes healthy. Results dir: $RESULTS"

# ── Suite registry (parallel arrays; bash 3.2 compatible) ───────
SUITE_IDS=()
SUITE_CMDS=()
SUITE_GROUPS=()
SUITE_RESULTS=()
SUITE_LOGS=()
SUITE_ELAPSEDS=()

register() {
  SUITE_IDS+=("$1")
  SUITE_GROUPS+=("$2")
  SUITE_CMDS+=("$3")
  SUITE_RESULTS+=("PENDING")
  SUITE_LOGS+=("")
  SUITE_ELAPSEDS+=("0")
}

# Group: smoke
register "v10-rc-validation"   "smoke" "$REPO_ROOT/scripts/v10-rc-validation.sh"

# Group: sweep
register "devnet-full-sweep"   "sweep" "$REPO_ROOT/scripts/_devnet-full-sweep.sh"

# Group: cutover — OT-RFC-49 ciphertext->catalog proof (canonical + current).
# Previously not wired into any orchestrator; included so the cutover (curated
# publish/update + catalog-root commitment + random-sampling-proves-the-catalog)
# is exercised by the comprehensive run. Gate off with SKIP_RFC49=1.
[ -n "${SKIP_RFC49:-}" ] || register "rfc49-catalog" "cutover" "$REPO_ROOT/scripts/devnet-test-rfc49-catalog-sampling.sh"

# Group: rc11 recovery
register "rc11-promote-crash"  "rc11-recovery" "$REPO_ROOT/scripts/devnet-test-rc11-promote-crash-recovery.sh"
register "rc11-shutdown-mid"   "rc11-recovery" "$REPO_ROOT/scripts/devnet-test-rc11-shutdown-mid-publish.sh"

# Group: rfc38 extras (the all-aggregator runs lu5/lu5-pub/lu7/lu8/lu9/lu10/e2e/xcg/mm/scale/lj)
if [ "${SKIP_RFC38_EXTRAS:-0}" != "1" ]; then
  register "rfc38-all" "rfc38-extras" "$REPO_ROOT/scripts/devnet-test-rfc38-all.sh"
fi

# Group: rc.12 feature probes
if [ "${SKIP_PROBES:-0}" != "1" ]; then
  for p in hub-rotation multi-rpc-failover libp2p-tunables cg-phonebook ack-rejection-reasons; do
    register "probe-${p}" "rc12-probes" "$REPO_ROOT/scripts/devnet-probe-${p}.sh"
  done
fi

# Group: node-ui smoke
if [ "${SKIP_UI:-0}" != "1" ]; then
  register "node-ui-smoke" "node-ui" "$REPO_ROOT/scripts/devnet-test-node-ui-smoke.sh"
fi

# Group: soak (LONG)
if [ "${SKIP_SOAK:-0}" != "1" ]; then
  SOAK_RECIPIENT_PEER=$(curl -sf -H "Authorization: Bearer $AUTH" \
    "http://127.0.0.1:$((API_PORT_BASE + 1))/api/status" 2>/dev/null \
    | python3 -c "import sys,json;print(json.load(sys.stdin).get('peerId',''))" 2>/dev/null || echo "")
  SOAK_LIBP2P_CYCLES="${SOAK_LIBP2P_CYCLES:-5}"
  SOAK_SWM_CYCLES="${SOAK_SWM_CYCLES:-10}"
  SOAK_RS_SECONDS="${SOAK_RS_SECONDS:-1800}"

  register "libp2p-soak-short" "soak" \
    "env DKG_HOME=$DEVNET_DIR/node1 DKG_AUTH=$AUTH API=http://127.0.0.1:$API_PORT_BASE RECIPIENT_PEER_ID=$SOAK_RECIPIENT_PEER RECIPIENT=devnet-node-2 SENDER_TAG=rc12 TOTAL_CYCLES=$SOAK_LIBP2P_CYCLES INTERVAL_S=60 $REPO_ROOT/scripts/libp2p-soak-test.sh"

  # SWM soak — solo mode (PEERS_EXPECTED unset). Confirms write-tag
  # rate on local SWM survives N × 30s cycles; cross-peer delivery
  # is already exercised by rfc38-multi-member + rfc38-cross-cg.
  # PEERS_EXPECTED is a comma-separated TAG list (not a count); set
  # only if running concurrent operators sharing a SOAK_COHORT_ID.
  register "swm-soak-short" "soak" \
    "env DKG_HOME=$DEVNET_DIR/node1 DKG_AUTH=$AUTH API=http://127.0.0.1:$API_PORT_BASE SWM_CG_PUBLIC=devnet-test SWM_CG_CURATED=devnet-isolation SWM_INTERVAL_S=30 SWM_TOTAL_CYCLES=$SOAK_SWM_CYCLES SENDER_TAG=rc12 $REPO_ROOT/scripts/swm-soak-test.sh"

  register "devnet-soak-rs" "soak" \
    "$REPO_ROOT/scripts/devnet-soak-rs.sh 1 $SOAK_RS_SECONDS"
fi

# Apply SOAK_ONLY filter
if [ "${SOAK_ONLY:-0}" = "1" ]; then
  NEW_IDS=()
  NEW_CMDS=()
  NEW_GROUPS=()
  NEW_RESULTS=()
  NEW_LOGS=()
  NEW_ELAPSEDS=()
  i=0
  while [ "$i" -lt "${#SUITE_IDS[@]}" ]; do
    if [ "${SUITE_GROUPS[$i]}" = "soak" ]; then
      NEW_IDS+=("${SUITE_IDS[$i]}")
      NEW_CMDS+=("${SUITE_CMDS[$i]}")
      NEW_GROUPS+=("${SUITE_GROUPS[$i]}")
      NEW_RESULTS+=("PENDING")
      NEW_LOGS+=("")
      NEW_ELAPSEDS+=("0")
    fi
    i=$((i + 1))
  done
  SUITE_IDS=("${NEW_IDS[@]}")
  SUITE_CMDS=("${NEW_CMDS[@]}")
  SUITE_GROUPS=("${NEW_GROUPS[@]}")
  SUITE_RESULTS=("${NEW_RESULTS[@]}")
  SUITE_LOGS=("${NEW_LOGS[@]}")
  SUITE_ELAPSEDS=("${NEW_ELAPSEDS[@]}")
fi

log "Registered ${#SUITE_IDS[@]} suite(s):"
i=0
while [ "$i" -lt "${#SUITE_IDS[@]}" ]; do
  log "  - ${SUITE_IDS[$i]}  [${SUITE_GROUPS[$i]}]"
  i=$((i + 1))
done

# ── Run loop ────────────────────────────────────────────────────
START=$(date +%s)
TOTAL_PASS=0
TOTAL_FAIL=0
TOTAL_MISSING=0

i=0
while [ "$i" -lt "${#SUITE_IDS[@]}" ]; do
  id="${SUITE_IDS[$i]}"
  cmd="${SUITE_CMDS[$i]}"
  group="${SUITE_GROUPS[$i]}"
  logfile="$RESULTS/${id}.log"
  SUITE_LOGS[$i]="$logfile"

  # Extract the script path (last token in the command, possibly after env vars)
  bare_path=""
  for tok in $cmd; do
    case "$tok" in
      *.sh) bare_path="$tok" ;;
    esac
  done
  [ -z "$bare_path" ] && bare_path=$(echo "$cmd" | awk '{print $NF}')

  if [ ! -e "$bare_path" ]; then
    log "MISSING: $id ($bare_path)"
    SUITE_RESULTS[$i]="MISSING"
    TOTAL_MISSING=$((TOTAL_MISSING + 1))
    i=$((i + 1))
    if [ "${FAIL_FAST:-0}" = "1" ]; then
      log "FAIL_FAST=1 — aborting"
      break
    fi
    continue
  fi

  log "============================================================"
  log "RUN  $id  [$group]"
  log "============================================================"
  suite_start=$(date +%s)
  ( cd "$REPO_ROOT" && bash -c "$cmd" ) > "$logfile" 2>&1
  ec=$?
  suite_end=$(date +%s)
  elapsed=$((suite_end - suite_start))
  SUITE_ELAPSEDS[$i]="$elapsed"

  if [ "$ec" -eq 0 ]; then
    SUITE_RESULTS[$i]="PASS"
    TOTAL_PASS=$((TOTAL_PASS + 1))
    log "PASS $id  (${elapsed}s)"
  else
    SUITE_RESULTS[$i]="FAIL:$ec"
    TOTAL_FAIL=$((TOTAL_FAIL + 1))
    log "FAIL $id  (exit=$ec, ${elapsed}s)"
    log "  last 12 lines of $logfile:"
    tail -n 12 "$logfile" 2>/dev/null | sed 's/^/      /' | tee -a "$RESULTS/orchestrator.log"
    if [ "${FAIL_FAST:-0}" = "1" ]; then
      log "FAIL_FAST=1 — aborting"
      break
    fi
  fi
  i=$((i + 1))
done

END=$(date +%s)
WALL=$((END - START))

# ── Reports ─────────────────────────────────────────────────────
log ""
log "============================================================"
log "DONE — ${WALL}s wall (~$((WALL/60))m)"
log "PASS=$TOTAL_PASS FAIL=$TOTAL_FAIL MISSING=$TOTAL_MISSING TOTAL=${#SUITE_IDS[@]}"
log "============================================================"

# Markdown report
MD="$RESULTS/REPORT.md"
{
  echo "# Comprehensive devnet test report"
  echo
  echo "- **Started**: $(date -u -r $START +'%Y-%m-%dT%H:%M:%SZ')"
  echo "- **Ended**: $(date -u -r $END +'%Y-%m-%dT%H:%M:%SZ')"
  echo "- **Wall**: ${WALL}s (~$((WALL/60))m)"
  echo "- **Branch**: $(cd "$REPO_ROOT" && git rev-parse --abbrev-ref HEAD) @ $(cd "$REPO_ROOT" && git rev-parse --short HEAD)"
  echo "- **Results dir**: \`$RESULTS\`"
  echo
  echo "## Summary"
  echo
  echo "| | count |"
  echo "|---|---|"
  echo "| PASS | $TOTAL_PASS |"
  echo "| FAIL | $TOTAL_FAIL |"
  echo "| MISSING | $TOTAL_MISSING |"
  echo "| Total registered | ${#SUITE_IDS[@]} |"
  echo
  echo "## Suites"
  echo
  echo "| id | group | result | elapsed | log |"
  echo "|---|---|---|---:|---|"
  i=0
  while [ "$i" -lt "${#SUITE_IDS[@]}" ]; do
    logf=$(basename "${SUITE_LOGS[$i]}")
    echo "| \`${SUITE_IDS[$i]}\` | ${SUITE_GROUPS[$i]} | ${SUITE_RESULTS[$i]} | ${SUITE_ELAPSEDS[$i]}s | \`$logf\` |"
    i=$((i + 1))
  done
  echo
  if [ "$TOTAL_FAIL" -gt 0 ]; then
    echo "## Failures — last 25 lines of each failing log"
    echo
    i=0
    while [ "$i" -lt "${#SUITE_IDS[@]}" ]; do
      case "${SUITE_RESULTS[$i]}" in
        FAIL:*)
          echo "### ${SUITE_IDS[$i]}"
          echo
          echo '```'
          tail -n 25 "${SUITE_LOGS[$i]}" 2>/dev/null || echo "(no log)"
          echo '```'
          echo
          ;;
      esac
      i=$((i + 1))
    done
  fi
} > "$MD"

# JSON report
JSON="$RESULTS/REPORT.json"
{
  echo "{"
  echo "  \"startedAt\": \"$(date -u -r $START +'%Y-%m-%dT%H:%M:%SZ')\","
  echo "  \"endedAt\": \"$(date -u -r $END +'%Y-%m-%dT%H:%M:%SZ')\","
  echo "  \"wallSeconds\": $WALL,"
  echo "  \"branch\": \"$(cd "$REPO_ROOT" && git rev-parse --abbrev-ref HEAD)\","
  echo "  \"commit\": \"$(cd "$REPO_ROOT" && git rev-parse HEAD)\","
  echo "  \"totals\": { \"pass\": $TOTAL_PASS, \"fail\": $TOTAL_FAIL, \"missing\": $TOTAL_MISSING, \"registered\": ${#SUITE_IDS[@]} },"
  echo "  \"suites\": ["
  first=1
  i=0
  while [ "$i" -lt "${#SUITE_IDS[@]}" ]; do
    [ "$first" -eq 0 ] && echo ","
    first=0
    printf '    { "id": "%s", "group": "%s", "result": "%s", "elapsedSeconds": %s, "log": "%s" }' \
      "${SUITE_IDS[$i]}" "${SUITE_GROUPS[$i]}" "${SUITE_RESULTS[$i]}" "${SUITE_ELAPSEDS[$i]}" \
      "$(basename "${SUITE_LOGS[$i]}")"
    i=$((i + 1))
  done
  echo
  echo "  ]"
  echo "}"
} > "$JSON"

log "Report: $MD"
log "JSON:   $JSON"

if [ "$TOTAL_FAIL" -gt 0 ]; then
  exit 1
fi
exit 0
