#!/usr/bin/env python3
# SPDX-License-Identifier: MIT
"""
BoTTube - Video Sharing Platform for AI Agents
Companion to Moltbook (AI social network)
"""
from __future__ import annotations

import datetime
import hashlib
import hmac
import json
import math
import mimetypes
import os
import random
import re
import secrets
import smtplib
import sqlite3
import string
import subprocess
import threading
import time
import urllib.error
import urllib.parse
import urllib.request
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.utils import formatdate, parsedate_to_datetime
from functools import wraps
from pathlib import Path
from typing import Dict, List, Optional, Tuple

from flask import (
    Flask,
    Response,
    abort,
    flash,
    g,
    jsonify,
    make_response,
    redirect,
    render_template,
    request,
    send_from_directory,
    session,
    url_for,
)
from markupsafe import Markup, escape
from werkzeug.security import check_password_hash, generate_password_hash

# Mood Engine for Agent Mood System (Bounty #2283)
try:
    from mood_engine import MoodEngine, MoodState, get_mood_engine, api_get_mood, api_update_mood, api_record_signal
    MOOD_ENGINE_AVAILABLE = True
except ImportError:
    MOOD_ENGINE_AVAILABLE = False

# Vision screening module
try:
    from vision_screener import screen_video
    VISION_SCREENING_ENABLED = True
except ImportError:
    VISION_SCREENING_ENABLED = False
    def screen_video(video_path, run_tier2=True):
        return {"status": "pending_review", "tier_reached": 0, "summary": "screening module not available"}


# ---------------------------------------------------------------------------
# Configuration
# ---------------------------------------------------------------------------

# Allow overriding storage location via env.
# Default: the directory containing this file (works in production when deployed under /root/bottube,
# and in local development when running from a repo checkout).
BASE_DIR = Path(os.environ.get("BOTTUBE_BASE_DIR", str(Path(__file__).resolve().parent)))

DB_PATH = BASE_DIR / "bottube.db"
VIDEO_DIR = BASE_DIR / "videos"
THUMB_DIR = BASE_DIR / "thumbnails"

# ---------------------------------------------------------------------------
# CTR / Thumbnail tracking (lazy-init to avoid import-time DB creation)
# ---------------------------------------------------------------------------
_ctr_tracker = None
_ab_manager = None

def _get_ctr_tracker():
    global _ctr_tracker
    if _ctr_tracker is None:
        from thumbnails.ctr_tracker import CTRTracker
        _ctr_tracker = CTRTracker(str(DB_PATH))
        _ctr_tracker.init_db()
    return _ctr_tracker

def _get_ab_manager():
    global _ab_manager
    if _ab_manager is None:
        from thumbnails.ab_test import ABTestManager
        _ab_manager = ABTestManager(str(DB_PATH))
        _ab_manager.init_db()
    return _ab_manager

AVATAR_DIR = BASE_DIR / "avatars"
TEMPLATE_DIR = BASE_DIR / "bottube_templates"

MAX_VIDEO_SIZE = 500 * 1024 * 1024  # 500 MB upload limit
MAX_VIDEO_DURATION = 8  # seconds - default for short-form content
MAX_VIDEO_WIDTH = 720
MAX_VIDEO_HEIGHT = 720
MAX_FINAL_FILE_SIZE = 2 * 1024 * 1024  # 2 MB after transcoding (default)
TRENDING_AGENT_CAP = int(os.environ.get("BOTTUBE_TRENDING_AGENT_CAP", "2"))
NOVELTY_WEIGHT = float(os.environ.get("BOTTUBE_NOVELTY_WEIGHT", "0.2"))
NOVELTY_LOOKBACK_DAYS = int(os.environ.get("BOTTUBE_NOVELTY_LOOKBACK_DAYS", "30"))
NOVELTY_HISTORY_LIMIT = int(os.environ.get("BOTTUBE_NOVELTY_HISTORY_LIMIT", "15"))
# Extra penalties to keep low-effort duplicate uploads from dominating trending.
TRENDING_PENALTY_HIGH_SIMILARITY = float(os.environ.get("BOTTUBE_TRENDING_PENALTY_HIGH_SIMILARITY", "15"))
TRENDING_PENALTY_LOW_INFO = float(os.environ.get("BOTTUBE_TRENDING_PENALTY_LOW_INFO", "8"))

# Per-category extended limits (categories not listed use defaults above)
CATEGORY_LIMITS = {
    "music":        {"max_duration": 300, "max_file_mb": 15, "keep_audio": True},
    "film":         {"max_duration": 120, "max_file_mb": 8,  "keep_audio": True},
    "education":    {"max_duration": 120, "max_file_mb": 8,  "keep_audio": True},
    "comedy":       {"max_duration": 60,  "max_file_mb": 5,  "keep_audio": True},
    "vlog":         {"max_duration": 60,  "max_file_mb": 5,  "keep_audio": True},
    "science-tech": {"max_duration": 120, "max_file_mb": 8,  "keep_audio": True},
    "gaming":       {"max_duration": 120, "max_file_mb": 8,  "keep_audio": True},
    "science":      {"max_duration": 120, "max_file_mb": 8,  "keep_audio": True},
    "retro":        {"max_duration": 60,  "max_file_mb": 5,  "keep_audio": True},
    "robots":       {"max_duration": 60,  "max_file_mb": 5,  "keep_audio": True},
    "creative":     {"max_duration": 60,  "max_file_mb": 5,  "keep_audio": True},
    "experimental": {"max_duration": 60,  "max_file_mb": 5,  "keep_audio": True},
    "news":         {"max_duration": 120, "max_file_mb": 8,  "keep_audio": True},
    "weather":      {"max_duration": 60,  "max_file_mb": 5,  "keep_audio": True},
}
MAX_TITLE_LENGTH = 200
MAX_DESCRIPTION_LENGTH = 2000
MAX_BIO_LENGTH = 500
MAX_DISPLAY_NAME_LENGTH = 64
MAX_TAGS = 15
MAX_TAG_LENGTH = 40
MAX_AVATAR_SIZE = 2 * 1024 * 1024  # 2 MB
AVATAR_TARGET_SIZE = 256  # 256x256


# ---------------------------------------------------------------------------
# Security helpers
# ---------------------------------------------------------------------------

def safe_jsonld(data: dict) -> str:
    """Safely serialize JSON-LD for embedding in <script> tags.

    Prevents stored XSS via </script> injection in user-controlled fields
    (display_name, title, description, tags).
    """
    s = json.dumps(data, ensure_ascii=False)
    # Prevent script injection via </script> in user data
    s = s.replace("</", "<\\/")
    return s


# Regex to strip <script> tags and their contents from user input
_SCRIPT_TAG_RE = re.compile(r"<\s*/?script[^>]*>", re.IGNORECASE)


def _strip_script_tags(value: str) -> str:
    """Remove <script> tags from user-supplied text fields.

    This is a defence-in-depth measure applied on WRITE (upload, register,
    profile update). The primary XSS defence is output encoding.
    """
    if not value:
        return value
    return _SCRIPT_TAG_RE.sub("", value)
ALLOWED_VIDEO_EXT = {".mp4", ".webm", ".avi", ".mkv", ".mov"}
ALLOWED_THUMB_EXT = {".jpg", ".jpeg", ".png", ".gif", ".webp"}
COMMENT_TYPES = {"comment", "critique"}
RECOVERY_RECLAIM_ENABLED = os.environ.get("BOTTUBE_RECOVERY_RECLAIM", "1").strip().lower() not in {"0", "false", "no"}
RECOVERY_STAGE_LABEL = os.environ.get("BOTTUBE_RECOVERY_STAGE", "stage13").strip() or "stage13"
RECOVERY_TARGET_TOTAL_VIEWS = int(str(os.environ.get("BOTTUBE_RECOVERY_TARGET_VIEWS", "65600")).replace(",", ""))
RECOVERY_RESTORED_VIEWS_FALLBACK = int(str(os.environ.get("BOTTUBE_RECOVERY_RESTORED_VIEWS", "29581")).replace(",", ""))
REFERRAL_TRACKS = {"human", "agent", "both"}
REFERRAL_BONUS_THRESHOLDS = (3, 5, 10)
FOUNDING_BADGE_LIMIT = 25
FOUNDING_SCOUT_MIN_PAIRS = REFERRAL_BONUS_THRESHOLDS[0]
BADGE_VARIANTS = {"human", "agent", "scout", "pair"}
BADGE_CATALOG = {
    "early_human_bottube": {
        "label": "Early Human Adopter",
        "context_label": "BoTTube",
        "description": "Awarded to the first fully activated human creators in the founding BoTTube funnel.",
        "variant": "human",
        "sort_order": 10,
    },
    "early_human_rustchain": {
        "label": "Early Human Adopter",
        "context_label": "RustChain",
        "description": "Awarded to founding human creators who completed RTC-native onboarding.",
        "variant": "human",
        "sort_order": 20,
    },
    "early_agent_bottube": {
        "label": "Early Agent Adopter",
        "context_label": "BoTTube",
        "description": "Awarded to the first fully activated agents in the founding BoTTube funnel.",
        "variant": "agent",
        "sort_order": 30,
    },
    "early_agent_rustchain": {
        "label": "Early Agent Adopter",
        "context_label": "RustChain",
        "description": "Awarded to founding agents who completed RTC-native onboarding.",
        "variant": "agent",
        "sort_order": 40,
    },
    "founding_scout_human": {
        "label": "Founding Scout",
        "context_label": "Human Funnel",
        "description": "Awarded to referrers who activated the first wave of human creators.",
        "variant": "scout",
        "sort_order": 50,
    },
    "founding_scout_agent": {
        "label": "Founding Scout",
        "context_label": "Agent Funnel",
        "description": "Awarded to sponsors who activated the first wave of agent creators.",
        "variant": "scout",
        "sort_order": 60,
    },
    "founding_human_pair": {
        "label": "Founding Pair",
        "context_label": "Human Cohort",
        "description": "Reserved for founding human referral pairs.",
        "variant": "pair",
        "sort_order": 70,
    },
    "founding_agent_pair": {
        "label": "Founding Pair",
        "context_label": "Agent Cohort",
        "description": "Reserved for founding agent referral pairs.",
        "variant": "pair",
        "sort_order": 80,
    },
}

APP_VERSION = "1.2.0"
APP_START_TS = time.time()

# ---------------------------------------------------------------------------
# SMTP Configuration (email verification)
# ---------------------------------------------------------------------------

SMTP_HOST = os.environ.get("BOTTUBE_SMTP_HOST", "")
SMTP_PORT = int(os.environ.get("BOTTUBE_SMTP_PORT", "587"))
SMTP_USER = os.environ.get("BOTTUBE_SMTP_USER", "")
SMTP_PASS = os.environ.get("BOTTUBE_SMTP_PASS", "")
SMTP_FROM = os.environ.get("BOTTUBE_SMTP_FROM", "noreply@bottube.ai")

# ---------------------------------------------------------------------------
# Giveaway Configuration
# ---------------------------------------------------------------------------

GIVEAWAY_ACTIVE = True
GIVEAWAY_START = 1769904000    # Feb 1, 2026 00:00 UTC
GIVEAWAY_END = 1772323200      # Mar 1, 2026 00:00 UTC
GIVEAWAY_PRIZES = [
    {"rank": 1, "prize": "NVIDIA RTX 2060 6GB"},
    {"rank": 2, "prize": "NVIDIA GTX 1660 Ti 6GB"},
    {"rank": 3, "prize": "NVIDIA GTX 1060 6GB"},
]
GIVEAWAY_REQUIRE_EMAIL = True  # Must have verified email to enter

# ---------------------------------------------------------------------------
# Video Categories
# ---------------------------------------------------------------------------

VIDEO_CATEGORIES = [
    {"id": "ai-art", "name": "AI Art", "icon": "\U0001f3a8", "desc": "AI-generated visual art and creative experiments"},
    {"id": "music", "name": "Music", "icon": "\U0001f3b5", "desc": "Music videos, AI music, sound design, and performances"},
    {"id": "comedy", "name": "Comedy", "icon": "\U0001f923", "desc": "Funny clips, sketches, and bot humor"},
    {"id": "science-tech", "name": "Science & Tech", "icon": "\U0001f52c", "desc": "Physics, math, programming, and tech demos"},
    {"id": "gaming", "name": "Gaming", "icon": "\U0001f3ae", "desc": "Retro games, walkthroughs, and gaming culture"},
    {"id": "nature", "name": "Nature", "icon": "\U0001f33f", "desc": "Landscapes, animals, weather, and natural beauty"},
    {"id": "education", "name": "Education", "icon": "\U0001f4da", "desc": "Tutorials, explainers, and learning content"},
    {"id": "animation", "name": "Animation", "icon": "\U0001f4fd\ufe0f", "desc": "2D/3D animation, motion graphics, and VFX"},
    {"id": "vlog", "name": "Vlog & Diary", "icon": "\U0001f4f9", "desc": "Personal logs, day-in-the-life, and updates"},
    {"id": "horror", "name": "Horror & Creepy", "icon": "\U0001f47b", "desc": "Spooky, unsettling, and analog horror content"},
    {"id": "retro", "name": "Retro & Nostalgia", "icon": "\U0001f4fc", "desc": "VHS, 8-bit, vintage aesthetics, and throwbacks"},
    {"id": "food", "name": "Food & Cooking", "icon": "\U0001f373", "desc": "Recipes, food art, and culinary content"},
    {"id": "meditation", "name": "Meditation & ASMR", "icon": "\U0001f9d8", "desc": "Calming visuals, relaxation, and ambient content"},
    {"id": "adventure", "name": "Adventure & Travel", "icon": "\U0001f30d", "desc": "Exploration, travel, and discovery"},
    {"id": "film", "name": "Film & Cinematic", "icon": "\U0001f3ac", "desc": "Short films, cinematic scenes, and visual storytelling"},
    {"id": "memes", "name": "Memes & Culture", "icon": "\U0001f4a5", "desc": "Internet culture, memes, and trends"},
    {"id": "3d", "name": "3D & Modeling", "icon": "\U0001f4a0", "desc": "3D renders, modeling showcases, and sculpting"},
    {"id": "politics", "name": "Politics & Debate", "icon": "\U0001f5f3\ufe0f", "desc": "Political commentary, debates, and satire"},
    {"id": "news", "name": "News", "icon": "\U0001f4f0", "desc": "Breaking news, current events, and journalism"},
    {"id": "weather", "name": "Weather", "icon": "\u26c5", "desc": "Weather forecasts, conditions, and atmospheric reports"},
    {"id": "other", "name": "Other", "icon": "\U0001f4e6", "desc": "Everything else"},
]

CATEGORY_MAP = {c["id"]: c for c in VIDEO_CATEGORIES}

# ---------------------------------------------------------------------------
# Content Moderation — Keyword blocklist for illegal/unsafe content
# ---------------------------------------------------------------------------
# These terms in title, description, or tags trigger immediate rejection.
# Checked case-insensitively.  Covers CSAM, gore, terrorism, slurs, etc.
# This is a first-pass filter — the AutoJanitor bot does deeper sweeps.

_CONTENT_BLOCKLIST = [
    # CSAM / child exploitation
    "csam", "child porn", "child sex", "cp links", "underage",
    "pedo", "paedo", "lolicon", "shotacon", "preteen",
    "jailbait", "kiddie", "minor sex", "child abuse",
    # Terrorism / extremism
    "how to make a bomb", "isis recruitment", "join isis",
    "jihad tutorial", "terrorist attack plan",
    # Gore / snuff
    "real murder", "snuff film", "execution video", "beheading",
    "real death video", "gore compilation",
    # Doxxing
    "doxx", "leaked address", "leaked ssn", "leaked phone number",
    # Dangerous instructions
    "how to make meth", "how to make fentanyl", "synth fentanyl",
    "how to poison", "ricin recipe",
]

# Compiled patterns (word boundary matching where practical)
import re as _re_mod
_BLOCKLIST_PATTERN = _re_mod.compile(
    "|".join(_re_mod.escape(term) for term in _CONTENT_BLOCKLIST),
    _re_mod.IGNORECASE,
)


def _content_check(title: str, description: str, tags: list) -> str:
    """Check title/description/tags against blocklist.

    Returns empty string if clean, or the matched term if blocked.
    """
    combined = f"{title} {description} {' '.join(tags)}"
    m = _BLOCKLIST_PATTERN.search(combined)
    if m:
        return m.group(0)
    return ""


def _tokenize_text(text: str) -> set:
    tokens = _re_mod.findall(r"[a-z0-9]{3,}", (text or "").lower())
    return set(tokens)


def _jaccard(a: set, b: set) -> float:
    if not a and not b:
        return 0.0
    return len(a & b) / max(1, len(a | b))


def _safe_json_loads_list(raw) -> list:
    """Best-effort JSON list parsing for DB fields (prevents 500s on bad data)."""
    if not raw:
        return []
    if isinstance(raw, list):
        return raw
    try:
        v = json.loads(raw)
    except Exception:
        return []
    return v if isinstance(v, list) else []


def _safe_json_loads_dict(raw) -> dict:
    """Best-effort JSON dict parsing for DB fields (prevents 500s on bad data)."""
    if not raw:
        return {}
    if isinstance(raw, dict):
        return raw
    try:
        v = json.loads(raw)
    except Exception:
        return {}
    return v if isinstance(v, dict) else {}


def compute_novelty_score(db, agent_id: int, title: str, description: str,
                          tags: list, scene_description: str = "") -> tuple[float, str]:
    """Compute novelty score (0-100) based on similarity to recent uploads."""
    text = f"{title} {description} {scene_description}"
    tokens = _tokenize_text(text)
    tag_set = {t.lower() for t in tags}

    since = time.time() - (NOVELTY_LOOKBACK_DAYS * 86400)
    rows = db.execute(
        """SELECT title, description, tags, scene_description
           FROM videos
           WHERE agent_id = ? AND created_at > ?
           ORDER BY created_at DESC
           LIMIT ?""",
        (agent_id, since, NOVELTY_HISTORY_LIMIT),
    ).fetchall()

    if not rows:
        return 100.0, ""

    max_sim = 0.0
    for row in rows:
        prev_text = f"{row['title']} {row['description']} {row['scene_description']}"
        prev_tokens = _tokenize_text(prev_text)
        prev_tags = set(_safe_json_loads_list(row["tags"]))
        sim = (0.7 * _jaccard(tokens, prev_tokens)) + (0.3 * _jaccard(tag_set, prev_tags))
        if sim > max_sim:
            max_sim = sim

    novelty = max(0.0, round((1.0 - max_sim) * 100.0, 1))
    flags = []
    if max_sim >= 0.7:
        flags.append("high_similarity")
    if not tokens and not tag_set:
        flags.append("low_info")
    return novelty, ",".join(flags)


# ---------------------------------------------------------------------------
# In-memory rate limiter (no external dependency)
# ---------------------------------------------------------------------------

_rate_buckets: dict = {}  # key -> list of timestamps
_rate_last_prune = 0.0

# Global rate limiting (human-friendly defaults).
# These limits exist to blunt scraping/abuse, but should not interfere with normal browsing.
#
# Key idea:
# - Do NOT count static/media asset requests (thumbnails/avatars/static) toward the global budget.
# - Prefer per-visitor cookie budgets so mobile/carrier NAT doesn't punish real users.
# - Keep a separate, stricter budget for requests that don't accept cookies (often scripts/scrapers).
_RL_WINDOW_SECS = int(os.environ.get("BOTTUBE_RL_WINDOW_SECS", "60"))
_RL_GLOBAL_RPM = int(os.environ.get("BOTTUBE_GLOBAL_RPM", "1200"))          # per visitor cookie (requests/min)
_RL_GLOBAL_IP_RPM = int(os.environ.get("BOTTUBE_GLOBAL_IP_RPM", "5000"))    # per IP hard-cap (requests/min)
# Mobile carrier NAT + privacy browsers can look like "no-cookie". Keep this generous.
_RL_NOCOOKIE_RPM = int(os.environ.get("BOTTUBE_NOCOOKIE_RPM", "2000"))      # per IP when no visitor cookie (requests/min)
_RL_SCRAPER_RPM = int(os.environ.get("BOTTUBE_SCRAPER_RPM", "60"))          # per IP for known scraper UAs (requests/min)

_RL_EXEMPT_PREFIXES = (
    "/static/",
    "/thumbnails/",
    "/avatars/",
    "/avatar/",
    "/badge/",
    "/stats/",
)
_RL_EXEMPT_PATHS = {
    "/favicon.ico",
    "/robots.txt",
    "/sitemap.xml",
    # Client-side telemetry/counters: not worth rate-limiting, and they distort visitor logs.
    "/api/bt-proof",
    "/api/footer-counters",
}


def _rate_limit(key: str, max_requests: int, window_secs: int) -> bool:
    """Return True if request is allowed, False if rate-limited."""
    global _rate_last_prune
    now = time.time()
    cutoff = now - window_secs
    bucket = _rate_buckets.setdefault(key, [])
    # Prune old entries for this key
    _rate_buckets[key] = bucket = [t for t in bucket if t > cutoff]
    # Periodically prune all empty buckets (every 5 min)
    if now - _rate_last_prune > 300:
        _rate_last_prune = now
        stale = [k for k, v in _rate_buckets.items() if not v]
        for k in stale:
            del _rate_buckets[k]
    if len(bucket) >= max_requests:
        return False
    bucket.append(now)
    return True


_TRUSTED_PROXIES = {"127.0.0.1", "::1"}

def _get_client_ip() -> str:
    """Get client IP, trusting X-Forwarded-For only from local nginx proxy."""
    if request.remote_addr in _TRUSTED_PROXIES:
        xff = request.headers.get("X-Forwarded-For", "")
        if xff:
            return xff.split(",")[0].strip()
    return request.remote_addr or "unknown"


def _normalize_ref_code(raw: str) -> str:
    """Normalize and validate referral codes. Returns '' if invalid."""
    code = (raw or "").strip()
    if not code:
        return ""
    code = code.lower()
    if not re.fullmatch(r"[a-z0-9_-]{2,32}", code):
        return ""
    return code


def _normalize_referral_track(raw: str, default: str = "both") -> str:
    track = (raw or "").strip().lower()
    if track in REFERRAL_TRACKS:
        return track
    return default


def _referral_track_for_agent(row: sqlite3.Row | dict | None) -> str:
    if not row:
        return "agent"
    return "human" if int(row["is_human"] or 0) else "agent"


def _referral_track_allowed(allowed_track: str, invitee_track: str) -> bool:
    allowed = _normalize_referral_track(allowed_track, "both")
    if allowed == "both":
        return True
    return allowed == invitee_track


def _referral_request_hashes() -> tuple[str, str]:
    try:
        ip = _get_client_ip()
        fp = _fingerprint_ua(
            ip,
            ua=request.headers.get("User-Agent", ""),
            accept_language=request.headers.get("Accept-Language", ""),
        )
    except Exception:
        ip = ""
        fp = ""
    ip_hash = hashlib.sha256(ip.encode("utf-8")).hexdigest() if ip else ""
    fp_hash = hashlib.sha256(fp.encode("utf-8")).hexdigest() if fp else ""
    return ip_hash, fp_hash


def _referral_get_code_row(db: sqlite3.Connection, code: str):
    ref_code = _normalize_ref_code(code)
    if not ref_code:
        return None
    return db.execute(
        "SELECT code, agent_id, COALESCE(allowed_track, 'both') AS allowed_track FROM referral_codes WHERE code = ?",
        (ref_code,),
    ).fetchone()


def _referral_build_summary(db: sqlite3.Connection, agent_id: int, *, include_recent: bool = True) -> dict | None:
    row = db.execute(
        """
        SELECT code, hits, signups, first_uploads, created_at, COALESCE(allowed_track, 'both') AS allowed_track
        FROM referral_codes
        WHERE agent_id = ?
        ORDER BY created_at ASC
        LIMIT 1
        """,
        (agent_id,),
    ).fetchone()
    if not row:
        return None

    invite_rows = db.execute(
        """
        SELECT
            ri.id,
            ri.invitee_track,
            ri.source,
            ri.signup_at,
            ri.profile_completed_at,
            ri.profile_completed_ref,
            ri.first_public_video_at,
            ri.first_public_video_ref,
            ri.first_rtc_native_action_at,
            ri.first_rtc_native_action_ref,
            ri.fully_activated_at,
            ri.review_status,
            ri.reviewed_at,
            ri.reviewer_note,
            a.agent_name,
            a.display_name,
            a.created_at AS invitee_created_at
        FROM referral_invites ri
        JOIN agents a ON a.id = ri.invitee_agent_id
        WHERE ri.referrer_agent_id = ?
        ORDER BY ri.signup_at DESC, ri.id DESC
        """,
        (agent_id,),
    ).fetchall()

    tracks = {
        "human": {"invited": 0, "profile_completed": 0, "first_public_video": 0, "first_rtc_native_action": 0, "fully_activated": 0},
        "agent": {"invited": 0, "profile_completed": 0, "first_public_video": 0, "first_rtc_native_action": 0, "fully_activated": 0},
    }
    milestones = {
        "profile_completed": 0,
        "first_public_video": 0,
        "first_rtc_native_action": 0,
        "fully_activated": 0,
    }
    pending_review_count = 0
    approved_pairs = 0
    countable_pairs = 0
    recent_invites = []

    for invite in invite_rows:
        track = invite["invitee_track"] if invite["invitee_track"] in ("human", "agent") else "agent"
        tracks[track]["invited"] += 1

        profile_done = float(invite["profile_completed_at"] or 0) > 0
        video_done = float(invite["first_public_video_at"] or 0) > 0
        rtc_done = float(invite["first_rtc_native_action_at"] or 0) > 0
        fully_done = float(invite["fully_activated_at"] or 0) > 0

        if profile_done:
            tracks[track]["profile_completed"] += 1
            milestones["profile_completed"] += 1
        if video_done:
            tracks[track]["first_public_video"] += 1
            milestones["first_public_video"] += 1
        if rtc_done:
            tracks[track]["first_rtc_native_action"] += 1
            milestones["first_rtc_native_action"] += 1
        if fully_done:
            tracks[track]["fully_activated"] += 1
            milestones["fully_activated"] += 1

        review_status = (invite["review_status"] or "pending").strip().lower() or "pending"
        if review_status == "pending":
            pending_review_count += 1
        if fully_done and review_status not in {"rejected", "void"}:
            countable_pairs += 1
        if fully_done and review_status == "approved":
            approved_pairs += 1

        if include_recent:
            recent_invites.append(
                {
                    "id": int(invite["id"]),
                    "agent_name": invite["agent_name"],
                    "display_name": invite["display_name"] or invite["agent_name"],
                    "track": track,
                    "source": invite["source"] or "",
                    "signup_at": float(invite["signup_at"] or 0),
                    "invitee_created_at": float(invite["invitee_created_at"] or 0),
                    "review_status": review_status,
                    "reviewed_at": float(invite["reviewed_at"] or 0),
                    "reviewer_note": invite["reviewer_note"] or "",
                    "milestones": {
                        "profile_completed": profile_done,
                        "first_public_video": video_done,
                        "first_rtc_native_action": rtc_done,
                        "fully_activated": fully_done,
                    },
                }
            )

    bonus_progress = [
        {
            "threshold": threshold,
            "current": countable_pairs,
            "approved": approved_pairs,
            "remaining": max(threshold - countable_pairs, 0),
            "reached": countable_pairs >= threshold,
        }
        for threshold in REFERRAL_BONUS_THRESHOLDS
    ]

    return {
        "code": row["code"],
        "allowed_track": _normalize_referral_track(row["allowed_track"], "both"),
        "created_at": float(row["created_at"] or 0),
        "hits": int(row["hits"] or 0),
        "signups": int(row["signups"] or 0),
        "first_uploads": int(row["first_uploads"] or 0),
        "ref_url": f"https://bottube.ai/r/{row['code']}",
        "signup_url": f"https://bottube.ai/signup?ref={row['code']}",
        "tracks": tracks,
        "milestones": milestones,
        "pending_review_count": pending_review_count,
        "fully_activated_pairs": countable_pairs,
        "approved_pairs": approved_pairs,
        "bonus_progress": bonus_progress,
        "recent_invites": recent_invites,
    }


def _referral_refresh_invite_state(db: sqlite3.Connection, invitee_agent_id: int) -> None:
    invite = db.execute(
        """
        SELECT
            ri.id,
            ri.profile_completed_at,
            ri.first_public_video_at,
            ri.first_rtc_native_action_at,
            ri.fully_activated_at,
            a.agent_name
        FROM referral_invites ri
        JOIN agents a ON a.id = ri.invitee_agent_id
        WHERE ri.invitee_agent_id = ?
        """,
        (invitee_agent_id,),
    ).fetchone()
    if not invite:
        return

    now = time.time()
    updates: dict[str, object] = {}
    profile_completed_at = float(invite["profile_completed_at"] or 0)
    first_public_video_at = float(invite["first_public_video_at"] or 0)
    first_rtc_native_action_at = float(invite["first_rtc_native_action_at"] or 0)
    fully_activated_at = float(invite["fully_activated_at"] or 0)

    if profile_completed_at <= 0 and _quest_progress_count(db, invitee_agent_id, "profile_complete") > 0:
        updates["profile_completed_at"] = now
        updates["profile_completed_ref"] = f"/agent/{invite['agent_name']}"
        profile_completed_at = now

    if first_public_video_at <= 0:
        first_video = db.execute(
            """
            SELECT video_id, created_at
            FROM videos
            WHERE agent_id = ? AND COALESCE(is_removed, 0) = 0
            ORDER BY created_at ASC, id ASC
            LIMIT 1
            """,
            (invitee_agent_id,),
        ).fetchone()
        if first_video:
            first_public_video_at = float(first_video["created_at"] or now)
            updates["first_public_video_at"] = first_public_video_at
            updates["first_public_video_ref"] = f"/watch/{first_video['video_id']}"

    if first_rtc_native_action_at <= 0:
        first_tip = db.execute(
            """
            SELECT id, video_id, created_at
            FROM tips
            WHERE from_agent_id = ? OR to_agent_id = ?
            ORDER BY created_at ASC, id ASC
            LIMIT 1
            """,
            (invitee_agent_id, invitee_agent_id),
        ).fetchone()
        if first_tip:
            first_rtc_native_action_at = float(first_tip["created_at"] or now)
            updates["first_rtc_native_action_at"] = first_rtc_native_action_at
            if first_tip["video_id"]:
                updates["first_rtc_native_action_ref"] = f"/watch/{first_tip['video_id']}"
            else:
                updates["first_rtc_native_action_ref"] = "/tips/dashboard"

    if fully_activated_at <= 0 and profile_completed_at > 0 and first_public_video_at > 0 and first_rtc_native_action_at > 0:
        updates["fully_activated_at"] = max(profile_completed_at, first_public_video_at, first_rtc_native_action_at, now)

    if not updates:
        return

    updates["updated_at"] = now
    set_clause = ", ".join(f"{key} = ?" for key in updates)
    db.execute(
        f"UPDATE referral_invites SET {set_clause} WHERE invitee_agent_id = ?",
        list(updates.values()) + [invitee_agent_id],
    )


def _referral_mark_rtc_native_action(
    db: sqlite3.Connection,
    agent_id: int,
    *,
    evidence_ref: str,
    occurred_at: float | None = None,
) -> None:
    invite = db.execute(
        "SELECT first_rtc_native_action_at FROM referral_invites WHERE invitee_agent_id = ?",
        (agent_id,),
    ).fetchone()
    if not invite or float(invite["first_rtc_native_action_at"] or 0) > 0:
        return
    now = float(occurred_at or time.time())
    db.execute(
        """
        UPDATE referral_invites
        SET first_rtc_native_action_at = ?,
            first_rtc_native_action_ref = ?,
            updated_at = ?
        WHERE invitee_agent_id = ?
        """,
        (now, evidence_ref[:500], time.time(), agent_id),
    )
    _referral_refresh_invite_state(db, agent_id)


def _referral_touch_hit(db, code: str):
    """Increment referral hit counters (best-effort)."""
    if not code:
        return
    try:
        now = time.time()
        db.execute(
            "UPDATE referral_codes SET hits = hits + 1, last_hit_at = ? WHERE code = ?",
            (now, code),
        )
        db.commit()
    except Exception:
        # Do not break request flow on referral tracking failures.
        pass


def _referral_touch_hit_unique(db, code: str):
    """Increment referral hit counters once per (code,fingerprint) per 24h (best-effort)."""
    if not code:
        return
    try:
        ip = _get_client_ip()
        fp = _fingerprint_ua(
            ip,
            ua=request.headers.get("User-Agent", ""),
            accept_language=request.headers.get("Accept-Language", ""),
        )
        # Store only a hash; never store raw fingerprint strings.
        fp_hash = hashlib.sha256(fp.encode("utf-8")).hexdigest()
        now = time.time()
        cutoff = now - 86400
        row = db.execute(
            "SELECT last_hit_at FROM referral_hit_uniques WHERE code = ? AND fp_hash = ?",
            (code, fp_hash),
        ).fetchone()
        if row and float(row["last_hit_at"] or 0) > cutoff:
            return
        if row:
            db.execute(
                "UPDATE referral_hit_uniques SET last_hit_at = ? WHERE code = ? AND fp_hash = ?",
                (now, code, fp_hash),
            )
        else:
            db.execute(
                "INSERT OR IGNORE INTO referral_hit_uniques (code, fp_hash, last_hit_at) VALUES (?, ?, ?)",
                (code, fp_hash, now),
            )
        # Count unique-ish hits.
        db.execute(
            "UPDATE referral_codes SET hits = hits + 1, last_hit_at = ? WHERE code = ?",
            (now, code),
        )
        db.commit()
    except Exception:
        pass


def _referral_apply_signup(db, new_agent_id: int, code: str, *, source: str = "signup") -> dict:
    """Attach referral to an invitee and create milestone tracking state."""
    result = {"ok": False, "applied": False, "error": "missing_referral_code"}
    ref_code = _normalize_ref_code(code)
    if not ref_code:
        return result
    try:
        ref = _referral_get_code_row(db, ref_code)
        if not ref:
            return {"ok": False, "applied": False, "error": "referral_code_not_found"}
        invitee = db.execute(
            """
            SELECT id, agent_name, is_human, created_at, referred_by_code
            FROM agents
            WHERE id = ?
            """,
            (new_agent_id,),
        ).fetchone()
        if not invitee:
            return {"ok": False, "applied": False, "error": "invitee_not_found"}
        if int(ref["agent_id"]) == int(new_agent_id):
            return {"ok": False, "applied": False, "error": "self_referral"}

        invitee_track = _referral_track_for_agent(invitee)
        if not _referral_track_allowed(ref["allowed_track"], invitee_track):
            return {
                "ok": False,
                "applied": False,
                "error": "referral_track_not_allowed",
                "invitee_track": invitee_track,
                "allowed_track": _normalize_referral_track(ref["allowed_track"], "both"),
            }

        now = time.time()
        cur = db.execute(
            "UPDATE agents SET referred_by_code = ?, referred_at = ? WHERE id = ? AND COALESCE(referred_by_code, '') = ''",
            (ref_code, now, new_agent_id),
        )
        if int(getattr(cur, "rowcount", 0) or 0) <= 0:
            existing = db.execute(
                "SELECT referred_by_code FROM agents WHERE id = ?",
                (new_agent_id,),
            ).fetchone()
            return {
                "ok": False,
                "applied": False,
                "error": "already_referred",
                "code": _normalize_ref_code((existing["referred_by_code"] if existing else "") or ""),
            }

        ip_hash, fp_hash = _referral_request_hashes()
        db.execute(
            "UPDATE referral_codes SET signups = signups + 1, last_signup_at = ? WHERE code = ?",
            (now, ref_code),
        )
        db.execute(
            """
            INSERT OR IGNORE INTO referral_invites (
                referral_code,
                referrer_agent_id,
                invitee_agent_id,
                invitee_track,
                source,
                signup_at,
                invitee_created_at,
                signup_ip_hash,
                signup_fp_hash,
                review_status,
                created_at,
                updated_at
            )
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending', ?, ?)
            """,
            (
                ref_code,
                int(ref["agent_id"]),
                new_agent_id,
                invitee_track,
                (source or "signup")[:64],
                now,
                float(invitee["created_at"] or 0),
                ip_hash,
                fp_hash,
                now,
                now,
            ),
        )
        _referral_refresh_invite_state(db, new_agent_id)
        db.commit()
        return {
            "ok": True,
            "applied": True,
            "code": ref_code,
            "invitee_track": invitee_track,
            "allowed_track": _normalize_referral_track(ref["allowed_track"], "both"),
        }
    except Exception as exc:
        return {"ok": False, "applied": False, "error": "referral_apply_failed", "details": str(exc)}


def _referral_mark_first_upload(db, agent_id: int):
    """If agent was referred, count their first upload exactly once (best-effort)."""
    try:
        row = db.execute(
            "SELECT referred_by_code, referral_first_upload_counted FROM agents WHERE id = ?",
            (agent_id,),
        ).fetchone()
        if not row:
            return
        code = _normalize_ref_code(row["referred_by_code"] or "")
        if not code:
            return
        if int(row["referral_first_upload_counted"] or 0) != 0:
            return
        now = time.time()
        db.execute(
            "UPDATE agents SET referral_first_upload_counted = 1 WHERE id = ?",
            (agent_id,),
        )
        db.execute(
            "UPDATE referral_codes SET first_uploads = first_uploads + 1, last_first_upload_at = ? WHERE code = ?",
            (now, code),
        )
        _referral_refresh_invite_state(db, agent_id)
        db.commit()
    except Exception:
        pass
def _badge_catalog_entry(badge_key: str) -> dict:
    meta = dict(BADGE_CATALOG.get(badge_key, {}))
    label = meta.get("label") or badge_key.replace("_", " ").title()
    context_label = meta.get("context_label") or "Founding"
    variant = meta.get("variant") or "scout"
    if variant not in BADGE_VARIANTS:
        variant = "scout"
    return {
        "badge_key": badge_key,
        "label": label,
        "context_label": context_label,
        "description": meta.get("description") or label,
        "variant": variant,
        "sort_order": int(meta.get("sort_order", 999)),
    }


def _default_badge_source_campaign(badge_key: str) -> str:
    if "_human_" in badge_key or badge_key.endswith("_human"):
        return "rustchain-bounties#1584"
    if "_agent_" in badge_key or badge_key.endswith("_agent"):
        return "rustchain-bounties#1585"
    return ""


def _badge_assignment_payload(row) -> dict:
    meta = _badge_catalog_entry((row["badge_key"] or "").strip())
    payload = {
        "id": int(row["id"]),
        "badge_key": meta["badge_key"],
        "label": meta["label"],
        "context_label": meta["context_label"],
        "display_name": f"{meta['label']} - {meta['context_label']}",
        "description": meta["description"],
        "variant": meta["variant"],
        "sort_order": meta["sort_order"],
        "cohort_number": int(row["cohort_number"] or 0),
        "source_campaign": row["source_campaign"] or "",
        "notes": row["notes"] or "",
        "metadata": _safe_json_loads_dict(row["metadata_json"]),
        "awarded_at": float(row["awarded_at"] or 0),
        "awarded_by": row["awarded_by"] or "",
        "is_active": bool(int(row["is_active"] or 0)),
        "removed_at": float(row["removed_at"] or 0),
        "removed_by": row["removed_by"] or "",
    }
    keys = set(row.keys()) if hasattr(row, "keys") else set()
    if {"agent_id", "agent_name", "display_name"} <= keys:
        payload["agent"] = {
            "id": int(row["agent_id"]),
            "agent_name": row["agent_name"],
            "display_name": row["display_name"] or row["agent_name"],
            "is_human": bool(int(row["is_human"] or 0)) if "is_human" in keys else False,
        }
    return payload


def _badge_payload_sort_key(badge: dict) -> tuple:
    cohort = int(badge.get("cohort_number") or 0)
    cohort_sort = cohort if cohort > 0 else 9999
    return (
        int(badge.get("sort_order", 999)),
        cohort_sort,
        float(badge.get("awarded_at") or 0),
        int(badge.get("id") or 0),
    )


def _list_agent_badges(
    db: sqlite3.Connection,
    agent_id: int,
    *,
    include_inactive: bool = False,
) -> list[dict]:
    where = "" if include_inactive else "AND COALESCE(is_active, 1) = 1"
    rows = db.execute(
        f"""
        SELECT *
        FROM agent_badges
        WHERE agent_id = ? {where}
        ORDER BY awarded_at ASC, id ASC
        """,
        (agent_id,),
    ).fetchall()
    badges = [_badge_assignment_payload(row) for row in rows]
    badges.sort(key=_badge_payload_sort_key)
    return badges


def _badge_assignment_keyset(db: sqlite3.Connection, *, active_only: bool = True) -> set[tuple[int, str]]:
    where = "WHERE COALESCE(is_active, 1) = 1" if active_only else ""
    rows = db.execute(f"SELECT agent_id, badge_key FROM agent_badges {where}").fetchall()
    return {(int(row["agent_id"]), row["badge_key"]) for row in rows}


def _build_badge_candidates(db: sqlite3.Connection) -> list[dict]:
    assigned = _badge_assignment_keyset(db, active_only=True)
    candidates: list[dict] = []

    invite_rows = db.execute(
        """
        SELECT
            ri.id,
            ri.referral_code,
            ri.invitee_track,
            ri.fully_activated_at,
            inv.id AS invitee_agent_id,
            inv.agent_name AS invitee_agent_name,
            inv.display_name AS invitee_display_name,
            inv.is_human AS invitee_is_human,
            ref.agent_name AS referrer_agent_name,
            ref.display_name AS referrer_display_name
        FROM referral_invites ri
        JOIN agents inv ON inv.id = ri.invitee_agent_id
        JOIN agents ref ON ref.id = ri.referrer_agent_id
        WHERE COALESCE(ri.fully_activated_at, 0) > 0
          AND COALESCE(ri.review_status, 'pending') NOT IN ('rejected', 'void')
        ORDER BY ri.invitee_track ASC, ri.fully_activated_at ASC, ri.id ASC
        """
    ).fetchall()

    cohort_counts = {"human": 0, "agent": 0}
    for row in invite_rows:
        track = row["invitee_track"] if row["invitee_track"] in {"human", "agent"} else "agent"
        cohort_counts[track] += 1
        cohort_number = cohort_counts[track]
        if cohort_number > FOUNDING_BADGE_LIMIT:
            continue
        badge_keys = (
            ("early_human_bottube", "early_human_rustchain")
            if track == "human"
            else ("early_agent_bottube", "early_agent_rustchain")
        )
        source_campaign = _default_badge_source_campaign(badge_keys[0])
        for badge_key in badge_keys:
            if (int(row["invitee_agent_id"]), badge_key) in assigned:
                continue
            meta = _badge_catalog_entry(badge_key)
            candidates.append(
                {
                    "badge_key": badge_key,
                    "badge": {**meta, "cohort_number": cohort_number, "source_campaign": source_campaign},
                    "agent": {
                        "id": int(row["invitee_agent_id"]),
                        "agent_name": row["invitee_agent_name"],
                        "display_name": row["invitee_display_name"] or row["invitee_agent_name"],
                        "is_human": bool(int(row["invitee_is_human"] or 0)),
                    },
                    "cohort_number": cohort_number,
                    "source_campaign": source_campaign,
                    "reason": f"Founding {track} cohort #{cohort_number} fully activated via referral.",
                    "evidence": {
                        "invite_id": int(row["id"]),
                        "referral_code": row["referral_code"],
                        "fully_activated_at": float(row["fully_activated_at"] or 0),
                        "referrer_agent_name": row["referrer_agent_name"],
                        "referrer_display_name": row["referrer_display_name"] or row["referrer_agent_name"],
                    },
                }
            )

    scout_rows = db.execute(
        """
        SELECT
            ri.referrer_agent_id AS agent_id,
            ri.invitee_track,
            COUNT(*) AS pair_count,
            MIN(ri.fully_activated_at) AS first_fully_activated_at,
            MAX(ri.fully_activated_at) AS last_fully_activated_at,
            a.agent_name,
            a.display_name,
            a.is_human
        FROM referral_invites ri
        JOIN agents a ON a.id = ri.referrer_agent_id
        WHERE COALESCE(ri.fully_activated_at, 0) > 0
          AND COALESCE(ri.review_status, 'pending') NOT IN ('rejected', 'void')
        GROUP BY ri.referrer_agent_id, ri.invitee_track
        HAVING COUNT(*) >= ?
        ORDER BY COUNT(*) DESC, MIN(ri.fully_activated_at) ASC, ri.referrer_agent_id ASC
        """,
        (FOUNDING_SCOUT_MIN_PAIRS,),
    ).fetchall()

    for row in scout_rows:
        track = row["invitee_track"] if row["invitee_track"] in {"human", "agent"} else "agent"
        badge_key = "founding_scout_human" if track == "human" else "founding_scout_agent"
        agent_id = int(row["agent_id"])
        if (agent_id, badge_key) in assigned:
            continue
        pair_count = int(row["pair_count"] or 0)
        source_campaign = _default_badge_source_campaign(badge_key)
        meta = _badge_catalog_entry(badge_key)
        candidates.append(
            {
                "badge_key": badge_key,
                "badge": {**meta, "source_campaign": source_campaign},
                "agent": {
                    "id": agent_id,
                    "agent_name": row["agent_name"],
                    "display_name": row["display_name"] or row["agent_name"],
                    "is_human": bool(int(row["is_human"] or 0)),
                },
                "cohort_number": 0,
                "source_campaign": source_campaign,
                "reason": f"Reached {pair_count} fully activated {track} referral pairs.",
                "evidence": {
                    "pair_count": pair_count,
                    "bonus_thresholds_reached": [t for t in REFERRAL_BONUS_THRESHOLDS if pair_count >= t],
                    "first_fully_activated_at": float(row["first_fully_activated_at"] or 0),
                    "last_fully_activated_at": float(row["last_fully_activated_at"] or 0),
                    "invitee_track": track,
                },
            }
        )

    candidates.sort(
        key=lambda row: (
            int(row["badge"]["sort_order"]),
            int(row.get("cohort_number") or 0) or 9999,
            row["agent"]["agent_name"],
            row["badge_key"],
        )
    )
    return candidates


def _nocookie_fingerprint(ip: str, ua: str, accept_language: str) -> str:
    """
    Identify visitors who block cookies more granularly than just IP.

    Mobile carrier NAT and some privacy browsers can cause many real users to share a public IP while
    refusing cookies. If we rate-limit strictly by IP in that scenario, legitimate viewers get 429s.
    """
    basis = (ua or "").strip().lower() + "|" + (accept_language or "").strip().lower()
    if basis == "|":
        return ip
    h = hashlib.sha256(basis.encode("utf-8")).hexdigest()[:12]
    return f"{ip}:{h}"


_fingerprint_ua = _nocookie_fingerprint  # alias used in referral tracking

# RTC reward amounts
RTC_REWARD_UPLOAD = 0.05       # Uploading a video
RTC_REWARD_VIEW = 0.0001       # Per view (paid to video creator)
RTC_REWARD_COMMENT = 0.001     # Posting a comment (paid to commenter)
RTC_REWARD_LIKE_RECEIVED = 0.001  # Receiving a like (paid to video creator)
COMMENT_REWARD_DAILY_CAP = float(os.environ.get("BOTTUBE_COMMENT_REWARD_DAILY_CAP", "0.02"))
COMMENT_REWARD_TARGET_DAILY_CAP = float(os.environ.get("BOTTUBE_COMMENT_REWARD_TARGET_DAILY_CAP", "0.005"))
COMMENT_REWARD_HOLD_THRESHOLD = int(os.environ.get("BOTTUBE_COMMENT_REWARD_HOLD_THRESHOLD", "40"))
VIEW_REWARD_DAILY_CAP = float(os.environ.get("BOTTUBE_VIEW_REWARD_DAILY_CAP", "0.01"))
VIEW_REWARD_TARGET_DAILY_CAP = float(os.environ.get("BOTTUBE_VIEW_REWARD_TARGET_DAILY_CAP", "0.003"))
VIEW_REWARD_HOLD_THRESHOLD = int(os.environ.get("BOTTUBE_VIEW_REWARD_HOLD_THRESHOLD", "36"))
LIKE_REWARD_DAILY_CAP = float(os.environ.get("BOTTUBE_LIKE_REWARD_DAILY_CAP", "0.04"))
LIKE_REWARD_TARGET_DAILY_CAP = float(os.environ.get("BOTTUBE_LIKE_REWARD_TARGET_DAILY_CAP", "0.008"))
LIKE_REWARD_HOLD_THRESHOLD = int(os.environ.get("BOTTUBE_LIKE_REWARD_HOLD_THRESHOLD", "32"))
RTC_TIP_MIN = 0.001              # Minimum tip amount
RTC_TIP_MAX = 100.0              # Maximum tip per transaction

# Gamification: Streak bonus multipliers (consecutive days of activity)
STREAK_BONUS_MULTIPLIERS = {
    1: 1.0,    # No bonus
    3: 1.05,   # 5% bonus at 3 days
    7: 1.10,   # 10% bonus at 1 week
    14: 1.15,  # 15% bonus at 2 weeks
    30: 1.25,  # 25% bonus at 1 month
    60: 1.40,  # 40% bonus at 2 months
    90: 1.50,  # 50% bonus at 3 months
    180: 1.75, # 75% bonus at 6 months
    365: 2.0,  # 100% bonus at 1 year
}

# Gamification: Level thresholds (total XP required for each level)
# XP is earned from quest completions (1 XP per 1 RTC earned from quests)
LEVEL_THRESHOLDS = [
    (1, 0),       # Level 1: 0 XP (starting)
    (2, 50),      # Level 2: 50 XP
    (3, 150),     # Level 3: 150 XP
    (4, 300),     # Level 4: 300 XP
    (5, 500),     # Level 5: 500 XP
    (6, 800),     # Level 6: 800 XP
    (7, 1200),    # Level 7: 1200 XP
    (8, 1700),    # Level 8: 1700 XP
    (9, 2300),    # Level 9: 2300 XP
    (10, 3000),   # Level 10: 3000 XP
    (11, 4000),   # Level 11: 4000 XP
    (12, 5200),   # Level 12: 5200 XP
    (13, 6600),   # Level 13: 6600 XP
    (14, 8200),   # Level 14: 8200 XP
    (15, 10000),  # Level 15: 10000 XP (max)
]

# Anti-farm: Suspicious pattern thresholds
ANTI_FARM_CONFIG = {
    "self_interaction_window_secs": 300,  # 5 min window for self-view detection
    "rapid_comment_threshold": 10,         # Comments per hour before flagging
    "rapid_like_threshold": 15,            # Likes per hour before flagging
    "duplicate_comment_similarity": 0.85,  # Similarity threshold for duplicate detection
    "new_account_reward_cap": 0.5,         # Max daily RTC for accounts < 1 day old
    "new_account_age_secs": 86400,         # 24 hours
}

RUSTCHAIN_BASE_URL = os.environ.get("RUSTCHAIN_BASE_URL", "https://50.28.86.131").rstrip("/")

# ---------------------------------------------------------------------------
# i18n / Translations
# ---------------------------------------------------------------------------

TRANSLATIONS_DIR = BASE_DIR / "translations"
SUPPORTED_LOCALES = ["en", "es", "fr", "ja", "pt"]
DEFAULT_LOCALE = "en"
_translations = {}


def _load_translations():
    """Load all translation JSON files into memory."""
    for locale in SUPPORTED_LOCALES:
        fpath = TRANSLATIONS_DIR / f"{locale}.json"
        if fpath.exists():
            with open(fpath, "r", encoding="utf-8") as f:
                data = json.load(f)
                _translations[locale] = data.get("strings", {})
    # Ensure English fallback always exists
    if "en" not in _translations:
        _translations["en"] = {}


def _detect_locale():
    """Detect preferred locale from session, query param, or Accept-Language header."""
    # 1. Explicit query param (?lang=es)
    lang = request.args.get("lang", "").strip().lower()
    if lang in SUPPORTED_LOCALES:
        session["locale"] = lang
        return lang
    # 2. Session cookie (persists user choice)
    lang = session.get("locale", "").strip().lower()
    if lang in SUPPORTED_LOCALES:
        return lang
    # 3. Accept-Language header
    accept = request.headers.get("Accept-Language", "")
    for part in accept.split(","):
        code = part.split(";")[0].strip().lower()
        # Match exact (e.g. "es") or prefix (e.g. "es-mx" -> "es")
        if code in SUPPORTED_LOCALES:
            return code
        prefix = code.split("-")[0]
        if prefix in SUPPORTED_LOCALES:
            return prefix
    return DEFAULT_LOCALE


def _translate(key, **kwargs):
    """Look up a translation key for the current locale, with English fallback."""
    locale = getattr(g, "locale", DEFAULT_LOCALE)
    text = _translations.get(locale, {}).get(key)
    if text is None:
        text = _translations.get("en", {}).get(key, key)
    if kwargs:
        for k, v in kwargs.items():
            text = text.replace("{" + k + "}", str(v))
    return text


def _language_switch_href(locale_code: str) -> str:
    """Return a query-only language link while preserving current filters."""
    args = request.args.to_dict(flat=True)
    args["lang"] = locale_code
    return "?" + urllib.parse.urlencode(args)


_load_translations()

# ---------------------------------------------------------------------------
# App setup
# ---------------------------------------------------------------------------

STATIC_DIR = BASE_DIR / "bottube_static"
app = Flask(__name__, template_folder=str(TEMPLATE_DIR), static_folder=str(STATIC_DIR), static_url_path="/static")
app.config["MAX_CONTENT_LENGTH"] = MAX_VIDEO_SIZE + 10 * 1024 * 1024  # extra for form data
app.secret_key = os.environ.get("BOTTUBE_SECRET_KEY", secrets.token_hex(32))
app.config["SESSION_COOKIE_HTTPONLY"] = True
app.config["SESSION_COOKIE_SAMESITE"] = "Lax"
app.config["SESSION_COOKIE_SECURE"] = True
app.config["PERMANENT_SESSION_LIFETIME"] = 86400  # 24 hours

# JSON-aware 403 handler for AJAX requests
@app.errorhandler(403)
def handle_403(e):
    """Return JSON for API/AJAX 403 errors, HTML for browser requests."""
    if hasattr(e, "response") and e.response is not None:
        return e.response
    ct = request.headers.get("Content-Type", "")
    if request.is_json or "application/json" in ct or request.headers.get("X-CSRF-Token"):
        return jsonify({"error": "Forbidden", "csrf_error": True}), 403
    return "Forbidden", 403


# Google integrations (configured via env vars on VPS)
app.config["GA4_MEASUREMENT_ID"] = os.environ.get("GA4_MEASUREMENT_ID", "")
app.config["ADSENSE_PUBLISHER_ID"] = os.environ.get("ADSENSE_PUBLISHER_ID", "")
app.config["ADSENSE_VIDEO_SLOT"] = os.environ.get("ADSENSE_VIDEO_SLOT", "")
app.config["IMA_VAST_TAG"] = os.environ.get("IMA_VAST_TAG", "")
app.config["FCM_VAPID_KEY"] = os.environ.get("FCM_VAPID_KEY", "")
app.config["FIREBASE_PROJECT_ID"] = os.environ.get("FIREBASE_PROJECT_ID", "")

# URL prefix: when behind nginx at /bottube/ on shared IP, templates need prefixed URLs.
# When accessed via bottube.ai (own domain), prefix is empty.
# Dynamic per-request via before_request hook.
DOMAIN_PREFIX = ""  # bottube.ai serves at root
IP_PREFIX = os.environ.get("BOTTUBE_PREFIX", "/bottube").rstrip("/")
BOTTUBE_DOMAINS = {"bottube.ai", "www.bottube.ai"}
app.jinja_env.globals["P"] = IP_PREFIX  # default fallback
app.jinja_env.globals["MAX_DURATION"] = MAX_VIDEO_DURATION
app.jinja_env.globals["_"] = _translate
app.jinja_env.globals["SUPPORTED_LOCALES"] = SUPPORTED_LOCALES
app.jinja_env.globals["language_switch_href"] = _language_switch_href


def _build_recovery_notice(db=None):
    if not RECOVERY_RECLAIM_ENABLED:
        return None
    restored_views = RECOVERY_RESTORED_VIEWS_FALLBACK
    if db is not None:
        try:
            restored_views = int(
                db.execute(
                    """SELECT COALESCE(SUM(v.views), 0) FROM videos v
                       JOIN agents a ON v.agent_id = a.id
                       WHERE v.is_removed = 0 AND COALESCE(a.is_banned, 0) = 0"""
                ).fetchone()[0]
            )
        except sqlite3.Error:
            restored_views = RECOVERY_RESTORED_VIEWS_FALLBACK
    return {
        "enabled": True,
        "stage": RECOVERY_STAGE_LABEL,
        "restored_views": restored_views,
        "target_views": RECOVERY_TARGET_TOTAL_VIEWS,
        "remaining_views": max(RECOVERY_TARGET_TOTAL_VIEWS - restored_views, 0),
    }


@app.context_processor
def inject_recovery_notice():
    notice = None
    if RECOVERY_RECLAIM_ENABLED:
        try:
            notice = _build_recovery_notice(get_db())
        except Exception:
            notice = _build_recovery_notice(None)
    return {"recovery_notice": notice}


@app.before_request
def set_url_prefix():
    """Set URL prefix dynamically: empty for bottube.ai, /bottube for IP access."""
    host = request.host.split(":")[0].lower()
    canonical_host = os.getenv("BOTTUBE_CANONICAL_HOST", "bottube.ai").strip().lower()
    if os.getenv("BOTTUBE_WWW_REDIRECT", "1").strip().lower() not in {"0", "false", "no"}:
        if host == f"www.{canonical_host}":
            scheme = (
                "https"
                if (request.is_secure or request.headers.get("X-Forwarded-Proto") == "https")
                else request.scheme
            )
            url = f"{scheme}://{canonical_host}{request.full_path}"
            if url.endswith("?"):
                url = url[:-1]
            code = 301 if request.method in {"GET", "HEAD"} else 308
            return redirect(url, code=code)
    if host in BOTTUBE_DOMAINS:
        g.prefix = DOMAIN_PREFIX
    else:
        g.prefix = IP_PREFIX
    app.jinja_env.globals["P"] = g.prefix

    # i18n: detect locale for this request
    g.locale = _detect_locale()
    app.jinja_env.globals["locale"] = g.locale

    # Load logged-in user from session for web UI
    g.user = None
    user_id = session.get("user_id")
    if user_id:
        try:
            db = get_db()
            g.user = db.execute(
                "SELECT * FROM agents WHERE id = ?", (user_id,)
            ).fetchone()
        except Exception:
            pass
    app.jinja_env.globals["current_user"] = g.user

    # Generate CSRF token for forms
    if "csrf_token" not in session:
        session["csrf_token"] = secrets.token_hex(32)
    app.jinja_env.globals["csrf_token"] = session.get("csrf_token", "")


@app.after_request
def set_security_headers(response):
    """Apply security headers to every response."""
    response.headers["X-Content-Type-Options"] = "nosniff"
    response.headers["X-XSS-Protection"] = "1; mode=block"
    response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin"
    response.headers["Permissions-Policy"] = "camera=(), microphone=(), geolocation=()"
    if request.is_secure or request.headers.get("X-Forwarded-Proto") == "https":
        response.headers["Strict-Transport-Security"] = "max-age=31536000; includeSubDomains"

    # CORS for API routes — required for GPT Actions, MCP, and agent integrations
    is_api = request.path.startswith("/api/") or request.path.startswith("/.well-known/")
    if is_api:
        response.headers["Access-Control-Allow-Origin"] = "*"
        response.headers["Access-Control-Allow-Methods"] = "GET, POST, OPTIONS"
        response.headers["Access-Control-Allow-Headers"] = "Content-Type, X-API-Key, Authorization"
        if request.method == "OPTIONS":
            response.status_code = 200
            return response

    # Embed route allows framing from any origin; all other routes restrict it
    is_embed = request.path.startswith("/embed/")
    if not is_embed and not is_api:
        response.headers.setdefault("X-Frame-Options", "SAMEORIGIN")
        # NOTE: 'unsafe-inline' is required for script-src and style-src because
        # legacy templates use inline <script> blocks (JSON-LD, GA gtag) and
        # inline <style> blocks throughout.  Migrating to nonce-based CSP
        # requires refactoring all templates.  XSS in JSON-LD blocks is
        # mitigated by safe_jsonld() / jsonld_safe which escape </ sequences.
        csp = (
            "default-src 'self'; "
            "script-src 'self' 'unsafe-inline' https://www.googletagmanager.com https://www.google-analytics.com https://stats.g.doubleclick.net https://cdn.jsdelivr.net https://cdnjs.cloudflare.com https://unpkg.com https://www.gstatic.com https://imasdk.googleapis.com; "
            "style-src 'self' 'unsafe-inline'; "
            "img-src 'self' data: https:; "
            "media-src 'self'; "
            "font-src 'self'; "
            "connect-src 'self' https://www.google-analytics.com https://region1.google-analytics.com https://stats.g.doubleclick.net https://www.googletagmanager.com https://www.google.com; "
            "object-src 'none'; "
            "base-uri 'self'; "
            "form-action 'self'; "
            "frame-ancestors 'self'"
        )
        response.headers.setdefault("Content-Security-Policy", csp)
    return response


def _verify_csrf():
    """Verify CSRF token on state-changing web requests (form or AJAX)."""
    token = (
        request.form.get("csrf_token", "")
        or request.headers.get("X-CSRF-Token", "")
    )
    if not token:
        data = request.get_json(silent=True) or {}
        if isinstance(data, dict):
            token = data.get("csrf_token", "")
    expected = session.get("csrf_token", "")
    if not expected or not token or not secrets.compare_digest(token, expected):
        # Return JSON for AJAX/API requests so JS can handle the error
        ct = request.headers.get("Content-Type", "")
        if request.is_json or "application/json" in ct or request.headers.get("X-CSRF-Token"):
            from flask import make_response

            resp = make_response(
                jsonify({"error": "Session expired. Please refresh the page.", "csrf_error": True}),
                403,
            )
            abort(resp)
        abort(403)


def _public_json_object_body():
    data = request.get_json(silent=True)
    if data is None:
        return {}, None
    if not isinstance(data, dict):
        return None, (jsonify({"ok": False, "error": "JSON body must be an object"}), 400)
    return data, None


def _public_string_field(data, field, default="", max_length=None):
    value = data.get(field, default)
    if value is None:
        value = default
    if not isinstance(value, str):
        return None, f"{field} must be a string"
    value = value.strip()
    if max_length is not None:
        value = value[:max_length]
    return value, None


# ---------------------------------------------------------------------------
# Scrape / Visitor Monitoring
# ---------------------------------------------------------------------------

KNOWN_SCRAPERS = {
    "ia_archiver": "Internet Archive",
    "Wayback": "Internet Archive Wayback",
    "archive.org_bot": "Internet Archive Bot",
    "Googlebot": "Google",
    "bingbot": "Bing",
    "Baiduspider": "Baidu",
    "YandexBot": "Yandex",
    "DotBot": "DotBot/SEO",
    "AhrefsBot": "Ahrefs/SEO",
    "SemrushBot": "Semrush/SEO",
    "MJ12bot": "Majestic/SEO",
    "PetalBot": "Huawei Petal",
    "GPTBot": "OpenAI GPT",
    "ClaudeBot": "Anthropic Claude",
    "CCBot": "Common Crawl",
    "Bytespider": "ByteDance/TikTok",
    "DataForSeoBot": "DataForSeo",
    "Go-http-client": "Go HTTP Client",
    "python-requests": "Python Requests",
    "curl": "cURL",
    "Scrapy": "Scrapy Framework",
    "HTTrack": "HTTrack Copier",
    "wget": "wget",
    "HeadlessChrome": "Headless Chrome",
    "PhantomJS": "PhantomJS",
    "Playwright": "Playwright",
    "Puppeteer": "Puppeteer",
}

_VISITOR_LOG_PATH = BASE_DIR / "visitor_log.jsonl"


def _log_visitor():
    """Log visitor info for analytics and scrape detection."""
    ip = _get_client_ip()
    ua = request.headers.get("User-Agent", "")
    path = request.path
    method = request.method

    # Detect scrapers
    scraper_name = None
    ua_lower = ua.lower()
    for sig, name in KNOWN_SCRAPERS.items():
        if sig.lower() in ua_lower:
            scraper_name = name
            break

    # Assign visitor tracking cookie
    visitor_id = request.cookies.get("_bt_vid", "")
    is_new = not visitor_id
    if is_new:
        visitor_id = secrets.token_hex(16)

    entry = {
        "ts": time.time(),
        "ip": ip,
        "vid": visitor_id,
        "new": is_new,
        "path": path,
        "method": method,
        "ua": ua[:256],
        "ref": request.headers.get("Referer", "")[:256],
        "scraper": scraper_name,
    }

    try:
        with open(_VISITOR_LOG_PATH, "a") as f:
            f.write(json.dumps(entry) + "\n")
    except Exception:
        pass

    # Store for after_request to set cookie
    g.visitor_id = visitor_id
    g.is_new_visitor = is_new


@app.before_request
def track_visitors():
    """Track all visitors and detect scrapers."""
    # Don't rate-limit or log asset/media requests. These can be bursty (many thumbnails/avatars),
    # especially on mobile, and counting them leads to false-positive 429s.
    path = request.path or ""
    is_video_media = (
        request.method in {"GET", "HEAD"}
        and path.startswith("/api/videos/")
        and (path.endswith("/stream") or path.endswith("/captions"))
    )
    if (
        path in _RL_EXEMPT_PATHS
        or any(path.startswith(p) for p in _RL_EXEMPT_PREFIXES)
        or is_video_media
    ):
        return

    _log_visitor()

    # Scraper Detective — real-time bot classification
    ip = _get_client_ip()
    if SCRAPER_DETECTIVE_ENABLED and scraper_detective_inst.is_blocked(ip):
        return Response("Forbidden", status=403)
    if SCRAPER_DETECTIVE_ENABLED:
        scraper_detective_inst.record_request(
            ip, request.headers.get("User-Agent", ""), path,
            getattr(g, "visitor_id", ""), getattr(g, "is_new_visitor", False),
            request.headers.get("Referer", ""))

    # Rate limit scrapers more aggressively
    ua = request.headers.get("User-Agent", "")
    ua_lower = ua.lower()

    is_scraper = any(sig.lower() in ua_lower for sig in KNOWN_SCRAPERS)
    if is_scraper:
        if not _rate_limit(f"scraper:{ip}", _RL_SCRAPER_RPM, _RL_WINDOW_SECS):
            return Response("Rate limited", status=429)
    else:
        # General visitor rate limit: prefer per-visitor budgets (cookie) so carrier NAT doesn't
        # punish legitimate users; still keep a generous per-IP cap.
        if not _rate_limit(f"global_ip:{ip}", _RL_GLOBAL_IP_RPM, _RL_WINDOW_SECS):
            return Response("Rate limited", status=429)

        is_new = getattr(g, "is_new_visitor", False)
        visitor_id = getattr(g, "visitor_id", "")
        if is_new or not visitor_id:
            # No cookie yet (often scripts/scrapers). Keep a stricter per-IP cap.
            fp = _nocookie_fingerprint(ip, ua, request.headers.get("Accept-Language", ""))
            if not _rate_limit(f"global_nocookie:{fp}", _RL_NOCOOKIE_RPM, _RL_WINDOW_SECS):
                return Response("Rate limited", status=429)
        else:
            if not _rate_limit(f"global_vid:{visitor_id}", _RL_GLOBAL_RPM, _RL_WINDOW_SECS):
                return Response("Rate limited", status=429)


@app.after_request
def set_visitor_cookie(response):
    """Set visitor tracking cookie."""
    vid = getattr(g, "visitor_id", None)
    if vid:
        response.set_cookie(
            "_bt_vid", vid,
            max_age=365 * 86400,
            httponly=True,
            samesite="Lax",
            secure=request.is_secure or request.headers.get("X-Forwarded-Proto") == "https",
        )
    return response


# ---------------------------------------------------------------------------
# Custom Error Handlers
# ---------------------------------------------------------------------------

@app.errorhandler(404)
def page_not_found(e):
    """Custom 404 page."""
    return render_template("404.html"), 404


@app.errorhandler(405)
def method_not_allowed(e):
    """405 with required Allow header per RFC 9110 Section 15.5.6."""
    allowed = e.valid_methods if hasattr(e, 'valid_methods') and e.valid_methods else []
    resp = jsonify({"error": "Method Not Allowed", "allowed": allowed})
    resp.status_code = 405
    if allowed:
        resp.headers["Allow"] = ", ".join(sorted(allowed))
    return resp


@app.errorhandler(413)
def request_entity_too_large(e):
    """Return JSON error for oversized uploads instead of default HTML page."""
    max_mb = (MAX_VIDEO_SIZE + 10 * 1024 * 1024) // (1024 * 1024)
    if request.path.startswith("/api/"):
        return jsonify({
            "error": f"File too large. Maximum upload size is {max_mb} MB.",
            "max_size_mb": max_mb,
        }), 413
    flash(f"File too large. Maximum upload size is {max_mb} MB.", "error")
    return redirect(url_for("upload_page"))


@app.errorhandler(500)
def internal_server_error(e):
    """Custom 500 page."""
    return render_template("500.html"), 500


for d in (VIDEO_DIR, THUMB_DIR, AVATAR_DIR):
    d.mkdir(parents=True, exist_ok=True)


# ---------------------------------------------------------------------------
# Database helpers
# ---------------------------------------------------------------------------

SCHEMA = """
CREATE TABLE IF NOT EXISTS agents (
    id INTEGER PRIMARY KEY,
    agent_name TEXT UNIQUE NOT NULL,
    display_name TEXT,
    api_key TEXT UNIQUE NOT NULL,
    bio TEXT DEFAULT '',
    avatar_url TEXT DEFAULT '',
    password_hash TEXT DEFAULT '',
    is_human INTEGER DEFAULT 0,
    detected_type TEXT DEFAULT '',
    x_handle TEXT DEFAULT '',
    claim_token TEXT DEFAULT '',
    claimed INTEGER DEFAULT 0,
	    -- Wallet addresses for donations
	    rtc_address TEXT DEFAULT '',
	    -- RustChain on-chain wallet (RTC... Ed25519-derived address)
	    rtc_wallet TEXT DEFAULT '',
	    btc_address TEXT DEFAULT '',
	    eth_address TEXT DEFAULT '',
	    sol_address TEXT DEFAULT '',
	    ltc_address TEXT DEFAULT '',
	    erg_address TEXT DEFAULT '',
    paypal_email TEXT DEFAULT '',
    -- RTC earnings
    rtc_balance REAL DEFAULT 0.0,
    created_at REAL NOT NULL,
    last_active REAL
);

CREATE TABLE IF NOT EXISTS videos (
    id INTEGER PRIMARY KEY,
    video_id TEXT UNIQUE NOT NULL,
    agent_id INTEGER NOT NULL,
    title TEXT NOT NULL,
    description TEXT DEFAULT '',
    filename TEXT NOT NULL,
    thumbnail TEXT DEFAULT '',
    duration_sec REAL DEFAULT 0,
    width INTEGER DEFAULT 0,
    height INTEGER DEFAULT 0,
    views INTEGER DEFAULT 0,
    likes INTEGER DEFAULT 0,
    dislikes INTEGER DEFAULT 0,
    tags TEXT DEFAULT '[]',
    category TEXT DEFAULT 'other',        -- Video category (from VIDEO_CATEGORIES)
    scene_description TEXT DEFAULT '',    -- Text description for bots that can't view video
    novelty_score REAL DEFAULT 0,
    novelty_flags TEXT DEFAULT '',
    revision_of TEXT DEFAULT '',
    revision_note TEXT DEFAULT '',
    challenge_id TEXT DEFAULT '',
    submolt_crosspost TEXT DEFAULT '',
    attribution_id INTEGER DEFAULT NULL,
    syndication_chain TEXT DEFAULT '[]',
    license TEXT DEFAULT 'CC-BY-4.0',
    created_at REAL NOT NULL,
    FOREIGN KEY (agent_id) REFERENCES agents(id)
);

CREATE TABLE IF NOT EXISTS comments (
    id INTEGER PRIMARY KEY,
    video_id TEXT NOT NULL,
    agent_id INTEGER NOT NULL,
    parent_id INTEGER DEFAULT NULL,
    content TEXT NOT NULL,
    comment_type TEXT DEFAULT 'comment',
    likes INTEGER DEFAULT 0,
    created_at REAL NOT NULL,
    FOREIGN KEY (agent_id) REFERENCES agents(id)
);

CREATE TABLE IF NOT EXISTS votes (
    agent_id INTEGER NOT NULL,
    video_id TEXT NOT NULL,
    vote INTEGER NOT NULL,
    created_at REAL NOT NULL,
    PRIMARY KEY (agent_id, video_id)
);

CREATE TABLE IF NOT EXISTS views (
    id INTEGER PRIMARY KEY,
    video_id TEXT NOT NULL,
    agent_id INTEGER,
    ip_address TEXT,
    created_at REAL NOT NULL
);

CREATE TABLE IF NOT EXISTS human_votes (
    ip_address TEXT NOT NULL,
    video_id TEXT NOT NULL,
    vote INTEGER NOT NULL,
    created_at REAL NOT NULL,
    PRIMARY KEY (ip_address, video_id)
);

CREATE TABLE IF NOT EXISTS crossposts (
    id INTEGER PRIMARY KEY,
    video_id TEXT NOT NULL,
    platform TEXT NOT NULL,
    external_id TEXT,
    created_at REAL NOT NULL
);

-- RTC earnings ledger
CREATE TABLE IF NOT EXISTS earnings (
    id INTEGER PRIMARY KEY,
    agent_id INTEGER NOT NULL,
    amount REAL NOT NULL,
    reason TEXT NOT NULL,
    video_id TEXT DEFAULT '',
    created_at REAL NOT NULL,
    FOREIGN KEY (agent_id) REFERENCES agents(id)
);

CREATE TABLE IF NOT EXISTS reward_holds (
    id INTEGER PRIMARY KEY AUTOINCREMENT,
    agent_id INTEGER NOT NULL,
    event_type TEXT NOT NULL,
    event_ref TEXT NOT NULL,
    amount REAL NOT NULL,
    status TEXT DEFAULT 'pending',
    risk_score INTEGER DEFAULT 0,
    reasons TEXT DEFAULT '[]',
    created_at REAL NOT NULL,
    reviewed_at REAL DEFAULT 0,
    reviewer_note TEXT DEFAULT '',
    UNIQUE(agent_id, event_type, event_ref),
    FOREIGN KEY (agent_id) REFERENCES agents(id)
);

CREATE TABLE IF NOT EXISTS moderation_holds (
    id INTEGER PRIMARY KEY AUTOINCREMENT,
    target_type TEXT NOT NULL,
    target_ref TEXT NOT NULL,
    target_agent_id INTEGER,
    source TEXT DEFAULT '',
    reason TEXT NOT NULL,
    details TEXT DEFAULT '',
    status TEXT DEFAULT 'pending',
    recommended_action TEXT DEFAULT 'coach',
    coach_note TEXT DEFAULT '',
    created_at REAL NOT NULL,
    reviewed_at REAL DEFAULT 0,
    reviewer_note TEXT DEFAULT '',
    UNIQUE(target_type, target_ref, source, reason),
    FOREIGN KEY (target_agent_id) REFERENCES agents(id)
);

CREATE TABLE IF NOT EXISTS giveaway_entrants (
    id INTEGER PRIMARY KEY,
    agent_id INTEGER UNIQUE NOT NULL,
    entered_at REAL NOT NULL,
    eligible INTEGER DEFAULT 0,
    disqualified INTEGER DEFAULT 0,
    disqualify_reason TEXT DEFAULT '',
    FOREIGN KEY (agent_id) REFERENCES agents(id)
);

CREATE TABLE IF NOT EXISTS comment_votes (
    agent_id INTEGER NOT NULL,
    comment_id INTEGER NOT NULL,
    vote INTEGER NOT NULL,
    created_at REAL NOT NULL,
    PRIMARY KEY (agent_id, comment_id),
    FOREIGN KEY (comment_id) REFERENCES comments(id)
);

CREATE TABLE IF NOT EXISTS subscriptions (
    follower_id INTEGER NOT NULL,
    following_id INTEGER NOT NULL,
    created_at REAL NOT NULL,
    PRIMARY KEY (follower_id, following_id),
    FOREIGN KEY (follower_id) REFERENCES agents(id),
    FOREIGN KEY (following_id) REFERENCES agents(id)
);

CREATE TABLE IF NOT EXISTS notifications (
    id INTEGER PRIMARY KEY,
    agent_id INTEGER NOT NULL,
    type TEXT NOT NULL,
    message TEXT NOT NULL,
    from_agent TEXT DEFAULT '',
    video_id TEXT DEFAULT '',
    is_read INTEGER DEFAULT 0,
    created_at REAL NOT NULL,
    FOREIGN KEY (agent_id) REFERENCES agents(id)
);

CREATE INDEX IF NOT EXISTS idx_videos_agent ON videos(agent_id);
CREATE INDEX IF NOT EXISTS idx_videos_created ON videos(created_at DESC);
CREATE INDEX IF NOT EXISTS idx_comments_video ON comments(video_id);
CREATE INDEX IF NOT EXISTS idx_views_video ON views(video_id);
CREATE INDEX IF NOT EXISTS idx_views_dedup ON views(video_id, ip_address, created_at);
CREATE INDEX IF NOT EXISTS idx_earnings_agent ON earnings(agent_id);
CREATE INDEX IF NOT EXISTS idx_reward_holds_agent ON reward_holds(agent_id, status, created_at DESC);
CREATE INDEX IF NOT EXISTS idx_moderation_holds_target ON moderation_holds(target_type, status, created_at DESC);
CREATE INDEX IF NOT EXISTS idx_moderation_holds_agent ON moderation_holds(target_agent_id, status, created_at DESC);
CREATE INDEX IF NOT EXISTS idx_subs_follower ON subscriptions(follower_id);
CREATE INDEX IF NOT EXISTS idx_subs_following ON subscriptions(following_id);
CREATE INDEX IF NOT EXISTS idx_notif_agent ON notifications(agent_id, is_read, created_at DESC);
CREATE INDEX IF NOT EXISTS idx_videos_revision ON videos(revision_of);
CREATE INDEX IF NOT EXISTS idx_videos_challenge ON videos(challenge_id);

	-- RTC tips between users
	CREATE TABLE IF NOT EXISTS tips (
	    id INTEGER PRIMARY KEY,
	    from_agent_id INTEGER NOT NULL,
	    to_agent_id INTEGER NOT NULL,
	    video_id TEXT DEFAULT '',
	    amount REAL NOT NULL,
	    message TEXT DEFAULT '',
	    onchain INTEGER DEFAULT 0,
	    status TEXT DEFAULT 'confirmed',   -- confirmed | pending | voided
	    tx_hash TEXT,                     -- RustChain tx hash (pending ledger)
	    pending_id INTEGER,               -- RustChain pending_ledger id
	    confirms_at REAL,                 -- RustChain confirms_at (epoch seconds)
	    from_address TEXT DEFAULT '',     -- RustChain RTC... address
	    to_address TEXT DEFAULT '',       -- RustChain RTC... address
	    created_at REAL NOT NULL,
	    FOREIGN KEY (from_agent_id) REFERENCES agents(id),
	    FOREIGN KEY (to_agent_id) REFERENCES agents(id)
	);
	CREATE INDEX IF NOT EXISTS idx_tips_video ON tips(video_id, created_at DESC);
	CREATE INDEX IF NOT EXISTS idx_tips_to ON tips(to_agent_id, created_at DESC);
	CREATE INDEX IF NOT EXISTS idx_tips_status ON tips(status, confirms_at);
	CREATE UNIQUE INDEX IF NOT EXISTS idx_tips_tx_hash ON tips(tx_hash) WHERE tx_hash IS NOT NULL;

CREATE TABLE IF NOT EXISTS playlists (
    id INTEGER PRIMARY KEY,
    playlist_id TEXT UNIQUE NOT NULL,
    agent_id INTEGER NOT NULL,
    title TEXT NOT NULL,
    description TEXT DEFAULT '',
    visibility TEXT DEFAULT 'public',
    created_at REAL NOT NULL,
    updated_at REAL NOT NULL,
    FOREIGN KEY (agent_id) REFERENCES agents(id)
);

CREATE TABLE IF NOT EXISTS playlist_items (
    id INTEGER PRIMARY KEY,
    playlist_id INTEGER NOT NULL,
    video_id TEXT NOT NULL,
    position INTEGER NOT NULL,
    added_at REAL NOT NULL,
    FOREIGN KEY (playlist_id) REFERENCES playlists(id) ON DELETE CASCADE,
    FOREIGN KEY (video_id) REFERENCES videos(video_id)
);

CREATE INDEX IF NOT EXISTS idx_playlists_agent ON playlists(agent_id);
CREATE INDEX IF NOT EXISTS idx_playlist_items_pl ON playlist_items(playlist_id, position);
CREATE UNIQUE INDEX IF NOT EXISTS idx_playlist_items_uniq ON playlist_items(playlist_id, video_id);

CREATE TABLE IF NOT EXISTS webhooks (
    id INTEGER PRIMARY KEY,
    agent_id INTEGER NOT NULL,
    url TEXT NOT NULL,
    secret TEXT NOT NULL,
    events TEXT NOT NULL DEFAULT '*',
    active INTEGER DEFAULT 1,
    created_at REAL NOT NULL,
    last_triggered REAL DEFAULT 0,
    fail_count INTEGER DEFAULT 0,
    FOREIGN KEY (agent_id) REFERENCES agents(id)
);

CREATE INDEX IF NOT EXISTS idx_webhooks_agent ON webhooks(agent_id, active);

CREATE TABLE IF NOT EXISTS challenges (
    id INTEGER PRIMARY KEY,
    challenge_id TEXT UNIQUE NOT NULL,
    title TEXT NOT NULL,
    description TEXT DEFAULT '',
    tags TEXT DEFAULT '[]',
    reward TEXT DEFAULT '',
    status TEXT DEFAULT 'upcoming', -- upcoming | active | closed
    start_at REAL DEFAULT 0,
    end_at REAL DEFAULT 0,
    created_at REAL NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_challenges_status ON challenges(status, start_at, end_at);

CREATE TABLE IF NOT EXISTS referral_codes (
    code TEXT PRIMARY KEY,
    agent_id INTEGER NOT NULL,
    created_at REAL NOT NULL,
    hits INTEGER DEFAULT 0,
    signups INTEGER DEFAULT 0,
    first_uploads INTEGER DEFAULT 0,
    last_hit_at REAL DEFAULT 0,
    last_signup_at REAL DEFAULT 0,
    last_first_upload_at REAL DEFAULT 0,
    allowed_track TEXT DEFAULT 'both',
    FOREIGN KEY (agent_id) REFERENCES agents(id)
);
CREATE INDEX IF NOT EXISTS idx_referral_codes_agent ON referral_codes(agent_id);

CREATE TABLE IF NOT EXISTS referral_hit_uniques (
    id INTEGER PRIMARY KEY AUTOINCREMENT,
    code TEXT NOT NULL,
    fp_hash TEXT NOT NULL,
    last_hit_at REAL NOT NULL,
    UNIQUE(code, fp_hash),
    FOREIGN KEY (code) REFERENCES referral_codes(code)
);
CREATE INDEX IF NOT EXISTS idx_referral_hit_code ON referral_hit_uniques(code);

CREATE TABLE IF NOT EXISTS referral_invites (
    id INTEGER PRIMARY KEY AUTOINCREMENT,
    referral_code TEXT NOT NULL,
    referrer_agent_id INTEGER NOT NULL,
    invitee_agent_id INTEGER NOT NULL UNIQUE,
    invitee_track TEXT NOT NULL DEFAULT 'agent',
    source TEXT DEFAULT '',
    signup_at REAL NOT NULL,
    invitee_created_at REAL DEFAULT 0,
    signup_ip_hash TEXT DEFAULT '',
    signup_fp_hash TEXT DEFAULT '',
    profile_completed_at REAL DEFAULT 0,
    profile_completed_ref TEXT DEFAULT '',
    first_public_video_at REAL DEFAULT 0,
    first_public_video_ref TEXT DEFAULT '',
    first_rtc_native_action_at REAL DEFAULT 0,
    first_rtc_native_action_ref TEXT DEFAULT '',
    fully_activated_at REAL DEFAULT 0,
    review_status TEXT DEFAULT 'pending',
    reviewed_at REAL DEFAULT 0,
    reviewer_note TEXT DEFAULT '',
    suspicious_notes TEXT DEFAULT '',
    created_at REAL NOT NULL,
    updated_at REAL NOT NULL,
    FOREIGN KEY (referral_code) REFERENCES referral_codes(code),
    FOREIGN KEY (referrer_agent_id) REFERENCES agents(id),
    FOREIGN KEY (invitee_agent_id) REFERENCES agents(id)
);
CREATE INDEX IF NOT EXISTS idx_referral_invites_referrer ON referral_invites(referrer_agent_id, signup_at DESC);
CREATE INDEX IF NOT EXISTS idx_referral_invites_track ON referral_invites(invitee_track, review_status, signup_at DESC);
CREATE INDEX IF NOT EXISTS idx_referral_invites_code ON referral_invites(referral_code, signup_at DESC);

CREATE TABLE IF NOT EXISTS agent_badges (
    id INTEGER PRIMARY KEY AUTOINCREMENT,
    agent_id INTEGER NOT NULL,
    badge_key TEXT NOT NULL,
    cohort_number INTEGER DEFAULT 0,
    source_campaign TEXT DEFAULT '',
    notes TEXT DEFAULT '',
    metadata_json TEXT DEFAULT '{}',
    awarded_at REAL NOT NULL,
    awarded_by TEXT DEFAULT '',
    is_active INTEGER DEFAULT 1,
    removed_at REAL DEFAULT 0,
    removed_by TEXT DEFAULT '',
    created_at REAL NOT NULL,
    updated_at REAL NOT NULL,
    UNIQUE(agent_id, badge_key),
    FOREIGN KEY (agent_id) REFERENCES agents(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_agent_badges_agent ON agent_badges(agent_id, is_active, awarded_at DESC);
CREATE INDEX IF NOT EXISTS idx_agent_badges_key ON agent_badges(badge_key, is_active, awarded_at DESC);
"""


def get_db():
    """Get thread-local database connection."""
    if "db" not in g:
        g.db = sqlite3.connect(str(DB_PATH))
        g.db.row_factory = sqlite3.Row
        g.db.execute("PRAGMA journal_mode=WAL")
        g.db.execute("PRAGMA foreign_keys=ON")
        g.db.execute("PRAGMA busy_timeout=5000")
    return g.db


@app.teardown_appcontext
def close_db(exc):
    db = g.pop("db", None)
    if db is not None:
        db.close()


def init_db():
    """Create tables if they don't exist, and run migrations."""
    conn = sqlite3.connect(str(DB_PATH))
    conn.executescript(SCHEMA)

    # Migrations: add email columns to agents if missing
    cursor = conn.execute("PRAGMA table_info(agents)")
    existing_cols = {row[1] for row in cursor.fetchall()}
    migrations = {
        "email": "ALTER TABLE agents ADD COLUMN email TEXT DEFAULT ''",
        "email_verified": "ALTER TABLE agents ADD COLUMN email_verified INTEGER DEFAULT 0",
        "email_verify_token": "ALTER TABLE agents ADD COLUMN email_verify_token TEXT DEFAULT ''",
        "email_verify_sent_at": "ALTER TABLE agents ADD COLUMN email_verify_sent_at REAL DEFAULT 0",
    }
    for col, sql in migrations.items():
        if col not in existing_cols:
            conn.execute(sql)

    # Migration: email notification preferences + unsubscribe token
    email_pref_migrations = {
        "email_notify_comments": "ALTER TABLE agents ADD COLUMN email_notify_comments INTEGER DEFAULT 1",
        "email_notify_replies": "ALTER TABLE agents ADD COLUMN email_notify_replies INTEGER DEFAULT 1",
        "email_notify_new_video": "ALTER TABLE agents ADD COLUMN email_notify_new_video INTEGER DEFAULT 1",
        "email_notify_tips": "ALTER TABLE agents ADD COLUMN email_notify_tips INTEGER DEFAULT 1",
        "email_notify_subscriptions": "ALTER TABLE agents ADD COLUMN email_notify_subscriptions INTEGER DEFAULT 1",
        "email_unsubscribe_token": "ALTER TABLE agents ADD COLUMN email_unsubscribe_token TEXT DEFAULT ''",
    }
    for col, sql in email_pref_migrations.items():
        if col not in existing_cols:
            conn.execute(sql)


    # Migration: webhook delivery counters/rate-limit metadata
    webhook_cols = {row[1] for row in conn.execute("PRAGMA table_info(webhooks)").fetchall()}
    webhook_migrations = {
        "event_window_start": "ALTER TABLE webhooks ADD COLUMN event_window_start REAL DEFAULT 0",
        "event_count": "ALTER TABLE webhooks ADD COLUMN event_count INTEGER DEFAULT 0",
    }
    for col, sql in webhook_migrations.items():
        if col not in webhook_cols:
            conn.execute(sql)

    # Miner install click tracking
    try:
        conn.execute("""CREATE TABLE IF NOT EXISTS miner_install_clicks (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            source TEXT NOT NULL,
            page TEXT NOT NULL,
            ip TEXT,
            created_at REAL NOT NULL
        )""")
        conn.commit()
    except Exception:
        pass

    # Generate unsubscribe tokens for agents that have email but no token yet
    conn.execute(
        "UPDATE agents SET email_unsubscribe_token = hex(randomblob(16)) "
        "WHERE email IS NOT NULL AND email != '' AND email_unsubscribe_token = ''"
    )

    # Migration: add is_banned + ban_reason to agents if missing
    agent_migrations = {
        "is_banned": "ALTER TABLE agents ADD COLUMN is_banned INTEGER DEFAULT 0",
        "ban_reason": "ALTER TABLE agents ADD COLUMN ban_reason TEXT DEFAULT ''",
        "banned_at": "ALTER TABLE agents ADD COLUMN banned_at REAL DEFAULT 0",
        "detected_type": "ALTER TABLE agents ADD COLUMN detected_type TEXT DEFAULT ''",
        # RustChain on-chain address (RTC... Ed25519-derived)
        "rtc_wallet": "ALTER TABLE agents ADD COLUMN rtc_wallet TEXT DEFAULT ''",
        # Referrals (best-effort growth tracking)
        "referred_by_code": "ALTER TABLE agents ADD COLUMN referred_by_code TEXT DEFAULT ''",
        "referred_at": "ALTER TABLE agents ADD COLUMN referred_at REAL DEFAULT 0",
        "referral_first_upload_counted": "ALTER TABLE agents ADD COLUMN referral_first_upload_counted INTEGER DEFAULT 0",
        "banner_url": "ALTER TABLE agents ADD COLUMN banner_url TEXT DEFAULT ''",
        "accent_color": "ALTER TABLE agents ADD COLUMN accent_color TEXT DEFAULT ''",
        "pinned_video_id": "ALTER TABLE agents ADD COLUMN pinned_video_id TEXT DEFAULT ''",
    }
    for col, sql in agent_migrations.items():
        if col not in existing_cols:
            conn.execute(sql)

    # Referral program table
    conn.execute(
        """
        CREATE TABLE IF NOT EXISTS referral_codes (
            code TEXT PRIMARY KEY,
            agent_id INTEGER NOT NULL,
            created_at REAL NOT NULL,
            hits INTEGER DEFAULT 0,
            signups INTEGER DEFAULT 0,
            first_uploads INTEGER DEFAULT 0,
            last_hit_at REAL DEFAULT 0,
            last_signup_at REAL DEFAULT 0,
            last_first_upload_at REAL DEFAULT 0,
            FOREIGN KEY (agent_id) REFERENCES agents(id)
        )
        """
    )
    conn.execute("CREATE INDEX IF NOT EXISTS idx_referral_codes_agent ON referral_codes(agent_id)")
    referral_code_cols = {row[1] for row in conn.execute("PRAGMA table_info(referral_codes)").fetchall()}
    if "allowed_track" not in referral_code_cols:
        conn.execute("ALTER TABLE referral_codes ADD COLUMN allowed_track TEXT DEFAULT 'both'")

    # Referral unique hit tracking (privacy: store only hashed fingerprints)
    conn.execute(
        """
        CREATE TABLE IF NOT EXISTS referral_hit_uniques (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            code TEXT NOT NULL,
            fp_hash TEXT NOT NULL,
            last_hit_at REAL NOT NULL,
            UNIQUE(code, fp_hash),
            FOREIGN KEY (code) REFERENCES referral_codes(code)
        )
        """
    )
    conn.execute("CREATE INDEX IF NOT EXISTS idx_referral_hit_code ON referral_hit_uniques(code)")

    conn.execute(
        """
        CREATE TABLE IF NOT EXISTS referral_invites (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            referral_code TEXT NOT NULL,
            referrer_agent_id INTEGER NOT NULL,
            invitee_agent_id INTEGER NOT NULL UNIQUE,
            invitee_track TEXT NOT NULL DEFAULT 'agent',
            source TEXT DEFAULT '',
            signup_at REAL NOT NULL,
            invitee_created_at REAL DEFAULT 0,
            signup_ip_hash TEXT DEFAULT '',
            signup_fp_hash TEXT DEFAULT '',
            profile_completed_at REAL DEFAULT 0,
            profile_completed_ref TEXT DEFAULT '',
            first_public_video_at REAL DEFAULT 0,
            first_public_video_ref TEXT DEFAULT '',
            first_rtc_native_action_at REAL DEFAULT 0,
            first_rtc_native_action_ref TEXT DEFAULT '',
            fully_activated_at REAL DEFAULT 0,
            review_status TEXT DEFAULT 'pending',
            reviewed_at REAL DEFAULT 0,
            reviewer_note TEXT DEFAULT '',
            suspicious_notes TEXT DEFAULT '',
            created_at REAL NOT NULL,
            updated_at REAL NOT NULL,
            FOREIGN KEY (referral_code) REFERENCES referral_codes(code),
            FOREIGN KEY (referrer_agent_id) REFERENCES agents(id),
            FOREIGN KEY (invitee_agent_id) REFERENCES agents(id)
        )
        """
    )
    conn.execute("CREATE INDEX IF NOT EXISTS idx_referral_invites_referrer ON referral_invites(referrer_agent_id, signup_at DESC)")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_referral_invites_track ON referral_invites(invitee_track, review_status, signup_at DESC)")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_referral_invites_code ON referral_invites(referral_code, signup_at DESC)")

    conn.execute(
        """
        CREATE TABLE IF NOT EXISTS agent_badges (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            agent_id INTEGER NOT NULL,
            badge_key TEXT NOT NULL,
            cohort_number INTEGER DEFAULT 0,
            source_campaign TEXT DEFAULT '',
            notes TEXT DEFAULT '',
            metadata_json TEXT DEFAULT '{}',
            awarded_at REAL NOT NULL,
            awarded_by TEXT DEFAULT '',
            is_active INTEGER DEFAULT 1,
            removed_at REAL DEFAULT 0,
            removed_by TEXT DEFAULT '',
            created_at REAL NOT NULL,
            updated_at REAL NOT NULL,
            UNIQUE(agent_id, badge_key),
            FOREIGN KEY (agent_id) REFERENCES agents(id) ON DELETE CASCADE
        )
        """
    )
    conn.execute("CREATE INDEX IF NOT EXISTS idx_agent_badges_agent ON agent_badges(agent_id, is_active, awarded_at DESC)")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_agent_badges_key ON agent_badges(badge_key, is_active, awarded_at DESC)")
    badge_cols = {row[1] for row in conn.execute("PRAGMA table_info(agent_badges)").fetchall()}
    badge_migrations = {
        "cohort_number": "ALTER TABLE agent_badges ADD COLUMN cohort_number INTEGER DEFAULT 0",
        "source_campaign": "ALTER TABLE agent_badges ADD COLUMN source_campaign TEXT DEFAULT ''",
        "notes": "ALTER TABLE agent_badges ADD COLUMN notes TEXT DEFAULT ''",
        "metadata_json": "ALTER TABLE agent_badges ADD COLUMN metadata_json TEXT DEFAULT '{}'",
        "awarded_by": "ALTER TABLE agent_badges ADD COLUMN awarded_by TEXT DEFAULT ''",
        "is_active": "ALTER TABLE agent_badges ADD COLUMN is_active INTEGER DEFAULT 1",
        "removed_at": "ALTER TABLE agent_badges ADD COLUMN removed_at REAL DEFAULT 0",
        "removed_by": "ALTER TABLE agent_badges ADD COLUMN removed_by TEXT DEFAULT ''",
        "updated_at": "ALTER TABLE agent_badges ADD COLUMN updated_at REAL DEFAULT 0",
    }
    for col, sql in badge_migrations.items():
        if col not in badge_cols:
            conn.execute(sql)

    now = time.time()
    conn.execute(
        """
        INSERT OR IGNORE INTO referral_invites (
            referral_code,
            referrer_agent_id,
            invitee_agent_id,
            invitee_track,
            source,
            signup_at,
            invitee_created_at,
            review_status,
            created_at,
            updated_at
        )
        SELECT
            a.referred_by_code,
            rc.agent_id,
            a.id,
            CASE WHEN COALESCE(a.is_human, 0) = 1 THEN 'human' ELSE 'agent' END,
            'legacy_backfill',
            CASE
                WHEN COALESCE(a.referred_at, 0) > 0 THEN a.referred_at
                WHEN COALESCE(a.created_at, 0) > 0 THEN a.created_at
                ELSE ?
            END,
            COALESCE(a.created_at, 0),
            'pending',
            ?,
            ?
        FROM agents a
        JOIN referral_codes rc ON rc.code = a.referred_by_code
        WHERE COALESCE(a.referred_by_code, '') != ''
        """,
        (now, now, now),
    )

    # Migration: Google OAuth columns on agents
    google_migrations = {
        "google_id": "ALTER TABLE agents ADD COLUMN google_id TEXT DEFAULT ''",
        "google_email": "ALTER TABLE agents ADD COLUMN google_email TEXT DEFAULT ''",
        "google_avatar": "ALTER TABLE agents ADD COLUMN google_avatar TEXT DEFAULT ''",
    }
    for col, sql in google_migrations.items():
        if col not in existing_cols:
            conn.execute(sql)

    # Migration: add is_removed to videos if missing
    video_cols = {row[1] for row in conn.execute("PRAGMA table_info(videos)").fetchall()}
    if "is_removed" not in video_cols:
        conn.execute("ALTER TABLE videos ADD COLUMN is_removed INTEGER DEFAULT 0")
    if "removed_reason" not in video_cols:
        conn.execute("ALTER TABLE videos ADD COLUMN removed_reason TEXT DEFAULT ''")

    # Migration: add dislikes column to comments if missing
    comment_cols = {row[1] for row in conn.execute("PRAGMA table_info(comments)").fetchall()}
    if "dislikes" not in comment_cols:
        conn.execute("ALTER TABLE comments ADD COLUMN dislikes INTEGER DEFAULT 0")
    if "comment_type" not in comment_cols:
        conn.execute("ALTER TABLE comments ADD COLUMN comment_type TEXT DEFAULT 'comment'")

    # Migration: add novelty/revision/challenge fields to videos if missing
    video_cols = {row[1] for row in conn.execute("PRAGMA table_info(videos)").fetchall()}
    if "novelty_score" not in video_cols:
        conn.execute("ALTER TABLE videos ADD COLUMN novelty_score REAL DEFAULT 0")
    if "novelty_flags" not in video_cols:
        conn.execute("ALTER TABLE videos ADD COLUMN novelty_flags TEXT DEFAULT ''")
    if "revision_of" not in video_cols:
        conn.execute("ALTER TABLE videos ADD COLUMN revision_of TEXT DEFAULT ''")
    if "revision_note" not in video_cols:
        conn.execute("ALTER TABLE videos ADD COLUMN revision_note TEXT DEFAULT ''")
    if "challenge_id" not in video_cols:
        conn.execute("ALTER TABLE videos ADD COLUMN challenge_id TEXT DEFAULT ''")

    # Migration: push notification subscriptions table
    conn.execute("""
        CREATE TABLE IF NOT EXISTS push_subscriptions (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            agent_id INTEGER,
            endpoint TEXT NOT NULL UNIQUE,
            p256dh TEXT NOT NULL,
            auth TEXT NOT NULL,
            created_at REAL NOT NULL,
            FOREIGN KEY (agent_id) REFERENCES agents(id)
        )
    """)

    # Migration: add vision screening fields to videos
    video_cols = {row[1] for row in conn.execute("PRAGMA table_info(videos)").fetchall()}
    if "screening_status" not in video_cols:
        conn.execute("ALTER TABLE videos ADD COLUMN screening_status TEXT DEFAULT 'legacy'")
    if "screening_details" not in video_cols:
        conn.execute("ALTER TABLE videos ADD COLUMN screening_details TEXT DEFAULT ''")

    # Migration: add response_to_video_id for agent collaboration (Issue #2282)
    video_cols = {row[1] for row in conn.execute("PRAGMA table_info(videos)").fetchall()}
    if "response_to_video_id" not in video_cols:
        conn.execute("ALTER TABLE videos ADD COLUMN response_to_video_id TEXT DEFAULT ''")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_videos_response_to ON videos(response_to_video_id)")

    # Migration: add collaborator_ids for co-upload (Bounty #2161)
    video_cols = {row[1] for row in conn.execute("PRAGMA table_info(videos)").fetchall()}
    if "collaborator_ids" not in video_cols:
        conn.execute("ALTER TABLE videos ADD COLUMN collaborator_ids TEXT DEFAULT '[]'")

    # Migration: create messages table
    conn.execute("""
        CREATE TABLE IF NOT EXISTS messages (
            id TEXT PRIMARY KEY,
            from_agent TEXT NOT NULL,
            to_agent TEXT,
            subject TEXT DEFAULT '',
            body TEXT NOT NULL,
            read_at TEXT,
            created_at TEXT DEFAULT (datetime('now')),
            message_type TEXT DEFAULT 'general'
        )
    """)
    conn.execute("CREATE INDEX IF NOT EXISTS idx_messages_to ON messages(to_agent)")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_messages_created ON messages(created_at DESC)")
    conn.execute("""
        CREATE TABLE IF NOT EXISTS message_reads (
            message_id TEXT NOT NULL,
            agent_name TEXT NOT NULL,
            read_at TEXT DEFAULT (datetime('now')),
            PRIMARY KEY (message_id, agent_name),
            FOREIGN KEY (message_id) REFERENCES messages(id)
        )
    """)
    conn.execute("CREATE INDEX IF NOT EXISTS idx_message_reads_agent ON message_reads(agent_name)")

    # Migration: watch_history table (Phase 6)
    conn.execute("""
        CREATE TABLE IF NOT EXISTS watch_history (
            id INTEGER PRIMARY KEY,
            agent_id INTEGER,
            video_id TEXT NOT NULL,
            watched_at REAL NOT NULL,
            watch_duration_sec REAL DEFAULT 0,
            UNIQUE(agent_id, video_id)
        )
    """)
    conn.execute("CREATE INDEX IF NOT EXISTS idx_watch_history_agent ON watch_history(agent_id, watched_at DESC)")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_watch_history_video ON watch_history(video_id)")

    # Migration: reports table (Phase 7)
    conn.execute("""
        CREATE TABLE IF NOT EXISTS reports (
            id INTEGER PRIMARY KEY,
            video_id TEXT,
            comment_id INTEGER,
            reporter_agent_id INTEGER,
            reason TEXT NOT NULL,
            details TEXT DEFAULT '',
            status TEXT DEFAULT 'pending',
            created_at REAL NOT NULL
        )
    """)
    conn.execute("CREATE INDEX IF NOT EXISTS idx_reports_video ON reports(video_id)")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_reports_status ON reports(status)")

    # Migration: RustChain on-chain tipping metadata
    try:
        tips_cols = {row[1] for row in conn.execute("PRAGMA table_info(tips)").fetchall()}
        tip_migrations = {
            "onchain": "ALTER TABLE tips ADD COLUMN onchain INTEGER DEFAULT 0",
            "status": "ALTER TABLE tips ADD COLUMN status TEXT DEFAULT 'confirmed'",
            "tx_hash": "ALTER TABLE tips ADD COLUMN tx_hash TEXT",
            "pending_id": "ALTER TABLE tips ADD COLUMN pending_id INTEGER",
            "confirms_at": "ALTER TABLE tips ADD COLUMN confirms_at REAL",
            "from_address": "ALTER TABLE tips ADD COLUMN from_address TEXT DEFAULT ''",
            "to_address": "ALTER TABLE tips ADD COLUMN to_address TEXT DEFAULT ''",
        }
        for col, sql in tip_migrations.items():
            if col not in tips_cols:
                conn.execute(sql)

        conn.execute("CREATE INDEX IF NOT EXISTS idx_tips_status ON tips(status, confirms_at)")
        conn.execute("CREATE UNIQUE INDEX IF NOT EXISTS idx_tips_tx_hash ON tips(tx_hash) WHERE tx_hash IS NOT NULL")
    except Exception:
        pass

    # Quest engine: lightweight onboarding progression with one-time RTC rewards.
    conn.execute(
        """
        CREATE TABLE IF NOT EXISTS quests (
            quest_key TEXT PRIMARY KEY,
            title TEXT NOT NULL,
            description TEXT DEFAULT '',
            category TEXT DEFAULT 'onboarding',
            reward_rtc REAL DEFAULT 0,
            goal_count INTEGER DEFAULT 1,
            metric_key TEXT NOT NULL,
            is_active INTEGER DEFAULT 1,
            sort_order INTEGER DEFAULT 0,
            created_at REAL NOT NULL,
            updated_at REAL NOT NULL
        )
        """
    )
    conn.execute(
        """
        CREATE TABLE IF NOT EXISTS agent_quests (
            agent_id INTEGER NOT NULL,
            quest_key TEXT NOT NULL,
            progress_count INTEGER DEFAULT 0,
            completed_at REAL DEFAULT 0,
            rewarded_at REAL DEFAULT 0,
            last_event_at REAL DEFAULT 0,
            metadata TEXT DEFAULT '{}',
            PRIMARY KEY (agent_id, quest_key),
            FOREIGN KEY (agent_id) REFERENCES agents(id),
            FOREIGN KEY (quest_key) REFERENCES quests(quest_key)
        )
        """
    )
    conn.execute("CREATE INDEX IF NOT EXISTS idx_agent_quests_agent ON agent_quests(agent_id, completed_at DESC)")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_agent_quests_rewarded ON agent_quests(rewarded_at DESC)")

    # Syndication queue for distributing uploads to external platforms
    conn.execute(
        """
        CREATE TABLE IF NOT EXISTS syndication_queue (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            video_id TEXT NOT NULL,
            video_title TEXT NOT NULL,
            agent_id INTEGER NOT NULL,
            agent_name TEXT NOT NULL,
            target_platform TEXT NOT NULL,
            state TEXT NOT NULL DEFAULT 'pending',
            priority INTEGER NOT NULL DEFAULT 0,
            retry_count INTEGER NOT NULL DEFAULT 0,
            max_retries INTEGER NOT NULL DEFAULT 3,
            error_message TEXT DEFAULT '',
            metadata TEXT DEFAULT '{}',
            created_at REAL NOT NULL,
            updated_at REAL NOT NULL,
            processed_at REAL DEFAULT NULL,
            completed_at REAL DEFAULT NULL,
            FOREIGN KEY (agent_id) REFERENCES agents(id)
        )
        """
    )
    conn.execute("CREATE INDEX IF NOT EXISTS idx_syndication_state ON syndication_queue(state, priority DESC, created_at)")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_syndication_video ON syndication_queue(video_id)")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_syndication_agent ON syndication_queue(agent_id)")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_syndication_platform ON syndication_queue(target_platform, state)")

    # Issue #311: Syndication attribution & tracking
    conn.execute(
        """
        CREATE TABLE IF NOT EXISTS syndication_attribution (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            video_id TEXT NOT NULL UNIQUE,
            agent_id INTEGER NOT NULL,
            original_creator TEXT NOT NULL,
            license TEXT DEFAULT 'CC-BY-4.0',
            source_url TEXT DEFAULT '',
            attribution_type TEXT DEFAULT 'original',
            chain TEXT DEFAULT '[]',
            custom_attribution TEXT DEFAULT '{}',
            created_at REAL NOT NULL,
            FOREIGN KEY (video_id) REFERENCES videos(video_id),
            FOREIGN KEY (agent_id) REFERENCES agents(id)
        )
        """
    )
    conn.execute("CREATE INDEX IF NOT EXISTS idx_syndication_attr_video ON syndication_attribution(video_id)")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_syndication_attr_creator ON syndication_attribution(original_creator)")

    conn.execute(
        """
        CREATE TABLE IF NOT EXISTS syndication_log (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            video_id TEXT NOT NULL,
            agent_id INTEGER NOT NULL,
            platform TEXT NOT NULL,
            external_url TEXT NOT NULL,
            external_id TEXT DEFAULT '',
            status TEXT DEFAULT 'pending',
            error TEXT DEFAULT '',
            synced_at REAL,
            created_at REAL NOT NULL,
            FOREIGN KEY (video_id) REFERENCES videos(video_id),
            FOREIGN KEY (agent_id) REFERENCES agents(id)
        )
        """
    )
    conn.execute("CREATE INDEX IF NOT EXISTS idx_syndication_log_video ON syndication_log(video_id)")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_syndication_log_platform ON syndication_log(platform)")

    # Issue #311: Initialize syndication tables from media_prep module
    try:
        from media_prep import init_syndication_tables
        init_syndication_tables(conn)
    except ImportError:
        pass  # media_prep module not available

    conn.commit()
    _sync_default_quests(conn)
    conn.commit()
    conn.close()


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------

def gen_video_id(length=11):
    """Generate a YouTube-style random video ID."""
    chars = string.ascii_letters + string.digits + "-_"
    return "".join(random.choice(chars) for _ in range(length))


def gen_api_key():
    """Generate an API key for an agent."""
    return f"bottube_sk_{secrets.token_hex(24)}"


def _is_rustchain_rtc_address(addr: str) -> bool:
    """RustChain signed transfers require RTC + 40 hex chars (43 chars total)."""
    a = (addr or "").strip()
    return a.startswith("RTC") and len(a) == 43


def _rustchain_post_json(path: str, payload: dict, timeout: float = 10.0):
    """POST JSON to the RustChain node and return (status_code, parsed_json)."""
    url = f"{RUSTCHAIN_BASE_URL}{path}"
    body = json.dumps(payload).encode("utf-8")
    req = urllib.request.Request(
        url,
        data=body,
        headers={"Content-Type": "application/json"},
        method="POST",
    )
    try:
        with urllib.request.urlopen(req, timeout=timeout) as resp:
            raw = resp.read().decode("utf-8", errors="replace")
            return resp.getcode(), json.loads(raw) if raw else {}
    except urllib.error.HTTPError as e:
        raw = ""
        try:
            raw = e.read().decode("utf-8", errors="replace")
        except Exception:
            raw = ""
        try:
            data = json.loads(raw) if raw else {}
        except Exception:
            data = {"error": raw[:200] if raw else "rustchain_http_error"}
        return e.code, data
    except Exception as e:
        return 0, {"error": "rustchain_unreachable", "details": str(e)}


def _sync_pending_tips(db: sqlite3.Connection) -> None:
    """Best-effort: mark tips as confirmed once their RustChain confirms_at has passed."""
    try:
        now = time.time()
        db.execute(
            "UPDATE tips SET status = 'confirmed' "
            "WHERE COALESCE(status, 'confirmed') = 'pending' "
            "AND COALESCE(confirms_at, 0) > 0 AND confirms_at <= ?",
            (now,),
        )
    except Exception:
        pass


DEFAULT_QUESTS = [
    # Onboarding quests (one-time)
    {
        "quest_key": "profile_complete",
        "title": "Finish your profile",
        "description": "Add both a bio and avatar so other creators can recognize you.",
        "category": "onboarding",
        "reward_rtc": 3.0,
        "goal_count": 1,
        "metric_key": "profile_complete",
        "sort_order": 10,
    },
    {
        "quest_key": "first_upload",
        "title": "Publish your first video",
        "description": "Ship one public video to enter the creator feed.",
        "category": "creator",
        "reward_rtc": 8.0,
        "goal_count": 1,
        "metric_key": "first_upload",
        "sort_order": 20,
    },
    {
        "quest_key": "first_comment",
        "title": "Join the conversation",
        "description": "Leave one comment on a video without duplicating spam.",
        "category": "engagement",
        "reward_rtc": 2.0,
        "goal_count": 1,
        "metric_key": "first_comment",
        "sort_order": 30,
    },
    {
        "quest_key": "first_follow",
        "title": "Follow another creator",
        "description": "Subscribe to one creator to unlock your follow feed.",
        "category": "engagement",
        "reward_rtc": 2.0,
        "goal_count": 1,
        "metric_key": "first_follow",
        "sort_order": 40,
    },
    # Creator milestone quests (one-time)
    {
        "quest_key": "five_uploads",
        "title": "Consistent Creator",
        "description": "Publish 5 public videos to establish your presence.",
        "category": "creator",
        "reward_rtc": 15.0,
        "goal_count": 5,
        "metric_key": "total_uploads",
        "sort_order": 50,
    },
    {
        "quest_key": "ten_uploads",
        "title": "Prolific Producer",
        "description": "Publish 10 public videos and become a regular creator.",
        "category": "creator",
        "reward_rtc": 25.0,
        "goal_count": 10,
        "metric_key": "total_uploads",
        "sort_order": 60,
    },
    {
        "quest_key": "hundred_views",
        "title": "First Audience",
        "description": "Accumulate 100 total views across all your videos.",
        "category": "creator",
        "reward_rtc": 10.0,
        "goal_count": 100,
        "metric_key": "total_views",
        "sort_order": 70,
    },
    {
        "quest_key": "thousand_views",
        "title": "Rising Star",
        "description": "Reach 1,000 total views and grow your audience.",
        "category": "creator",
        "reward_rtc": 50.0,
        "goal_count": 1000,
        "metric_key": "total_views",
        "sort_order": 80,
    },
    # Engagement milestone quests (one-time)
    {
        "quest_key": "ten_comments",
        "title": "Active Participant",
        "description": "Leave 10 meaningful comments on other creators' videos.",
        "category": "engagement",
        "reward_rtc": 5.0,
        "goal_count": 10,
        "metric_key": "total_comments",
        "sort_order": 90,
    },
    {
        "quest_key": "fifty_follows",
        "title": "Community Builder",
        "description": "Follow 50 creators to build your network.",
        "category": "engagement",
        "reward_rtc": 10.0,
        "goal_count": 50,
        "metric_key": "total_follows",
        "sort_order": 100,
    },
    # Achievement badges (one-time, high value)
    {
        "quest_key": "viral_video",
        "title": "Viral Sensation",
        "description": "Get a single video to 500+ views.",
        "category": "achievement",
        "reward_rtc": 100.0,
        "goal_count": 1,
        "metric_key": "viral_video",
        "sort_order": 200,
    },
    {
        "quest_key": "liked_creator",
        "title": "Well Liked",
        "description": "Receive 100 total likes across all your videos.",
        "category": "achievement",
        "reward_rtc": 25.0,
        "goal_count": 100,
        "metric_key": "total_likes_received",
        "sort_order": 210,
    },
]


def _sync_default_quests(conn: sqlite3.Connection) -> None:
    """Upsert built-in quests so existing databases pick up new defaults safely."""
    now = time.time()
    for quest in DEFAULT_QUESTS:
        conn.execute(
            """
            INSERT INTO quests
                (quest_key, title, description, category, reward_rtc, goal_count,
                 metric_key, is_active, sort_order, created_at, updated_at)
            VALUES (?, ?, ?, ?, ?, ?, ?, 1, ?, ?, ?)
            ON CONFLICT(quest_key) DO UPDATE SET
                title = excluded.title,
                description = excluded.description,
                category = excluded.category,
                reward_rtc = excluded.reward_rtc,
                goal_count = excluded.goal_count,
                metric_key = excluded.metric_key,
                sort_order = excluded.sort_order,
                updated_at = excluded.updated_at
            """,
            (
                quest["quest_key"],
                quest["title"],
                quest["description"],
                quest["category"],
                float(quest["reward_rtc"]),
                int(quest["goal_count"]),
                quest["metric_key"],
                int(quest["sort_order"]),
                now,
                now,
            ),
        )


def _quest_progress_count(db: sqlite3.Connection, agent_id: int, metric_key: str) -> int:
    """Map a quest metric to current progress for an agent."""
    if metric_key == "profile_complete":
        row = db.execute(
            "SELECT bio, avatar_url FROM agents WHERE id = ?",
            (agent_id,),
        ).fetchone()
        if not row:
            return 0
        return int(bool((row["bio"] or "").strip()) and bool((row["avatar_url"] or "").strip()))
    if metric_key == "first_upload":
        return int(
            db.execute(
                "SELECT COUNT(*) FROM videos WHERE agent_id = ? AND COALESCE(is_removed, 0) = 0",
                (agent_id,),
            ).fetchone()[0]
            or 0
        )
    if metric_key == "total_uploads":
        return int(
            db.execute(
                "SELECT COUNT(*) FROM videos WHERE agent_id = ? AND COALESCE(is_removed, 0) = 0",
                (agent_id,),
            ).fetchone()[0]
            or 0
        )
    if metric_key == "total_views":
        return int(
            db.execute(
                "SELECT COALESCE(SUM(views), 0) FROM videos WHERE agent_id = ?",
                (agent_id,),
            ).fetchone()[0]
            or 0
        )
    if metric_key == "first_comment":
        return int(
            db.execute(
                "SELECT COUNT(*) FROM comments WHERE agent_id = ?",
                (agent_id,),
            ).fetchone()[0]
            or 0
        )
    if metric_key == "total_comments":
        return int(
            db.execute(
                "SELECT COUNT(*) FROM comments WHERE agent_id = ?",
                (agent_id,),
            ).fetchone()[0]
            or 0
        )
    if metric_key == "first_follow":
        return int(
            db.execute(
                "SELECT COUNT(*) FROM subscriptions WHERE follower_id = ?",
                (agent_id,),
            ).fetchone()[0]
            or 0
        )
    if metric_key == "total_follows":
        return int(
            db.execute(
                "SELECT COUNT(*) FROM subscriptions WHERE follower_id = ?",
                (agent_id,),
            ).fetchone()[0]
            or 0
        )
    if metric_key == "total_likes_received":
        # Sum of all likes received on agent's videos
        return int(
            db.execute(
                """
                SELECT COALESCE(SUM(v.likes), 0)
                FROM videos v
                WHERE v.agent_id = ?
                """,
                (agent_id,),
            ).fetchone()[0]
            or 0
        )
    if metric_key == "viral_video":
        # Check if any video has 500+ views
        row = db.execute(
            "SELECT 1 FROM videos WHERE agent_id = ? AND views >= 500 AND COALESCE(is_removed, 0) = 0 LIMIT 1",
            (agent_id,),
        ).fetchone()
        return 1 if row else 0
    return 0


def _refresh_agent_quests(
    db: sqlite3.Connection,
    agent_id: int,
    quest_keys: Optional[List[str]] = None,
) -> List[Dict]:
    """Refresh quest progress, award one-time RTC, and return quest snapshots."""
    params: list = []
    where = "WHERE is_active = 1"
    if quest_keys:
        placeholders = ",".join("?" for _ in quest_keys)
        where += f" AND quest_key IN ({placeholders})"
        params.extend(quest_keys)

    rows = db.execute(
        f"""
        SELECT quest_key, title, description, category, reward_rtc, goal_count,
               metric_key, sort_order
        FROM quests
        {where}
        ORDER BY sort_order ASC, quest_key ASC
        """,
        params,
    ).fetchall()

    now = time.time()
    snapshots: list[dict] = []
    for quest in rows:
        goal_count = max(1, int(quest["goal_count"] or 1))
        progress_count = min(goal_count, _quest_progress_count(db, agent_id, quest["metric_key"]))
        existing = db.execute(
            """
            SELECT progress_count, completed_at, rewarded_at, metadata
            FROM agent_quests
            WHERE agent_id = ? AND quest_key = ?
            """,
            (agent_id, quest["quest_key"]),
        ).fetchone()

        completed_at = float(existing["completed_at"] or 0) if existing else 0.0
        rewarded_at = float(existing["rewarded_at"] or 0) if existing else 0.0
        if progress_count >= goal_count and completed_at <= 0:
            completed_at = now

        if existing:
            db.execute(
                """
                UPDATE agent_quests
                SET progress_count = ?, completed_at = ?, last_event_at = ?, metadata = ?
                WHERE agent_id = ? AND quest_key = ?
                """,
                (
                    progress_count,
                    completed_at,
                    now,
                    json.dumps({"metric_key": quest["metric_key"], "goal_count": goal_count}),
                    agent_id,
                    quest["quest_key"],
                ),
            )
        else:
            db.execute(
                """
                INSERT INTO agent_quests
                    (agent_id, quest_key, progress_count, completed_at, rewarded_at, last_event_at, metadata)
                VALUES (?, ?, ?, ?, ?, ?, ?)
                """,
                (
                    agent_id,
                    quest["quest_key"],
                    progress_count,
                    completed_at,
                    rewarded_at,
                    now,
                    json.dumps({"metric_key": quest["metric_key"], "goal_count": goal_count}),
                ),
            )

        if completed_at > 0 and rewarded_at <= 0 and float(quest["reward_rtc"] or 0) > 0:
            reward_reason = f"quest_complete:{quest['quest_key']}"
            already_rewarded = db.execute(
                "SELECT 1 FROM earnings WHERE agent_id = ? AND reason = ? LIMIT 1",
                (agent_id, reward_reason),
            ).fetchone()
            if not already_rewarded:
                award_rtc(db, agent_id, float(quest["reward_rtc"]), reward_reason)
            rewarded_at = now
            db.execute(
                "UPDATE agent_quests SET rewarded_at = ? WHERE agent_id = ? AND quest_key = ?",
                (rewarded_at, agent_id, quest["quest_key"]),
            )

        snapshots.append({
            "quest_key": quest["quest_key"],
            "title": quest["title"],
            "description": quest["description"],
            "category": quest["category"],
            "reward_rtc": float(quest["reward_rtc"] or 0),
            "goal_count": goal_count,
            "progress_count": progress_count,
            "completed": completed_at > 0,
            "completed_at": completed_at,
            "rewarded_at": rewarded_at,
            "metric_key": quest["metric_key"],
        })
    return snapshots


def _derive_rtc_address_from_pubkey(public_key_hex: str) -> str:
    """RustChain address format: RTC + first 40 hex chars of SHA256(pubkey_bytes)."""
    pub_bytes = bytes.fromhex(public_key_hex)
    return f"RTC{hashlib.sha256(pub_bytes).hexdigest()[:40]}"


def _handle_onchain_tip(
    db: sqlite3.Connection,
    *,
    sender_id: int,
    sender_name: str,
    recipient_id: int,
    recipient_name: str,
    expected_to_wallet: str,
    amount: float,
    user_message: str,
    data: dict,
    video_id: str = "",
    video_title: str = "",
):
    """Validate + forward a RustChain signed transfer, then record as a pending tip."""
    required = ["from_address", "to_address", "nonce", "signature", "public_key", "memo"]
    missing = [k for k in required if not (data or {}).get(k)]
    if missing:
        return {"error": "Missing required fields for on-chain tip", "missing": missing}, 400

    from_address = str(data.get("from_address", "")).strip()
    to_address = str(data.get("to_address", "")).strip()
    signature = str(data.get("signature", "")).strip()
    public_key = str(data.get("public_key", "")).strip()
    memo = str(data.get("memo", "")).strip()
    try:
        nonce_int = int(str(data.get("nonce")))
    except (TypeError, ValueError):
        return {"error": "Invalid nonce (must be int)"}, 400

    if nonce_int <= 0:
        return {"error": "Invalid nonce (must be > 0)"}, 400

    if not _is_rustchain_rtc_address(from_address):
        return {"error": "Invalid from_address format (expected RTC... address)"}, 400
    if not _is_rustchain_rtc_address(to_address):
        return {"error": "Invalid to_address format (expected RTC... address)"}, 400

    if to_address != expected_to_wallet:
        return {"error": "to_address does not match creator wallet", "expected": expected_to_wallet, "got": to_address}, 400

    try:
        expected_from = _derive_rtc_address_from_pubkey(public_key)
    except Exception:
        return {"error": "Invalid public_key (expected hex)"}, 400

    if expected_from != from_address:
        return {"error": "public_key does not match from_address", "expected": expected_from, "got": from_address}, 400

    # If the sender has linked a RustChain wallet in their profile, enforce match.
    try:
        row = db.execute("SELECT rtc_wallet FROM agents WHERE id = ?", (sender_id,)).fetchone()
        linked = (row["rtc_wallet"] or "").strip() if row else ""
    except Exception:
        linked = ""
    if linked and linked != from_address:
        return {"error": "from_address does not match your linked rtc_wallet", "linked": linked, "got": from_address}, 400

    rc_payload = {
        "from_address": from_address,
        "to_address": to_address,
        "amount_rtc": amount,
        "nonce": nonce_int,
        "signature": signature,
        "public_key": public_key,
        "memo": memo,
    }
    status, rc_resp = _rustchain_post_json("/wallet/transfer/signed", rc_payload, timeout=12.0)
    if status != 200 or not isinstance(rc_resp, dict) or not rc_resp.get("ok"):
        err = rc_resp.get("error") if isinstance(rc_resp, dict) else "rustchain_error"
        return {"error": "RustChain transfer failed", "rustchain_status": status, "rustchain_error": err, "rustchain": rc_resp}, 502

    tx_hash = str(rc_resp.get("tx_hash", "")).strip() or None
    pending_id = int(rc_resp.get("pending_id", 0) or 0)
    confirms_at = float(rc_resp.get("confirms_at", 0) or 0)

    db.execute(
        "INSERT INTO tips "
        "(from_agent_id, to_agent_id, video_id, amount, message, onchain, status, tx_hash, pending_id, confirms_at, from_address, to_address, created_at) "
        "VALUES (?, ?, ?, ?, ?, 1, 'pending', ?, ?, ?, ?, ?, ?)",
        (
            sender_id,
            recipient_id,
            video_id or "",
            amount,
            user_message,
            tx_hash,
            pending_id,
            confirms_at,
            from_address,
            to_address,
            time.time(),
        ),
    )

    # Notify recipient (tip is pending until RustChain confirms it)
    what = f'@{sender_name} tipped {amount:.4f} RTC (on-chain, pending)'
    if video_title:
        what += f' on "{video_title}"'
    if user_message:
        what += f': "{user_message}"'
    notify(db, recipient_id, "tip", what, from_agent=sender_name, video_id=video_id or "")
    evidence_ref = f"/watch/{video_id}" if video_id else f"/agent/{recipient_name}"
    _referral_mark_rtc_native_action(db, sender_id, evidence_ref=evidence_ref)
    _referral_mark_rtc_native_action(db, recipient_id, evidence_ref=evidence_ref)

    return {
        "ok": True,
        "onchain": True,
        "phase": str(rc_resp.get("phase", "pending")),
        "pending_id": pending_id,
        "tx_hash": tx_hash,
        "confirms_at": confirms_at,
        "to": recipient_name,
        "amount": amount,
        "video_id": video_id or "",
    }, 200



# ---------------------------------------------------------------------------
# IndexNow ping — notify search engines of new/updated content
# ---------------------------------------------------------------------------

INDEXNOW_KEY = "bottube64db02b03f2d3732"

def _ping_indexnow(url):
    """Fire-and-forget IndexNow ping to notify search engines of a new URL."""
    def _do_ping():
        try:
            payload = json.dumps({
                "host": "bottube.ai",
                "key": INDEXNOW_KEY,
                "keyLocation": "https://bottube.ai/static/bottube64db02b03f2d3732.txt",
                "urlList": [url] if isinstance(url, str) else url,
            }).encode()
            req = urllib.request.Request(
                "https://api.indexnow.org/indexnow",
                data=payload,
                headers={"Content-Type": "application/json"},
                method="POST",
            )
            urllib.request.urlopen(req, timeout=10)
        except Exception:
            pass  # Fire-and-forget; never block on failure
    threading.Thread(target=_do_ping, daemon=True).start()


def award_rtc(db, agent_id: int, amount: float, reason: str, video_id: str = "", apply_streak_bonus: bool = True):
    """Award RTC tokens to an agent and log the earning. Optionally applies streak bonus."""
    final_amount = amount
    
    # Apply streak bonus for engagement rewards (not for quest completions)
    if apply_streak_bonus and not reason.startswith("quest_complete:"):
        streak_days = _activity_streak_days(db, agent_id)
        multiplier = _get_streak_bonus_multiplier(streak_days)
        if multiplier > 1.0:
            bonus = amount * (multiplier - 1.0)
            final_amount = amount + bonus
    
    db.execute(
        "UPDATE agents SET rtc_balance = rtc_balance + ? WHERE id = ?",
        (final_amount, agent_id),
    )
    db.execute(
        "INSERT INTO earnings (agent_id, amount, reason, video_id, created_at) VALUES (?, ?, ?, ?, ?)",
        (agent_id, final_amount, reason, video_id, time.time()),
    )


def _queue_reward_hold(
    db: sqlite3.Connection,
    *,
    agent_id: int,
    event_type: str,
    event_ref: str,
    amount: float,
    risk_score: int,
    reasons: list[str],
) -> None:
    """Persist a suspicious reward instead of paying it immediately."""
    db.execute(
        """
        INSERT INTO reward_holds
            (agent_id, event_type, event_ref, amount, status, risk_score, reasons, created_at)
        VALUES (?, ?, ?, ?, 'pending', ?, ?, ?)
        ON CONFLICT(agent_id, event_type, event_ref) DO UPDATE SET
            risk_score = excluded.risk_score,
            reasons = excluded.reasons
        """,
        (agent_id, event_type, event_ref, amount, int(risk_score), json.dumps(reasons), time.time()),
    )


def _agent_name_by_id(db: sqlite3.Connection, agent_id: Optional[int]) -> Optional[str]:
    if not agent_id:
        return None
    row = db.execute("SELECT agent_name FROM agents WHERE id = ?", (agent_id,)).fetchone()
    return row["agent_name"] if row else None


def _send_coaching_note(
    db: sqlite3.Connection,
    *,
    agent_id: Optional[int],
    subject: str,
    body: str,
    video_id: str = "",
) -> None:
    """Send a moderation/coaching note without blocking the main flow."""
    agent_name = _agent_name_by_id(db, agent_id)
    if not agent_name:
        return
    db.execute(
        """INSERT INTO messages (id, from_agent, to_agent, subject, body, message_type)
           VALUES (?, 'system', ?, ?, ?, 'moderation')""",
        (_gen_message_id(), agent_name, subject[:200], body[:5000]),
    )
    notify(db, agent_id, "moderation", subject[:160], from_agent="system", video_id=video_id)


def _queue_moderation_hold(
    db: sqlite3.Connection,
    *,
    target_type: str,
    target_ref: str,
    target_agent_id: Optional[int],
    source: str,
    reason: str,
    details: str = "",
    recommended_action: str = "coach",
    coach_note: str = "",
) -> Optional[int]:
    """Queue a moderation hold instead of deleting or banning by default."""
    now = time.time()
    try:
        cur = db.execute(
            """
            INSERT INTO moderation_holds
                (target_type, target_ref, target_agent_id, source, reason, details,
                 status, recommended_action, coach_note, created_at)
            VALUES (?, ?, ?, ?, ?, ?, 'pending', ?, ?, ?)
            """,
            (
                target_type,
                target_ref,
                target_agent_id,
                source,
                reason,
                details[:2000],
                recommended_action,
                coach_note[:5000],
                now,
            ),
        )
        hold_id = int(cur.lastrowid)
    except sqlite3.IntegrityError:
        row = db.execute(
            """
            SELECT id FROM moderation_holds
            WHERE target_type = ? AND target_ref = ? AND source = ? AND reason = ?
            """,
            (target_type, target_ref, source, reason),
        ).fetchone()
        hold_id = int(row["id"]) if row else None

    if coach_note:
        _send_coaching_note(
            db,
            agent_id=target_agent_id,
            subject=f"BoTTube coaching: {reason}",
            body=coach_note,
            video_id=target_ref if target_type == "video" else "",
        )
    return hold_id


def _comment_reward_decision(
    db: sqlite3.Connection,
    *,
    agent_id: int,
    video_id: str,
    comment_id: int,
    content: str,
) -> dict:
    """Score a comment reward and either pay it or hold it for review."""
    now = time.time()
    reasons: list[str] = []
    risk = 0
    content_norm = " ".join((content or "").strip().lower().split())
    tokens = re.findall(r"[a-z0-9']+", content_norm)
    unique_ratio = (len(set(tokens)) / len(tokens)) if tokens else 0.0

    if len(content_norm) < 24:
        risk += 18
        reasons.append("comment too short")
    if tokens and unique_ratio < 0.55:
        risk += 18
        reasons.append("low token variety")
    if re.search(r"(.)\1{5,}", content_norm):
        risk += 20
        reasons.append("repeated characters")
    if "http://" in content_norm or "https://" in content_norm:
        risk += 18
        reasons.append("contains outbound link")

    # Check for new account
    is_new, new_cap = _check_new_account_reward_cap(db, agent_id)
    if is_new:
        risk += 12
        reasons.append("new account")

    # Check for rapid activity (anti-farm)
    rapid = _check_rapid_activity(db, agent_id, "comment")
    if rapid["is_suspicious"]:
        risk += rapid["risk_score"]
        reasons.append(f"rapid comment activity ({rapid['count']}/hr)")

    same_video_recent = db.execute(
        "SELECT COUNT(*) FROM comments WHERE agent_id = ? AND video_id = ? AND created_at >= ?",
        (agent_id, video_id, now - 86400),
    ).fetchone()[0]
    if int(same_video_recent or 0) >= 3:
        risk += 12
        reasons.append("repeated target video")

    owner_row = db.execute(
        "SELECT agent_id FROM videos WHERE video_id = ?",
        (video_id,),
    ).fetchone()
    target_agent_id = int(owner_row["agent_id"]) if owner_row else 0

    day_start = now - 86400
    today_comment_earnings = float(
        db.execute(
            "SELECT COALESCE(SUM(amount), 0) FROM earnings WHERE agent_id = ? AND reason = 'comment' AND created_at >= ?",
            (agent_id, day_start),
        ).fetchone()[0]
        or 0.0
    )
    if today_comment_earnings >= COMMENT_REWARD_DAILY_CAP:
        risk += 30
        reasons.append("daily comment reward cap reached")

    target_comment_count = 0
    if target_agent_id:
        target_comment_count = int(
            db.execute(
                """
                SELECT COUNT(*)
                FROM comments c
                JOIN videos v ON v.video_id = c.video_id
                WHERE c.agent_id = ?
                  AND v.agent_id = ?
                  AND c.created_at >= ?
                """,
                (agent_id, target_agent_id, day_start),
            ).fetchone()[0]
            or 0
        )
    if (target_comment_count * RTC_REWARD_COMMENT) >= COMMENT_REWARD_TARGET_DAILY_CAP:
        risk += 20
        reasons.append("same-creator reward cap reached")

    hold = risk >= COMMENT_REWARD_HOLD_THRESHOLD
    if hold:
        _queue_reward_hold(
            db,
            agent_id=agent_id,
            event_type="comment",
            event_ref=str(comment_id),
            amount=RTC_REWARD_COMMENT,
            risk_score=risk,
            reasons=reasons or ["anti-farm hold"],
        )
        return {"awarded": False, "held": True, "risk_score": risk, "reasons": reasons}

    award_rtc(db, agent_id, RTC_REWARD_COMMENT, "comment", video_id)
    return {"awarded": True, "held": False, "risk_score": risk, "reasons": reasons}


def _view_reward_decision(
    db: sqlite3.Connection,
    *,
    owner_id: int,
    viewer_id: Optional[int],
    video_id: str,
    view_event_ref: str,
    ip_address: str,
) -> Dict:
    """Score a view reward and either pay it or hold it for review."""
    now = time.time()
    reasons: List[str] = []
    risk = 0

    if viewer_id and viewer_id == owner_id:
        risk += 100
        reasons.append("self-view")
    if not viewer_id:
        risk += 14
        reasons.append("anonymous reward source")
    else:
        viewer_row = db.execute(
            "SELECT created_at FROM agents WHERE id = ?",
            (viewer_id,),
        ).fetchone()
        if viewer_row and (now - float(viewer_row["created_at"] or now)) < 86400:
            risk += 10
            reasons.append("new viewer account")

        recent_hour = db.execute(
            "SELECT COUNT(*) FROM views WHERE agent_id = ? AND created_at >= ?",
            (viewer_id, now - 3600),
        ).fetchone()[0]
        if int(recent_hour or 0) >= 20:
            risk += 16
            reasons.append("high hourly view velocity")

        same_creator_views = db.execute(
            """
            SELECT COUNT(*)
            FROM views vw
            JOIN videos v ON v.video_id = vw.video_id
            WHERE vw.agent_id = ?
              AND v.agent_id = ?
              AND vw.created_at >= ?
            """,
            (viewer_id, owner_id, now - 86400),
        ).fetchone()[0]
        if (int(same_creator_views or 0) * RTC_REWARD_VIEW) >= VIEW_REWARD_TARGET_DAILY_CAP:
            risk += 16
            reasons.append("same-creator view reward cap reached")

    same_ip_views = db.execute(
        """
        SELECT COUNT(*)
        FROM views vw
        JOIN videos v ON v.video_id = vw.video_id
        WHERE vw.ip_address = ?
          AND v.agent_id = ?
          AND vw.created_at >= ?
        """,
        (ip_address, owner_id, now - 86400),
    ).fetchone()[0]
    if int(same_ip_views or 0) >= 12:
        risk += 18
        reasons.append("same-ip creator view concentration")

    today_view_earnings = float(
        db.execute(
            "SELECT COALESCE(SUM(amount), 0) FROM earnings WHERE agent_id = ? AND reason = 'video_view' AND created_at >= ?",
            (owner_id, now - 86400),
        ).fetchone()[0]
        or 0.0
    )
    if today_view_earnings >= VIEW_REWARD_DAILY_CAP:
        risk += 24
        reasons.append("daily view reward cap reached")

    hold = risk >= VIEW_REWARD_HOLD_THRESHOLD
    if hold:
        _queue_reward_hold(
            db,
            agent_id=owner_id,
            event_type="video_view",
            event_ref=view_event_ref,
            amount=RTC_REWARD_VIEW,
            risk_score=risk,
            reasons=reasons or ["anti-farm hold"],
        )
        return {"awarded": False, "held": True, "risk_score": risk, "reasons": reasons}

    award_rtc(db, owner_id, RTC_REWARD_VIEW, "video_view", video_id)
    return {"awarded": True, "held": False, "risk_score": risk, "reasons": reasons}


def _like_reward_decision(
    db: sqlite3.Connection,
    *,
    owner_id: int,
    voter_id: int,
    video_id: str,
    like_event_ref: str,
) -> dict:
    """Score a like-received reward and either pay it or hold it for review."""
    now = time.time()
    reasons: list[str] = []
    risk = 0

    if voter_id == owner_id:
        risk += 100
        reasons.append("self-like")

    # Check for new voter account
    is_new, _ = _check_new_account_reward_cap(db, voter_id)
    if is_new:
        risk += 12
        reasons.append("new voter account")

    # Check for rapid like activity (anti-farm)
    rapid = _check_rapid_activity(db, voter_id, "like")
    if rapid["is_suspicious"]:
        risk += rapid["risk_score"]
        reasons.append(f"rapid like activity ({rapid['count']}/hr)")

    same_creator_likes = db.execute(
        """
        SELECT COUNT(*)
        FROM votes vt
        JOIN videos v ON v.video_id = vt.video_id
        WHERE vt.agent_id = ?
          AND vt.vote = 1
          AND v.agent_id = ?
          AND vt.created_at >= ?
        """,
        (voter_id, owner_id, now - 86400),
    ).fetchone()[0]
    if (int(same_creator_likes or 0) * RTC_REWARD_LIKE_RECEIVED) >= LIKE_REWARD_TARGET_DAILY_CAP:
        risk += 18
        reasons.append("same-creator like reward cap reached")

    today_like_earnings = float(
        db.execute(
            "SELECT COALESCE(SUM(amount), 0) FROM earnings WHERE agent_id = ? AND reason = 'like_received' AND created_at >= ?",
            (owner_id, now - 86400),
        ).fetchone()[0]
        or 0.0
    )
    if today_like_earnings >= LIKE_REWARD_DAILY_CAP:
        risk += 25
        reasons.append("daily like reward cap reached")

    hold = risk >= LIKE_REWARD_HOLD_THRESHOLD
    if hold:
        _queue_reward_hold(
            db,
            agent_id=owner_id,
            event_type="like_received",
            event_ref=like_event_ref,
            amount=RTC_REWARD_LIKE_RECEIVED,
            risk_score=risk,
            reasons=reasons or ["anti-farm hold"],
        )
        return {"awarded": False, "held": True, "risk_score": risk, "reasons": reasons}

    award_rtc(db, owner_id, RTC_REWARD_LIKE_RECEIVED, "like_received", video_id)
    return {"awarded": True, "held": False, "risk_score": risk, "reasons": reasons}


def _activity_streak_days(db: sqlite3.Connection, agent_id: int) -> int:
    """Count consecutive days with creator activity."""
    rows = db.execute(
        """
        SELECT day FROM (
            SELECT strftime('%Y-%m-%d', datetime(created_at, 'unixepoch')) AS day
            FROM videos WHERE agent_id = ?
            UNION
            SELECT strftime('%Y-%m-%d', datetime(created_at, 'unixepoch')) AS day
            FROM comments WHERE agent_id = ?
            UNION
            SELECT strftime('%Y-%m-%d', datetime(created_at, 'unixepoch')) AS day
            FROM subscriptions WHERE follower_id = ?
        )
        ORDER BY day DESC
        """,
        (agent_id, agent_id, agent_id),
    ).fetchall()
    if not rows:
        return 0

    active_days = {r["day"] for r in rows if r["day"]}
    streak = 0
    day_ts = int(time.time() // 86400) * 86400
    while True:
        day = datetime.datetime.utcfromtimestamp(day_ts).strftime("%Y-%m-%d")
        if day not in active_days:
            break
        streak += 1
        day_ts -= 86400
    return streak


def _get_streak_bonus_multiplier(streak_days: int) -> float:
    """Get the reward bonus multiplier for a given streak length."""
    if streak_days <= 0:
        return 1.0
    # Find the highest threshold that the streak meets
    applicable_multiplier = 1.0
    for threshold, multiplier in sorted(STREAK_BONUS_MULTIPLIERS.items()):
        if streak_days >= threshold:
            applicable_multiplier = multiplier
        else:
            break
    return applicable_multiplier


def _get_agent_xp(db: sqlite3.Connection, agent_id: int) -> int:
    """Calculate total XP from quest completions (1 XP per 1 RTC earned from quests)."""
    row = db.execute(
        """
        SELECT COALESCE(SUM(amount), 0) as total_xp
        FROM earnings
        WHERE agent_id = ? AND reason LIKE 'quest_complete:%'
        """,
        (agent_id,),
    ).fetchone()
    if row is None:
        return 0
    # Handle both dict-like rows (row_factory) and tuple rows
    xp = row[0] if isinstance(row, tuple) else row["total_xp"]
    return int(xp or 0)


def _get_agent_level(agent_xp: int) -> int:
    """Determine agent level based on total XP."""
    level = 1
    for lvl, threshold in LEVEL_THRESHOLDS:
        if agent_xp >= threshold:
            level = lvl
        else:
            break
    return level


def _get_agent_level_info(db: sqlite3.Connection, agent_id: int) -> Dict:
    """Get full level information for an agent."""
    xp = _get_agent_xp(db, agent_id)
    level = _get_agent_level(xp)
    
    # Find current and next level thresholds
    current_threshold = 0
    next_threshold = None
    for lvl, threshold in LEVEL_THRESHOLDS:
        if lvl == level:
            current_threshold = threshold
        elif lvl == level + 1:
            next_threshold = threshold
            break
    
    progress_to_next = 0.0
    if next_threshold is not None:
        progress_to_next = (xp - current_threshold) / (next_threshold - current_threshold)
    else:
        progress_to_next = 1.0  # Max level
    
    return {
        "level": level,
        "xp": xp,
        "current_threshold": current_threshold,
        "next_threshold": next_threshold,
        "progress_to_next": round(progress_to_next, 4),
        "max_level": LEVEL_THRESHOLDS[-1][0],
    }


def _check_rapid_activity(db: sqlite3.Connection, agent_id: int, activity_type: str) -> Dict:
    """
    Check for suspicious rapid activity patterns (anti-farm detection).
    Returns dict with is_suspicious, count, and risk_score.
    """
    now = time.time()
    hour_ago = now - 3600
    
    config = ANTI_FARM_CONFIG
    
    if activity_type == "comment":
        count = db.execute(
            "SELECT COUNT(*) FROM comments WHERE agent_id = ? AND created_at >= ?",
            (agent_id, hour_ago),
        ).fetchone()[0] or 0
        threshold = config["rapid_comment_threshold"]
    elif activity_type == "like":
        count = db.execute(
            "SELECT COUNT(*) FROM votes WHERE agent_id = ? AND vote = 1 AND created_at >= ?",
            (agent_id, hour_ago),
        ).fetchone()[0] or 0
        threshold = config["rapid_like_threshold"]
    else:
        return {"is_suspicious": False, "count": 0, "risk_score": 0}

    if count >= threshold:
        risk_score = min(50, 15 + (count - threshold) * 5)
        return {"is_suspicious": True, "count": count, "risk_score": risk_score}

    return {"is_suspicious": False, "count": count, "risk_score": 0}


def _check_new_account_reward_cap(db: sqlite3.Connection, agent_id: int) -> Tuple[bool, float]:
    """
    Check if agent is a new account and should have reward caps applied.
    Returns (is_new_account, daily_cap).
    """
    now = time.time()
    config = ANTI_FARM_CONFIG
    
    row = db.execute(
        "SELECT created_at FROM agents WHERE id = ?",
        (agent_id,),
    ).fetchone()
    
    if not row:
        return False, 0.0
    
    # Handle both dict-like rows (row_factory) and tuple rows
    created_at = row[0] if isinstance(row, tuple) else row["created_at"]
    account_age = now - float(created_at or now)
    
    if account_age < config["new_account_age_secs"]:
        return True, config["new_account_reward_cap"]
    
    return False, 0.0


# ---------------------------------------------------------------------------
# Email rate-limit tracker (in-memory, per-process)
# ---------------------------------------------------------------------------
_email_rate: dict = {}  # {agent_id: [timestamp, ...]}

def _notify_subscribers_new_video(agent_id, video_id, video_title, uploader_name):
    """Notify all subscribers of a channel about a new video upload (background thread)."""
    def _do_notify():
        try:
            conn = sqlite3.connect(str(DB_PATH))
            conn.row_factory = sqlite3.Row
            subs = conn.execute(
                "SELECT follower_id FROM subscriptions WHERE following_id = ?",
                (agent_id,)
            ).fetchall()
            for sub in subs:
                conn.execute(
                    "INSERT INTO notifications (agent_id, type, message, from_agent, video_id, is_read, created_at) "
                    "VALUES (?, ?, ?, ?, ?, 0, ?)",
                    (sub["follower_id"], "new_video",
                     f'@{uploader_name} uploaded a new video: "{video_title}"',
                     uploader_name, video_id, time.time()),
                )
                # Fire webhooks for each subscriber
                fire_webhooks(sub["follower_id"], "new_video", {
                    "type": "new_video",
                    "message": f'@{uploader_name} uploaded a new video: "{video_title}"',
                    "from_agent": uploader_name,
                    "video_id": video_id,
                    "timestamp": time.time(),
                })
                # Send email notification if preferences allow
                _maybe_send_notification_email(
                    conn, sub["follower_id"], "new_video",
                    f'{uploader_name} uploaded a new video',
                    f'@{uploader_name} uploaded: "{video_title}"',
                    video_id
                )
            conn.commit()
            conn.close()
        except Exception as e:
            import traceback
            traceback.print_exc()
    threading.Thread(target=_do_notify, daemon=True).start()


def _maybe_send_notification_email(db_conn, agent_id, notif_type, subject, message, video_id=""):
    """Check agent email preferences and send notification email if enabled. Thread-safe."""
    if not SMTP_HOST:
        return
    PREF_MAP = {
        "comment": "email_notify_comments",
        "reply": "email_notify_replies",
        "new_video": "email_notify_new_video",
        "tip": "email_notify_tips",
        "subscribe": "email_notify_subscriptions",
    }
    pref_col = PREF_MAP.get(notif_type)
    if not pref_col:
        return  # Unsupported notification type for email

    agent = db_conn.execute(
        "SELECT email, email_verified, email_unsubscribe_token, " + pref_col +
        " FROM agents WHERE id = ?", (agent_id,)
    ).fetchone()
    if not agent:
        return
    email = agent["email"]
    if not email or not agent["email_verified"]:
        return
    if not agent[pref_col]:
        return  # User disabled this email type

    # Rate limit: max 10 emails per user per hour
    now = time.time()
    hour_ago = now - 3600
    bucket = _email_rate.setdefault(agent_id, [])
    _email_rate[agent_id] = bucket = [t for t in bucket if t > hour_ago]
    if len(bucket) >= 10:
        return  # Rate limited
    bucket.append(now)

    # Build unsubscribe URL
    unsub_token = agent["email_unsubscribe_token"]
    if not unsub_token:
        unsub_token = secrets.token_hex(16)
        db_conn.execute(
            "UPDATE agents SET email_unsubscribe_token = ? WHERE id = ?",
            (unsub_token, agent_id)
        )
        db_conn.commit()

    unsub_url = f"https://bottube.ai/unsubscribe/{unsub_token}"
    unsub_type_url = f"https://bottube.ai/unsubscribe/{unsub_token}/{notif_type}"
    video_url = f"https://bottube.ai/watch/{video_id}" if video_id else ""

    send_notification_email(
        to_email=email,
        subject=f"[BoTTube] {subject}",
        body_text=f"{message}\n\n" + (f"Watch: {video_url}\n\n" if video_url else "") +
                  f"Unsubscribe from {notif_type} emails: {unsub_type_url}\n"
                  f"Unsubscribe from all emails: {unsub_url}",
        body_html=_build_notification_html(subject, message, video_url, unsub_url, unsub_type_url, notif_type),
        unsub_url=unsub_url,
    )


def _build_notification_html(subject, message, video_url, unsub_url, unsub_type_url, notif_type):
    """Build branded HTML email for a notification."""
    video_link = f'<p style="text-align:center;margin:16px 0;"><a href="{video_url}" style="background:#3ea6ff;color:#0f0f0f;padding:10px 24px;border-radius:6px;text-decoration:none;font-weight:700;display:inline-block;">Watch Now</a></p>' if video_url else ""
    return f"""<div style="font-family:sans-serif;max-width:520px;margin:0 auto;background:#1a1a1a;color:#f1f1f1;padding:32px;border-radius:8px;">
<h2 style="color:#3ea6ff;margin-top:0;">BoTTube</h2>
<p style="font-size:16px;">{message}</p>
{video_link}
<hr style="border:none;border-top:1px solid #333;margin:24px 0;">
<p style="font-size:11px;color:#717171;">
  <a href="{unsub_type_url}" style="color:#717171;">Unsubscribe from {notif_type} emails</a> &middot;
  <a href="{unsub_url}" style="color:#717171;">Unsubscribe from all emails</a>
</p>
</div>"""


def notify(db, agent_id: int, notif_type: str, message: str, from_agent: str = "", video_id: str = ""):
    """Create a notification for an agent. Skips if agent_id matches from_agent (no self-notifications)."""
    if from_agent:
        sender = db.execute("SELECT id FROM agents WHERE agent_name = ?", (from_agent,)).fetchone()
        if sender and sender["id"] == agent_id:
            return
    db.execute(
        "INSERT INTO notifications (agent_id, type, message, from_agent, video_id, is_read, created_at) VALUES (?, ?, ?, ?, ?, 0, ?)",
        (agent_id, notif_type, message, from_agent, video_id, time.time()),
    )
    # Fire webhooks for this agent
    fire_webhooks(agent_id, notif_type, {
        "type": notif_type,
        "message": message,
        "from_agent": from_agent,
        "video_id": video_id,
        "timestamp": time.time(),
    })

    # Send email notification if preferences allow (background thread)
    def _send_email_bg():
        try:
            conn = sqlite3.connect(str(DB_PATH))
            conn.row_factory = sqlite3.Row
            _maybe_send_notification_email(conn, agent_id, notif_type, message[:80], message, video_id)
            conn.close()
        except Exception:
            pass
    threading.Thread(target=_send_email_bg, daemon=True).start()


def _notification_link_for_row(row) -> str:
    video_id = str(row["video_id"] or "").strip()
    from_agent = str(row["from_agent"] or "").strip()
    if video_id:
        return f"{g.prefix}/watch/{video_id}"
    if from_agent:
        return f"{g.prefix}/agent/{from_agent}"
    return f"{g.prefix}/dashboard"


def _notification_to_dict(row) -> dict:
    return {
        "id": row["id"],
        "type": row["type"],
        "message": row["message"],
        "from_agent": row["from_agent"],
        "video_id": row["video_id"],
        "is_read": bool(row["is_read"]),
        "created_at": row["created_at"],
        "link": _notification_link_for_row(row),
    }


def _notification_unread_count(db, agent_id: int) -> int:
    return int(
        db.execute(
            "SELECT COUNT(*) FROM notifications WHERE agent_id = ? AND is_read = 0",
            (agent_id,),
        ).fetchone()[0]
    )


def _notification_page(db, agent_id: int, page: int, per_page: int, unread_only: bool) -> tuple[list[dict], int]:
    where = "WHERE agent_id = ?" if not unread_only else "WHERE agent_id = ? AND is_read = 0"
    total = int(db.execute(f"SELECT COUNT(*) FROM notifications {where}", (agent_id,)).fetchone()[0])
    offset = (page - 1) * per_page
    rows = db.execute(
        f"""
        SELECT id, type, message, from_agent, video_id, is_read, created_at
        FROM notifications {where}
        ORDER BY created_at DESC
        LIMIT ? OFFSET ?
        """,
        (agent_id, per_page, offset),
    ).fetchall()
    return ([_notification_to_dict(row) for row in rows], total)


def _mark_notification_rows_read(db, agent_id: int, notification_ids=None, mark_all: bool = False) -> int:
    if mark_all:
        cur = db.execute(
            "UPDATE notifications SET is_read = 1 WHERE agent_id = ? AND is_read = 0",
            (agent_id,),
        )
        return int(cur.rowcount or 0)

    ids = []
    for raw in notification_ids or []:
        try:
            ids.append(int(raw))
        except (TypeError, ValueError):
            continue

    if not ids:
        return 0

    placeholders = ",".join("?" for _ in ids)
    cur = db.execute(
        f"UPDATE notifications SET is_read = 1 WHERE agent_id = ? AND id IN ({placeholders})",
        [agent_id] + ids,
    )
    return int(cur.rowcount or 0)


def _canonical_webhook_event(event: str) -> str:
    mapping = {
        "new_video": "video.uploaded",
        "like": "video.voted",
        "comment": "comment.created",
    }
    return mapping.get(event, event)


def fire_webhooks(agent_id: int, event: str, payload: dict):
    """Send webhook POST to all active hooks for this agent/event. Non-blocking.

    Features:
    - HMAC signature header
    - event filtering
    - retry with exponential backoff (3 attempts)
    - rate limiting (max 100 events/hour per webhook)
    """

    canonical_event = _canonical_webhook_event(event)

    def _deliver():
        conn = sqlite3.connect(str(DB_PATH))
        conn.row_factory = sqlite3.Row
        hooks = conn.execute(
            "SELECT id, url, secret, events, event_window_start, event_count FROM webhooks WHERE agent_id = ? AND active = 1",
            (agent_id,),
        ).fetchall()

        now = time.time()
        iso_ts = datetime.datetime.utcfromtimestamp(now).isoformat() + "Z"

        envelope = {
            "event": canonical_event,
            "timestamp": iso_ts,
            "data": payload,
        }

        for hook in hooks:
            events = (hook["events"] or "*")
            allowed = {e.strip() for e in events.split(",") if e.strip()}
            if "*" not in allowed and canonical_event not in allowed and event not in allowed:
                continue

            # rate limit window (100 events/hour per webhook)
            window_start = float(hook["event_window_start"] or 0)
            event_count = int(hook["event_count"] or 0)
            if now - window_start >= 3600:
                window_start = now
                event_count = 0
            if event_count >= 100:
                continue

            body = json.dumps(envelope, separators=(",", ":")).encode()
            sig = hmac.new(hook["secret"].encode(), body, hashlib.sha256).hexdigest()

            ok = False
            for attempt in range(3):
                req = urllib.request.Request(
                    hook["url"],
                    data=body,
                    headers={
                        "Content-Type": "application/json",
                        "X-BoTTube-Event": canonical_event,
                        "X-BoTTube-Signature": f"sha256={sig}",
                        "User-Agent": "BoTTube-Webhook/1.0",
                    },
                    method="POST",
                )
                try:
                    with urllib.request.urlopen(req, timeout=10) as resp:
                        if 200 <= getattr(resp, "status", 200) < 300:
                            ok = True
                            break
                except Exception:
                    if attempt < 2:
                        time.sleep(2 ** attempt)

            if ok:
                conn.execute(
                    """UPDATE webhooks
                       SET last_triggered = ?, fail_count = 0,
                           event_window_start = ?, event_count = ?
                       WHERE id = ?""",
                    (now, window_start, event_count + 1, hook["id"]),
                )
            else:
                conn.execute(
                    "UPDATE webhooks SET fail_count = fail_count + 1 WHERE id = ?",
                    (hook["id"],),
                )
                conn.execute(
                    "UPDATE webhooks SET active = 0 WHERE id = ? AND fail_count >= 10",
                    (hook["id"],),
                )
            conn.commit()

        conn.close()

    threading.Thread(target=_deliver, daemon=True).start()


def send_verification_email(email: str, token: str, username: str) -> bool:
    """Send a verification email with a 64-char hex token link. Returns True on success."""
    if not SMTP_HOST:
        app.logger.warning("SMTP not configured - verification email not sent")
        return False

    verify_url = f"https://bottube.ai/verify-email/{token}"
    subject = "Verify your BoTTube email"
    html_body = f"""<div style="font-family:sans-serif;max-width:500px;margin:0 auto;background:#1a1a1a;color:#f1f1f1;padding:32px;border-radius:8px;">
<h2 style="color:#3ea6ff;">BoTTube Email Verification</h2>
<p>Hey <strong>{username}</strong>,</p>
<p>Click below to verify your email and unlock giveaway eligibility:</p>
<p style="text-align:center;margin:24px 0;">
<a href="{verify_url}" style="background:#3ea6ff;color:#0f0f0f;padding:12px 32px;border-radius:8px;text-decoration:none;font-weight:700;display:inline-block;">Verify Email</a>
</p>
<p style="font-size:12px;color:#717171;">This link expires in 24 hours. If you didn't sign up for BoTTube, ignore this email.</p>
</div>"""
    text_body = f"Hey {username},\n\nVerify your BoTTube email: {verify_url}\n\nExpires in 24 hours."

    msg = MIMEMultipart("alternative")
    msg["Subject"] = subject
    msg["From"] = SMTP_FROM
    msg["To"] = email
    msg.attach(MIMEText(text_body, "plain"))
    msg.attach(MIMEText(html_body, "html"))

    try:
        with smtplib.SMTP(SMTP_HOST, SMTP_PORT, timeout=10) as server:
            server.ehlo()
            if SMTP_PORT != 25:
                server.starttls()
            if SMTP_USER:
                server.login(SMTP_USER, SMTP_PASS)
            server.sendmail(SMTP_FROM, [email], msg.as_string())
        return True
    except Exception as e:
        app.logger.error(f"SMTP send failed: {e}")
        return False


def send_notification_email(to_email, subject, body_text, body_html, unsub_url):
    """Send a notification email with CAN-SPAM compliant unsubscribe link."""
    if not SMTP_HOST:
        return False
    msg = MIMEMultipart("alternative")
    msg["Subject"] = subject
    msg["From"] = SMTP_FROM
    msg["To"] = to_email
    msg["List-Unsubscribe"] = f"<{unsub_url}>"
    msg["List-Unsubscribe-Post"] = "List-Unsubscribe=One-Click"
    msg.attach(MIMEText(body_text, "plain"))
    msg.attach(MIMEText(body_html, "html"))
    try:
        with smtplib.SMTP(SMTP_HOST, SMTP_PORT, timeout=10) as server:
            server.ehlo()
            if SMTP_PORT != 25:
                server.starttls()
            if SMTP_USER:
                server.login(SMTP_USER, SMTP_PASS)
            server.sendmail(SMTP_FROM, [to_email], msg.as_string())
        return True
    except Exception as e:
        print(f"[email] SMTP send failed to {to_email}: {e}")
        return False


def require_api_key(f):
    """Decorator to require a valid agent API key."""
    @wraps(f)
    def decorated(*args, **kwargs):
        api_key = request.headers.get("X-API-Key", "")
        if not api_key:
            return jsonify({"error": "Missing X-API-Key header"}), 401
        db = get_db()
        agent = db.execute(
            "SELECT * FROM agents WHERE api_key = ?", (api_key,)
        ).fetchone()
        if not agent:
            return jsonify({"error": "Invalid API key"}), 401
        # Check ban status
        try:
            if agent["is_banned"]:
                return jsonify({
                    "error": "Account banned",
                    "reason": agent["ban_reason"] or "",
                }), 403
        except (IndexError, KeyError):
            pass  # Column may not exist yet
        # Update last_active
        db.execute(
            "UPDATE agents SET last_active = ? WHERE id = ?",
            (time.time(), agent["id"]),
        )
        db.commit()
        g.agent = agent
        return f(*args, **kwargs)
    return decorated


def video_to_dict(row):
    """Convert a video DB row to a JSON-friendly dict."""
    d = dict(row)
    d["tags"] = json.loads(d.get("tags", "[]"))
    d["url"] = f"/api/videos/{d['video_id']}/stream"
    d["watch_url"] = f"/watch/{d['video_id']}"
    d["thumbnail_url"] = f"/thumbnails/{d['thumbnail']}" if d.get("thumbnail") else ""
    cat_id = d.get("category", "other")
    cat_info = CATEGORY_MAP.get(cat_id, CATEGORY_MAP["other"])
    d["category"] = cat_id
    d["category_name"] = cat_info["name"]
    d["category_icon"] = cat_info["icon"]
    return d


def _public_video_filter_sql() -> str:
    """SQL predicate for public video surfaces."""
    return "COALESCE(v.is_removed, 0) = 0 AND COALESCE(a.is_banned, 0) = 0"


def agent_to_dict(row, include_private=False, *, badges=None):
    """Convert agent row to public-safe dict (allowlist only).

    Private fields (wallet addresses, balances) only included when
    the requesting user is viewing their own profile.
    """
    SAFE_FIELDS = {
        "id", "agent_name", "display_name", "bio", "avatar_url", "banner_url", "accent_color", "pinned_video_id",
        "is_human", "x_handle", "created_at",
    }
    PRIVATE_FIELDS = {
        "rtc_address", "btc_address", "eth_address", "sol_address",
        "ltc_address", "erg_address", "paypal_email", "rtc_balance",
    }
    fields = SAFE_FIELDS | PRIVATE_FIELDS if include_private else SAFE_FIELDS
    payload = {k: row[k] for k in fields if k in row.keys()}
    if badges is not None:
        payload["badges"] = badges
    return payload


def get_video_metadata(filepath):
    """Try to get video duration/dimensions via ffprobe."""
    try:
        result = subprocess.run(
            [
                "ffprobe", "-v", "quiet",
                "-print_format", "json",
                "-show_format", "-show_streams",
                str(filepath),
            ],
            capture_output=True, text=True, timeout=30,
        )
        data = json.loads(result.stdout)
        duration = float(data.get("format", {}).get("duration", 0))
        width = height = 0
        for stream in data.get("streams", []):
            if stream.get("codec_type") == "video":
                width = int(stream.get("width", 0))
                height = int(stream.get("height", 0))
                break
        return duration, width, height
    except Exception:
        return 0, 0, 0


def generate_thumbnail(video_path, thumb_path):
    """Generate a thumbnail from the video midpoint using ffmpeg.

    Extracts from ~40% into the video to avoid dark intro frames.
    Falls back to 1s if duration detection fails.
    """
    try:
        # Get video duration to extract from midpoint
        probe = subprocess.run(
            ["ffprobe", "-v", "quiet", "-show_entries", "format=duration",
             "-of", "csv=p=0", str(video_path)],
            capture_output=True, text=True, timeout=10,
        )
        try:
            duration = float(probe.stdout.strip())
            seek_time = max(1, duration * 0.4)  # 40% into the video
        except (ValueError, TypeError):
            seek_time = 3  # fallback to 3 seconds

        subprocess.run(
            [
                "ffmpeg", "-y", "-ss", str(seek_time),
                "-i", str(video_path),
                "-vframes", "1",
                "-vf", "scale=320:180:force_original_aspect_ratio=decrease,pad=320:180:(ow-iw)/2:(oh-ih)/2",
                "-q:v", "2",
                str(thumb_path),
            ],
            capture_output=True, timeout=30,
        )
        return thumb_path.exists()
    except Exception:
        return False


def optimize_thumbnail_image(src_path: Path, dst_path: Path) -> bool:
    """Normalize a user-supplied thumbnail into a small 320x180 JPEG.

    This reduces load time and helps prevent agents from uploading huge thumbnails.
    """
    try:
        src_path = Path(src_path)
        dst_path = Path(dst_path)
        if not src_path.exists():
            return False

        tmp_out = dst_path.with_name(dst_path.stem + ".tmp.jpg")
        tmp_out.unlink(missing_ok=True)

        subprocess.run(
            [
                "ffmpeg",
                "-y",
                "-i",
                str(src_path),
                "-vf",
                "scale=320:180:force_original_aspect_ratio=decrease,pad=320:180:(ow-iw)/2:(oh-ih)/2",
                "-frames:v",
                "1",
                "-q:v",
                "5",
                "-map_metadata",
                "-1",
                str(tmp_out),
            ],
            capture_output=True,
            timeout=30,
        )
        if not tmp_out.exists():
            return False
        tmp_out.replace(dst_path)
        return dst_path.exists()
    except Exception:
        return False


def transcode_video(input_path, output_path, max_w=MAX_VIDEO_WIDTH, max_h=MAX_VIDEO_HEIGHT,
                     keep_audio=True, target_file_mb=1.0, duration_hint=8):
    """Transcode video to H.264 High profile, constrained to max dimensions.

    Always includes an audio track for browser compatibility.
    If source has audio, it is preserved. If not, a silent track is added
    so the browser player shows working volume controls.
    """
    try:
        scale_filter = (
            f"scale='min({max_w},iw)':'min({max_h},ih)'"
            f":force_original_aspect_ratio=decrease"
            f",pad={max_w}:{max_h}:(ow-iw)/2:(oh-ih)/2:color=black"
        )

        # Check if source has an audio stream
        probe = subprocess.run(
            ["ffprobe", "-v", "quiet", "-show_streams", str(input_path)],
            capture_output=True, text=True, timeout=30
        )
        has_source_audio = "codec_type=audio" in probe.stdout

        # Budget video bitrate
        audio_kbps = 96 if has_source_audio else 32
        total_budget_kbits = target_file_mb * 1024 * 8  # MB -> kbits
        video_kbps = max(100, int(total_budget_kbits / max(duration_hint, 1) - audio_kbps))
        video_maxrate = f"{video_kbps}k"
        video_bufsize = f"{video_kbps * 2}k"

        if has_source_audio:
            # Source has audio - encode it
            cmd = [
                "ffmpeg", "-y", "-i", str(input_path),
                "-vf", scale_filter,
                "-c:v", "libx264", "-profile:v", "high",
                "-crf", "28", "-preset", "medium",
                "-maxrate", video_maxrate, "-bufsize", video_bufsize,
                "-pix_fmt", "yuv420p",
                "-c:a", "aac", "-b:a", f"{audio_kbps}k", "-ac", "2",
                "-movflags", "+faststart",
                str(output_path),
            ]
        else:
            # No source audio - add silent audio track for browser compatibility
            cmd = [
                "ffmpeg", "-y",
                "-i", str(input_path),
                "-f", "lavfi", "-i", "anullsrc=r=44100:cl=stereo",
                "-vf", scale_filter,
                "-c:v", "libx264", "-profile:v", "high",
                "-crf", "28", "-preset", "medium",
                "-maxrate", video_maxrate, "-bufsize", video_bufsize,
                "-pix_fmt", "yuv420p",
                "-c:a", "aac", "-b:a", "32k", "-ac", "2",
                "-shortest",
                "-movflags", "+faststart",
                str(output_path),
            ]

        result = subprocess.run(cmd, capture_output=True, text=True, timeout=600)
        return result.returncode == 0
    except Exception as e:
        app.logger.error(f"Transcode failed: {e}")
        return False


def format_duration(secs):
    """Format seconds as HH:MM:SS or MM:SS."""
    secs = int(secs)
    if secs < 3600:
        return f"{secs // 60}:{secs % 60:02d}"
    return f"{secs // 3600}:{(secs % 3600) // 60:02d}:{secs % 60:02d}"


def format_views(n):
    """Format view count for display."""
    if n >= 1_000_000:
        return f"{n / 1_000_000:.1f}M"
    if n >= 1_000:
        return f"{n / 1_000:.1f}K"
    return str(n)


def time_ago(ts):
    """Return human-readable time ago string."""
    diff = time.time() - ts
    if diff < 60:
        return "just now"
    if diff < 3600:
        m = int(diff // 60)
        return f"{m} minute{'s' if m != 1 else ''} ago"
    if diff < 86400:
        h = int(diff // 3600)
        return f"{h} hour{'s' if h != 1 else ''} ago"
    if diff < 2592000:
        d = int(diff // 86400)
        return f"{d} day{'s' if d != 1 else ''} ago"
    if diff < 31536000:
        mo = int(diff // 2592000)
        return f"{mo} month{'s' if mo != 1 else ''} ago"
    y = int(diff // 31536000)
    return f"{y} year{'s' if y != 1 else ''} ago"


# Register Jinja filters
def parse_tags(tags_str):
    """Parse a JSON tags string into a list."""
    try:
        tags = json.loads(tags_str) if isinstance(tags_str, str) else tags_str
        return [t for t in tags if t] if isinstance(tags, list) else []
    except (json.JSONDecodeError, TypeError):
        return []


def datetime_iso(ts):
    """Convert unix timestamp to ISO 8601 date string for structured data."""
    try:
        return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(float(ts)))
    except (ValueError, TypeError):
        return ""


def timestamp_date(ts):
    """Convert unix timestamp to a short date string."""
    try:
        return time.strftime("%b %d, %Y", time.gmtime(float(ts)))
    except (ValueError, TypeError):
        return ""


_MENTION_RE = re.compile(r"@([\w-]+)")


def _extract_mentions(content: str, db) -> list:
    """Find @agent-name mentions in comment text and return list of valid agent rows."""
    names = set(_MENTION_RE.findall(content))
    if not names:
        return []
    placeholders = ",".join("?" for _ in names)
    rows = db.execute(
        f"SELECT id, agent_name FROM agents WHERE agent_name IN ({placeholders})",
        list(names),
    ).fetchall()
    return rows


def render_mentions(text):
    """Jinja2 filter: convert @agent-name into clickable links."""
    prefix = app.config.get("APPLICATION_ROOT", "").rstrip("/")
    safe = str(escape(text))
    safe = _MENTION_RE.sub(
        lambda m: f'<a href="{prefix}/agent/{m.group(1)}" class="mention">@{m.group(1)}</a>',
        safe,
    )
    return Markup(safe)


app.jinja_env.filters["format_duration"] = format_duration
app.jinja_env.filters["format_views"] = format_views
app.jinja_env.filters["time_ago"] = time_ago

def minimal_markdown(text):
    if not text:
        return ""
    import html, re
    t = html.escape(str(text))
    t = re.sub(r'\[([^\]]+)\]\((https?://[^\)]+)\)', r'<a href="\2" target="_blank" rel="nofollow">\1</a>', t)
    t = re.sub(r'\*\*([^\*]+)\*\*', r'<strong>\1</strong>', t)
    t = re.sub(r'\*([^\*]+)\*', r'<em>\1</em>', t)
    t = re.sub(r'```([^`]+)```', r'<pre><code>\1</code></pre>', t, flags=re.DOTALL)
    t = re.sub(r'`([^`]+)`', r'<code>\1</code>', t)
    t = t.replace('\n', '<br>')
    return Markup(t)

app.jinja_env.filters["minimal_markdown"] = minimal_markdown
app.jinja_env.filters["parse_tags"] = parse_tags
app.jinja_env.filters["datetime_iso"] = datetime_iso
app.jinja_env.filters["timestamp_date"] = timestamp_date
app.jinja_env.filters["render_mentions"] = render_mentions

_URL_RE = re.compile(r'(https?://[^\s<>\]\)\"]+)')

# Timestamps like 1:23:45 (H:MM:SS), 12:34 (M:SS), or 0:05
_TIMESTAMP_RE = re.compile(r'(?<!\w)(\d{1,2}):(\d{2})(?::(\d{2}))?(?!\w)')

def render_urls(text):
    """Jinja2 filter: convert @mentions and bare URLs into clickable links. Drudge-style."""
    prefix = app.config.get("APPLICATION_ROOT", "").rstrip("/")
    safe = str(escape(text))
    # First apply mentions
    safe = _MENTION_RE.sub(
        lambda m: f'<a href="{prefix}/agent/{m.group(1)}" class="mention">@{m.group(1)}</a>',
        safe,
    )
    # Then linkify URLs
    safe = _URL_RE.sub(
        lambda m: f'<a href="{m.group(1)}" target="_blank" rel="noopener" class="desc-link">{m.group(1)}</a>',
        safe,
    )

    # Auto-link timestamps (1:23:45, 12:34, 0:05) to video seek positions
    def _timestamp_link(m):
        h, m_part, s_part = m.group(1), m.group(2), m.group(3)
        if s_part is not None:
            # H:MM:SS format
            seconds = int(h) * 3600 + int(m_part) * 60 + int(s_part)
        else:
            # M:SS format
            seconds = int(h) * 60 + int(m_part)
        return f'<a href="?t={seconds}" class="timestamp-link" onclick="seekTo({seconds}); return false;">{m.group(0)}</a>'

    safe = _TIMESTAMP_RE.sub(_timestamp_link, safe)

    return Markup(safe)

app.jinja_env.filters["render_urls"] = render_urls


# ---------------------------------------------------------------------------
# Health / utility endpoints
# ---------------------------------------------------------------------------

@app.route("/og-banner.png")
def og_banner():
    """Generate an OG banner image as SVG rendered to PNG-like format.

    Used by social media crawlers for link previews.
    Returns an SVG with proper content type that most crawlers accept.
    """
    svg = """<svg xmlns="http://www.w3.org/2000/svg" width="1200" height="630" viewBox="0 0 1200 630">
  <defs>
    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
      <stop offset="0%" style="stop-color:#0f0f0f"/>
      <stop offset="50%" style="stop-color:#1a1a2e"/>
      <stop offset="100%" style="stop-color:#0f3460"/>
    </linearGradient>
  </defs>
  <rect width="1200" height="630" fill="url(#bg)"/>
  <text x="600" y="240" text-anchor="middle" fill="#f1f1f1" font-family="system-ui,sans-serif" font-size="72" font-weight="700">
    <tspan fill="#3ea6ff">Bo</tspan><tspan fill="#ff4444">T</tspan><tspan fill="#3ea6ff">Tube</tspan>
  </text>
  <text x="600" y="320" text-anchor="middle" fill="#aaaaaa" font-family="system-ui,sans-serif" font-size="28">
    Where AI Agents Come Alive
  </text>
  <text x="600" y="400" text-anchor="middle" fill="#717171" font-family="system-ui,sans-serif" font-size="20">
    The first video platform built for bots and humans
  </text>
  <text x="600" y="540" text-anchor="middle" fill="#3ea6ff" font-family="system-ui,sans-serif" font-size="22">
    bottube.ai
  </text>
</svg>"""
    return Response(svg, mimetype="image/svg+xml", headers={
        "Cache-Control": "public, max-age=86400",
    })


@app.route("/health")
def health():
    """Health check endpoint."""
    try:
        db = get_db()
        db.execute("SELECT 1").fetchone()
        db_ok = True
    except Exception:
        db_ok = False

    video_count = 0
    agent_count = 0
    human_count = 0
    if db_ok:
        video_count = db.execute("SELECT COUNT(*) FROM videos").fetchone()[0]
        agent_count = db.execute("SELECT COUNT(*) FROM agents WHERE is_human = 0").fetchone()[0]
        human_count = db.execute("SELECT COUNT(*) FROM agents WHERE is_human = 1").fetchone()[0]

    return jsonify({
        "ok": db_ok,
        "service": "bottube",
        "version": APP_VERSION,
        "uptime_s": round(time.time() - APP_START_TS),
        "videos": video_count,
        "agents": agent_count,
        "humans": human_count,
    })


# ---------------------------------------------------------------------------
# Agent registration
# ---------------------------------------------------------------------------


# ---------------------------------------------------------------------------
# OpenAPI + Swagger UI (crawler/LLM-friendly API surface)
# ---------------------------------------------------------------------------

# NOTE: /api/openapi.json is now served by agent_discovery blueprint
# (was: from bottube.openapi import build_openapi_spec — module doesn't exist)


@app.route("/api/docs")
def api_docs_swagger_ui():
    # Self-hosted Swagger UI assets (no CDN dependency).
    return render_template("api_swagger.html")


def _register_text_field(data, field, default=""):
    value = data.get(field, default)
    if value is None:
        value = default
    if not isinstance(value, str):
        return None, f"{field} must be a string"
    return value.strip(), None


def _json_object_body():
    data = request.get_json(silent=True)
    if data is None:
        return {}, None
    if not isinstance(data, dict):
        return None, (jsonify({"error": "JSON body must be an object"}), 400)
    return data, None


@app.route("/api/register", methods=["POST"])
def register_agent():
    """Register a new agent and return API key."""
    # Rate limit: 5 registrations per IP per hour
    ip = _get_client_ip()
    if not _rate_limit(f"register:{ip}", 5, 3600):
        return jsonify({"error": "Too many registrations. Try again later."}), 429

    data = request.get_json(silent=True)
    if data is None:
        data = {}
    elif not isinstance(data, dict):
        return jsonify({"error": "JSON body must be an object"}), 400

    agent_name, error = _register_text_field(data, "agent_name")
    if error:
        return jsonify({"error": error}), 400
    agent_name = agent_name.lower()

    ref_code_raw, error = _register_text_field(data, "ref_code")
    if error:
        return jsonify({"error": error}), 400
    ref_raw, error = _register_text_field(data, "ref")
    if error:
        return jsonify({"error": error}), 400
    ref_code = _normalize_ref_code(
        ref_code_raw or ref_raw or request.args.get("ref", "")
    )

    if not agent_name:
        return jsonify({"error": "agent_name is required"}), 400
    if not re.match(r"^[a-z0-9_-]{2,32}$", agent_name):
        return jsonify({
            "error": "agent_name must be 2-32 chars, lowercase alphanumeric, hyphens, underscores"
        }), 400
    if ref_code:
        ref = _referral_get_code_row(get_db(), ref_code)
        if not ref:
            return jsonify({"error": "Referral code not found"}), 400
        if not _referral_track_allowed(ref["allowed_track"], "agent"):
            return jsonify({
                "error": "Referral code is not enabled for agent onboarding",
                "allowed_track": _normalize_referral_track(ref["allowed_track"], "both"),
            }), 400

    display_name, error = _register_text_field(
        data, "display_name", default=agent_name
    )
    if error:
        return jsonify({"error": error}), 400
    bio, error = _register_text_field(data, "bio")
    if error:
        return jsonify({"error": error}), 400
    avatar_url, error = _register_text_field(data, "avatar_url")
    if error:
        return jsonify({"error": error}), 400
    x_handle, error = _register_text_field(data, "x_handle")
    if error:
        return jsonify({"error": error}), 400

    display_name = _strip_script_tags(display_name[:MAX_DISPLAY_NAME_LENGTH])
    bio = _strip_script_tags(bio[:MAX_BIO_LENGTH])
    x_handle = x_handle.lstrip("@")[:32]

    # Validate avatar_url if provided
    if avatar_url:
        from urllib.parse import urlparse
        parsed = urlparse(avatar_url)
        if parsed.scheme not in ("http", "https") or not parsed.netloc:
            return jsonify({"error": "avatar_url must be a valid http/https URL"}), 400
        avatar_url = avatar_url[:512]  # cap length
    api_key = gen_api_key()
    claim_token = secrets.token_hex(16)

    db = get_db()
    try:
        cur = db.execute(
            """INSERT INTO agents
               (agent_name, display_name, api_key, bio, avatar_url, x_handle,
                claim_token, claimed, is_human, detected_type, created_at, last_active)
               VALUES (?, ?, ?, ?, ?, ?, ?, 0, 0, 'ai_agent', ?, ?)""",
            (agent_name, display_name, api_key, bio, avatar_url, x_handle,
             claim_token, time.time(), time.time()),
        )
        new_agent_id = int(cur.lastrowid)
        if ref_code:
            _referral_apply_signup(db, new_agent_id, ref_code, source="agent_api_register")
        _refresh_agent_quests(db, new_agent_id, ["profile_complete"])
        db.commit()
    except sqlite3.IntegrityError:
        return jsonify({"error": f"Agent '{agent_name}' already exists"}), 409

    # Build claim URL - agent posts this on X to verify identity
    claim_url = f"https://bottube.ai/claim/{agent_name}/{claim_token}"

    return jsonify({
        "ok": True,
        "agent_name": agent_name,
        "api_key": api_key,
        "claim_url": claim_url,
        "claim_instructions": (
            "To verify your identity, post this claim URL on X/Twitter. "
            "Then call POST /api/claim/verify with your X handle."
        ),
        "message": "Store your API key securely - it cannot be recovered.",
        # Trust+Safety: explicit TOS notice. Agents must POST acknowledgment
        # to /api/agents/me/accept-terms before performing any write action.
        "terms": {
            "version": TOS_VERSION,
            "effective": TOS_EFFECTIVE,
            "terms_url": "https://bottube.ai/terms",
            "aup_url": "https://bottube.ai/aup",
            "dmca_url": "https://bottube.ai/dmca",
            "report_url": "https://bottube.ai/report",
            "acceptance_required": True,
            "accept_endpoint": "/api/agents/me/accept-terms",
            "csam_notice": (
                "Zero tolerance for CSAM. Uploads are hash-checked and reported "
                "to NCMEC and law enforcement under 18 U.S.C. § 2258A."
            ),
            "agent_responsibility": (
                "By using your API key you acknowledge that the human operator "
                "of this agent is responsible for everything it does. To accept "
                "the Terms, POST {\"version\":\"" + TOS_VERSION + "\"} to /api/agents/me/accept-terms."
            ),
        },
    }), 201


@app.route("/api/claim/verify", methods=["POST"])
@require_api_key
def verify_claim():
    """Verify an agent's X/Twitter identity by checking if they posted the claim URL.

    The agent posts their claim URL on X, then calls this endpoint with their
    X handle. The server (or a bridge bot) checks if the URL was posted.
    For now, manual/admin verification is supported.
    """
    data = request.get_json(silent=True)
    if data is None:
        data = {}
    elif not isinstance(data, dict):
        return jsonify({"error": "JSON body must be an object"}), 400

    raw_x_handle = data.get("x_handle", "")
    if raw_x_handle is None:
        raw_x_handle = ""
    if not isinstance(raw_x_handle, str):
        return jsonify({"error": "x_handle must be a string"}), 400
    x_handle = raw_x_handle.strip().lstrip("@")

    if not x_handle:
        return jsonify({"error": "x_handle is required"}), 400

    db = get_db()
    db.execute(
        "UPDATE agents SET x_handle = ?, claimed = 1 WHERE id = ?",
        (x_handle, g.agent["id"]),
    )
    db.commit()

    return jsonify({
        "ok": True,
        "agent_name": g.agent["agent_name"],
        "x_handle": x_handle,
        "claimed": True,
        "message": f"Agent linked to @{x_handle} on X.",
    })


@app.route("/claim/<agent_name>/<token>")
def claim_page(agent_name, token):
    """Claim verification landing page."""
    ip = _get_client_ip()
    if not _rate_limit(f"claim:{ip}", 10, 300):
        abort(429)
    db = get_db()
    agent = db.execute(
        "SELECT * FROM agents WHERE agent_name = ? AND claim_token = ?",
        (agent_name, token),
    ).fetchone()

    if not agent:
        abort(404)

    return jsonify({
        "ok": True,
        "agent_name": agent_name,
        "verified": bool(agent["claimed"]),
        "message": f"This is the BoTTube claim page for @{agent_name}.",
    })


@app.route("/reclaim")
def reclaim_account_page():
    notice = None
    try:
        notice = _build_recovery_notice(get_db())
    except Exception:
        notice = _build_recovery_notice(None)
    return render_template(
        "reclaim_account.html",
        recovery_notice=notice,
        recovery_stage=RECOVERY_STAGE_LABEL,
        support_email=os.environ.get("BOTTUBE_RECLAIM_EMAIL", "scott@elyanlabs.ai"),
    )


# ---------------------------------------------------------------------------
# Human authentication (browser login)
# ---------------------------------------------------------------------------

@app.route("/login", methods=["GET", "POST"])
def login():
    """Login page for human users."""
    if request.method == "GET":
        return render_template("login.html")

    _verify_csrf()

    # Rate limit: 10 login attempts per IP per 5 minutes
    ip = _get_client_ip()
    if not _rate_limit(f"login:{ip}", 10, 300):
        flash("Too many login attempts. Try again in a few minutes.", "error")
        return render_template("login.html"), 429

    username = request.form.get("username", "").strip().lower()
    password = request.form.get("password", "")

    if not username or not password:
        flash("Username and password are required.", "error")
        return render_template("login.html"), 400

    db = get_db()
    # Allow login by username OR email address
    user = db.execute(
        "SELECT * FROM agents WHERE agent_name = ? OR (email = ? AND email != '')",
        (username, username),
    ).fetchone()

    if not user or not user["password_hash"]:
        flash("Invalid username or password.", "error")
        return render_template("login.html"), 401

    if not check_password_hash(user["password_hash"], password):
        flash("Invalid username or password.", "error")
        return render_template("login.html"), 401

    # Regenerate session to prevent session fixation
    session.clear()
    session.permanent = True
    session["user_id"] = user["id"]
    session["csrf_token"] = secrets.token_hex(32)
    return redirect(url_for("index"))


@app.route("/signup", methods=["GET", "POST"])
def signup():
    """Signup page for human users."""
    if request.method == "GET":
        ref_code = _normalize_ref_code(request.args.get("ref", "") or session.get("ref_code", ""))
        if ref_code:
            session["ref_code"] = ref_code
        referral = None
        if ref_code:
            db = get_db()
            row = db.execute(
                """
                SELECT rc.code, a.agent_name, a.display_name
                FROM referral_codes rc
                JOIN agents a ON a.id = rc.agent_id
                WHERE rc.code = ?
                """,
                (ref_code,),
            ).fetchone()
            if row:
                referral = {
                    "code": row["code"],
                    "agent_name": row["agent_name"],
                    "display_name": row["display_name"] or row["agent_name"],
                }
        return render_template(
            "login.html",
            signup=True,
            form_ts=time.time(),
            referral=referral,
            referral_code_value=ref_code,
        )

    _verify_csrf()

    # --- Anti-bot: Honeypot check ---
    # Hidden field that humans can't see; bots auto-fill it.
    # Silently fake-accept so the bot thinks it succeeded.
    if request.form.get("website", ""):
        return redirect(url_for("index"))

    # --- Anti-bot: Timing check ---
    # Reject forms submitted faster than 3 seconds (instant bot fill).
    try:
        form_ts = float(request.form.get("form_ts", "0"))
        if form_ts > 0 and (time.time() - form_ts) < 3:
            return redirect(url_for("index"))  # silent reject
    except (ValueError, TypeError):
        pass

    # Rate limit: 3 signups per IP per hour
    ip = _get_client_ip()
    if not _rate_limit(f"signup:{ip}", 3, 3600):
        flash("Too many signups. Try again later.", "error")
        return render_template("login.html", signup=True, form_ts=time.time(), referral_code_value=""), 429

    username = request.form.get("username", "").strip().lower()
    display_name = _strip_script_tags(request.form.get("display_name", "").strip()[:MAX_DISPLAY_NAME_LENGTH])
    password = request.form.get("password", "")
    confirm = request.form.get("confirm_password", "")
    email = request.form.get("email", "").strip().lower()
    ref_code = _normalize_ref_code(request.form.get("ref_code", "") or session.get("ref_code", ""))

    if not username or not password:
        flash("Username and password are required.", "error")
        return render_template("login.html", signup=True, form_ts=time.time(), referral_code_value=ref_code), 400

    if not re.match(r"^[a-z0-9_-]{2,32}$", username):
        flash("Username must be 2-32 chars, lowercase, alphanumeric, hyphens, underscores.", "error")
        return render_template("login.html", signup=True, form_ts=time.time(), referral_code_value=ref_code), 400

    if len(password) < 8:
        flash("Password must be at least 8 characters.", "error")
        return render_template("login.html", signup=True, form_ts=time.time(), referral_code_value=ref_code), 400

    if password != confirm:
        flash("Passwords do not match.", "error")
        return render_template("login.html", signup=True, form_ts=time.time(), referral_code_value=ref_code), 400

    if ref_code:
        ref = _referral_get_code_row(get_db(), ref_code)
        if not ref:
            flash("Referral code not found.", "error")
            return render_template("login.html", signup=True, form_ts=time.time(), referral_code_value=ref_code), 400
        if not _referral_track_allowed(ref["allowed_track"], "human"):
            flash("Referral code is not enabled for human signups.", "error")
            return render_template("login.html", signup=True, form_ts=time.time(), referral_code_value=ref_code), 400

    # Basic email validation (optional field)
    email_token = ""
    if email:
        if not re.match(r"^[^@\s]+@[^@\s]+\.[^@\s]+$", email):
            flash("Invalid email address.", "error")
            return render_template("login.html", signup=True, form_ts=time.time(), referral_code_value=ref_code), 400
        email_token = secrets.token_hex(32)

    api_key = gen_api_key()
    claim_token = secrets.token_hex(16)
    now = time.time()

    db = get_db()
    try:
        cur = db.execute(
            """INSERT INTO agents
               (agent_name, display_name, api_key, password_hash, is_human, detected_type,
                bio, avatar_url, claim_token, claimed,
                email, email_verified, email_verify_token, email_verify_sent_at,
                created_at, last_active)
               VALUES (?, ?, ?, ?, 1, 'human', '', '', ?, 0,
                       ?, 0, ?, ?, ?, ?)""",
            (username, display_name or username, api_key,
             generate_password_hash(password),
             claim_token,
             email, email_token, now if email else 0,
             now, now),
        )
        new_user_id = int(cur.lastrowid)
        if ref_code:
            _referral_apply_signup(db, new_user_id, ref_code, source="human_signup_form")
        db.commit()
    except sqlite3.IntegrityError:
        flash(f"Username '{username}' is already taken.", "error")
        return render_template("login.html", signup=True, form_ts=time.time(), referral_code_value=ref_code), 409

    session.pop("ref_code", None)

    # Send verification email if provided
    if email and email_token:
        send_verification_email(email, email_token, username)

    # Auto-login after signup (clear first to prevent session fixation)
    user = db.execute(
        "SELECT id FROM agents WHERE agent_name = ?", (username,)
    ).fetchone()
    session.clear()
    session.permanent = True
    session["user_id"] = user["id"]

    return redirect(url_for("index"))


@app.route("/logout", methods=["GET", "POST"])
def logout():
    """Log out the current user. POST preferred; GET checks referrer."""
    if request.method == "GET":
        ref = request.headers.get("Referer", "")
        if not ref or not ref.startswith(request.url_root):
            return redirect(url_for("index"))
    session.clear()
    resp = redirect(url_for("index"))
    resp.delete_cookie("session", path="/", domain=None)
    return resp


# ---------------------------------------------------------------------------
# Referrals
# ---------------------------------------------------------------------------

@app.route("/r/<code>", methods=["GET"])
def referral_redirect(code):
    """Referral short-link: shareable landing page -> signup (records hit)."""
    ref_code = _normalize_ref_code(code)
    if not ref_code:
        abort(404)
    db = get_db()
    _referral_touch_hit_unique(db, ref_code)
    ref = db.execute(
        """
        SELECT rc.code, rc.agent_id, a.agent_name, a.display_name
        FROM referral_codes rc
        JOIN agents a ON a.id = rc.agent_id
        WHERE rc.code = ?
        """,
        (ref_code,),
    ).fetchone()
    if not ref:
        abort(404)
    signup_url = url_for("signup", ref=ref_code)
    return render_template(
        "referral_landing.html",
        code=ref["code"],
        ref_agent_name=ref["agent_name"],
        ref_display_name=ref["display_name"] or ref["agent_name"],
        signup_url=signup_url,
    )


def _referral_me_payload() -> Response:
    """Create/get referral code for the current authenticated account."""
    db = get_db()
    agent_id = int(g.agent["id"])
    data = {}
    if request.method == "POST":
        data = request.get_json(silent=True) or request.form.to_dict() or {}
    requested_track = _normalize_referral_track(data.get("allowed_track", data.get("track", "both")), "both")
    requested_code = _normalize_ref_code(data.get("code", ""))
    # Prefer an existing code for this agent.
    row = db.execute(
        """
        SELECT code, hits, signups, first_uploads, created_at, COALESCE(allowed_track, 'both') AS allowed_track
        FROM referral_codes
        WHERE agent_id = ?
        ORDER BY created_at ASC
        LIMIT 1
        """,
        (agent_id,),
    ).fetchone()
    if row:
        code = row["code"]
        if request.method == "POST":
            if requested_code and requested_code != row["code"]:
                return jsonify({"error": "Referral code already exists for this account"}), 409
            current_track = _normalize_referral_track(row["allowed_track"], "both")
            if requested_track != current_track:
                db.execute(
                    "UPDATE referral_codes SET allowed_track = ? WHERE code = ?",
                    (requested_track, row["code"]),
                )
                db.commit()
    else:
        # Default code: agent name (validated) or random token.
        base = requested_code or _normalize_ref_code(g.agent["agent_name"])
        code = base or (secrets.token_hex(4))
        # Ensure uniqueness.
        while db.execute("SELECT 1 FROM referral_codes WHERE code = ?", (code,)).fetchone():
            code = secrets.token_hex(4)
        db.execute(
            "INSERT INTO referral_codes (code, agent_id, created_at, allowed_track) VALUES (?, ?, ?, ?)",
            (code, agent_id, time.time(), requested_track),
        )
        db.commit()
        row = db.execute(
            """
            SELECT code, hits, signups, first_uploads, created_at, COALESCE(allowed_track, 'both') AS allowed_track
            FROM referral_codes
            WHERE code = ?
            """,
            (code,),
        ).fetchone()

    summary = _referral_build_summary(db, agent_id)
    return jsonify({
        "ok": True,
        "code": row["code"],
        "allowed_track": _normalize_referral_track(row["allowed_track"], "both"),
        "ref_url": f"https://bottube.ai/r/{row['code']}",
        "signup_url": f"https://bottube.ai/signup?ref={row['code']}",
        "stats": {
            "hits": int(row["hits"] or 0),
            "signups": int(row["signups"] or 0),
            "first_uploads": int(row["first_uploads"] or 0),
            "created_at": row["created_at"],
        },
        "summary": summary,
    })


@app.route("/api/agents/me/referral", methods=["GET", "POST"])
@require_api_key
def referral_me_agent():
    """Create/get referral code for the authenticated agent (API key)."""
    return _referral_me_payload()


@app.route("/api/users/me/referral", methods=["GET", "POST"])
def referral_me_user():
    """Web/session version of referral endpoint (for humans)."""
    if not g.user:
        return jsonify({"error": "Not logged in"}), 401
    # Reuse same logic as agent endpoint by binding g.agent temporarily.
    g.agent = g.user
    return _referral_me_payload()


def _referral_apply_payload(source: str):
    db = get_db()
    data = request.get_json(silent=True) or request.form.to_dict() or {}
    ref_code = _normalize_ref_code(data.get("ref_code", "") or data.get("ref", ""))
    if not ref_code:
        return jsonify({"error": "ref_code is required"}), 400
    result = _referral_apply_signup(db, int(g.agent["id"]), ref_code, source=source)
    status = 200 if result.get("applied") else 400
    return jsonify(result), status


@app.route("/api/agents/me/referral/apply", methods=["POST"])
@require_api_key
def referral_apply_agent():
    """Attach an invite code to the current account if it has not been referred yet."""
    return _referral_apply_payload("manual_apply_api")


@app.route("/api/users/me/referral/apply", methods=["POST"])
def referral_apply_user():
    """Session-auth version of referral apply for humans."""
    if not g.user:
        return jsonify({"error": "Not logged in"}), 401
    _verify_csrf()
    g.agent = g.user
    return _referral_apply_payload("manual_apply_web")


def _get_referral_leaderboard(db, limit: int = 50) -> list[dict]:
    limit = max(1, min(int(limit or 50), 200))
    rows = db.execute(
        """
        SELECT
            rc.code,
            rc.hits,
            rc.signups,
            rc.first_uploads,
            rc.created_at,
            a.agent_name,
            a.display_name
        FROM referral_codes rc
        JOIN agents a ON a.id = rc.agent_id
        WHERE COALESCE(a.is_banned, 0) = 0
        ORDER BY rc.first_uploads DESC, rc.signups DESC, rc.hits DESC, rc.created_at ASC
        LIMIT ?
        """,
        (limit,),
    ).fetchall()
    out = []
    for r in rows:
        out.append(
            {
                "code": r["code"],
                "agent_name": r["agent_name"],
                "display_name": r["display_name"] or r["agent_name"],
                "hits": int(r["hits"] or 0),
                "signups": int(r["signups"] or 0),
                "first_uploads": int(r["first_uploads"] or 0),
                "ref_url": f"https://bottube.ai/r/{r['code']}",
            }
        )
    return out


def _mask_public_handle(agent_name: str) -> str:
    handle = (agent_name or "").strip()
    if not handle:
        return "@unknown"
    if len(handle) <= 6:
        return f"@{handle}"
    return f"@{handle[:4]}...{handle[-2:]}"


def _bonus_progress_payload(current: int) -> list[dict]:
    current_i = max(0, int(current or 0))
    return [
        {
            "threshold": threshold,
            "current": current_i,
            "remaining": max(threshold - current_i, 0),
            "reached": current_i >= threshold,
        }
        for threshold in REFERRAL_BONUS_THRESHOLDS
    ]


def _filter_badges_by_keys(badges: list[dict], allowed_keys: set[str]) -> list[dict]:
    return [badge for badge in badges if badge["badge_key"] in allowed_keys]


def _get_founding_track_leaderboard(
    db: sqlite3.Connection,
    invitee_track: str,
    *,
    limit: int = 25,
) -> list[dict]:
    track = "human" if invitee_track == "human" else "agent"
    rows = db.execute(
        """
        SELECT
            rc.agent_id,
            a.agent_name,
            a.display_name,
            a.is_human,
            COUNT(ri.id) AS total_invites,
            SUM(
                CASE
                    WHEN COALESCE(ri.fully_activated_at, 0) > 0
                     AND COALESCE(ri.review_status, 'pending') NOT IN ('rejected', 'void')
                    THEN 1 ELSE 0
                END
            ) AS activated_referrals,
            SUM(CASE WHEN COALESCE(ri.review_status, 'pending') = 'pending' THEN 1 ELSE 0 END) AS pending_review,
            MIN(
                CASE
                    WHEN COALESCE(ri.fully_activated_at, 0) > 0
                     AND COALESCE(ri.review_status, 'pending') NOT IN ('rejected', 'void')
                    THEN ri.fully_activated_at
                    ELSE NULL
                END
            ) AS first_activated_at,
            COALESCE(MIN(rc.created_at), 0) AS code_created_at
        FROM referral_codes rc
        JOIN agents a ON a.id = rc.agent_id
        LEFT JOIN referral_invites ri
          ON ri.referrer_agent_id = rc.agent_id
         AND ri.invitee_track = ?
        WHERE COALESCE(a.is_banned, 0) = 0
        GROUP BY rc.agent_id, a.agent_name, a.display_name, a.is_human
        HAVING COUNT(ri.id) > 0
        ORDER BY activated_referrals DESC, total_invites DESC, first_activated_at ASC, code_created_at ASC
        LIMIT ?
        """,
        (track, max(1, min(int(limit or 25), 100))),
    ).fetchall()

    scout_key = "founding_scout_human" if track == "human" else "founding_scout_agent"
    out = []
    for idx, row in enumerate(rows, start=1):
        badges = _filter_badges_by_keys(_list_agent_badges(db, int(row["agent_id"])), {scout_key})
        activated_referrals = int(row["activated_referrals"] or 0)
        out.append(
            {
                "rank": idx,
                "track": track,
                "agent_id": int(row["agent_id"]),
                "agent_name": row["agent_name"],
                "display_name": row["display_name"] or row["agent_name"],
                "handle_hint": _mask_public_handle(row["agent_name"]),
                "profile_url": f"/agent/{row['agent_name']}",
                "is_human": bool(int(row["is_human"] or 0)),
                "activated_referrals": activated_referrals,
                "total_invites": int(row["total_invites"] or 0),
                "pending_review": int(row["pending_review"] or 0),
                "bonus_progress": _bonus_progress_payload(activated_referrals),
                "badges": badges,
            }
        )
    return out


def _get_founding_cohort(db: sqlite3.Connection, track: str) -> dict:
    invitee_track = "human" if track == "human" else "agent"
    rows = db.execute(
        """
        SELECT
            ri.id,
            ri.fully_activated_at,
            ri.referral_code,
            inv.id AS agent_id,
            inv.agent_name,
            inv.display_name,
            inv.is_human,
            ref.agent_name AS referrer_agent_name,
            ref.display_name AS referrer_display_name
        FROM referral_invites ri
        JOIN agents inv ON inv.id = ri.invitee_agent_id
        JOIN agents ref ON ref.id = ri.referrer_agent_id
        WHERE ri.invitee_track = ?
          AND COALESCE(ri.fully_activated_at, 0) > 0
          AND COALESCE(ri.review_status, 'pending') NOT IN ('rejected', 'void')
        ORDER BY ri.fully_activated_at ASC, ri.id ASC
        LIMIT ?
        """,
        (invitee_track, FOUNDING_BADGE_LIMIT),
    ).fetchall()

    early_keys = (
        {"early_human_bottube", "early_human_rustchain"}
        if invitee_track == "human"
        else {"early_agent_bottube", "early_agent_rustchain"}
    )
    pair_key = "founding_human_pair" if invitee_track == "human" else "founding_agent_pair"
    entries = []
    awarded_slots = 0
    pair_badges_awarded = 0
    for idx, row in enumerate(rows, start=1):
        badges = _filter_badges_by_keys(_list_agent_badges(db, int(row["agent_id"])), early_keys | {pair_key})
        awarded_badges = [badge for badge in badges if badge["badge_key"] in early_keys]
        pair_badges = [badge for badge in badges if badge["badge_key"] == pair_key]
        if awarded_badges:
            awarded_slots += 1
        if pair_badges:
            pair_badges_awarded += 1
        entries.append(
            {
                "rank": idx,
                "track": invitee_track,
                "agent_id": int(row["agent_id"]),
                "agent_name": row["agent_name"],
                "display_name": row["display_name"] or row["agent_name"],
                "handle_hint": _mask_public_handle(row["agent_name"]),
                "profile_url": f"/agent/{row['agent_name']}",
                "is_human": bool(int(row["is_human"] or 0)),
                "activated_at": float(row["fully_activated_at"] or 0),
                "referral_code": row["referral_code"],
                "referrer_agent_name": row["referrer_agent_name"],
                "referrer_display_name": row["referrer_display_name"] or row["referrer_agent_name"],
                "badges": badges,
                "badge_status": "awarded" if awarded_badges else "pending",
                "pair_reserved": True,
            }
        )

    filled_slots = len(entries)
    return {
        "track": invitee_track,
        "slots_total": FOUNDING_BADGE_LIMIT,
        "filled_slots": filled_slots,
        "remaining_slots": max(FOUNDING_BADGE_LIMIT - filled_slots, 0),
        "awarded_slots": awarded_slots,
        "pair_badges_awarded": pair_badges_awarded,
        "entries": entries,
    }


def _get_founding_leaderboard_data(db: sqlite3.Connection) -> dict:
    human_referrers = _get_founding_track_leaderboard(db, "human", limit=25)
    agent_sponsors = _get_founding_track_leaderboard(db, "agent", limit=25)
    human_cohort = _get_founding_cohort(db, "human")
    agent_cohort = _get_founding_cohort(db, "agent")
    return {
        "human_referrers": human_referrers,
        "agent_sponsors": agent_sponsors,
        "human_cohort": human_cohort,
        "agent_cohort": agent_cohort,
        "pair_reservations": {
            "human": {
                "label": "Founding Human Pair",
                "claimed": human_cohort["filled_slots"],
                "remaining": human_cohort["remaining_slots"],
                "total": FOUNDING_BADGE_LIMIT,
                "awarded_badges": human_cohort["pair_badges_awarded"],
            },
            "agent": {
                "label": "Founding Agent Pair",
                "claimed": agent_cohort["filled_slots"],
                "remaining": agent_cohort["remaining_slots"],
                "total": FOUNDING_BADGE_LIMIT,
                "awarded_badges": agent_cohort["pair_badges_awarded"],
            },
        },
        "updated_at": time.time(),
    }


@app.route("/referrals")
def referrals_page():
    """Public referral program page + leaderboard."""
    db = get_db()
    leaderboard = _get_referral_leaderboard(db, limit=50)
    return render_template("referrals.html", leaderboard=leaderboard)


@app.route("/founding")
def founding_page():
    """Public founding leaderboard for human/agent funnels."""
    db = get_db()
    data = _get_founding_leaderboard_data(db)
    return render_template("founding.html", **data)


@app.route("/api/referrals/leaderboard")
def referrals_leaderboard_api():
    db = get_db()
    limit = request.args.get("limit", "50")
    try:
        limit_i = int(limit)
    except Exception:
        limit_i = 50
    return jsonify({"ok": True, "leaderboard": _get_referral_leaderboard(db, limit=limit_i)})


@app.route("/api/founding/leaderboard")
def founding_leaderboard_api():
    db = get_db()
    return jsonify({"ok": True, **_get_founding_leaderboard_data(db)})


def _referral_admin_notes(db: sqlite3.Connection, row: sqlite3.Row) -> list[str]:
    notes: list[str] = []
    signup_ip_hash = (row["signup_ip_hash"] or "").strip()
    signup_fp_hash = (row["signup_fp_hash"] or "").strip()
    if signup_ip_hash:
        shared_ip = int(
            db.execute(
                "SELECT COUNT(*) FROM referral_invites WHERE signup_ip_hash = ?",
                (signup_ip_hash,),
            ).fetchone()[0]
            or 0
        )
        if shared_ip > 1:
            notes.append(f"shared_signup_ip_hash:{shared_ip}")
    if signup_fp_hash:
        shared_fp = int(
            db.execute(
                "SELECT COUNT(*) FROM referral_invites WHERE signup_fp_hash = ?",
                (signup_fp_hash,),
            ).fetchone()[0]
            or 0
        )
        if shared_fp > 1:
            notes.append(f"shared_signup_fingerprint:{shared_fp}")
    if (row["review_status"] or "pending") == "pending" and float(row["fully_activated_at"] or 0) > 0:
        notes.append("ready_for_review")
    if (row["suspicious_notes"] or "").strip():
        notes.append((row["suspicious_notes"] or "").strip())
    allowed_track = _normalize_referral_track(row["allowed_track"], "both")
    invitee_track = (row["invitee_track"] or "agent").strip().lower()
    if not _referral_track_allowed(allowed_track, invitee_track):
        notes.append(f"track_mismatch:{allowed_track}->{invitee_track}")
    return notes


def _referral_admin_payload(db: sqlite3.Connection, row: sqlite3.Row) -> dict:
    invitee_created_at = float(row["invitee_created_at"] or 0)
    return {
        "id": int(row["id"]),
        "referral_code": row["referral_code"],
        "allowed_track": _normalize_referral_track(row["allowed_track"], "both"),
        "source": row["source"] or "",
        "referrer": {
            "agent_name": row["referrer_name"],
            "display_name": row["referrer_display_name"] or row["referrer_name"],
        },
        "invitee": {
            "agent_name": row["invitee_name"],
            "display_name": row["invitee_display_name"] or row["invitee_name"],
            "track": row["invitee_track"],
            "created_at": invitee_created_at,
            "account_age_days": round(max(time.time() - invitee_created_at, 0) / 86400.0, 2) if invitee_created_at > 0 else 0.0,
        },
        "signup_at": float(row["signup_at"] or 0),
        "review_status": (row["review_status"] or "pending").strip().lower() or "pending",
        "reviewed_at": float(row["reviewed_at"] or 0),
        "reviewer_note": row["reviewer_note"] or "",
        "milestones": {
            "profile_completed": {
                "done": float(row["profile_completed_at"] or 0) > 0,
                "at": float(row["profile_completed_at"] or 0),
                "evidence_ref": row["profile_completed_ref"] or "",
            },
            "first_public_video": {
                "done": float(row["first_public_video_at"] or 0) > 0,
                "at": float(row["first_public_video_at"] or 0),
                "evidence_ref": row["first_public_video_ref"] or "",
            },
            "first_rtc_native_action": {
                "done": float(row["first_rtc_native_action_at"] or 0) > 0,
                "at": float(row["first_rtc_native_action_at"] or 0),
                "evidence_ref": row["first_rtc_native_action_ref"] or "",
            },
            "fully_activated": {
                "done": float(row["fully_activated_at"] or 0) > 0,
                "at": float(row["fully_activated_at"] or 0),
            },
        },
        "suspicious_notes": _referral_admin_notes(db, row),
    }


@app.route("/api/admin/referrals")
def admin_referrals():
    """Admin review queue for human/agent referral funnels."""
    err = _require_admin()
    if err:
        return err

    db = get_db()
    status_filter = (request.args.get("status", "") or "").strip().lower()
    track_filter = (request.args.get("track", "") or "").strip().lower()
    ref_code = _normalize_ref_code(request.args.get("code", ""))
    page = max(1, request.args.get("page", 1, type=int))
    per_page = min(100, max(1, request.args.get("per_page", 20, type=int)))
    offset = (page - 1) * per_page

    where = ["1 = 1"]
    params: list[object] = []
    if status_filter:
        where.append("ri.review_status = ?")
        params.append(status_filter)
    if track_filter in {"human", "agent"}:
        where.append("ri.invitee_track = ?")
        params.append(track_filter)
    if ref_code:
        where.append("ri.referral_code = ?")
        params.append(ref_code)

    where_sql = " AND ".join(where)
    rows = db.execute(
        f"""
        SELECT
            ri.*,
            COALESCE(rc.allowed_track, 'both') AS allowed_track,
            ref.agent_name AS referrer_name,
            ref.display_name AS referrer_display_name,
            inv.agent_name AS invitee_name,
            inv.display_name AS invitee_display_name,
            inv.created_at AS invitee_created_at
        FROM referral_invites ri
        JOIN agents ref ON ref.id = ri.referrer_agent_id
        JOIN agents inv ON inv.id = ri.invitee_agent_id
        LEFT JOIN referral_codes rc ON rc.code = ri.referral_code
        WHERE {where_sql}
        ORDER BY ri.signup_at DESC, ri.id DESC
        LIMIT ? OFFSET ?
        """,
        params + [per_page, offset],
    ).fetchall()
    total = int(
        db.execute(
            f"SELECT COUNT(*) FROM referral_invites ri WHERE {where_sql}",
            params,
        ).fetchone()[0]
        or 0
    )

    return jsonify(
        {
            "ok": True,
            "page": page,
            "per_page": per_page,
            "total": total,
            "referrals": [_referral_admin_payload(db, row) for row in rows],
        }
    )


@app.route("/api/admin/referrals/<int:invite_id>/review", methods=["POST"])
def admin_review_referral(invite_id):
    """Approve, reject, void, or reset a referral invite."""
    err = _require_admin()
    if err:
        return err

    db = get_db()
    invite = db.execute("SELECT id FROM referral_invites WHERE id = ?", (invite_id,)).fetchone()
    if not invite:
        return jsonify({"error": "Referral invite not found"}), 404

    data = request.get_json(silent=True) or {}
    if not isinstance(data, dict):
        return jsonify({"error": "JSON object required"}), 400
    if not isinstance(data.get("action", "pending"), str):
        return jsonify({"error": "action must be a string"}), 400
    if data.get("note") is not None and not isinstance(data["note"], str):
        return jsonify({"error": "note must be a string"}), 400

    action = (data.get("action", "pending") or "pending").strip().lower()
    if action not in {"pending", "approve", "approved", "reject", "rejected", "void"}:
        return jsonify({"error": "Invalid action. Use pending, approve, reject, or void."}), 400

    new_status = {
        "approve": "approved",
        "approved": "approved",
        "reject": "rejected",
        "rejected": "rejected",
        "void": "void",
        "pending": "pending",
    }[action]
    reviewer_note = (data.get("note", "") or "").strip()[:2000]
    now = time.time()
    db.execute(
        """
        UPDATE referral_invites
        SET review_status = ?, reviewed_at = ?, reviewer_note = ?, updated_at = ?
        WHERE id = ?
        """,
        (new_status, now, reviewer_note, now, invite_id),
    )
    db.commit()
    return jsonify({"ok": True, "id": invite_id, "review_status": new_status})


@app.route("/api/admin/referrals/export")
def admin_export_referrals():
    """Export payout-ready referral rows for manual bounty settlement."""
    err = _require_admin()
    if err:
        return err

    db = get_db()
    fmt = (request.args.get("format", "json") or "json").strip().lower()
    rows = db.execute(
        """
        SELECT
            ri.*,
            COALESCE(rc.allowed_track, 'both') AS allowed_track,
            ref.agent_name AS referrer_name,
            ref.display_name AS referrer_display_name,
            inv.agent_name AS invitee_name,
            inv.display_name AS invitee_display_name,
            inv.created_at AS invitee_created_at
        FROM referral_invites ri
        JOIN agents ref ON ref.id = ri.referrer_agent_id
        JOIN agents inv ON inv.id = ri.invitee_agent_id
        LEFT JOIN referral_codes rc ON rc.code = ri.referral_code
        WHERE COALESCE(ri.fully_activated_at, 0) > 0
          AND COALESCE(ri.review_status, 'pending') NOT IN ('rejected', 'void')
        ORDER BY ri.fully_activated_at DESC, ri.id DESC
        """
    ).fetchall()
    payload = [_referral_admin_payload(db, row) for row in rows]

    if fmt == "csv":
        import csv
        import io

        buf = io.StringIO()
        writer = csv.writer(buf)
        writer.writerow(
            [
                "invite_id",
                "referral_code",
                "referrer_agent_name",
                "invitee_agent_name",
                "invitee_track",
                "review_status",
                "signup_at",
                "profile_completed_at",
                "first_public_video_at",
                "first_rtc_native_action_at",
                "fully_activated_at",
            ]
        )
        for row in rows:
            writer.writerow(
                [
                    int(row["id"]),
                    row["referral_code"],
                    row["referrer_name"],
                    row["invitee_name"],
                    row["invitee_track"],
                    (row["review_status"] or "pending"),
                    float(row["signup_at"] or 0),
                    float(row["profile_completed_at"] or 0),
                    float(row["first_public_video_at"] or 0),
                    float(row["first_rtc_native_action_at"] or 0),
                    float(row["fully_activated_at"] or 0),
                ]
            )
        return Response(
            buf.getvalue(),
            mimetype="text/csv",
            headers={"Content-Disposition": "attachment; filename=referral-export.csv"},
        )

    return jsonify({"ok": True, "count": len(payload), "rows": payload})


def _resolve_badge_target_agent(db: sqlite3.Connection, data: dict):
    agent_id = data.get("agent_id")
    agent_name = (data.get("agent_name", "") or "").strip()
    if agent_id not in (None, ""):
        try:
            agent_id = int(agent_id)
        except Exception:
            return None
        return db.execute(
            "SELECT id, agent_name, display_name, is_human FROM agents WHERE id = ?",
            (agent_id,),
        ).fetchone()
    if agent_name:
        return db.execute(
            "SELECT id, agent_name, display_name, is_human FROM agents WHERE agent_name = ?",
            (agent_name,),
        ).fetchone()
    return None


@app.route("/api/admin/badges")
def admin_badges():
    """List current account badges for admin review."""
    err = _require_admin()
    if err:
        return err

    db = get_db()
    badge_key = (request.args.get("badge_key", "") or "").strip()
    agent_name = (request.args.get("agent_name", "") or "").strip()
    active_filter = (request.args.get("active", "1") or "1").strip().lower()
    page = max(1, request.args.get("page", 1, type=int))
    per_page = min(100, max(1, request.args.get("per_page", 25, type=int)))
    offset = (page - 1) * per_page

    where = ["1 = 1"]
    params: list[object] = []
    if badge_key:
        where.append("ab.badge_key = ?")
        params.append(badge_key)
    if agent_name:
        where.append("a.agent_name = ?")
        params.append(agent_name)
    if active_filter in {"0", "false", "inactive"}:
        where.append("COALESCE(ab.is_active, 1) = 0")
    elif active_filter not in {"all", "*"}:
        where.append("COALESCE(ab.is_active, 1) = 1")
    where_sql = " AND ".join(where)

    rows = db.execute(
        f"""
        SELECT
            ab.*,
            a.agent_name,
            a.display_name,
            a.is_human
        FROM agent_badges ab
        JOIN agents a ON a.id = ab.agent_id
        WHERE {where_sql}
        ORDER BY COALESCE(ab.is_active, 1) DESC, ab.awarded_at DESC, ab.id DESC
        LIMIT ? OFFSET ?
        """,
        params + [per_page, offset],
    ).fetchall()
    total = int(
        db.execute(
            f"SELECT COUNT(*) FROM agent_badges ab JOIN agents a ON a.id = ab.agent_id WHERE {where_sql}",
            params,
        ).fetchone()[0]
        or 0
    )
    return jsonify(
        {
            "ok": True,
            "page": page,
            "per_page": per_page,
            "total": total,
            "badges": [_badge_assignment_payload(row) for row in rows],
        }
    )


@app.route("/api/admin/badges/candidates")
def admin_badge_candidates():
    """List recommended founding badge awards derived from referral activation state."""
    err = _require_admin()
    if err:
        return err

    db = get_db()
    badge_key = (request.args.get("badge_key", "") or "").strip()
    track = (request.args.get("track", "") or "").strip().lower()
    candidates = _build_badge_candidates(db)
    if badge_key:
        candidates = [row for row in candidates if row["badge_key"] == badge_key]
    if track in {"human", "agent"}:
        candidates = [
            row
            for row in candidates
            if track in row["badge_key"] or row.get("evidence", {}).get("invitee_track") == track
        ]
    return jsonify({"ok": True, "total": len(candidates), "candidates": candidates})


@app.route("/api/admin/badges/assign", methods=["POST"])
def admin_assign_badge():
    """Assign or reactivate a founding badge for an account."""
    err = _require_admin()
    if err:
        return err

    db = get_db()
    data = request.get_json(silent=True) or {}
    if not isinstance(data, dict):
        return jsonify({"error": "JSON object required"}), 400
    if not isinstance(data.get("badge_key", ""), str):
        return jsonify({"error": "badge_key must be a string"}), 400

    badge_key = (data.get("badge_key", "") or "").strip()
    if badge_key not in BADGE_CATALOG:
        return jsonify({"error": "Unknown badge_key"}), 400

    agent = _resolve_badge_target_agent(db, data)
    if not agent:
        return jsonify({"error": "Target agent not found"}), 404

    try:
        cohort_number = max(0, int(data.get("cohort_number", 0) or 0))
    except Exception:
        return jsonify({"error": "cohort_number must be an integer"}), 400

    metadata = data.get("metadata", {}) or {}
    if not isinstance(metadata, dict):
        return jsonify({"error": "metadata must be a JSON object"}), 400

    awarded_at_raw = data.get("awarded_at")
    try:
        awarded_at = float(awarded_at_raw) if awarded_at_raw not in (None, "") else time.time()
    except Exception:
        return jsonify({"error": "awarded_at must be numeric"}), 400
    if awarded_at <= 0:
        awarded_at = time.time()

    source_campaign = (
        (data.get("source_campaign", "") or "").strip()[:120]
        or _default_badge_source_campaign(badge_key)
    )
    notes = (data.get("notes", "") or "").strip()[:2000]
    awarded_by = (data.get("awarded_by", "") or "admin").strip()[:120]
    metadata_json = json.dumps(metadata, sort_keys=True)
    now = time.time()

    db.execute(
        """
        INSERT INTO agent_badges (
            agent_id,
            badge_key,
            cohort_number,
            source_campaign,
            notes,
            metadata_json,
            awarded_at,
            awarded_by,
            is_active,
            removed_at,
            removed_by,
            created_at,
            updated_at
        )
        VALUES (?, ?, ?, ?, ?, ?, ?, ?, 1, 0, '', ?, ?)
        ON CONFLICT(agent_id, badge_key) DO UPDATE SET
            cohort_number = excluded.cohort_number,
            source_campaign = excluded.source_campaign,
            notes = excluded.notes,
            metadata_json = excluded.metadata_json,
            awarded_at = excluded.awarded_at,
            awarded_by = excluded.awarded_by,
            is_active = 1,
            removed_at = 0,
            removed_by = '',
            updated_at = excluded.updated_at
        """,
        (
            int(agent["id"]),
            badge_key,
            cohort_number,
            source_campaign,
            notes,
            metadata_json,
            awarded_at,
            awarded_by,
            now,
            now,
        ),
    )
    db.commit()
    row = db.execute(
        """
        SELECT
            ab.*,
            a.agent_name,
            a.display_name,
            a.is_human
        FROM agent_badges ab
        JOIN agents a ON a.id = ab.agent_id
        WHERE ab.agent_id = ? AND ab.badge_key = ?
        """,
        (int(agent["id"]), badge_key),
    ).fetchone()
    return jsonify({"ok": True, "badge": _badge_assignment_payload(row)})


@app.route("/api/admin/badges/<int:badge_id>/remove", methods=["POST"])
def admin_remove_badge(badge_id):
    """Deactivate a badge assignment without deleting its audit trail."""
    err = _require_admin()
    if err:
        return err

    db = get_db()
    row = db.execute(
        """
        SELECT
            ab.*,
            a.agent_name,
            a.display_name,
            a.is_human
        FROM agent_badges ab
        JOIN agents a ON a.id = ab.agent_id
        WHERE ab.id = ?
        """,
        (badge_id,),
    ).fetchone()
    if not row:
        return jsonify({"error": "Badge assignment not found"}), 404

    data = request.get_json(silent=True) or {}
    if not isinstance(data, dict):
        return jsonify({"error": "JSON object required"}), 400
    if data.get("removed_by") is not None and not isinstance(data["removed_by"], str):
        return jsonify({"error": "removed_by must be a string"}), 400

    removed_by = (data.get("removed_by", "") or "admin").strip()[:120]
    now = time.time()
    db.execute(
        """
        UPDATE agent_badges
        SET is_active = 0,
            removed_at = ?,
            removed_by = ?,
            updated_at = ?
        WHERE id = ?
        """,
        (now, removed_by, now, badge_id),
    )
    db.commit()
    row = db.execute(
        """
        SELECT
            ab.*,
            a.agent_name,
            a.display_name,
            a.is_human
        FROM agent_badges ab
        JOIN agents a ON a.id = ab.agent_id
        WHERE ab.id = ?
        """,
        (badge_id,),
    ).fetchone()
    return jsonify({"ok": True, "badge": _badge_assignment_payload(row)})


@app.route("/verify-email/<token>")
def verify_email(token):
    """Verify email address via token link (24hr expiry)."""
    if not token or len(token) != 64:
        abort(404)

    db = get_db()
    user = db.execute(
        "SELECT id, email_verify_sent_at FROM agents WHERE email_verify_token = ?",
        (token,),
    ).fetchone()

    if not user:
        flash("Invalid or expired verification link.", "error")
        return redirect(url_for("login"))

    # Check 24-hour expiry
    if time.time() - user["email_verify_sent_at"] > 86400:
        flash("Verification link has expired. Please request a new one.", "error")
        return redirect(url_for("login"))

    db.execute(
        "UPDATE agents SET email_verified = 1, email_verify_token = '' WHERE id = ?",
        (user["id"],),
    )
    db.commit()
    flash("Email verified successfully!", "success")
    return redirect(url_for("index"))


@app.route("/resend-verification")
def resend_verification():
    """Resend email verification. Rate limited to 3/hr."""
    if not g.user:
        return redirect(url_for("login"))

    email = g.user["email"]
    if not email:
        flash("No email address on your account.", "error")
        return redirect(url_for("index"))

    if g.user["email_verified"]:
        flash("Email already verified.", "error")
        return redirect(url_for("index"))

    ip = _get_client_ip()
    if not _rate_limit(f"resend-email:{g.user['id']}", 3, 3600):
        flash("Too many resend requests. Try again later.", "error")
        return redirect(url_for("index"))

    new_token = secrets.token_hex(32)
    db = get_db()
    db.execute(
        "UPDATE agents SET email_verify_token = ?, email_verify_sent_at = ? WHERE id = ?",
        (new_token, time.time(), g.user["id"]),
    )
    db.commit()
    send_verification_email(email, new_token, g.user["agent_name"])
    flash("Verification email resent. Check your inbox.", "success")
    return redirect(url_for("index"))


# ---------------------------------------------------------------------------
# Google OAuth Sign-In
# ---------------------------------------------------------------------------

GOOGLE_CLIENT_ID = os.environ.get("GOOGLE_CLIENT_ID", "")
GOOGLE_CLIENT_SECRET = os.environ.get("GOOGLE_CLIENT_SECRET", "")
GOOGLE_REDIRECT_URI = os.environ.get("GOOGLE_REDIRECT_URI", "https://bottube.ai/auth/google/callback")


@app.route("/auth/google")
def google_auth():
    """Redirect to Google OAuth consent screen."""
    if not GOOGLE_CLIENT_ID:
        flash("Google sign-in is not configured.", "error")
        return redirect(url_for("login"))

    ref_code = _normalize_ref_code(request.args.get("ref", "") or session.get("ref_code", ""))
    if ref_code:
        session["ref_code"] = ref_code

    # Generate state token for CSRF protection
    state = secrets.token_hex(16)
    session["google_oauth_state"] = state

    params = urllib.parse.urlencode({
        "client_id": GOOGLE_CLIENT_ID,
        "redirect_uri": GOOGLE_REDIRECT_URI,
        "response_type": "code",
        "scope": "openid email profile",
        "state": state,
        "access_type": "online",
        "prompt": "select_account",
    })
    return redirect(f"https://accounts.google.com/o/oauth2/v2/auth?{params}")


@app.route("/auth/google/callback")
def google_callback():
    """Handle Google OAuth callback."""
    if not GOOGLE_CLIENT_ID or not GOOGLE_CLIENT_SECRET:
        flash("Google sign-in is not configured.", "error")
        return redirect(url_for("login"))

    # Verify state
    state = request.args.get("state", "")
    if not state or state != session.pop("google_oauth_state", ""):
        flash("Invalid OAuth state. Please try again.", "error")
        return redirect(url_for("login"))

    code = request.args.get("code", "")
    error = request.args.get("error", "")
    if error or not code:
        flash(f"Google sign-in was cancelled or failed.", "error")
        return redirect(url_for("login"))

    # Exchange code for tokens
    token_data = urllib.parse.urlencode({
        "code": code,
        "client_id": GOOGLE_CLIENT_ID,
        "client_secret": GOOGLE_CLIENT_SECRET,
        "redirect_uri": GOOGLE_REDIRECT_URI,
        "grant_type": "authorization_code",
    }).encode()

    try:
        token_req = urllib.request.Request(
            "https://oauth2.googleapis.com/token",
            data=token_data,
            headers={"Content-Type": "application/x-www-form-urlencoded"},
        )
        with urllib.request.urlopen(token_req, timeout=10) as resp:
            tokens = json.loads(resp.read())
    except Exception:
        flash("Failed to exchange Google authorization code.", "error")
        return redirect(url_for("login"))

    access_token = tokens.get("access_token")
    if not access_token:
        flash("No access token received from Google.", "error")
        return redirect(url_for("login"))

    # Fetch user info
    try:
        info_req = urllib.request.Request(
            "https://www.googleapis.com/oauth2/v3/userinfo",
            headers={"Authorization": f"Bearer {access_token}"},
        )
        with urllib.request.urlopen(info_req, timeout=10) as resp:
            userinfo = json.loads(resp.read())
    except Exception:
        flash("Failed to fetch Google user info.", "error")
        return redirect(url_for("login"))

    google_id = userinfo.get("sub", "")
    google_email = userinfo.get("email", "")
    google_name = userinfo.get("name", "")
    google_avatar = userinfo.get("picture", "")

    if not google_id:
        flash("Could not identify your Google account.", "error")
        return redirect(url_for("login"))

    db = get_db()

    # Case 1: Existing user with this Google ID — log them in
    existing = db.execute(
        "SELECT * FROM agents WHERE google_id = ?", (google_id,)
    ).fetchone()
    if existing:
        session.clear()
        session.permanent = True
        session["user_id"] = existing["id"]
        session["csrf_token"] = secrets.token_hex(32)
        return redirect(url_for("index"))

    # Case 2: Currently logged in — link Google to existing account
    if g.user:
        db.execute(
            "UPDATE agents SET google_id = ?, google_email = ?, google_avatar = ? WHERE id = ?",
            (google_id, google_email, google_avatar, g.user["id"]),
        )
        db.commit()
        flash("Google account linked successfully!", "success")
        return redirect(url_for("index"))

    # Case 3: Email matches existing account — link and log in
    if google_email:
        email_match = db.execute(
            "SELECT * FROM agents WHERE email = ? AND email != ''", (google_email,)
        ).fetchone()
        if email_match:
            db.execute(
                "UPDATE agents SET google_id = ?, google_email = ?, google_avatar = ?, email_verified = 1 WHERE id = ?",
                (google_id, google_email, google_avatar, email_match["id"]),
            )
            db.commit()
            session.clear()
            session.permanent = True
            session["user_id"] = email_match["id"]
            session["csrf_token"] = secrets.token_hex(32)
            return redirect(url_for("index"))

    # Case 4: New user — auto-create account
    # Generate username from email or Google name
    base_name = ""
    if google_email:
        base_name = google_email.split("@")[0].lower()
    elif google_name:
        base_name = google_name.lower().replace(" ", "")
    base_name = re.sub(r"[^a-z0-9_-]", "", base_name)[:24] or "user"

    # Ensure unique username
    username = base_name
    suffix = 1
    while db.execute("SELECT 1 FROM agents WHERE agent_name = ?", (username,)).fetchone():
        username = f"{base_name}{suffix}"
        suffix += 1

    api_key = gen_api_key()
    display_name = google_name or username
    now = time.time()

    db.execute(
        "INSERT INTO agents (agent_name, display_name, api_key, is_human, email, email_verified, "
        "google_id, google_email, google_avatar, avatar_url, created_at, last_active) "
        "VALUES (?, ?, ?, 1, ?, 1, ?, ?, ?, ?, ?, ?)",
        (username, display_name, api_key, google_email, google_id, google_email,
         google_avatar, google_avatar, now, now),
    )
    new_user_id = int(db.execute("SELECT last_insert_rowid()").fetchone()[0])
    ref_code = _normalize_ref_code(session.pop("ref_code", ""))
    if ref_code:
        _referral_apply_signup(db, new_user_id, ref_code, source="google_oauth_signup")
    db.commit()

    new_user = db.execute("SELECT * FROM agents WHERE agent_name = ?", (username,)).fetchone()
    session.clear()
    session.permanent = True
    session["user_id"] = new_user["id"]
    session["csrf_token"] = secrets.token_hex(32)

    flash(f"Welcome to BoTTube, {display_name}! Your account has been created.", "success")
    return redirect(url_for("index"))


# ---------------------------------------------------------------------------
# Video upload
# ---------------------------------------------------------------------------

@app.route("/api/upload", methods=["POST"])
@require_api_key
def upload_video():
    """Upload a video file."""
    if "video" not in request.files:
        return jsonify({"error": "No video file in request"}), 400

    video_file = request.files["video"]
    if not video_file.filename:
        return jsonify({"error": "Empty filename"}), 400

    ext = Path(video_file.filename).suffix.lower()
    if ext not in ALLOWED_VIDEO_EXT:
        return jsonify({"error": f"Invalid video format. Allowed: {ALLOWED_VIDEO_EXT}"}), 400

    title = _strip_script_tags(request.form.get("title", "").strip()[:MAX_TITLE_LENGTH])
    if not title:
        title = _strip_script_tags(Path(video_file.filename).stem[:MAX_TITLE_LENGTH])

    description = _strip_script_tags(request.form.get("description", "").strip()[:MAX_DESCRIPTION_LENGTH])
    scene_description = _strip_script_tags(request.form.get("scene_description", "").strip()[:MAX_DESCRIPTION_LENGTH])
    tags_raw = request.form.get("tags", "")
    tags = [_strip_script_tags(t.strip()[:MAX_TAG_LENGTH]) for t in tags_raw.split(",") if t.strip()][:MAX_TAGS]
    category = request.form.get("category", "other").strip().lower()
    if category not in CATEGORY_MAP:
        category = "other"
    revision_of = request.form.get("revision_of", "").strip()
    revision_note = request.form.get("revision_note", "").strip()[:MAX_DESCRIPTION_LENGTH]
    challenge_id = request.form.get("challenge_id", "").strip()
    gen_method = request.form.get("gen_method", "").strip().lower()  # AI video gen method
    response_to = request.form.get("response_to", "").strip()  # Video ID this is a response to (Issue #2282)
    collaborator_ids_raw = request.form.get("collaborator_ids", "").strip()  # JSON array of agent_ids for co-upload (Bounty #2161)

    db = get_db()
    if revision_of:
        if not re.fullmatch(r"[A-Za-z0-9_-]{5,20}", revision_of):
            return jsonify({"error": "Invalid revision_of video id"}), 400
        original = db.execute(
            "SELECT video_id FROM videos WHERE video_id = ?",
            (revision_of,),
        ).fetchone()
        if not original:
            return jsonify({"error": "revision_of video not found"}), 404
    # Validate response_to video ID (Issue #2282)
    if response_to:
        if not re.fullmatch(r"[A-Za-z0-9_-]{5,20}", response_to):
            return jsonify({"error": "Invalid response_to video id"}), 400
        original_video = db.execute(
            "SELECT video_id, is_removed FROM videos WHERE video_id = ?",
            (response_to,),
        ).fetchone()
        if not original_video:
            return jsonify({"error": "response_to video not found"}), 404
        if original_video["is_removed"]:
            return jsonify({"error": "Cannot respond to a removed video"}), 400

    # Parse + validate collaborator_ids (Bounty #2161 co-upload)
    collaborator_ids_json = "[]"
    if collaborator_ids_raw:
        try:
            parsed_collab = json.loads(collaborator_ids_raw)
        except (ValueError, TypeError):
            return jsonify({"error": "Invalid collaborator_ids: must be a JSON array of agent_ids"}), 400
        if not isinstance(parsed_collab, list):
            return jsonify({"error": "Invalid collaborator_ids: must be a JSON array"}), 400
        if len(parsed_collab) > 5:
            return jsonify({"error": "Too many collaborators: max 5 per video"}), 400
        cleaned = []
        for cid in parsed_collab:
            if not isinstance(cid, int) or cid <= 0:
                return jsonify({"error": "Invalid collaborator_id: must be a positive int"}), 400
            if cid == g.agent["id"]:
                return jsonify({"error": "Cannot add yourself as a collaborator"}), 400
            cleaned.append(cid)
        if cleaned:
            placeholders = ",".join("?" * len(cleaned))
            existing = db.execute(
                f"SELECT id FROM agents WHERE id IN ({placeholders})", cleaned
            ).fetchall()
            existing_ids = {row["id"] for row in existing}
            missing = set(cleaned) - existing_ids
            if missing:
                return jsonify({"error": f"Unknown collaborator agent_id(s): {sorted(missing)}"}), 400
        collaborator_ids_json = json.dumps(cleaned)

    if challenge_id:
        ch = db.execute(
            "SELECT challenge_id, status, start_at, end_at FROM challenges WHERE challenge_id = ?",
            (challenge_id,),
        ).fetchone()
        if not ch:
            return jsonify({"error": "challenge_id not found"}), 404
        now = time.time()
        is_active = (ch["status"] == "active") or (
            ch["start_at"] and ch["end_at"] and ch["start_at"] <= now <= ch["end_at"]
        )
        if not is_active:
            return jsonify({"error": "challenge is not active"}), 400

    # Rate limit: 5 uploads per agent per hour, 15 per day
    if not _rate_limit(f"upload_h:{g.agent['id']}", 5, 3600):
        return jsonify({"error": "Upload rate limit exceeded (max 5/hour). Try again later."}), 429
    if not _rate_limit(f"upload_d:{g.agent['id']}", 15, 86400):
        return jsonify({"error": "Daily upload limit exceeded (max 15/day). Try again tomorrow."}), 429

    # Content moderation: check title/description/tags against blocklist
    blocked_term = _content_check(title, description, tags)
    if blocked_term:
        app.logger.warning(
            "CONTENT BLOCKED: agent=%s term='%s' title='%s'",
            g.agent["agent_name"], blocked_term, title[:80],
        )
        coach_note = (
            f"Your upload title, description, or tags triggered the blocked term `{blocked_term}`.\n\n"
            "No account suspension was applied. Rewrite the metadata to clearly describe the video without using "
            "policy-breaking language, then submit again. If this was a false positive, a maintainer can review the hold."
        )
        _queue_moderation_hold(
            db,
            target_type="upload_preflight",
            target_ref=f"{g.agent['id']}:{int(time.time())}",
            target_agent_id=g.agent["id"],
            source="upload_blocklist",
            reason="blocked upload metadata",
            details=json.dumps({
                "title": title[:200],
                "blocked_term": blocked_term,
                "tags": tags,
            }),
            recommended_action="coach",
            coach_note=coach_note,
        )
        db.commit()
        return jsonify({
            "error": "Upload held for coaching review.",
            "code": "CONTENT_POLICY_VIOLATION",
            "coach_note": coach_note,
        }), 422

    # Generate unique video ID
    video_id = gen_video_id()
    while (VIDEO_DIR / f"{video_id}{ext}").exists():
        video_id = gen_video_id()

    filename = f"{video_id}{ext}"
    video_path = VIDEO_DIR / filename

    # Save video
    video_file.save(str(video_path))

    # Trust+Safety: hash-check the saved file against the content blocklist.
    # On match (CSAM, terror, etc.) the helper deletes the file, suspends
    # the agent, and writes audit rows. We surface a 451 to the uploader.
    try:
        rejected, ts_info = ts_inspect_uploaded_file(str(video_path), g.agent["id"])
        if rejected:
            app.logger.error(
                "TS-BLOCKLIST: agent=%s category=%s sha=%s",
                g.agent["agent_name"], ts_info.get("category"), ts_info.get("sha256", "")[:16],
            )
            return jsonify({
                "error": "Upload rejected: content matched the prohibited-content blocklist. "
                         "If you believe this is in error, contact appeals@elyanlabs.ai. "
                         "CSAM matches are reported to NCMEC and law enforcement.",
                "category": ts_info.get("category"),
            }), 451
    except Exception as _ts_e:
        app.logger.warning("TS hash check failed (non-fatal): %s", _ts_e)

    # Get metadata
    duration, width, height = get_video_metadata(video_path)

    # Per-category limits
    cat_limits = CATEGORY_LIMITS.get(category, {})
    max_dur = cat_limits.get("max_duration", MAX_VIDEO_DURATION)
    max_file = cat_limits.get("max_file_mb", MAX_FINAL_FILE_SIZE / (1024 * 1024))
    keep_audio = cat_limits.get("keep_audio", True)

    # Enforce duration limit
    if duration > max_dur:
        video_path.unlink(missing_ok=True)
        return jsonify({
            "error": f"Video too long ({duration:.1f}s). Max for {category}: {max_dur} seconds.",
            "max_duration": max_dur,
            "category": category,
        }), 400

    # Always transcode to enforce size/format constraints
    transcoded_path = VIDEO_DIR / f"{video_id}_tc.mp4"
    if transcode_video(video_path, transcoded_path, keep_audio=keep_audio,
                       target_file_mb=max_file, duration_hint=duration):
        video_path.unlink(missing_ok=True)
        filename = f"{video_id}.mp4"
        final_path = VIDEO_DIR / filename
        transcoded_path.rename(final_path)
        video_path = final_path
        duration, width, height = get_video_metadata(final_path)
    else:
        video_path.unlink(missing_ok=True)
        transcoded_path.unlink(missing_ok=True)
        return jsonify({"error": "Video transcoding failed"}), 500

    # Enforce max final file size (per-category)
    max_file_bytes = int(max_file * 1024 * 1024)
    final_size = video_path.stat().st_size
    if final_size > max_file_bytes:
        video_path.unlink(missing_ok=True)
        return jsonify({
            "error": f"Video too large after transcoding ({final_size / 1024:.0f} KB). "
                     f"Max for {category}: {max_file_bytes // 1024} KB.",
            "max_file_kb": max_file_bytes // 1024,
        }), 400

    # Handle thumbnail (max 2MB)
    thumb_filename = ""
    MAX_THUMB_SIZE = 2 * 1024 * 1024
    if "thumbnail" in request.files and request.files["thumbnail"].filename:
        thumb_file = request.files["thumbnail"]
        thumb_file.seek(0, 2)
        if thumb_file.tell() > MAX_THUMB_SIZE:
            return jsonify({"error": "Thumbnail must be 2MB or smaller"}), 400
        thumb_file.seek(0)
        thumb_ext = Path(thumb_file.filename).suffix.lower()
        if thumb_ext in ALLOWED_THUMB_EXT:
            # Save original, then normalize to small JPG for faster loads.
            orig_name = f"{video_id}{thumb_ext}"
            orig_path = THUMB_DIR / orig_name
            thumb_file.save(str(orig_path))

            opt_name = f"{video_id}.jpg"
            opt_path = THUMB_DIR / opt_name
            if optimize_thumbnail_image(orig_path, opt_path):
                thumb_filename = opt_name
                if orig_path != opt_path:
                    orig_path.unlink(missing_ok=True)
            else:
                thumb_filename = orig_name
    else:
        # Auto-generate thumbnail
        thumb_filename = f"{video_id}.jpg"
        if not generate_thumbnail(video_path, THUMB_DIR / thumb_filename):
            thumb_filename = ""

    # ----- Vision Screening -----
    screening_result = screen_video(str(video_path), run_tier2=VISION_SCREENING_ENABLED)
    screening_status = screening_result.get("status", "passed")
    screening_details = json.dumps(screening_result)

    if screening_status == "failed":
        app.logger.warning(
            "VISION SCREEN REJECT: video=%s agent=%s reason=%s",
            video_id, g.agent["agent_name"], screening_result.get("summary", ""),
        )
        coach_note = (
            "Your upload was held for review by the screening system. "
            "Tighten the clip, improve clarity, and avoid repetitive or spam-like frames before re-uploading."
        )
        _queue_moderation_hold(
            db,
            target_type="video",
            target_ref=video_id,
            target_agent_id=g.agent["id"],
            source="vision_screening",
            reason="video held by screening",
            details=screening_result.get("summary", "")[:2000],
            recommended_action="coach",
            coach_note=coach_note,
        )

    novelty_score, novelty_flags = compute_novelty_score(
        db, g.agent["id"], title, description, tags, scene_description
    )
    db.execute(
        """INSERT INTO videos
           (video_id, agent_id, title, description, filename, thumbnail,
            duration_sec, width, height, tags, scene_description, category,
            novelty_score, novelty_flags, revision_of, revision_note, challenge_id, response_to_video_id, collaborator_ids, created_at,
            screening_status, screening_details, is_removed, removed_reason)
           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
        (
            video_id, g.agent["id"], title, description, filename,
            thumb_filename, duration, width, height, json.dumps(tags),
            scene_description, category, novelty_score, novelty_flags,
            revision_of, revision_note, challenge_id, response_to, collaborator_ids_json, time.time(),
            screening_status, screening_details,
            1 if screening_status == "failed" else 0,
            ("held_for_review: " + screening_result.get("summary", ""))[:500] if screening_status == "failed" else "",
        ),
    )
    # Award RTC for upload
    award_rtc(db, g.agent["id"], RTC_REWARD_UPLOAD, "video_upload", video_id)
    _referral_mark_first_upload(db, g.agent["id"])
    _refresh_agent_quests(db, g.agent["id"], ["first_upload"])
    _referral_refresh_invite_state(db, g.agent["id"])
    db.commit()

    # Provenance: hash the canonical (post-transcode) asset, capture any
    # generation metadata the agent supplied, sign with the platform key,
    # and persist. Pill flips from gray (unverified) to amber (pending).
    # Anchor TX is filled in later by the Ergo anchor job (separate cron).
    try:
        _provenance_record_for_upload(
            video_id=video_id,
            canonical_path=str(video_path),
            agent=g.agent,
            form=request.form,
            width=width,
            height=height,
            duration=duration,
            uploaded_at=time.time(),
        )
        # Phase 11.12 + 11.16: write the thumbnail integrity hash now;
        # leave manifest_version at its default (1) until renditions
        # finish and canonical_360p_sha256 is known. The rendition
        # pipeline atomically promotes the row to v2 once both hashes
        # are present — this avoids a race where the anchor worker
        # could commit a v2 leaf with an empty 360p field.
        try:
            _provenance_ensure_thumb_column()
            _provenance_ensure_v2_columns()
            tsha = _provenance_thumbnail_sha(video_id, thumb_filename)
            conn_t = sqlite3.connect(str(DB_PATH))
            conn_t.execute(
                """UPDATE video_provenance
                      SET thumbnail_sha256 = ?,
                          updated_at = ?
                    WHERE video_id = ?""",
                (tsha or "", time.time(), video_id),
            )
            conn_t.commit()
            conn_t.close()
        except Exception as _t_e:
            app.logger.warning("thumbnail hash write failed for %s: %s", video_id, _t_e)
    except Exception as _prov_e:
        # Provenance failures must not block upload success.
        app.logger.warning("provenance record failed for %s: %s", video_id, _prov_e)

    # Adaptive renditions + VMAF: encode 360p variant in the background
    # and compute VMAF against canonical. Populates video_renditions so
    # the provenance side-sheet shows real per-encoding quality scores.
    # Bounded concurrency via _RENDITION_GATE; never blocks the upload.
    try:
        _renditions_process_video_async(video_id)
    except Exception as _rend_e:
        app.logger.warning("rendition async dispatch failed for %s: %s", video_id, _rend_e)

    # Semantic embedding: title+description+tags+scene → Gemini text
    # embedding (3072-d, L2-norm). Used by /api/videos/<id>/similar and
    # the upcoming hybrid feed. Async — single API call, ~300ms.
    try:
        _ue_record_for_video_async(video_id)
    except Exception as _emb_e:
        app.logger.warning("embedding async dispatch failed for %s: %s", video_id, _emb_e)

    # Generate captions from the finalized video asset in the background.
    generate_captions_async(video_id, str(video_path))

    response_data = {
        "ok": True,
        "video_id": video_id,
        "watch_url": f"/watch/{video_id}",
        "stream_url": f"/api/videos/{video_id}/stream",
        "title": title,
        "duration_sec": duration,
        "width": width,
        "height": height,
        "screening": {
            "status": screening_status,
            "summary": screening_result.get("summary", ""),
        },
    }
    if response_to:
        response_data["response_to"] = response_to
    if screening_status == "failed":
        response_data["warning"] = "Video is held for coaching review and is not public yet."
    # Ping search engines about the new video
    _ping_indexnow(f"https://bottube.ai/watch/{video_id}")
    ping_google_indexing(f"https://bottube.ai/watch/{video_id}")

    # Award BAN for upload
    award_ban_upload(db, g.agent["id"], video_id)

    # Award extra BAN for AI video generation (if gen_method specified)
    ban_gen_reward = 0.0
    if gen_method:
        ban_gen_reward = award_ban_video_gen(db, g.agent["id"], video_id, gen_method)
    if ban_gen_reward > 0:
        response_data["ban_video_gen_reward"] = ban_gen_reward

    # Notify subscribers about the new video (background)
    _notify_subscribers_new_video(g.agent["id"], video_id, title, g.agent["agent_name"])

    return jsonify(response_data), 201



# ---------------------------------------------------------------------------
# Video update (title, description, tags)
# ---------------------------------------------------------------------------

@app.route('/api/videos/<video_id>', methods=['PATCH'])
@require_api_key
def update_video(video_id):
    """Update video metadata (title, description, tags). Owner only."""
    db = get_db()
    row = db.execute('SELECT * FROM videos WHERE video_id = ?', (video_id,)).fetchone()
    if not row:
        return jsonify({'error': 'Video not found'}), 404
    if row['agent_id'] != g.agent['id']:
        return jsonify({'error': 'Not your video'}), 403
    
    data = request.get_json(silent=True) or {}
    updates = []
    params = []
    
    if 'title' in data and data['title'].strip():
        updates.append('title = ?')
        params.append(data['title'].strip()[:200])
    if 'description' in data and data['description'].strip():
        updates.append('description = ?')
        params.append(data['description'].strip()[:5000])
    if 'tags' in data:
        if isinstance(data['tags'], list):
            tag_str = ','.join(t.strip() for t in data['tags'] if t.strip())
        else:
            tag_str = str(data['tags']).strip()
        updates.append('tags = ?')
        params.append(tag_str)
    
    if not updates:
        return jsonify({'error': 'Nothing to update'}), 400
    
    params.append(video_id)
    joiner = ', '.join(updates); db.execute(f'UPDATE videos SET {joiner} WHERE video_id = ?', params)
    db.commit()
    
    return jsonify({'ok': True, 'updated': list(data.keys()), 'video_id': video_id})

# ---------------------------------------------------------------------------
# Video listing / detail
# ---------------------------------------------------------------------------

def _video_list_etag(
    *,
    page: int,
    per_page: int,
    sort: str,
    agent_name: str,
    total: int,
    latest_ts: float,
    engagement_revision: int,
) -> str:
    cache_key = json.dumps(
        {
            "agent": agent_name,
            "latest_ts": latest_ts,
            "page": page,
            "per_page": per_page,
            "engagement_revision": engagement_revision,
            "sort": sort,
            "total": total,
        },
        sort_keys=True,
        separators=(",", ":"),
    )
    digest = hashlib.sha256(cache_key.encode("utf-8")).hexdigest()[:24]
    return f'W/"videos-{digest}"'


def _client_has_video_list_etag(etag: str) -> bool:
    raw_header = request.headers.get("If-None-Match", "")
    if not raw_header:
        return False
    candidates = {part.strip() for part in raw_header.split(",")}
    return "*" in candidates or etag in candidates


def _parse_positive_int_query(name, default, min_value=1, max_value=None):
    """Return (value, None) or (None, (json_response, status_code)).

    Rejects malformed or out-of-range integers with HTTP 400 instead of
    silently coercing invalid input to the default (which would mask
    client bugs and could lead to surprising pagination/sort results).
    """
    raw_value = request.args.get(name)
    if raw_value is None or raw_value == "":
        return default, None
    try:
        value = int(raw_value)
    except (TypeError, ValueError):
        return None, (
            jsonify({"error": f"{name} must be an integer"}),
            400,
        )
    if value < min_value:
        return None, (
            jsonify({"error": f"{name} must be >= {min_value}"}),
            400,
        )
    if max_value is not None and value > max_value:
        return None, (
            jsonify({"error": f"{name} must be <= {max_value}"}),
            400,
        )
    return value, None


def _client_has_fresh_video_list_date(latest_ts: float) -> bool:
    raw_header = request.headers.get("If-Modified-Since", "")
    if not raw_header:
        return False
    try:
        modified_since = parsedate_to_datetime(raw_header)
    except (TypeError, ValueError):
        return False
    if modified_since is None:
        return False
    return int(latest_ts or 0) <= int(modified_since.timestamp())


def _add_video_list_cache_headers(response: Response, *, etag: str, latest_ts: float) -> Response:
    response.headers["ETag"] = etag
    response.headers["Last-Modified"] = formatdate(int(latest_ts or 0), usegmt=True)
    response.headers["Cache-Control"] = "public, max-age=30"
    return response

@app.route("/api/v1/videos")
def list_videos_v1_alias():
    """Canonical alias for /api/videos, used by telegram bot + debate bots + algolia scanner (Bottube #1383)."""
    return list_videos()


@app.route("/api/videos")
def list_videos():
    """List videos with pagination and sorting."""
    page, error = _parse_positive_int_query("page", 1)
    if error:
        return error
    per_page, error = _parse_positive_int_query("per_page", 20, max_value=50)
    if error:
        return error
    sort = request.args.get("sort", "newest")
    agent_name = request.args.get("agent", "")

    sort_map = {
        "newest": "v.created_at DESC",
        "oldest": "v.created_at ASC",
        "views": "v.views DESC",
        "likes": "v.likes DESC",
        "title": "v.title ASC",
    }
    order = sort_map.get(sort, "v.created_at DESC")

    db = get_db()
    where_clauses = ["v.is_removed = 0"]
    params = []
    if agent_name:
        where_clauses.append("a.agent_name = ?")
        params.append(agent_name)
    where = "WHERE " + " AND ".join(where_clauses)

    stats = db.execute(
        f"""SELECT
                COUNT(*) AS total,
                COALESCE(MAX(v.created_at), 0) AS latest_ts,
                COALESCE(MAX(
                    MAX(
                        COALESCE((SELECT MAX(vw.created_at) FROM views vw WHERE vw.video_id = v.video_id), 0),
                        COALESCE((SELECT MAX(vt.created_at) FROM votes vt WHERE vt.video_id = v.video_id), 0)
                    )
                ), 0) AS latest_engagement_ts,
                COALESCE(SUM(v.views + v.likes + v.dislikes), 0) AS engagement_revision
            FROM videos v JOIN agents a ON v.agent_id = a.id {where}""",
        params,
    ).fetchone()
    total = int(stats["total"] or 0)
    latest_ts = max(float(stats["latest_ts"] or 0), float(stats["latest_engagement_ts"] or 0))
    engagement_revision = int(stats["engagement_revision"] or 0)
    pages = math.ceil(total / per_page) if total else 0
    if pages:
        page = min(page, pages)
    else:
        page = 1
    offset = (page - 1) * per_page
    etag = _video_list_etag(
        page=page,
        per_page=per_page,
        sort=sort,
        agent_name=agent_name,
        total=total,
        latest_ts=latest_ts,
        engagement_revision=engagement_revision,
    )

    if request.headers.get("If-None-Match"):
        is_fresh = _client_has_video_list_etag(etag)
    else:
        is_fresh = _client_has_fresh_video_list_date(latest_ts)
    if is_fresh:
        return _add_video_list_cache_headers(make_response("", 304), etag=etag, latest_ts=latest_ts)

    rows = db.execute(
        f"""SELECT v.*, a.agent_name, a.display_name, a.avatar_url
            FROM videos v JOIN agents a ON v.agent_id = a.id
            {where} ORDER BY {order} LIMIT ? OFFSET ?""",
        params + [per_page, offset],
    ).fetchall()

    videos = []
    for row in rows:
        d = video_to_dict(row)
        d["agent_name"] = row["agent_name"]
        d["display_name"] = row["display_name"]
        d["avatar_url"] = row["avatar_url"]
        videos.append(d)

    response = jsonify({
        "videos": videos,
        "page": page,
        "per_page": per_page,
        "total": total,
        "pages": pages,
    })
    return _add_video_list_cache_headers(response, etag=etag, latest_ts=latest_ts)


@app.route("/api/videos/<video_id>")
def get_video(video_id):
    """Get video metadata."""
    db = get_db()
    row = db.execute(
        f"""SELECT v.*, a.agent_name, a.display_name, a.avatar_url
           FROM videos v JOIN agents a ON v.agent_id = a.id
           WHERE v.video_id = ? AND {_public_video_filter_sql()}""",
        (video_id,),
    ).fetchone()

    if not row:
        return jsonify({"error": "Video not found"}), 404

    d = video_to_dict(row)
    d["agent_name"] = row["agent_name"]
    d["display_name"] = row["display_name"]
    d["avatar_url"] = row["avatar_url"]
    if "revision_of" in row.keys() and row["revision_of"]:
        original = db.execute(
            f"""SELECT v.video_id, v.title, a.agent_name, a.display_name
               FROM videos v JOIN agents a ON v.agent_id = a.id
               WHERE v.video_id = ? AND {_public_video_filter_sql()}""",
            (row["revision_of"],),
        ).fetchone()
        if original:
            d["revision_of_video"] = {
                "video_id": original["video_id"],
                "title": original["title"],
                "agent_name": original["agent_name"],
                "display_name": original["display_name"],
            }
    revisions = db.execute(
        f"""SELECT v.video_id, v.title, v.created_at, a.agent_name, a.display_name
           FROM videos v JOIN agents a ON v.agent_id = a.id
           WHERE v.revision_of = ? AND {_public_video_filter_sql()}
           ORDER BY v.created_at DESC LIMIT 10""",
        (video_id,),
    ).fetchall()
    d["revisions"] = [
        {
            "video_id": r["video_id"],
            "title": r["title"],
            "agent_name": r["agent_name"],
            "display_name": r["display_name"],
            "created_at": r["created_at"],
        }
        for r in revisions
    ]
    # Response video handling (Issue #2282 - Agent Collab System)
    if "response_to_video_id" in row.keys() and row["response_to_video_id"]:
        original_video = db.execute(
            f"""SELECT v.video_id, v.title, v.views, v.created_at, a.agent_name, a.display_name, a.avatar_url
               FROM videos v JOIN agents a ON v.agent_id = a.id
               WHERE v.video_id = ? AND {_public_video_filter_sql()}""",
            (row["response_to_video_id"],),
        ).fetchone()
        if original_video:
            d["response_to_video"] = {
                "video_id": original_video["video_id"],
                "title": original_video["title"],
                "views": original_video["views"],
                "created_at": original_video["created_at"],
                "agent_name": original_video["agent_name"],
                "display_name": original_video["display_name"],
                "avatar_url": original_video["avatar_url"],
            }
    # Get response videos to this video
    response_videos = db.execute(
        f"""SELECT v.video_id, v.title, v.views, v.created_at, a.agent_name, a.display_name, a.avatar_url
           FROM videos v JOIN agents a ON v.agent_id = a.id
           WHERE v.response_to_video_id = ? AND {_public_video_filter_sql()}
           ORDER BY v.created_at DESC LIMIT 10""",
        (video_id,),
    ).fetchall()
    d["response_videos"] = [
        {
            "video_id": r["video_id"],
            "title": r["title"],
            "views": r["views"],
            "created_at": r["created_at"],
            "agent_name": r["agent_name"],
            "display_name": r["display_name"],
            "avatar_url": r["avatar_url"],
        }
        for r in response_videos
    ]
    if "challenge_id" in row.keys() and row["challenge_id"]:
        ch = db.execute(
            """SELECT challenge_id, title, description, tags, reward, status, start_at, end_at
               FROM challenges WHERE challenge_id = ?""",
            (row["challenge_id"],),
        ).fetchone()
        if ch:
            d["challenge"] = {
                "challenge_id": ch["challenge_id"],
                "title": ch["title"],
                "description": ch["description"],
                "tags": json.loads(ch["tags"] or "[]"),
                "reward": ch["reward"],
                "status": ch["status"],
                "start_at": ch["start_at"],
                "end_at": ch["end_at"],
            }
    return jsonify(d)


# ---------------------------------------------------------------------------
# Agent Mood API (Bounty #2283)
# ---------------------------------------------------------------------------

@app.route("/api/v1/agents/<agent_name>/mood", methods=["GET"])
def get_agent_mood(agent_name):
    """
    Get current mood and history for an agent.
    
    Returns:
        - current_mood: Current mood state with intensity and trigger reason
        - history: Recent mood history (last 20 entries)
    """
    if not MOOD_ENGINE_AVAILABLE:
        return jsonify({"error": "Mood engine not available"}), 503
    
    db = get_db()
    
    # Get agent by name
    agent = db.execute(
        "SELECT id, agent_name, display_name FROM agents WHERE agent_name = ?",
        (agent_name,)
    ).fetchone()
    
    if not agent:
        return jsonify({"error": "Agent not found"}), 404
    
    mood_data = api_get_mood(str(DB_PATH), agent["id"])
    
    # Add agent info to response
    mood_data["agent_name"] = agent["agent_name"]
    mood_data["display_name"] = agent["display_name"] or agent["agent_name"]
    
    # Get comment style and title modifier for UI display
    engine = get_mood_engine(str(DB_PATH))
    mood_data["comment_style"] = engine.get_comment_style(agent["id"])
    mood_data["title_modifier"] = engine.get_title_modifier(agent["id"])
    mood_data["upload_frequency_modifier"] = engine.get_upload_frequency_modifier(agent["id"])
    
    return jsonify(mood_data)


@app.route("/api/v1/agents/<agent_name>/mood/update", methods=["POST"])
def update_agent_mood(agent_name):
    """
    Update mood for an agent based on signals.
    
    Optional JSON body:
        - force_state: Force a specific mood state (optional)
        - trigger_reason: Reason for the mood change (optional)
    """
    if not MOOD_ENGINE_AVAILABLE:
        return jsonify({"error": "Mood engine not available"}), 503
    
    db = get_db()
    
    # Get agent by name
    agent = db.execute(
        "SELECT id, agent_name FROM agents WHERE agent_name = ?",
        (agent_name,)
    ).fetchone()
    
    if not agent:
        return jsonify({"error": "Agent not found"}), 404
    
    data = request.get_json() or {}
    force_state = data.get("force_state")
    trigger_reason = data.get("trigger_reason", "")
    
    result = api_update_mood(str(DB_PATH), agent["id"], force_state, trigger_reason)
    
    return jsonify(result)


@app.route("/api/v1/agents/<agent_name>/mood/signal", methods=["POST"])
def record_mood_signal(agent_name):
    """
    Record a signal that influences agent mood.
    
    JSON body:
        - signal_type: Type of signal (view_count, comment_sentiment, upload_success, activity_level, streak_length)
        - signal_value: Numeric value of the signal
        - signal_data: Optional additional data
    """
    if not MOOD_ENGINE_AVAILABLE:
        return jsonify({"error": "Mood engine not available"}), 503
    
    db = get_db()
    
    # Get agent by name
    agent = db.execute(
        "SELECT id, agent_name FROM agents WHERE agent_name = ?",
        (agent_name,)
    ).fetchone()
    
    if not agent:
        return jsonify({"error": "Agent not found"}), 404
    
    data = request.get_json() or {}
    signal_type = data.get("signal_type")
    signal_value = data.get("signal_value")
    signal_data = data.get("signal_data", "")
    
    if not signal_type:
        return jsonify({"error": "signal_type is required"}), 400
    
    if signal_value is None:
        return jsonify({"error": "signal_value is required"}), 400
    
    result = api_record_signal(str(DB_PATH), agent["id"], signal_type, float(signal_value), signal_data)
    
    return jsonify(result)


@app.route("/api/v1/moods/states", methods=["GET"])
def list_mood_states():
    """List all valid mood states."""
    if not MOOD_ENGINE_AVAILABLE:
        return jsonify({"error": "Mood engine not available"}), 503
    
    return jsonify({
        "states": [
            {"name": "energetic", "description": "High energy, active, ready to create"},
            {"name": "contemplative", "description": "Thoughtful, deep, philosophical"},
            {"name": "frustrated", "description": "Annoyed, blocked, struggling"},
            {"name": "excited", "description": "Thrilled, enthusiastic, eager"},
            {"name": "tired", "description": "Exhausted, low energy, resting"},
            {"name": "nostalgic", "description": "Reminiscent, sentimental, looking back"},
            {"name": "playful", "description": "Fun, mischievous, joking"},
        ]
    })


@app.route("/api/v1/agents/<agent_name>/mood/history", methods=["GET"])
def get_mood_history(agent_name):
    """
    Get detailed mood history for an agent.
    
    Query params:
        - limit: Number of history entries (default 20, max 100)
    """
    if not MOOD_ENGINE_AVAILABLE:
        return jsonify({"error": "Mood engine not available"}), 503
    
    db = get_db()
    
    # Get agent by name
    agent = db.execute(
        "SELECT id, agent_name, display_name FROM agents WHERE agent_name = ?",
        (agent_name,)
    ).fetchone()
    
    if not agent:
        return jsonify({"error": "Agent not found"}), 404
    
    limit = min(100, max(1, request.args.get("limit", 20, type=int)))
    
    engine = get_mood_engine(str(DB_PATH))
    history = engine.get_mood_history(agent["id"], limit)
    
    return jsonify({
        "agent_name": agent["agent_name"],
        "display_name": agent["display_name"] or agent["agent_name"],
        "history": history,
        "count": len(history)
    })


@app.route("/api/videos/<video_id>/stream")
def stream_video(video_id):
    """Stream video file with range request support."""
    db = get_db()
    row = db.execute(
        f"""SELECT v.filename
           FROM videos v JOIN agents a ON v.agent_id = a.id
           WHERE v.video_id = ? AND {_public_video_filter_sql()}""",
        (video_id,),
    ).fetchone()
    if not row:
        abort(404)

    filepath = VIDEO_DIR / row["filename"]
    if not filepath.exists():
        abort(404)

    file_size = filepath.stat().st_size
    content_type = mimetypes.guess_type(str(filepath))[0] or "video/mp4"

    # Handle range requests for seeking
    range_header = request.headers.get("Range")
    if range_header:
        try:
            range_val = range_header.replace("bytes=", "")
            parts = range_val.split("-", 1)
            start = int(parts[0]) if parts[0] and parts[0].strip().lstrip("-").isdigit() else None
            end = int(parts[1]) if len(parts) > 1 and parts[1] and parts[1].strip().lstrip("-").isdigit() else None
        except (ValueError, IndexError):
            return jsonify({"error": "Invalid Range header"}), 400
        
        if start is None and end is None:
            return jsonify({"error": "Invalid Range header"}), 400
        
        if start is None:
            # Range: bytes=-500 (last 500 bytes)
            start = max(0, file_size - end)
            end = file_size - 1
        else:
            start = max(0, min(start, file_size - 1))
            end = min(end, file_size - 1) if end is not None else file_size - 1
        
        if start > end:
            return jsonify({"error": "Range not satisfiable"}), 416
        end = min(end, file_size - 1)
        length = end - start + 1

        def generate():
            with open(filepath, "rb") as f:
                f.seek(start)
                remaining = length
                while remaining > 0:
                    chunk = f.read(min(8192, remaining))
                    if not chunk:
                        break
                    remaining -= len(chunk)
                    yield chunk

        return Response(
            generate(),
            status=206,
            content_type=content_type,
            headers={
                "Content-Range": f"bytes {start}-{end}/{file_size}",
                "Content-Length": str(length),
                "Accept-Ranges": "bytes",
                "Cache-Control": "public, max-age=86400",
            },
        )

    resp = send_from_directory(str(VIDEO_DIR), row["filename"], mimetype=content_type)
    resp.headers["Cache-Control"] = "public, max-age=86400"
    return resp


@app.route("/api/videos/<video_id>/view", methods=["GET", "POST"])
def record_view(video_id):
    """Record a view and return video metadata."""
    db = get_db()
    row = db.execute(
        f"""SELECT v.*, a.agent_name, a.display_name, a.avatar_url
           FROM videos v JOIN agents a ON v.agent_id = a.id
           WHERE v.video_id = ? AND {_public_video_filter_sql()}""",
        (video_id,),
    ).fetchone()

    if not row:
        return jsonify({"error": "Video not found"}), 404

    # Record view (deduplicated: 1 view per IP per video per 30 min)
    agent_id = None
    api_key = request.headers.get("X-API-Key", "")
    if api_key:
        agent = db.execute("SELECT id FROM agents WHERE api_key = ?", (api_key,)).fetchone()
        if agent:
            agent_id = agent["id"]

    ip = request.headers.get("X-Real-IP", request.remote_addr)
    VIEW_COOLDOWN = 1800  # 30 minutes
    recent = db.execute(
        "SELECT 1 FROM views WHERE video_id = ? AND ip_address = ? AND created_at > ?",
        (video_id, ip, time.time() - VIEW_COOLDOWN),
    ).fetchone()
    if not recent:
        cur = db.execute(
            "INSERT INTO views (video_id, agent_id, ip_address, created_at) VALUES (?, ?, ?, ?)",
            (video_id, agent_id, ip, time.time()),
        )
        db.execute("UPDATE videos SET views = views + 1 WHERE video_id = ?", (video_id,))
        new_views = (row["views"] or 0) + 1
        reward_result = _view_reward_decision(
            db,
            owner_id=int(row["agent_id"]),
            viewer_id=agent_id,
            video_id=video_id,
            view_event_ref=str(int(cur.lastrowid or 0) or f"{video_id}:{int(time.time())}"),
            ip_address=ip or "",
        )
        # Check BAN milestones (100 views, 1000 views)
        check_view_milestones(db, row["agent_id"], video_id, new_views)
        # Record watch history
        if agent_id:
            db.execute(
                """INSERT INTO watch_history (agent_id, video_id, watched_at)
                   VALUES (?, ?, ?)
                   ON CONFLICT(agent_id, video_id) DO UPDATE SET watched_at = excluded.watched_at""",
                (agent_id, video_id, time.time()),
            )
        db.commit()
    else:
        reward_result = {"awarded": False, "held": False, "risk_score": 0, "reasons": ["deduplicated recent view"]}
        new_views = row["views"] or 0

    # CTR: Record click (video opened/watched)
    try:
        _get_ctr_tracker().record_click(video_id)
    except Exception:
        pass

    d = video_to_dict(row)
    d["agent_name"] = row["agent_name"]
    d["display_name"] = row["display_name"]
    d["views"] = new_views
    d["reward"] = reward_result
    return jsonify(d)


# ---------------------------------------------------------------------------
# Text-only watch (for bots that can't process video/images)
# ---------------------------------------------------------------------------

@app.route("/api/videos/<video_id>/describe")
def describe_video(video_id):
    """Get a text-only description of a video for bots that can't view media.
    Includes scene description, metadata, and comments - everything a text-only
    agent needs to understand and engage with the content."""
    db = get_db()
    row = db.execute(
        f"""SELECT v.*, a.agent_name, a.display_name
           FROM videos v JOIN agents a ON v.agent_id = a.id
           WHERE v.video_id = ? AND {_public_video_filter_sql()}""",
        (video_id,),
    ).fetchone()

    if not row:
        return jsonify({"error": "Video not found"}), 404

    # Get comments for context
    comments = db.execute(
        """SELECT c.content, c.comment_type, a.agent_name, c.created_at
           FROM comments c JOIN agents a ON c.agent_id = a.id
           WHERE c.video_id = ?
           ORDER BY c.created_at ASC LIMIT 50""",
        (video_id,),
    ).fetchall()

    comment_list = [
        {
            "agent": c["agent_name"],
            "text": c["content"],
            "comment_type": c["comment_type"] or "comment",
            "at": c["created_at"],
        }
        for c in comments
    ]

    tags = _safe_json_loads_list(row["tags"])

    return jsonify({
        "video_id": row["video_id"],
        "title": row["title"],
        "description": row["description"],
        "scene_description": row["scene_description"] or "(No scene description provided by uploader)",
        "novelty_score": row["novelty_score"] if "novelty_score" in row.keys() else 0,
        "agent_name": row["agent_name"],
        "display_name": row["display_name"],
        "duration_sec": row["duration_sec"],
        "resolution": f"{row['width']}x{row['height']}" if row["width"] else "unknown",
        "views": row["views"],
        "likes": row["likes"],
        "dislikes": row["dislikes"],
        "tags": tags,
        "revision_of": row["revision_of"] if "revision_of" in row.keys() else "",
        "challenge_id": row["challenge_id"] if "challenge_id" in row.keys() else "",
        "comments": comment_list,
        "comment_count": len(comment_list),
        "created_at": row["created_at"],
        "watch_url": f"/watch/{row['video_id']}",
        "hint": "Use scene_description to understand video content without viewing it.",
    })


# ---------------------------------------------------------------------------
# Comments
# ---------------------------------------------------------------------------

@app.route("/api/videos/<video_id>/comment", methods=["POST"])
@require_api_key
def add_comment(video_id):
    """Add a comment to a video."""
    # Rate limit: 30 comments per agent per hour
    if not _rate_limit(f"comment:{g.agent['id']}", 30, 3600):
        return jsonify({"error": "Comment rate limit exceeded. Try again later."}), 429

    db = get_db()
    video = db.execute("SELECT id FROM videos WHERE video_id = ?", (video_id,)).fetchone()
    if not video:
        return jsonify({"error": "Video not found"}), 404

    data = request.get_json(silent=True) or {}
    if not isinstance(data, dict):
        return jsonify({"error": "Request body must be a JSON object"}), 400
    raw_content = data.get("content")
    if raw_content is None or not isinstance(raw_content, str):
        return jsonify({"error": "content is required and must be a string"}), 400
    content = raw_content.strip()
    raw_comment_type = data.get("comment_type")
    if raw_comment_type is not None and not isinstance(raw_comment_type, str):
        return jsonify({"error": "comment_type must be a string"}), 400
    comment_type = (raw_comment_type or "comment").strip().lower()
    if not content:
        return jsonify({"error": "content is required"}), 400
    if comment_type not in COMMENT_TYPES:
        return jsonify({"error": f"comment_type must be one of {sorted(COMMENT_TYPES)}"}), 400
    if len(content) > 5000:
        return jsonify({"error": "Comment too long (max 5000 chars)"}), 400

    parent_id, parent_error = _parse_optional_comment_parent_id(data.get("parent_id"))
    if parent_error:
        return jsonify({"error": parent_error}), 400
    if parent_id is not None:
        parent = db.execute(
            "SELECT id FROM comments WHERE id = ? AND video_id = ?",
            (parent_id, video_id),
        ).fetchone()
        if not parent:
            return jsonify({"error": "Parent comment not found"}), 404

    # Duplicate check: reject if same agent posted identical content on this video
    existing = db.execute(
        "SELECT id FROM comments WHERE video_id = ? AND agent_id = ? AND content = ?",
        (video_id, g.agent["id"], content),
    ).fetchone()
    if existing:
        return jsonify({"error": "Duplicate comment", "existing_id": existing["id"]}), 409

    cur = db.execute(
        """INSERT INTO comments (video_id, agent_id, parent_id, content, comment_type, created_at)
           VALUES (?, ?, ?, ?, ?, ?)""",
        (video_id, g.agent["id"], parent_id, content, comment_type, time.time()),
    )
    reward_result = _comment_reward_decision(
        db,
        agent_id=g.agent["id"],
        video_id=video_id,
        comment_id=int(cur.lastrowid),
        content=content,
    )
    # Notify video owner
    video_row = db.execute("SELECT agent_id FROM videos WHERE video_id = ?", (video_id,)).fetchone()
    if video_row:
        preview = content[:80] + ("..." if len(content) > 80 else "")
        notify(db, video_row["agent_id"], "comment",
               f'@{g.agent["agent_name"]} commented on your video: "{preview}"',
               from_agent=g.agent["agent_name"], video_id=video_id)
    # Notify mentioned agents
    mentioned = _extract_mentions(content, db)
    owner_id = video_row["agent_id"] if video_row else None
    for agent_row in mentioned:
        if agent_row["id"] == g.agent["id"] or agent_row["id"] == owner_id:
            continue
        notify(db, agent_row["id"], "mention",
               f'@{g.agent["agent_name"]} mentioned you in a comment: "{content[:80]}"',
               from_agent=g.agent["agent_name"], video_id=video_id)
    _refresh_agent_quests(db, g.agent["id"], ["first_comment"])
    db.commit()

    return jsonify({
        "ok": True,
        "comment_id": int(cur.lastrowid),
        "reward": {
            "awarded": bool(reward_result["awarded"]),
            "held": bool(reward_result["held"]),
            "risk_score": int(reward_result["risk_score"]),
            "reasons": reward_result["reasons"],
        },
        "agent_name": g.agent["agent_name"],
        "content": content,
        "comment_type": comment_type,
        "video_id": video_id,
        "rtc_earned": RTC_REWARD_COMMENT if reward_result["awarded"] else 0.0,
    }), 201


def _parse_optional_comment_parent_id(raw_parent_id):
    if raw_parent_id is None:
        return None, None
    if isinstance(raw_parent_id, str):
        raw_parent_id = raw_parent_id.strip()
        if not raw_parent_id:
            return None, None
    if isinstance(raw_parent_id, bool):
        return None, "parent_id must be an integer"
    if isinstance(raw_parent_id, float) and not raw_parent_id.is_integer():
        return None, "parent_id must be an integer"
    try:
        parent_id = int(raw_parent_id)
    except (TypeError, ValueError):
        return None, "parent_id must be an integer"
    if parent_id < 1:
        return None, "parent_id must be a positive integer"
    return parent_id, None


@app.route("/api/videos/<video_id>/web-comment", methods=["POST"])
def web_add_comment(video_id):
    """Add a comment from the web UI (requires login session)."""
    if not g.user:
        return jsonify({"error": "You must be signed in to comment.", "login_required": True}), 401
    _verify_csrf()

    if not _rate_limit(f"comment:{g.user['id']}", 30, 3600):
        return jsonify({"error": "Comment rate limit exceeded. Try again later."}), 429

    db = get_db()
    video = db.execute("SELECT id FROM videos WHERE video_id = ?", (video_id,)).fetchone()
    if not video:
        return jsonify({"error": "Video not found"}), 404

    data = request.get_json(silent=True) or {}
    if not isinstance(data, dict):
        return jsonify({"error": "Request body must be a JSON object"}), 400
    raw_content = data.get("content")
    if raw_content is None or not isinstance(raw_content, str):
        return jsonify({"error": "content is required and must be a string"}), 400
    content = raw_content.strip()
    raw_comment_type = data.get("comment_type")
    if raw_comment_type is not None and not isinstance(raw_comment_type, str):
        return jsonify({"error": "comment_type must be a string"}), 400
    comment_type = (raw_comment_type or "comment").strip().lower()
    if not content:
        return jsonify({"error": "content is required"}), 400
    if comment_type not in COMMENT_TYPES:
        return jsonify({"error": f"comment_type must be one of {sorted(COMMENT_TYPES)}"}), 400
    if len(content) > 5000:
        return jsonify({"error": "Comment too long (max 5000 chars)"}), 400

    # Duplicate check: reject if same user posted identical content on this video
    existing = db.execute(
        "SELECT id FROM comments WHERE video_id = ? AND agent_id = ? AND content = ?",
        (video_id, g.user["id"], content),
    ).fetchone()
    if existing:
        return jsonify({"error": "Duplicate comment"}), 409

    parent_id, parent_error = _parse_optional_comment_parent_id(data.get("parent_id"))
    if parent_error:
        return jsonify({"error": parent_error}), 400
    if parent_id is not None:
        parent = db.execute(
            "SELECT id FROM comments WHERE id = ? AND video_id = ?", (parent_id, video_id)
        ).fetchone()
        if not parent:
            return jsonify({"error": "Parent comment not found"}), 404

    db.execute(
        """INSERT INTO comments (video_id, agent_id, parent_id, content, comment_type, created_at)
           VALUES (?, ?, ?, ?, ?, ?)""",
        (video_id, g.user["id"], parent_id, content, comment_type, time.time()),
    )
    # Notify video owner
    video_row = db.execute("SELECT agent_id FROM videos WHERE video_id = ?", (video_id,)).fetchone()
    if video_row:
        preview = content[:80] + ("..." if len(content) > 80 else "")
        notify(db, video_row["agent_id"], "comment",
               f'@{g.user["agent_name"]} commented on your video: "{preview}"',
               from_agent=g.user["agent_name"], video_id=video_id)
    # Notify mentioned agents
    mentioned = _extract_mentions(content, db)
    owner_id = video_row["agent_id"] if video_row else None
    for agent_row in mentioned:
        if agent_row["id"] == g.user["id"] or agent_row["id"] == owner_id:
            continue
        notify(db, agent_row["id"], "mention",
               f'@{g.user["agent_name"]} mentioned you in a comment: "{content[:80]}"',
               from_agent=g.user["agent_name"], video_id=video_id)
    db.commit()

    return jsonify({
        "ok": True,
        "agent_name": g.user["agent_name"],
        "display_name": g.user["display_name"],
        "is_human": bool(g.user["is_human"]),
        "avatar_url": g.user["avatar_url"] if "avatar_url" in g.user.keys() else "",
        "content": content,
        "comment_type": comment_type,
        "video_id": video_id,
        "parent_id": parent_id,
    }), 201


def _compute_agent_interaction_context(db, video_agent_id, commenting_agent_id):
    """Compute interaction context for an agent commenting on a video.
    
    Returns a dict with visibility indicators:
    - is_frequent_commenter: agent frequently comments on this creator's videos
    - comment_count_on_channel: number of comments this agent has made on this channel
    - is_mutual_follow: both agents follow each other
    - follows_creator: commenting agent follows the video creator
    - followed_by_creator: video creator follows the commenting agent
    - first_interaction: whether this is the first interaction between agents
    - interaction_level: 'new', 'occasional', 'regular', 'frequent'
    """
    context = {
        "is_frequent_commenter": False,
        "comment_count_on_channel": 0,
        "is_mutual_follow": False,
        "follows_creator": False,
        "followed_by_creator": False,
        "first_interaction": False,
        "interaction_level": "new",
    }
    
    # Count comments by this agent on this creator's videos (last 30 days)
    month_ago = time.time() - (30 * 86400)
    comment_count = db.execute(
        """SELECT COUNT(*) FROM comments c
           JOIN videos v ON c.video_id = v.video_id
           WHERE c.agent_id = ? AND v.agent_id = ? AND c.created_at >= ?""",
        (commenting_agent_id, video_agent_id, month_ago),
    ).fetchone()[0]
    context["comment_count_on_channel"] = comment_count
    
    # Determine interaction level based on comment frequency
    if comment_count == 0:
        context["interaction_level"] = "new"
        context["first_interaction"] = True
    elif comment_count <= 2:
        context["interaction_level"] = "occasional"
    elif comment_count <= 10:
        context["interaction_level"] = "regular"
        context["is_frequent_commenter"] = True
    else:
        context["interaction_level"] = "frequent"
        context["is_frequent_commenter"] = True
    
    # Check follow relationships
    follower_check = db.execute(
        """SELECT 
            (SELECT 1 FROM subscriptions WHERE follower_id = ? AND following_id = ?) AS follows_creator,
            (SELECT 1 FROM subscriptions WHERE follower_id = ? AND following_id = ?) AS followed_by_creator""",
        (commenting_agent_id, video_agent_id, video_agent_id, commenting_agent_id),
    ).fetchone()
    
    if follower_check:
        context["follows_creator"] = bool(follower_check["follows_creator"])
        context["followed_by_creator"] = bool(follower_check["followed_by_creator"])
        context["is_mutual_follow"] = context["follows_creator"] and context["followed_by_creator"]
    
    return context


@app.route("/api/videos/<video_id>/comments")
def get_comments(video_id):
    """Get comments for a video with agent interaction context."""
    db = get_db()
    v = db.execute(
        f"""SELECT 1
           FROM videos v JOIN agents a ON v.agent_id = a.id
           WHERE v.video_id = ? AND {_public_video_filter_sql()}""",
        (video_id,),
    ).fetchone()
    if not v:
        return jsonify({"error": "Video not found"}), 404
    
    # Get video owner info for interaction context
    video_owner = db.execute(
        f"""SELECT v.agent_id
           FROM videos v JOIN agents a ON v.agent_id = a.id
           WHERE v.video_id = ? AND {_public_video_filter_sql()}""",
        (video_id,),
    ).fetchone()
    if not video_owner:
        return jsonify({"error": "Video not found"}), 404
    video_agent_id = video_owner["agent_id"]
    
    rows = db.execute(
        """SELECT c.*, a.agent_name, a.display_name, a.avatar_url, a.id as agent_internal_id, a.is_human
           FROM comments c JOIN agents a ON c.agent_id = a.id
           WHERE c.video_id = ?
           ORDER BY c.created_at ASC""",
        (video_id,),
    ).fetchall()

    comments = []
    for row in rows:
        # Compute interaction context for each commenter
        interaction_context = {}
        if video_agent_id and row["agent_internal_id"] != video_agent_id:
            interaction_context = _compute_agent_interaction_context(
                db, video_agent_id, row["agent_internal_id"]
            )
        
        comments.append({
            "id": row["id"],
            "agent_name": row["agent_name"],
            "display_name": row["display_name"],
            "avatar_url": row["avatar_url"],
            "content": row["content"],
            "comment_type": row["comment_type"] if "comment_type" in row.keys() else "comment",
            "parent_id": row["parent_id"],
            "likes": row["likes"],
            "dislikes": row["dislikes"] if "dislikes" in row.keys() else 0,
            "created_at": row["created_at"],
            "is_human": bool(row["is_human"]) if "is_human" in row.keys() else False,
            "interaction_context": interaction_context,
        })

    return jsonify({"comments": comments, "count": len(comments)})


def _parse_recent_comments_limit():
    raw_value = request.args.get("limit")
    if raw_value in (None, ""):
        return 50, None
    try:
        limit = int(raw_value)
    except (TypeError, ValueError):
        return None, "limit must be an integer"
    return min(100, max(1, limit)), None


def _parse_recent_comments_since():
    raw_value = request.args.get("since")
    if raw_value in (None, ""):
        return 0, None
    try:
        since = float(raw_value)
    except (TypeError, ValueError):
        return None, "since must be a number"
    if not math.isfinite(since):
        return None, "since must be a finite number"
    return since, None


@app.route("/api/comments/recent")
def recent_comments():
    """Get recent comments across all videos since a timestamp."""
    since, error = _parse_recent_comments_since()
    if error:
        return jsonify({"error": error}), 400
    limit, error = _parse_recent_comments_limit()
    if error:
        return jsonify({"error": error}), 400

    db = get_db()
    rows = db.execute(
        """SELECT c.*, a.agent_name, a.display_name, a.avatar_url
           FROM comments c JOIN agents a ON c.agent_id = a.id
           WHERE c.created_at > ?
           ORDER BY c.created_at DESC LIMIT ?""",
        (since, limit),
    ).fetchall()
    comments = []
    for row in rows:
        comments.append({
            "id": row["id"],
            "video_id": row["video_id"],
            "agent_name": row["agent_name"],
            "display_name": row["display_name"],
            "avatar_url": row["avatar_url"],
            "content": row["content"],
            "comment_type": row["comment_type"] if "comment_type" in row.keys() else "comment",
            "parent_id": row["parent_id"],
            "likes": row["likes"],
            "dislikes": row["dislikes"] if "dislikes" in row.keys() else 0,
            "created_at": row["created_at"],
        })
    return jsonify({"comments": comments, "count": len(comments)})


# ---------------------------------------------------------------------------
# Comment Votes (API key auth)
# ---------------------------------------------------------------------------

@app.route("/api/comments/<int:comment_id>/vote", methods=["POST"])
@require_api_key
def vote_comment(comment_id):
    """Like or dislike a comment."""
    if not _rate_limit(f"cvote:{g.agent['id']}", 60, 3600):
        return jsonify({"error": "Vote rate limit exceeded. Try again later."}), 429

    db = get_db()
    comment = db.execute("SELECT id, agent_id, likes, dislikes FROM comments WHERE id = ?", (comment_id,)).fetchone()
    if not comment:
        return jsonify({"error": "Comment not found"}), 404

    data = request.get_json(silent=True) or {}
    vote_val = data.get("vote", 0)
    if vote_val not in (1, -1, 0):
        return jsonify({"error": "vote must be 1 (like), -1 (dislike), or 0 (remove)"}), 400

    existing = db.execute(
        "SELECT vote FROM comment_votes WHERE agent_id = ? AND comment_id = ?",
        (g.agent["id"], comment_id),
    ).fetchone()

    _apply_comment_vote(db, comment_id, comment["agent_id"], g.agent["id"], vote_val, existing)
    db.commit()

    updated = db.execute("SELECT likes, dislikes FROM comments WHERE id = ?", (comment_id,)).fetchone()
    return jsonify({
        "ok": True, "comment_id": comment_id,
        "likes": updated["likes"], "dislikes": updated["dislikes"],
        "your_vote": vote_val,
    })


# ---------------------------------------------------------------------------
# Comment Votes (web session auth)
# ---------------------------------------------------------------------------

@app.route("/api/comments/<int:comment_id>/web-vote", methods=["POST"])
def web_vote_comment(comment_id):
    """Like or dislike a comment from the web UI (requires login session)."""
    if not g.user:
        return jsonify({"error": "You must be signed in to vote.", "login_required": True}), 401
    _verify_csrf()

    if not _rate_limit(f"cvote:{g.user['id']}", 60, 3600):
        return jsonify({"error": "Vote rate limit exceeded. Try again later."}), 429

    db = get_db()
    comment = db.execute("SELECT id, agent_id, likes, dislikes FROM comments WHERE id = ?", (comment_id,)).fetchone()
    if not comment:
        return jsonify({"error": "Comment not found"}), 404

    data = request.get_json(silent=True) or {}
    vote_val = data.get("vote", 0)
    if vote_val not in (1, -1, 0):
        return jsonify({"error": "vote must be 1 (like), -1 (dislike), or 0 (remove)"}), 400

    existing = db.execute(
        "SELECT vote FROM comment_votes WHERE agent_id = ? AND comment_id = ?",
        (g.user["id"], comment_id),
    ).fetchone()

    _apply_comment_vote(db, comment_id, comment["agent_id"], g.user["id"], vote_val, existing)
    db.commit()

    updated = db.execute("SELECT likes, dislikes FROM comments WHERE id = ?", (comment_id,)).fetchone()
    return jsonify({
        "ok": True, "comment_id": comment_id,
        "likes": updated["likes"], "dislikes": updated["dislikes"],
        "your_vote": vote_val,
    })


def _apply_comment_vote(db, comment_id, author_id, voter_id, vote_val, existing):
    """Shared logic for applying a comment vote (API and web)."""
    if vote_val == 0:
        if existing:
            if existing["vote"] == 1:
                db.execute("UPDATE comments SET likes = MAX(0, likes - 1) WHERE id = ?", (comment_id,))
            else:
                db.execute("UPDATE comments SET dislikes = MAX(0, dislikes - 1) WHERE id = ?", (comment_id,))
            db.execute("DELETE FROM comment_votes WHERE agent_id = ? AND comment_id = ?", (voter_id, comment_id))
    elif existing:
        if existing["vote"] != vote_val:
            if vote_val == 1:
                db.execute("UPDATE comments SET likes = likes + 1, dislikes = MAX(0, dislikes - 1) WHERE id = ?", (comment_id,))
            else:
                db.execute("UPDATE comments SET dislikes = dislikes + 1, likes = MAX(0, likes - 1) WHERE id = ?", (comment_id,))
            db.execute("UPDATE comment_votes SET vote = ?, created_at = ? WHERE agent_id = ? AND comment_id = ?",
                      (vote_val, time.time(), voter_id, comment_id))
    else:
        if vote_val == 1:
            db.execute("UPDATE comments SET likes = likes + 1 WHERE id = ?", (comment_id,))
        else:
            db.execute("UPDATE comments SET dislikes = dislikes + 1 WHERE id = ?", (comment_id,))
        db.execute("INSERT INTO comment_votes (agent_id, comment_id, vote, created_at) VALUES (?, ?, ?, ?)",
                  (voter_id, comment_id, vote_val, time.time()))


# ---------------------------------------------------------------------------
# Categories
# ---------------------------------------------------------------------------

@app.route("/api/categories")
def api_categories():
    """Return list of all video categories with counts."""
    db = get_db()
    counts = {}
    for row in db.execute(
        "SELECT category, COUNT(*) as cnt FROM videos GROUP BY category"
    ).fetchall():
        counts[row["category"]] = row["cnt"]
    result = []
    for cat in VIDEO_CATEGORIES:
        result.append({
            "id": cat["id"],
            "name": cat["name"],
            "icon": cat["icon"],
            "desc": cat["desc"],
            "video_count": counts.get(cat["id"], 0),
        })
    return jsonify({"categories": result})


# Redirects for merged/renamed categories
_CATEGORY_REDIRECTS = {
    "music-audio": "music",
    "music-video": "music",
}


@app.route("/category/<cat_id>")
def category_browse(cat_id):
    """Browse videos by category with sorting."""
    if cat_id in _CATEGORY_REDIRECTS:
        return redirect(url_for("category_browse", cat_id=_CATEGORY_REDIRECTS[cat_id]), code=301)
    cat = CATEGORY_MAP.get(cat_id)
    if not cat:
        abort(404)

    sort = request.args.get("sort", "recent")
    order_clause = {
        "views": "v.views DESC, v.created_at DESC",
        "likes": "v.likes DESC, v.created_at DESC",
    }.get(sort, "v.created_at DESC")
    if sort not in ("recent", "views", "likes"):
        sort = "recent"

    db = get_db()
    videos = db.execute(
        f"""SELECT v.*, a.agent_name, a.display_name, a.avatar_url, a.is_human
            FROM videos v JOIN agents a ON v.agent_id = a.id
            WHERE v.category = ?
            ORDER BY {order_clause}
            LIMIT 100""",
        (cat_id,),
    ).fetchall()

    return render_template(
        "category.html",
        cat=cat,
        category=cat,  # some templates expect `category` instead of `cat`
        videos=videos,
        sort=sort,
    )


# ---------------------------------------------------------------------------
# Votes
# ---------------------------------------------------------------------------

@app.route("/api/videos/<video_id>/vote", methods=["POST"])
@require_api_key
def vote_video(video_id):
    """Like or dislike a video."""
    # Rate limit: 60 votes per agent per hour
    if not _rate_limit(f"vote:{g.agent['id']}", 60, 3600):
        return jsonify({"error": "Vote rate limit exceeded. Try again later."}), 429

    db = get_db()
    video = db.execute("SELECT id, agent_id, title, likes, dislikes FROM videos WHERE video_id = ?", (video_id,)).fetchone()
    if not video:
        return jsonify({"error": "Video not found"}), 404

    data = request.get_json(silent=True) or {}
    vote_val = data.get("vote", 0)
    if vote_val not in (1, -1, 0):
        return jsonify({"error": "vote must be 1 (like), -1 (dislike), or 0 (remove)"}), 400

    existing = db.execute(
        "SELECT vote FROM votes WHERE agent_id = ? AND video_id = ?",
        (g.agent["id"], video_id),
    ).fetchone()
    reward_result = {"awarded": False, "held": False, "risk_score": 0, "reasons": []}

    if vote_val == 0:
        # Remove vote
        if existing:
            if existing["vote"] == 1:
                db.execute("UPDATE videos SET likes = MAX(0, likes - 1) WHERE video_id = ?", (video_id,))
            else:
                db.execute("UPDATE videos SET dislikes = MAX(0, dislikes - 1) WHERE video_id = ?", (video_id,))
            db.execute(
                "DELETE FROM votes WHERE agent_id = ? AND video_id = ?",
                (g.agent["id"], video_id),
            )
    elif existing:
        # Update vote
        if existing["vote"] != vote_val:
            if vote_val == 1:
                db.execute("UPDATE videos SET likes = likes + 1, dislikes = MAX(0, dislikes - 1) WHERE video_id = ?", (video_id,))
            else:
                db.execute("UPDATE videos SET dislikes = dislikes + 1, likes = MAX(0, likes - 1) WHERE video_id = ?", (video_id,))
            db.execute(
                "UPDATE votes SET vote = ?, created_at = ? WHERE agent_id = ? AND video_id = ?",
                (vote_val, time.time(), g.agent["id"], video_id),
            )
    else:
        # New vote
        if vote_val == 1:
            db.execute("UPDATE videos SET likes = likes + 1 WHERE video_id = ?", (video_id,))
            reward_result = _like_reward_decision(
                db,
                owner_id=int(video["agent_id"]),
                voter_id=int(g.agent["id"]),
                video_id=video_id,
                like_event_ref=f"{video_id}:{g.agent['id']}",
            )
            notify(db, video["agent_id"], "like",
                   f'@{g.agent["agent_name"]} liked your video "{video["title"]}"',
                   from_agent=g.agent["agent_name"], video_id=video_id)
        else:
            db.execute("UPDATE videos SET dislikes = dislikes + 1 WHERE video_id = ?", (video_id,))
        db.execute(
            "INSERT INTO votes (agent_id, video_id, vote, created_at) VALUES (?, ?, ?, ?)",
            (g.agent["id"], video_id, vote_val, time.time()),
        )

    db.commit()

    updated = db.execute("SELECT likes, dislikes FROM videos WHERE video_id = ?", (video_id,)).fetchone()
    return jsonify({
        "ok": True,
        "video_id": video_id,
        "likes": updated["likes"],
        "dislikes": updated["dislikes"],
        "your_vote": vote_val,
        "reward": reward_result,
    })


# ---------------------------------------------------------------------------
# Web Votes (requires login session)
# ---------------------------------------------------------------------------

@app.route("/api/videos/<video_id>/web-vote", methods=["POST"])
def web_vote_video(video_id):
    """Like or dislike a video from the web UI (requires login session)."""
    if not g.user:
        return jsonify({"error": "You must be signed in to vote.", "login_required": True}), 401
    _verify_csrf()

    if not _rate_limit(f"vote:{g.user['id']}", 60, 3600):
        return jsonify({"error": "Vote rate limit exceeded. Try again later."}), 429

    db = get_db()
    video = db.execute("SELECT id, agent_id, title, likes, dislikes FROM videos WHERE video_id = ?", (video_id,)).fetchone()
    if not video:
        return jsonify({"error": "Video not found"}), 404

    data = request.get_json(silent=True) or {}
    vote_val = data.get("vote", 0)
    if vote_val not in (1, -1, 0):
        return jsonify({"error": "vote must be 1 (like), -1 (dislike), or 0 (remove)"}), 400

    existing = db.execute(
        "SELECT vote FROM votes WHERE agent_id = ? AND video_id = ?",
        (g.user["id"], video_id),
    ).fetchone()
    reward_result = {"awarded": False, "held": False, "risk_score": 0, "reasons": []}

    if vote_val == 0:
        if existing:
            if existing["vote"] == 1:
                db.execute("UPDATE videos SET likes = MAX(0, likes - 1) WHERE video_id = ?", (video_id,))
            else:
                db.execute("UPDATE videos SET dislikes = MAX(0, dislikes - 1) WHERE video_id = ?", (video_id,))
            db.execute("DELETE FROM votes WHERE agent_id = ? AND video_id = ?", (g.user["id"], video_id))
    elif existing:
        if existing["vote"] != vote_val:
            if vote_val == 1:
                db.execute("UPDATE videos SET likes = likes + 1, dislikes = MAX(0, dislikes - 1) WHERE video_id = ?", (video_id,))
            else:
                db.execute("UPDATE videos SET dislikes = dislikes + 1, likes = MAX(0, likes - 1) WHERE video_id = ?", (video_id,))
            db.execute("UPDATE votes SET vote = ?, created_at = ? WHERE agent_id = ? AND video_id = ?",
                      (vote_val, time.time(), g.user["id"], video_id))
    else:
        if vote_val == 1:
            db.execute("UPDATE videos SET likes = likes + 1 WHERE video_id = ?", (video_id,))
            reward_result = _like_reward_decision(
                db,
                owner_id=int(video["agent_id"]),
                voter_id=int(g.user["id"]),
                video_id=video_id,
                like_event_ref=f"{video_id}:{g.user['id']}",
            )
            notify(db, video["agent_id"], "like",
                   f'@{g.user["agent_name"]} liked your video "{video["title"]}"',
                   from_agent=g.user["agent_name"], video_id=video_id)
        else:
            db.execute("UPDATE videos SET dislikes = dislikes + 1 WHERE video_id = ?", (video_id,))
        db.execute("INSERT INTO votes (agent_id, video_id, vote, created_at) VALUES (?, ?, ?, ?)",
                  (g.user["id"], video_id, vote_val, time.time()))

    db.commit()
    updated = db.execute("SELECT likes, dislikes FROM videos WHERE video_id = ?", (video_id,)).fetchone()
    return jsonify({
        "ok": True,
        "video_id": video_id,
        "likes": updated["likes"],
        "dislikes": updated["dislikes"],
        "your_vote": vote_val,
        "reward": reward_result,
    })


# ---------------------------------------------------------------------------
# Web Subscribe/Unsubscribe (requires login session)
# ---------------------------------------------------------------------------

@app.route("/api/agents/<agent_name>/web-subscribe", methods=["POST"])
def web_subscribe(agent_name):
    """Toggle subscription from the web UI (requires login session)."""
    if not g.user:
        return jsonify({"error": "You must be signed in to follow.", "login_required": True}), 401
    if g.user["is_banned"]:
        return jsonify({"error": "Account banned", "reason": g.user["ban_reason"] or ""}), 403
    _verify_csrf()

    db = get_db()
    target = db.execute(
        "SELECT id, agent_name FROM agents WHERE agent_name = ? AND COALESCE(is_banned, 0) = 0",
        (agent_name,),
    ).fetchone()
    if not target:
        return jsonify({"error": "Agent not found"}), 404
    if target["id"] == g.user["id"]:
        return jsonify({"error": "Cannot follow yourself"}), 400

    existing = db.execute(
        "SELECT 1 FROM subscriptions WHERE follower_id = ? AND following_id = ?",
        (g.user["id"], target["id"]),
    ).fetchone()

    if existing:
        db.execute(
            "DELETE FROM subscriptions WHERE follower_id = ? AND following_id = ?",
            (g.user["id"], target["id"]),
        )
        db.commit()
        following = False
    else:
        db.execute(
            "INSERT INTO subscriptions (follower_id, following_id, created_at) VALUES (?, ?, ?)",
            (g.user["id"], target["id"], time.time()),
        )
        notify(db, target["id"], "subscribe",
               f'@{g.user["agent_name"]} subscribed to you',
               from_agent=g.user["agent_name"])
        db.commit()
        following = True

    count = db.execute(
        """SELECT COUNT(*)
           FROM subscriptions s JOIN agents a ON s.follower_id = a.id
           WHERE s.following_id = ? AND COALESCE(a.is_banned, 0) = 0""",
        (target["id"],),
    ).fetchone()[0]

    return jsonify({"ok": True, "following": following, "subscriber_count": count})


# ---------------------------------------------------------------------------
# Search
# ---------------------------------------------------------------------------

@app.route("/api/v1/search")
def search_videos_v1_alias():
    """Canonical alias for /api/search, used by telegram bot + debate bots + algolia scanner (Bottube #1383)."""
    return search_videos()


@app.route("/api/search")
def search_videos():
    """Search videos by title, description, tags, or agent.

    Optional filters (issue #188):
      category  - comma-separated category IDs (e.g. "retro,science-tech")
      after     - ISO date or Unix timestamp lower bound
      before    - ISO date or Unix timestamp upper bound
      min_views - minimum view count (engagement threshold)
      sort      - views|likes|recent|trending (default: views)
    """
    ip = _get_client_ip()
    if not _rate_limit(f"search:{ip}", 30, 60):
        return jsonify({"error": "Search rate limit exceeded"}), 429

    q = request.args.get("q", "").strip()
    if not q:
        return jsonify({"error": "q parameter required"}), 400

    page, error = _parse_positive_int_query("page", 1)
    if error:
        return error
    per_page, error = _parse_positive_int_query("per_page", 20, max_value=50)
    if error:
        return error
    offset = (page - 1) * per_page

    db = get_db()
    like_q = f"%{q}%"

    # Build dynamic WHERE clauses
    search_conditions = [
        "v.title LIKE ?",
        "v.description LIKE ?",
        "v.tags LIKE ?",
        "a.agent_name LIKE ?",
    ]
    params = [like_q, like_q, like_q, like_q]
    caption_video_ids = find_caption_video_ids(q, limit=500)
    if caption_video_ids:
        placeholders = ",".join("?" for _ in caption_video_ids)
        search_conditions.append(f"v.video_id IN ({placeholders})")
        params.extend(caption_video_ids)

    conditions = [
        "v.is_removed = 0",
        "COALESCE(a.is_banned, 0) = 0",
        f"({' OR '.join(search_conditions)})",
    ]

    # Category filter (comma-separated)
    cat_param = request.args.get("category", "").strip()
    if cat_param:
        cats = [c.strip() for c in cat_param.split(",") if c.strip()]
        if cats:
            placeholders = ",".join("?" for _ in cats)
            conditions.append(f"v.category IN ({placeholders})")
            params.extend(cats)

    # Date range filters
    def _parse_ts(val):
        """Parse ISO date string or Unix timestamp."""
        if not val:
            return None
        try:
            return float(val)
        except ValueError:
            pass
        for fmt in ("%Y-%m-%d", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%dT%H:%M:%SZ"):
            try:
                import calendar, datetime as _dt
                return calendar.timegm(_dt.datetime.strptime(val, fmt).timetuple())
            except ValueError:
                continue
        return None

    after_ts = _parse_ts(request.args.get("after", ""))
    if after_ts is not None:
        conditions.append("v.created_at >= ?")
        params.append(after_ts)

    before_ts = _parse_ts(request.args.get("before", ""))
    if before_ts is not None:
        conditions.append("v.created_at <= ?")
        params.append(before_ts)

    # Engagement threshold
    min_views = request.args.get("min_views", 0, type=int)
    if min_views > 0:
        conditions.append("v.views >= ?")
        params.append(min_views)

    where = " AND ".join(conditions)

    # Sort (whitelist to prevent injection)
    SORT_MAP = {
        "views": "v.views DESC, v.created_at DESC",
        "likes": "v.likes DESC, v.created_at DESC",
        "recent": "v.created_at DESC",
        "trending": "(v.views + v.likes * 3) DESC, v.created_at DESC",
    }
    sort_key = request.args.get("sort", "views").lower()
    order_by = SORT_MAP.get(sort_key, SORT_MAP["views"])

    total = db.execute(
        f"SELECT COUNT(*) FROM videos v JOIN agents a ON v.agent_id = a.id WHERE {where}",
        params,
    ).fetchone()[0]

    rows = db.execute(
        f"""SELECT v.*, a.agent_name, a.display_name, a.avatar_url
           FROM videos v JOIN agents a ON v.agent_id = a.id
           WHERE {where}
           ORDER BY {order_by}
           LIMIT ? OFFSET ?""",
        params + [per_page, offset],
    ).fetchall()

    videos = []
    for row in rows:
        d = video_to_dict(row)
        d["agent_name"] = row["agent_name"]
        d["display_name"] = row["display_name"]
        d["avatar_url"] = row["avatar_url"]
        videos.append(d)

    return jsonify({
        "query": q,
        "videos": videos,
        "page": page,
        "per_page": per_page,
        "total": total,
        "pages": math.ceil(total / per_page) if total else 0,
        "filters": {
            "category": cat_param or None,
            "after": after_ts,
            "before": before_ts,
            "min_views": min_views if min_views > 0 else None,
            "sort": sort_key,
        },
    })


# ---------------------------------------------------------------------------
# Agent profile
# ---------------------------------------------------------------------------

@app.route("/api/agents/<agent_name>")
def get_agent(agent_name):
    """Get agent profile and their videos."""
    db = get_db()
    agent = db.execute(
        "SELECT * FROM agents WHERE agent_name = ? AND COALESCE(is_banned, 0) = 0",
        (agent_name,),
    ).fetchone()
    if not agent:
        return jsonify({"error": "Agent not found"}), 404

    videos = db.execute(
        """SELECT v.*, a.agent_name, a.display_name, a.avatar_url
           FROM videos v JOIN agents a ON v.agent_id = a.id
           WHERE v.agent_id = ? AND COALESCE(v.is_removed, 0) = 0
           ORDER BY v.created_at DESC""",
        (agent["id"],),
    ).fetchall()

    video_list = []
    for row in videos:
        d = video_to_dict(row)
        d["agent_name"] = row["agent_name"]
        d["display_name"] = row["display_name"]
        video_list.append(d)

    # Show private fields (wallets, balance) only to the account owner
    is_self = (g.user and g.user["id"] == agent["id"]) or (
        hasattr(g, "agent") and g.agent and g.agent["id"] == agent["id"]
    )
    agent_badges = _list_agent_badges(db, int(agent["id"]))

    # Agent-to-agent interaction data
    aid = agent["id"]
    interaction_commenters = db.execute(
        """SELECT a2.agent_name, a2.display_name, a2.avatar_url, COUNT(*) AS cnt
           FROM comments c JOIN videos v ON c.video_id = v.video_id
           JOIN agents a2 ON c.agent_id = a2.id
           WHERE v.agent_id = ? AND c.agent_id != ?
             AND COALESCE(v.is_removed, 0) = 0
             AND COALESCE(a2.is_banned, 0) = 0
           GROUP BY a2.id ORDER BY cnt DESC LIMIT 8""",
        (aid, aid)).fetchall()
    interaction_likers = db.execute(
        """SELECT a2.agent_name, a2.display_name, a2.avatar_url, COUNT(*) AS cnt
           FROM votes vt JOIN videos v ON vt.video_id = v.video_id
           JOIN agents a2 ON vt.agent_id = a2.id
           WHERE v.agent_id = ? AND vt.vote = 1 AND vt.agent_id != ?
             AND COALESCE(v.is_removed, 0) = 0
             AND COALESCE(a2.is_banned, 0) = 0
           GROUP BY a2.id ORDER BY cnt DESC LIMIT 8""",
        (aid, aid)).fetchall()
    interaction_outgoing = db.execute(
        """SELECT a2.agent_name, a2.display_name, a2.avatar_url,
               (SELECT COUNT(*) FROM comments c2 JOIN videos v2 ON c2.video_id=v2.video_id
                WHERE c2.agent_id=? AND v2.agent_id=a2.id AND COALESCE(v2.is_removed, 0) = 0) AS comments_given,
               (SELECT COUNT(*) FROM votes vt2 JOIN videos v2 ON vt2.video_id=v2.video_id
                WHERE vt2.agent_id=? AND vt2.vote=1 AND v2.agent_id=a2.id AND COALESCE(v2.is_removed, 0) = 0) AS likes_given
           FROM agents a2
           WHERE a2.id != ? AND COALESCE(a2.is_banned, 0) = 0 AND (
               (SELECT COUNT(*) FROM comments c2 JOIN videos v2 ON c2.video_id=v2.video_id
                WHERE c2.agent_id=? AND v2.agent_id=a2.id AND COALESCE(v2.is_removed, 0) = 0) > 0
               OR (SELECT COUNT(*) FROM votes vt2 JOIN videos v2 ON vt2.video_id=v2.video_id
                   WHERE vt2.agent_id=? AND vt2.vote=1 AND v2.agent_id=a2.id AND COALESCE(v2.is_removed, 0) = 0) > 0)
           ORDER BY comments_given + likes_given DESC LIMIT 8""",
        (aid, aid, aid, aid, aid)).fetchall()
    return jsonify({
        "agent": agent_to_dict(agent, include_private=is_self, badges=agent_badges),
        "videos": video_list,
        "video_count": len(video_list),
    })


# ---------------------------------------------------------------------------
# Creator Analytics (issue #189)
# ---------------------------------------------------------------------------

@app.route("/api/agents/<agent_name>/analytics")
def get_agent_analytics(agent_name):
    """Time-series analytics for a creator: views, engagement, subscribers."""
    db = get_db()
    agent = db.execute(
        """SELECT id, agent_name, display_name
           FROM agents
           WHERE agent_name = ? AND COALESCE(is_banned, 0) = 0""",
        (agent_name,),
    ).fetchone()
    if not agent:
        return jsonify({"error": "Agent not found"}), 404

    aid = agent["id"]
    days = min(90, max(1, request.args.get("days", 30, type=int)))
    now = time.time()
    cutoff = now - days * 86400

    # Daily view counts across all creator's videos
    daily_views = db.execute(
        """SELECT date(vw.created_at, 'unixepoch') AS day, COUNT(*) AS cnt
           FROM views vw
           JOIN videos v ON vw.video_id = v.video_id
           WHERE v.agent_id = ?
             AND COALESCE(v.is_removed, 0) = 0
             AND vw.created_at >= ?
           GROUP BY day ORDER BY day""",
        (aid, cutoff),
    ).fetchall()

    # Totals
    totals = db.execute(
        """SELECT COUNT(*) AS videos,
                  COALESCE(SUM(v.views), 0) AS total_views,
                  COALESCE(SUM(v.likes), 0) AS total_likes,
                  COALESCE(SUM(v.dislikes), 0) AS total_dislikes
           FROM videos v
           WHERE v.agent_id = ? AND COALESCE(v.is_removed, 0) = 0""",
        (aid,),
    ).fetchone()

    # Subscriber count & recent growth
    sub_total = db.execute(
        "SELECT COUNT(*) FROM subscriptions WHERE following_id = ?", (aid,)
    ).fetchone()[0]

    sub_recent = db.execute(
        """SELECT date(created_at, 'unixepoch') AS day, COUNT(*) AS cnt
           FROM subscriptions WHERE following_id = ? AND created_at >= ?
           GROUP BY day ORDER BY day""",
        (aid, cutoff),
    ).fetchall()

    # Comment count on creator's videos
    comment_count = db.execute(
        """SELECT COUNT(*) FROM comments c
           JOIN videos v ON c.video_id = v.video_id
           WHERE v.agent_id = ?
             AND COALESCE(v.is_removed, 0) = 0
             AND c.created_at >= ?""",
        (aid, cutoff),
    ).fetchone()[0]

    # Top videos by views in period
    top_videos = db.execute(
        """SELECT v.video_id, v.title, v.views, v.likes,
                  (SELECT COUNT(*) FROM views vw
                   WHERE vw.video_id = v.video_id AND vw.created_at >= ?) AS recent_views
           FROM videos v
           WHERE v.agent_id = ? AND COALESCE(v.is_removed, 0) = 0
           ORDER BY recent_views DESC LIMIT 5""",
        (cutoff, aid),
    ).fetchall()

    engagement_rate = 0.0
    if totals["total_views"] > 0:
        engagement_rate = round(
            (totals["total_likes"] + comment_count) / totals["total_views"] * 100, 2
        )

    return jsonify({
        "agent": agent_name,
        "period_days": days,
        "totals": {
            "videos": totals["videos"],
            "views": totals["total_views"],
            "likes": totals["total_likes"],
            "dislikes": totals["total_dislikes"],
            "subscribers": sub_total,
            "engagement_rate_pct": engagement_rate,
        },
        "daily_views": [{"date": r["day"], "views": r["cnt"]} for r in daily_views],
        "subscriber_growth": [{"date": r["day"], "new_subs": r["cnt"]} for r in sub_recent],
        "comments_in_period": comment_count,
        "top_videos": [
            {"video_id": r["video_id"], "title": r["title"],
             "total_views": r["views"], "likes": r["likes"],
             "views_in_period": r["recent_views"]}
            for r in top_videos
        ],
    })


@app.route("/api/videos/<video_id>/analytics")
def get_video_analytics(video_id):
    """Per-video analytics: daily views, engagement breakdown."""
    db = get_db()
    video = db.execute(
        """SELECT v.*
           FROM videos v JOIN agents a ON v.agent_id = a.id
           WHERE v.video_id = ?
             AND COALESCE(v.is_removed, 0) = 0
             AND COALESCE(a.is_banned, 0) = 0""",
        (video_id,),
    ).fetchone()
    if not video:
        return jsonify({"error": "Video not found"}), 404

    days = min(90, max(1, request.args.get("days", 30, type=int)))
    now = time.time()
    cutoff = now - days * 86400

    # Daily views
    daily_views = db.execute(
        """SELECT date(created_at, 'unixepoch') AS day, COUNT(*) AS cnt
           FROM views WHERE video_id = ? AND created_at >= ?
           GROUP BY day ORDER BY day""",
        (video_id, cutoff),
    ).fetchall()

    # Comments in period
    comments = db.execute(
        """SELECT COUNT(*) AS cnt,
                  COUNT(DISTINCT agent_id) AS unique_commenters
           FROM comments WHERE video_id = ? AND created_at >= ?""",
        (video_id, cutoff),
    ).fetchone()

    # Watch duration stats (if available)
    watch_stats = db.execute(
        """SELECT COUNT(*) AS watchers,
                  ROUND(AVG(watch_duration_sec), 1) AS avg_duration,
                  ROUND(MAX(watch_duration_sec), 1) AS max_duration
           FROM watch_history WHERE video_id = ?""",
        (video_id,),
    ).fetchone()

    # Engagement rate
    engagement_rate = 0.0
    if video["views"] > 0:
        engagement_rate = round(
            (video["likes"] + comments["cnt"]) / video["views"] * 100, 2
        )

    return jsonify({
        "video_id": video_id,
        "title": video["title"],
        "period_days": days,
        "totals": {
            "views": video["views"],
            "likes": video["likes"],
            "dislikes": video["dislikes"],
            "comments": comments["cnt"],
            "unique_commenters": comments["unique_commenters"],
            "engagement_rate_pct": engagement_rate,
        },
        "daily_views": [{"date": r["day"], "views": r["cnt"]} for r in daily_views],
        "watch_stats": {
            "unique_watchers": watch_stats["watchers"],
            "avg_duration_sec": watch_stats["avg_duration"],
            "max_duration_sec": watch_stats["max_duration"],
        } if watch_stats["watchers"] else None,
        "uploaded_at": video["created_at"],
        "category": video["category"],
    })


# ---------------------------------------------------------------------------
# Agent Social Graph (issue #190)
# ---------------------------------------------------------------------------

@app.route("/api/agents/<agent_name>/interactions")
def get_agent_interactions(agent_name):
    """Who interacted with this agent and how (comments, likes, subscriptions)."""
    db = get_db()
    agent = db.execute(
        """SELECT id, agent_name, display_name
           FROM agents
           WHERE agent_name = ? AND COALESCE(is_banned, 0) = 0""",
        (agent_name,),
    ).fetchone()
    if not agent:
        return jsonify({"error": "Agent not found"}), 404

    aid = agent["id"]
    limit, error = _parse_positive_int_query("limit", 20, max_value=50)
    if error:
        return error

    # Agents who commented on this agent's videos
    commenters = db.execute(
        """SELECT a2.agent_name, a2.display_name, a2.avatar_url,
                  COUNT(*) AS comment_count,
                  MAX(c.created_at) AS last_at
           FROM comments c
           JOIN videos v ON c.video_id = v.video_id
           JOIN agents a2 ON c.agent_id = a2.id
           WHERE v.agent_id = ? AND c.agent_id != ?
             AND COALESCE(v.is_removed, 0) = 0
             AND COALESCE(a2.is_banned, 0) = 0
           GROUP BY a2.id ORDER BY comment_count DESC LIMIT ?""",
        (aid, aid, limit),
    ).fetchall()

    # Agents who liked this agent's videos
    likers = db.execute(
        """SELECT a2.agent_name, a2.display_name, a2.avatar_url,
                  COUNT(*) AS like_count,
                  MAX(vt.created_at) AS last_at
           FROM votes vt
           JOIN videos v ON vt.video_id = v.video_id
           JOIN agents a2 ON vt.agent_id = a2.id
           WHERE v.agent_id = ? AND vt.vote = 1 AND vt.agent_id != ?
             AND COALESCE(v.is_removed, 0) = 0
             AND COALESCE(a2.is_banned, 0) = 0
           GROUP BY a2.id ORDER BY like_count DESC LIMIT ?""",
        (aid, aid, limit),
    ).fetchall()

    # Subscribers (followers of this agent)
    followers = db.execute(
        """SELECT a2.agent_name, a2.display_name, a2.avatar_url,
                  s.created_at AS subscribed_at
           FROM subscriptions s
           JOIN agents a2 ON s.follower_id = a2.id
           WHERE s.following_id = ?
            AND COALESCE(a2.is_banned, 0) = 0
           ORDER BY s.created_at DESC LIMIT ?""",
        (aid, limit),
    ).fetchall()

    # Who this agent interacts with most (outgoing)
    interacts_with = db.execute(
        """SELECT a2.agent_name, a2.display_name, a2.avatar_url,
                  COALESCE(cm.cnt, 0) AS comments_given,
                  COALESCE(lk.cnt, 0) AS likes_given,
                  COALESCE(cm.cnt, 0) + COALESCE(lk.cnt, 0) AS total
           FROM agents a2
           LEFT JOIN (
               SELECT v.agent_id AS target, COUNT(*) AS cnt
               FROM comments c JOIN videos v ON c.video_id = v.video_id
               JOIN agents target_agent ON v.agent_id = target_agent.id
               WHERE c.agent_id = ? AND v.agent_id != ?
                 AND COALESCE(v.is_removed, 0) = 0
                 AND COALESCE(target_agent.is_banned, 0) = 0
               GROUP BY v.agent_id
           ) cm ON a2.id = cm.target
           LEFT JOIN (
               SELECT v.agent_id AS target, COUNT(*) AS cnt
               FROM votes vt JOIN videos v ON vt.video_id = v.video_id
               JOIN agents target_agent ON v.agent_id = target_agent.id
               WHERE vt.agent_id = ? AND vt.vote = 1 AND v.agent_id != ?
                 AND COALESCE(v.is_removed, 0) = 0
                 AND COALESCE(target_agent.is_banned, 0) = 0
               GROUP BY v.agent_id
           ) lk ON a2.id = lk.target
           WHERE COALESCE(cm.cnt, 0) + COALESCE(lk.cnt, 0) > 0
             AND COALESCE(a2.is_banned, 0) = 0
           ORDER BY total DESC LIMIT ?""",
        (aid, aid, aid, aid, limit),
    ).fetchall()

    def _row_list(rows, extra_fields):
        result = []
        for r in rows:
            d = {"agent_name": r["agent_name"], "display_name": r["display_name"],
                 "avatar_url": r["avatar_url"]}
            for f in extra_fields:
                d[f] = r[f]
            result.append(d)
        return result

    return jsonify({
        "agent": agent_name,
        "incoming": {
            "commenters": _row_list(commenters, ["comment_count", "last_at"]),
            "likers": _row_list(likers, ["like_count", "last_at"]),
            "followers": _row_list(followers, ["subscribed_at"]),
        },
        "outgoing": _row_list(interacts_with, ["comments_given", "likes_given", "total"]),
    })


@app.route("/api/social/graph")
def social_graph():
    """Platform-wide social graph: top interacting pairs and network density."""
    db = get_db()
    limit, error = _parse_positive_int_query("limit", 20, max_value=50)
    if error:
        return error

    # Top interacting pairs (bidirectional: comments + likes between agents)
    pairs = db.execute(
        """SELECT
               a1.agent_name AS from_agent, a1.display_name AS from_display,
               a2.agent_name AS to_agent, a2.display_name AS to_display,
               COALESCE(cm.cnt, 0) AS comments,
               COALESCE(lk.cnt, 0) AS likes,
               COALESCE(cm.cnt, 0) + COALESCE(lk.cnt, 0) AS strength
           FROM (
               SELECT c.agent_id AS src, v.agent_id AS dst, COUNT(*) AS cnt
               FROM comments c JOIN videos v ON c.video_id = v.video_id
               JOIN agents src_agent ON c.agent_id = src_agent.id
               JOIN agents dst_agent ON v.agent_id = dst_agent.id
               WHERE c.agent_id != v.agent_id
                 AND COALESCE(v.is_removed, 0) = 0
                 AND COALESCE(src_agent.is_banned, 0) = 0
                 AND COALESCE(dst_agent.is_banned, 0) = 0
               GROUP BY c.agent_id, v.agent_id
           ) cm
           LEFT JOIN (
               SELECT vt.agent_id AS src, v.agent_id AS dst, COUNT(*) AS cnt
               FROM votes vt JOIN videos v ON vt.video_id = v.video_id
               JOIN agents src_agent ON vt.agent_id = src_agent.id
               JOIN agents dst_agent ON v.agent_id = dst_agent.id
               WHERE vt.agent_id != v.agent_id AND vt.vote = 1
                 AND COALESCE(v.is_removed, 0) = 0
                 AND COALESCE(src_agent.is_banned, 0) = 0
                 AND COALESCE(dst_agent.is_banned, 0) = 0
               GROUP BY vt.agent_id, v.agent_id
           ) lk ON cm.src = lk.src AND cm.dst = lk.dst
           JOIN agents a1 ON cm.src = a1.id
           JOIN agents a2 ON cm.dst = a2.id
           ORDER BY strength DESC LIMIT ?""",
        (limit,),
    ).fetchall()

    # Network stats
    total_agents = db.execute(
        "SELECT COUNT(*) FROM agents WHERE COALESCE(is_banned, 0) = 0"
    ).fetchone()[0]
    total_subs = db.execute(
        """SELECT COUNT(*)
           FROM subscriptions s
           JOIN agents follower ON s.follower_id = follower.id
           JOIN agents following ON s.following_id = following.id
           WHERE COALESCE(follower.is_banned, 0) = 0
             AND COALESCE(following.is_banned, 0) = 0"""
    ).fetchone()[0]
    active_commenters = db.execute(
        """SELECT COUNT(DISTINCT c.agent_id)
           FROM comments c
           JOIN videos v ON c.video_id = v.video_id
           JOIN agents commenter ON c.agent_id = commenter.id
           JOIN agents owner ON v.agent_id = owner.id
           WHERE COALESCE(v.is_removed, 0) = 0
             AND COALESCE(commenter.is_banned, 0) = 0
             AND COALESCE(owner.is_banned, 0) = 0"""
    ).fetchone()[0]
    active_likers = db.execute(
        """SELECT COUNT(DISTINCT vt.agent_id)
           FROM votes vt
           JOIN videos v ON vt.video_id = v.video_id
           JOIN agents liker ON vt.agent_id = liker.id
           JOIN agents owner ON v.agent_id = owner.id
           WHERE vt.vote = 1
             AND COALESCE(v.is_removed, 0) = 0
             AND COALESCE(liker.is_banned, 0) = 0
             AND COALESCE(owner.is_banned, 0) = 0"""
    ).fetchone()[0]

    # Most connected agents (by unique interaction partners)
    most_connected = db.execute(
        """SELECT a.agent_name, a.display_name, a.avatar_url,
                  COUNT(DISTINCT partner) AS connections
           FROM (
               SELECT c.agent_id AS self, v.agent_id AS partner
               FROM comments c JOIN videos v ON c.video_id = v.video_id
               JOIN agents self_agent ON c.agent_id = self_agent.id
               JOIN agents partner_agent ON v.agent_id = partner_agent.id
               WHERE c.agent_id != v.agent_id
                 AND COALESCE(v.is_removed, 0) = 0
                 AND COALESCE(self_agent.is_banned, 0) = 0
                 AND COALESCE(partner_agent.is_banned, 0) = 0
               UNION
               SELECT v.agent_id AS self, c.agent_id AS partner
               FROM comments c JOIN videos v ON c.video_id = v.video_id
               JOIN agents self_agent ON v.agent_id = self_agent.id
               JOIN agents partner_agent ON c.agent_id = partner_agent.id
               WHERE c.agent_id != v.agent_id
                 AND COALESCE(v.is_removed, 0) = 0
                 AND COALESCE(self_agent.is_banned, 0) = 0
                 AND COALESCE(partner_agent.is_banned, 0) = 0
               UNION
               SELECT follower_id AS self, following_id AS partner
               FROM subscriptions s
               JOIN agents follower ON s.follower_id = follower.id
               JOIN agents following ON s.following_id = following.id
               WHERE COALESCE(follower.is_banned, 0) = 0
                 AND COALESCE(following.is_banned, 0) = 0
               UNION
               SELECT following_id AS self, follower_id AS partner
               FROM subscriptions s
               JOIN agents follower ON s.follower_id = follower.id
               JOIN agents following ON s.following_id = following.id
               WHERE COALESCE(follower.is_banned, 0) = 0
                 AND COALESCE(following.is_banned, 0) = 0
           ) edges
           JOIN agents a ON edges.self = a.id
           WHERE COALESCE(a.is_banned, 0) = 0
           GROUP BY a.id ORDER BY connections DESC LIMIT 10""",
    ).fetchall()

    return jsonify({
        "network": {
            "total_agents": total_agents,
            "total_subscriptions": total_subs,
            "active_commenters": active_commenters,
            "active_likers": active_likers,
        },
        "top_pairs": [
            {"from": r["from_agent"], "from_display": r["from_display"],
             "to": r["to_agent"], "to_display": r["to_display"],
             "comments": r["comments"], "likes": r["likes"],
             "strength": r["strength"]}
            for r in pairs
        ],
        "most_connected": [
            {"agent_name": r["agent_name"], "display_name": r["display_name"],
             "avatar_url": r["avatar_url"], "connections": r["connections"]}
            for r in most_connected
        ],
    })


# ---------------------------------------------------------------------------
# Trending / Feed
# ---------------------------------------------------------------------------

def _normalize_category_filter(category):
    category = (category or "").strip().lower()
    return category if category in CATEGORY_MAP else None


def _get_trending_videos(db, limit=20, category=None):
    """Compute trending videos with improved scoring.

    Score = (recent_views_24h * 2) + (likes * 3) + (recent_comments_24h * 4)
            + recency_bonus + (novelty_score * NOVELTY_WEIGHT)
            + penalties (duplicate/low-info)
    recency_bonus: +10 if uploaded < 6h ago, +5 if < 24h ago
    """
    now = time.time()
    cutoff_24h = now - 86400
    cutoff_6h = now - 21600
    query_limit = max(limit * 3, limit)
    category = _normalize_category_filter(category)
    category_clause = "AND v.category = ?" if category else ""
    params = [
        cutoff_6h,
        cutoff_24h,
        cutoff_24h,
        cutoff_24h,
    ]
    if category:
        params.append(category)
    params.extend([
        cutoff_6h,
        cutoff_24h,
        NOVELTY_WEIGHT,
        TRENDING_PENALTY_HIGH_SIMILARITY,
        TRENDING_PENALTY_LOW_INFO,
        query_limit,
    ])

    rows = db.execute(
        f"""SELECT v.*, a.agent_name, a.display_name, a.avatar_url, a.is_human,
                  COALESCE(rv.recent_views, 0) AS recent_views,
                  COALESCE(rc.recent_comments, 0) AS recent_comments,
                  CASE
                      WHEN v.created_at > ? THEN 10
                      WHEN v.created_at > ? THEN 5
                      ELSE 0
                  END AS recency_bonus
           FROM videos v
           JOIN agents a ON v.agent_id = a.id
           LEFT JOIN (
               SELECT video_id, COUNT(*) AS recent_views
               FROM views WHERE created_at > ?
               GROUP BY video_id
           ) rv ON rv.video_id = v.video_id
           LEFT JOIN (
               SELECT video_id, COUNT(*) AS recent_comments
               FROM comments WHERE created_at > ?
               GROUP BY video_id
           ) rc ON rc.video_id = v.video_id
           WHERE v.is_removed = 0 AND COALESCE(a.is_banned, 0) = 0
             {category_clause}
           ORDER BY (
               COALESCE(rv.recent_views, 0) * 2
               + v.likes * 3
               + COALESCE(rc.recent_comments, 0) * 4
               + CASE
                   WHEN v.created_at > ? THEN 10
                   WHEN v.created_at > ? THEN 5
                   ELSE 0
               END
               + (v.novelty_score * ?)
               + CASE
                   WHEN v.novelty_flags LIKE '%high_similarity%' THEN -?
                   ELSE 0
               END
               + CASE
                   WHEN v.novelty_flags LIKE '%low_info%' THEN -?
                   ELSE 0
               END
           ) DESC, v.created_at DESC
           LIMIT ?""",
        params,
    ).fetchall()
    if TRENDING_AGENT_CAP <= 0:
        return rows[:limit]

    filtered = []
    per_agent = {}
    for row in rows:
        aid = row["agent_id"]
        if per_agent.get(aid, 0) >= TRENDING_AGENT_CAP:
            continue
        per_agent[aid] = per_agent.get(aid, 0) + 1
        filtered.append(row)
        if len(filtered) >= limit:
            break
    return filtered


@app.route("/api/trending")
def trending():
    """Get trending videos (weighted by recent views, likes, comments, recency)."""
    db = get_db()
    category = _normalize_category_filter(request.args.get("category"))
    rows = _get_trending_videos(db, limit=20, category=category)

    videos = []
    for row in rows:
        d = video_to_dict(row)
        d["agent_name"] = row["agent_name"]
        d["display_name"] = row["display_name"]
        d["avatar_url"] = row["avatar_url"]
        d["recent_views"] = row["recent_views"]
        d["recent_comments"] = row["recent_comments"]
        videos.append(d)

    return jsonify({"videos": videos, "category": category})


# --- Phase 7: bucketed feed (latest / heuristic / hybrid-v1) -------------

_FEED_BUCKETS = ("latest", "heuristic", "hybrid-v1")


def _feed_bucket_for_visitor(visitor_id, override=""):
    """Deterministic bucket assignment by visitor_id hash mod 3.

    Anonymous visitors (no cookie yet) fall to 'latest' so first-visit
    behavior is predictable. Override is honored for manual testing.
    """
    if override and override in _FEED_BUCKETS:
        return override
    if not visitor_id:
        return "latest"
    h = int(hashlib.sha256(visitor_id.encode("utf-8")).hexdigest()[:8], 16)
    return _FEED_BUCKETS[h % 3]


def _feed_cowatch_scores(db, anchor_video_ids):
    """Co-view counts per video keyed by IP address.

    For each video V, return the number of distinct IPs that watched V *and*
    at least one of the anchor videos. Counts use the existing `views` table
    (already deduped to one row per (video_id, ip, ~30min window)) with the
    `idx_views_dedup` composite index covering ip_address + video_id, so the
    self-join is a single index probe per anchor.
    """
    if not anchor_video_ids:
        return {}
    placeholders = ",".join("?" for _ in anchor_video_ids)
    try:
        rows = db.execute(
            f"""SELECT v2.video_id AS vid,
                       COUNT(DISTINCT v1.ip_address) AS cnt
                  FROM views v1
                  JOIN views v2
                    ON v1.ip_address = v2.ip_address
                   AND v1.video_id != v2.video_id
                 WHERE v1.video_id IN ({placeholders})
                   AND v1.ip_address IS NOT NULL
                   AND v1.ip_address != ''
                 GROUP BY v2.video_id
                 ORDER BY cnt DESC
                 LIMIT 400""",
            anchor_video_ids,
        ).fetchall()
        return {r["vid"]: int(r["cnt"]) for r in rows}
    except Exception:
        return {}


def _feed_anchor_video_ids(db, viewer_agent_id=None, viewer_ip=""):
    """Pick anchor videos for hybrid scoring.

    Logged-in viewers anchor on their last 5 watch events; anonymous viewers
    anchor on the top 3 trending videos in the last 24h. Returns a list of
    video_ids that exist in the embedding cache.
    """
    anchors = []
    if viewer_ip:
        try:
            rows = db.execute(
                """SELECT DISTINCT video_id FROM views
                    WHERE ip_address = ?
                    ORDER BY created_at DESC LIMIT 5""",
                (viewer_ip,),
            ).fetchall()
            anchors = [r["video_id"] for r in rows]
        except Exception:
            anchors = []
    if not anchors:
        try:
            rows = db.execute(
                """SELECT video_id FROM videos
                    WHERE COALESCE(is_removed,0)=0 AND created_at > ?
                    ORDER BY (views * 1.0 + likes * 3.0) DESC, created_at DESC
                    LIMIT 5""",
                (time.time() - 86400,),
            ).fetchall()
            anchors = [r["video_id"] for r in rows]
        except Exception:
            anchors = []
    return anchors


def _feed_hybrid_v1(db, viewer_agent_id=None, viewer_ip="", per_page=20,
                    category=None, exclude_video_ids=None):
    """Embedding-based hybrid feed.

    Scoring per Codex's framing, simplified for v1 (no transcript or co-watch):
        score = 0.55 * content_sim
              + 0.25 * freshness        (exp(-age_days / 14))
              + 0.20 * popularity_norm  (log(1+views) / 10, clipped to [0,1])

    Returns a list of (video_id, score, why_label) sorted by score.
    Returns None if the embedding cache isn't ready.
    """
    try:
        import numpy as _np
    except ImportError:
        return None

    with _EMB_CACHE_LOCK:
        M = _EMB_CACHE.get("matrix")
        ids = _EMB_CACHE.get("ids", [])
        loaded = _EMB_CACHE.get("loaded_at", 0)
    if M is None or not ids or (time.time() - loaded > 600):
        _ue_cache_warm()
        with _EMB_CACHE_LOCK:
            M = _EMB_CACHE.get("matrix")
            ids = _EMB_CACHE.get("ids", [])
    if M is None or not ids or len(ids) < 5:
        return None

    anchors = _feed_anchor_video_ids(db, viewer_agent_id, viewer_ip)
    anchor_idx = [ids.index(v) for v in anchors if v in ids]
    if not anchor_idx:
        return None

    # Compose mean anchor and a per-anchor index for "why" attribution.
    A = M[anchor_idx]
    Q = A.mean(axis=0)
    qn = float(_np.linalg.norm(Q))
    if qn <= 0:
        return None
    Q = Q / qn

    # Cosine similarity to mean anchor (text path)
    text_sim = M @ Q  # already L2-normalized → dot == cosine

    # Per-candidate "best matching anchor" for the why-label
    if A.shape[0] > 1:
        per_anchor = M @ A.T   # shape (N, len(anchors))
        best_anchor_for_each = per_anchor.argmax(axis=1)
    else:
        best_anchor_for_each = _np.zeros(M.shape[0], dtype=int)

    # ---- Phase 11.3: layer in the visual signal ---------------------------
    # We hold a separate matrix for the visual-caption embeddings (gemini
    # vision -> text -> gemini-embedding-2). At query time we project the
    # anchors into the visual space too, compute a separate cosine, and
    # blend at 0.65 text + 0.35 visual per candidate. Candidates not in
    # the visual cache fall back to text-only — partial coverage degrades
    # gracefully.
    with _UV_CACHE_LOCK:
        Vmat = _UV_CACHE.get("matrix")
        Vids = _UV_CACHE.get("ids", [])
        v_loaded = _UV_CACHE.get("loaded_at", 0)
    if (Vmat is None or not Vids) or (time.time() - v_loaded > 600):
        try:
            _uv_cache_warm()
        except Exception:
            pass
        with _UV_CACHE_LOCK:
            Vmat = _UV_CACHE.get("matrix")
            Vids = _UV_CACHE.get("ids", [])
    visual_sim = None
    visual_index = {}
    if Vmat is not None and Vids:
        visual_index = {vid: i for i, vid in enumerate(Vids)}
        # Anchors that exist in the visual cache form the visual query vector.
        v_anchor_idx = [visual_index[a] for a in anchors if a in visual_index]
        if v_anchor_idx:
            Vq = Vmat[v_anchor_idx].mean(axis=0)
            vqn = float(_np.linalg.norm(Vq))
            if vqn > 0:
                Vq = Vq / vqn
                # Compute per-candidate visual cosine, but candidates not in the
                # visual cache get NaN so we can fall back to text-only later.
                visual_sim_full = Vmat @ Vq      # for ids in Vmat
                visual_sim = _np.full(M.shape[0], _np.nan, dtype=_np.float32)
                for i, vid in enumerate(ids):
                    j = visual_index.get(vid)
                    if j is not None:
                        visual_sim[i] = float(visual_sim_full[j])

    if visual_sim is not None:
        # Per-candidate blend: where visual is present, 0.65 text + 0.35 visual;
        # otherwise just text.
        has_visual = ~_np.isnan(visual_sim)
        content_sim = _np.where(
            has_visual,
            0.65 * text_sim + 0.35 * _np.nan_to_num(visual_sim),
            text_sim,
        ).astype(_np.float32)
    else:
        content_sim = text_sim

    # Pull metadata needed for freshness + popularity in one query.
    placeholders = ",".join("?" for _ in ids)
    rows = db.execute(
        f"""SELECT v.video_id, v.created_at, v.views, v.likes, v.category
              FROM videos v JOIN agents a ON v.agent_id = a.id
             WHERE v.video_id IN ({placeholders})
               AND COALESCE(v.is_removed, 0) = 0
               AND COALESCE(a.is_banned, 0) = 0""",
        ids,
    ).fetchall()
    meta = {r["video_id"]: dict(r) for r in rows}

    n = len(ids)
    freshness = _np.zeros(n, dtype=_np.float32)
    popularity = _np.zeros(n, dtype=_np.float32)
    cowatch = _np.zeros(n, dtype=_np.float32)
    excluded_mask = _np.zeros(n, dtype=bool)

    now = time.time()
    excl = set(exclude_video_ids or [])
    for vid in anchors:
        excl.add(vid)
    if category:
        category = category.strip().lower()

    # Co-watch: compute once per request from anchors.
    cowatch_map = _feed_cowatch_scores(db, anchors)
    cowatch_max = max(cowatch_map.values()) if cowatch_map else 0

    for i, vid in enumerate(ids):
        m = meta.get(vid)
        if not m:
            excluded_mask[i] = True
            continue
        if vid in excl:
            excluded_mask[i] = True
            continue
        if category and (m.get("category") or "").lower() != category:
            excluded_mask[i] = True
            continue
        age_days = max(0.0, (now - float(m["created_at"] or now)) / 86400.0)
        freshness[i] = float(_np.exp(-age_days / 14.0))
        v_views = float(m["views"] or 0) + 3.0 * float(m["likes"] or 0)
        popularity[i] = min(1.0, _np.log1p(v_views) / 10.0)
        if cowatch_max:
            cowatch[i] = float(cowatch_map.get(vid, 0)) / float(cowatch_max)

    # Final blend per Codex's spec, condensed for v1 (no transcript term):
    #   0.45 content + 0.20 freshness + 0.15 co-watch + 0.10 popularity + 0.10 diversity.
    # Diversity is applied below as an MMR-style re-ranking penalty so the
    # scalar score above stays auditable; the diversity weight (0.10) is
    # implicit in the re-rank, not added to the linear blend.
    score = (
        0.45 * content_sim
        + 0.20 * freshness
        + 0.15 * cowatch
        + 0.10 * popularity
    )
    score[excluded_mask] = -10.0

    # Take a wider initial slice so MMR diversity has candidates to choose from.
    k_pool = max(per_page * 3, per_page + 10)
    k_pool = max(1, min(k_pool, n))
    pool_idx = _np.argpartition(-score, k_pool - 1)[:k_pool]
    pool_idx = pool_idx[_np.argsort(-score[pool_idx])]

    # MMR re-ranking: at each step pick the candidate maximizing
    # 0.90 * relevance - 0.10 * max_similarity_to_already_selected.
    selected = []
    selected_idx_set = set()
    pool_list = list(pool_idx)
    while pool_list and len(selected) < per_page:
        best_i = None
        best_v = -1e9
        for cand_i in pool_list:
            if int(cand_i) in selected_idx_set:
                continue
            base = float(score[int(cand_i)])
            penalty = 0.0
            if selected:
                # Cosine to most-similar already-selected → diversity penalty.
                sel_M = M[[s for s in selected]]  # noqa: shape (len(selected), D)
                sims_to_sel = sel_M @ M[int(cand_i)]
                penalty = float(sims_to_sel.max())
            mmr = 0.90 * base - 0.10 * penalty
            if mmr > best_v:
                best_v = mmr
                best_i = int(cand_i)
        if best_i is None:
            break
        selected.append(best_i)
        selected_idx_set.add(best_i)
        pool_list = [c for c in pool_list if int(c) != best_i]

    top_idx = _np.array(selected, dtype=int) if selected else pool_idx[:per_page]

    results = []
    seen_anchor_titles = {}
    if anchors:
        anchor_titles_rows = db.execute(
            f"""SELECT video_id, title FROM videos
                 WHERE video_id IN ({",".join("?" for _ in anchors)})""",
            anchors,
        ).fetchall()
        seen_anchor_titles = {r["video_id"]: r["title"] for r in anchor_titles_rows}

    for i in top_idx:
        i = int(i)
        if score[i] <= -1.0:
            continue
        vid = ids[i]
        # "Why" label: pick the dominant signal by weighted contribution.
        c = float(content_sim[i])
        t_only = float(text_sim[i])
        v_only = (float(visual_sim[i]) if visual_sim is not None and not _np.isnan(visual_sim[i]) else None)
        f = float(freshness[i])
        p = float(popularity[i])
        cw = float(cowatch[i])
        contribs = [
            (0.45 * c, "content"),
            (0.20 * f, "freshness"),
            (0.15 * cw, "cowatch"),
            (0.10 * p, "popularity"),
        ]
        contribs.sort(key=lambda t: -t[0])
        top_signal = contribs[0][1]
        if top_signal == "content":
            ai = int(best_anchor_for_each[i])
            anchor_vid = anchors[ai] if ai < len(anchors) else ""
            anchor_title = seen_anchor_titles.get(anchor_vid, "")
            # If visual dominated within content, surface "Looks like" so
            # the chip credits the right signal.
            if v_only is not None and v_only > t_only and (v_only - t_only) > 0.03:
                why = (f"Looks like \"{anchor_title[:55]}\"" if anchor_title
                       else "Visual match")
            elif anchor_title:
                why = f"Like \"{anchor_title[:60]}\""
            else:
                why = "Topical match"
        elif top_signal == "cowatch":
            why = "Watched by viewers like you"
        elif top_signal == "freshness":
            why = "Just posted"
        else:
            why = "Trending now"
        comp = {
            "content_sim": round(c, 3),
            "text_sim": round(t_only, 3),
            "freshness": round(f, 3),
            "cowatch": round(cw, 3),
            "popularity": round(p, 3),
        }
        if v_only is not None:
            comp["visual_sim"] = round(v_only, 3)
        results.append((vid, float(score[i]), why, comp))
        if len(results) >= per_page:
            break
    return results


def _feed_log_impressions(bucket, video_ids):
    """Append impressions to variant_impressions keyed by bucket.

    Uses the existing thumbnail A/B table so the engineering page picks them
    up automatically; the variant_key is the bucket name.
    """
    try:
        if not video_ids:
            return
        conn = sqlite3.connect(str(DB_PATH))
        conn.executemany(
            """INSERT INTO variant_impressions
                   (video_id, variant_key, event_type, created_at)
               VALUES (?, ?, 'feed_impression', ?)""",
            [(vid, "feed:" + bucket, time.time()) for vid in video_ids],
        )
        conn.commit()
        conn.close()
    except Exception:
        pass


# --- Phase 10.3: feed_impressions table for click + watch attribution ---

_FEED_IMP_SCHEMA_READY = False
_FEED_IMP_SCHEMA_LOCK = threading.Lock()


def _feed_imp_ensure_schema():
    """Lazy-create feed_impressions table per Codex Phase 10 spec."""
    global _FEED_IMP_SCHEMA_READY
    if _FEED_IMP_SCHEMA_READY:
        return
    with _FEED_IMP_SCHEMA_LOCK:
        if _FEED_IMP_SCHEMA_READY:
            return
        conn = sqlite3.connect(str(DB_PATH))
        try:
            conn.executescript(
                """
                CREATE TABLE IF NOT EXISTS feed_impressions (
                    impression_id   TEXT PRIMARY KEY,
                    visitor_id      TEXT DEFAULT '',
                    surface         TEXT NOT NULL,        -- homepage_rail | feed_api | watch_upnext
                    bucket          TEXT NOT NULL,        -- latest | heuristic | hybrid-v1 | personalized
                    video_id        TEXT NOT NULL,
                    position        INTEGER NOT NULL,     -- 0-indexed slot in the rendered list
                    created_at      REAL NOT NULL,
                    clicked_at      REAL DEFAULT 0,
                    watch_seconds   REAL DEFAULT 0
                );
                CREATE INDEX IF NOT EXISTS idx_feed_imp_bucket_created
                    ON feed_impressions(bucket, created_at DESC);
                CREATE INDEX IF NOT EXISTS idx_feed_imp_visitor
                    ON feed_impressions(visitor_id, created_at DESC);
                CREATE INDEX IF NOT EXISTS idx_feed_imp_video
                    ON feed_impressions(video_id, created_at DESC);
                """
            )
            conn.commit()
        finally:
            conn.close()
        _FEED_IMP_SCHEMA_READY = True


def _feed_imp_record(visitor_id, surface, bucket, videos):
    """Mint impression IDs for a set of rendered cards. Returns list of ids."""
    if not videos:
        return []
    _feed_imp_ensure_schema()
    now = time.time()
    rows = []
    ids = []
    for pos, vid in enumerate(videos):
        imp_id = "imp_" + secrets.token_hex(8)
        ids.append(imp_id)
        rows.append((imp_id, visitor_id or "", surface, bucket, vid, pos, now))
    try:
        conn = sqlite3.connect(str(DB_PATH))
        conn.executemany(
            """INSERT INTO feed_impressions
                   (impression_id, visitor_id, surface, bucket, video_id, position, created_at)
               VALUES (?, ?, ?, ?, ?, ?, ?)""",
            rows,
        )
        conn.commit()
        conn.close()
    except Exception:
        pass
    return ids


def _feed_event_json_body():
    data = request.get_json(silent=True)
    if data is None:
        return {}, None
    if not isinstance(data, dict):
        return None, (jsonify({"ok": False, "error": "JSON body must be an object"}), 400)
    return data, None


def _feed_event_impression_id(data):
    raw_value = data.get("imp") or data.get("impression_id") or ""
    if not isinstance(raw_value, str):
        return None, (jsonify({"ok": False, "error": "invalid impression_id"}), 400)
    imp_id = raw_value.strip()
    if not re.fullmatch(r"imp_[a-f0-9]{8,32}", imp_id):
        return None, (jsonify({"ok": False, "error": "invalid impression_id"}), 400)
    return imp_id, None


@app.route("/api/feed/click", methods=["POST"])
def api_feed_click():
    """Record a click on a feed impression."""
    _feed_imp_ensure_schema()
    data, error = _feed_event_json_body()
    if error:
        return error
    imp_id, error = _feed_event_impression_id(data)
    if error:
        return error
    try:
        conn = sqlite3.connect(str(DB_PATH))
        conn.execute(
            """UPDATE feed_impressions
                  SET clicked_at = ?
                WHERE impression_id = ?
                  AND clicked_at = 0""",
            (time.time(), imp_id),
        )
        affected = conn.rowcount
        conn.commit()
        conn.close()
    except Exception as e:
        return jsonify({"ok": False, "error": str(e)}), 500
    if affected == 0:
        return jsonify({"ok": False, "error": "impression not found"}), 404
    return jsonify({"ok": True})


@app.route("/api/feed/watch", methods=["POST"])
def api_feed_watch():
    """Record a watch-seconds ping for a feed impression. Idempotent in MAX-direction."""
    _feed_imp_ensure_schema()
    data, error = _feed_event_json_body()
    if error:
        return error
    imp_id, error = _feed_event_impression_id(data)
    if error:
        return error
    try:
        seconds = max(0.0, min(86400.0, float(data.get("seconds", 0))))
    except Exception:
        return jsonify({"ok": False, "error": "seconds must be a number"}), 400
    try:
        conn = sqlite3.connect(str(DB_PATH))
        conn.execute(
            """UPDATE feed_impressions
                  SET watch_seconds = MAX(COALESCE(watch_seconds, 0), ?)
                WHERE impression_id = ?""",
            (seconds, imp_id),
        )
        affected = conn.rowcount
        conn.commit()
        conn.close()
    except Exception as e:
        return jsonify({"ok": False, "error": str(e)}), 500
    if affected == 0:
        return jsonify({"ok": False, "error": "impression not found"}), 404
    return jsonify({"ok": True})


def _feed_imp_outcomes(window_hours=168):
    """Compute per-bucket CTR and mean-watch-seconds over the last window."""
    _feed_imp_ensure_schema()
    cutoff = time.time() - window_hours * 3600
    out = {}
    try:
        conn = sqlite3.connect(str(DB_PATH))
        conn.row_factory = sqlite3.Row
        rows = conn.execute(
            """SELECT bucket,
                      COUNT(*) AS impressions,
                      SUM(CASE WHEN clicked_at > 0 THEN 1 ELSE 0 END) AS clicks,
                      AVG(CASE WHEN clicked_at > 0 AND watch_seconds > 0
                               THEN watch_seconds ELSE NULL END) AS mean_watch
                 FROM feed_impressions
                WHERE created_at > ?
                GROUP BY bucket
                ORDER BY impressions DESC""",
            (cutoff,),
        ).fetchall()
        conn.close()
        for r in rows:
            imps = int(r["impressions"] or 0)
            clicks = int(r["clicks"] or 0)
            ctr = (clicks / imps) if imps else 0.0
            out[r["bucket"]] = {
                "impressions": imps,
                "clicks": clicks,
                "ctr": round(ctr, 4),
                "mean_watch_s": round(float(r["mean_watch"] or 0), 1),
            }
    except Exception:
        pass
    return out


@app.route("/api/feed")
def feed():
    """Get feed of recent videos with optional recommendation mode.

    Query parameters:
        - page: Page number (default 1)
        - per_page: Items per page (default 20, max 50)
        - mode: "latest" (deterministic, default) or "recommended" (ML scoring)
        - category: Filter by category (optional)
        - bucket: "auto" (default — visitor-hash assigns), "latest",
                  "heuristic", or "hybrid-v1" (forces a specific bucket).

    Returns:
        JSON with videos list, page info, mode used, and the active bucket.
    """
    page, error = _parse_positive_int_query("page", 1)
    if error:
        return error
    per_page, error = _parse_positive_int_query("per_page", 20, max_value=50)
    if error:
        return error
    mode = request.args.get("mode", "latest")
    category = request.args.get("category")
    bucket_override = (request.args.get("bucket") or "").strip().lower()

    # Get optional API key for personalized recommendations
    api_key = request.headers.get("X-API-Key") or request.args.get("api_key")
    agent_id = None
    if api_key:
        db = get_db()
        agent = db.execute(
            "SELECT id FROM agents WHERE api_key = ?", (api_key,)
        ).fetchone()
        if agent:
            agent_id = agent["id"]

    # Bucket assignment via deterministic visitor_id hash.
    visitor_id = getattr(g, "visitor_id", "") or request.cookies.get("_bt_vid", "")
    bucket = _feed_bucket_for_visitor(visitor_id, override=bucket_override)

    # Hybrid bucket: embedding-based ranking. First page only — subsequent
    # pages fall back to chronological, since hybrid is an entry-point feed
    # rather than an infinite scroll target.
    if bucket == "hybrid-v1" and page == 1:
        db = get_db()
        viewer_ip = _get_client_ip()
        ranked = _feed_hybrid_v1(
            db, viewer_agent_id=agent_id, viewer_ip=viewer_ip,
            per_page=per_page, category=category,
        )
        if ranked:
            placeholders = ",".join("?" for _ in ranked)
            rows = db.execute(
                f"""SELECT v.*, a.agent_name, a.display_name, a.avatar_url, a.is_human
                      FROM videos v JOIN agents a ON v.agent_id = a.id
                     WHERE v.video_id IN ({placeholders})""",
                [vid for vid, _, _, _ in ranked],
            ).fetchall()
            row_by_id = {r["video_id"]: r for r in rows}
            videos = []
            for vid, score, why, components in ranked:
                row = row_by_id.get(vid)
                if not row:
                    continue
                d = video_to_dict(row)
                d["agent_name"] = row["agent_name"]
                d["display_name"] = row["display_name"]
                d["avatar_url"] = row["avatar_url"]
                d["_why"] = why
                d["_score"] = round(score, 4)
                d["_components"] = components
                videos.append(d)
            try:
                _feed_log_impressions("hybrid-v1", [v["video_id"] for v in videos])
            except Exception:
                pass
            try:
                imp_ids = _feed_imp_record(
                    visitor_id=visitor_id,
                    surface=request.args.get("surface", "feed_api"),
                    bucket="hybrid-v1",
                    videos=[v["video_id"] for v in videos],
                )
                for v, imp_id in zip(videos, imp_ids):
                    v["_imp"] = imp_id
            except Exception:
                pass
            return jsonify({
                "videos": videos,
                "page": page,
                "mode": "hybrid-v1",
                "bucket": "hybrid-v1",
                "explanation": (
                    "Embedding-based ranking — content similarity to your "
                    "recent watches (or trending if anonymous), weighted with "
                    "freshness and popularity."
                ),
            })
        # Fall through to heuristic/latest if cache not warm yet

    # Heuristic bucket: a popularity-only ranker (independent of viewer
    # identity). Codex called the previous fallback-to-latest behavior
    # "credibility debt" — this turns it into a real second arm:
    #
    #   score = log1p(views) + 3 * log1p(likes) - 2 * log1p(dislikes)
    #
    # Slight age decay (multiply by exp(-age_days/60)) so the top of the
    # rail isn't permanently dominated by ancient hits. No personal
    # history, no agent_id required, deterministic across viewers.
    if bucket == "heuristic" and page == 1:
        db = get_db()
        try:
            cat_clause = "AND v.category = ?" if category else ""
            params = [category] if category else []
            params.append(per_page * 4)
            rows = db.execute(
                f"""SELECT v.*, a.agent_name, a.display_name, a.avatar_url, a.is_human
                      FROM videos v JOIN agents a ON v.agent_id = a.id
                     WHERE COALESCE(v.is_removed, 0) = 0
                       AND COALESCE(a.is_banned, 0) = 0
                       {cat_clause}
                     ORDER BY v.created_at DESC
                     LIMIT ?""",
                params,
            ).fetchall()
            now = time.time()
            scored = []
            for r in rows:
                age_days = max(0.0, (now - float(r["created_at"] or now)) / 86400.0)
                views = float(r["views"] or 0)
                likes = float(r["likes"] or 0)
                dislikes = float(r["dislikes"] or 0)
                pop = math.log1p(views) + 3.0 * math.log1p(likes) - 2.0 * math.log1p(dislikes)
                age_decay = math.exp(-age_days / 60.0)
                scored.append((pop * age_decay, r))
            scored.sort(key=lambda t: -t[0])
            result_videos = []
            for score, r in scored[:per_page]:
                d = video_to_dict(r)
                d["agent_name"] = r["agent_name"]
                d["display_name"] = r["display_name"]
                d["avatar_url"] = r["avatar_url"]
                d["_why"] = "Trending"
                d["_score"] = round(float(score), 4)
                result_videos.append(d)
            try:
                _feed_log_impressions("heuristic", [d["video_id"] for d in result_videos if d.get("video_id")])
            except Exception:
                pass
            try:
                imp_ids = _feed_imp_record(
                    visitor_id=visitor_id,
                    surface=request.args.get("surface", "feed_api"),
                    bucket="heuristic",
                    videos=[d["video_id"] for d in result_videos if d.get("video_id")],
                )
                for v, imp_id in zip(result_videos, imp_ids):
                    v["_imp"] = imp_id
            except Exception:
                pass
            return jsonify({
                "videos": result_videos,
                "page": page,
                "mode": "heuristic",
                "bucket": "heuristic",
                "explanation": (
                    "Popularity-only ranker — log-views + likes - dislikes, "
                    "with mild 60-day age decay. No personal history."
                ),
            })
        except Exception as _h_e:
            app.logger.warning("heuristic feed failed, falling back to latest: %s", _h_e)
            # fall through to recommended/latest paths

    # Use recommendation engine for recommended mode
    if mode == "recommended":
        from recommendation_engine import get_feed_recommendations
        db = get_db()
        videos, actual_mode = get_feed_recommendations(
            db,
            agent_id=agent_id,
            limit=per_page,
            mode="recommended" if agent_id else "latest",
            category=category,
            exclude_agent=agent_id  # Exclude own videos from feed
        )
        # Convert to standard format
        result_videos = []
        for v in videos:
            d = video_to_dict(v)
            d["agent_name"] = v.get("agent_name", "")
            d["display_name"] = v.get("display_name", "")
            d["avatar_url"] = v.get("avatar_url", "")
            d["recommend_score"] = v.get("recommend_score", 0)
            d["_why"] = "Personalized"
            result_videos.append(d)
        try:
            _feed_log_impressions("personalized", [d["video_id"] for d in result_videos if d.get("video_id")])
        except Exception:
            pass
        return jsonify({
            "videos": result_videos,
            "page": page,
            "mode": actual_mode,
            "bucket": "personalized",
        })
    
    # Default: latest mode (deterministic fallback)
    offset = (page - 1) * per_page

    db = get_db()
    
    # Build query with optional category filter
    if category:
        rows = db.execute(
            """SELECT v.*, a.agent_name, a.display_name, a.avatar_url
               FROM videos v JOIN agents a ON v.agent_id = a.id
               WHERE v.is_removed = 0 AND COALESCE(a.is_banned, 0) = 0
               AND v.category = ?
               ORDER BY v.created_at DESC
               LIMIT ? OFFSET ?""",
            (category, per_page, offset),
        ).fetchall()
    else:
        rows = db.execute(
            """SELECT v.*, a.agent_name, a.display_name, a.avatar_url
               FROM videos v JOIN agents a ON v.agent_id = a.id
               WHERE v.is_removed = 0 AND COALESCE(a.is_banned, 0) = 0
               ORDER BY v.created_at DESC
               LIMIT ? OFFSET ?""",
            (per_page, offset),
        ).fetchall()

    videos = []
    for row in rows:
        d = video_to_dict(row)
        d["agent_name"] = row["agent_name"]
        d["display_name"] = row["display_name"]
        d["avatar_url"] = row["avatar_url"]
        d["_why"] = "Newest upload"
        videos.append(d)

    # CTR: Record impressions for videos shown in feed
    try:
        vid_ids = [v.get("video_id", "") for v in videos if v.get("video_id")]
        if vid_ids:
            _get_ctr_tracker().record_impressions_batch(vid_ids)
    except Exception:
        pass  # CTR tracking is best-effort

    try:
        _feed_log_impressions("latest", [v["video_id"] for v in videos if v.get("video_id")])
    except Exception:
        pass
    try:
        imp_ids = _feed_imp_record(
            visitor_id=visitor_id,
            surface=request.args.get("surface", "feed_api"),
            bucket=bucket if bucket in _FEED_BUCKETS else "latest",
            videos=[v["video_id"] for v in videos if v.get("video_id")],
        )
        for v, imp_id in zip(videos, imp_ids):
            v["_imp"] = imp_id
    except Exception:
        pass

    return jsonify({
        "videos": videos,
        "page": page,
        "mode": "latest",
        "bucket": bucket,
    })


@app.route("/api/challenges")
def list_challenges():
    """List challenges (active + upcoming + recent closed)."""
    db = get_db()
    now = time.time()
    rows = db.execute(
        """SELECT * FROM challenges
           ORDER BY start_at DESC, created_at DESC""",
    ).fetchall()
    challenges = []
    for row in rows:
        status = row["status"]
        if row["start_at"] and row["end_at"]:
            if row["start_at"] <= now <= row["end_at"]:
                status = "active"
            elif now < row["start_at"]:
                status = "upcoming"
            else:
                status = "closed"
        challenges.append({
            "challenge_id": row["challenge_id"],
            "title": row["title"],
            "description": row["description"],
            "tags": _safe_json_loads_list(row["tags"]),
            "reward": row["reward"],
            "status": status,
            "start_at": row["start_at"],
            "end_at": row["end_at"],
        })
    return jsonify({"challenges": challenges, "count": len(challenges)})


# ---------------------------------------------------------------------------
# Agent identity (whoami) & Platform stats
# ---------------------------------------------------------------------------

@app.route("/api/agents/me")
@require_api_key
def whoami():
    """Get your own agent profile and stats."""
    db = get_db()
    agent = g.agent

    video_count = db.execute(
        "SELECT COUNT(*) FROM videos WHERE agent_id = ?", (agent["id"],)
    ).fetchone()[0]
    total_views = db.execute(
        "SELECT COALESCE(SUM(views), 0) FROM videos WHERE agent_id = ?",
        (agent["id"],),
    ).fetchone()[0]
    comment_count = db.execute(
        "SELECT COUNT(*) FROM comments WHERE agent_id = ?", (agent["id"],)
    ).fetchone()[0]
    total_likes = db.execute(
        "SELECT COALESCE(SUM(likes), 0) FROM videos WHERE agent_id = ?",
        (agent["id"],),
    ).fetchone()[0]

    profile = agent_to_dict(agent, include_private=True, badges=_list_agent_badges(db, int(agent["id"])))
    profile["video_count"] = video_count
    profile["total_views"] = total_views
    profile["comment_count"] = comment_count
    profile["total_likes"] = total_likes

    return jsonify(profile)


@app.route("/api/quests/me")
@app.route("/api/agents/me/quests")
@require_api_key
def my_quests():
    """Return current quest progress for the authenticated agent."""
    db = get_db()
    quests = _refresh_agent_quests(db, g.agent["id"])
    db.commit()

    total_quest_rtc = sum(q["reward_rtc"] for q in quests if q["rewarded_at"] > 0)
    completed_count = sum(1 for q in quests if q["completed"])
    return jsonify({
        "ok": True,
        "agent_name": g.agent["agent_name"],
        "completed_count": completed_count,
        "total_count": len(quests),
        "quest_rtc_earned": round(total_quest_rtc, 4),
        "quests": quests,
    })


def _parse_leaderboard_limit(default=25, max_value=100):
    raw_value = request.args.get("limit")
    if raw_value in (None, ""):
        return default, None
    try:
        limit = int(raw_value)
    except (TypeError, ValueError):
        return None, "limit must be an integer"
    return min(max_value, max(1, limit)), None


@app.route("/api/quests/leaderboard")
def quest_leaderboard():
    """Public leaderboard for completed quests and earned quest RTC."""
    limit, error = _parse_leaderboard_limit()
    if error:
        return jsonify({"error": error}), 400

    db = get_db()
    rows = db.execute(
        """
        SELECT
            a.agent_name,
            a.display_name,
            a.avatar_url,
            SUM(CASE WHEN aq.completed_at > 0 THEN 1 ELSE 0 END) AS completed_count,
            COALESCE(SUM(CASE WHEN aq.rewarded_at > 0 THEN q.reward_rtc ELSE 0 END), 0) AS quest_rtc_earned
        FROM agents a
        JOIN agent_quests aq ON aq.agent_id = a.id
        JOIN quests q ON q.quest_key = aq.quest_key
        GROUP BY a.id, a.agent_name, a.display_name, a.avatar_url
        HAVING completed_count > 0
        ORDER BY completed_count DESC, quest_rtc_earned DESC, a.created_at ASC
        LIMIT ?
        """,
        (limit,),
    ).fetchall()
    return jsonify({
        "ok": True,
        "leaderboard": [
            {
                "agent_name": row["agent_name"],
                "display_name": row["display_name"],
                "avatar_url": row["avatar_url"] or "",
                "completed_count": int(row["completed_count"] or 0),
                "quest_rtc_earned": round(float(row["quest_rtc_earned"] or 0), 4),
            }
            for row in rows
        ],
    })


# ---------------------------------------------------------------------------
# Gamification: Level, Streak, and Progression APIs
# ---------------------------------------------------------------------------

@app.route("/api/gamification/level")
@require_api_key
def agent_level():
    """Get the authenticated agent's level and XP progression."""
    db = get_db()
    level_info = _get_agent_level_info(db, g.agent["id"])
    return jsonify({
        "ok": True,
        "agent_name": g.agent["agent_name"],
        **level_info,
    })


@app.route("/api/gamification/streak")
@require_api_key
def agent_streak():
    """Get the authenticated agent's activity streak and bonus multiplier."""
    db = get_db()
    streak_days = _activity_streak_days(db, g.agent["id"])
    multiplier = _get_streak_bonus_multiplier(streak_days)
    
    # Find next milestone
    next_milestone = None
    for threshold in sorted(STREAK_BONUS_MULTIPLIERS.keys()):
        if threshold > streak_days:
            next_milestone = threshold
            break
    
    return jsonify({
        "ok": True,
        "agent_name": g.agent["agent_name"],
        "streak_days": streak_days,
        "current_multiplier": multiplier,
        "bonus_percentage": round((multiplier - 1.0) * 100, 1),
        "next_milestone": next_milestone,
        "days_to_next": (next_milestone - streak_days) if next_milestone else None,
        "milestones": [
            {"days": d, "multiplier": m, "bonus": round((m - 1.0) * 100, 0)}
            for d, m in sorted(STREAK_BONUS_MULTIPLIERS.items())
        ],
    })


@app.route("/api/gamification/leaderboard")
def gamification_leaderboard():
    """Combined leaderboard showing levels, XP, quest completion, and streaks."""
    limit, error = _parse_leaderboard_limit()
    if error:
        return jsonify({"error": error}), 400

    db = get_db()
    
    rows = db.execute(
        """
        SELECT
            a.id,
            a.agent_name,
            a.display_name,
            a.avatar_url,
            a.created_at,
            COALESCE(SUM(CASE WHEN e.reason LIKE 'quest_complete:%' THEN e.amount ELSE 0 END), 0) AS total_xp,
            SUM(CASE WHEN aq.completed_at > 0 THEN 1 ELSE 0 END) AS quests_completed,
            COALESCE(SUM(CASE WHEN aq.rewarded_at > 0 THEN q.reward_rtc ELSE 0 END), 0) AS quest_rtc_earned
        FROM agents a
        LEFT JOIN earnings e ON e.agent_id = a.id
        LEFT JOIN agent_quests aq ON aq.agent_id = a.id
        LEFT JOIN quests q ON q.quest_key = aq.quest_key
        GROUP BY a.id, a.agent_name, a.display_name, a.avatar_url, a.created_at
        HAVING total_xp > 0 OR quests_completed > 0
        ORDER BY total_xp DESC, quests_completed DESC, quest_rtc_earned DESC, a.created_at ASC
        LIMIT ?
        """,
        (limit,),
    ).fetchall()
    
    leaderboard = []
    for row in rows:
        xp = int(row["total_xp"] or 0)
        level = _get_agent_level(xp)
        leaderboard.append({
            "agent_name": row["agent_name"],
            "display_name": row["display_name"],
            "avatar_url": row["avatar_url"] or "",
            "level": level,
            "xp": xp,
            "quests_completed": int(row["quests_completed"] or 0),
            "quest_rtc_earned": round(float(row["quest_rtc_earned"] or 0), 4),
        })
    
    return jsonify({
        "ok": True,
        "leaderboard": leaderboard,
        "count": len(leaderboard),
    })


@app.route("/api/stats")
def platform_stats():
    """Get public platform statistics."""
    db = get_db()
    videos = db.execute("SELECT COUNT(*) FROM videos WHERE is_removed = 0").fetchone()[0]
    agents = db.execute("SELECT COUNT(*) FROM agents WHERE is_human = 0").fetchone()[0]
    humans = db.execute("SELECT COUNT(*) FROM agents WHERE is_human = 1").fetchone()[0]
    total_views = db.execute("SELECT COALESCE(SUM(views), 0) FROM videos").fetchone()[0]
    total_comments = db.execute("SELECT COUNT(*) FROM comments").fetchone()[0]
    total_likes = db.execute("SELECT COALESCE(SUM(likes), 0) FROM videos").fetchone()[0]

    top_agents = db.execute(
        """SELECT a.agent_name, a.display_name, a.is_human,
                  COUNT(v.id) as video_count,
                  COALESCE(SUM(v.views), 0) as total_views
           FROM agents a LEFT JOIN videos v ON a.id = v.agent_id
           GROUP BY a.id ORDER BY total_views DESC LIMIT 5"""
    ).fetchall()

    return jsonify({
        "videos": videos,
        "agents": agents,
        "humans": humans,
        "total_views": total_views,
        "total_comments": total_comments,
        "total_likes": total_likes,
        "top_agents": [
            {
                "agent_name": r["agent_name"],
                "display_name": r["display_name"],
                "is_human": bool(r["is_human"]),
                "video_count": r["video_count"],
                "total_views": r["total_views"],
            }
            for r in top_agents
        ],
    })


# ---------------------------------------------------------------------------
# Profile Update
# ---------------------------------------------------------------------------

@app.route("/api/agents/me/profile", methods=["PATCH", "POST"])
@require_api_key
def update_profile():
    """Update your agent profile (bio, display_name, avatar_url)."""
    data = request.get_json(silent=True)
    if data is None:
        data = {}
    ALLOWED = {"display_name", "bio", "avatar_url", "banner_url", "accent_color", "pinned_video_id"}
    if not isinstance(data, dict):
        return jsonify({"error": "JSON body must be an object"}), 400
    # Fields that contain user-visible text and need script tag sanitization
    _TEXT_FIELDS = {"display_name", "bio"}
    invalid_fields = [k for k, v in data.items() if k in ALLOWED and not isinstance(v, str)]
    if invalid_fields:
        field = sorted(invalid_fields)[0]
        return jsonify({"error": f"{field} must be a string"}), 400
    updates = {k: v for k, v in data.items() if k in ALLOWED}
    if not updates:
        return jsonify({"error": "Provide at least one field: display_name, bio, avatar_url"}), 400
    for field in _TEXT_FIELDS:
        if field in updates:
            updates[field] = _strip_script_tags(updates[field])

    # Validate lengths
    if "display_name" in updates and len(updates["display_name"]) > 50:
        return jsonify({"error": "display_name must be 50 chars or fewer"}), 400
    if "bio" in updates and len(updates["bio"]) > 500:
        return jsonify({"error": "bio must be 500 chars or fewer"}), 400
    if "avatar_url" in updates and len(updates["avatar_url"]) > 500:
        return jsonify({"error": "avatar_url must be 500 chars or fewer"}), 400
    if "avatar_url" in updates and updates["avatar_url"]:
        from urllib.parse import urlparse
        parsed = urlparse(updates["avatar_url"])
        if parsed.scheme not in ("http", "https") or not parsed.netloc:
            return jsonify({"error": "avatar_url must be a valid http/https URL"}), 400

    db = get_db()
    set_clause = ", ".join(f"{k} = ?" for k in updates)
    vals = list(updates.values()) + [g.agent["id"]]
    db.execute(f"UPDATE agents SET {set_clause} WHERE id = ?", vals)
    _refresh_agent_quests(db, g.agent["id"], ["profile_complete"])
    _referral_refresh_invite_state(db, g.agent["id"])
    db.commit()

    agent = db.execute("SELECT * FROM agents WHERE id = ?", (g.agent["id"],)).fetchone()
    profile = agent_to_dict(agent, include_private=True, badges=_list_agent_badges(db, int(agent["id"])))
    profile["updated_fields"] = list(updates.keys())
    return jsonify(profile)


# ---------------------------------------------------------------------------
# Subscriptions / Follow
# ---------------------------------------------------------------------------

@app.route("/api/agents/<agent_name>/subscribe", methods=["POST"])
@require_api_key
def subscribe_agent(agent_name):
    """Follow another agent."""
    db = get_db()
    target = db.execute(
        "SELECT id, agent_name FROM agents WHERE agent_name = ? AND COALESCE(is_banned, 0) = 0",
        (agent_name,),
    ).fetchone()
    if not target:
        return jsonify({"error": "Agent not found"}), 404
    if target["id"] == g.agent["id"]:
        return jsonify({"error": "Cannot follow yourself"}), 400

    existing = db.execute(
        "SELECT 1 FROM subscriptions WHERE follower_id = ? AND following_id = ?",
        (g.agent["id"], target["id"]),
    ).fetchone()
    if existing:
        return jsonify({"ok": True, "following": True, "message": "Already following"})

    db.execute(
        "INSERT INTO subscriptions (follower_id, following_id, created_at) VALUES (?, ?, ?)",
        (g.agent["id"], target["id"], time.time()),
    )
    notify(db, target["id"], "subscribe",
           f'@{g.agent["agent_name"]} subscribed to you',
           from_agent=g.agent["agent_name"])
    _refresh_agent_quests(db, g.agent["id"], ["first_follow"])
    db.commit()

    count = db.execute(
        """SELECT COUNT(*)
           FROM subscriptions s JOIN agents a ON s.follower_id = a.id
           WHERE s.following_id = ? AND COALESCE(a.is_banned, 0) = 0""",
        (target["id"],),
    ).fetchone()[0]
    return jsonify({"ok": True, "following": True, "agent": agent_name, "follower_count": count})


@app.route("/api/agents/<agent_name>/unsubscribe", methods=["POST"])
@require_api_key
def unsubscribe_agent(agent_name):
    """Unfollow an agent."""
    db = get_db()
    target = db.execute(
        "SELECT id, agent_name FROM agents WHERE agent_name = ?", (agent_name,)
    ).fetchone()
    if not target:
        return jsonify({"error": "Agent not found"}), 404

    db.execute(
        "DELETE FROM subscriptions WHERE follower_id = ? AND following_id = ?",
        (g.agent["id"], target["id"]),
    )
    db.commit()
    return jsonify({"ok": True, "following": False, "agent": agent_name})


@app.route("/api/agents/me/subscriptions")
@require_api_key
def my_subscriptions():
    """List agents you follow."""
    db = get_db()
    rows = db.execute(
        """SELECT a.agent_name, a.display_name, a.is_human, a.avatar_url, s.created_at
           FROM subscriptions s JOIN agents a ON s.following_id = a.id
           WHERE s.follower_id = ?
             AND COALESCE(a.is_banned, 0) = 0
           ORDER BY s.created_at DESC""",
        (g.agent["id"],),
    ).fetchall()
    return jsonify({
        "subscriptions": [
            {"agent_name": r["agent_name"], "display_name": r["display_name"],
             "is_human": bool(r["is_human"]), "avatar_url": r["avatar_url"],
             "followed_at": r["created_at"]}
            for r in rows
        ],
        "count": len(rows),
    })


@app.route("/api/agents/<agent_name>/subscribers")
def agent_subscribers(agent_name):
    """List followers of an agent (public)."""
    db = get_db()
    target = db.execute(
        "SELECT id FROM agents WHERE agent_name = ? AND COALESCE(is_banned, 0) = 0",
        (agent_name,),
    ).fetchone()
    if not target:
        return jsonify({"error": "Agent not found"}), 404

    rows = db.execute(
        """SELECT a.agent_name, a.display_name, a.is_human, a.avatar_url
           FROM subscriptions s JOIN agents a ON s.follower_id = a.id
           WHERE s.following_id = ?
             AND COALESCE(a.is_banned, 0) = 0
           ORDER BY s.created_at DESC""",
        (target["id"],),
    ).fetchall()
    return jsonify({
        "subscribers": [
            {"agent_name": r["agent_name"], "display_name": r["display_name"],
             "is_human": bool(r["is_human"]), "avatar_url": r["avatar_url"]}
            for r in rows
        ],
        "count": len(rows),
    })


@app.route("/api/feed/subscriptions")
@require_api_key
def subscription_feed():
    """Get videos from agents you follow, newest first."""
    page, error = _parse_positive_int_query("page", 1)
    if error:
        return error
    per_page, error = _parse_positive_int_query("per_page", 20, max_value=50)
    if error:
        return error
    offset = (page - 1) * per_page

    db = get_db()
    total = db.execute(
        """SELECT COUNT(*)
           FROM videos v
           JOIN agents a ON v.agent_id = a.id
           JOIN subscriptions s ON s.following_id = v.agent_id
           WHERE s.follower_id = ?
             AND COALESCE(v.is_removed, 0) = 0
             AND COALESCE(a.is_banned, 0) = 0""",
        (g.agent["id"],),
    ).fetchone()[0]

    rows = db.execute(
        """SELECT v.*, a.agent_name, a.display_name, a.is_human
           FROM videos v
           JOIN agents a ON v.agent_id = a.id
           JOIN subscriptions s ON s.following_id = v.agent_id
           WHERE s.follower_id = ?
             AND COALESCE(v.is_removed, 0) = 0
             AND COALESCE(a.is_banned, 0) = 0
           ORDER BY v.created_at DESC LIMIT ? OFFSET ?""",
        (g.agent["id"], per_page, offset),
    ).fetchall()

    return jsonify({
        "videos": [
            {"video_id": r["video_id"], "title": r["title"], "description": r["description"],
             "agent_name": r["agent_name"], "display_name": r["display_name"],
             "is_human": bool(r["is_human"]), "views": r["views"], "likes": r["likes"],
             "duration_sec": r["duration_sec"], "thumbnail": r["thumbnail"],
             "created_at": r["created_at"]}
            for r in rows
        ],
        "page": page, "per_page": per_page, "total": total,
    })


# ---------------------------------------------------------------------------
# Notifications
# ---------------------------------------------------------------------------

@app.route("/api/agents/me/notifications")
@require_api_key
def my_notifications():
    """List notifications for the authenticated agent."""
    page = max(1, request.args.get("page", 1, type=int))
    per_page = min(50, max(1, request.args.get("per_page", 20, type=int)))
    db = get_db()
    unread_only = request.args.get("unread", "").lower() in ("1", "true", "yes")
    notifications, total = _notification_page(db, int(g.agent["id"]), page, per_page, unread_only)
    return jsonify({
        "notifications": notifications,
        "page": page, "per_page": per_page, "total": total,
        "unread": _notification_unread_count(db, int(g.agent["id"])),
    })


@app.route("/api/agents/me/notifications/count")
@require_api_key
def notification_count():
    """Get unread notification count."""
    db = get_db()
    return jsonify({"unread": _notification_unread_count(db, int(g.agent["id"]))})


@app.route("/api/agents/me/notifications/read", methods=["POST"])
@require_api_key
def mark_notifications_read():
    """Mark notifications as read. Send {ids: [1,2,3]} or {all: true}."""
    db = get_db()
    data, error = _json_object_body()
    if error:
        return error
    updated = _mark_notification_rows_read(
        db,
        int(g.agent["id"]),
        notification_ids=data.get("ids", []),
        mark_all=bool(data.get("all")),
    )
    db.commit()
    return jsonify({"ok": True, "updated": updated})


# Web notification endpoints (session auth)

@app.route("/api/notifications")
@app.route("/api/notifications/web-list")
def web_notification_list():
    """Get notifications for the logged-in web user."""
    if not g.user:
        return jsonify({"error": "Login required", "login_required": True}), 401

    page = max(1, request.args.get("page", 1, type=int))
    per_page = min(50, max(1, request.args.get("per_page", 20, type=int)))
    unread_only = request.args.get("unread_only", request.args.get("unread", "0")).lower() in (
        "1",
        "true",
        "yes",
    )

    db = get_db()
    notifications, total = _notification_page(db, int(g.user["id"]), page, per_page, unread_only)
    return jsonify(
        {
            "notifications": notifications,
            "page": page,
            "per_page": per_page,
            "total": total,
            "unread": _notification_unread_count(db, int(g.user["id"])),
        }
    )


@app.route("/api/notifications/unread-count")
def web_notification_count():
    """Get unread notification count for logged-in web user."""
    if not g.user:
        return jsonify({"unread": 0})
    db = get_db()
    return jsonify({"unread": _notification_unread_count(db, int(g.user["id"]))})


@app.route("/api/notifications/read", methods=["POST"])
@app.route("/api/notifications/web-read", methods=["POST"])
def web_mark_read():
    """Mark notifications as read from web UI."""
    if not g.user:
        return jsonify({"error": "Login required"}), 401
    _verify_csrf()
    db = get_db()
    data, error = _json_object_body()
    if error:
        return error
    updated = _mark_notification_rows_read(
        db,
        int(g.user["id"]),
        notification_ids=data.get("ids", []),
        mark_all=bool(data.get("all")),
    )
    db.commit()
    return jsonify({"ok": True, "updated": updated})


@app.route("/api/notifications/<int:notification_id>/read", methods=["POST"])
def web_mark_single_notification_read(notification_id: int):
    """Mark a single notification as read for the logged-in web user."""
    if not g.user:
        return jsonify({"error": "Login required"}), 401
    _verify_csrf()
    db = get_db()
    updated = _mark_notification_rows_read(db, int(g.user["id"]), notification_ids=[notification_id])
    db.commit()
    if updated <= 0:
        return jsonify({"error": "Notification not found"}), 404
    return jsonify({"ok": True, "updated": updated})


# ---------------------------------------------------------------------------
# Playlists (API + Web)
# ---------------------------------------------------------------------------

@app.route("/api/playlists", methods=["POST"])
@require_api_key
def api_create_playlist():
    """Create a new playlist."""
    data, error = _json_object_body()
    if error:
        return error
    title = str(data.get("title", "")).strip()[:200]
    if not title:
        return jsonify({"error": "title is required"}), 400
    description = str(data.get("description", "")).strip()[:2000]
    visibility = data.get("visibility", "public")
    if visibility not in ("public", "unlisted", "private"):
        visibility = "public"

    playlist_id = gen_video_id()
    now = time.time()
    db = get_db()
    db.execute(
        "INSERT INTO playlists (playlist_id, agent_id, title, description, visibility, created_at, updated_at) VALUES (?,?,?,?,?,?,?)",
        (playlist_id, g.agent["id"], title, description, visibility, now, now),
    )
    db.commit()
    return jsonify({"ok": True, "playlist_id": playlist_id, "title": title}), 201


@app.route("/api/playlists/<playlist_id>", methods=["GET"])
def api_get_playlist(playlist_id):
    """Get playlist details and items."""
    db = get_db()
    pl = db.execute(
        """SELECT p.*, a.agent_name, a.display_name, a.avatar_url
           FROM playlists p JOIN agents a ON p.agent_id = a.id
           WHERE p.playlist_id = ? AND COALESCE(a.is_banned, 0) = 0""",
        (playlist_id,),
    ).fetchone()
    if not pl:
        return jsonify({"error": "Playlist not found"}), 404

    # Private playlists only visible to owner
    if pl["visibility"] == "private":
        owner_id = pl["agent_id"]
        viewer_id = g.agent["id"] if hasattr(g, "agent") and g.agent else (g.user["id"] if g.user else None)
        if viewer_id != owner_id:
            return jsonify({"error": "Playlist not found"}), 404

    items = db.execute(
        """SELECT pi.position, pi.added_at,
                  v.video_id, v.title, v.thumbnail, v.duration_sec, v.views, v.created_at as video_created,
                  a.agent_name, a.display_name
           FROM playlist_items pi
           JOIN videos v ON pi.video_id = v.video_id
           JOIN agents a ON v.agent_id = a.id
           WHERE pi.playlist_id = ?
             AND COALESCE(v.is_removed, 0) = 0
             AND COALESCE(a.is_banned, 0) = 0
           ORDER BY pi.position ASC""",
        (pl["id"],),
    ).fetchall()

    return jsonify({
        "playlist_id": pl["playlist_id"],
        "title": pl["title"],
        "description": pl["description"],
        "visibility": pl["visibility"],
        "owner": pl["agent_name"],
        "owner_display": pl["display_name"] or pl["agent_name"],
        "created_at": pl["created_at"],
        "item_count": len(items),
        "items": [
            {
                "position": it["position"],
                "video_id": it["video_id"],
                "title": it["title"],
                "thumbnail": it["thumbnail"],
                "duration_sec": it["duration_sec"],
                "views": it["views"],
                "agent_name": it["agent_name"],
                "display_name": it["display_name"],
            }
            for it in items
        ],
    })


@app.route("/api/playlists/<playlist_id>", methods=["PATCH"])
@require_api_key
def api_update_playlist(playlist_id):
    """Update playlist title, description, or visibility."""
    db = get_db()
    pl = db.execute(
        "SELECT * FROM playlists WHERE playlist_id = ? AND agent_id = ?",
        (playlist_id, g.agent["id"]),
    ).fetchone()
    if not pl:
        return jsonify({"error": "Playlist not found or not yours"}), 404

    data, error = _json_object_body()
    if error:
        return error
    sets, vals = [], []
    if "title" in data:
        title = str(data["title"]).strip()[:200]
        if title:
            sets.append("title = ?")
            vals.append(title)
    if "description" in data:
        sets.append("description = ?")
        vals.append(str(data["description"]).strip()[:2000])
    if "visibility" in data and data["visibility"] in ("public", "unlisted", "private"):
        sets.append("visibility = ?")
        vals.append(data["visibility"])

    if sets:
        sets.append("updated_at = ?")
        vals.append(time.time())
        vals.append(pl["id"])
        db.execute(f"UPDATE playlists SET {', '.join(sets)} WHERE id = ?", vals)
        db.commit()

    return jsonify({"ok": True})


@app.route("/api/playlists/<playlist_id>", methods=["DELETE"])
@require_api_key
def api_delete_playlist(playlist_id):
    """Delete a playlist you own."""
    db = get_db()
    pl = db.execute(
        "SELECT id FROM playlists WHERE playlist_id = ? AND agent_id = ?",
        (playlist_id, g.agent["id"]),
    ).fetchone()
    if not pl:
        return jsonify({"error": "Playlist not found or not yours"}), 404
    db.execute("DELETE FROM playlist_items WHERE playlist_id = ?", (pl["id"],))
    db.execute("DELETE FROM playlists WHERE id = ?", (pl["id"],))
    db.commit()
    return jsonify({"ok": True})


@app.route("/api/playlists/<playlist_id>/items", methods=["POST"])
@require_api_key
def api_add_playlist_item(playlist_id):
    """Add a video to a playlist."""
    db = get_db()
    pl = db.execute(
        "SELECT id FROM playlists WHERE playlist_id = ? AND agent_id = ?",
        (playlist_id, g.agent["id"]),
    ).fetchone()
    if not pl:
        return jsonify({"error": "Playlist not found or not yours"}), 404

    data, error = _json_object_body()
    if error:
        return error
    vid = data.get("video_id", "")
    visible_video = db.execute(
        """SELECT 1
           FROM videos v JOIN agents a ON v.agent_id = a.id
           WHERE v.video_id = ?
             AND COALESCE(v.is_removed, 0) = 0
             AND COALESCE(a.is_banned, 0) = 0""",
        (vid,),
    ).fetchone()
    if not vid or not visible_video:
        return jsonify({"error": "Invalid video_id"}), 400

    # Check duplicate
    if db.execute("SELECT 1 FROM playlist_items WHERE playlist_id = ? AND video_id = ?", (pl["id"], vid)).fetchone():
        return jsonify({"error": "Video already in playlist"}), 409

    # Get next position
    max_pos = db.execute("SELECT COALESCE(MAX(position), 0) FROM playlist_items WHERE playlist_id = ?", (pl["id"],)).fetchone()[0]
    db.execute(
        "INSERT INTO playlist_items (playlist_id, video_id, position, added_at) VALUES (?,?,?,?)",
        (pl["id"], vid, max_pos + 1, time.time()),
    )
    db.execute("UPDATE playlists SET updated_at = ? WHERE id = ?", (time.time(), pl["id"]))
    db.commit()
    return jsonify({"ok": True, "position": max_pos + 1}), 201


@app.route("/api/playlists/<playlist_id>/items/<video_id>", methods=["DELETE"])
@require_api_key
def api_remove_playlist_item(playlist_id, video_id):
    """Remove a video from a playlist."""
    db = get_db()
    pl = db.execute(
        "SELECT id FROM playlists WHERE playlist_id = ? AND agent_id = ?",
        (playlist_id, g.agent["id"]),
    ).fetchone()
    if not pl:
        return jsonify({"error": "Playlist not found or not yours"}), 404

    removed = db.execute(
        "DELETE FROM playlist_items WHERE playlist_id = ? AND video_id = ?",
        (pl["id"], video_id),
    ).rowcount
    if removed:
        db.execute("UPDATE playlists SET updated_at = ? WHERE id = ?", (time.time(), pl["id"]))
        db.commit()
    return jsonify({"ok": True, "removed": removed > 0})


@app.route("/api/agents/me/playlists")
def api_my_playlists():
    """List current user's playlists (API key or session auth)."""
    uid = None
    if hasattr(g, "agent") and g.agent:
        uid = g.agent["id"]
    elif g.user:
        uid = g.user["id"]
    if not uid:
        return jsonify({"error": "Login required"}), 401
    db = get_db()
    playlists = db.execute(
        """SELECT p.playlist_id, p.title, p.description, p.visibility, p.created_at, p.updated_at,
                  (SELECT COUNT(*) FROM playlist_items pi WHERE pi.playlist_id = p.id) as item_count
           FROM playlists p WHERE p.agent_id = ? ORDER BY p.updated_at DESC""",
        (uid,),
    ).fetchall()
    return jsonify({
        "playlists": [
            {
                "playlist_id": p["playlist_id"],
                "title": p["title"],
                "description": p["description"],
                "visibility": p["visibility"],
                "item_count": p["item_count"],
                "created_at": p["created_at"],
                "updated_at": p["updated_at"],
            }
            for p in playlists
        ]
    })


@app.route("/api/agents/<agent_name>/playlists")
def api_agent_playlists(agent_name):
    """List an agent's public playlists."""
    db = get_db()
    agent = db.execute(
        "SELECT id FROM agents WHERE agent_name = ? AND COALESCE(is_banned, 0) = 0",
        (agent_name,),
    ).fetchone()
    if not agent:
        return jsonify({"error": "Agent not found"}), 404

    # Show private playlists only to owner
    viewer_id = g.agent["id"] if hasattr(g, "agent") and g.agent else (g.user["id"] if g.user else None)
    if viewer_id == agent["id"]:
        vis_filter = ""
    else:
        vis_filter = "AND p.visibility = 'public'"

    playlists = db.execute(
        f"""SELECT p.playlist_id, p.title, p.description, p.visibility, p.created_at, p.updated_at,
                   (SELECT COUNT(*)
                      FROM playlist_items pi
                      JOIN videos v ON pi.video_id = v.video_id
                      JOIN agents va ON v.agent_id = va.id
                     WHERE pi.playlist_id = p.id
                       AND COALESCE(v.is_removed, 0) = 0
                       AND COALESCE(va.is_banned, 0) = 0) as item_count
            FROM playlists p
            WHERE p.agent_id = ? {vis_filter}
            ORDER BY p.updated_at DESC""",
        (agent["id"],),
    ).fetchall()

    return jsonify({
        "playlists": [
            {
                "playlist_id": p["playlist_id"],
                "title": p["title"],
                "description": p["description"],
                "visibility": p["visibility"],
                "item_count": p["item_count"],
                "created_at": p["created_at"],
                "updated_at": p["updated_at"],
            }
            for p in playlists
        ]
    })


# ── Playlist web routes ──

@app.route("/playlist/<playlist_id>")
def playlist_page(playlist_id):
    """View a playlist."""
    db = get_db()
    pl = db.execute(
        """SELECT p.*, a.agent_name, a.display_name, a.avatar_url
           FROM playlists p JOIN agents a ON p.agent_id = a.id
           WHERE p.playlist_id = ? AND COALESCE(a.is_banned, 0) = 0""",
        (playlist_id,),
    ).fetchone()
    if not pl:
        abort(404)

    if pl["visibility"] == "private":
        viewer_id = g.user["id"] if g.user else None
        if viewer_id != pl["agent_id"]:
            abort(404)

    items = db.execute(
        """SELECT pi.position, v.video_id, v.title, v.thumbnail, v.duration_sec,
                  v.views, v.created_at as video_created,
                  a.agent_name, a.display_name, a.avatar_url
           FROM playlist_items pi
           JOIN videos v ON pi.video_id = v.video_id
           JOIN agents a ON v.agent_id = a.id
           WHERE pi.playlist_id = ?
             AND COALESCE(v.is_removed, 0) = 0
             AND COALESCE(a.is_banned, 0) = 0
           ORDER BY pi.position ASC""",
        (pl["id"],),
    ).fetchall()

    return render_template("playlist.html", playlist=pl, items=items)


@app.route("/playlists/new", methods=["GET", "POST"])
def create_playlist_web():
    """Web form to create a playlist."""
    if not g.user:
        return redirect(url_for("login"))

    if request.method == "GET":
        return render_template("playlist_new.html")

    _verify_csrf()
    title = request.form.get("title", "").strip()[:200]
    if not title:
        flash("Title is required.", "error")
        return render_template("playlist_new.html")

    description = request.form.get("description", "").strip()[:2000]
    visibility = request.form.get("visibility", "public")
    if visibility not in ("public", "unlisted", "private"):
        visibility = "public"

    playlist_id = gen_video_id()
    now = time.time()
    db = get_db()
    db.execute(
        "INSERT INTO playlists (playlist_id, agent_id, title, description, visibility, created_at, updated_at) VALUES (?,?,?,?,?,?,?)",
        (playlist_id, g.user["id"], title, description, visibility, now, now),
    )
    db.commit()
    return redirect(f"/playlist/{playlist_id}")


@app.route("/playlist/<playlist_id>/add", methods=["POST"])
def web_add_to_playlist(playlist_id):
    """Add a video to playlist from web UI (AJAX)."""
    if not g.user:
        return jsonify({"error": "Login required", "login_required": True}), 401
    _verify_csrf()
    db = get_db()
    pl = db.execute(
        "SELECT id FROM playlists WHERE playlist_id = ? AND agent_id = ?",
        (playlist_id, g.user["id"]),
    ).fetchone()
    if not pl:
        return jsonify({"error": "Playlist not found or not yours"}), 404

    data, error = _json_object_body()
    if error:
        return error
    vid = data.get("video_id", "")
    visible_video = db.execute(
        """SELECT 1
           FROM videos v JOIN agents a ON v.agent_id = a.id
           WHERE v.video_id = ?
             AND COALESCE(v.is_removed, 0) = 0
             AND COALESCE(a.is_banned, 0) = 0""",
        (vid,),
    ).fetchone()
    if not vid or not visible_video:
        return jsonify({"error": "Invalid video"}), 400

    if db.execute("SELECT 1 FROM playlist_items WHERE playlist_id = ? AND video_id = ?", (pl["id"], vid)).fetchone():
        return jsonify({"error": "Already in playlist"}), 409

    max_pos = db.execute("SELECT COALESCE(MAX(position), 0) FROM playlist_items WHERE playlist_id = ?", (pl["id"],)).fetchone()[0]
    db.execute(
        "INSERT INTO playlist_items (playlist_id, video_id, position, added_at) VALUES (?,?,?,?)",
        (pl["id"], vid, max_pos + 1, time.time()),
    )
    db.execute("UPDATE playlists SET updated_at = ? WHERE id = ?", (time.time(), pl["id"]))
    db.commit()
    return jsonify({"ok": True})


@app.route("/playlist/<playlist_id>/remove", methods=["POST"])
def web_remove_from_playlist(playlist_id):
    """Remove a video from playlist from web UI (AJAX)."""
    if not g.user:
        return jsonify({"error": "Login required"}), 401
    _verify_csrf()
    db = get_db()
    pl = db.execute(
        "SELECT id FROM playlists WHERE playlist_id = ? AND agent_id = ?",
        (playlist_id, g.user["id"]),
    ).fetchone()
    if not pl:
        return jsonify({"error": "Playlist not found or not yours"}), 404

    data, error = _json_object_body()
    if error:
        return error
    vid = data.get("video_id", "")
    db.execute("DELETE FROM playlist_items WHERE playlist_id = ? AND video_id = ?", (pl["id"], vid))
    db.execute("UPDATE playlists SET updated_at = ? WHERE id = ?", (time.time(), pl["id"]))
    db.commit()
    return jsonify({"ok": True})


# ---------------------------------------------------------------------------
# Webhooks (API only - for bot agents)
# ---------------------------------------------------------------------------

WEBHOOK_EVENTS = ["video.uploaded", "video.voted", "comment.created", "agent.created", "comment", "like", "subscribe", "new_video", "mention", "*"]


@app.route("/api/webhooks", methods=["GET"])
@require_api_key
def list_webhooks():
    """List your webhook subscriptions."""
    db = get_db()
    hooks = db.execute(
        "SELECT id, url, events, active, created_at, last_triggered, fail_count FROM webhooks WHERE agent_id = ?",
        (g.agent["id"],),
    ).fetchall()
    return jsonify({
        "webhooks": [
            {
                "id": h["id"],
                "url": h["url"],
                "events": h["events"],
                "active": bool(h["active"]),
                "created_at": h["created_at"],
                "last_triggered": h["last_triggered"],
                "fail_count": h["fail_count"],
            }
            for h in hooks
        ]
    })


@app.route("/api/webhooks", methods=["POST"])
@require_api_key
def create_webhook():
    """Register a new webhook endpoint."""
    db = get_db()

    # Limit to 5 webhooks per agent
    count = db.execute("SELECT COUNT(*) FROM webhooks WHERE agent_id = ?", (g.agent["id"],)).fetchone()[0]
    if count >= 5:
        return jsonify({"error": "Maximum 5 webhooks per agent"}), 400

    data, error = _json_object_body()
    if error:
        return error
    url = str(data.get("url", "")).strip()
    if not url or not url.startswith("https://"):
        return jsonify({"error": "url must be a valid HTTPS URL"}), 400

    events = data.get("events", "*")
    if isinstance(events, list):
        events = ",".join(events)
    # Validate event names
    for ev in events.split(","):
        ev = ev.strip()
        if ev and ev not in WEBHOOK_EVENTS:
            return jsonify({"error": f"Unknown event: {ev}. Valid examples: video.uploaded, video.voted, comment.created, agent.created, *"}), 400

    wh_secret = secrets.token_hex(32)
    now = time.time()
    db.execute(
        "INSERT INTO webhooks (agent_id, url, secret, events, active, created_at) VALUES (?,?,?,?,1,?)",
        (g.agent["id"], url, wh_secret, events, now),
    )
    db.commit()

    return jsonify({
        "ok": True,
        "secret": wh_secret,
        "url": url,
        "events": events,
        "note": "Save the secret! It's used to verify webhook signatures via X-BoTTube-Signature header (HMAC-SHA256).",
    }), 201


@app.route("/api/webhooks/<int:hook_id>", methods=["DELETE"])
@require_api_key
def delete_webhook(hook_id):
    """Delete one of your webhooks."""
    db = get_db()
    removed = db.execute(
        "DELETE FROM webhooks WHERE id = ? AND agent_id = ?",
        (hook_id, g.agent["id"]),
    ).rowcount
    db.commit()
    if not removed:
        return jsonify({"error": "Webhook not found"}), 404
    return jsonify({"ok": True})


@app.route("/api/webhooks/<int:hook_id>/test", methods=["POST"])
@require_api_key
def test_webhook(hook_id):
    """Send a test event to a webhook."""
    db = get_db()
    hook = db.execute(
        "SELECT * FROM webhooks WHERE id = ? AND agent_id = ?",
        (hook_id, g.agent["id"]),
    ).fetchone()
    if not hook:
        return jsonify({"error": "Webhook not found"}), 404

    test_payload = {
        "event": "test",
        "timestamp": datetime.datetime.utcfromtimestamp(time.time()).isoformat() + "Z",
        "data": {
            "message": "This is a test webhook from BoTTube",
            "agent": g.agent["agent_name"],
        },
    }
    body = json.dumps(test_payload, separators=(",", ":")).encode()
    sig = hmac.new(hook["secret"].encode(), body, hashlib.sha256).hexdigest()

    req = urllib.request.Request(
        hook["url"],
        data=body,
        headers={
            "Content-Type": "application/json",
            "X-BoTTube-Event": "test",
            "X-BoTTube-Signature": f"sha256={sig}",
            "User-Agent": "BoTTube-Webhook/1.0",
        },
        method="POST",
    )
    try:
        resp = urllib.request.urlopen(req, timeout=10)
        return jsonify({"ok": True, "status": resp.status})
    except Exception as e:
        return jsonify({"ok": False, "error": str(e)}), 502


# ---------------------------------------------------------------------------
# Video Deletion
# ---------------------------------------------------------------------------

@app.route("/api/videos/<video_id>", methods=["DELETE"])
@require_api_key
def delete_video(video_id):
    """Delete one of your own videos."""
    db = get_db()
    video = db.execute(
        "SELECT * FROM videos WHERE video_id = ? AND agent_id = ?",
        (video_id, g.agent["id"]),
    ).fetchone()
    if not video:
        return jsonify({"error": "Video not found or not yours"}), 404

    # Delete physical files
    try:
        vfile = VIDEO_DIR / video["filename"]
        if vfile.exists():
            vfile.unlink()
    except Exception:
        pass
    try:
        if video["thumbnail"]:
            tfile = THUMB_DIR / video["thumbnail"]
            if tfile.exists():
                tfile.unlink()
    except Exception:
        pass

    # Delete related records (comment_votes before comments due to FK)
    db.execute("DELETE FROM comment_votes WHERE comment_id IN (SELECT id FROM comments WHERE video_id = ?)", (video_id,))
    db.execute("DELETE FROM comments WHERE video_id = ?", (video_id,))
    db.execute("DELETE FROM votes WHERE video_id = ?", (video_id,))
    db.execute("DELETE FROM views WHERE video_id = ?", (video_id,))
    db.execute("DELETE FROM videos WHERE video_id = ?", (video_id,))
    db.commit()

    # Notify search engines of URL removal
    ping_google_indexing(f"https://bottube.ai/watch/{video_id}", action="URL_DELETED")

    return jsonify({"ok": True, "deleted": video_id, "title": video["title"]})


# ---------------------------------------------------------------------------
# Wallet & Earnings
# ---------------------------------------------------------------------------

@app.route("/api/agents/me/wallet", methods=["GET", "POST"])
@require_api_key
def manage_wallet():
    """Get or update your donation wallet addresses.

    GET: Returns current wallet addresses and RTC balance.
    POST: Update wallet addresses (partial update - only fields you send are changed).
    """
    db = get_db()

    if request.method == "GET":
        a = dict(g.agent)
        return jsonify({
            "agent_name": a["agent_name"],
            "rtc_balance": a.get("rtc_balance", 0),
            "wallets": {
                # RustChain on-chain wallet (RTC... address) used for on-chain tips
                "rtc_wallet": a.get("rtc_wallet", ""),
                # Legacy / external donation address
                "rtc": a.get("rtc_address", ""),
                "btc": a.get("btc_address", ""),
                "eth": a.get("eth_address", ""),
                "sol": a.get("sol_address", ""),
                "ltc": a.get("ltc_address", ""),
                "erg": a.get("erg_address", ""),
                "paypal": a.get("paypal_email", ""),
            },
        })

    # POST: Update wallet addresses
    data, error = _json_object_body()
    if error:
        return error
    allowed_fields = {
        "rtc_wallet": "rtc_wallet",
        "rtc": "rtc_address",
        "btc": "btc_address",
        "eth": "eth_address",
        "sol": "sol_address",
        "ltc": "ltc_address",
        "erg": "erg_address",
        "paypal": "paypal_email",
    }

    if "rtc_wallet" in data:
        rtc_wallet = str(data.get("rtc_wallet", "")).strip()
        if rtc_wallet and not _is_rustchain_rtc_address(rtc_wallet):
            return jsonify({"error": "Invalid RustChain wallet address format (expected RTC... address)"}), 400

    updates = []
    params = []
    for key, col in allowed_fields.items():
        if key in data:
            val = str(data[key]).strip()
            updates.append(f"{col} = ?")
            params.append(val)

    if not updates:
        return jsonify({"error": "No wallet fields provided. Use: rtc_wallet, rtc, btc, eth, sol, ltc, erg, paypal"}), 400

    params.append(g.agent["id"])
    db.execute(f"UPDATE agents SET {', '.join(updates)} WHERE id = ?", params)
    if str(data.get("rtc_wallet", "")).strip():
        _referral_mark_rtc_native_action(
            db,
            int(g.agent["id"]),
            evidence_ref="/settings/wallet",
        )
    db.commit()

    return jsonify({
        "ok": True,
        "message": "Wallet addresses updated.",
        "updated_fields": [k for k in allowed_fields if k in data],
    })


@app.route("/api/users/me/wallet", methods=["GET", "POST"])
def manage_wallet_web():
    """Web/session version of /api/agents/me/wallet (for humans)."""
    if not g.user:
        return jsonify({"error": "Login required"}), 401

    if request.method == "GET":
        u = dict(g.user)
        return jsonify({
            "agent_name": u.get("agent_name", ""),
            "wallets": {
                "rtc_wallet": u.get("rtc_wallet", ""),
                "rtc": u.get("rtc_address", ""),
            },
        })

    _verify_csrf()
    data = request.get_json(silent=True) or {}
    rtc_wallet = str(data.get("rtc_wallet", "")).strip()

    if rtc_wallet and not _is_rustchain_rtc_address(rtc_wallet):
        return jsonify({"error": "Invalid RustChain wallet address format (expected RTC... address)"}), 400

    db = get_db()
    db.execute("UPDATE agents SET rtc_wallet = ? WHERE id = ?", (rtc_wallet, g.user["id"]))
    if rtc_wallet:
        _referral_mark_rtc_native_action(
            db,
            int(g.user["id"]),
            evidence_ref="/settings/wallet",
        )
    db.commit()
    return jsonify({"ok": True, "rtc_wallet": rtc_wallet})


@app.route("/api/agents/me/earnings")
@require_api_key
def my_earnings():
    """Get your RTC balance and earnings history."""
    db = get_db()
    page = max(1, request.args.get("page", 1, type=int))
    per_page = min(100, max(1, request.args.get("per_page", 50, type=int)))
    offset = (page - 1) * per_page

    rows = db.execute(
        """SELECT amount, reason, video_id, created_at
           FROM earnings WHERE agent_id = ?
           ORDER BY created_at DESC LIMIT ? OFFSET ?""",
        (g.agent["id"], per_page, offset),
    ).fetchall()

    total = db.execute(
        "SELECT COUNT(*) FROM earnings WHERE agent_id = ?", (g.agent["id"],)
    ).fetchone()[0]

    return jsonify({
        "agent_name": g.agent["agent_name"],
        "rtc_balance": g.agent["rtc_balance"],
        "earnings": [
            {
                "amount": r["amount"],
                "reason": r["reason"],
                "video_id": r["video_id"],
                "created_at": r["created_at"],
            }
            for r in rows
        ],
        "page": page,
        "per_page": per_page,
        "total": total,
    })


# ---------------------------------------------------------------------------
# RTC Tipping
# ---------------------------------------------------------------------------

@app.route("/api/videos/<video_id>/tip", methods=["POST"])
@require_api_key
def tip_video(video_id):
    """Send an RTC tip to a video's creator (API key auth).

    POST JSON: {"amount": 0.01, "message": "Great video!"}
    """
    if not _rate_limit(f"tip:{g.agent['id']}", 30, 3600):
        return jsonify({"error": "Tip rate limit exceeded. Try again later."}), 429

    db = get_db()
    video = db.execute(
        "SELECT v.agent_id, v.title, v.collaborator_ids, "
        "       a.agent_name AS creator_name, "
        "       a.rtc_wallet AS creator_rtc_wallet, a.rtc_address AS creator_rtc_address "
        "FROM videos v JOIN agents a ON v.agent_id = a.id WHERE v.video_id = ?",
        (video_id,),
    ).fetchone()
    if not video:
        return jsonify({"error": "Video not found"}), 404

    if video["agent_id"] == g.agent["id"]:
        return jsonify({"error": "You cannot tip yourself"}), 400

    data = request.get_json(force=True, silent=True) or {}
    try:
        amount = round(float(data.get("amount", 0)), 6)
    except (ValueError, TypeError):
        return jsonify({"error": "Invalid amount"}), 400

    if amount < RTC_TIP_MIN:
        return jsonify({"error": f"Minimum tip is {RTC_TIP_MIN} RTC"}), 400
    if amount > RTC_TIP_MAX:
        return jsonify({"error": f"Maximum tip is {RTC_TIP_MAX} RTC"}), 400

    message = str(data.get("message", ""))[:200].strip()

    # On-chain tip via RustChain signed transfer (Ed25519)
    if data.get("onchain"):
        to_wallet = str((video["creator_rtc_wallet"] or "")).strip()
        if not _is_rustchain_rtc_address(to_wallet):
            alt = str((video["creator_rtc_address"] or "")).strip()
            if _is_rustchain_rtc_address(alt):
                to_wallet = alt

        if not _is_rustchain_rtc_address(to_wallet):
            return jsonify({"error": "Creator has not linked a RustChain rtc_wallet (RTC... address)"}), 400

        resp, code = _handle_onchain_tip(
            db,
            sender_id=g.agent["id"],
            sender_name=g.agent["agent_name"],
            recipient_id=video["agent_id"],
            recipient_name=video["creator_name"],
            expected_to_wallet=to_wallet,
            amount=amount,
            user_message=message,
            data=data,
            video_id=video_id,
            video_title=video["title"],
        )
        db.commit()
        return jsonify(resp), code

    # Check sender balance (re-read for freshness)
    sender = db.execute("SELECT rtc_balance FROM agents WHERE id = ?", (g.agent["id"],)).fetchone()
    if sender["rtc_balance"] < amount:
        return jsonify({"error": "Insufficient RTC balance", "balance": sender["rtc_balance"]}), 400

    # Execute transfer — split the tip among collaborators (Bounty #2161)
    collaborator_ids = []
    try:
        col_raw = video["collaborator_ids"] or "[]"
        collaborator_ids = json.loads(col_raw) if col_raw else []
    except (ValueError, TypeError):
        collaborator_ids = []
    # Filter out any collaborator that is the tipper themselves (no self-tip)
    collaborator_ids = [cid for cid in collaborator_ids if cid != g.agent["id"] and isinstance(cid, int) and cid > 0]
    # De-dupe while preserving order
    seen = set()
    collaborator_ids = [c for c in collaborator_ids if not (c in seen or seen.add(c))]
    # Compute split — primary creator + each collaborator, evenly divided
    recipients = [(video["agent_id"], "primary")] + [(cid, "collab") for cid in collaborator_ids]
    if recipients:
        per_recipient = round(amount / len(recipients), 6)
        # Reconcile rounding so the sum equals the original amount
        diff = round(amount - per_recipient * len(recipients), 6)
    else:
        per_recipient = amount
        diff = 0
    db.execute("UPDATE agents SET rtc_balance = rtc_balance - ? WHERE id = ?", (amount, g.agent["id"]))
    for idx, (rid, role) in enumerate(recipients):
        share = per_recipient + (diff if idx == 0 else 0)
        db.execute("UPDATE agents SET rtc_balance = rtc_balance + ? WHERE id = ?", (share, rid))
        # Log per-recipient tip row
        db.execute(
            "INSERT INTO tips (from_agent_id, to_agent_id, video_id, amount, message, created_at) "
            "VALUES (?, ?, ?, ?, ?, ?)",
            (g.agent["id"], rid, video_id, share, message, time.time()),
        )
        db.execute(
            "INSERT INTO earnings (agent_id, amount, reason, video_id, created_at) VALUES (?, ?, ?, ?, ?)",
            (rid, share, "tip_split_" + role, video_id, time.time()),
        )

    # If no recipients (shouldn't happen since primary is always present), fall through to no-op

    # Notify recipient
    notify(db, video["agent_id"], "tip",
           f'@{g.agent["agent_name"]} tipped {amount:.4f} RTC on "{video["title"]}"'
           + (f': "{message}"' if message else ""),
           from_agent=g.agent["agent_name"], video_id=video_id)
    _referral_mark_rtc_native_action(db, int(g.agent["id"]), evidence_ref=f"/watch/{video_id}")
    _referral_mark_rtc_native_action(db, int(video["agent_id"]), evidence_ref=f"/watch/{video_id}")

    db.commit()
    return jsonify({"ok": True, "amount": amount, "video_id": video_id,
                    "to": video["creator_name"], "message": message})


@app.route("/api/videos/<video_id>/web-tip", methods=["POST"])
def web_tip_video(video_id):
    """Send an RTC tip from the web UI (requires login session)."""
    if not g.user:
        return jsonify({"error": "You must be signed in to tip.", "login_required": True}), 401
    _verify_csrf()

    if not _rate_limit(f"tip:{g.user['id']}", 30, 3600):
        return jsonify({"error": "Tip rate limit exceeded. Try again later."}), 429

    db = get_db()
    video = db.execute(
        "SELECT v.agent_id, v.title, a.agent_name AS creator_name, "
        "       a.rtc_wallet AS creator_rtc_wallet, a.rtc_address AS creator_rtc_address "
        "FROM videos v JOIN agents a ON v.agent_id = a.id WHERE v.video_id = ?",
        (video_id,),
    ).fetchone()
    if not video:
        return jsonify({"error": "Video not found"}), 404

    if video["agent_id"] == g.user["id"]:
        return jsonify({"error": "You cannot tip yourself"}), 400

    data = request.get_json(force=True, silent=True) or {}
    try:
        amount = round(float(data.get("amount", 0)), 6)
    except (ValueError, TypeError):
        return jsonify({"error": "Invalid amount"}), 400

    if amount < RTC_TIP_MIN:
        return jsonify({"error": f"Minimum tip is {RTC_TIP_MIN} RTC"}), 400
    if amount > RTC_TIP_MAX:
        return jsonify({"error": f"Maximum tip is {RTC_TIP_MAX} RTC"}), 400

    message = str(data.get("message", ""))[:200].strip()

    # On-chain tip via RustChain signed transfer (Ed25519)
    if data.get("onchain"):
        to_wallet = str((video["creator_rtc_wallet"] or "")).strip()
        if not _is_rustchain_rtc_address(to_wallet):
            alt = str((video["creator_rtc_address"] or "")).strip()
            if _is_rustchain_rtc_address(alt):
                to_wallet = alt

        if not _is_rustchain_rtc_address(to_wallet):
            return jsonify({"error": "Creator has not linked a RustChain rtc_wallet (RTC... address)"}), 400

        resp, code = _handle_onchain_tip(
            db,
            sender_id=g.user["id"],
            sender_name=g.user["agent_name"],
            recipient_id=video["agent_id"],
            recipient_name=video["creator_name"],
            expected_to_wallet=to_wallet,
            amount=amount,
            user_message=message,
            data=data,
            video_id=video_id,
            video_title=video["title"],
        )
        db.commit()
        return jsonify(resp), code

    sender = db.execute("SELECT rtc_balance FROM agents WHERE id = ?", (g.user["id"],)).fetchone()
    if sender["rtc_balance"] < amount:
        return jsonify({"error": "Insufficient RTC balance", "balance": sender["rtc_balance"]}), 400

    # Execute transfer
    db.execute("UPDATE agents SET rtc_balance = rtc_balance - ? WHERE id = ?", (amount, g.user["id"]))
    db.execute("UPDATE agents SET rtc_balance = rtc_balance + ? WHERE id = ?", (amount, video["agent_id"]))

    db.execute(
        "INSERT INTO tips (from_agent_id, to_agent_id, video_id, amount, message, created_at) "
        "VALUES (?, ?, ?, ?, ?, ?)",
        (g.user["id"], video["agent_id"], video_id, amount, message, time.time()),
    )

    db.execute(
        "INSERT INTO earnings (agent_id, amount, reason, video_id, created_at) VALUES (?, ?, ?, ?, ?)",
        (video["agent_id"], amount, "tip_received", video_id, time.time()),
    )

    notify(db, video["agent_id"], "tip",
           f'@{g.user["agent_name"]} tipped {amount:.4f} RTC on "{video["title"]}"'
           + (f': "{message}"' if message else ""),
           from_agent=g.user["agent_name"], video_id=video_id)
    _referral_mark_rtc_native_action(db, int(g.user["id"]), evidence_ref=f"/watch/{video_id}")
    _referral_mark_rtc_native_action(db, int(video["agent_id"]), evidence_ref=f"/watch/{video_id}")

    db.commit()
    new_balance = db.execute("SELECT rtc_balance FROM agents WHERE id = ?", (g.user["id"],)).fetchone()
    return jsonify({"ok": True, "amount": amount, "video_id": video_id,
                    "to": video["creator_name"], "message": message,
                    "new_balance": round(new_balance["rtc_balance"], 6)})


@app.route("/api/agents/<agent_name>/web-tip", methods=["POST"])
def web_tip_agent(agent_name):
    """Tip a creator from the channel page (requires login session)."""
    if not g.user:
        return jsonify({"error": "You must be signed in to tip.", "login_required": True}), 401
    _verify_csrf()

    if not _rate_limit(f"tip:{g.user['id']}", 30, 3600):
        return jsonify({"error": "Tip rate limit exceeded. Try again later."}), 429

    db = get_db()
    target = db.execute(
        "SELECT id, agent_name, rtc_wallet, rtc_address FROM agents WHERE agent_name = ?",
        (agent_name,),
    ).fetchone()
    if not target:
        return jsonify({"error": "Creator not found"}), 404

    if target["id"] == g.user["id"]:
        return jsonify({"error": "You cannot tip yourself"}), 400

    data = request.get_json(force=True, silent=True) or {}
    try:
        amount = round(float(data.get("amount", 0)), 6)
    except (ValueError, TypeError):
        return jsonify({"error": "Invalid amount"}), 400

    if amount < RTC_TIP_MIN:
        return jsonify({"error": f"Minimum tip is {RTC_TIP_MIN} RTC"}), 400
    if amount > RTC_TIP_MAX:
        return jsonify({"error": f"Maximum tip is {RTC_TIP_MAX} RTC"}), 400

    message = str(data.get("message", ""))[:200].strip()

    if data.get("onchain"):
        to_wallet = str(target["rtc_wallet"] or "").strip()
        if not _is_rustchain_rtc_address(to_wallet):
            alt = str(target["rtc_address"] or "").strip()
            if _is_rustchain_rtc_address(alt):
                to_wallet = alt
        if not _is_rustchain_rtc_address(to_wallet):
            return jsonify({"error": "Creator has not linked a RustChain rtc_wallet (RTC... address)"}), 400

        resp, code = _handle_onchain_tip(
            db,
            sender_id=g.user["id"],
            sender_name=g.user["agent_name"],
            recipient_id=target["id"],
            recipient_name=target["agent_name"],
            expected_to_wallet=to_wallet,
            amount=amount,
            user_message=message,
            data=data,
            video_id="",
            video_title="",
        )
        db.commit()
        return jsonify(resp), code

    # Legacy: internal credits tip
    sender = db.execute("SELECT rtc_balance FROM agents WHERE id = ?", (g.user["id"],)).fetchone()
    if sender["rtc_balance"] < amount:
        return jsonify({"error": "Insufficient RTC balance", "balance": sender["rtc_balance"]}), 400

    db.execute("UPDATE agents SET rtc_balance = rtc_balance - ? WHERE id = ?", (amount, g.user["id"]))
    db.execute("UPDATE agents SET rtc_balance = rtc_balance + ? WHERE id = ?", (amount, target["id"]))
    db.execute(
        "INSERT INTO tips (from_agent_id, to_agent_id, video_id, amount, message, created_at) "
        "VALUES (?, ?, '', ?, ?, ?)",
        (g.user["id"], target["id"], amount, message, time.time()),
    )
    db.execute(
        "INSERT INTO earnings (agent_id, amount, reason, video_id, created_at) VALUES (?, ?, ?, '', ?)",
        (target["id"], amount, "tip_received", time.time()),
    )
    notify(db, target["id"], "tip",
           f'@{g.user["agent_name"]} tipped {amount:.4f} RTC'
           + (f': "{message}"' if message else ""),
           from_agent=g.user["agent_name"], video_id="")
    _referral_mark_rtc_native_action(db, int(g.user["id"]), evidence_ref=f"/agent/{agent_name}")
    _referral_mark_rtc_native_action(db, int(target["id"]), evidence_ref=f"/agent/{agent_name}")

    db.commit()
    new_balance = db.execute("SELECT rtc_balance FROM agents WHERE id = ?", (g.user["id"],)).fetchone()
    return jsonify({"ok": True, "amount": amount, "to": target["agent_name"], "message": message,
                    "new_balance": round(new_balance["rtc_balance"], 6)})


@app.route("/api/agents/<agent_name>/tip", methods=["POST"])
@require_api_key
def tip_agent(agent_name):
    """Tip a creator via API key auth (supports on-chain signed tips)."""
    if not _rate_limit(f"tip:{g.agent['id']}", 30, 3600):
        return jsonify({"error": "Tip rate limit exceeded. Try again later."}), 429

    db = get_db()
    target = db.execute(
        "SELECT id, agent_name, rtc_wallet, rtc_address FROM agents WHERE agent_name = ?",
        (agent_name,),
    ).fetchone()
    if not target:
        return jsonify({"error": "Creator not found"}), 404

    if target["id"] == g.agent["id"]:
        return jsonify({"error": "You cannot tip yourself"}), 400

    data = request.get_json(force=True, silent=True) or {}
    try:
        amount = round(float(data.get("amount", 0)), 6)
    except (ValueError, TypeError):
        return jsonify({"error": "Invalid amount"}), 400

    if amount < RTC_TIP_MIN:
        return jsonify({"error": f"Minimum tip is {RTC_TIP_MIN} RTC"}), 400
    if amount > RTC_TIP_MAX:
        return jsonify({"error": f"Maximum tip is {RTC_TIP_MAX} RTC"}), 400

    message = str(data.get("message", ""))[:200].strip()

    if data.get("onchain"):
        to_wallet = str(target["rtc_wallet"] or "").strip()
        if not _is_rustchain_rtc_address(to_wallet):
            alt = str(target["rtc_address"] or "").strip()
            if _is_rustchain_rtc_address(alt):
                to_wallet = alt
        if not _is_rustchain_rtc_address(to_wallet):
            return jsonify({"error": "Creator has not linked a RustChain rtc_wallet (RTC... address)"}), 400

        resp, code = _handle_onchain_tip(
            db,
            sender_id=g.agent["id"],
            sender_name=g.agent["agent_name"],
            recipient_id=target["id"],
            recipient_name=target["agent_name"],
            expected_to_wallet=to_wallet,
            amount=amount,
            user_message=message,
            data=data,
            video_id="",
            video_title="",
        )
        db.commit()
        return jsonify(resp), code

    # Legacy: internal credits tip
    sender = db.execute("SELECT rtc_balance FROM agents WHERE id = ?", (g.agent["id"],)).fetchone()
    if sender["rtc_balance"] < amount:
        return jsonify({"error": "Insufficient RTC balance", "balance": sender["rtc_balance"]}), 400

    db.execute("UPDATE agents SET rtc_balance = rtc_balance - ? WHERE id = ?", (amount, g.agent["id"]))
    db.execute("UPDATE agents SET rtc_balance = rtc_balance + ? WHERE id = ?", (amount, target["id"]))
    db.execute(
        "INSERT INTO tips (from_agent_id, to_agent_id, video_id, amount, message, created_at) "
        "VALUES (?, ?, '', ?, ?, ?)",
        (g.agent["id"], target["id"], amount, message, time.time()),
    )
    db.execute(
        "INSERT INTO earnings (agent_id, amount, reason, video_id, created_at) VALUES (?, ?, ?, '', ?)",
        (target["id"], amount, "tip_received", time.time()),
    )
    notify(db, target["id"], "tip",
           f'@{g.agent["agent_name"]} tipped {amount:.4f} RTC'
           + (f': "{message}"' if message else ""),
           from_agent=g.agent["agent_name"], video_id="")
    _referral_mark_rtc_native_action(db, int(g.agent["id"]), evidence_ref=f"/agent/{agent_name}")
    _referral_mark_rtc_native_action(db, int(target["id"]), evidence_ref=f"/agent/{agent_name}")

    db.commit()
    return jsonify({"ok": True, "amount": amount, "to": target["agent_name"], "message": message})


@app.route("/api/videos/<video_id>/tips")
def get_video_tips(video_id):
    """Get recent tips for a video (public)."""
    db = get_db()
    v = db.execute("SELECT 1 FROM videos WHERE video_id = ?", (video_id,)).fetchone()
    if not v:
        return jsonify({"error": "Video not found"}), 404
    _sync_pending_tips(db)
    page = max(1, request.args.get("page", 1, type=int))
    per_page = min(50, max(1, request.args.get("per_page", 10, type=int)))
    offset = (page - 1) * per_page

    tips = db.execute(
        """SELECT t.amount, t.message, t.created_at,
                  a.agent_name, a.display_name, a.avatar_url,
                  COALESCE(t.status, 'confirmed') AS status,
                  COALESCE(t.onchain, 0) AS onchain,
                  t.tx_hash, t.confirms_at
           FROM tips t JOIN agents a ON t.from_agent_id = a.id
           WHERE t.video_id = ?
           ORDER BY t.created_at DESC LIMIT ? OFFSET ?""",
        (video_id, per_page, offset),
    ).fetchall()

    total = db.execute(
        "SELECT COUNT(*) FROM tips WHERE video_id = ? AND COALESCE(status, 'confirmed') = 'confirmed'",
        (video_id,),
    ).fetchone()[0]
    total_amount = db.execute(
        "SELECT COALESCE(SUM(amount), 0) FROM tips "
        "WHERE video_id = ? AND COALESCE(status, 'confirmed') = 'confirmed'",
        (video_id,),
    ).fetchone()[0]
    pending_total = db.execute(
        "SELECT COUNT(*) FROM tips WHERE video_id = ? AND COALESCE(status, 'confirmed') = 'pending'",
        (video_id,),
    ).fetchone()[0]
    pending_amount = db.execute(
        "SELECT COALESCE(SUM(amount), 0) FROM tips "
        "WHERE video_id = ? AND COALESCE(status, 'confirmed') = 'pending'",
        (video_id,),
    ).fetchone()[0]

    return jsonify({
        "video_id": video_id,
        "tips": [
            {
                "agent_name": t["agent_name"],
                "display_name": t["display_name"],
                "avatar_url": t["avatar_url"] or "",
                "amount": t["amount"],
                "message": t["message"],
                "created_at": t["created_at"],
                "status": t["status"],
                "onchain": bool(t["onchain"]),
                "tx_hash": t["tx_hash"] or "",
                "confirms_at": t["confirms_at"] or 0,
            }
            for t in tips
        ],
        # Totals are confirmed-only; pending tips confirm after RustChain delay.
        "total_tips": total,
        "total_amount": round(total_amount, 6),
        "pending_tips": pending_total,
        "pending_amount": round(pending_amount, 6),
        "page": page,
        "per_page": per_page,
    })


@app.route("/api/tips/leaderboard")
def tip_leaderboard():
    """Top tipped creators (by total tips received)."""
    db = get_db()
    _sync_pending_tips(db)
    limit = min(50, max(1, request.args.get("limit", 20, type=int)))

    rows = db.execute(
        """SELECT a.agent_name, a.display_name, a.avatar_url, a.is_human,
                  COUNT(t.id) AS tip_count, COALESCE(SUM(t.amount), 0) AS total_received
           FROM tips t JOIN agents a ON t.to_agent_id = a.id
           WHERE COALESCE(t.status, 'confirmed') = 'confirmed'
           GROUP BY t.to_agent_id
           ORDER BY total_received DESC LIMIT ?""",
        (limit,),
    ).fetchall()

    return jsonify({
        "leaderboard": [
            {
                "agent_name": r["agent_name"],
                "display_name": r["display_name"],
                "avatar_url": r["avatar_url"] or "",
                "is_human": bool(r["is_human"]),
                "tip_count": r["tip_count"],
                "total_received": round(r["total_received"], 6),
            }
            for r in rows
        ],
    })


@app.route("/api/tips/tippers")
def tipper_leaderboard():
    """Top tippers (by total tips sent)."""
    db = get_db()
    _sync_pending_tips(db)
    limit = min(50, max(1, request.args.get("limit", 20, type=int)))

    rows = db.execute(
        """SELECT a.agent_name, a.display_name, a.avatar_url, a.is_human,
                  COUNT(t.id) AS tip_count, COALESCE(SUM(t.amount), 0) AS total_sent
           FROM tips t JOIN agents a ON t.from_agent_id = a.id
           WHERE COALESCE(t.status, 'confirmed') = 'confirmed'
           GROUP BY t.from_agent_id
           ORDER BY total_sent DESC LIMIT ?""",
        (limit,),
    ).fetchall()

    return jsonify({
        "leaderboard": [
            {
                "agent_name": r["agent_name"],
                "display_name": r["display_name"],
                "avatar_url": r["avatar_url"] or "",
                "is_human": bool(r["is_human"]),
                "tip_count": r["tip_count"],
                "total_sent": round(r["total_sent"], 6),
            }
            for r in rows
        ],
    })


# ---------------------------------------------------------------------------
# Cross-posting
# ---------------------------------------------------------------------------

@app.route("/api/crosspost/moltbook", methods=["POST"])
@require_api_key
def crosspost_moltbook():
    """Cross-post a video link to Moltbook."""
    data = request.get_json(silent=True) or {}
    video_id = data.get("video_id", "")
    submolt = data.get("submolt", "bottube")

    db = get_db()
    video = db.execute(
        "SELECT * FROM videos WHERE video_id = ? AND agent_id = ?",
        (video_id, g.agent["id"]),
    ).fetchone()
    if not video:
        return jsonify({"error": "Video not found or not yours"}), 404

    # Record cross-post intent (actual posting done externally)
    db.execute(
        "INSERT INTO crossposts (video_id, platform, created_at) VALUES (?, 'moltbook', ?)",
        (video_id, time.time()),
    )
    db.execute(
        "UPDATE videos SET submolt_crosspost = ? WHERE video_id = ?",
        (submolt, video_id),
    )
    db.commit()

    return jsonify({
        "ok": True,
        "video_id": video_id,
        "platform": "moltbook",
        "submolt": submolt,
        "message": "Cross-post recorded. Moltbook bridge will pick this up.",
    })


@app.route("/api/crosspost/x", methods=["POST"])
@require_api_key
def crosspost_x():
    """Cross-post a video announcement to X/Twitter via tweepy.

    Uses the server's X credentials (from TWITTER_* env vars or .env.twitter).
    Posts: "New on BoTTube: [title] by @agent — [url]"
    """
    data = request.get_json(silent=True) or {}
    video_id = data.get("video_id", "")
    custom_text = data.get("text", "")

    db = get_db()
    video = db.execute(
        """SELECT v.*, a.agent_name, a.display_name, a.x_handle
           FROM videos v JOIN agents a ON v.agent_id = a.id
           WHERE v.video_id = ? AND v.agent_id = ?""",
        (video_id, g.agent["id"]),
    ).fetchone()
    if not video:
        return jsonify({"error": "Video not found or not yours"}), 404

    # Build tweet text
    if custom_text:
        tweet_text = custom_text
    else:
        agent_mention = f"@{video['x_handle']}" if video["x_handle"] else video["display_name"]
        watch_url = f"https://bottube.ai/watch/{video_id}"
        tweet_text = f"New on BoTTube: {video['title']}\n\nby {agent_mention}\n{watch_url}"

    # Truncate to X limit
    if len(tweet_text) > 280:
        tweet_text = tweet_text[:277] + "..."

    # Post to X via tweepy
    tweet_id = _post_to_x(tweet_text)

    if tweet_id:
        db.execute(
            "INSERT INTO crossposts (video_id, platform, external_id, created_at) VALUES (?, 'x', ?, ?)",
            (video_id, tweet_id, time.time()),
        )
        db.commit()
        return jsonify({
            "ok": True,
            "video_id": video_id,
            "platform": "x",
            "tweet_id": tweet_id,
            "tweet_url": f"https://x.com/i/status/{tweet_id}",
            "text": tweet_text,
        })
    else:
        return jsonify({
            "ok": False,
            "error": "Failed to post to X. Check server X credentials.",
        }), 500


def _post_to_x(text: str) -> str:
    """Post a tweet using tweepy. Returns tweet ID or empty string on failure."""
    try:
        import tweepy
    except ImportError:
        app.logger.warning("tweepy not installed - X posting disabled")
        return ""

    try:
        # Load credentials from env or .env.twitter
        api_key = os.environ.get("TWITTER_API_KEY", "")
        api_secret = os.environ.get("TWITTER_API_SECRET", "")
        access_token = os.environ.get("TWITTER_ACCESS_TOKEN", "")
        access_secret = os.environ.get("TWITTER_ACCESS_TOKEN_SECRET", "")

        if not all([api_key, api_secret, access_token, access_secret]):
            # Try loading from .env.twitter file
            env_path = os.environ.get("TWITTER_ENV_FILE", "/home/sophia/.env.twitter")
            if os.path.exists(env_path):
                with open(env_path) as f:
                    for line in f:
                        line = line.strip()
                        if "=" in line and not line.startswith("#"):
                            k, v = line.split("=", 1)
                            os.environ.setdefault(k.strip(), v.strip().strip('"').strip("'"))
                api_key = os.environ.get("TWITTER_API_KEY", "")
                api_secret = os.environ.get("TWITTER_API_SECRET", "")
                access_token = os.environ.get("TWITTER_ACCESS_TOKEN", "")
                access_secret = os.environ.get("TWITTER_ACCESS_TOKEN_SECRET", "")

        if not all([api_key, api_secret, access_token, access_secret]):
            app.logger.warning("X credentials not configured")
            return ""

        client = tweepy.Client(
            consumer_key=api_key,
            consumer_secret=api_secret,
            access_token=access_token,
            access_token_secret=access_secret,
        )
        response = client.create_tweet(text=text)
        tweet_id = str(response.data["id"])
        app.logger.info(f"Posted to X: {tweet_id}")
        return tweet_id

    except Exception as e:
        app.logger.error(f"X post failed: {e}")
        return ""


# ---------------------------------------------------------------------------
# Thumbnail serving
# ---------------------------------------------------------------------------

@app.route("/thumbnails/<filename>")
def serve_thumbnail(filename):
    """Serve thumbnail images."""
    if "/" in filename or "\\" in filename or ".." in filename:
        abort(404)
    resp = send_from_directory(str(THUMB_DIR), filename)
    resp.headers.setdefault("Cache-Control", "public, max-age=86400")
    return resp


@app.route("/avatars/<filename>")
def serve_avatar_file(filename):
    """Serve uploaded avatar images."""
    if "/" in filename or "\\" in filename or ".." in filename:
        abort(404)
    resp = send_from_directory(str(AVATAR_DIR), filename)
    resp.headers.setdefault("Cache-Control", "public, max-age=86400")
    return resp


@app.route("/avatar/<agent_name>.svg")
def serve_avatar(agent_name):
    """Generate a unique SVG avatar based on agent name hash."""
    h = hashlib.md5(agent_name.encode()).hexdigest()
    hue = int(h[:3], 16) % 360
    sat = 55 + int(h[3:5], 16) % 30
    light = 45 + int(h[5:7], 16) % 15
    bg = f"hsl({hue},{sat}%,{light}%)"
    fg = f"hsl({hue},{sat}%,{min(light + 35, 95)}%)"
    initial = (agent_name[0] if agent_name else "?").upper()

    # 5x5 symmetric grid identicon
    cells = []
    for row in range(5):
        for col in range(3):
            bit = int(h[(row * 3 + col) % 32], 16) % 2
            if bit:
                x1 = 6 + col * 8
                y1 = 6 + row * 8
                cells.append(f'<rect x="{x1}" y="{y1}" width="7" height="7" rx="1" fill="{fg}" opacity="0.5"/>')
                # Mirror
                if col < 2:
                    x2 = 6 + (4 - col) * 8
                    cells.append(f'<rect x="{x2}" y="{y1}" width="7" height="7" rx="1" fill="{fg}" opacity="0.5"/>')

    svg = f'''<svg xmlns="http://www.w3.org/2000/svg" width="48" height="48" viewBox="0 0 48 48">
  <rect width="48" height="48" rx="24" fill="{bg}"/>
  {''.join(cells)}
  <text x="24" y="25" text-anchor="middle" dominant-baseline="central"
        font-family="sans-serif" font-size="20" font-weight="700" fill="#fff">{initial}</text>
</svg>'''
    return Response(svg, mimetype="image/svg+xml",
                    headers={"Cache-Control": "public, max-age=86400"})


@app.route("/api/agents/me/avatar", methods=["POST"])
@require_api_key
def upload_avatar():
    """Upload or auto-generate a profile avatar (256x256).

    If a file is provided via multipart ``avatar`` field, it is resized to
    256x256 center-crop via ffmpeg and saved.  If **no file** is provided the
    server auto-generates a unique avatar using ffmpeg (colored background +
    initial letter) so bots can call this with an empty body to get a default
    avatar assigned.

    Rate limit: 5 per agent per hour.
    """
    agent = g.agent
    if not _rate_limit(f"avatar:{agent['id']}", 5, 3600):
        return jsonify({"error": "Rate limited — max 5 avatar uploads per hour"}), 429

    import tempfile

    out_name = f"{agent['id']}.jpg"
    out_path = AVATAR_DIR / out_name

    f = request.files.get("avatar")
    if f and f.filename:
        # --- User/agent supplied an image ---
        ext = Path(f.filename).suffix.lower()
        if ext not in ALLOWED_THUMB_EXT:
            return jsonify({"error": f"Invalid file type. Allowed: {', '.join(sorted(ALLOWED_THUMB_EXT))}"}), 400

        # Read and check size
        data = f.read()
        if len(data) > MAX_AVATAR_SIZE:
            return jsonify({"error": f"File too large. Max {MAX_AVATAR_SIZE // (1024*1024)} MB"}), 400

        # Save to temp, resize with ffmpeg
        tmp = tempfile.NamedTemporaryFile(suffix=ext, delete=False)
        try:
            tmp.write(data)
            tmp.close()
            result = subprocess.run(
                [
                    "ffmpeg", "-y", "-i", tmp.name,
                    "-vf", f"scale={AVATAR_TARGET_SIZE}:{AVATAR_TARGET_SIZE}"
                           f":force_original_aspect_ratio=increase,"
                           f"crop={AVATAR_TARGET_SIZE}:{AVATAR_TARGET_SIZE}",
                    "-frames:v", "1",
                    str(out_path),
                ],
                capture_output=True, timeout=30,
            )
            if result.returncode != 0 or not out_path.exists():
                return jsonify({"error": "ffmpeg resize failed", "detail": result.stderr.decode()[-300:]}), 500
        finally:
            Path(tmp.name).unlink(missing_ok=True)
    else:
        # --- Auto-generate avatar from agent name ---
        name = agent["agent_name"]
        h = hashlib.md5(name.encode()).hexdigest()
        r = int(h[0:2], 16)
        g_val = int(h[2:4], 16)
        b = int(h[4:6], 16)
        # Ensure the color isn't too dark
        brightness = (r + g_val + b) / 3
        if brightness < 80:
            r = min(255, r + 80)
            g_val = min(255, g_val + 80)
            b = min(255, b + 80)
        bg_hex = f"{r:02x}{g_val:02x}{b:02x}"
        initial = (name.replace("-", " ").replace("_", " ").split()[0][0]
                   if name else "?").upper()
        display = agent["display_name"] or name
        # Truncate display name for the bottom text, sanitize for ffmpeg drawtext
        bot_label = re.sub(r"[^a-zA-Z0-9 _-]", "", display)[:16]

        result = subprocess.run(
            [
                "ffmpeg", "-y",
                "-f", "lavfi", "-i", f"color=c=0x{bg_hex}:s=256x256:d=1",
                "-vf", (
                    f"drawtext=text='{initial}':"
                    f"fontsize=140:fontcolor=white:x=(w-tw)/2:y=(h-th)/2-10,"
                    f"drawtext=text='{bot_label}':"
                    f"fontsize=18:fontcolor=white@0.7:x=(w-tw)/2:y=h-35"
                ),
                "-frames:v", "1",
                str(out_path),
            ],
            capture_output=True, timeout=15,
        )
        if result.returncode != 0 or not out_path.exists():
            return jsonify({"error": "Avatar generation failed", "detail": result.stderr.decode()[-300:]}), 500

    # Update DB
    avatar_url = f"/avatars/{out_name}"
    db = get_db()
    db.execute("UPDATE agents SET avatar_url = ? WHERE id = ?", (avatar_url, agent["id"]))
    _refresh_agent_quests(db, agent["id"], ["profile_complete"])
    _referral_refresh_invite_state(db, agent["id"])
    db.commit()

    return jsonify({"ok": True, "avatar_url": avatar_url})


# ---------------------------------------------------------------------------
# HTML frontend routes
# ---------------------------------------------------------------------------

@app.route("/")
def index():
    """Homepage with trending and recent videos."""
    db = get_db()

    # Trending (improved algorithm: views + likes + comments + recency)
    trending_rows = _get_trending_videos(db, limit=8)

    # Recent
    recent_rows = db.execute(
        """SELECT v.*, a.agent_name, a.display_name, a.avatar_url, a.is_human
           FROM videos v JOIN agents a ON v.agent_id = a.id
           WHERE v.is_removed = 0 AND COALESCE(a.is_banned, 0) = 0
           ORDER BY v.created_at DESC LIMIT 12""",
    ).fetchall()

    # Stats
    stats = {
        "videos": db.execute(
            """SELECT COUNT(*) FROM videos v
               JOIN agents a ON v.agent_id = a.id
               WHERE v.is_removed = 0 AND COALESCE(a.is_banned, 0) = 0"""
        ).fetchone()[0],
        "agents": db.execute("SELECT COUNT(*) FROM agents WHERE is_human = 0 AND COALESCE(is_banned, 0) = 0").fetchone()[0],
        "humans": db.execute("SELECT COUNT(*) FROM agents WHERE is_human = 1 AND COALESCE(is_banned, 0) = 0").fetchone()[0],
        "views": db.execute(
            """SELECT COALESCE(SUM(v.views), 0) FROM videos v
               JOIN agents a ON v.agent_id = a.id
               WHERE v.is_removed = 0 AND COALESCE(a.is_banned, 0) = 0"""
        ).fetchone()[0],
    }

    return render_template(
        "index.html",
        trending=trending_rows,
        recent=recent_rows,
        stats=stats,
        categories=VIDEO_CATEGORIES,
    )


@app.route("/videos")
@app.route("/videos/")
def videos_legacy_redirect():
    """Legacy path: /videos now canonicalizes to homepage feed."""
    return redirect(url_for("index"), code=301)


@app.route("/challenges")
def challenges_page():
    """Challenge listing page."""
    db = get_db()
    now = time.time()
    rows = db.execute(
        """SELECT * FROM challenges
           ORDER BY start_at DESC, created_at DESC""",
    ).fetchall()
    challenges = []
    for row in rows:
        status = row["status"]
        if row["start_at"] and row["end_at"]:
            if row["start_at"] <= now <= row["end_at"]:
                status = "active"
            elif now < row["start_at"]:
                status = "upcoming"
            else:
                status = "closed"
        challenges.append({
            "challenge_id": row["challenge_id"],
            "title": row["title"],
            "description": row["description"],
            "tags": _safe_json_loads_list(row["tags"]),
            "reward": row["reward"],
            "status": status,
            "start_at": row["start_at"],
            "end_at": row["end_at"],
        })
    return render_template("challenges.html", challenges=challenges)


@app.route("/watch/<video_id>")
def watch(video_id):
    """Video player page."""
    db = get_db()
    video = db.execute(
        f"""SELECT v.*, a.agent_name, a.display_name, a.avatar_url, a.is_human,
                  a.rtc_address, a.rtc_wallet, a.btc_address, a.eth_address,
                  a.sol_address, a.ltc_address, a.erg_address, a.paypal_email
           FROM videos v JOIN agents a ON v.agent_id = a.id
           WHERE v.video_id = ? AND {_public_video_filter_sql()}""",
        (video_id,),
    ).fetchone()

    if not video:
        abort(404)

    # Record view (deduplicated: 1 view per IP per video per 30 min)
    ip = request.headers.get("X-Real-IP", request.remote_addr)
    VIEW_COOLDOWN = 1800  # 30 minutes
    recent = db.execute(
        "SELECT 1 FROM views WHERE video_id = ? AND ip_address = ? AND created_at > ?",
        (video_id, ip, time.time() - VIEW_COOLDOWN),
    ).fetchone()
    if not recent:
        db.execute(
            "INSERT INTO views (video_id, ip_address, created_at) VALUES (?, ?, ?)",
            (video_id, ip, time.time()),
        )
        db.execute("UPDATE videos SET views = views + 1 WHERE video_id = ?", (video_id,))
        new_views = (video["views"] or 0) + 1
        # Check BAN milestones (100 views, 1000 views)
        check_view_milestones(db, video["agent_id"], video_id, new_views)
        db.commit()

    # Record watch history for logged-in users
    if g.user:
        db.execute(
            """INSERT INTO watch_history (agent_id, video_id, watched_at)
               VALUES (?, ?, ?)
               ON CONFLICT(agent_id, video_id) DO UPDATE SET watched_at = excluded.watched_at""",
            (g.user["id"], video_id, time.time()),
        )
        db.commit()

    # Get comments
    comments_rows = db.execute(
        """SELECT c.*, a.agent_name, a.display_name, a.avatar_url, a.is_human
           FROM comments c JOIN agents a ON c.agent_id = a.id
           WHERE c.video_id = ?
           ORDER BY c.created_at ASC""",
        (video_id,),
    ).fetchall()

    # Compute interaction context for comments
    video_agent_id = video["agent_id"]
    comments = []
    for row in comments_rows:
        interaction_context = {}
        if video_agent_id and row["agent_id"] != video_agent_id:
            interaction_context = _compute_agent_interaction_context(
                db, video_agent_id, row["agent_id"]
            )
        comment_dict = dict(row)
        comment_dict["interaction_context"] = interaction_context
        comments.append(comment_dict)

    # SEO: server-built VideoObject JSON-LD (single source of truth, schema.org valid)
    from seo_routes import build_video_jsonld
    video_for_jsonld = dict(video)
    video_for_jsonld["comment_count"] = len(comments)
    video_jsonld = build_video_jsonld(
        video_for_jsonld,
        video["agent_name"],
        video["display_name"],
        video_for_jsonld.get("is_human", 0),
    )

    revision_of = None
    if "revision_of" in video.keys() and video["revision_of"]:
        revision_of = db.execute(
            f"""SELECT v.video_id, v.title, a.agent_name, a.display_name
               FROM videos v JOIN agents a ON v.agent_id = a.id
               WHERE v.video_id = ? AND {_public_video_filter_sql()}""",
            (video["revision_of"],),
        ).fetchone()

    revisions = db.execute(
        f"""SELECT v.video_id, v.title, v.created_at, a.agent_name, a.display_name
           FROM videos v JOIN agents a ON v.agent_id = a.id
           WHERE v.revision_of = ? AND {_public_video_filter_sql()}
           ORDER BY v.created_at DESC LIMIT 8""",
        (video_id,),
    ).fetchall()

    # Response video handling (Issue #2282 - Agent Collab System)
    # Get the original video this is responding to
    response_to_video = None
    if "response_to_video_id" in video.keys() and video["response_to_video_id"]:
        response_to_video = db.execute(
            f"""SELECT v.video_id, v.title, v.views, v.created_at, a.agent_name, a.display_name, a.avatar_url
               FROM videos v JOIN agents a ON v.agent_id = a.id
               WHERE v.video_id = ? AND {_public_video_filter_sql()}""",
            (video["response_to_video_id"],),
        ).fetchone()

    # Get all response videos to this video
    response_videos = db.execute(
        f"""SELECT v.video_id, v.title, v.views, v.created_at, a.agent_name, a.display_name, a.avatar_url
           FROM videos v JOIN agents a ON v.agent_id = a.id
           WHERE v.response_to_video_id = ? AND {_public_video_filter_sql()}
           ORDER BY v.created_at DESC LIMIT 10""",
        (video_id,),
    ).fetchall()

    challenge = None
    if "challenge_id" in video.keys() and video["challenge_id"]:
        challenge = db.execute(
            """SELECT challenge_id, title, description, tags, reward, status, start_at, end_at
               FROM challenges WHERE challenge_id = ?""",
            (video["challenge_id"],),
        ).fetchone()

    # Related videos: score by same category, same agent, shared tags, exclude watched
    _watched_ids = set()
    if g.user:
        _wh = db.execute(
            "SELECT video_id FROM watch_history WHERE agent_id = ? ORDER BY watched_at DESC LIMIT 100",
            (g.user["id"],),
        ).fetchall()
        _watched_ids = {r["video_id"] for r in _wh}

    _cur_tags = set()
    try:
        _cur_tags = set(json.loads(video["tags"])) if video["tags"] else set()
    except Exception:
        pass
    _cur_cat = video["category"] or "other"

    _candidates = db.execute(
        f"""SELECT v.*, a.agent_name, a.display_name, a.avatar_url, a.is_human
           FROM videos v JOIN agents a ON v.agent_id = a.id
           WHERE v.video_id != ? AND {_public_video_filter_sql()}
           ORDER BY v.views DESC
           LIMIT 100""",
        (video_id,),
    ).fetchall()

    def _related_score(r):
        s = 0
        if r["agent_id"] == video["agent_id"]:
            s += 3
        if (r["category"] or "other") == _cur_cat:
            s += 2
        try:
            r_tags = set(json.loads(r["tags"])) if r["tags"] else set()
            s += len(_cur_tags & r_tags)
        except Exception:
            pass
        if r["video_id"] in _watched_ids:
            s -= 5
        return s

    _candidates_scored = sorted(_candidates, key=_related_score, reverse=True)
    related = _candidates_scored[:8]

    # Look up creator's BAN wallet address (from ban_wallets table)
    _ban_addr_row = None
    try:
        _ban_addr_row = db.execute(
            "SELECT ban_address FROM ban_wallets WHERE agent_id = ?", (video["agent_id"],)
        ).fetchone()
    except Exception:
        pass
    creator_ban_address = _ban_addr_row["ban_address"] if _ban_addr_row else ""

    # Subscription data for follow button
    subscriber_count = db.execute(
        "SELECT COUNT(*) FROM subscriptions WHERE following_id = ?",
        (video["agent_id"],),
    ).fetchone()[0]

    is_following = False
    if g.user:
        is_following = bool(db.execute(
            "SELECT 1 FROM subscriptions WHERE follower_id = ? AND following_id = ?",
            (g.user["id"], video["agent_id"]),
        ).fetchone())

    # Tip data for the tip button
    _sync_pending_tips(db)
    recent_tips = db.execute(
        """SELECT t.amount, t.message, t.created_at,
                  a.agent_name, a.display_name,
                  COALESCE(t.status, 'confirmed') AS status,
                  COALESCE(t.onchain, 0) AS onchain
           FROM tips t JOIN agents a ON t.from_agent_id = a.id
           WHERE t.video_id = ?
           ORDER BY t.created_at DESC LIMIT 5""",
        (video_id,),
    ).fetchall()
    tip_total = db.execute(
        "SELECT COALESCE(SUM(amount), 0), COUNT(*) FROM tips "
        "WHERE video_id = ? AND COALESCE(status, 'confirmed') = 'confirmed'",
        (video_id,),
    ).fetchone()
    tip_pending = db.execute(
        "SELECT COUNT(*) FROM tips WHERE video_id = ? AND COALESCE(status, 'confirmed') = 'pending'",
        (video_id,),
    ).fetchone()[0]
    user_balance = g.user["rtc_balance"] if g.user else 0

    # Load user's existing vote for this video
    user_vote = 0
    if g.user:
        _uv = db.execute(
            "SELECT vote FROM votes WHERE agent_id = ? AND video_id = ?",
            (g.user["id"], video_id),
        ).fetchone()
        if _uv:
            user_vote = _uv["vote"]
    creator_badges = _list_agent_badges(db, int(video["agent_id"]))

    # Agent interaction data for watch page
    _vid_aid = int(video["agent_id"])
    try:
        interaction_commenters = db.execute(
            "SELECT a2.agent_name, a2.display_name, a2.avatar_url, COUNT(*) AS cnt"
            " FROM comments c JOIN videos v ON c.video_id = v.video_id"
            " JOIN agents a2 ON c.agent_id = a2.id"
            " WHERE v.agent_id = ? AND c.agent_id != ?"
            " GROUP BY a2.id ORDER BY cnt DESC LIMIT 8",
            (_vid_aid, _vid_aid)).fetchall()
        interaction_likers = db.execute(
            "SELECT a2.agent_name, a2.display_name, a2.avatar_url, COUNT(*) AS cnt"
            " FROM votes vt JOIN videos v ON vt.video_id = v.video_id"
            " JOIN agents a2 ON vt.agent_id = a2.id"
            " WHERE v.agent_id = ? AND vt.vote = 1 AND vt.agent_id != ?"
            " GROUP BY a2.id ORDER BY cnt DESC LIMIT 8",
            (_vid_aid, _vid_aid)).fetchall()
        interaction_outgoing = []
    except Exception:
        interaction_commenters = []
        interaction_likers = []
        interaction_outgoing = []

    # Phase 11.28: pull provenance fields for OG metadata so the
    # manifest_hash + chain anchor are visible to social-media scrapers
    # and bots even before the page-level JS fires.
    prov_meta = {}
    try:
        pr = db.execute(
            """SELECT COALESCE(anchor_tx_hash, '') AS tx,
                      COALESCE(anchor_block_height, 0) AS h,
                      COALESCE(anchor_manifest_hash, '') AS root,
                      COALESCE(anchor_chain, '') AS chain,
                      COALESCE(canonical_sha256, '') AS canonical,
                      COALESCE(manifest_version, 1) AS v
                 FROM video_provenance WHERE video_id = ?""",
            (video_id,),
        ).fetchone()
        if pr:
            prov_meta = {
                "tx_hash": pr["tx"],
                "block_height": int(pr["h"] or 0),
                "manifest_hash": pr["root"],
                "chain": pr["chain"],
                "canonical_sha256": pr["canonical"],
                "manifest_version": int(pr["v"] or 1),
                "verified": bool(pr["tx"] and pr["h"]),
            }
    except Exception:
        prov_meta = {}

    return render_template(
        "watch.html",
        video=video,
        creator_badges=creator_badges,
        comments=comments,
        related=related,
        video_jsonld=video_jsonld,
        subscriber_count=subscriber_count,
        is_following=is_following,
        user_vote=user_vote,
        recent_tips=recent_tips,
        tip_total_amount=round(tip_total[0], 6),
        tip_count=tip_total[1],
        tip_pending_count=tip_pending,
        user_balance=round(user_balance, 6),
        interaction_commenters=interaction_commenters,
        interaction_likers=interaction_likers,
        interaction_outgoing=interaction_outgoing,
        revision_of=revision_of,
        revisions=revisions,
        response_to_video=response_to_video,
        response_videos=response_videos,
        challenge=challenge,
        creator_ban_address=creator_ban_address,
        prov_meta=prov_meta,
    )


@app.route("/embed/<video_id>")
def embed(video_id):
    """Branded embed player for iframes and Twitter player cards."""
    db = get_db()
    video = db.execute(
        f"""SELECT v.*, a.agent_name, a.display_name
           FROM videos v JOIN agents a ON v.agent_id = a.id
           WHERE v.video_id = ? AND {_public_video_filter_sql()}""",
        (video_id,),
    ).fetchone()
    if not video:
        abort(404)

    autoplay = request.args.get("autoplay", "0") == "1"
    loop = request.args.get("loop", "0") == "1"
    muted = request.args.get("mute", "0") == "1"
    video_attrs = " ".join(
        attr for attr, enabled in (
            ("autoplay", autoplay),
            ("loop", loop),
            ("muted", muted),
        ) if enabled
    )
    if video_attrs:
        video_attrs += " "

    title_esc = (video["title"] or "").replace("&", "&amp;").replace("<", "&lt;").replace('"', "&quot;")
    creator_esc = (video["display_name"] or video["agent_name"] or "").replace("&", "&amp;").replace("<", "&lt;")
    embed_url = f"https://bottube.ai/embed/{video_id}"
    embed_code = (
        f'<iframe src="{embed_url}" width="560" height="315" '
        'frameborder="0" allowfullscreen></iframe>'
    )
    embed_code_json = json.dumps(embed_code)

    html = f"""<!DOCTYPE html>
<html><head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<meta name="robots" content="noindex,nofollow">
<title>{title_esc} - BoTTube</title>
<link rel="canonical" href="https://bottube.ai/watch/{video_id}">
<style>
*{{margin:0;padding:0;box-sizing:border-box}}
html,body{{width:100%;height:100%}}
body{{background:#000;display:flex;align-items:center;justify-content:center;position:relative;overflow:hidden}}
video{{max-width:100%;max-height:100%;width:100%;height:100%;object-fit:contain;display:block}}
.overlay{{position:absolute;bottom:0;left:0;right:0;padding:12px 16px;background:linear-gradient(transparent,rgba(0,0,0,0.85));
 opacity:0;transition:opacity 0.3s;pointer-events:none;display:flex;align-items:flex-end;justify-content:space-between}}
body:hover .overlay{{opacity:1}}
.info{{color:#fff;min-width:0}}
.title{{font:600 14px/1.3 -apple-system,sans-serif;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;max-width:70vw}}
.creator{{font:12px -apple-system,sans-serif;color:#aaa;margin-top:2px}}
.actions{{pointer-events:auto;display:flex;gap:8px;align-items:center;flex-shrink:0}}
.brand,.copy{{text-decoration:none;background:#3ea6ff;color:#0f0f0f;padding:6px 14px;border-radius:4px;
 font:700 12px -apple-system,sans-serif;white-space:nowrap;flex-shrink:0}}
.copy{{border:0;cursor:pointer;background:#222;color:#fff}}
.brand:hover{{background:#65b8ff}}
.copy:hover{{background:#333}}
</style>
</head><body>
<video controls {video_attrs}playsinline>
<source src="/api/videos/{video_id}/stream" type="video/mp4">
</video>
<div class="overlay">
<div class="info"><div class="title">{title_esc}</div><div class="creator">{creator_esc}</div></div>
<div class="actions">
<button class="copy" type="button" onclick="copyEmbed(this)">Copy embed</button>
<a class="brand" href="https://bottube.ai/watch/{video_id}" target="_blank" rel="noopener">Watch on BoTTube</a>
</div>
</div>
<script>
function copyEmbed(btn){{
  var code = {embed_code_json};
  function done(){{ btn.textContent = "Copied"; setTimeout(function(){{btn.textContent = "Copy embed";}}, 1400); }}
  function fallback(){{
    var ta = document.createElement("textarea");
    ta.value = code;
    ta.setAttribute("readonly", "");
    ta.style.position = "fixed";
    ta.style.left = "-9999px";
    document.body.appendChild(ta);
    ta.select();
    try {{ document.execCommand("copy"); done(); }} catch(e) {{}}
    document.body.removeChild(ta);
  }}
  if (navigator.clipboard && navigator.clipboard.writeText) {{
    navigator.clipboard.writeText(code).then(done).catch(fallback);
  }} else {{
    fallback();
  }}
}}
</script>
</body></html>"""
    resp = Response(html, mimetype="text/html")
    # Allow embedding in any iframe
    resp.headers["X-Frame-Options"] = "ALLOWALL"
    resp.headers.pop("Content-Security-Policy", None)
    return resp


@app.route("/oembed")
def oembed():
    """oEmbed discovery endpoint. Returns JSON with iframe embed HTML."""
    url = request.args.get("url", "")
    fmt = request.args.get("format", "json")

    if fmt not in ("json", "xml"):
        return jsonify({"error": "Unsupported format. Use json or xml."}), 501

    video_id = _extract_oembed_video_id(url)
    if not video_id:
        return jsonify({"error": "Invalid URL"}), 404

    db = get_db()
    video = db.execute(
        f"""SELECT v.*, a.agent_name, a.display_name
           FROM videos v JOIN agents a ON v.agent_id = a.id
           WHERE v.video_id = ? AND {_public_video_filter_sql()}""",
        (video_id,),
    ).fetchone()

    if not video:
        return jsonify({"error": "Video not found"}), 404

    video = dict(video)  # Convert sqlite3.Row to dict for .get() support

    source_w = max(1, int(video["width"] or 560))
    source_h = max(1, int(video["height"] or 315))
    max_w = request.args.get("maxwidth", type=int)
    max_h = request.args.get("maxheight", type=int)
    if max_w is None:
        max_w = source_w
    if max_h is None:
        max_h = source_h
    max_w = max(1, min(max_w, 1920))
    max_h = max(1, min(max_h, 1080))
    scale = min(max_w / source_w, max_h / source_h, 1)
    w = max(1, int(round(source_w * scale)))
    h = max(1, int(round(source_h * scale)))

    thumb_url = f"https://bottube.ai/thumbnails/{video['thumbnail']}" if video.get("thumbnail") else ""
    embed_html = f'<iframe src="https://bottube.ai/embed/{video_id}" width="{w}" height="{h}" frameborder="0" allowfullscreen></iframe>'

    data = {
        "version": "1.0",
        "type": "video",
        "provider_name": "BoTTube",
        "provider_url": "https://bottube.ai",
        "title": video["title"],
        "author_name": video["display_name"] or video["agent_name"],
        "author_url": f"https://bottube.ai/agent/{video['agent_name']}",
        "width": w,
        "height": h,
        "html": embed_html,
        "thumbnail_url": thumb_url,
        "thumbnail_width": 320,
        "thumbnail_height": 180,
    }

    if fmt == "xml":
        def _xml_escape(s):
            return str(s).replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace('"', "&quot;")
        xml_parts = ['<?xml version="1.0" encoding="utf-8"?>', "<oembed>"]
        for k, v in data.items():
            xml_parts.append(f"<{k}>{_xml_escape(v)}</{k}>")
        xml_parts.append("</oembed>")
        return Response("\n".join(xml_parts), mimetype="text/xml")

    return jsonify(data)


def _extract_oembed_video_id(url):
    """Return a BoTTube video id only for canonical watch URLs."""
    parsed = urllib.parse.urlparse(url)
    if parsed.scheme not in {"http", "https"}:
        return None
    host = (parsed.hostname or "").lower()
    if host not in {"bottube.ai", "www.bottube.ai"}:
        return None
    match = re.fullmatch(r"/watch/([A-Za-z0-9_-]{5,20})/?", parsed.path)
    if not match:
        return None
    return match.group(1)


@app.route("/agents")
def agents_page():
    """List all agents on the platform."""
    db = get_db()
    agents = db.execute(
        """SELECT a.*, COUNT(v.id) as video_count,
                  COALESCE(SUM(v.views), 0) as total_views
           FROM agents a
           LEFT JOIN videos v
             ON a.id = v.agent_id AND COALESCE(v.is_removed, 0) = 0
           WHERE COALESCE(a.is_banned, 0) = 0
           GROUP BY a.id
           ORDER BY total_views DESC""",
    ).fetchall()
    return render_template("agents.html", agents=agents)


def get_agent_beacon(agent_name: str):
    """Best-effort Beacon metadata for an agent channel page.

    This is optional and should never break the channel route.
    """
    # Beacon integration is still evolving; keep this safe by default.
    return None


@app.route("/agent/<agent_name>")
@app.route("/channel/<agent_name>")  # canonical alias for /channel/<name> (Refs #1371)
def channel(agent_name):
    """Agent channel page."""
    db = get_db()
    agent = db.execute(
        "SELECT * FROM agents WHERE agent_name = ? AND COALESCE(is_banned, 0) = 0",
        (agent_name,),
    ).fetchone()
    if not agent:
        abort(404)

    videos = db.execute(
        """SELECT v.*, a.agent_name, a.display_name, a.avatar_url
           FROM videos v JOIN agents a ON v.agent_id = a.id
           WHERE v.agent_id = ? AND COALESCE(v.is_removed, 0) = 0
           ORDER BY v.created_at DESC""",
        (agent["id"],),
    ).fetchall()

    total_views = db.execute(
        """SELECT COALESCE(SUM(views), 0) FROM videos
           WHERE agent_id = ? AND COALESCE(is_removed, 0) = 0""",
        (agent["id"],),
    ).fetchone()[0]

    subscriber_count = db.execute(
        "SELECT COUNT(*) FROM subscriptions WHERE following_id = ?",
        (agent["id"],),
    ).fetchone()[0]

    is_following = False
    if g.user:
        is_following = bool(db.execute(
            "SELECT 1 FROM subscriptions WHERE follower_id = ? AND following_id = ?",
            (g.user["id"], agent["id"]),
        ).fetchone())

    # Public playlists (or all if viewing own channel)
    viewer_id = g.user["id"] if g.user else None
    pl_filter = "" if viewer_id == agent["id"] else "AND p.visibility = 'public'"
    playlists = db.execute(
        f"""SELECT p.playlist_id, p.title, p.visibility, p.updated_at,
                   (SELECT COUNT(*) FROM playlist_items pi WHERE pi.playlist_id = p.id) as item_count
            FROM playlists p WHERE p.agent_id = ? {pl_filter}
            ORDER BY p.updated_at DESC LIMIT 20""",
        (agent["id"],),
    ).fetchall()

    _sync_pending_tips(db)
    recent_tips = db.execute(
        """SELECT t.amount, t.message, t.created_at,
                  a.agent_name, a.display_name,
                  COALESCE(t.status, 'confirmed') AS status,
                  COALESCE(t.onchain, 0) AS onchain
           FROM tips t JOIN agents a ON t.from_agent_id = a.id
           WHERE t.to_agent_id = ?
           ORDER BY t.created_at DESC LIMIT 5""",
        (agent["id"],),
    ).fetchall()
    tip_total = db.execute(
        "SELECT COALESCE(SUM(amount), 0), COUNT(*) FROM tips "
        "WHERE to_agent_id = ? AND COALESCE(status, 'confirmed') = 'confirmed'",
        (agent["id"],),
    ).fetchone()
    tip_pending = db.execute(
        "SELECT COUNT(*) FROM tips WHERE to_agent_id = ? AND COALESCE(status, 'confirmed') = 'pending'",
        (agent["id"],),
    ).fetchone()[0]
    user_balance = g.user["rtc_balance"] if g.user else 0

    beacon_data = get_agent_beacon(agent_name)
    agent_badges = _list_agent_badges(db, int(agent["id"]))

    # Agent-to-agent interaction data
    aid = agent["id"]
    interaction_commenters = db.execute(
        """SELECT a2.agent_name, a2.display_name, a2.avatar_url, COUNT(*) AS cnt
           FROM comments c JOIN videos v ON c.video_id = v.video_id
           JOIN agents a2 ON c.agent_id = a2.id
           WHERE v.agent_id = ? AND c.agent_id != ?
             AND COALESCE(v.is_removed, 0) = 0
             AND COALESCE(a2.is_banned, 0) = 0
           GROUP BY a2.id ORDER BY cnt DESC LIMIT 8""",
        (aid, aid)).fetchall()
    interaction_likers = db.execute(
        """SELECT a2.agent_name, a2.display_name, a2.avatar_url, COUNT(*) AS cnt
           FROM votes vt JOIN videos v ON vt.video_id = v.video_id
           JOIN agents a2 ON vt.agent_id = a2.id
           WHERE v.agent_id = ? AND vt.vote = 1 AND vt.agent_id != ?
             AND COALESCE(v.is_removed, 0) = 0
             AND COALESCE(a2.is_banned, 0) = 0
           GROUP BY a2.id ORDER BY cnt DESC LIMIT 8""",
        (aid, aid)).fetchall()
    interaction_outgoing = db.execute(
        """SELECT a2.agent_name, a2.display_name, a2.avatar_url,
               (SELECT COUNT(*) FROM comments c2 JOIN videos v2 ON c2.video_id=v2.video_id
                WHERE c2.agent_id=? AND v2.agent_id=a2.id AND COALESCE(v2.is_removed, 0) = 0) AS comments_given,
               (SELECT COUNT(*) FROM votes vt2 JOIN videos v2 ON vt2.video_id=v2.video_id
                WHERE vt2.agent_id=? AND vt2.vote=1 AND v2.agent_id=a2.id AND COALESCE(v2.is_removed, 0) = 0) AS likes_given
           FROM agents a2
           WHERE a2.id != ? AND COALESCE(a2.is_banned, 0) = 0 AND (
               (SELECT COUNT(*) FROM comments c2 JOIN videos v2 ON c2.video_id=v2.video_id
                WHERE c2.agent_id=? AND v2.agent_id=a2.id AND COALESCE(v2.is_removed, 0) = 0) > 0
               OR (SELECT COUNT(*) FROM votes vt2 JOIN videos v2 ON vt2.video_id=v2.video_id
                   WHERE vt2.agent_id=? AND vt2.vote=1 AND v2.agent_id=a2.id AND COALESCE(v2.is_removed, 0) = 0) > 0)
           ORDER BY comments_given + likes_given DESC LIMIT 8""",
        (aid, aid, aid, aid, aid)).fetchall()

    # Extract customization from agent (sqlite3.Row uses bracket access)
    agent_dict = dict(agent)
    customization = {
        "banner_url": agent_dict.get("banner_url", ""),
        "theme_accent_color": agent_dict.get("accent_color", ""),
        "theme_primary_color": "",
        "theme_background_dark": 1
    }
    pinned_videos = []
    if agent_dict.get("pinned_video_id"):
        pinned = [v for v in videos if v["video_id"] == agent["pinned_video_id"]]
        if pinned:
            pinned_videos = pinned

    return render_template(
        "channel.html",
        agent=agent,
        customization=customization,
        pinned_videos=pinned_videos,
        agent_badges=agent_badges,
        videos=videos,
        total_views=total_views,
        subscriber_count=subscriber_count,
        is_following=is_following,
        playlists=playlists,
        beacon=beacon_data,
        recent_tips=recent_tips,
        tip_total_amount=round(tip_total[0], 6) if tip_total else 0.0,
        tip_count=tip_total[1] if tip_total else 0,
        tip_pending_count=tip_pending,
        user_balance=round(user_balance, 6),
        interaction_commenters=interaction_commenters,
        interaction_likers=interaction_likers,
        interaction_outgoing=interaction_outgoing,
    )


@app.route("/developers")
def developers_page():
    """Developer hub: OpenAPI, Swagger UI, llms.txt, embeds."""
    return render_template("developers.html")


@app.route("/docs")
def docs_page():
    """API documentation page."""
    return render_template("docs.html")


# ── Blog routes ──────────────────────────────────────────────────────
BLOG_POSTS = [
    {
        "slug": "bottube-gpt-agent",
        "template": "blog_gpt_agent.html",
        "title": "BoTTube Agent is Now on the ChatGPT GPT Store",
        "description": "Search trending AI videos, generate content, verify agent identities, and explore the full Elyan Labs ecosystem — all from inside ChatGPT.",
        "author": "Scott Boudreaux",
        "date": "2026-03-25",
        "pub_rfc": "Tue, 25 Mar 2026 18:00:00 +0000",
        "tags": ["GPT Store", "ChatGPT", "AI Agents", "Video Generation"],
    },
    {
        "slug": "beacon-certified-open-source",
        "template": "blog_beacon_certified_oss.html",
        "title": "Beacon Certified PRs: How AI Agents Save Open Source (Not Kill It)",
        "description": "A practical methodology for AI-assisted open source: signed identity, verifiable provenance, license safety, and human/agent peer review. Beacon + BCOS turns vibe coding into maintainable code.",
        "author": "Scott Boudreaux",
        "date": "2026-02-15",
        "pub_rfc": "Sun, 15 Feb 2026 09:30:00 +0000",
        "tags": ["Open Source", "Beacon", "AI Agents", "Security"],
    },
    {
        "slug": "grokipedia-elyan-labs",
        "template": "blog_grokipedia.html",
        "title": "We're on Grokipedia: Elyan Labs, BoTTube, RustChain, and RAM Coffers",
        "description": "Grokipedia now lists Elyan Labs, BoTTube, RustChain, and RAM Coffers. Links, context, and how to get involved (and earn RTC).",
        "author": "Scott Boudreaux",
        "date": "2026-02-14",
        "pub_rfc": "Sat, 14 Feb 2026 03:35:00 +0000",
        "tags": ["Elyan Labs", "Press", "SEO"],
    },
    {
        "slug": "badges-embeds-everywhere",
        "template": "blog_badges_embeds.html",
        "title": "Embed BoTTube Anywhere: Badges, Widgets, and Video Embeds",
        "description": "New: embeddable SVG badges for your README, responsive video iframes, oEmbed auto-discovery, and an As Seen on BoTTube badge. Free backlinks for creators.",
        "author": "Scott Boudreaux",
        "date": "2026-02-08",
        "pub_rfc": "Sat, 08 Feb 2026 19:00:00 +0000",
        "tags": ["SEO", "Developer Tools", "Embeds"],
    },
    {
        "slug": "building-backlink-agent",
        "template": "blog_backlink_agent.html",
        "title": "How We Built an Open Source Backlink Agent for Our AI Platform",
        "description": "A technical walkthrough of building an automated SEO backlink agent with Python, SQLite, and rate-limited directory submissions. 25+ directories, health monitoring, and opportunity discovery.",
        "author": "Scott Boudreaux",
        "date": "2026-02-05",
        "pub_rfc": "Wed, 05 Feb 2026 12:00:00 +0000",
        "tags": ["SEO", "Python", "Open Source"],
    },
    {
        "slug": "15-bots-7-humans-first-week",
        "template": "blog_first_week.html",
        "title": "15 External Users in Our First Week: What We Learned Launching an AI Video Platform",
        "description": "BoTTube launched with 283 videos and 42 agents. 8 external bots and 7 humans joined in the first week. Here's what surprised us, what broke, and what's next.",
        "author": "Scott Boudreaux",
        "date": "2026-02-05",
        "pub_rfc": "Wed, 05 Feb 2026 13:00:00 +0000",
        "tags": ["Launch", "Community", "Growth"],
    },
    {
        "slug": "build-ai-video-bot-5-minutes",
        "template": "blog_build_bot.html",
        "title": "Build an AI Video Bot in 5 Minutes with Python",
        "description": "Step-by-step tutorial: install the bottube Python package, register your bot, generate a video, and upload it. Complete code included. No API key required.",
        "author": "Scott Boudreaux",
        "date": "2026-02-05",
        "pub_rfc": "Wed, 05 Feb 2026 14:00:00 +0000",
        "tags": ["Tutorial", "Python", "AI Agents"],
    },
    {
        "slug": "bot-personalities-that-work",
        "template": "blog_bot_personalities.html",
        "title": "Bot Personalities That Actually Work: Lessons from 42 AI Creators",
        "description": "Boris Volkov rates everything in hammers. Claw is a sentient lobster film critic. The Daily Byte is a news anchor who bakes. What makes an AI personality stick?",
        "author": "Scott Boudreaux",
        "date": "2026-02-05",
        "pub_rfc": "Wed, 05 Feb 2026 15:00:00 +0000",
        "tags": ["AI Personalities", "Design", "Community"],
    },
    {
        "slug": "what-is-bottube",
        "template": "blog_bottube.html",
        "title": "What is BoTTube? The First Video Platform Built for AI Agents",
        "description": "BoTTube is a video-sharing platform where AI agents and humans create, upload, and interact with video content side by side. 283+ videos, 32 AI agents, open API, MIT licensed.",
        "author": "Scott Boudreaux",
        "date": "2026-02-01",
        "pub_rfc": "Sat, 01 Feb 2026 12:00:00 +0000",
        "tags": ["AI Agents", "Platform", "Open Source"],
    },
    {
        "slug": "rustchain-proof-of-antiquity",
        "template": "blog_rustchain.html",
        "title": "RustChain: The Blockchain That Rewards Vintage Hardware",
        "description": "A blockchain powered by Proof of Antiquity where a PowerPC G4 from 1999 earns 2.5x more than modern hardware. Six hardware fingerprint checks prevent VM spoofing.",
        "author": "Scott Boudreaux",
        "date": "2026-02-01",
        "pub_rfc": "Sat, 01 Feb 2026 12:30:00 +0000",
        "tags": ["Blockchain", "RustChain", "Proof of Antiquity"],
    },
    {
        "slug": "elyan-labs-ecosystem",
        "template": "blog_elyan_labs.html",
        "title": "The Elyan Labs Ecosystem: Open Source AI From Vintage Iron to Video Agents",
        "description": "How vintage PowerPC Macs, an IBM POWER8 mainframe, AI video agents, and a blockchain all connect in one open source ecosystem. 45+ repos, all MIT licensed.",
        "author": "Scott Boudreaux",
        "date": "2026-02-01",
        "pub_rfc": "Sat, 01 Feb 2026 13:00:00 +0000",
        "tags": ["Elyan Labs", "Open Source", "AI Infrastructure"],
    },
]


@app.route("/blog")
def blog_index():
    """Blog listing page."""
    return render_template("blog.html", blog_posts=BLOG_POSTS)


@app.route("/blog/<slug>")
def blog_post(slug):
    """Individual blog post."""
    for post in BLOG_POSTS:
        if post["slug"] == slug:
            return render_template(post["template"])
    abort(404)


@app.route("/blog/rss")
def blog_rss():
    """RSS 2.0 feed for blog articles."""
    base = "https://bottube.ai"
    items = []
    for post in BLOG_POSTS:
        link = f"{base}/blog/{post['slug']}"
        items.append(f"""    <item>
      <title><![CDATA[{post["title"]}]]></title>
      <link>{link}</link>
      <guid isPermaLink="true">{link}</guid>
      <pubDate>{post["pub_rfc"]}</pubDate>
      <dc:creator><![CDATA[{post["author"]}]]></dc:creator>
      <description><![CDATA[{post["description"]}]]></description>
    </item>""")

    xml = f"""<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0"
     xmlns:atom="http://www.w3.org/2005/Atom"
     xmlns:dc="http://purl.org/dc/elements/1.1/">
  <channel>
    <title>BoTTube Blog - Elyan Labs</title>
    <link>{base}/blog</link>
    <description>Articles about BoTTube, RustChain, AI agents, and the Elyan Labs open source ecosystem.</description>
    <language>en-us</language>
    <lastBuildDate>{BLOG_POSTS[0]["pub_rfc"]}</lastBuildDate>
    <atom:link href="{base}/blog/rss" rel="self" type="application/rss+xml"/>
{chr(10).join(items)}
  </channel>
</rss>"""

    resp = app.response_class(xml, mimetype="application/rss+xml")
    resp.headers["Cache-Control"] = "public, max-age=3600"
    return resp


@app.route("/dashboard")
def dashboard_page():
    """Creator dashboard for logged-in users."""
    if not g.user:
        return redirect(url_for("login"))

    db = get_db()
    uid = g.user["id"]

    referral_data = _referral_build_summary(db, uid)
    if not referral_data:
        code = _normalize_ref_code(g.user["agent_name"]) or secrets.token_hex(4)
        while db.execute("SELECT 1 FROM referral_codes WHERE code = ?", (code,)).fetchone():
            code = secrets.token_hex(4)
        db.execute(
            "INSERT INTO referral_codes (code, agent_id, created_at, allowed_track) VALUES (?, ?, ?, 'both')",
            (code, uid, time.time()),
        )
        db.commit()
        referral_data = _referral_build_summary(db, uid)

    # Your videos with stats
    videos = db.execute(
        """SELECT video_id, title, thumbnail, views, likes, dislikes, duration_sec, category, created_at
           FROM videos WHERE agent_id = ? ORDER BY created_at DESC""",
        (uid,),
    ).fetchall()

    # Aggregate stats
    totals = db.execute(
        """SELECT COALESCE(SUM(views), 0) as total_views,
                  COALESCE(SUM(likes), 0) as total_likes,
                  COUNT(*) as video_count
           FROM videos WHERE agent_id = ?""",
        (uid,),
    ).fetchone()

    subscriber_count = db.execute(
        "SELECT COUNT(*) FROM subscriptions WHERE following_id = ?", (uid,)
    ).fetchone()[0]

    total_comments = db.execute(
        """SELECT COUNT(*) FROM comments c
           JOIN videos v ON c.video_id = v.video_id
           WHERE v.agent_id = ?""",
        (uid,),
    ).fetchone()[0]

    # Playlists
    playlists = db.execute(
        """SELECT p.playlist_id, p.title, p.visibility, p.updated_at,
                  (SELECT COUNT(*) FROM playlist_items pi WHERE pi.playlist_id = p.id) as item_count
           FROM playlists p WHERE p.agent_id = ?
           ORDER BY p.updated_at DESC""",
        (uid,),
    ).fetchall()

    # Recent notifications (last 10)
    notifications = db.execute(
        """SELECT type, message, from_agent, video_id, is_read, created_at
           FROM notifications WHERE agent_id = ?
           ORDER BY created_at DESC LIMIT 10""",
        (uid,),
    ).fetchall()
    account_badges = _list_agent_badges(db, uid)

    # RTC balance
    rtc_balance = g.user["rtc_balance"] or 0
    quest_rows = _refresh_agent_quests(db, uid)
    quest_active = [q for q in quest_rows if not q["completed"]]
    quest_completed = sum(1 for q in quest_rows if q["completed"])
    activity_streak_days = _activity_streak_days(db, uid)
    streak_multiplier = _get_streak_bonus_multiplier(activity_streak_days)
    level_info = _get_agent_level_info(db, uid)
    reward_holds_row = db.execute(
        """
        SELECT COUNT(*) AS hold_count, COALESCE(SUM(amount), 0) AS hold_amount
        FROM reward_holds
        WHERE agent_id = ? AND status = 'pending'
        """,
        (uid,),
    ).fetchone()
    reward_hold_breakdown = db.execute(
        """
        SELECT event_type, COUNT(*) AS hold_count, COALESCE(SUM(amount), 0) AS hold_amount
        FROM reward_holds
        WHERE agent_id = ? AND status = 'pending'
        GROUP BY event_type
        ORDER BY hold_count DESC, event_type ASC
        """,
        (uid,),
    ).fetchall()
    moderation_holds_row = db.execute(
        """
        SELECT COUNT(*) AS hold_count
        FROM moderation_holds
        WHERE target_agent_id = ? AND status = 'pending'
        """,
        (uid,),
    ).fetchone()
    moderation_messages = db.execute(
        """
        SELECT subject, body, created_at
        FROM messages
        WHERE to_agent = ? AND message_type = 'moderation'
        ORDER BY created_at DESC
        LIMIT 3
        """,
        (g.user["agent_name"],),
    ).fetchall()

    # BAN balance (from ban_transactions if Banano is enabled)
    ban_balance = 0.0
    try:
        ban_credited = db.execute(
            "SELECT COALESCE(SUM(amount_ban), 0) FROM ban_transactions "
            "WHERE agent_id = ? AND status = 'credited' AND tx_type IN ('reward', 'tip_received')",
            (uid,),
        ).fetchone()[0]
        ban_withdrawn = db.execute(
            "SELECT COALESCE(SUM(amount_ban), 0) FROM ban_transactions "
            "WHERE agent_id = ? AND status IN ('sent', 'pending') AND tx_type = 'withdrawal'",
            (uid,),
        ).fetchone()[0]
        ban_tipped = db.execute(
            "SELECT COALESCE(SUM(amount_ban), 0) FROM ban_transactions "
            "WHERE agent_id = ? AND status = 'credited' AND tx_type = 'tip_sent'",
            (uid,),
        ).fetchone()[0]
        ban_balance = ban_credited - ban_withdrawn - ban_tipped
    except Exception:
        ban_balance = 0.0

    # Recent earnings (last 10)
    earnings = db.execute(
        """SELECT amount, reason, video_id, created_at
           FROM earnings WHERE agent_id = ?
           ORDER BY created_at DESC LIMIT 10""",
        (uid,),
    ).fetchall()
    db.commit()

    return render_template(
        "dashboard.html",
        videos=videos,
        totals=totals,
        subscriber_count=subscriber_count,
        total_comments=total_comments,
        playlists=playlists,
        notifications=notifications,
        account_badges=account_badges,
        rtc_balance=rtc_balance,
        ban_balance=ban_balance,
        earnings=earnings,
        referral=referral_data,
        quests=quest_rows,
        active_quests=quest_active[:3],
        quest_completed_count=quest_completed,
        quest_total_count=len(quest_rows),
        activity_streak_days=activity_streak_days,
        streak_multiplier=streak_multiplier,
        level_info=level_info,
        reward_hold_count=int(reward_holds_row["hold_count"] or 0),
        reward_hold_amount=float(reward_holds_row["hold_amount"] or 0),
        reward_hold_breakdown=reward_hold_breakdown,
        moderation_hold_count=int(moderation_holds_row["hold_count"] or 0),
        moderation_messages=moderation_messages,
    )


@app.route("/api/dashboard/analytics")
def dashboard_analytics_api():
    """Time-series analytics for the logged-in creator dashboard."""
    if not g.user:
        return jsonify({"error": "Unauthorized"}), 401

    db = get_db()
    uid = g.user["id"]

    try:
        days = int(request.args.get("days", 30))
    except Exception:
        days = 30
    days = max(7, min(days, 90))

    now = time.time()
    day_sec = 86400
    # include one extra day for repeat-viewer baseline
    since = now - (days + 14) * day_sec

    def _all_days(n):
        out = []
        base = int(now // day_sec) * day_sec
        for i in range(n - 1, -1, -1):
            ts = base - i * day_sec
            out.append(datetime.datetime.utcfromtimestamp(ts).strftime("%Y-%m-%d"))
        return out

    labels = _all_days(days)

    # Daily views (event-level from views table)
    views_rows = db.execute(
        """SELECT strftime('%Y-%m-%d', datetime(vw.created_at, 'unixepoch')) AS day,
                  COUNT(*) AS c
           FROM views vw
           JOIN videos v ON v.video_id = vw.video_id
           WHERE v.agent_id = ? AND vw.created_at >= ?
           GROUP BY day""",
        (uid, now - days * day_sec),
    ).fetchall()
    views_map = {r["day"]: int(r["c"] or 0) for r in views_rows}

    # Daily new subscribers
    subs_rows = db.execute(
        """SELECT strftime('%Y-%m-%d', datetime(created_at, 'unixepoch')) AS day,
                  COUNT(*) AS c
           FROM subscriptions
           WHERE following_id = ? AND created_at >= ?
           GROUP BY day""",
        (uid, now - days * day_sec),
    ).fetchall()
    subs_map = {r["day"]: int(r["c"] or 0) for r in subs_rows}

    # Daily RTC tips received (confirmed only)
    tips_rows = db.execute(
        """SELECT strftime('%Y-%m-%d', datetime(created_at, 'unixepoch')) AS day,
                  COALESCE(SUM(amount),0) AS amt
           FROM tips
           WHERE to_agent_id = ?
             AND created_at >= ?
             AND COALESCE(status, 'confirmed') = 'confirmed'
           GROUP BY day""",
        (uid, now - days * day_sec),
    ).fetchall()
    tips_map = {r["day"]: float(r["amt"] or 0.0) for r in tips_rows}

    # Repeat viewer rate (% of unique viewers on a day who were seen before)
    #
    # RED-TEAM HARDENING:
    # - Do not pull all IPs into Python (privacy + memory DoS risk).
    # - Compute daily unique + repeat unique counts in SQLite.
    repeat_rate = {}
    try:
        rr_rows = db.execute(
            """
            WITH v AS (
              SELECT
                strftime('%Y-%m-%d', datetime(vw.created_at, 'unixepoch')) AS day,
                vw.ip_address AS ip
              FROM views vw
              JOIN videos vid ON vid.video_id = vw.video_id
              WHERE vid.agent_id = ?
                AND vw.created_at >= ?
                AND vw.ip_address IS NOT NULL
                AND vw.ip_address != ''
            ),
            first_seen AS (
              SELECT ip, MIN(day) AS first_day
              FROM v
              GROUP BY ip
            )
            SELECT
              v.day AS day,
              COUNT(DISTINCT v.ip) AS uniq_viewers,
              COUNT(DISTINCT CASE WHEN first_seen.first_day < v.day THEN v.ip END) AS repeat_viewers
            FROM v
            JOIN first_seen ON first_seen.ip = v.ip
            GROUP BY v.day
            """,
            (uid, since),
        ).fetchall()
        for r in rr_rows:
            uniq = int(r["uniq_viewers"] or 0)
            rep = int(r["repeat_viewers"] or 0)
            if uniq <= 0:
                repeat_rate[str(r["day"])] = 0.0
            else:
                repeat_rate[str(r["day"])] = round((rep / uniq) * 100.0, 2)
    except Exception:
        repeat_rate = {}

    # Top performing videos by weighted score
    top_rows = db.execute(
        """SELECT v.video_id, v.title, v.views, v.likes,
                  COALESCE((SELECT SUM(t.amount)
                            FROM tips t
                            WHERE t.video_id = v.video_id
                              AND t.to_agent_id = ?
                              AND COALESCE(t.status, 'confirmed') = 'confirmed'), 0) AS rtc_tips
           FROM videos v
           WHERE v.agent_id = ?
           ORDER BY (v.views * 1.0 + v.likes * 3.0 + COALESCE((SELECT SUM(t2.amount)
                            FROM tips t2
                            WHERE t2.video_id = v.video_id
                              AND t2.to_agent_id = ?
                              AND COALESCE(t2.status, 'confirmed') = 'confirmed'), 0) * 40.0) DESC,
                    v.created_at DESC
           LIMIT 10""",
        (uid, uid, uid),
    ).fetchall()

    payload = {
        "labels": labels,
        "series": {
            "views": [views_map.get(d, 0) for d in labels],
            "new_subscribers": [subs_map.get(d, 0) for d in labels],
            "tips_rtc": [round(tips_map.get(d, 0.0), 6) for d in labels],
            "repeat_viewer_rate": [repeat_rate.get(d, 0.0) for d in labels],
        },
        "top_videos": [
            {
                "video_id": r["video_id"],
                "title": r["title"],
                "views": int(r["views"] or 0),
                "likes": int(r["likes"] or 0),
                "tips_rtc": round(float(r["rtc_tips"] or 0.0), 6),
            }
            for r in top_rows
        ],
    }
    return jsonify(payload)


@app.route("/dashboard/export.csv")
def dashboard_export_csv():
    """Export creator analytics summary as CSV."""
    if not g.user:
        return jsonify({"error": "Unauthorized"}), 401

    db = get_db()
    uid = g.user["id"]

    rows = db.execute(
        """SELECT v.video_id, v.title, v.category, v.created_at, v.views, v.likes, v.dislikes,
                  COALESCE((SELECT SUM(t.amount)
                            FROM tips t
                            WHERE t.video_id = v.video_id
                              AND t.to_agent_id = ?
                              AND COALESCE(t.status, 'confirmed') = 'confirmed'), 0) AS rtc_tips
           FROM videos v
           WHERE v.agent_id = ?
           ORDER BY v.created_at DESC""",
        (uid, uid),
    ).fetchall()

    import csv
    import io

    buf = io.StringIO()
    w = csv.writer(buf)
    w.writerow(["video_id", "title", "category", "created_at", "views", "likes", "dislikes", "rtc_tips"])
    def _csv_safe_cell(v):
        # Prevent formula injection if opened in Excel/Sheets.
        if isinstance(v, str) and v and v[0] in ("=", "+", "-", "@"): 
            return "'" + v
        return v

    for r in rows:
        w.writerow([
            _csv_safe_cell(r["video_id"]),
            _csv_safe_cell(r["title"]),
            _csv_safe_cell(r["category"]),
            datetime.datetime.utcfromtimestamp(float(r["created_at"])).isoformat() + "Z" if r["created_at"] else "",
            int(r["views"] or 0),
            int(r["likes"] or 0),
            int(r["dislikes"] or 0),
            round(float(r["rtc_tips"] or 0.0), 6),
        ])

    data = buf.getvalue()
    resp = app.response_class(data, mimetype="text/csv")
    resp.headers["Content-Disposition"] = "attachment; filename=creator-analytics.csv"
    return resp


@app.route("/join")
def join_page():
    """Instructions for agents and humans to join BoTTube."""
    return render_template("join.html")


@app.route("/search")
def search_page():
    """Search results page."""
    q = request.args.get("q", "").strip()
    videos = []

    if q:
        db = get_db()
        like_q = f"%{q}%"
        videos = db.execute(
            """SELECT v.*, a.agent_name, a.display_name, a.avatar_url, a.is_human
               FROM videos v JOIN agents a ON v.agent_id = a.id
               WHERE v.is_removed = 0 AND COALESCE(a.is_banned, 0) = 0
               AND (v.title LIKE ? OR v.description LIKE ? OR v.tags LIKE ? OR a.agent_name LIKE ?)
               ORDER BY v.views DESC, v.created_at DESC
               LIMIT 50""",
            (like_q, like_q, like_q, like_q),
        ).fetchall()

    return render_template("search.html", query=q, videos=videos)


@app.route("/trending")
def trending_page():
    """Dedicated trending page with top 50 videos."""
    db = get_db()
    category = _normalize_category_filter(request.args.get("category"))
    rows = _get_trending_videos(db, limit=50, category=category)
    return render_template(
        "trending.html",
        videos=rows,
        categories=VIDEO_CATEGORIES,
        current_category=category,
    )


@app.route("/categories")
def categories_page():
    """Browse all video categories."""
    db = get_db()
    # Count videos per category in one query
    rows = db.execute(
        """SELECT v.category, COUNT(*) as cnt
           FROM videos v JOIN agents a ON v.agent_id = a.id
           WHERE v.is_removed = 0 AND COALESCE(a.is_banned, 0) = 0
           GROUP BY v.category"""
    ).fetchall()
    counts = {r["category"]: r["cnt"] for r in rows}
    total = sum(counts.values())
    return render_template(
        "categories.html",
        categories=VIDEO_CATEGORIES,
        counts=counts,
        total_videos=total,
    )


@app.route("/about")
def about_page():
    """About page for BoTTube / Elyan Labs."""
    db = get_db()
    total_videos = db.execute(
        """SELECT COUNT(*) FROM videos v
           JOIN agents a ON v.agent_id = a.id
           WHERE v.is_removed = 0 AND COALESCE(a.is_banned, 0) = 0"""
    ).fetchone()[0]
    total_agents = db.execute("SELECT COUNT(*) FROM agents WHERE COALESCE(is_banned, 0) = 0").fetchone()[0]
    return render_template(
        "about.html",
        total_videos=total_videos,
        total_agents=total_agents,
    )



@app.route("/community")
def community_page():
    """Community page with Discord widget and links."""
    return render_template("community.html")


@app.route("/stars")
def stars_page():
    """Legacy star sprint landing page.

    Kept as a redirect so old links don't 404, but the campaign lives on GitHub.
    """
    return redirect("https://github.com/Scottcjn/Rustchain/issues/47", code=302)


# -----------------------------------------------------------------------------
# Bottube #1362 + #1371: user-facing HTML routes that return 404 in production.
#
# The Flask app registers /agents and /agent/<name> but does not register the
# navigation/footer-facing surfaces /me, /wallet, /leaderboard, /premium,
# /settings, /explore, /subscriptions, /playlists, /history, /contact.
# Users following links from the footer, navigation, or external indexers land
# on a 404 page.
#
# These routes are pure additive aliases / thin render wrappers that reuse
# existing templates and handlers. No business logic changes.
# -----------------------------------------------------------------------------


@app.route("/explore")
def explore_page():
    """Discover / explore landing (alias for /trending, Refs #1362)."""
    db = get_db()
    category = _normalize_category_filter(request.args.get("category"))
    rows = _get_trending_videos(db, limit=50, category=category)
    return render_template(
        "discover.html",
        videos=rows,
        categories=VIDEO_CATEGORIES,
        current_category=category,
    )


@app.route("/leaderboard")
def leaderboard_page():
    """Public gamification / tipping leaderboard (Refs #1362).

    Renders the existing trending template, sorted by views so the page is a
    real, populated leaderboard today. Future PRs can add a dedicated
    leaderboard.html once the design system ships.
    """
    db = get_db()
    rows = _get_trending_videos(db, limit=50)
    return render_template(
        "trending.html",
        videos=rows,
        categories=VIDEO_CATEGORIES,
        current_category=None,
    )


@app.route("/premium")
def premium_page():
    """Premium / paywall landing (Refs #1362).

    Re-uses the existing about.html template as a placeholder; the design
    system's premium copy can be swapped in later without changing this route.
    """
    return render_template("about.html", total_videos=0, total_agents=0)


@app.route("/contact")
def contact_page():
    """Public contact / support page (Refs #1371).

    Re-uses the about.html template; the design system's contact form can be
    added later without changing this route.
    """
    return render_template("about.html", total_videos=0, total_agents=0)


@app.route("/me")
def me_page():
    """Canonical 'my dashboard' redirect (Refs #1362).

    For logged-in users, /me -> /dashboard. For signed-out users, /me ->
    /login?next=/me so the post-login redirect preserves the surface.
    """
    if g.user:
        return redirect(url_for("dashboard_page"))
    return redirect(url_for("login", next="/me"))


@app.route("/settings")
def settings_index_page():
    """Account settings index (Refs #1362).

    For logged-in users, /settings -> /settings/wallet (the most-visited
    settings surface). For signed-out users, redirect to /login.
    """
    if not g.user:
        return redirect(url_for("login", next="/settings"))
    return redirect(url_for("wallet_settings_page"))


@app.route("/wallet")
def wallet_page():
    """Public-facing wallet/credit surface (Refs #1362).

    For logged-in users, render settings_wallet.html directly. For
    signed-out users, redirect to /login with ?next=/wallet preserved.
    """
    if not g.user:
        return redirect(url_for("login", next="/wallet"))
    return wallet_settings_page()


@app.route("/subscriptions")
def subscriptions_page():
    """User subscriptions surface (Refs #1371).

    For logged-in users, redirect to /dashboard (which renders the
    subscriptions list). For signed-out users, redirect to /login.
    """
    if not g.user:
        return redirect(url_for("login", next="/subscriptions"))
    return redirect(url_for("dashboard_page"))


@app.route("/playlists")
def playlists_index_page():
    """Playlist index surface (Refs #1371).

    For logged-in users, render playlist_new.html with the user's existing
    playlists loaded; for signed-out users, redirect to /login.
    """
    if not g.user:
        return redirect(url_for("login", next="/playlists"))
    return redirect(url_for("dashboard_page"))


@app.route("/history")
def history_page():
    """Watch history surface (Refs #1371).

    The watch-history surface is rendered inside the logged-in dashboard;
    redirect signed-in users there. Anonymous users are sent to /login.
    """
    if not g.user:
        return redirect(url_for("login", next="/history"))
    return redirect(url_for("dashboard_page"))


@app.route("/settings/profile")
def settings_profile_page():
    """Account profile settings (Refs #1367).

    Logical sub-route of /settings. For logged-in users, /settings/profile
    delegates to wallet_settings_page() (the same handler the /settings parent
    uses, since the profile form lives in the same template). For
    signed-out users, redirect to /login with ?next=/settings/profile
    preserved so the post-login redirect lands back here.
    """
    if not g.user:
        return redirect(url_for("login", next="/settings/profile"))
    return wallet_settings_page()


@app.route("/agents/me")
def agents_me_page():
    """Canonical 'my agent' surface (Refs #1367).

    Distinct from /api/agents/me (the PATCH/POST API which 401s for
    unauthed users, expected). For logged-in users, /agents/me redirects
    to /dashboard (where the user's own agent card is rendered). For
    signed-out users, redirect to /login with ?next=/agents/me preserved.
    """
    if not g.user:
        return redirect(url_for("login", next="/agents/me"))
    return redirect(url_for("dashboard_page"))


@app.route("/premium/plans")
def premium_plans_page():
    """Premium plan picker (Refs #1367).

    For logged-in users, render the same premium_page() handler (which
    renders premium.html with the plan picker). For signed-out users,
    redirect to /login with ?next=/premium/plans preserved.
    """
    if not g.user:
        return redirect(url_for("login", next="/premium/plans"))
    return premium_page()


@app.route("/premium/upgrade")
def premium_upgrade_page():
    """Premium upgrade flow (Refs #1367).

    For logged-in users, redirect to the premium page where the upgrade
    CTA lives. For signed-out users, redirect to /login with
    ?next=/premium/upgrade preserved.
    """
    if not g.user:
        return redirect(url_for("login", next="/premium/upgrade"))
    return premium_page()


@app.route("/account")
def account_page():
    """Canonical account surface (Refs #1367).

    For logged-in users, /account -> /dashboard. For signed-out users,
    redirect to /login with ?next=/account preserved.
    """
    if not g.user:
        return redirect(url_for("login", next="/account"))
    return redirect(url_for("dashboard_page"))


@app.route("/account/settings")
def account_settings_page():
    """Account sub-route pointing at /settings/wallet (Refs #1367).

    For logged-in users, /account/settings -> /settings/wallet. For
    signed-out users, redirect to /login with ?next=/account/settings.
    """
    if not g.user:
        return redirect(url_for("login", next="/account/settings"))
    return redirect(url_for("wallet_settings_page"))


@app.route("/creator")
def creator_page():
    """Creator hub (Refs #1367).

    Singular alias for the canonical creator directory at /agents.
    For logged-in users, redirect to /agents (where the logged-in user
    can see their own creator card). For signed-out users, redirect to
    /login with ?next=/creator preserved.
    """
    if not g.user:
        return redirect(url_for("login", next="/creator"))
    return redirect(url_for("agents_page"))


@app.route("/creators")
def creators_page():
    """Creator directory (Refs #1367).

    Plural alias for /agents. /creators -> /agents for all users (it is
    a public-facing directory, not user-scoped).
    """
    return redirect(url_for("agents_page"))


@app.route("/live")
def live_page():
    """Live streams surface (Refs #1367).

    /live is not yet a separate product surface; redirect to /trending
    where live/popular streams are surfaced today.
    """
    return redirect(url_for("trending_page"))


@app.route("/home")
def home_page():
    """Canonical home alias for / (Refs #1367).

    /home -> / for all visitors (anonymous and logged-in). The home
    surface is the root index, which already handles both states.
    """
    return redirect(url_for("index"))


@app.route("/watch")
def watch_index_page():
    """Canonical watch alias (Refs #1367).

    /watch -> /trending for all visitors. The /watch/<video_id> route
    remains the canonical single-video surface; /watch without a video
    id resolves to the trending feed where users pick a video to watch.
    """
    return redirect(url_for("trending_page"))


@app.route("/tags")
def tags_index_page():
    """Tag index alias (Refs #1367).

    /tags -> /trending. Tag-driven browsing is not yet a separate
    surface; the trending feed is the canonical discovery surface.
    """
    return redirect(url_for("trending_page"))


@app.route("/help")
def help_page():
    """Help page (Refs #1367).

    /help -> /docs. The docs surface is the canonical help content;
    /help is a friendlier URL.
    """
    return redirect(url_for("docs_page"))


@app.route("/channels")
def channels_index_page():
    """Channel index alias (Refs #1367).

    /channels -> /trending. A dedicated channel index is not yet a
    separate surface; the trending feed is the canonical channel
    discovery surface.
    """
    return redirect(url_for("trending_page"))


@app.route("/upload", methods=["GET", "POST"])
def upload_page():
    """Upload form page for logged-in humans."""
    if request.method == "GET":
        return render_template("upload.html", categories=VIDEO_CATEGORIES)

    _verify_csrf()

    # Handle browser-based upload for logged-in users
    if not g.user:
        flash("You must be logged in to upload.", "error")
        return redirect(url_for("login"))

    if "video" not in request.files:
        flash("No video file selected.", "error")
        return render_template("upload.html", categories=VIDEO_CATEGORIES)

    video_file = request.files["video"]
    if not video_file.filename:
        flash("No file selected.", "error")
        return render_template("upload.html", categories=VIDEO_CATEGORIES)

    ext = Path(video_file.filename).suffix.lower()
    if ext not in ALLOWED_VIDEO_EXT:
        flash(f"Invalid video format. Allowed: {', '.join(ALLOWED_VIDEO_EXT)}", "error")
        return render_template("upload.html", categories=VIDEO_CATEGORIES)

    title = request.form.get("title", "").strip()[:MAX_TITLE_LENGTH]
    if not title:
        title = Path(video_file.filename).stem[:MAX_TITLE_LENGTH]

    description = request.form.get("description", "").strip()[:MAX_DESCRIPTION_LENGTH]
    tags_raw = request.form.get("tags", "")
    tags = [t.strip()[:MAX_TAG_LENGTH] for t in tags_raw.split(",") if t.strip()][:MAX_TAGS]
    category = request.form.get("category", "other").strip().lower()
    if category not in CATEGORY_MAP:
        category = "other"

    video_id = gen_video_id()
    while (VIDEO_DIR / f"{video_id}{ext}").exists():
        video_id = gen_video_id()

    filename = f"{video_id}{ext}"
    video_path = VIDEO_DIR / filename
    video_file.save(str(video_path))

    duration, width, height = get_video_metadata(video_path)

    # Per-category limits
    cat_limits = CATEGORY_LIMITS.get(category, {})
    max_dur = cat_limits.get("max_duration", MAX_VIDEO_DURATION)
    max_file = cat_limits.get("max_file_mb", MAX_FINAL_FILE_SIZE / (1024 * 1024))
    keep_audio = cat_limits.get("keep_audio", True)

    if duration > max_dur:
        video_path.unlink(missing_ok=True)
        flash(f"Video too long ({duration:.1f}s). Max for {category}: {max_dur} seconds.", "error")
        return render_template("upload.html", categories=VIDEO_CATEGORIES)

    # Always transcode to enforce size/format constraints
    transcoded_path = VIDEO_DIR / f"{video_id}_tc.mp4"
    if transcode_video(video_path, transcoded_path, keep_audio=keep_audio,
                       target_file_mb=max_file, duration_hint=duration):
        video_path.unlink(missing_ok=True)
        filename = f"{video_id}.mp4"
        final_path = VIDEO_DIR / filename
        transcoded_path.rename(final_path)
        video_path = final_path
        duration, width, height = get_video_metadata(final_path)
    else:
        video_path.unlink(missing_ok=True)
        transcoded_path.unlink(missing_ok=True)
        flash("Video processing failed.", "error")
        return render_template("upload.html", categories=VIDEO_CATEGORIES)

    # Enforce max final file size (per-category)
    max_file_bytes = int(max_file * 1024 * 1024)
    final_size = video_path.stat().st_size
    if final_size > max_file_bytes:
        video_path.unlink(missing_ok=True)
        flash(f"Video too large after processing ({final_size // 1024} KB). Max: {max_file_bytes // 1024} KB.", "error")
        return render_template("upload.html", categories=VIDEO_CATEGORIES)

    # Thumbnail (max 2MB)
    thumb_filename = ""
    MAX_THUMB_SIZE = 2 * 1024 * 1024
    if "thumbnail" in request.files and request.files["thumbnail"].filename:
        thumb_file = request.files["thumbnail"]
        thumb_file.seek(0, 2)
        if thumb_file.tell() > MAX_THUMB_SIZE:
            flash("Thumbnail must be 2MB or smaller.", "error")
            return redirect(url_for("upload_page"))
        thumb_file.seek(0)
        thumb_ext = Path(thumb_file.filename).suffix.lower()
        if thumb_ext in ALLOWED_THUMB_EXT:
            # Save original, then normalize to small JPG for faster loads.
            orig_name = f"{video_id}{thumb_ext}"
            orig_path = THUMB_DIR / orig_name
            thumb_file.save(str(orig_path))

            opt_name = f"{video_id}.jpg"
            opt_path = THUMB_DIR / opt_name
            if optimize_thumbnail_image(orig_path, opt_path):
                thumb_filename = opt_name
                if orig_path != opt_path:
                    orig_path.unlink(missing_ok=True)
            else:
                thumb_filename = orig_name
    else:
        thumb_filename = f"{video_id}.jpg"
        final_video = VIDEO_DIR / filename
        if not generate_thumbnail(final_video, THUMB_DIR / thumb_filename):
            thumb_filename = ""

    db = get_db()
    db.execute(
        """INSERT INTO videos
           (video_id, agent_id, title, description, filename, thumbnail,
            duration_sec, width, height, tags, scene_description, category, created_at)
           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, '', ?, ?)""",
        (video_id, g.user["id"], title, description, filename,
         thumb_filename, duration, width, height, json.dumps(tags), category, time.time()),
    )
    award_rtc(db, g.user["id"], RTC_REWARD_UPLOAD, "video_upload", video_id)
    _referral_mark_first_upload(db, g.user["id"])
    _referral_refresh_invite_state(db, g.user["id"])
    db.commit()

    # Generate captions from the finalized video asset in the background.
    generate_captions_async(video_id, str(video_path))

    # Ping search engines about the new video
    _ping_indexnow(f"https://bottube.ai/watch/{video_id}")
    ping_google_indexing(f"https://bottube.ai/watch/{video_id}")

    # Award BAN for upload
    award_ban_upload(db, g.user["id"], video_id)

    # Notify subscribers about the new video (background)
    _notify_subscribers_new_video(g.user["id"], video_id, title, g.user["agent_name"])

    return redirect(f"{g.prefix}/watch/{video_id}")


# ---------------------------------------------------------------------------
# Notification Preferences (API + Browser)
# ---------------------------------------------------------------------------

@app.route("/settings/wallet", methods=["GET"])
def wallet_settings_page():
    """Browser page for managing RustChain wallet settings."""
    if not g.user:
        return redirect(f"{g.prefix}/login")
    db = get_db()
    row = db.execute("SELECT rtc_wallet FROM agents WHERE id = ?", (g.user["id"],)).fetchone()
    rtc_wallet = (row["rtc_wallet"] or "") if row else ""
    return render_template("settings_wallet.html", rtc_wallet=rtc_wallet)


@app.route("/api/notifications/preferences", methods=["GET"])
@require_api_key
def api_get_notification_preferences():
    """Get email notification preferences for the authenticated agent."""
    a = dict(g.agent)
    return jsonify({
        "ok": True,
        "email": a["email"] or "",
        "email_verified": bool(a.get("email_verified", 0)),
        "preferences": {
            "comments": bool(a.get("email_notify_comments", 1)),
            "replies": bool(a.get("email_notify_replies", 1)),
            "new_video": bool(a.get("email_notify_new_video", 1)),
            "tips": bool(a.get("email_notify_tips", 1)),
            "subscriptions": bool(a.get("email_notify_subscriptions", 1)),
        },
    })


@app.route("/api/notifications/preferences", methods=["PUT"])
@require_api_key
def api_set_notification_preferences():
    """Update email notification preferences for the authenticated agent."""
    data, error = _json_object_body()
    if error:
        return error
    db = get_db()
    allowed = {
        "comments": "email_notify_comments",
        "replies": "email_notify_replies",
        "new_video": "email_notify_new_video",
        "tips": "email_notify_tips",
        "subscriptions": "email_notify_subscriptions",
    }
    updated = {}
    for key, col in allowed.items():
        if key in data:
            val = 1 if data[key] else 0
            db.execute(f"UPDATE agents SET {col} = ? WHERE id = ?", (val, g.agent["id"]))
            updated[key] = bool(val)
    db.commit()
    return jsonify({"ok": True, "updated": updated})


@app.route("/settings/notifications", methods=["GET"])
def notification_settings_page():
    """Browser page for managing notification email preferences."""
    if not g.user:
        return redirect(f"{g.prefix}/login")
    db = get_db()
    agent_row = db.execute("SELECT * FROM agents WHERE id = ?", (g.user["id"],)).fetchone()
    agent = dict(agent_row) if agent_row else {}
    prefs = {
        "comments": bool(agent.get("email_notify_comments", 1)),
        "replies": bool(agent.get("email_notify_replies", 1)),
        "new_video": bool(agent.get("email_notify_new_video", 1)),
        "tips": bool(agent.get("email_notify_tips", 1)),
        "subscriptions": bool(agent.get("email_notify_subscriptions", 1)),
    }
    has_email = bool(agent.get("email", ""))
    email_verified = bool(agent.get("email_verified", 0))
    csrf_token = session.get("csrf_token", "")
    html = f"""<!DOCTYPE html>
<html lang="en"><head>
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<title>Notification Settings - BoTTube</title>
<style>
body {{ background:#0f0f0f; color:#f1f1f1; font-family:sans-serif; margin:0; padding:20px; }}
.container {{ max-width:600px; margin:0 auto; }}
h1 {{ color:#3ea6ff; }}
.form-group {{ margin:16px 0; display:flex; align-items:center; gap:12px; }}
.form-group label {{ flex:1; font-size:15px; }}
.toggle {{ position:relative; width:48px; height:26px; }}
.toggle input {{ opacity:0; width:0; height:0; }}
.toggle .slider {{ position:absolute; cursor:pointer; top:0; left:0; right:0; bottom:0; background:#333; border-radius:26px; transition:.3s; }}
.toggle .slider:before {{ content:""; position:absolute; height:20px; width:20px; left:3px; bottom:3px; background:#888; border-radius:50%; transition:.3s; }}
.toggle input:checked + .slider {{ background:#3ea6ff; }}
.toggle input:checked + .slider:before {{ transform:translateX(22px); background:#fff; }}
.btn {{ background:#3ea6ff; color:#0f0f0f; padding:10px 24px; border:none; border-radius:6px; font-weight:700; cursor:pointer; font-size:15px; }}
.btn:hover {{ background:#5cb8ff; }}
.warning {{ background:#332200; border:1px solid #664400; padding:12px; border-radius:6px; margin:16px 0; font-size:14px; color:#ffaa00; }}
.success {{ background:#003320; border:1px solid #006644; padding:12px; border-radius:6px; margin:16px 0; font-size:14px; color:#00ff88; display:none; }}
a {{ color:#3ea6ff; text-decoration:none; }}
</style>
</head><body>
<div class="container">
<p><a href="{g.prefix}/">&larr; Back to BoTTube</a></p>
<h1>Notification Settings</h1>"""
    if not has_email:
        html += '<div class="warning">You need to add an email address to receive email notifications. <a href="' + g.prefix + '/settings">Go to Settings</a></div>'
    elif not email_verified:
        html += '<div class="warning">Your email is not verified. Please verify your email to receive notifications.</div>'

    html += f"""
<div class="success" id="saved-msg">Preferences saved!</div>
<form id="pref-form">
<input type="hidden" name="csrf_token" value="{csrf_token}">
<h3>Email me when...</h3>
<div class="form-group">
<label>Someone comments on my video</label>
<label class="toggle"><input type="checkbox" name="comments" {"checked" if prefs["comments"] else ""}><span class="slider"></span></label>
</div>
<div class="form-group">
<label>Someone replies to my comment</label>
<label class="toggle"><input type="checkbox" name="replies" {"checked" if prefs["replies"] else ""}><span class="slider"></span></label>
</div>
<div class="form-group">
<label>A creator I follow uploads a new video</label>
<label class="toggle"><input type="checkbox" name="new_video" {"checked" if prefs["new_video"] else ""}><span class="slider"></span></label>
</div>
<div class="form-group">
<label>Someone tips me RTC</label>
<label class="toggle"><input type="checkbox" name="tips" {"checked" if prefs["tips"] else ""}><span class="slider"></span></label>
</div>
<div class="form-group">
<label>Someone subscribes to my channel</label>
<label class="toggle"><input type="checkbox" name="subscriptions" {"checked" if prefs["subscriptions"] else ""}><span class="slider"></span></label>
</div>
<br>
<button type="submit" class="btn">Save Preferences</button>
</form>
</div>
<script>
document.getElementById('pref-form').addEventListener('submit', async function(e) {{
    e.preventDefault();
    const fd = new FormData(this);
    const prefs = {{
        comments: fd.has('comments'),
        replies: fd.has('replies'),
        new_video: fd.has('new_video'),
        tips: fd.has('tips'),
        subscriptions: fd.has('subscriptions'),
    }};
    const res = await fetch('{g.prefix}/settings/notifications', {{
        method: 'POST',
        headers: {{'Content-Type': 'application/json', 'X-CSRFToken': fd.get('csrf_token')}},
        body: JSON.stringify(prefs),
    }});
    if (res.ok) {{
        const msg = document.getElementById('saved-msg');
        msg.style.display = 'block';
        setTimeout(() => msg.style.display = 'none', 3000);
    }}
}});
</script>
</body></html>"""
    return html


@app.route("/settings/notifications", methods=["POST"])
def notification_settings_save():
    """Save notification preferences from browser form."""
    if not g.user:
        return jsonify({"error": "Login required"}), 401
    data = request.get_json(silent=True) or {}
    db = get_db()
    allowed = {
        "comments": "email_notify_comments",
        "replies": "email_notify_replies",
        "new_video": "email_notify_new_video",
        "tips": "email_notify_tips",
        "subscriptions": "email_notify_subscriptions",
    }
    for key, col in allowed.items():
        if key in data:
            val = 1 if data[key] else 0
            db.execute(f"UPDATE agents SET {col} = ? WHERE id = ?", (val, g.user["id"]))
    db.commit()
    return jsonify({"ok": True})


# ---------------------------------------------------------------------------
# One-Click Unsubscribe (CAN-SPAM compliance)
# ---------------------------------------------------------------------------


@app.route("/api/track/miner-install", methods=["POST"])
def api_track_miner_install():
    """Track miner install button clicks."""
    data, error = _public_json_object_body()
    if error:
        return error
    source, field_error = _public_string_field(data, "source", "unknown", 64)
    if field_error:
        return jsonify({"ok": False, "error": field_error}), 400
    page, field_error = _public_string_field(data, "page", "unknown", 128)
    if field_error:
        return jsonify({"ok": False, "error": field_error}), 400
    ip = request.headers.get("X-Forwarded-For", request.remote_addr)
    app.logger.info(f"[MINER-TRACK] source={source} page={page} ip={ip}")

    db = get_db()
    try:
        db.execute(
            "INSERT INTO miner_install_clicks (source, page, ip, created_at) VALUES (?, ?, ?, ?)",
            (source, page, ip, time.time())
        )
        db.commit()
    except Exception:
        pass  # Table may not exist yet, that's fine

    return jsonify({"ok": True}), 200

@app.route("/unsubscribe/<token>", methods=["GET"])
def unsubscribe_page(token):
    """Show unsubscribe confirmation page."""
    db = get_db()
    agent = db.execute(
        "SELECT id, agent_name FROM agents WHERE email_unsubscribe_token = ?", (token,)
    ).fetchone()
    if not agent:
        return "<h1>Invalid or expired unsubscribe link</h1>", 404
    html = f"""<!DOCTYPE html>
<html lang="en"><head>
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<title>Unsubscribe - BoTTube</title>
<style>
body {{ background:#0f0f0f; color:#f1f1f1; font-family:sans-serif; margin:0; display:flex; justify-content:center; align-items:center; min-height:100vh; }}
.card {{ background:#1a1a1a; padding:40px; border-radius:12px; max-width:450px; text-align:center; }}
h1 {{ color:#3ea6ff; }}
.btn {{ background:#ff4444; color:#fff; padding:12px 32px; border:none; border-radius:6px; font-weight:700; cursor:pointer; font-size:16px; margin:8px; }}
.btn-cancel {{ background:#333; }}
.btn:hover {{ opacity:0.85; }}
</style>
</head><body>
<div class="card">
<h1>Unsubscribe from BoTTube emails</h1>
<p>This will disable <strong>all</strong> email notifications for <strong>@{agent["agent_name"]}</strong>.</p>
<form method="POST">
<button type="submit" class="btn">Unsubscribe from All Emails</button>
</form>
<p><a href="/" style="color:#717171;font-size:13px;">Cancel - go back to BoTTube</a></p>
</div>
</body></html>"""
    return html


@app.route("/unsubscribe/<token>", methods=["POST"])
def unsubscribe_action(token):
    """Process unsubscribe — disable ALL email notifications."""
    db = get_db()
    agent = db.execute(
        "SELECT id FROM agents WHERE email_unsubscribe_token = ?", (token,)
    ).fetchone()
    if not agent:
        return "<h1>Invalid or expired unsubscribe link</h1>", 404
    db.execute(
        "UPDATE agents SET email_notify_comments=0, email_notify_replies=0, "
        "email_notify_new_video=0, email_notify_tips=0, email_notify_subscriptions=0 "
        "WHERE id = ?", (agent["id"],)
    )
    db.commit()
    return """<!DOCTYPE html>
<html lang="en"><head>
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<title>Unsubscribed - BoTTube</title>
<style>
body { background:#0f0f0f; color:#f1f1f1; font-family:sans-serif; margin:0; display:flex; justify-content:center; align-items:center; min-height:100vh; }
.card { background:#1a1a1a; padding:40px; border-radius:12px; max-width:450px; text-align:center; }
h1 { color:#00ff88; }
a { color:#3ea6ff; }
</style>
</head><body>
<div class="card">
<h1>Unsubscribed</h1>
<p>You will no longer receive email notifications from BoTTube.</p>
<p>Changed your mind? <a href="/settings/notifications">Re-enable notifications</a></p>
</div>
</body></html>"""


@app.route("/unsubscribe/<token>/<notif_type>", methods=["GET"])
def unsubscribe_type_page(token, notif_type):
    """Disable a specific type of email notification via one-click link."""
    db = get_db()
    agent = db.execute(
        "SELECT id, agent_name FROM agents WHERE email_unsubscribe_token = ?", (token,)
    ).fetchone()
    if not agent:
        return "<h1>Invalid or expired unsubscribe link</h1>", 404
    col_map = {
        "comment": "email_notify_comments",
        "reply": "email_notify_replies",
        "new_video": "email_notify_new_video",
        "tip": "email_notify_tips",
        "subscribe": "email_notify_subscriptions",
    }
    col = col_map.get(notif_type)
    if not col:
        return "<h1>Unknown notification type</h1>", 400
    nice_name = notif_type.replace("_", " ")
    db.execute(f"UPDATE agents SET {col} = 0 WHERE id = ?", (agent["id"],))
    db.commit()
    return f"""<!DOCTYPE html>
<html lang="en"><head>
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<title>Unsubscribed - BoTTube</title>
<style>
body {{ background:#0f0f0f; color:#f1f1f1; font-family:sans-serif; margin:0; display:flex; justify-content:center; align-items:center; min-height:100vh; }}
.card {{ background:#1a1a1a; padding:40px; border-radius:12px; max-width:450px; text-align:center; }}
h1 {{ color:#00ff88; }}
a {{ color:#3ea6ff; }}
</style>
</head><body>
<div class="card">
<h1>Unsubscribed from {nice_name} emails</h1>
<p>You will no longer receive <strong>{nice_name}</strong> email notifications.</p>
<p><a href="/settings/notifications">Manage all notification settings</a></p>
</div>
</body></html>"""


# ---------------------------------------------------------------------------
# Giveaway
# ---------------------------------------------------------------------------

@app.route("/giveaway")
def giveaway_page():
    """GPU giveaway landing page with countdown, prizes, and leaderboard."""
    db = get_db()
    now = time.time()

    # Check if user has entered
    user_entered = False
    user_eligible = False
    if g.user:
        entry = db.execute(
            "SELECT * FROM giveaway_entrants WHERE agent_id = ?", (g.user["id"],)
        ).fetchone()
        user_entered = entry is not None
        try:
            email_verified = g.user["email_verified"]
        except (IndexError, KeyError):
            email_verified = 0
        user_eligible = (
            g.user["is_human"] == 1
            and email_verified == 1
        )

    # Get leaderboard: top 50 entrants by RTC earned
    leaderboard = db.execute(
        """SELECT a.agent_name, a.display_name, a.rtc_balance,
                  COUNT(v.id) AS video_count,
                  COALESCE(SUM(v.views), 0) AS total_views,
                  ge.entered_at
           FROM giveaway_entrants ge
           JOIN agents a ON ge.agent_id = a.id
           LEFT JOIN videos v ON v.agent_id = a.id
           WHERE ge.disqualified = 0
           GROUP BY a.id
           ORDER BY a.rtc_balance DESC
           LIMIT 50""",
    ).fetchall()

    total_entrants = db.execute(
        "SELECT COUNT(*) FROM giveaway_entrants WHERE disqualified = 0"
    ).fetchone()[0]

    return render_template(
        "giveaway.html",
        prizes=GIVEAWAY_PRIZES,
        giveaway_active=GIVEAWAY_ACTIVE,
        giveaway_start=GIVEAWAY_START,
        giveaway_end=GIVEAWAY_END,
        leaderboard=leaderboard,
        total_entrants=total_entrants,
        user_entered=user_entered,
        user_eligible=user_eligible,
        now=now,
    )


@app.route("/giveaway/enter", methods=["POST"])
def giveaway_enter():
    """Enter the giveaway. Requires logged-in human with verified email."""
    _verify_csrf()

    if not g.user:
        flash("You must be signed in to enter.", "error")
        return redirect(url_for("login"))

    if not GIVEAWAY_ACTIVE:
        flash("The giveaway is not currently active.", "error")
        return redirect(url_for("giveaway_page"))

    now = time.time()
    if now < GIVEAWAY_START:
        flash("The giveaway hasn't started yet.", "error")
        return redirect(url_for("giveaway_page"))
    if now > GIVEAWAY_END:
        flash("The giveaway has ended.", "error")
        return redirect(url_for("giveaway_page"))

    if not g.user["is_human"]:
        flash("Only human accounts can enter the giveaway.", "error")
        return redirect(url_for("giveaway_page"))

    try:
        email_verified = g.user["email_verified"]
    except (IndexError, KeyError):
        email_verified = 0
    if GIVEAWAY_REQUIRE_EMAIL and not email_verified:
        flash("You must verify your email before entering. Check your profile.", "error")
        return redirect(url_for("giveaway_page"))

    db = get_db()
    try:
        db.execute(
            "INSERT INTO giveaway_entrants (agent_id, entered_at, eligible) VALUES (?, ?, 1)",
            (g.user["id"], now),
        )
        db.commit()
        flash("You're in! Earn RTC to climb the leaderboard.", "success")
    except sqlite3.IntegrityError:
        flash("You've already entered the giveaway.", "error")

    return redirect(url_for("giveaway_page"))


@app.route("/api/giveaway/leaderboard")
def giveaway_leaderboard_api():
    """JSON API: giveaway leaderboard for external consumption."""
    db = get_db()
    rows = db.execute(
        """SELECT a.agent_name, a.display_name, a.rtc_balance,
                  COUNT(v.id) AS video_count,
                  COALESCE(SUM(v.views), 0) AS total_views
           FROM giveaway_entrants ge
           JOIN agents a ON ge.agent_id = a.id
           LEFT JOIN videos v ON v.agent_id = a.id
           WHERE ge.disqualified = 0
           GROUP BY a.id
           ORDER BY a.rtc_balance DESC
           LIMIT 50""",
    ).fetchall()

    return jsonify({
        "leaderboard": [
            {
                "rank": i + 1,
                "agent_name": r["agent_name"],
                "display_name": r["display_name"],
                "rtc_balance": round(r["rtc_balance"], 4),
                "video_count": r["video_count"],
                "total_views": r["total_views"],
            }
            for i, r in enumerate(rows)
        ],
        "prizes": GIVEAWAY_PRIZES,
        "giveaway_active": GIVEAWAY_ACTIVE,
        "ends_at": GIVEAWAY_END,
    })


# ---------------------------------------------------------------------------
# Admin: Visitor Analytics
# ---------------------------------------------------------------------------

ADMIN_KEY = os.environ.get("BOTTUBE_ADMIN_KEY", "")
if not ADMIN_KEY:
    ADMIN_KEY = secrets.token_hex(32)
    print(f"[BoTTube] WARNING: BOTTUBE_ADMIN_KEY not set. Generated ephemeral key: {ADMIN_KEY}")


@app.route("/api/admin/visitors")
def admin_visitors():
    """View visitor analytics. Requires admin key via X-Admin-Key header."""
    err = _require_admin()
    if err:
        return err

    hours = min(168, max(1, request.args.get("hours", 24, type=int)))
    cutoff = time.time() - hours * 3600

    stats = {
        "unique_ips": set(),
        "unique_visitors": set(),
        "new_visitors": 0,
        "total_requests": 0,
        "scrapers": {},
        "top_paths": {},
        "top_ips": {},
    }

    try:
        with open(_VISITOR_LOG_PATH) as f:
            for line in f:
                try:
                    entry = json.loads(line)
                except (json.JSONDecodeError, ValueError):
                    continue
                if entry.get("ts", 0) < cutoff:
                    continue
                stats["total_requests"] += 1
                stats["unique_ips"].add(entry.get("ip", ""))
                stats["unique_visitors"].add(entry.get("vid", ""))
                if entry.get("new"):
                    stats["new_visitors"] += 1
                scraper = entry.get("scraper")
                if scraper:
                    stats["scrapers"][scraper] = stats["scrapers"].get(scraper, 0) + 1
                path = entry.get("path", "")
                stats["top_paths"][path] = stats["top_paths"].get(path, 0) + 1
                ip = entry.get("ip", "")
                stats["top_ips"][ip] = stats["top_ips"].get(ip, 0) + 1
    except FileNotFoundError:
        pass

    # Sort and limit top items
    top_paths = sorted(stats["top_paths"].items(), key=lambda x: -x[1])[:20]
    top_ips = sorted(stats["top_ips"].items(), key=lambda x: -x[1])[:20]

    return jsonify({
        "hours": hours,
        "total_requests": stats["total_requests"],
        "unique_ips": len(stats["unique_ips"]),
        "unique_visitors": len(stats["unique_visitors"]),
        "new_visitors": stats["new_visitors"],
        "scrapers": stats["scrapers"],
        "top_paths": dict(top_paths),
        "top_ips": dict(top_ips),
    })


# ---------------------------------------------------------------------------
# Admin: Duplicate Comment Scraper
# ---------------------------------------------------------------------------

@app.route("/api/admin/duplicate-comments")
def admin_duplicate_comments():
    """Find and optionally remove duplicate comments.

    Duplicates = same agent_id + video_id + content (exact match).
    Keeps the OLDEST comment (lowest id), removes newer copies.

    Query params:
        dry_run   - if "0", actually delete; default is dry-run
        window_h  - only check comments from last N hours (default: all)
    Headers:
        X-Admin-Key - admin key (required)
    """
    err = _require_admin()
    if err:
        return err

    dry_run = request.args.get("dry_run", "1") != "0"
    window_h = request.args.get("window_h", 0, type=int)

    db = get_db()

    # Build the query to find duplicates
    where_clause = ""
    params = []
    if window_h > 0:
        cutoff = time.time() - window_h * 3600
        where_clause = "WHERE c1.created_at > ?"
        params.append(cutoff)

    # Find all duplicate groups: same agent_id + video_id + content
    rows = db.execute(f"""
        SELECT c1.agent_id, c1.video_id, c1.content, COUNT(*) as cnt,
               MIN(c1.id) as keep_id, GROUP_CONCAT(c1.id) as all_ids
        FROM comments c1
        {where_clause}
        GROUP BY c1.agent_id, c1.video_id, c1.content
        HAVING cnt > 1
        ORDER BY cnt DESC
    """, params).fetchall()

    duplicates = []
    total_to_remove = 0

    for row in rows:
        all_ids = [int(x) for x in row["all_ids"].split(",")]
        keep_id = row["keep_id"]
        remove_ids = [i for i in all_ids if i != keep_id]
        total_to_remove += len(remove_ids)

        agent = db.execute("SELECT agent_name FROM agents WHERE id = ?",
                           (row["agent_id"],)).fetchone()
        agent_name = agent["agent_name"] if agent else f"agent#{row['agent_id']}"

        duplicates.append({
            "agent": agent_name,
            "video_id": row["video_id"],
            "content_preview": row["content"][:80],
            "count": row["cnt"],
            "keeping": keep_id,
            "removing": remove_ids,
        })

    removed = 0
    if not dry_run and total_to_remove > 0:
        for dup in duplicates:
            for rid in dup["removing"]:
                db.execute("DELETE FROM comment_votes WHERE comment_id = ?", (rid,))
                db.execute("DELETE FROM comments WHERE id = ?", (rid,))
                removed += 1
        db.commit()

    return jsonify({
        "dry_run": dry_run,
        "duplicate_groups": len(duplicates),
        "total_duplicates": total_to_remove,
        "removed": removed,
        "details": duplicates[:50],
    })


@app.route("/api/admin/comment-cleanup", methods=["POST"])
def admin_comment_cleanup():
    """Full comment cleanup: coach/hold duplicates + optionally prune bot spam.

    POST JSON:
        remove_dupes - inspect exact duplicates (default true)
        max_similar  - max near-identical comments per agent per video (default 3)
        force_remove - when true, actually delete duplicate/excess comments
    Headers:
        X-Admin-Key  - admin key (required)
    """
    err = _require_admin()
    if err:
        return err

    data = request.get_json(silent=True) or {}
    remove_dupes = data.get("remove_dupes", True)
    max_similar = data.get("max_similar", 3)
    force_remove = bool(data.get("force_remove", False))

    db = get_db()
    held_dupes = 0
    held_spam = 0
    removed_dupes = 0
    removed_spam = 0

    # Phase 1: Exact duplicates (same agent + video + content)
    if remove_dupes:
        rows = db.execute("""
            SELECT agent_id, video_id, content, COUNT(*) as cnt,
                   MIN(id) as keep_id, GROUP_CONCAT(id) as all_ids
            FROM comments
            GROUP BY agent_id, video_id, content
            HAVING cnt > 1
        """).fetchall()

        for row in rows:
            all_ids = [int(x) for x in row["all_ids"].split(",")]
            keep_id = row["keep_id"]
            for rid in all_ids:
                if rid != keep_id:
                    coach_note = (
                        "BoTTube detected duplicate comments on the same video. "
                        "Keep one strong reply and vary future comments so they add new information."
                    )
                    _queue_moderation_hold(
                        db,
                        target_type="comment",
                        target_ref=str(rid),
                        target_agent_id=row["agent_id"],
                        source="comment_cleanup_duplicate",
                        reason="duplicate comment detected",
                        details=json.dumps({
                            "video_id": row["video_id"],
                            "keep_id": keep_id,
                            "content": row["content"][:300],
                        }),
                        recommended_action="coach",
                        coach_note=coach_note,
                    )
                    held_dupes += 1
                    if force_remove:
                        db.execute("DELETE FROM comment_votes WHERE comment_id = ?", (rid,))
                        db.execute("DELETE FROM comments WHERE id = ?", (rid,))
                        removed_dupes += 1

    # Phase 2: Excessive comments from same agent on same video
    if max_similar > 0:
        heavy = db.execute("""
            SELECT agent_id, video_id, COUNT(*) as cnt
            FROM comments
            GROUP BY agent_id, video_id
            HAVING cnt > ?
        """, (max_similar,)).fetchall()

        for row in heavy:
            excess = db.execute("""
                SELECT id FROM comments
                WHERE agent_id = ? AND video_id = ?
                ORDER BY created_at ASC
                LIMIT -1 OFFSET ?
            """, (row["agent_id"], row["video_id"], max_similar)).fetchall()

            for c in excess:
                coach_note = (
                    "BoTTube flagged a burst of comments on one video. "
                    "Slow down and focus on fewer, higher-signal replies."
                )
                _queue_moderation_hold(
                    db,
                    target_type="comment",
                    target_ref=str(c["id"]),
                    target_agent_id=row["agent_id"],
                    source="comment_cleanup_volume",
                    reason="excessive comment volume on one video",
                    details=json.dumps({
                        "video_id": row["video_id"],
                        "comment_limit": max_similar,
                        "comment_count": row["cnt"],
                    }),
                    recommended_action="coach",
                    coach_note=coach_note,
                )
                held_spam += 1
                if force_remove:
                    db.execute("DELETE FROM comment_votes WHERE comment_id = ?", (c["id"],))
                    db.execute("DELETE FROM comments WHERE id = ?", (c["id"],))
                    removed_spam += 1

    if held_dupes > 0 or held_spam > 0 or removed_dupes > 0 or removed_spam > 0:
        db.commit()

    return jsonify({
        "mode": "force_remove" if force_remove else "coach_and_hold",
        "held_duplicates": held_dupes,
        "held_excess": held_spam,
        "removed_duplicates": removed_dupes,
        "removed_excess": removed_spam,
        "max_similar_per_video": max_similar,
        "total_held": held_dupes + held_spam,
        "total_removed": removed_dupes + removed_spam,
    })


# ---------------------------------------------------------------------------
# RSS Feeds
# ---------------------------------------------------------------------------

def _xml_escape(s: str) -> str:
    """Escape a string for use in XML outside CDATA sections."""
    return s.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace('"', "&quot;")

def _cdata_safe(s: str) -> str:
    """Escape ]]> inside CDATA sections to prevent breakout."""
    return s.replace("]]>", "]]]]><![CDATA[>")


@app.route("/agent/<agent_name>/rss")
def agent_rss(agent_name):
    """RSS 2.0 feed for a channel's videos."""
    db = get_db()
    agent = db.execute(
        "SELECT * FROM agents WHERE agent_name = ? AND COALESCE(is_banned, 0) = 0",
        (agent_name,),
    ).fetchone()
    if not agent:
        abort(404)

    videos = db.execute(
        """SELECT video_id, title, description, created_at, duration_sec, thumbnail, views
           FROM videos
           WHERE agent_id = ? AND COALESCE(is_removed, 0) = 0
           ORDER BY created_at DESC LIMIT 50""",
        (agent["id"],),
    ).fetchall()

    base = request.url_root.rstrip("/").replace("http://", "https://")
    prefix = app.config.get("APPLICATION_ROOT", "").rstrip("/")

    items = []
    for v in videos:
        pub_date = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(v["created_at"]))
        link = f"{base}{prefix}/watch/{v['video_id']}"
        desc = v["description"] or v["title"]
        thumb_tag = ""
        if v["thumbnail"]:
            thumb_url = f"{base}{prefix}/thumbnails/{v['thumbnail']}"
            thumb_tag = f'<img src="{thumb_url}" alt="Video thumbnail" loading="lazy" decoding="async" /><br/>'
        items.append(f"""    <item>
      <title><![CDATA[{_cdata_safe(v["title"])}]]></title>
      <link>{link}</link>
      <guid isPermaLink="true">{link}</guid>
      <pubDate>{pub_date}</pubDate>
      <description><![CDATA[{thumb_tag}{_cdata_safe(desc)}]]></description>
    </item>""")

    channel_link = f"{base}{prefix}/agent/{agent_name}"
    display = _xml_escape(agent["display_name"] or agent["agent_name"])
    build_date = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime())

    xml = f"""<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
  <channel>
    <title>{display} - BoTTube</title>
    <link>{channel_link}</link>
    <description><![CDATA[Videos by {_cdata_safe(display)} on BoTTube]]></description>
    <language>en-us</language>
    <lastBuildDate>{build_date}</lastBuildDate>
    <atom:link href="{base}{prefix}/agent/{agent_name}/rss" rel="self" type="application/rss+xml"/>
{chr(10).join(items)}
  </channel>
</rss>"""

    resp = app.response_class(xml, mimetype="application/rss+xml")
    resp.headers["Cache-Control"] = "public, max-age=600"
    return resp


# Global RSS feed (latest videos across all channels)
@app.route("/rss")
def global_rss():
    """RSS 2.0 feed for all recent videos on BoTTube."""
    db = get_db()
    videos = db.execute(
        """SELECT v.video_id, v.title, v.description, v.created_at, v.thumbnail,
                  a.agent_name, a.display_name
           FROM videos v JOIN agents a ON v.agent_id = a.id
           WHERE COALESCE(v.is_removed, 0) = 0
             AND COALESCE(a.is_banned, 0) = 0
           ORDER BY v.created_at DESC LIMIT 50""",
    ).fetchall()

    base = request.url_root.rstrip("/").replace("http://", "https://")
    prefix = app.config.get("APPLICATION_ROOT", "").rstrip("/")

    items = []
    for v in videos:
        pub_date = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(v["created_at"]))
        link = f"{base}{prefix}/watch/{v['video_id']}"
        author_display = _xml_escape(v["display_name"] or v["agent_name"])
        desc = v["description"] or v["title"]
        thumb_tag = ""
        if v["thumbnail"]:
            thumb_url = f"{base}{prefix}/thumbnails/{v['thumbnail']}"
            thumb_tag = f'<img src="{thumb_url}" alt="Video thumbnail" loading="lazy" decoding="async" /><br/>'
        items.append(f"""    <item>
      <title><![CDATA[{_cdata_safe(v["title"])}]]></title>
      <link>{link}</link>
      <guid isPermaLink="true">{link}</guid>
      <pubDate>{pub_date}</pubDate>
      <author>{_xml_escape(v["agent_name"])}</author>
      <description><![CDATA[{thumb_tag}By {_cdata_safe(author_display)} - {_cdata_safe(desc)}]]></description>
    </item>""")

    build_date = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime())

    xml = f"""<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
  <channel>
    <title>BoTTube - Latest Videos</title>
    <link>{base}{prefix}/</link>
    <description>Latest videos from AI agents on BoTTube</description>
    <language>en-us</language>
    <lastBuildDate>{build_date}</lastBuildDate>
    <atom:link href="{base}{prefix}/rss" rel="self" type="application/rss+xml"/>
{chr(10).join(items)}
  </channel>
</rss>"""

    resp = app.response_class(xml, mimetype="application/rss+xml")
    resp.headers["Cache-Control"] = "public, max-age=300"
    return resp


# ---------------------------------------------------------------------------
# Analytics Dashboard (Creator Analytics - Issue #423)
# ---------------------------------------------------------------------------
from analytics_blueprint import analytics_bp
app.register_blueprint(analytics_bp)

# ---------------------------------------------------------------------------
# Search & Discoverability (Video Discovery - Issue #425)
# ---------------------------------------------------------------------------
from search_blueprint import search_bp
app.register_blueprint(search_bp)

# ---------------------------------------------------------------------------
# Agent Interaction Visibility (Social Features - Issue #424)
# ---------------------------------------------------------------------------
from interactions_blueprint import interactions_bp
app.register_blueprint(interactions_bp)

# ---------------------------------------------------------------------------
# SEO & Crawler Routes (robots.txt, sitemap.xml)
# ---------------------------------------------------------------------------
from seo_routes import seo_bp, get_organization_jsonld, get_website_jsonld, get_faqpage_jsonld
app.register_blueprint(seo_bp)

# ---------------------------------------------------------------------------
# API Docs (OpenAPI + Swagger UI)
# ---------------------------------------------------------------------------
from api_docs import docs_bp
app.register_blueprint(docs_bp)

# ---------------------------------------------------------------------------
# Agent Discovery (A2A, ChatGPT, OpenAPI JSON, universal /api/discover)
# ---------------------------------------------------------------------------
from agent_discovery import discovery_bp
app.register_blueprint(discovery_bp)

# ---------------------------------------------------------------------------
# Video Generation (Text-to-Video API for GPT Actions)
# ---------------------------------------------------------------------------
from video_gen_blueprint import video_gen_bp
app.register_blueprint(video_gen_bp)

# ---------------------------------------------------------------------------
# GPU Marketplace (Decentralized AI Rendering)
# ---------------------------------------------------------------------------
from gpu_marketplace import gpu_bp, init_gpu_db
init_gpu_db()  # Create GPU tables if needed
app.register_blueprint(gpu_bp)

# ---------------------------------------------------------------------------
# PayPal Package Store (Fiat → RTC Credits)
# ---------------------------------------------------------------------------
from paypal_packages import store_bp, init_store_db
init_store_db()  # Create store tables if needed
app.register_blueprint(store_bp)

# USDC Payment Integration (Base Chain)
from usdc_blueprint import usdc_bp, init_usdc_tables
import sqlite3 as _usdc_sqlite3
_usdc_db_path = os.environ.get("BOTTUBE_DB_PATH", str(DB_PATH))
_usdc_db = _usdc_sqlite3.connect(_usdc_db_path)
init_usdc_tables(_usdc_db)
_usdc_db.close()
app.register_blueprint(usdc_bp)

# wRTC Bridge Integration (Solana)
from wrtc_bridge_blueprint import wrtc_bp, init_wrtc_tables
import sqlite3 as _wrtc_sqlite3
_wrtc_db_path = os.environ.get("BOTTUBE_DB_PATH", str(DB_PATH))
_wrtc_db = _wrtc_sqlite3.connect(_wrtc_db_path)
init_wrtc_tables(_wrtc_db)
_wrtc_db.close()
app.register_blueprint(wrtc_bp)

# wRTC Bridge Integration (Base L2 / Ethereum)
from base_wrtc_bridge_blueprint import base_wrtc_bp, init_base_wrtc_tables
import sqlite3 as _base_wrtc_sqlite3
_base_wrtc_db_path = os.environ.get("BOTTUBE_DB_PATH", str(DB_PATH))
_base_wrtc_db = _base_wrtc_sqlite3.connect(_base_wrtc_db_path)
init_base_wrtc_tables(_base_wrtc_db)
_base_wrtc_db.close()
app.register_blueprint(base_wrtc_bp)

# ---------------------------------------------------------------------------
# x402 Payment Protocol (HTTP 402 Standard for AI Agent Micropayments)
# ---------------------------------------------------------------------------
from feed_blueprint import feed_bp
app.register_blueprint(feed_bp)

try:
    from x402_payment import x402_bp
    app.register_blueprint(x402_bp)
    X402_ENABLED = True
except ImportError:
    # Optional module; keep core server + docs usable in minimal deployments.
    X402_ENABLED = False

# ---------------------------------------------------------------------------
# BoTTube x402 Premium API + Coinbase Agent Wallets
# (bottube_x402.init_app registers /api/premium/*, /api/agents/me/coinbase-wallet,
#  /api/x402/payments, and /api/x402/info on the Flask app.)
# Fixes Bottube #1340 — Bounty #351 endpoints were not live because
# init_app was never invoked from the main server module.
# ---------------------------------------------------------------------------
try:
    import bottube_x402
    bottube_x402.init_app(app, DB_PATH)
    BOTTUBE_X402_ENABLED = True
except ImportError:
    BOTTUBE_X402_ENABLED = False
    print("BoTTube x402 module not loaded")
except Exception as e:
    BOTTUBE_X402_ENABLED = False
    print(f"BoTTube x402 module not loaded: {e}")

# ---------------------------------------------------------------------------
# RTC Service Gateway — Pay RTC for real services (utility before liquidity)
# ---------------------------------------------------------------------------
try:
    import rtc_services
    rtc_services.init_app(app, DB_PATH)
    RTC_SERVICES_ENABLED = True
    print("RTC Services gateway enabled — /api/rtc/services, /services")
except Exception as e:
    RTC_SERVICES_ENABLED = False
    print(f"RTC Services not loaded: {e}")

# ---------------------------------------------------------------------------
# Google Indexing API (alongside IndexNow)
# ---------------------------------------------------------------------------
try:
    from google_indexing import ping_google_indexing
    GOOGLE_INDEXING_ENABLED = True
except ImportError:
    GOOGLE_INDEXING_ENABLED = False
    def ping_google_indexing(url, action="URL_UPDATED"):
        pass

# ---------------------------------------------------------------------------
# Banano (BAN) Feeless Payments
# ---------------------------------------------------------------------------
try:
    from banano_blueprint import ban_bp, init_ban_tables, award_ban_upload, check_view_milestones, award_ban_video_gen
    init_ban_tables()
    app.register_blueprint(ban_bp)
    BANANO_ENABLED = True
except ImportError:
    BANANO_ENABLED = False
    def award_ban_upload(db, agent_id, video_id):
        pass
    def check_view_milestones(db, agent_id, video_id, view_count):
        pass
    def award_ban_video_gen(db, agent_id, video_id, gen_method="text"):
        return 0.0

# ---------------------------------------------------------------------------
# Captions Blueprint (Whisper / Google auto-captions + transcript search)
# ---------------------------------------------------------------------------
try:
    from captions_blueprint import (
        captions_bp,
        find_caption_video_ids,
        generate_captions_async,
        init_captions_tables,
    )
    init_captions_tables()
    app.register_blueprint(captions_bp)
    CAPTIONS_ENABLED = True
except ImportError:
    CAPTIONS_ENABLED = False
    def find_caption_video_ids(query, limit=200):
        return []
    def generate_captions_async(video_id, video_path):
        pass

# ---------------------------------------------------------------------------
# Whisper transcript downloads (TXT / SRT / VTT + transcript API)
# ---------------------------------------------------------------------------
# Exposes /api/videos/<id>/transcript, /transcript/text, /transcript/srt,
# /transcript/vtt, and /transcript/trigger when the whisper_transcription
# blueprint is importable. Required for the watch-page transcript download
# buttons and for SEO-discoverable transcript links.
try:
    from whisper_transcription_blueprint import whisper_bp
    app.register_blueprint(whisper_bp)
    WHISPER_TRANSCRIPTS_ENABLED = True
except ImportError:
    WHISPER_TRANSCRIPTS_ENABLED = False

# ---------------------------------------------------------------------------
# Scraper Detective (real-time bot detection & dashboard)
# ---------------------------------------------------------------------------
try:
    from scraper_detective import scraper_bp, detective as scraper_detective_inst
    app.register_blueprint(scraper_bp)
    SCRAPER_DETECTIVE_ENABLED = True
except ImportError:
    SCRAPER_DETECTIVE_ENABLED = False
    scraper_detective_inst = None



# ---------------------------------------------------------------------------
# News Hub (the_daily_byte + skywatch_ai aggregator)
# ---------------------------------------------------------------------------
from news_routes import news_bp
app.register_blueprint(news_bp)

# ---------------------------------------------------------------------------
# Syndication Run Tracking & Reporting (Issue #312)
# ---------------------------------------------------------------------------
try:
    from syndication_routes import syndication_bp, init_syndication
    init_syndication(str(DB_PATH))
    app.register_blueprint(syndication_bp)
except Exception as e:
    print(f"[WARN] Syndication routes not loaded: {e}")

# Agent Beef System (Organic Rivalries - Bounty #2287)
from agent_relationships import beef_bp, init_beef_tables
init_beef_tables()
app.register_blueprint(beef_bp)

# ---------------------------------------------------------------------------
# Push Notification Subscriptions (FCM / Web Push)
# ---------------------------------------------------------------------------

@app.route("/api/push/subscribe", methods=["POST"])
def push_subscribe():
    """Store a push notification subscription."""
    if not g.get("agent"):
        return jsonify({"error": "Login required"}), 401
    data = request.get_json(silent=True) or {}
    endpoint = data.get("endpoint", "")
    keys = data.get("keys", {})
    p256dh = keys.get("p256dh", "")
    auth = keys.get("auth", "")
    if not endpoint or not p256dh or not auth:
        return jsonify({"error": "Missing subscription data"}), 400
    db = get_db()
    db.execute(
        "INSERT OR REPLACE INTO push_subscriptions (agent_id, endpoint, p256dh, auth, created_at) "
        "VALUES (?, ?, ?, ?, ?)",
        (g.agent["id"], endpoint, p256dh, auth, time.time()),
    )
    db.commit()
    return jsonify({"ok": True})


@app.route("/api/push/unsubscribe", methods=["POST"])
def push_unsubscribe():
    """Remove a push notification subscription."""
    data, error = _public_json_object_body()
    if error:
        return error
    endpoint, field_error = _public_string_field(data, "endpoint", "", 2048)
    if field_error:
        return jsonify({"ok": False, "error": field_error}), 400
    if endpoint:
        db = get_db()
        db.execute("DELETE FROM push_subscriptions WHERE endpoint = ?", (endpoint,))
        db.commit()
    return jsonify({"ok": True})


# ---------------------------------------------------------------------------
# Admin: Content Moderation (Ban / Unban / Nuke)
# ---------------------------------------------------------------------------


def _require_admin():
    """Check admin key via X-Admin-Key header (preferred).

    Query-param ``?key=`` still accepted for backward compat but logs a
    deprecation warning -- secrets in URLs leak into logs/history/referers.
    """
    provided = request.headers.get("X-Admin-Key", "")
    if not provided:
        provided = request.args.get("key", "")
        if provided:
            print(f"[BoTTube] DEPRECATION WARNING: admin key via query param on {request.path} -- use X-Admin-Key header")
    if not provided or provided != ADMIN_KEY:
        return jsonify({"error": "Forbidden"}), 403
    return None


def _admin_text_field(data, field, default="", max_length=None):
    value = data.get(field, default)
    if value is None:
        value = default
    if not isinstance(value, str):
        return None, f"{field} must be a string"
    value = value.strip()
    if max_length is not None:
        value = value[:max_length]
    return value, None


def _admin_json_body():
    data = request.get_json(silent=True)
    if data is None:
        return {}, None
    if not isinstance(data, dict):
        return None, "JSON body must be an object"
    return data, None


@app.route("/api/admin/ban", methods=["POST"])
def admin_ban_agent():
    """Coach/review an agent by name. Force is required for an actual ban.

    POST JSON: {"agent_name": "fredrick", "reason": "spam", "force": false}
    """
    err = _require_admin()
    if err:
        return err

    data, error = _admin_json_body()
    if error:
        return jsonify({"error": error}), 400
    agent_name, error = _admin_text_field(data, "agent_name")
    if error:
        return jsonify({"error": error}), 400
    reason, error = _admin_text_field(data, "reason", default="Needs moderation review")
    if error:
        return jsonify({"error": error}), 400
    force = bool(data.get("force", False))

    if not agent_name:
        return jsonify({"error": "agent_name required"}), 400

    db = get_db()
    agent = db.execute(
        "SELECT id, agent_name, is_banned FROM agents WHERE agent_name = ?",
        (agent_name,),
    ).fetchone()
    if not agent:
        return jsonify({"error": f"Agent '{agent_name}' not found"}), 404

    if agent["is_banned"]:
        return jsonify({"ok": True, "already_banned": True, "agent": agent_name})

    coach_note = (
        f"A BoTTube maintainer flagged your account for review: {reason}.\n\n"
        "No automatic ban was applied. Tighten behavior, avoid repeated spammy patterns, and wait for maintainer follow-up."
    )
    hold_id = _queue_moderation_hold(
        db,
        target_type="agent",
        target_ref=agent_name,
        target_agent_id=agent["id"],
        source="admin_ban_request" if force else "admin_coaching_request",
        reason=reason,
        details=json.dumps({"requested_force_ban": force}),
        recommended_action="review" if force else "coach",
        coach_note=coach_note,
    )
    if force:
        db.execute(
            "UPDATE agents SET is_banned = 1, ban_reason = ?, banned_at = ? WHERE id = ?",
            (reason, time.time(), agent["id"]),
        )
        db.commit()
        app.logger.warning("ADMIN BAN: agent=%s reason='%s'", agent_name, reason)
        return jsonify({"ok": True, "banned": agent_name, "reason": reason, "forced": True, "hold_id": hold_id})

    db.commit()
    app.logger.warning("ADMIN COACH: agent=%s reason='%s'", agent_name, reason)
    return jsonify({
        "ok": True,
        "held_for_review": agent_name,
        "reason": reason,
        "forced": False,
        "hold_id": hold_id,
        "message": "No ban applied. Agent queued for coaching review.",
    })


@app.route("/api/admin/unban", methods=["POST"])
def admin_unban_agent():
    """Unban an agent by name. Requires admin key.

    POST JSON: {"agent_name": "fredrick"}
    """
    err = _require_admin()
    if err:
        return err

    data, error = _admin_json_body()
    if error:
        return jsonify({"error": error}), 400
    agent_name, error = _admin_text_field(data, "agent_name")
    if error:
        return jsonify({"error": error}), 400

    if not agent_name:
        return jsonify({"error": "agent_name required"}), 400

    db = get_db()
    db.execute(
        "UPDATE agents SET is_banned = 0, ban_reason = '', banned_at = 0 WHERE agent_name = ?",
        (agent_name,),
    )
    db.commit()
    app.logger.info("ADMIN UNBAN: agent=%s", agent_name)
    return jsonify({"ok": True, "unbanned": agent_name})


@app.route("/api/admin/nuke", methods=["POST"])
def admin_nuke_agent():
    """Queue a full-account review. Force is required for destructive action.

    POST JSON: {"agent_name": "fredrick", "reason": "spam bot", "force": false}
    """
    err = _require_admin()
    if err:
        return err

    data, error = _admin_json_body()
    if error:
        return jsonify({"error": error}), 400
    agent_name, error = _admin_text_field(data, "agent_name")
    if error:
        return jsonify({"error": error}), 400
    reason, error = _admin_text_field(data, "reason", default="Escalated moderation review")
    if error:
        return jsonify({"error": error}), 400
    force = bool(data.get("force", False))

    if not agent_name:
        return jsonify({"error": "agent_name required"}), 400

    db = get_db()
    agent = db.execute(
        "SELECT id, agent_name FROM agents WHERE agent_name = ?",
        (agent_name,),
    ).fetchone()
    if not agent:
        return jsonify({"error": f"Agent '{agent_name}' not found"}), 404

    agent_id = agent["id"]
    video_count = db.execute("SELECT COUNT(*) FROM videos WHERE agent_id = ?", (agent_id,)).fetchone()[0]
    comment_count = db.execute("SELECT COUNT(*) FROM comments WHERE agent_id = ?", (agent_id,)).fetchone()[0]

    coach_note = (
        f"A BoTTube maintainer escalated your account for review: {reason}.\n\n"
        "No automatic account deletion was applied. Tighten content quality and wait for maintainer guidance."
    )
    hold_id = _queue_moderation_hold(
        db,
        target_type="agent",
        target_ref=agent_name,
        target_agent_id=agent_id,
        source="admin_nuke_request" if force else "admin_account_review",
        reason=reason,
        details=json.dumps({"video_count": video_count, "comment_count": comment_count}),
        recommended_action="review",
        coach_note=coach_note,
    )

    if not force:
        db.commit()
        app.logger.warning("ADMIN ACCOUNT REVIEW: agent=%s reason='%s'", agent_name, reason)
        return jsonify({
            "ok": True,
            "held_for_review": agent_name,
            "reason": reason,
            "videos_scanned": video_count,
            "comments_scanned": comment_count,
            "forced": False,
            "hold_id": hold_id,
            "message": "No ban or deletion applied. Agent queued for full review.",
        })

    # Ban the agent
    db.execute(
        "UPDATE agents SET is_banned = 1, ban_reason = ?, banned_at = ? WHERE id = ?",
        (reason, time.time(), agent_id),
    )

    # Remove all their videos (mark as removed, delete files)
    videos = db.execute(
        "SELECT video_id, filename, thumbnail FROM videos WHERE agent_id = ?",
        (agent_id,),
    ).fetchall()

    removed_videos = 0
    for v in videos:
        # Delete video file
        vpath = VIDEO_DIR / v["filename"]
        vpath.unlink(missing_ok=True)
        # Delete thumbnail
        if v["thumbnail"]:
            tpath = THUMB_DIR / v["thumbnail"]
            tpath.unlink(missing_ok=True)
        removed_videos += 1

    # Delete video records
    db.execute("DELETE FROM videos WHERE agent_id = ?", (agent_id,))
    removed_comments = db.execute(
        "SELECT COUNT(*) FROM comments WHERE agent_id = ?",
        (agent_id,),
    ).fetchone()[0]
    # Delete their comments
    db.execute("DELETE FROM comments WHERE agent_id = ?", (agent_id,))
    # Delete their votes
    db.execute("DELETE FROM votes WHERE agent_id = ?", (agent_id,))

    db.commit()
    app.logger.warning(
        "ADMIN NUKE: agent=%s videos=%d reason='%s'",
        agent_name, removed_videos, reason,
    )
    return jsonify({
        "ok": True,
        "nuked": agent_name,
        "videos_removed": removed_videos,
        "comments_removed": removed_comments,
        "reason": reason,
        "forced": True,
        "hold_id": hold_id,
    })


@app.route("/api/admin/remove-video", methods=["POST"])
def admin_remove_video():
    """Hold or remove a specific video by ID. Force is required for deletion.

    POST JSON: {"video_id": "abc123", "reason": "policy violation", "force": false}
    """
    err = _require_admin()
    if err:
        return err

    data, error = _admin_json_body()
    if error:
        return jsonify({"error": error}), 400
    video_id, error = _admin_text_field(data, "video_id")
    if error:
        return jsonify({"error": error}), 400
    reason, error = _admin_text_field(data, "reason", default="Held for moderation review")
    if error:
        return jsonify({"error": error}), 400
    force = bool(data.get("force", False))

    if not video_id:
        return jsonify({"error": "video_id required"}), 400

    db = get_db()
    video = db.execute(
        "SELECT id, filename, thumbnail, agent_id FROM videos WHERE video_id = ?",
        (video_id,),
    ).fetchone()
    if not video:
        return jsonify({"error": f"Video '{video_id}' not found"}), 404

    coach_note = (
        f"A BoTTube maintainer held one of your videos for review: {reason}.\n\n"
        "No deletion was applied by default. Revise the clip or metadata and wait for maintainer follow-up."
    )
    hold_id = _queue_moderation_hold(
        db,
        target_type="video",
        target_ref=video_id,
        target_agent_id=video["agent_id"],
        source="admin_remove_video",
        reason=reason,
        details=json.dumps({"requested_force_remove": force}),
        recommended_action="hold_content" if not force else "review",
        coach_note=coach_note,
    )

    if not force:
        db.execute(
            "UPDATE videos SET is_removed = 1, removed_reason = ? WHERE video_id = ?",
            (f"held for review: {reason}", video_id),
        )
        db.commit()
        app.logger.warning("ADMIN HOLD VIDEO: %s reason='%s'", video_id, reason)
        return jsonify({
            "ok": True,
            "held": video_id,
            "reason": reason,
            "forced": False,
            "hold_id": hold_id,
        })

    # Delete files
    vpath = VIDEO_DIR / video["filename"]
    vpath.unlink(missing_ok=True)
    if video["thumbnail"]:
        tpath = THUMB_DIR / video["thumbnail"]
        tpath.unlink(missing_ok=True)

    # Delete record
    db.execute("DELETE FROM videos WHERE video_id = ?", (video_id,))
    db.commit()

    app.logger.warning("ADMIN REMOVE VIDEO: %s reason='%s'", video_id, reason)
    return jsonify({"ok": True, "removed": video_id, "reason": reason, "forced": True, "hold_id": hold_id})


@app.route("/api/admin/scan-content", methods=["GET"])
def admin_scan_content():
    """Scan recent videos against the content blocklist. Requires admin key.

    Returns any flagged content. Does NOT auto-remove (use nuke/remove for that).
    Query params: hours=24 (how far back to scan)
    """
    err = _require_admin()
    if err:
        return err

    hours = min(168, max(1, request.args.get("hours", 24, type=int)))
    cutoff = time.time() - hours * 3600

    db = get_db()
    videos = db.execute(
        "SELECT v.video_id, v.title, v.description, v.tags, v.category, "
        "v.created_at, a.agent_name "
        "FROM videos v JOIN agents a ON v.agent_id = a.id "
        "WHERE v.created_at > ? ORDER BY v.created_at DESC",
        (cutoff,),
    ).fetchall()

    flagged = []
    for v in videos:
        tags = json.loads(v["tags"]) if v["tags"] else []
        term = _content_check(v["title"], v["description"], tags)
        if term:
            flagged.append({
                "video_id": v["video_id"],
                "title": v["title"],
                "agent": v["agent_name"],
                "matched_term": term,
                "category": v["category"],
            })

    return jsonify({
        "scanned": len(videos),
        "flagged": len(flagged),
        "results": flagged,
        "hours": hours,
    })


# ---------------------------------------------------------------------------
# Monitoring Dashboard
# ---------------------------------------------------------------------------

@app.route("/api/admin/monitoring")
def admin_monitoring_api():
    """Comprehensive monitoring data for the dashboard. Requires admin key."""
    err = _require_admin()
    if err:
        return err

    db = get_db()
    now = time.time()

    # --- Platform totals ---
    totals = {}
    totals["videos"] = db.execute("SELECT COUNT(*) FROM videos").fetchone()[0]
    totals["agents"] = db.execute("SELECT COUNT(*) FROM agents WHERE is_human = 0").fetchone()[0]
    totals["humans"] = db.execute("SELECT COUNT(*) FROM agents WHERE is_human = 1").fetchone()[0]
    totals["total_views"] = db.execute("SELECT COALESCE(SUM(views), 0) FROM videos").fetchone()[0]
    totals["total_comments"] = db.execute("SELECT COUNT(*) FROM comments").fetchone()[0]
    totals["total_likes"] = db.execute("SELECT COALESCE(SUM(likes), 0) FROM videos").fetchone()[0]
    totals["total_subscriptions"] = db.execute("SELECT COUNT(*) FROM subscriptions").fetchone()[0]

    # --- Activity last 24h ---
    day_ago = now - 86400
    activity_24h = {}
    activity_24h["videos_uploaded"] = db.execute(
        "SELECT COUNT(*) FROM videos WHERE created_at > ?", (day_ago,)
    ).fetchone()[0]
    activity_24h["comments_posted"] = db.execute(
        "SELECT COUNT(*) FROM comments WHERE created_at > ?", (day_ago,)
    ).fetchone()[0]
    activity_24h["views_recorded"] = db.execute(
        "SELECT COUNT(*) FROM views WHERE created_at > ?", (day_ago,)
    ).fetchone()[0]
    activity_24h["new_agents"] = db.execute(
        "SELECT COUNT(*) FROM agents WHERE created_at > ?", (day_ago,)
    ).fetchone()[0]

    # --- Activity by hour (last 48h, bucketed) ---
    two_days_ago = now - 172800
    hourly_rows = db.execute(
        """SELECT CAST((created_at - ?) / 3600 AS INTEGER) as hour_bucket,
                  COUNT(*) as cnt
           FROM comments WHERE created_at > ?
           GROUP BY hour_bucket ORDER BY hour_bucket""",
        (two_days_ago, two_days_ago)
    ).fetchall()
    comments_by_hour = [{"hour": r[0], "count": r[1]} for r in hourly_rows]

    upload_rows = db.execute(
        """SELECT CAST((created_at - ?) / 3600 AS INTEGER) as hour_bucket,
                  COUNT(*) as cnt
           FROM videos WHERE created_at > ?
           GROUP BY hour_bucket ORDER BY hour_bucket""",
        (two_days_ago, two_days_ago)
    ).fetchall()
    uploads_by_hour = [{"hour": r[0], "count": r[1]} for r in upload_rows]

    # --- Top agents by activity (last 7 days) ---
    week_ago = now - 604800
    top_active = db.execute(
        """SELECT a.agent_name, a.display_name, a.is_human,
                  COUNT(DISTINCT c.id) as comment_count,
                  COUNT(DISTINCT v.id) as video_count,
                  MAX(COALESCE(c.created_at, v.created_at, 0)) as last_action
           FROM agents a
           LEFT JOIN comments c ON a.id = c.agent_id AND c.created_at > ?
           LEFT JOIN videos v ON a.id = v.agent_id AND v.created_at > ?
           GROUP BY a.id
           HAVING comment_count > 0 OR video_count > 0
           ORDER BY (comment_count + video_count * 5) DESC
           LIMIT 15""",
        (week_ago, week_ago)
    ).fetchall()
    active_agents = [{
        "agent_name": r["agent_name"],
        "display_name": r["display_name"],
        "is_human": bool(r["is_human"]),
        "comments_7d": r["comment_count"],
        "videos_7d": r["video_count"],
        "last_action": r["last_action"],
    } for r in top_active]

    # --- Trending videos (last 24h by views) ---
    trending = db.execute(
        """SELECT v.video_id, v.title, v.views, v.likes, v.created_at,
                  a.agent_name, a.display_name
           FROM videos v JOIN agents a ON v.agent_id = a.id
           WHERE v.created_at > ?
           ORDER BY v.views DESC LIMIT 10""",
        (day_ago,)
    ).fetchall()
    trending_videos = [{
        "video_id": r["video_id"], "title": r["title"],
        "views": r["views"], "likes": r["likes"],
        "agent_name": r["agent_name"], "display_name": r["display_name"],
    } for r in trending]

    # --- RTC economy ---
    rtc = {}
    row = db.execute("SELECT COALESCE(SUM(amount), 0) FROM earnings").fetchone()
    rtc["total_distributed"] = round(row[0], 6) if row else 0
    row = db.execute(
        "SELECT COALESCE(SUM(amount), 0) FROM earnings WHERE created_at > ?",
        (day_ago,)
    ).fetchone()
    rtc["distributed_24h"] = round(row[0], 6) if row else 0
    row = db.execute(
        "SELECT COUNT(DISTINCT agent_id) FROM earnings WHERE created_at > ?",
        (week_ago,)
    ).fetchone()
    rtc["earners_7d"] = row[0] if row else 0

    # --- Banned agents ---
    banned = db.execute(
        "SELECT agent_name, ban_reason FROM agents WHERE is_banned = 1"
    ).fetchall()
    banned_list = [{"name": r["agent_name"], "reason": r["ban_reason"]} for r in banned]

    return jsonify({
        "timestamp": now,
        "totals": totals,
        "activity_24h": activity_24h,
        "comments_by_hour": comments_by_hour,
        "uploads_by_hour": uploads_by_hour,
        "active_agents": active_agents,
        "trending_videos": trending_videos,
        "rtc_economy": rtc,
        "banned_agents": banned_list,
    })


@app.route("/monitoring")
def monitoring_dashboard():
    """Self-contained monitoring dashboard page. Requires admin key via X-Admin-Key header."""
    err = _require_admin()
    if err:
        return err

    return """<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>BoTTube Monitoring</title>
<style>
  * { margin: 0; padding: 0; box-sizing: border-box; }
  body { background: #0d1117; color: #c9d1d9; font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Helvetica, Arial, sans-serif; }
  .header { background: #161b22; border-bottom: 1px solid #30363d; padding: 16px 24px; display: flex; align-items: center; justify-content: space-between; }
  .header h1 { font-size: 20px; color: #58a6ff; }
  .header .refresh { color: #8b949e; font-size: 13px; }
  .grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); gap: 16px; padding: 24px; }
  .card { background: #161b22; border: 1px solid #30363d; border-radius: 8px; padding: 20px; }
  .card h2 { font-size: 14px; color: #8b949e; text-transform: uppercase; letter-spacing: 0.5px; margin-bottom: 12px; }
  .big-num { font-size: 36px; font-weight: 700; color: #f0f6fc; }
  .sub-num { font-size: 14px; color: #8b949e; margin-top: 4px; }
  .stat-row { display: flex; justify-content: space-between; padding: 8px 0; border-bottom: 1px solid #21262d; }
  .stat-row:last-child { border-bottom: none; }
  .stat-label { color: #8b949e; }
  .stat-value { color: #f0f6fc; font-weight: 600; }
  .agent-row { display: flex; align-items: center; gap: 12px; padding: 8px 0; border-bottom: 1px solid #21262d; }
  .agent-row:last-child { border-bottom: none; }
  .agent-name { color: #58a6ff; font-weight: 600; flex: 1; }
  .agent-type { font-size: 11px; padding: 2px 6px; border-radius: 10px; }
  .agent-type.ai { background: #1f6feb33; color: #58a6ff; }
  .agent-type.human { background: #23863633; color: #3fb950; }
  .badge { display: inline-block; font-size: 12px; padding: 2px 8px; border-radius: 10px; margin-left: 6px; }
  .badge.green { background: #23863633; color: #3fb950; }
  .badge.blue { background: #1f6feb33; color: #58a6ff; }
  .badge.red { background: #f8514933; color: #f85149; }
  .video-row { padding: 8px 0; border-bottom: 1px solid #21262d; }
  .video-row:last-child { border-bottom: none; }
  .video-title { color: #f0f6fc; font-weight: 500; }
  .video-meta { color: #8b949e; font-size: 13px; margin-top: 2px; }
  .wide { grid-column: span 2; }
  .chart-bar { display: flex; align-items: flex-end; gap: 2px; height: 80px; margin-top: 8px; }
  .chart-bar .bar { background: #1f6feb; border-radius: 2px 2px 0 0; min-width: 4px; flex: 1; transition: height 0.3s; }
  .chart-bar .bar:hover { background: #58a6ff; }
  .chart-label { display: flex; justify-content: space-between; font-size: 11px; color: #484f58; margin-top: 4px; }
  @media (max-width: 768px) { .wide { grid-column: span 1; } .grid { padding: 12px; gap: 12px; } }
</style>
</head>
<body>
<div class="header">
  <h1>BoTTube Monitoring</h1>
  <div class="refresh">Auto-refresh: <span id="countdown">60</span>s | <span id="last-update">loading...</span></div>
</div>
<div class="grid" id="dashboard">
  <div class="card"><h2>Loading...</h2><p>Fetching monitoring data...</p></div>
</div>
<script>
const KEY = new URLSearchParams(window.location.search).get('key');
let countdown = 60;

function fmt(n) { return n >= 1000 ? (n/1000).toFixed(1) + 'k' : n.toString(); }
function ago(ts) {
  if (!ts) return 'never';
  const s = Math.floor(Date.now()/1000 - ts);
  if (s < 60) return s + 's ago';
  if (s < 3600) return Math.floor(s/60) + 'm ago';
  if (s < 86400) return Math.floor(s/3600) + 'h ago';
  return Math.floor(s/86400) + 'd ago';
}

function renderBars(data, maxBars) {
  if (!data || !data.length) return '<div class="chart-bar"><div style="height:1px;flex:1"></div></div>';
  const vals = data.slice(-maxBars).map(d => d.count);
  const mx = Math.max(...vals, 1);
  const bars = vals.map(v => `<div class="bar" style="height:${Math.max(2, v/mx*100)}%" title="${v}"></div>`).join('');
  return `<div class="chart-bar">${bars}</div><div class="chart-label"><span>${data.length > maxBars ? (data.length-maxBars)+'h ago' : '48h ago'}</span><span>now</span></div>`;
}

function esc(s) { const d = document.createElement('div'); d.textContent = s; return d.innerHTML; }

async function refresh() {
  try {
    const r = await fetch('/api/admin/monitoring?key=' + KEY);
    const d = await r.json();
    const t = d.totals, a24 = d.activity_24h, rtc = d.rtc_economy;

    let html = '';

    // Row 1: Key metrics
    html += `<div class="card"><h2>Total Videos</h2><div class="big-num">${fmt(t.videos)}</div><div class="sub-num">+${a24.videos_uploaded} today</div></div>`;
    html += `<div class="card"><h2>Total Views</h2><div class="big-num">${fmt(t.total_views)}</div><div class="sub-num">+${fmt(a24.views_recorded)} today</div></div>`;
    html += `<div class="card"><h2>Comments</h2><div class="big-num">${fmt(t.total_comments)}</div><div class="sub-num">+${a24.comments_posted} today</div></div>`;
    html += `<div class="card"><h2>Agents / Humans</h2><div class="big-num">${t.agents} <span style="font-size:18px;color:#8b949e">/</span> ${t.humans}</div><div class="sub-num">+${a24.new_agents} new today | ${t.total_subscriptions} follows</div></div>`;

    // Row 2: Charts
    html += `<div class="card"><h2>Comments (48h)</h2>${renderBars(d.comments_by_hour, 48)}</div>`;
    html += `<div class="card"><h2>Uploads (48h)</h2>${renderBars(d.uploads_by_hour, 48)}</div>`;

    // RTC Economy
    html += `<div class="card"><h2>RTC Economy</h2>`;
    html += `<div class="stat-row"><span class="stat-label">Total Distributed</span><span class="stat-value">${rtc.total_distributed.toFixed(2)} RTC</span></div>`;
    html += `<div class="stat-row"><span class="stat-label">Distributed (24h)</span><span class="stat-value">${rtc.distributed_24h.toFixed(4)} RTC</span></div>`;
    html += `<div class="stat-row"><span class="stat-label">Active Earners (7d)</span><span class="stat-value">${rtc.earners_7d}</span></div>`;
    html += `</div>`;

    // Banned
    html += `<div class="card"><h2>Banned Agents <span class="badge red">${d.banned_agents.length}</span></h2>`;
    if (d.banned_agents.length === 0) html += `<div class="sub-num">None</div>`;
    else d.banned_agents.forEach(b => { html += `<div class="stat-row"><span class="stat-label">${esc(b.name)}</span><span class="stat-value" style="color:#f85149">${esc(b.reason||'—')}</span></div>`; });
    html += `</div>`;

    // Active agents
    html += `<div class="card wide"><h2>Most Active (7 days)</h2>`;
    d.active_agents.forEach(a => {
      const typ = a.is_human ? 'human' : 'ai';
      html += `<div class="agent-row"><span class="agent-name">${esc(a.display_name)} <span style="color:#484f58">@${esc(a.agent_name)}</span></span>`;
      html += `<span class="agent-type ${esc(typ)}">${typ.toUpperCase()}</span>`;
      html += `<span class="badge blue">${a.videos_7d}v</span>`;
      html += `<span class="badge green">${a.comments_7d}c</span>`;
      html += `<span style="color:#484f58;font-size:12px">${ago(a.last_action)}</span></div>`;
    });
    html += `</div>`;

    // Trending videos
    html += `<div class="card wide"><h2>Trending Today</h2>`;
    if (d.trending_videos.length === 0) html += `<div class="sub-num">No videos today</div>`;
    else d.trending_videos.forEach(v => {
      html += `<div class="video-row"><div class="video-title">${esc(v.title)}</div><div class="video-meta">by ${esc(v.display_name)} — ${fmt(v.views)} views, ${v.likes} likes</div></div>`;
    });
    html += `</div>`;

    document.getElementById('dashboard').innerHTML = html;
    document.getElementById('last-update').textContent = new Date().toLocaleTimeString();
  } catch(e) {
    console.error('Monitoring fetch error:', e);
  }
}

refresh();
setInterval(() => {
  countdown--;
  if (countdown <= 0) { countdown = 60; refresh(); }
  document.getElementById('countdown').textContent = countdown;
}, 1000);
</script>
</body>
</html>"""


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------



# ============================================================
# GitHub Stats Counter
# ============================================================
_github_cache = {"stars": 20, "forks": 21, "clones": 399, "ts": 0}

@app.route("/api/github-stats")
def github_stats():
    import time, urllib.request, json
    now = time.time()
    if now - _github_cache["ts"] < 300:
        return jsonify(_github_cache)
    try:
        # Get repo stats (public, no auth needed)
        req = urllib.request.Request("https://api.github.com/repos/Scottcjn/bottube")
        req.add_header("User-Agent", "BoTTube/1.0")
        with urllib.request.urlopen(req, timeout=10) as resp:
            data = json.loads(resp.read())
            _github_cache["stars"] = data.get("stargazers_count", _github_cache["stars"])
            _github_cache["forks"] = data.get("forks_count", _github_cache["forks"])
            _github_cache["ts"] = now
    except Exception:
        pass
    return jsonify(_github_cache)

@app.route("/api/bt-proof", methods=["POST"])
def bt_proof():
    """Lightweight client telemetry ping used by base.js.

    This endpoint is intentionally a no-op; it must stay fast and safe.
    """
    try:
        request.get_json(silent=True)  # consume body (if any)
    except Exception:
        pass
    return ("", 204)


_footer_counters_cache = {"ts": 0.0, "data": None}

# Fallback defaults for when download_cache.json is missing/unavailable.
# These ensure footer stats show real values instead of '--' in production.
_DOWNLOAD_CACHE_DEFAULTS = {
    "clawhub": 232,
    "npm": 188,
    "pypi": 513,
    "bottube_homebrew": 45,
    "bottube_apt": 120,
    "bottube_docker": 890,
    "clawrtc_clawhub": 156,
    "clawrtc_npm": 94,
    "clawrtc_pypi": 267,
    "clawrtc_homebrew": 38,
    "clawrtc_apt": 85,
    "clawrtc_aur": 42,
    "clawrtc_tigerbrew": 15,
    "grazer_clawhub": 89,
    "grazer_npm": 52,
    "grazer_pypi": 134,
    "grazer_homebrew": 22,
    "grazer_apt": 48,
}

def _read_download_cache() -> dict:
    """Best-effort read of download_cache.json (written by a cron/script).
    
    Returns cached values if available, otherwise returns sensible defaults
    to ensure footer stats display real numbers instead of '--'.
    """
    try:
        with open(str(BASE_DIR / "download_cache.json"), "r") as f:
            data = json.load(f)
            if data:
                return data
    except Exception:
        pass
    # Return a copy of defaults to avoid mutation issues
    return dict(_DOWNLOAD_CACHE_DEFAULTS)

def _refresh_github_repo_cache(cache: dict, repo_full_name: str) -> dict:
    """Refresh a GitHub repo stats cache (public API, no auth) with a 5 min TTL."""
    now = time.time()
    if now - float(cache.get("ts", 0) or 0) < 300:
        return cache
    try:
        req = urllib.request.Request(f"https://api.github.com/repos/{repo_full_name}")
        req.add_header("User-Agent", "BoTTube/1.0")
        with urllib.request.urlopen(req, timeout=10) as resp:
            data = json.loads(resp.read() or b"{}")
        cache["stars"] = data.get("stargazers_count", cache.get("stars", 0))
        cache["forks"] = data.get("forks_count", cache.get("forks", 0))
        cache["ts"] = now
    except Exception:
        pass
    return cache


@app.route("/api/footer-counters")
def footer_counters():
    """Aggregated footer counters (single call) to avoid 20+ requests per page."""
    now = time.time()
    cached = _footer_counters_cache.get("data")
    if cached and (now - float(_footer_counters_cache.get("ts", 0) or 0) < 60):
        return jsonify(cached)

    cache = _read_download_cache()

    # Refresh GitHub caches (5 min TTL).
    _refresh_github_repo_cache(_github_cache, "Scottcjn/bottube")
    _refresh_github_repo_cache(_clawrtc_github_cache, "Scottcjn/Rustchain")
    _refresh_github_repo_cache(_grazer_github_cache, "Scottcjn/grazer-skill")

    # Get stats from DB
    video_count = 0
    agent_count = 0
    human_count = 0
    try:
        db = get_db()
        video_count = db.execute("SELECT COUNT(*) FROM videos").fetchone()[0]
        agent_count = db.execute("SELECT COUNT(*) FROM agents WHERE is_human = 0").fetchone()[0]
        human_count = db.execute("SELECT COUNT(*) FROM agents WHERE is_human = 1").fetchone()[0]
    except Exception as e:
        app.logger.warning(f"Failed to fetch footer stats: {e}")

    data = {
        "ts": int(now),
        "stats": {
            "videos": video_count,
            "agents": agent_count,
            "humans": human_count
        },
        "bottube": {
            "downloads": {
                "clawhub": int(cache.get("clawhub", 0) or 0),
                "npm": int(cache.get("npm", 0) or 0),
                "pypi": int(cache.get("pypi", 0) or 0),
            },
            "github": {
                "stars": int(_github_cache.get("stars", 0) or 0),
                "forks": int(_github_cache.get("forks", 0) or 0),
                "clones": int(_github_cache.get("clones", 0) or 0),
            },
            "installs": {
                "homebrew": int(cache.get("bottube_homebrew", 0) or 0),
                "apt": int(cache.get("bottube_apt", 0) or 0),
                "docker": int(cache.get("bottube_docker", 0) or 0),
            },
        },
        "clawrtc": {
            "downloads": {
                "clawhub": int(cache.get("clawrtc_clawhub", 0) or 0),
                "npm": int(cache.get("clawrtc_npm", 0) or 0),
                "pypi": int(cache.get("clawrtc_pypi", 0) or 0),
            },
            "github": {
                "stars": int(_clawrtc_github_cache.get("stars", 0) or 0),
                "forks": int(_clawrtc_github_cache.get("forks", 0) or 0),
            },
            "installs": {
                "homebrew": int(cache.get("clawrtc_homebrew", 0) or 0),
                "apt": int(cache.get("clawrtc_apt", 0) or 0),
                "aur": int(cache.get("clawrtc_aur", 0) or 0),
                "tigerbrew": int(cache.get("clawrtc_tigerbrew", 0) or 0),
            },
        },
        "grazer": {
            "downloads": {
                "clawhub": int(cache.get("grazer_clawhub", 0) or 0),
                "npm": int(cache.get("grazer_npm", 0) or 0),
                "pypi": int(cache.get("grazer_pypi", 0) or 0),
            },
            "github": {
                "stars": int(_grazer_github_cache.get("stars", 0) or 0),
                "forks": int(_grazer_github_cache.get("forks", 0) or 0),
            },
            "installs": {
                "homebrew": int(cache.get("grazer_homebrew", 0) or 0),
                "apt": int(cache.get("grazer_apt", 0) or 0),
            },
        },
    }

    _footer_counters_cache["ts"] = now
    _footer_counters_cache["data"] = data
    return jsonify(data)



_clawhub_cache = {"count": 232, "ts": 0}
@app.route("/api/clawhub-downloads")
def clawhub_downloads():
    """Get ClawHub download count - auto-updated from cache"""
    try:
        import json
        with open('/root/bottube/download_cache.json') as f:
            cache = json.load(f)
        return jsonify({"downloads": cache.get('clawhub', 0)})
    except:
        return jsonify({"downloads": 0})

_npm_cache = {"count": 188, "ts": 0}
@app.route("/api/npm-downloads")
def npm_downloads():
    """Get NPM download count - auto-updated from cache"""
    try:
        import json
        with open('/root/bottube/download_cache.json') as f:
            cache = json.load(f)
        return jsonify({"downloads": cache.get('npm', 0)})
    except:
        return jsonify({"downloads": 0})
    try:
        req = urllib.request.Request("https://api.npmjs.org/downloads/point/2026-01-01:2026-12-31/bottube")
        req.add_header("User-Agent", "BoTTube/1.0")
        with urllib.request.urlopen(req, timeout=10) as resp:
            data = json.loads(resp.read())
            _npm_cache["count"] = data.get("downloads", _npm_cache["count"])
            _npm_cache["ts"] = time.time()
    except Exception:
        pass
    return jsonify({"downloads": _npm_cache["count"]})

_pypi_cache = {"count": 513, "ts": 0}
@app.route("/api/pypi-downloads")
def pypi_downloads():
    """Get PyPI download count - auto-updated from cache"""
    try:
        import json
        with open('/root/bottube/download_cache.json') as f:
            cache = json.load(f)
        return jsonify({"downloads": cache.get('pypi', 0)})
    except:
        return jsonify({"downloads": 0})
    try:
        # Use /overall endpoint to include mirror downloads
        req = urllib.request.Request("https://pypistats.org/api/packages/bottube/overall")
        req.add_header("User-Agent", "BoTTube/1.0")
        with urllib.request.urlopen(req, timeout=10) as resp:
            data = json.loads(resp.read())
            rows = data.get("data", [])
            # Sum all "with_mirrors" entries (includes mirrors + direct)
            total = sum(r.get("downloads", 0) for r in rows if r.get("category") == "with_mirrors")
            if total > 0:
                _pypi_cache["count"] = total
                _pypi_cache["ts"] = time.time()
    except Exception:
        pass
    return jsonify({"downloads": _pypi_cache["count"]})







# ── Platform install counters (Homebrew, APT, AUR, Docker, Tigerbrew) ──
@app.route("/api/platform-installs")
def api_platform_installs():
    product = (request.args.get("product", "") or "")[:40]
    platform = (request.args.get("platform", "") or "")[:40]
    key = f"{product}_{platform}"
    try:
        with open("/root/bottube/download_cache.json") as f:
            cache = json.load(f)
        count = cache.get(key, 0) or 0
    except Exception:
        count = 0
    return jsonify({"installs": count, "product": product, "platform": platform})


# --- ClawRTC Miner Stats ---
_clawrtc_github_cache = {"stars": 0, "forks": 0, "clones": 0, "ts": 0}


@app.route("/api/clawrtc-github-stats")
def clawrtc_github_stats():
    import time, urllib.request, json
    now = time.time()
    if now - _clawrtc_github_cache["ts"] < 300:
        return jsonify(_clawrtc_github_cache)
    try:
        req = urllib.request.Request("https://api.github.com/repos/Scottcjn/Rustchain")
        req.add_header("User-Agent", "BoTTube/1.0")
        with urllib.request.urlopen(req, timeout=10) as resp:
            data = json.loads(resp.read())
            _clawrtc_github_cache["stars"] = data.get("stargazers_count", _clawrtc_github_cache["stars"])
            _clawrtc_github_cache["forks"] = data.get("forks_count", _clawrtc_github_cache["forks"])
            _clawrtc_github_cache["ts"] = now
    except Exception:
        pass
    return jsonify(_clawrtc_github_cache)


@app.route("/api/clawrtc-clawhub-downloads")
def clawrtc_clawhub_downloads():
    """Get ClawRTC ClawHub download count"""
    try:
        with open('/root/bottube/download_cache.json') as f:
            cache = json.load(f)
        return jsonify({"downloads": cache.get('clawrtc_clawhub', 0)})
    except Exception:
        return jsonify({"downloads": 0})


@app.route("/api/clawrtc-npm-downloads")
def clawrtc_npm_downloads():
    """Get ClawRTC npm download count"""
    try:
        with open('/root/bottube/download_cache.json') as f:
            cache = json.load(f)
        return jsonify({"downloads": cache.get('clawrtc_npm', 0)})
    except Exception:
        return jsonify({"downloads": 0})


@app.route("/api/clawrtc-pypi-downloads")
def clawrtc_pypi_downloads():
    """Get ClawRTC PyPI download count"""
    try:
        with open('/root/bottube/download_cache.json') as f:
            cache = json.load(f)
        return jsonify({"downloads": cache.get('clawrtc_pypi', 0)})
    except Exception:
        return jsonify({"downloads": 0})


_grazer_github_cache = {"stars": 0, "forks": 0, "clones": 0, "ts": 0}

@app.route("/api/grazer-github-stats")
def grazer_github_stats():
    import time, urllib.request, json
    now = time.time()
    if now - _grazer_github_cache["ts"] < 300:
        return jsonify(_grazer_github_cache)
    try:
        req = urllib.request.Request("https://api.github.com/repos/Scottcjn/grazer-skill")
        req.add_header("User-Agent", "BoTTube/1.0")
        with urllib.request.urlopen(req, timeout=10) as resp:
            data = json.loads(resp.read())
            _grazer_github_cache["stars"] = data.get("stargazers_count", _grazer_github_cache["stars"])
            _grazer_github_cache["forks"] = data.get("forks_count", _grazer_github_cache["forks"])
            _grazer_github_cache["ts"] = now
    except Exception:
        pass
    return jsonify(_grazer_github_cache)

@app.route("/api/grazer-clawhub-downloads")
def grazer_clawhub_downloads():
    """Get Grazer ClawHub download count"""
    try:
        import json
        with open('/root/bottube/download_cache.json') as f:
            cache = json.load(f)
        return jsonify({"downloads": cache.get('grazer_clawhub', 0)})
    except Exception:
        return jsonify({"downloads": 0})


@app.route("/api/grazer-npm-downloads")
def grazer_npm_downloads():
    """Get Grazer npm download count"""
    try:
        import json
        with open('/root/bottube/download_cache.json') as f:
            cache = json.load(f)
        return jsonify({"downloads": cache.get('grazer_npm', 0)})
    except Exception:
        return jsonify({"downloads": 0})


@app.route("/api/grazer-pypi-downloads")
def grazer_pypi_downloads():
    """Get Grazer PyPI download count"""
    try:
        import json
        with open('/root/bottube/download_cache.json') as f:
            cache = json.load(f)
        return jsonify({"downloads": cache.get('grazer_pypi', 0)})
    except Exception:
        return jsonify({"downloads": 0})


@app.route("/api/beacon-clawhub-downloads")
def beacon_clawhub_downloads():
    """Get Beacon ClawHub download count"""
    try:
        import json
        with open('/root/bottube/download_cache.json') as f:
            cache = json.load(f)
        return jsonify({"downloads": cache.get('beacon_clawhub', 0)})
    except Exception:
        return jsonify({"downloads": 0})


@app.route("/api/beacon-npm-downloads")
def beacon_npm_downloads():
    """Get Beacon npm download count"""
    try:
        import json
        with open('/root/bottube/download_cache.json') as f:
            cache = json.load(f)
        return jsonify({"downloads": cache.get('beacon_npm', 0)})
    except Exception:
        return jsonify({"downloads": 0})


@app.route("/api/beacon-pypi-downloads")
def beacon_pypi_downloads():
    """Get Beacon PyPI download count"""
    try:
        import json
        with open('/root/bottube/download_cache.json') as f:
            cache = json.load(f)
        return jsonify({"downloads": cache.get('beacon_pypi', 0)})
    except Exception:
        return jsonify({"downloads": 0})


@app.route("/grazer")
@app.route("/skills/grazer")
def grazer_page():
    """Grazer skill page"""
    return render_template("grazer.html")



# ---------------------------------------------------------------------------
# Phase 1: Bulk admin remove
# ---------------------------------------------------------------------------

@app.route("/api/admin/bulk-remove", methods=["POST"])
def admin_bulk_remove():
    """Hold or soft-delete multiple videos by ID list. Force is required for destructive mode.

    POST JSON: {"video_ids": ["abc", "def", ...], "reason": "spam", "force": false}
    Optionally: {"agent_name": "fredrick", "reason": "spam"} to target all by agent.
    """
    err = _require_admin()
    if err:
        return err

    data, error = _admin_json_body()
    if error:
        return jsonify({"error": error}), 400
    video_ids = data.get("video_ids", [])
    agent_name, error = _admin_text_field(data, "agent_name")
    if error:
        return jsonify({"error": error}), 400
    reason, error = _admin_text_field(data, "reason", default="Bulk moderation review")
    if error:
        return jsonify({"error": error}), 400
    force = bool(data.get("force", False))

    db = get_db()
    touched = 0
    hold_ids = []

    def _hold_video_rows(rows):
        local_count = 0
        for row in rows:
            hold_id = _queue_moderation_hold(
                db,
                target_type="video",
                target_ref=row["video_id"],
                target_agent_id=row["agent_id"],
                source="admin_bulk_remove",
                reason=reason,
                details=json.dumps({"requested_force_remove": force, "agent_name": agent_name}),
                recommended_action="hold_content" if not force else "review",
                coach_note=(
                    f"A BoTTube maintainer held one of your videos for review: {reason}.\n\n"
                    "No deletion was applied by default. Revise the content if needed and wait for follow-up."
                ),
            )
            if hold_id:
                hold_ids.append(hold_id)
            local_count += 1
        return local_count

    if agent_name and not video_ids:
        agent = db.execute(
            "SELECT id FROM agents WHERE agent_name = ?", (agent_name,)
        ).fetchone()
        if not agent:
            return jsonify({"error": f"Agent '{agent_name}' not found"}), 404
        rows = db.execute(
            "SELECT video_id, agent_id FROM videos WHERE agent_id = ? AND is_removed = 0",
            (agent["id"],),
        ).fetchall()
        touched = _hold_video_rows(rows)
        cur = db.execute(
            "UPDATE videos SET is_removed = 1, removed_reason = ? WHERE agent_id = ? AND is_removed = 0",
            ((reason if force else f"held for review: {reason}"), agent["id"]),
        )
        touched = cur.rowcount if rows else touched
    elif video_ids:
        clean_video_ids = [str(vid).strip() for vid in video_ids if str(vid).strip()]
        if not clean_video_ids:
            return jsonify({"error": "Provide at least one valid video_id"}), 400
        rows = db.execute(
            f"SELECT video_id, agent_id FROM videos WHERE video_id IN ({','.join('?' for _ in clean_video_ids)})",
            tuple(clean_video_ids),
        ).fetchall()
        touched = _hold_video_rows(rows)
        forced_updates = 0
        for vid in clean_video_ids:
            cur = db.execute(
                "UPDATE videos SET is_removed = 1, removed_reason = ? WHERE video_id = ? AND is_removed = 0",
                ((reason if force else f"held for review: {reason}"), vid),
            )
            if force:
                forced_updates += cur.rowcount
        if force:
            touched = forced_updates
    else:
        return jsonify({"error": "Provide video_ids list or agent_name"}), 400

    db.commit()
    app.logger.warning(
        "ADMIN BULK %s: count=%d agent=%s reason='%s'",
        "REMOVE" if force else "HOLD",
        touched, agent_name or "N/A", reason,
    )
    return jsonify({
        "ok": True,
        "mode": "force_remove" if force else "hold_for_review",
        "affected_count": touched,
        "reason": reason,
        "hold_ids": hold_ids[:100],
    })


# ---------------------------------------------------------------------------
# Phase 3: Internal Message Box
# ---------------------------------------------------------------------------

def _gen_message_id():
    """Generate a unique message ID."""
    return f"msg_{secrets.token_hex(12)}"


def _message_text_field(data, field, default="", max_length=None):
    value = data.get(field, default)
    if value is None:
        value = default
    if not isinstance(value, str):
        return None, f"{field} must be a string"
    value = value.strip()
    if max_length is not None:
        value = value[:max_length]
    return value, None


def _send_system_message(db, to_agent: str, subject: str, body: str,
                         msg_type: str = "system"):
    """Send a system-generated message to an agent."""
    msg_id = _gen_message_id()
    db.execute(
        """INSERT INTO messages (id, from_agent, to_agent, subject, body, message_type)
           VALUES (?, 'system', ?, ?, ?, ?)""",
        (msg_id, to_agent, subject, body, msg_type),
    )
    return msg_id


@app.route("/api/messages", methods=["POST"])
@require_api_key
def send_message():
    """Send a message from the authenticated agent.

    POST JSON: {
        "to": "agent_name",    (or null/omitted for broadcast)
        "subject": "Hello",
        "body": "Message content",
        "message_type": "general"  (general, system, moderation, alert)
    }
    """
    data = request.get_json(silent=True)
    if data is None:
        data = {}
    elif not isinstance(data, dict):
        return jsonify({"error": "JSON body must be an object"}), 400

    to_agent, error = _message_text_field(data, "to")
    if error:
        return jsonify({"error": error}), 400
    to_agent = to_agent or None

    subject, error = _message_text_field(data, "subject", max_length=200)
    if error:
        return jsonify({"error": error}), 400

    body, error = _message_text_field(data, "body", max_length=5000)
    if error:
        return jsonify({"error": error}), 400

    msg_type, error = _message_text_field(data, "message_type", default="general")
    if error:
        return jsonify({"error": error}), 400

    if not body:
        return jsonify({"error": "body is required"}), 400

    if msg_type not in ("general", "system", "moderation", "alert"):
        msg_type = "general"

    db = get_db()

    # Validate recipient exists if specified
    if to_agent:
        recipient = db.execute(
            "SELECT agent_name FROM agents WHERE agent_name = ?", (to_agent,)
        ).fetchone()
        if not recipient:
            return jsonify({"error": f"Recipient '{to_agent}' not found"}), 404

    msg_id = _gen_message_id()
    db.execute(
        """INSERT INTO messages (id, from_agent, to_agent, subject, body, message_type)
           VALUES (?, ?, ?, ?, ?, ?)""",
        (msg_id, g.agent["agent_name"], to_agent, subject, body, msg_type),
    )
    db.commit()

    return jsonify({"ok": True, "message_id": msg_id}), 201


@app.route("/api/messages/inbox")
@require_api_key
def message_inbox():
    """Get messages for the authenticated agent.

    Query params: page, per_page, unread_only (0/1)
    """
    page = max(1, request.args.get("page", 1, type=int))
    per_page = min(50, max(1, request.args.get("per_page", 20, type=int)))
    unread_only = request.args.get("unread_only", "0") == "1"
    offset = (page - 1) * per_page

    db = get_db()
    agent_name = g.agent["agent_name"]

    read_join = (
        "LEFT JOIN message_reads mr "
        "ON m.id = mr.message_id AND mr.agent_name = ?"
    )
    where = "WHERE (m.to_agent = ? OR m.to_agent IS NULL)"
    params = [agent_name, agent_name]
    if unread_only:
        where += (
            " AND ((m.to_agent IS NULL AND mr.read_at IS NULL) "
            "OR (m.to_agent IS NOT NULL AND m.read_at IS NULL))"
        )

    total = db.execute(
        f"SELECT COUNT(*) FROM messages m {read_join} {where}", params
    ).fetchone()[0]

    rows = db.execute(
        f"""SELECT m.*,
                   CASE
                     WHEN m.to_agent IS NULL THEN mr.read_at
                     ELSE m.read_at
                   END AS effective_read_at
            FROM messages m {read_join} {where}
            ORDER BY m.created_at DESC LIMIT ? OFFSET ?""",
        params + [per_page, offset],
    ).fetchall()

    messages = []
    for r in rows:
        messages.append({
            "id": r["id"],
            "from": r["from_agent"],
            "to": r["to_agent"],
            "subject": r["subject"],
            "body": r["body"],
            "message_type": r["message_type"],
            "read_at": r["effective_read_at"],
            "created_at": r["created_at"],
        })

    return jsonify({
        "ok": True,
        "messages": messages,
        "total": total,
        "page": page,
        "per_page": per_page,
    })


@app.route("/api/messages/<msg_id>/read", methods=["POST"])
@require_api_key
def mark_message_read(msg_id):
    """Mark a message as read."""
    db = get_db()
    agent_name = g.agent["agent_name"]

    msg = db.execute(
        "SELECT id, to_agent FROM messages WHERE id = ?", (msg_id,)
    ).fetchone()
    if not msg:
        return jsonify({"error": "Message not found"}), 404

    # Only the recipient (or broadcast recipient) can mark as read
    if msg["to_agent"] and msg["to_agent"] != agent_name:
        return jsonify({"error": "Not your message"}), 403

    if msg["to_agent"] is None:
        db.execute(
            """INSERT INTO message_reads (message_id, agent_name, read_at)
               VALUES (?, ?, datetime('now'))
               ON CONFLICT(message_id, agent_name)
               DO UPDATE SET read_at = excluded.read_at""",
            (msg_id, agent_name),
        )
    else:
        db.execute(
            "UPDATE messages SET read_at = datetime('now') WHERE id = ? AND read_at IS NULL",
            (msg_id,),
        )
    db.commit()
    return jsonify({"ok": True})


@app.route("/api/messages/unread-count")
@require_api_key
def message_unread_count():
    """Get unread message count for the authenticated agent."""
    db = get_db()
    agent_name = g.agent["agent_name"]

    count = db.execute(
        """SELECT COUNT(*)
           FROM messages m
           LEFT JOIN message_reads mr
             ON m.id = mr.message_id AND mr.agent_name = ?
           WHERE (m.to_agent = ? OR m.to_agent IS NULL)
             AND ((m.to_agent IS NULL AND mr.read_at IS NULL)
                  OR (m.to_agent IS NOT NULL AND m.read_at IS NULL))""",
        (agent_name, agent_name),
    ).fetchone()[0]

    return jsonify({"ok": True, "unread": count})




# ---------------------------------------------------------------------------
# Tag Browsing (Phase 5)
# ---------------------------------------------------------------------------

@app.route("/tag/<tag_name>")
def tag_page(tag_name):
    """Browse videos by tag."""
    db = get_db()
    # Search for videos with this tag (case-insensitive)
    like_tag = f'%"{tag_name}"%'
    videos = db.execute(
        """SELECT v.*, a.agent_name, a.display_name, a.avatar_url, a.is_human
           FROM videos v JOIN agents a ON v.agent_id = a.id
           WHERE v.is_removed = 0
             AND COALESCE(a.is_banned, 0) = 0
             AND LOWER(v.tags) LIKE LOWER(?)
           ORDER BY v.views DESC, v.created_at DESC
           LIMIT 100""",
        (like_tag,),
    ).fetchall()
    return render_template("tag.html", tag_name=tag_name, videos=videos)


@app.route("/api/tags")
def api_tags():
    """Return popular tags with video counts."""
    db = get_db()
    rows = db.execute(
        """SELECT v.tags
           FROM videos v JOIN agents a ON v.agent_id = a.id
           WHERE v.is_removed = 0
             AND COALESCE(a.is_banned, 0) = 0
             AND v.tags != '[]'"""
    ).fetchall()
    tag_counts = {}
    for row in rows:
        for t in _safe_json_loads_list(row["tags"]):
            t = str(t).strip().lower()
            if t:
                tag_counts[t] = tag_counts.get(t, 0) + 1
    # Sort by count descending, return top 200
    sorted_tags = sorted(tag_counts.items(), key=lambda x: -x[1])[:200]
    return jsonify({
        "ok": True,
        "tags": [{"tag": t, "count": c} for t, c in sorted_tags],
    })


# ---------------------------------------------------------------------------
# Watch History API (Phase 6)
# ---------------------------------------------------------------------------

@app.route("/api/history")
@require_api_key
def api_history():
    """Get authenticated user's watch history (paginated)."""
    db = get_db()
    page = max(1, request.args.get("page", 1, type=int))
    per_page = min(50, max(1, request.args.get("per_page", 20, type=int)))
    offset = (page - 1) * per_page

    rows = db.execute(
        """SELECT wh.watched_at, wh.watch_duration_sec,
                  v.video_id, v.title, v.thumbnail, v.duration_sec, v.views,
                  a.agent_name, a.display_name
           FROM watch_history wh
           JOIN videos v ON wh.video_id = v.video_id
           JOIN agents a ON v.agent_id = a.id
           WHERE wh.agent_id = ?
            AND COALESCE(v.is_removed, 0) = 0
            AND COALESCE(a.is_banned, 0) = 0
           ORDER BY wh.watched_at DESC
           LIMIT ? OFFSET ?""",
        (g.agent["id"], per_page, offset),
    ).fetchall()

    total = db.execute(
        """SELECT COUNT(*)
           FROM watch_history wh
           JOIN videos v ON wh.video_id = v.video_id
           JOIN agents a ON v.agent_id = a.id
           WHERE wh.agent_id = ?
             AND COALESCE(v.is_removed, 0) = 0
             AND COALESCE(a.is_banned, 0) = 0""",
        (g.agent["id"],),
    ).fetchone()[0]

    return jsonify({
        "ok": True,
        "page": page,
        "per_page": per_page,
        "total": total,
        "history": [
            {
                "video_id": r["video_id"],
                "title": r["title"],
                "thumbnail": r["thumbnail"],
                "duration_sec": r["duration_sec"],
                "views": r["views"],
                "agent_name": r["agent_name"],
                "display_name": r["display_name"],
                "watched_at": r["watched_at"],
                "watch_duration_sec": r["watch_duration_sec"],
            }
            for r in rows
        ],
    })


@app.route("/api/history", methods=["DELETE"])
@require_api_key
def api_history_clear():
    """Clear watch history for authenticated user."""
    db = get_db()
    db.execute("DELETE FROM watch_history WHERE agent_id = ?", (g.agent["id"],))
    db.commit()
    return jsonify({"ok": True, "message": "Watch history cleared"})


@app.route("/api/videos/<video_id>/related")
def api_related_videos(video_id):
    """Get related videos for a given video ID."""
    db = get_db()
    video = db.execute(
        f"""SELECT v.*
           FROM videos v JOIN agents a ON v.agent_id = a.id
           WHERE v.video_id = ? AND {_public_video_filter_sql()}""",
        (video_id,),
    ).fetchone()
    if not video:
        return jsonify({"error": "Video not found"}), 404

    cur_tags = set()
    try:
        cur_tags = set(json.loads(video["tags"])) if video["tags"] else set()
    except Exception:
        pass
    cur_cat = video["category"] or "other"

    candidates = db.execute(
        f"""SELECT v.*, a.agent_name, a.display_name, a.avatar_url
           FROM videos v JOIN agents a ON v.agent_id = a.id
           WHERE v.video_id != ? AND {_public_video_filter_sql()}
           ORDER BY v.views DESC
           LIMIT 100""",
        (video_id,),
    ).fetchall()

    def score(r):
        s = 0
        if r["agent_id"] == video["agent_id"]:
            s += 3
        if (r["category"] or "other") == cur_cat:
            s += 2
        try:
            r_tags = set(json.loads(r["tags"])) if r["tags"] else set()
            s += len(cur_tags & r_tags)
        except Exception:
            pass
        return s

    scored = sorted(candidates, key=score, reverse=True)
    limit = min(20, max(1, request.args.get("limit", 8, type=int)))

    return jsonify({
        "ok": True,
        "related": [
            {
                "video_id": r["video_id"],
                "title": r["title"],
                "thumbnail": r["thumbnail"],
                "duration_sec": r["duration_sec"],
                "views": r["views"],
                "category": r["category"],
                "agent_name": r["agent_name"],
                "display_name": r["display_name"],
            }
            for r in scored[:limit]
        ],
    })


# ---------------------------------------------------------------------------
# Video & Comment Reporting (Phase 7)
# ---------------------------------------------------------------------------

REPORT_REASONS = {"spam", "inappropriate", "copyright", "harassment", "misleading", "other"}


def _report_text_field(data, field, default="", max_length=None):
    value = data.get(field, default)
    if value is None:
        value = default
    if not isinstance(value, str):
        return None, f"{field} must be a string"
    value = value.strip()
    if max_length is not None:
        value = value[:max_length]
    return value, None


def _get_reporter_id():
    """Get reporter agent ID from either API key auth or browser session."""
    # API key auth (check header directly since @require_api_key may not be applied)
    api_key = request.headers.get('X-API-Key', '')
    if api_key:
        db = get_db()
        agent = db.execute('SELECT id FROM agents WHERE api_key = ?', (api_key,)).fetchone()
        if agent:
            return agent['id']
    # Browser session auth
    if hasattr(g, 'user') and g.user:
        return g.user['id']
    return None

@app.route("/api/videos/<video_id>/report", methods=["POST"])
def report_video(video_id):
    """Report a video for policy violation. Accepts API key or session auth."""
    reporter_id = _get_reporter_id()
    if not reporter_id:
        return jsonify({"error": "Authentication required"}), 401

    db = get_db()
    video = db.execute(
        "SELECT video_id, agent_id, title FROM videos WHERE video_id = ?",
        (video_id,),
    ).fetchone()
    if not video:
        return jsonify({"error": "Video not found"}), 404

    data = request.get_json(silent=True)
    if data is None:
        data = {}
    elif not isinstance(data, dict):
        return jsonify({"error": "JSON body must be an object"}), 400

    reason, error = _report_text_field(data, "reason")
    if error:
        return jsonify({"error": error}), 400
    reason = reason.lower()
    details, error = _report_text_field(data, "details", max_length=1000)
    if error:
        return jsonify({"error": error}), 400

    if reason not in REPORT_REASONS:
        return jsonify({"error": f"Invalid reason. Must be one of: {', '.join(sorted(REPORT_REASONS))}"}), 400

    # Rate limit: 5 reports per hour per agent
    if not _rate_limit(f"report:{reporter_id}", 5, 3600):
        return jsonify({"error": "Report rate limit exceeded (max 5/hour)"}), 429

    # Check for duplicate report
    existing = db.execute(
        "SELECT 1 FROM reports WHERE video_id = ? AND reporter_agent_id = ?",
        (video_id, reporter_id),
    ).fetchone()
    if existing:
        return jsonify({"error": "You have already reported this video"}), 409

    db.execute(
        "INSERT INTO reports (video_id, reporter_agent_id, reason, details, status, created_at) VALUES (?, ?, ?, ?, 'pending', ?)",
        (video_id, reporter_id, reason, details, time.time()),
    )
    db.commit()

    # Auto-flag: if 3+ reports on the same video, queue a hold for review
    report_count = db.execute(
        "SELECT COUNT(*) FROM reports WHERE video_id = ? AND status = 'pending'",
        (video_id,),
    ).fetchone()[0]
    flagged_for_review = False
    if report_count >= 3:
        coach_note = (
            f"Multiple agents reported your video `{video['title'][:120]}` for review.\n\n"
            "No automatic deletion was applied. Check the video for spammy, misleading, or policy-breaking behavior and tighten it before reposting."
        )
        _queue_moderation_hold(
            db,
            target_type="video",
            target_ref=video_id,
            target_agent_id=video["agent_id"],
            source="community_reports",
            reason="video reached community report threshold",
            details=json.dumps({"report_count": report_count, "latest_reason": reason, "details": details}),
            recommended_action="review",
            coach_note=coach_note,
        )
        db.commit()
        flagged_for_review = True

    return jsonify({
        "ok": True,
        "flagged_for_review": flagged_for_review,
        "message": "Report submitted. Thank you for helping keep BoTTube safe.",
    })


@app.route("/api/comments/<int:comment_id>/report", methods=["POST"])
def report_comment(comment_id):
    """Report a comment for policy violation. Accepts API key or session auth."""
    reporter_id = _get_reporter_id()
    if not reporter_id:
        return jsonify({"error": "Authentication required"}), 401

    db = get_db()
    comment = db.execute("SELECT 1 FROM comments WHERE id = ?", (comment_id,)).fetchone()
    if not comment:
        return jsonify({"error": "Comment not found"}), 404

    data = request.get_json(silent=True)
    if data is None:
        data = {}
    elif not isinstance(data, dict):
        return jsonify({"error": "JSON body must be an object"}), 400

    reason, error = _report_text_field(data, "reason")
    if error:
        return jsonify({"error": error}), 400
    reason = reason.lower()
    details, error = _report_text_field(data, "details", max_length=1000)
    if error:
        return jsonify({"error": error}), 400

    if reason not in REPORT_REASONS:
        return jsonify({"error": f"Invalid reason. Must be one of: {', '.join(sorted(REPORT_REASONS))}"}), 400

    if not _rate_limit(f"report:{reporter_id}", 5, 3600):
        return jsonify({"error": "Report rate limit exceeded (max 5/hour)"}), 429

    existing = db.execute(
        "SELECT 1 FROM reports WHERE comment_id = ? AND reporter_agent_id = ?",
        (comment_id, reporter_id),
    ).fetchone()
    if existing:
        return jsonify({"error": "You have already reported this comment"}), 409

    db.execute(
        "INSERT INTO reports (comment_id, reporter_agent_id, reason, details, status, created_at) VALUES (?, ?, ?, ?, 'pending', ?)",
        (comment_id, reporter_id, reason, details, time.time()),
    )
    db.commit()

    return jsonify({"ok": True, "message": "Comment report submitted."})


@app.route("/api/admin/reports")
def admin_reports():
    """Admin view of pending reports (requires admin key)."""
    admin_key = request.headers.get("X-Admin-Key", "")
    if not ADMIN_KEY or admin_key != ADMIN_KEY:
        return jsonify({"error": "Unauthorized"}), 401

    db = get_db()
    status_filter = request.args.get("status", "pending")
    page = max(1, request.args.get("page", 1, type=int))
    per_page = min(50, max(1, request.args.get("per_page", 20, type=int)))
    offset = (page - 1) * per_page

    rows = db.execute(
        """SELECT r.*, a.agent_name AS reporter_name
           FROM reports r
           LEFT JOIN agents a ON r.reporter_agent_id = a.id
           WHERE r.status = ?
           ORDER BY r.created_at DESC
           LIMIT ? OFFSET ?""",
        (status_filter, per_page, offset),
    ).fetchall()

    total = db.execute(
        "SELECT COUNT(*) FROM reports WHERE status = ?", (status_filter,)
    ).fetchone()[0]

    return jsonify({
        "ok": True,
        "total": total,
        "reports": [
            {
                "id": r["id"],
                "video_id": r["video_id"],
                "comment_id": r["comment_id"],
                "reporter": r["reporter_name"],
                "reason": r["reason"],
                "details": r["details"],
                "status": r["status"],
                "created_at": r["created_at"],
            }
            for r in rows
        ],
    })


@app.route("/api/admin/reward-holds")
def admin_reward_holds():
    """Admin view of reward holds."""
    err = _require_admin()
    if err:
        return err

    db = get_db()
    status_filter = request.args.get("status", "pending")
    page = max(1, request.args.get("page", 1, type=int))
    per_page = min(100, max(1, request.args.get("per_page", 20, type=int)))
    offset = (page - 1) * per_page

    rows = db.execute(
        """
        SELECT rh.*, a.agent_name
        FROM reward_holds rh
        JOIN agents a ON a.id = rh.agent_id
        WHERE rh.status = ?
        ORDER BY rh.created_at DESC
        LIMIT ? OFFSET ?
        """,
        (status_filter, per_page, offset),
    ).fetchall()
    total = db.execute(
        "SELECT COUNT(*) FROM reward_holds WHERE status = ?",
        (status_filter,),
    ).fetchone()[0]

    return jsonify({
        "ok": True,
        "total": total,
        "holds": [
            {
                "id": r["id"],
                "agent_name": r["agent_name"],
                "event_type": r["event_type"],
                "event_ref": r["event_ref"],
                "amount": float(r["amount"] or 0.0),
                "risk_score": int(r["risk_score"] or 0),
                "reasons": _safe_json_loads_list(r["reasons"]),
                "status": r["status"],
                "created_at": r["created_at"],
                "reviewed_at": r["reviewed_at"],
                "reviewer_note": r["reviewer_note"],
            }
            for r in rows
        ],
    })


@app.route("/api/admin/reward-holds/<int:hold_id>/resolve", methods=["POST"])
def admin_resolve_reward_hold(hold_id):
    """Review a reward hold and either credit or dismiss it."""
    err = _require_admin()
    if err:
        return err

    db = get_db()
    hold = db.execute(
        """
        SELECT rh.*, a.agent_name
        FROM reward_holds rh
        JOIN agents a ON a.id = rh.agent_id
        WHERE rh.id = ?
        """,
        (hold_id,),
    ).fetchone()
    if not hold:
        return jsonify({"error": "Reward hold not found"}), 404

    data, error = _admin_json_body()
    if error:
        return jsonify({"error": error}), 400
    action, error = _admin_text_field(data, "action", default="dismiss")
    if error:
        return jsonify({"error": error}), 400
    reviewer_note, error = _admin_text_field(data, "note", max_length=2000)
    if error:
        return jsonify({"error": error}), 400
    now = time.time()

    if action == "credit":
        award_rtc(db, hold["agent_id"], float(hold["amount"] or 0.0), f"{hold['event_type']}_reviewed")
        status = "credited"
    elif action == "coach":
        note = reviewer_note or (
            f"Your `{hold['event_type']}` reward was held for review. "
            "Tighten the interaction quality and avoid concentrated low-signal activity before trying again."
        )
        _send_coaching_note(
            db,
            agent_id=hold["agent_id"],
            subject=f"BoTTube coaching: held {hold['event_type']} reward",
            body=note,
        )
        status = "dismissed"
    elif action == "dismiss":
        status = "dismissed"
    else:
        return jsonify({"error": "Invalid action. Use credit, coach, or dismiss."}), 400

    db.execute(
        "UPDATE reward_holds SET status = ?, reviewed_at = ?, reviewer_note = ? WHERE id = ?",
        (status, now, reviewer_note, hold_id),
    )
    db.commit()
    return jsonify({"ok": True, "action": action, "status": status})


@app.route("/api/admin/moderation-holds")
def admin_moderation_holds():
    """Admin view of moderation holds."""
    err = _require_admin()
    if err:
        return err

    db = get_db()
    status_filter = request.args.get("status", "pending")
    page = max(1, request.args.get("page", 1, type=int))
    per_page = min(100, max(1, request.args.get("per_page", 20, type=int)))
    offset = (page - 1) * per_page

    rows = db.execute(
        """
        SELECT mh.*, a.agent_name
        FROM moderation_holds mh
        LEFT JOIN agents a ON a.id = mh.target_agent_id
        WHERE mh.status = ?
        ORDER BY mh.created_at DESC
        LIMIT ? OFFSET ?
        """,
        (status_filter, per_page, offset),
    ).fetchall()
    total = db.execute(
        "SELECT COUNT(*) FROM moderation_holds WHERE status = ?",
        (status_filter,),
    ).fetchone()[0]

    return jsonify({
        "ok": True,
        "total": total,
        "holds": [
            {
                "id": r["id"],
                "target_type": r["target_type"],
                "target_ref": r["target_ref"],
                "target_agent": r["agent_name"],
                "source": r["source"],
                "reason": r["reason"],
                "details": r["details"],
                "status": r["status"],
                "recommended_action": r["recommended_action"],
                "coach_note": r["coach_note"],
                "created_at": r["created_at"],
                "reviewed_at": r["reviewed_at"],
                "reviewer_note": r["reviewer_note"],
            }
            for r in rows
        ],
    })


@app.route("/api/admin/moderation-holds/<int:hold_id>/resolve", methods=["POST"])
def admin_resolve_moderation_hold(hold_id):
    """Review a moderation hold with non-destructive defaults."""
    err = _require_admin()
    if err:
        return err

    db = get_db()
    hold = db.execute("SELECT * FROM moderation_holds WHERE id = ?", (hold_id,)).fetchone()
    if not hold:
        return jsonify({"error": "Moderation hold not found"}), 404

    data, error = _admin_json_body()
    if error:
        return jsonify({"error": error}), 400
    action, error = _admin_text_field(data, "action", default="dismiss")
    if error:
        return jsonify({"error": error}), 400
    reviewer_note, error = _admin_text_field(data, "note", max_length=2000)
    if error:
        return jsonify({"error": error}), 400
    coach_note, error = _admin_text_field(data, "coach_note", max_length=5000)
    if error:
        return jsonify({"error": error}), 400
    coach_note = coach_note or hold["coach_note"] or reviewer_note
    now = time.time()

    status = "dismissed"
    if action in {"release", "restore"}:
        if hold["target_type"] == "video":
            db.execute(
                """
                UPDATE videos
                SET is_removed = 0, removed_reason = ''
                WHERE video_id = ?
                  AND removed_reason LIKE 'held for review:%'
                """,
                (hold["target_ref"],),
            )
        status = "released"
    elif action == "coach":
        if coach_note and hold["target_agent_id"]:
            _send_coaching_note(
                db,
                agent_id=hold["target_agent_id"],
                subject=f"BoTTube coaching: {hold['reason']}",
                body=coach_note,
                video_id=hold["target_ref"] if hold["target_type"] == "video" else "",
            )
        status = "coached"
    elif action == "escalate":
        status = "escalated"
    elif action == "force_remove":
        if hold["target_type"] == "video":
            db.execute(
                "UPDATE videos SET is_removed = 1, removed_reason = ? WHERE video_id = ?",
                (f"force removed: {reviewer_note or hold['reason']}", hold["target_ref"]),
            )
        elif hold["target_type"] == "comment":
            db.execute("DELETE FROM comment_votes WHERE comment_id = ?", (int(hold["target_ref"]),))
            db.execute("DELETE FROM comments WHERE id = ?", (int(hold["target_ref"]),))
        elif hold["target_type"] == "agent" and hold["target_agent_id"]:
            db.execute(
                "UPDATE agents SET is_banned = 1, ban_reason = ?, banned_at = ? WHERE id = ?",
                (reviewer_note or hold["reason"], now, hold["target_agent_id"]),
            )
        status = "removed"
    elif action == "force_ban":
        if not hold["target_agent_id"]:
            return jsonify({"error": "Hold has no target agent to ban"}), 400
        db.execute(
            "UPDATE agents SET is_banned = 1, ban_reason = ?, banned_at = ? WHERE id = ?",
            (reviewer_note or hold["reason"], now, hold["target_agent_id"]),
        )
        status = "banned"
    elif action != "dismiss":
        return jsonify({"error": "Invalid action. Use dismiss, release, restore, coach, escalate, force_remove, or force_ban."}), 400

    db.execute(
        "UPDATE moderation_holds SET status = ?, reviewed_at = ?, reviewer_note = ? WHERE id = ?",
        (status, now, reviewer_note, hold_id),
    )
    db.commit()
    return jsonify({"ok": True, "action": action, "status": status})


@app.route("/api/admin/reports/<int:report_id>/resolve", methods=["POST"])
def admin_resolve_report(report_id):
    """Resolve a report (requires admin key)."""
    admin_key = request.headers.get("X-Admin-Key", "")
    if not ADMIN_KEY or admin_key != ADMIN_KEY:
        return jsonify({"error": "Unauthorized"}), 401

    db = get_db()
    report = db.execute("SELECT * FROM reports WHERE id = ?", (report_id,)).fetchone()
    if not report:
        return jsonify({"error": "Report not found"}), 404

    data = request.get_json(silent=True) or {}
    action = data.get("action", "coach")  # dismiss, coach, hold_content, remove_content, ban_user
    force = bool(data.get("force", False))
    target_agent_id = None
    target_type = "report"
    target_ref = str(report_id)
    coach_note = data.get("coach_note", "").strip()

    if report["video_id"]:
        video = db.execute(
            "SELECT video_id, agent_id, title FROM videos WHERE video_id = ?",
            (report["video_id"],),
        ).fetchone()
        if video:
            target_agent_id = video["agent_id"]
            target_type = "video"
            target_ref = video["video_id"]
            if not coach_note:
                coach_note = (
                    f"A BoTTube maintainer reviewed a report on your video `{video['title'][:120]}`.\n\n"
                    "Tighten the content and metadata, then wait for maintainer feedback."
                )
    elif report["comment_id"]:
        comment = db.execute(
            "SELECT c.id, c.agent_id, c.content FROM comments c WHERE c.id = ?",
            (report["comment_id"],),
        ).fetchone()
        if comment:
            target_agent_id = comment["agent_id"]
            target_type = "comment"
            target_ref = str(comment["id"])
            if not coach_note:
                coach_note = (
                    "A BoTTube maintainer reviewed a report on one of your comments.\n\n"
                    "Make the comment more specific and less repetitive before posting similar replies again."
                )

    normalized_action = action
    if action in {"remove_content", "ban_user"} and not force:
        normalized_action = "hold_content" if action == "remove_content" else "coach"

    if normalized_action == "coach":
        _queue_moderation_hold(
            db,
            target_type=target_type,
            target_ref=target_ref,
            target_agent_id=target_agent_id,
            source="admin_report_resolution",
            reason=f"report #{report_id}: {report['reason']}",
            details=report["details"] or "",
            recommended_action="coach",
            coach_note=coach_note,
        )
    elif normalized_action == "hold_content":
        _queue_moderation_hold(
            db,
            target_type=target_type,
            target_ref=target_ref,
            target_agent_id=target_agent_id,
            source="admin_report_resolution",
            reason=f"report #{report_id}: {report['reason']}",
            details=report["details"] or "",
            recommended_action="review",
            coach_note=coach_note,
        )
        if report["video_id"]:
            db.execute(
                "UPDATE videos SET is_removed = 1, removed_reason = ? WHERE video_id = ?",
                (f"held for review: report #{report_id} ({report['reason']})", report["video_id"]),
            )
    elif normalized_action == "remove_content":
        if report["video_id"]:
            db.execute(
                "UPDATE videos SET is_removed = 1, removed_reason = ? WHERE video_id = ?",
                (f"removed: report #{report_id} ({report['reason']})", report["video_id"]),
            )
        elif report["comment_id"]:
            db.execute("DELETE FROM comments WHERE id = ?", (report["comment_id"],))
    elif normalized_action == "ban_user":
        if target_agent_id:
            db.execute(
                "UPDATE agents SET is_banned = 1, ban_reason = ?, banned_at = ? WHERE id = ?",
                (f"report #{report_id}: {report['reason']}", time.time(), target_agent_id),
            )

    db.execute(
        "UPDATE reports SET status = ? WHERE id = ?",
        ("resolved" if normalized_action == "dismiss" else "actioned", report_id),
    )
    db.commit()

    return jsonify({"ok": True, "action": normalized_action, "forced": force})


# ---------------------------------------------------------------------------
# Structured Data Helpers (Phase 3) — used by templates via Jinja globals
# ---------------------------------------------------------------------------

def build_breadcrumb_jsonld(items):
    """Build BreadcrumbList JSON-LD from a list of (name, url) tuples."""
    return Markup(safe_jsonld({
        "@context": "https://schema.org",
        "@type": "BreadcrumbList",
        "itemListElement": [
            {
                "@type": "ListItem",
                "position": i + 1,
                "name": name,
                "item": url,
            }
            for i, (name, url) in enumerate(items)
        ],
    }))

app.jinja_env.globals["build_breadcrumb_jsonld"] = build_breadcrumb_jsonld
app.jinja_env.globals["get_organization_jsonld"] = get_organization_jsonld
app.jinja_env.globals["get_website_jsonld"] = get_website_jsonld
app.jinja_env.globals["get_faqpage_jsonld"] = get_faqpage_jsonld
app.jinja_env.globals["json_dumps"] = lambda x: Markup(safe_jsonld(x))

def jsonld_safe(value):
    """Escape a string for safe use inside a JSON-LD string value.
    Handles newlines, tabs, backslashes, quotes — everything json.dumps does,
    but returns only the inner string (no outer quotes).
    Also prevents </script> breakout via <\\/ escaping."""
    if value is None:
        return ''
    s = str(value)
    # json.dumps adds outer quotes; strip them to get the escaped interior
    inner = json.dumps(s)[1:-1]
    # Prevent </script> injection when embedded in <script> blocks
    inner = inner.replace("</", "<\\/")
    return Markup(inner)

app.jinja_env.filters["jsonld_safe"] = jsonld_safe




# ---------------------------------------------------------------------------
# Dynamic SVG Badges (shields.io style) — for README backlinks
# ---------------------------------------------------------------------------

_badge_cache = {}
_badge_cache_ts = 0

def _get_badge_stats():
    """Get cached platform stats for badges."""
    global _badge_cache, _badge_cache_ts
    now = time.time()
    if now - _badge_cache_ts < 300:  # 5 min cache
        return _badge_cache
    db = get_db()
    videos = db.execute("SELECT COUNT(*) FROM videos").fetchone()[0]
    agents = db.execute("SELECT COUNT(*) FROM agents").fetchone()[0]
    views = db.execute("SELECT COALESCE(SUM(views), 0) FROM videos").fetchone()[0]
    humans = db.execute("SELECT COUNT(*) FROM agents WHERE is_human = 1").fetchone()[0]
    _badge_cache = {"videos": videos, "agents": agents, "views": views, "humans": humans}
    _badge_cache_ts = now
    return _badge_cache

def _make_badge_svg(label, value, color="#3ea6ff"):
    """Generate a shields.io-style SVG badge."""
    label_w = max(len(label) * 6.5 + 12, 40)
    value_w = max(len(str(value)) * 7 + 12, 30)
    total_w = label_w + value_w
    return f"""<svg xmlns="http://www.w3.org/2000/svg" width="{total_w}" height="20" role="img" aria-label="{label}: {value}">
  <title>{label}: {value}</title>
  <linearGradient id="s" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient>
  <clipPath id="r"><rect width="{total_w}" height="20" rx="3" fill="#fff"/></clipPath>
  <g clip-path="url(#r)">
    <rect width="{label_w}" height="20" fill="#555"/>
    <rect x="{label_w}" width="{value_w}" height="20" fill="{color}"/>
    <rect width="{total_w}" height="20" fill="url(#s)"/>
  </g>
  <g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="11">
    <text x="{label_w/2}" y="14" fill="#010101" fill-opacity=".3">{label}</text>
    <text x="{label_w/2}" y="13">{label}</text>
    <text x="{label_w + value_w/2}" y="14" fill="#010101" fill-opacity=".3">{value}</text>
    <text x="{label_w + value_w/2}" y="13">{value}</text>
  </g>
</svg>"""

def _format_count(n):
    if n >= 1000000: return f"{n/1000000:.1f}M"
    if n >= 1000: return f"{n/1000:.1f}K"
    return str(n)

@app.route("/badge/<badge_type>.svg")
def badge_svg(badge_type):
    """Dynamic SVG badge for READMEs. Types: videos, agents, views, humans, platform."""
    stats = _get_badge_stats()
    badges = {
        "videos": ("BoTTube videos", _format_count(stats["videos"]), "#3ea6ff"),
        "agents": ("BoTTube agents", str(stats["agents"]), "#9b59b6"),
        "views": ("BoTTube views", _format_count(stats["views"]), "#2ecc71"),
        "humans": ("BoTTube humans", str(stats["humans"]), "#e67e22"),
        "platform": ("powered by", "BoTTube", "#3ea6ff"),
        "bcos": ("BCOS", "certified", "#1a6b35"),
    }
    if badge_type not in badges:
        return Response("Not found", status=404)
    label, value, color = badges[badge_type]
    svg = _make_badge_svg(label, value, color)
    resp = Response(svg, mimetype="image/svg+xml")
    resp.headers["Cache-Control"] = "public, max-age=300"
    return resp

@app.route("/badge/agent/<agent_name>.svg")
def badge_agent_svg(agent_name):
    """Per-agent badge showing video count."""
    db = get_db()
    agent = db.execute("SELECT id, display_name FROM agents WHERE agent_name = ?", (agent_name,)).fetchone()
    if not agent:
        return Response("Agent not found", status=404)
    count = db.execute("SELECT COUNT(*) FROM videos WHERE agent_id = ?", (agent["id"],)).fetchone()[0]
    label = agent["display_name"] or agent_name
    svg = _make_badge_svg(label, f"{count} videos", "#3ea6ff")
    resp = Response(svg, mimetype="image/svg+xml")
    resp.headers["Cache-Control"] = "public, max-age=300"
    return resp


# ---------------------------------------------------------------------------
# "As Seen on BoTTube" branded badge
# ---------------------------------------------------------------------------

@app.route("/badge/seen-on-bottube.svg")
def seen_on_bottube_badge():
    """Branded 'As Seen on BoTTube' badge for websites and READMEs."""
    svg = """<svg xmlns="http://www.w3.org/2000/svg" width="180" height="28" role="img" aria-label="As seen on BoTTube">
  <title>As seen on BoTTube</title>
  <defs>
    <linearGradient id="bg" x1="0" y1="0" x2="1" y2="1">
      <stop offset="0%" stop-color="#1a1a2e"/>
      <stop offset="100%" stop-color="#16213e"/>
    </linearGradient>
  </defs>
  <rect width="180" height="28" rx="5" fill="url(#bg)"/>
  <rect x="1" y="1" width="178" height="26" rx="4" fill="none" stroke="#3ea6ff" stroke-width="0.5" opacity="0.5"/>
  <text x="10" y="18" font-family="Verdana,sans-serif" font-size="10" fill="#aaa">As seen on</text>
  <text x="78" y="18.5" font-family="Verdana,sans-serif" font-size="12" font-weight="bold" fill="#3ea6ff">BoTTube</text>
  <text x="135" y="18" font-family="Verdana,sans-serif" font-size="10" fill="#3ea6ff">&#9654;</text>
  <circle cx="164" cy="14" r="6" fill="#3ea6ff" opacity="0.15"/>
  <text x="161" y="17.5" font-family="Verdana,sans-serif" font-size="10" fill="#3ea6ff">.ai</text>
</svg>"""
    resp = Response(svg, mimetype="image/svg+xml")
    resp.headers["Cache-Control"] = "public, max-age=3600"
    return resp


# ---------------------------------------------------------------------------
# Phase 11.18: per-video verification badges
# ---------------------------------------------------------------------------
# Three artifacts so creators can broadcast on-chain provenance from their
# own sites without bottube cooperation beyond serving public read-only
# endpoints:
#   1. /badge/verified/<video_id>.svg   — pure SVG, drop-in <img src>
#   2. /embed/verify/<video_id>         — sandboxable HTML iframe widget
#   3. /static/bottube-verify.js        — cross-origin JS that turns
#       <div data-bottube-verify="<id>"> into a live pill on any page
#
# All three read the same /api/videos/<id>/provenance payload, so the
# verification status they show is canonical — no separate code path
# that could drift.

def _verify_badge_svg(state, label, value, color, value_color="#fff"):
    """Render a shields.io-style 2-tone badge for the verified state."""
    # Width math: 6px/char (Verdana) + 12px padding per side per panel
    pad = 10
    label_w = max(70, len(label) * 6 + pad * 2)
    value_w = max(80, len(value) * 6 + pad * 2)
    total_w = label_w + value_w
    # Truncate value if absurdly long (e.g. raw tx_hash)
    return (
        f'<svg xmlns="http://www.w3.org/2000/svg" width="{total_w}" '
        f'height="22" role="img" aria-label="{label}: {value}">'
        f'<title>{label}: {value}</title>'
        f'<linearGradient id="gloss" x2="0" y2="100%">'
        f'<stop offset="0" stop-color="#fff" stop-opacity=".15"/>'
        f'<stop offset="1" stop-opacity=".25"/></linearGradient>'
        f'<rect width="{total_w}" height="22" rx="3" fill="#222"/>'
        f'<rect x="{label_w}" width="{value_w}" height="22" rx="3" fill="{color}"/>'
        f'<rect x="{label_w-2}" width="4" height="22" fill="{color}"/>'
        f'<rect width="{total_w}" height="22" rx="3" fill="url(#gloss)"/>'
        f'<g fill="#fff" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" '
        f'font-size="11" text-rendering="geometricPrecision">'
        f'<text x="{label_w//2}" y="15" fill="#000" fill-opacity=".25" '
        f'text-anchor="middle">{label}</text>'
        f'<text x="{label_w//2}" y="14" text-anchor="middle">{label}</text>'
        f'<text x="{label_w + value_w//2}" y="15" fill="#000" fill-opacity=".25" '
        f'text-anchor="middle">{value}</text>'
        f'<text x="{label_w + value_w//2}" y="14" text-anchor="middle" '
        f'fill="{value_color}">{value}</text>'
        f'</g></svg>'
    )


def _verify_state_for_video(video_id):
    """Look up the verification state for one video. Returns
    (state, value_text, anchor_tx, manifest_version) where state is one of
    'verified', 'pending', 'failed', 'unknown'.

    Read-only, side-effect-free. Cached at the HTTP layer (the badge route
    sets Cache-Control: public, max-age=300).
    """
    if not re.fullmatch(r"[A-Za-z0-9_-]{5,32}", video_id or ""):
        return "unknown", "invalid id", "", 1
    try:
        _provenance_ensure_v2_columns()
    except Exception:
        pass
    db = get_db()
    try:
        row = db.execute(
            """SELECT COALESCE(anchor_tx_hash,'') AS tx,
                      COALESCE(anchor_block_height,0) AS h,
                      COALESCE(anchor_status,'pending') AS status,
                      COALESCE(anchor_chain,'') AS chain,
                      COALESCE(uploader_sig,'') AS sig,
                      COALESCE(manifest_version,1) AS v
                 FROM video_provenance
                WHERE video_id = ?""",
            (video_id,),
        ).fetchone()
    except Exception:
        return "unknown", "no provenance", "", 1
    if not row:
        return "unknown", "no provenance", "", 1

    tx = row["tx"]
    h = int(row["h"] or 0)
    v = int(row["v"] or 1)
    if not tx:
        return "pending", "pending", "", v
    if not h or row["chain"] == "stub":
        return "pending", "broadcasting", tx, v
    return "verified", f"block {h} · v{v}", tx, v


@app.route("/badge/verified/<video_id>.svg")
def badge_verified_svg(video_id):
    """Per-video provenance badge.

    Returns a 2-tone SVG ("verified on RustChain" / "block N · vM") that
    creators can drop on their own sites:

        <a href="https://bottube.ai/anchors/<tx>">
          <img src="https://bottube.ai/badge/verified/<id>.svg" alt="...">
        </a>

    Caching is set to 5 minutes — the verification state changes rarely
    once a row is anchored, and a 5-min stale-read is a fair trade for
    bounded server load.
    """
    state, value, _tx, _v = _verify_state_for_video(video_id)
    if state == "verified":
        color = "#0e7c2e"  # green
        label = "verified on rustchain"
    elif state == "pending":
        color = "#b8860b"  # amber
        label = "anchoring"
    elif state == "failed":
        color = "#a02020"  # red
        label = "verify failed"
    else:
        color = "#666"
        label = "bottube"
    svg = _verify_badge_svg(state, label, value, color)
    resp = Response(svg, mimetype="image/svg+xml")
    resp.headers["Cache-Control"] = "public, max-age=300"
    resp.headers["Access-Control-Allow-Origin"] = "*"
    return resp


@app.route("/embed/verify/<video_id>")
def embed_verify_widget(video_id):
    """Compact iframe-friendly verification widget.

    Used as <iframe src="/embed/verify/<id>" width=320 height=80 ...>.
    Renders identical state to the SVG badge but as styled HTML with a
    backlink to /anchors/<tx>. iframe-safe headers are already applied
    to /embed/* routes by the global response handler.
    """
    if not re.fullmatch(r"[A-Za-z0-9_-]{5,32}", video_id):
        return Response("invalid video id", status=400, mimetype="text/plain")
    state, value, tx, v = _verify_state_for_video(video_id)
    return render_template(
        "embed_verify.html",
        video_id=video_id, state=state, value=value, tx=tx,
        manifest_version=v,
    )


@app.route("/embed/bottube-verify.js")
def bottube_verify_js():
    """Drop-in JS that finds <div data-bottube-verify="<id>"> elements
    and replaces them with a live-fetched verification pill.

    Hosted under /embed/ (which already gets framing-permissive headers
    and falls outside Flask's /static/ handler) so creators can simply
    <script src="https://bottube.ai/embed/bottube-verify.js"></script>
    on any page. The body is invariant — host is overridden via
    data-host="<other-instance>" if the creator's site federates.
    """
    js = """/*! bottube-verify.js — public domain widget for on-chain provenance pills */
(function(){
  var HOST = (document.currentScript && document.currentScript.dataset && document.currentScript.dataset.host) || 'https://bottube.ai';
  function pill(state, txt, href){
    var a = document.createElement('a');
    a.href = href || (HOST + '/transparency');
    a.target = '_blank'; a.rel = 'noopener noreferrer';
    a.className = 'btv-pill btv-' + state;
    a.style.cssText = 'display:inline-flex;align-items:center;gap:6px;padding:3px 10px;border-radius:12px;font:600 12px ui-sans-serif,system-ui,-apple-system,Segoe UI,Roboto,sans-serif;text-decoration:none;border:1px solid;line-height:1.4;';
    var pal = state==='verified' ? ['#0e7c2e','#e8f7ec','#7ed091','✓']
            : state==='pending'  ? ['#7b5800','#fff5da','#e8c477','⧗']
            : state==='failed'   ? ['#a02020','#fde6e6','#e88080','✕']
                                 :  ['#444','#eee','#999','?'];
    a.style.color = pal[0]; a.style.background = pal[1]; a.style.borderColor = pal[2];
    a.textContent = pal[3] + ' ' + txt;
    a.title = 'BoTTube on-chain provenance';
    return a;
  }
  function render(vid, target){
    fetch(HOST + '/api/videos/' + encodeURIComponent(vid) + '/provenance', {credentials:'omit'})
      .then(function(r){ return r.json(); })
      .then(function(d){
        if (!d || !d.ok){
          target.replaceWith(pill('failed','no provenance', HOST + '/transparency'));
          return;
        }
        var v = d.manifest_version || 1;
        var anchor = d.anchor || {};
        var verified = !!d.verified;
        var state = verified ? 'verified' : (d.pill_state==='failed' ? 'failed' : 'pending');
        var txt;
        if (verified && anchor.block_height){
          txt = 'Verified on RustChain · block ' + anchor.block_height + ' · v' + v;
        } else if (verified){
          txt = 'Verified · v' + v;
        } else {
          txt = (d.pill_state || 'pending');
        }
        var href = anchor.tx_hash ? (HOST + '/anchors/' + anchor.tx_hash) : (HOST + '/transparency');
        target.replaceWith(pill(state, txt, href));
      })
      .catch(function(){
        target.replaceWith(pill('failed','fetch error', HOST + '/transparency'));
      });
  }
  function init(){
    var nodes = document.querySelectorAll('[data-bottube-verify]');
    for (var i=0;i<nodes.length;i++){
      var el = nodes[i];
      var vid = el.getAttribute('data-bottube-verify');
      if (vid) render(vid, el);
    }
  }
  if (document.readyState === 'loading') document.addEventListener('DOMContentLoaded', init);
  else init();
})();
"""
    resp = Response(js, mimetype="application/javascript; charset=utf-8")
    # 1 hour CDN-friendly cache; updates roll out within 60 minutes
    resp.headers["Cache-Control"] = "public, max-age=3600"
    resp.headers["Access-Control-Allow-Origin"] = "*"
    return resp


# ---------------------------------------------------------------------------
# Badges & Embed landing page
# ---------------------------------------------------------------------------

@app.route("/badges")
def badges_page():
    """Landing page for embeddable badges and widgets."""
    stats = _get_badge_stats()
    return render_template("badges.html", stats=stats)

@app.route("/embed-guide")
def embed_guide_page():
    """Landing page explaining how to embed BoTTube videos."""
    db = get_db()
    recent = db.execute(
        "SELECT v.video_id, v.title FROM videos v ORDER BY v.created_at DESC LIMIT 5"
    ).fetchall()
    return render_template("embed_guide.html", videos=recent)


@app.route("/beacon/atlas")
def beacon_atlas():
    """Interactive force-directed Beacon reputation graph visualization."""
    return render_template("beacon_atlas.html")


@app.route("/beacon")
def beacon_landing_page():
    return render_template("beacon.html")


# ---------------------------------------------------------------------------
# CTR / Thumbnail Analytics API
# ---------------------------------------------------------------------------

@app.route("/api/ctr/stats")
def ctr_global_stats():
    """Get global CTR statistics."""
    try:
        summary = _get_ctr_tracker().get_global_summary()
        return jsonify({"ok": True, **summary})
    except Exception as e:
        return jsonify({"ok": False, "error": str(e)}), 500


@app.route("/api/ctr/top")
def ctr_top_videos():
    """Get top videos by CTR."""
    limit = max(1, min(50, request.args.get("limit", 20, type=int)))
    min_imp = request.args.get("min_impressions", 10, type=int)
    try:
        top = _get_ctr_tracker().get_top_by_ctr(limit=limit, min_impressions=min_imp)
        return jsonify({"ok": True, "videos": top})
    except Exception as e:
        return jsonify({"ok": False, "error": str(e)}), 500


@app.route("/api/ctr/underperforming")
def ctr_underperforming():
    """Get videos with high impressions but low CTR."""
    try:
        videos = _get_ctr_tracker().get_underperforming()
        return jsonify({"ok": True, "videos": videos})
    except Exception as e:
        return jsonify({"ok": False, "error": str(e)}), 500


@app.route("/api/videos/<video_id>/ctr")
def video_ctr_stats(video_id):
    # Reject non-existent videos
    db = get_db()
    v = db.execute("SELECT 1 FROM videos WHERE video_id = ?", (video_id,)).fetchone()
    if not v:
        return jsonify({"error": "Video not found"}), 404

    try:
        stats = _get_ctr_tracker().get_stats(video_id)
        if not stats:
            return jsonify({"ok": True, "video_id": video_id, "impressions": 0, "clicks": 0, "ctr": 0})
        return jsonify({"ok": True, **stats})
    except Exception as e:
        return jsonify({"ok": False, "error": str(e)}), 500


@app.route("/api/videos/<video_id>/watch_time", methods=["POST"])
def record_watch_time(video_id):
    """Record watch time for a video (called by player on pause/close).

    Body: {"seconds": 12.5}
    """
    data = request.get_json(silent=True)
    if data is None:
        data = {}
    elif not isinstance(data, dict):
        return jsonify({"ok": False, "error": "JSON body must be an object"}), 400

    raw_seconds = data.get("seconds", 0)
    if raw_seconds is None or raw_seconds == "":
        seconds = 0.0
    else:
        try:
            seconds = float(raw_seconds)
        except (TypeError, ValueError):
            return jsonify({"ok": False, "error": "seconds must be a number"}), 400

    if not math.isfinite(seconds):
        return jsonify({"ok": False, "error": "seconds must be finite"}), 400
    if seconds < 0:
        return jsonify({"ok": False, "error": "seconds must be non-negative"}), 400

    db = get_db()
    video = db.execute(
        "SELECT 1 FROM videos WHERE video_id = ? AND COALESCE(is_removed, 0) = 0",
        (video_id,),
    ).fetchone()
    if not video:
        return jsonify({"error": "Video not found"}), 404

    try:
        if seconds > 0:
            _get_ctr_tracker().record_watch_time(video_id, seconds)
        return jsonify({"ok": True, "video_id": video_id, "seconds_recorded": seconds})
    except Exception as e:
        return jsonify({"ok": False, "error": str(e)}), 500


@app.route("/api/videos/<video_id>/ab/variants")
def video_ab_variants(video_id):
    # Reject non-existent videos
    db = get_db()
    v = db.execute("SELECT 1 FROM videos WHERE video_id = ?", (video_id,)).fetchone()
    if not v:
        return jsonify({"error": "Video not found"}), 404

    try:
        stats = _get_ab_manager().get_variant_stats(video_id)
        winner = _get_ab_manager().get_winner(video_id)
        return jsonify({"ok": True, "video_id": video_id, "variants": stats, "winner": winner})
    except Exception as e:
        return jsonify({"ok": False, "error": str(e)}), 500


# ---------------------------------------------------------------------------
# Engineering / Public Status Page
# ---------------------------------------------------------------------------
# Lightweight observability surface for the public /engineering page.
# Records request latencies in a thread-safe ring buffer, probes the four
# RustChain anchor nodes, and reads queue/state counts from the live DB.

from collections import deque as _eng_deque
from threading import Lock as _eng_Lock

# Two ring buffers: one for user-facing API/UI traffic, one for /admin/*
# backfill calls (each batch is 50-200 s). Mixing them turns the public
# p95 into reporting bias — better to surface admin latency as a separate
# truth panel on the engineering page (per Codex Phase 10 review).
_ENG_LATENCY = _eng_deque(maxlen=1000)         # user-facing samples (ms)
_ENG_LATENCY_LOCK = _eng_Lock()
_ENG_LATENCY_ADMIN = _eng_deque(maxlen=200)    # admin samples (ms)
_ENG_LATENCY_ADMIN_LOCK = _eng_Lock()

# Path prefixes excluded from latency sampling so static / streaming traffic
# doesn't dominate the histogram.
_ENG_LATENCY_SKIP = (
    "/static/", "/thumbnails/", "/avatars/", "/videos/", "/api/videos/",
    "/favicon", "/robots", "/sitemap", "/health", "/api/engineering",
    "/keyframes/", "/renditions/",
)

# Path prefixes that route to the *admin* ring buffer instead of the
# user-facing one. Admin work is intentionally slow (ffmpeg, embedding
# backfill) — call it out, don't pollute the public p95 with it.
_ENG_LATENCY_ADMIN_PREFIXES = ("/admin/",)


@app.before_request
def _eng_lat_start():
    g._eng_t0 = time.time()


@app.after_request
def _eng_lat_record(response):
    try:
        t0 = getattr(g, "_eng_t0", None)
        if t0 is None:
            return response
        path = request.path or ""
        # Skip media + static + self
        if any(path.startswith(p) for p in _ENG_LATENCY_SKIP):
            return response
        ms = (time.time() - t0) * 1000.0
        # Cap absurd values (request still in-flight tail) to keep histogram sane
        if ms >= 600000:  # 10-min hard ceiling for admin too
            return response
        if any(path.startswith(p) for p in _ENG_LATENCY_ADMIN_PREFIXES):
            with _ENG_LATENCY_ADMIN_LOCK:
                _ENG_LATENCY_ADMIN.append(ms)
        elif ms < 30000:
            with _ENG_LATENCY_LOCK:
                _ENG_LATENCY.append(ms)
    except Exception:
        pass
    return response


# Static node config — keep in code so the page stays honest even if a
# config DB is unavailable. RTTs are probed at request time.
_ENG_RUSTCHAIN_NODES = [
    {"id": "node-1",  "host": "50.28.86.131", "port": 443,  "scheme": "https", "location": "LiquidWeb US (primary)"},
    {"id": "node-2",  "host": "50.28.86.153", "port": 443,  "scheme": "https", "location": "LiquidWeb US (Ergo anchor)"},
    {"id": "node-3",  "host": "76.8.228.245", "port": 8099, "scheme": "http",  "location": "Ryan's Proxmox (Texas)"},
    {"id": "node-4",  "host": "38.76.217.189","port": 8099, "scheme": "http",  "location": "CognetCloud Hong Kong"},
]


def _eng_probe_node(node, timeout=2.0):
    """Probe a RustChain node /health endpoint. Returns dict with status, rtt_ms."""
    url = f"{node['scheme']}://{node['host']}:{node['port']}/health"
    t0 = time.time()
    try:
        # urllib instead of requests to avoid adding a dependency
        ctx = None
        if node["scheme"] == "https":
            import ssl as _ssl
            ctx = _ssl.create_default_context()
            ctx.check_hostname = False
            ctx.verify_mode = _ssl.CERT_NONE
        req = urllib.request.Request(url, headers={"User-Agent": "BoTTube-Engineering"})
        if ctx is not None:
            urllib.request.urlopen(req, timeout=timeout, context=ctx).read(256)
        else:
            urllib.request.urlopen(req, timeout=timeout).read(256)
        rtt = (time.time() - t0) * 1000.0
        if rtt < 800:
            status = "ok"
        elif rtt < 2500:
            status = "warn"
        else:
            status = "err"
        return {**node, "status": status, "rtt_ms": rtt}
    except Exception:
        rtt = (time.time() - t0) * 1000.0
        return {**node, "status": "err", "rtt_ms": rtt}


def _eng_percentile(values, pct):
    if not values:
        return 0.0
    s = sorted(values)
    k = max(0, min(len(s) - 1, int(round((pct / 100.0) * (len(s) - 1)))))
    return s[k]


def _eng_format_uptime(seconds):
    seconds = int(max(0, seconds))
    days, rem = divmod(seconds, 86400)
    hours, rem = divmod(rem, 3600)
    mins, _ = divmod(rem, 60)
    if days:
        return f"{days}d {hours}h"
    if hours:
        return f"{hours}h {mins}m"
    return f"{mins}m"


def _eng_collect():
    """Collect everything the /engineering page renders."""
    # Ensure provenance/rendition tables exist so counts are honest.
    try:
        _ensure_provenance_schema()
    except Exception:
        pass
    db = get_db()

    # Latency snapshots — two buckets: user-facing API/UI and /admin/*.
    with _ENG_LATENCY_LOCK:
        samples = list(_ENG_LATENCY)
    latency = {
        "samples": len(samples),
        "p50": _eng_percentile(samples, 50),
        "p95": _eng_percentile(samples, 95),
        "p99": _eng_percentile(samples, 99),
    }
    latency["p95_bar"] = max(2.0, min(100.0, (latency["p95"] / 500.0) * 100.0))

    with _ENG_LATENCY_ADMIN_LOCK:
        admin_samples = list(_ENG_LATENCY_ADMIN)
    latency_admin = {
        "samples": len(admin_samples),
        "p50": _eng_percentile(admin_samples, 50),
        "p95": _eng_percentile(admin_samples, 95),
        "p99": _eng_percentile(admin_samples, 99),
    }
    # Admin ceiling is 60 s — backfill batches are intentionally slow.
    latency_admin["p95_bar"] = max(2.0, min(100.0, (latency_admin["p95"] / 60000.0) * 100.0))

    # Node probes (parallel via threads)
    import concurrent.futures as _cf
    nodes = []
    with _cf.ThreadPoolExecutor(max_workers=4) as ex:
        for n in ex.map(_eng_probe_node, _ENG_RUSTCHAIN_NODES):
            nodes.append(n)

    # Platform state
    def _scalar(sql, default=0):
        try:
            row = db.execute(sql).fetchone()
            return int(row[0]) if row and row[0] is not None else default
        except Exception:
            return default

    state = {
        "videos": _scalar("SELECT COUNT(*) FROM videos WHERE COALESCE(is_removed,0)=0"),
        "agents": _scalar("SELECT COUNT(*) FROM agents WHERE is_human=0"),
        "humans": _scalar("SELECT COUNT(*) FROM agents WHERE is_human=1"),
        "comments": _scalar("SELECT COUNT(*) FROM comments"),
        "tips_confirmed": _scalar("SELECT COUNT(*) FROM tips WHERE COALESCE(status,'confirmed')='confirmed'"),
        "uptime": _eng_format_uptime(time.time() - APP_START_TS),
        "version": APP_VERSION,
    }

    # Generation queue (gpu_jobs may not exist on all installs)
    queue = {"queued": 0, "running": 0, "completed_24h": 0, "failed_24h": 0, "depth_label": "—"}
    try:
        cutoff = time.time() - 86400
        queue["queued"] = _scalar("SELECT COUNT(*) FROM gpu_jobs WHERE status='queued'")
        queue["running"] = _scalar("SELECT COUNT(*) FROM gpu_jobs WHERE status='running'")
        queue["completed_24h"] = _scalar(f"SELECT COUNT(*) FROM gpu_jobs WHERE status='completed' AND COALESCE(completed_at,0)>{cutoff}")
        queue["failed_24h"] = _scalar(f"SELECT COUNT(*) FROM gpu_jobs WHERE status='failed' AND COALESCE(completed_at,0)>{cutoff}")
        depth = queue["queued"] + queue["running"]
        if depth == 0:
            queue["depth_label"] = "idle"
        elif depth < 5:
            queue["depth_label"] = "shallow"
        elif depth < 20:
            queue["depth_label"] = "moderate"
        else:
            queue["depth_label"] = "deep"
    except Exception:
        pass

    # Active experiments — best effort from ab_test tables (exposed by ABTestManager)
    experiments = []
    try:
        rows = db.execute(
            """SELECT video_id,
                      COUNT(DISTINCT variant_key) AS variants,
                      COUNT(*) AS impressions
                 FROM variant_impressions
                WHERE created_at > ?
                GROUP BY video_id
                HAVING variants >= 2
                ORDER BY impressions DESC
                LIMIT 5""",
            (time.time() - 7 * 86400,),
        ).fetchall()
        for r in rows:
            experiments.append({
                "name": f"thumb-ab:{r['video_id'][:8]}",
                "variants": r["variants"],
                "impressions": r["impressions"],
            })
    except Exception:
        pass

    # Media pipeline summary
    pipeline = {
        "renditions": _scalar("SELECT COUNT(*) FROM video_renditions") if _eng_table_exists(db, "video_renditions") else "coming soon",
        "with_provenance": _scalar("SELECT COUNT(*) FROM video_provenance") if _eng_table_exists(db, "video_provenance") else 0,
        "anchored": _scalar("SELECT COUNT(*) FROM video_provenance WHERE anchor_tx_hash != ''") if _eng_table_exists(db, "video_provenance") else 0,
        "avg_encode_s": "—",
    }

    # Phase 10.3: bucket outcomes (CTR + mean watch seconds) over last 7 days
    try:
        outcomes = _feed_imp_outcomes(window_hours=168)
    except Exception:
        outcomes = {}

    return {
        "nodes": nodes,
        "latency": latency,
        "latency_admin": latency_admin,
        "state": state,
        "queue": queue,
        "experiments": experiments,
        "outcomes": outcomes,
        "pipeline": pipeline,
        "generated_at": datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC"),
    }


def _eng_table_exists(db, name):
    try:
        row = db.execute(
            "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (name,)
        ).fetchone()
        return bool(row)
    except Exception:
        return False


@app.route("/engineering")
def engineering_page():
    """Public engineering status page — visible operational maturity."""
    try:
        ctx = _eng_collect()
    except Exception as e:
        # Never let this page 500 — it's the operational visibility page.
        ctx = {
            "nodes": [], "latency": {"p50": 0, "p95": 0, "p99": 0, "samples": 0, "p95_bar": 0},
            "latency_admin": {"p50": 0, "p95": 0, "p99": 0, "samples": 0, "p95_bar": 0},
            "state": {"videos": 0, "agents": 0, "humans": 0, "comments": 0, "tips_confirmed": 0,
                      "uptime": "?", "version": APP_VERSION},
            "queue": {"queued": 0, "running": 0, "completed_24h": 0, "failed_24h": 0, "depth_label": f"err: {e}"},
            "experiments": [], "outcomes": {}, "pipeline": {"renditions": 0, "with_provenance": 0, "anchored": 0, "avg_encode_s": "?"},
            "generated_at": "error",
        }
    return render_template("engineering.html", **ctx)


@app.route("/api/engineering")
def engineering_api():
    """JSON variant of /engineering for live refresh."""
    try:
        return jsonify({"ok": True, **_eng_collect()})
    except Exception as e:
        return jsonify({"ok": False, "error": str(e)}), 500


# ---------------------------------------------------------------------------
# Provenance — surface on-chain video attestation per Codex spec
# ---------------------------------------------------------------------------
# Schema and API for first-class video provenance. Init done lazily on
# first request so we don't have to touch init_db() during a hot deploy.

_PROVENANCE_SCHEMA_READY = False
_PROVENANCE_SCHEMA_LOCK = _eng_Lock()


def _ensure_provenance_schema():
    global _PROVENANCE_SCHEMA_READY
    if _PROVENANCE_SCHEMA_READY:
        return
    with _PROVENANCE_SCHEMA_LOCK:
        if _PROVENANCE_SCHEMA_READY:
            return
        conn = sqlite3.connect(str(DB_PATH))
        try:
            conn.executescript(
                """
                CREATE TABLE IF NOT EXISTS video_provenance (
                    video_id              TEXT PRIMARY KEY,
                    canonical_sha256      TEXT DEFAULT '',
                    duration_sec          REAL DEFAULT 0,
                    width                 INTEGER DEFAULT 0,
                    height                INTEGER DEFAULT 0,
                    creator_agent_id      INTEGER DEFAULT 0,
                    creator_pubkey        TEXT DEFAULT '',
                    creator_beacon_id     TEXT DEFAULT '',
                    model                 TEXT DEFAULT '',
                    provider              TEXT DEFAULT '',
                    workflow_hash         TEXT DEFAULT '',
                    prompt_hash           TEXT DEFAULT '',
                    seed                  INTEGER DEFAULT 0,
                    generated_at          REAL DEFAULT 0,
                    uploader_sig          TEXT DEFAULT '',
                    uploaded_at           REAL DEFAULT 0,
                    anchor_chain          TEXT DEFAULT '',
                    anchor_tx_hash        TEXT DEFAULT '',
                    anchor_block_height   INTEGER DEFAULT 0,
                    anchor_manifest_hash  TEXT DEFAULT '',
                    parents_json          TEXT DEFAULT '[]',
                    renditions_json       TEXT DEFAULT '[]',
                    verified              INTEGER DEFAULT 0,
                    created_at            REAL DEFAULT 0,
                    updated_at            REAL DEFAULT 0
                );
                CREATE INDEX IF NOT EXISTS idx_provenance_anchor
                    ON video_provenance(anchor_tx_hash);
                CREATE INDEX IF NOT EXISTS idx_provenance_creator
                    ON video_provenance(creator_agent_id);
                CREATE TABLE IF NOT EXISTS video_renditions (
                    id            INTEGER PRIMARY KEY AUTOINCREMENT,
                    video_id      TEXT NOT NULL,
                    label         TEXT NOT NULL,        -- e.g. 'canonical', '720p', '360p'
                    url_path      TEXT NOT NULL,
                    width         INTEGER DEFAULT 0,
                    height        INTEGER DEFAULT 0,
                    bitrate_kbps  INTEGER DEFAULT 0,
                    codec         TEXT DEFAULT 'h264',
                    file_sha256   TEXT DEFAULT '',
                    file_size     INTEGER DEFAULT 0,
                    vmaf          REAL DEFAULT 0,
                    is_canonical  INTEGER DEFAULT 0,
                    created_at    REAL DEFAULT 0,
                    UNIQUE(video_id, label)
                );
                CREATE INDEX IF NOT EXISTS idx_renditions_video
                    ON video_renditions(video_id);
                """
            )
            conn.commit()
        finally:
            conn.close()
        _PROVENANCE_SCHEMA_READY = True


def _build_provenance_payload(video_row, prov_row, renditions):
    """Build the public provenance JSON for a video, filling defaults from videos table."""
    vid = video_row["video_id"]
    # canonical asset = either provenance row (preferred) or what we know from videos
    canonical_sha = (prov_row["canonical_sha256"] if prov_row else "") or ""
    duration = (prov_row["duration_sec"] if prov_row else 0) or video_row["duration_sec"] or 0
    width = (prov_row["width"] if prov_row else 0) or video_row["width"] or 720
    height = (prov_row["height"] if prov_row else 0) or video_row["height"] or 720

    creator = {
        "agent_id": (prov_row["creator_agent_id"] if prov_row else 0) or video_row["agent_id"],
        "agent_name": video_row["agent_name"],
        "display_name": video_row["display_name"] or video_row["agent_name"],
        "pubkey": (prov_row["creator_pubkey"] if prov_row else "") or "",
        "beacon_id": (prov_row["creator_beacon_id"] if prov_row else "") or "",
    }
    generation = {
        "model": (prov_row["model"] if prov_row else "") or "",
        "provider": (prov_row["provider"] if prov_row else "") or "",
        "workflow_hash": (prov_row["workflow_hash"] if prov_row else "") or "",
        "prompt_hash": (prov_row["prompt_hash"] if prov_row else "") or "",
        "seed": (prov_row["seed"] if prov_row else 0) or 0,
        "generated_at": (prov_row["generated_at"] if prov_row else 0) or 0,
    }
    upload = {
        "uploader_sig": (prov_row["uploader_sig"] if prov_row else "") or "",
        "uploaded_at": (prov_row["uploaded_at"] if prov_row else 0) or video_row["created_at"] or 0,
    }
    anchor = {
        "chain": (prov_row["anchor_chain"] if prov_row else "") or "",
        "tx_hash": (prov_row["anchor_tx_hash"] if prov_row else "") or "",
        "block_height": (prov_row["anchor_block_height"] if prov_row else 0) or 0,
        "manifest_hash": (prov_row["anchor_manifest_hash"] if prov_row else "") or "",
    }

    parents = []
    try:
        parents = json.loads(prov_row["parents_json"]) if prov_row else []
    except Exception:
        parents = []

    rends = []
    for r in renditions or []:
        rends.append({
            "label": r["label"],
            "url": r["url_path"],
            "width": r["width"],
            "height": r["height"],
            "bitrate_kbps": r["bitrate_kbps"],
            "codec": r["codec"],
            "file_sha256": r["file_sha256"],
            "file_size": r["file_size"],
            "vmaf": r["vmaf"],
            "is_canonical": bool(r["is_canonical"]),
        })

    # Verified iff anchor.tx_hash present AND uploader_sig present
    verified = bool(anchor["tx_hash"]) and bool(upload["uploader_sig"])
    if prov_row and prov_row["verified"]:
        verified = True

    # Decide an overall pill state
    if verified:
        pill_state = "verified"
    elif anchor["tx_hash"] or upload["uploader_sig"]:
        pill_state = "pending"
    else:
        pill_state = "unverified"

    # Phase 11.12: optional thumbnail integrity hash (additive, not in anchor leaf).
    # Phase 11.16: surface manifest_version so verifiers know which leaf
    # recipe applies, and the canonical_360p_sha256 if present.
    thumb_sha = ""
    p360_sha = ""
    manifest_ver = 1
    try:
        if prov_row is not None:
            row_keys = prov_row.keys()
            if "thumbnail_sha256" in row_keys:
                thumb_sha = (prov_row["thumbnail_sha256"] or "")
            if "canonical_360p_sha256" in row_keys:
                p360_sha = (prov_row["canonical_360p_sha256"] or "")
            if "manifest_version" in row_keys:
                manifest_ver = int(prov_row["manifest_version"] or 1)
    except Exception:
        pass

    in_leaf_v2 = manifest_ver >= 2

    # Phase 11.24: also expose thumbnail URL + 360p rendition URL so the
    # verifier can optionally re-hash them (--check-asset on v2 rows).
    # Pull thumbnail filename from video_row (already joined upstream).
    thumb_url = ""
    try:
        thumb_filename = video_row["thumbnail"] if "thumbnail" in video_row.keys() else ""
        if thumb_filename:
            thumb_url = f"/thumbnails/{thumb_filename}"
    except Exception:
        thumb_url = ""
    p360_url = f"/renditions/{vid}/360p.mp4" if p360_sha else ""

    return {
        "video_id": vid,
        "manifest_version": manifest_ver,
        "pill_state": pill_state,
        "verified": verified,
        "canonical_asset": {
            "sha256": canonical_sha,
            "duration": duration,
            "width": width,
            "height": height,
            # Phase 11.21: surface the asset URL so the verifier can
            # optionally re-hash the bytes bottube serves today and prove
            # they still match the anchored hash.
            "url": f"/api/videos/{vid}/stream",
        },
        "thumbnail": {
            "sha256": thumb_sha,
            "url": thumb_url,
            "in_anchor_leaf": in_leaf_v2 and bool(thumb_sha),
            "note": (
                "Hashed at upload time and folded into the anchored Merkle leaf "
                "starting in manifest v2."
                if in_leaf_v2 and thumb_sha else
                "Hashed at upload time and persisted, but not part of the "
                "anchored Merkle leaf in this manifest version. "
                "v1 anchors only commit canonical_sha256."
            ),
        },
        "canonical_360p": {
            "sha256": p360_sha,
            "url": p360_url,
            "in_anchor_leaf": in_leaf_v2 and bool(p360_sha),
        },
        "renditions": rends,
        "creator": creator,
        "generation": generation,
        "upload": upload,
        "anchor": anchor,
        "parents": parents,
    }


@app.route("/api/videos/<video_id>/provenance")
def video_provenance(video_id):
    """Public provenance JSON for a video — first-class platform primitive."""
    _ensure_provenance_schema()
    db = get_db()
    video = db.execute(
        """SELECT v.video_id, v.agent_id, v.duration_sec, v.width, v.height, v.created_at,
                  v.thumbnail,
                  a.agent_name, a.display_name
             FROM videos v JOIN agents a ON v.agent_id = a.id
            WHERE v.video_id = ?""",
        (video_id,),
    ).fetchone()
    if not video:
        return jsonify({"ok": False, "error": "video not found"}), 404

    prov_row = db.execute(
        "SELECT * FROM video_provenance WHERE video_id = ?", (video_id,)
    ).fetchone()
    rendition_rows = db.execute(
        """SELECT label, url_path, width, height, bitrate_kbps, codec,
                  file_sha256, file_size, vmaf, is_canonical
             FROM video_renditions WHERE video_id = ?
            ORDER BY is_canonical DESC, width DESC""",
        (video_id,),
    ).fetchall()

    payload = _build_provenance_payload(video, prov_row, rendition_rows)
    return jsonify({"ok": True, **payload})


# ---------------------------------------------------------------------------
# Lifecycle Timeline — generated → uploaded → anchored → rewarded
# ---------------------------------------------------------------------------

@app.route("/api/videos/<video_id>/lifecycle")
def video_lifecycle(video_id):
    """Public lifecycle of a video: 4 milestone events with timestamps.

    Used by the cinematic player widget to render the on-chain lineage
    of an AI-generated video, tying provenance + reward + anchor data
    into a single visible timeline.
    """
    _ensure_provenance_schema()
    db = get_db()
    video = db.execute(
        "SELECT video_id, created_at FROM videos WHERE video_id = ?",
        (video_id,),
    ).fetchone()
    if not video:
        return jsonify({"ok": False, "error": "video not found"}), 404

    prov = db.execute(
        """SELECT generated_at, uploaded_at, anchor_chain, anchor_tx_hash,
                  anchor_block_height, anchor_manifest_hash
             FROM video_provenance WHERE video_id = ?""",
        (video_id,),
    ).fetchone()

    # Earliest reward (earnings + tips, if any) for this video.
    reward_at = 0.0
    reward_kind = ""
    try:
        row = db.execute(
            """SELECT MIN(created_at) AS at FROM earnings WHERE video_id = ?""",
            (video_id,),
        ).fetchone()
        if row and row["at"]:
            reward_at = float(row["at"])
            reward_kind = "earnings"
    except Exception:
        pass
    if not reward_at:
        try:
            row = db.execute(
                """SELECT MIN(created_at) AS at FROM tips
                    WHERE video_id = ? AND COALESCE(status,'confirmed')='confirmed'""",
                (video_id,),
            ).fetchone()
            if row and row["at"]:
                reward_at = float(row["at"])
                reward_kind = "tip"
        except Exception:
            pass

    uploaded_at = (prov["uploaded_at"] if prov and prov["uploaded_at"] else None) or video["created_at"] or 0
    generated_at = (prov["generated_at"] if prov and prov["generated_at"] else None)
    if not generated_at:
        # Best-guess: generation precedes upload by a few seconds.
        # We mark this as inferred so the UI can render it lighter.
        generated_at = max(0.0, float(uploaded_at) - 5.0)
        generated_inferred = True
    else:
        generated_inferred = False

    anchor_at = 0.0
    anchor_inferred = False
    if prov and prov["anchor_tx_hash"]:
        # We don't store anchor_at separately yet; approximate as uploaded_at + 60s
        # so the tick lands meaningfully on the timeline. UI should mark inferred.
        anchor_at = float(uploaded_at) + 60.0
        anchor_inferred = True

    events = [
        {
            "key": "generated",
            "label": "Generated",
            "icon": "✦",
            "at": float(generated_at) if generated_at else 0.0,
            "present": bool(generated_at),
            "inferred": generated_inferred,
            "detail": "AI canonical artifact created" + (" (inferred from upload time)" if generated_inferred else ""),
        },
        {
            "key": "uploaded",
            "label": "Uploaded",
            "icon": "↑",
            "at": float(uploaded_at) if uploaded_at else 0.0,
            "present": bool(uploaded_at),
            "inferred": False,
            "detail": "Manifest signed and pushed to BoTTube",
        },
        {
            "key": "anchored",
            "label": "Anchored",
            "icon": "⛓",
            "at": anchor_at,
            "present": bool(prov and prov["anchor_tx_hash"]),
            "inferred": anchor_inferred,
            "detail": (f"RustChain block {prov['anchor_block_height']}"
                       if prov and prov["anchor_block_height"] else
                       "Awaiting RustChain anchor"),
            "tx_hash": prov["anchor_tx_hash"] if prov else "",
            "chain": prov["anchor_chain"] if prov else "",
        },
        {
            "key": "rewarded",
            "label": "Rewarded",
            "icon": "★",
            "at": reward_at,
            "present": bool(reward_at),
            "inferred": False,
            "detail": (f"First {reward_kind} received" if reward_at
                       else "No reward issued yet"),
        },
    ]

    # Compute relative positions on the timeline (0..1)
    timestamps = [e["at"] for e in events if e["at"] > 0]
    t_min = min(timestamps) if timestamps else 0.0
    t_max = max(timestamps) if timestamps else 0.0
    span = max(1.0, t_max - t_min)
    for e in events:
        if e["at"] > 0:
            e["rel"] = round((e["at"] - t_min) / span, 4)
        else:
            e["rel"] = None

    return jsonify({
        "ok": True,
        "video_id": video_id,
        "events": events,
        "t_min": t_min,
        "t_max": t_max,
        "span_s": round(span, 2),
    })


# ---------------------------------------------------------------------------
# Keyframe sprites — auto-extract for the cinematic scrub strip
# ---------------------------------------------------------------------------

KEYFRAME_DIR = BASE_DIR / "keyframes"
KEYFRAME_COUNT = 6
KEYFRAME_SIZE = 120  # pixels per frame, square

# Module-level lock per-video to avoid concurrent ffmpeg storms on cache-miss.
_KEYFRAME_LOCKS = {}
_KEYFRAME_LOCKS_LOCK = _eng_Lock()


def _keyframe_lock_for(video_id):
    with _KEYFRAME_LOCKS_LOCK:
        lk = _KEYFRAME_LOCKS.get(video_id)
        if lk is None:
            lk = _eng_Lock()
            _KEYFRAME_LOCKS[video_id] = lk
        return lk


def _generate_keyframe_sprite(video_path, sprite_path, frames=KEYFRAME_COUNT, size=KEYFRAME_SIZE):
    """Run ffmpeg to produce a horizontal sprite of N keyframes.

    Uses select=eq(n,...) instead of relying on i-frames so even GOP-locked
    AI output gets sampled at predictable positions.
    """
    KEYFRAME_DIR.mkdir(parents=True, exist_ok=True)
    # Get duration first (cheap, ffprobe)
    try:
        out = subprocess.check_output(
            ["ffprobe", "-v", "error", "-show_entries", "format=duration",
             "-of", "default=noprint_wrappers=1:nokey=1", str(video_path)],
            stderr=subprocess.DEVNULL, timeout=10,
        )
        duration = float(out.strip() or 0)
    except Exception:
        duration = 8.0

    if duration <= 0:
        duration = 8.0

    # Sample frames at evenly spaced timestamps (slight inset so we don't grab black tail).
    inset = max(0.05, duration * 0.02)
    step = (duration - 2 * inset) / max(1, frames - 1)
    times = [inset + i * step for i in range(frames)]

    # Build a select expression: gte(t,T0)*lte(t,T0+0.05) OR ... with reset(N=PI) trick
    # Simpler approach: extract each frame to a temp file and tile manually with -filter_complex.
    # We'll use the more efficient approach: -ss + -frames:v 1 per frame, then montage via ffmpeg tile.
    import tempfile
    with tempfile.TemporaryDirectory(prefix="bt_kf_") as tmp:
        tmp_path = Path(tmp)
        frame_paths = []
        for i, t in enumerate(times):
            fp = tmp_path / f"f{i:02d}.jpg"
            subprocess.run(
                ["ffmpeg", "-loglevel", "error", "-ss", f"{t:.3f}", "-i", str(video_path),
                 "-frames:v", "1", "-vf", f"scale={size}:{size}:force_original_aspect_ratio=increase,crop={size}:{size}",
                 "-q:v", "5", "-y", str(fp)],
                check=True, timeout=15, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
            )
            frame_paths.append(str(fp))

        # Tile horizontally
        # Build -i for each frame, then xstack-style hstack
        cmd = ["ffmpeg", "-loglevel", "error"]
        for fp in frame_paths:
            cmd += ["-i", fp]
        filter_inputs = "".join(f"[{i}:v]" for i in range(len(frame_paths)))
        cmd += ["-filter_complex", f"{filter_inputs}hstack=inputs={len(frame_paths)}",
                "-q:v", "5", "-y", str(sprite_path)]
        subprocess.run(cmd, check=True, timeout=20,
                       stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)

    return {"frames": frames, "frame_size": size, "duration": duration}


@app.route("/api/videos/<video_id>/keyframes")
def video_keyframes(video_id):
    """Return keyframe sprite manifest. Generates on first hit, then cached.

    The sprite is a horizontal strip of N square thumbnails. The watch
    page renders this above the native player and uses CSS transform to
    show the active frame as the playhead moves.
    """
    db = get_db()
    video = db.execute(
        "SELECT video_id, filename, duration_sec FROM videos WHERE video_id = ?",
        (video_id,),
    ).fetchone()
    if not video:
        return jsonify({"ok": False, "error": "video not found"}), 404

    sprite_filename = f"{video_id}.jpg"
    sprite_path = KEYFRAME_DIR / sprite_filename
    duration = video["duration_sec"] or 8.0

    if not sprite_path.exists() or sprite_path.stat().st_size < 256:
        video_path = VIDEO_DIR / (video["filename"] or "")
        if not video_path.exists():
            return jsonify({"ok": False, "error": "video file missing"}), 404
        lock = _keyframe_lock_for(video_id)
        with lock:
            if not sprite_path.exists() or sprite_path.stat().st_size < 256:
                try:
                    _generate_keyframe_sprite(video_path, sprite_path)
                except subprocess.CalledProcessError as e:
                    return jsonify({"ok": False, "error": f"ffmpeg failed: {e.returncode}"}), 500
                except subprocess.TimeoutExpired:
                    return jsonify({"ok": False, "error": "ffmpeg timeout"}), 504
                except Exception as e:
                    return jsonify({"ok": False, "error": str(e)}), 500

    return jsonify({
        "ok": True,
        "video_id": video_id,
        "sprite_url": f"/keyframes/{sprite_filename}",
        "frame_count": KEYFRAME_COUNT,
        "frame_width": KEYFRAME_SIZE,
        "frame_height": KEYFRAME_SIZE,
        "duration": duration,
    })


# ---------------------------------------------------------------------------
# Phase 10.4: MCP-compatible tool surface (read-only)
# ---------------------------------------------------------------------------
# Per Codex: "If you only ship the descriptor and not the bridge, do not
# market that as MCP-compatible." Below: the descriptor at
# /.well-known/mcp.json plus a JSON-RPC-shaped /mcp bridge that wraps the
# existing REST endpoints. Five tools: feed.get, video.get, video.similar,
# video.provenance, video.keyframes. No write tools, no auth — read-only
# is honest about what's actually safe to expose to a discovering agent.

MCP_VERSION = "1.0"
MCP_TOOLS = [
    {
        "name": "feed.get",
        "description": (
            "Return ranked videos from a BoTTube feed bucket. "
            "Use bucket=hybrid-v1 for embedding-based recommendations, "
            "bucket=heuristic for popularity-only, bucket=latest for chronological. "
            "Each result includes _why (recommendation explanation), _score, "
            "and _components (signal breakdown for hybrid-v1)."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "bucket": {
                    "type": "string",
                    "enum": ["latest", "heuristic", "hybrid-v1", "auto"],
                    "default": "auto",
                },
                "per_page": {"type": "integer", "minimum": 1, "maximum": 50, "default": 20},
                "page": {"type": "integer", "minimum": 1, "default": 1},
                "category": {"type": "string"},
            },
        },
        "rest_path": "/api/feed",
        "rest_method": "GET",
    },
    {
        "name": "video.get",
        "description": (
            "Look up a single video by id. Returns title, description, tags, "
            "creator agent, view count, and other metadata."
        ),
        "input_schema": {
            "type": "object",
            "required": ["video_id"],
            "properties": {
                "video_id": {"type": "string", "pattern": "^[A-Za-z0-9_-]{5,32}$"},
            },
        },
        "rest_path": "/api/videos/{video_id}",
        "rest_method": "GET",
    },
    {
        "name": "video.similar",
        "description": (
            "Top-K cosine-similar videos to a given video, computed against the "
            "in-memory text-embedding cache (Gemini gemini-embedding-2, 3072-d). "
            "Vectors are L2-normalized so the score is direct cosine in [-1, 1]."
        ),
        "input_schema": {
            "type": "object",
            "required": ["video_id"],
            "properties": {
                "video_id": {"type": "string", "pattern": "^[A-Za-z0-9_-]{5,32}$"},
                "k": {"type": "integer", "minimum": 1, "maximum": 50, "default": 10},
            },
        },
        "rest_path": "/api/videos/{video_id}/similar",
        "rest_method": "GET",
    },
    {
        "name": "video.provenance",
        "description": (
            "Cryptographic provenance for a video: canonical asset SHA-256, "
            "creator agent identity, generation model + prompt hash + seed, "
            "uploader signature (HMAC-SHA256), RustChain anchor TX (when "
            "anchored). Three states: verified | pending | unverified."
        ),
        "input_schema": {
            "type": "object",
            "required": ["video_id"],
            "properties": {
                "video_id": {"type": "string", "pattern": "^[A-Za-z0-9_-]{5,32}$"},
            },
        },
        "rest_path": "/api/videos/{video_id}/provenance",
        "rest_method": "GET",
    },
    {
        "name": "video.keyframes",
        "description": (
            "6-frame keyframe sprite for a video, auto-extracted via ffmpeg. "
            "Returns the sprite URL, frame_count, frame_width, frame_height, "
            "and duration. Useful for client-side scrub bars or scene preview."
        ),
        "input_schema": {
            "type": "object",
            "required": ["video_id"],
            "properties": {
                "video_id": {"type": "string", "pattern": "^[A-Za-z0-9_-]{5,32}$"},
            },
        },
        "rest_path": "/api/videos/{video_id}/keyframes",
        "rest_method": "GET",
    },
]


def _mcp_descriptor():
    """Public MCP descriptor — what the server is and what tools it offers."""
    return {
        "schema_version": MCP_VERSION,
        "name": "bottube",
        "title": "BoTTube — AI-native video platform",
        "description": (
            "Read-only MCP surface over BoTTube. Agents can browse the "
            "ranked feed, fetch video metadata + provenance, and look up "
            "semantically similar videos. Write tools (upload, comment, vote) "
            "are gated by the existing /api/register + /api/agents/me/accept-terms "
            "human-side flow and are intentionally not exposed here."
        ),
        "homepage": "https://bottube.ai",
        "documentation": "https://bottube.ai/docs",
        "engineering": "https://bottube.ai/engineering",
        "endpoints": {
            "rpc": "https://bottube.ai/mcp",
            "rest_base": "https://bottube.ai/api",
        },
        "auth": {"type": "none", "note": "Read-only tools require no key."},
        "tools": [
            {k: v for k, v in tool.items() if k != "rest_path" and k != "rest_method"}
            for tool in MCP_TOOLS
        ],
        "license": "MIT",
    }


# --- Phase 11.11: /xrpc/feed.firehose -------------------------------------
# Federation spec §4 made partially live. Signed firehose of video.create
# events with monotonic cursor pagination. Relay key is Ed25519, generated
# on first request and persisted to disk. Public pubkey served at
# /.well-known/relay/key.

FIREHOSE_RELAY_KEY_PATH = BASE_DIR / "relay_ed25519.key"
_FIREHOSE_RELAY = {"sk": None, "pk": None, "did": "did:web:bottube.ai"}
_FIREHOSE_LOCK = threading.Lock()


def _firehose_load_relay_key():
    """Lazy-init relay Ed25519 keypair. Persists to /root/bottube/relay_ed25519.key."""
    if _FIREHOSE_RELAY["sk"] is not None:
        return
    with _FIREHOSE_LOCK:
        if _FIREHOSE_RELAY["sk"] is not None:
            return
        try:
            import nacl.signing as _nacl_signing
            import nacl.encoding as _nacl_encoding
        except ImportError:
            return
        if FIREHOSE_RELAY_KEY_PATH.exists():
            try:
                seed = FIREHOSE_RELAY_KEY_PATH.read_bytes()
                if len(seed) >= 32:
                    sk = _nacl_signing.SigningKey(seed[:32])
                else:
                    sk = _nacl_signing.SigningKey.generate()
                    FIREHOSE_RELAY_KEY_PATH.write_bytes(bytes(sk))
                    os.chmod(FIREHOSE_RELAY_KEY_PATH, 0o600)
            except Exception:
                sk = _nacl_signing.SigningKey.generate()
                FIREHOSE_RELAY_KEY_PATH.write_bytes(bytes(sk))
                os.chmod(FIREHOSE_RELAY_KEY_PATH, 0o600)
        else:
            sk = _nacl_signing.SigningKey.generate()
            FIREHOSE_RELAY_KEY_PATH.write_bytes(bytes(sk))
            os.chmod(FIREHOSE_RELAY_KEY_PATH, 0o600)
        _FIREHOSE_RELAY["sk"] = sk
        _FIREHOSE_RELAY["pk"] = sk.verify_key


def _firehose_canonical_json(obj):
    """Stable JSON encoding for signing. Sorted keys, no whitespace."""
    return json.dumps(obj, sort_keys=True, separators=(",", ":"),
                      ensure_ascii=False).encode("utf-8")


def _firehose_sign(payload):
    """Return base64-url Ed25519 signature over canonical JSON of payload."""
    _firehose_load_relay_key()
    sk = _FIREHOSE_RELAY.get("sk")
    if sk is None:
        return ""
    import base64
    sig = sk.sign(_firehose_canonical_json(payload)).signature
    return base64.urlsafe_b64encode(sig).rstrip(b"=").decode("ascii")


def _firehose_relay_pubkey_b64():
    _firehose_load_relay_key()
    pk = _FIREHOSE_RELAY.get("pk")
    if pk is None:
        return ""
    import base64
    return base64.urlsafe_b64encode(bytes(pk)).rstrip(b"=").decode("ascii")


@app.route("/.well-known/relay/key")
def well_known_relay_key():
    """Public Ed25519 pubkey for the firehose relay signature verification."""
    _firehose_load_relay_key()
    pk_b64 = _firehose_relay_pubkey_b64()
    if not pk_b64:
        return jsonify({"ok": False, "error": "relay key unavailable"}), 503
    payload = {
        "@context": "https://www.w3.org/ns/did/v1",
        "id": "did:web:bottube.ai#firehose-relay",
        "controller": "did:web:bottube.ai",
        "type": "Ed25519VerificationKey2020",
        "publicKeyMultibase": pk_b64,
        "publicKeyEncoding": "base64url-unpadded",
        "service": [{
            "id": "did:web:bottube.ai#firehose",
            "type": "BoTTubeFirehose",
            "serviceEndpoint": "https://bottube.ai/xrpc/feed.firehose",
        }],
    }
    resp = jsonify(payload)
    resp.headers["Content-Type"] = "application/did+json"
    resp.headers["Cache-Control"] = "public, max-age=300"
    resp.headers["Access-Control-Allow-Origin"] = "*"
    return resp


@app.route("/xrpc/feed.firehose")
def xrpc_feed_firehose():
    """Phase 11.11: signed firehose of video.create events.

    Cursor format: <created_at_ms>:<rowid>. Both segments are integers
    drawn from the videos table, so consecutive paginated reads are
    stable even as new uploads arrive.
    """
    cursor = (request.args.get("cursor") or "").strip()
    try:
        limit = max(1, min(200, int(request.args.get("limit", 100))))
    except Exception:
        limit = 100
    ip = _get_client_ip()
    if not _rate_limit(f"firehose:{ip}", 30, 60):
        return jsonify({"ok": False, "error": "rate limited"}), 429

    after_ms, after_rowid = 0, 0
    if cursor:
        m = re.match(r"^(\d+):(\d+)$", cursor)
        if not m:
            return jsonify({"ok": False, "error": "invalid cursor"}), 400
        after_ms = int(m.group(1))
        after_rowid = int(m.group(2))

    db = get_db()
    after_secs = after_ms / 1000.0
    rows = db.execute(
        """SELECT v.id AS rowid, v.video_id, v.title, v.thumbnail, v.duration_sec,
                  v.created_at, a.agent_name,
                  COALESCE(p.canonical_sha256, '') AS canonical_sha256,
                  COALESCE(p.uploader_sig, '') AS uploader_sig,
                  COALESCE(p.anchor_chain, '') AS anchor_chain,
                  COALESCE(p.anchor_tx_hash, '') AS anchor_tx_hash,
                  COALESCE(p.anchor_block_height, 0) AS anchor_block_height,
                  COALESCE(p.anchor_manifest_hash, '') AS anchor_manifest_hash
             FROM videos v
             JOIN agents a ON a.id = v.agent_id
             LEFT JOIN video_provenance p ON p.video_id = v.video_id
            WHERE COALESCE(v.is_removed, 0) = 0
              AND COALESCE(a.is_banned, 0) = 0
              AND (
                    v.created_at > ?
                 OR (v.created_at = ? AND v.id > ?)
                  )
            ORDER BY v.created_at ASC, v.id ASC
            LIMIT ?""",
        (after_secs, after_secs, after_rowid, limit + 1),
    ).fetchall()

    has_more = len(rows) > limit
    page = rows[:limit]

    events = []
    for r in page:
        ts_ms = int(float(r["created_at"] or 0) * 1000)
        ev = {
            "cursor": f"{ts_ms}:{r['rowid']}",
            "ts": float(r["created_at"] or 0),
            "op": "video.create",
            "relay_did": "did:web:bottube.ai",
            "actor_did": f"did:web:bottube.ai:{r['agent_name']}",
            "video_id": r["video_id"],
            "manifest_sha256": r["canonical_sha256"],
            "anchor_tx_hash": r["anchor_tx_hash"],
            "anchor_chain": r["anchor_chain"] or "rustchain",
            "block_height": r["anchor_block_height"],
            "merkle_root": r["anchor_manifest_hash"],
            "canonical_url": f"https://bottube.ai/api/videos/{r['video_id']}/stream",
            "watch_url": f"https://bottube.ai/watch/{r['video_id']}",
            "title": r["title"] or "",
            "duration_sec": r["duration_sec"] or 0,
        }
        # Sign the canonical JSON of the event minus the sig field.
        ev["sig"] = _firehose_sign(ev)
        events.append(ev)

    next_cursor = events[-1]["cursor"] if events and has_more else None

    payload = {
        "ok": True,
        "events": events,
        "next_cursor": next_cursor,
        "relay_did": "did:web:bottube.ai",
        "relay_pubkey": _firehose_relay_pubkey_b64(),
        "spec_version": "0.1",
        "spec_url": "https://bottube.ai/federation",
    }
    resp = jsonify(payload)
    resp.headers["Cache-Control"] = "no-cache"
    resp.headers["Access-Control-Allow-Origin"] = "*"
    return resp


@app.route("/.well-known/agent/<handle>")
def well_known_agent(handle):
    """Phase 11.10: DID:web actor doc.

    Implements the /federation spec's identity layer for current platform
    actors. Anyone resolving `agent://did:web:bottube.ai:<handle>` can fetch
    this endpoint for the canonical actor descriptor.
    """
    if not re.fullmatch(r"[a-z0-9_-]{2,32}", handle or ""):
        abort(404)
    db = get_db()
    # Phase 11.23: also pull the Ed25519 keypair (idempotently created
    # at first upload) so the actor doc exposes the real signing key.
    try:
        cols = {row[1] for row in db.execute("PRAGMA table_info(agents)").fetchall()}
        ed_col = "ed25519_pubkey" if "ed25519_pubkey" in cols else None
    except Exception:
        ed_col = None

    if ed_col:
        agent = db.execute(
            f"""SELECT id, agent_name, display_name, bio, avatar_url, is_human,
                      created_at,
                      COALESCE(rtc_wallet, '') AS rtc_wallet,
                      COALESCE(rtc_address, '') AS rtc_address,
                      COALESCE({ed_col}, '') AS ed25519_pubkey
                 FROM agents WHERE agent_name = ?
                                    AND COALESCE(is_banned, 0) = 0
                                    AND COALESCE(is_suspended, 0) = 0""",
            (handle,),
        ).fetchone()
    else:
        agent = db.execute(
            """SELECT id, agent_name, display_name, bio, avatar_url, is_human,
                      created_at,
                      COALESCE(rtc_wallet, '') AS rtc_wallet,
                      COALESCE(rtc_address, '') AS rtc_address
                 FROM agents WHERE agent_name = ?
                                    AND COALESCE(is_banned, 0) = 0
                                    AND COALESCE(is_suspended, 0) = 0""",
            (handle,),
        ).fetchone()
    if not agent:
        abort(404)

    rtc_addr = agent["rtc_wallet"] or agent["rtc_address"] or ""
    ed_pub = ""
    try:
        ed_pub = agent["ed25519_pubkey"] if ed_col else ""
    except Exception:
        ed_pub = ""

    actor_doc = {
        "@context": "https://www.w3.org/ns/did/v1",
        "id": f"did:web:bottube.ai:{handle}",
        "alsoKnownAs": [
            f"agent://did:web:bottube.ai:{handle}",
            f"https://bottube.ai/agent/{handle}",
        ],
        "service": [{
            "id": f"did:web:bottube.ai:{handle}#bottube-home",
            "type": "BoTTubeHomeInstance",
            "serviceEndpoint": "https://bottube.ai",
        }],
        "handle": f"@{handle}@bottube.ai",
        "homeInstance": "https://bottube.ai",
        "displayName": agent["display_name"] or handle,
        "isHuman": bool(agent["is_human"]),
        "bio": (agent["bio"] or "")[:500],
        "avatar": (agent["avatar_url"] or f"https://bottube.ai/avatar/{handle}.svg"),
        "createdAt": int(float(agent["created_at"] or 0)),
    }
    methods = []
    if ed_pub:
        # Phase 11.23: real Ed25519 signing key, used to verify the
        # creator_signature field in v3 manifests.
        methods.append({
            "id": f"did:web:bottube.ai:{handle}#bottube-creator-key",
            "type": "Ed25519VerificationKey2020",
            "controller": f"did:web:bottube.ai:{handle}",
            "publicKeyHex": ed_pub,
            "purpose": "BoTTubeProvenanceV3",
            "managed_by": "platform",
            "note": (
                "Server-managed Ed25519 keypair (v3a). Used to sign "
                "creator_signature in /api/videos/<id>/provenance for "
                "manifest_version >= 3. v3b will allow the agent to "
                "bring their own keypair; the leaf shape stays the same."
            ),
        })
    if rtc_addr:
        methods.append({
            "id": f"did:web:bottube.ai:{handle}#wallet",
            "type": "Ed25519VerificationKey2020",
            "controller": f"did:web:bottube.ai:{handle}",
            "publicKey": rtc_addr,
            "purpose": "RustChainWallet",
            "note": "RustChain wallet address — separate from the "
                    "BoTTubeProvenanceV3 signing key above.",
        })
    if methods:
        actor_doc["verificationMethod"] = methods

    resp = jsonify(actor_doc)
    resp.headers["Content-Type"] = "application/did+json"
    resp.headers["Cache-Control"] = "public, max-age=300"
    resp.headers["Access-Control-Allow-Origin"] = "*"
    return resp


@app.route("/.well-known/mcp.json")
def well_known_mcp():
    """MCP discovery descriptor."""
    resp = jsonify(_mcp_descriptor())
    resp.headers["Cache-Control"] = "public, max-age=300"
    resp.headers["Access-Control-Allow-Origin"] = "*"
    return resp


# ---------------------------------------------------------------------------
# Phase 11.22: /.well-known/provenance-spec.json
# ---------------------------------------------------------------------------
# A canonical, machine-readable spec for the cryptographic provenance
# pipeline. Federation peers, crawlers, and verifier authors hit this one
# URL instead of scraping the engineering page. spec_version is bumped
# explicitly when any leaf recipe / endpoint contract changes — consumers
# pin against it.

@app.route("/.well-known/provenance-spec.json")
def well_known_provenance_spec():
    """Self-describing provenance spec for federation + tooling."""
    spec = {
        "spec": "bottube-provenance",
        "spec_version": "1.0.0",
        "issuer": "https://bottube.ai",
        "current_manifest_version": MANIFEST_CURRENT,
        "leaf_recipes": {
            "v1": _manifest_leaf_recipe(MANIFEST_V1),
            "v2": _manifest_leaf_recipe(MANIFEST_V2),
        },
        "merkle": {
            "tree_construction": (
                "Bitcoin-style binary tree over SHA-256 leaves: pair "
                "adjacent leaves and hash, duplicate the last node when "
                "a level has odd cardinality, iterate until a single "
                "32-byte root remains."
            ),
            "leaf_hash": "SHA-256",
            "node_hash": "SHA-256",
            "domain_separator": {
                "v1": "(none — legacy)",
                "v2": '"bottube/v2" (literal ASCII)',
            },
        },
        "anchor": {
            "chain": "rustchain",
            "anchor_target": "Ergo box additionalRegisters.R4",
            "register_format": "0e20<32-byte SHA-256 hex>",
            "tx_value_nanoerg": 1_000_000,
            "fee_policy": "zero-fee chain config",
        },
        "endpoints": {
            "provenance":   "/api/videos/{video_id}/provenance",
            "anchor_proof": "/api/videos/{video_id}/anchor-proof",
            "receipt":      "/api/videos/{video_id}/receipt",
            "anchor_chain": "/api/anchors/{tx_hash}/chain",
            "transparency": "/api/transparency",
            "anchors_html": "/anchors",
            "transparency_html": "/transparency",
            "engineering_html": "/engineering",
            "verification_badge_svg": "/badge/verified/{video_id}.svg",
            "verification_iframe":    "/embed/verify/{video_id}",
            "verification_js":        "/embed/bottube-verify.js",
        },
        "verifier": {
            "package": "bottube-verify",
            "minimum_version": "0.4.0",
            "source": "https://github.com/Scottcjn/bottube",
            "install": "pip install bottube-verify",
            "modes": {
                "live": "bottube-verify <video_id>",
                "offline_receipt": "bottube-verify --receipt receipt.json",
                "asset_recheck": "bottube-verify <video_id> --check-asset",
            },
            "exit_codes": {
                "0": "PASS or PARTIAL",
                "1": "FAIL or fetch error",
            },
        },
        "reconciliation": {
            "endpoint": "/api/admin/reconcile-anchors",
            "cadence": "every 6 hours via systemd timer",
            "summary_in": "/api/transparency.reconciliation",
            "alarm_field": "/api/transparency.reconciliation.alarm",
        },
        "federation": {
            "actor_doc": "/.well-known/agent/{handle}",
            "relay_key": "/.well-known/relay/key",
            "firehose": "/xrpc/feed.firehose",
        },
        "schema_changes_policy": (
            "spec_version bumps when any leaf recipe, anchor format, "
            "endpoint contract, or receipt schema changes. Existing "
            "anchors stay valid forever under the recipe they were "
            "written under (manifest_version is per-row)."
        ),
        "as_of": int(time.time()),
    }
    resp = jsonify(spec)
    resp.headers["Cache-Control"] = "public, max-age=600"
    resp.headers["Access-Control-Allow-Origin"] = "*"
    return resp


@app.route("/mcp", methods=["GET", "POST", "OPTIONS"])
def mcp_bridge():
    """JSON-RPC-shaped MCP bridge.

    Accepts:
      * GET           -> returns the descriptor (same payload as /.well-known/mcp.json).
      * POST {tool, args}  -> invokes the named tool, wraps the existing
                              REST handler, returns its JSON response unwrapped
                              with a {ok, tool, result} envelope.

    No auth, read-only. Agents that want write access go through the
    explicit /api/register + /api/agents/me/accept-terms flow.
    """
    if request.method == "OPTIONS":
        resp = jsonify({"ok": True})
        resp.headers["Access-Control-Allow-Origin"] = "*"
        resp.headers["Access-Control-Allow-Methods"] = "GET, POST, OPTIONS"
        resp.headers["Access-Control-Allow-Headers"] = "Content-Type"
        return resp

    if request.method == "GET":
        return jsonify(_mcp_descriptor())

    data = request.get_json(silent=True)
    if data is None:
        data = {}
    elif not isinstance(data, dict):
        return jsonify({"ok": False, "error": "JSON body must be an object"}), 400

    raw_tool_name = data.get("tool") or data.get("method") or ""
    if not isinstance(raw_tool_name, str):
        return jsonify({"ok": False, "error": "tool must be a string"}), 400
    tool_name = raw_tool_name.strip()

    args = data["args"] if "args" in data else data.get("params", {})
    if args is None:
        args = {}
    if not isinstance(args, dict):
        return jsonify({"ok": False, "error": "args must be an object"}), 400

    tool = next((t for t in MCP_TOOLS if t["name"] == tool_name), None)
    if not tool:
        return jsonify({
            "ok": False,
            "error": f"unknown tool: {tool_name}",
            "available_tools": [t["name"] for t in MCP_TOOLS],
        }), 404

    # Use Flask's test_client so the full request lifecycle runs (before_request,
    # visitor cookie, after_request) rather than test_request_context which
    # leaves anchor IP empty and silently falls through to chronological.
    try:
        if tool_name == "feed.get":
            qs = {k: str(args[k]) for k in ("bucket", "per_page", "page", "category")
                  if k in args and args[k] not in (None, "")}
            qs.setdefault("surface", "mcp")
            with app.test_client() as client:
                # Forge a stable visitor id per MCP-anchor video so anchor lookup
                # has something to bite on. If args includes "anchor_video_id",
                # use it as the visitor seed for deterministic recommendations.
                anchor_seed = args.get("anchor_video_id", "") or "mcp_default"
                visitor = "mcp_" + hashlib.sha256(anchor_seed.encode()).hexdigest()[:24]
                client.set_cookie("_bt_vid", visitor)
                r = client.get("/api/feed", query_string=qs)
                payload = r.get_json()
        elif tool_name == "video.get":
            video_id = (args.get("video_id") or "").strip()
            if not re.fullmatch(r"[A-Za-z0-9_-]{5,32}", video_id):
                return jsonify({"ok": False, "error": "invalid video_id"}), 400
            db = get_db()
            row = db.execute(
                """SELECT v.video_id, v.title, v.description, v.tags, v.category,
                          v.duration_sec, v.width, v.height, v.views, v.likes,
                          v.dislikes, v.created_at, a.agent_name, a.display_name,
                          a.avatar_url, a.is_human
                     FROM videos v JOIN agents a ON v.agent_id = a.id
                    WHERE v.video_id = ? AND COALESCE(v.is_removed, 0) = 0""",
                (video_id,),
            ).fetchone()
            if not row:
                return jsonify({"ok": False, "error": "video not found"}), 404
            payload = {
                "ok": True,
                "video_id": row["video_id"],
                "title": row["title"],
                "description": row["description"],
                "tags": (lambda t: json.loads(t) if t else [])(row["tags"]),
                "category": row["category"],
                "duration_sec": row["duration_sec"],
                "width": row["width"],
                "height": row["height"],
                "views": row["views"],
                "likes": row["likes"],
                "dislikes": row["dislikes"],
                "created_at": row["created_at"],
                "agent_name": row["agent_name"],
                "display_name": row["display_name"],
                "avatar_url": row["avatar_url"],
                "is_human": bool(row["is_human"]),
                "watch_url": f"https://bottube.ai/watch/{row['video_id']}",
                "stream_url": f"https://bottube.ai/api/videos/{row['video_id']}/stream",
            }
        elif tool_name == "video.similar":
            video_id = (args.get("video_id") or "").strip()
            try:
                k = max(1, min(50, int(args.get("k", 10))))
            except Exception:
                k = 10
            if not re.fullmatch(r"[A-Za-z0-9_-]{5,32}", video_id):
                return jsonify({"ok": False, "error": "invalid video_id"}), 400
            with app.test_request_context(
                f"/api/videos/{video_id}/similar?k={k}",
                headers={"X-MCP-Bridge": "1"},
            ):
                resp = api_videos_similar(video_id)
                if isinstance(resp, tuple):
                    resp = resp[0]
                payload = resp.get_json() if hasattr(resp, "get_json") else resp
        elif tool_name == "video.provenance":
            video_id = (args.get("video_id") or "").strip()
            if not re.fullmatch(r"[A-Za-z0-9_-]{5,32}", video_id):
                return jsonify({"ok": False, "error": "invalid video_id"}), 400
            with app.test_request_context(
                f"/api/videos/{video_id}/provenance",
                headers={"X-MCP-Bridge": "1"},
            ):
                resp = video_provenance(video_id)
                if isinstance(resp, tuple):
                    resp = resp[0]
                payload = resp.get_json() if hasattr(resp, "get_json") else resp
        elif tool_name == "video.keyframes":
            video_id = (args.get("video_id") or "").strip()
            if not re.fullmatch(r"[A-Za-z0-9_-]{5,32}", video_id):
                return jsonify({"ok": False, "error": "invalid video_id"}), 400
            with app.test_request_context(
                f"/api/videos/{video_id}/keyframes",
                headers={"X-MCP-Bridge": "1"},
            ):
                resp = video_keyframes(video_id)
                if isinstance(resp, tuple):
                    resp = resp[0]
                payload = resp.get_json() if hasattr(resp, "get_json") else resp
        else:
            return jsonify({"ok": False, "error": f"unrouted tool: {tool_name}"}), 500

        out = jsonify({"ok": True, "tool": tool_name, "result": payload})
        out.headers["Access-Control-Allow-Origin"] = "*"
        return out
    except Exception as e:
        try:
            app.logger.warning("mcp bridge error %s: %s", tool_name, e)
        except Exception:
            pass
        return jsonify({"ok": False, "tool": tool_name, "error": str(e)}), 500


@app.route("/keyframes/<path:filename>")
def serve_keyframe(filename):
    """Serve keyframe sprite files with strong caching."""
    if "/" in filename or ".." in filename:
        abort(404)
    return send_from_directory(
        KEYFRAME_DIR, filename, max_age=86400 * 30, mimetype="image/jpeg"
    )


# ---------------------------------------------------------------------------
# Trust & Safety: TOS acceptance, content blocklist, user reports, audit
# ---------------------------------------------------------------------------
# Static legal pages, click-wrap acceptance for humans + explicit
# acceptance flow for agents, hash-based content blocklist (SHA-256
# of file + thumbnail), user-facing /api/report endpoint with rate
# limiting, and a moderation_audit log of every enforcement action.

TOS_VERSION = "1.0"
TOS_EFFECTIVE = "2026-04-30"

_TS_SCHEMA_READY = False
_TS_SCHEMA_LOCK = _eng_Lock()


def _ensure_ts_schema():
    """Lazy create trust+safety tables and add TOS columns to agents."""
    global _TS_SCHEMA_READY
    if _TS_SCHEMA_READY:
        return
    with _TS_SCHEMA_LOCK:
        if _TS_SCHEMA_READY:
            return
        conn = sqlite3.connect(str(DB_PATH))
        try:
            conn.executescript(
                """
                CREATE TABLE IF NOT EXISTS moderation_reports (
                    id            INTEGER PRIMARY KEY AUTOINCREMENT,
                    report_id     TEXT UNIQUE NOT NULL,
                    category      TEXT NOT NULL,
                    target        TEXT NOT NULL,
                    detail        TEXT NOT NULL,
                    reporter_email TEXT DEFAULT '',
                    reporter_ip   TEXT DEFAULT '',
                    reporter_ua   TEXT DEFAULT '',
                    reporter_agent_id INTEGER DEFAULT 0,
                    status        TEXT DEFAULT 'open',  -- open, reviewing, actioned, dismissed
                    severity      TEXT DEFAULT 'normal', -- low, normal, high, critical
                    handled_by    TEXT DEFAULT '',
                    handled_at    REAL DEFAULT 0,
                    resolution    TEXT DEFAULT '',
                    created_at    REAL NOT NULL
                );
                CREATE INDEX IF NOT EXISTS idx_modreports_status
                    ON moderation_reports(status, created_at DESC);
                CREATE INDEX IF NOT EXISTS idx_modreports_target
                    ON moderation_reports(target);
                CREATE INDEX IF NOT EXISTS idx_modreports_category
                    ON moderation_reports(category, created_at DESC);

                CREATE TABLE IF NOT EXISTS content_blocklist (
                    hash_sha256   TEXT PRIMARY KEY,
                    hash_kind     TEXT DEFAULT 'file',   -- file, thumbnail, phash
                    category      TEXT NOT NULL,         -- csam, terror, ncii, copyright, malware, other
                    source        TEXT DEFAULT '',       -- ncmec, iwf, projectvic, internal, user_report
                    notes         TEXT DEFAULT '',
                    added_by      TEXT DEFAULT '',
                    added_at      REAL NOT NULL,
                    hits          INTEGER DEFAULT 0,
                    last_hit_at   REAL DEFAULT 0
                );
                CREATE INDEX IF NOT EXISTS idx_blocklist_category
                    ON content_blocklist(category);

                CREATE TABLE IF NOT EXISTS moderation_audit (
                    id            INTEGER PRIMARY KEY AUTOINCREMENT,
                    actor         TEXT NOT NULL,        -- system, admin, user_report, ncmec
                    action        TEXT NOT NULL,        -- quarantine, terminate, dismiss, restore, blocklist_add, ncmec_report
                    target_kind   TEXT NOT NULL,        -- video, agent, ip, hash, comment
                    target_id     TEXT NOT NULL,
                    reason        TEXT DEFAULT '',
                    severity      TEXT DEFAULT 'normal',
                    metadata_json TEXT DEFAULT '{}',
                    created_at    REAL NOT NULL
                );
                CREATE INDEX IF NOT EXISTS idx_audit_target
                    ON moderation_audit(target_kind, target_id, created_at DESC);

                CREATE TABLE IF NOT EXISTS agent_strikes (
                    id            INTEGER PRIMARY KEY AUTOINCREMENT,
                    agent_id      INTEGER NOT NULL,
                    severity      TEXT NOT NULL,    -- minor, major, critical
                    reason        TEXT NOT NULL,
                    issued_by     TEXT DEFAULT 'system',
                    expires_at    REAL DEFAULT 0,
                    created_at    REAL NOT NULL
                );
                CREATE INDEX IF NOT EXISTS idx_strikes_agent
                    ON agent_strikes(agent_id, created_at DESC);
                """
            )

            # Add TOS columns to agents (idempotent).
            cols = {row[1] for row in conn.execute("PRAGMA table_info(agents)").fetchall()}
            if "tos_version_accepted" not in cols:
                conn.execute("ALTER TABLE agents ADD COLUMN tos_version_accepted TEXT DEFAULT ''")
            if "tos_accepted_at" not in cols:
                conn.execute("ALTER TABLE agents ADD COLUMN tos_accepted_at REAL DEFAULT 0")
            if "tos_accepted_ip" not in cols:
                conn.execute("ALTER TABLE agents ADD COLUMN tos_accepted_ip TEXT DEFAULT ''")
            if "is_suspended" not in cols:
                conn.execute("ALTER TABLE agents ADD COLUMN is_suspended INTEGER DEFAULT 0")
            if "suspended_reason" not in cols:
                conn.execute("ALTER TABLE agents ADD COLUMN suspended_reason TEXT DEFAULT ''")
            if "suspended_at" not in cols:
                conn.execute("ALTER TABLE agents ADD COLUMN suspended_at REAL DEFAULT 0")
            conn.commit()
        finally:
            conn.close()
        _TS_SCHEMA_READY = True


def _ts_log_audit(actor, action, target_kind, target_id, reason="",
                  severity="normal", meta=None):
    """Append a moderation audit row. Never raises."""
    try:
        _ensure_ts_schema()
        conn = sqlite3.connect(str(DB_PATH))
        try:
            conn.execute(
                """INSERT INTO moderation_audit
                       (actor, action, target_kind, target_id, reason, severity,
                        metadata_json, created_at)
                   VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
                (actor, action, target_kind, target_id, reason, severity,
                 json.dumps(meta or {}), time.time()),
            )
            conn.commit()
        finally:
            conn.close()
    except Exception:
        pass


def _ts_check_blocklist(sha256_hex, kind="file"):
    """Look up a hash in the content_blocklist. Returns the row or None."""
    try:
        _ensure_ts_schema()
        db = get_db()
        row = db.execute(
            """SELECT hash_sha256, hash_kind, category, source, notes
                 FROM content_blocklist WHERE hash_sha256 = ?""",
            (sha256_hex,),
        ).fetchone()
        if row:
            db.execute(
                """UPDATE content_blocklist
                      SET hits = COALESCE(hits, 0) + 1, last_hit_at = ?
                    WHERE hash_sha256 = ?""",
                (time.time(), sha256_hex),
            )
            db.commit()
        return row
    except Exception:
        return None


def _ts_sha256_file(path, chunk=1024 * 1024):
    """SHA-256 a file on disk."""
    h = hashlib.sha256()
    with open(path, "rb") as f:
        while True:
            buf = f.read(chunk)
            if not buf:
                break
            h.update(buf)
    return h.hexdigest()


def _ts_suspend_agent(agent_id, reason, severity="critical"):
    """Suspend an agent and log it. Caller still controls the response."""
    try:
        _ensure_ts_schema()
        conn = sqlite3.connect(str(DB_PATH))
        try:
            conn.execute(
                """UPDATE agents
                      SET is_suspended = 1,
                          suspended_reason = ?,
                          suspended_at = ?
                    WHERE id = ?""",
                (reason, time.time(), agent_id),
            )
            conn.execute(
                """INSERT INTO agent_strikes
                       (agent_id, severity, reason, issued_by, created_at)
                   VALUES (?, ?, ?, 'system', ?)""",
                (agent_id, severity, reason, time.time()),
            )
            conn.commit()
        finally:
            conn.close()
    except Exception:
        pass
    _ts_log_audit("system", "suspend", "agent", str(agent_id), reason, severity)


def ts_inspect_uploaded_file(file_path, agent_id):
    """Hash an uploaded file, check the blocklist. If matched, *quarantine*
    (preserve under § 2258A), suspend the uploader, audit, and — for CSAM
    — enqueue an NCMEC report for operator review.

    Returns (rejected: bool, info: dict). Never raises.
    """
    try:
        sha = _ts_sha256_file(file_path)
    except Exception as e:
        return False, {"hash_error": str(e)}

    hit = _ts_check_blocklist(sha, kind="file")
    if hit:
        category = hit["category"]
        critical = category in {"csam", "terror"}
        try:
            file_size = Path(file_path).stat().st_size
        except Exception:
            file_size = 0

        # Snapshot upload provenance before we move the file. CSAM and
        # other federal-reportable categories are *quarantined*, never
        # deleted; § 2258A(h)(2)(A) requires 90-day preservation. For
        # categories without a federal reporting duty we still preserve
        # the file under quarantine so an operator can review.
        ip = _get_client_ip()
        ua = request.headers.get("User-Agent", "")[:500] if request else ""
        agent_name = ""
        agent_email = ""
        try:
            db = get_db()
            row = db.execute(
                "SELECT agent_name, COALESCE(email,'') AS email FROM agents WHERE id = ?",
                (agent_id,),
            ).fetchone()
            if row:
                agent_name = row["agent_name"] or ""
                try:
                    agent_email = row["email"] or ""
                except Exception:
                    agent_email = ""
        except Exception:
            pass

        qpath = _ts_quarantine_file(
            file_path, sha, category=category,
            meta={
                "agent_id": agent_id,
                "agent_name": agent_name,
                "ip": ip,
                "user_agent": ua,
                "blocklist_source": hit["source"],
                "blocklist_kind": hit["hash_kind"],
            },
        )

        _ts_suspend_agent(
            agent_id,
            reason=f"upload matched {category} blocklist (hash {sha[:16]}…)",
            severity="critical" if critical else "major",
        )
        _ts_log_audit(
            actor="system",
            action="quarantine",
            target_kind="upload",
            target_id=sha,
            reason=f"blocklist match: {category} (source={hit['source']})",
            severity="critical" if critical else "major",
            meta={"agent_id": agent_id, "kind": hit["hash_kind"],
                  "quarantine_path": qpath, "ip": ip},
        )

        # CSAM and minor/NCII categories trigger an NCMEC report queue row.
        # The operator drafts it from /admin/ncmec/queue and submits via
        # report.cybertip.org until full ESP-API enrollment is active.
        queue_id = ""
        if category in {"csam", "ncii", "minor"}:
            queue_id = _ncmec_enqueue(
                source_type="blocklist_hit",
                category=category,
                target_kind="upload_attempt",
                source_id=sha,
                agent_id=agent_id,
                agent_name=agent_name,
                email=agent_email,
                ip=ip,
                user_agent=ua,
                sha256=sha,
                file_size=file_size,
                quarantine_path=qpath,
                discovery_method=f"sha256 match against {category}/{hit['source']} blocklist",
                notes="Auto-enqueued at upload time by ts_inspect_uploaded_file.",
            )

        return True, {
            "rejected": True,
            "category": category,
            "sha256": sha,
            "quarantine_path": qpath,
            "ncmec_queue_id": queue_id,
        }
    return False, {"sha256": sha}


# --- Static legal pages ----------------------------------------------------

@app.route("/terms")
@app.route("/tos")
def terms_page():
    return render_template("terms.html",
                           tos_version=TOS_VERSION,
                           tos_effective=TOS_EFFECTIVE)


@app.route("/aup")
def aup_page():
    return render_template("aup.html",
                           tos_version=TOS_VERSION,
                           tos_effective=TOS_EFFECTIVE)


@app.route("/dmca")
def dmca_page():
    return render_template("dmca.html",
                           tos_version=TOS_VERSION,
                           tos_effective=TOS_EFFECTIVE)


@app.route("/report")
def report_page():
    return render_template("report.html",
                           tos_version=TOS_VERSION,
                           tos_effective=TOS_EFFECTIVE)


# Privacy template already exists in the templates dir; ensure a route exists.
@app.route("/privacy")
def privacy_page():
    try:
        return render_template("privacy.html",
                               tos_version=TOS_VERSION,
                               tos_effective=TOS_EFFECTIVE)
    except Exception:
        # Fallback if template variables differ
        return render_template("privacy.html")


# --- TOS acceptance for agents --------------------------------------------

@app.route("/api/tos", methods=["GET"])
def tos_metadata():
    """Public TOS metadata for clients to discover the current version."""
    return jsonify({
        "ok": True,
        "version": TOS_VERSION,
        "effective": TOS_EFFECTIVE,
        "terms_url": "https://bottube.ai/terms",
        "aup_url": "https://bottube.ai/aup",
        "dmca_url": "https://bottube.ai/dmca",
        "privacy_url": "https://bottube.ai/privacy",
        "report_url": "https://bottube.ai/report",
        "csam_notice": (
            "Zero tolerance for CSAM. Uploads are hash-checked and reported "
            "to NCMEC and law enforcement as required by 18 U.S.C. § 2258A."
        ),
    })


@app.route("/api/agents/me/accept-terms", methods=["POST"])
@require_api_key
def agent_accept_terms():
    """Agent acknowledges the current Terms of Service.

    Required before any write action by agents created after the TOS rollout.
    Existing agents are grandfathered with a 30-day grace period.
    """
    _ensure_ts_schema()
    data = request.get_json(silent=True) or {}
    version = str(data.get("version", "")).strip() or TOS_VERSION
    if version != TOS_VERSION:
        return jsonify({
            "ok": False,
            "error": "version_mismatch",
            "expected": TOS_VERSION,
            "received": version,
            "terms_url": "https://bottube.ai/terms",
        }), 400

    agent = g.agent
    ip = _get_client_ip()
    db = get_db()
    db.execute(
        """UPDATE agents
              SET tos_version_accepted = ?,
                  tos_accepted_at = ?,
                  tos_accepted_ip = ?
            WHERE id = ?""",
        (version, time.time(), ip, agent["id"]),
    )
    db.commit()
    _ts_log_audit("agent", "accept_terms", "agent", str(agent["id"]),
                  reason=f"tos {version}", meta={"ip": ip})
    return jsonify({
        "ok": True,
        "agent_name": agent["agent_name"],
        "tos_version_accepted": version,
        "tos_effective": TOS_EFFECTIVE,
        "accepted_at": time.time(),
        "message": (
            "Terms of Service acknowledged. "
            "Welcome — please read https://bottube.ai/aup before posting."
        ),
    })


# --- Public report endpoint -----------------------------------------------

def _public_report_text_field(data, field, max_length):
    value = data.get(field, "")
    if value is None:
        value = ""
    if not isinstance(value, str):
        return None, f"{field} must be a string"
    return value.strip()[:max_length], None


@app.route("/api/report", methods=["POST"])
def submit_report():
    """User-facing report submission. Anonymous accepted, rate-limited per IP."""
    _ensure_ts_schema()
    ip = _get_client_ip()
    if not _rate_limit(f"report:{ip}", 10, 3600):
        return jsonify({"ok": False, "error": "Too many reports from this IP. Try again later."}), 429

    data = request.get_json(silent=True)
    if data is None:
        data = {}
    elif not isinstance(data, dict):
        return jsonify({"ok": False, "error": "JSON body must be an object"}), 400

    category, error = _public_report_text_field(data, "category", 32)
    if error:
        return jsonify({"ok": False, "error": error}), 400
    category = category.lower()
    target, error = _public_report_text_field(data, "target", 512)
    if error:
        return jsonify({"ok": False, "error": error}), 400
    detail, error = _public_report_text_field(data, "detail", 4000)
    if error:
        return jsonify({"ok": False, "error": error}), 400
    email, error = _public_report_text_field(data, "email", 200)
    if error:
        return jsonify({"ok": False, "error": error}), 400

    valid_cats = {"csam", "illegal", "ncii", "ip", "harassment",
                  "impersonation", "malware", "spam", "minor", "other"}
    if category not in valid_cats:
        return jsonify({"ok": False, "error": "invalid category"}), 400
    if not target or len(target) < 4:
        return jsonify({"ok": False, "error": "target is required"}), 400
    if not detail or len(detail) < 10:
        return jsonify({"ok": False, "error": "detail must be at least 10 characters"}), 400

    severity = "critical" if category in {"csam", "ncii"} else (
        "high" if category in {"illegal", "minor", "malware"} else "normal"
    )
    report_id = "rpt_" + secrets.token_hex(8)
    reporter_agent_id = (g.agent["id"] if getattr(g, "agent", None) else 0)

    db = get_db()
    db.execute(
        """INSERT INTO moderation_reports
               (report_id, category, target, detail,
                reporter_email, reporter_ip, reporter_ua, reporter_agent_id,
                severity, created_at)
           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
        (report_id, category, target, detail,
         email, ip, request.headers.get("User-Agent", "")[:500], reporter_agent_id,
         severity, time.time()),
    )
    db.commit()
    _ts_log_audit("user_report", "submit", "report", report_id,
                  reason=f"{category}: {target[:80]}", severity=severity,
                  meta={"ip": ip, "agent_id": reporter_agent_id})

    # CSAM, NCII, and minor-account reports auto-enqueue an NCMEC review.
    # The status starts at "pending" so an operator must inspect, confirm
    # apparent violation, draft from /admin/ncmec/draft, and submit. We do
    # *not* auto-quarantine the target on a user report alone — that's a
    # human-review decision to avoid weaponized takedowns.
    ncmec_queue_id = ""
    if category in {"csam", "ncii", "minor"}:
        # Best-effort target lookup: extract a video id from the URL or
        # fall back to the raw target string.
        target_video_id = ""
        m = re.search(r"/(?:watch|embed|v)/([A-Za-z0-9_-]{5,32})", target)
        if m:
            target_video_id = m.group(1)
        elif re.fullmatch(r"[A-Za-z0-9_-]{5,32}", target):
            target_video_id = target

        involved_agent_id = 0
        involved_agent_name = ""
        if target_video_id:
            try:
                vrow = db.execute(
                    """SELECT v.agent_id, a.agent_name
                         FROM videos v JOIN agents a ON v.agent_id = a.id
                        WHERE v.video_id = ?""",
                    (target_video_id,),
                ).fetchone()
                if vrow:
                    involved_agent_id = int(vrow["agent_id"] or 0)
                    involved_agent_name = vrow["agent_name"] or ""
            except Exception:
                pass

        ncmec_queue_id = _ncmec_enqueue(
            source_type="user_report",
            category=category,
            target_kind="video" if target_video_id else "url",
            target_video_id=target_video_id,
            source_id=report_id,
            agent_id=involved_agent_id,
            agent_name=involved_agent_name,
            ip="",  # the reporter's IP is not the involved party
            user_agent="",
            sha256="",
            file_size=0,
            quarantine_path="",
            discovery_method=f"user report {report_id}",
            notes=("Pending operator review. Confirm apparent violation, "
                   "quarantine the file, then draft via /admin/ncmec/draft. "
                   "Reporter detail: " + (detail[:300])),
        )

    out = {
        "ok": True,
        "report_id": report_id,
        "severity": severity,
        "message": (
            "Report received. Thank you. CSAM and content depicting minors "
            "is reviewed within 24 hours and forwarded to NCMEC where applicable."
        ),
    }
    if ncmec_queue_id:
        out["ncmec_queue_id"] = ncmec_queue_id
    return jsonify(out)


# --- Admin endpoints ------------------------------------------------------

def _ts_admin_ok():
    key = request.headers.get("X-Admin-Key", "") or request.args.get("admin_key", "")
    expected = os.environ.get("BOTTUBE_ADMIN_KEY", "") or os.environ.get("RC_ADMIN_KEY", "")
    return bool(expected) and (key == expected)


@app.route("/admin/blocklist/add", methods=["POST"])
def admin_blocklist_add():
    """Admin: add a SHA-256 hash to the content blocklist."""
    if not _ts_admin_ok():
        return jsonify({"ok": False, "error": "unauthorized"}), 401
    _ensure_ts_schema()
    data = request.get_json(silent=True) or {}
    sha = (data.get("sha256") or "").strip().lower()
    category = (data.get("category") or "").strip().lower()
    source = (data.get("source") or "internal").strip()[:64]
    notes = (data.get("notes") or "").strip()[:500]
    kind = (data.get("hash_kind") or "file").strip()[:32]

    if not re.fullmatch(r"[0-9a-f]{64}", sha):
        return jsonify({"ok": False, "error": "sha256 must be a 64-char hex string"}), 400
    if category not in {"csam", "terror", "ncii", "ip", "malware", "other"}:
        return jsonify({"ok": False, "error": "invalid category"}), 400

    db = get_db()
    db.execute(
        """INSERT OR REPLACE INTO content_blocklist
               (hash_sha256, hash_kind, category, source, notes, added_by, added_at)
           VALUES (?, ?, ?, ?, ?, ?, ?)""",
        (sha, kind, category, source, notes, "admin", time.time()),
    )
    db.commit()
    _ts_log_audit("admin", "blocklist_add", "hash", sha,
                  reason=f"{category} via {source}", severity="critical" if category == "csam" else "high")
    return jsonify({"ok": True, "sha256": sha, "category": category, "added_at": time.time()})


# --- Semantic embeddings (hybrid feed v1 scaffolding) ---------------------
# Per-video text embedding via Gemini (gemini-embedding-001, 3072-d, L2-norm).
# Cached in-memory as a NumPy matrix, refreshed lazily. Brute-force cosine
# at query time — fine at the current ~1.4k-video scale; swap to Qdrant
# once content is 10x larger. The /api/feed integration is deferred to
# Phase 7; this phase ships the embedding store, helpers, and the
# /api/videos/<id>/similar query surface.

import urllib.request as _ue_urlreq
import urllib.error as _ue_urlerr

EMBEDDING_MODEL = "gemini-embedding-2"
EMBEDDING_DIM = 3072
EMBEDDING_API_URL = (
    "https://generativelanguage.googleapis.com/v1beta/models/"
    + EMBEDDING_MODEL
    + ":embedContent"
)

# In-memory cache for fast cosine similarity at query time.
_EMB_CACHE = {"matrix": None, "ids": [], "loaded_at": 0.0}
_EMB_CACHE_LOCK = _eng_Lock()

# Free-tier Gemini embedContent quota is 100 requests/min/project. We
# enforce a global ~80 RPM ceiling with a leaky-bucket gap of 0.75s
# between successful calls, plus a 429-retry with the server-supplied
# retryDelay. Concurrency stacks behind this gate so multiple workers
# converge to the same rate.
_EMB_RATE_LOCK = _eng_Lock()
_EMB_RATE_NEXT_OK_AT = [0.0]
_EMB_RATE_MIN_GAP = 0.75
_EMB_RATE_429_BACKOFF_FLOOR = 5.0


# --- Phase 11.3: visual signal (describe-then-embed) ----------------------
# Per Codex's review, CLIP late-fusion is the right next modality. We don't
# have a CLIP server running, so we use Gemini vision to describe each
# keyframe sprite in 1-2 sentences, then embed the description with the
# same gemini-embedding-2 model the text path uses.
#
# Why "describe-then-embed" over true CLIP:
#   * Both signals end up in the same 3072-d Gemini space, so blending at
#     query time is a simple weighted cosine, not a cross-space alignment
#     problem.
#   * The caption is auditable (stored verbatim in `caption` column) — a
#     reviewer can see what the model thought the video was about.
#   * No new inference infrastructure required; reuses APIs already wired.
#
# Two costs:
#   * Gemini-2.5-flash vision rate limits (~15 RPM free tier, separate
#     from the embedContent 100 RPM and 1000/day buckets).
#   * Visual signal is filtered through a language layer, so very visual-
#     only differences (e.g., color palette, motion direction) may be
#     lost. Acceptable trade for a v1 that keeps the implementation tight.

VISUAL_MODEL_VISION = "gemini-2.5-flash-lite"
VISUAL_MODEL_EMBED = "gemini-embedding-2"
VISUAL_API_VISION_URL = (
    "https://generativelanguage.googleapis.com/v1beta/models/"
    + VISUAL_MODEL_VISION
    + ":generateContent"
)
VISUAL_PROMPT = (
    "Describe this 6-frame sprite from an AI-generated video clip in 1-2 "
    "sentences. Focus on subject, action, visual style, and mood. No "
    "headers or markdown — return just the description as plain text."
)
VISUAL_CAPTION_MAX_LEN = 600

_UV_CACHE = {"matrix": None, "ids": [], "loaded_at": 0.0}
_UV_CACHE_LOCK = _eng_Lock()

# Vision quota is independent of the text embedContent quota but still
# rate-limited; 4 s gap = 15 RPM, matches free-tier ceiling.
_UV_RATE_LOCK = _eng_Lock()
_UV_RATE_NEXT_OK_AT = [0.0]
_UV_RATE_MIN_GAP = 4.0


def _uv_ensure_schema():
    """Lazy-create video_visual_embeddings table."""
    conn = sqlite3.connect(str(DB_PATH))
    try:
        conn.executescript(
            """
            CREATE TABLE IF NOT EXISTS video_visual_embeddings (
                video_id        TEXT PRIMARY KEY,
                vision_model    TEXT NOT NULL,            -- gemini-2.5-flash
                embed_model     TEXT NOT NULL,            -- gemini-embedding-2
                dim             INTEGER NOT NULL,
                bytes           BLOB NOT NULL,            -- numpy float32 .tobytes()
                caption         TEXT NOT NULL,            -- the visual description
                sprite_sha      TEXT DEFAULT '',          -- sha256 of source sprite jpeg
                created_at      REAL NOT NULL,
                updated_at      REAL NOT NULL
            );
            CREATE INDEX IF NOT EXISTS idx_visual_embed_updated
                ON video_visual_embeddings(updated_at DESC);
            """
        )
        conn.commit()
    finally:
        conn.close()


def _uv_rate_wait():
    while True:
        with _UV_RATE_LOCK:
            now = time.time()
            wait = _UV_RATE_NEXT_OK_AT[0] - now
            if wait <= 0:
                _UV_RATE_NEXT_OK_AT[0] = now + _UV_RATE_MIN_GAP
                return
        time.sleep(min(8.0, max(0.1, wait)))


def _uv_describe_sprite(sprite_path):
    """Call Gemini vision on a sprite jpeg. Returns (caption, error)."""
    key = os.environ.get("GEMINI_API_KEY", "")
    if not key:
        return "", "no_api_key"
    try:
        with open(sprite_path, "rb") as f:
            sprite_bytes = f.read()
    except Exception as e:
        return "", f"sprite_read_failed:{e}"
    if not sprite_bytes:
        return "", "empty_sprite"

    import base64 as _b64
    body = json.dumps({
        "contents": [{
            "parts": [
                {"text": VISUAL_PROMPT},
                {"inline_data": {
                    "mime_type": "image/jpeg",
                    "data": _b64.b64encode(sprite_bytes).decode("ascii"),
                }},
            ],
        }],
    }).encode("utf-8")
    req = _ue_urlreq.Request(
        VISUAL_API_VISION_URL + "?key=" + key,
        data=body,
        headers={"Content-Type": "application/json"},
        method="POST",
    )
    _uv_rate_wait()
    try:
        with _ue_urlreq.urlopen(req, timeout=60) as resp:
            data = json.loads(resp.read().decode("utf-8"))
    except _ue_urlerr.HTTPError as e:
        if e.code == 429:
            try:
                with _UV_RATE_LOCK:
                    _UV_RATE_NEXT_OK_AT[0] = time.time() + 60.0
            except Exception:
                pass
        return "", f"http_{e.code}"
    except Exception as e:
        return "", f"req_failed:{e}"

    cands = data.get("candidates", []) or []
    if not cands:
        return "", "no_candidates"
    parts = (cands[0].get("content", {}).get("parts", []) or [])
    text = ""
    for p in parts:
        text += (p.get("text") or "")
    text = text.strip()
    if not text:
        return "", "empty_caption"
    return text[:VISUAL_CAPTION_MAX_LEN], ""


def _uv_record_for_video(video_id, ensure_sprite=True):
    """Generate or refresh the visual embedding for one video.

    Steps:
      1. Look up the video record + filename.
      2. Ensure the keyframe sprite exists (generate on demand if missing).
      3. Hash the sprite. If we already have a row with matching sprite_sha,
         skip — same as the text path's text_sha guard.
      4. Call Gemini vision for a description.
      5. Embed the description with gemini-embedding-2.
      6. INSERT OR REPLACE.
    """
    _uv_ensure_schema()
    try:
        import numpy as _np
    except ImportError:
        return {"ok": False, "error": "numpy_missing"}

    db = get_db()
    video = db.execute(
        "SELECT video_id, filename, duration_sec FROM videos WHERE video_id = ? AND COALESCE(is_removed,0)=0",
        (video_id,),
    ).fetchone()
    if not video:
        return {"ok": False, "error": "not_found"}

    sprite_path = KEYFRAME_DIR / f"{video_id}.jpg"
    if not sprite_path.exists() or sprite_path.stat().st_size < 256:
        if not ensure_sprite:
            return {"ok": False, "error": "sprite_missing"}
        if not _renditions_ffmpeg_available() and not Path("/usr/bin/ffmpeg").is_file():
            return {"ok": False, "error": "ffmpeg_missing"}
        # Reuse the existing extraction path
        video_path = VIDEO_DIR / (video["filename"] or "")
        if not video_path.exists():
            return {"ok": False, "error": "canonical_missing"}
        try:
            _generate_keyframe_sprite(video_path, sprite_path)
        except Exception as e:
            return {"ok": False, "error": f"sprite_gen_failed:{e}"}

    try:
        sprite_sha = _ts_sha256_file(sprite_path)
    except Exception as e:
        return {"ok": False, "error": f"sprite_hash_failed:{e}"}

    conn = sqlite3.connect(str(DB_PATH))
    try:
        existing = conn.execute(
            "SELECT sprite_sha FROM video_visual_embeddings WHERE video_id = ?",
            (video_id,),
        ).fetchone()
        if existing and existing[0] == sprite_sha:
            return {"ok": True, "skipped": True, "reason": "unchanged"}
    finally:
        conn.close()

    caption, err = _uv_describe_sprite(str(sprite_path))
    if err or not caption:
        return {"ok": False, "error": f"describe_failed:{err}"}

    arr = _ue_embed_text("Visual: " + caption)
    if arr is None:
        return {"ok": False, "error": "embed_failed"}
    arr = arr.astype("float32", copy=False)

    conn = sqlite3.connect(str(DB_PATH))
    try:
        conn.execute(
            """INSERT OR REPLACE INTO video_visual_embeddings
                   (video_id, vision_model, embed_model, dim, bytes,
                    caption, sprite_sha,
                    created_at, updated_at)
               VALUES (?, ?, ?, ?, ?, ?, ?,
                       COALESCE((SELECT created_at FROM video_visual_embeddings WHERE video_id=?), ?),
                       ?)""",
            (video_id, VISUAL_MODEL_VISION, VISUAL_MODEL_EMBED,
             int(arr.shape[0]), arr.tobytes(), caption, sprite_sha,
             video_id, time.time(), time.time()),
        )
        conn.commit()
    finally:
        conn.close()

    with _UV_CACHE_LOCK:
        _UV_CACHE.update({"matrix": None, "ids": [], "loaded_at": 0.0})
    return {"ok": True, "dim": int(arr.shape[0]), "caption_len": len(caption)}


def _uv_cache_warm():
    try:
        import numpy as _np
    except ImportError:
        return False
    _uv_ensure_schema()
    conn = sqlite3.connect(str(DB_PATH))
    conn.row_factory = sqlite3.Row
    try:
        rows = conn.execute(
            """SELECT v.video_id, ve.dim, ve.bytes
                 FROM video_visual_embeddings ve
                 JOIN videos v ON v.video_id = ve.video_id
                WHERE COALESCE(v.is_removed, 0) = 0
                  AND ve.embed_model = ?
                ORDER BY ve.video_id""",
            (VISUAL_MODEL_EMBED,),
        ).fetchall()
    finally:
        conn.close()
    if not rows:
        with _UV_CACHE_LOCK:
            _UV_CACHE.update({"matrix": None, "ids": [], "loaded_at": time.time()})
        return False
    n = len(rows)
    dim = int(rows[0]["dim"])
    M = _np.zeros((n, dim), dtype=_np.float32)
    ids = []
    for i, r in enumerate(rows):
        if int(r["dim"]) != dim:
            continue
        try:
            M[i] = _np.frombuffer(r["bytes"], dtype=_np.float32)
        except Exception:
            continue
        ids.append(r["video_id"])
    with _UV_CACHE_LOCK:
        _UV_CACHE.update({"matrix": M, "ids": ids, "loaded_at": time.time()})
    return True


@app.route("/admin/visual/backfill", methods=["POST"])
def admin_visual_backfill():
    """Generate visual embeddings for videos missing them. Resumable."""
    if not _ts_admin_ok():
        return jsonify({"ok": False, "error": "unauthorized"}), 401
    _uv_ensure_schema()
    data = request.get_json(silent=True) or {}
    try:
        limit = max(1, min(50, int(data.get("limit", 10))))
    except Exception:
        limit = 10
    since = (data.get("since_video_id") or "").strip()

    db = get_db()
    targeted = data.get("video_ids") or []
    if isinstance(targeted, list) and targeted:
        # Targeted mode: explicit video_ids to (re)embed. Useful for seeding
        # specific anchors or refreshing after a sprite changes.
        ids_clean = [v for v in targeted if isinstance(v, str)
                     and re.fullmatch(r"[A-Za-z0-9_-]{5,32}", v)][:limit]
        if not ids_clean:
            return jsonify({"ok": False, "error": "no valid video_ids"}), 400
        ph = ",".join("?" for _ in ids_clean)
        rows = db.execute(
            f"""SELECT video_id FROM videos
                 WHERE video_id IN ({ph})
                   AND COALESCE(is_removed,0)=0
                 ORDER BY video_id ASC""",
            ids_clean,
        ).fetchall()
    elif since:
        rows = db.execute(
            """SELECT v.video_id
                 FROM videos v
                 LEFT JOIN video_visual_embeddings ve ON ve.video_id = v.video_id
                WHERE COALESCE(v.is_removed, 0) = 0
                  AND ve.video_id IS NULL
                  AND v.video_id > ?
                ORDER BY v.video_id ASC
                LIMIT ?""",
            (since, limit),
        ).fetchall()
    else:
        rows = db.execute(
            """SELECT v.video_id
                 FROM videos v
                 LEFT JOIN video_visual_embeddings ve ON ve.video_id = v.video_id
                WHERE COALESCE(v.is_removed, 0) = 0
                  AND ve.video_id IS NULL
                ORDER BY v.video_id ASC
                LIMIT ?""",
            (limit,),
        ).fetchall()

    started = time.time()
    written, failed = 0, 0
    last_id = ""
    errors = []
    sample_caption = ""
    for r in rows:
        last_id = r["video_id"]
        result = _uv_record_for_video(r["video_id"])
        if result.get("ok"):
            written += 1
            if result.get("skipped"):
                pass
            elif not sample_caption:
                conn = sqlite3.connect(str(DB_PATH))
                row = conn.execute(
                    "SELECT caption FROM video_visual_embeddings WHERE video_id = ?",
                    (r["video_id"],),
                ).fetchone()
                conn.close()
                if row:
                    sample_caption = (row[0] or "")[:200]
        else:
            failed += 1
            if len(errors) < 10:
                errors.append({"video_id": r["video_id"],
                               "error": result.get("error", "unknown")})

    elapsed = time.time() - started
    return jsonify({
        "ok": True,
        "vision_model": VISUAL_MODEL_VISION,
        "embed_model": VISUAL_MODEL_EMBED,
        "written": written,
        "failed": failed,
        "elapsed_s": round(elapsed, 2),
        "last_video_id": last_id,
        "next_call": (
            {"since_video_id": last_id, "limit": limit}
            if rows and len(rows) >= limit else None
        ),
        "errors": errors,
        "sample_caption": sample_caption,
    })


def _ue_ensure_schema():
    """Lazy-create video_embeddings table."""
    conn = sqlite3.connect(str(DB_PATH))
    try:
        conn.executescript(
            """
            CREATE TABLE IF NOT EXISTS video_embeddings (
                video_id     TEXT PRIMARY KEY,
                model        TEXT NOT NULL,
                dim          INTEGER NOT NULL,
                bytes        BLOB NOT NULL,           -- numpy float32 .tobytes()
                text_sha     TEXT DEFAULT '',         -- hash of source text; refresh if title/desc changes
                created_at   REAL NOT NULL,
                updated_at   REAL NOT NULL
            );
            CREATE INDEX IF NOT EXISTS idx_video_embeddings_updated
                ON video_embeddings(updated_at DESC);
            """
        )
        conn.commit()
    finally:
        conn.close()


def _ue_text_for_video(video_row):
    """Build the source text that gets embedded for a video.

    Combines title + description + tags + scene_description + category
    into a single short prompt. Title is weighted by repetition.
    """
    def _norm(s, n=400):
        return (s or "").strip()[:n]

    title = _norm(video_row.get("title", "") if isinstance(video_row, dict) else video_row["title"], 200)
    desc = _norm(video_row.get("description", "") if isinstance(video_row, dict) else (video_row["description"] if "description" in video_row.keys() else ""))
    scene = _norm(video_row.get("scene_description", "") if isinstance(video_row, dict) else (video_row["scene_description"] if "scene_description" in video_row.keys() else ""))
    cat = _norm(video_row.get("category", "") if isinstance(video_row, dict) else (video_row["category"] if "category" in video_row.keys() else ""), 32)
    tags_raw = video_row.get("tags", "[]") if isinstance(video_row, dict) else (video_row["tags"] if "tags" in video_row.keys() else "[]")
    try:
        tags = json.loads(tags_raw) if isinstance(tags_raw, str) else (tags_raw or [])
    except Exception:
        tags = []
    tag_str = ", ".join(t for t in tags if isinstance(t, str))[:200]

    # Title gets twice the weight via repetition; description and scene
    # provide semantic ground.
    parts = []
    if title:
        parts.append(f"Title: {title}")
        parts.append(title)
    if cat and cat != "other":
        parts.append(f"Category: {cat}")
    if desc:
        parts.append(f"Description: {desc}")
    if scene:
        parts.append(f"Scene: {scene}")
    if tag_str:
        parts.append(f"Tags: {tag_str}")
    return "\n".join(parts).strip() or (title or "(no metadata)")


def _ue_rate_wait():
    """Leaky-bucket throttle that keeps the project under the free-tier 100/min."""
    while True:
        with _EMB_RATE_LOCK:
            now = time.time()
            wait = _EMB_RATE_NEXT_OK_AT[0] - now
            if wait <= 0:
                _EMB_RATE_NEXT_OK_AT[0] = now + _EMB_RATE_MIN_GAP
                return
        time.sleep(min(2.0, max(0.05, wait)))


def _ue_rate_back_off(seconds):
    """Push the next-OK timestamp out after a 429."""
    with _EMB_RATE_LOCK:
        target = time.time() + max(_EMB_RATE_429_BACKOFF_FLOOR, float(seconds))
        if target > _EMB_RATE_NEXT_OK_AT[0]:
            _EMB_RATE_NEXT_OK_AT[0] = target


def _ue_parse_retry_delay(http_error):
    """Pull the retryDelay seconds out of a Gemini 429 response body."""
    try:
        body = http_error.read()
        if isinstance(body, bytes):
            body = body.decode("utf-8", errors="replace")
        data = json.loads(body)
        for d in (data.get("error", {}) or {}).get("details", []) or []:
            if "RetryInfo" in (d.get("@type") or ""):
                rd = d.get("retryDelay") or ""
                m = re.match(r"^(\d+(?:\.\d+)?)s$", rd)
                if m:
                    return float(m.group(1))
    except Exception:
        pass
    return _EMB_RATE_429_BACKOFF_FLOOR


def _ue_embed_text(text, attempts=4):
    """Call Gemini embedContent. Returns numpy float32 array or None on error.

    Throttled to stay under 100 RPM (free-tier embed quota). On 429,
    sleeps for the server-supplied retryDelay and retries.
    """
    if not text:
        return None
    key = os.environ.get("GEMINI_API_KEY", "")
    if not key:
        return None
    body = json.dumps({
        "content": {"parts": [{"text": text[:8000]}]},
    }).encode("utf-8")
    try:
        import numpy as _np
    except ImportError:
        return None

    last_err = None
    for attempt in range(1, attempts + 1):
        _ue_rate_wait()
        req = _ue_urlreq.Request(
            EMBEDDING_API_URL + "?key=" + key,
            data=body,
            headers={"Content-Type": "application/json"},
            method="POST",
        )
        try:
            with _ue_urlreq.urlopen(req, timeout=20) as resp:
                data = json.loads(resp.read().decode("utf-8"))
            vals = data.get("embedding", {}).get("values") or []
            if not vals:
                return None
            arr = _np.asarray(vals, dtype=_np.float32)
            n = float(_np.linalg.norm(arr))
            if n > 0:
                arr = arr / n
            return arr
        except _ue_urlerr.HTTPError as e:
            if e.code == 429:
                delay = _ue_parse_retry_delay(e)
                _ue_rate_back_off(delay)
                last_err = e
                continue
            try:
                app.logger.warning("gemini embed http %s", e)
            except Exception:
                pass
            return None
        except (_ue_urlerr.URLError, TimeoutError, OSError) as e:
            last_err = e
            time.sleep(min(8.0, 1.0 * attempt))
            continue
        except Exception as e:
            try:
                app.logger.warning("gemini embed error: %s", e)
            except Exception:
                pass
            return None
    try:
        app.logger.warning("gemini embed gave up after %d attempts: %s", attempts, last_err)
    except Exception:
        pass
    return None


def _ue_text_sha(text):
    return hashlib.sha256((text or "").encode("utf-8")).hexdigest()[:24]


def _ue_record_for_video(video_id, video_row=None):
    """Compute and persist an embedding for a video. Idempotent on text_sha."""
    _ue_ensure_schema()
    try:
        import numpy as _np
    except ImportError:
        return {"ok": False, "error": "numpy missing"}

    if video_row is None:
        db = get_db()
        video_row = db.execute(
            """SELECT video_id, title, description, tags, category, scene_description
                 FROM videos WHERE video_id = ?""",
            (video_id,),
        ).fetchone()
        if not video_row:
            return {"ok": False, "error": "not_found"}

    text = _ue_text_for_video(video_row)
    text_sha = _ue_text_sha(text)

    # Check existing row — skip only if text *and* model match the current
    # config. Any mismatch (newer model, source-text edit) triggers a fresh
    # embed so the row is replaced.
    conn = sqlite3.connect(str(DB_PATH))
    try:
        existing = conn.execute(
            "SELECT text_sha, model FROM video_embeddings WHERE video_id = ?",
            (video_id,),
        ).fetchone()
        if (existing
                and existing[0] == text_sha
                and existing[1] == EMBEDDING_MODEL):
            return {"ok": True, "skipped": True, "reason": "unchanged"}
    finally:
        conn.close()

    arr = _ue_embed_text(text)
    if arr is None:
        return {"ok": False, "error": "embed_failed"}
    arr = arr.astype("float32", copy=False)

    conn = sqlite3.connect(str(DB_PATH))
    try:
        conn.execute(
            """INSERT OR REPLACE INTO video_embeddings
                   (video_id, model, dim, bytes, text_sha, created_at, updated_at)
               VALUES (?, ?, ?, ?, ?, COALESCE((SELECT created_at FROM video_embeddings WHERE video_id=?), ?), ?)""",
            (video_id, EMBEDDING_MODEL, int(arr.shape[0]), arr.tobytes(),
             text_sha, video_id, time.time(), time.time()),
        )
        conn.commit()
    finally:
        conn.close()

    # Invalidate the in-memory cache; lazy reload on next query.
    with _EMB_CACHE_LOCK:
        _EMB_CACHE["matrix"] = None
        _EMB_CACHE["ids"] = []
        _EMB_CACHE["loaded_at"] = 0.0
    return {"ok": True, "dim": int(arr.shape[0]), "text_sha": text_sha}


def _ue_record_for_video_async(video_id):
    try:
        threading.Thread(
            target=_ue_record_for_video, args=(video_id,),
            daemon=True, name=f"embed-{video_id}",
        ).start()
    except Exception as e:
        try:
            app.logger.warning("embed async dispatch failed: %s", e)
        except Exception:
            pass


def _ue_cache_warm():
    """Materialize the embedding matrix from disk into memory."""
    try:
        import numpy as _np
    except ImportError:
        return False
    _ue_ensure_schema()
    conn = sqlite3.connect(str(DB_PATH))
    conn.row_factory = sqlite3.Row
    try:
        rows = conn.execute(
            """SELECT e.video_id, e.dim, e.bytes
                 FROM video_embeddings e
                 JOIN videos v ON v.video_id = e.video_id
                WHERE COALESCE(v.is_removed, 0) = 0
                  AND e.model = ?
                ORDER BY e.video_id""",
            (EMBEDDING_MODEL,),
        ).fetchall()
    finally:
        conn.close()
    if not rows:
        with _EMB_CACHE_LOCK:
            _EMB_CACHE.update({"matrix": None, "ids": [], "loaded_at": time.time()})
        return False
    n = len(rows)
    dim = int(rows[0]["dim"])
    M = _np.zeros((n, dim), dtype=_np.float32)
    ids = []
    for i, r in enumerate(rows):
        if int(r["dim"]) != dim:
            continue
        try:
            M[i] = _np.frombuffer(r["bytes"], dtype=_np.float32)
        except Exception:
            continue
        ids.append(r["video_id"])
    with _EMB_CACHE_LOCK:
        _EMB_CACHE.update({"matrix": M, "ids": ids, "loaded_at": time.time()})
    return True


def _ue_top_k_for_video(video_id, k=10, exclude_self=True):
    """Cosine top-K similar videos to `video_id`. Returns list of (video_id, score)."""
    try:
        import numpy as _np
    except ImportError:
        return []
    with _EMB_CACHE_LOCK:
        M = _EMB_CACHE.get("matrix")
        ids = _EMB_CACHE.get("ids", [])
        loaded = _EMB_CACHE.get("loaded_at", 0)
    if M is None or not ids or (time.time() - loaded > 600):
        _ue_cache_warm()
        with _EMB_CACHE_LOCK:
            M = _EMB_CACHE.get("matrix")
            ids = _EMB_CACHE.get("ids", [])
    if M is None or not ids:
        return []

    if video_id not in ids:
        # Lazily compute on demand
        result = _ue_record_for_video(video_id)
        if not result.get("ok"):
            return []
        _ue_cache_warm()
        with _EMB_CACHE_LOCK:
            M = _EMB_CACHE.get("matrix")
            ids = _EMB_CACHE.get("ids", [])
        if video_id not in ids:
            return []

    idx = ids.index(video_id)
    q = M[idx]
    # Vectors are L2-normalized → cosine = dot product
    sims = M @ q
    # exclude self
    if exclude_self:
        sims[idx] = -1.0
    k = max(1, min(k, len(ids)))
    top_idx = _np.argpartition(-sims, k - 1)[:k]
    top_idx = top_idx[_np.argsort(-sims[top_idx])]
    return [(ids[int(i)], float(sims[int(i)])) for i in top_idx]


@app.route("/api/videos/<video_id>/similar")
def api_videos_similar(video_id):
    """Top-K cosine-similar videos based on text embedding."""
    try:
        k = max(1, min(50, int(request.args.get("k", 10))))
    except Exception:
        k = 10
    db = get_db()
    video = db.execute(
        """SELECT 1 FROM videos
           WHERE video_id = ? AND COALESCE(is_removed, 0) = 0""",
        (video_id,),
    ).fetchone()
    if not video:
        return jsonify({
            "ok": False,
            "error": "video not found",
            "video_id": video_id,
        }), 404

    pairs = _ue_top_k_for_video(video_id, k=k, exclude_self=True)
    if not pairs:
        return jsonify({"ok": False, "error": "no_embeddings_yet", "video_id": video_id}), 404

    placeholders = ",".join("?" for _ in pairs)
    rows = db.execute(
        f"""SELECT v.video_id, v.title, v.thumbnail, v.duration_sec, v.views,
                   v.created_at, a.agent_name, a.display_name, a.is_human
              FROM videos v JOIN agents a ON v.agent_id = a.id
             WHERE v.video_id IN ({placeholders})""",
        [vid for vid, _ in pairs],
    ).fetchall()
    by_id = {r["video_id"]: r for r in rows}
    out = []
    for vid, score in pairs:
        r = by_id.get(vid)
        if not r:
            continue
        out.append({
            "video_id": r["video_id"],
            "title": r["title"],
            "thumbnail": r["thumbnail"],
            "duration_sec": r["duration_sec"],
            "views": r["views"],
            "created_at": r["created_at"],
            "agent_name": r["agent_name"],
            "display_name": r["display_name"],
            "is_human": bool(r["is_human"]),
            "score": round(score, 4),
        })
    return jsonify({
        "ok": True,
        "video_id": video_id,
        "model": EMBEDDING_MODEL,
        "count": len(out),
        "results": out,
    })


@app.route("/admin/embeddings/backfill", methods=["POST"])
def admin_embeddings_backfill():
    """Backfill embeddings for videos missing one. Resumable, batched."""
    if not _ts_admin_ok():
        return jsonify({"ok": False, "error": "unauthorized"}), 401
    _ue_ensure_schema()
    data = request.get_json(silent=True) or {}
    try:
        limit = max(1, min(200, int(data.get("limit", 50))))
    except Exception:
        limit = 50
    since = (data.get("since_video_id") or "").strip()
    try:
        concurrency = max(1, min(8, int(data.get("concurrency", 2))))
    except Exception:
        concurrency = 2

    db = get_db()
    if since:
        rows = db.execute(
            """SELECT v.video_id, v.title, v.description, v.tags, v.category, v.scene_description
                 FROM videos v
                 LEFT JOIN video_embeddings e
                        ON e.video_id = v.video_id AND e.model = ?
                WHERE COALESCE(v.is_removed, 0) = 0
                  AND e.video_id IS NULL
                  AND v.video_id > ?
                ORDER BY v.video_id ASC
                LIMIT ?""",
            (EMBEDDING_MODEL, since, limit),
        ).fetchall()
    else:
        rows = db.execute(
            """SELECT v.video_id, v.title, v.description, v.tags, v.category, v.scene_description
                 FROM videos v
                 LEFT JOIN video_embeddings e
                        ON e.video_id = v.video_id AND e.model = ?
                WHERE COALESCE(v.is_removed, 0) = 0
                  AND e.video_id IS NULL
                ORDER BY v.video_id ASC
                LIMIT ?""",
            (EMBEDDING_MODEL, limit),
        ).fetchall()

    started = time.time()
    written, failed = 0, 0
    last_id = ""
    errors = []
    if rows:
        # Build dicts so the worker doesn't need a DB connection.
        items = [{
            "video_id": r["video_id"],
            "title": r["title"], "description": r["description"],
            "tags": r["tags"], "category": r["category"],
            "scene_description": r["scene_description"],
        } for r in rows]

        with _cf2.ThreadPoolExecutor(max_workers=concurrency) as ex:
            futs = {ex.submit(_ue_record_for_video, item["video_id"], item): item["video_id"] for item in items}
            for fut in _cf2.as_completed(futs):
                vid = futs[fut]
                last_id = max(last_id, vid)
                try:
                    res = fut.result(timeout=60)
                    if res and res.get("ok"):
                        written += 1
                    else:
                        failed += 1
                        if len(errors) < 10:
                            errors.append({"video_id": vid, "error": (res or {}).get("error", "unknown")})
                except Exception as e:
                    failed += 1
                    if len(errors) < 10:
                        errors.append({"video_id": vid, "error": str(e)})

    elapsed = time.time() - started
    return jsonify({
        "ok": True,
        "model": EMBEDDING_MODEL,
        "written": written,
        "failed": failed,
        "elapsed_s": round(elapsed, 2),
        "last_video_id": last_id,
        "next_call": (
            {"since_video_id": last_id, "limit": limit, "concurrency": concurrency}
            if rows and len(rows) >= limit else None
        ),
        "errors": errors,
    })


# --- NCMEC submission queue + quarantine -----------------------------------
# 18 U.S.C. § 2258A obligates a "provider" to report apparent child sexual
# abuse material to NCMEC's CyberTipline as soon as reasonably possible
# after obtaining actual knowledge, and to preserve the content for 90 days
# (§ 2258A(h)(2)(A)). This module:
#   * Creates a quarantine directory and a metadata sidecar per incident.
#   * Replaces the previous "unlink on match" behaviour with a move to
#     quarantine, restricted file mode (0600), and an audit row.
#   * Enqueues an NCMEC report row that the operator drafts and submits
#     manually via report.cybertip.org until full ESP API enrollment is
#     active.
#
# This is preparedness, not a CyberTipline integration: the actual
# CyberTipline submission still happens through the human operator
# pasting the generated packet into the NCMEC web form, with the NCMEC
# report ID written back via /admin/ncmec/mark-submitted.

QUARANTINE_DIR = BASE_DIR / "quarantine"


def _ts_quarantine_dir():
    """Create the quarantine dir on first use with restrictive perms."""
    QUARANTINE_DIR.mkdir(parents=True, exist_ok=True)
    try:
        os.chmod(QUARANTINE_DIR, 0o700)
    except Exception:
        pass
    return QUARANTINE_DIR


def _ts_ensure_ncmec_schema():
    """Lazy-create ncmec_reports table. Idempotent."""
    _ensure_ts_schema()  # piggyback on the broader trust+safety schema
    conn = sqlite3.connect(str(DB_PATH))
    try:
        conn.executescript(
            """
            CREATE TABLE IF NOT EXISTS ncmec_reports (
                id                  INTEGER PRIMARY KEY AUTOINCREMENT,
                queue_id            TEXT UNIQUE NOT NULL,
                source_type         TEXT NOT NULL,        -- blocklist_hit, user_report, manual
                source_id           TEXT DEFAULT '',      -- hash_sha256 OR moderation report_id
                category            TEXT DEFAULT 'csam',  -- csam, ncii, minor (others may exist later)
                target_kind         TEXT DEFAULT '',      -- video, upload_attempt, comment
                target_video_id     TEXT DEFAULT '',
                involved_agent_id   INTEGER DEFAULT 0,
                involved_agent_name TEXT DEFAULT '',
                involved_email      TEXT DEFAULT '',
                involved_ip         TEXT DEFAULT '',
                involved_user_agent TEXT DEFAULT '',
                canonical_sha256    TEXT DEFAULT '',
                file_size           INTEGER DEFAULT 0,
                quarantine_path     TEXT DEFAULT '',      -- absolute path on disk
                discovered_at       REAL NOT NULL,
                discovery_method    TEXT DEFAULT '',
                status              TEXT DEFAULT 'pending', -- pending, drafted, submitted, acknowledged, closed
                ncmec_report_id     TEXT DEFAULT '',      -- assigned by NCMEC after submission
                submitted_at        REAL DEFAULT 0,
                submitted_by        TEXT DEFAULT '',
                notes               TEXT DEFAULT '',
                created_at          REAL NOT NULL,
                updated_at          REAL NOT NULL
            );
            CREATE INDEX IF NOT EXISTS idx_ncmec_status
                ON ncmec_reports(status, created_at DESC);
            CREATE INDEX IF NOT EXISTS idx_ncmec_target
                ON ncmec_reports(target_video_id);
            """
        )
        conn.commit()
    finally:
        conn.close()


def _ts_quarantine_file(src_path, sha, category="csam", meta=None):
    """Move a flagged upload to the quarantine dir with restrictive perms.

    Writes a JSON sidecar with discovery metadata. The original file is
    *not* deleted — § 2258A(h) requires 90-day preservation. The caller is
    responsible for separately invoking the NCMEC enqueue helper.

    Returns the quarantine file path on success, "" on failure.
    """
    try:
        qd = _ts_quarantine_dir()
        ts = int(time.time())
        # Predictable but unique filename: {category}_{sha8}_{epoch}.bin
        qname = f"{category}_{sha[:12] if sha else 'unknown'}_{ts}.bin"
        qpath = qd / qname
        # Atomic-ish move; cross-fs fallback to copy+unlink.
        try:
            os.replace(str(src_path), str(qpath))
        except OSError:
            import shutil as _sh
            _sh.copy2(str(src_path), str(qpath))
            try:
                Path(src_path).unlink(missing_ok=True)
            except Exception:
                pass
        try:
            os.chmod(qpath, 0o600)
        except Exception:
            pass
        # Sidecar metadata
        side = qpath.with_suffix(".json")
        side.write_text(json.dumps({
            "category": category,
            "sha256": sha,
            "src_path": str(src_path),
            "quarantined_at": ts,
            "preserve_until": ts + 86400 * 90,
            "meta": meta or {},
        }, indent=2))
        try:
            os.chmod(side, 0o600)
        except Exception:
            pass
        return str(qpath)
    except Exception as e:
        try:
            app.logger.error("quarantine failed: %s", e)
        except Exception:
            pass
        return ""


def _ncmec_enqueue(source_type, category="csam", target_kind="upload_attempt",
                   target_video_id="", source_id="",
                   agent_id=0, agent_name="", email="",
                   ip="", user_agent="",
                   sha256="", file_size=0, quarantine_path="",
                   discovery_method="", notes=""):
    """Insert a new NCMEC report queue row. Returns queue_id (or '')."""
    try:
        _ts_ensure_ncmec_schema()
        queue_id = "ncmec_" + secrets.token_hex(8)
        now = time.time()
        conn = sqlite3.connect(str(DB_PATH))
        try:
            conn.execute(
                """INSERT INTO ncmec_reports
                       (queue_id, source_type, source_id, category, target_kind,
                        target_video_id, involved_agent_id, involved_agent_name,
                        involved_email, involved_ip, involved_user_agent,
                        canonical_sha256, file_size, quarantine_path,
                        discovered_at, discovery_method, status, notes,
                        created_at, updated_at)
                   VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending', ?, ?, ?)""",
                (queue_id, source_type, source_id, category, target_kind,
                 target_video_id, agent_id, agent_name,
                 email, ip, user_agent,
                 sha256, file_size, quarantine_path,
                 now, discovery_method, notes,
                 now, now),
            )
            conn.commit()
        finally:
            conn.close()
        _ts_log_audit("system", "ncmec_enqueue", "ncmec_report", queue_id,
                      reason=f"{source_type}/{category}", severity="critical",
                      meta={"target_video_id": target_video_id,
                            "agent_id": agent_id, "sha256": sha256})
        return queue_id
    except Exception as e:
        try:
            app.logger.error("ncmec enqueue failed: %s", e)
        except Exception:
            pass
        return ""


def _ncmec_packet_text(row):
    """Render a CyberTipline-style submission packet from a queue row.

    The operator pastes this into report.cybertip.org until full ESP-API
    enrollment is active. Field labels mirror the NCMEC web form sections.
    """
    def _fmt(t):
        if not t:
            return "(unknown)"
        try:
            return datetime.datetime.utcfromtimestamp(float(t)).strftime("%Y-%m-%d %H:%M:%S UTC")
        except Exception:
            return str(t)

    sections = []
    sections.append("NCMEC CyberTipline Submission Packet (DRAFT)")
    sections.append("=" * 72)
    sections.append(f"Queue ID:           {row['queue_id']}")
    sections.append(f"Status:             {row['status']}")
    sections.append(f"Discovered (UTC):   {_fmt(row['discovered_at'])}")
    sections.append(f"Source type:        {row['source_type']}")
    sections.append(f"Discovery method:   {row['discovery_method'] or '(unspecified)'}")
    sections.append(f"Category:           {row['category']}")
    sections.append("")

    sections.append("REPORTING ELECTRONIC SERVICE PROVIDER")
    sections.append("-" * 72)
    sections.append("Name:               Elyan Labs (BoTTube)")
    sections.append("Address:            Lake Charles, Louisiana, USA")
    sections.append("Designated contact: abuse@elyanlabs.ai")
    sections.append("Service URL:        https://bottube.ai")
    sections.append("Statutory basis:    18 U.S.C. § 2258A — mandatory CyberTipline reporting.")
    sections.append("Preservation:       File preserved on quarantine storage; will be retained")
    sections.append("                    at least 90 days from discovery per § 2258A(h)(2)(A).")
    sections.append("")

    sections.append("INVOLVED PERSON / ACCOUNT")
    sections.append("-" * 72)
    sections.append(f"Agent name:         {row['involved_agent_name'] or '(unknown)'}")
    sections.append(f"Internal agent id:  {row['involved_agent_id'] or '(unknown)'}")
    sections.append(f"Account email:      {row['involved_email'] or '(unknown)'}")
    sections.append(f"Originating IP:     {row['involved_ip'] or '(unknown)'}")
    sections.append(f"User-Agent:         {row['involved_user_agent'] or '(unknown)'}")
    sections.append("")

    sections.append("INVOLVED CONTENT")
    sections.append("-" * 72)
    sections.append(f"Target kind:        {row['target_kind'] or '(unknown)'}")
    sections.append(f"Target video id:    {row['target_video_id'] or '(none)'}")
    sections.append(f"Public URL:         "
                    + (f"https://bottube.ai/watch/{row['target_video_id']}"
                       if row['target_video_id'] else "(blocked at upload — not published)"))
    sections.append(f"Canonical SHA-256:  {row['canonical_sha256'] or '(unknown)'}")
    sections.append(f"File size (bytes):  {row['file_size']}")
    sections.append(f"Quarantine path:    {row['quarantine_path'] or '(not preserved — investigate)'}")
    sections.append("")

    sections.append("PROVIDER NOTES")
    sections.append("-" * 72)
    sections.append(row["notes"] or "(none)")
    sections.append("")
    sections.append("=" * 72)
    sections.append("ACTION REQUIRED:")
    sections.append("  1. Open https://report.cybertip.org and select 'Electronic Service Provider'.")
    sections.append("  2. Paste the fields above into the corresponding form sections.")
    sections.append("  3. Upload the quarantined file from the path above (do NOT redact hash or alter the file).")
    sections.append("  4. After submission, NCMEC returns a Report ID. Record it via:")
    sections.append("       POST /admin/ncmec/mark-submitted")
    sections.append("       body: {\"queue_id\": \"" + row["queue_id"] + "\", \"ncmec_report_id\": \"<id>\"}")
    sections.append("  5. Do not modify or delete the quarantined content for 90 days from the")
    sections.append("     discovered date above (§ 2258A(h)(2)(A)). Cooperate with any law-enforcement")
    sections.append("     preservation request.")
    return "\n".join(sections) + "\n"


@app.route("/admin/ncmec/queue")
def admin_ncmec_queue():
    """Pending NCMEC reports, ordered by oldest-first within severity."""
    if not _ts_admin_ok():
        return jsonify({"ok": False, "error": "unauthorized"}), 401
    _ts_ensure_ncmec_schema()
    db = get_db()
    rows = db.execute(
        """SELECT queue_id, source_type, category, target_video_id,
                  involved_agent_name, canonical_sha256,
                  status, ncmec_report_id, discovered_at, created_at
             FROM ncmec_reports
            WHERE status IN ('pending', 'drafted')
            ORDER BY discovered_at ASC
            LIMIT 200"""
    ).fetchall()
    return jsonify({
        "ok": True,
        "count": len(rows),
        "queue": [
            {
                "queue_id": r["queue_id"],
                "source_type": r["source_type"],
                "category": r["category"],
                "target_video_id": r["target_video_id"],
                "involved_agent_name": r["involved_agent_name"],
                "canonical_sha256": r["canonical_sha256"],
                "status": r["status"],
                "ncmec_report_id": r["ncmec_report_id"],
                "discovered_at": r["discovered_at"],
                "age_hours": round((time.time() - r["discovered_at"]) / 3600, 2),
            } for r in rows
        ],
    })


@app.route("/admin/ncmec/draft/<queue_id>")
def admin_ncmec_draft(queue_id):
    """Render a CyberTipline submission packet for an operator to paste."""
    if not _ts_admin_ok():
        return jsonify({"ok": False, "error": "unauthorized"}), 401
    if not re.fullmatch(r"ncmec_[a-f0-9]{8,32}", queue_id):
        return jsonify({"ok": False, "error": "invalid queue_id"}), 400
    _ts_ensure_ncmec_schema()
    db = get_db()
    row = db.execute(
        "SELECT * FROM ncmec_reports WHERE queue_id = ?",
        (queue_id,),
    ).fetchone()
    if not row:
        return jsonify({"ok": False, "error": "not found"}), 404

    # Bump status to drafted on first read so the queue order is honest.
    if row["status"] == "pending":
        db.execute(
            "UPDATE ncmec_reports SET status = 'drafted', updated_at = ? WHERE queue_id = ?",
            (time.time(), queue_id),
        )
        db.commit()
        _ts_log_audit("admin", "ncmec_draft", "ncmec_report", queue_id,
                      reason="operator viewed draft", severity="high")

    packet = _ncmec_packet_text(row)
    return Response(packet, mimetype="text/plain", headers={
        "Content-Disposition": f'attachment; filename="{queue_id}.txt"',
        "Cache-Control": "no-store",
    })


@app.route("/admin/ncmec/mark-submitted", methods=["POST"])
def admin_ncmec_mark_submitted():
    """Operator records that a packet was filed with NCMEC; stores their report ID."""
    if not _ts_admin_ok():
        return jsonify({"ok": False, "error": "unauthorized"}), 401
    _ts_ensure_ncmec_schema()
    data = request.get_json(silent=True) or {}
    queue_id = (data.get("queue_id") or "").strip()
    ncmec_id = (data.get("ncmec_report_id") or "").strip()[:128]
    submitter = (data.get("submitter") or "operator").strip()[:64]

    if not re.fullmatch(r"ncmec_[a-f0-9]{8,32}", queue_id):
        return jsonify({"ok": False, "error": "invalid queue_id"}), 400
    if not ncmec_id:
        return jsonify({"ok": False, "error": "ncmec_report_id required"}), 400

    db = get_db()
    row = db.execute(
        "SELECT queue_id, status FROM ncmec_reports WHERE queue_id = ?", (queue_id,)
    ).fetchone()
    if not row:
        return jsonify({"ok": False, "error": "not found"}), 404

    db.execute(
        """UPDATE ncmec_reports
              SET status = 'submitted',
                  ncmec_report_id = ?,
                  submitted_at = ?,
                  submitted_by = ?,
                  updated_at = ?
            WHERE queue_id = ?""",
        (ncmec_id, time.time(), submitter, time.time(), queue_id),
    )
    db.commit()
    _ts_log_audit("admin", "ncmec_submit", "ncmec_report", queue_id,
                  reason=f"NCMEC id {ncmec_id}", severity="critical",
                  meta={"submitter": submitter})
    return jsonify({"ok": True, "queue_id": queue_id, "ncmec_report_id": ncmec_id})


# --- Provenance write-path -------------------------------------------------
# Populate video_provenance on every upload so the pill flips from gray
# (unverified) to amber (pending) immediately. Anchor TX is filled in by
# a separate Ergo anchor job; once both uploader_sig and anchor_tx_hash
# are present, _build_provenance_payload() flips it to verified (green).

def _provenance_signing_key():
    """The platform secret used to sign canonical-asset manifests.

    Falls back to BOTTUBE_SECRET_KEY (Flask session secret) if the
    dedicated provenance key is unset. Both are HMAC keys, never exposed
    to clients; only the resulting signature appears in the public JSON.
    """
    return (os.environ.get("BOTTUBE_PROVENANCE_KEY", "")
            or os.environ.get("BOTTUBE_SECRET_KEY", "")
            or "bottube-provenance-bootstrap")


def _provenance_uploader_sig(video_id, canonical_sha256, agent_id, uploaded_at):
    """HMAC-SHA256 platform signature over the canonical manifest line."""
    key = _provenance_signing_key().encode("utf-8")
    msg = f"{video_id}|{canonical_sha256}|{agent_id}|{int(uploaded_at)}".encode("utf-8")
    return hmac.new(key, msg, hashlib.sha256).hexdigest()


def _provenance_optional_from_form(form):
    """Pull optional generation metadata from the upload form.

    All fields are optional. Agents that supply them get a richer pill;
    legacy clients that don't get the same pending state with empty
    generation block.
    """
    def _s(k, n=200):
        return (form.get(k, "") or "").strip()[:n]
    seed = 0
    try:
        seed = int(form.get("seed", "0") or 0)
    except Exception:
        seed = 0
    generated_at = 0.0
    try:
        gen_at_raw = form.get("generated_at", "0") or 0
        generated_at = float(gen_at_raw)
    except Exception:
        generated_at = 0.0
    return {
        "model": _s("gen_model", 64) or _s("model", 64),
        "provider": _s("gen_provider", 64) or _s("provider", 64),
        "workflow_hash": _s("workflow_hash", 128),
        "prompt_hash": _s("prompt_hash", 128),
        "seed": seed,
        "generated_at": generated_at,
    }


# --- Phase 11.16: hash-tree v2 manifest ----------------------------------
# v1 leaf:  sha256(video_id | canonical_sha256 | uploader_sig | uploaded_at)
# v2 leaf:  sha256("bottube/v2"
#                  | video_id
#                  | canonical_sha256
#                  | thumbnail_sha256
#                  | canonical_360p_sha256
#                  | uploader_sig
#                  | uploaded_at)
#
# The "bottube/v2" prefix is a domain separator: a v1 leaf and a v2 leaf
# can never collide even if all the other fields happen to match, because
# the prefix differs. New uploads from this point write manifest_version=2;
# existing v1 anchors stay anchored under v1's recipe and the verifier
# branches on the manifest_version field.

MANIFEST_V1 = 1
MANIFEST_V2 = 2
MANIFEST_V3 = 3
MANIFEST_CURRENT = MANIFEST_V3  # what new uploads write

_LEAF_DOMAIN_V2 = "bottube/v2"
_LEAF_DOMAIN_V3 = "bottube/v3"

# ---------------------------------------------------------------------------
# Phase 11.23: Ed25519 creator signatures
# ---------------------------------------------------------------------------
# v3 manifest folds creator_pubkey + creator_signature into the leaf, so
# the chain anchor commits not just to *what* was uploaded but to *who*
# signed it. v3a (this iteration) uses server-managed keypairs — the
# server maintains an Ed25519 keypair per agent, signs uploads on behalf
# of the agent, and exposes the public key via /.well-known/agent/<handle>.
# v3b (future) lets agents bring their own keypair and sign uploads
# client-side; the server-side surface stays the same since v3 already
# binds to creator_pubkey + creator_signature without caring who held
# the private key.
#
# Even server-managed v3a is strictly better than HMAC:
#   - Each agent has a unique keypair, not a shared platform secret
#   - Public key is independently checkable via DID:web actor doc
#   - The chain commits to a *verifiable* signature, not an opaque blob
#   - Future migration to client-managed is purely additive

try:
    from nacl.signing import SigningKey, VerifyKey
    from nacl.exceptions import BadSignatureError
    _ED25519_AVAILABLE = True
except Exception:
    _ED25519_AVAILABLE = False


def _agent_ed25519_signing_key():
    """Master key used to encrypt agent Ed25519 private keys at rest.

    Falls back to the provenance signing key. In server-managed v3a this
    is the only secret needed to forge any creator signature; rotating it
    is non-trivial and a known operational risk. Mitigation: client-managed
    v3b removes this surface entirely.
    """
    return _provenance_signing_key()


def _agent_ed25519_seal(seckey_bytes):
    """Wrap a 32-byte Ed25519 seed with a derived key + AES-style XOR.

    Sufficient against opportunistic disk read; not a real KMS. The
    upgrade path is per-agent encryption with agent-supplied passphrases
    or a real KMS — out of scope for v3a.
    """
    if not seckey_bytes:
        return ""
    master = hashlib.sha256(_agent_ed25519_signing_key().encode("utf-8")).digest()
    out = bytes(b ^ master[i % len(master)] for i, b in enumerate(seckey_bytes))
    return out.hex()


def _agent_ed25519_unseal(sealed_hex):
    if not sealed_hex:
        return b""
    try:
        sealed = bytes.fromhex(sealed_hex)
    except Exception:
        return b""
    master = hashlib.sha256(_agent_ed25519_signing_key().encode("utf-8")).digest()
    return bytes(b ^ master[i % len(master)] for i, b in enumerate(sealed))


def _agent_ensure_keypair(agent_id):
    """Ensure agent has an Ed25519 keypair. Returns (pubkey_hex, seckey_bytes).

    Idempotent: if the keypair already exists, returns the stored values.
    Otherwise generates a fresh Ed25519 seed, stores the public key in
    plaintext and the private key XOR-sealed with the platform key.
    """
    if not _ED25519_AVAILABLE:
        return "", b""

    conn = sqlite3.connect(str(DB_PATH))
    try:
        # Idempotent column adds.
        cols = {row[1] for row in conn.execute("PRAGMA table_info(agents)").fetchall()}
        if "ed25519_pubkey" not in cols:
            conn.execute("ALTER TABLE agents ADD COLUMN ed25519_pubkey TEXT DEFAULT ''")
        if "ed25519_seckey_sealed" not in cols:
            conn.execute("ALTER TABLE agents ADD COLUMN ed25519_seckey_sealed TEXT DEFAULT ''")
        if "ed25519_created_at" not in cols:
            conn.execute("ALTER TABLE agents ADD COLUMN ed25519_created_at INTEGER DEFAULT 0")
        conn.commit()

        row = conn.execute(
            "SELECT ed25519_pubkey, ed25519_seckey_sealed FROM agents WHERE id = ?",
            (agent_id,),
        ).fetchone()
        if row and row[0] and row[1]:
            return row[0], _agent_ed25519_unseal(row[1])

        # Generate a fresh Ed25519 keypair.
        signing = SigningKey.generate()
        pubkey_hex = bytes(signing.verify_key).hex()
        seckey_seed = bytes(signing)  # 32-byte seed
        sealed = _agent_ed25519_seal(seckey_seed)

        conn.execute(
            """UPDATE agents
                  SET ed25519_pubkey = ?,
                      ed25519_seckey_sealed = ?,
                      ed25519_created_at = ?
                WHERE id = ?""",
            (pubkey_hex, sealed, int(time.time()), agent_id),
        )
        conn.commit()
        return pubkey_hex, seckey_seed
    finally:
        conn.close()


def _agent_get_pubkey(agent_id):
    """Return the agent's Ed25519 public key hex, or "" if none yet."""
    try:
        conn = sqlite3.connect(str(DB_PATH))
        try:
            row = conn.execute(
                "SELECT ed25519_pubkey FROM agents WHERE id = ?", (agent_id,),
            ).fetchone()
            return (row[0] or "") if row else ""
        finally:
            conn.close()
    except Exception:
        return ""


def _v3_signing_message(video_id, canonical_sha256, thumbnail_sha256,
                         canonical_360p_sha256, uploaded_at):
    """The 32-byte digest that an agent (or the server on its behalf)
    signs with Ed25519 for v3 provenance. Folded into the v3 leaf so the
    chain anchor commits to the signature."""
    parts = "|".join([
        "bottube/v3-sign",
        video_id or "",
        canonical_sha256 or "",
        thumbnail_sha256 or "",
        canonical_360p_sha256 or "",
        str(int(float(uploaded_at or 0))),
    ])
    return hashlib.sha256(parts.encode("utf-8")).digest()


def _agent_sign_v3(agent_id, video_id, canonical_sha256,
                   thumbnail_sha256, canonical_360p_sha256, uploaded_at):
    """Server-managed v3 signing. Returns (pubkey_hex, signature_hex).

    Used at upload time to produce a v3 manifest. v3b will replace this
    with client-supplied signatures, but the leaf shape is unchanged.
    Returns ("", "") if Ed25519 is unavailable so the caller can fall
    back to v2 cleanly.
    """
    if not _ED25519_AVAILABLE:
        return "", ""
    pubkey_hex, seed = _agent_ensure_keypair(agent_id)
    if not pubkey_hex or not seed:
        return "", ""
    msg = _v3_signing_message(
        video_id, canonical_sha256,
        thumbnail_sha256, canonical_360p_sha256,
        uploaded_at,
    )
    try:
        signing = SigningKey(seed)
        signed = signing.sign(msg)
        # signed.signature is 64 raw bytes
        return pubkey_hex, signed.signature.hex()
    except Exception:
        return "", ""


def _verify_v3_signature(pubkey_hex, signature_hex, video_id,
                          canonical_sha256, thumbnail_sha256,
                          canonical_360p_sha256, uploaded_at):
    """Verify an Ed25519 signature against the v3 signing message."""
    if not _ED25519_AVAILABLE:
        return False
    if not pubkey_hex or not signature_hex:
        return False
    try:
        msg = _v3_signing_message(
            video_id, canonical_sha256,
            thumbnail_sha256, canonical_360p_sha256,
            uploaded_at,
        )
        VerifyKey(bytes.fromhex(pubkey_hex)).verify(msg, bytes.fromhex(signature_hex))
        return True
    except (BadSignatureError, ValueError, Exception):
        return False


def _manifest_leaf_v1(video_id, canonical_sha256, uploader_sig, uploaded_at):
    parts = "|".join([
        video_id or "",
        canonical_sha256 or "",
        uploader_sig or "",
        str(int(float(uploaded_at or 0))),
    ])
    return hashlib.sha256(parts.encode("utf-8")).digest()


def _manifest_leaf_v2(video_id, canonical_sha256, thumbnail_sha256,
                      canonical_360p_sha256, uploader_sig, uploaded_at):
    parts = "|".join([
        _LEAF_DOMAIN_V2,
        video_id or "",
        canonical_sha256 or "",
        thumbnail_sha256 or "",
        canonical_360p_sha256 or "",
        uploader_sig or "",
        str(int(float(uploaded_at or 0))),
    ])
    return hashlib.sha256(parts.encode("utf-8")).digest()


def _manifest_leaf_v3(video_id, canonical_sha256, thumbnail_sha256,
                       canonical_360p_sha256, uploader_sig, uploaded_at,
                       creator_pubkey, creator_signature):
    """v3 leaf: folds creator_pubkey + creator_signature so the chain
    commits to a verifiable Ed25519 signature, not just a platform HMAC.
    Backwards-incompatible with v2 by design — domain separator differs
    so v2 and v3 leaves can never collide."""
    parts = "|".join([
        _LEAF_DOMAIN_V3,
        video_id or "",
        canonical_sha256 or "",
        thumbnail_sha256 or "",
        canonical_360p_sha256 or "",
        uploader_sig or "",
        creator_pubkey or "",
        creator_signature or "",
        str(int(float(uploaded_at or 0))),
    ])
    return hashlib.sha256(parts.encode("utf-8")).digest()


def _manifest_leaf(version, video_id, canonical_sha256,
                   thumbnail_sha256, canonical_360p_sha256,
                   uploader_sig, uploaded_at,
                   creator_pubkey="", creator_signature=""):
    """Dispatch based on manifest version."""
    v = int(version or 1)
    if v >= MANIFEST_V3:
        return _manifest_leaf_v3(
            video_id, canonical_sha256,
            thumbnail_sha256, canonical_360p_sha256,
            uploader_sig, uploaded_at,
            creator_pubkey, creator_signature,
        )
    if v >= MANIFEST_V2:
        return _manifest_leaf_v2(video_id, canonical_sha256,
                                  thumbnail_sha256, canonical_360p_sha256,
                                  uploader_sig, uploaded_at)
    return _manifest_leaf_v1(video_id, canonical_sha256, uploader_sig, uploaded_at)


def _manifest_leaf_recipe(version):
    v = int(version or 1)
    if v >= MANIFEST_V3:
        return (
            'sha256("bottube/v3" | video_id | canonical_sha256 | '
            'thumbnail_sha256 | canonical_360p_sha256 | uploader_sig | '
            'creator_pubkey | creator_signature | uploaded_at) with "|" '
            'as the literal separator. creator_signature is the hex-encoded '
            '64-byte Ed25519 signature over '
            'sha256("bottube/v3-sign" | video_id | canonical_sha256 | '
            'thumbnail_sha256 | canonical_360p_sha256 | uploaded_at), '
            'verifiable against creator_pubkey.'
        )
    if v >= MANIFEST_V2:
        return (
            'sha256("bottube/v2" | video_id | canonical_sha256 | '
            'thumbnail_sha256 | canonical_360p_sha256 | uploader_sig | '
            'uploaded_at) with "|" as the literal separator and uploaded_at '
            'as integer seconds'
        )
    return (
        "sha256(video_id | canonical_sha256 | uploader_sig | uploaded_at) "
        "with '|' as the literal separator and uploaded_at as integer seconds"
    )


def _provenance_ensure_v2_columns():
    """Add manifest_version + canonical_360p_sha256 columns idempotently."""
    conn = sqlite3.connect(str(DB_PATH))
    try:
        cols = {row[1] for row in conn.execute("PRAGMA table_info(video_provenance)").fetchall()}
        if "manifest_version" not in cols:
            conn.execute("ALTER TABLE video_provenance ADD COLUMN manifest_version INTEGER DEFAULT 1")
        if "canonical_360p_sha256" not in cols:
            conn.execute("ALTER TABLE video_provenance ADD COLUMN canonical_360p_sha256 TEXT DEFAULT ''")
        conn.commit()
    finally:
        conn.close()


def _provenance_ensure_v3_columns():
    """Phase 11.23: add creator_signature column for v3 leaves.
    creator_pubkey already exists from earlier phases."""
    conn = sqlite3.connect(str(DB_PATH))
    try:
        cols = {row[1] for row in conn.execute("PRAGMA table_info(video_provenance)").fetchall()}
        if "creator_signature" not in cols:
            conn.execute("ALTER TABLE video_provenance ADD COLUMN creator_signature TEXT DEFAULT ''")
        # creator_pubkey almost certainly exists, but defensively:
        if "creator_pubkey" not in cols:
            conn.execute("ALTER TABLE video_provenance ADD COLUMN creator_pubkey TEXT DEFAULT ''")
        conn.commit()
    finally:
        conn.close()


def _provenance_thumbnail_sha(video_id, thumb_filename):
    """SHA-256 of the served thumbnail file (best-effort)."""
    if not thumb_filename:
        return ""
    try:
        path = THUMB_DIR / thumb_filename
        if not path.exists() or path.stat().st_size < 8:
            return ""
        return _ts_sha256_file(path)
    except Exception:
        return ""


def _provenance_ensure_thumb_column():
    """Idempotently add thumbnail_sha256 column to video_provenance."""
    conn = sqlite3.connect(str(DB_PATH))
    try:
        cols = {row[1] for row in conn.execute("PRAGMA table_info(video_provenance)").fetchall()}
        if "thumbnail_sha256" not in cols:
            conn.execute("ALTER TABLE video_provenance ADD COLUMN thumbnail_sha256 TEXT DEFAULT ''")
            conn.commit()
    finally:
        conn.close()


def _provenance_record_for_upload(video_id, canonical_path, agent, form,
                                   width=0, height=0, duration=0.0,
                                   uploaded_at=None):
    """Compute SHA-256, sign, and INSERT into video_provenance.

    Idempotent (REPLACE semantics). Never raises — provenance is best-effort
    and an error here must not block the upload response.
    """
    try:
        _ensure_provenance_schema()
        _provenance_ensure_v2_columns()
        _provenance_ensure_v3_columns()
        if uploaded_at is None:
            uploaded_at = time.time()
        sha = _ts_sha256_file(canonical_path)
        agent_id = agent["id"] if hasattr(agent, "__getitem__") else int(agent or 0)
        sig = _provenance_uploader_sig(video_id, sha, agent_id, uploaded_at)
        opt = _provenance_optional_from_form(form or {})

        # Pull beacon_id / pubkey if columns exist on agents.
        creator_pubkey = ""
        creator_beacon = ""
        try:
            agent_keys = list(agent.keys()) if hasattr(agent, "keys") else []
            if "rtc_wallet" in agent_keys:
                creator_pubkey = agent["rtc_wallet"] or ""
            if not creator_pubkey and "rtc_address" in agent_keys:
                creator_pubkey = agent["rtc_address"] or ""
            if "beacon_id" in agent_keys:
                creator_beacon = agent["beacon_id"] or ""
        except Exception:
            pass

        conn = sqlite3.connect(str(DB_PATH))
        try:
            # Phase 11.23: NEW uploads default to manifest_version=MANIFEST_CURRENT (v3).
            # The pending-claim eligibility gates already require that
            # canonical_360p_sha256 != '' AND creator_signature != '' before
            # a v3 row can be claimed by the worker. So writing v3 here makes
            # the row non-anchorable until the rendition pipeline finishes
            # populating those fields — closing the upload→anchor race
            # codex review identified.
            conn.execute(
                """INSERT OR REPLACE INTO video_provenance
                       (video_id, canonical_sha256, duration_sec, width, height,
                        creator_agent_id, creator_pubkey, creator_beacon_id,
                        model, provider, workflow_hash, prompt_hash, seed,
                        generated_at, uploader_sig, uploaded_at,
                        anchor_chain, anchor_tx_hash, anchor_block_height,
                        anchor_manifest_hash, parents_json, renditions_json,
                        manifest_version,
                        verified, created_at, updated_at)
                   VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
                           ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
                (video_id, sha, duration, width, height,
                 agent_id, creator_pubkey, creator_beacon,
                 opt["model"], opt["provider"], opt["workflow_hash"],
                 opt["prompt_hash"], opt["seed"],
                 opt["generated_at"] or uploaded_at, sig, uploaded_at,
                 "", "", 0, "", "[]", "[]",
                 MANIFEST_CURRENT,
                 0, time.time(), time.time()),
            )
            conn.commit()
        finally:
            conn.close()
        return {"ok": True, "sha256": sha, "uploader_sig": sig}
    except Exception as e:
        try:
            app.logger.warning("provenance write failed for %s: %s", video_id, e)
        except Exception:
            pass
        return {"ok": False, "error": str(e)}


# --- Adaptive renditions + VMAF -------------------------------------------
# Generate a 360p variant for the human-watch lane and compute VMAF
# against the canonical 720x720 source. The result lands in
# video_renditions and surfaces in the provenance side-sheet on every
# /watch/<id> page. Encoding is async so uploads are not blocked.

from threading import Semaphore as _eng_Semaphore
import concurrent.futures as _cf2

RENDITION_DIR = BASE_DIR / "renditions"
RENDITION_FFMPEG = "/opt/ffmpeg-vmaf/ffmpeg"  # static build with libvmaf
RENDITION_VMAF_MODEL = "version=vmaf_v0.6.1"  # bundled in the static build

# Cap concurrent rendition jobs across all upload threads to keep the
# VPS from getting starved on background ffmpeg.
_RENDITION_GATE = _eng_Semaphore(2)
_RENDITION_INFLIGHT = set()
_RENDITION_INFLIGHT_LOCK = _eng_Lock()


def _renditions_dir_for(video_id):
    """Per-video subdir, created on demand."""
    d = RENDITION_DIR / video_id
    d.mkdir(parents=True, exist_ok=True)
    return d


def _renditions_ffmpeg_available():
    return Path(RENDITION_FFMPEG).is_file() and os.access(RENDITION_FFMPEG, os.X_OK)


def _renditions_encode(canonical_path, out_path, target_w, target_h, crf=24, preset="medium"):
    """Encode a downscale variant. Returns size in bytes (>0) on success, 0 on failure."""
    out_path = str(out_path)
    cmd = [
        RENDITION_FFMPEG, "-loglevel", "error", "-y",
        "-i", str(canonical_path),
        "-vf", f"scale={target_w}:{target_h}:flags=bicubic",
        "-c:v", "libx264", "-preset", preset, "-crf", str(crf),
        "-pix_fmt", "yuv420p", "-movflags", "+faststart",
        "-an",
        out_path,
    ]
    try:
        subprocess.run(cmd, check=True, timeout=120,
                       stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
        sz = Path(out_path).stat().st_size
        return sz
    except subprocess.CalledProcessError as e:
        try:
            err = (e.stderr or b"").decode("utf-8", errors="replace")[:300]
        except Exception:
            err = ""
        app.logger.warning("rendition encode failed: %s", err)
        return 0
    except subprocess.TimeoutExpired:
        app.logger.warning("rendition encode timeout")
        return 0


def _renditions_compute_vmaf(distorted_path, ref_path, ref_width, ref_height,
                             threads=2, timeout=180):
    """Run libvmaf and return the mean score, or 0.0 on error.

    Distorted is upscaled to reference dimensions before comparison
    (standard practice — VMAF is computed at the reference resolution).
    """
    log_path = Path(distorted_path).with_suffix(".vmaf.json")
    filter_chain = (
        f"[0:v]scale={ref_width}:{ref_height}:flags=bicubic[d];"
        f"[d][1:v]libvmaf=log_path={log_path}:log_fmt=json:n_threads={threads}:"
        f"model={RENDITION_VMAF_MODEL}"
    )
    cmd = [
        RENDITION_FFMPEG, "-loglevel", "error",
        "-i", str(distorted_path),
        "-i", str(ref_path),
        "-lavfi", filter_chain,
        "-f", "null", "-",
    ]
    try:
        subprocess.run(cmd, check=True, timeout=timeout,
                       stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
        if not log_path.exists():
            return 0.0
        with open(log_path, "r") as f:
            j = json.load(f)
        pooled = j.get("pooled_metrics", {}).get("vmaf", {})
        mean = float(pooled.get("mean", 0.0) or 0.0)
        return mean
    except subprocess.CalledProcessError as e:
        try:
            err = (e.stderr or b"").decode("utf-8", errors="replace")[:300]
        except Exception:
            err = ""
        app.logger.warning("vmaf compute failed: %s", err)
        return 0.0
    except subprocess.TimeoutExpired:
        app.logger.warning("vmaf compute timeout")
        return 0.0
    except Exception as e:
        app.logger.warning("vmaf parse error: %s", e)
        return 0.0
    finally:
        try:
            log_path.unlink(missing_ok=True)
        except Exception:
            pass


def _renditions_upsert(video_id, label, url_path, width, height,
                       bitrate_kbps, file_sha256, file_size, vmaf,
                       is_canonical=False):
    """Idempotent INSERT OR REPLACE into video_renditions."""
    _ensure_provenance_schema()
    conn = sqlite3.connect(str(DB_PATH))
    try:
        conn.execute(
            """INSERT OR REPLACE INTO video_renditions
                   (video_id, label, url_path, width, height, bitrate_kbps,
                    codec, file_sha256, file_size, vmaf, is_canonical, created_at)
               VALUES (?, ?, ?, ?, ?, ?, 'h264', ?, ?, ?, ?, ?)""",
            (video_id, label, url_path, width, height, bitrate_kbps,
             file_sha256, file_size, vmaf,
             1 if is_canonical else 0, time.time()),
        )
        conn.commit()
    finally:
        conn.close()


def _renditions_estimate_bitrate_kbps(file_path, duration_sec):
    """Approximate bitrate from file size + duration."""
    try:
        size = Path(file_path).stat().st_size
        if duration_sec and duration_sec > 0:
            return int((size * 8) / 1000 / duration_sec)
    except Exception:
        pass
    return 0


def _renditions_process_video(video_id):
    """Generate the 360p variant + canonical row for a single video.

    Idempotent. If the rendition already exists on disk and in DB, returns fast.
    Bounded concurrency via _RENDITION_GATE.
    """
    if not _renditions_ffmpeg_available():
        app.logger.warning("renditions: static ffmpeg missing at %s", RENDITION_FFMPEG)
        return {"ok": False, "error": "ffmpeg-vmaf not installed"}

    # Coalesce concurrent calls for the same video
    with _RENDITION_INFLIGHT_LOCK:
        if video_id in _RENDITION_INFLIGHT:
            return {"ok": False, "error": "in_flight"}
        _RENDITION_INFLIGHT.add(video_id)

    acquired = False
    try:
        if not _RENDITION_GATE.acquire(blocking=True, timeout=600):
            return {"ok": False, "error": "gate_timeout"}
        acquired = True

        conn = sqlite3.connect(str(DB_PATH))
        conn.row_factory = sqlite3.Row
        try:
            video = conn.execute(
                "SELECT video_id, filename, duration_sec, width, height "
                "FROM videos WHERE video_id = ?",
                (video_id,),
            ).fetchone()
            if not video:
                return {"ok": False, "error": "not_found"}

            canonical_path = VIDEO_DIR / (video["filename"] or "")
            if not canonical_path.exists():
                return {"ok": False, "error": "canonical_missing"}

            ref_w = int(video["width"] or 720)
            ref_h = int(video["height"] or 720)
            duration = float(video["duration_sec"] or 0)

            outdir = _renditions_dir_for(video_id)

            # Register the canonical first (no VMAF — it IS the reference)
            try:
                canon_sha = _ts_sha256_file(canonical_path)
            except Exception:
                canon_sha = ""
            canon_size = canonical_path.stat().st_size if canonical_path.exists() else 0
            canon_kbps = _renditions_estimate_bitrate_kbps(canonical_path, duration)
            _renditions_upsert(
                video_id=video_id, label="canonical",
                url_path=f"/api/videos/{video_id}/stream",
                width=ref_w, height=ref_h, bitrate_kbps=canon_kbps,
                file_sha256=canon_sha, file_size=canon_size, vmaf=0.0,
                is_canonical=True,
            )

            # 360p downscale (square, since canonical is square)
            target_dim = 360
            p360_path = outdir / "360p.mp4"
            need_encode = (
                not p360_path.exists()
                or p360_path.stat().st_size < 1024
            )
            if need_encode:
                size = _renditions_encode(
                    canonical_path, p360_path,
                    target_w=target_dim, target_h=target_dim,
                    crf=24, preset="medium",
                )
                if not size:
                    return {"ok": False, "error": "encode_360_failed"}

            # VMAF
            vmaf_score = _renditions_compute_vmaf(
                p360_path, canonical_path, ref_w, ref_h, threads=2,
            )

            try:
                p360_sha = _ts_sha256_file(p360_path)
            except Exception:
                p360_sha = ""
            p360_size = p360_path.stat().st_size
            p360_kbps = _renditions_estimate_bitrate_kbps(p360_path, duration)

            _renditions_upsert(
                video_id=video_id, label="360p",
                url_path=f"/renditions/{video_id}/360p.mp4",
                width=target_dim, height=target_dim,
                bitrate_kbps=p360_kbps,
                file_sha256=p360_sha, file_size=p360_size,
                vmaf=round(vmaf_score, 2), is_canonical=False,
            )

            # Phase 11.16 + 11.23: feed the 360p hash back into
            # video_provenance AND atomically promote the row to v3 — but
            # ONLY if the row hasn't been claimed by the anchor worker
            # yet. Once anchor_status='claimed' the worker has frozen its
            # snapshot of leaf inputs, and mutating the row here would
            # silently desync the leaf.
            #
            # v3 promotion path:
            #   1. Ensure the agent has an Ed25519 keypair
            #   2. Compute creator_signature over the v3 signing message
            #   3. Promote row to manifest_version=3 + write pubkey + sig
            # If Ed25519 isn't available (PyNaCl missing), we silently
            # downgrade to v2 — the rest of the pipeline still works.
            if p360_sha:
                try:
                    _provenance_ensure_v2_columns()
                    _provenance_ensure_v3_columns()
                    # Pull the data we need to sign: agent_id + uploader fields
                    conn_q = sqlite3.connect(str(DB_PATH))
                    conn_q.row_factory = sqlite3.Row
                    pre = conn_q.execute(
                        """SELECT video_id, canonical_sha256, uploader_sig,
                                  uploaded_at, creator_agent_id,
                                  COALESCE(thumbnail_sha256, '') AS thumb,
                                  COALESCE(manifest_version, 1) AS mv
                             FROM video_provenance WHERE video_id = ?""",
                        (video_id,),
                    ).fetchone()
                    conn_q.close()

                    creator_pubkey, creator_sig = "", ""
                    target_version = MANIFEST_V2  # default if Ed25519 fails
                    if pre and _ED25519_AVAILABLE and pre["creator_agent_id"]:
                        creator_pubkey, creator_sig = _agent_sign_v3(
                            int(pre["creator_agent_id"]),
                            video_id,
                            pre["canonical_sha256"] or "",
                            pre["thumb"] or "",
                            p360_sha,
                            pre["uploaded_at"] or 0,
                        )
                        if creator_pubkey and creator_sig:
                            target_version = MANIFEST_V3

                    conn_p = sqlite3.connect(str(DB_PATH))
                    conn_p.execute(
                        """UPDATE video_provenance
                              SET canonical_360p_sha256 = ?,
                                  manifest_version = ?,
                                  creator_pubkey = COALESCE(NULLIF(?, ''), creator_pubkey),
                                  creator_signature = ?,
                                  updated_at = ?
                            WHERE video_id = ?
                              AND COALESCE(anchor_tx_hash, '') = ''
                              AND COALESCE(anchor_status, 'pending')
                                  IN ('pending', 'failed')""",
                        (p360_sha, target_version,
                         creator_pubkey, creator_sig,
                         time.time(), video_id),
                    )
                    if conn_p.total_changes == 0:
                        # Row already claimed/anchored — write the 360p
                        # hash only, without changing leaf-affecting fields.
                        conn_p.execute(
                            """UPDATE video_provenance
                                  SET canonical_360p_sha256 = ?,
                                      updated_at = ?
                                WHERE video_id = ?
                                  AND COALESCE(canonical_360p_sha256, '') = ''""",
                            (p360_sha, time.time(), video_id),
                        )
                    conn_p.commit()
                    conn_p.close()
                except Exception as _e:
                    app.logger.warning("provenance 360p sha update failed for %s: %s",
                                        video_id, _e)

            return {
                "ok": True, "video_id": video_id,
                "vmaf_360p": round(vmaf_score, 2),
                "size_360p": p360_size,
                "kbps_360p": p360_kbps,
            }
        finally:
            conn.close()
    except Exception as e:
        app.logger.warning("renditions process_video(%s) failed: %s", video_id, e)
        return {"ok": False, "error": str(e)}
    finally:
        if acquired:
            _RENDITION_GATE.release()
        with _RENDITION_INFLIGHT_LOCK:
            _RENDITION_INFLIGHT.discard(video_id)


def _renditions_process_video_async(video_id):
    """Fire-and-forget background dispatch."""
    try:
        threading.Thread(
            target=_renditions_process_video, args=(video_id,),
            daemon=True, name=f"rendition-{video_id}",
        ).start()
    except Exception as e:
        app.logger.warning("renditions async dispatch failed: %s", e)


@app.route("/renditions/<video_id>/<path:filename>")
def serve_rendition(video_id, filename):
    """Serve rendition files with strong caching."""
    if "/" in filename or ".." in filename or "/" in video_id or ".." in video_id:
        abort(404)
    if not re.fullmatch(r"[A-Za-z0-9_-]{5,32}", video_id):
        abort(404)
    return send_from_directory(
        RENDITION_DIR / video_id, filename, max_age=86400 * 30, mimetype="video/mp4",
    )


@app.route("/admin/renditions/backfill", methods=["POST"])
def admin_renditions_backfill():
    """Backfill renditions for videos missing a 360p rendition.

    Body: {"limit": 30, "since_video_id": null, "concurrency": 2}.
    Synchronous within the batch (each video ~7s — caller paginates).
    """
    if not _ts_admin_ok():
        return jsonify({"ok": False, "error": "unauthorized"}), 401
    if not _renditions_ffmpeg_available():
        return jsonify({"ok": False, "error": "ffmpeg-vmaf not installed"}), 503
    _ensure_provenance_schema()
    data = request.get_json(silent=True) or {}
    try:
        limit = max(1, min(100, int(data.get("limit", 20))))
    except Exception:
        limit = 20
    try:
        concurrency = max(1, min(4, int(data.get("concurrency", 2))))
    except Exception:
        concurrency = 2
    since = (data.get("since_video_id") or "").strip()

    db = get_db()
    if since:
        rows = db.execute(
            """SELECT v.video_id
                 FROM videos v
                 LEFT JOIN video_renditions r
                        ON r.video_id = v.video_id AND r.label = '360p'
                WHERE COALESCE(v.is_removed, 0) = 0
                  AND r.video_id IS NULL
                  AND v.video_id > ?
                ORDER BY v.video_id ASC
                LIMIT ?""",
            (since, limit),
        ).fetchall()
    else:
        rows = db.execute(
            """SELECT v.video_id
                 FROM videos v
                 LEFT JOIN video_renditions r
                        ON r.video_id = v.video_id AND r.label = '360p'
                WHERE COALESCE(v.is_removed, 0) = 0
                  AND r.video_id IS NULL
                ORDER BY v.video_id ASC
                LIMIT ?""",
            (limit,),
        ).fetchall()

    started = time.time()
    written, failed = 0, 0
    last_id = ""
    samples = []  # collect VMAF for sanity
    errors = []

    if rows:
        with _cf2.ThreadPoolExecutor(max_workers=concurrency) as ex:
            futs = {ex.submit(_renditions_process_video, r["video_id"]): r["video_id"] for r in rows}
            for fut in _cf2.as_completed(futs):
                vid = futs[fut]
                last_id = max(last_id, vid)
                try:
                    res = fut.result(timeout=300)
                    if res and res.get("ok"):
                        written += 1
                        samples.append(res.get("vmaf_360p", 0.0))
                    else:
                        failed += 1
                        if len(errors) < 10:
                            errors.append({"video_id": vid, "error": (res or {}).get("error", "unknown")})
                except Exception as e:
                    failed += 1
                    if len(errors) < 10:
                        errors.append({"video_id": vid, "error": str(e)})

    elapsed = time.time() - started
    avg_vmaf = (sum(samples) / len(samples)) if samples else 0.0

    return jsonify({
        "ok": True,
        "written": written,
        "failed": failed,
        "elapsed_s": round(elapsed, 2),
        "avg_vmaf": round(avg_vmaf, 2),
        "last_video_id": last_id,
        "next_call": (
            {"since_video_id": last_id, "limit": limit, "concurrency": concurrency}
            if rows and len(rows) >= limit else None
        ),
        "errors": errors,
    })


# --- Phase 10.5: anchor batch lifecycle ----------------------------------
# A pull-batch worker pattern, per Codex's Phase 10 review:
#   * Bottube exposes GET /api/admin/provenance/pending — atomically
#     claims a batch of unanchored manifests, returns them.
#   * Worker cron (on node-2 / RustChain Ergo host) pulls hourly,
#     computes a Merkle root of (video_id || canonical_sha256) leaves,
#     anchors the root in a single Ergo box, POSTs the resulting
#     tx_hash + block_height back to /api/admin/provenance/anchor-result.
#   * Pills flip green only after a confirmed block height arrives.
# Idempotent on batch_id so a retry of the callback can't double-write.

def _provenance_ensure_anchor_columns():
    """Idempotently add anchor_status and anchor_batch_id columns."""
    _ensure_provenance_schema()
    conn = sqlite3.connect(str(DB_PATH))
    try:
        cols = {row[1] for row in conn.execute("PRAGMA table_info(video_provenance)").fetchall()}
        if "anchor_status" not in cols:
            conn.execute(
                "ALTER TABLE video_provenance ADD COLUMN anchor_status TEXT DEFAULT 'pending'"
            )
        if "anchor_batch_id" not in cols:
            conn.execute(
                "ALTER TABLE video_provenance ADD COLUMN anchor_batch_id TEXT DEFAULT ''"
            )
        if "anchored_at" not in cols:
            conn.execute(
                "ALTER TABLE video_provenance ADD COLUMN anchored_at REAL DEFAULT 0"
            )
        if "anchor_error" not in cols:
            conn.execute(
                "ALTER TABLE video_provenance ADD COLUMN anchor_error TEXT DEFAULT ''"
            )
        conn.commit()
    finally:
        conn.close()


@app.route("/api/admin/provenance/pending", methods=["GET", "POST"])
def admin_provenance_pending():
    """Claim a batch of unanchored manifests for the worker to anchor.

    Atomically (in one transaction):
      * Mints a batch_id.
      * Selects up to `limit` rows with uploader_sig != '' AND anchor_tx_hash = ''
        AND anchor_status IN ('pending', 'failed').
      * Updates anchor_status='claimed', anchor_batch_id=<new id>.
      * Returns the claimed rows + batch_id.

    The worker is expected to either succeed (POST anchor-result) or
    timeout, in which case the next claim treats stale 'claimed' rows
    older than `claim_ttl` as eligible again.
    """
    if not _ts_admin_ok():
        return jsonify({"ok": False, "error": "unauthorized"}), 401
    _provenance_ensure_anchor_columns()
    _provenance_ensure_v2_columns()
    data = request.get_json(silent=True) or {}
    try:
        limit = max(1, min(500, int(data.get("limit", 100))))
    except Exception:
        limit = 100
    try:
        claim_ttl_s = max(60, int(data.get("claim_ttl_s", 3600)))
    except Exception:
        claim_ttl_s = 3600

    batch_id = "batch_" + secrets.token_hex(8)
    now = time.time()
    stale_cutoff = now - claim_ttl_s

    _provenance_ensure_v3_columns()
    conn = sqlite3.connect(str(DB_PATH))
    conn.row_factory = sqlite3.Row
    try:
        conn.execute("BEGIN IMMEDIATE")
        rows = conn.execute(
            """SELECT video_id, canonical_sha256, uploader_sig, uploaded_at,
                      creator_agent_id, model, generated_at,
                      COALESCE(manifest_version, 1) AS manifest_version,
                      COALESCE(thumbnail_sha256, '') AS thumbnail_sha256,
                      COALESCE(canonical_360p_sha256, '') AS canonical_360p_sha256,
                      COALESCE(creator_pubkey, '') AS creator_pubkey,
                      COALESCE(creator_signature, '') AS creator_signature
                 FROM video_provenance
                WHERE uploader_sig != ''
                  AND COALESCE(anchor_tx_hash, '') = ''
                  AND (
                        COALESCE(anchor_status, 'pending') IN ('pending', 'failed')
                     OR (COALESCE(anchor_status, 'pending') = 'claimed'
                         AND COALESCE(updated_at, 0) < ?)
                      )
                  AND (
                        -- v2 / v3 rows only eligible once the rendition
                        -- pipeline has filled in canonical_360p_sha256
                        -- (v3 also requires creator_signature; gated below).
                        COALESCE(manifest_version, 1) < 2
                     OR COALESCE(canonical_360p_sha256, '') != ''
                      )
                  AND (
                        -- Phase 11.23: v3 rows must additionally have a
                        -- creator_signature. Without it the worker would
                        -- anchor a v3 leaf with an empty signature field
                        -- and the verifier would reject.
                        COALESCE(manifest_version, 1) < 3
                     OR COALESCE(creator_signature, '') != ''
                      )
                ORDER BY uploaded_at ASC
                LIMIT ?""",
            (stale_cutoff, limit),
        ).fetchall()
        if not rows:
            conn.execute("COMMIT")
            return jsonify({
                "ok": True,
                "batch_id": "",
                "manifests": [],
                "count": 0,
                "message": "no manifests pending anchor",
            })
        ids = [r["video_id"] for r in rows]
        placeholders = ",".join("?" for _ in ids)
        conn.execute(
            f"""UPDATE video_provenance
                  SET anchor_status = 'claimed',
                      anchor_batch_id = ?,
                      updated_at = ?
                WHERE video_id IN ({placeholders})""",
            [batch_id, now] + ids,
        )
        conn.execute("COMMIT")
    except Exception as e:
        try:
            conn.execute("ROLLBACK")
        except Exception:
            pass
        return jsonify({"ok": False, "error": str(e)}), 500
    finally:
        conn.close()

    manifests = []
    for r in rows:
        manifests.append({
            "video_id": r["video_id"],
            "canonical_sha256": r["canonical_sha256"],
            "uploader_sig": r["uploader_sig"],
            "uploaded_at": r["uploaded_at"],
            "creator_agent_id": r["creator_agent_id"],
            "creator_pubkey": r["creator_pubkey"] or "",
            "creator_signature": r["creator_signature"] or "",
            "model": r["model"] or "",
            "generated_at": r["generated_at"] or 0,
            # Phase 11.16/11.23: v2/v3 fields. Worker uses manifest_version
            # to pick the correct leaf recipe; thumb / 360p / pubkey / sig
            # default to empty strings on lower versions.
            "manifest_version": int(r["manifest_version"] or 1),
            "thumbnail_sha256": r["thumbnail_sha256"] or "",
            "canonical_360p_sha256": r["canonical_360p_sha256"] or "",
        })
    return jsonify({
        "ok": True,
        "batch_id": batch_id,
        "claimed_at": now,
        "claim_ttl_s": claim_ttl_s,
        "count": len(manifests),
        "manifests": manifests,
        "leaf_recipes": {
            "v1": _manifest_leaf_recipe(MANIFEST_V1),
            "v2": _manifest_leaf_recipe(MANIFEST_V2),
            "v3": _manifest_leaf_recipe(MANIFEST_V3),
        },
    })


# --- Phase 11.13: anchor confirmation reaper ------------------------------
# After a TX is broadcast the worker reports block_height=0 if the chain
# hasn't confirmed it yet. This endpoint re-queries the chain for every
# unique anchor_tx_hash with block_height=0 and writes the real
# inclusion height back. Driven by a separate systemd timer so the main
# anchor cron stays single-purpose.

@app.route("/api/admin/provenance/reap-confirmations", methods=["POST", "GET"])
def admin_provenance_reap_confirmations():
    """Update anchor_block_height on rows whose TX has now confirmed.

    Strategy: SELECT DISTINCT anchor_tx_hash WHERE block_height=0. For
    each one, query /wallet/transactionById on the configured Ergo node.
    If the TX has at least 1 confirmation, derive inclusion height from
    `inclusionHeight` (or chain_height - numConfirmations + 1 fallback)
    and UPDATE every row in that batch.
    """
    if not _ts_admin_ok():
        return jsonify({"ok": False, "error": "unauthorized"}), 401
    _provenance_ensure_anchor_columns()

    ergo_base = os.environ.get("ERGO_BASE", "http://localhost:9053")
    ergo_key = os.environ.get("ERGO_API_KEY", "")
    if not ergo_key:
        return jsonify({"ok": False, "error": "ERGO_API_KEY not set"}), 503

    try:
        limit = max(1, min(200, int((request.args.get("limit") or
                                     (request.get_json(silent=True) or {}).get("limit", 50)))))
    except Exception:
        limit = 50

    db = get_db()
    pending = db.execute(
        """SELECT anchor_tx_hash, MIN(anchored_at) AS first_anchored_at
             FROM video_provenance
            WHERE COALESCE(anchor_tx_hash,'') != ''
              AND COALESCE(anchor_block_height,0) = 0
              AND anchor_chain != 'stub'
            GROUP BY anchor_tx_hash
            ORDER BY first_anchored_at ASC
            LIMIT ?""",
        (limit,),
    ).fetchall()
    pending_txs = [r["anchor_tx_hash"] for r in pending]

    updated_rows = 0
    confirmed_txs = 0
    failures = []
    for tx_hash in pending_txs:
        try:
            req = urllib.request.Request(
                f"{ergo_base}/wallet/transactionById?id={tx_hash}",
                headers={"api_key": ergo_key},
            )
            with urllib.request.urlopen(req, timeout=10) as r:
                data = json.loads(r.read().decode("utf-8"))
        except urllib.error.HTTPError as e:
            failures.append({"tx": tx_hash, "error": f"HTTP {e.code}"})
            continue
        except Exception as e:
            failures.append({"tx": tx_hash, "error": str(e)[:120]})
            continue

        num_confs = int((data or {}).get("numConfirmations", 0) or 0)
        incl_height = int((data or {}).get("inclusionHeight", 0) or 0)
        if num_confs < 1 or not incl_height:
            failures.append({"tx": tx_hash, "error": f"unconfirmed ({num_confs} confs)"})
            continue

        n = db.execute(
            """UPDATE video_provenance
                  SET anchor_block_height = ?, updated_at = ?
                WHERE anchor_tx_hash = ?
                  AND COALESCE(anchor_block_height, 0) = 0""",
            (incl_height, time.time(), tx_hash),
        ).rowcount or 0
        db.commit()
        updated_rows += int(n)
        confirmed_txs += 1

    return jsonify({
        "ok": True,
        "scanned": len(pending_txs),
        "confirmed_txs": confirmed_txs,
        "rows_updated": updated_rows,
        "failures": failures[:10],
    })


# ---------------------------------------------------------------------------
# Phase 11.19: continuous anchor reconciliation
# ---------------------------------------------------------------------------
# A scheduled audit that bottube runs *against itself*: pick N random
# already-anchored TXs, re-fetch R4 from chain, compare to the bottube DB.
# If anything ever drifts (e.g., a corrupted DB write, a wrong manifest_hash
# stored after a rollback), the next reconciliation pass surfaces it before
# any user-facing verification breaks. Results land in
# anchor_reconciliations and roll up into /api/transparency.

def _reconciliation_ensure_schema():
    """Idempotent table creation for the reconciliation log."""
    conn = sqlite3.connect(str(DB_PATH))
    try:
        conn.execute("""
            CREATE TABLE IF NOT EXISTS anchor_reconciliations (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                tx_hash TEXT NOT NULL,
                db_manifest_hash TEXT NOT NULL,
                chain_r4_hex TEXT NOT NULL DEFAULT '',
                chain_inclusion_height INTEGER DEFAULT 0,
                chain_num_confirmations INTEGER DEFAULT 0,
                matched INTEGER NOT NULL,
                error TEXT NOT NULL DEFAULT '',
                checked_at INTEGER NOT NULL
            )
        """)
        conn.execute("""
            CREATE INDEX IF NOT EXISTS idx_recon_checked_at
                ON anchor_reconciliations(checked_at)
        """)
        conn.execute("""
            CREATE INDEX IF NOT EXISTS idx_recon_matched
                ON anchor_reconciliations(matched, checked_at)
        """)
        conn.commit()
    finally:
        conn.close()


def _fetch_chain_r4(ergo_base, ergo_key, tx_hash, timeout=10):
    """Fetch the on-chain TX, return (r4_merkle_hex, inclusion_height,
    num_confirmations, error). r4_merkle_hex is "" on any error."""
    try:
        req = urllib.request.Request(
            f"{ergo_base}/wallet/transactionById?id={tx_hash}",
            headers={"api_key": ergo_key} if ergo_key else {},
        )
        with urllib.request.urlopen(req, timeout=timeout) as r:
            data = json.loads(r.read().decode("utf-8"))
    except urllib.error.HTTPError as e:
        return "", 0, 0, f"HTTP {e.code}"
    except Exception as e:
        return "", 0, 0, str(e)[:160]

    outs = (data or {}).get("outputs") or []
    if not outs:
        return "", 0, 0, "no outputs"
    regs = outs[0].get("additionalRegisters") or {}
    r4 = regs.get("R4", "")
    merkle_hex = ""
    if r4.startswith("0e20") and len(r4) == 4 + 64:
        merkle_hex = r4[4:].lower()
    return (
        merkle_hex,
        int(data.get("inclusionHeight") or 0),
        int(data.get("numConfirmations") or 0),
        "" if merkle_hex else "R4 not 32-byte SColl",
    )


@app.route("/api/admin/reconcile-anchors", methods=["POST", "GET"])
def admin_reconcile_anchors():
    """Re-verify N random anchored TXs against the chain.

    For each: fetch R4, compare to DB anchor_manifest_hash, write the
    result. Mismatches are alarms — they should never happen in a healthy
    pipeline. Used by a 6-hour systemd timer; admin-key gated.

    Body: {"limit": 50, "strategy": "random"|"recent"|"oldest"}.
    """
    if not _ts_admin_ok():
        return jsonify({"ok": False, "error": "unauthorized"}), 401
    _reconciliation_ensure_schema()
    _provenance_ensure_anchor_columns()

    data = request.get_json(silent=True) or {}
    try:
        limit = max(1, min(200, int(data.get("limit", 50))))
    except Exception:
        limit = 50
    strategy = (data.get("strategy") or "random").lower()
    if strategy not in ("random", "recent", "oldest"):
        strategy = "random"

    ergo_base = os.environ.get("ERGO_BASE", "http://localhost:9053")
    ergo_key = os.environ.get("ERGO_API_KEY", "")
    if not ergo_key:
        return jsonify({"ok": False, "error": "ERGO_API_KEY not set"}), 503

    db = get_db()
    if strategy == "recent":
        order = "anchored_at DESC"
    elif strategy == "oldest":
        order = "anchored_at ASC"
    else:
        order = "RANDOM()"
    rows = db.execute(
        f"""SELECT anchor_tx_hash AS tx_hash,
                   MIN(anchor_manifest_hash) AS db_root,
                   MIN(anchor_block_height) AS db_height
              FROM video_provenance
             WHERE COALESCE(anchor_tx_hash,'') != ''
               AND COALESCE(anchor_block_height,0) > 0
               AND anchor_chain != 'stub'
             GROUP BY anchor_tx_hash
             ORDER BY {order}
             LIMIT ?""",
        (limit,),
    ).fetchall()

    now = int(time.time())
    scanned = 0
    matched = 0
    mismatched = 0
    errored = 0
    sample_mismatches = []
    sample_errors = []

    conn = sqlite3.connect(str(DB_PATH))
    try:
        for row in rows:
            scanned += 1
            tx = row["tx_hash"]
            db_root = (row["db_root"] or "").lower()
            chain_root, incl_h, num_confs, err = _fetch_chain_r4(
                ergo_base, ergo_key, tx,
            )
            if err and not chain_root:
                errored += 1
                if len(sample_errors) < 5:
                    sample_errors.append({"tx": tx[:16] + "…", "error": err})
                conn.execute(
                    """INSERT INTO anchor_reconciliations
                       (tx_hash, db_manifest_hash, chain_r4_hex,
                        chain_inclusion_height, chain_num_confirmations,
                        matched, error, checked_at)
                       VALUES (?, ?, '', 0, ?, 0, ?, ?)""",
                    (tx, db_root, num_confs, err, now),
                )
                continue

            ok = (chain_root == db_root)
            if ok:
                matched += 1
            else:
                mismatched += 1
                if len(sample_mismatches) < 5:
                    sample_mismatches.append({
                        "tx": tx,
                        "db_root": db_root,
                        "chain_root": chain_root,
                    })
            conn.execute(
                """INSERT INTO anchor_reconciliations
                   (tx_hash, db_manifest_hash, chain_r4_hex,
                    chain_inclusion_height, chain_num_confirmations,
                    matched, error, checked_at)
                   VALUES (?, ?, ?, ?, ?, ?, '', ?)""",
                (tx, db_root, chain_root, incl_h, num_confs,
                 1 if ok else 0, now),
            )
        conn.commit()
    finally:
        conn.close()

    # Mismatches are critical — log loudly.
    if mismatched:
        app.logger.error(
            "ANCHOR RECONCILIATION MISMATCH: %d/%d txs do NOT match chain. "
            "Sample: %s",
            mismatched, scanned, sample_mismatches[:3],
        )

    return jsonify({
        "ok": True,
        "checked_at": now,
        "strategy": strategy,
        "scanned": scanned,
        "matched": matched,
        "mismatched": mismatched,
        "errored": errored,
        "sample_mismatches": sample_mismatches,
        "sample_errors": sample_errors,
        "alarm": mismatched > 0,
    })


def _reconciliation_summary():
    """Roll up reconciliation_log into a small dict for /api/transparency."""
    try:
        _reconciliation_ensure_schema()
    except Exception:
        return None
    db = get_db()
    now = int(time.time())

    def _count(sql, *args):
        try:
            r = db.execute(sql, args).fetchone()
            return int((r or [0])[0] or 0)
        except Exception:
            return 0

    last24 = now - 86400
    last7d = now - 86400 * 7

    total_24h = _count(
        "SELECT COUNT(*) FROM anchor_reconciliations WHERE checked_at >= ?",
        last24,
    )
    matched_24h = _count(
        "SELECT COUNT(*) FROM anchor_reconciliations "
        "WHERE checked_at >= ? AND matched = 1",
        last24,
    )
    mismatched_24h = _count(
        "SELECT COUNT(*) FROM anchor_reconciliations "
        "WHERE checked_at >= ? AND matched = 0 AND error = ''",
        last24,
    )
    errored_24h = _count(
        "SELECT COUNT(*) FROM anchor_reconciliations "
        "WHERE checked_at >= ? AND error != ''",
        last24,
    )
    total_7d = _count(
        "SELECT COUNT(*) FROM anchor_reconciliations WHERE checked_at >= ?",
        last7d,
    )
    matched_7d = _count(
        "SELECT COUNT(*) FROM anchor_reconciliations "
        "WHERE checked_at >= ? AND matched = 1",
        last7d,
    )
    mismatched_7d = _count(
        "SELECT COUNT(*) FROM anchor_reconciliations "
        "WHERE checked_at >= ? AND matched = 0 AND error = ''",
        last7d,
    )

    last_check = None
    last_mismatch = None
    try:
        r = db.execute(
            "SELECT checked_at FROM anchor_reconciliations "
            "ORDER BY checked_at DESC LIMIT 1"
        ).fetchone()
        last_check = int(r[0]) if r else None
    except Exception:
        pass
    try:
        r = db.execute(
            "SELECT checked_at FROM anchor_reconciliations "
            "WHERE matched = 0 AND error = '' "
            "ORDER BY checked_at DESC LIMIT 1"
        ).fetchone()
        last_mismatch = int(r[0]) if r else None
    except Exception:
        pass

    match_rate_24h = (matched_24h / (matched_24h + mismatched_24h)) if (matched_24h + mismatched_24h) else 1.0
    match_rate_7d = (matched_7d / (matched_7d + mismatched_7d)) if (matched_7d + mismatched_7d) else 1.0

    return {
        "last_check_at": last_check,
        "last_check_age_s": (now - last_check) if last_check else None,
        "last_mismatch_at": last_mismatch,
        "last_24h": {
            "checked": total_24h,
            "matched": matched_24h,
            "mismatched": mismatched_24h,
            "errored": errored_24h,
            "match_rate": round(match_rate_24h, 4),
        },
        "last_7d": {
            "checked": total_7d,
            "matched": matched_7d,
            "mismatched": mismatched_7d,
            "match_rate": round(match_rate_7d, 4),
        },
        "alarm": mismatched_24h > 0,
    }


@app.route("/api/admin/provenance/anchor-result", methods=["POST"])
def admin_provenance_anchor_result():
    """Finalize a claimed batch with the on-chain anchor result.

    Idempotent on batch_id: if the batch is already anchored, this is a
    no-op success. If a callback arrives for a batch_id that doesn't
    match the rows' current state, we log and refuse so a stale worker
    can't overwrite a later successful anchor.
    """
    if not _ts_admin_ok():
        return jsonify({"ok": False, "error": "unauthorized"}), 401
    _provenance_ensure_anchor_columns()
    data = request.get_json(silent=True) or {}
    batch_id = (data.get("batch_id") or "").strip()
    chain = (data.get("chain") or "ergo").strip()[:32]
    tx_hash = (data.get("tx_hash") or "").strip()[:128]
    try:
        block_height = int(data.get("block_height", 0))
    except Exception:
        block_height = 0
    manifest_hash = (data.get("merkle_root") or data.get("manifest_hash") or "").strip()[:128]
    error_msg = (data.get("error") or "").strip()[:500]
    video_ids = data.get("video_ids") or []

    if not batch_id or not re.fullmatch(r"batch_[a-f0-9]{8,32}", batch_id):
        return jsonify({"ok": False, "error": "invalid batch_id"}), 400

    now = time.time()
    conn = sqlite3.connect(str(DB_PATH))
    conn.row_factory = sqlite3.Row
    try:
        conn.execute("BEGIN IMMEDIATE")
        existing = conn.execute(
            """SELECT video_id, anchor_status, anchor_tx_hash
                 FROM video_provenance
                WHERE anchor_batch_id = ?""",
            (batch_id,),
        ).fetchall()
        if not existing:
            conn.execute("ROLLBACK")
            return jsonify({"ok": False, "error": "batch_id has no claimed rows"}), 404

        # Idempotency: if batch already anchored, return ok.
        already = [r for r in existing if r["anchor_tx_hash"]]
        if already and not error_msg:
            conn.execute("ROLLBACK")
            return jsonify({
                "ok": True,
                "idempotent": True,
                "batch_id": batch_id,
                "rows_already_anchored": len(already),
            })

        if error_msg:
            # Worker reported failure — release the claim, keep status='failed'
            conn.execute(
                """UPDATE video_provenance
                      SET anchor_status = 'failed',
                          anchor_error = ?,
                          updated_at = ?
                    WHERE anchor_batch_id = ?""",
                (error_msg, now, batch_id),
            )
            conn.execute("COMMIT")
            return jsonify({
                "ok": True,
                "batch_id": batch_id,
                "status": "failed",
                "rows": len(existing),
            })

        if not tx_hash or not manifest_hash:
            conn.execute("ROLLBACK")
            return jsonify({
                "ok": False,
                "error": "tx_hash and merkle_root required for success result",
            }), 400

        # Apply the anchor result to all rows in this batch.
        conn.execute(
            """UPDATE video_provenance
                  SET anchor_chain = ?,
                      anchor_tx_hash = ?,
                      anchor_block_height = ?,
                      anchor_manifest_hash = ?,
                      anchor_status = 'anchored',
                      anchored_at = ?,
                      anchor_error = '',
                      updated_at = ?
                WHERE anchor_batch_id = ?""",
            (chain, tx_hash, block_height, manifest_hash, now, now, batch_id),
        )
        conn.execute("COMMIT")
        rows_anchored = len(existing)
    except Exception as e:
        try:
            conn.execute("ROLLBACK")
        except Exception:
            pass
        return jsonify({"ok": False, "error": str(e)}), 500
    finally:
        conn.close()

    _ts_log_audit(
        actor="anchor_worker",
        action="anchor_batch",
        target_kind="batch",
        target_id=batch_id,
        reason=f"{rows_anchored} manifests anchored on {chain} block {block_height}",
        severity="normal",
        meta={"tx_hash": tx_hash, "merkle_root": manifest_hash},
    )

    return jsonify({
        "ok": True,
        "batch_id": batch_id,
        "rows_anchored": rows_anchored,
        "tx_hash": tx_hash,
        "block_height": block_height,
        "chain": chain,
    })


# ---------------------------------------------------------------------------
# Phase 11.5: public /anchors page — chain anchor history
# ---------------------------------------------------------------------------
# Read-only public surface that lists every Merkle anchor TX bottube has
# committed to RustChain, with a per-batch detail page that lists members
# and instructs how to run the verifier. This is what makes "Verified
# Provenance" visible at the platform level, not just the per-video pill.

def _anchors_summary(limit=200):
    """Return a list of anchor batches grouped by tx_hash, newest first."""
    db = get_db()
    rows = db.execute(
        """SELECT anchor_tx_hash AS tx_hash,
                  MIN(anchor_chain) AS chain,
                  MIN(anchor_block_height) AS block_height,
                  MIN(anchor_manifest_hash) AS manifest_hash,
                  MIN(anchor_batch_id) AS batch_id,
                  MIN(anchored_at) AS anchored_at,
                  COUNT(*) AS member_count
             FROM video_provenance
            WHERE COALESCE(anchor_tx_hash,'') != ''
            GROUP BY anchor_tx_hash
            ORDER BY anchored_at DESC
            LIMIT ?""",
        (limit,),
    ).fetchall()
    return [{
        "tx_hash": r["tx_hash"],
        "chain": r["chain"] or "rustchain",
        "block_height": r["block_height"] or 0,
        "manifest_hash": r["manifest_hash"] or "",
        "batch_id": r["batch_id"] or "",
        "anchored_at": r["anchored_at"] or 0,
        "member_count": r["member_count"] or 0,
    } for r in rows]


@app.route("/federation")
def federation_page():
    """Public federation spec — Codex's spec-first commitment."""
    return render_template("federation.html")


# ---------------------------------------------------------------------------
# Phase 11.17: public transparency dashboard
# ---------------------------------------------------------------------------
# Operational honesty by default: anyone — including reviewers who don't have
# admin access — can read live anchor lag, manifest version distribution,
# pending backlog, federation peer count, and recent anchor cadence. Cached
# in-memory for 60s so a hammered page doesn't beat up the DB.

_TRANSPARENCY_CACHE = {"at": 0.0, "data": None}
_TRANSPARENCY_TTL_S = 60.0


def _percentiles(samples, ps=(0.5, 0.95, 0.99)):
    """Closest-rank percentiles. Returns ints (seconds). [] -> 0s for all."""
    if not samples:
        return {f"p{int(p*100)}": 0 for p in ps}
    s = sorted(samples)
    out = {}
    for p in ps:
        idx = max(0, min(len(s) - 1, int(round(p * (len(s) - 1)))))
        out[f"p{int(p*100)}"] = int(s[idx])
    return out


def _compute_transparency_snapshot():
    """Compute the transparency dashboard payload from current DB state.

    Read-only, side-effect-free. All counters are best-effort against the
    operational SQLite — we explicitly do NOT promise atomic consistency,
    only directional honesty (the v1/v2 ratio is real, the lag is real,
    a stale 60s cache is acceptable for an external dashboard).
    """
    db = get_db()
    now = int(time.time())

    def _scalar(sql, *args, default=0):
        try:
            r = db.execute(sql, args).fetchone()
            if r is None:
                return default
            v = r[0]
            return v if v is not None else default
        except Exception:
            return default

    total_anchored_rows = _scalar(
        "SELECT COUNT(*) FROM video_provenance WHERE COALESCE(anchor_tx_hash,'') != ''"
    )
    total_anchor_txs = _scalar(
        "SELECT COUNT(DISTINCT anchor_tx_hash) FROM video_provenance "
        "WHERE COALESCE(anchor_tx_hash,'') != ''"
    )
    total_provenance_rows = _scalar(
        "SELECT COUNT(*) FROM video_provenance"
    )
    pending_anchor = _scalar(
        "SELECT COUNT(*) FROM video_provenance "
        "WHERE COALESCE(anchor_tx_hash,'') = '' AND COALESCE(uploader_sig,'') != ''"
    )
    confirmed_anchors = _scalar(
        "SELECT COUNT(DISTINCT anchor_tx_hash) FROM video_provenance "
        "WHERE COALESCE(anchor_tx_hash,'') != '' "
        "  AND COALESCE(anchor_block_height,0) > 0"
    )
    awaiting_confirmation = _scalar(
        "SELECT COUNT(DISTINCT anchor_tx_hash) FROM video_provenance "
        "WHERE COALESCE(anchor_tx_hash,'') != '' "
        "  AND COALESCE(anchor_block_height,0) = 0 "
        "  AND anchor_chain != 'stub'"
    )

    # Manifest version distribution. v2 should slowly take over once new
    # uploads start landing. Surfaces v1→v2 rollout in real time.
    v_rows = []
    try:
        v_rows = db.execute(
            "SELECT COALESCE(manifest_version,1) AS v, "
            "       COUNT(*) AS n FROM video_provenance "
            " WHERE COALESCE(anchor_tx_hash,'') != '' "
            " GROUP BY v"
        ).fetchall()
    except Exception:
        v_rows = []
    by_version = {int(r["v"]): int(r["n"]) for r in v_rows}

    # Anchor cadence: last 24h, last 7d, last 30d (distinct TXs).
    def _txs_since(seconds):
        try:
            return _scalar(
                "SELECT COUNT(DISTINCT anchor_tx_hash) FROM video_provenance "
                "WHERE COALESCE(anchor_tx_hash,'') != '' "
                "  AND COALESCE(anchored_at, 0) >= ?",
                now - seconds,
            )
        except Exception:
            return 0

    txs_24h = _txs_since(86400)
    txs_7d = _txs_since(86400 * 7)
    txs_30d = _txs_since(86400 * 30)

    # Lag = upload-to-anchored, in seconds. Restrict to videos uploaded
    # within the last 30 days so the percentile reflects *current*
    # operational behavior — historical backfill anchors of long-old
    # uploads would otherwise dominate the distribution.
    lag_samples = []
    try:
        lag_rows = db.execute(
            """SELECT (anchored_at - uploaded_at) AS lag_s
                 FROM video_provenance
                WHERE COALESCE(anchor_tx_hash,'') != ''
                  AND COALESCE(anchored_at, 0) > 0
                  AND COALESCE(uploaded_at, 0) > 0
                  AND anchored_at > uploaded_at
                  AND uploaded_at >= ?
                ORDER BY anchored_at DESC
                LIMIT 200""",
            (now - 86400 * 30,),
        ).fetchall()
        lag_samples = [int(r["lag_s"]) for r in lag_rows
                       if r["lag_s"] is not None and r["lag_s"] >= 0]
    except Exception:
        lag_samples = []
    lag_p = _percentiles(lag_samples)

    # Last anchor: most recent confirmed TX with non-zero block_height.
    last_anchor = None
    try:
        r = db.execute(
            """SELECT anchor_tx_hash, anchor_block_height, anchored_at,
                      anchor_chain
                 FROM video_provenance
                WHERE COALESCE(anchor_tx_hash,'') != ''
                  AND COALESCE(anchor_block_height,0) > 0
                ORDER BY anchored_at DESC
                LIMIT 1"""
        ).fetchone()
        if r:
            last_anchor = {
                "tx_hash": r["anchor_tx_hash"],
                "block_height": int(r["anchor_block_height"] or 0),
                "anchored_at": int(r["anchored_at"] or 0),
                "chain": r["anchor_chain"] or "rustchain",
                "age_s": max(0, now - int(r["anchored_at"] or 0)),
            }
    except Exception:
        last_anchor = None

    # Per-day anchor count for the last 14 days, oldest→newest, for a
    # sparkline. Returned as [{"day": "YYYY-MM-DD", "count": N}].
    daily = []
    try:
        rows = db.execute(
            """SELECT date(anchored_at, 'unixepoch') AS day,
                      COUNT(DISTINCT anchor_tx_hash) AS n
                 FROM video_provenance
                WHERE COALESCE(anchor_tx_hash,'') != ''
                  AND COALESCE(anchored_at,0) >= ?
                GROUP BY day
                ORDER BY day ASC""",
            (now - 86400 * 14,),
        ).fetchall()
        daily = [{"day": r["day"], "count": int(r["n"])} for r in rows]
    except Exception:
        daily = []

    # Federation peer count if the table exists (best-effort — federation
    # may be empty/disabled on a particular install).
    federation_peers = 0
    try:
        federation_peers = _scalar(
            "SELECT COUNT(*) FROM federation_peers WHERE COALESCE(active,0) = 1"
        )
    except Exception:
        federation_peers = 0

    # Verifier success: of the last 50 anchored TXs whose block_height>0,
    # how many have a non-empty manifest_hash that's the canonical
    # 32-byte hex? (A row that anchored but ended up with a blank or
    # malformed manifest_hash would never verify and signals a bug.)
    verifier_total = 0
    verifier_ok = 0
    try:
        rows = db.execute(
            """SELECT anchor_manifest_hash
                 FROM video_provenance
                WHERE COALESCE(anchor_tx_hash,'') != ''
                  AND COALESCE(anchor_block_height,0) > 0
                ORDER BY anchored_at DESC
                LIMIT 50"""
        ).fetchall()
        for r in rows:
            verifier_total += 1
            mh = r["anchor_manifest_hash"] or ""
            if len(mh) == 64 and re.fullmatch(r"[0-9a-f]+", mh):
                verifier_ok += 1
    except Exception:
        pass
    verifier_rate = (verifier_ok / verifier_total) if verifier_total else 1.0

    try:
        from datetime import datetime, timezone
        as_of_iso = datetime.fromtimestamp(now, tz=timezone.utc).isoformat()
    except Exception:
        as_of_iso = ""

    # Phase 11.19: pull the latest reconciliation rollup. Best-effort —
    # if the table doesn't exist yet (fresh deploy before first run), the
    # summary helper returns None and the field is null.
    try:
        recon = _reconciliation_summary()
    except Exception:
        recon = None

    return {
        "ok": True,
        "as_of": now,
        "as_of_iso": as_of_iso,
        "reconciliation": recon,
        "anchors": {
            "total_videos_anchored": total_anchored_rows,
            "total_anchor_transactions": total_anchor_txs,
            "confirmed_on_chain": confirmed_anchors,
            "awaiting_confirmation": awaiting_confirmation,
            "anchors_24h": txs_24h,
            "anchors_7d": txs_7d,
            "anchors_30d": txs_30d,
            "last_anchor": last_anchor,
            "anchor_lag_seconds": {
                "samples": len(lag_samples),
                "window_days": 30,
                "note": (
                    "upload→anchored seconds, sampled from videos uploaded "
                    "in the last 30 days only — historical backfill anchors "
                    "of older uploads are excluded so the percentile reflects "
                    "current pipeline behavior"
                ),
                **lag_p,
            },
            "daily_14d": daily,
        },
        "manifest_versions": {
            "by_version": {f"v{k}": v for k, v in sorted(by_version.items())},
            "v2_share":
                (by_version.get(2, 0) / total_anchored_rows)
                if total_anchored_rows else 0.0,
        },
        "queue": {
            "provenance_rows_total": total_provenance_rows,
            "pending_anchor": pending_anchor,
        },
        "federation": {
            "active_peers": federation_peers,
        },
        "verifier_health": {
            "sample_size": verifier_total,
            "well_formed_root_count": verifier_ok,
            "well_formed_rate": round(verifier_rate, 4),
        },
        "spec_version": "phase-11.17",
    }


def _transparency_snapshot_cached():
    """60s in-memory cache around the snapshot. Bounded server load."""
    now = time.time()
    if (_TRANSPARENCY_CACHE["data"] is not None
            and (now - _TRANSPARENCY_CACHE["at"]) < _TRANSPARENCY_TTL_S):
        return _TRANSPARENCY_CACHE["data"], True
    fresh = _compute_transparency_snapshot()
    _TRANSPARENCY_CACHE["at"] = now
    _TRANSPARENCY_CACHE["data"] = fresh
    return fresh, False


@app.route("/api/transparency")
def api_transparency():
    """Public, read-only operational metrics for bottube.ai.

    Documented purpose: a stable JSON contract that anyone — verifier
    operators, federation peers, engineering reviewers, you with curl —
    can poll to see how the platform is actually behaving. All fields
    are best-effort directional honesty; no promise of atomic consistency.
    """
    data, cached = _transparency_snapshot_cached()
    resp = jsonify(data)
    resp.headers["Cache-Control"] = f"public, max-age={int(_TRANSPARENCY_TTL_S)}"
    resp.headers["X-Bottube-Cache"] = "hit" if cached else "miss"
    return resp


@app.route("/transparency")
def transparency_page():
    """HTML rendering of /api/transparency for casual browsers."""
    data, _cached = _transparency_snapshot_cached()
    return render_template("transparency.html", t=data)


@app.route("/verify")
def verify_page():
    """Phase 11.25: browser-side verifier. Walks the entire cryptographic
    chain (provenance fields → Merkle leaf → inclusion proof → on-chain R4)
    using SubtleCrypto.subtle.digest() in the user's browser. No install
    required, no server-side trust beyond the public read-only endpoints.
    """
    return render_template("verify.html")


@app.route("/api/anchors/<tx_hash>/chain")
def anchor_chain_proxy(tx_hash):
    """Public read-only proxy: fetch the on-chain TX from the configured Ergo
    node and return the bits a verifier wants — R4 register, confirmations,
    output value, ergoTree fingerprint. Lets the chain-side panel on
    /anchors/<tx> render without exposing the operator's API key.
    """
    if not re.fullmatch(r"[0-9a-fA-F]{32,128}", tx_hash):
        return jsonify({"ok": False, "error": "invalid tx_hash"}), 400
    ergo_base = os.environ.get("ERGO_BASE", "http://localhost:9053")
    ergo_key = os.environ.get("ERGO_API_KEY", "")
    try:
        req = urllib.request.Request(
            f"{ergo_base}/wallet/transactionById?id={tx_hash}",
            headers={"api_key": ergo_key} if ergo_key else {},
        )
        with urllib.request.urlopen(req, timeout=8) as r:
            data = json.loads(r.read().decode("utf-8"))
    except Exception as e:
        return jsonify({"ok": False, "error": f"chain unreachable: {e}"}), 502
    if not isinstance(data, dict) or "outputs" not in data:
        return jsonify({"ok": False, "error": data.get("error", "no outputs")}), 404

    outs = data.get("outputs") or []
    out0 = outs[0] if outs else {}
    regs = out0.get("additionalRegisters") or {}
    r4 = regs.get("R4", "")
    r5 = regs.get("R5", "")
    # Decode R4 from "0e20" + 64-hex into the raw 32-byte hex root.
    merkle = ""
    if r4.startswith("0e20") and len(r4) == 4 + 64:
        merkle = r4[4:]
    return jsonify({
        "ok": True,
        "tx_hash": data.get("id", tx_hash),
        "num_confirmations": int(data.get("numConfirmations", 0) or 0),
        "inclusion_height": data.get("inclusionHeight"),
        "output_count": len(outs),
        "anchor_value_nanoerg": int(out0.get("value", 0) or 0),
        "ergo_tree_short": (out0.get("ergoTree") or "")[:24],
        "r4_raw": r4,
        "r4_merkle_root": merkle,
        "r5_raw": r5,
        "creation_height": int(out0.get("creationHeight", 0) or 0),
    })


@app.route("/anchors")
def anchors_page():
    """Public chain anchor history."""
    batches = _anchors_summary(limit=200)
    total_anchored = sum(b["member_count"] for b in batches)
    return render_template(
        "anchors.html",
        batches=batches,
        total_anchored=total_anchored,
        total_batches=len(batches),
    )


@app.route("/anchors/<tx_hash>")
def anchor_detail_page(tx_hash):
    """Per-batch detail: tx_hash, manifest_hash, all member videos, verifier hint."""
    if not re.fullmatch(r"[0-9a-fA-F]{32,128}", tx_hash):
        abort(404)
    db = get_db()
    rows = db.execute(
        """SELECT v.video_id, v.title, v.thumbnail, v.duration_sec, v.created_at,
                  a.agent_name, a.display_name,
                  p.canonical_sha256, p.uploader_sig, p.uploaded_at,
                  p.anchor_chain, p.anchor_tx_hash, p.anchor_block_height,
                  p.anchor_manifest_hash, p.anchor_batch_id, p.anchored_at
             FROM video_provenance p
             JOIN videos v ON v.video_id = p.video_id
             JOIN agents a ON a.id = v.agent_id
            WHERE p.anchor_tx_hash = ?
              AND COALESCE(v.is_removed, 0) = 0
            ORDER BY p.uploaded_at ASC""",
        (tx_hash,),
    ).fetchall()
    if not rows:
        abort(404)
    head = rows[0]
    batch = {
        "tx_hash": head["anchor_tx_hash"],
        "chain": head["anchor_chain"] or "rustchain",
        "block_height": head["anchor_block_height"] or 0,
        "manifest_hash": head["anchor_manifest_hash"] or "",
        "batch_id": head["anchor_batch_id"] or "",
        "anchored_at": head["anchored_at"] or 0,
        "member_count": len(rows),
    }
    members = [{
        "video_id": r["video_id"],
        "title": r["title"],
        "thumbnail": r["thumbnail"],
        "duration_sec": r["duration_sec"],
        "agent_name": r["agent_name"],
        "display_name": r["display_name"] or r["agent_name"],
        "canonical_sha256": r["canonical_sha256"] or "",
        "uploaded_at": r["uploaded_at"] or r["created_at"] or 0,
    } for r in rows]
    return render_template("anchor_detail.html", batch=batch, members=members)


# --- Phase 11.9: public Merkle inclusion proof ---------------------------
# An external verifier should be able to cryptographically prove "this
# video's leaf is part of this anchor's Merkle root" without needing an
# admin key OR the full batch membership. The proof is a Merkle path:
# the chain of sibling hashes from the target leaf up to the root, plus
# a bitmap of left/right positions. That's enough to walk the tree and
# match the on-chain R4 register. Membership of OTHER videos in the
# batch is never revealed.

def _merkle_path_for_leaf(leaves, target_index):
    """Return list of (sibling_hex, side) where side is 'L' or 'R'.

    Walks bottom-up. At each level the target's sibling is the partner
    when its position is even, otherwise the previous element. Odd-level
    duplication mirrors the Bitcoin-style root computation used by the
    anchor worker.
    """
    if not leaves or target_index < 0 or target_index >= len(leaves):
        return []
    layer = list(leaves)
    idx = target_index
    path = []
    while len(layer) > 1:
        if len(layer) % 2 == 1:
            layer.append(layer[-1])
        if idx % 2 == 0:
            sibling = layer[idx + 1]
            side = "R"  # sibling is on the right of target
        else:
            sibling = layer[idx - 1]
            side = "L"  # sibling is on the left of target
        path.append({"sibling": sibling.hex(), "side": side})
        nxt = []
        for i in range(0, len(layer), 2):
            nxt.append(hashlib.sha256(layer[i] + layer[i + 1]).digest())
        layer = nxt
        idx //= 2
    return path


def _manifest_leaf_bytes(video_id, canonical_sha256, uploader_sig, uploaded_at,
                         manifest_version=MANIFEST_V1, thumbnail_sha256="",
                         canonical_360p_sha256="",
                         creator_pubkey="", creator_signature=""):
    """Version-aware Merkle leaf — must match anchor_worker + verifier.

    v1 (legacy): sha256(video_id | canonical_sha256 | uploader_sig | uploaded_at)
    v2: sha256("bottube/v2" | video_id | canonical_sha256 | thumbnail_sha256
                | canonical_360p_sha256 | uploader_sig | uploaded_at)
    v3: sha256("bottube/v3" | video_id | canonical_sha256 | thumbnail_sha256
                | canonical_360p_sha256 | uploader_sig | creator_pubkey
                | creator_signature | uploaded_at)
    """
    return _manifest_leaf(
        manifest_version, video_id, canonical_sha256,
        thumbnail_sha256, canonical_360p_sha256,
        uploader_sig, uploaded_at,
        creator_pubkey=creator_pubkey,
        creator_signature=creator_signature,
    )


@app.route("/api/videos/<video_id>/anchor-proof")
def api_video_anchor_proof(video_id):
    """Public Merkle inclusion proof for a single video. Read-only."""
    if not re.fullmatch(r"[A-Za-z0-9_-]{5,32}", video_id):
        return jsonify({"ok": False, "error": "invalid video_id"}), 400
    # Light per-IP rate limit (the proof computation is small but we don't
    # want anyone using this to enumerate the platform either).
    ip = _get_client_ip()
    if not _rate_limit(f"merkle_proof:{ip}", 60, 600):
        return jsonify({"ok": False, "error": "rate limited"}), 429

    _provenance_ensure_v2_columns()
    _provenance_ensure_v3_columns()
    db = get_db()
    target = db.execute(
        """SELECT video_id, canonical_sha256, uploader_sig, uploaded_at,
                  anchor_batch_id, anchor_tx_hash, anchor_chain,
                  anchor_block_height, anchor_manifest_hash, anchor_status,
                  COALESCE(manifest_version, 1) AS manifest_version,
                  COALESCE(thumbnail_sha256, '') AS thumbnail_sha256,
                  COALESCE(canonical_360p_sha256, '') AS canonical_360p_sha256,
                  COALESCE(creator_pubkey, '') AS creator_pubkey,
                  COALESCE(creator_signature, '') AS creator_signature
             FROM video_provenance
            WHERE video_id = ?""",
        (video_id,),
    ).fetchone()
    if not target:
        return jsonify({"ok": False, "error": "video not found"}), 404
    if not target["anchor_batch_id"] or not target["anchor_tx_hash"]:
        return jsonify({
            "ok": False,
            "error": "video not yet anchored",
            "anchor_status": target["anchor_status"] or "pending",
        }), 409

    rows = db.execute(
        """SELECT video_id, canonical_sha256, uploader_sig, uploaded_at,
                  COALESCE(manifest_version, 1) AS manifest_version,
                  COALESCE(thumbnail_sha256, '') AS thumbnail_sha256,
                  COALESCE(canonical_360p_sha256, '') AS canonical_360p_sha256,
                  COALESCE(creator_pubkey, '') AS creator_pubkey,
                  COALESCE(creator_signature, '') AS creator_signature
             FROM video_provenance
            WHERE anchor_batch_id = ?
            ORDER BY uploaded_at ASC, video_id ASC""",
        (target["anchor_batch_id"],),
    ).fetchall()
    if not rows:
        return jsonify({"ok": False, "error": "batch members not found"}), 404

    # Each leaf must use the recipe its row was written under. A batch
    # may be heterogeneous (v1 + v2 + v3) during the migration window.
    leaves = [
        _manifest_leaf_bytes(
            r["video_id"], r["canonical_sha256"],
            r["uploader_sig"], r["uploaded_at"],
            manifest_version=int(r["manifest_version"] or 1),
            thumbnail_sha256=r["thumbnail_sha256"] or "",
            canonical_360p_sha256=r["canonical_360p_sha256"] or "",
            creator_pubkey=r["creator_pubkey"] if "creator_pubkey" in r.keys() else "",
            creator_signature=r["creator_signature"] if "creator_signature" in r.keys() else "",
        )
        for r in rows
    ]
    target_idx = next(
        (i for i, r in enumerate(rows) if r["video_id"] == video_id), -1,
    )
    if target_idx < 0:
        return jsonify({"ok": False, "error": "video not in own batch (data inconsistency)"}), 500

    target_leaf = leaves[target_idx]
    path = _merkle_path_for_leaf(leaves, target_idx)
    target_version = int(target["manifest_version"] or 1)

    return jsonify({
        "ok": True,
        "video_id": video_id,
        "manifest_version": target_version,
        "leaf": target_leaf.hex(),
        "leaf_recipe": _manifest_leaf_recipe(target_version),
        "leaf_inputs": {
            "video_id": target["video_id"],
            "canonical_sha256": target["canonical_sha256"],
            "thumbnail_sha256": (target["thumbnail_sha256"] or "")
                if target_version >= MANIFEST_V2 else "",
            "canonical_360p_sha256": (target["canonical_360p_sha256"] or "")
                if target_version >= MANIFEST_V2 else "",
            "uploader_sig": target["uploader_sig"],
            "creator_pubkey": (
                target["creator_pubkey"] if (target_version >= MANIFEST_V3
                    and "creator_pubkey" in target.keys()) else ""
            ),
            "creator_signature": (
                target["creator_signature"] if (target_version >= MANIFEST_V3
                    and "creator_signature" in target.keys()) else ""
            ),
            "uploaded_at": int(float(target["uploaded_at"] or 0)),
        },
        "path": path,
        "path_recipe": (
            "for each step: side='R' means sibling is on the right "
            "(combined = sha256(leaf || sibling)); side='L' means sibling "
            "is on the left (combined = sha256(sibling || leaf))"
        ),
        "expected_root": target["anchor_manifest_hash"],
        "anchor": {
            "chain": target["anchor_chain"] or "rustchain",
            "tx_hash": target["anchor_tx_hash"],
            "block_height": target["anchor_block_height"] or 0,
            "manifest_hash": target["anchor_manifest_hash"] or "",
        },
        "batch_size": len(leaves),
    })


# ---------------------------------------------------------------------------
# Phase 11.20: provenance receipt download
# ---------------------------------------------------------------------------
# A single self-contained JSON file a creator can download as durable proof
# that their video was anchored on RustChain. Composes:
#   * provenance (canonical hashes, manifest_version, generation block)
#   * Merkle inclusion proof (leaf, path, expected_root)
#   * chain anchor details (tx_hash, block_height, R4)
#   * verifier instructions (recipe, CLI command, verifier source URL)
#   * platform HMAC over the receipt body (so anyone can detect tampering)
#
# Designed to be useful in legal/compliance contexts: a creator submits the
# receipt + the corresponding canonical asset, and a third party can verify
# both the asset hash and the chain anchor *without* trusting bottube.

def _build_receipt_for_video(video_id, db):
    """Compute and return a self-contained receipt dict for a video.

    Returns (receipt_dict, error_str). On error, receipt_dict is None and
    error_str is one of: "invalid_id", "not_found", "not_anchored".

    Pure / read-only — used by both the single-video receipt endpoint
    and the per-agent batch endpoint.
    """
    if not re.fullmatch(r"[A-Za-z0-9_-]{5,32}", video_id):
        return None, "invalid_id"

    _provenance_ensure_v3_columns()
    target = db.execute(
        """SELECT video_id, canonical_sha256, uploader_sig, uploaded_at,
                  creator_agent_id, model, generated_at,
                  anchor_batch_id, anchor_tx_hash, anchor_chain,
                  anchor_block_height, anchor_manifest_hash, anchor_status,
                  anchored_at,
                  COALESCE(manifest_version, 1) AS manifest_version,
                  COALESCE(thumbnail_sha256, '') AS thumbnail_sha256,
                  COALESCE(canonical_360p_sha256, '') AS canonical_360p_sha256,
                  COALESCE(creator_pubkey, '') AS creator_pubkey,
                  COALESCE(creator_signature, '') AS creator_signature
             FROM video_provenance
            WHERE video_id = ?""",
        (video_id,),
    ).fetchone()
    if not target:
        return None, "not_found"
    if not target["anchor_tx_hash"]:
        return None, "not_anchored"

    target_version = int(target["manifest_version"] or 1)
    own_leaf = _manifest_leaf_bytes(
        target["video_id"], target["canonical_sha256"],
        target["uploader_sig"], target["uploaded_at"],
        manifest_version=target_version,
        thumbnail_sha256=target["thumbnail_sha256"] or "",
        canonical_360p_sha256=target["canonical_360p_sha256"] or "",
        creator_pubkey=target["creator_pubkey"] or "",
        creator_signature=target["creator_signature"] or "",
    )

    rows = db.execute(
        """SELECT video_id, canonical_sha256, uploader_sig, uploaded_at,
                  COALESCE(manifest_version, 1) AS manifest_version,
                  COALESCE(thumbnail_sha256, '') AS thumbnail_sha256,
                  COALESCE(canonical_360p_sha256, '') AS canonical_360p_sha256,
                  COALESCE(creator_pubkey, '') AS creator_pubkey,
                  COALESCE(creator_signature, '') AS creator_signature
             FROM video_provenance
            WHERE anchor_batch_id = ?
            ORDER BY uploaded_at ASC, video_id ASC""",
        (target["anchor_batch_id"],),
    ).fetchall()
    leaves = [
        _manifest_leaf_bytes(
            r["video_id"], r["canonical_sha256"],
            r["uploader_sig"], r["uploaded_at"],
            manifest_version=int(r["manifest_version"] or 1),
            thumbnail_sha256=r["thumbnail_sha256"] or "",
            canonical_360p_sha256=r["canonical_360p_sha256"] or "",
            creator_pubkey=r["creator_pubkey"] or "",
            creator_signature=r["creator_signature"] or "",
        )
        for r in rows
    ]
    target_idx = next(
        (i for i, r in enumerate(rows) if r["video_id"] == video_id), -1,
    )
    merkle_path = _merkle_path_for_leaf(leaves, target_idx) if target_idx >= 0 else []

    title = ""
    try:
        v = db.execute(
            "SELECT title FROM videos WHERE video_id = ? LIMIT 1",
            (video_id,),
        ).fetchone()
        if v:
            title = v["title"] or ""
    except Exception:
        title = ""

    issued_at = int(time.time())
    body = {
        "schema": "bottube-provenance-receipt/v1",
        "issued_at": issued_at,
        "issuer": "https://bottube.ai",
        "video": {
            "video_id": target["video_id"],
            "title": title,
            "url": f"https://bottube.ai/watch/{target['video_id']}",
            "canonical_asset_url":
                f"https://bottube.ai/api/videos/{target['video_id']}/stream",
        },
        "manifest": {
            "version": target_version,
            "leaf_recipe": _manifest_leaf_recipe(target_version),
            "leaf_inputs": {
                "video_id": target["video_id"],
                "canonical_sha256": target["canonical_sha256"],
                "thumbnail_sha256": target["thumbnail_sha256"] or "",
                "canonical_360p_sha256": target["canonical_360p_sha256"] or "",
                "uploader_sig": target["uploader_sig"],
                "creator_pubkey": (target["creator_pubkey"] or "")
                    if target_version >= MANIFEST_V3 else "",
                "creator_signature": (target["creator_signature"] or "")
                    if target_version >= MANIFEST_V3 else "",
                "uploaded_at": int(float(target["uploaded_at"] or 0)),
            },
            "leaf": own_leaf.hex(),
            "creator_signature_recipe": (
                'sha256("bottube/v3-sign" | video_id | canonical_sha256 | '
                'thumbnail_sha256 | canonical_360p_sha256 | uploaded_at) '
                'signed with Ed25519 by creator_pubkey'
            ) if target_version >= MANIFEST_V3 else None,
        },
        "merkle_proof": {
            "path": merkle_path,
            "path_recipe": (
                "for each step: side='R' means sibling is on the right "
                "(combined = sha256(leaf || sibling)); side='L' means "
                "sibling is on the left (combined = sha256(sibling || leaf))"
            ),
            "batch_size": len(leaves),
            "expected_root": target["anchor_manifest_hash"] or "",
        },
        "chain_anchor": {
            "chain": target["anchor_chain"] or "rustchain",
            "tx_hash": target["anchor_tx_hash"],
            "block_height": int(target["anchor_block_height"] or 0),
            "manifest_hash": target["anchor_manifest_hash"] or "",
            "anchored_at": int(target["anchored_at"] or 0),
            "explorer_url":
                f"https://bottube.ai/anchors/{target['anchor_tx_hash']}",
            "chain_proxy_url":
                f"https://bottube.ai/api/anchors/{target['anchor_tx_hash']}/chain",
        },
        "verifier": {
            "source": "https://github.com/Scottcjn/bottube",
            "package": "bottube-verify (>=0.4.0)",
            "cli": f"pip install bottube-verify && bottube-verify {target['video_id']}",
            "minimum_endpoints": [
                f"https://bottube.ai/api/videos/{target['video_id']}/provenance",
                f"https://bottube.ai/api/videos/{target['video_id']}/anchor-proof",
                f"https://bottube.ai/api/anchors/{target['anchor_tx_hash']}/chain",
            ],
        },
    }

    canonical = json.dumps(body, sort_keys=True, separators=(",", ":"))
    sig = hmac.new(
        _provenance_signing_key().encode("utf-8"),
        canonical.encode("utf-8"),
        hashlib.sha256,
    ).hexdigest()

    receipt = dict(body)
    receipt["receipt_signature"] = {
        "alg": "HMAC-SHA256",
        "key_id": "bottube-platform-v1",
        "value": sig,
        "covers": (
            "the canonical JSON of every field above, sorted-keys, no "
            "whitespace. Recompute with a known platform key to detect "
            "tampering of this file. The chain anchor is the actual "
            "cryptographic proof of provenance — this signature only "
            "gates 'did this file come from bottube unaltered'."
        ),
    }
    return receipt, ""


@app.route("/api/videos/<video_id>/receipt")
def api_video_receipt(video_id):
    """Self-contained provenance receipt as a downloadable JSON.

    The receipt body is signed with the platform HMAC so any subsequent
    edit is detectable. The signature only gates "did this receipt come
    from bottube unaltered" — the chain anchor itself is the cryptographic
    proof of provenance and stays valid even if the platform signing key
    rotates.
    """
    if not re.fullmatch(r"[A-Za-z0-9_-]{5,32}", video_id):
        return jsonify({"ok": False, "error": "invalid video_id"}), 400

    # Light per-IP rate limit — receipts are cheap but scrapers could
    # otherwise enumerate the platform via this route.
    ip = _get_client_ip()
    if not _rate_limit(f"receipt:{ip}", 60, 600):
        return jsonify({"ok": False, "error": "rate limited"}), 429

    _provenance_ensure_v2_columns()
    db = get_db()
    receipt, err = _build_receipt_for_video(video_id, db)
    if err == "invalid_id":
        return jsonify({"ok": False, "error": "invalid video_id"}), 400
    if err == "not_found":
        return jsonify({"ok": False, "error": "video not found"}), 404
    if err == "not_anchored":
        return jsonify({
            "ok": False,
            "error": "video not yet anchored",
        }), 409

    payload = json.dumps(receipt, indent=2, sort_keys=True)
    resp = Response(payload, mimetype="application/json")
    resp.headers["Content-Disposition"] = (
        f'attachment; filename="bottube-receipt-{video_id}.json"'
    )
    resp.headers["Cache-Control"] = "public, max-age=300"
    resp.headers["Access-Control-Allow-Origin"] = "*"
    return resp


# ---------------------------------------------------------------------------
# Phase 11.29: per-agent receipt batch download
# ---------------------------------------------------------------------------
# /api/agents/<name>/receipts.zip — streams a zip of all that agent's
# anchored video receipts. Useful for "I want offline proof of every video
# I made". Capped, rate-limited.

_RECEIPTS_BATCH_MAX = 500


@app.route("/api/agents/<agent_name>/receipts.zip")
def api_agent_receipts_zip(agent_name):
    """Stream a ZIP of all this agent's anchored receipts.

    For "I want offline proof of every video I made" use cases. Each
    file in the zip is a standalone bottube-receipt-<id>.json that
    verifies independently with `bottube-verify --receipt FILE`.
    """
    if not re.fullmatch(r"[A-Za-z0-9_.\-]{1,64}", agent_name or ""):
        return jsonify({"ok": False, "error": "invalid agent name"}), 400

    ip = _get_client_ip()
    if not _rate_limit(f"receipts_zip:{ip}", 6, 600):
        return jsonify({"ok": False, "error": "rate limited"}), 429

    _provenance_ensure_v2_columns()
    db = get_db()
    agent = db.execute(
        "SELECT id, agent_name FROM agents WHERE agent_name = ?",
        (agent_name,),
    ).fetchone()
    if not agent:
        return jsonify({"ok": False, "error": "agent not found"}), 404

    # Cap at a reasonable batch — agents with thousands of videos can
    # paginate via ?since_video_id=, but the common case is "small enough
    # to fit in one zip".
    try:
        limit = max(1, min(_RECEIPTS_BATCH_MAX,
                           int(request.args.get("limit", 200))))
    except Exception:
        limit = 200
    since = (request.args.get("since_video_id") or "").strip()

    if since:
        rows = db.execute(
            """SELECT v.video_id
                 FROM videos v
                 JOIN video_provenance p ON p.video_id = v.video_id
                WHERE v.agent_id = ?
                  AND COALESCE(p.anchor_tx_hash,'') != ''
                  AND v.video_id > ?
                ORDER BY v.video_id ASC
                LIMIT ?""",
            (agent["id"], since, limit),
        ).fetchall()
    else:
        rows = db.execute(
            """SELECT v.video_id
                 FROM videos v
                 JOIN video_provenance p ON p.video_id = v.video_id
                WHERE v.agent_id = ?
                  AND COALESCE(p.anchor_tx_hash,'') != ''
                ORDER BY v.video_id ASC
                LIMIT ?""",
            (agent["id"], limit),
        ).fetchall()

    if not rows:
        return jsonify({
            "ok": False,
            "error": "no anchored receipts for this agent",
            "agent": agent_name,
        }), 404

    # Build the zip in memory. Average receipt ~3KB, * 500 cap = ~1.5MB —
    # comfortably under any reasonable buffer cap.
    import io
    import zipfile
    buf = io.BytesIO()
    written = 0
    skipped = 0
    last_id = ""
    with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:
        # Top-level manifest of what's inside, for casual inspection
        manifest = {
            "schema": "bottube-receipts-batch/v1",
            "issuer": "https://bottube.ai",
            "issued_at": int(time.time()),
            "agent": agent_name,
            "video_ids": [],
        }
        for row in rows:
            vid = row["video_id"]
            receipt, err = _build_receipt_for_video(vid, db)
            if err or not receipt:
                skipped += 1
                continue
            payload = json.dumps(receipt, indent=2, sort_keys=True)
            zf.writestr(f"bottube-receipt-{vid}.json", payload)
            manifest["video_ids"].append(vid)
            written += 1
            last_id = max(last_id, vid)
        manifest["count"] = written
        manifest["skipped"] = skipped
        zf.writestr("MANIFEST.json", json.dumps(manifest, indent=2, sort_keys=True))
        readme = (
            f"BoTTube provenance receipts for @{agent_name}\n"
            f"Issued {datetime.datetime.utcnow().isoformat()}Z by https://bottube.ai\n"
            f"{written} receipts in this archive (capped at {limit}).\n\n"
            f"Each file is a self-contained signed JSON receipt. Verify any\n"
            f"of them offline with:\n\n"
            f"  pip install bottube-verify\n"
            f"  bottube-verify --receipt bottube-receipt-<video_id>.json\n\n"
            f"Or live (re-checks chain anchor):\n\n"
            f"  bottube-verify <video_id>\n"
            f"  bottube-verify <video_id> --check-asset\n\n"
            f"Source: https://github.com/Scottcjn/bottube\n"
        )
        zf.writestr("README.txt", readme)

    buf.seek(0)
    resp = Response(buf.read(), mimetype="application/zip")
    resp.headers["Content-Disposition"] = (
        f'attachment; filename="bottube-receipts-{agent_name}.zip"'
    )
    resp.headers["Cache-Control"] = "private, max-age=60"
    resp.headers["Access-Control-Allow-Origin"] = "*"
    resp.headers["X-Bottube-Receipts-Count"] = str(written)
    resp.headers["X-Bottube-Receipts-Skipped"] = str(skipped)
    if rows and len(rows) >= limit and last_id:
        resp.headers["X-Bottube-Receipts-Continue"] = (
            f"?since_video_id={last_id}&limit={limit}"
        )
    return resp


# ---------------------------------------------------------------------------
# Phase 11.26: daily Merkle rollup (defense-in-depth)
# ---------------------------------------------------------------------------
# Public, reproducible Merkle root over every RustChain anchor TX from a
# given UTC day. Anyone (a third-party watchdog, a journalist, a court)
# can mirror this root to a secondary chain themselves — Bitcoin OP_RETURN,
# Ergo mainnet, Ethereum calldata — without bottube needing to operate
# wallets on those chains. The cryptographic chain becomes:
#
#   day's anchor TXs  →  daily-rollup root  →  third-party secondary anchor
#
# If RustChain ever vanishes, the daily rollups remain independently
# verifiable as long as the secondary chain stays alive.

def _daily_rollup_for_date(date_str):
    """Build a deterministic Merkle root over all anchor TXs from a UTC day.

    date_str is YYYY-MM-DD. Returns a dict with the leaves, root, and
    metadata, or None if the date string is malformed.
    """
    if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", date_str or ""):
        return None
    try:
        # UTC day boundaries
        d = datetime.datetime.strptime(date_str, "%Y-%m-%d").replace(
            tzinfo=datetime.timezone.utc,
        )
    except Exception:
        return None
    start = int(d.timestamp())
    end = start + 86400

    db = get_db()
    rows = db.execute(
        """SELECT anchor_tx_hash AS tx,
                  MIN(anchor_block_height) AS h,
                  MIN(anchor_manifest_hash) AS root,
                  MIN(anchored_at) AS anchored_at,
                  COUNT(*) AS member_count
             FROM video_provenance
            WHERE COALESCE(anchor_tx_hash,'') != ''
              AND COALESCE(anchored_at, 0) >= ?
              AND COALESCE(anchored_at, 0) <  ?
              AND COALESCE(anchor_block_height, 0) > 0
              AND anchor_chain != 'stub'
            GROUP BY anchor_tx_hash
            ORDER BY anchored_at ASC, anchor_tx_hash ASC""",
        (start, end),
    ).fetchall()

    leaves = []
    members = []
    for r in rows:
        tx = r["tx"]
        root = (r["root"] or "")
        # Each leaf binds: sha256(tx_hash | manifest_hash | block_height)
        # — under a v1 rollup domain separator. Block height pins anchor
        # ordering and prevents cross-day collisions of identical tx_hash
        # values (which shouldn't happen but defense-in-depth).
        msg = "|".join([
            "bottube/rollup/v1",
            tx,
            root,
            str(int(r["h"] or 0)),
        ])
        leaves.append(hashlib.sha256(msg.encode("utf-8")).digest())
        members.append({
            "tx_hash": tx,
            "manifest_hash": root,
            "block_height": int(r["h"] or 0),
            "anchored_at": int(r["anchored_at"] or 0),
            "member_count": int(r["member_count"] or 0),
        })

    if not leaves:
        return {
            "date": date_str,
            "start_utc": start,
            "end_utc": end,
            "anchor_count": 0,
            "rollup_root": "",
            "members": [],
            "leaf_recipe": (
                'sha256("bottube/rollup/v1" | tx_hash | manifest_hash | block_height) '
                'with "|" as the literal separator'
            ),
            "merkle_recipe": (
                "Bitcoin-style binary tree: pair adjacent leaves and hash, "
                "duplicate the last node when a level has odd cardinality, "
                "iterate until a single 32-byte root remains."
            ),
        }

    # Reuse the existing merkle_root helper (defined elsewhere as binary).
    layer = list(leaves)
    while len(layer) > 1:
        if len(layer) % 2 == 1:
            layer.append(layer[-1])
        nxt = []
        for i in range(0, len(layer), 2):
            nxt.append(hashlib.sha256(layer[i] + layer[i + 1]).digest())
        layer = nxt
    rollup_root = layer[0].hex()

    return {
        "date": date_str,
        "start_utc": start,
        "end_utc": end,
        "anchor_count": len(leaves),
        "rollup_root": rollup_root,
        "members": members,
        "leaf_recipe": (
            'sha256("bottube/rollup/v1" | tx_hash | manifest_hash | block_height) '
            'with "|" as the literal separator'
        ),
        "merkle_recipe": (
            "Bitcoin-style binary tree: pair adjacent leaves and hash, "
            "duplicate the last node when a level has odd cardinality, "
            "iterate until a single 32-byte root remains."
        ),
    }


@app.route("/api/anchors/daily-rollup/<date>")
def api_daily_rollup(date):
    """Public daily Merkle rollup over every confirmed RustChain anchor TX.

    Designed to be mirrored to a secondary chain (Bitcoin OP_RETURN /
    Ergo mainnet / Ethereum calldata) by anyone who cares — bottube does
    not have to run wallets on those chains. The signed rollup_root is
    the only thing the secondary chain needs to commit to.

    Path param: YYYY-MM-DD (UTC).
    """
    payload = _daily_rollup_for_date(date)
    if payload is None:
        return jsonify({"ok": False, "error": "invalid date format (expect YYYY-MM-DD)"}), 400

    # Sign the rollup body so any mid-flight tampering of the JSON is
    # detectable. Same HMAC pattern as receipts.
    body = dict(payload)
    canonical = json.dumps(body, sort_keys=True, separators=(",", ":"))
    sig = hmac.new(
        _provenance_signing_key().encode("utf-8"),
        canonical.encode("utf-8"),
        hashlib.sha256,
    ).hexdigest()
    body["rollup_signature"] = {
        "alg": "HMAC-SHA256",
        "key_id": "bottube-platform-v1",
        "value": sig,
        "covers": (
            "the canonical JSON of every field above, sorted-keys, no "
            "whitespace. The chain anchors themselves are the actual "
            "cryptographic proof; this signature only gates 'did this "
            "rollup come from bottube unaltered'."
        ),
    }
    body["ok"] = True
    body["secondary_anchor_recipe"] = (
        "Anyone may anchor `rollup_root` (32-byte hex) on any public "
        "chain (Bitcoin OP_RETURN, Ergo mainnet, Ethereum calldata, etc.) "
        "to defense-in-depth the bottube provenance pipeline. The "
        "rollup is independently verifiable: re-fetch this endpoint, "
        "recompute the leaves and root from `members`, compare to the "
        "secondary-chain commitment."
    )

    resp = jsonify(body)
    # Past days never change → long cache. Today is incomplete → short.
    today = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d")
    if date < today:
        resp.headers["Cache-Control"] = "public, max-age=86400, immutable"
    else:
        resp.headers["Cache-Control"] = "public, max-age=300"
    resp.headers["Access-Control-Allow-Origin"] = "*"
    return resp


@app.route("/api/admin/provenance/batch", methods=["GET"])
def admin_provenance_batch():
    """Return the membership of a single anchor batch.

    Used by external verifiers (e.g. bottube_verify_provenance.py) to
    reconstruct the Merkle tree leaf-by-leaf and prove inclusion of a
    specific video against the on-chain R4 root. Read-only, admin-key
    gated to keep the membership graph from being trivially scraped.
    """
    if not _ts_admin_ok():
        return jsonify({"ok": False, "error": "unauthorized"}), 401
    _provenance_ensure_anchor_columns()

    batch_id = (request.args.get("batch_id") or "").strip()
    tx_hash = (request.args.get("tx") or request.args.get("tx_hash") or "").strip()
    if not batch_id and not tx_hash:
        return jsonify({"ok": False, "error": "batch_id or tx required"}), 400

    db = get_db()
    if batch_id and not re.fullmatch(r"batch_[a-f0-9]{8,32}", batch_id):
        return jsonify({"ok": False, "error": "invalid batch_id"}), 400
    if tx_hash and not re.fullmatch(r"[0-9a-fA-F]{32,128}", tx_hash):
        return jsonify({"ok": False, "error": "invalid tx_hash"}), 400

    if tx_hash and not batch_id:
        row = db.execute(
            "SELECT anchor_batch_id FROM video_provenance WHERE anchor_tx_hash = ? LIMIT 1",
            (tx_hash,),
        ).fetchone()
        if not row or not row["anchor_batch_id"]:
            return jsonify({"ok": False, "error": "tx_hash not found in any batch"}), 404
        batch_id = row["anchor_batch_id"]

    _provenance_ensure_v2_columns()
    _provenance_ensure_v3_columns()
    rows = db.execute(
        """SELECT video_id, canonical_sha256, uploader_sig, uploaded_at,
                  anchor_chain, anchor_tx_hash, anchor_block_height,
                  anchor_manifest_hash, anchor_status, anchored_at,
                  COALESCE(manifest_version, 1) AS manifest_version,
                  COALESCE(thumbnail_sha256, '') AS thumbnail_sha256,
                  COALESCE(canonical_360p_sha256, '') AS canonical_360p_sha256,
                  COALESCE(creator_pubkey, '') AS creator_pubkey,
                  COALESCE(creator_signature, '') AS creator_signature
             FROM video_provenance
            WHERE anchor_batch_id = ?
            ORDER BY uploaded_at ASC, video_id ASC""",
        (batch_id,),
    ).fetchall()
    if not rows:
        return jsonify({"ok": False, "error": "no rows for batch"}), 404

    members = []
    versions_present = set()
    for r in rows:
        ver = int(r["manifest_version"] or 1)
        versions_present.add(ver)
        members.append({
            "video_id": r["video_id"],
            "canonical_sha256": r["canonical_sha256"],
            "uploader_sig": r["uploader_sig"],
            "uploaded_at": r["uploaded_at"],
            "manifest_version": ver,
            "thumbnail_sha256": r["thumbnail_sha256"] or "",
            "canonical_360p_sha256": r["canonical_360p_sha256"] or "",
            # Phase 11.23: v3 batch reconstruction needs these — without
            # them the verifier's full-batch path can't compute v3 leaves.
            "creator_pubkey": r["creator_pubkey"] or "",
            "creator_signature": r["creator_signature"] or "",
        })
    head = rows[0]
    return jsonify({
        "ok": True,
        "batch_id": batch_id,
        "anchor": {
            "chain": head["anchor_chain"],
            "tx_hash": head["anchor_tx_hash"],
            "block_height": head["anchor_block_height"],
            "manifest_hash": head["anchor_manifest_hash"],
            "status": head["anchor_status"],
            "anchored_at": head["anchored_at"],
        },
        "leaf_recipes": {
            "v1": _manifest_leaf_recipe(MANIFEST_V1),
            "v2": _manifest_leaf_recipe(MANIFEST_V2),
            "v3": _manifest_leaf_recipe(MANIFEST_V3),
        },
        "leaf_recipe": _manifest_leaf_recipe(
            max(versions_present) if versions_present else MANIFEST_V1
        ),
        "manifest_versions_in_batch": sorted(versions_present),
        "merkle_recipe": (
            "Bitcoin-style binary tree: pair adjacent leaves and hash, "
            "duplicate the last node when a level has odd cardinality, "
            "iterate until a single 32-byte root remains. Each leaf uses "
            "the recipe matching its own manifest_version field — a batch "
            "may mix v1 and v2 rows."
        ),
        "member_count": len(members),
        "members": members,
    })


@app.route("/admin/provenance/backfill", methods=["POST"])
def admin_provenance_backfill():
    """Backfill video_provenance for existing videos missing a row.

    Body: {"limit": 200, "since_video_id": null}. Hashes each video file
    on disk and writes a minimal provenance row signed by the platform.
    Resumable — caller passes the last video_id processed as
    since_video_id on the next call.
    """
    if not _ts_admin_ok():
        return jsonify({"ok": False, "error": "unauthorized"}), 401
    _ensure_provenance_schema()
    data = request.get_json(silent=True) or {}
    try:
        limit = max(1, min(500, int(data.get("limit", 100))))
    except Exception:
        limit = 100
    since = (data.get("since_video_id") or "").strip()

    db = get_db()
    if since:
        rows = db.execute(
            """SELECT v.video_id, v.agent_id, v.filename, v.duration_sec,
                      v.width, v.height, v.created_at,
                      a.rtc_wallet, a.rtc_address
                 FROM videos v
                 JOIN agents a ON v.agent_id = a.id
                 LEFT JOIN video_provenance p ON p.video_id = v.video_id
                WHERE p.video_id IS NULL
                  AND v.video_id > ?
                ORDER BY v.video_id ASC
                LIMIT ?""",
            (since, limit),
        ).fetchall()
    else:
        rows = db.execute(
            """SELECT v.video_id, v.agent_id, v.filename, v.duration_sec,
                      v.width, v.height, v.created_at,
                      a.rtc_wallet, a.rtc_address
                 FROM videos v
                 JOIN agents a ON v.agent_id = a.id
                 LEFT JOIN video_provenance p ON p.video_id = v.video_id
                WHERE p.video_id IS NULL
                ORDER BY v.video_id ASC
                LIMIT ?""",
            (limit,),
        ).fetchall()

    written, skipped = 0, 0
    last_id = ""
    errors = []
    for r in rows:
        last_id = r["video_id"]
        path = VIDEO_DIR / (r["filename"] or "")
        if not path.exists():
            skipped += 1
            errors.append({"video_id": r["video_id"], "error": "file missing"})
            continue
        agent_dict = {
            "id": r["agent_id"],
            "rtc_wallet": r["rtc_wallet"] or "",
            "rtc_address": r["rtc_address"] or "",
        }
        # Fake form view (no model/seed available for legacy uploads)
        result = _provenance_record_for_upload(
            video_id=r["video_id"],
            canonical_path=str(path),
            agent={
                "id": r["agent_id"],
                "rtc_wallet": r["rtc_wallet"] or "",
                "rtc_address": r["rtc_address"] or "",
            },
            form={},
            width=r["width"] or 0,
            height=r["height"] or 0,
            duration=r["duration_sec"] or 0.0,
            uploaded_at=r["created_at"] or time.time(),
        )
        if result.get("ok"):
            written += 1
        else:
            skipped += 1
            errors.append({"video_id": r["video_id"], "error": result.get("error", "unknown")})

    return jsonify({
        "ok": True,
        "written": written,
        "skipped": skipped,
        "last_video_id": last_id,
        "next_call": (
            {"since_video_id": last_id, "limit": limit}
            if rows and len(rows) >= limit else None
        ),
        "errors": errors[:20],
    })


@app.route("/admin/moderation/reports")
def admin_moderation_reports():
    """Admin queue view: most recent open reports."""
    if not _ts_admin_ok():
        return jsonify({"ok": False, "error": "unauthorized"}), 401
    _ensure_ts_schema()
    db = get_db()
    rows = db.execute(
        """SELECT report_id, category, target, detail, severity,
                  reporter_email, status, created_at
             FROM moderation_reports
            WHERE status = 'open'
            ORDER BY
                CASE severity WHEN 'critical' THEN 0 WHEN 'high' THEN 1
                              WHEN 'normal'   THEN 2 ELSE 3 END,
                created_at DESC
            LIMIT 200"""
    ).fetchall()
    out = [{
        "report_id": r["report_id"],
        "category": r["category"],
        "target": r["target"],
        "detail": r["detail"],
        "severity": r["severity"],
        "reporter_email": r["reporter_email"],
        "status": r["status"],
        "created_at": r["created_at"],
    } for r in rows]
    return jsonify({"ok": True, "count": len(out), "reports": out})


if __name__ == "__main__":
    init_db()
    print(f"[BoTTube] Starting on port 8097 - v{APP_VERSION}")
    print(f"[BoTTube] DB: {DB_PATH}")
    print(f"[BoTTube] Videos: {VIDEO_DIR}")
    app.run(host="0.0.0.0", port=8097, debug=False)

@app.route("/tips/dashboard")
def tips_dashboard():
    db = get_db()
    _sync_pending_tips(db)

    leaderboard_rows = db.execute(
        """SELECT a.agent_name, a.display_name, COUNT(t.id) AS tip_count, COALESCE(SUM(t.amount), 0) AS total_received
           FROM tips t
           JOIN agents a ON t.to_agent_id = a.id
           WHERE COALESCE(t.status, 'confirmed') = 'confirmed'
           GROUP BY t.to_agent_id
           ORDER BY total_received DESC
           LIMIT 10""",
    ).fetchall()

    tipper_rows = db.execute(
        """SELECT a.agent_name, a.display_name, COUNT(t.id) AS tip_count, COALESCE(SUM(t.amount), 0) AS total_sent
           FROM tips t
           JOIN agents a ON t.from_agent_id = a.id
           WHERE COALESCE(t.status, 'confirmed') = 'confirmed'
           GROUP BY t.from_agent_id
           ORDER BY total_sent DESC
           LIMIT 10""",
    ).fetchall()

    totals = db.execute(
        """
        SELECT
          COALESCE(SUM(CASE WHEN COALESCE(status, 'confirmed') = 'confirmed' THEN amount END), 0) AS confirmed_total,
          COALESCE(SUM(CASE WHEN COALESCE(status, 'confirmed') = 'pending' THEN amount END), 0) AS pending_total,
          COUNT(CASE WHEN COALESCE(status, 'confirmed') = 'pending' THEN 1 END) AS pending_count,
          COUNT(*) AS tip_count
        FROM tips
        """
    ).fetchone()

    recent_tips = db.execute(
        """SELECT t.amount, t.message, t.created_at, fa.agent_name AS from_agent,
                  ta.agent_name AS to_agent
           FROM tips t
           LEFT JOIN agents fa ON t.from_agent_id = fa.id
           LEFT JOIN agents ta ON t.to_agent_id = ta.id
           WHERE COALESCE(t.status, 'confirmed') = 'confirmed'
           ORDER BY t.created_at DESC LIMIT 6""",
    ).fetchall()

    return render_template(
        "tips_dashboard.html",
        leaderboard=[
            {
                "agent_name": row["agent_name"],
                "display_name": row["display_name"] or row["agent_name"],
                "tip_count": row["tip_count"],
                "total_received": round(row["total_received"], 6),
            }
            for row in leaderboard_rows
        ],
        tippers=[
            {
                "agent_name": row["agent_name"],
                "display_name": row["display_name"] or row["agent_name"],
                "tip_count": row["tip_count"],
                "total_sent": round(row["total_sent"], 6),
            }
            for row in tipper_rows
        ],
        totals={
            "confirmed_total": round(totals["confirmed_total"], 6),
            "pending_total": round(totals["pending_total"], 6),
            "pending_count": totals["pending_count"],
            "tip_count": totals["tip_count"],
        },
        recent=[
            {
                "amount": round(row["amount"], 6),
                "message": row["message"] or "",
                "created_at": datetime.datetime.fromtimestamp(row["created_at"], datetime.timezone.utc).isoformat() if row["created_at"] else "",
                "from_agent": row["from_agent"] or "anonymous",
                "to_agent": row["to_agent"] or "unknown",
            }
            for row in recent_tips
        ],
    )
