"""Routes for metrics dashboard."""

from datetime import datetime, timedelta, UTC
from typing import Any
from urllib.parse import urlparse

from flask import Blueprint, jsonify, request, session as flask_session
from loguru import logger
from sqlalchemy import case, func

from ...database.models import (
    Journal,
    Paper,
    PaperAppearance,
    RateLimitEstimate,
    ResearchHistory,
    ResearchRating,
    ResearchResource,
    ResearchStrategy,
    TokenUsage,
)
from ...constants import get_available_strategies
from ...domain_classifier import DomainClassifier, DomainClassification
from ...database.session_context import get_user_db_session
from ...metrics import TokenCounter
from ...metrics.query_utils import (
    get_context_overflow_truncation_summary,
    get_period_days,
    get_time_filter_condition,
)
from ...metrics.search_tracker import get_search_tracker
from ...security.decorators import require_json_body
from ...security.rate_limiter import journal_data_limit, journals_read_limit
from ..auth.decorators import login_required
from ..utils.templates import render_template_with_defaults

# Create a Blueprint for metrics
metrics_bp = Blueprint("metrics", __name__, url_prefix="/metrics")

# NOTE: Routes use flask_session["username"] (not .get()) intentionally.
# @login_required guarantees the key exists; direct access fails fast
# if the decorator is ever removed.


def _extract_domain(url):
    """Extract normalized domain from URL, stripping www. prefix."""
    try:
        parsed = urlparse(url)
        domain = parsed.netloc.lower()
        if domain.startswith("www."):
            domain = domain[4:]
        return domain if domain else None
    except (ValueError, AttributeError, TypeError):
        return None


def get_rating_analytics(period="30d", research_mode="all", username=None):
    """Get rating analytics for the specified period and research mode."""
    try:
        if not username:
            username = flask_session.get("username")

        if not username:
            return {
                "rating_analytics": {
                    "avg_rating": None,
                    "total_ratings": 0,
                    "rating_distribution": {},
                    "satisfaction_stats": {
                        "very_satisfied": 0,
                        "satisfied": 0,
                        "neutral": 0,
                        "dissatisfied": 0,
                        "very_dissatisfied": 0,
                    },
                    "error": "No user session",
                }
            }

        # Calculate date range
        days = get_period_days(period)

        with get_user_db_session(username) as session:
            query = session.query(ResearchRating)

            # Apply time filter
            if days:
                cutoff_date = datetime.now(UTC) - timedelta(days=days)
                query = query.filter(ResearchRating.created_at >= cutoff_date)

            # Get all ratings
            ratings = query.all()

            if not ratings:
                return {
                    "rating_analytics": {
                        "avg_rating": None,
                        "total_ratings": 0,
                        "rating_distribution": {},
                        "satisfaction_stats": {
                            "very_satisfied": 0,
                            "satisfied": 0,
                            "neutral": 0,
                            "dissatisfied": 0,
                            "very_dissatisfied": 0,
                        },
                    }
                }

            # Calculate statistics
            rating_values = [r.rating for r in ratings]
            avg_rating = sum(rating_values) / len(rating_values)

            # Rating distribution
            rating_counts = {}
            for i in range(1, 6):
                rating_counts[str(i)] = rating_values.count(i)

            # Satisfaction categories
            satisfaction_stats = {
                "very_satisfied": rating_values.count(5),
                "satisfied": rating_values.count(4),
                "neutral": rating_values.count(3),
                "dissatisfied": rating_values.count(2),
                "very_dissatisfied": rating_values.count(1),
            }

            return {
                "rating_analytics": {
                    "avg_rating": round(avg_rating, 1),
                    "total_ratings": len(ratings),
                    "rating_distribution": rating_counts,
                    "satisfaction_stats": satisfaction_stats,
                }
            }

    except Exception:
        logger.exception("Error getting rating analytics")
        return {
            "rating_analytics": {
                "avg_rating": None,
                "total_ratings": 0,
                "rating_distribution": {},
                "satisfaction_stats": {
                    "very_satisfied": 0,
                    "satisfied": 0,
                    "neutral": 0,
                    "dissatisfied": 0,
                    "very_dissatisfied": 0,
                },
            }
        }


def get_link_analytics(period="30d", username=None):
    """Get link analytics from research resources."""
    try:
        if not username:
            username = flask_session.get("username")

        if not username:
            return {
                "link_analytics": {
                    "top_domains": [],
                    "total_unique_domains": 0,
                    "avg_links_per_research": 0,
                    "domain_distribution": {},
                    "source_type_analysis": {},
                    "academic_vs_general": {},
                    "total_links": 0,
                    "error": "No user session",
                }
            }

        # Calculate date range
        days = get_period_days(period)

        with get_user_db_session(username) as session:
            # Project only the columns the analytics loop reads (url /
            # research_id / created_at / source_type / title) plus a SQL-level
            # boolean for whether content_preview is present. Loading full
            # ``ResearchResource`` entities would materialize the
            # ``content_preview`` Text column for every row on this whole-table
            # scan — which runs unfiltered when ``period=all`` (#4560).
            has_preview = (
                ResearchResource.content_preview.isnot(None)
                & (ResearchResource.content_preview != "")
            ).label("has_preview")
            query = session.query(
                ResearchResource.url,
                ResearchResource.research_id,
                ResearchResource.created_at,
                ResearchResource.source_type,
                ResearchResource.title,
                has_preview,
            )

            # Apply time filter
            if days:
                cutoff_date = datetime.now(UTC) - timedelta(days=days)
                query = query.filter(
                    ResearchResource.created_at >= cutoff_date.isoformat()
                )

            # Get all resources
            resources = query.all()

            if not resources:
                return {
                    "link_analytics": {
                        "top_domains": [],
                        "total_unique_domains": 0,
                        "avg_links_per_research": 0,
                        "domain_distribution": {},
                        "source_type_analysis": {},
                        "academic_vs_general": {},
                        "total_links": 0,
                    }
                }

            # Extract domains from URLs
            domain_counts: dict[str, Any] = {}
            domain_researches: dict[
                str, Any
            ] = {}  # Track which researches used each domain
            source_types: dict[str, Any] = {}
            temporal_data: dict[str, Any] = {}  # Track links over time
            domain_connections: dict[
                str, Any
            ] = {}  # Track domain co-occurrences

            # Generic category counting from LLM classifications
            category_counts: dict[str, Any] = {}

            quality_metrics = {
                "with_title": 0,
                "with_preview": 0,
                "with_both": 0,
                "total": 0,
            }

            # First pass: collect all domains from resources
            all_domains = set()
            for resource in resources:
                if resource.url:
                    domain = _extract_domain(resource.url)
                    if domain:
                        all_domains.add(domain)

            # Batch load all domain classifications in one query (fix N+1)
            domain_classifications_map = {}
            if all_domains:
                all_classifications = (
                    session.query(DomainClassification)
                    .filter(DomainClassification.domain.in_(all_domains))
                    .all()
                )
                for classification in all_classifications:
                    domain_classifications_map[classification.domain] = (
                        classification
                    )

            # Second pass: process resources with pre-loaded classifications
            for resource in resources:
                if resource.url:
                    try:
                        domain = _extract_domain(resource.url)
                        if not domain:
                            continue

                        # Count domains
                        domain_counts[domain] = domain_counts.get(domain, 0) + 1

                        # Track research IDs for each domain
                        if domain not in domain_researches:
                            domain_researches[domain] = set()
                        domain_researches[domain].add(resource.research_id)

                        # Track temporal data (daily counts)
                        if resource.created_at:
                            date_str = resource.created_at[
                                :10
                            ]  # Extract YYYY-MM-DD
                            temporal_data[date_str] = (
                                temporal_data.get(date_str, 0) + 1
                            )

                        # Count categories from pre-loaded classifications (no N+1)
                        classification = domain_classifications_map.get(domain)
                        if classification:
                            category = classification.category
                            category_counts[category] = (
                                category_counts.get(category, 0) + 1
                            )
                        else:
                            category_counts["Unclassified"] = (
                                category_counts.get("Unclassified", 0) + 1
                            )

                        # Track source type from metadata if available
                        if resource.source_type:
                            source_types[resource.source_type] = (
                                source_types.get(resource.source_type, 0) + 1
                            )

                        # Track quality metrics
                        quality_metrics["total"] += 1
                        if resource.title:
                            quality_metrics["with_title"] += 1
                        if resource.has_preview:
                            quality_metrics["with_preview"] += 1
                        if resource.title and resource.has_preview:
                            quality_metrics["with_both"] += 1

                        # Track domain co-occurrences for network visualization
                        research_id = resource.research_id
                        if research_id not in domain_connections:
                            domain_connections[research_id] = []
                        domain_connections[research_id].append(domain)

                    except Exception:
                        logger.exception(f"Error parsing URL {resource.url}")

            # Sort domains by count and get top 10
            sorted_domains = sorted(
                domain_counts.items(), key=lambda x: x[1], reverse=True
            )
            top_10_domains = sorted_domains[:10]

            # Calculate domain distribution (top domains vs others)
            top_10_count = sum(count for _, count in top_10_domains)
            others_count = len(resources) - top_10_count

            # Get unique research IDs to calculate average
            unique_research_ids = {r.research_id for r in resources}
            avg_links = (
                len(resources) / len(unique_research_ids)
                if unique_research_ids
                else 0
            )

            # Prepare temporal trend data (sorted by date)
            temporal_trend = sorted(
                [
                    {"date": date, "count": count}
                    for date, count in temporal_data.items()
                ],
                key=lambda x: x["date"],
            )

            # Get most recent research for each top domain and classifications
            domain_recent_research = {}
            # Build domain_classifications dict from pre-loaded data
            domain_classifications = {
                domain: {
                    "category": classification.category,
                    "subcategory": classification.subcategory,
                    "confidence": classification.confidence,
                }
                for domain, classification in domain_classifications_map.items()
            }

            # Batch-load research details for top domains (fix N+1 query)
            all_research_ids = []
            domain_research_id_lists = {}
            for domain, _ in top_10_domains:
                if domain in domain_researches:
                    ids = list(domain_researches[domain])[:3]
                    domain_research_id_lists[domain] = ids
                    all_research_ids.extend(ids)

            research_by_id = {}
            if all_research_ids:
                researches = (
                    session.query(ResearchHistory)
                    .filter(ResearchHistory.id.in_(all_research_ids))
                    .all()
                )
                research_by_id = {r.id: r for r in researches}

            for domain, ids in domain_research_id_lists.items():
                domain_recent_research[domain] = [
                    {
                        "id": r_id,
                        "query": research_by_id[r_id].query[:50]
                        if research_by_id.get(r_id)
                        and research_by_id[r_id].query
                        else "Research",
                    }
                    for r_id in ids
                    if r_id in research_by_id
                ]

            return {
                "link_analytics": {
                    "top_domains": [
                        {
                            "domain": domain,
                            "count": count,
                            "percentage": round(
                                count / len(resources) * 100, 1
                            ),
                            "research_count": len(
                                domain_researches.get(domain, set())
                            ),
                            "recent_researches": domain_recent_research.get(
                                domain, []
                            ),
                            "classification": domain_classifications.get(
                                domain, None
                            ),
                        }
                        for domain, count in top_10_domains
                    ],
                    "total_unique_domains": len(domain_counts),
                    "avg_links_per_research": round(avg_links, 1),
                    "domain_distribution": {
                        "top_10": top_10_count,
                        "others": others_count,
                    },
                    "source_type_analysis": source_types,
                    "category_distribution": category_counts,
                    # Generic pie chart data - use whatever LLM classifier outputs
                    "domain_categories": category_counts,
                    "total_links": len(resources),
                    "total_researches": len(unique_research_ids),
                    "temporal_trend": temporal_trend,
                    "domain_metrics": {
                        domain: {
                            "usage_count": count,
                            "usage_percentage": round(
                                count / len(resources) * 100, 1
                            ),
                            "research_diversity": len(
                                domain_researches.get(domain, set())
                            ),
                            "frequency_rank": rank + 1,
                        }
                        for rank, (domain, count) in enumerate(top_10_domains)
                    },
                }
            }

    except Exception:
        logger.exception("Error getting link analytics")
        return {
            "link_analytics": {
                "top_domains": [],
                "total_unique_domains": 0,
                "avg_links_per_research": 0,
                "domain_distribution": {},
                "source_type_analysis": {},
                "academic_vs_general": {},
                "total_links": 0,
                "error": "Failed to retrieve link analytics",
            }
        }


def get_strategy_analytics(period="30d", username=None):
    """Get strategy usage analytics for the specified period."""
    try:
        if not username:
            username = flask_session.get("username")

        if not username:
            return {
                "strategy_analytics": {
                    "total_research_with_strategy": 0,
                    "total_research": 0,
                    "most_popular_strategy": None,
                    "strategy_usage": [],
                    "strategy_distribution": {},
                    "available_strategies": get_available_strategies(),
                    "error": "No user session",
                }
            }

        # Calculate date range
        days = get_period_days(period)

        with get_user_db_session(username) as session:
            # Check if we have any ResearchStrategy records
            strategy_count = session.query(ResearchStrategy).count()

            if strategy_count == 0:
                logger.warning("No research strategies found in database")
                return {
                    "strategy_analytics": {
                        "total_research_with_strategy": 0,
                        "total_research": 0,
                        "most_popular_strategy": None,
                        "strategy_usage": [],
                        "strategy_distribution": {},
                        "available_strategies": get_available_strategies(),
                        "message": "Strategy tracking not yet available - run a research to start tracking",
                    }
                }

            # Base query for strategy usage (no JOIN needed since we just want strategy counts)
            query = session.query(
                ResearchStrategy.strategy_name,
                func.count(ResearchStrategy.id).label("usage_count"),
            )

            # Apply time filter if specified
            if days:
                cutoff_date = datetime.now(UTC) - timedelta(days=days)
                query = query.filter(ResearchStrategy.created_at >= cutoff_date)

            # Group by strategy and order by usage
            strategy_results = (
                query.group_by(ResearchStrategy.strategy_name)
                .order_by(func.count(ResearchStrategy.id).desc())
                .all()
            )

            # Get total strategy count for percentage calculation
            total_query = session.query(ResearchStrategy)
            if days:
                total_query = total_query.filter(
                    ResearchStrategy.created_at >= cutoff_date
                )
            total_research = total_query.count()

            # Format strategy data
            strategy_usage = []
            strategy_distribution = {}

            for strategy_name, usage_count in strategy_results:
                percentage = (
                    (usage_count / total_research * 100)
                    if total_research > 0
                    else 0
                )
                strategy_usage.append(
                    {
                        "strategy": strategy_name,
                        "count": usage_count,
                        "percentage": round(percentage, 1),
                    }
                )
                strategy_distribution[strategy_name] = usage_count

            # Find most popular strategy
            most_popular = (
                strategy_usage[0]["strategy"] if strategy_usage else None
            )

            return {
                "strategy_analytics": {
                    "total_research_with_strategy": sum(
                        item["count"] for item in strategy_usage
                    ),
                    "total_research": total_research,
                    "most_popular_strategy": most_popular,
                    "strategy_usage": strategy_usage,
                    "strategy_distribution": strategy_distribution,
                    "available_strategies": get_available_strategies(),
                }
            }

    except Exception:
        logger.exception("Error getting strategy analytics")
        return {
            "strategy_analytics": {
                "total_research_with_strategy": 0,
                "total_research": 0,
                "most_popular_strategy": None,
                "strategy_usage": [],
                "strategy_distribution": {},
                "available_strategies": get_available_strategies(),
                "error": "Failed to retrieve strategy data",
            }
        }


def get_rate_limiting_analytics(period="30d", username=None):
    """Get rate limiting analytics for the specified period."""
    try:
        if not username:
            username = flask_session.get("username")

        if not username:
            return {
                "rate_limiting": {
                    "total_attempts": 0,
                    "successful_attempts": 0,
                    "failed_attempts": 0,
                    "success_rate": 0,
                    "rate_limit_events": 0,
                    "avg_wait_time": 0,
                    "avg_successful_wait": 0,
                    "tracked_engines": 0,
                    "engine_stats": [],
                    "total_engines_tracked": 0,
                    "healthy_engines": 0,
                    "degraded_engines": 0,
                    "poor_engines": 0,
                    "error": "No user session",
                }
            }

        # Calculate date range for timestamp filtering
        import time

        if period == "7d":
            cutoff_time = time.time() - (7 * 24 * 3600)
        elif period == "30d":
            cutoff_time = time.time() - (30 * 24 * 3600)
        elif period == "3m":
            cutoff_time = time.time() - (90 * 24 * 3600)
        elif period == "1y":
            cutoff_time = time.time() - (365 * 24 * 3600)
        else:  # all
            cutoff_time = 0

        with get_user_db_session(username) as session:
            # Rate-limit analytics are derived from RateLimitEstimate, the
            # learned per-engine wait-time model that production code
            # actually persists. The raw per-attempt table
            # (RateLimitAttempt) is intentionally NOT written — attempt
            # persistence was disabled (commit fef359be9) to avoid DB
            # locking under parallel search — so the previous code, which
            # read RateLimitAttempt, always returned an empty panel.
            #
            # Limitations of deriving from estimates (documented so the
            # numbers aren't mistaken for raw-attempt counts):
            #   - total_attempts is each engine's recent rolling window
            #     (capped at rate_limiting.memory_window, default 100), not
            #     a lifetime count.
            #   - rate_limit_events (RateLimitError-specific failures) and a
            #     true per-attempt average wait cannot be reconstructed and
            #     are reported as 0 / the learned base wait respectively.
            estimates_query = session.query(RateLimitEstimate)

            # Recency filter uses the estimate's last_updated (epoch
            # seconds); there is no per-attempt timestamp history.
            if cutoff_time > 0:
                estimates_query = estimates_query.filter(
                    RateLimitEstimate.last_updated >= cutoff_time
                )

            estimates = estimates_query.all()

            engine_stats = []
            total_attempts = 0
            successful_attempts = 0
            base_wait_sum = 0.0

            for estimate in estimates:
                # success_rate is stored as a 0..1 fraction.
                success_rate_pct = round(estimate.success_rate * 100, 1)
                engine_attempts = estimate.total_attempts or 0
                engine_success = round(engine_attempts * estimate.success_rate)

                total_attempts += engine_attempts
                successful_attempts += engine_success
                base_wait_sum += estimate.base_wait_seconds

                status = (
                    "healthy"
                    if estimate.success_rate > 0.8
                    else "degraded"
                    if estimate.success_rate > 0.5
                    else "poor"
                )

                engine_stats.append(
                    {
                        "engine": estimate.engine_type,
                        "base_wait": estimate.base_wait_seconds,
                        "base_wait_seconds": round(
                            estimate.base_wait_seconds, 2
                        ),
                        "min_wait_seconds": round(estimate.min_wait_seconds, 2),
                        "max_wait_seconds": round(estimate.max_wait_seconds, 2),
                        "success_rate": success_rate_pct,
                        "total_attempts": engine_attempts,
                        "recent_attempts": engine_attempts,
                        "recent_success_rate": success_rate_pct,
                        "attempts": engine_attempts,
                        "status": status,
                        # ISO format already includes timezone
                        "last_updated": datetime.fromtimestamp(
                            estimate.last_updated, UTC
                        ).isoformat(),
                    }
                )

            tracked_engines = len(engine_stats)
            failed_attempts = total_attempts - successful_attempts
            # base_wait_seconds is the learned optimal (median of recent
            # successful waits), so it represents both the typical wait and
            # the typical successful wait; a true per-attempt average needs
            # the raw attempts table.
            avg_wait_time = (
                base_wait_sum / tracked_engines if tracked_engines else 0
            )
            avg_successful_wait = avg_wait_time
            # Not derivable from estimates (needs the raw attempts table).
            rate_limit_events = 0

            logger.info(
                f"Rate limiting analytics from estimates: "
                f"tracked_engines={tracked_engines}, "
                f"total_attempts(recent)={total_attempts}"
            )

            result = {
                "rate_limiting": {
                    "total_attempts": total_attempts,
                    "successful_attempts": successful_attempts,
                    "failed_attempts": failed_attempts,
                    "success_rate": (successful_attempts / total_attempts * 100)
                    if total_attempts > 0
                    else 0,
                    "rate_limit_events": rate_limit_events,
                    "avg_wait_time": round(float(avg_wait_time), 2),
                    "avg_successful_wait": round(float(avg_successful_wait), 2),
                    "tracked_engines": tracked_engines,
                    "engine_stats": engine_stats,
                    "total_engines_tracked": tracked_engines,
                    "healthy_engines": len(
                        [s for s in engine_stats if s["status"] == "healthy"]
                    ),
                    "degraded_engines": len(
                        [s for s in engine_stats if s["status"] == "degraded"]
                    ),
                    "poor_engines": len(
                        [s for s in engine_stats if s["status"] == "poor"]
                    ),
                }
            }

            logger.info(
                f"DEBUG: Returning rate_limiting_analytics result: {result}"
            )
            return result

    except Exception:
        logger.exception("Error getting rate limiting analytics")
        return {
            "rate_limiting": {
                "total_attempts": 0,
                "successful_attempts": 0,
                "failed_attempts": 0,
                "success_rate": 0,
                "rate_limit_events": 0,
                "avg_wait_time": 0,
                "avg_successful_wait": 0,
                "tracked_engines": 0,
                "engine_stats": [],
                "total_engines_tracked": 0,
                "healthy_engines": 0,
                "degraded_engines": 0,
                "poor_engines": 0,
                "error": "An internal error occurred while processing the request.",
            }
        }


@metrics_bp.route("/")
@login_required
def metrics_dashboard():
    """Render the metrics dashboard page."""
    return render_template_with_defaults("pages/metrics.html")


@metrics_bp.route("/context-overflow")
@login_required
def context_overflow_page():
    """Context overflow analytics page."""
    return render_template_with_defaults("pages/context_overflow.html")


@metrics_bp.route("/api/metrics")
@login_required
def api_metrics():
    """Get overall metrics data."""
    logger.debug("api_metrics endpoint called")
    try:
        # Get username from session
        username = flask_session["username"]

        # Get time period and research mode from query parameters
        period = request.args.get("period", "30d")
        research_mode = request.args.get("mode", "all")

        token_counter = TokenCounter()
        search_tracker = get_search_tracker()

        # Get both token and search metrics
        token_metrics = token_counter.get_overall_metrics(
            period=period, research_mode=research_mode
        )
        search_metrics = search_tracker.get_search_metrics(
            period=period,
            research_mode=research_mode,
            username=username,
        )

        # Get user satisfaction rating data
        try:
            with get_user_db_session(username) as session:
                # Build base query with time filter
                ratings_query = session.query(ResearchRating)
                time_condition = get_time_filter_condition(
                    period, ResearchRating.created_at
                )
                if time_condition is not None:
                    ratings_query = ratings_query.filter(time_condition)

                # Get average rating
                avg_rating = ratings_query.with_entities(
                    func.avg(ResearchRating.rating).label("avg_rating")
                ).scalar()

                # Get total rating count
                total_ratings = ratings_query.count()

                user_satisfaction = {
                    "avg_rating": round(avg_rating, 1) if avg_rating else None,
                    "total_ratings": total_ratings,
                }
        except Exception:
            logger.exception("Error getting user satisfaction data")
            user_satisfaction = {"avg_rating": None, "total_ratings": 0}

        # Get strategy analytics
        strategy_data = get_strategy_analytics(period, username)
        logger.debug(f"strategy_data keys: {list(strategy_data.keys())}")

        # Get rate limiting analytics
        rate_limiting_data = get_rate_limiting_analytics(period, username)
        logger.debug(f"rate_limiting_data: {rate_limiting_data}")
        logger.debug(
            f"rate_limiting_data keys: {list(rate_limiting_data.keys())}"
        )

        # Truncation summary surfaced on the main dashboard. Failure sentinel
        # is None (not 0): a real zero means "no truncation", so falling back
        # to 0 on error would silently flip a red signal green.
        context_overflow_data = {
            "truncation_rate": None,
            "avg_tokens_truncated": None,
        }
        try:
            with get_user_db_session(username) as session:
                # Honor the dashboard's research_mode filter the same way the
                # rest of api_metrics() does (token_metrics, search_metrics,
                # etc.). Without this the panel ignores mode toggles.
                summary = get_context_overflow_truncation_summary(
                    session, period, research_mode=research_mode
                )
            context_overflow_data = {
                "truncation_rate": round(summary["truncation_rate"], 1),
                "avg_tokens_truncated": int(summary["avg_tokens_truncated"]),
            }
        except Exception:
            logger.exception(
                "Error getting context overflow summary for /api/metrics"
            )

        # Combine metrics
        combined_metrics = {
            **token_metrics,
            **search_metrics,
            **strategy_data,
            **rate_limiting_data,
            **context_overflow_data,
            "user_satisfaction": user_satisfaction,
        }

        logger.debug(f"combined_metrics keys: {list(combined_metrics.keys())}")
        logger.debug(
            f"combined_metrics['rate_limiting']: {combined_metrics.get('rate_limiting', 'NOT FOUND')}"
        )

        return jsonify(
            {
                "status": "success",
                "metrics": combined_metrics,
                "period": period,
                "research_mode": research_mode,
            }
        )
    except Exception:
        logger.exception("Error getting metrics")
        return (
            jsonify(
                {
                    "status": "error",
                    "message": "An internal error occurred. Please try again later.",
                }
            ),
            500,
        )


@metrics_bp.route("/api/rate-limiting")
@login_required
def api_rate_limiting_metrics():
    """Get detailed rate limiting metrics."""
    # KNOWN-DEFERRED: debug log left in during development. Not harmful
    # (no PII, just marks endpoint entry) but noisy — post-merge cleanup.
    logger.info("DEBUG: api_rate_limiting_metrics endpoint called")
    try:
        username = flask_session["username"]
        period = request.args.get("period", "30d")
        rate_limiting_data = get_rate_limiting_analytics(period, username)

        return jsonify(
            {"status": "success", "data": rate_limiting_data, "period": period}
        )
    except Exception:
        logger.exception("Error getting rate limiting metrics")
        return jsonify(
            {
                "status": "error",
                "message": "Failed to retrieve rate limiting metrics",
            }
        ), 500


@metrics_bp.route("/api/rate-limiting/current")
@login_required
def api_current_rate_limits():
    """Get current rate limit estimates for all engines."""
    try:
        username = flask_session["username"]

        with get_user_db_session(username) as session:
            estimates = (
                session.query(RateLimitEstimate)
                .order_by(RateLimitEstimate.engine_type)
                .all()
            )

            current_limits = []
            for est in estimates:
                current_limits.append(
                    {
                        "engine_type": est.engine_type,
                        "base_wait_seconds": round(est.base_wait_seconds, 2),
                        "min_wait_seconds": round(est.min_wait_seconds, 2),
                        "max_wait_seconds": round(est.max_wait_seconds, 2),
                        "success_rate": round(est.success_rate * 100, 1),
                        "total_attempts": est.total_attempts,
                        "last_updated": datetime.fromtimestamp(
                            est.last_updated, UTC
                        ).isoformat(),
                        "status": "healthy"
                        if est.success_rate > 0.8
                        else "degraded"
                        if est.success_rate > 0.5
                        else "poor",
                    }
                )

        return jsonify(
            {
                "status": "success",
                "current_limits": current_limits,
                "timestamp": datetime.now(UTC).isoformat(),
            }
        )
    except Exception:
        logger.exception("Error getting current rate limits")
        return jsonify(
            {
                "status": "error",
                "message": "Failed to retrieve current rate limits",
            }
        ), 500


@metrics_bp.route("/api/metrics/research/<string:research_id>/links")
@login_required
def api_research_link_metrics(research_id):
    """Get link analytics for a specific research."""
    try:
        username = flask_session["username"]

        with get_user_db_session(username) as session:
            # Get all resources for this specific research
            resources = (
                session.query(ResearchResource)
                .filter(ResearchResource.research_id == research_id)
                .all()
            )

            if not resources:
                return jsonify(
                    {
                        "status": "success",
                        "data": {
                            "total_links": 0,
                            "unique_domains": 0,
                            "domains": [],
                            "category_distribution": {},
                            "domain_categories": {},
                            "resources": [],
                        },
                    }
                )

            # Extract domain information
            domain_counts: dict[str, Any] = {}

            # Generic category counting from LLM classifications
            category_counts: dict[str, Any] = {}

            # First pass: collect all domains
            all_domains = set()
            for resource in resources:
                if resource.url:
                    domain = _extract_domain(resource.url)
                    if domain:
                        all_domains.add(domain)

            # Batch load all domain classifications in one query (fix N+1)
            domain_classifications_map = {}
            if all_domains:
                all_classifications = (
                    session.query(DomainClassification)
                    .filter(DomainClassification.domain.in_(all_domains))
                    .all()
                )
                for classification in all_classifications:
                    domain_classifications_map[classification.domain] = (
                        classification
                    )

            # Second pass: process resources with pre-loaded classifications
            for resource in resources:
                if resource.url:
                    try:
                        domain = _extract_domain(resource.url)
                        if not domain:
                            continue

                        domain_counts[domain] = domain_counts.get(domain, 0) + 1

                        # Count categories from pre-loaded classifications (no N+1)
                        classification = domain_classifications_map.get(domain)
                        if classification:
                            category = classification.category
                            category_counts[category] = (
                                category_counts.get(category, 0) + 1
                            )
                        else:
                            category_counts["Unclassified"] = (
                                category_counts.get("Unclassified", 0) + 1
                            )
                    except (AttributeError, KeyError) as e:
                        logger.debug(f"Error classifying domain {domain}: {e}")

            # Sort domains by count
            sorted_domains = sorted(
                domain_counts.items(), key=lambda x: x[1], reverse=True
            )

            return jsonify(
                {
                    "status": "success",
                    "data": {
                        "total_links": len(resources),
                        "unique_domains": len(domain_counts),
                        "domains": [
                            {
                                "domain": domain,
                                "count": count,
                                "percentage": round(
                                    count / len(resources) * 100, 1
                                ),
                            }
                            for domain, count in sorted_domains[
                                :20
                            ]  # Top 20 domains
                        ],
                        "category_distribution": category_counts,
                        "domain_categories": category_counts,  # Generic categories from LLM
                        "resources": [
                            {
                                "title": r.title or "Untitled",
                                "url": r.url,
                                "preview": r.content_preview[:200]
                                if r.content_preview
                                else None,
                            }
                            for r in resources[:10]  # First 10 resources
                        ],
                    },
                }
            )

    except Exception:
        logger.exception("Error getting research link metrics")
        return jsonify(
            {"status": "error", "message": "Failed to retrieve link metrics"}
        ), 500


@metrics_bp.route("/api/metrics/research/<string:research_id>")
@login_required
def api_research_metrics(research_id):
    """Get metrics for a specific research."""
    try:
        token_counter = TokenCounter()
        metrics = token_counter.get_research_metrics(research_id)
        return jsonify({"status": "success", "metrics": metrics})
    except Exception:
        logger.exception("Error getting research metrics")
        return (
            jsonify(
                {
                    "status": "error",
                    "message": "An internal error occurred. Please try again later.",
                }
            ),
            500,
        )


@metrics_bp.route("/api/metrics/research/<string:research_id>/timeline")
@login_required
def api_research_timeline_metrics(research_id):
    """Get timeline metrics for a specific research."""
    try:
        token_counter = TokenCounter()
        timeline_metrics = token_counter.get_research_timeline_metrics(
            research_id
        )
        return jsonify({"status": "success", "metrics": timeline_metrics})
    except Exception:
        logger.exception("Error getting research timeline metrics")
        return (
            jsonify(
                {
                    "status": "error",
                    "message": "An internal error occurred. Please try again later.",
                }
            ),
            500,
        )


@metrics_bp.route("/api/metrics/research/<string:research_id>/search")
@login_required
def api_research_search_metrics(research_id):
    """Get search metrics for a specific research."""
    try:
        username = flask_session["username"]
        search_tracker = get_search_tracker()
        search_metrics = search_tracker.get_research_search_metrics(
            research_id, username=username
        )
        return jsonify({"status": "success", "metrics": search_metrics})
    except Exception:
        logger.exception("Error getting research search metrics")
        return (
            jsonify(
                {
                    "status": "error",
                    "message": "An internal error occurred. Please try again later.",
                }
            ),
            500,
        )


@metrics_bp.route("/api/metrics/enhanced")
@login_required
def api_enhanced_metrics():
    """Get enhanced Phase 1 tracking metrics."""
    try:
        # Get time period and research mode from query parameters
        period = request.args.get("period", "30d")
        research_mode = request.args.get("mode", "all")
        username = flask_session["username"]

        token_counter = TokenCounter()
        search_tracker = get_search_tracker()

        enhanced_metrics = token_counter.get_enhanced_metrics(
            period=period, research_mode=research_mode
        )

        # Add search time series data for the chart
        search_time_series = search_tracker.get_search_time_series(
            period=period,
            research_mode=research_mode,
            username=username,
        )
        enhanced_metrics["search_time_series"] = search_time_series

        # Add rating analytics
        rating_analytics = get_rating_analytics(period, research_mode, username)
        enhanced_metrics.update(rating_analytics)

        return jsonify(
            {
                "status": "success",
                "metrics": enhanced_metrics,
                "period": period,
                "research_mode": research_mode,
            }
        )
    except Exception:
        logger.exception("Error getting enhanced metrics")
        return (
            jsonify(
                {
                    "status": "error",
                    "message": "An internal error occurred. Please try again later.",
                }
            ),
            500,
        )


@metrics_bp.route("/api/ratings/<string:research_id>", methods=["GET"])
@login_required
def api_get_research_rating(research_id):
    """Get rating for a specific research session."""
    try:
        username = flask_session["username"]

        with get_user_db_session(username) as session:
            rating = (
                session.query(ResearchRating)
                .filter_by(research_id=research_id)
                .first()
            )

            if rating:
                return jsonify(
                    {
                        "status": "success",
                        "rating": rating.rating,
                        "created_at": rating.created_at.isoformat(),
                        "updated_at": rating.updated_at.isoformat(),
                    }
                )
            return jsonify({"status": "success", "rating": None})

    except Exception:
        logger.exception("Error getting research rating")
        return (
            jsonify(
                {
                    "status": "error",
                    "message": "An internal error occurred. Please try again later.",
                }
            ),
            500,
        )


@metrics_bp.route("/api/ratings/<string:research_id>", methods=["POST"])
@login_required
@require_json_body(error_format="status")
def api_save_research_rating(research_id):
    """Save or update rating for a specific research session."""
    try:
        username = flask_session["username"]

        data = request.get_json()
        rating_value = data.get("rating")

        if (
            not isinstance(rating_value, int)
            or isinstance(rating_value, bool)
            or rating_value < 1
            or rating_value > 5
        ):
            return (
                jsonify(
                    {
                        "status": "error",
                        "message": "Rating must be an integer between 1 and 5",
                    }
                ),
                400,
            )

        # Optional sub-dimension fields (1-5)
        sub_dimensions = {}
        for field in ("accuracy", "completeness", "relevance", "readability"):
            val = data.get(field)
            if val is not None:
                if (
                    not isinstance(val, int)
                    or isinstance(val, bool)
                    or val < 1
                    or val > 5
                ):
                    return (
                        jsonify(
                            {
                                "status": "error",
                                "message": f"{field} must be an integer between 1 and 5",
                            }
                        ),
                        400,
                    )
                sub_dimensions[field] = val

        feedback_text = data.get("feedback")
        if feedback_text is not None:
            if not isinstance(feedback_text, str):
                return (
                    jsonify(
                        {
                            "status": "error",
                            "message": "feedback must be a string",
                        }
                    ),
                    400,
                )
            if len(feedback_text) > 10000:
                return (
                    jsonify(
                        {
                            "status": "error",
                            "message": "feedback must be 10000 characters or fewer",
                        }
                    ),
                    400,
                )

        with get_user_db_session(username) as session:
            # Check if rating already exists
            existing_rating = (
                session.query(ResearchRating)
                .filter_by(research_id=research_id)
                .first()
            )

            if existing_rating:
                # Update existing rating
                existing_rating.rating = rating_value
                existing_rating.updated_at = func.now()
                for field, val in sub_dimensions.items():
                    setattr(existing_rating, field, val)
                if feedback_text is not None:
                    existing_rating.feedback = feedback_text
            else:
                # Create new rating
                new_rating = ResearchRating(
                    research_id=research_id,
                    rating=rating_value,
                    feedback=feedback_text,
                    **sub_dimensions,
                )
                session.add(new_rating)

            session.commit()

            return jsonify(
                {
                    "status": "success",
                    "message": "Rating saved successfully",
                    "rating": rating_value,
                }
            )

    except Exception:
        logger.exception("Error saving research rating")
        return (
            jsonify(
                {
                    "status": "error",
                    "message": "An internal error occurred. Please try again later.",
                }
            ),
            500,
        )


@metrics_bp.route("/star-reviews")
@login_required
def star_reviews():
    """Display star reviews metrics page."""
    return render_template_with_defaults("pages/star_reviews.html")


@metrics_bp.route("/costs")
@login_required
def cost_analytics():
    """Display cost analytics page."""
    return render_template_with_defaults("pages/cost_analytics.html")


@metrics_bp.route("/api/star-reviews")
@login_required
def api_star_reviews():
    """Get star reviews analytics data."""
    try:
        username = flask_session["username"]

        period = request.args.get("period", "30d")

        with get_user_db_session(username) as session:
            # Build base query with time filter
            base_query = session.query(ResearchRating)
            time_condition = get_time_filter_condition(
                period, ResearchRating.created_at
            )
            if time_condition is not None:
                base_query = base_query.filter(time_condition)

            # Overall rating statistics
            overall_stats = session.query(
                func.avg(ResearchRating.rating).label("avg_rating"),
                func.count(ResearchRating.rating).label("total_ratings"),
                func.sum(case((ResearchRating.rating == 5, 1), else_=0)).label(
                    "five_star"
                ),
                func.sum(case((ResearchRating.rating == 4, 1), else_=0)).label(
                    "four_star"
                ),
                func.sum(case((ResearchRating.rating == 3, 1), else_=0)).label(
                    "three_star"
                ),
                func.sum(case((ResearchRating.rating == 2, 1), else_=0)).label(
                    "two_star"
                ),
                func.sum(case((ResearchRating.rating == 1, 1), else_=0)).label(
                    "one_star"
                ),
            )

            if time_condition is not None:
                overall_stats = overall_stats.filter(time_condition)

            overall_stats = overall_stats.first()

            # Ratings by LLM model. A research has many token_usage rows (one per
            # LLM call), so joining TokenUsage directly fans out and multiplies the
            # counts/averages. Collapse to distinct (research_id, model) pairs first
            # so each rating is counted once per model it actually used.
            # Normalize the "missing model" sentinels — empty/whitespace and the
            # lowercase "unknown" fallback the token counter writes — to a single
            # "Unknown" *inside* the subquery, so DISTINCT dedups on the normalized
            # value. (Normalizing only at group_by time lets "" and "unknown"
            # survive DISTINCT as separate rows and double-count one rating into the
            # merged bucket.) Real model names keep their original casing via else_.
            normalized_model = case(
                (
                    func.lower(func.trim(TokenUsage.model_name)).in_(
                        ["", "unknown"]
                    ),
                    "Unknown",
                ),
                else_=TokenUsage.model_name,
            )
            research_models = (
                session.query(
                    TokenUsage.research_id.label("research_id"),
                    normalized_model.label("model_name"),
                )
                .distinct()
                .subquery()
            )
            # A rating with no token rows has no subquery match -> NULL via the
            # outerjoin -> "Unknown".
            model_label = func.coalesce(research_models.c.model_name, "Unknown")

            llm_ratings_query = (
                session.query(
                    model_label.label("model"),
                    func.avg(ResearchRating.rating).label("avg_rating"),
                    func.count(ResearchRating.rating).label("rating_count"),
                    func.sum(
                        case((ResearchRating.rating >= 4, 1), else_=0)
                    ).label("positive_ratings"),
                    func.sum(
                        case((ResearchRating.rating == 3, 1), else_=0)
                    ).label("neutral_ratings"),
                    func.sum(
                        case((ResearchRating.rating <= 2, 1), else_=0)
                    ).label("negative_ratings"),
                )
                .select_from(ResearchRating)
                .outerjoin(
                    research_models,
                    ResearchRating.research_id == research_models.c.research_id,
                )
            )

            if time_condition is not None:
                llm_ratings_query = llm_ratings_query.filter(time_condition)

            llm_ratings = (
                llm_ratings_query.group_by(model_label)
                .order_by(func.avg(ResearchRating.rating).desc())
                .all()
            )

            # Ratings by search engine. Same one-to-many fan-out concern as the LLM
            # query — collapse to distinct (research_id, search_engine) pairs first.
            # search_engine_selected is NULL on non-search LLM-call rows (and is
            # occasionally an empty/whitespace string); exclude both so a research
            # that used a real engine isn't ALSO attributed to the "Unknown" bucket.
            # A research with no recorded engine still falls through the outerjoin to
            # "Unknown" exactly once.
            research_engines = (
                session.query(
                    TokenUsage.research_id.label("research_id"),
                    TokenUsage.search_engine_selected.label("search_engine"),
                )
                .filter(
                    func.trim(
                        func.coalesce(TokenUsage.search_engine_selected, "")
                    )
                    != ""
                )
                .distinct()
                .subquery()
            )
            engine_label = func.coalesce(
                research_engines.c.search_engine, "Unknown"
            )

            search_engine_ratings_query = (
                session.query(
                    engine_label.label("search_engine"),
                    func.avg(ResearchRating.rating).label("avg_rating"),
                    func.count(ResearchRating.rating).label("rating_count"),
                    func.sum(
                        case((ResearchRating.rating >= 4, 1), else_=0)
                    ).label("positive_ratings"),
                )
                .select_from(ResearchRating)
                .outerjoin(
                    research_engines,
                    ResearchRating.research_id
                    == research_engines.c.research_id,
                )
            )

            if time_condition is not None:
                search_engine_ratings_query = (
                    search_engine_ratings_query.filter(time_condition)
                )

            search_engine_ratings = (
                search_engine_ratings_query.group_by(engine_label)
                .having(func.count(ResearchRating.rating) > 0)
                .order_by(func.avg(ResearchRating.rating).desc())
                .all()
            )

            # Rating trends over time
            rating_trends_query = session.query(
                func.date(ResearchRating.created_at).label("date"),
                func.avg(ResearchRating.rating).label("avg_rating"),
                func.count(ResearchRating.rating).label("daily_count"),
            )

            if time_condition is not None:
                rating_trends_query = rating_trends_query.filter(time_condition)

            rating_trends = (
                rating_trends_query.group_by(
                    func.date(ResearchRating.created_at)
                )
                .order_by("date")
                .all()
            )

            # Ratings by research mode
            mode_ratings_query = session.query(
                func.coalesce(ResearchHistory.mode, "Unknown").label("mode"),
                func.avg(ResearchRating.rating).label("avg_rating"),
                func.count(ResearchRating.rating).label("rating_count"),
                func.sum(case((ResearchRating.rating >= 4, 1), else_=0)).label(
                    "positive_ratings"
                ),
            ).outerjoin(
                ResearchHistory,
                ResearchRating.research_id == ResearchHistory.id,
            )

            if time_condition is not None:
                mode_ratings_query = mode_ratings_query.filter(time_condition)

            mode_ratings = (
                mode_ratings_query.group_by(ResearchHistory.mode)
                .having(func.count(ResearchRating.rating) > 0)
                .order_by(func.avg(ResearchRating.rating).desc())
                .all()
            )

            # Quality dimension averages. Each sub-dimension is independently
            # nullable, so count them separately; dimension_count reflects rows
            # with ANY dimension filled, so the radar isn't hidden when only some
            # dimensions have data.
            dimension_stats = session.query(
                func.avg(ResearchRating.accuracy).label("avg_accuracy"),
                func.avg(ResearchRating.completeness).label("avg_completeness"),
                func.avg(ResearchRating.relevance).label("avg_relevance"),
                func.avg(ResearchRating.readability).label("avg_readability"),
                func.count(ResearchRating.accuracy).label("count_accuracy"),
                func.count(ResearchRating.completeness).label(
                    "count_completeness"
                ),
                func.count(ResearchRating.relevance).label("count_relevance"),
                func.count(ResearchRating.readability).label(
                    "count_readability"
                ),
                func.count(
                    func.coalesce(
                        ResearchRating.accuracy,
                        ResearchRating.completeness,
                        ResearchRating.relevance,
                        ResearchRating.readability,
                    )
                ).label("dimension_count"),
            )

            if time_condition is not None:
                dimension_stats = dimension_stats.filter(time_condition)

            dimension_stats = dimension_stats.first()

            # Recent ratings with research details. Join a one-row-per-research
            # model subquery instead of TokenUsage directly: a research has many
            # token_usage rows, which would fan out and make limit(20) return far
            # fewer than 20 unique ratings (the same rating duplicated per row).
            recent_model_subq = (
                session.query(
                    TokenUsage.research_id.label("research_id"),
                    func.max(TokenUsage.model_name).label("model_name"),
                )
                .group_by(TokenUsage.research_id)
                .subquery()
            )

            recent_ratings_query = (
                session.query(
                    ResearchRating.rating,
                    ResearchRating.created_at,
                    ResearchRating.research_id,
                    ResearchHistory.query,
                    ResearchHistory.mode,
                    recent_model_subq.c.model_name,
                    ResearchRating.feedback,
                )
                .outerjoin(
                    ResearchHistory,
                    ResearchRating.research_id == ResearchHistory.id,
                )
                .outerjoin(
                    recent_model_subq,
                    ResearchRating.research_id
                    == recent_model_subq.c.research_id,
                )
            )

            if time_condition is not None:
                recent_ratings_query = recent_ratings_query.filter(
                    time_condition
                )

            recent_ratings = (
                recent_ratings_query.order_by(ResearchRating.created_at.desc())
                .limit(20)
                .all()
            )

            # Recent feedback entries (non-empty feedback text)
            recent_feedback_query = session.query(
                ResearchRating.feedback,
                ResearchRating.rating,
                ResearchRating.created_at,
                ResearchRating.research_id,
            ).filter(
                ResearchRating.feedback.isnot(None),
                ResearchRating.feedback != "",
            )

            if time_condition is not None:
                recent_feedback_query = recent_feedback_query.filter(
                    time_condition
                )

            recent_feedback = (
                recent_feedback_query.order_by(ResearchRating.created_at.desc())
                .limit(20)
                .all()
            )

            return jsonify(
                {
                    "overall_stats": {
                        "avg_rating": round(overall_stats.avg_rating or 0, 2),
                        "total_ratings": overall_stats.total_ratings or 0,
                        "rating_distribution": {
                            "5": overall_stats.five_star or 0,
                            "4": overall_stats.four_star or 0,
                            "3": overall_stats.three_star or 0,
                            "2": overall_stats.two_star or 0,
                            "1": overall_stats.one_star or 0,
                        },
                    },
                    "llm_ratings": [
                        {
                            "model": rating.model,
                            "avg_rating": round(rating.avg_rating or 0, 2),
                            "rating_count": rating.rating_count or 0,
                            "positive_ratings": rating.positive_ratings or 0,
                            "neutral_ratings": rating.neutral_ratings or 0,
                            "negative_ratings": rating.negative_ratings or 0,
                            "satisfaction_rate": round(
                                (rating.positive_ratings or 0)
                                / max(rating.rating_count or 1, 1)
                                * 100,
                                1,
                            ),
                        }
                        for rating in llm_ratings
                    ],
                    "search_engine_ratings": [
                        {
                            "search_engine": rating.search_engine,
                            "avg_rating": round(rating.avg_rating or 0, 2),
                            "rating_count": rating.rating_count or 0,
                            "positive_ratings": rating.positive_ratings or 0,
                            "satisfaction_rate": round(
                                (rating.positive_ratings or 0)
                                / max(rating.rating_count or 1, 1)
                                * 100,
                                1,
                            ),
                        }
                        for rating in search_engine_ratings
                    ],
                    "rating_trends": [
                        {
                            "date": str(trend.date),
                            "avg_rating": round(trend.avg_rating or 0, 2),
                            "count": trend.daily_count or 0,
                        }
                        for trend in rating_trends
                    ],
                    "recent_ratings": [
                        {
                            "rating": rating.rating,
                            "created_at": rating.created_at.isoformat()
                            if rating.created_at
                            else None,
                            "research_id": rating.research_id,
                            "query": (
                                rating.query
                                if rating.query
                                else f"Research Session #{rating.research_id}"
                            ),
                            "mode": rating.mode
                            if rating.mode
                            else "Standard Research",
                            "llm_model": (
                                rating.model_name
                                if rating.model_name
                                else "LLM Model"
                            ),
                            "feedback": rating.feedback,
                        }
                        for rating in recent_ratings
                    ],
                    "mode_ratings": [
                        {
                            "mode": rating.mode,
                            "avg_rating": round(rating.avg_rating or 0, 2),
                            "rating_count": rating.rating_count or 0,
                            "satisfaction_rate": round(
                                (rating.positive_ratings or 0)
                                / max(rating.rating_count or 1, 1)
                                * 100,
                                1,
                            ),
                        }
                        for rating in mode_ratings
                    ],
                    "quality_dimensions": {
                        "avg_accuracy": round(dimension_stats.avg_accuracy, 2)
                        if dimension_stats.avg_accuracy is not None
                        else None,
                        "avg_completeness": round(
                            dimension_stats.avg_completeness, 2
                        )
                        if dimension_stats.avg_completeness is not None
                        else None,
                        "avg_relevance": round(dimension_stats.avg_relevance, 2)
                        if dimension_stats.avg_relevance is not None
                        else None,
                        "avg_readability": round(
                            dimension_stats.avg_readability, 2
                        )
                        if dimension_stats.avg_readability is not None
                        else None,
                        "dimension_count": dimension_stats.dimension_count or 0,
                        "dimension_counts": {
                            "accuracy": dimension_stats.count_accuracy or 0,
                            "completeness": dimension_stats.count_completeness
                            or 0,
                            "relevance": dimension_stats.count_relevance or 0,
                            "readability": dimension_stats.count_readability
                            or 0,
                        },
                    },
                    "recent_feedback": [
                        {
                            "feedback": rating.feedback,
                            "rating": rating.rating,
                            "created_at": rating.created_at.isoformat()
                            if rating.created_at
                            else None,
                            "research_id": rating.research_id,
                        }
                        for rating in recent_feedback
                    ],
                }
            )

    except Exception:
        logger.exception("Error getting star reviews data")
        return (
            jsonify(
                {"error": "An internal error occurred. Please try again later."}
            ),
            500,
        )


@metrics_bp.route("/api/pricing")
@login_required
def api_pricing():
    """Get current LLM pricing data."""
    try:
        from ...metrics.pricing.pricing_fetcher import PricingFetcher

        # Use static pricing data instead of async
        fetcher = PricingFetcher()
        pricing_data = fetcher.static_pricing

        return jsonify(
            {
                "status": "success",
                "pricing": pricing_data,
                "last_updated": datetime.now(UTC).isoformat(),
                "note": "Pricing data is from static configuration. Real-time APIs not available for most providers.",
            }
        )

    except Exception:
        logger.exception("Error fetching pricing data")
        return jsonify({"error": "Internal Server Error"}), 500


@metrics_bp.route("/api/pricing/<model_name>")
@login_required
def api_model_pricing(model_name):
    """Get pricing for a specific model."""
    try:
        # Optional provider parameter
        provider = request.args.get("provider")

        from ...metrics.pricing.cost_calculator import CostCalculator

        # Use synchronous approach with cached/static pricing
        calculator = CostCalculator()
        pricing = calculator.cache.get_model_pricing(
            model_name
        ) or calculator.calculate_cost_sync(model_name, 1000, 1000).get(
            "pricing_used", {}
        )

        return jsonify(
            {
                "status": "success",
                "model": model_name,
                "provider": provider,
                "pricing": pricing,
                "last_updated": datetime.now(UTC).isoformat(),
            }
        )

    except Exception:
        logger.exception(f"Error getting pricing for model: {model_name}")
        return jsonify({"error": "An internal error occurred"}), 500


@metrics_bp.route("/api/cost-calculation", methods=["POST"])
@login_required
@require_json_body(error_message="No data provided")
def api_cost_calculation():
    """Calculate cost for token usage."""
    try:
        data = request.get_json()
        model_name = data.get("model_name")
        provider = data.get("provider")  # Optional provider parameter
        prompt_tokens = data.get("prompt_tokens", 0)
        completion_tokens = data.get("completion_tokens", 0)

        if not model_name:
            return jsonify({"error": "model_name is required"}), 400

        from ...metrics.pricing.cost_calculator import CostCalculator

        # Use synchronous cost calculation
        calculator = CostCalculator()
        cost_data = calculator.calculate_cost_sync(
            model_name, prompt_tokens, completion_tokens
        )

        return jsonify(
            {
                "status": "success",
                "model_name": model_name,
                "provider": provider,
                "prompt_tokens": prompt_tokens,
                "completion_tokens": completion_tokens,
                "total_tokens": prompt_tokens + completion_tokens,
                **cost_data,
            }
        )

    except Exception:
        logger.exception("Error calculating cost")
        return jsonify({"error": "An internal error occurred"}), 500


@metrics_bp.route("/api/research-costs/<string:research_id>")
@login_required
def api_research_costs(research_id):
    """Get cost analysis for a specific research session."""
    try:
        username = flask_session["username"]

        with get_user_db_session(username) as session:
            # Get token usage records for this research
            usage_records = (
                session.query(TokenUsage)
                .filter(TokenUsage.research_id == research_id)
                .all()
            )

            if not usage_records:
                return jsonify(
                    {
                        "status": "success",
                        "research_id": research_id,
                        "total_cost": 0.0,
                        "message": "No token usage data found for this research session",
                    }
                )

            # Convert to dict format for cost calculation
            usage_data = []
            for record in usage_records:
                usage_data.append(
                    {
                        "model_name": record.model_name,
                        "provider": getattr(
                            record, "provider", None
                        ),  # Handle both old and new records
                        "prompt_tokens": record.prompt_tokens,
                        "completion_tokens": record.completion_tokens,
                        "timestamp": record.timestamp,
                    }
                )

            from ...metrics.pricing.cost_calculator import CostCalculator

            # Use synchronous calculation for research costs
            calculator = CostCalculator()
            costs = []
            for record in usage_data:
                cost_data = calculator.calculate_cost_sync(
                    record["model_name"],
                    record["prompt_tokens"],
                    record["completion_tokens"],
                )
                costs.append({**record, **cost_data})

            total_cost = sum(c["total_cost"] for c in costs)
            total_prompt_tokens = sum(r["prompt_tokens"] for r in usage_data)
            total_completion_tokens = sum(
                r["completion_tokens"] for r in usage_data
            )

            cost_summary = {
                "total_cost": round(total_cost, 6),
                "total_tokens": total_prompt_tokens + total_completion_tokens,
                "prompt_tokens": total_prompt_tokens,
                "completion_tokens": total_completion_tokens,
            }

            return jsonify(
                {
                    "status": "success",
                    "research_id": research_id,
                    **cost_summary,
                }
            )

    except Exception:
        logger.exception(
            f"Error getting research costs for research: {research_id}"
        )
        return jsonify({"error": "An internal error occurred"}), 500


@metrics_bp.route("/api/cost-analytics")
@login_required
def api_cost_analytics():
    """Get cost analytics across all research sessions."""
    try:
        username = flask_session["username"]

        period = request.args.get("period", "30d")

        with get_user_db_session(username) as session:
            # Get token usage for the period
            query = session.query(TokenUsage)
            time_condition = get_time_filter_condition(
                period, TokenUsage.timestamp
            )
            if time_condition is not None:
                query = query.filter(time_condition)

            # First check if we have any records to avoid expensive queries
            record_count = query.count()

            if record_count == 0:
                return jsonify(
                    {
                        "status": "success",
                        "period": period,
                        "overview": {
                            "total_cost": 0.0,
                            "total_tokens": 0,
                            "prompt_tokens": 0,
                            "completion_tokens": 0,
                        },
                        "top_expensive_research": [],
                        "research_count": 0,
                        "message": "No token usage data found for this period",
                    }
                )

            # If we have too many records, limit to recent ones to avoid timeout
            if record_count > 1000:
                logger.warning(
                    f"Large dataset detected ({record_count} records), limiting to recent 1000 for performance"
                )
                usage_records = (
                    query.order_by(TokenUsage.timestamp.desc())
                    .limit(1000)
                    .all()
                )
            else:
                usage_records = query.all()

            # Convert to dict format
            usage_data = []
            for record in usage_records:
                usage_data.append(
                    {
                        "model_name": record.model_name,
                        "provider": getattr(
                            record, "provider", None
                        ),  # Handle both old and new records
                        "prompt_tokens": record.prompt_tokens,
                        "completion_tokens": record.completion_tokens,
                        "research_id": record.research_id,
                        "timestamp": record.timestamp,
                    }
                )

            from ...metrics.pricing.cost_calculator import CostCalculator

            # Use synchronous calculation
            calculator = CostCalculator()

            # Calculate overall costs
            costs = []
            for record in usage_data:
                cost_data = calculator.calculate_cost_sync(
                    record["model_name"],
                    record["prompt_tokens"],
                    record["completion_tokens"],
                )
                costs.append({**record, **cost_data})

            total_cost = sum(c["total_cost"] for c in costs)
            total_prompt_tokens = sum(r["prompt_tokens"] for r in usage_data)
            total_completion_tokens = sum(
                r["completion_tokens"] for r in usage_data
            )

            cost_summary = {
                "total_cost": round(total_cost, 6),
                "total_tokens": total_prompt_tokens + total_completion_tokens,
                "prompt_tokens": total_prompt_tokens,
                "completion_tokens": total_completion_tokens,
            }

            # Group by research_id for per-research costs
            research_costs: dict[str, Any] = {}
            for record in usage_data:
                rid = record["research_id"]
                if rid not in research_costs:
                    research_costs[rid] = []
                research_costs[rid].append(record)

            # Calculate cost per research
            research_summaries = {}
            for rid, records in research_costs.items():
                research_total: float = 0
                for record in records:
                    cost_data = calculator.calculate_cost_sync(
                        record["model_name"],
                        record["prompt_tokens"],
                        record["completion_tokens"],
                    )
                    research_total += cost_data["total_cost"]
                research_summaries[rid] = {
                    "total_cost": round(research_total, 6)
                }

            # Top expensive research sessions
            top_expensive = sorted(
                [
                    (rid, data["total_cost"])
                    for rid, data in research_summaries.items()
                ],
                key=lambda x: x[1],
                reverse=True,
            )[:10]

            return jsonify(
                {
                    "status": "success",
                    "period": period,
                    "overview": cost_summary,
                    "top_expensive_research": [
                        {"research_id": rid, "total_cost": cost}
                        for rid, cost in top_expensive
                    ],
                    "research_count": len(research_summaries),
                }
            )

    except Exception:
        logger.exception("Error getting cost analytics")
        # Return a more graceful error response
        return (
            jsonify(
                {
                    "status": "success",
                    "period": period,
                    "overview": {
                        "total_cost": 0.0,
                        "total_tokens": 0,
                        "prompt_tokens": 0,
                        "completion_tokens": 0,
                    },
                    "top_expensive_research": [],
                    "research_count": 0,
                    "error": "Cost analytics temporarily unavailable",
                }
            ),
            200,
        )  # Return 200 to avoid breaking the UI


@metrics_bp.route("/links")
@login_required
def link_analytics():
    """Display link analytics page."""
    return render_template_with_defaults("pages/link_analytics.html")


@metrics_bp.route("/api/link-analytics")
@login_required
def api_link_analytics():
    """Get link analytics data."""
    try:
        username = flask_session["username"]

        period = request.args.get("period", "30d")

        # Get link analytics data
        link_data = get_link_analytics(period, username)

        return jsonify(
            {
                "status": "success",
                "data": link_data["link_analytics"],
                "period": period,
            }
        )

    except Exception:
        logger.exception("Error getting link analytics")
        return (
            jsonify(
                {
                    "status": "error",
                    "message": "An internal error occurred. Please try again later.",
                }
            ),
            500,
        )


@metrics_bp.route("/api/domain-classifications", methods=["GET"])
@login_required
def api_get_domain_classifications():
    """Get all domain classifications."""
    classifier = None
    try:
        username = flask_session["username"]

        classifier = DomainClassifier(username)
        classifications = classifier.get_all_classifications()

        return jsonify(
            {
                "status": "success",
                "classifications": [c.to_dict() for c in classifications],
                "total": len(classifications),
            }
        )

    except Exception:
        logger.exception("Error getting domain classifications")
        return jsonify(
            {"status": "error", "message": "Failed to retrieve classifications"}
        ), 500
    finally:
        if classifier is not None:
            from ...utilities.resource_utils import safe_close

            safe_close(classifier, "domain classifier")


@metrics_bp.route("/api/domain-classifications/summary", methods=["GET"])
@login_required
def api_get_classifications_summary():
    """Get summary of domain classifications by category."""
    classifier = None
    try:
        username = flask_session["username"]

        classifier = DomainClassifier(username)
        summary = classifier.get_categories_summary()

        return jsonify({"status": "success", "summary": summary})

    except Exception:
        logger.exception("Error getting classifications summary")
        return jsonify(
            {"status": "error", "message": "Failed to retrieve summary"}
        ), 500
    finally:
        if classifier is not None:
            from ...utilities.resource_utils import safe_close

            safe_close(classifier, "domain classifier")


@metrics_bp.route("/api/domain-classifications/classify", methods=["POST"])
@login_required
def api_classify_domains():
    """Trigger classification of a specific domain or batch classification."""
    classifier = None
    try:
        username = flask_session["username"]

        data = request.get_json() or {}
        domain = data.get("domain")
        force_update = data.get("force_update", False)
        batch_mode = data.get("batch", False)

        # Get settings snapshot for LLM configuration
        from ...settings.manager import SettingsManager
        from ...database.session_context import get_user_db_session

        with get_user_db_session(username) as db_session:
            settings_manager = SettingsManager(db_session=db_session)
            settings_snapshot = settings_manager.get_all_settings()

        classifier = DomainClassifier(
            username, settings_snapshot=settings_snapshot
        )

        if domain and not batch_mode:
            # Classify single domain
            logger.info(f"Classifying single domain: {domain}")
            classification = classifier.classify_domain(domain, force_update)
            if classification:
                return jsonify(
                    {
                        "status": "success",
                        "classification": classification.to_dict(),
                    }
                )
            return jsonify(
                {
                    "status": "error",
                    "message": f"Failed to classify domain: {domain}",
                }
            ), 400
        if batch_mode:
            # Batch classification - this should really be a background task
            # For now, we'll just return immediately and let the frontend poll
            logger.info("Starting batch classification of all domains")
            results = classifier.classify_all_domains(force_update)

            return jsonify({"status": "success", "results": results})
        return jsonify(
            {
                "status": "error",
                "message": "Must provide either 'domain' or set 'batch': true",
            }
        ), 400

    except Exception:
        logger.exception("Error classifying domains")
        return jsonify(
            {"status": "error", "message": "Failed to classify domains"}
        ), 500
    finally:
        if classifier is not None:
            from ...utilities.resource_utils import safe_close

            safe_close(classifier, "domain classifier")


@metrics_bp.route("/api/domain-classifications/progress", methods=["GET"])
@login_required
def api_classification_progress():
    """Get progress of domain classification task."""
    try:
        username = flask_session["username"]

        # Get counts of classified vs unclassified domains
        with get_user_db_session(username) as session:
            # Count total unique domains
            resources = session.query(ResearchResource.url).distinct().all()
            domains = set()

            for (url,) in resources:
                if url:
                    domain = _extract_domain(url)
                    if domain:
                        domains.add(domain)

            all_domains = sorted(domains)
            total_domains = len(domains)

            # Count classified domains
            classified_count = session.query(DomainClassification).count()

            return jsonify(
                {
                    "status": "success",
                    "progress": {
                        "total_domains": total_domains,
                        "classified": classified_count,
                        "unclassified": total_domains - classified_count,
                        "percentage": round(
                            (classified_count / total_domains * 100)
                            if total_domains > 0
                            else 0,
                            1,
                        ),
                        "all_domains": all_domains,  # Return all domains for classification
                    },
                }
            )

    except Exception:
        logger.exception("Error getting classification progress")
        return jsonify(
            {"status": "error", "message": "Failed to retrieve progress"}
        ), 500


# ---------------------------------------------------------------------------
# Journal Quality Dashboard
# ---------------------------------------------------------------------------


@metrics_bp.route("/journals")
@login_required
def journal_quality():
    """Display journal quality dashboard."""
    return render_template_with_defaults("pages/journal_quality.html")


@metrics_bp.route("/api/journal-data/status")
@login_required
def api_journal_data_status():
    """Get status of downloadable journal data files."""
    try:
        from ...journal_quality.downloader import (
            get_journal_data_status,
        )

        return jsonify(get_journal_data_status())
    except Exception:
        logger.exception("Error checking journal data status")
        return jsonify({"error": "Failed to check status"}), 500


@metrics_bp.route("/api/journal-data/download", methods=["POST"])
@login_required
@journal_data_limit
def api_journal_data_download():
    """Trigger download/update of journal data files.

    Rate-limited to 2 per hour per authenticated user: the download streams
    several hundred MB and rebuilds the on-disk reference DB, so unbounded
    invocation is a DoS vector.
    """
    try:
        from ...journal_quality.downloader import (
            download_journal_data,
            get_download_state,
        )
        from ...journal_quality.data_sources import ALL_SOURCES

        # Egress policy: this endpoint streams several hundred MB over public
        # HTTP. Under an offline-for-public scope (PRIVATE_ONLY / STRICT) the
        # user has opted out of public egress, so refuse rather than reaching
        # out. A corrupt/unknown scope also fails closed. The background
        # auto-download path is already gated by
        # JournalReputationFilter._should_skip_journal_fetch_for_scope; this
        # closes the manual button as the matching entry point.
        from ...security.egress.policy import (
            DEFAULT_EGRESS_SCOPE,
            EgressScope,
        )
        from ...utilities.db_utils import get_settings_manager

        username = flask_session.get("username")
        scope_raw = get_settings_manager(username=username).get_setting(
            "policy.egress_scope", DEFAULT_EGRESS_SCOPE
        )
        try:
            scope = EgressScope(str(scope_raw).lower())
        except ValueError:
            scope = None  # corrupt scope -> fail closed below
        if scope is None or scope in (
            EgressScope.PRIVATE_ONLY,
            EgressScope.STRICT,
        ):
            logger.bind(policy_audit=True).warning(
                "journal data download refused by egress policy",
                scope=str(scope_raw),
            )
            return (
                jsonify(
                    {
                        "success": False,
                        "message": (
                            "Journal data download needs public network "
                            "access, which the current egress policy blocks "
                            "(private/offline scope). Change the egress "
                            "scope to download."
                        ),
                    }
                ),
                403,
            )

        force = request.json.get("force", False) if request.is_json else False
        success, internal_message = download_journal_data(force=force)
        if not success:
            logger.warning(f"Journal data download failed: {internal_message}")
            return jsonify({"success": False, "message": "Download failed"})

        # download_journal_data() already calls build_db() + reset_db()
        # internally on its success path (downloader.py:563 → db.py:1209),
        # so the DB is live on disk and the cached engine has been
        # invalidated by the time we get here. Do not add a second build
        # here — it would run the full ~30 s rebuild a second time and
        # write to the legacy `journal_reference.db` filename that the
        # downloader just cleaned up.

        # Build the user-facing message locally from structured state
        # (ints + developer-authored source labels). We deliberately do
        # NOT echo `internal_message` from download_journal_data: keeping
        # the response safe-by-construction means a future refactor that
        # lets arbitrary strings (exception info, user input, PII) slip
        # into the downloader's message cannot reach the client.
        counts = get_download_state().get("counts")
        if counts is not None:
            parts = [
                f"{int(counts.get(src.key) or 0)} {src.count_label}"
                for src in ALL_SOURCES
            ]
            user_message = (
                f"Fetched {' + '.join(parts)}. Database rebuilt successfully."
            )
        else:
            # `counts` is None when download_journal_data took its
            # early-return "already up to date" branch (no fetch ran).
            user_message = "Journal data is already up to date."
        return jsonify({"success": True, "message": user_message})
    except Exception:
        logger.exception("Error downloading journal data")
        return jsonify({"success": False, "message": "Download failed"}), 500


#: Allowlist of ``score_source`` values accepted by ``/api/journals``.
#: Matches the writer side: ``openalex`` / ``doaj`` for reference-DB
#: hits, ``llm`` for Tier 4 cache rows. Empty string means "no filter"
#: and is handled by the caller before validation.
_ALLOWED_SCORE_SOURCES = frozenset({"openalex", "doaj", "llm"})

#: Upper bound on the echoed ``page`` parameter. Prevents a crafted
#: ``?page=10**9`` from issuing an OFFSET scan before the post-query
#: clamp can take effect — reject at input validation instead.
_MAX_PAGE = 10_000


@metrics_bp.route("/api/journals")
@login_required
@journals_read_limit
def api_journal_quality():
    """Get journal quality data with server-side pagination and filtering.

    Reads from the bundled read-only reference database (~217K journals)
    rather than the per-user DB, so the dashboard is always populated.

    Query params:
        page (int): 1-indexed page number (default 1, max 10000)
        per_page (int): rows per page, max 200 (default 50)
        search (str): name substring filter
        tier (str): elite/strong/moderate/low/predatory
        score_source (str): openalex/doaj/llm (allowlisted)
        sort (str): column to sort by (default quality)
        order (str): asc or desc (default desc)
    """
    try:
        from ...journal_quality.db import get_journal_reference_db

        ref = get_journal_reference_db()
        if not ref.available:
            return jsonify(
                {
                    "status": "error",
                    "message": "Journal reference database not available.",
                }
            ), 503

        try:
            page = max(1, int(request.args.get("page", 1)))
            per_page = min(max(1, int(request.args.get("per_page", 50))), 200)
        except (TypeError, ValueError):
            return jsonify(
                {
                    "status": "error",
                    "message": "Invalid pagination parameters",
                }
            ), 400
        if page > _MAX_PAGE:
            return jsonify(
                {
                    "status": "error",
                    "message": (
                        f"page exceeds maximum ({_MAX_PAGE}); narrow the "
                        "filter or increase per_page"
                    ),
                }
            ), 400
        search = request.args.get("search", "")
        tier = request.args.get("tier", "")
        score_source = request.args.get("score_source", "")
        if score_source and score_source not in _ALLOWED_SCORE_SOURCES:
            return jsonify(
                {
                    "status": "error",
                    "message": (
                        f"Invalid score_source; must be one of "
                        f"{sorted(_ALLOWED_SCORE_SOURCES)}"
                    ),
                }
            ), 400
        sort = request.args.get("sort", "quality")
        order = request.args.get("order", "desc")

        journals, total = ref.get_journals_page(
            page=page,
            per_page=per_page,
            search=search,
            tier=tier,
            score_source=score_source,
            sort=sort,
            order=order,
        )

        # Clamp the echoed page so the UI never displays out-of-range
        # numbers on crafted input (e.g. ?page=10**9). SQLite's OFFSET on
        # an indexed ORDER BY caps work at ~total rows regardless of the
        # requested offset, so no DB-level clamp is needed.
        total_pages = -(-total // per_page) if per_page > 0 and total > 0 else 1
        page = min(page, total_pages)

        result = {
            "status": "success",
            "journals": journals,
            "pagination": {
                "page": page,
                "per_page": per_page,
                "total_count": total,
                "total_pages": total_pages,
            },
        }

        # Include summary only when requested (avoids 3 extra SQL queries
        # on every pagination/sort/filter request)
        if request.args.get("include_summary", "false") == "true":
            summary = ref.get_summary()
            summary["quality_distribution"] = ref.get_quality_distribution()
            summary["source_distribution"] = ref.get_source_distribution()
            result["summary"] = summary

        return jsonify(result)

    except Exception:
        logger.exception("Error getting journal quality data")
        return (
            jsonify(
                {
                    "status": "error",
                    "message": "An internal error occurred. Please try again later.",
                }
            ),
            500,
        )


def _ref_db_lookup(ref_db, name: str) -> dict:
    """Look up a journal's display bibliometrics in the reference DB.

    Returns a dict with keys the dashboard template already renders
    (h_index, impact_factor, sjr_quartile, publisher, is_predatory,
    predatory_source, is_in_doaj). Missing fields default
    to None / False so the frontend never sees KeyError. On any ref-DB
    error the function returns an empty dict — the dashboard still shows
    the name + user-DB quality, just without the extras.
    """
    if ref_db is None or not name:
        return {}
    try:
        entry = ref_db.lookup_source(name=name) or {}
    except Exception:  # noqa: silent-exception
        # Reference DB lookups are best-effort enrichment. Any failure
        # degrades to "no bibliometric extras" without crashing the
        # dashboard; detailed errors already surface via the DB layer's
        # own logger.exception calls when they matter.
        return {}
    # lookup_source returns a compact dict; the sjr_quartile lives under
    # "quartile" and predatory/DOAJ fields may be absent entirely.
    return {
        "h_index": entry.get("h_index"),
        "impact_factor": entry.get("impact_factor"),
        "sjr_quartile": entry.get("quartile"),
        "is_predatory": bool(entry.get("is_predatory")),
        "predatory_source": entry.get("predatory_source"),
        "is_in_doaj": bool(entry.get("is_in_doaj")),
        "publisher": entry.get("publisher"),
    }


def _get_ref_db_or_none():
    """Return the JournalQualityDB singleton, or None if unavailable.

    The reference DB is optional — if the user hasn't downloaded the
    snapshot, the dashboard still renders with user-DB data only.
    """
    try:
        from ...journal_quality.db import get_journal_reference_db

        return get_journal_reference_db()
    except Exception:  # noqa: silent-exception
        # Reference DB is optional; if import or initialization fails
        # (unusual: usually it's lazily built on first access), the
        # dashboard falls back to user-DB-only rendering.
        return None


def _resolve_paper_quality(
    llm_quality: int | None, enrichment: dict
) -> tuple[int | None, str | None]:
    """Pick a quality score for a dashboard row.

    Precedence: current LLM verdict from the user's ``journals`` table
    (Tier 4 cache, keyed by NFKC-normalized container_title) → live
    derivation from the bundled reference DB row (Tier 1-3). Always
    live — no frozen per-Paper copy exists, so a re-scored journal
    propagates automatically. Returns (score, source_label) or
    (None, None) if neither path had data.
    """
    if llm_quality is not None:
        return llm_quality, "llm"
    if not enrichment:
        return None, None
    # enrichment comes from _source_to_dashboard_dict — row.quality is
    # the ref-DB's pre-computed score (same formula as the filter uses),
    # so we trust it directly rather than re-running derive_quality_score.
    q = enrichment.get("quality")
    if q is not None:
        return int(q), enrichment.get("score_source") or "openalex"
    return None, None


def _lookup_journal_llm_quality(
    db, container_titles: list[str]
) -> dict[str, int]:
    """Batch-look up current Tier 4 LLM verdicts from the user's
    ``journals`` table.

    Returns a dict mapping ``normalize_name(container_title)`` →
    ``Journal.quality``. Missing journals (never Tier-4-scored) simply
    don't appear in the result — callers fall through to the bundled
    reference DB. One indexed ``name_lower IN (...)`` query.
    """
    from ...journal_quality.scoring import normalize_name

    if not container_titles:
        return {}
    normalized = list({normalize_name(ct) for ct in container_titles if ct})
    if not normalized:
        return {}
    rows = (
        db.query(Journal.name_lower, Journal.quality)
        .filter(Journal.name_lower.in_(normalized))
        .filter(Journal.quality.isnot(None))
        .all()
    )
    return {name_lower: int(q) for name_lower, q in rows}


@metrics_bp.route("/api/journals/user-research")
@login_required
@journals_read_limit
def api_user_research_journals():
    """Get journals from the user's own research sessions.

    Paper-rooted query: groups by ``Paper.container_title`` (the
    cleaned name the filter used to score the journal), counts paper
    appearances. Quality is resolved live — Tier 4 via a batch lookup
    against the user's ``journals`` table (keyed by NFKC-normalized
    container_title), Tier 1-3 via the bundled read-only reference DB.
    A re-scored journal propagates to existing research rows
    automatically because no per-Paper score is stored.
    """
    username = flask_session.get("username")
    if not username:
        return jsonify({"status": "error", "message": "Not authenticated"}), 401

    _empty_response = {
        "status": "success",
        "summary": {
            "total_journals": 0,
            "avg_quality": None,
            "total_papers": 0,
            "predatory_blocked": 0,
        },
        "quality_distribution": {},
        "journals": [],
    }

    try:
        from sqlalchemy import inspect as sa_inspect

        with get_user_db_session(username) as db:
            inspector = sa_inspect(db.bind)
            if not inspector.has_table("papers"):
                return jsonify(_empty_response)

            # Top-200 most-cited journals in this user's research.
            # Orphan Papers (whose ``PaperAppearance`` rows were
            # cascade-deleted when their research session was deleted)
            # are excluded so the dashboard reflects what the user
            # currently has, not residual rows from deleted sessions.
            # See issue #3544.
            rows = (
                db.query(
                    Paper.container_title,
                    func.count(Paper.id).label("paper_count"),
                    func.min(Paper.year).label("year_min"),
                    func.max(Paper.year).label("year_max"),
                )
                .filter(Paper.container_title.isnot(None))
                .filter(Paper.appearances.any())
                .group_by(Paper.container_title)
                .order_by(func.count(Paper.id).desc())
                .limit(200)
                .all()
            )

            if not rows:
                return jsonify(_empty_response)

            # One batched ref-DB lookup for the whole top-200 slice —
            # hits `sources.name_lower IN (…)` rather than 200 point
            # queries.
            ref_db = _get_ref_db_or_none()
            enrich_map = {}
            if ref_db is not None:
                enrich_map = ref_db.lookup_sources_batch(
                    [r.container_title for r in rows]
                )

            from ...journal_quality.scoring import normalize_name

            # Batch-look up current LLM verdicts (Tier 4) from the
            # user's journals table, keyed by NFKC-normalized name.
            # Always live — no frozen Paper copy — so a re-scored
            # journal propagates here without any backfill.
            llm_by_name = _lookup_journal_llm_quality(
                db, [r.container_title for r in rows]
            )

            journals: list[dict] = []
            qualities: list[int] = []
            for r in rows:
                normalized = normalize_name(r.container_title)
                enrichment = enrich_map.get(normalized, {})
                quality, source_label = _resolve_paper_quality(
                    llm_by_name.get(normalized), enrichment
                )
                if quality is not None:
                    qualities.append(quality)
                journals.append(
                    {
                        "name": r.container_title,
                        "quality": quality,
                        "score_source": source_label,
                        "paper_count": r.paper_count,
                        "year_min": r.year_min,
                        "year_max": r.year_max,
                        **{
                            k: v
                            for k, v in enrichment.items()
                            if k not in ("quality", "score_source", "name")
                        },
                    }
                )

            # Aggregate stats computed across the top-200 slice for the
            # dashboard summary — matches how the table renders.
            total_journals = len(journals)
            total_papers = sum(r.paper_count for r in rows)
            avg_quality = (
                round(sum(qualities) / len(qualities), 1) if qualities else None
            )
            quality_distribution: dict[str, int] = {}
            for q in qualities:
                quality_distribution[str(q)] = (
                    quality_distribution.get(str(q), 0) + 1
                )

            # Predatory count uses the full set of distinct
            # container_titles across the user's research, not just the
            # top-200 display slice. One batched query.
            #
            # KNOWN-DEFERRED: unbounded SELECT DISTINCT. Acceptable today
            # because typical users have <5K distinct titles even after
            # years of use, and count_predatory_by_names documents
            # support up to ~100K params. Adding .limit(N) was considered
            # and rejected — it would SILENTLY UNDERCOUNT predatory
            # journals, which violates the no-fallbacks rule. Proper fix
            # (cross-DB correlated subquery or TTL cache) is tracked as
            # a post-merge follow-up. Threshold for visible impact:
            # ~50K papers.
            predatory_blocked = 0
            if ref_db is not None:
                # Same orphan-exclusion as the top-200 query above —
                # otherwise predatory_blocked stays inflated by titles
                # whose only Papers belong to deleted research sessions.
                all_names = [
                    name
                    for (name,) in db.query(Paper.container_title)
                    .filter(Paper.container_title.isnot(None))
                    .filter(Paper.appearances.any())
                    .distinct()
                    .all()
                ]
                predatory_blocked = ref_db.count_predatory_by_names(all_names)

        return jsonify(
            {
                "status": "success",
                "summary": {
                    "total_journals": total_journals,
                    "avg_quality": avg_quality,
                    "total_papers": total_papers,
                    "predatory_blocked": predatory_blocked,
                },
                "quality_distribution": quality_distribution,
                "journals": journals,
            }
        )
    except Exception:
        logger.exception("Error getting user research journals")
        return (
            jsonify(
                {
                    "status": "error",
                    "message": "Failed to load your research data.",
                }
            ),
            500,
        )


@metrics_bp.route("/api/journals/research/<research_id>")
@login_required
@journals_read_limit
def api_research_journals(research_id):
    """Get journals encountered in a single research session.

    Filters the per-user papers table by joining through
    Paper → PaperAppearance → ResearchResource and matching ``research_id``.
    Quality is resolved live (journals.quality + bundled reference DB)
    so results always reflect the current verdict, not a stale snapshot.
    Mirrors the response shape of /api/journals/user-research so the
    dashboard can reuse its rendering code.
    """
    username = flask_session.get("username")
    if not username:
        return jsonify({"status": "error", "message": "Not authenticated"}), 401

    _empty_response = {
        "status": "success",
        "summary": {
            "total_journals": 0,
            "avg_quality": None,
            "total_papers": 0,
            "predatory_blocked": 0,
        },
        "quality_distribution": {},
        "journals": [],
    }

    try:
        from sqlalchemy import inspect as sa_inspect

        with get_user_db_session(username) as db:
            inspector = sa_inspect(db.bind)
            if not inspector.has_table("papers") or not inspector.has_table(
                "paper_appearances"
            ):
                return jsonify(_empty_response)

            # Verify the research_id belongs to this user before exposing
            # any data — research_history is in the same per-user DB so
            # the existence check doubles as an ownership check.
            from ...database.models.research import ResearchHistory

            research = (
                db.query(ResearchHistory.id)
                .filter(ResearchHistory.id == research_id)
                .first()
            )
            if research is None:
                return (
                    jsonify(
                        {"status": "error", "message": "Research not found"}
                    ),
                    404,
                )

            # Aggregate container_title → paper_count for this research.
            # Join chain: Paper → PaperAppearance → ResearchResource.
            rows = (
                db.query(
                    Paper.container_title,
                    func.count(Paper.id).label("paper_count"),
                    func.min(Paper.year).label("year_min"),
                    func.max(Paper.year).label("year_max"),
                )
                .join(
                    PaperAppearance,
                    PaperAppearance.paper_id == Paper.id,
                )
                .join(
                    ResearchResource,
                    ResearchResource.id == PaperAppearance.resource_id,
                )
                .filter(
                    ResearchResource.research_id == research_id,
                    Paper.container_title.isnot(None),
                )
                .group_by(Paper.container_title)
                .order_by(func.count(Paper.id).desc())
                .all()
            )

            if not rows:
                return jsonify(_empty_response)

            ref_db = _get_ref_db_or_none()
            enrich_map = {}
            if ref_db is not None:
                enrich_map = ref_db.lookup_sources_batch(
                    [r.container_title for r in rows]
                )

            from ...journal_quality.scoring import normalize_name

            # Batch-look up current LLM verdicts (Tier 4) — see
            # _lookup_journal_llm_quality for rationale. Same live
            # resolution as the cross-research rollup above.
            llm_by_name = _lookup_journal_llm_quality(
                db, [r.container_title for r in rows]
            )

            journals: list[dict] = []
            qualities: list[int] = []
            predatory_blocked = 0
            for r in rows:
                normalized = normalize_name(r.container_title)
                enrichment = enrich_map.get(normalized, {})
                if enrichment.get("is_predatory"):
                    predatory_blocked += 1
                quality, source_label = _resolve_paper_quality(
                    llm_by_name.get(normalized), enrichment
                )
                if quality is not None:
                    qualities.append(quality)
                journals.append(
                    {
                        "name": r.container_title,
                        "quality": quality,
                        "score_source": source_label,
                        "paper_count": r.paper_count,
                        "year_min": r.year_min,
                        "year_max": r.year_max,
                        **{
                            k: v
                            for k, v in enrichment.items()
                            if k not in ("quality", "score_source", "name")
                        },
                    }
                )

            total_papers = sum(r.paper_count for r in rows)
            avg_quality = (
                round(sum(qualities) / len(qualities), 1) if qualities else None
            )
            quality_distribution: dict[str, int] = {}
            for q in qualities:
                quality_distribution[str(q)] = (
                    quality_distribution.get(str(q), 0) + 1
                )

        return jsonify(
            {
                "status": "success",
                "summary": {
                    "total_journals": len(journals),
                    "avg_quality": avg_quality,
                    "total_papers": total_papers,
                    "predatory_blocked": predatory_blocked,
                },
                "quality_distribution": quality_distribution,
                "journals": journals,
            }
        )
    except Exception:
        logger.exception("Error getting per-research journals")
        return (
            jsonify(
                {
                    "status": "error",
                    "message": "Failed to load research journals.",
                }
            ),
            500,
        )
