"""
Flask API endpoints for news system.
Converted from FastAPI to match LDR's Flask architecture.
"""

from functools import wraps
from typing import Any
from flask import Blueprint, request, jsonify
from loguru import logger

from . import api
from .folder_manager import FolderManager
from ..database.models import SubscriptionFolder
from ..web.auth.decorators import login_required
from ..database.session_context import get_user_db_session
from ..settings.env_registry import get_env_setting
from ..utilities.db_utils import get_settings_manager
from ..llm.providers.base import normalize_provider
from ..security.decorators import require_json_body

# Hard ceiling for user-supplied ``limit`` query params on the news
# endpoints in this module. Matches the ``max_value`` of the
# ``news.feed.default_limit`` setting so a direct API caller cannot request a
# larger page than the UI can configure. Shared by the feed and
# subscription-history endpoints here so their caps stay in lockstep.
NEWS_FEED_MAX_LIMIT = 100


def scheduler_control_required(f):
    """Decorator that gates global scheduler control behind a setting.

    The news scheduler is a global singleton — starting, stopping, or
    triggering it affects all users.  This decorator checks the
    ``news.scheduler.allow_api_control`` setting (env var
    ``LDR_NEWS_SCHEDULER_ALLOW_API_CONTROL``, default ``false``) and
    returns 403 when the setting is disabled.

    Must be placed *after* ``@login_required`` in the decorator stack.
    """

    @wraps(f)
    def wrapper(*args, **kwargs):
        if not get_env_setting("news.scheduler.allow_api_control", False):
            from flask import session as flask_session

            username = flask_session.get("username", "unknown")
            remote_addr = request.remote_addr
            logger.warning(
                "Scheduler API control blocked for endpoint {} (user={}, ip={})",
                f.__name__,
                username,
                remote_addr,
            )
            return (
                jsonify(
                    {
                        "error": "Scheduler API control is disabled. "
                        "Contact your administrator to enable it."
                    }
                ),
                403,
            )
        return f(*args, **kwargs)

    return wrapper


def safe_error_message(e: Exception, context: str = "") -> str:
    """
    Return a safe error message that doesn't expose internal details.

    Args:
        e: The exception
        context: Optional context about what was being attempted

    Returns:
        A generic error message safe for external users
    """
    # Log the actual error for debugging
    logger.exception(f"Error in {context}")

    # Return generic messages based on exception type
    if isinstance(e, ValueError):
        return "Invalid input provided"
    if isinstance(e, KeyError):
        return "Required data missing"
    if isinstance(e, TypeError):
        return "Invalid data format"
    # Generic message for production
    return f"An error occurred{f' while {context}' if context else ''}"


def _is_job_owned_by_user(job, username, scheduler):
    """Check if an APScheduler job belongs to a specific user."""
    # Primary: all news scheduler jobs pass username as first arg
    if hasattr(job, "args") and job.args and job.args[0] == username:
        return True
    # Fallback: check the tracked scheduled_jobs set
    if hasattr(scheduler, "user_sessions"):
        session_info = scheduler.user_sessions.get(username, {})
        if job.id in session_info.get("scheduled_jobs", set()):
            return True
    return False


def _call_start_research_internal(request_data: dict) -> dict:
    """Start a research run by invoking the research route handler in-process.

    Both manual subscription runs (``run_subscription_now``) and the overdue
    sweep (``check_overdue_subscriptions``) previously issued a loopback HTTP
    POST to ``/research/api/start`` via ``safe_post``. That endpoint lives on a
    CSRF-protected blueprint (only ``api_v1`` is exempt — see
    ``web/app_factory.py``), and a server-to-server request cannot carry a CSRF
    token, so every loopback failed with HTTP 400 ("The CSRF token is
    missing"). Forwarding the user's session cookie did not help — CSRF is
    checked before authentication. The result: both the "run now" button and
    the overdue endpoint were broken (only the scheduler path, which calls the
    programmatic API directly, still worked).

    Calling the view function directly skips the HTTP layer (and therefore the
    CSRF ``before_request`` hook) entirely, and removes any need to relay the
    session cookie. Must be called from within the authenticated request
    context of the caller; that context's session and DB session are
    propagated into the nested request context so ``start_research`` can resolve
    the user's DB password (it reads ``session["session_id"]`` -> password
    store, or ``g.user_password``) and reuse the open connection.

    Returns the route's JSON body as a dict (with at least a ``status`` key).
    """
    from flask import current_app, g, session
    from ..web.routes.research_routes import start_research
    from ..database.session_context import get_g_db_session

    host_url = request.host_url.rstrip("/")
    # Snapshot the caller's auth context. Copying only session["username"]
    # would make start_research() -> resolve_user_password() fail with
    # "session expired" on encrypted databases, because the DB password is
    # keyed by session["session_id"] in the password store.
    outer_session = dict(session)
    outer_user_password = getattr(g, "user_password", None)
    db_session = get_g_db_session()

    # Pushing a request context for the same app reuses the existing app
    # context, so ``g`` is shared with the caller and ``session`` is fresh.
    # session.update() is therefore required; the g.* assignments below are
    # usually no-ops but kept so this stays correct if a fresh app context is
    # ever pushed (e.g. a different invocation path or a future Flask change).
    with current_app.test_request_context(
        "/research/api/start",
        method="POST",
        json=request_data,
        base_url=host_url,
    ):
        session.update(outer_session)
        if outer_user_password is not None:
            g.user_password = outer_user_password  # gitleaks:allow
        if db_session is not None:
            g.db_session = db_session

        result = start_research()

        # start_research returns either a Response or (Response, status_code).
        # The target path is under /api/, so @login_required returns a JSON 401
        # (never an HTML redirect) and every other return is jsonify(...), so
        # get_json() always yields a dict — keep this route under /api/.
        resp_obj = result[0] if isinstance(result, tuple) else result
        data: dict = resp_obj.get_json()
        return data


# Create Blueprint - no url_prefix here since parent blueprint already has /news
news_api_bp = Blueprint("news_api", __name__, url_prefix="/api")
# NOTE: Routes use session["username"] (not .get()) intentionally.
# @login_required guarantees the key exists; direct access fails fast
# if the decorator is ever removed.

# Components are initialized in api.py


def get_user_id():
    """Get current user ID from session"""
    from ..web.auth.decorators import current_user

    username = current_user()

    if not username:
        # For news, we need authenticated users
        return None

    return username


@news_api_bp.route("/feed", methods=["GET"])
@login_required
def get_news_feed() -> Any:
    """
    Get personalized news feed for user.

    Query params:
        user_id: User identifier (default: anonymous)
        limit: Maximum number of cards to return (default: 20)
        use_cache: Whether to use cached news (default: true)
        strategy: Override default recommendation strategy
        focus: Optional focus area for news
    """
    try:
        # Get current user (login_required ensures we have one)
        user_id = get_user_id()
        logger.info(f"News feed requested by user: {user_id}")

        # Get query parameters
        settings_manager = get_settings_manager()
        default_limit = settings_manager.get_setting("news.feed.default_limit")
        limit = int(request.args.get("limit", default_limit))
        limit = max(1, min(limit, NEWS_FEED_MAX_LIMIT))
        use_cache = request.args.get("use_cache", "true").lower() == "true"
        strategy = request.args.get("strategy")
        focus = request.args.get("focus")
        subscription_id = request.args.get("subscription_id")

        logger.info(
            f"News feed params: limit={limit}, subscription_id={subscription_id}, focus={focus}"
        )

        # Call the direct API function (now synchronous)
        result = api.get_news_feed(
            user_id=user_id,
            limit=limit,
            use_cache=use_cache,
            focus=focus,
            search_strategy=strategy,
            subscription_id=subscription_id,
        )

        # Check for errors in result
        if "error" in result and result.get("news_items") == []:
            # Sanitize error message before returning to client
            safe_msg = safe_error_message(
                Exception(result["error"]), context="get_news_feed"
            )
            return jsonify(
                {"error": safe_msg, "news_items": []}
            ), 400 if "must be between" in result["error"] else 500

        # Debug: Log the result before returning
        logger.info(
            f"API returning {len(result.get('news_items', []))} news items"
        )
        if result.get("news_items"):
            logger.info(
                f"First item ID: {result['news_items'][0].get('id', 'NO_ID')}"
            )

        return jsonify(result)

    except Exception as e:
        return jsonify(
            {
                "error": safe_error_message(e, "getting news feed"),
                "news_items": [],
            }
        ), 500


@news_api_bp.route("/subscribe", methods=["POST"])
@login_required
@require_json_body(error_message="No JSON data provided")
def create_subscription() -> Any:
    """
    Create a new subscription for user.

    JSON body:
        query: Search query or topic
        subscription_type: "search" or "topic" (default: "search")
        refresh_minutes: Refresh interval in minutes (default: from settings)
    """
    try:
        data = request.get_json(force=True)
    except Exception:
        # Handle invalid JSON
        return jsonify({"error": "Invalid JSON data"}), 400

    try:
        # Get current user
        user_id = get_user_id()

        # Extract parameters
        query = data.get("query")
        subscription_type = data.get("subscription_type", "search")
        refresh_minutes = data.get(
            "refresh_minutes"
        )  # Will use default from api.py

        # Extract model configuration (optional)
        model_provider = normalize_provider(data.get("model_provider"))
        model = data.get("model")
        search_strategy = data.get("search_strategy", "news_aggregation")
        custom_endpoint = data.get("custom_endpoint")

        # Extract additional fields
        name = data.get("name")
        folder_id = data.get("folder_id")
        is_active = data.get("is_active", True)
        search_engine = data.get("search_engine")
        search_iterations = data.get("search_iterations")
        questions_per_iteration = data.get("questions_per_iteration")

        # Validate required fields
        if not query:
            return jsonify({"error": "query is required"}), 400

        # Call the direct API function
        result = api.create_subscription(
            user_id=user_id,
            query=query,
            subscription_type=subscription_type,
            refresh_minutes=refresh_minutes,
            model_provider=model_provider,
            model=model,
            search_strategy=search_strategy,
            custom_endpoint=custom_endpoint,
            name=name,
            folder_id=folder_id,
            is_active=is_active,
            search_engine=search_engine,
            search_iterations=search_iterations,
            questions_per_iteration=questions_per_iteration,
        )

        return jsonify(result)

    except ValueError as e:
        return jsonify(
            {"error": safe_error_message(e, "creating subscription")}
        ), 400
    except Exception as e:
        return jsonify(
            {"error": safe_error_message(e, "creating subscription")}
        ), 500


@news_api_bp.route("/vote", methods=["POST"])
@login_required
@require_json_body(error_message="No JSON data provided")
def vote_on_news() -> Any:
    """
    Submit vote on a news item.

    JSON body:
        card_id: ID of the news card
        vote: "up" or "down"
    """
    try:
        data = request.get_json()

        # Get current user
        user_id = get_user_id()

        card_id = data.get("card_id")
        vote = data.get("vote")

        # Validate
        if not all([card_id, vote]):
            return jsonify({"error": "card_id and vote are required"}), 400

        # Call the direct API function
        result = api.submit_feedback(
            card_id=card_id, user_id=user_id, vote=vote
        )

        return jsonify(result)

    except ValueError as e:
        error_msg = str(e)
        if "not found" in error_msg.lower():
            return jsonify({"error": "Resource not found"}), 404
        return jsonify({"error": safe_error_message(e, "submitting vote")}), 400
    except Exception as e:
        return jsonify({"error": safe_error_message(e, "submitting vote")}), 500


@news_api_bp.route("/feedback/batch", methods=["POST"])
@login_required
@require_json_body(error_message="No JSON data provided")
def get_batch_feedback() -> Any:
    """
    Get feedback (votes) for multiple news cards.
    JSON body:
        card_ids: List of card IDs
    """
    try:
        data = request.get_json()
        card_ids = data.get("card_ids", [])
        if not card_ids:
            return jsonify({"votes": {}})

        # Get current user
        user_id = get_user_id()

        # Call the direct API function
        result = api.get_votes_for_cards(card_ids=card_ids, user_id=user_id)

        return jsonify(result)

    except ValueError as e:
        error_msg = str(e)
        if "not found" in error_msg.lower():
            return jsonify({"error": "Resource not found"}), 404
        return jsonify({"error": safe_error_message(e, "getting votes")}), 400
    except Exception as e:
        logger.exception("Error getting batch feedback")
        return jsonify({"error": safe_error_message(e, "getting votes")}), 500


@news_api_bp.route("/feedback/<card_id>", methods=["POST"])
@login_required
@require_json_body(error_message="No JSON data provided")
def submit_feedback(card_id: str) -> Any:
    """
    Submit feedback (vote) for a news card.

    JSON body:
        vote: "up" or "down"
    """
    try:
        data = request.get_json()

        # Get current user
        user_id = get_user_id()
        vote = data.get("vote")

        # Validate
        if not vote:
            return jsonify({"error": "vote is required"}), 400

        # Call the direct API function
        result = api.submit_feedback(
            card_id=card_id, user_id=user_id, vote=vote
        )

        return jsonify(result)

    except ValueError as e:
        error_msg = str(e)
        if "not found" in error_msg.lower():
            return jsonify({"error": "Resource not found"}), 404
        if "must be" in error_msg.lower():
            return jsonify({"error": "Invalid input value"}), 400
        return jsonify(
            {"error": safe_error_message(e, "submitting feedback")}
        ), 400
    except Exception as e:
        return jsonify(
            {"error": safe_error_message(e, "submitting feedback")}
        ), 500


@news_api_bp.route("/research/<card_id>", methods=["POST"])
@login_required
def research_news_item(card_id: str) -> Any:
    """
    Perform deeper research on a news item.

    JSON body:
        depth: "quick", "detailed", or "report" (default: "quick")
    """
    try:
        data = request.get_json() or {}
        depth = data.get("depth", "quick")

        # Call the API function which handles the research
        result = api.research_news_item(card_id, depth)

        return jsonify(result)

    except Exception as e:
        return jsonify(
            {"error": safe_error_message(e, "researching news item")}
        ), 500


@news_api_bp.route("/subscriptions/current", methods=["GET"])
@login_required
def get_current_user_subscriptions() -> Any:
    """Get all subscriptions for current user."""
    try:
        # Get current user
        user_id = get_user_id()

        # Ensure we have a database session for the user
        # This will trigger register_activity
        logger.debug(f"Getting news feed for user {user_id}")

        # Use the API function
        result = api.get_subscriptions(user_id)
        if "error" in result:
            logger.error(
                f"Error getting subscriptions for user {user_id}: {result['error']}"
            )
            return jsonify({"error": "Failed to retrieve subscriptions"}), 500
        return jsonify(result)

    except Exception as e:
        return jsonify(
            {"error": safe_error_message(e, "getting subscriptions")}
        ), 500


@news_api_bp.route("/subscriptions/<subscription_id>", methods=["GET"])
@login_required
def get_subscription(subscription_id: str) -> Any:
    """Get a single subscription by ID."""
    try:
        # Handle null or invalid subscription IDs
        if (
            subscription_id == "null"
            or subscription_id == "undefined"
            or not subscription_id
        ):
            return jsonify({"error": "Invalid subscription ID"}), 400

        # Get the subscription
        subscription = api.get_subscription(subscription_id)

        if not subscription:
            return jsonify({"error": "Subscription not found"}), 404

        return jsonify(subscription)

    except Exception as e:
        return jsonify(
            {"error": safe_error_message(e, "getting subscription")}
        ), 500


@news_api_bp.route("/subscriptions/<subscription_id>", methods=["PUT"])
@login_required
@require_json_body(error_message="No JSON data provided")
def update_subscription(subscription_id: str) -> Any:
    """Update a subscription."""
    try:
        data = request.get_json(force=True)
    except Exception:
        return jsonify({"error": "Invalid JSON data"}), 400

    try:
        # Prepare update data
        update_data = {}

        # Map fields from request to storage format
        field_mapping = {
            "query": "query_or_topic",
            "name": "name",
            "refresh_minutes": "refresh_interval_minutes",
            "is_active": "is_active",
            "folder_id": "folder_id",
            "model_provider": "model_provider",
            "model": "model",
            "search_strategy": "search_strategy",
            "custom_endpoint": "custom_endpoint",
            "search_engine": "search_engine",
            "search_iterations": "search_iterations",
            "questions_per_iteration": "questions_per_iteration",
        }

        for request_field, storage_field in field_mapping.items():
            if request_field in data:
                update_data[storage_field] = data[request_field]

        # Update subscription
        result = api.update_subscription(subscription_id, update_data)

        if "error" in result:
            # Sanitize error message before returning to client
            original_error = result["error"]
            result["error"] = safe_error_message(
                Exception(original_error), "updating subscription"
            )
            if "not found" in original_error.lower():
                return jsonify(result), 404
            return jsonify(result), 400

        return jsonify(result)

    except Exception as e:
        return jsonify(
            {"error": safe_error_message(e, "updating subscription")}
        ), 500


@news_api_bp.route("/subscriptions/<subscription_id>", methods=["DELETE"])
@login_required
def delete_subscription(subscription_id: str) -> Any:
    """Delete a subscription."""
    try:
        # Call the direct API function
        success = api.delete_subscription(subscription_id)

        if success:
            return jsonify(
                {
                    "status": "success",
                    "message": f"Subscription {subscription_id} deleted",
                }
            )
        return jsonify({"error": "Subscription not found"}), 404

    except Exception as e:
        return jsonify(
            {"error": safe_error_message(e, "deleting subscription")}
        ), 500


@news_api_bp.route("/subscriptions/<subscription_id>/run", methods=["POST"])
@login_required
def run_subscription_now(subscription_id: str) -> Any:
    """Manually trigger a subscription to run now."""
    try:
        from flask import session
        from .core.utils import get_local_date_string
        from .subscription_runner import (
            advance_refresh_schedule,
            build_subscription_request_data,
        )
        from ..database.session_context import get_user_db_session
        from ..database.models.news import NewsSubscription
        from ..settings.manager import SettingsManager
        from datetime import datetime, timezone

        username = session["username"]

        # Load the subscription from the user's database. Reading the ORM row
        # directly (rather than the trimmed api.get_subscriptions() dict, which
        # drops model_provider/model/search_strategy/search_engine) ensures the
        # manual run honors the subscription's saved model config, matching the
        # overdue sweep and the scheduler.
        # Read the subscription + build the payload, then release the read
        # transaction before the blocking POST below. The per-user encrypted DB
        # uses deferred isolation, so a SELECT holds a SHARED lock on the
        # SQLCipher file until the transaction ends — and the session is
        # request-cached, so it is NOT closed on `with` exit. We therefore
        # rollback() explicitly to drop the read lock before the (up to 30s)
        # HTTP call, then reopen a short session to advance.
        with get_user_db_session(username) as db:
            sub = (
                db.query(NewsSubscription)
                .filter(NewsSubscription.id == subscription_id)
                .first()
            )
            if not sub:
                return jsonify({"error": "Subscription not found"}), 404

            subscription_pk = sub.id
            # Snapshot next_refresh for the post-POST compare-and-set (below):
            # a fast-failing run's failure handler may reset next_refresh on the
            # worker thread, and the advance must not clobber that.
            prev_next_refresh = sub.next_refresh
            settings_manager = SettingsManager(db)
            current_date = get_local_date_string(settings_manager)

            request_data = build_subscription_request_data(
                query_template=sub.query_or_topic,
                current_date=current_date,
                triggered_by="manual",
                subscription_id=sub.id,
                model_provider=sub.model_provider,
                model=sub.model,
                search_strategy=sub.search_strategy,
                search_engine=sub.search_engine,
                custom_endpoint=sub.custom_endpoint,
                title=sub.name,
            )
            # End the read transaction so the SHARED lock is released before the
            # blocking POST (exiting the `with` does not — the session is
            # request-cached and not closed on exit).
            db.rollback()

        # Start the research in-process. A loopback HTTP POST to
        # /research/api/start cannot pass CSRF validation (see
        # _call_start_research_internal), so call the route handler directly.
        # The read transaction was already released above so the research
        # route can reuse the connection without contending for the lock.
        data = _call_start_research_internal(request_data)

        if data.get("status") in ("success", "queued"):
            # Advance the schedule so a subscription that was also overdue
            # is not immediately re-run by the scheduler while this run is
            # in flight. If the run later fails, the research failure
            # handler resets next_refresh so the scheduler retries it.
            # Compare-and-set: only advance if next_refresh is unchanged
            # since we read it pre-spawn. A fast-failing run can reset it
            # (worker thread) before we get here; in that case leave the
            # reset in place rather than clobbering it and re-hiding the
            # failed subscription for a full interval.
            with get_user_db_session(username) as db:
                sub = (
                    db.query(NewsSubscription)
                    .filter(NewsSubscription.id == subscription_pk)
                    .first()
                )
                if sub and sub.next_refresh == prev_next_refresh:
                    advance_refresh_schedule(sub, datetime.now(timezone.utc))
                    db.commit()
            return jsonify(
                {
                    "status": "success",
                    "message": "Research started",
                    "research_id": data.get("research_id"),
                    "url": f"/progress/{data.get('research_id')}",
                }
            )
        return jsonify(
            {
                "error": data.get(
                    "message",
                    data.get("error", "Failed to start research"),
                )
            }
        ), 500

    except Exception as e:
        return jsonify(
            {"error": safe_error_message(e, "running subscription")}
        ), 500


@news_api_bp.route("/subscriptions/<subscription_id>/history", methods=["GET"])
@login_required
def get_subscription_history(subscription_id: str) -> Any:
    """Get research history for a subscription."""
    try:
        settings_manager = get_settings_manager()
        default_limit = settings_manager.get_setting("news.feed.default_limit")
        limit = int(request.args.get("limit", default_limit))
        limit = max(1, min(limit, NEWS_FEED_MAX_LIMIT))
        result = api.get_subscription_history(subscription_id, limit)
        if "error" in result:
            logger.error(
                f"Error getting subscription history: {result['error']}"
            )
            return jsonify(
                {
                    "error": "Failed to retrieve subscription history",
                    "history": [],
                }
            ), 500
        return jsonify(result)
    except Exception as e:
        return jsonify(
            {"error": safe_error_message(e, "getting subscription history")}
        ), 500


@news_api_bp.route("/preferences", methods=["POST"])
@login_required
@require_json_body(error_message="No JSON data provided")
def save_preferences() -> Any:
    """Save user preferences for news."""
    try:
        data = request.get_json()

        # Get current user
        user_id = get_user_id()
        preferences = data.get("preferences", {})

        # Call the direct API function
        result = api.save_news_preferences(user_id, preferences)

        return jsonify(result)

    except Exception as e:
        return jsonify(
            {"error": safe_error_message(e, "saving preferences")}
        ), 500


@news_api_bp.route("/categories", methods=["GET"])
@login_required
def get_categories() -> Any:
    """Get news category distribution."""
    try:
        # Call the direct API function
        result = api.get_news_categories()

        return jsonify(result)

    except Exception as e:
        return jsonify(
            {"error": safe_error_message(e, "getting categories")}
        ), 500


@news_api_bp.route("/scheduler/status", methods=["GET"])
@login_required
def get_scheduler_status() -> Any:
    """Get activity-based scheduler status."""
    try:
        logger.info("Scheduler status endpoint called")
        from flask import session
        from ..scheduler.background import get_background_job_scheduler

        # Get scheduler instance
        scheduler = get_background_job_scheduler()
        username = session["username"]
        show_all = get_env_setting("news.scheduler.allow_api_control", False)
        logger.info(
            f"Scheduler instance obtained: is_running={scheduler.is_running}"
        )

        # Build status manually to avoid potential deadlock
        if show_all:
            active_users = (
                len(scheduler.user_sessions)
                if hasattr(scheduler, "user_sessions")
                else 0
            )
        else:
            active_users = (
                1
                if hasattr(scheduler, "user_sessions")
                and username in scheduler.user_sessions
                else 0
            )

        status = {
            "scheduler_available": True,  # APScheduler is installed and working
            "is_running": scheduler.is_running,
            "config": scheduler.config.copy()
            if hasattr(scheduler, "config")
            else {},
            "active_users": active_users,
            "total_scheduled_jobs": 0,
        }

        # Count scheduled jobs
        if hasattr(scheduler, "user_sessions"):
            if show_all:
                total_jobs = sum(
                    len(sess.get("scheduled_jobs", set()))
                    for sess in scheduler.user_sessions.values()
                )
            else:
                user_session = scheduler.user_sessions.get(username, {})
                total_jobs = len(user_session.get("scheduled_jobs", set()))
            status["total_scheduled_jobs"] = total_jobs

        # Also count actual APScheduler jobs
        if hasattr(scheduler, "scheduler") and scheduler.scheduler:
            try:
                apscheduler_jobs = scheduler.scheduler.get_jobs()
                if not show_all:
                    apscheduler_jobs = [
                        j
                        for j in apscheduler_jobs
                        if _is_job_owned_by_user(j, username, scheduler)
                    ]
                status["apscheduler_job_count"] = len(apscheduler_jobs)
                status["apscheduler_jobs"] = [
                    {
                        "id": job.id,
                        "name": job.name,
                        "next_run": job.next_run_time.isoformat()
                        if job.next_run_time
                        else None,
                    }
                    for job in apscheduler_jobs[
                        :10
                    ]  # Limit to first 10 for display
                ]
            except Exception:
                logger.exception("Error getting APScheduler jobs")
                status["apscheduler_job_count"] = 0

        logger.info(f"Status built: {list(status.keys())}")

        # Add scheduled_jobs field that JS expects
        status["scheduled_jobs"] = status.get("total_scheduled_jobs", 0)

        logger.info(
            f"Returning status: is_running={status.get('is_running')}, active_users={status.get('active_users')}"
        )
        return jsonify(status)

    except Exception as e:
        return jsonify(
            {"error": safe_error_message(e, "getting scheduler status")}
        ), 500


@news_api_bp.route("/scheduler/start", methods=["POST"])
@login_required
@scheduler_control_required
def start_scheduler() -> Any:
    """Start the subscription scheduler."""
    try:
        from flask import current_app
        from ..scheduler.background import get_background_job_scheduler

        # Get scheduler instance
        scheduler = get_background_job_scheduler()

        if scheduler.is_running:
            return jsonify({"message": "Scheduler is already running"}), 200

        # Start the scheduler
        scheduler.start()

        # Update app reference
        current_app.background_job_scheduler = scheduler  # type: ignore[attr-defined,unused-ignore]

        logger.info("News scheduler started via API")
        return jsonify(
            {
                "status": "success",
                "message": "Scheduler started",
                "active_users": len(scheduler.user_sessions),
            }
        )

    except Exception as e:
        return jsonify(
            {"error": safe_error_message(e, "starting scheduler")}
        ), 500


@news_api_bp.route("/scheduler/stop", methods=["POST"])
@login_required
@scheduler_control_required
def stop_scheduler() -> Any:
    """Stop the subscription scheduler."""
    try:
        from flask import current_app

        if (
            hasattr(current_app, "background_job_scheduler")
            and current_app.background_job_scheduler
        ):
            scheduler = current_app.background_job_scheduler
            if scheduler.is_running:
                scheduler.stop()
                logger.info("News scheduler stopped via API")
                return jsonify(
                    {"status": "success", "message": "Scheduler stopped"}
                )
            return jsonify({"message": "Scheduler is not running"}), 200
        return jsonify({"message": "No scheduler instance found"}), 404

    except Exception as e:
        return jsonify(
            {"error": safe_error_message(e, "stopping scheduler")}
        ), 500


@news_api_bp.route("/scheduler/check-now", methods=["POST"])
@login_required
@scheduler_control_required
def check_subscriptions_now() -> Any:
    """Manually trigger subscription checking."""
    try:
        from flask import current_app

        if (
            not hasattr(current_app, "background_job_scheduler")
            or not current_app.background_job_scheduler
        ):
            return jsonify({"error": "Scheduler not initialized"}), 503

        scheduler = current_app.background_job_scheduler
        if not scheduler.is_running:
            return jsonify({"error": "Scheduler is not running"}), 503

        # Run the check subscriptions task immediately
        scheduler_instance = current_app.background_job_scheduler

        # Get count of due subscriptions
        from ..database.models import NewsSubscription as BaseSubscription
        from datetime import datetime, timedelta, timezone

        with get_user_db_session() as session:
            now = datetime.now(timezone.utc)
            count = (
                session.query(BaseSubscription)
                .filter(BaseSubscription.due_filter(now))
                .count()
            )

        # Trigger the check asynchronously via APScheduler with app context
        username = get_user_id()
        if not username:
            return jsonify({"error": "No authenticated user"}), 401

        scheduler_instance.scheduler.add_job(
            func=scheduler_instance._wrap_job(
                scheduler_instance._check_user_overdue_subscriptions
            ),
            args=[username],
            trigger="date",
            run_date=datetime.now(timezone.utc) + timedelta(seconds=1),
            id=f"manual_check_{username}",
            replace_existing=True,
        )

        return jsonify(
            {
                "status": "success",
                "message": f"Checking {count} due subscriptions",
                "count": count,
            }
        )

    except Exception as e:
        return jsonify(
            {"error": safe_error_message(e, "checking subscriptions")}
        ), 500


@news_api_bp.route("/scheduler/cleanup-now", methods=["POST"])
@login_required
@scheduler_control_required
def trigger_cleanup() -> Any:
    """Manually trigger cleanup job."""
    try:
        from ..scheduler.background import get_background_job_scheduler
        from datetime import datetime, UTC, timedelta

        scheduler = get_background_job_scheduler()

        if not scheduler.is_running:
            return jsonify({"error": "Scheduler is not running"}), 400

        # Schedule cleanup to run in 1 second
        scheduler.scheduler.add_job(
            scheduler._wrap_job(scheduler._run_cleanup_with_tracking),
            "date",
            run_date=datetime.now(UTC) + timedelta(seconds=1),
            id="manual_cleanup_trigger",
            replace_existing=True,
        )

        return jsonify(
            {
                "status": "triggered",
                "message": "Cleanup job will run within seconds",
            }
        )

    except Exception as e:
        return jsonify(
            {"error": safe_error_message(e, "triggering cleanup")}
        ), 500


@news_api_bp.route("/scheduler/users", methods=["GET"])
@login_required
def get_active_users() -> Any:
    """Get summary of active user sessions."""
    try:
        from flask import session
        from ..scheduler.background import get_background_job_scheduler

        scheduler = get_background_job_scheduler()
        username = session["username"]
        users_summary = scheduler.get_user_sessions_summary()

        show_all = get_env_setting("news.scheduler.allow_api_control", False)
        if not show_all:
            users_summary = [
                u for u in users_summary if u.get("user_id") == username
            ]

        return jsonify(
            {"active_users": len(users_summary), "users": users_summary}
        )

    except Exception as e:
        return jsonify(
            {"error": safe_error_message(e, "getting active users")}
        ), 500


@news_api_bp.route("/scheduler/stats", methods=["GET"])
@login_required
def scheduler_stats() -> Any:
    """Get scheduler statistics and state."""
    try:
        from ..scheduler.background import get_background_job_scheduler
        from flask import session

        scheduler = get_background_job_scheduler()
        username = session["username"]

        # Debug info
        debug_info = {
            "current_user": username,
            "scheduler_running": scheduler.is_running,
            "user_sessions": {},
            "apscheduler_jobs": [],
        }

        show_all = get_env_setting("news.scheduler.allow_api_control", False)

        # Get user session info
        if hasattr(scheduler, "user_sessions"):
            for user, session_info in scheduler.user_sessions.items():
                if not show_all and user != username:
                    continue
                debug_info["user_sessions"][user] = {
                    "has_password": bool(
                        scheduler._credential_store.retrieve(user)
                    ),
                    "last_activity": session_info.get(
                        "last_activity"
                    ).isoformat()
                    if session_info.get("last_activity")
                    else None,
                    "scheduled_jobs_count": len(
                        session_info.get("scheduled_jobs", set())
                    ),
                }

        # Get APScheduler jobs
        if hasattr(scheduler, "scheduler") and scheduler.scheduler:
            jobs = scheduler.scheduler.get_jobs()
            if not show_all:
                jobs = [
                    j
                    for j in jobs
                    if _is_job_owned_by_user(j, username, scheduler)
                ]
            debug_info["apscheduler_jobs"] = [
                {
                    "id": job.id,
                    "name": job.name,
                    "next_run": job.next_run_time.isoformat()
                    if job.next_run_time
                    else None,
                    "trigger": str(job.trigger),
                }
                for job in jobs
            ]

        return jsonify(debug_info)

    except Exception as e:
        return jsonify(
            {"error": safe_error_message(e, "getting scheduler stats")}
        ), 500


@news_api_bp.route("/check-overdue", methods=["POST"])
@login_required
def check_overdue_subscriptions():
    """Check and run all overdue subscriptions for the current user."""
    try:
        from flask import session
        from .subscription_runner import (
            advance_refresh_schedule,
            build_subscription_request_data,
        )
        from ..database.session_context import get_user_db_session
        from ..database.models.news import NewsSubscription
        from datetime import datetime, UTC

        username = session["username"]

        # Get overdue subscriptions
        overdue_count = 0
        results = []
        with get_user_db_session(username) as db:
            now = datetime.now(UTC)
            overdue_subs = (
                db.query(NewsSubscription)
                .filter(NewsSubscription.due_filter(now))
                .all()
            )

            logger.info(
                f"Found {len(overdue_subs)} overdue subscriptions for {username}"
            )

            # Get timezone-aware current date using settings
            from .core.utils import get_local_date_string
            from ..settings.manager import SettingsManager

            settings_manager = SettingsManager(db)
            current_date = get_local_date_string(settings_manager)

            for sub in overdue_subs:
                # Capture identity up front as plain strings. This loop shares
                # one DB session across every start_research call, and
                # start_research's error path does not roll back — so a failed
                # run can leave the session in a PendingRollbackError state.
                # Reading sub.* again in an error branch would then raise (the
                # row was expired by an earlier commit), collapsing the whole
                # sweep. Snapshotting here keeps the error branches session-free.
                sub_id = str(sub.id)
                sub_label = sub.name or sub.query_or_topic[:50]
                try:
                    # Run the subscription using the same pattern as run_subscription_now
                    logger.info(
                        f"Running overdue subscription: {sub_label[:30]}"
                    )

                    # Snapshot for the post-run compare-and-set (see below).
                    prev_next_refresh = sub.next_refresh

                    request_data = build_subscription_request_data(
                        query_template=sub.query_or_topic,
                        current_date=current_date,
                        triggered_by="overdue_check",
                        subscription_id=sub.id,
                        model_provider=sub.model_provider,
                        model=sub.model,
                        search_strategy=sub.search_strategy,
                        search_engine=sub.search_engine,
                        custom_endpoint=sub.custom_endpoint,
                        title=sub.name,
                    )

                    # Start the research in-process. A loopback HTTP POST to
                    # /research/api/start cannot pass CSRF validation (see
                    # _call_start_research_internal), so call the route handler
                    # directly — this also removes the session-cookie relay
                    # that the loopback used to attempt authentication.
                    result = _call_start_research_internal(request_data)

                    if result.get("status") in ("success", "queued"):
                        overdue_count += 1

                        # Update subscription's last/next refresh times.
                        # Compare-and-set: re-read and skip the advance if a
                        # fast-failing run already reset next_refresh (worker
                        # thread), so we don't clobber the reset and re-hide it.
                        db.refresh(sub)
                        if sub.next_refresh == prev_next_refresh:
                            advance_refresh_schedule(sub, datetime.now(UTC))
                            db.commit()

                        results.append(
                            {
                                "id": sub_id,
                                "name": sub_label,
                                "research_id": result.get("research_id"),
                            }
                        )
                    else:
                        # start_research failed and may have left the shared
                        # session dirty; reset it so the next subscription runs.
                        db.rollback()
                        results.append(
                            {
                                "id": sub_id,
                                "name": sub_label,
                                # start_research reports failures under
                                # "message"; keep "error" as a fallback for
                                # any other shape.
                                "error": result.get(
                                    "message",
                                    result.get(
                                        "error", "Failed to start research"
                                    ),
                                ),
                            }
                        )
                except Exception as e:
                    # Recover the shared session (a failed start_research commit
                    # can leave it in a PendingRollbackError state) so the
                    # remaining overdue subscriptions in the sweep still run.
                    db.rollback()
                    logger.exception(f"Error running subscription {sub_id}")
                    results.append(
                        {
                            "id": sub_id,
                            "name": sub_label,
                            "error": safe_error_message(
                                e, "running subscription"
                            ),
                        }
                    )

        return jsonify(
            {
                "status": "success",
                "overdue_found": len(overdue_subs),
                "started": overdue_count,
                "results": results,
            }
        )

    except Exception as e:
        return jsonify(
            {"error": safe_error_message(e, "checking overdue subscriptions")}
        ), 500


# Folder and subscription management routes
@news_api_bp.route("/subscription/folders", methods=["GET"])
@login_required
def get_folders():
    """Get all folders for the current user"""
    try:
        user_id = get_user_id()

        with get_user_db_session() as session:
            manager = FolderManager(session)
            folders = manager.get_user_folders(user_id)

            return jsonify([folder.to_dict() for folder in folders])

    except Exception as e:
        return jsonify({"error": safe_error_message(e, "getting folders")}), 500


@news_api_bp.route("/subscription/folders", methods=["POST"])
@login_required
@require_json_body()
def create_folder():
    """Create a new folder"""
    try:
        data = request.json
        if not data.get("name"):
            return jsonify({"error": "Folder name is required"}), 400

        with get_user_db_session() as session:
            manager = FolderManager(session)

            # Check if folder already exists
            existing = (
                session.query(SubscriptionFolder)
                .filter_by(name=data["name"])
                .first()
            )
            if existing:
                return jsonify({"error": "Folder already exists"}), 409

            folder = manager.create_folder(
                name=data["name"],
                description=data.get("description"),
            )

            return jsonify(folder.to_dict()), 201

    except Exception as e:
        return jsonify({"error": safe_error_message(e, "creating folder")}), 500


@news_api_bp.route("/subscription/folders/<folder_id>", methods=["PUT"])
@login_required
@require_json_body()
def update_folder(folder_id):
    """Update a folder"""
    try:
        data = request.json
        with get_user_db_session() as session:
            manager = FolderManager(session)
            folder = manager.update_folder(folder_id, **data)

            if not folder:
                return jsonify({"error": "Folder not found"}), 404

            return jsonify(folder.to_dict())

    except Exception as e:
        return jsonify({"error": safe_error_message(e, "updating folder")}), 500


@news_api_bp.route("/subscription/folders/<folder_id>", methods=["DELETE"])
@login_required
def delete_folder(folder_id):
    """Delete a folder"""
    try:
        move_to = request.args.get("move_to")

        with get_user_db_session() as session:
            manager = FolderManager(session)
            success = manager.delete_folder(folder_id, move_to)

            if not success:
                return jsonify({"error": "Folder not found"}), 404

            return jsonify({"status": "deleted"}), 200

    except Exception as e:
        return jsonify({"error": safe_error_message(e, "deleting folder")}), 500


@news_api_bp.route("/subscription/subscriptions/organized", methods=["GET"])
@login_required
def get_subscriptions_organized():
    """Get subscriptions organized by folder"""
    try:
        user_id = get_user_id()

        with get_user_db_session() as session:
            manager = FolderManager(session)
            organized = manager.get_subscriptions_by_folder(user_id)

            # get_subscriptions_by_folder already returns JSON-friendly dicts
            # ({"folders": [{"folder": {...}, "subscriptions": [...]}, ...],
            # "uncategorized": [...]}). The previous code called .to_dict() on
            # those plain dicts (AttributeError -> HTTP 500). Flatten into the
            # {folder_name: [subscription, ...]} map the subscriptions UI
            # consumes (Object.values(...) for the "all" view, keyed lookup per
            # folder), with ungrouped subscriptions under "uncategorized".
            result = {}
            for entry in organized.get("folders", []):
                folder_name = entry["folder"].get("name") or entry[
                    "folder"
                ].get("id")
                result[folder_name] = entry["subscriptions"]
            # Merge (don't overwrite) so a user folder literally named
            # "uncategorized" doesn't have its subscriptions dropped by the
            # ungrouped bucket. In the normal case this just sets the key.
            result.setdefault("uncategorized", []).extend(
                organized.get("uncategorized", [])
            )

            return jsonify(result)

    except Exception as e:
        return jsonify(
            {"error": safe_error_message(e, "getting organized subscriptions")}
        ), 500


@news_api_bp.route(
    "/subscription/subscriptions/<subscription_id>", methods=["PUT"]
)
@login_required
@require_json_body()
def update_subscription_folder(subscription_id):
    """Update a subscription (mainly for folder assignment)"""
    try:
        data = request.json
        logger.info(
            f"Updating subscription {subscription_id} with data: {data}"
        )

        with get_user_db_session() as session:
            # Manually handle the update to ensure next_refresh is recalculated
            from ..database.models.news import (
                NewsSubscription as BaseSubscription,
            )
            from datetime import datetime, timedelta, timezone

            sub = (
                session.query(BaseSubscription)
                .filter_by(id=subscription_id)
                .first()
            )
            if not sub:
                return jsonify({"error": "Subscription not found"}), 404

            # `status` is the single source of truth for active/paused (the
            # scheduler keys on it, not the legacy is_active column). Translate
            # an is_active toggle into status and keep both out of the blind
            # setattr loop below -- otherwise a body like {"is_active": false}
            # would flip only the legacy column while status stayed "active"
            # and the scheduler would keep running the subscription. Mirrors
            # api.update_subscription's translation.
            if "is_active" in data:
                sub.status = "active" if data["is_active"] else "paused"
            if "status" in data:
                sub.status = data["status"]

            # Update remaining fields
            for key, value in data.items():
                if hasattr(sub, key) and key not in [
                    "id",
                    "user_id",
                    "created_at",
                    "is_active",
                    "status",
                ]:
                    setattr(sub, key, value)

            # Recalculate next_refresh if refresh_interval_minutes changed
            if "refresh_interval_minutes" in data:
                new_minutes = data["refresh_interval_minutes"]
                if sub.last_refresh:
                    sub.next_refresh = sub.last_refresh + timedelta(
                        minutes=new_minutes
                    )
                else:
                    sub.next_refresh = datetime.now(timezone.utc) + timedelta(
                        minutes=new_minutes
                    )
                logger.info(f"Recalculated next_refresh: {sub.next_refresh}")

            sub.updated_at = datetime.now(timezone.utc)
            session.commit()

            # NewsSubscription has no to_dict(); serialize the fields the UI
            # needs explicitly. is_active is derived from status (the source of
            # truth) so the response stays consistent with the toggle above.
            result = {
                "id": sub.id,
                "name": sub.name,
                "status": sub.status,
                "is_active": sub.status == "active",
                "folder_id": sub.folder_id,
                "refresh_interval_minutes": sub.refresh_interval_minutes,
                "next_refresh": sub.next_refresh.isoformat()
                if sub.next_refresh
                else None,
                "last_refresh": sub.last_refresh.isoformat()
                if sub.last_refresh
                else None,
            }
            logger.info(
                f"Updated subscription result: refresh_interval_minutes={result.get('refresh_interval_minutes')}, next_refresh={result.get('next_refresh')}"
            )
            return jsonify(result)

    except Exception as e:
        return jsonify(
            {"error": safe_error_message(e, "updating subscription")}
        ), 500


@news_api_bp.route("/subscription/stats", methods=["GET"])
@login_required
def get_subscription_stats():
    """Get subscription statistics"""
    try:
        user_id = get_user_id()

        with get_user_db_session() as session:
            manager = FolderManager(session)
            stats = manager.get_subscription_stats(user_id)

            return jsonify(stats)

    except Exception as e:
        return jsonify({"error": safe_error_message(e, "getting stats")}), 500


# Error handlers
@news_api_bp.errorhandler(400)
def bad_request(e):
    return jsonify({"error": "Bad request"}), 400


@news_api_bp.errorhandler(404)
def not_found(e):
    return jsonify({"error": "Resource not found"}), 404


@news_api_bp.errorhandler(500)
def internal_error(e):
    return jsonify({"error": "Internal server error"}), 500


@news_api_bp.route("/search-history", methods=["GET"])
@login_required
def get_search_history():
    """Get search history for current user."""
    try:
        # Get username from session
        from ..web.auth.decorators import current_user

        username = current_user()
        if not username:
            # Not authenticated, return empty history
            return jsonify({"search_history": []})

        # Get search history from user's encrypted database
        from ..database.session_context import get_user_db_session
        from ..database.models import UserNewsSearchHistory

        # Get password from Flask g object (set by middleware)
        from flask import g

        password = getattr(g, "user_password", None)

        with get_user_db_session(username, password) as db_session:
            history = (
                db_session.query(UserNewsSearchHistory)
                .order_by(UserNewsSearchHistory.created_at.desc())
                .limit(20)
                .all()
            )

            return jsonify(
                {"search_history": [item.to_dict() for item in history]}
            )

    except Exception as e:
        return jsonify(
            {"error": safe_error_message(e, "getting search history")}
        ), 500


@news_api_bp.route("/search-history", methods=["POST"])
@login_required
def add_search_history():
    """Add a search to the history."""
    try:
        # Get username from session
        from ..web.auth.decorators import current_user

        username = current_user()
        if not username:
            # Not authenticated
            return jsonify({"error": "Authentication required"}), 401

        data = request.get_json()
        logger.info(
            f"add_search_history received data keys: {list(data.keys()) if data else 'None'}"
        )
        if not data or not data.get("query"):
            logger.warning("Invalid search history data: missing query")
            return jsonify({"error": "query is required"}), 400

        # Add to user's encrypted database
        from ..database.session_context import get_user_db_session
        from ..database.models import UserNewsSearchHistory

        # Get password from Flask g object (set by middleware)
        from flask import g

        password = getattr(g, "user_password", None)

        with get_user_db_session(username, password) as db_session:
            search_history = UserNewsSearchHistory(
                query=data["query"],
                search_type=data.get("type", "filter"),
                result_count=data.get("resultCount", 0),
            )
            db_session.add(search_history)
            db_session.commit()

            return jsonify({"status": "success", "id": search_history.id})

    except Exception as e:
        logger.exception("Error adding search history")
        return jsonify(
            {"error": safe_error_message(e, "adding search history")}
        ), 500


@news_api_bp.route("/search-history", methods=["DELETE"])
@login_required
def clear_search_history():
    """Clear all search history for current user."""
    try:
        # Get username from session
        from ..web.auth.decorators import current_user

        username = current_user()
        if not username:
            return jsonify({"status": "success"})

        # Clear from user's encrypted database
        from ..database.session_context import get_user_db_session
        from ..database.models import UserNewsSearchHistory

        # Get password from Flask g object (set by middleware)
        from flask import g

        password = getattr(g, "user_password", None)

        with get_user_db_session(username, password) as db_session:
            db_session.query(UserNewsSearchHistory).delete()
            db_session.commit()

            return jsonify({"status": "success"})

    except Exception as e:
        return jsonify(
            {"error": safe_error_message(e, "clearing search history")}
        ), 500
