"""
Activity-based news subscription scheduler for per-user encrypted databases.
Tracks user activity and temporarily stores credentials for automatic updates.
"""

import random
import threading
from dataclasses import dataclass
from datetime import datetime, timedelta, UTC
from functools import wraps
from typing import Any, Callable, Dict, List

from cachetools import TTLCache
from loguru import logger
from ..settings.logger import log_settings
from ..settings.manager import SnapshotSettingsContext

from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.jobstores.base import JobLookupError
from ..constants import ResearchStatus
from ..database.credential_store_base import CredentialStoreBase
from ..database.session_context import safe_rollback
from ..database.thread_local_session import thread_cleanup

# RAG indexing imports
from ..research_library.services.library_rag_service import LibraryRAGService
from ..database.library_init import get_default_library_id
from ..database.models.library import Document, DocumentCollection


SCHEDULER_AVAILABLE = True  # Always available since it's a required dependency


class SchedulerCredentialStore(CredentialStoreBase):
    """Credential store for the news scheduler.

    Stores user passwords with TTL expiration so that background scheduler
    jobs can access encrypted per-user databases.
    """

    def __init__(self, ttl_hours: int = 48):
        super().__init__(ttl_hours * 3600)

    def store(self, username: str, password: str) -> None:
        """Store password for a user."""
        self._store_credentials(
            username, {"username": username, "password": password}
        )

    def retrieve(self, username: str) -> str | None:
        """Retrieve password for a user. Returns None if expired/missing."""
        result = self._retrieve_credentials(username, remove=False)
        return result[1] if result else None

    def clear(self, username: str) -> None:
        """Clear stored password for a user."""
        self.clear_entry(username)


@dataclass(frozen=True)
class DocumentSchedulerSettings:
    """
    Immutable settings snapshot for document scheduler.

    Thread-safe: This is a frozen dataclass that can be safely passed
    to and used from background threads.
    """

    enabled: bool = True
    interval_seconds: int = 1800
    download_pdfs: bool = False
    extract_text: bool = True
    generate_rag: bool = False
    last_run: str = ""

    @classmethod
    def defaults(cls) -> "DocumentSchedulerSettings":
        """Return default settings."""
        return cls()


class BackgroundJobScheduler:
    """
    Singleton scheduler that manages news subscriptions for active users.

    This scheduler:
    - Monitors user activity through database access
    - Temporarily stores user credentials in memory
    - Automatically schedules subscription checks
    - Cleans up inactive users after configurable period
    """

    _instance = None
    _lock = threading.Lock()

    def __new__(cls):
        """Ensure singleton instance."""
        if cls._instance is None:
            with cls._lock:
                if cls._instance is None:
                    cls._instance = super().__new__(cls)
        return cls._instance

    def __init__(self):
        """Initialize the scheduler (only runs once due to singleton)."""
        # Skip if already initialized
        if hasattr(self, "_initialized"):
            return

        # User session tracking
        self.user_sessions = {}  # user_id -> {last_activity, scheduled_jobs}
        self.lock = threading.Lock()

        # Credential store with TTL-based expiration
        self._credential_store = SchedulerCredentialStore(ttl_hours=48)

        # Scheduler instance
        self.scheduler = BackgroundScheduler()

        # Configuration (will be loaded from settings)
        self.config = self._load_default_config()

        # State
        self.is_running = False
        self._app = None  # Flask app reference for background job contexts

        # Settings cache: username -> DocumentSchedulerSettings
        # TTL of 300 seconds (5 minutes) reduces database queries
        self._settings_cache: TTLCache = TTLCache(maxsize=100, ttl=300)
        self._settings_cache_lock = threading.Lock()

        self._initialized = True
        logger.info("News scheduler initialized")

    def _load_default_config(self) -> Dict[str, Any]:
        """Load default configuration (will be overridden by settings manager)."""
        return {
            "enabled": True,
            "retention_hours": 48,
            "cleanup_interval_hours": 1,
            "max_jitter_seconds": 300,
            "max_concurrent_jobs": 10,
            "subscription_batch_size": 5,
            "activity_check_interval_minutes": 5,
        }

    def initialize_with_settings(self, settings_manager):
        """Initialize configuration from settings manager."""
        try:
            # Load all scheduler settings
            self.settings_manager = settings_manager
            self.config = {
                "enabled": self._get_setting("news.scheduler.enabled", True),
                "retention_hours": self._get_setting(
                    "news.scheduler.retention_hours", 48
                ),
                "cleanup_interval_hours": self._get_setting(
                    "news.scheduler.cleanup_interval_hours", 1
                ),
                "max_jitter_seconds": self._get_setting(
                    "news.scheduler.max_jitter_seconds", 300
                ),
                "max_concurrent_jobs": self._get_setting(
                    "news.scheduler.max_concurrent_jobs", 10
                ),
                "subscription_batch_size": self._get_setting(
                    "news.scheduler.batch_size", 5
                ),
                "activity_check_interval_minutes": self._get_setting(
                    "news.scheduler.activity_check_interval", 5
                ),
            }
            log_settings(self.config, "Scheduler configuration loaded")
        except Exception:
            logger.exception("Error loading scheduler settings")
            # Keep default config

    def _get_setting(self, key: str, default: Any) -> Any:
        """Get setting with fallback to default."""
        if hasattr(self, "settings_manager") and self.settings_manager:
            return self.settings_manager.get_setting(key, default=default)
        return default

    def set_app(self, app) -> None:
        """Store a reference to the Flask app for creating app contexts in background jobs."""
        self._app = app

    def _wrap_job(self, func: Callable) -> Callable:
        """Wrap a scheduler job function so it runs inside a Flask app context.

        APScheduler runs jobs in a thread pool without Flask context.
        This wrapper pushes an app context before the job runs and pops it after.
        """

        @wraps(func)
        def wrapper(*args, **kwargs):
            if self._app is not None:
                with self._app.app_context():
                    return func(*args, **kwargs)
            else:
                logger.warning(
                    f"No Flask app set on scheduler; running {func.__name__} without app context"
                )
                return func(*args, **kwargs)

        return wrapper

    def _get_document_scheduler_settings(
        self, username: str, force_refresh: bool = False
    ) -> DocumentSchedulerSettings:
        """
        Get document scheduler settings for a user with TTL caching.

        This is the single source of truth for document scheduler settings.
        Settings are cached for 5 minutes by default to reduce database queries.

        Args:
            username: User to get settings for
            force_refresh: If True, bypass cache and fetch fresh settings

        Returns:
            DocumentSchedulerSettings dataclass (frozen/immutable for thread-safety)
        """
        # Fast path: check cache without modifying it
        if not force_refresh:
            with self._settings_cache_lock:
                cached = self._settings_cache.get(username)
                if cached is not None:
                    logger.debug(f"[SETTINGS_CACHE] Cache hit for {username}")
                    cached_settings: DocumentSchedulerSettings = cached
                    return cached_settings

        # Cache miss - need to fetch from database
        logger.debug(
            f"[SETTINGS_CACHE] Cache miss for {username}, fetching from DB"
        )

        # Get password from session
        session_info = self.user_sessions.get(username)
        if not session_info:
            logger.warning(
                f"[SETTINGS_CACHE] No session info for {username}, using defaults"
            )
            return DocumentSchedulerSettings.defaults()

        password = self._credential_store.retrieve(username)
        if not password:
            logger.warning(
                f"[SETTINGS_CACHE] Credentials expired for {username}, using defaults"
            )
            return DocumentSchedulerSettings.defaults()

        # Fetch settings from database (outside lock to avoid blocking)
        try:
            from ..database.session_context import get_user_db_session
            from ..settings.manager import SettingsManager

            with get_user_db_session(username, password) as db:
                sm = SettingsManager(db)

                settings = DocumentSchedulerSettings(
                    enabled=sm.get_setting("document_scheduler.enabled", True),
                    interval_seconds=sm.get_setting(
                        "document_scheduler.interval_seconds", 1800
                    ),
                    download_pdfs=sm.get_setting(
                        "document_scheduler.download_pdfs", False
                    ),
                    extract_text=sm.get_setting(
                        "document_scheduler.extract_text", True
                    ),
                    generate_rag=sm.get_setting(
                        "document_scheduler.generate_rag", False
                    ),
                    last_run=sm.get_setting("document_scheduler.last_run", ""),
                )

            # Store in cache
            with self._settings_cache_lock:
                self._settings_cache[username] = settings
                logger.debug(f"[SETTINGS_CACHE] Cached settings for {username}")

            return settings

        except Exception:
            logger.exception(
                f"[SETTINGS_CACHE] Error fetching settings for {username}"
            )
            return DocumentSchedulerSettings.defaults()

    def invalidate_user_settings_cache(self, username: str) -> bool:
        """
        Invalidate cached settings for a specific user.

        Call this when user settings change or user logs out.

        Args:
            username: User whose cache to invalidate

        Returns:
            True if cache entry was removed, False if not found
        """
        with self._settings_cache_lock:
            if username in self._settings_cache:
                del self._settings_cache[username]
                logger.debug(
                    f"[SETTINGS_CACHE] Invalidated cache for {username}"
                )
                return True
            return False

    def invalidate_all_settings_cache(self) -> int:
        """
        Invalidate all cached settings.

        Call this when doing bulk settings updates or during config reload.

        Returns:
            Number of cache entries cleared
        """
        with self._settings_cache_lock:
            count = len(self._settings_cache)
            self._settings_cache.clear()
            logger.info(
                f"[SETTINGS_CACHE] Cleared all settings cache ({count} entries)"
            )
            return count

    def start(self):
        """Start the scheduler."""
        if not self.config.get("enabled", True):
            logger.info("News scheduler is disabled in settings")
            return

        if self.is_running:
            logger.warning("Scheduler is already running")
            return

        if self._app is None:
            raise RuntimeError(
                "BackgroundJobScheduler.set_app() must be called before start()"
            )

        # Schedule cleanup job
        self.scheduler.add_job(
            self._wrap_job(self._run_cleanup_with_tracking),
            "interval",
            hours=self.config["cleanup_interval_hours"],
            id="cleanup_inactive_users",
            name="Cleanup Inactive User Sessions",
            jitter=60,  # Add some jitter to cleanup
        )

        # Schedule configuration reload
        self.scheduler.add_job(
            self._wrap_job(self._reload_config),
            "interval",
            minutes=30,
            id="reload_config",
            name="Reload Configuration",
        )

        # Start the scheduler
        self.scheduler.start()
        self.is_running = True

        # Schedule initial cleanup after a delay
        self.scheduler.add_job(
            self._wrap_job(self._run_cleanup_with_tracking),
            "date",
            run_date=datetime.now(UTC) + timedelta(seconds=30),
            id="initial_cleanup",
        )

        logger.info("News scheduler started")

    def stop(self):
        """Stop the scheduler."""
        if self.is_running:
            self.scheduler.shutdown(wait=True)
            self.is_running = False

            # Clear all user sessions and credentials
            with self.lock:
                for username in self.user_sessions:
                    self._credential_store.clear(username)
                self.user_sessions.clear()

            logger.info("News scheduler stopped")

    def update_user_info(self, username: str, password: str):
        """
        Update user info in scheduler. Called on every database interaction.

        Args:
            username: User's username
            password: User's password
        """
        logger.info(
            f"[SCHEDULER] update_user_info called for {username}, is_running={self.is_running}, active_users={len(self.user_sessions)}"
        )
        logger.debug(
            f"[SCHEDULER] Current active users: {list(self.user_sessions.keys())}"
        )

        if not self.is_running:
            logger.warning(
                f"[SCHEDULER] Scheduler not running, cannot update user {username}"
            )
            return

        with self.lock:
            # Store password in credential store (inside lock to prevent
            # race where concurrent calls leave mismatched credentials)
            self._credential_store.store(username, password)

            now = datetime.now(UTC)

            if username not in self.user_sessions:
                # New user - create session info
                logger.info(f"[SCHEDULER] New user in scheduler: {username}")
                self.user_sessions[username] = {
                    "last_activity": now,
                    "scheduled_jobs": set(),
                }
                logger.debug(
                    f"[SCHEDULER] Created session for {username}, scheduling subscriptions"
                )
                # Schedule their subscriptions
                self._schedule_user_subscriptions(username)
            else:
                # Existing user - update info
                logger.info(
                    f"[SCHEDULER] Updating existing user {username} activity, will reschedule"
                )
                old_activity = self.user_sessions[username]["last_activity"]
                activity_delta = now - old_activity
                logger.debug(
                    f"[SCHEDULER] User {username} last activity: {old_activity}, delta: {activity_delta}"
                )

                self.user_sessions[username]["last_activity"] = now
                logger.debug(
                    f"[SCHEDULER] Updated {username} session info, scheduling subscriptions"
                )
                # Reschedule their subscriptions in case they changed
                self._schedule_user_subscriptions(username)

    def unregister_user(self, username: str):
        """
        Unregister a user and clean up their scheduled jobs.
        Called when user logs out.
        """
        with self.lock:
            if username in self.user_sessions:
                logger.info(f"Unregistering user {username}")

                # Remove all scheduled jobs for this user
                session_info = self.user_sessions[username]
                for job_id in session_info["scheduled_jobs"].copy():
                    try:
                        self.scheduler.remove_job(job_id)
                    except JobLookupError:
                        pass

                # Remove user session and clear credentials atomically
                del self.user_sessions[username]
                self._credential_store.clear(username)

        # Invalidate settings cache for this user (outside lock)
        self.invalidate_user_settings_cache(username)
        logger.info(f"User {username} unregistered successfully")

    def _schedule_user_subscriptions(self, username: str):
        """Schedule all active subscriptions for a user."""
        logger.info(f"_schedule_user_subscriptions called for {username}")
        try:
            session_info = self.user_sessions.get(username)
            if not session_info:
                logger.warning(f"No session info found for {username}")
                return

            password = self._credential_store.retrieve(username)
            if not password:
                logger.warning(
                    f"Credentials expired for {username}, skipping subscription scheduling"
                )
                return
            logger.debug(f"Got password for {username}: present")

            # Get user's subscriptions from their encrypted database
            from ..database.session_context import get_user_db_session
            from ..database.models.news import NewsSubscription

            with get_user_db_session(username, password) as db:
                subscriptions = (
                    db.query(NewsSubscription).filter_by(is_active=True).all()
                )
                logger.debug(
                    f"Query executed, found {len(subscriptions)} results"
                )

                # Log details of each subscription
                for sub in subscriptions:
                    logger.debug(
                        f"Subscription {sub.id}: name='{sub.name}', is_active={sub.is_active}, status='{sub.status}', refresh_interval={sub.refresh_interval_minutes} minutes"
                    )

            logger.info(
                f"Found {len(subscriptions)} active subscriptions for {username}"
            )

            # Clear old jobs for this user
            for job_id in session_info["scheduled_jobs"].copy():
                try:
                    self.scheduler.remove_job(job_id)
                    session_info["scheduled_jobs"].remove(job_id)
                except JobLookupError:
                    pass

            # Schedule each subscription with jitter
            for sub in subscriptions:
                job_id = f"{username}_{sub.id}"

                # Calculate jitter
                # Security: random jitter to distribute subscription timing, not security-sensitive
                max_jitter = int(self.config.get("max_jitter_seconds", 300))
                jitter = random.randint(0, max_jitter)

                # Determine trigger based on frequency
                refresh_minutes = sub.refresh_interval_minutes

                if refresh_minutes <= 60:  # 60 minutes or less
                    # For hourly or more frequent, use interval trigger
                    trigger = "interval"
                    trigger_args = {
                        "minutes": refresh_minutes,
                        "jitter": jitter,
                        "start_date": datetime.now(UTC),  # Start immediately
                    }
                else:
                    # For less frequent, calculate next run time
                    now = datetime.now(UTC)
                    if sub.next_refresh:
                        # Ensure timezone-aware for comparison with now (UTC)
                        next_refresh_aware = sub.next_refresh
                        if next_refresh_aware.tzinfo is None:
                            logger.warning(
                                f"Subscription {sub.id} has naive (non-tz-aware) "
                                f"next_refresh datetime, assuming UTC"
                            )
                            next_refresh_aware = next_refresh_aware.replace(
                                tzinfo=UTC
                            )
                        if next_refresh_aware <= now:
                            # Subscription is overdue - run it immediately with small jitter
                            logger.info(
                                f"Subscription {sub.id} is overdue, scheduling immediate run"
                            )
                            next_run = now + timedelta(seconds=jitter)
                        else:
                            next_run = next_refresh_aware
                    else:
                        next_run = now + timedelta(
                            minutes=refresh_minutes, seconds=jitter
                        )

                    trigger = "date"
                    trigger_args = {"run_date": next_run}

                # Add the job
                self.scheduler.add_job(
                    func=self._wrap_job(self._check_subscription),
                    args=[username, sub.id],
                    trigger=trigger,
                    id=job_id,
                    name=f"Check {sub.name or sub.query_or_topic[:30]}",
                    replace_existing=True,
                    **trigger_args,
                )

                session_info["scheduled_jobs"].add(job_id)
                logger.info(f"Scheduled job {job_id} with {trigger} trigger")

        except Exception:
            logger.exception(f"Error scheduling subscriptions for {username}")

        # Add document processing for this user
        self._schedule_document_processing(username)

    def _schedule_document_processing(self, username: str):
        """Schedule document processing for a user."""
        logger.info(
            f"[DOC_SCHEDULER] Scheduling document processing for {username}"
        )
        logger.debug(
            f"[DOC_SCHEDULER] Current user sessions: {list(self.user_sessions.keys())}"
        )

        try:
            session_info = self.user_sessions.get(username)
            if not session_info:
                logger.warning(
                    f"[DOC_SCHEDULER] No session info found for {username}"
                )
                logger.debug(
                    f"[DOC_SCHEDULER] Available sessions: {list(self.user_sessions.keys())}"
                )
                return

            logger.debug(
                f"[DOC_SCHEDULER] Retrieved session for {username}, scheduler running: {self.is_running}"
            )

            # Get user's document scheduler settings (cached)
            settings = self._get_document_scheduler_settings(username)

            if not settings.enabled:
                logger.info(
                    f"[DOC_SCHEDULER] Document scheduler disabled for user {username}"
                )
                return

            logger.info(
                f"[DOC_SCHEDULER] User {username} document settings: enabled={settings.enabled}, "
                f"interval={settings.interval_seconds}s, pdfs={settings.download_pdfs}, "
                f"text={settings.extract_text}, rag={settings.generate_rag}"
            )

            # Schedule document processing job
            job_id = f"{username}_document_processing"
            logger.debug(f"[DOC_SCHEDULER] Preparing to schedule job {job_id}")

            # Remove existing document job if any
            try:
                self.scheduler.remove_job(job_id)
                session_info["scheduled_jobs"].discard(job_id)
                logger.debug(f"[DOC_SCHEDULER] Removed existing job {job_id}")
            except JobLookupError:
                logger.debug(
                    f"[DOC_SCHEDULER] No existing job {job_id} to remove"
                )
                pass  # Job doesn't exist, that's fine

            # Add new document processing job
            logger.debug(
                f"[DOC_SCHEDULER] Adding new document processing job with interval {settings.interval_seconds}s"
            )
            self.scheduler.add_job(
                func=self._wrap_job(self._process_user_documents),
                args=[username],
                trigger="interval",
                seconds=settings.interval_seconds,
                id=job_id,
                name=f"Process Documents for {username}",
                jitter=30,  # Add small jitter to prevent multiple users from processing simultaneously
                max_instances=1,  # Prevent overlapping document processing for same user
                replace_existing=True,
            )

            session_info["scheduled_jobs"].add(job_id)
            logger.info(
                f"[DOC_SCHEDULER] Scheduled document processing job {job_id} for {username} with {settings.interval_seconds}s interval"
            )
            logger.debug(
                f"[DOC_SCHEDULER] User {username} now has {len(session_info['scheduled_jobs'])} scheduled jobs: {list(session_info['scheduled_jobs'])}"
            )

            # Verify job was added
            job = self.scheduler.get_job(job_id)
            if job:
                logger.info(
                    f"[DOC_SCHEDULER] Successfully verified job {job_id} exists, next run: {job.next_run_time}"
                )
            else:
                logger.error(
                    f"[DOC_SCHEDULER] Failed to verify job {job_id} exists!"
                )

        except Exception:
            logger.exception(
                f"Error scheduling document processing for {username}"
            )

    @thread_cleanup
    def _process_user_documents(self, username: str):
        """Process documents for a user."""
        logger.info(f"[DOC_SCHEDULER] Processing documents for user {username}")
        start_time = datetime.now(UTC)

        try:
            session_info = self.user_sessions.get(username)
            if not session_info:
                logger.warning(
                    f"[DOC_SCHEDULER] No session info found for user {username}"
                )
                return

            password = self._credential_store.retrieve(username)
            if not password:
                logger.warning(
                    f"[DOC_SCHEDULER] Credentials expired for user {username}"
                )
                return
            logger.debug(
                f"[DOC_SCHEDULER] Starting document processing for {username}"
            )

            # Get user's document scheduler settings (cached)
            settings = self._get_document_scheduler_settings(username)

            logger.info(
                f"[DOC_SCHEDULER] Processing settings for {username}: "
                f"pdfs={settings.download_pdfs}, text={settings.extract_text}, rag={settings.generate_rag}"
            )

            if not any(
                [
                    settings.download_pdfs,
                    settings.extract_text,
                    settings.generate_rag,
                ]
            ):
                logger.info(
                    f"[DOC_SCHEDULER] No processing options enabled for user {username}"
                )
                return

            # Parse last_run from cached settings
            last_run = (
                datetime.fromisoformat(settings.last_run)
                if settings.last_run
                else None
            )

            logger.info(f"[DOC_SCHEDULER] Last run for {username}: {last_run}")

            # Need database session for queries and updates
            from ..database.session_context import get_user_db_session
            from ..database.models.research import ResearchHistory
            from ..settings.manager import SettingsManager

            with get_user_db_session(username, password) as db:
                settings_manager = SettingsManager(db)

                # Query for completed research since last run
                logger.debug(
                    f"[DOC_SCHEDULER] Querying for completed research since {last_run}"
                )
                query = db.query(ResearchHistory).filter(
                    ResearchHistory.status == ResearchStatus.COMPLETED,
                    ResearchHistory.completed_at.is_not(
                        None
                    ),  # Ensure completed_at is not null
                )

                if last_run:
                    query = query.filter(
                        ResearchHistory.completed_at > last_run
                    )

                # Limit to recent research to prevent overwhelming
                query = query.order_by(
                    ResearchHistory.completed_at.desc()
                ).limit(20)

                research_sessions = query.all()
                logger.debug(
                    f"[DOC_SCHEDULER] Query executed, found {len(research_sessions)} sessions"
                )

                if not research_sessions:
                    logger.info(
                        f"[DOC_SCHEDULER] No new completed research sessions found for user {username}"
                    )
                    return

                logger.info(
                    f"[DOC_SCHEDULER] Found {len(research_sessions)} research sessions to process for {username}"
                )

                # Log details of each research session
                for i, research in enumerate(
                    research_sessions[:5]
                ):  # Log first 5 details
                    title_safe = (
                        (research.title[:50] + "...")
                        if research.title
                        else "No title"
                    )
                    completed_safe = (
                        research.completed_at
                        if research.completed_at
                        else "No completion time"
                    )
                    logger.debug(
                        f"[DOC_SCHEDULER] Session {i + 1}: id={research.id}, title={title_safe}, completed={completed_safe}"
                    )

                    # Handle completed_at which might be a string or datetime
                    completed_at_obj = None
                    if research.completed_at:
                        if isinstance(research.completed_at, str):
                            try:
                                completed_at_obj = datetime.fromisoformat(
                                    research.completed_at.replace("Z", "+00:00")
                                )
                            except (ValueError, TypeError, AttributeError):
                                completed_at_obj = None
                        else:
                            completed_at_obj = research.completed_at

                    logger.debug(
                        f"[DOC_SCHEDULER]   - completed_at type: {type(research.completed_at)}"
                    )
                    logger.debug(
                        f"[DOC_SCHEDULER]   - completed_at timezone: {completed_at_obj.tzinfo if completed_at_obj else 'None'}"
                    )
                    logger.debug(f"[DOC_SCHEDULER]   - last_run: {last_run}")
                    logger.debug(
                        f"[DOC_SCHEDULER]   - completed_at > last_run: {completed_at_obj > last_run if last_run and completed_at_obj else 'N/A'}"
                    )

                processed_count = 0
                for research in research_sessions:
                    try:
                        logger.info(
                            f"[DOC_SCHEDULER] Processing research {research.id} for user {username}"
                        )

                        # Set search context so rate limiting works in both
                        # download_pdfs and extract_text paths
                        from ..utilities.thread_context import (
                            set_search_context,
                        )

                        set_search_context(
                            {
                                "research_id": str(research.id),
                                "username": username,
                                "user_password": password,
                                "research_phase": "document_scheduler",
                            }
                        )

                        # Call actual processing APIs
                        if settings.download_pdfs:
                            logger.info(
                                f"[DOC_SCHEDULER] Downloading PDFs for research {research.id}"
                            )
                            try:
                                # Use the DownloadService to queue PDF downloads
                                from ..research_library.services.download_service import (
                                    DownloadService,
                                )

                                with DownloadService(
                                    username=username, password=password
                                ) as download_service:
                                    queued_count = download_service.queue_research_downloads(
                                        research.id
                                    )
                                    logger.info(
                                        f"[DOC_SCHEDULER] Queued {queued_count} PDF downloads for research {research.id}"
                                    )
                            except Exception:
                                # Recover the shared thread-local session
                                # before continuing — without rollback the
                                # next phase (text extract / RAG) and the
                                # post-loop last_run commit run on a
                                # poisoned session (issue #3827).
                                safe_rollback(db, "DOC_SCHEDULER PDF download")
                                logger.exception(
                                    f"[DOC_SCHEDULER] Failed to download PDFs for research {research.id}"
                                )

                        if settings.extract_text:
                            logger.info(
                                f"[DOC_SCHEDULER] Extracting text for research {research.id}"
                            )
                            try:
                                # Use the DownloadService to extract text for all resources
                                from ..research_library.services.download_service import (
                                    DownloadService,
                                )
                                from ..database.models.research import (
                                    ResearchResource,
                                )

                                from ..research_library.utils import (
                                    is_downloadable_url,
                                )

                                with DownloadService(
                                    username=username, password=password
                                ) as download_service:
                                    # Get all resources for this research (reuse existing db session)
                                    all_resources = (
                                        db.query(ResearchResource)
                                        .filter_by(research_id=research.id)
                                        .all()
                                    )
                                    # Filter: only process downloadable resources (academic/PDF)
                                    resources = [
                                        r
                                        for r in all_resources
                                        if is_downloadable_url(r.url)
                                    ]
                                    processed_count = 0
                                    for resource in resources:
                                        # We need to pass the password to the download service
                                        # The DownloadService creates its own database sessions, so we need to ensure password is available
                                        try:
                                            success, error = (
                                                download_service.download_as_text(
                                                    resource.id
                                                )
                                            )
                                            if success:
                                                processed_count += 1
                                                logger.info(
                                                    f"[DOC_SCHEDULER] Successfully extracted text for resource {resource.id}"
                                                )
                                            else:
                                                logger.warning(
                                                    f"[DOC_SCHEDULER] Failed to extract text for resource {resource.id}: {error}"
                                                )
                                        except Exception as resource_error:
                                            # Roll back FIRST so the next
                                            # iteration's queries don't
                                            # cascade on a poisoned session
                                            # (issue #3827).
                                            safe_rollback(
                                                db,
                                                "DOC_SCHEDULER resource",
                                            )
                                            logger.exception(
                                                f"[DOC_SCHEDULER] Error processing resource {resource.id}: {resource_error}"
                                            )
                                    logger.info(
                                        f"[DOC_SCHEDULER] Text extraction completed for research {research.id}: {processed_count}/{len(resources)} resources processed"
                                    )
                            except Exception:
                                safe_rollback(
                                    db, "DOC_SCHEDULER text extraction"
                                )
                                logger.exception(
                                    f"[DOC_SCHEDULER] Failed to extract text for research {research.id}"
                                )

                        if settings.generate_rag:
                            logger.info(
                                f"[DOC_SCHEDULER] Generating RAG embeddings for research {research.id}"
                            )
                            try:
                                # Get embedding settings from user configuration
                                embedding_model = settings_manager.get_setting(
                                    "local_search_embedding_model",
                                    "all-MiniLM-L6-v2",
                                )
                                embedding_provider = (
                                    settings_manager.get_setting(
                                        "local_search_embedding_provider",
                                        "sentence_transformers",
                                    )
                                )
                                chunk_size = int(
                                    settings_manager.get_setting(
                                        "local_search_chunk_size", 1000
                                    )
                                )
                                chunk_overlap = int(
                                    settings_manager.get_setting(
                                        "local_search_chunk_overlap", 200
                                    )
                                )

                                # Initialize RAG service with user's embedding configuration
                                with LibraryRAGService(
                                    username=username,
                                    embedding_model=embedding_model,
                                    embedding_provider=embedding_provider,
                                    chunk_size=chunk_size,
                                    chunk_overlap=chunk_overlap,
                                    db_password=password,
                                ) as rag_service:
                                    # Get default Library collection ID
                                    library_collection_id = (
                                        get_default_library_id(
                                            username, password
                                        )
                                    )

                                    # Query for unindexed documents from this research session
                                    documents_to_index = (
                                        db.query(Document.id, Document.title)
                                        .outerjoin(
                                            DocumentCollection,
                                            (
                                                DocumentCollection.document_id
                                                == Document.id
                                            )
                                            & (
                                                DocumentCollection.collection_id
                                                == library_collection_id
                                            ),
                                        )
                                        .filter(
                                            Document.research_id == research.id,
                                            Document.text_content.isnot(None),
                                            (
                                                DocumentCollection.indexed.is_(
                                                    False
                                                )
                                                | DocumentCollection.id.is_(
                                                    None
                                                )
                                            ),
                                        )
                                        .all()
                                    )

                                    if not documents_to_index:
                                        logger.info(
                                            f"[DOC_SCHEDULER] No unindexed documents found for research {research.id}"
                                        )
                                    else:
                                        # Index each document
                                        indexed_count = 0
                                        for (
                                            doc_id,
                                            doc_title,
                                        ) in documents_to_index:
                                            try:
                                                result = rag_service.index_document(
                                                    document_id=doc_id,
                                                    collection_id=library_collection_id,
                                                    force_reindex=False,
                                                )
                                                if (
                                                    result["status"]
                                                    == "success"
                                                ):
                                                    indexed_count += 1
                                                    logger.info(
                                                        f"[DOC_SCHEDULER] Indexed document {doc_id} ({doc_title}) "
                                                        f"with {result.get('chunk_count', 0)} chunks"
                                                    )
                                            except Exception as doc_error:
                                                logger.exception(
                                                    f"[DOC_SCHEDULER] Failed to index document {doc_id}: {doc_error}"
                                                )

                                        logger.info(
                                            f"[DOC_SCHEDULER] RAG indexing completed for research {research.id}: "
                                            f"{indexed_count}/{len(documents_to_index)} documents indexed"
                                        )
                            except Exception:
                                safe_rollback(db, "DOC_SCHEDULER RAG")
                                logger.exception(
                                    f"[DOC_SCHEDULER] Failed to generate RAG embeddings for research {research.id}"
                                )

                        processed_count += 1
                        logger.debug(
                            f"[DOC_SCHEDULER] Successfully queued processing for research {research.id}"
                        )

                    except Exception:
                        safe_rollback(db, "DOC_SCHEDULER research")
                        logger.exception(
                            f"[DOC_SCHEDULER] Error processing research {research.id} for user {username}"
                        )

                # Update last run time in user's settings.
                # Intentionally NOT wrapped in try/finally: if upstream setup
                # fails (DB open, SettingsManager init, initial query),
                # last_run should stay put so the next tick retries.
                # Advancing here would mask a persistent failure (corrupted
                # DB, wrong password). See closed PR #3288.
                current_time = datetime.now(UTC).isoformat()
                settings_manager.set_setting(
                    "document_scheduler.last_run", current_time, commit=True
                )
                logger.debug(
                    f"[DOC_SCHEDULER] Updated last run time for {username} to {current_time}"
                )

                end_time = datetime.now(UTC)
                duration = (end_time - start_time).total_seconds()
                logger.info(
                    f"[DOC_SCHEDULER] Completed document processing for user {username}: {processed_count} sessions processed in {duration:.2f}s"
                )

        except Exception:
            logger.exception(
                f"[DOC_SCHEDULER] Error processing documents for user {username}"
            )

    def get_document_scheduler_status(self, username: str) -> Dict[str, Any]:
        """Get document scheduler status for a specific user."""
        try:
            session_info = self.user_sessions.get(username)
            if not session_info:
                return {
                    "enabled": False,
                    "message": "User not found in scheduler",
                }

            # Get user's document scheduler settings (cached)
            settings = self._get_document_scheduler_settings(username)

            # Check if user has document processing job
            job_id = f"{username}_document_processing"
            has_job = job_id in session_info.get("scheduled_jobs", set())

            return {
                "enabled": settings.enabled,
                "interval_seconds": settings.interval_seconds,
                "processing_options": {
                    "download_pdfs": settings.download_pdfs,
                    "extract_text": settings.extract_text,
                    "generate_rag": settings.generate_rag,
                },
                "last_run": settings.last_run,
                "has_scheduled_job": has_job,
                "user_active": username in self.user_sessions,
            }

        except Exception as e:
            logger.exception(
                f"Error getting document scheduler status for user {username}"
            )
            return {
                "enabled": False,
                "message": f"Failed to retrieve scheduler status: {type(e).__name__}",
            }

    def trigger_document_processing(self, username: str) -> bool:
        """Trigger immediate document processing for a user."""
        logger.info(
            f"[DOC_SCHEDULER] Manual trigger requested for user {username}"
        )
        try:
            session_info = self.user_sessions.get(username)
            if not session_info:
                logger.warning(
                    f"[DOC_SCHEDULER] User {username} not found in scheduler"
                )
                logger.debug(
                    f"[DOC_SCHEDULER] Available users: {list(self.user_sessions.keys())}"
                )
                return False

            if not self.is_running:
                logger.warning(
                    f"[DOC_SCHEDULER] Scheduler not running, cannot trigger document processing for {username}"
                )
                return False

            # Trigger immediate processing
            job_id = f"{username}_document_processing_manual"
            logger.debug(f"[DOC_SCHEDULER] Scheduling manual job {job_id}")

            self.scheduler.add_job(
                func=self._wrap_job(self._process_user_documents),
                args=[username],
                trigger="date",
                run_date=datetime.now(UTC) + timedelta(seconds=1),
                id=job_id,
                name=f"Manual Document Processing for {username}",
                replace_existing=True,
            )

            # Verify job was added
            job = self.scheduler.get_job(job_id)
            if job:
                logger.info(
                    f"[DOC_SCHEDULER] Successfully triggered manual document processing for user {username}, job {job_id}, next run: {job.next_run_time}"
                )
            else:
                logger.error(
                    f"[DOC_SCHEDULER] Failed to verify manual job {job_id} was added!"
                )
                return False

            return True

        except Exception:
            logger.exception(
                f"[DOC_SCHEDULER] Error triggering document processing for user {username}"
            )
            return False

    @thread_cleanup
    def _check_user_overdue_subscriptions(self, username: str):
        """Check and immediately run any overdue subscriptions for a user."""
        try:
            session_info = self.user_sessions.get(username)
            if not session_info:
                return

            password = self._credential_store.retrieve(username)
            if not password:
                return

            # Get user's overdue subscriptions
            from ..database.session_context import get_user_db_session
            from ..database.models.news import NewsSubscription
            from datetime import timezone

            with get_user_db_session(username, password) as db:
                now = datetime.now(timezone.utc)
                overdue_subs = (
                    db.query(NewsSubscription)
                    .filter(
                        NewsSubscription.is_active.is_(True),
                        NewsSubscription.next_refresh.is_not(None),
                        NewsSubscription.next_refresh <= now,
                    )
                    .all()
                )

            if overdue_subs:
                logger.info(
                    f"Found {len(overdue_subs)} overdue subscriptions for {username}"
                )

                for sub in overdue_subs:
                    # Run immediately with small random delay
                    # Security: random delay to stagger overdue jobs, not security-sensitive
                    delay_seconds = random.randint(1, 30)
                    job_id = (
                        f"overdue_{username}_{sub.id}_{int(now.timestamp())}"
                    )

                    self.scheduler.add_job(
                        func=self._wrap_job(self._check_subscription),
                        args=[username, sub.id],
                        trigger="date",
                        run_date=now + timedelta(seconds=delay_seconds),
                        id=job_id,
                        name=f"Overdue: {sub.name or sub.query_or_topic[:30]}",
                        replace_existing=True,
                    )

                    logger.info(
                        f"Scheduled overdue subscription {sub.id} to run in {delay_seconds} seconds"
                    )

        except Exception:
            logger.exception(
                f"Error checking overdue subscriptions for {username}"
            )

    @thread_cleanup
    def _check_subscription(self, username: str, subscription_id: int):
        """Check and refresh a single subscription."""
        logger.info(
            f"_check_subscription called for user {username}, subscription {subscription_id}"
        )
        try:
            session_info = self.user_sessions.get(username)
            if not session_info:
                # User no longer active, cancel job
                job_id = f"{username}_{subscription_id}"
                try:
                    self.scheduler.remove_job(job_id)
                except JobLookupError:
                    pass
                return

            password = self._credential_store.retrieve(username)
            if not password:
                logger.warning(
                    f"Credentials expired for {username}, skipping subscription check"
                )
                return

            # Get subscription details
            from ..database.session_context import get_user_db_session
            from ..database.models.news import NewsSubscription

            with get_user_db_session(username, password) as db:
                sub = db.query(NewsSubscription).get(subscription_id)
                if not sub or not sub.is_active:
                    logger.info(
                        f"Subscription {subscription_id} not active, skipping"
                    )
                    return

                # Prepare query with date replacement using user's timezone
                query = sub.query_or_topic
                if "YYYY-MM-DD" in query:
                    from local_deep_research.news.core.utils import (
                        get_local_date_string,
                    )
                    from ..settings.manager import SettingsManager

                    settings_manager = SettingsManager(db)
                    local_date = get_local_date_string(settings_manager)
                    query = query.replace("YYYY-MM-DD", local_date)

                # Update last/next refresh times
                sub.last_refresh = datetime.now(UTC)
                sub.next_refresh = datetime.now(UTC) + timedelta(
                    minutes=sub.refresh_interval_minutes
                )
                db.commit()

                subscription_data = {
                    "id": sub.id,
                    "name": sub.name,
                    "query": query,
                    "original_query": sub.query_or_topic,
                    "model_provider": sub.model_provider,
                    "model": sub.model,
                    "search_strategy": sub.search_strategy,
                    "search_engine": sub.search_engine,
                }

            logger.info(
                f"Refreshing subscription {subscription_id}: {subscription_data['name']}"
            )

            # Trigger research synchronously using requests with proper auth
            self._trigger_subscription_research_sync(
                username, subscription_data
            )

            # Reschedule for next interval if using interval trigger
            job_id = f"{username}_{subscription_id}"
            job = self.scheduler.get_job(job_id)
            if job and job.trigger.__class__.__name__ == "DateTrigger":
                # For date triggers, reschedule
                # Security: random jitter to distribute subscription timing, not security-sensitive
                next_run = datetime.now(UTC) + timedelta(
                    minutes=sub.refresh_interval_minutes,
                    seconds=random.randint(
                        0, int(self.config.get("max_jitter_seconds", 300))
                    ),
                )
                self.scheduler.add_job(
                    func=self._wrap_job(self._check_subscription),
                    args=[username, subscription_id],
                    trigger="date",
                    run_date=next_run,
                    id=job_id,
                    replace_existing=True,
                )

        except Exception:
            logger.exception(f"Error checking subscription {subscription_id}")

    @thread_cleanup
    def _trigger_subscription_research_sync(
        self, username: str, subscription: Dict[str, Any]
    ):
        """Trigger research for a subscription using programmatic API."""
        from ..config.thread_settings import set_settings_context

        try:
            # Get user's password from session info
            session_info = self.user_sessions.get(username)
            if not session_info:
                logger.error(f"No session info for user {username}")
                return

            password = self._credential_store.retrieve(username)
            if not password:
                logger.error(f"Credentials expired for user {username}")
                return

            # Generate research ID
            import uuid

            research_id = str(uuid.uuid4())

            logger.info(
                f"Starting research {research_id} for subscription {subscription['id']}"
            )

            # Get user settings for research
            from ..database.session_context import get_user_db_session
            from ..settings.manager import SettingsManager

            with get_user_db_session(username, password) as db:
                settings_manager = SettingsManager(db)
                settings_snapshot = settings_manager.get_settings_snapshot()

                # Use the search engine from the subscription if specified
                search_engine = subscription.get("search_engine")

                if search_engine:
                    settings_snapshot["search.tool"] = {
                        "value": search_engine,
                        "ui_element": "select",
                    }
                    logger.info(
                        f"Using subscription's search engine: '{search_engine}' for {subscription['id']}"
                    )
                else:
                    # Use the user's default search tool from their settings
                    default_search_tool = settings_snapshot.get(
                        "search.tool", "auto"
                    )
                    logger.info(
                        f"Using user's default search tool: '{default_search_tool}' for {subscription['id']}"
                    )

                logger.debug(
                    f"Settings snapshot has {len(settings_snapshot)} settings"
                )
                # Log a few key settings to verify they're present
                logger.debug(
                    f"Key settings: llm.model={settings_snapshot.get('llm.model')}, llm.provider={settings_snapshot.get('llm.provider')}, search.tool={settings_snapshot.get('search.tool')}"
                )

            # Set up research parameters
            query = subscription["query"]

            # Build metadata for news search
            metadata = {
                "is_news_search": True,
                "search_type": "news_analysis",
                "display_in": "news_feed",
                "subscription_id": subscription["id"],
                "triggered_by": "scheduler",
                "subscription_name": subscription["name"],
                "title": subscription["name"] if subscription["name"] else None,
                "scheduled_at": datetime.now(UTC).isoformat(),
                "original_query": subscription["original_query"],
                "user_id": username,
            }

            # Use programmatic API with settings context
            from ..api.research_functions import quick_summary

            # Create and set settings context for this thread
            settings_context = SnapshotSettingsContext(settings_snapshot)
            set_settings_context(settings_context)

            # Get search strategy from subscription data (for the API call)
            search_strategy = subscription.get(
                "search_strategy", "news_aggregation"
            )

            # Call quick_summary with appropriate parameters
            result = quick_summary(
                query=query,
                research_id=research_id,
                username=username,
                user_password=password,
                settings_snapshot=settings_snapshot,
                search_strategy=search_strategy,
                model_name=subscription.get("model"),
                provider=subscription.get("model_provider"),
                iterations=1,  # Single iteration for news
                metadata=metadata,
                search_original_query=False,  # Don't send long subscription prompts to search engines
            )

            logger.info(
                f"Completed research {research_id} for subscription {subscription['id']}"
            )

            # Store the research result in the database
            self._store_research_result(
                username,
                password,
                research_id,
                subscription["id"],
                result,
                subscription,
            )

        except Exception:
            logger.exception(
                f"Error triggering research for subscription {subscription['id']}"
            )

    def _store_research_result(
        self,
        username: str,
        password: str,
        research_id: str,
        subscription_id: int,
        result: Dict[str, Any],
        subscription: Dict[str, Any],
    ):
        """Store research result in database for news display."""
        try:
            from ..database.session_context import get_user_db_session
            from ..database.models import ResearchHistory
            from ..settings.manager import SettingsManager
            import json

            # Convert result to JSON-serializable format
            def make_serializable(obj):
                """Convert non-serializable objects to dictionaries."""
                if hasattr(obj, "dict"):
                    return obj.dict()
                if hasattr(obj, "__dict__"):
                    return {
                        k: make_serializable(v)
                        for k, v in obj.__dict__.items()
                        if not k.startswith("_")
                    }
                if isinstance(obj, (list, tuple)):
                    return [make_serializable(item) for item in obj]
                if isinstance(obj, dict):
                    return {k: make_serializable(v) for k, v in obj.items()}
                return obj

            serializable_result = make_serializable(result)

            with get_user_db_session(username, password) as db:
                # Get user settings to store in metadata
                settings_manager = SettingsManager(db)
                settings_snapshot = settings_manager.get_settings_snapshot()

                # Get the report content - check both 'report' and 'summary' fields
                report_content = serializable_result.get(
                    "report"
                ) or serializable_result.get("summary")
                logger.debug(
                    f"Report content length: {len(report_content) if report_content else 0} chars"
                )

                # Extract sources/links from the result
                sources = serializable_result.get("sources", [])

                # First add the sources/references section if we have sources
                if report_content and sources:
                    # Import utilities for formatting links
                    from ..utilities.search_utilities import (
                        format_links_to_markdown,
                    )

                    # Format the links/citations
                    formatted_links = format_links_to_markdown(sources)

                    # Add references section to the report
                    if formatted_links:
                        report_content = f"{report_content}\n\n## Sources\n\n{formatted_links}"

                # Then format citations in the report content
                if report_content:
                    # Import citation formatter
                    from ..text_optimization.citation_formatter import (
                        CitationFormatter,
                        CitationMode,
                    )
                    from ..config.search_config import (
                        get_setting_from_snapshot,
                    )

                    # Get citation format from settings
                    citation_format = get_setting_from_snapshot(
                        "report.citation_format", "domain_id_hyperlinks"
                    )
                    mode_map = {
                        "number_hyperlinks": CitationMode.NUMBER_HYPERLINKS,
                        "domain_hyperlinks": CitationMode.DOMAIN_HYPERLINKS,
                        "domain_id_hyperlinks": CitationMode.DOMAIN_ID_HYPERLINKS,
                        "domain_id_always_hyperlinks": CitationMode.DOMAIN_ID_ALWAYS_HYPERLINKS,
                        "source_tagged_hyperlinks": CitationMode.SOURCE_TAGGED_HYPERLINKS,
                        "no_hyperlinks": CitationMode.NO_HYPERLINKS,
                    }
                    mode = mode_map.get(
                        citation_format, CitationMode.DOMAIN_ID_HYPERLINKS
                    )
                    formatter = CitationFormatter(mode=mode)

                    # Format citations within the content
                    report_content = formatter.format_document(report_content)

                if not report_content:
                    # If neither field exists, use the full result as JSON
                    report_content = json.dumps(serializable_result)

                # Generate headline and topics for news searches
                from ..news.utils.headline_generator import generate_headline
                from ..news.utils.topic_generator import generate_topics

                query_text = result.get(
                    "query", subscription.get("query", "News Update")
                )

                # Generate headline from the actual research findings
                logger.info(
                    f"Generating headline for subscription {subscription_id}"
                )
                generated_headline = generate_headline(
                    query=query_text,
                    findings=report_content,
                    max_length=200,  # Allow longer headlines for news
                )

                # Generate topics from the findings
                logger.info(
                    f"Generating topics for subscription {subscription_id}"
                )
                generated_topics = generate_topics(
                    query=query_text,
                    findings=report_content,
                    category=subscription.get("name", "News"),
                    max_topics=6,
                )

                logger.info(
                    f"Generated headline: {generated_headline}, topics: {generated_topics}"
                )

                # Get subscription name for metadata
                subscription_name = subscription.get("name", "")

                # Use generated headline as title, or fallback
                if generated_headline:
                    title = generated_headline
                else:
                    if subscription_name:
                        title = f"{subscription_name} - {datetime.now(UTC).isoformat(timespec='minutes')}"
                    else:
                        title = f"{query_text[:60]}... - {datetime.now(UTC).isoformat(timespec='minutes')}"

                # Create research history entry
                history_entry = ResearchHistory(
                    id=research_id,
                    query=result.get("query", ""),
                    mode="news_subscription",
                    status="completed",
                    created_at=datetime.now(UTC).isoformat(),
                    completed_at=datetime.now(UTC).isoformat(),
                    title=title,
                    research_meta={
                        "subscription_id": subscription_id,
                        "triggered_by": "scheduler",
                        "is_news_search": True,
                        "username": username,
                        "subscription_name": subscription_name,  # Store subscription name for display
                        "settings_snapshot": settings_snapshot,  # Store settings snapshot for later retrieval
                        "generated_headline": generated_headline,  # Store generated headline for news display
                        "generated_topics": generated_topics,  # Store topics for categorization
                    },
                )
                db.add(history_entry)
                db.commit()

                # Store the report content using storage abstraction
                from ..storage import get_report_storage

                # Use storage to save the report (report_content already retrieved above)
                storage = get_report_storage(session=db)
                storage.save_report(
                    research_id=research_id,
                    content=report_content,
                    username=username,
                )

                logger.info(
                    f"Stored research result {research_id} for subscription {subscription_id}"
                )

        except Exception:
            logger.exception("Error storing research result")

    def _run_cleanup_with_tracking(self):
        """Wrapper that tracks cleanup execution."""

        try:
            cleaned_count = self._cleanup_inactive_users()

            logger.info(
                f"Cleanup successful: removed {cleaned_count} inactive users"
            )

        except Exception:
            logger.exception("Cleanup job failed")

    def _cleanup_inactive_users(self) -> int:
        """Remove users inactive for longer than retention period."""
        retention_hours = self.config.get("retention_hours", 48)
        cutoff = datetime.now(UTC) - timedelta(hours=retention_hours)

        cleaned_count = 0

        with self.lock:
            inactive_users = [
                user_id
                for user_id, session in self.user_sessions.items()
                if session["last_activity"] < cutoff
            ]

            for user_id in inactive_users:
                # Remove all scheduled jobs
                for job_id in self.user_sessions[user_id][
                    "scheduled_jobs"
                ].copy():
                    try:
                        self.scheduler.remove_job(job_id)
                    except JobLookupError:
                        pass

                # Clear credentials and session data
                self._credential_store.clear(user_id)
                del self.user_sessions[user_id]
                cleaned_count += 1
                logger.info(f"Cleaned up inactive user {user_id}")

        return cleaned_count

    def _reload_config(self):
        """Reload configuration from settings manager."""
        if not hasattr(self, "settings_manager") or not self.settings_manager:
            return

        try:
            old_retention = self.config.get("retention_hours", 48)

            # Reload all settings
            for key in self.config:
                if key == "enabled":
                    continue  # Don't change enabled state while running

                full_key = f"news.scheduler.{key}"
                self.config[key] = self._get_setting(full_key, self.config[key])

            # Handle changes that need immediate action
            if old_retention != self.config["retention_hours"]:
                logger.info(
                    f"Retention period changed from {old_retention} "
                    f"to {self.config['retention_hours']} hours"
                )
                # Trigger immediate cleanup with new retention
                self.scheduler.add_job(
                    self._wrap_job(self._run_cleanup_with_tracking),
                    "date",
                    run_date=datetime.now(UTC) + timedelta(seconds=5),
                    id="immediate_cleanup_config_change",
                )

            # Clear settings cache to pick up any user setting changes
            self.invalidate_all_settings_cache()

        except Exception:
            logger.exception("Error reloading configuration")

    def get_status(self) -> Dict[str, Any]:
        """Get scheduler status information."""
        with self.lock:
            active_users = len(self.user_sessions)
            total_jobs = sum(
                len(session["scheduled_jobs"])
                for session in self.user_sessions.values()
            )

        # Get next run time for cleanup job
        next_cleanup = None
        if self.is_running:
            job = self.scheduler.get_job("cleanup_inactive_users")
            if job:
                next_cleanup = job.next_run_time

        return {
            "is_running": self.is_running,
            "config": self.config,
            "active_users": active_users,
            "total_scheduled_jobs": total_jobs,
            "next_cleanup": next_cleanup.isoformat() if next_cleanup else None,
            "memory_usage": self._estimate_memory_usage(),
        }

    def _estimate_memory_usage(self) -> int:
        """Estimate memory usage of user sessions."""

        # Rough estimate: username (50) + password (100) + metadata (200) per user
        per_user_estimate = 350
        return len(self.user_sessions) * per_user_estimate

    def get_user_sessions_summary(self) -> List[Dict[str, Any]]:
        """Get summary of active user sessions (without passwords)."""
        with self.lock:
            summary = []
            for user_id, session in self.user_sessions.items():
                summary.append(
                    {
                        "user_id": user_id,
                        "last_activity": session["last_activity"].isoformat(),
                        "scheduled_jobs": len(session["scheduled_jobs"]),
                        "time_since_activity": str(
                            datetime.now(UTC) - session["last_activity"]
                        ),
                    }
                )
            return summary


# Singleton instance getter
_scheduler_instance = None


def get_background_job_scheduler() -> BackgroundJobScheduler:
    """Get the singleton news scheduler instance."""
    global _scheduler_instance
    if _scheduler_instance is None:
        _scheduler_instance = BackgroundJobScheduler()
    return _scheduler_instance
