"""
Routes for Research Library and Download Manager

Provides web endpoints for:
- Library browsing and management
- Download manager interface
- API endpoints for downloads and queries
"""

import json
import math
from io import BytesIO
from pathlib import Path
from flask import (
    Blueprint,
    g,
    jsonify,
    request,
    session,
    Response,
    send_file,
    stream_with_context,
)
from loguru import logger

from ...security.decorators import require_json_body
from ...web.auth.decorators import login_required
from ...web.utils.templates import render_template_with_defaults
from ...database.session_context import get_user_db_session, safe_rollback
from ...database.models.research import ResearchResource
from ...database.models.library import (
    Document as Document,
    DocumentStatus,
    DownloadQueue as LibraryDownloadQueue,
    Collection,
)
from ...library.download_management import ResourceFilter
from ..services.download_service import DownloadService
from ..services.library_service import LibraryService
from ..services.pdf_storage_manager import PDFStorageManager
from ..utils import (
    get_document_for_resource,
    handle_api_error,
    is_downloadable_domain,
    is_downloadable_url,
)
from ...utilities.db_utils import get_settings_manager
from ...config.paths import get_library_directory
from ...web.exceptions import AuthenticationRequiredError

# Create Blueprint
library_bp = Blueprint("library", __name__, url_prefix="/library")

# NOTE: Routes use session["username"] (not .get()) intentionally.
# @login_required guarantees the key exists; direct access fails fast
# if the decorator is ever removed.


# Error handler for authentication errors
@library_bp.errorhandler(Exception)
def handle_web_api_exception(error):
    """Handle WebAPIException and its subclasses."""
    from ...web.exceptions import WebAPIException

    if isinstance(error, WebAPIException):
        return jsonify(error.to_dict()), error.status_code
    # Re-raise other exceptions
    raise error


def get_authenticated_user_password(
    username: str, flask_session_id: str | None = None
) -> str:
    """
    Get authenticated user password from session store with fallback to g.user_password.

    Args:
        username: The username to get password for
        flask_session_id: Optional Flask session ID. If not provided, uses session.get("session_id")

    Returns:
        str: The user's password

    Raises:
        AuthenticationRequiredError: If no password is available for the user
    """
    from ...database.session_passwords import session_password_store

    session_id = flask_session_id or session.get("session_id")

    # Try session password store first
    try:
        user_password = session_password_store.get_session_password(
            username, session_id
        )
        if user_password:
            logger.debug(
                f"Retrieved user password from session store for user {username}"
            )
            return user_password
    except Exception:
        logger.exception("Failed to get user password from session store")

    # Fallback to g.user_password (set by middleware if temp_auth was used)
    user_password = getattr(g, "user_password", None)
    if user_password:
        logger.debug(
            f"Retrieved user password from g.user_password fallback for user {username}"
        )
        return user_password

    # No password available
    logger.error(f"No user password available for user {username}")
    raise AuthenticationRequiredError(
        message="Authentication required: Please refresh the page and log in again to access encrypted database features.",
    )


# ============= Page Routes =============


@library_bp.route("/")
@login_required
def library_page():
    """Main library page showing downloaded documents."""
    username = session["username"]
    service = LibraryService(username)

    # Get library settings
    from ...utilities.db_utils import get_settings_manager

    settings = get_settings_manager()
    pdf_storage_mode = settings.get_setting(
        "research_library.pdf_storage_mode", "database"
    )
    # Enable PDF storage button if mode is not "none"
    enable_pdf_storage = pdf_storage_mode != "none"
    shared_library = settings.get_setting(
        "research_library.shared_library", False
    )

    # Get statistics
    stats = service.get_library_stats()

    # Get documents with optional filters
    domain_filter = request.args.get("domain")
    research_filter = request.args.get("research")
    collection_filter = request.args.get("collection")  # New collection filter
    date_filter = request.args.get("date")

    # Resolve collection_id once to avoid redundant DB lookups
    from ...database.library_init import get_default_library_id

    resolved_collection = collection_filter or get_default_library_id(username)

    # Pagination
    per_page = 100
    total_docs = service.count_documents(
        research_id=research_filter,
        domain=domain_filter,
        collection_id=resolved_collection,
        date_filter=date_filter,
    )
    total_pages = max(1, math.ceil(total_docs / per_page))
    page = request.args.get("page", 1, type=int)
    page = max(1, min(page, total_pages))
    offset = (page - 1) * per_page

    documents = service.get_documents(
        research_id=research_filter,
        domain=domain_filter,
        collection_id=resolved_collection,
        date_filter=date_filter,
        limit=per_page,
        offset=offset,
    )

    # Get unique domains for filter dropdown
    unique_domains = service.get_unique_domains()

    # Get research list for filter dropdown
    research_list = service.get_research_list_for_dropdown()

    # Get collections list for filter dropdown
    collections = service.get_all_collections()

    # Find default library collection ID for semantic search
    default_collection_id = next(
        (c["id"] for c in collections if c.get("is_default")), None
    )

    return render_template_with_defaults(
        "pages/library.html",
        stats=stats,
        documents=documents,
        unique_domains=unique_domains,
        research_list=research_list,
        collections=collections,
        selected_collection=collection_filter,
        default_collection_id=default_collection_id,
        storage_path=stats.get("storage_path", ""),
        enable_pdf_storage=enable_pdf_storage,
        pdf_storage_mode=pdf_storage_mode,
        shared_library=shared_library,
        page=page,
        total_pages=total_pages,
        selected_date=date_filter,
        selected_research=research_filter,
        selected_domain=domain_filter,
    )


@library_bp.route("/document/<string:document_id>")
@login_required
def document_details_page(document_id):
    """Document details page showing all metadata and links."""
    username = session["username"]
    service = LibraryService(username)

    # Get document details
    document = service.get_document_by_id(document_id)

    if not document:
        return "Document not found", 404

    return render_template_with_defaults(
        "pages/document_details.html", document=document
    )


@library_bp.route("/download-manager")
@login_required
def download_manager_page():
    """Download manager page for selecting and downloading research PDFs."""
    username = session["username"]
    service = LibraryService(username)

    # Get library settings
    from ...utilities.db_utils import get_settings_manager

    settings = get_settings_manager()
    pdf_storage_mode = settings.get_setting(
        "research_library.pdf_storage_mode", "database"
    )
    # Enable PDF storage button if mode is not "none"
    enable_pdf_storage = pdf_storage_mode != "none"
    shared_library = settings.get_setting(
        "research_library.shared_library", False
    )

    # Summary stats over ALL sessions (also used for page count)
    per_page = 50
    summary = service.get_download_manager_summary_stats()
    total_pages = max(1, math.ceil(summary["total_researches"] / per_page))

    # Pagination with upper-bound clamp
    page = request.args.get("page", 1, type=int)
    page = max(1, min(page, total_pages))
    offset = (page - 1) * per_page

    # Get paginated research sessions
    research_list = service.get_research_list_with_stats(
        limit=per_page, offset=offset
    )

    # Batch-fetch PDF previews and domain breakdowns (single query)
    research_ids = [r["id"] for r in research_list]
    previews = service.get_pdf_previews_batch(research_ids)
    for research in research_list:
        rid = research["id"]
        data = previews.get(rid, {"pdf_sources": [], "domains": {}})
        research["pdf_sources"] = data["pdf_sources"]
        research["domains"] = data["domains"]

    return render_template_with_defaults(
        "pages/download_manager.html",
        research_list=research_list,
        total_researches=summary["total_researches"],
        total_resources=summary["total_resources"],
        already_downloaded=summary["already_downloaded"],
        available_to_download=summary["available_to_download"],
        enable_pdf_storage=enable_pdf_storage,
        pdf_storage_mode=pdf_storage_mode,
        shared_library=shared_library,
        page=page,
        total_pages=total_pages,
    )


# ============= API Routes =============


@library_bp.route("/api/stats")
@login_required
def get_library_stats():
    """Get library statistics."""
    username = session["username"]
    service = LibraryService(username)
    stats = service.get_library_stats()
    return jsonify(stats)


@library_bp.route("/api/collections/list")
@login_required
def get_collections_list():
    """Get list of all collections for dropdown selection."""
    username = session["username"]

    with get_user_db_session(username) as db_session:
        collections = (
            db_session.query(Collection).order_by(Collection.name).all()
        )

        return jsonify(
            {
                "success": True,
                "collections": [
                    {
                        "id": col.id,
                        "name": col.name,
                        "description": col.description,
                    }
                    for col in collections
                ],
            }
        )


@library_bp.route("/api/documents")
@login_required
def get_documents():
    """Get documents with filtering."""
    username = session["username"]
    service = LibraryService(username)

    # Get filter parameters
    research_id = request.args.get("research_id")
    domain = request.args.get("domain")
    file_type = request.args.get("file_type")
    favorites_only = request.args.get("favorites") == "true"
    search_query = request.args.get("search")
    # Clamp pagination. SQLite treats LIMIT -1 as "no limit", so an
    # unclamped negative limit (e.g. ?limit=-1) would bypass pagination and
    # load the whole collection — including large Document.text_content
    # bodies — into memory (#4560). Mirrors the clamp in history_routes.
    limit = request.args.get("limit", 100, type=int)
    limit = max(1, min(limit, 1000))
    offset = max(0, request.args.get("offset", 0, type=int))

    documents = service.get_documents(
        research_id=research_id,
        domain=domain,
        file_type=file_type,
        favorites_only=favorites_only,
        search_query=search_query,
        limit=limit,
        offset=offset,
    )

    return jsonify({"documents": documents})


@library_bp.route(
    "/api/document/<string:document_id>/favorite", methods=["POST"]
)
@login_required
def toggle_favorite(document_id):
    """Toggle favorite status of a document."""
    username = session["username"]
    service = LibraryService(username)
    is_favorite = service.toggle_favorite(document_id)
    return jsonify({"favorite": is_favorite})


@library_bp.route("/api/document/<string:document_id>", methods=["DELETE"])
@login_required
def delete_document(document_id):
    """Delete a document from library."""
    username = session["username"]
    service = LibraryService(username)
    success = service.delete_document(document_id)
    return jsonify({"success": success})


@library_bp.route("/api/document/<string:document_id>/pdf-url")
@login_required
def get_pdf_url(document_id):
    """Get URL for viewing PDF."""
    # Return URL that will serve the PDF
    return jsonify(
        {
            "url": f"/library/api/document/{document_id}/pdf",
            "title": "Document",  # Could fetch actual title
        }
    )


@library_bp.route("/document/<string:document_id>/pdf")
@login_required
def view_pdf_page(document_id):
    """Page for viewing PDF file - uses PDFStorageManager for retrieval."""
    username = session["username"]

    with get_user_db_session(username) as db_session:
        # Get document from database
        document = db_session.query(Document).filter_by(id=document_id).first()

        if not document:
            logger.warning(
                f"Document ID {document_id} not found in database for user {username}"
            )
            return "Document not found", 404

        logger.info(
            f"Document {document_id}: title='{document.title}', "
            f"file_path={document.file_path}"
        )

        # Get settings for PDF storage manager
        settings = get_settings_manager(db_session)
        storage_mode = settings.get_setting(
            "research_library.pdf_storage_mode", "none"
        )
        library_root = (
            Path(
                settings.get_setting(
                    "research_library.storage_path",
                    str(get_library_directory()),
                )
            )
            .expanduser()
            .resolve()
        )

        # Use PDFStorageManager to load PDF (handles database and filesystem)
        pdf_manager = PDFStorageManager(library_root, storage_mode)
        pdf_bytes = pdf_manager.load_pdf(document, db_session)

        if pdf_bytes:
            logger.info(
                f"Serving PDF for document {document_id} ({len(pdf_bytes)} bytes)"
            )
            return send_file(
                BytesIO(pdf_bytes),
                mimetype="application/pdf",
                as_attachment=False,
                download_name=document.filename or "document.pdf",
            )

        # No PDF found anywhere
        logger.warning(f"No PDF available for document {document_id}")
        return "PDF not available", 404


@library_bp.route("/api/document/<string:document_id>/pdf")
@login_required
def serve_pdf_api(document_id):
    """API endpoint for serving PDF file (kept for backward compatibility)."""
    return view_pdf_page(document_id)


@library_bp.route("/document/<string:document_id>/txt")
@login_required
def view_text_page(document_id):
    """Page for viewing text content."""
    username = session["username"]

    with get_user_db_session(username) as db_session:
        # Get document by ID (text now stored in Document.text_content)
        document = db_session.query(Document).filter_by(id=document_id).first()

        if not document:
            logger.warning(f"Document not found for document ID {document_id}")
            return "Document not found", 404

        if not document.text_content:
            logger.warning(f"Document {document_id} has no text content")
            return "Text content not available", 404

        logger.info(
            f"Serving text content for document {document_id}: {len(document.text_content)} characters"
        )

        # Render as HTML page
        return render_template_with_defaults(
            "pages/document_text.html",
            document_id=document_id,
            title=document.title or "Document Text",
            text_content=document.text_content,
            extraction_method=document.extraction_method,
            word_count=document.word_count,
        )


@library_bp.route("/api/document/<string:document_id>/text")
@login_required
def serve_text_api(document_id):
    """API endpoint for serving text content (kept for backward compatibility)."""
    username = session["username"]

    with get_user_db_session(username) as db_session:
        # Get document by ID (text now stored in Document.text_content)
        document = db_session.query(Document).filter_by(id=document_id).first()

        if not document:
            logger.warning(f"Document not found for document ID {document_id}")
            return jsonify({"error": "Document not found"}), 404

        if not document.text_content:
            logger.warning(f"Document {document_id} has no text content")
            return jsonify({"error": "Text content not available"}), 404

        logger.info(
            f"Serving text content for document {document_id}: {len(document.text_content)} characters"
        )

        return jsonify(
            {
                "text_content": document.text_content,
                "title": document.title or "Document",
                "extraction_method": document.extraction_method,
                "word_count": document.word_count,
            }
        )


@library_bp.route("/api/open-folder", methods=["POST"])
@login_required
def open_folder():
    """Open folder containing a document.

    Security: This endpoint is disabled for server deployments.
    It only makes sense for desktop usage where the server and client are on the same machine.
    """
    return jsonify(
        {
            "status": "error",
            "message": "This feature is disabled. It is only available in desktop mode.",
        }
    ), 403


@library_bp.route("/api/download/<int:resource_id>", methods=["POST"])
@login_required
def download_single_resource(resource_id):
    """Download a single resource."""
    username = session["username"]
    user_password = get_authenticated_user_password(username)

    with DownloadService(username, user_password) as service:
        success, error = service.download_resource(resource_id)
        if success:
            return jsonify({"success": True})
        logger.warning(f"Download failed for resource {resource_id}: {error}")
        return jsonify(
            {
                "success": False,
                "error": "Download failed. Please try again or contact support.",
            }
        ), 500


@library_bp.route("/api/download-text/<int:resource_id>", methods=["POST"])
@login_required
def download_text_single(resource_id):
    """Download a single resource as text file."""
    try:
        username = session["username"]
        user_password = get_authenticated_user_password(username)

        with DownloadService(username, user_password) as service:
            success, error = service.download_as_text(resource_id)

            # Sanitize error message - don't expose internal details
            if not success:
                if error:
                    logger.warning(
                        f"Download as text failed for resource {resource_id}: {error}"
                    )
                return jsonify(
                    {"success": False, "error": "Failed to download resource"}
                )

            return jsonify({"success": True, "error": None})
    except AuthenticationRequiredError:
        raise  # Let blueprint error handler return 401
    except Exception as e:
        return handle_api_error(
            f"downloading resource {resource_id} as text", e
        )


@library_bp.route("/api/download-all-text", methods=["POST"])
@login_required
def download_all_text():
    """Download all undownloaded resources as text files."""
    username = session["username"]
    # Capture Flask session ID to avoid scoping issues in nested function
    flask_session_id = session.get("session_id")

    def generate():
        # Get user password for database operations
        try:
            user_password = get_authenticated_user_password(
                username, flask_session_id
            )
        except AuthenticationRequiredError:
            logger.warning(
                f"Authentication unavailable for user {username} - password not in session store"
            )
            yield f"data: {json.dumps({'progress': 0, 'current': 0, 'total': 0, 'error': 'Authentication required', 'complete': True})}\n\n"
            return

        download_service = DownloadService(username, user_password)
        try:
            # Get all undownloaded resources
            with get_user_db_session(username) as session:
                # Project only the columns the loop uses (id/url/title) rather
                # than loading full ``ResearchResource`` entities — the
                # ``content_preview`` Text column would otherwise materialize
                # for every resource on this whole-table scan (#4560).
                all_resources = session.query(
                    ResearchResource.id,
                    ResearchResource.url,
                    ResearchResource.title,
                ).all()
                # Filter to only downloadable resources (academic/PDF)
                resources = [
                    r for r in all_resources if is_downloadable_url(r.url)
                ]

                # Filter resources that need text extraction
                txt_path = Path(download_service.library_root) / "txt"
                resources_to_process = []

                # Pre-scan directory once to get all existing resource IDs
                existing_resource_ids = set()
                if txt_path.exists():
                    for txt_file in txt_path.glob("*.txt"):
                        # Extract resource ID from filename pattern *_{id}.txt
                        parts = txt_file.stem.rsplit("_", 1)
                        if len(parts) == 2:
                            try:
                                existing_resource_ids.add(int(parts[1]))
                            except ValueError:
                                pass

                for resource in resources:
                    # Check if text file already exists using preloaded set
                    if resource.id not in existing_resource_ids:
                        resources_to_process.append(resource)

                total = len(resources_to_process)
                current = 0

                logger.info(f"Found {total} resources needing text extraction")

                for resource in resources_to_process:
                    current += 1
                    progress = (
                        int((current / total) * 100) if total > 0 else 100
                    )

                    file_name = (
                        resource.title[:50]
                        if resource
                        else f"document_{current}.txt"
                    )

                    try:
                        success, error = download_service.download_as_text(
                            resource.id
                        )

                        if success:
                            status = "success"
                            error_msg = None
                        else:
                            status = "failed"
                            error_msg = error or "Text extraction failed"

                    except Exception as e:
                        logger.exception(
                            f"Error extracting text for resource {resource.id}"
                        )
                        status = "failed"
                        error_msg = (
                            f"Text extraction failed - {type(e).__name__}"
                        )

                    # Send update
                    update = {
                        "progress": progress,
                        "current": current,
                        "total": total,
                        "file": file_name,
                        "url": resource.url,  # Add the URL for UI display
                        "status": status,
                        "error": error_msg,
                    }
                    yield f"data: {json.dumps(update)}\n\n"

                # Send completion
                yield f"data: {json.dumps({'complete': True, 'total': total})}\n\n"
        finally:
            from ...utilities.resource_utils import safe_close

            safe_close(download_service, "download service")

    return Response(
        stream_with_context(generate()), mimetype="text/event-stream"
    )


@library_bp.route("/api/download-research/<research_id>", methods=["POST"])
@login_required
def download_research_pdfs(research_id):
    """Queue all PDFs from a research session for download."""
    username = session["username"]
    user_password = get_authenticated_user_password(username)

    with DownloadService(username, user_password) as service:
        # Get optional collection_id from request body
        data = request.json or {}
        collection_id = data.get("collection_id")

        queued = service.queue_research_downloads(research_id, collection_id)

        # Start processing queue (in production, this would be a background task)
        # For now, we'll process synchronously
        # TODO: Integrate with existing queue processor

        return jsonify({"success": True, "queued": queued})


@library_bp.route("/api/download-bulk", methods=["POST"])
@login_required
@require_json_body()
def download_bulk():
    """Download PDFs or extract text from multiple research sessions."""
    username = session["username"]
    data = request.json
    research_ids = data.get("research_ids", [])
    mode = data.get("mode", "pdf")  # pdf or text_only
    collection_id = data.get(
        "collection_id"
    )  # Optional: target collection for downloads

    if not research_ids:
        return jsonify({"error": "No research IDs provided"}), 400

    # Capture Flask session ID to avoid scoping issues in nested function
    flask_session_id = session.get("session_id")

    def generate():
        """Generate progress updates as Server-Sent Events."""
        # Get user password for database operations
        try:
            user_password = get_authenticated_user_password(
                username, flask_session_id
            )
        except AuthenticationRequiredError:
            logger.warning(
                f"Authentication unavailable for user {username} - password not in session store"
            )
            yield f"data: {json.dumps({'progress': 0, 'current': 0, 'total': 0, 'error': 'Authentication required', 'complete': True})}\n\n"
            return

        download_service = DownloadService(username, user_password)
        try:
            total = 0
            current = 0
            queue_failures = 0

            # Pre-queue and count in one merged loop. queue_research_downloads
            # opens its own session, commits, and handles dedup internally:
            # it skips resources that are already PENDING or already backed by
            # a COMPLETED Document, and re-queues any remaining row (e.g. a
            # prior FAILED attempt) back to PENDING. Calling it BEFORE counting
            # ensures the total reflects what will actually be processed
            # (issue #4660).
            for research_id in research_ids:
                try:
                    download_service.queue_research_downloads(
                        research_id, collection_id
                    )
                except Exception:
                    queue_failures += 1
                    logger.exception(
                        f"Error queueing downloads for "
                        f"user={username} research={research_id} "
                        f"collection={collection_id}"
                    )

                with get_user_db_session(username) as session:
                    count = (
                        session.query(LibraryDownloadQueue)
                        .filter_by(
                            research_id=research_id,
                            status=DocumentStatus.PENDING,
                        )
                        .count()
                    )
                total += count
                logger.debug(
                    f"Research {research_id}: {count} pending items in queue"
                )

            logger.info(f"Total pending downloads across all research: {total}")
            yield f"data: {json.dumps({'progress': 0, 'current': 0, 'total': total})}\n\n"

            # If the pre-pass yielded nothing to process, surface it as an
            # error so the UI alerts instead of silently completing with
            # "0 / 0 files" success. Distinguish queue failure from a
            # legitimate "nothing left to download" state.
            if total == 0:
                if queue_failures > 0:
                    error_msg = (
                        f"Download failed to start: queueing failed for "
                        f"{queue_failures} of {len(research_ids)} "
                        f"research session(s). Check server logs."
                    )
                else:
                    error_msg = (
                        "No new papers to download for the selected "
                        "research session(s)."
                    )
                yield f"data: {json.dumps({'progress': 100, 'current': 0, 'total': 0, 'complete': True, 'error': error_msg})}\n\n"
                return

            # Process each research
            for research_id in research_ids:
                # Get queued downloads for this research
                with get_user_db_session(username) as session:
                    # Get pending queue items for this research (pre-pass
                    # above already ensured items are queued)
                    queue_items = (
                        session.query(LibraryDownloadQueue)
                        .filter_by(
                            research_id=research_id,
                            status=DocumentStatus.PENDING,
                        )
                        .all()
                    )

                    # Process each queued item
                    for queue_item in queue_items:
                        current += 1

                        progress = (
                            int((current / total) * 100) if total > 0 else 100
                        )

                        # Get resource info
                        resource = session.query(ResearchResource).get(
                            queue_item.resource_id
                        )
                        file_name = (
                            resource.title[:50]
                            if resource
                            else f"document_{current}.pdf"
                        )

                        # Attempt actual download with error handling
                        skip_reason = None
                        status = "skipped"  # Default to skipped
                        success = False
                        error_msg = None

                        try:
                            logger.debug(
                                f"Attempting {'PDF download' if mode == 'pdf' else 'text extraction'} for resource {queue_item.resource_id}"
                            )

                            # Call appropriate service method based on mode
                            if mode == "pdf":
                                result = download_service.download_resource(
                                    queue_item.resource_id
                                )
                            else:  # text_only
                                result = download_service.download_as_text(
                                    queue_item.resource_id
                                )

                            # Handle new tuple return format
                            if isinstance(result, tuple):
                                success, skip_reason = result
                            else:
                                success = result
                                skip_reason = None

                            status = "success" if success else "skipped"
                            if skip_reason and not success:
                                error_msg = skip_reason
                                logger.info(
                                    f"{'Download' if mode == 'pdf' else 'Text extraction'} skipped for resource {queue_item.resource_id}: {skip_reason}"
                                )

                            logger.debug(
                                f"{'Download' if mode == 'pdf' else 'Text extraction'} result: success={success}, status={status}, skip_reason={skip_reason}"
                            )
                        except Exception as e:
                            # Roll back FIRST: the next loop iteration's
                            # session.query(ResearchResource).get(...) at the
                            # top of the for-body runs BEFORE the next
                            # try/except, so a poisoned session here would
                            # cascade into PendingRollbackError on the next
                            # item before this handler ever runs again
                            # (issue #3827).
                            safe_rollback(session, "SSE download")
                            # Log error but continue processing
                            error_msg = str(e)
                            error_type = type(e).__name__
                            logger.warning(
                                f"CAUGHT Download exception for resource {queue_item.resource_id}: {error_type}: {error_msg}"
                            )
                            # Check if this is a skip reason (not a real error)
                            # Use error category + categorized message for user display
                            if any(
                                phrase in error_msg.lower()
                                for phrase in [
                                    "paywall",
                                    "subscription",
                                    "not available",
                                    "not found",
                                    "no free",
                                    "embargoed",
                                    "forbidden",
                                    "not accessible",
                                ]
                            ):
                                status = "skipped"
                                skip_reason = f"Document not accessible (paywall or access restriction) - {error_type}"
                            elif any(
                                phrase in error_msg.lower()
                                for phrase in [
                                    "failed to download",
                                    "could not",
                                    "invalid",
                                    "server",
                                ]
                            ):
                                status = "failed"
                                skip_reason = f"Download failed - {error_type}"
                            else:
                                status = "failed"
                                skip_reason = (
                                    f"Processing failed - {error_type}"
                                )
                            success = False

                        # Ensure skip_reason is set if we have an error message
                        if error_msg and not skip_reason:
                            skip_reason = f"Processing failed - {error_type}"
                            logger.debug(
                                f"Setting skip_reason from error_msg: {error_msg}"
                            )

                        # Send progress update
                        update_data = {
                            "progress": progress,
                            "current": current,
                            "total": total,
                            "file": file_name,
                            "status": status,
                        }
                        # Add skip reason if available
                        if skip_reason:
                            update_data["error"] = skip_reason
                            logger.info(
                                f"Sending skip reason to UI: {skip_reason}"
                            )

                        yield f"data: {json.dumps(update_data)}\n\n"

            yield f"data: {json.dumps({'progress': 100, 'current': current, 'total': total, 'complete': True})}\n\n"
        finally:
            from ...utilities.resource_utils import safe_close

            safe_close(download_service, "download service")

    return Response(
        stream_with_context(generate()),
        mimetype="text/event-stream",
        headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
    )


@library_bp.route("/api/research-list")
@login_required
def get_research_list():
    """Get list of research sessions for dropdowns."""
    username = session["username"]
    service = LibraryService(username)
    research_list = service.get_research_list_for_dropdown()
    return jsonify({"research": research_list})


@library_bp.route("/api/sync-library", methods=["POST"])
@login_required
def sync_library():
    """Sync library database with filesystem."""
    username = session["username"]
    service = LibraryService(username)
    stats = service.sync_library_with_filesystem()
    return jsonify(stats)


@library_bp.route("/api/mark-redownload", methods=["POST"])
@login_required
@require_json_body()
def mark_for_redownload():
    """Mark documents for re-download."""
    username = session["username"]
    service = LibraryService(username)

    data = request.json
    document_ids = data.get("document_ids", [])

    if not document_ids:
        return jsonify({"error": "No document IDs provided"}), 400

    count = service.mark_for_redownload(document_ids)
    return jsonify({"success": True, "marked": count})


@library_bp.route("/api/queue-all-undownloaded", methods=["POST"])
@login_required
def queue_all_undownloaded():
    """Queue all articles that haven't been downloaded yet."""
    username = session["username"]

    logger.info(f"queue_all_undownloaded called for user {username}")

    with get_user_db_session(username) as db_session:
        # Find all resources that don't have a completed download. Project only
        # the columns the filter/loop use (id/url/research_id) instead of full
        # ``ResearchResource`` entities so the ``content_preview`` Text column
        # isn't materialized for every row on this whole-table scan (#4560).
        undownloaded = (
            db_session.query(
                ResearchResource.id,
                ResearchResource.url,
                ResearchResource.research_id,
            )
            .outerjoin(
                Document,
                (
                    (ResearchResource.id == Document.resource_id)
                    | (ResearchResource.document_id == Document.id)
                )
                & (Document.status == "completed"),
            )
            .filter(Document.id.is_(None))
            .all()
        )

        logger.info(f"Found {len(undownloaded)} total undownloaded resources")

        # Get user password for encrypted database access
        user_password = get_authenticated_user_password(username)

        resource_filter = ResourceFilter(username, user_password)
        filter_results = resource_filter.filter_downloadable_resources(
            undownloaded
        )

        # Get detailed filtering summary
        filter_summary = resource_filter.get_filter_summary(undownloaded)
        skipped_info = resource_filter.get_skipped_resources_info(undownloaded)

        logger.info(f"Filter results: {filter_summary.to_dict()}")

        queued_count = 0
        research_ids = set()
        skipped_count = 0

        # Convert filter_results to dict for O(1) lookup instead of O(n²)
        filter_results_by_id = {r.resource_id: r for r in filter_results}

        for resource in undownloaded:
            # Check if resource passed the smart filter
            filter_result = filter_results_by_id.get(resource.id)

            if not filter_result or not filter_result.can_retry:
                skipped_count += 1
                if filter_result:
                    logger.debug(
                        f"Skipping resource {resource.id} due to retry policy: {filter_result.reason}"
                    )
                else:
                    logger.debug(
                        f"Skipping resource {resource.id} - no filter result available"
                    )
                continue

            # Check if it's downloadable using proper URL parsing
            if not resource.url:
                skipped_count += 1
                continue

            is_downloadable = is_downloadable_domain(resource.url)

            # Log what we're checking
            if resource.url and "pubmed" in resource.url.lower():
                logger.info(f"Found PubMed URL: {resource.url[:100]}")

            if not is_downloadable:
                skipped_count += 1
                logger.debug(
                    f"Skipping non-downloadable URL: {resource.url[:100] if resource.url else 'None'}"
                )
                continue

            # Check if already in queue (any status)
            existing_queue = (
                db_session.query(LibraryDownloadQueue)
                .filter_by(resource_id=resource.id)
                .first()
            )

            if existing_queue:
                # If it exists but isn't pending, reset it to pending
                if existing_queue.status != DocumentStatus.PENDING:
                    existing_queue.status = DocumentStatus.PENDING
                    existing_queue.completed_at = None
                    queued_count += 1
                    research_ids.add(resource.research_id)
                    logger.debug(
                        f"Reset queue entry for resource {resource.id} to pending"
                    )
                else:
                    # Already pending, still count it
                    queued_count += 1
                    research_ids.add(resource.research_id)
                    logger.debug(
                        f"Resource {resource.id} already pending in queue"
                    )
            else:
                # Add new entry to queue
                queue_entry = LibraryDownloadQueue(
                    resource_id=resource.id,
                    research_id=resource.research_id,
                    priority=0,
                    status=DocumentStatus.PENDING,
                )
                db_session.add(queue_entry)
                queued_count += 1
                research_ids.add(resource.research_id)
                logger.debug(
                    f"Added new queue entry for resource {resource.id}"
                )

        db_session.commit()

        logger.info(
            f"Queued {queued_count} articles for download, skipped {skipped_count} resources (including {filter_summary.permanently_failed_count} permanently failed and {filter_summary.temporarily_failed_count} temporarily failed)"
        )

        # Note: Removed synchronous download processing here to avoid blocking the HTTP request
        # Downloads will be processed via the SSE streaming endpoint or background tasks

        return jsonify(
            {
                "success": True,
                "queued": queued_count,
                "research_ids": list(research_ids),
                "total_undownloaded": len(undownloaded),
                "skipped": skipped_count,
                "filter_summary": filter_summary.to_dict(),
                "skipped_details": skipped_info,
            }
        )


@library_bp.route("/api/get-research-sources/<research_id>", methods=["GET"])
@login_required
def get_research_sources(research_id):
    """Get all sources for a research with snippets."""
    username = session["username"]

    sources = []
    with get_user_db_session(username) as db_session:
        # Get all resources for this research
        resources = (
            db_session.query(ResearchResource)
            .filter_by(research_id=research_id)
            .order_by(ResearchResource.created_at)
            .all()
        )

        for idx, resource in enumerate(resources, 1):
            # Check if document exists
            document = get_document_for_resource(db_session, resource)

            # Get domain from URL
            domain = ""
            if resource.url:
                try:
                    from urllib.parse import urlparse

                    domain = urlparse(resource.url).hostname or ""
                except (ValueError, AttributeError):
                    # urlparse can raise ValueError for malformed URLs
                    pass

            source_data = {
                "number": idx,
                "resource_id": resource.id,
                "url": resource.url,
                "title": resource.title or f"Source {idx}",
                "snippet": resource.content_preview or "",
                "domain": domain,
                "relevance_score": getattr(resource, "relevance_score", None),
                "downloaded": False,
                "document_id": None,
                "file_type": None,
            }

            if document and document.status == "completed":
                source_data.update(
                    {
                        "downloaded": True,
                        "document_id": document.id,
                        "file_type": document.file_type,
                        "download_date": document.created_at.isoformat()
                        if document.created_at
                        else None,
                    }
                )

            sources.append(source_data)

    return jsonify({"success": True, "sources": sources, "total": len(sources)})


@library_bp.route("/api/check-downloads", methods=["POST"])
@login_required
@require_json_body()
def check_downloads():
    """Check download status for a list of URLs."""
    username = session["username"]
    data = request.json
    research_id = data.get("research_id")
    urls = data.get("urls", [])

    if not research_id or not urls:
        return jsonify({"error": "Missing research_id or urls"}), 400

    download_status = {}

    with get_user_db_session(username) as db_session:
        # Get all resources for this research
        resources = (
            db_session.query(ResearchResource)
            .filter_by(research_id=research_id)
            .filter(ResearchResource.url.in_(urls))
            .all()
        )

        for resource in resources:
            # Check if document exists
            document = get_document_for_resource(db_session, resource)

            if document and document.status == "completed":
                download_status[resource.url] = {
                    "downloaded": True,
                    "document_id": document.id,
                    "file_type": document.file_type,
                    "title": document.title or resource.title,
                }
            else:
                download_status[resource.url] = {
                    "downloaded": False,
                    "resource_id": resource.id,
                }

    return jsonify({"download_status": download_status})


@library_bp.route("/api/download-source", methods=["POST"])
@login_required
@require_json_body()
def download_source():
    """Download a single source from a research."""
    username = session["username"]
    user_password = get_authenticated_user_password(username)
    data = request.json
    research_id = data.get("research_id")
    url = data.get("url")

    if not research_id or not url:
        return jsonify({"error": "Missing research_id or url"}), 400

    # Check if URL is downloadable
    if not is_downloadable_domain(url):
        return jsonify({"error": "URL is not from a downloadable domain"}), 400

    with get_user_db_session(username) as db_session:
        # Find the resource
        resource = (
            db_session.query(ResearchResource)
            .filter_by(research_id=research_id, url=url)
            .first()
        )

        if not resource:
            return jsonify({"error": "Resource not found"}), 404

        # Check if already downloaded
        existing = get_document_for_resource(db_session, resource)

        if existing and existing.status == "completed":
            return jsonify(
                {
                    "success": True,
                    "message": "Already downloaded",
                    "document_id": existing.id,
                }
            )

        # Add to download queue
        queue_entry = (
            db_session.query(LibraryDownloadQueue)
            .filter_by(resource_id=resource.id)
            .first()
        )

        if not queue_entry:
            queue_entry = LibraryDownloadQueue(
                resource_id=resource.id,
                research_id=resource.research_id,
                priority=1,  # Higher priority for manual downloads
                status=DocumentStatus.PENDING,
            )
            db_session.add(queue_entry)
        else:
            queue_entry.status = DocumentStatus.PENDING
            queue_entry.priority = 1

        db_session.commit()

        # Start download immediately
        with DownloadService(username, user_password) as service:
            success, message = service.download_resource(resource.id)

            if success:
                return jsonify(
                    {"success": True, "message": "Download completed"}
                )
            # Log internal message, but show only generic message to user
            return jsonify({"success": False, "message": "Download failed"})
