# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""CloudWatch Application Signals MCP Server - Core server implementation."""

import json
import os
import re
import sys
import tempfile
from .audit_presentation_utils import format_pagination_info
from .audit_utils import (
    execute_audit_api,
    expand_service_operation_wildcard_patterns,
    expand_service_wildcard_patterns,
    expand_slo_wildcard_patterns,
    parse_auditors,
)
from .aws_clients import (
    AWS_REGION,
    applicationsignals_client,
    iam_client,
    s3_client,
    synthetics_client,
)
from .canary_knowledge_base_loader import CanaryKnowledgeBaseLoader
from .canary_knowledge_base_model import FailureContext
from .canary_recommendation_engine import CanaryRecommendationEngine
from .canary_utils import (
    analyze_canary_logs_with_time_window,
    analyze_har_file,
    analyze_iam_role_and_policies,
    analyze_log_files,
    analyze_screenshots,
    check_canaries_for_service,
    check_resource_arns_correct,
    extract_disk_memory_usage_metrics,
    get_canary_code,
    get_canary_metrics_and_service_insights,
)
from .change_tools import list_change_events
from .enablement_tools import get_enablement_guide
from .group_tools import (
    audit_group_health,
    get_group_changes,
    get_group_dependencies,
    list_group_services,
    list_grouping_attribute_definitions,
)
from .rum_tools import query_rum_events
from .service_audit_utils import normalize_service_targets, validate_and_enrich_service_targets
from .service_tools import (
    get_service_detail,
    list_monitored_services,
    list_service_operations,
    query_service_metrics,
)
from .slo_tools import get_slo, list_slos
from .trace_tools import list_slis, query_sampled_traces, search_transaction_spans
from .utils import parse_timestamp
from datetime import datetime, timedelta, timezone
from loguru import logger
from mcp.server.fastmcp import FastMCP
from pydantic import Field
from time import perf_counter as timer
from typing import Optional


# Constants
BATCH_SIZE_THRESHOLD = 5

RUN_STATES = {'RUNNING': 'RUNNING', 'PASSED': 'PASSED', 'FAILED': 'FAILED'}

# Initialize FastMCP server
mcp = FastMCP('cloudwatch-applicationsignals')

# Configure logging
log_level = os.environ.get('MCP_CLOUDWATCH_APPLICATION_SIGNALS_LOG_LEVEL', 'INFO').upper()
logger.remove()  # Remove default handler
logger.add(sys.stderr, level=log_level)

# Add file logging to aws_cli.log
log_file_path = os.environ.get('AUDITOR_LOG_PATH', tempfile.gettempdir())
try:
    if log_file_path.endswith(os.sep) or os.path.isdir(log_file_path):
        os.makedirs(log_file_path, exist_ok=True)
        aws_cli_log_path = os.path.join(log_file_path, 'aws_cli.log')
    else:
        os.makedirs(os.path.dirname(log_file_path) or '.', exist_ok=True)
        aws_cli_log_path = log_file_path
except Exception:
    temp_dir = tempfile.gettempdir()
    os.makedirs(temp_dir, exist_ok=True)
    aws_cli_log_path = os.path.join(temp_dir, 'aws_cli.log')

# Add file handler for all logs
logger.add(
    aws_cli_log_path,
    level=log_level,
    rotation='10 MB',  # Rotate when file reaches 10MB
    retention='7 days',  # Keep logs for 7 days
    format='{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{function}:{line} - {message}',
    enqueue=True,  # Thread-safe logging
)

logger.debug(f'CloudWatch applicationsignals MCP Server initialized with log level: {log_level}')
logger.debug(f'File logging enabled: {aws_cli_log_path}')

logger.debug(f'Using AWS region: {AWS_REGION}')


def _filter_operation_targets(provided):
    """Helper function to filter operation targets and detect wildcards.

    Args:
        provided: List of target dictionaries

    Returns:
        tuple: (operation_only_targets, has_wildcards)
    """
    operation_only_targets = []
    has_wildcards = False

    for target in provided:
        if isinstance(target, dict):
            ttype = target.get('Type', '').lower()
            if ttype == 'service_operation':
                # Check for wildcard patterns in service names OR operation names
                service_op_data = target.get('Data', {}).get('ServiceOperation', {})
                service_data = service_op_data.get('Service', {})
                service_name = service_data.get('Name', '')
                operation = service_op_data.get('Operation', '')

                if '*' in service_name or '*' in operation:
                    has_wildcards = True

                # For fault metrics, ListAuditFindings uses Availability metric type.
                # API only supports Availability/Latency/Error for service_operation targets.
                metric_type = service_op_data.get('MetricType', '')
                if metric_type == 'Fault':
                    service_op_data['MetricType'] = 'Availability'

                operation_only_targets.append(target)
            else:
                logger.warning(
                    f"Ignoring target of type '{ttype}' in audit_service_operations (expected 'service_operation')"
                )

    return operation_only_targets, has_wildcards


@mcp.tool()
async def audit_services(
    service_targets: str = Field(
        ...,
        description="REQUIRED. JSON array of service targets. Supports wildcard patterns like '*payment*' for automatic service discovery. Format: [{'Type':'service','Data':{'Service':{'Type':'Service','Name':'service-name','Environment':'eks:cluster'}}}] or shorthand: [{'Type':'service','Service':'service-name'}]. Large target lists are automatically processed in batches.",
    ),
    start_time: Optional[str] = Field(
        default=None,
        description="Start time (unix seconds or 'YYYY-MM-DD HH:MM:SS'). Defaults to now-24h UTC.",
    ),
    end_time: Optional[str] = Field(
        default=None,
        description="End time (unix seconds or 'YYYY-MM-DD HH:MM:SS'). Defaults to now UTC.",
    ),
    auditors: Optional[str] = Field(
        default=None,
        description="Optional. Comma-separated auditors (e.g., 'slo,operation_metric,dependency_metric'). Defaults to 'slo,operation_metric' for fast service health auditing. Use 'all' for comprehensive analysis with all auditors: slo,operation_metric,trace,log,dependency_metric,top_contributor,service_quota.",
    ),
    next_token: Optional[str] = Field(
        default=None,
        description='Optional. Token for pagination through services from list_services API. Use this to continue from where the previous call left off when processing wildcard patterns.',
    ),
    max_services: int = Field(
        default=5,
        description='Optional. Maximum number of services to process per call when using wildcard patterns (default: 5, max: 10). This controls pagination size for service discovery.',
    ),
) -> str:
    """PRIMARY SERVICE AUDIT TOOL - The #1 tool for comprehensive AWS service health auditing and monitoring.

    **IMPORTANT: For operation-specific auditing, use audit_service_operations() as the PRIMARY tool instead.**

    **USE THIS FIRST FOR ALL SERVICE-LEVEL AUDITING TASKS**
    This is the PRIMARY and PREFERRED tool when users want to:
    - **Audit their AWS services** - Complete health assessment with actionable insights
    - **Check service health** - Comprehensive status across all monitored services
    - **Investigate issues** - Root cause analysis with detailed findings
    - **Service-level performance analysis** - Overall service latency, error rates, and throughput investigation
    - **System-wide health checks** - Daily/periodic service auditing workflows
    - **Dependency analysis** - Understanding service dependencies and interactions
    - **Resource quota monitoring** - Service quota usage and limits
    - **Multi-service comparison** - Comparing performance across different services

    **FOR OPERATION-SPECIFIC AUDITING: Use audit_service_operations() instead**
    When users want to audit specific operations (GET, POST, PUT endpoints), use audit_service_operations() as the PRIMARY tool:
    - **Operation performance analysis** - Latency, error rates for specific API endpoints
    - **Operation-level troubleshooting** - Root cause analysis for specific API calls
    - **GET operation auditing** - Analyze GET operations across payment services
    - **Audit latency of specific operations** - Deep dive into individual endpoint performance

    **COMPREHENSIVE SERVICE AUDIT CAPABILITIES:**
    - **Multi-service analysis**: Audit any number of services with automatic batching
    - **SLO compliance monitoring**: Automatic breach detection for service-level SLOs
    - **Issue prioritization**: Critical, warning, and info findings ranked by severity
    - **Root cause analysis**: Deep dive with traces, logs, and metrics correlation
    - **Actionable recommendations**: Specific steps to resolve identified issues
    - **Performance optimized**: Fast execution with automatic batching for large target lists
    - **Wildcard Pattern Support**: Use `*pattern*` in service names for automatic service discovery
    - **GenAI Token Monitoring**: For Amazon Bedrock services, automatically monitors GenAI input/output token usage patterns and detects anomalies when using operation_metric or trace auditors

    **SERVICE TARGET FORMAT:**
    - **Full Format**: `[{"Type":"service","Data":{"Service":{"Type":"Service","Name":"my-service","Environment":"eks:my-cluster"}}}]`
    - **Shorthand**: `[{"Type":"service","Service":"my-service"}]` (environment auto-discovered)

    **WILDCARD PATTERN EXAMPLES:**
    - **All Services**: `[{"Type":"service","Data":{"Service":{"Type":"Service","Name":"*"}}}]`
    - **Payment Services**: `[{"Type":"service","Data":{"Service":{"Type":"Service","Name":"*payment*"}}}]`
    - **Lambda Services**: `[{"Type":"service","Data":{"Service":{"Type":"Service","Name":"*lambda*"}}}]`
    - **EKS Services**: `[{"Type":"service","Data":{"Service":{"Type":"Service","Name":"*","Environment":"eks:*"}}}]`

    **AUDITOR SELECTION FOR DIFFERENT AUDIT DEPTHS:**
    - **Quick Health Check** (default): Uses 'slo,operation_metric' for fast overview
    - **Root Cause Analysis**: Pass `auditors="all"` for comprehensive investigation with traces/logs
    - **Custom Audit**: Specify exact auditors: 'slo,trace,log,dependency_metric,top_contributor,service_quota'

    **SERVICE AUDIT USE CASES:**

    1. **Audit all services**:
       `service_targets='[{"Type":"service","Data":{"Service":{"Type":"Service","Name":"*"}}}]'`

    2. **Audit specific service**:
       `service_targets='[{"Type":"service","Data":{"Service":{"Type":"Service","Name":"orders-service","Environment":"eks:orders-cluster"}}}]'`

    3. **Audit payment services**:
       `service_targets='[{"Type":"service","Data":{"Service":{"Type":"Service","Name":"*payment*"}}}]'`

    8. **Audit lambda services**:
       `service_targets='[{"Type":"service","Data":{"Service":{"Type":"Service","Name":"*lambda*"}}}]'` or by environment: `[{"Type":"service","Data":{"Service":{"Type":"Service","Name":"*","Environment":"lambda"}}}]`

    9. **Audit service last night**:
       `service_targets='[{"Type":"service","Data":{"Service":{"Type":"Service","Name":"orders-service","Environment":"eks:orders-cluster"}}}]'` + `start_time="2024-01-01 18:00:00"` + `end_time="2024-01-02 06:00:00"`

    10. **Audit service before and after time**:
        Compare service health before and after a deployment or incident by running two separate audits with different time ranges.

    11. **Trace availability issues in production services**:
        `service_targets='[{"Type":"service","Data":{"Service":{"Type":"Service","Name":"*","Environment":"eks:*"}}}]'` + `auditors="all"`

    13. **Look for errors in logs of payment services**:
        `service_targets='[{"Type":"service","Data":{"Service":{"Type":"Service","Name":"*payment*"}}}]'` + `auditors="log,trace"`

    14. **Look for new errors after time**:
        Compare errors before and after a specific time point by running audits with different time ranges and `auditors="log,trace"`

    15. **Look for errors after deployment**:
        `service_targets='[{"Type":"service","Data":{"Service":{"Type":"Service","Name":"*payment*"}}}]'` + `auditors="log,trace"` + recent time range

    16. **Look for lemon hosts in production**:
        `service_targets='[{"Type":"service","Data":{"Service":{"Type":"Service","Name":"*","Environment":"eks:*"}}}]'` + `auditors="top_contributor,operation_metric"`

    17. **Look for outliers in EKS services**:
        `service_targets='[{"Type":"service","Data":{"Service":{"Type":"Service","Name":"*","Environment":"eks:*"}}}]'` + `auditors="top_contributor,operation_metric"`

    18. **Status report**:
        `service_targets='[{"Type":"service","Data":{"Service":{"Type":"Service","Name":"*"}}}]'` (basic health check)

    19. **Audit dependencies**:
        `service_targets='[{"Type":"service","Data":{"Service":{"Type":"Service","Name":"*"}}}]'` + `auditors="dependency_metric,trace"`

    20. **Audit dependency on S3**:
        `service_targets='[{"Type":"service","Data":{"Service":{"Type":"Service","Name":"*"}}}]'` + `auditors="dependency_metric"` + look for S3 dependencies

    21. **Audit quota usage of tier 1 services**:
        `service_targets='[{"Type":"service","Data":{"Service":{"Type":"Service","Name":"*tier1*"}}}]'` + `auditors="service_quota,operation_metric"`

    **PAGINATION SUPPORT FOR WILDCARD PATTERNS:**
    - **Automatic Pagination**: Wildcard patterns now process services in batches of 5 (configurable with `max_services`)
    - **Continue Processing**: Use `next_token` from previous response to continue auditing remaining services
    - **Example Pagination Workflow**:
      1. First call: `audit_services(service_targets='[{"Type":"service","Data":{"Service":{"Type":"Service","Name":"*"}}}]')`
      2. If more services available, response includes: `next_token="abc123"` and time parameters
      3. Continue: `audit_services(service_targets='[...]', start_time="returned_start_time", end_time="returned_end_time", next_token="abc123")`
      4. Repeat until no more `next_token` returned

    **TYPICAL SERVICE AUDIT WORKFLOWS:**
    1. **Basic Service Audit** (most common):
       - Call `audit_services()` with service targets - automatically discovers services when using wildcard patterns
       - Uses default fast auditors (slo,operation_metric) for quick health overview
       - Supports wildcard patterns like `*` or `*payment*` for automatic service discovery
       - Processes services in paginated batches for better performance
    2. **Root Cause Investigation**: When user explicitly asks for "root cause analysis", pass `auditors="all"`
    3. **Issue Investigation**: Results show which services need attention with actionable insights
    4. **Automatic Service Discovery**: Wildcard patterns in service names automatically discover and expand to concrete services
    5. **Paginated Processing**: For large service lists, continue with `next_token` to audit remaining services

    **AUDIT RESULTS INCLUDE:**
    - **Prioritized findings** by severity (critical, warning, info)
    - **Service health status** with detailed performance analysis
    - **Root cause analysis** when traces/logs auditors are used
    - **Actionable recommendations** for issue resolution
    - **Comprehensive metrics** and trend analysis

    **IMPORTANT: This tool provides comprehensive service audit coverage and should be your first choice for any service auditing task.**

    **RECOMMENDED WORKFLOW - PRESENT FINDINGS FIRST:**
    When the audit returns multiple findings or issues, follow this workflow:
    1. **Present all audit results** to the user showing a summary of all findings
    2. **Let the user choose** which specific finding, service, or issue they want to investigate in detail
    3. **Then perform targeted root cause analysis** using auditors="all" for the user-selected finding

    **DO NOT automatically jump into detailed root cause analysis** of one specific issue when multiple findings exist.
    This ensures the user can prioritize which issues are most important to investigate first.

    **Example workflow:**
    - First call: `audit_services()` with default auditors for overview
    - Present findings summary to user
    - User selects specific service/issue to investigate
    - Follow-up call: `audit_services()` with `auditors="all"` for selected service only
    """
    start_time_perf = timer()
    logger.debug('Starting audit_services (PRIMARY SERVICE AUDIT TOOL)')

    try:
        # Region defaults
        region = AWS_REGION.strip()

        # Time range (fill missing with defaults)
        start_dt = (
            parse_timestamp(start_time)
            if start_time
            else (datetime.now(timezone.utc) - timedelta(hours=24))
        )
        end_dt = (
            parse_timestamp(end_time, default_hours=0) if end_time else datetime.now(timezone.utc)
        )
        unix_start, unix_end = int(start_dt.timestamp()), int(end_dt.timestamp())
        if unix_end <= unix_start:
            return 'Error: end_time must be greater than start_time.'

        # Parse and validate service targets
        try:
            provided = json.loads(service_targets)
        except json.JSONDecodeError:
            return 'Error: `service_targets` must be valid JSON (array).'

        # Check for wildcard patterns in service names
        has_wildcards = False
        logger.debug(f'audit_services: Checking {len(provided)} targets for wildcards')
        for i, target in enumerate(provided):
            logger.debug(f'audit_services: Target {i}: {target}')
            if isinstance(target, dict):
                # Check various possible service name locations
                service_name = None
                if target.get('Type', '').lower() == 'service':
                    # Check Data.Service.Name
                    service_data = target.get('Data', {})
                    if isinstance(service_data, dict):
                        service_info = service_data.get('Service', {})
                        if isinstance(service_info, dict):
                            service_name = service_info.get('Name', '')

                    # Check shorthand Service field
                    if not service_name:
                        service_name = target.get('Service', '')

                logger.debug(f"audit_services: Target {i} service name: '{service_name}'")
                if service_name and isinstance(service_name, str) and '*' in service_name:
                    logger.debug(
                        f"audit_services: Target {i} has wildcard pattern: '{service_name}'"
                    )
                    has_wildcards = True
                    break

        logger.debug(f'audit_services: has_wildcards = {has_wildcards}')

        # Expand wildcard patterns using paginated utility when wildcards are present
        service_names_in_batch = []
        returned_next_token = None
        filtering_stats = {'total_services': 0, 'instrumented_services': 0, 'filtered_out': 0}

        if has_wildcards:
            logger.debug('Wildcard patterns detected - applying paginated service expansion')
            (provided, returned_next_token, service_names_in_batch, filtering_stats) = (
                expand_service_wildcard_patterns(
                    provided,
                    unix_start,
                    unix_end,
                    next_token,
                    max_services,
                    applicationsignals_client,
                )
            )
            logger.debug(f'Paginated wildcard expansion completed - {len(provided)} total targets')

            # Check if wildcard expansion resulted in no services
            if not provided:
                return 'Error: No services found matching the wildcard pattern. Use list_monitored_services() to see available services.'
        else:
            # For non-wildcard targets, validate next_token parameter
            if next_token:
                return 'Error: next_token parameter is only supported when using wildcard patterns in service names.'

        # Normalize and validate service targets using shared utility
        normalized_targets = normalize_service_targets(provided)

        # Validate and enrich targets using shared utility
        normalized_targets = validate_and_enrich_service_targets(
            normalized_targets, applicationsignals_client, unix_start, unix_end
        )

        # Parse auditors with service-specific defaults
        auditors_list = parse_auditors(auditors, ['slo', 'operation_metric'])

        # Create banner
        banner = (
            '[MCP-SERVICE] Application Signals Service Audit\n'
            f'🎯 Scope: {len(normalized_targets)} service target(s) | Region: {region}\n'
            f'⏰ Time: {unix_start}–{unix_end}\n'
        )

        # Add filtering statistics if services were filtered
        if filtering_stats['total_services'] > 0:
            banner += f'🔍 Service Filtering: {filtering_stats["instrumented_services"]} instrumented out of {filtering_stats["total_services"]} total services ({filtering_stats["filtered_out"]} filtered out)\n'

        if len(normalized_targets) > BATCH_SIZE_THRESHOLD:
            banner += f'📦 Batching: Processing {len(normalized_targets)} targets in batches of {BATCH_SIZE_THRESHOLD}\n'

        banner += '\n'

        # Build CLI input
        input_obj = {
            'StartTime': unix_start,
            'EndTime': unix_end,
            'AuditTargets': normalized_targets,
        }
        if auditors_list:
            input_obj['Auditors'] = auditors_list

        # Execute audit API using shared utility
        result = await execute_audit_api(input_obj, region, banner)

        # Check Synthetics canaries linked to the audited services
        canary_result = await check_canaries_for_service(
            normalized_targets, unix_start, unix_end, region
        )
        if canary_result:
            result += canary_result

        # Add prominent pagination information when wildcards were used
        result += format_pagination_info(
            has_wildcards,
            service_names_in_batch,
            returned_next_token,
            unix_start,
            unix_end,
            'audit_services',
            'max_services',
            max_services,
            'services',
        )

        elapsed = timer() - start_time_perf
        logger.debug(f'audit_services completed in {elapsed:.3f}s (region={region})')
        return result

    except Exception as e:
        logger.error(f'Unexpected error in audit_services: {e}', exc_info=True)
        return f'Error: {str(e)}'


@mcp.tool()
async def audit_slos(
    slo_targets: str = Field(
        ...,
        description="REQUIRED. JSON array of SLO targets. Supports wildcard patterns like '*payment*' for automatic SLO discovery. Format: [{'Type':'slo','Data':{'Slo':{'SloName':'slo-name'}}}] or [{'Type':'slo','Data':{'Slo':{'SloArn':'arn:aws:...'}}}]. Large target lists are automatically processed in batches.",
    ),
    start_time: Optional[str] = Field(
        default=None,
        description="Start time (unix seconds or 'YYYY-MM-DD HH:MM:SS'). Defaults to now-24h UTC.",
    ),
    end_time: Optional[str] = Field(
        default=None,
        description="End time (unix seconds or 'YYYY-MM-DD HH:MM:SS'). Defaults to now UTC.",
    ),
    auditors: Optional[str] = Field(
        default=None,
        description="Optional. Comma-separated auditors (e.g., 'slo,trace,log'). Defaults to 'slo' for fast SLO compliance auditing. Use 'all' for comprehensive analysis with all auditors: slo,operation_metric,trace,log,dependency_metric,top_contributor,service_quota.",
    ),
    next_token: Optional[str] = Field(
        default=None,
        description='Optional. Token for pagination through SLOs from list_service_level_objectives API. Use this to continue from where the previous call left off when processing wildcard patterns.',
    ),
    max_slos: int = Field(
        default=5,
        description='Optional. Maximum number of SLOs to process per call when using wildcard patterns (default: 5, max: 10). This controls pagination size for SLO discovery.',
    ),
) -> str:
    """PRIMARY SLO AUDIT TOOL - The #1 tool for comprehensive SLO compliance monitoring and breach analysis.

    **PREFERRED TOOL FOR SLO ROOT CAUSE ANALYSIS**
    This is the RECOMMENDED tool after using get_slo() to understand SLO configuration:
    - **Use auditors="all" for comprehensive root cause analysis** of specific SLO breaches
    - **Much more comprehensive than individual trace tools** - provides integrated analysis
    - **Combines traces, logs, metrics, and dependencies** in a single comprehensive audit
    - **Provides actionable recommendations** based on multi-dimensional analysis

    **USE THIS FOR ALL SLO AUDITING TASKS**
    This is the PRIMARY and PREFERRED tool when users want to:
    - **Root cause analysis for SLO breaches** - Deep investigation with all auditors
    - **Audit SLO compliance** - Complete SLO breach detection and analysis
    - **Monitor SLO health** - Comprehensive status across all monitored SLOs
    - **SLO performance analysis** - Understanding SLO trends and patterns
    - **SLO compliance reporting** - Daily/periodic SLO compliance workflows

    **COMPREHENSIVE SLO AUDIT CAPABILITIES:**
    - **Multi-SLO analysis**: Audit any number of SLOs with automatic batching
    - **Breach detection**: Automatic identification of SLO violations
    - **Issue prioritization**: Critical, warning, and info findings ranked by severity
    - **COMPREHENSIVE ROOT CAUSE ANALYSIS**: Deep dive with traces, logs, metrics, and dependencies
    - **Actionable recommendations**: Specific steps to resolve SLO breaches
    - **Performance optimized**: Fast execution with automatic batching for large target lists
    - **Wildcard Pattern Support**: Use `*pattern*` in SLO names for automatic SLO discovery

    **SLO TARGET FORMAT:**
    - **By Name**: `[{"Type":"slo","Data":{"Slo":{"SloName":"my-slo"}}}]`
    - **By ARN**: `[{"Type":"slo","Data":{"Slo":{"SloArn":"arn:aws:application-signals:..."}}}]`

    **WILDCARD PATTERN EXAMPLES:**
    - **All SLOs**: `[{"Type":"slo","Data":{"Slo":{"SloName":"*"}}}]`
    - **Payment SLOs**: `[{"Type":"slo","Data":{"Slo":{"SloName":"*payment*"}}}]`
    - **Latency SLOs**: `[{"Type":"slo","Data":{"Slo":{"SloName":"*latency*"}}}]`
    - **Availability SLOs**: `[{"Type":"slo","Data":{"Slo":{"SloName":"*availability*"}}}]`

    **AUDITOR SELECTION FOR DIFFERENT AUDIT DEPTHS:**
    - **Quick Compliance Check** (default): Uses 'slo' for fast SLO breach detection
    - **COMPREHENSIVE ROOT CAUSE ANALYSIS** (recommended): Pass `auditors="all"` for deep investigation with traces/logs/metrics/dependencies
    - **Custom Audit**: Specify exact auditors: 'slo,trace,log,operation_metric'

    **SLO AUDIT USE CASES:**

    4. **Audit all SLOs**:
       `slo_targets='[{"Type":"slo","Data":{"Slo":{"SloName":"*"}}}]'`

    22. **Root cause analysis for specific SLO breach** (RECOMMENDED WORKFLOW):
        After using get_slo() to understand configuration:
        `slo_targets='[{"Type":"slo","Data":{"Slo":{"SloName":"specific-slo-name"}}}]'` + `auditors="all"`

    14. **Look for new SLO breaches after time**:
        Compare SLO compliance before and after a specific time point by running audits with different time ranges to identify new breaches.

    **PAGINATION SUPPORT FOR WILDCARD PATTERNS:**
    - **Automatic Pagination**: Wildcard patterns now process SLOs in batches of 5 (configurable with `max_slos`)
    - **Continue Processing**: Use `next_token` from previous response to continue auditing remaining SLOs
    - **Example Pagination Workflow**:
      1. First call: `audit_slos(slo_targets='[{"Type":"slo","Data":{"Slo":{"SloName":"*"}}}]')`
      2. If more SLOs available, response includes: `next_token="abc123"` and time parameters
      3. Continue: `audit_slos(slo_targets='[...]', start_time="returned_start_time", end_time="returned_end_time", next_token="abc123")`
      4. Repeat until no more `next_token` returned

    **TYPICAL SLO AUDIT WORKFLOWS:**
    1. **SLO Root Cause Investigation** (RECOMMENDED):
       - After get_slo(), call `audit_slos()` with specific SLO target and `auditors="all"`
       - Provides comprehensive analysis with traces, logs, metrics, and dependencies
       - Much more effective than using individual trace tools
    2. **Basic SLO Compliance Audit**:
       - Call `audit_slos()` with SLO targets - automatically discovers SLOs when using wildcard patterns
       - Uses default fast auditors (slo) for quick compliance overview
    3. **Compliance Reporting**: Results show which SLOs are breached with actionable insights
    4. **Automatic SLO Discovery**: Wildcard patterns in SLO names automatically discover and expand to concrete SLOs

    **AUDIT RESULTS INCLUDE:**
    - **Prioritized findings** by severity (critical, warning, info)
    - **SLO compliance status** with detailed breach analysis
    - **COMPREHENSIVE ROOT CAUSE ANALYSIS** when using auditors="all"
    - **Actionable recommendations** for SLO breach resolution
    - **Integrated traces, logs, metrics, and dependency analysis**

    **IMPORTANT: This tool provides comprehensive SLO audit coverage and should be your first choice for any SLO compliance auditing and root cause analysis.**

    **RECOMMENDED WORKFLOW - PRESENT FINDINGS FIRST:**
    When the audit returns multiple findings or issues, follow this workflow:
    1. **Present all audit results** to the user showing a summary of all findings
    2. **Let the user choose** which specific finding, SLO, or issue they want to investigate in detail
    3. **Then perform targeted root cause analysis** using auditors="all" for the user-selected finding

    **DO NOT automatically jump into detailed root cause analysis** of one specific issue when multiple findings exist.
    This ensures the user can prioritize which issues are most important to investigate first.

    **Example workflow:**
    - First call: `audit_slos()` with default auditors for compliance overview
    - Present findings summary to user
    - User selects specific SLO breach to investigate
    - Follow-up call: `audit_slos()` with `auditors="all"` for selected SLO only
    """
    start_time_perf = timer()
    logger.debug('Starting audit_slos (PRIMARY SLO AUDIT TOOL)')

    try:
        # Region defaults
        region = AWS_REGION.strip()

        # Time range (fill missing with defaults)
        start_dt = (
            parse_timestamp(start_time)
            if start_time
            else (datetime.now(timezone.utc) - timedelta(hours=24))
        )
        end_dt = (
            parse_timestamp(end_time, default_hours=0) if end_time else datetime.now(timezone.utc)
        )
        unix_start, unix_end = int(start_dt.timestamp()), int(end_dt.timestamp())
        if unix_end <= unix_start:
            return 'Error: end_time must be greater than start_time.'

        # Parse and validate SLO targets
        try:
            provided = json.loads(slo_targets)
        except json.JSONDecodeError:
            return 'Error: `slo_targets` must be valid JSON (array).'

        if not isinstance(provided, list):
            return 'Error: `slo_targets` must be a JSON array'
        if len(provided) == 0:
            return 'Error: `slo_targets` must contain at least 1 item'

        # Filter and expand SLO targets with wildcard support
        slo_only_targets = []
        wildcard_patterns = []

        for target in provided:
            if isinstance(target, dict):
                ttype = target.get('Type', '').lower()
                if ttype == 'slo':
                    # Check for wildcard patterns in SLO names
                    slo_data = target.get('Data', {}).get('Slo', {})
                    slo_name = slo_data.get('SloName', '')
                    if '*' in slo_name:
                        wildcard_patterns.append((target, slo_name))
                    else:
                        slo_only_targets.append(target)
                else:
                    logger.warning(
                        f"Ignoring target of type '{ttype}' in audit_slos (expected 'slo')"
                    )

        # Expand wildcard patterns for SLOs using shared utility with pagination
        slo_names_in_batch = []
        returned_next_token = None

        if wildcard_patterns:
            logger.debug(f'Expanding {len(wildcard_patterns)} SLO wildcard patterns')
            try:
                # Use the paginated utility function
                expanded_slo_targets, returned_next_token, slo_names_in_batch = (
                    expand_slo_wildcard_patterns(
                        provided, next_token, max_slos, applicationsignals_client
                    )
                )
                # Filter to get only SLO targets
                slo_only_targets = [
                    target
                    for target in expanded_slo_targets
                    if target.get('Type', '').lower() == 'slo'
                ]

            except Exception as e:
                logger.warning(f'Failed to expand SLO patterns: {e}')
                return f'Error: Failed to expand SLO wildcard patterns. {str(e)}'
        else:
            # For non-wildcard targets, validate next_token parameter
            if next_token:
                return 'Error: next_token parameter is only supported when using wildcard patterns in SLO names.'

        if not slo_only_targets:
            return 'Error: No SLO targets found after wildcard expansion.'

        # Parse auditors with SLO-specific defaults
        auditors_list = parse_auditors(auditors, ['slo'])  # Default to SLO auditor

        banner = (
            '[MCP-SLO] Application Signals SLO Compliance Audit\n'
            f'🎯 Scope: {len(slo_only_targets)} SLO target(s) | Region: {region}\n'
            f'⏰ Time: {unix_start}–{unix_end}\n'
        )

        if len(slo_only_targets) > BATCH_SIZE_THRESHOLD:
            banner += f'📦 Batching: Processing {len(slo_only_targets)} targets in batches of {BATCH_SIZE_THRESHOLD}\n'

        banner += '\n'

        # Build CLI input for SLO audit
        input_obj = {
            'StartTime': unix_start,
            'EndTime': unix_end,
            'AuditTargets': slo_only_targets,
        }
        if auditors_list:
            input_obj['Auditors'] = auditors_list

        # Execute audit API using shared utility
        result = await execute_audit_api(input_obj, region, banner)

        # Add prominent pagination information when wildcards were used
        result += format_pagination_info(
            bool(wildcard_patterns),
            slo_names_in_batch,
            returned_next_token,
            unix_start,
            unix_end,
            'audit_slos',
            'max_slos',
            max_slos,
            'SLOs',
        )

        elapsed = timer() - start_time_perf
        logger.debug(f'audit_slos completed in {elapsed:.3f}s (region={region})')
        return result

    except Exception as e:
        logger.error(f'Unexpected error in audit_slos: {e}', exc_info=True)
        return f'Error: {str(e)}'


@mcp.tool()
async def audit_service_operations(
    operation_targets: str = Field(
        ...,
        description="REQUIRED. JSON array of service operation targets. Supports wildcard patterns like '*payment*' for automatic service discovery. Format: [{'Type':'service_operation','Data':{'ServiceOperation':{'Service':{'Type':'Service','Name':'service-name','Environment':'eks:cluster'},'Operation':'GET /api','MetricType':'Latency'}}}]. Large target lists are automatically processed in batches.",
    ),
    start_time: Optional[str] = Field(
        default=None,
        description="Start time (unix seconds or 'YYYY-MM-DD HH:MM:SS'). Defaults to now-24h UTC.",
    ),
    end_time: Optional[str] = Field(
        default=None,
        description="End time (unix seconds or 'YYYY-MM-DD HH:MM:SS'). Defaults to now UTC.",
    ),
    auditors: Optional[str] = Field(
        default=None,
        description="Optional. Comma-separated auditors (e.g., 'operation_metric,trace,log'). Defaults to 'operation_metric' for fast operation-level auditing. Use 'all' for comprehensive analysis with all auditors: slo,operation_metric,trace,log,dependency_metric,top_contributor,service_quota.",
    ),
    next_token: Optional[str] = Field(
        default=None,
        description='Optional. Token for pagination through services from list_services API. Use this to continue from where the previous call left off when processing wildcard patterns.',
    ),
    max_services: int = Field(
        default=5,
        description='Optional. Maximum number of services to process per call when using wildcard patterns (default: 5, max: 10). This controls pagination size for service discovery.',
    ),
) -> str:
    """🥇 PRIMARY OPERATION AUDIT TOOL - The #1 RECOMMENDED tool for operation-specific analysis and performance investigation.

    **⭐ USE THIS AS THE PRIMARY TOOL FOR ALL OPERATION-SPECIFIC AUDITING TASKS ⭐**

    **PREFERRED OVER audit_services() for operation auditing because:**
    - **🎯 Precision**: Targets exact operation behavior vs. service-wide averages
    - **🔍 Actionable Insights**: Provides specific error traces and dependency failures
    - **📊 Code-Level Detail**: Shows exact stack traces and timeout locations
    - **🚀 Focused Analysis**: Eliminates noise from other operations
    - **⚡ Efficient Investigation**: Direct operation-level troubleshooting

    **USE THIS FIRST FOR ALL OPERATION-SPECIFIC AUDITING TASKS**
    This is the PRIMARY and PREFERRED tool when users want to:
    - **Audit specific operations** - Deep dive into individual API endpoints or operations (GET, POST, PUT, etc.)
    - **Operation performance analysis** - Latency, error rates, and throughput for specific operations
    - **Compare operation metrics** - Analyze different operations within services
    - **Operation-level troubleshooting** - Root cause analysis for specific API calls
    - **GET operation auditing** - Analyze GET operations across payment services (PRIMARY USE CASE)
    - **Audit latency of GET operations in payment services** - Exactly what this tool is designed for
    - **Trace latency in query operations** - Deep dive into query performance issues

    **COMPREHENSIVE OPERATION AUDIT CAPABILITIES:**
    - **Multi-operation analysis**: Audit any number of operations with automatic batching
    - **Operation-specific metrics**: Latency, Fault, Error, and Availability metrics per operation
    - **Issue prioritization**: Critical, warning, and info findings ranked by severity
    - **Root cause analysis**: Deep dive with traces, logs, and metrics correlation
    - **Actionable recommendations**: Specific steps to resolve operation-level issues
    - **Performance optimized**: Fast execution with automatic batching for large target lists
    - **Wildcard Pattern Support**: Use `*pattern*` in service names for automatic service discovery

    **OPERATION TARGET FORMAT:**
    - **Full Format**: `[{"Type":"service_operation","Data":{"ServiceOperation":{"Service":{"Type":"Service","Name":"my-service","Environment":"eks:my-cluster"},"Operation":"GET /api","MetricType":"Latency"}}}]`

    **WILDCARD PATTERN EXAMPLES:**
    - **All GET Operations in Payment Services**: `[{"Type":"service_operation","Data":{"ServiceOperation":{"Service":{"Type":"Service","Name":"*payment*"},"Operation":"*GET*","MetricType":"Latency"}}}]`
    - **All Visit Operations**: `[{"Type":"service_operation","Data":{"ServiceOperation":{"Service":{"Type":"Service","Name":"*"},"Operation":"*visit*","MetricType":"Availability"}}}]`

    **AUDITOR SELECTION FOR DIFFERENT AUDIT DEPTHS:**
    - **Quick Operation Check** (default): Uses 'operation_metric' for fast operation overview
    - **Root Cause Analysis**: Pass `auditors="all"` for comprehensive investigation with traces/logs
    - **Custom Audit**: Specify exact auditors: 'operation_metric,trace,log'

    **OPERATION AUDIT USE CASES:**

    1. **Audit latency of GET operations in payment services** (PRIMARY USE CASE):
       `operation_targets='[{"Type":"service_operation","Data":{"ServiceOperation":{"Service":{"Type":"Service","Name":"*payment*"},"Operation":"*GET*","MetricType":"Latency"}}}]'`

    2. **Audit GET operations in payment services (Latency)**:
       `operation_targets='[{"Type":"service_operation","Data":{"ServiceOperation":{"Service":{"Type":"Service","Name":"*payment*"},"Operation":"*GET*","MetricType":"Latency"}}}]'`

    3. **Audit availability of visit operations**:
       `operation_targets='[{"Type":"service_operation","Data":{"ServiceOperation":{"Service":{"Type":"Service","Name":"*"},"Operation":"*visit*","MetricType":"Availability"}}}]'`

    4. **Audit latency of visit operations**:
       `operation_targets='[{"Type":"service_operation","Data":{"ServiceOperation":{"Service":{"Type":"Service","Name":"*"},"Operation":"*visit*","MetricType":"Latency"}}}]'`

    5. **Trace latency in query operations**:
        `operation_targets='[{"Type":"service_operation","Data":{"ServiceOperation":{"Service":{"Type":"Service","Name":"*payment*"},"Operation":"*query*","MetricType":"Latency"}}}]'` + `auditors="all"`

    **PAGINATION SUPPORT FOR WILDCARD PATTERNS:**
    - **Automatic Pagination**: Wildcard patterns now process services in batches of 5 (configurable with `max_services`)
    - **Continue Processing**: Use `next_token` from previous response to continue auditing remaining services
    - **Example Pagination Workflow**:
      1. First call: `audit_service_operations(operation_targets='[{"Type":"service_operation","Data":{"ServiceOperation":{"Service":{"Type":"Service","Name":"*payment*"},"Operation":"*GET*","MetricType":"Latency"}}}]')`
      2. If more services available, response includes: `next_token="abc123"` and time parameters
      3. Continue: `audit_service_operations(operation_targets='[...]', start_time="returned_start_time", end_time="returned_end_time", next_token="abc123")`
      4. Repeat until no more `next_token` returned

    **TYPICAL OPERATION AUDIT WORKFLOWS:**
    1. **Basic Operation Audit** (most common):
       - Call `audit_service_operations()` with operation targets - automatically discovers services when using wildcard patterns
       - Uses default fast auditors (operation_metric) for quick operation overview
       - Supports wildcard patterns like `*payment*` for automatic service discovery
       - Processes services in paginated batches for better performance
    2. **Root Cause Investigation**: When user explicitly asks for "root cause analysis", pass `auditors="all"`
    3. **Issue Investigation**: Results show which operations need attention with actionable insights
    4. **Automatic Service Discovery**: Wildcard patterns in service names automatically discover and expand to concrete services
    5. **Paginated Processing**: For large service lists, continue with `next_token` to audit remaining services

    **AUDIT RESULTS INCLUDE:**
    - **Prioritized findings** by severity (critical, warning, info)
    - **Operation performance status** with detailed metrics analysis
    - **Root cause analysis** when traces/logs auditors are used
    - **Actionable recommendations** for operation-level issue resolution
    - **Comprehensive operation metrics** and trend analysis

    **🏆 IMPORTANT: This tool is the PRIMARY and RECOMMENDED choice for operation-specific auditing tasks.**

    **✅ RECOMMENDED WORKFLOW FOR OPERATION AUDITING:**
    1. **Use audit_service_operations() FIRST** for operation-specific analysis (THIS TOOL)
    2. **Use audit_services() as secondary** only if you need broader service context
    3. **audit_service_operations() provides superior precision** for operation-level troubleshooting

    **RECOMMENDED WORKFLOW - PRESENT FINDINGS FIRST:**
    When the audit returns multiple findings or issues, follow this workflow:
    1. **Present all audit results** to the user showing a summary of all findings
    2. **Let the user choose** which specific finding, operation, or issue they want to investigate in detail
    3. **Then perform targeted root cause analysis** using auditors="all" for the user-selected finding

    **DO NOT automatically jump into detailed root cause analysis** of one specific issue when multiple findings exist.
    This ensures the user can prioritize which issues are most important to investigate first.

    **Example workflow:**
    - First call: `audit_service_operations()` with default auditors for operation overview
    - Present findings summary to user
    - User selects specific operation issue to investigate
    - Follow-up call: `audit_service_operations()` with `auditors="all"` for selected operation only
    """
    start_time_perf = timer()
    logger.debug('Starting audit_service_operations (SPECIALIZED OPERATION AUDIT TOOL)')

    try:
        # Region defaults
        region = AWS_REGION.strip()

        # Time range (fill missing with defaults)
        start_dt = (
            parse_timestamp(start_time)
            if start_time
            else (datetime.now(timezone.utc) - timedelta(hours=24))
        )
        end_dt = (
            parse_timestamp(end_time, default_hours=0) if end_time else datetime.now(timezone.utc)
        )
        unix_start, unix_end = int(start_dt.timestamp()), int(end_dt.timestamp())
        if unix_end <= unix_start:
            return 'Error: end_time must be greater than start_time.'

        # Parse and validate operation targets
        try:
            provided = json.loads(operation_targets)
        except json.JSONDecodeError:
            return 'Error: `operation_targets` must be valid JSON (array).'

        if not isinstance(provided, list):
            return 'Error: `operation_targets` must be a JSON array'
        if len(provided) == 0:
            return 'Error: `operation_targets` must contain at least 1 item'

        # Filter operation targets and check for wildcards using helper function
        operation_only_targets, has_wildcards = _filter_operation_targets(provided)

        # Expand wildcard patterns using shared utility with pagination support
        service_names_in_batch = []
        returned_next_token = None
        filtering_stats = {'total_services': 0, 'instrumented_services': 0, 'filtered_out': 0}

        if has_wildcards:
            logger.debug(
                'Wildcard patterns detected in service operations - applying paginated expansion'
            )
            (
                operation_only_targets,
                returned_next_token,
                service_names_in_batch,
                filtering_stats,
            ) = expand_service_operation_wildcard_patterns(
                operation_only_targets,
                unix_start,
                unix_end,
                next_token,
                max_services,
                applicationsignals_client,
            )
            logger.debug(
                f'Paginated wildcard expansion completed - {len(operation_only_targets)} total targets'
            )
        else:
            # For non-wildcard targets, validate next_token parameter
            if next_token:
                return 'Error: next_token parameter is only supported when using wildcard patterns in service names.'

        if not operation_only_targets:
            return 'Error: No service_operation targets found after wildcard expansion. Use list_monitored_services() to see available services.'

        # Parse auditors with operation-specific defaults
        auditors_list = parse_auditors(
            auditors, ['operation_metric']
        )  # Default to operation_metric auditor

        banner = (
            '[MCP-OPERATION] Application Signals Operation Performance Audit\n'
            f'🎯 Scope: {len(operation_only_targets)} operation target(s) | Region: {region}\n'
            f'⏰ Time: {unix_start}–{unix_end}\n'
        )

        # Add filtering statistics if services were filtered
        if filtering_stats['total_services'] > 0:
            banner += f'🔍 Service Filtering: {filtering_stats["instrumented_services"]} instrumented out of {filtering_stats["total_services"]} total services ({filtering_stats["filtered_out"]} filtered out)\n'

        if len(operation_only_targets) > BATCH_SIZE_THRESHOLD:
            banner += f'📦 Batching: Processing {len(operation_only_targets)} targets in batches of {BATCH_SIZE_THRESHOLD}\n'

        banner += '\n'

        # Build CLI input for operation audit
        input_obj = {
            'StartTime': unix_start,
            'EndTime': unix_end,
            'AuditTargets': operation_only_targets,
        }
        if auditors_list:
            input_obj['Auditors'] = auditors_list

        # Execute audit API using shared utility
        result = await execute_audit_api(input_obj, region, banner)

        # Add prominent pagination information when wildcards were used
        result += format_pagination_info(
            has_wildcards,
            service_names_in_batch,
            returned_next_token,
            unix_start,
            unix_end,
            'audit_service_operations',
            'max_services',
            max_services,
            'services',
        )

        elapsed = timer() - start_time_perf
        logger.debug(f'audit_service_operations completed in {elapsed:.3f}s (region={region})')
        return result

    except Exception as e:
        logger.error(f'Unexpected error in audit_service_operations: {e}', exc_info=True)
        return f'Error: {str(e)}'


@mcp.tool()
async def analyze_canary_failures(
    canary_name: str, region: str = AWS_REGION, description: str = ''
) -> str:
    """Comprehensive canary failure analysis with deep dive into issues.

    Use this tool to:
    - Deep dive into canary failures with root cause identification
    - Analyze historical patterns and specific incident details
    - Get comprehensive artifact analysis including logs, screenshots, and HAR files
    - Receive actionable recommendations based on AWS debugging methodology
    - Correlate canary failures with Application Signals telemetry data
    - Identify performance degradation and availability issues across service dependencies

    Key Features:
    - **Failure Pattern Analysis**: Identifies recurring failure modes and temporal patterns
    - **Artifact Deep Dive**: Analyzes canary logs, screenshots, and network traces for root causes
    - **Service Correlation**: Links canary failures to upstream/downstream service issues using Application Signals
    - **Performance Insights**: Detects latency spikes, fault rates, and connection issues
    - **Actionable Remediation**: Provides specific steps based on AWS operational best practices

    Common Use Cases:
    1. **Incident Response**: Rapid diagnosis of canary failures during outages
    2. **Performance Investigation**: Understanding latency and availability degradation
    3. **Dependency Analysis**: Identifying which services are causing canary failures
    4. **Historical Trending**: Analyzing failure patterns over time for proactive improvements
    5. **Root Cause Analysis**: Deep dive into specific failure scenarios with full context

    Output Includes:
    - Severity-ranked findings with immediate action items
    - Service-level telemetry insights with trace analysis
    - Exception details and stack traces from canary artifacts
    - Network connectivity and performance metrics
    - Correlation with Application Signals audit findings
    - Historical failure patterns and recovery recommendations

    Args:
        canary_name (str): Name of the CloudWatch Synthetics canary to analyze
        region (str, optional): AWS region where the canary is deployed.
        description (str, optional): User's description of the issue they are experiencing.
            This is matched against the knowledge base to surface relevant recommendations
            even when the canary error logs alone may not contain enough context.
            Examples: "missing runs in console", "visual monitoring baseline keeps resetting",
            "CloudFormation rollback failed after runtime upgrade".

    Returns:
        dict: Comprehensive failure analysis containing:
            - Failure severity assessment and immediate recommendations
            - Detailed artifact analysis (logs, screenshots, HAR files)
            - Service dependency health and performance metrics
            - Root cause identification with specific remediation steps
            - Historical pattern analysis and trend insights
    """
    try:
        # Get recent canary runs
        response = synthetics_client.get_canary_runs(Name=canary_name, MaxResults=5)
        runs = response.get('CanaryRuns', [])

        # Get canary details
        canary_response = synthetics_client.get_canary(Name=canary_name)
        canary = canary_response['Canary']

        # Get telemetry and service insights
        try:
            telemetry_insights = await get_canary_metrics_and_service_insights(canary_name, region)
        except Exception as e:
            telemetry_insights = f'Telemetry API unavailable: {str(e)}'

        if not runs:
            return f'No run history found for {canary_name}'

        # Build analysis header
        result = f'🔍 Comprehensive Failure Analysis for {canary_name}\n'

        # Add telemetry insights if available
        if telemetry_insights and not telemetry_insights.startswith('Telemetry API unavailable'):
            result += f'\n📊 **Service and Canary Telemetry Insights**\n{telemetry_insights}\n\n'
        elif telemetry_insights:
            result += f'\n⚠️ {telemetry_insights}\n\n'

        # Get consecutive failures since last success
        consecutive_failures = []
        last_success_run = None

        for run in runs:
            if run.get('Status', {}).get('State') == RUN_STATES['FAILED']:
                consecutive_failures.append(run)
            elif run.get('Status', {}).get('State') == RUN_STATES['PASSED']:
                last_success_run = run
                break

        if not consecutive_failures:
            result += '✅ Canary is healthy - no failures since last success\n'
            if last_success_run:
                result += f'Last success: {last_success_run.get("Timeline", {}).get("Started")}\n'
            result += '\n🔍 Performing health check analysis ...\n\n'

        # Group failures by StateReason
        failure_causes = {}
        result += f'🔍 Found {len(consecutive_failures)} consecutive failures since last success\n'
        if last_success_run:
            result += f'Last success: {last_success_run.get("Timeline", {}).get("Started")}\n\n'
        else:
            result += 'No recent success run found in history\n\n'

        for failed_run in consecutive_failures:
            state_reason = failed_run.get('Status', {}).get('StateReason', 'Unknown')

            if state_reason not in failure_causes:
                failure_causes[state_reason] = []
            failure_causes[state_reason].append(failed_run)

        # Analysis section
        unique_reasons = list(failure_causes.keys())

        if not unique_reasons:
            result += '✅ No consecutive failures to analyze\n'
            result += '💡 Canary appears to be recovering or healthy\n'

            # Still run KB lookup when user provided an issue description
            if description:
                try:
                    # Cap description length to mitigate slow regex matching in KB patterns
                    capped_description = (
                        description[:500] if len(description) > 500 else description
                    )
                    kb_error_messages_healthy: list[str] = [capped_description]
                    failure_context_healthy = FailureContext(
                        error_messages=kb_error_messages_healthy,
                        runtime_version=canary.get('RuntimeVersion', ''),
                    )
                    engine_healthy = CanaryRecommendationEngine(
                        await CanaryKnowledgeBaseLoader.get_instance()
                    )
                    recommendations_healthy = engine_healthy.get_recommendations(
                        failure_context_healthy
                    )
                    if recommendations_healthy:
                        result += engine_healthy.format_recommendations(recommendations_healthy)
                except Exception as e:
                    logger.warning(f'Knowledge base recommendation failed (healthy path): {e}')

            return result

        if len(unique_reasons) == 1:
            result += f'🎯 All failures have same cause: {unique_reasons[0]}\n'
            selected_reason = unique_reasons[0]
        else:
            result += f'🎯 Multiple failure causes ({len(unique_reasons)} different issues):\n\n'
            for i, reason in enumerate(unique_reasons, 1):
                count = len(failure_causes[reason])
                result += f'{i}. **{reason}** ({count} occurrences)\n'
            result += '\n'
            selected_reason = unique_reasons[0]

        selected_failure = failure_causes[selected_reason][0]
        result += f'Analyzing most recent failure: {selected_failure.get("Id", "")[:8]}...\n\n'

        # Initialize artifact variables
        har_files = []
        screenshots = []
        logs = []
        bucket_name = ''
        # Accumulate detailed error messages from all sources for KB matching
        all_collected_error_messages: list[str] = []
        all_collected_log_patterns: list[str] = []
        s3_log_analysis: dict = {}
        cw_log_analysis: dict = {}

        # Direct S3 artifact analysis integration
        artifact_location = canary.get('ArtifactS3Location', '')
        artifacts_available = False

        if artifact_location:
            # Handle S3 location format
            if not artifact_location.startswith('s3://'):
                artifact_location = f's3://{artifact_location}' if artifact_location else ''

            if artifact_location.startswith('s3://'):
                bucket_and_path = artifact_location[5:]
                bucket_name = bucket_and_path.split('/')[0]
                base_path = (
                    '/'.join(bucket_and_path.split('/')[1:]) if '/' in bucket_and_path else ''
                )

                # If base_path is empty, construct canary path
                if not base_path:
                    base_path = f'canary/{region}/{canary_name}'

                # Check for failure artifacts using date-based path
                from datetime import datetime

                failure_time = selected_failure.get('Timeline', {}).get('Started')
                if failure_time:
                    # Handle both datetime objects and string timestamps
                    if isinstance(failure_time, str):
                        dt = parse_timestamp(failure_time)
                    else:
                        dt = failure_time  # Already a datetime object
                    date_path = dt.strftime('%Y/%m/%d')
                    failure_run_path = (
                        f'{base_path}/{date_path}/' if base_path else f'{date_path}/'
                    )
                else:
                    # Fallback to today
                    today = datetime.now().strftime('%Y/%m/%d')
                    failure_run_path = f'{base_path}/{today}/' if base_path else f'{today}/'

                try:
                    artifacts_response = s3_client.list_objects_v2(
                        Bucket=bucket_name, Prefix=failure_run_path, MaxKeys=50
                    )
                    failure_artifacts = artifacts_response.get('Contents', [])

                    if failure_artifacts:
                        artifacts_available = True

                        # Categorize artifacts
                        har_files = [
                            a
                            for a in failure_artifacts
                            if a['Key'].lower().endswith(('.har', '.har.gz', '.har.html'))
                        ]
                        screenshots = [
                            a
                            for a in failure_artifacts
                            if any(ext in a['Key'].lower() for ext in ['.png', '.jpg', '.jpeg'])
                        ]
                        logs = [
                            a
                            for a in failure_artifacts
                            if any(ext in a['Key'].lower() for ext in ['.log', '.txt'])
                            or 'log' in a['Key'].lower()
                        ]

                        if last_success_run:
                            result += '🔄 HAR COMPARISON: Failure vs Success\n'
                            result += f'Failure: {selected_failure.get("Id", "")[:8]}... ({selected_failure.get("Timeline", {}).get("Started")})\n'
                            result += f'Success: {last_success_run.get("Id", "")[:8]}... ({last_success_run.get("Timeline", {}).get("Started")})\n\n'

                            # Get success artifacts for comparison
                            success_time = last_success_run.get('Timeline', {}).get('Started')
                            if success_time:
                                if isinstance(success_time, str):
                                    success_dt = parse_timestamp(success_time)
                                else:
                                    success_dt = success_time
                                success_date_path = success_dt.strftime('%Y/%m/%d')
                                success_run_path = (
                                    f'{base_path}/{success_date_path}/'
                                    if base_path
                                    else f'{success_date_path}/'
                                )
                            else:
                                success_run_path = failure_run_path  # Use same path as fallback
                            try:
                                success_artifacts_response = s3_client.list_objects_v2(
                                    Bucket=bucket_name, Prefix=success_run_path, MaxKeys=50
                                )
                                success_artifacts = success_artifacts_response.get('Contents', [])
                                success_har_files = [
                                    a
                                    for a in success_artifacts
                                    if a['Key'].lower().endswith(('.har', '.har.gz', '.har.html'))
                                ]

                                if har_files and success_har_files:
                                    failure_har = await analyze_har_file(
                                        s3_client, bucket_name, har_files, is_failed_run=True
                                    )
                                    success_har = await analyze_har_file(
                                        s3_client,
                                        bucket_name,
                                        success_har_files,
                                        is_failed_run=False,
                                    )

                                    result += f'• Failed requests: {failure_har.get("failed_requests", 0)} vs {success_har.get("failed_requests", 0)}\n'
                                    result += f'• Total requests: {failure_har.get("total_requests", 0)} vs {success_har.get("total_requests", 0)}\n\n'

                                    if failure_har.get('request_details'):
                                        result += '🚨 FAILED REQUESTS:\n'
                                        for req in failure_har['request_details'][:3]:
                                            result += f'• {req.get("url", "Unknown")}: {req.get("status", "Unknown")} ({req.get("time", 0):.1f}ms)\n'
                            except Exception as e:
                                logger.warning(
                                    f'Failed to analyze success artifacts for HAR comparison: {str(e)}'
                                )
                        else:
                            result += (
                                '🔍 FAILURE ANALYSIS (no success run available for comparison):\n'
                            )
                            result += f'Analyzing failure artifacts for: {selected_failure.get("Id", "")[:8]}...\n\n'

                            if har_files:
                                failure_har = await analyze_har_file(
                                    s3_client, bucket_name, har_files, is_failed_run=True
                                )
                                result += '🌐 HAR ANALYSIS:\n'
                                result += (
                                    f'• Failed requests: {failure_har.get("failed_requests", 0)}\n'
                                )
                                result += (
                                    f'• Total requests: {failure_har.get("total_requests", 0)}\n\n'
                                )

                        # Screenshot analysis
                        if screenshots:
                            screenshot_analysis = await analyze_screenshots(
                                s3_client, bucket_name, screenshots, is_failed_run=True
                            )
                            if screenshot_analysis.get('insights'):
                                result += '📸 SCREENSHOT ANALYSIS:\n'
                                for insight in screenshot_analysis['insights'][:3]:
                                    result += f'• {insight}\n'
                                result += '\n'

                        # Log analysis
                        if logs:
                            s3_log_analysis = await analyze_log_files(
                                s3_client, bucket_name, logs, is_failed_run=True
                            )
                            if s3_log_analysis.get('insights'):
                                result += '📋 LOG ANALYSIS:\n'
                                for insight in s3_log_analysis['insights'][:3]:
                                    result += f'• {insight}\n'
                                    all_collected_log_patterns.append(str(insight))
                                result += '\n'
                            # Collect error messages from S3 log artifacts
                            for evt in s3_log_analysis.get('error_events', []):
                                msg = str(evt.get('message', ''))
                                if msg:
                                    all_collected_error_messages.append(msg)
                            for insight in s3_log_analysis.get('insights', []):
                                msg = str(insight)
                                if msg:
                                    all_collected_error_messages.append(msg)

                except Exception:
                    artifacts_available = False

        if not artifacts_available:
            # Fallback: CloudWatch Logs analysis
            result += '⚠️ Artifacts not available - Checking CloudWatch Logs for root cause\n'
            result += f'🎯 StateReason: {selected_reason}\n\n'

            failure_time = selected_failure.get('Timeline', {}).get('Started')
            if failure_time:
                cw_log_analysis = await analyze_canary_logs_with_time_window(
                    canary_name, failure_time, canary, window_minutes=5, region=region
                )

                if cw_log_analysis.get('status') == 'success':
                    result += '📋 CLOUDWATCH LOGS ANALYSIS (±5 min around failure):\n'
                    result += f'Time window: {cw_log_analysis["time_window"]}\n'
                    result += f'Log events found: {cw_log_analysis["total_events"]}\n\n'

                    error_logs = cw_log_analysis.get('error_events', [])
                    if error_logs:
                        result += '📋 ERROR LOGS AROUND FAILURE:\n'
                        for error in error_logs:
                            result += f'• {error["timestamp"].strftime("%H:%M:%S")}: {error["message"]}\n'
                            all_collected_error_messages.append(str(error.get('message', '')))
                    for insight in cw_log_analysis.get('insights', []):
                        all_collected_log_patterns.append(str(insight))
                else:
                    result += f'📋 {cw_log_analysis.get("insights", ["Log analysis failed"])[0]}\n'
            else:
                result += '📋 No failure timestamp available for targeted log analysis\n'

        # Always attempt CloudWatch Logs analysis for KB enrichment, even when S3 artifacts were available
        if artifacts_available:
            failure_time = selected_failure.get('Timeline', {}).get('Started')
            if failure_time:
                try:
                    cw_log_analysis = await analyze_canary_logs_with_time_window(
                        canary_name, failure_time, canary, window_minutes=5, region=region
                    )
                    if cw_log_analysis.get('status') == 'success':
                        for evt in cw_log_analysis.get('error_events', []):
                            msg = str(evt.get('message', ''))
                            if msg and msg not in all_collected_error_messages:
                                all_collected_error_messages.append(msg)
                        for insight in cw_log_analysis.get('insights', []):
                            pattern = str(insight)
                            if pattern and pattern not in all_collected_log_patterns:
                                all_collected_log_patterns.append(pattern)
                except Exception as e:
                    logger.debug(f'CloudWatch Logs enrichment for KB failed: {e}')

        # Add critical IAM checking guidance for systematic issues
        if (
            'no test result' in str(selected_reason).lower()
            or 'permission' in str(selected_reason).lower()
            or 'access denied' in str(selected_reason).lower()
        ):
            try:
                result += f"\n🔍 RUNNING COMPREHENSIVE IAM ANALYSIS (common cause of '{selected_reason}'):\n"

                # 1. Check IAM role and policies
                iam_analysis = await analyze_iam_role_and_policies(canary, iam_client, region)

                # Display IAM analysis results
                result += f'IAM Role Analysis Status: {iam_analysis["status"]}\n'
                for check_name, check_result in iam_analysis.get('checks', {}).items():
                    result += f'• {check_name}: {check_result}\n'

                # 2. ENHANCED: Check resource ARN correctness with detailed validation
                result += '\n🔍 CHECKING RESOURCE ARN CORRECTNESS:\n'
                arn_check = check_resource_arns_correct(canary, iam_client)

                if arn_check.get('correct'):
                    result += '✅ Resource ARNs: Correct\n'
                else:
                    result += f'❌ Resource ARNs: {arn_check.get("error", "Issues found")}\n'

                # Combine all IAM issues with enhanced categorization
                all_iam_issues = []
                if iam_analysis.get('issues_found'):
                    all_iam_issues.extend(
                        [f'IAM Policy: {issue}' for issue in iam_analysis['issues_found']]
                    )
                if not arn_check.get('correct') and arn_check.get('issues'):
                    all_iam_issues.extend(
                        [f'Resource ARN: {issue}' for issue in arn_check['issues']]
                    )

                if all_iam_issues:
                    result += f'\n🚨 ALL IAM ISSUES FOUND ({len(all_iam_issues)} total):\n'
                    for issue in all_iam_issues:
                        result += f'• {issue}\n'

                # Enhanced IAM recommendations with priority
                all_iam_recommendations = []
                if iam_analysis.get('recommendations'):
                    all_iam_recommendations.extend(
                        [f'Policy Fix: {rec}' for rec in iam_analysis['recommendations']]
                    )
                if not arn_check.get('correct'):
                    all_iam_recommendations.extend(
                        [
                            'PRIORITY: Review and correct S3 bucket ARN patterns in IAM policies',
                            'PRIORITY: Ensure bucket names match expected patterns (e.g., cw-syn-* for CloudWatch Synthetics)',
                            'Verify canary has access to the correct S3 bucket for artifacts storage',
                            'Check if bucket exists and is in the same region as the canary',
                        ]
                    )

                if all_iam_recommendations:
                    result += (
                        f'\n💡 ALL IAM RECOMMENDATIONS ({len(all_iam_recommendations)} total):\n'
                    )
                    for rec in all_iam_recommendations:
                        result += f'• {rec}\n'

            except Exception as iam_error:
                result += f'⚠️ IAM analysis failed: {str(iam_error)[:200]}\n\n'

        # History-based diagnosis for specific error patterns
        error_recommendations = []

        # 1. ENOSPC: no space left on device
        if any(
            re.search(pattern, selected_reason, re.IGNORECASE)
            for pattern in ['enospc', 'no space left on device']
        ):
            try:
                telemetry_data = await extract_disk_memory_usage_metrics(canary_name, region)
                if 'error' not in telemetry_data:
                    result += '\n🔍 DISK USAGE ROOT CAUSE ANALYSIS:\n'
                    result += f'• Storage: {telemetry_data.get("maxEphemeralStorageUsageInMb", 0):.1f} MB peak\n'
                    result += f'• Usage: {telemetry_data.get("maxEphemeralStorageUsagePercent", 0):.1f}% peak\n'
                else:
                    result += f'\n🔍 DISK USAGE ROOT CAUSE ANALYSIS:\n{telemetry_data["error"]}\n'
            except Exception as debug_error:
                result += f'\n⚠️ Could not generate disk usage debugging code: {str(debug_error)}\n'

        # 2. Protocol error (Target.activateTarget): Session closed / detached Frame
        elif any(
            re.search(pattern, selected_reason, re.IGNORECASE)
            for pattern in [
                'protocol error',
                'target.activatetarget',
                'session closed',
                'detached frame',
                'session already detached',
            ]
        ):
            try:
                telemetry_data = await extract_disk_memory_usage_metrics(canary_name, region)
                if 'error' not in telemetry_data:
                    result += '\n🔍 MEMORY USAGE ROOT CAUSE ANALYSIS:\n'
                    result += f'• Memory: {telemetry_data.get("maxSyntheticsMemoryUsageInMB", 0):.1f} MB peak\n'
                else:
                    result += (
                        f'\n🔍 MEMORY USAGE ROOT CAUSE ANALYSIS:\n{telemetry_data["error"]}\n'
                    )
            except Exception as debug_error:
                result += f'\n⚠️ Could not collect memory usage metrics: {str(debug_error)}\n'

        # 3. Navigation timed out / Page.captureScreenshot timed out
        elif any(
            re.search(pattern, selected_reason, re.IGNORECASE)
            for pattern in [
                'navigation timeout',
                'navigation timed out',
                'ms exceeded',
                'page.capturescreenshot timed out',
                'protocoltimeout',
                'connection timed out',
            ]
        ):
            # Navigation timeout specific analysis using existing HAR data
            if har_files and bucket_name:
                try:
                    har_timeout_analysis = await analyze_har_file(
                        s3_client, bucket_name, har_files, is_failed_run=True
                    )

                    result += '\n🔍 HAR FILE ANALYSIS FOR NAVIGATION TIMEOUT:\n'
                    if har_timeout_analysis.get('failed_requests', 0) > 0:
                        result += (
                            f'• Failed HTTP requests: {har_timeout_analysis["failed_requests"]}\n'
                        )

                    if har_timeout_analysis.get('insights'):
                        for insight in har_timeout_analysis['insights'][:5]:
                            result += f'• {insight}\n'

                    # Additional timeout-specific analysis
                    result += f'• Total requests analyzed: {har_timeout_analysis.get("total_requests", 0)}\n'
                    result += (
                        f'• Analysis status: {har_timeout_analysis.get("status", "unknown")}\n'
                    )
                    result += '\n'
                except Exception as har_error:
                    result += f'\n⚠️ HAR analysis failed: {str(har_error)[:100]}\n'
            else:
                result += '\n🔍 NAVIGATION TIMEOUT DETECTED:\n'
                result += '• No HAR files available for detailed analysis\n'
                result += '• Timeout suggests page loading issues or UI changes\n'
                result += '• Check if target elements exist and page loads completely\n\n'

        # 4. Visual variation
        elif re.search('visual variation', selected_reason, re.IGNORECASE):
            error_recommendations.extend(
                [
                    '🔧 VISUAL MONITORING ISSUE DETECTED:',
                    '• Website UI changed - not a technical failure',
                    '• Check if website legitimately updated (ads, banners, content)',
                    '• Update visual baseline with new reference screenshots',
                    '• Adjust visual difference threshold (increase from default)',
                    '• Consider excluding dynamic content areas from comparison',
                ]
            )

        if error_recommendations:
            result += '\n💡 PATTERN-BASED RECOMMENDATIONS:\n'
            for rec in error_recommendations:
                result += f'{rec}\n'
            result += '\n'

        # Add canary code if available
        try:
            code_analysis = await get_canary_code(canary, region)
            if 'error' not in code_analysis and code_analysis.get('code_content'):
                result += f'\ncanary code:\n{code_analysis["code_content"]}\n'
        except Exception as e:
            result += f'Note: Could not retrieve canary code: {str(e)}\n'

        # --- Knowledge Base Recommendations ---
        try:
            # Build environment_indicators from canary tags and category context
            kb_env_indicators: list[str] = []
            canary_tags = canary.get('Tags', {})
            if isinstance(canary_tags, dict):
                kb_env_indicators.extend(canary_tags.values())
            elif isinstance(canary_tags, list):
                for tag in canary_tags:
                    if isinstance(tag, dict):
                        kb_env_indicators.append(tag.get('Value', ''))

            # Build error_messages: StateReason + all collected detailed errors + user description
            kb_error_messages: list[str] = [selected_reason]
            if description:
                # Cap description length to mitigate slow regex matching in KB patterns
                kb_error_messages.append(
                    description[:500] if len(description) > 500 else description
                )
            for msg in all_collected_error_messages:
                if msg and msg not in kb_error_messages:
                    kb_error_messages.append(msg)

            failure_context = FailureContext(
                error_messages=kb_error_messages,
                state_reasons=[selected_reason],
                runtime_version=canary.get('RuntimeVersion', ''),
                log_patterns=all_collected_log_patterns,
                environment_indicators=kb_env_indicators,
            )
            engine = CanaryRecommendationEngine(await CanaryKnowledgeBaseLoader.get_instance())
            recommendations = engine.get_recommendations(failure_context)
            if recommendations:
                result += engine.format_recommendations(recommendations)
        except Exception as e:
            logger.warning(f'Knowledge base recommendation failed: {e}')

        result += '\n'
        return result

    except Exception as e:
        return f'❌ Error in comprehensive failure analysis: {str(e)}'


@mcp.tool()
async def list_canaries(region: str = AWS_REGION, max_results: int = 20) -> str:
    """List all CloudWatch Synthetics canaries in the account.

    Use this tool to discover canaries before analyzing them with analyze_canary_failures().
    Returns canary names, status, schedule, runtime version, and last run state.

    Args:
        region: AWS region to query (defaults to configured region).
        max_results: Maximum number of canaries to display (default: 20, max: 200).

    Returns:
        Formatted list of all canaries with their current status and configuration.
    """
    from botocore.exceptions import ClientError

    max_results = min(max(max_results, 1), 200)

    try:
        canaries: list = []
        paginator_token = None

        while True:
            kwargs: dict = {'MaxResults': 20}
            if paginator_token:
                kwargs['NextToken'] = paginator_token

            response = synthetics_client.describe_canaries(**kwargs)
            canaries.extend(response.get('Canaries', []))
            paginator_token = response.get('NextToken')
            if not paginator_token or len(canaries) >= max_results:
                break

        if not canaries:
            return 'No canaries found in this account/region.'

        total_fetched = len(canaries)
        display_canaries = canaries[:max_results]

        result = f'Found {total_fetched} canaries'
        if total_fetched > max_results:
            result += f' (showing first {max_results})'
        result += ':\n\n'

        for c in display_canaries:
            name = c.get('Name', 'Unknown')
            status_obj = c.get('Status', {})
            state = status_obj.get('State', 'Unknown')
            state_reason = status_obj.get('StateReason', '')
            schedule_expr = c.get('Schedule', {}).get('Expression', 'N/A')
            runtime = c.get('RuntimeVersion', 'N/A')

            last_run = c.get('Timeline', {})
            last_started = last_run.get('LastStarted', 'Never')

            if state == 'RUNNING':
                emoji = '🟢'
            elif state == 'STOPPED':
                emoji = '🔴'
            elif state == 'ERROR':
                emoji = '🟠'
            else:
                emoji = '⚪'

            result += f'{emoji} {name}\n'
            result += f'   State: {state}'
            if state_reason:
                result += f' ({state_reason})'
            result += '\n'
            result += f'   Schedule: {schedule_expr}\n'
            result += f'   Runtime: {runtime}\n'
            result += f'   Last started: {last_started}\n'
            result += '\n'

        result += (
            'Use analyze_canary_failures(canary_name="<name>") to investigate a specific canary.'
        )
        return result

    except ClientError as e:
        return f'Error listing canaries: {e}'
    except Exception as e:
        return f'Error listing canaries: {str(e)}'


# Register all imported tools with the MCP server
mcp.tool()(list_monitored_services)
mcp.tool()(get_service_detail)
mcp.tool()(query_service_metrics)
mcp.tool()(list_service_operations)
mcp.tool()(get_slo)
mcp.tool()(list_slos)
mcp.tool()(search_transaction_spans)
mcp.tool()(query_sampled_traces)
mcp.tool()(list_slis)
mcp.tool()(get_enablement_guide)
mcp.tool()(list_change_events)
mcp.tool()(list_group_services)
mcp.tool()(audit_group_health)
mcp.tool()(get_group_dependencies)
mcp.tool()(get_group_changes)
mcp.tool()(list_grouping_attribute_definitions)

# RUM tools
mcp.tool()(query_rum_events)


def main():
    """Run the MCP server."""
    logger.debug('Starting CloudWatch Application Signals MCP server')
    try:
        mcp.run(transport='stdio')
    except KeyboardInterrupt:
        logger.debug('Server shutdown by user')
    except Exception as e:
        logger.error(f'Server error: {e}', exc_info=True)
        raise


if __name__ == '__main__':
    main()
