# Copyright 2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# SPDX-License-Identifier: Apache-2.0

"""Tests for Vertex AI Evaluators."""

from unittest.mock import AsyncMock, MagicMock, patch

import pytest

from genkit.plugins.google_genai.evaluators import (
    VertexAIEvaluationMetricType,
    create_vertex_evaluators,
)
from genkit.plugins.google_genai.evaluators.evaluation import (
    EvaluatorFactory,
    VertexAIEvaluationMetricConfig,
    _is_config,
    _stringify,
)


def test_vertex_ai_evaluation_metric_type_values() -> None:
    """Test that VertexAIEvaluationMetricType has expected values."""
    assert VertexAIEvaluationMetricType.BLEU == 'BLEU'
    assert VertexAIEvaluationMetricType.ROUGE == 'ROUGE'
    assert VertexAIEvaluationMetricType.FLUENCY == 'FLUENCY'
    assert VertexAIEvaluationMetricType.SAFETY == 'SAFETY'
    assert VertexAIEvaluationMetricType.GROUNDEDNESS == 'GROUNDEDNESS'
    assert VertexAIEvaluationMetricType.SUMMARIZATION_QUALITY == 'SUMMARIZATION_QUALITY'
    assert VertexAIEvaluationMetricType.SUMMARIZATION_HELPFULNESS == 'SUMMARIZATION_HELPFULNESS'
    assert VertexAIEvaluationMetricType.SUMMARIZATION_VERBOSITY == 'SUMMARIZATION_VERBOSITY'


def test_vertex_ai_evaluation_metric_type_is_str_enum() -> None:
    """Test that metric types can be used as strings."""
    metric = VertexAIEvaluationMetricType.FLUENCY
    assert isinstance(metric, str)
    assert metric == 'FLUENCY'


def test_vertex_ai_evaluation_metric_config_basic() -> None:
    """Test VertexAIEvaluationMetricConfig model."""
    config = VertexAIEvaluationMetricConfig(
        type=VertexAIEvaluationMetricType.BLEU,
        metric_spec={'use_sentence_level': True},
    )
    assert config.type == VertexAIEvaluationMetricType.BLEU
    assert config.metric_spec == {'use_sentence_level': True}


def test_vertex_ai_evaluation_metric_config_defaults() -> None:
    """Test VertexAIEvaluationMetricConfig default values."""
    config = VertexAIEvaluationMetricConfig(type=VertexAIEvaluationMetricType.SAFETY)
    assert config.type == VertexAIEvaluationMetricType.SAFETY
    assert config.metric_spec is None


def test_stringify_string_input() -> None:
    """Test _stringify with string input returns unchanged."""
    result = _stringify('hello world')
    assert result == 'hello world'


def test_stringify_dict_input() -> None:
    """Test _stringify with dict input returns JSON."""
    result = _stringify({'key': 'value'})
    assert result == '{"key": "value"}'


def test_stringify_list_input() -> None:
    """Test _stringify with list input returns JSON."""
    result = _stringify(['a', 'b', 'c'])
    assert result == '["a", "b", "c"]'


def test_stringify_number_input() -> None:
    """Test _stringify with number input returns JSON."""
    result = _stringify(42)
    assert result == '42'


def test_is_config_with_metric_type() -> None:
    """Test _is_config returns False for metric type."""
    metric = VertexAIEvaluationMetricType.FLUENCY
    assert _is_config(metric) is False


def test_is_config_with_metric_config() -> None:
    """Test _is_config returns True for metric config."""
    config = VertexAIEvaluationMetricConfig(type=VertexAIEvaluationMetricType.FLUENCY)
    assert _is_config(config) is True


def test_evaluator_factory_initialization() -> None:
    """Test EvaluatorFactory can be initialized."""
    factory = EvaluatorFactory(
        project_id='test-project',
        location='us-central1',
    )
    assert factory.project_id == 'test-project'
    assert factory.location == 'us-central1'


@pytest.mark.asyncio
async def test_evaluator_factory_evaluate_instances_structure() -> None:
    """Test that evaluate_instances makes correct API call structure."""
    factory = EvaluatorFactory(
        project_id='test-project',
        location='us-central1',
    )

    mock_credentials = MagicMock()
    mock_credentials.token = 'mock-token'
    mock_credentials.expired = False

    mock_response_data = {
        'fluencyResult': {
            'score': 4.5,
            'explanation': 'Very fluent text',
        }
    }

    with patch('genkit.plugins.google_genai.evaluators.evaluation.google_auth_default') as mock_auth:
        mock_auth.return_value = (mock_credentials, 'test-project')

        # Mock get_cached_client to return a mock client
        mock_client = AsyncMock()
        mock_response = MagicMock()
        mock_response.status_code = 200
        mock_response.json.return_value = mock_response_data
        mock_client.post = AsyncMock(return_value=mock_response)
        mock_client.is_closed = False

        with patch('genkit.plugins.google_genai.evaluators.evaluation.get_cached_client', return_value=mock_client):
            result = await factory.evaluate_instances({'fluencyInput': {'prediction': 'Test'}})

            assert result == mock_response_data
            mock_client.post.assert_called_once()


@pytest.mark.asyncio
async def test_evaluator_factory_evaluate_instances_error_handling() -> None:
    """Test that evaluate_instances raises GenkitError on API failure."""
    from genkit._core._error import GenkitError

    factory = EvaluatorFactory(
        project_id='test-project',
        location='us-central1',
    )

    mock_credentials = MagicMock()
    mock_credentials.token = 'mock-token'
    mock_credentials.expired = False

    with patch('genkit.plugins.google_genai.evaluators.evaluation.google_auth_default') as mock_auth:
        mock_auth.return_value = (mock_credentials, 'test-project')

        # Mock get_cached_client to return a mock client
        mock_client = AsyncMock()
        mock_response = MagicMock()
        mock_response.status_code = 500
        mock_response.text = 'Internal Server Error'
        mock_client.post = AsyncMock(return_value=mock_response)
        mock_client.is_closed = False

        with patch('genkit.plugins.google_genai.evaluators.evaluation.get_cached_client', return_value=mock_client):
            with pytest.raises(GenkitError) as exc_info:
                await factory.evaluate_instances({'input': 'test'})

            assert exc_info.value.status == 'INTERNAL'


def test_create_vertex_evaluators_with_metric_types() -> None:
    """Test create_vertex_evaluators with simple metric types."""
    mock_registry = MagicMock()
    mock_registry.define_evaluator = MagicMock()

    metrics = [
        VertexAIEvaluationMetricType.FLUENCY,
        VertexAIEvaluationMetricType.SAFETY,
    ]

    create_vertex_evaluators(
        registry=mock_registry,
        metrics=metrics,
        project_id='test-project',
        location='us-central1',
    )

    assert mock_registry.define_evaluator.call_count == 2


def test_create_vertex_evaluators_with_metric_configs() -> None:
    """Test create_vertex_evaluators with metric configs."""
    mock_registry = MagicMock()
    mock_registry.define_evaluator = MagicMock()

    metrics = [
        VertexAIEvaluationMetricConfig(
            type=VertexAIEvaluationMetricType.BLEU,
            metric_spec={'use_sentence_level': True},
        ),
    ]

    create_vertex_evaluators(
        registry=mock_registry,
        metrics=metrics,
        project_id='test-project',
        location='us-central1',
    )

    mock_registry.define_evaluator.assert_called_once()


def test_create_vertex_evaluators_names_format() -> None:
    """Test that evaluator names follow vertexai/{metric} format."""
    mock_registry = MagicMock()
    evaluator_names: list[str] = []

    def capture_name(*args: object, **kwargs: object) -> None:
        if 'name' in kwargs:
            name = kwargs['name']
            if isinstance(name, str):
                evaluator_names.append(name)

    mock_registry.define_evaluator = capture_name

    metrics = [
        VertexAIEvaluationMetricType.FLUENCY,
        VertexAIEvaluationMetricType.GROUNDEDNESS,
    ]

    create_vertex_evaluators(
        registry=mock_registry,
        metrics=metrics,
        project_id='test-project',
        location='us-central1',
    )

    assert 'vertexai/fluency' in evaluator_names
    assert 'vertexai/groundedness' in evaluator_names


def test_create_vertex_evaluators_empty_metrics() -> None:
    """Test create_vertex_evaluators with empty metrics list."""
    mock_registry = MagicMock()
    mock_registry.define_evaluator = MagicMock()

    create_vertex_evaluators(
        registry=mock_registry,
        metrics=[],
        project_id='test-project',
        location='us-central1',
    )

    mock_registry.define_evaluator.assert_not_called()


def test_all_metric_types_supported() -> None:
    """Test that all metric types are supported by create_vertex_evaluators."""
    mock_registry = MagicMock()
    mock_registry.define_evaluator = MagicMock()

    all_metrics = list(VertexAIEvaluationMetricType)

    create_vertex_evaluators(
        registry=mock_registry,
        metrics=all_metrics,
        project_id='test-project',
        location='us-central1',
    )

    assert mock_registry.define_evaluator.call_count == len(all_metrics)
