feat: Allow using registered metric resource names in evaluation

PiperOrigin-RevId: 880868820

Author: vertex-sdk-bot

tests/unit/vertexai/genai/replays/test_create_evaluation_run.py

@@ -238,6 +238,28 @@ def test_create_eval_run_with_inference_configs(client):
     assert evaluation_run.error is None
 
 
+def test_create_eval_run_with_metric_resource_name(client):
+    """Tests create_evaluation_run with metric_resource_name."""
+    client._api_client._http_options.api_version = "v1beta1"
+    client._api_client._http_options.base_url = (
+        "https://us-central1-autopush-aiplatform.sandbox.googleapis.com/"
+    )
+    metric_resource_name = "projects/977012026409/locations/us-central1/evaluationMetrics/6048334299558576128"
+    metric = types.EvaluationRunMetric(
+        metric="my_custom_metric",
+        metric_resource_name=metric_resource_name,
+    )
+    evaluation_run = client.evals.create_evaluation_run(
+        dataset=types.EvaluationDataset(
+            eval_dataset_df=INPUT_DF_WITH_CONTEXT_AND_HISTORY
+        ),
+        metrics=[metric],
+        dest=GCS_DEST,
+    )
+    assert isinstance(evaluation_run, types.EvaluationRun)
+    assert evaluation_run.evaluation_config.metrics[0].metric == "my_custom_metric"
+
+
 # Dataframe tests fail in replay mode because of UUID generation mismatch.
 # def test_create_eval_run_data_source_evaluation_dataset(client):
 #     """Tests that create_evaluation_run() creates a correctly structured

tests/unit/vertexai/genai/replays/test_public_generate_rubrics.py

@@ -143,19 +143,21 @@
 User prompt:
 {prompt}"""
 
+_PROMPTS_DF = pd.DataFrame(
+    {
+        "prompt": [
+            "Explain the theory of relativity in one sentence.",
+            "Write a short poem about a cat.",
+        ]
+    }
+)
+
 
 def test_public_method_generate_rubrics(client):
     """Tests the public generate_rubrics method."""
-    prompts_df = pd.DataFrame(
-        {
-            "prompt": [
-                "Explain the theory of relativity in one sentence.",
-                "Write a short poem about a cat.",
-            ]
-        }
-    )
+
     eval_dataset = client.evals.generate_rubrics(
-        src=prompts_df,
+        src=_PROMPTS_DF,
         prompt_template=_TEST_RUBRIC_GENERATION_PROMPT,
         rubric_group_name="text_quality_rubrics",
     )
@@ -176,6 +178,36 @@ def test_public_method_generate_rubrics(client):
     assert isinstance(first_rubric_group["text_quality_rubrics"][0], types.evals.Rubric)
 
 
+def test_public_method_generate_rubrics_with_metric(client):
+    """Tests the public generate_rubrics method with a metric."""
+    client._api_client._http_options.api_version = "v1beta1"
+    client._api_client._http_options.base_url = (
+        "https://us-central1-staging-aiplatform.sandbox.googleapis.com/"
+    )
+    metric_resource_name = "projects/977012026409/locations/us-central1/evaluationMetrics/6048334299558576128"
+    metric = types.Metric(
+        name="my_custom_metric", metric_resource_name=metric_resource_name
+    )
+    eval_dataset = client.evals.generate_rubrics(
+        src=_PROMPTS_DF, rubric_group_name="my_registered_rubrics", metric=metric
+    )
+    eval_dataset_df = eval_dataset.eval_dataset_df
+
+    assert isinstance(eval_dataset, types.EvaluationDataset)
+    assert isinstance(eval_dataset_df, pd.DataFrame)
+    assert "rubric_groups" in eval_dataset_df.columns
+    assert len(eval_dataset_df) == 2
+
+    first_rubric_group = eval_dataset_df["rubric_groups"][0]
+    assert isinstance(first_rubric_group, dict)
+    assert "my_registered_rubrics" in first_rubric_group
+    assert isinstance(first_rubric_group["my_registered_rubrics"], list)
+    assert first_rubric_group["my_registered_rubrics"]
+    assert isinstance(
+        first_rubric_group["my_registered_rubrics"][0], types.evals.Rubric
+    )
+
+
 pytestmark = pytest_helper.setup(
     file=__file__,
     globals_for_file=globals(),

vertexai/_genai/_evals_common.py

@@ -45,6 +45,7 @@
 from . import _gcs_utils
 from . import evals
 from . import types
+from . import _transformers as t
 
 logger = logging.getLogger(__name__)
 
@@ -1328,7 +1329,7 @@ def _resolve_dataset_inputs(
 
 
 def _resolve_evaluation_run_metrics(
-    metrics: list[types.EvaluationRunMetric], api_client: Any
+    metrics: Union[list[types.EvaluationRunMetric], list[types.Metric]], api_client: Any
 ) -> list[types.EvaluationRunMetric]:
     """Resolves a list of evaluation run metric instances, loading RubricMetric if necessary."""
     if not metrics:
@@ -1361,6 +1362,16 @@ def _resolve_evaluation_run_metrics(
                     e,
                 )
                 raise
+        elif isinstance(metric_instance, types.Metric):
+            config_dict = t.t_metrics([metric_instance])[0]
+            res_name = config_dict.pop("metric_resource_name", None)
+            resolved_metrics_list.append(
+                types.EvaluationRunMetric(
+                    metric=metric_instance.name,
+                    metric_config=config_dict if config_dict else None,
+                    metric_resource_name=res_name,
+                )
+            )
         else:
             try:
                 metric_name_str = str(metric_instance)

vertexai/_genai/_evals_metric_handlers.py

@@ -1027,7 +1027,7 @@ def get_metric_result(
             for attempt in range(_MAX_RETRIES):
                 try:
                     api_response = self.module._evaluate_instances(
-                        metrics=[self.metric],
+                        metrics_sources=[self.metric],
                         instance=payload.get("instance"),
                         autorater_config=payload.get("autorater_config"),
                     )
@@ -1164,7 +1164,7 @@ def get_metric_result(
             for attempt in range(_MAX_RETRIES):
                 try:
                     api_response = self.module._evaluate_instances(
-                        metrics=[self.metric],
+                        metrics_sources=[self.metric],
                         instance=payload.get("instance"),
                     )
                     break
@@ -1242,6 +1242,14 @@ def aggregate(
         )
 
 
+class RegisteredMetricHandler(PredefinedMetricHandler):
+    """Metric handler for registered metrics."""
+
+    def __init__(self, module: "evals.Evals", metric: types.Metric):
+        # Skip the parent check for SUPPORTED_PREDEFINED_METRICS
+        MetricHandler.__init__(self, module=module, metric=metric)
+
+
 _METRIC_HANDLER_MAPPING = [
     (
         lambda m: hasattr(m, "remote_custom_function") and m.remote_custom_function,
@@ -1251,6 +1259,10 @@ def aggregate(
         lambda m: m.custom_function and isinstance(m.custom_function, Callable),
         CustomMetricHandler,
     ),
+    (
+        lambda m: getattr(m, "metric_resource_name", None) is not None,
+        RegisteredMetricHandler,
+    ),
     (
         lambda m: m.name in ComputationMetricHandler.SUPPORTED_COMPUTATION_METRICS,
         ComputationMetricHandler,

vertexai/_genai/_transformers.py

@@ -14,6 +14,7 @@
 #
 
 """Transformers module for Vertex addons."""
+import re
 from typing import Any
 
 from google.genai._common import get_value_by_path as getv
@@ -31,15 +32,26 @@ def t_metrics(
     Args:
         metrics: A list of metrics used for evaluation.
         set_default_aggregation_metrics: Whether to set default aggregation metrics.
+
     Returns:
         A list of resolved metric payloads for the evaluation request.
     """
     metrics_payload = []
 
     for metric in metrics:
+        # Case 1: Registered Metric Resource Name
+        if isinstance(metric, str) and re.match(
+            r"^projects/[^/]+/location/[^/]+/evaluationMetric/[^/]+$", metric
+        ):
+            metrics_payload.append({"metric_resource_name": metric})
+            continue
+
+        # Case 2: Inline Metric Configuration
         metric_payload_item: dict[str, Any] = {}
+        if hasattr(metric, "metric_resource_name") and metric.metric_resource_name:
+            metric_payload_item["metric_resource_name"] = metric.metric_resource_name
 
-        metric_name = getv(metric, ["name"]).lower()
+        metric_name = getattr(metric, "name", "").lower()
 
         if set_default_aggregation_metrics:
             metric_payload_item["aggregation_metrics"] = [
@@ -79,9 +91,36 @@ def t_metrics(
                     "return_raw_output": return_raw_output
                 }
             metric_payload_item["pointwise_metric_spec"] = pointwise_spec
+        elif "metric_resource_name" in metric_payload_item:
+            # Valid case: Metric is identified by resource name; no inline spec required.
+            pass
         else:
             raise ValueError(
                 f"Unsupported metric type or invalid metric name: {metric_name}"
             )
-        metrics_payload.append(metric_payload_item)
+        metrics_payload.append({"metric": metric_payload_item})
     return metrics_payload
+
+
+def t_metric_sources(metrics: list[Any]) -> list[dict[str, Any]]:
+    """Prepares the MetricSource payload for the evaluation request."""
+    sources_payload = []
+    for metric in metrics:
+        # Check if the 'metric' is a resource name string or contains one
+        resource_name = getattr(metric, "metric_resource_name", None)
+        if (
+            not resource_name
+            and isinstance(metric, str)
+            and re.match(
+                r"^projects/[^/]+/location/[^/]+/evaluationMetric/[^/]+$", metric
+            )
+        ):
+            resource_name = metric
+
+        if resource_name:
+            sources_payload.append({"metric_resource_name": resource_name})
+        else:
+            # Fallback to existing Metric spec transformation
+            metric_payload = t_metrics([metric])[0]
+            sources_payload.append({"metric": metric_payload})
+    return sources_payload