feat: Add support for referencing registered metrics by resource name in evaluation run API

PiperOrigin-RevId: 878604099

Author: vertex-sdk-bot

vertexai/_genai/_evals_common.py

@@ -45,6 +45,7 @@
 from . import _gcs_utils
 from . import evals
 from . import types
+from . import _transformers as t
 
 logger = logging.getLogger(__name__)
 
@@ -1320,16 +1321,14 @@ def _resolve_dataset_inputs(
 
 
 def _resolve_evaluation_run_metrics(
-    metrics: list[types.EvaluationRunMetric], api_client: Any
+    metrics: list[Any], api_client: Any
 ) -> list[types.EvaluationRunMetric]:
     """Resolves a list of evaluation run metric instances, loading RubricMetric if necessary."""
     if not metrics:
         return []
     resolved_metrics_list = []
     for metric_instance in metrics:
-        if isinstance(metric_instance, types.EvaluationRunMetric):
-            resolved_metrics_list.append(metric_instance)
-        elif isinstance(
+        if isinstance(
             metric_instance, _evals_metric_loaders.LazyLoadedPrebuiltMetric
         ):
             try:
@@ -1353,6 +1352,18 @@ def _resolve_evaluation_run_metrics(
                     e,
                 )
                 raise
+        elif isinstance(metric_instance, types.EvaluationRunMetric):
+            resolved_metrics_list.append(metric_instance)
+        elif isinstance(metric_instance, types.Metric):
+            config_dict = t.t_metrics([metric_instance])[0]
+            res_name = config_dict.pop("metric_resource_name", None)
+            resolved_metrics_list.append(
+                types.EvaluationRunMetric(
+                    metric=metric_instance.name,
+                    metric_config=config_dict,
+                    metric_resource_name=res_name,
+                )
+            )
         else:
             try:
                 metric_name_str = str(metric_instance)

vertexai/_genai/_transformers.py

@@ -38,6 +38,8 @@ def t_metrics(
 
     for metric in metrics:
         metric_payload_item: dict[str, Any] = {}
+        if hasattr(metric, "metric_resource_name") and metric.metric_resource_name:
+            metric_payload_item["metric_resource_name"] = metric.metric_resource_name
 
         metric_name = getv(metric, ["name"]).lower()
 
@@ -79,6 +81,9 @@ def t_metrics(
                     "return_raw_output": return_raw_output
                 }
             metric_payload_item["pointwise_metric_spec"] = pointwise_spec
+        elif "metric_resource_name" in metric_payload_item:
+            # Valid case: Metric is identified by resource name; no inline spec required.
+            pass
         else:
             raise ValueError(
                 f"Unsupported metric type or invalid metric name: {metric_name}"

vertexai/_genai/evals.py

@@ -399,6 +399,13 @@ def _EvaluationRunMetric_from_vertex(
             _UnifiedMetric_from_vertex(getv(from_object, ["metricConfig"]), to_object),
         )
 
+    if getv(from_object, ["metricResourceName"]) is not None:
+        setv(
+            to_object,
+            ["metric_resource_name"],
+            getv(from_object, ["metricResourceName"]),
+        )
+
     return to_object
 
 
@@ -417,6 +424,13 @@ def _EvaluationRunMetric_to_vertex(
             _UnifiedMetric_to_vertex(getv(from_object, ["metric_config"]), to_object),
         )
 
+    if getv(from_object, ["metric_resource_name"]) is not None:
+        setv(
+            to_object,
+            ["metricResourceName"],
+            getv(from_object, ["metric_resource_name"]),
+        )
+
     return to_object
 
 

vertexai/_genai/types/common.py

@@ -2326,6 +2326,9 @@ class LLMBasedMetricSpec(_common.BaseModel):
         default=None,
         description="""Dynamically generate rubrics using this specification.""",
     )
+    metric_resource_name: Optional[str] = Field(
+        default=None, description="""The resource name of the metric definition. Example: projects/{project}/locations/{location}/evaluationMetrics/{evaluation_metric_id}"""
+    )
 
 
 class LLMBasedMetricSpecDict(TypedDict, total=False):
@@ -2350,6 +2353,9 @@ class LLMBasedMetricSpecDict(TypedDict, total=False):
     rubric_generation_spec: Optional[RubricGenerationSpecDict]
     """Dynamically generate rubrics using this specification."""
 
+    metric_resource_name: Optional[str]
+    """The resource name of the metric definition. Example: projects/{project}/locations/{location}/evaluationMetrics/{evaluation_metric_id}"""
+
 
 LLMBasedMetricSpecOrDict = Union[LLMBasedMetricSpec, LLMBasedMetricSpecDict]
 
@@ -2482,6 +2488,9 @@ class EvaluationRunMetric(_common.BaseModel):
     metric_config: Optional[UnifiedMetric] = Field(
         default=None, description="""The unified metric used for evaluation run."""
     )
+    metric_resource_name: Optional[str] = Field(
+        default=None, description="""The resource name of the metric definition. Example: projects/{project}/locations/{location}/evaluationMetrics/{evaluation_metric_id}"""
+    )
 
 
 class EvaluationRunMetricDict(TypedDict, total=False):
@@ -2493,6 +2502,9 @@ class EvaluationRunMetricDict(TypedDict, total=False):
     metric_config: Optional[UnifiedMetricDict]
     """The unified metric used for evaluation run."""
 
+    metric_resource_name: Optional[str]
+    """The resource name of the metric definition. Example: projects/{project}/locations/{location}/evaluationMetrics/{evaluation_metric_id}"""
+
 
 EvaluationRunMetricOrDict = Union[EvaluationRunMetric, EvaluationRunMetricDict]
 
@@ -4439,6 +4451,9 @@ class Metric(_common.BaseModel):
         default=None,
         description="""Optional steering instruction parameters for the automated predefined metric.""",
     )
+    metric_resource_name: Optional[str] = Field(
+        default=None, description="""The resource name of the metric definition. Example: projects/{project}/locations/{location}/evaluationMetrics/{evaluation_metric_id}"""
+    )
 
     # Allow extra fields to support metric-specific config fields.
     model_config = ConfigDict(extra="allow")
@@ -4643,6 +4658,9 @@ class MetricDict(TypedDict, total=False):
     metric_spec_parameters: Optional[dict[str, Any]]
     """Optional steering instruction parameters for the automated predefined metric."""
 
+    metric_resource_name: Optional[str]
+    """The resource name of the metric definition. Example: projects/{project}/locations/{location}/evaluationMetrics/{evaluation_metric_id}"""
+
 
 MetricOrDict = Union[Metric, MetricDict]