feat: Allow RegisteredMetricHandler to build payloads without a full inline metric definition

PiperOrigin-RevId: 885215504

Author: vertex-sdk-bot

tests/unit/vertexai/genai/replays/test_evaluate.py

@@ -353,7 +353,7 @@ def test_evaluation_agent_data(client):
         assert case_result.response_candidate_results is not None
 
 
-def test_metric_resource_name(client):
+def test_evaluation_metric_resource_name(client):
     """Tests with a metric resource name in types.Metric."""
     client._api_client._http_options.api_version = "v1beta1"
     client._api_client._http_options.base_url = (
@@ -375,8 +375,22 @@ def test_metric_resource_name(client):
     )
     assert isinstance(evaluation_result, types.EvaluationResult)
     assert evaluation_result.eval_case_results is not None
-    assert len(evaluation_result.eval_case_results) > 0
+    assert len(evaluation_result.eval_case_results) == 1
     assert evaluation_result.summary_metrics[0].metric_name == "my_custom_metric"
+    assert evaluation_result.summary_metrics[0].mean_score is not None
+    assert evaluation_result.summary_metrics[0].num_cases_valid == 1
+    assert evaluation_result.summary_metrics[0].num_cases_error == 0
+
+    case_result = evaluation_result.eval_case_results[0]
+    assert case_result.response_candidate_results is not None
+    assert len(case_result.response_candidate_results) == 1
+
+    metric_result = case_result.response_candidate_results[0].metric_results[
+        "my_custom_metric"
+    ]
+    assert metric_result.score is not None
+    assert metric_result.score > 0.5
+    assert metric_result.error_message is None
 
 
 pytestmark = pytest_helper.setup(

vertexai/_genai/_evals_metric_handlers.py

@@ -1281,74 +1281,78 @@ def aggregate(
         )
 
 
-class RegisteredMetricHandler(MetricHandler[types.MetricSource]):
+class RegisteredMetricHandler(MetricHandler[types.Metric]):
     """Metric handler for registered metrics."""
 
     def __init__(
         self,
         module: "evals.Evals",
-        metric: Union[types.MetricSource, types.MetricSourceDict],
+        metric: types.Metric,
     ):
         if isinstance(metric, dict):
             metric = types.MetricSource(**metric)
         super().__init__(module=module, metric=metric)
 
-    # TODO: b/489823454 - Unify _build_request_payload with PredefinedMetricHandler.
     def _build_request_payload(
         self, eval_case: types.EvalCase, response_index: int
     ) -> dict[str, Any]:
-        """Builds request payload for registered metric."""
-        if not self.metric.metric:
+        """Builds request payload for registered metric by assembling EvaluationInstance."""
+        response_content = _get_response_from_eval_case(
+            eval_case, response_index, self.metric_name
+        )
+
+        if not response_content and not getattr(eval_case, "agent_data", None):
             raise ValueError(
-                "Registered metric must have an underlying metric definition."
+                f"Response content missing for candidate {response_index}."
             )
-        return PredefinedMetricHandler(
-            self.module, metric=self.metric.metric
-        )._build_request_payload(eval_case, response_index)
+
+        reference_instance_data = None
+        if eval_case.reference:
+            reference_instance_data = PredefinedMetricHandler._content_to_instance_data(
+                eval_case.reference.response
+            )
+
+        extracted_prompt = _get_prompt_from_eval_case(eval_case)
+        prompt_instance_data = PredefinedMetricHandler._content_to_instance_data(
+            extracted_prompt
+        )
+
+        instance_payload = types.EvaluationInstance(
+            prompt=prompt_instance_data,
+            response=PredefinedMetricHandler._content_to_instance_data(
+                response_content
+            ),
+            reference=reference_instance_data,
+            rubric_groups=eval_case.rubric_groups,
+            agent_data=PredefinedMetricHandler._eval_case_to_agent_data(eval_case),
+        )
+
+        request_payload = {
+            "instance": instance_payload,
+        }
+        return request_payload
 
     @property
     def metric_name(self) -> str:
-        # Resolve name from resource name or internal metric name
-        if isinstance(self.metric, types.MetricSource):
-            if self.metric.metric and self.metric.metric.name:
-                return self.metric.metric.name
-            if self.metric.metric_resource_name:
-                return self.metric.metric_resource_name
-            return "unknown"
-        else:  # Should be Metric
-            metric_like = self.metric
-            if metric_like.name:
-                return metric_like.name
-            if metric_like.metric_resource_name:
-                return metric_like.metric_resource_name
-            return "unknown"
+        return self.metric.name
 
     @override
     def get_metric_result(
         self, eval_case: types.EvalCase, response_index: int
     ) -> types.EvalCaseMetricResult:
-        """Processes a single evaluation case for a registered metric."""
+        """Processes a single evaluation case using a MetricSource reference."""
         metric_name = self.metric_name
+        metric_source = types.MetricSource(
+            metric_resource_name=self.metric.metric_resource_name
+        )
+
         try:
             payload = self._build_request_payload(eval_case, response_index)
-            for attempt in range(_MAX_RETRIES):
-                try:
-                    api_response = self.module._evaluate_instances(
-                        metric_sources=[self.metric],
-                        instance=payload.get("instance"),
-                        autorater_config=payload.get("autorater_config"),
-                    )
-                    break
-                except genai_errors.ClientError as e:
-                    if e.code == 429:
-                        if attempt == _MAX_RETRIES - 1:
-                            return types.EvalCaseMetricResult(
-                                metric_name=metric_name,
-                                error_message=f"Judge model resource exhausted after {_MAX_RETRIES} retries: {e}",
-                            )
-                        time.sleep(2**attempt)
-                    else:
-                        raise e
+            api_response = self.module._evaluate_instances(
+                metric_sources=[metric_source],
+                instance=payload.get("instance"),
+                autorater_config=payload.get("autorater_config"),
+            )
 
             if api_response and api_response.metric_results:
                 result_data = api_response.metric_results[0]
@@ -1377,7 +1381,6 @@ def aggregate(
         self, eval_case_metric_results: list[types.EvalCaseMetricResult]
     ) -> types.AggregatedMetricResult:
         """Aggregates the metric results for a registered metric."""
-        logger.debug("Aggregating results for registered metric: %s", self.metric_name)
         return _default_aggregate_scores(
             self.metric_name, eval_case_metric_results, calculate_pass_rate=True
         )