Skip to content

Commit 2779e55

Browse files
vertex-sdk-botcopybara-github
authored andcommitted
feat: Add support for passing other data and agent data for custom code execution metric
PiperOrigin-RevId: 886400009
1 parent 0cff2d8 commit 2779e55

1 file changed

Lines changed: 22 additions & 3 deletions

File tree

vertexai/_genai/_evals_metric_handlers.py

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1296,7 +1296,7 @@ def __init__(
12961296
def _build_request_payload(
12971297
self, eval_case: types.EvalCase, response_index: int
12981298
) -> dict[str, Any]:
1299-
"""Builds request payload for registered metric by assembling EvaluationInstance."""
1299+
"""Builds the request parameters for evaluate instances request."""
13001300
response_content = _get_response_from_eval_case(
13011301
eval_case, response_index, self.metric_name
13021302
)
@@ -1317,20 +1317,39 @@ def _build_request_payload(
13171317
extracted_prompt
13181318
)
13191319

1320+
other_data_map: dict[str, Any] = {}
1321+
if hasattr(eval_case, "context") and eval_case.context:
1322+
if isinstance(eval_case.context, str):
1323+
other_data_map["context"] = types.evals.InstanceData(
1324+
text=eval_case.context
1325+
)
1326+
elif isinstance(eval_case.context, genai_types.Content):
1327+
other_data_map["context"] = (
1328+
PredefinedMetricHandler._content_to_instance_data(eval_case.context)
1329+
)
1330+
else:
1331+
logger.warning(
1332+
f"Unsupported type for context: {type(eval_case.context)}"
1333+
)
1334+
13201335
instance_payload = types.EvaluationInstance(
13211336
prompt=prompt_instance_data,
13221337
response=PredefinedMetricHandler._content_to_instance_data(
13231338
response_content
13241339
),
13251340
reference=reference_instance_data,
13261341
rubric_groups=eval_case.rubric_groups,
1342+
other_data=(
1343+
types.MapInstance(map_instance=other_data_map)
1344+
if other_data_map
1345+
else None
1346+
),
13271347
agent_data=PredefinedMetricHandler._eval_case_to_agent_data(eval_case),
13281348
)
13291349

1330-
request_payload = {
1350+
return {
13311351
"instance": instance_payload,
13321352
}
1333-
return request_payload
13341353

13351354
@property
13361355
def metric_name(self) -> str:

0 commit comments

Comments
 (0)