fix: GenAI Client(evals): Remove prompt filled by starting prompt

PiperOrigin-RevId: 882785935

Author: vertex-sdk-bot

tests/unit/vertexai/genai/test_evals.py

@@ -5961,6 +5961,9 @@ def test_generate_user_scenarios(self):
         assert eval_dataset.eval_dataset_df is not None
         assert len(eval_dataset.eval_dataset_df) == 2
         assert eval_dataset.eval_dataset_df.iloc[0]["starting_prompt"] == "Prompt 1"
+        assert eval_dataset.eval_dataset_df.iloc[0]["conversation_plan"] == "Plan 1"
+        assert eval_dataset.eval_dataset_df.iloc[1]["starting_prompt"] == "Prompt 2"
+        assert eval_dataset.eval_dataset_df.iloc[1]["conversation_plan"] == "Plan 2"
 
         self.mock_api_client.request.assert_called_once()
 
@@ -5984,10 +5987,87 @@ async def test_async_generate_user_scenarios(self):
 
         assert eval_dataset.eval_dataset_df is not None
         assert len(eval_dataset.eval_dataset_df) == 2
+        assert eval_dataset.eval_dataset_df.iloc[0]["starting_prompt"] == "Prompt 1"
+        assert eval_dataset.eval_dataset_df.iloc[0]["conversation_plan"] == "Plan 1"
+        assert eval_dataset.eval_dataset_df.iloc[1]["starting_prompt"] == "Prompt 2"
+        assert eval_dataset.eval_dataset_df.iloc[1]["conversation_plan"] == "Plan 2"
 
         self.mock_api_client.async_request.assert_called_once()
 
 
+class TestConvertRequestToDatasetRow:
+    """Unit tests for the _convert_request_to_dataset_row function."""
+
+    def test_convert_request_to_dataset_row_with_prompt_and_golden(self):
+        request = vertexai_genai_types.EvaluationItemRequest(
+            prompt=vertexai_genai_types.EvaluationPrompt(text="test prompt"),
+            golden_response=vertexai_genai_types.CandidateResponse(
+                text="golden response"
+            ),
+        )
+        result = _evals_common._convert_request_to_dataset_row(request)
+        assert result["prompt"] == "test prompt"
+        assert result["reference"] == vertexai_genai_types.CandidateResponse(
+            text="golden response"
+        )
+        assert result["intermediate_events"] == []
+        assert result["agent_data"] is None
+
+    def test_convert_request_to_dataset_row_with_user_scenario(self):
+        request = vertexai_genai_types.EvaluationItemRequest(
+            prompt=vertexai_genai_types.EvaluationPrompt(
+                user_scenario=vertexai_genai_types.evals.UserScenario(
+                    starting_prompt="start prompt", conversation_plan="convo plan"
+                )
+            )
+        )
+        result = _evals_common._convert_request_to_dataset_row(request)
+        assert result["starting_prompt"] == "start prompt"
+        assert result["conversation_plan"] == "convo plan"
+        assert result["prompt"] is None
+
+    def test_convert_request_to_dataset_row_with_candidate_events(self):
+        request = vertexai_genai_types.EvaluationItemRequest(
+            candidate_responses=[
+                vertexai_genai_types.CandidateResponse(
+                    candidate="test-candidate",
+                    text="candidate text",
+                    events=[
+                        genai_types.Content(
+                            parts=[genai_types.Part(text="event part")], role="model"
+                        )
+                    ],
+                )
+            ]
+        )
+        result = _evals_common._convert_request_to_dataset_row(request)
+        assert result["test-candidate"] == "candidate text"
+        assert result["intermediate_events"] == [
+            {
+                "event_id": "test-candidate",
+                "content": {
+                    "parts": [genai_types.Part(text="event part")],
+                    "role": "model",
+                },
+            }
+        ]
+        assert result["agent_data"] is None
+
+    def test_convert_request_to_dataset_row_with_agent_data(self):
+        mock_agent_data = vertexai_genai_types.evals.AgentData(turns=[])
+        request = vertexai_genai_types.EvaluationItemRequest(
+            candidate_responses=[
+                vertexai_genai_types.CandidateResponse(
+                    candidate="test-candidate", agent_data=mock_agent_data
+                )
+            ]
+        )
+        result = _evals_common._convert_request_to_dataset_row(request)
+        assert result["test-candidate"] is None
+        assert result["agent_data"] == mock_agent_data
+        assert result["intermediate_events"] == []
+
+
 class TestCreateEvaluationSetFromDataFrame:
     """Unit tests for the _create_evaluation_set_from_dataframe function."""
 

vertexai/_genai/_evals_common.py

@@ -2026,7 +2026,17 @@ def _convert_request_to_dataset_row(
         request.prompt.text if request.prompt and request.prompt.text else None
     )
     dict_row[_evals_constant.REFERENCE] = request.golden_response
+
+    if request.prompt and request.prompt.user_scenario:
+        dict_row[_evals_constant.STARTING_PROMPT] = (
+            request.prompt.user_scenario.starting_prompt
+        )
+        dict_row[_evals_constant.CONVERSATION_PLAN] = (
+            request.prompt.user_scenario.conversation_plan
+        )
+
     intermediate_events = []
+    agent_data = None
     if request.candidate_responses:
         for candidate in request.candidate_responses:
             if candidate.candidate is not None:
@@ -2041,7 +2051,10 @@ def _convert_request_to_dataset_row(
                             "content": content_dict,
                         }
                         intermediate_events.append(int_events_dict)
+        agent_data = request.candidate_responses[0].agent_data
+
     dict_row[_evals_constant.INTERMEDIATE_EVENTS] = intermediate_events
+    dict_row[_evals_constant.AGENT_DATA] = agent_data
     return dict_row
 
 

vertexai/_genai/_evals_utils.py

@@ -347,13 +347,7 @@ def _postprocess_user_scenarios_response(
     data_for_df = []
     if hasattr(response, "user_scenarios") and response.user_scenarios:
         for scenario in response.user_scenarios:
-            prompt_content = None
-            if scenario.starting_prompt:
-                prompt_content = genai_types.Content(
-                    parts=[genai_types.Part(text=scenario.starting_prompt)]
-                )
             eval_case = types.EvalCase(
-                prompt=prompt_content,
                 user_scenario=scenario,
             )
             eval_cases.append(eval_case)