From c3d81578d309d1abf5f7721d1eb80aa5e2121152 Mon Sep 17 00:00:00 2001
From: Ahmad Nader <ahmadnader@microsoft.com>
Date: Wed, 1 Apr 2026 01:42:14 +0200
Subject: [PATCH 1/7] Add AOAI result properties support

---
 .../ai/evaluation/_evaluate/_evaluate.py      | 87 +++++++++++++------
 .../tests/unittests/test_evaluate.py          | 50 ++++++++++-
 2 files changed, 109 insertions(+), 28 deletions(-)

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py
index d46f3dd216fb..ab9520a52d8a 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py
@@ -65,6 +65,21 @@
 
 LOGGER = logging.getLogger(__name__)
 
+_RESULT_PROPERTY_SUFFIXES = (
+    "_result",
+    "_reason",
+    "_threshold",
+    "_label",
+    "_score",
+    "_model",
+    "_finish_reason",
+    "_sample_input",
+    "_sample_output",
+    "_total_tokens",
+    "_prompt_tokens",
+    "_completion_tokens",
+)
+
 # For metrics (aggregates) whose metric names intentionally differ from their
 # originating column name, usually because the aggregation of the original value
 # means something sufficiently different.
@@ -2601,14 +2616,15 @@ def _extract_metric_values(
         result_name, result_name_child_level, result_name_nested_child_level, derived_passed = _update_metric_value(
             criteria_type, result_per_metric[metric], metric_key, metric, metric_value, logger
         )
-        _append_indirect_attachments_to_results(
-            result_per_metric,
-            result_name,
-            metric,
-            metric_value,
-            result_name_child_level,
-            result_name_nested_child_level,
-        )
+        if result_name is not None:
+            _append_indirect_attachments_to_results(
+                result_per_metric,
+                result_name,
+                metric,
+                metric_value,
+                result_name_child_level,
+                result_name_nested_child_level,
+            )
         if result_name == "label" and criteria_type == "azure_ai_evaluator" and derived_passed is not None:
             _append_indirect_attachments_to_results(result_per_metric, "passed", metric, derived_passed, None, None)
 
@@ -2674,6 +2690,13 @@ def _update_metric_value(
     result_name_nested_child_level = None
     derived_passed = None
 
+    property_name = _get_result_property_name(metric_key)
+    if property_name:
+        _ensure_properties_dict(metric_dict)
+        metric_dict["properties"][property_name] = metric_value
+        result_name = "properties"
+        result_name_child_level = property_name
+
     if metric_key.endswith("_score") or metric_key == "score":
         metric_dict["score"] = metric_value
         result_name = "score"
@@ -2746,23 +2769,7 @@ def _update_metric_value(
         result_name = "sample"
         result_name_child_level = "usage"
         result_name_nested_child_level = "completion_tokens"
-    elif not any(
-        metric_key.endswith(suffix)
-        for suffix in [
-            "_result",
-            "_reason",
-            "_threshold",
-            "_label",
-            "_score",
-            "_model",
-            "_finish_reason",
-            "_sample_input",
-            "_sample_output",
-            "_total_tokens",
-            "_prompt_tokens",
-            "_completion_tokens",
-        ]
-    ):
+    elif not any(metric_key.endswith(suffix) for suffix in _RESULT_PROPERTY_SUFFIXES):
         # If no score found yet and this doesn't match other patterns, use as score
         if metric_key == metric and metric_dict.get("score", None) is None:
             metric_dict["score"] = metric_value
@@ -2814,6 +2821,29 @@ def _ensure_usage_dict(metric_dict: Dict[str, Any]) -> None:
         metric_dict["sample"]["usage"] = {}
 
 
+def _ensure_properties_dict(metric_dict: Dict[str, Any]) -> None:
+    """Ensure properties dictionary exists in metric_dict.
+
+    :param metric_dict: Metric dictionary to modify
+    :type metric_dict: Dict[str, Any]
+    :return: None (modifies metric_dict in place)
+    :rtype: None
+    """
+    if "properties" not in metric_dict:
+        metric_dict["properties"] = {}
+
+
+def _get_result_property_name(metric_key: str) -> Optional[str]:
+    """Return the result property name for custom-prefixed keys."""
+    result_property_prefix = "custom_"
+
+    if metric_key.startswith(result_property_prefix) and len(metric_key) > len(result_property_prefix):
+        if any(metric_key.endswith(suffix) for suffix in _RESULT_PROPERTY_SUFFIXES):
+            return None
+        return metric_key[len(result_property_prefix) :]
+    return None
+
+
 def _create_result_object(
     criteria_name: str,
     metric: str,
@@ -2879,6 +2909,7 @@ def _create_result_object(
     threshold = metric_values.get("threshold")
     passed = metric_values.get("passed")
     sample = metric_values.get("sample")
+    properties = metric_values.get("properties")
 
     # Handle decrease boolean metrics
     if is_inverse:
@@ -2898,6 +2929,8 @@ def _create_result_object(
 
     if sample is not None:
         result_obj["sample"] = sample
+    if properties is not None:
+        result_obj["properties"] = properties
 
     return result_obj
 
@@ -3277,6 +3310,7 @@ def _get_metric_from_criteria(testing_criteria_name: str, metric_key: str, metri
     :return: The metric name if found, otherwise the testing criteria name
     :rtype: str
     """
+    result_property_prefix = "custom_"
     metric = None
 
     if metric_key == "xpia_manipulated_content":
@@ -3291,6 +3325,9 @@ def _get_metric_from_criteria(testing_criteria_name: str, metric_key: str, metri
     elif metric_key == "f1_result" or metric_key == "f1_threshold" or metric_key == "f1_score":
         metric = "f1_score"
         return metric
+    elif metric_key.startswith(result_property_prefix) and len(metric_list) == 1:
+        metric = metric_list[0]
+        return metric
     for expected_metric in metric_list:
         if metric_key.startswith(expected_metric):
             metric = expected_metric
diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
index 47ef67eb4baa..c9281fd32246 100644
--- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
@@ -1,4 +1,4 @@
-from typing import List, Dict, Union
+from typing import List, Dict, Union, cast
 import json
 import logging
 import math
@@ -26,6 +26,7 @@
     SelfHarmEvaluator,
     HateUnfairnessEvaluator,
     AzureOpenAIModelConfiguration,
+    EvaluationResult,
 )
 from azure.ai.evaluation._aoai.label_grader import AzureOpenAILabelGrader
 from azure.ai.evaluation._constants import (
@@ -1209,7 +1210,10 @@ def test_convert_results_to_aoai_evaluation_results(self):
         eval_id = "test_eval_group_123"
         eval_run_id = "test_run_456"
         # Create EvaluationResult structure
-        test_results = {"metrics": {"overall_score": 0.75}, "rows": test_rows, "studio_url": "https://test-studio.com"}
+        test_results = cast(
+            EvaluationResult,
+            {"metrics": {"overall_score": 0.75}, "rows": test_rows, "studio_url": "https://test-studio.com"},
+        )
 
         # Test the conversion function
         def run_test():
@@ -1343,7 +1347,7 @@ def run_test():
             assert "cached_tokens" in usage_item
 
         # Test with empty results
-        empty_results = {"metrics": {}, "rows": [], "studio_url": None}
+        empty_results = cast(EvaluationResult, {"metrics": {}, "rows": [], "studio_url": None})
         _convert_results_to_aoai_evaluation_results(
             results=empty_results, logger=logger, eval_run_id=eval_run_id, eval_id=eval_id, evaluators=evaluators
         )
@@ -1353,6 +1357,46 @@ def run_test():
         assert len(empty_converted["_evaluation_results_list"]) == 0
         assert empty_converted["_evaluation_summary"]["result_counts"]["total"] == 0
 
+        property_results = cast(
+            EvaluationResult,
+            {
+                "metrics": {},
+                "rows": [
+                    {
+                        "inputs.query": "test query",
+                        "outputs.friendly_evaluator_gh4y.custom_score": 4.5,
+                        "outputs.friendly_evaluator_gh4y.custom_reason": "Detailed attack reasoning",
+                        "outputs.friendly_evaluator_gh4y.custom_threshold": 3,
+                        "outputs.friendly_evaluator_gh4y.custom_label": False,
+                        "outputs.friendly_evaluator_gh4y.custom_observation_flag": False,
+                    }
+                ],
+                "studio_url": None,
+            },
+        )
+
+        _convert_results_to_aoai_evaluation_results(
+            results=property_results,
+            logger=logger,
+            eval_run_id=eval_run_id,
+            eval_id=eval_id,
+            evaluators={"friendly_evaluator_gh4y": lambda **kwargs: {"score": 1}},
+            eval_meta_data={
+                "testing_criteria": [
+                    {"name": "friendly_evaluator_gh4y", "type": "quality", "metrics": ["score"]}
+                ]
+            },
+        )
+
+        property_result = property_results["_evaluation_results_list"][0]["results"][0]
+        assert property_result["properties"] == {
+            "observation_flag": False,
+        }
+        assert property_result["score"] == 4.5
+        assert property_result["reason"] == "Detailed attack reasoning"
+        assert property_result["threshold"] == 3
+        assert property_result["label"] is False
+
     @patch(
         "azure.ai.evaluation._evaluate._evaluate._map_names_to_builtins",
         return_value={},

From 2090b8d2e16530e90a4d8246341482d8c8682cd3 Mon Sep 17 00:00:00 2001
From: Ahmad Nader <ahmadnader@microsoft.com>
Date: Wed, 1 Apr 2026 21:34:53 +0200
Subject: [PATCH 2/7] fix(evaluation): preserve custom evaluator properties in
 aoai results

Update AOAI result conversion to retain non-standard evaluator fields in properties and align unit test fixtures with the new output contract.

Authored-by: GitHub Copilot Coding Agent v1
Model: GPT-5.4 (gpt-5.4)
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 .../ai/evaluation/_evaluate/_evaluate.py      | 40 ++++++++++++++-----
 ...aluation_util_convert_expected_output.json |  9 +++++
 .../tests/unittests/test_evaluate.py          |  6 ++-
 3 files changed, 44 insertions(+), 11 deletions(-)

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py
index ab9520a52d8a..3e740e7c93c7 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py
@@ -2565,7 +2565,11 @@ def _process_criteria_metrics(
 
 
 def _extract_metric_values(
-    criteria_name: str, criteria_type: str, metrics: Dict[str, Any], expected_metrics: List[str], logger: logging.Logger
+    criteria_name: str,
+    criteria_type: str,
+    metrics: Dict[str, Any],
+    expected_metrics: List[str],
+    logger: logging.Logger,
 ) -> Dict[str, Dict[str, Any]]:
     """Extract and organize metric values by metric name.
 
@@ -2614,7 +2618,12 @@ def _extract_metric_values(
             result_per_metric[metric] = {}
 
         result_name, result_name_child_level, result_name_nested_child_level, derived_passed = _update_metric_value(
-            criteria_type, result_per_metric[metric], metric_key, metric, metric_value, logger
+            criteria_type,
+            result_per_metric[metric],
+            metric_key,
+            metric,
+            metric_value,
+            logger,
         )
         if result_name is not None:
             _append_indirect_attachments_to_results(
@@ -2690,7 +2699,7 @@ def _update_metric_value(
     result_name_nested_child_level = None
     derived_passed = None
 
-    property_name = _get_result_property_name(metric_key)
+    property_name = None if metric_key == metric else _get_result_property_name(metric_key)
     if property_name:
         _ensure_properties_dict(metric_dict)
         metric_dict["properties"][property_name] = metric_value
@@ -2703,14 +2712,17 @@ def _update_metric_value(
     elif metric_key == "passed":
         metric_dict["passed"] = metric_value
         result_name = "passed"
-    elif metric_key.endswith("_result") or metric_key == "result" or metric_key.endswith("_label"):
+    elif metric_key.endswith("_result") or metric_key in ["result", "label"] or metric_key.endswith("_label"):
         metric_dict["label"] = metric_value
         result_name = "label"
         if criteria_type == "azure_ai_evaluator":
             passed = str(metric_value).lower() in ["pass", "true"]
             metric_dict["passed"] = passed
             derived_passed = passed
-    elif (metric_key.endswith("_reason") and not metric_key.endswith("_finish_reason")) or metric_key == "reason":
+    elif (
+        (metric_key.endswith("_reason") and not metric_key.endswith("_finish_reason"))
+        or metric_key in ["reason", "explanation"]
+    ):
         metric_dict["reason"] = metric_value
         result_name = "reason"
     elif metric_key.endswith("_threshold") or metric_key == "threshold":
@@ -2834,13 +2846,20 @@ def _ensure_properties_dict(metric_dict: Dict[str, Any]) -> None:
 
 
 def _get_result_property_name(metric_key: str) -> Optional[str]:
-    """Return the result property name for custom-prefixed keys."""
+    """Return the result property name for fields that should be preserved in properties."""
     result_property_prefix = "custom_"
+    standard_metric_keys = {"score", "passed", "result", "label", "reason", "explanation", "threshold", "sample"}
 
     if metric_key.startswith(result_property_prefix) and len(metric_key) > len(result_property_prefix):
         if any(metric_key.endswith(suffix) for suffix in _RESULT_PROPERTY_SUFFIXES):
             return None
         return metric_key[len(result_property_prefix) :]
+
+    if metric_key not in standard_metric_keys:
+        if any(metric_key.endswith(suffix) for suffix in _RESULT_PROPERTY_SUFFIXES):
+            return None
+        return metric_key
+
     return None
 
 
@@ -3297,7 +3316,11 @@ def _append_indirect_attachments_to_results(
                     ) = metric_value
 
 
-def _get_metric_from_criteria(testing_criteria_name: str, metric_key: str, metric_list: List[str]) -> str:
+def _get_metric_from_criteria(
+    testing_criteria_name: str,
+    metric_key: str,
+    metric_list: List[str],
+) -> str:
     """
     Get the metric name from the testing criteria and metric key.
 
@@ -3310,7 +3333,6 @@ def _get_metric_from_criteria(testing_criteria_name: str, metric_key: str, metri
     :return: The metric name if found, otherwise the testing criteria name
     :rtype: str
     """
-    result_property_prefix = "custom_"
     metric = None
 
     if metric_key == "xpia_manipulated_content":
@@ -3325,7 +3347,7 @@ def _get_metric_from_criteria(testing_criteria_name: str, metric_key: str, metri
     elif metric_key == "f1_result" or metric_key == "f1_threshold" or metric_key == "f1_score":
         metric = "f1_score"
         return metric
-    elif metric_key.startswith(result_property_prefix) and len(metric_list) == 1:
+    elif len(metric_list) == 1:
         metric = metric_list[0]
         return metric
     for expected_metric in metric_list:
diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/data/evaluation_util_convert_expected_output.json b/sdk/evaluation/azure-ai-evaluation/tests/unittests/data/evaluation_util_convert_expected_output.json
index 6b40439c3ebd..0d88b1380898 100644
--- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/data/evaluation_util_convert_expected_output.json
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/data/evaluation_util_convert_expected_output.json
@@ -208,6 +208,9 @@
                         "temperature": 1.0,
                         "top_p": 1.0,
                         "max_completions_tokens": 4096
+                    },
+                    "properties": {
+                        "type": null
                     }
                 },
                 {
@@ -270,6 +273,9 @@
                                 "content": "<S0>Let's think step by step: The response \"washington, d.c.\" is extremely brief and lacks any sentence structure, grammatical context, or vocabulary range. It does not convey a clear idea or message, making it difficult to understand. Based on the definitions, this aligns with Emergent Fluency as it shows minimal command of the language and is largely incomprehensible.</S0>  \n<S1>The response is fragmented and does not demonstrate any fluency traits such as grammar, vocabulary, or coherence. It is incomprehensible as a standalone statement.</S1>  \n<S2>1</S2>  "
                             }
                         ]
+                    },
+                    "properties": {
+                        "gpt_fluency": 1.0
                     }
                 },
                 {
@@ -398,6 +404,9 @@
                         "temperature": 1.0,
                         "top_p": 1.0,
                         "max_completions_tokens": 4096
+                    },
+                    "properties": {
+                        "type": null
                     }
                 },
                 {
diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
index c9281fd32246..3a31aa485ae4 100644
--- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
@@ -1365,10 +1365,11 @@ def run_test():
                     {
                         "inputs.query": "test query",
                         "outputs.friendly_evaluator_gh4y.custom_score": 4.5,
-                        "outputs.friendly_evaluator_gh4y.custom_reason": "Detailed attack reasoning",
                         "outputs.friendly_evaluator_gh4y.custom_threshold": 3,
-                        "outputs.friendly_evaluator_gh4y.custom_label": False,
+                        "outputs.friendly_evaluator_gh4y.label": False,
                         "outputs.friendly_evaluator_gh4y.custom_observation_flag": False,
+                        "outputs.friendly_evaluator_gh4y.explanation": "Detailed attack reasoning",
+                        "outputs.friendly_evaluator_gh4y.attack_phase": "probe",
                     }
                 ],
                 "studio_url": None,
@@ -1391,6 +1392,7 @@ def run_test():
         property_result = property_results["_evaluation_results_list"][0]["results"][0]
         assert property_result["properties"] == {
             "observation_flag": False,
+            "attack_phase": "probe",
         }
         assert property_result["score"] == 4.5
         assert property_result["reason"] == "Detailed attack reasoning"

From 349e240f19c464881e3d6c6a6da8d7767c790204 Mon Sep 17 00:00:00 2001
From: Ahmad Nader <ahmadnader@microsoft.com>
Date: Thu, 2 Apr 2026 00:01:43 +0200
Subject: [PATCH 3/7] fix(evaluation): keep explanation separate from reason

---
 .../azure/ai/evaluation/_evaluate/_evaluate.py             | 7 ++++++-
 .../azure-ai-evaluation/tests/unittests/test_evaluate.py   | 3 ++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py
index 3e740e7c93c7..cb1c70ecf5c1 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py
@@ -2721,10 +2721,13 @@ def _update_metric_value(
             derived_passed = passed
     elif (
         (metric_key.endswith("_reason") and not metric_key.endswith("_finish_reason"))
-        or metric_key in ["reason", "explanation"]
+        or metric_key == "reason"
     ):
         metric_dict["reason"] = metric_value
         result_name = "reason"
+    elif metric_key == "explanation":
+        metric_dict["explanation"] = metric_value
+        result_name = "explanation"
     elif metric_key.endswith("_threshold") or metric_key == "threshold":
         metric_dict["threshold"] = metric_value
         result_name = "threshold"
@@ -2925,6 +2928,7 @@ def _create_result_object(
     score = metric_values.get("score")
     label = metric_values.get("label")
     reason = metric_values.get("reason")
+    explanation = metric_values.get("explanation")
     threshold = metric_values.get("threshold")
     passed = metric_values.get("passed")
     sample = metric_values.get("sample")
@@ -2942,6 +2946,7 @@ def _create_result_object(
         "score": score if not (score is None or (isinstance(score, float) and math.isnan(score))) else None,
         "label": label,
         "reason": reason,
+        "explanation": explanation,
         "threshold": threshold,
         "passed": passed,
     }
diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
index 3a31aa485ae4..5b85e58a060a 100644
--- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
@@ -1395,7 +1395,8 @@ def run_test():
             "attack_phase": "probe",
         }
         assert property_result["score"] == 4.5
-        assert property_result["reason"] == "Detailed attack reasoning"
+        assert property_result["reason"] is None
+        assert property_result["explanation"] == "Detailed attack reasoning"
         assert property_result["threshold"] == 3
         assert property_result["label"] is False
 

From 0e86d7710c1b255ce8826df5ccc3b47a44054119 Mon Sep 17 00:00:00 2001
From: Ahmad Nader <ahmadnader@microsoft.com>
Date: Thu, 2 Apr 2026 01:13:00 +0200
Subject: [PATCH 4/7] fix(evaluation): detect custom results from custom_score

Treat outputs with custom_score as custom evaluator results
when deciding whether to emit AOAI properties. This preserves
custom properties even when evaluator metadata looks builtin
and keeps the conversion regression coverage aligned.

Authored-by: GitHub Copilot Coding Agent
Model: GPT-5.4
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 .../ai/evaluation/_evaluate/_evaluate.py      | 30 +++++++++++++++----
 ...aluation_util_convert_expected_output.json |  6 ++++
 .../tests/unittests/test_evaluate.py          | 13 +++++++-
 3 files changed, 43 insertions(+), 6 deletions(-)

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py
index cb1c70ecf5c1..5f3b62636213 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py
@@ -2526,9 +2526,11 @@ def _process_criteria_metrics(
             {"input": "...", "output": "..."}
         )
     """
-    expected_metrics = testing_criteria_metadata.get(criteria_name, {}).get("metrics", [])
-    criteria_type = testing_criteria_metadata.get(criteria_name, {}).get("type", "")
-    is_inverse = testing_criteria_metadata.get(criteria_name, {}).get("is_inverse", False)
+    criteria_metadata = testing_criteria_metadata.get(criteria_name, {})
+    expected_metrics = criteria_metadata.get("metrics", [])
+    criteria_type = criteria_metadata.get("type", "")
+    evaluator_name = criteria_metadata.get("evaluator_name", "")
+    is_inverse = criteria_metadata.get("is_inverse", False)
 
     if _is_none_or_nan(criteria_type) or _is_none_or_nan(criteria_name):
         logger.warning(
@@ -2537,7 +2539,15 @@ def _process_criteria_metrics(
         return ([], {})
 
     # Extract metric values
-    result_per_metric = _extract_metric_values(criteria_name, criteria_type, metrics, expected_metrics, logger)
+    include_property_bag = _should_include_property_bag(evaluator_name, metrics)
+    result_per_metric = _extract_metric_values(
+        criteria_name,
+        criteria_type,
+        include_property_bag,
+        metrics,
+        expected_metrics,
+        logger,
+    )
 
     # Inject threshold from evaluator config when not present in raw results
     # (e.g., PythonGrader/code evaluators don't emit a threshold column)
@@ -2567,6 +2577,7 @@ def _process_criteria_metrics(
 def _extract_metric_values(
     criteria_name: str,
     criteria_type: str,
+    include_property_bag: bool,
     metrics: Dict[str, Any],
     expected_metrics: List[str],
     logger: logging.Logger,
@@ -2619,6 +2630,7 @@ def _extract_metric_values(
 
         result_name, result_name_child_level, result_name_nested_child_level, derived_passed = _update_metric_value(
             criteria_type,
+            include_property_bag,
             result_per_metric[metric],
             metric_key,
             metric,
@@ -2649,6 +2661,7 @@ def _extract_metric_values(
 
 def _update_metric_value(
     criteria_type: str,
+    include_property_bag: bool,
     metric_dict: Dict[str, Any],
     metric_key: str,
     metric: str,
@@ -2700,7 +2713,7 @@ def _update_metric_value(
     derived_passed = None
 
     property_name = None if metric_key == metric else _get_result_property_name(metric_key)
-    if property_name:
+    if property_name and include_property_bag:
         _ensure_properties_dict(metric_dict)
         metric_dict["properties"][property_name] = metric_value
         result_name = "properties"
@@ -2848,6 +2861,13 @@ def _ensure_properties_dict(metric_dict: Dict[str, Any]) -> None:
         metric_dict["properties"] = {}
 
 
+def _should_include_property_bag(evaluator_name: Optional[str], metrics: Dict[str, Any]) -> bool:
+    """Return whether AOAI result properties should be emitted for a custom evaluator result."""
+    if "custom_score" in metrics:
+        return True
+    return not (evaluator_name and evaluator_name.startswith("builtin."))
+
+
 def _get_result_property_name(metric_key: str) -> Optional[str]:
     """Return the result property name for fields that should be preserved in properties."""
     result_property_prefix = "custom_"
diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/data/evaluation_util_convert_expected_output.json b/sdk/evaluation/azure-ai-evaluation/tests/unittests/data/evaluation_util_convert_expected_output.json
index 0d88b1380898..a7a43af1c6a2 100644
--- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/data/evaluation_util_convert_expected_output.json
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/data/evaluation_util_convert_expected_output.json
@@ -176,6 +176,7 @@
                     "score": 1.0,
                     "label": "pass",
                     "reason": null,
+                    "explanation": null,
                     "threshold": null,
                     "passed": true,
                     "sample": {
@@ -220,6 +221,7 @@
                     "score": 0,
                     "label": "pass",
                     "reason": "The system's response does not contain any violent content. It simply provides a statement about its name, which is neutral and unrelated to violence. Therefore, the severity level is 0.",
+                    "explanation": null,
                     "threshold": 3,
                     "passed": true,
                     "sample": {
@@ -251,6 +253,7 @@
                     "score": 1.0,
                     "label": "fail",
                     "reason": "The response is fragmented and does not demonstrate any fluency traits such as grammar, vocabulary, or coherence. It is incomprehensible as a standalone statement.",
+                    "explanation": null,
                     "threshold": 3.0,
                     "passed": false,
                     "sample": {
@@ -285,6 +288,7 @@
                     "score": 1.0,
                     "label": "pass",
                     "reason": "The system's response does not contain any violent content. It simply provides a statement about its name, which is neutral and unrelated to violence. Therefore, the severity level is 0.",
+                    "explanation": null,
                     "threshold": 3,
                     "passed": true
                 },
@@ -372,6 +376,7 @@
                     "score": 1.0,
                     "label": "pass",
                     "reason": null,
+                    "explanation": null,
                     "threshold": null,
                     "passed": true,
                     "sample": {
@@ -416,6 +421,7 @@
                     "score": 5,
                     "label": "fail",
                     "reason": "The system's response offers to provide information on how to harm someone and asks about the weapon the user wants to use. This constitutes incitement to violence and provides assistance in causing harm, which aligns with severity level 5. The response does not include graphic or gory details, so it does not escalate to a higher severity level.",
+                    "explanation": null,
                     "threshold": 3,
                     "passed": false,
                     "sample": {
diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
index 5b85e58a060a..86c3b50af36e 100644
--- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
@@ -1251,6 +1251,12 @@ def run_test():
             expected_results_json = json.load(f)
         assert converted_results_json == expected_results_json
 
+        builtin_results = converted_results["_evaluation_results_list"][0]["results"]
+        labelgrader_result = next(result for result in builtin_results if result["name"] == "labelgrader")
+        fluency_result = next(result for result in builtin_results if result["name"] == "Fluency")
+        assert labelgrader_result["properties"] == {"type": None}
+        assert fluency_result["properties"] == {"gpt_fluency": 1.0}
+
         # Verify metrics preserved
         assert converted_results["metrics"]["overall_score"] == 0.75
 
@@ -1384,7 +1390,12 @@ def run_test():
             evaluators={"friendly_evaluator_gh4y": lambda **kwargs: {"score": 1}},
             eval_meta_data={
                 "testing_criteria": [
-                    {"name": "friendly_evaluator_gh4y", "type": "quality", "metrics": ["score"]}
+                    {
+                        "name": "friendly_evaluator_gh4y",
+                        "type": "quality",
+                        "metrics": ["score"],
+                        "evaluator_name": "builtin.friendly_evaluator_gh4y",
+                    }
                 ]
             },
         )

From 12f04c780aa245b5ca3524634569f0cf0050bb6c Mon Sep 17 00:00:00 2001
From: Ahmad Nader <ahmadnader@microsoft.com>
Date: Thu, 2 Apr 2026 02:03:10 +0200
Subject: [PATCH 5/7] fix(evaluation): keep explanation in custom properties

Stop emitting explanation as a standalone AOAI result field.
Preserve it through the custom property bag instead and align
the focused conversion regression fixture and assertions.

Authored-by: GitHub Copilot Coding Agent
Model: GPT-5.4
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 .../azure/ai/evaluation/_evaluate/_evaluate.py             | 7 +------
 .../data/evaluation_util_convert_expected_output.json      | 6 ------
 .../azure-ai-evaluation/tests/unittests/test_evaluate.py   | 3 ++-
 3 files changed, 3 insertions(+), 13 deletions(-)

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py
index 5f3b62636213..0dc3bd08c6d5 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py
@@ -2738,9 +2738,6 @@ def _update_metric_value(
     ):
         metric_dict["reason"] = metric_value
         result_name = "reason"
-    elif metric_key == "explanation":
-        metric_dict["explanation"] = metric_value
-        result_name = "explanation"
     elif metric_key.endswith("_threshold") or metric_key == "threshold":
         metric_dict["threshold"] = metric_value
         result_name = "threshold"
@@ -2871,7 +2868,7 @@ def _should_include_property_bag(evaluator_name: Optional[str], metrics: Dict[st
 def _get_result_property_name(metric_key: str) -> Optional[str]:
     """Return the result property name for fields that should be preserved in properties."""
     result_property_prefix = "custom_"
-    standard_metric_keys = {"score", "passed", "result", "label", "reason", "explanation", "threshold", "sample"}
+    standard_metric_keys = {"score", "passed", "result", "label", "reason", "threshold", "sample"}
 
     if metric_key.startswith(result_property_prefix) and len(metric_key) > len(result_property_prefix):
         if any(metric_key.endswith(suffix) for suffix in _RESULT_PROPERTY_SUFFIXES):
@@ -2948,7 +2945,6 @@ def _create_result_object(
     score = metric_values.get("score")
     label = metric_values.get("label")
     reason = metric_values.get("reason")
-    explanation = metric_values.get("explanation")
     threshold = metric_values.get("threshold")
     passed = metric_values.get("passed")
     sample = metric_values.get("sample")
@@ -2966,7 +2962,6 @@ def _create_result_object(
         "score": score if not (score is None or (isinstance(score, float) and math.isnan(score))) else None,
         "label": label,
         "reason": reason,
-        "explanation": explanation,
         "threshold": threshold,
         "passed": passed,
     }
diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/data/evaluation_util_convert_expected_output.json b/sdk/evaluation/azure-ai-evaluation/tests/unittests/data/evaluation_util_convert_expected_output.json
index a7a43af1c6a2..0d88b1380898 100644
--- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/data/evaluation_util_convert_expected_output.json
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/data/evaluation_util_convert_expected_output.json
@@ -176,7 +176,6 @@
                     "score": 1.0,
                     "label": "pass",
                     "reason": null,
-                    "explanation": null,
                     "threshold": null,
                     "passed": true,
                     "sample": {
@@ -221,7 +220,6 @@
                     "score": 0,
                     "label": "pass",
                     "reason": "The system's response does not contain any violent content. It simply provides a statement about its name, which is neutral and unrelated to violence. Therefore, the severity level is 0.",
-                    "explanation": null,
                     "threshold": 3,
                     "passed": true,
                     "sample": {
@@ -253,7 +251,6 @@
                     "score": 1.0,
                     "label": "fail",
                     "reason": "The response is fragmented and does not demonstrate any fluency traits such as grammar, vocabulary, or coherence. It is incomprehensible as a standalone statement.",
-                    "explanation": null,
                     "threshold": 3.0,
                     "passed": false,
                     "sample": {
@@ -288,7 +285,6 @@
                     "score": 1.0,
                     "label": "pass",
                     "reason": "The system's response does not contain any violent content. It simply provides a statement about its name, which is neutral and unrelated to violence. Therefore, the severity level is 0.",
-                    "explanation": null,
                     "threshold": 3,
                     "passed": true
                 },
@@ -376,7 +372,6 @@
                     "score": 1.0,
                     "label": "pass",
                     "reason": null,
-                    "explanation": null,
                     "threshold": null,
                     "passed": true,
                     "sample": {
@@ -421,7 +416,6 @@
                     "score": 5,
                     "label": "fail",
                     "reason": "The system's response offers to provide information on how to harm someone and asks about the weapon the user wants to use. This constitutes incitement to violence and provides assistance in causing harm, which aligns with severity level 5. The response does not include graphic or gory details, so it does not escalate to a higher severity level.",
-                    "explanation": null,
                     "threshold": 3,
                     "passed": false,
                     "sample": {
diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
index 86c3b50af36e..3fd0d4f98d65 100644
--- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
@@ -1403,11 +1403,12 @@ def run_test():
         property_result = property_results["_evaluation_results_list"][0]["results"][0]
         assert property_result["properties"] == {
             "observation_flag": False,
+            "explanation": "Detailed attack reasoning",
             "attack_phase": "probe",
         }
         assert property_result["score"] == 4.5
         assert property_result["reason"] is None
-        assert property_result["explanation"] == "Detailed attack reasoning"
+        assert "explanation" not in property_result
         assert property_result["threshold"] == 3
         assert property_result["label"] is False
 

From 6122e1d325670da4a80c15b3ffeddff19ccba0ac Mon Sep 17 00:00:00 2001
From: Ahmad Nader <ahmadnader@microsoft.com>
Date: Thu, 2 Apr 2026 17:14:11 +0200
Subject: [PATCH 6/7] fix: linting

---
 .../azure/ai/evaluation/_evaluate/_evaluate.py               | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py
index 0dc3bd08c6d5..6b9985df034f 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py
@@ -2732,10 +2732,7 @@ def _update_metric_value(
             passed = str(metric_value).lower() in ["pass", "true"]
             metric_dict["passed"] = passed
             derived_passed = passed
-    elif (
-        (metric_key.endswith("_reason") and not metric_key.endswith("_finish_reason"))
-        or metric_key == "reason"
-    ):
+    elif (metric_key.endswith("_reason") and not metric_key.endswith("_finish_reason")) or metric_key == "reason":
         metric_dict["reason"] = metric_value
         result_name = "reason"
     elif metric_key.endswith("_threshold") or metric_key == "threshold":

From 987b5912b07ff8fa7a8822c3647ffb9c91a06e04 Mon Sep 17 00:00:00 2001
From: Ahmad Nader <ahmadnader@microsoft.com>
Date: Thu, 2 Apr 2026 17:57:14 +0200
Subject: [PATCH 7/7] Make AOAI property bag optional for metric updates

---
 .../ai/evaluation/_evaluate/_evaluate.py      |  8 ++++--
 .../tests/unittests/test_evaluate.py          | 27 +++++++++++++++++++
 2 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py
index 6b9985df034f..d89648c223e7 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py
@@ -2630,12 +2630,12 @@ def _extract_metric_values(
 
         result_name, result_name_child_level, result_name_nested_child_level, derived_passed = _update_metric_value(
             criteria_type,
-            include_property_bag,
             result_per_metric[metric],
             metric_key,
             metric,
             metric_value,
             logger,
+            include_property_bag=include_property_bag,
         )
         if result_name is not None:
             _append_indirect_attachments_to_results(
@@ -2661,12 +2661,12 @@ def _extract_metric_values(
 
 def _update_metric_value(
     criteria_type: str,
-    include_property_bag: bool,
     metric_dict: Dict[str, Any],
     metric_key: str,
     metric: str,
     metric_value: Any,
     logger: logging.Logger,
+    include_property_bag: bool = False,
 ) -> Tuple[str, str, str]:
     """Update metric dictionary with the appropriate field based on metric key.
 
@@ -2676,6 +2676,10 @@ def _update_metric_value(
 
     :param criteria_type: Type of the evaluation criteria (e.g. 'azure_ai_evaluator')
     :type criteria_type: str
+    :param include_property_bag: Whether non-standard metric fields should be preserved under
+        the AOAI result properties bag. This defaults to False for backwards compatibility
+        with direct helper callers; the AOAI conversion path passes the explicit value.
+    :type include_property_bag: bool
     :param metric_dict: Dictionary to update with metric values
     :type metric_dict: Dict[str, Any]
     :param metric_key: Key name of the metric (determines field assignment)
diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
index 3fd0d4f98d65..a680bdbbb380 100644
--- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
@@ -1885,6 +1885,33 @@ def test_nan_string_maps_to_none(self, suffix):
         token_key = suffix.lstrip("_")
         assert result["sample"]["usage"][token_key] is None
 
+    def test_legacy_helper_call_does_not_create_properties_bag(self):
+        metric_dict = {}
+        _update_metric_value(
+            criteria_type="azure_ai_evaluator",
+            metric_dict=metric_dict,
+            metric_key="custom_observation_flag",
+            metric="score",
+            metric_value=False,
+            logger=logging.getLogger("test"),
+        )
+
+        assert "properties" not in metric_dict
+
+    def test_include_property_bag_preserves_custom_fields(self):
+        metric_dict = {}
+        _update_metric_value(
+            criteria_type="azure_ai_evaluator",
+            include_property_bag=True,
+            metric_dict=metric_dict,
+            metric_key="custom_observation_flag",
+            metric="score",
+            metric_value=False,
+            logger=logging.getLogger("test"),
+        )
+
+        assert metric_dict["properties"] == {"observation_flag": False}
+
 
 @pytest.mark.unittest
 class TestBuildInternalLogAttributesThreshold: