Nested ATIF lineage

examples/evaluation_and_profiling/simple_calculator_eval/README.md

@@ -40,6 +40,22 @@ This example demonstrates how to evaluate and profile AI agent performance using
 
 1. **Agent toolkit**: Ensure you have the Agent toolkit installed. If you have not already done so, follow the instructions in the [Install Guide](../../../docs/source/get-started/installation.md#install-from-source) to create the development environment and install NeMo Agent Toolkit.
 2. **Base workflow**: This example builds upon the Getting Started [Simple Calculator](../../getting_started/simple_calculator/) example. Make sure you are familiar with the example before proceeding.
+3. **Phoenix tracing backend**: Start Phoenix before running trajectory-based configurations in this example.
+
+```bash
+phoenix serve
+```
+
+If your environment does not include the `phoenix` CLI, install it with:
+
+```bash
+uv pip install arize-phoenix
+```
+
+You can run Phoenix from a separate virtual environment than the one used for
+NVIDIA NeMo Agent Toolkit evaluation runs. This is often preferable to avoid
+dependency and version conflicts between Phoenix packages and toolkit plus
+evaluator dependencies.
 
 ## Installation
 
@@ -77,3 +93,47 @@ The evaluation generates comprehensive metrics including:
 - **Question-by-Question Analysis**: Detailed breakdown of individual responses
 - **Performance Metrics**: Overall quality assessments
 - **Error Analysis**: Identification of common failure patterns
+
+### Running Nested Trajectory Evaluation
+
+Evaluate a workflow that performs a nested tool call (`power_of_two` -> `calculator__multiply`) and inspect how it appears in the ATIF trajectory output:
+
+```bash
+nat eval --config_file examples/evaluation_and_profiling/simple_calculator_eval/configs/config-nested-trajectory-eval.yml
+```
+
+This command:
+- Uses `examples/evaluation_and_profiling/simple_calculator_eval/data/simple_calculator_power_of_two.json`
+- Runs the built-in `trajectory` evaluator
+- Writes workflow trajectories to `.tmp/nat/examples/simple_calculator/nested-eval/workflow_output_atif.json`
+
+To inspect the call hierarchy from the generated ATIF file:
+
+```bash
+python packages/nvidia_nat_eval/scripts/print_atif_function_tree.py \
+  .tmp/nat/examples/simple_calculator/nested-eval/workflow_output_atif.json \
+  --view ancestry \
+  --item-id 1
+```
+
+### Running Branching Nested Trajectory Evaluation
+
+Evaluate a workflow where one top-level tool (`power_branch`) fans out to two internal tools (`square_via_multiply` and `cube_via_multiply_chain`) and each branch calls `calculator__multiply`.
+
+```bash
+nat eval --config_file examples/evaluation_and_profiling/simple_calculator_eval/configs/config-branching-nested-trajectory-eval.yml
+```
+
+This command:
+- Uses `examples/evaluation_and_profiling/simple_calculator_eval/data/simple_calculator_power_branch.json`
+- Runs the built-in `trajectory` evaluator
+- Writes trajectories to `.tmp/nat/examples/simple_calculator/branching-nested-eval/workflow_output_atif.json`
+
+To inspect one input item:
+
+```bash
+python packages/nvidia_nat_eval/scripts/print_atif_function_tree.py \
+  .tmp/nat/examples/simple_calculator/branching-nested-eval/workflow_output_atif.json \
+  --view ancestry \
+  --item-id 1
+```

examples/evaluation_and_profiling/simple_calculator_eval/pyproject.toml

@@ -34,10 +34,13 @@ classifiers = ["Programming Language :: Python"]
 
 [tool.setuptools_dynamic_dependencies]
 dependencies = [
-"nvidia-nat[eval,langchain,profiler,test] == {version}",
+  "nvidia-nat[eval,langchain,phoenix,profiler,test] == {version}",
   "nat_simple_calculator",
 ]
 
 [tool.uv.sources]
 nvidia-nat = { path = "../../..", editable = true }
 nat_simple_calculator = { path = "../../getting_started/simple_calculator", editable = true }
+
+[project.entry-points."nat.components"]
+nat_simple_calculator_eval = "nat_simple_calculator_eval.register"

examples/evaluation_and_profiling/simple_calculator_eval/src/nat_simple_calculator_eval/configs/config-branching-nested-trajectory-eval.yml

@@ -0,0 +1,84 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Branching nested trajectory evaluation example:
+#   react_agent -> power_branch -> {square_via_multiply, cube_via_multiply_chain}
+# and both internal branches call calculator__multiply.
+
+general:
+  telemetry:
+    tracing:
+      phoenix:
+        _type: phoenix
+        endpoint: http://localhost:6006/v1/traces
+        project: simple_calculator_branching_nested_eval
+
+function_groups:
+  calculator:
+    _type: calculator
+
+functions:
+  square_via_multiply:
+    _type: square_via_multiply
+    multiply_fn: calculator__multiply
+  cube_via_multiply_chain:
+    _type: cube_via_multiply_chain
+    multiply_fn: calculator__multiply
+  power_branch:
+    _type: power_branch
+    square_fn: square_via_multiply
+    cube_fn: cube_via_multiply_chain
+
+llms:
+  nim_llm:
+    _type: nim
+    model_name: nvidia/nemotron-3-nano-30b-a3b
+    temperature: 0.0
+    max_tokens: 1024
+    chat_template_kwargs:
+      enable_thinking: false
+  eval_llm:
+    _type: nim
+    model_name: mistralai/mixtral-8x22b-instruct-v0.1
+    temperature: 0.0
+    max_tokens: 1024
+
+workflow:
+  _type: react_agent
+  tool_names: [power_branch]
+  llm_name: nim_llm
+  verbose: true
+  parse_agent_response_max_retries: 3
+
+eval:
+  general:
+    max_concurrency: 1
+    output:
+      dir: .tmp/nat/examples/simple_calculator/branching-nested-eval
+      write_atif_workflow_output: true
+      cleanup: true
+    dataset:
+      _type: json
+      file_path: examples/evaluation_and_profiling/simple_calculator_eval/data/simple_calculator_power_branch.json
+      filter:
+        allowlist:
+          field:
+            id: [1]
+
+  evaluators:
+    trajectory_eval:
+      _type: trajectory
+      enable_atif_evaluator: true
+      llm_name: eval_llm

examples/evaluation_and_profiling/simple_calculator_eval/src/nat_simple_calculator_eval/configs/config-nested-trajectory-eval.yml

@@ -0,0 +1,79 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Nested trajectory evaluation example:
+#   react_agent -> power_of_two -> calculator__multiply
+#
+# This configuration writes ATIF workflow output so you can inspect how nested
+# tool calls are represented in trajectory steps.
+
+general:
+  telemetry:
+    tracing:
+      phoenix:
+        _type: phoenix
+        endpoint: http://localhost:6006/v1/traces
+        project: simple_calculator_nested_eval
+
+function_groups:
+  calculator:
+    _type: calculator
+
+functions:
+  power_of_two:
+    _type: power_of_two
+    multiply_fn: calculator__multiply
+
+llms:
+  nim_llm:
+    _type: nim
+    model_name: nvidia/nemotron-3-nano-30b-a3b
+    temperature: 0.0
+    max_tokens: 1024
+    chat_template_kwargs:
+      enable_thinking: false
+  eval_llm:
+    _type: nim
+    model_name: mistralai/mixtral-8x22b-instruct-v0.1
+    temperature: 0.0
+    max_tokens: 1024
+
+workflow:
+  _type: react_agent
+  tool_names: [power_of_two]
+  llm_name: nim_llm
+  verbose: true
+  parse_agent_response_max_retries: 3
+
+eval:
+  general:
+    max_concurrency: 1
+    output:
+      dir: .tmp/nat/examples/simple_calculator/nested-eval
+      write_atif_workflow_output: true
+      cleanup: true
+    dataset:
+      _type: json
+      file_path: examples/evaluation_and_profiling/simple_calculator_eval/data/simple_calculator_power_of_two.json
+      filter:
+        allowlist:
+          field:
+            id: [1]
+
+  evaluators:
+    trajectory_eval:
+      _type: trajectory
+      enable_atif_evaluator: true
+      llm_name: eval_llm

examples/evaluation_and_profiling/simple_calculator_eval/src/nat_simple_calculator_eval/configs/config-trajectory-eval.yml

@@ -0,0 +1,78 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Trajectory evaluation example:
+#   react_agent -> {calculator, current_datetime}
+#
+# This configuration writes ATIF workflow output so you can inspect
+# trajectory structure with standard calculator and datetime tool calls.
+
+general:
+  telemetry:
+    tracing:
+      phoenix:
+        _type: phoenix
+        endpoint: http://localhost:6006/v1/traces
+        project: simple_calculator_eval
+
+function_groups:
+  calculator:
+    _type: calculator
+
+functions:
+  current_datetime:
+    _type: current_datetime
+
+llms:
+  nim_llm:
+    _type: nim
+    model_name: nvidia/nemotron-3-nano-30b-a3b
+    temperature: 0.0
+    max_tokens: 1024
+    chat_template_kwargs:
+      enable_thinking: false
+  eval_llm:
+    _type: nim
+    model_name: mistralai/mixtral-8x22b-instruct-v0.1
+    temperature: 0.0
+    max_tokens: 1024
+
+workflow:
+  _type: react_agent
+  tool_names: [calculator, current_datetime]
+  llm_name: nim_llm
+  verbose: true
+  parse_agent_response_max_retries: 3
+
+eval:
+  general:
+    max_concurrency: 1
+    output:
+      dir: .tmp/nat/examples/simple_calculator/trajectory-eval
+      write_atif_workflow_output: true
+      cleanup: true
+    dataset:
+      _type: json
+      file_path: examples/getting_started/simple_calculator/data/simple_calculator.json
+      filter:
+        allowlist:
+          field:
+            id: [1]
+
+  evaluators:
+    trajectory_eval:
+      _type: trajectory
+      enable_atif_evaluator: true
+      llm_name: eval_llm

examples/evaluation_and_profiling/simple_calculator_eval/src/nat_simple_calculator_eval/configs/config-tunable-rag-eval-atif.yml

@@ -13,6 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+general:
+  telemetry:
+    tracing:
+      phoenix:
+        _type: phoenix
+        endpoint: http://localhost:6006/v1/traces
+        project: simple_calculator_tunable_rag_eval_atif
+
 function_groups:
   calculator:
     _type: calculator
@@ -27,6 +35,8 @@ llms:
     model_name: nvidia/nemotron-3-nano-30b-a3b
     temperature: 0.0
     max_tokens: 1024
+    chat_template_kwargs:
+      enable_thinking: false
   eval_llm:
     _type: nim
     model_name: mistralai/mixtral-8x22b-instruct-v0.1