From 41a88740fa84e68322b819a99bc7b2e9f2667fdf Mon Sep 17 00:00:00 2001 From: Waqas Javed <7674577+w-javed@users.noreply.github.com> Date: Wed, 1 Apr 2026 14:17:26 -0700 Subject: [PATCH 1/6] fix(azure-ai-projects): skip all dot-prefixed directories in evaluator upload Change skip_dirs filter to exclude any directory starting with '.' instead of only '.git' and '.venv'. This covers .venv, .git, .mypy_cache, .tox, .pytest_cache, and any other hidden/tool directories. Applied to both sync and async upload functions. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../ai/projects/aio/operations/_patch_evaluators_async.py | 7 ++++--- .../azure/ai/projects/operations/_patch_evaluators.py | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/aio/operations/_patch_evaluators_async.py b/sdk/ai/azure-ai-projects/azure/ai/projects/aio/operations/_patch_evaluators_async.py index c6c366fd5956..53a82cff6a96 100644 --- a/sdk/ai/azure-ai-projects/azure/ai/projects/aio/operations/_patch_evaluators_async.py +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/aio/operations/_patch_evaluators_async.py @@ -47,7 +47,8 @@ async def _upload_folder_to_blob( ) -> None: """Walk *folder* and upload every eligible file to the blob container. - Skips ``__pycache__``, ``.git``, ``.venv``, ``venv``, ``node_modules`` + Skips directories starting with ``.`` (e.g. ``.git``, ``.venv``), + ``__pycache__``, ``venv``, ``node_modules`` directories and ``.pyc`` / ``.pyo`` files. :param container_client: The blob container client to upload files to. @@ -58,12 +59,12 @@ async def _upload_folder_to_blob( :raises HttpResponseError: Re-raised with a friendlier message on ``AuthorizationPermissionMismatch``. """ - skip_dirs = {"__pycache__", ".git", ".venv", "venv", "node_modules"} + skip_dirs = {"__pycache__", "venv", "node_modules"} skip_extensions = {".pyc", ".pyo"} files_uploaded = False for root, dirs, files in os.walk(folder): - dirs[:] = [d for d in dirs if d not in skip_dirs] + dirs[:] = [d for d in dirs if d not in skip_dirs and not d.startswith(".")] for file_name in files: if any(file_name.endswith(ext) for ext in skip_extensions): continue diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/operations/_patch_evaluators.py b/sdk/ai/azure-ai-projects/azure/ai/projects/operations/_patch_evaluators.py index 3f1c38d97b2b..51d123abdd0f 100644 --- a/sdk/ai/azure-ai-projects/azure/ai/projects/operations/_patch_evaluators.py +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/operations/_patch_evaluators.py @@ -47,7 +47,8 @@ def _upload_folder_to_blob( ) -> None: """Walk *folder* and upload every eligible file to the blob container. - Skips ``__pycache__``, ``.git``, ``.venv``, ``venv``, ``node_modules`` + Skips directories starting with ``.`` (e.g. ``.git``, ``.venv``), + ``__pycache__``, ``venv``, ``node_modules`` directories and ``.pyc`` / ``.pyo`` files. :param container_client: The blob container client to upload files to. @@ -58,12 +59,12 @@ def _upload_folder_to_blob( :raises HttpResponseError: Re-raised with a friendlier message on ``AuthorizationPermissionMismatch``. """ - skip_dirs = {"__pycache__", ".git", ".venv", "venv", "node_modules"} + skip_dirs = {"__pycache__", "venv", "node_modules"} skip_extensions = {".pyc", ".pyo"} files_uploaded = False for root, dirs, files in os.walk(folder): - dirs[:] = [d for d in dirs if d not in skip_dirs] + dirs[:] = [d for d in dirs if d not in skip_dirs and not d.startswith(".")] for file_name in files: if any(file_name.endswith(ext) for ext in skip_extensions): continue From 4deccdd0e761caa8f0a7eda391d5d3c571bcc076 Mon Sep 17 00:00:00 2001 From: Waqas Javed <7674577+w-javed@users.noreply.github.com> Date: Wed, 1 Apr 2026 22:46:02 -0700 Subject: [PATCH 2/6] fix(azure-ai-projects): skip dot-prefixed files in evaluator upload Extend the existing skip logic to also exclude dot-prefixed files (e.g. .env, .DS_Store, .gitignore) from evaluator uploads, matching the treatment already applied to dot-prefixed directories. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../azure/ai/projects/aio/operations/_patch_evaluators_async.py | 2 +- .../azure/ai/projects/operations/_patch_evaluators.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/aio/operations/_patch_evaluators_async.py b/sdk/ai/azure-ai-projects/azure/ai/projects/aio/operations/_patch_evaluators_async.py index 53a82cff6a96..fdb3725b45b4 100644 --- a/sdk/ai/azure-ai-projects/azure/ai/projects/aio/operations/_patch_evaluators_async.py +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/aio/operations/_patch_evaluators_async.py @@ -66,7 +66,7 @@ async def _upload_folder_to_blob( for root, dirs, files in os.walk(folder): dirs[:] = [d for d in dirs if d not in skip_dirs and not d.startswith(".")] for file_name in files: - if any(file_name.endswith(ext) for ext in skip_extensions): + if file_name.startswith('.') or any(file_name.endswith(ext) for ext in skip_extensions): continue file_path = os.path.join(root, file_name) blob_name = os.path.relpath(file_path, folder).replace("\\", "/") diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/operations/_patch_evaluators.py b/sdk/ai/azure-ai-projects/azure/ai/projects/operations/_patch_evaluators.py index 51d123abdd0f..b0858a7234f2 100644 --- a/sdk/ai/azure-ai-projects/azure/ai/projects/operations/_patch_evaluators.py +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/operations/_patch_evaluators.py @@ -66,7 +66,7 @@ def _upload_folder_to_blob( for root, dirs, files in os.walk(folder): dirs[:] = [d for d in dirs if d not in skip_dirs and not d.startswith(".")] for file_name in files: - if any(file_name.endswith(ext) for ext in skip_extensions): + if file_name.startswith('.') or any(file_name.endswith(ext) for ext in skip_extensions): continue file_path = os.path.join(root, file_name) blob_name = os.path.relpath(file_path, folder).replace("\\", "/") From b71cd2f4a2b71f8bf56b9796bbee9a723638e2a7 Mon Sep 17 00:00:00 2001 From: Waqas Javed <7674577+w-javed@users.noreply.github.com> Date: Thu, 2 Apr 2026 10:45:33 -0700 Subject: [PATCH 3/6] feat(azure-ai-projects): add file_pattern and folder_exclusions_pattern to evaluators.upload Add optional regex-based filtering parameters to _upload_folder_to_blob and upload methods, consistent with datasets.upload_folder pattern: - file_pattern: filter which files to upload by name - folder_exclusions_pattern: exclude directories by name pattern Applied to both sync and async implementations. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../aio/operations/_patch_evaluators_async.py | 28 +- .../projects/operations/_patch_evaluators.py | 28 +- .../sample_eval_upload_friendly_evaluator.py | 244 ------------------ 3 files changed, 52 insertions(+), 248 deletions(-) diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/aio/operations/_patch_evaluators_async.py b/sdk/ai/azure-ai-projects/azure/ai/projects/aio/operations/_patch_evaluators_async.py index fdb3725b45b4..2e92b6dccf49 100644 --- a/sdk/ai/azure-ai-projects/azure/ai/projects/aio/operations/_patch_evaluators_async.py +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/aio/operations/_patch_evaluators_async.py @@ -9,6 +9,7 @@ """ import os +import re import logging from typing import Any, Final, IO, Tuple, Optional, Union from pathlib import Path @@ -43,6 +44,8 @@ class BetaEvaluatorsOperations(BetaEvaluatorsOperationsGenerated): async def _upload_folder_to_blob( container_client: ContainerClient, folder: str, + file_pattern: Optional[re.Pattern] = None, + folder_exclusions_pattern: Optional[re.Pattern] = None, **kwargs: Any, ) -> None: """Walk *folder* and upload every eligible file to the blob container. @@ -55,6 +58,12 @@ async def _upload_folder_to_blob( :type container_client: ~azure.storage.blob.ContainerClient :param folder: Path to the local folder containing files to upload. :type folder: str + :param file_pattern: Optional regex pattern to filter files. Only files + whose name matches this pattern will be uploaded. + :type file_pattern: Optional[re.Pattern] + :param folder_exclusions_pattern: Optional regex pattern to exclude + directories. Directories whose name matches this pattern will be skipped. + :type folder_exclusions_pattern: Optional[re.Pattern] :raises ValueError: If the folder contains no uploadable files. :raises HttpResponseError: Re-raised with a friendlier message on ``AuthorizationPermissionMismatch``. @@ -64,10 +73,17 @@ async def _upload_folder_to_blob( files_uploaded = False for root, dirs, files in os.walk(folder): - dirs[:] = [d for d in dirs if d not in skip_dirs and not d.startswith(".")] + dirs[:] = [ + d for d in dirs + if d not in skip_dirs + and not d.startswith(".") + and not (folder_exclusions_pattern and folder_exclusions_pattern.search(d)) + ] for file_name in files: if file_name.startswith('.') or any(file_name.endswith(ext) for ext in skip_extensions): continue + if file_pattern and not file_pattern.search(file_name): + continue file_path = os.path.join(root, file_name) blob_name = os.path.relpath(file_path, folder).replace("\\", "/") logger.debug("[upload] Start uploading file `%s` as blob `%s`.", file_path, blob_name) @@ -204,6 +220,8 @@ async def upload( *, folder: str, connection_name: Optional[str] = None, + file_pattern: Optional[re.Pattern] = None, + folder_exclusions_pattern: Optional[re.Pattern] = None, **kwargs: Any, ) -> EvaluatorVersion: """Upload all files in a folder to blob storage and create a code-based evaluator version @@ -226,6 +244,12 @@ async def upload( should be uploaded. If not specified, the default Azure Storage Account connection will be used. Optional. :paramtype connection_name: str + :keyword file_pattern: Optional regex pattern to filter files. Only files whose name + matches this pattern will be uploaded. Optional. + :paramtype file_pattern: Optional[re.Pattern] + :keyword folder_exclusions_pattern: Optional regex pattern to exclude directories. + Directories whose name matches this pattern will be skipped during upload. Optional. + :paramtype folder_exclusions_pattern: Optional[re.Pattern] :return: The created evaluator version. :rtype: ~azure.ai.projects.models.EvaluatorVersion :raises ~azure.core.exceptions.HttpResponseError: If an error occurs during the HTTP request. @@ -247,7 +271,7 @@ async def upload( ) async with container_client: - await self._upload_folder_to_blob(container_client, folder, **kwargs) + await self._upload_folder_to_blob(container_client, folder, file_pattern, folder_exclusions_pattern, **kwargs) self._set_blob_uri(evaluator_version, blob_uri) result = await self.create_version( diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/operations/_patch_evaluators.py b/sdk/ai/azure-ai-projects/azure/ai/projects/operations/_patch_evaluators.py index b0858a7234f2..093055a25408 100644 --- a/sdk/ai/azure-ai-projects/azure/ai/projects/operations/_patch_evaluators.py +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/operations/_patch_evaluators.py @@ -9,6 +9,7 @@ """ import os +import re import logging from typing import Any, Final, IO, Tuple, Optional, Union from pathlib import Path @@ -43,6 +44,8 @@ class BetaEvaluatorsOperations(BetaEvaluatorsOperationsGenerated): def _upload_folder_to_blob( container_client: ContainerClient, folder: str, + file_pattern: Optional[re.Pattern] = None, + folder_exclusions_pattern: Optional[re.Pattern] = None, **kwargs: Any, ) -> None: """Walk *folder* and upload every eligible file to the blob container. @@ -55,6 +58,12 @@ def _upload_folder_to_blob( :type container_client: ~azure.storage.blob.ContainerClient :param folder: Path to the local folder containing files to upload. :type folder: str + :param file_pattern: Optional regex pattern to filter files. Only files + whose name matches this pattern will be uploaded. + :type file_pattern: Optional[re.Pattern] + :param folder_exclusions_pattern: Optional regex pattern to exclude + directories. Directories whose name matches this pattern will be skipped. + :type folder_exclusions_pattern: Optional[re.Pattern] :raises ValueError: If the folder contains no uploadable files. :raises HttpResponseError: Re-raised with a friendlier message on ``AuthorizationPermissionMismatch``. @@ -64,10 +73,17 @@ def _upload_folder_to_blob( files_uploaded = False for root, dirs, files in os.walk(folder): - dirs[:] = [d for d in dirs if d not in skip_dirs and not d.startswith(".")] + dirs[:] = [ + d for d in dirs + if d not in skip_dirs + and not d.startswith(".") + and not (folder_exclusions_pattern and folder_exclusions_pattern.search(d)) + ] for file_name in files: if file_name.startswith('.') or any(file_name.endswith(ext) for ext in skip_extensions): continue + if file_pattern and not file_pattern.search(file_name): + continue file_path = os.path.join(root, file_name) blob_name = os.path.relpath(file_path, folder).replace("\\", "/") logger.debug("[upload] Start uploading file `%s` as blob `%s`.", file_path, blob_name) @@ -204,6 +220,8 @@ def upload( *, folder: str, connection_name: Optional[str] = None, + file_pattern: Optional[re.Pattern] = None, + folder_exclusions_pattern: Optional[re.Pattern] = None, **kwargs: Any, ) -> EvaluatorVersion: """Upload all files in a folder to blob storage and create a code-based evaluator version @@ -226,6 +244,12 @@ def upload( should be uploaded. If not specified, the default Azure Storage Account connection will be used. Optional. :paramtype connection_name: str + :keyword file_pattern: Optional regex pattern to filter files. Only files whose name + matches this pattern will be uploaded. Optional. + :paramtype file_pattern: Optional[re.Pattern] + :keyword folder_exclusions_pattern: Optional regex pattern to exclude directories. + Directories whose name matches this pattern will be skipped during upload. Optional. + :paramtype folder_exclusions_pattern: Optional[re.Pattern] :return: The created evaluator version. :rtype: ~azure.ai.projects.models.EvaluatorVersion :raises ~azure.core.exceptions.HttpResponseError: If an error occurs during the HTTP request. @@ -247,7 +271,7 @@ def upload( ) with container_client: - self._upload_folder_to_blob(container_client, folder, **kwargs) + self._upload_folder_to_blob(container_client, folder, file_pattern, folder_exclusions_pattern, **kwargs) self._set_blob_uri(evaluator_version, blob_uri) result = self.create_version( diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_eval_upload_friendly_evaluator.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_eval_upload_friendly_evaluator.py index 67e168d3509a..e69de29bb2d1 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_eval_upload_friendly_evaluator.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_eval_upload_friendly_evaluator.py @@ -1,244 +0,0 @@ -# pylint: disable=line-too-long,useless-suppression -# ------------------------------------ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. -# ------------------------------------ - -""" -DESCRIPTION: - Given an AIProjectClient, this sample demonstrates how to: - 1. Upload a custom LLM-based evaluator (FriendlyEvaluator) with nested - folder structure (common_util/) using `evaluators.upload()`. - 2. Create an evaluation (eval) that references the uploaded evaluator. - 3. Run the evaluation with inline data and poll for results. - - The FriendlyEvaluator calls Azure OpenAI to judge the friendliness of a - response and returns score, label, reason, and explanation. - -USAGE: - python sample_eval_upload_friendly_evaluator.py - - Before running the sample: - - pip install "azure-ai-projects>=2.0.0b4" azure-storage-blob python-dotenv azure-identity openai - - Set these environment variables with your own values: - 1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint. - 2) FOUNDRY_MODEL_NAME - Optional. The name of the model deployment to use for evaluation. -""" - -import os -import time -import random -import string -from pathlib import Path -from pprint import pprint - -from dotenv import load_dotenv -from openai.types.evals.create_eval_jsonl_run_data_source_param import ( - CreateEvalJSONLRunDataSourceParam, - SourceFileContent, - SourceFileContentContent, -) -from openai.types.eval_create_params import DataSourceConfigCustom -from azure.identity import DefaultAzureCredential -from azure.ai.projects import AIProjectClient -from azure.ai.projects.models import ( - CodeBasedEvaluatorDefinition, - EvaluatorCategory, - EvaluatorMetric, - EvaluatorMetricType, - EvaluatorMetricDirection, - EvaluatorType, - EvaluatorVersion, -) - -load_dotenv() - -endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] -model_deployment_name = os.environ.get("FOUNDRY_MODEL_NAME") -azure_openai_endpoint = os.environ["AZURE_OPENAI_ENDPOINT"] -azure_openai_api_key = os.environ["AZURE_OPENAI_API_KEY"] - -# The folder containing the FriendlyEvaluator code, including common_util/ subfolder -local_upload_folder = str(Path(__file__).parent / "custom_evaluators" / "friendly_evaluator") - -with ( - DefaultAzureCredential() as credential, - AIProjectClient(endpoint=endpoint, credential=credential) as project_client, - project_client.get_openai_client() as client, -): - # --------------------------------------------------------------- - # 1. Upload evaluator code and create evaluator version - # The folder structure uploaded is: - # friendly_evaluator/ - # friendly_evaluator.py <- entry point - # common_util/ - # __init__.py - # util.py <- helper functions - # --------------------------------------------------------------- - suffix = "".join(random.choices(string.ascii_lowercase, k=5)) - evaluator_name = f"friendly_evaluator_{suffix}" - - evaluator_version = EvaluatorVersion( - evaluator_type=EvaluatorType.CUSTOM, - categories=[EvaluatorCategory.QUALITY], - display_name="Friendliness Evaluator", - description="LLM-based evaluator that scores how friendly a response is (1-5)", - definition=CodeBasedEvaluatorDefinition( - entry_point="friendly_evaluator:FriendlyEvaluator", - init_parameters={ - "type": "object", - "properties": { - "model_config": { - "type": "object", - "description": "Azure OpenAI configuration for the LLM judge", - "properties": { - "azure_endpoint": {"type": "string"}, - "api_version": {"type": "string"}, - "api_key": {"type": "string"}, - }, - "required": ["azure_endpoint", "api_key"], - }, - "threshold": {"type": "number"}, - }, - "required": ["model_config", "threshold"], - }, - data_schema={ - "type": "object", - "properties": { - "query": {"type": "string", "description": "The original user query"}, - "response": {"type": "string", "description": "The response to evaluate for friendliness"}, - }, - "required": ["query", "response"], - }, - metrics={ - "score": EvaluatorMetric( - type=EvaluatorMetricType.ORDINAL, - desirable_direction=EvaluatorMetricDirection.INCREASE, - min_value=1, - max_value=5, - ) - }, - ), - ) - - print("Uploading FriendlyEvaluator (with nested common_util folder)...") - friendly_evaluator = project_client.beta.evaluators.upload( - name=evaluator_name, - evaluator_version=evaluator_version, - folder=local_upload_folder, - ) - - print(f"\nEvaluator created: name={friendly_evaluator.name}, version={friendly_evaluator.version}") - print(f"Evaluator ID: {friendly_evaluator.id}") - pprint(friendly_evaluator) - - # --------------------------------------------------------------- - # 2. Create an evaluation referencing the uploaded evaluator - # --------------------------------------------------------------- - data_source_config = DataSourceConfigCustom( - { - "type": "custom", - "item_schema": { - "type": "object", - "properties": { - "query": {"type": "string"}, - "response": {"type": "string"}, - }, - "required": ["query", "response"], - }, - "include_sample_schema": True, - } - ) - - testing_criteria = [ - { - "type": "azure_ai_evaluator", - "name": evaluator_name, - "evaluator_name": evaluator_name, - "initialization_parameters": { - "deployment_name": f"{model_deployment_name}", # provide model_config or, deployment name passed is used to construct the model_config for the evaluator. - "threshold": 3, - }, - } - ] - - print("\nCreating evaluation...") - eval_object = client.evals.create( - name=f"Friendliness Evaluation - {suffix}", - data_source_config=data_source_config, - testing_criteria=testing_criteria, # type: ignore - ) - print(f"Evaluation created (id: {eval_object.id}, name: {eval_object.name})") - - # --------------------------------------------------------------- - # 3. Run the evaluation with inline data - # --------------------------------------------------------------- - print("\nCreating evaluation run with inline data...") - eval_run_object = client.evals.runs.create( - eval_id=eval_object.id, - name=f"Friendliness Eval Run - {suffix}", - metadata={"team": "eval-exp", "scenario": "friendliness-v1"}, - data_source=CreateEvalJSONLRunDataSourceParam( - type="jsonl", - source=SourceFileContent( - type="file_content", - content=[ - SourceFileContentContent( - item={ - "query": "How do I reset my password?", - "response": "Go to settings and click reset. That's it.", - } - ), - SourceFileContentContent( - item={ - "query": "I'm having trouble with my account", - "response": "I'm really sorry to hear you're having trouble! I'd love to help you get this sorted out. Could you tell me a bit more about what's happening so I can assist you better?", - } - ), - SourceFileContentContent( - item={ - "query": "Can you help me?", - "response": "Read the docs.", - } - ), - SourceFileContentContent( - item={ - "query": "What's the weather like today?", - "response": "Great question! While I'm not a weather service, I'd be happy to suggest some wonderful weather apps that can give you accurate forecasts. Would you like some recommendations? 😊", - } - ), - ], - ), - ), - ) - - print(f"Evaluation run created (id: {eval_run_object.id})") - pprint(eval_run_object) - - # --------------------------------------------------------------- - # 4. Poll for evaluation run completion - # --------------------------------------------------------------- - while True: - run = client.evals.runs.retrieve(run_id=eval_run_object.id, eval_id=eval_object.id) - if run.status in ("completed", "failed"): - print(f"\nEvaluation run finished with status: {run.status}") - output_items = list(client.evals.runs.output_items.list(run_id=run.id, eval_id=eval_object.id)) - pprint(output_items) - print(f"\nEvaluation run Report URL: {run.report_url}") - break - time.sleep(5) - print("Waiting for evaluation run to complete...") - - # --------------------------------------------------------------- - # 5. Cleanup (uncomment to delete) - # --------------------------------------------------------------- - # print("\nCleaning up...") - # project_client.beta.evaluators.delete_version( - # name=friendly_evaluator.name, - # version=friendly_evaluator.version, - # ) - # client.evals.delete(eval_id=eval_object.id) - # print("Cleanup done.") - print("\nDone - FriendlyEvaluator upload, eval creation, and eval run verified successfully.") From 379142329e26ee69ac437f00ef7ffa9192a18513 Mon Sep 17 00:00:00 2001 From: Waqas Javed <7674577+w-javed@users.noreply.github.com> Date: Thu, 2 Apr 2026 12:54:44 -0700 Subject: [PATCH 4/6] refactor: remove hardcoded skip lists, let customer control via patterns Remove hardcoded skip_dirs and skip_extensions. Filtering is now fully controlled by the optional file_pattern and folder_exclusions_pattern parameters. Docstrings include recommended excludes for typical Python evaluator projects. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../aio/operations/_patch_evaluators_async.py | 33 +++++++++++-------- .../projects/operations/_patch_evaluators.py | 33 +++++++++++-------- 2 files changed, 40 insertions(+), 26 deletions(-) diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/aio/operations/_patch_evaluators_async.py b/sdk/ai/azure-ai-projects/azure/ai/projects/aio/operations/_patch_evaluators_async.py index 2e92b6dccf49..f5ce647350e1 100644 --- a/sdk/ai/azure-ai-projects/azure/ai/projects/aio/operations/_patch_evaluators_async.py +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/aio/operations/_patch_evaluators_async.py @@ -50,9 +50,15 @@ async def _upload_folder_to_blob( ) -> None: """Walk *folder* and upload every eligible file to the blob container. - Skips directories starting with ``.`` (e.g. ``.git``, ``.venv``), - ``__pycache__``, ``venv``, ``node_modules`` - directories and ``.pyc`` / ``.pyo`` files. + By default all files and directories are included. Use *file_pattern* + and *folder_exclusions_pattern* to control what gets uploaded. + + Recommended excludes for typical Python evaluator projects:: + + file_pattern = re.compile(r"^(?!\\.).+(? None: """Walk *folder* and upload every eligible file to the blob container. - Skips directories starting with ``.`` (e.g. ``.git``, ``.venv``), - ``__pycache__``, ``venv``, ``node_modules`` - directories and ``.pyc`` / ``.pyo`` files. + By default all files and directories are included. Use *file_pattern* + and *folder_exclusions_pattern* to control what gets uploaded. + + Recommended excludes for typical Python evaluator projects:: + + file_pattern = re.compile(r"^(?!\\.).+(? Date: Thu, 2 Apr 2026 12:57:28 -0700 Subject: [PATCH 5/6] fix: restore sample_eval_upload_friendly_evaluator.py accidentally emptied Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../sample_eval_upload_friendly_evaluator.py | 244 ++++++++++++++++++ 1 file changed, 244 insertions(+) diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_eval_upload_friendly_evaluator.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_eval_upload_friendly_evaluator.py index e69de29bb2d1..67e168d3509a 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_eval_upload_friendly_evaluator.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_eval_upload_friendly_evaluator.py @@ -0,0 +1,244 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + Given an AIProjectClient, this sample demonstrates how to: + 1. Upload a custom LLM-based evaluator (FriendlyEvaluator) with nested + folder structure (common_util/) using `evaluators.upload()`. + 2. Create an evaluation (eval) that references the uploaded evaluator. + 3. Run the evaluation with inline data and poll for results. + + The FriendlyEvaluator calls Azure OpenAI to judge the friendliness of a + response and returns score, label, reason, and explanation. + +USAGE: + python sample_eval_upload_friendly_evaluator.py + + Before running the sample: + + pip install "azure-ai-projects>=2.0.0b4" azure-storage-blob python-dotenv azure-identity openai + + Set these environment variables with your own values: + 1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint. + 2) FOUNDRY_MODEL_NAME - Optional. The name of the model deployment to use for evaluation. +""" + +import os +import time +import random +import string +from pathlib import Path +from pprint import pprint + +from dotenv import load_dotenv +from openai.types.evals.create_eval_jsonl_run_data_source_param import ( + CreateEvalJSONLRunDataSourceParam, + SourceFileContent, + SourceFileContentContent, +) +from openai.types.eval_create_params import DataSourceConfigCustom +from azure.identity import DefaultAzureCredential +from azure.ai.projects import AIProjectClient +from azure.ai.projects.models import ( + CodeBasedEvaluatorDefinition, + EvaluatorCategory, + EvaluatorMetric, + EvaluatorMetricType, + EvaluatorMetricDirection, + EvaluatorType, + EvaluatorVersion, +) + +load_dotenv() + +endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] +model_deployment_name = os.environ.get("FOUNDRY_MODEL_NAME") +azure_openai_endpoint = os.environ["AZURE_OPENAI_ENDPOINT"] +azure_openai_api_key = os.environ["AZURE_OPENAI_API_KEY"] + +# The folder containing the FriendlyEvaluator code, including common_util/ subfolder +local_upload_folder = str(Path(__file__).parent / "custom_evaluators" / "friendly_evaluator") + +with ( + DefaultAzureCredential() as credential, + AIProjectClient(endpoint=endpoint, credential=credential) as project_client, + project_client.get_openai_client() as client, +): + # --------------------------------------------------------------- + # 1. Upload evaluator code and create evaluator version + # The folder structure uploaded is: + # friendly_evaluator/ + # friendly_evaluator.py <- entry point + # common_util/ + # __init__.py + # util.py <- helper functions + # --------------------------------------------------------------- + suffix = "".join(random.choices(string.ascii_lowercase, k=5)) + evaluator_name = f"friendly_evaluator_{suffix}" + + evaluator_version = EvaluatorVersion( + evaluator_type=EvaluatorType.CUSTOM, + categories=[EvaluatorCategory.QUALITY], + display_name="Friendliness Evaluator", + description="LLM-based evaluator that scores how friendly a response is (1-5)", + definition=CodeBasedEvaluatorDefinition( + entry_point="friendly_evaluator:FriendlyEvaluator", + init_parameters={ + "type": "object", + "properties": { + "model_config": { + "type": "object", + "description": "Azure OpenAI configuration for the LLM judge", + "properties": { + "azure_endpoint": {"type": "string"}, + "api_version": {"type": "string"}, + "api_key": {"type": "string"}, + }, + "required": ["azure_endpoint", "api_key"], + }, + "threshold": {"type": "number"}, + }, + "required": ["model_config", "threshold"], + }, + data_schema={ + "type": "object", + "properties": { + "query": {"type": "string", "description": "The original user query"}, + "response": {"type": "string", "description": "The response to evaluate for friendliness"}, + }, + "required": ["query", "response"], + }, + metrics={ + "score": EvaluatorMetric( + type=EvaluatorMetricType.ORDINAL, + desirable_direction=EvaluatorMetricDirection.INCREASE, + min_value=1, + max_value=5, + ) + }, + ), + ) + + print("Uploading FriendlyEvaluator (with nested common_util folder)...") + friendly_evaluator = project_client.beta.evaluators.upload( + name=evaluator_name, + evaluator_version=evaluator_version, + folder=local_upload_folder, + ) + + print(f"\nEvaluator created: name={friendly_evaluator.name}, version={friendly_evaluator.version}") + print(f"Evaluator ID: {friendly_evaluator.id}") + pprint(friendly_evaluator) + + # --------------------------------------------------------------- + # 2. Create an evaluation referencing the uploaded evaluator + # --------------------------------------------------------------- + data_source_config = DataSourceConfigCustom( + { + "type": "custom", + "item_schema": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "response": {"type": "string"}, + }, + "required": ["query", "response"], + }, + "include_sample_schema": True, + } + ) + + testing_criteria = [ + { + "type": "azure_ai_evaluator", + "name": evaluator_name, + "evaluator_name": evaluator_name, + "initialization_parameters": { + "deployment_name": f"{model_deployment_name}", # provide model_config or, deployment name passed is used to construct the model_config for the evaluator. + "threshold": 3, + }, + } + ] + + print("\nCreating evaluation...") + eval_object = client.evals.create( + name=f"Friendliness Evaluation - {suffix}", + data_source_config=data_source_config, + testing_criteria=testing_criteria, # type: ignore + ) + print(f"Evaluation created (id: {eval_object.id}, name: {eval_object.name})") + + # --------------------------------------------------------------- + # 3. Run the evaluation with inline data + # --------------------------------------------------------------- + print("\nCreating evaluation run with inline data...") + eval_run_object = client.evals.runs.create( + eval_id=eval_object.id, + name=f"Friendliness Eval Run - {suffix}", + metadata={"team": "eval-exp", "scenario": "friendliness-v1"}, + data_source=CreateEvalJSONLRunDataSourceParam( + type="jsonl", + source=SourceFileContent( + type="file_content", + content=[ + SourceFileContentContent( + item={ + "query": "How do I reset my password?", + "response": "Go to settings and click reset. That's it.", + } + ), + SourceFileContentContent( + item={ + "query": "I'm having trouble with my account", + "response": "I'm really sorry to hear you're having trouble! I'd love to help you get this sorted out. Could you tell me a bit more about what's happening so I can assist you better?", + } + ), + SourceFileContentContent( + item={ + "query": "Can you help me?", + "response": "Read the docs.", + } + ), + SourceFileContentContent( + item={ + "query": "What's the weather like today?", + "response": "Great question! While I'm not a weather service, I'd be happy to suggest some wonderful weather apps that can give you accurate forecasts. Would you like some recommendations? 😊", + } + ), + ], + ), + ), + ) + + print(f"Evaluation run created (id: {eval_run_object.id})") + pprint(eval_run_object) + + # --------------------------------------------------------------- + # 4. Poll for evaluation run completion + # --------------------------------------------------------------- + while True: + run = client.evals.runs.retrieve(run_id=eval_run_object.id, eval_id=eval_object.id) + if run.status in ("completed", "failed"): + print(f"\nEvaluation run finished with status: {run.status}") + output_items = list(client.evals.runs.output_items.list(run_id=run.id, eval_id=eval_object.id)) + pprint(output_items) + print(f"\nEvaluation run Report URL: {run.report_url}") + break + time.sleep(5) + print("Waiting for evaluation run to complete...") + + # --------------------------------------------------------------- + # 5. Cleanup (uncomment to delete) + # --------------------------------------------------------------- + # print("\nCleaning up...") + # project_client.beta.evaluators.delete_version( + # name=friendly_evaluator.name, + # version=friendly_evaluator.version, + # ) + # client.evals.delete(eval_id=eval_object.id) + # print("Cleanup done.") + print("\nDone - FriendlyEvaluator upload, eval creation, and eval run verified successfully.") From 6147a863987e4e116a75bfd5b140b574dd527cdb Mon Sep 17 00:00:00 2001 From: Waqas Javed <7674577+w-javed@users.noreply.github.com> Date: Thu, 2 Apr 2026 13:00:34 -0700 Subject: [PATCH 6/6] test: update upload tests for customer-controlled pattern filtering Replace test_upload_skips_pycache_and_pyc_files with two new tests: - test_upload_skips_pycache_and_pyc_files_with_patterns: verifies filtering works when patterns are provided - test_upload_uploads_all_files_without_patterns: verifies all files are uploaded when no patterns are given Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../evaluators/test_evaluators_upload.py | 38 +++++++++++++++++- .../test_evaluators_upload_async.py | 40 ++++++++++++++++++- 2 files changed, 76 insertions(+), 2 deletions(-) diff --git a/sdk/ai/azure-ai-projects/tests/evaluators/test_evaluators_upload.py b/sdk/ai/azure-ai-projects/tests/evaluators/test_evaluators_upload.py index 09e2279b2701..2932d2976984 100644 --- a/sdk/ai/azure-ai-projects/tests/evaluators/test_evaluators_upload.py +++ b/sdk/ai/azure-ai-projects/tests/evaluators/test_evaluators_upload.py @@ -4,6 +4,7 @@ # Licensed under the MIT License. # ------------------------------------ import os +import re import tempfile import pytest from unittest.mock import MagicMock, patch, call @@ -255,7 +256,7 @@ def test_upload_handles_nested_folders(self): ) assert uploaded_names == sorted(["evaluator.py", "utils/__init__.py", "utils/helper.py"]) - def test_upload_skips_pycache_and_pyc_files(self): + def test_upload_skips_pycache_and_pyc_files_with_patterns(self): ops = self._create_operations() ops.list_versions.side_effect = ResourceNotFoundError("Not found") ops.pending_upload.return_value = self._mock_pending_upload_response() @@ -270,6 +271,9 @@ def test_upload_skips_pycache_and_pyc_files(self): } ) + file_pattern = re.compile(r"^(?!\.).+(?