diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/aio/operations/_patch_evaluators_async.py b/sdk/ai/azure-ai-projects/azure/ai/projects/aio/operations/_patch_evaluators_async.py index c6c366fd5956..2e92b6dccf49 100644 --- a/sdk/ai/azure-ai-projects/azure/ai/projects/aio/operations/_patch_evaluators_async.py +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/aio/operations/_patch_evaluators_async.py @@ -9,6 +9,7 @@ """ import os +import re import logging from typing import Any, Final, IO, Tuple, Optional, Union from pathlib import Path @@ -43,29 +44,45 @@ class BetaEvaluatorsOperations(BetaEvaluatorsOperationsGenerated): async def _upload_folder_to_blob( container_client: ContainerClient, folder: str, + file_pattern: Optional[re.Pattern] = None, + folder_exclusions_pattern: Optional[re.Pattern] = None, **kwargs: Any, ) -> None: """Walk *folder* and upload every eligible file to the blob container. - Skips ``__pycache__``, ``.git``, ``.venv``, ``venv``, ``node_modules`` + Skips directories starting with ``.`` (e.g. ``.git``, ``.venv``), + ``__pycache__``, ``venv``, ``node_modules`` directories and ``.pyc`` / ``.pyo`` files. :param container_client: The blob container client to upload files to. :type container_client: ~azure.storage.blob.ContainerClient :param folder: Path to the local folder containing files to upload. :type folder: str + :param file_pattern: Optional regex pattern to filter files. Only files + whose name matches this pattern will be uploaded. + :type file_pattern: Optional[re.Pattern] + :param folder_exclusions_pattern: Optional regex pattern to exclude + directories. Directories whose name matches this pattern will be skipped. + :type folder_exclusions_pattern: Optional[re.Pattern] :raises ValueError: If the folder contains no uploadable files. :raises HttpResponseError: Re-raised with a friendlier message on ``AuthorizationPermissionMismatch``. """ - skip_dirs = {"__pycache__", ".git", ".venv", "venv", "node_modules"} + skip_dirs = {"__pycache__", "venv", "node_modules"} skip_extensions = {".pyc", ".pyo"} files_uploaded = False for root, dirs, files in os.walk(folder): - dirs[:] = [d for d in dirs if d not in skip_dirs] + dirs[:] = [ + d for d in dirs + if d not in skip_dirs + and not d.startswith(".") + and not (folder_exclusions_pattern and folder_exclusions_pattern.search(d)) + ] for file_name in files: - if any(file_name.endswith(ext) for ext in skip_extensions): + if file_name.startswith('.') or any(file_name.endswith(ext) for ext in skip_extensions): + continue + if file_pattern and not file_pattern.search(file_name): continue file_path = os.path.join(root, file_name) blob_name = os.path.relpath(file_path, folder).replace("\\", "/") @@ -203,6 +220,8 @@ async def upload( *, folder: str, connection_name: Optional[str] = None, + file_pattern: Optional[re.Pattern] = None, + folder_exclusions_pattern: Optional[re.Pattern] = None, **kwargs: Any, ) -> EvaluatorVersion: """Upload all files in a folder to blob storage and create a code-based evaluator version @@ -225,6 +244,12 @@ async def upload( should be uploaded. If not specified, the default Azure Storage Account connection will be used. Optional. :paramtype connection_name: str + :keyword file_pattern: Optional regex pattern to filter files. Only files whose name + matches this pattern will be uploaded. Optional. + :paramtype file_pattern: Optional[re.Pattern] + :keyword folder_exclusions_pattern: Optional regex pattern to exclude directories. + Directories whose name matches this pattern will be skipped during upload. Optional. + :paramtype folder_exclusions_pattern: Optional[re.Pattern] :return: The created evaluator version. :rtype: ~azure.ai.projects.models.EvaluatorVersion :raises ~azure.core.exceptions.HttpResponseError: If an error occurs during the HTTP request. @@ -246,7 +271,7 @@ async def upload( ) async with container_client: - await self._upload_folder_to_blob(container_client, folder, **kwargs) + await self._upload_folder_to_blob(container_client, folder, file_pattern, folder_exclusions_pattern, **kwargs) self._set_blob_uri(evaluator_version, blob_uri) result = await self.create_version( diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/operations/_patch_evaluators.py b/sdk/ai/azure-ai-projects/azure/ai/projects/operations/_patch_evaluators.py index 3f1c38d97b2b..093055a25408 100644 --- a/sdk/ai/azure-ai-projects/azure/ai/projects/operations/_patch_evaluators.py +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/operations/_patch_evaluators.py @@ -9,6 +9,7 @@ """ import os +import re import logging from typing import Any, Final, IO, Tuple, Optional, Union from pathlib import Path @@ -43,29 +44,45 @@ class BetaEvaluatorsOperations(BetaEvaluatorsOperationsGenerated): def _upload_folder_to_blob( container_client: ContainerClient, folder: str, + file_pattern: Optional[re.Pattern] = None, + folder_exclusions_pattern: Optional[re.Pattern] = None, **kwargs: Any, ) -> None: """Walk *folder* and upload every eligible file to the blob container. - Skips ``__pycache__``, ``.git``, ``.venv``, ``venv``, ``node_modules`` + Skips directories starting with ``.`` (e.g. ``.git``, ``.venv``), + ``__pycache__``, ``venv``, ``node_modules`` directories and ``.pyc`` / ``.pyo`` files. :param container_client: The blob container client to upload files to. :type container_client: ~azure.storage.blob.ContainerClient :param folder: Path to the local folder containing files to upload. :type folder: str + :param file_pattern: Optional regex pattern to filter files. Only files + whose name matches this pattern will be uploaded. + :type file_pattern: Optional[re.Pattern] + :param folder_exclusions_pattern: Optional regex pattern to exclude + directories. Directories whose name matches this pattern will be skipped. + :type folder_exclusions_pattern: Optional[re.Pattern] :raises ValueError: If the folder contains no uploadable files. :raises HttpResponseError: Re-raised with a friendlier message on ``AuthorizationPermissionMismatch``. """ - skip_dirs = {"__pycache__", ".git", ".venv", "venv", "node_modules"} + skip_dirs = {"__pycache__", "venv", "node_modules"} skip_extensions = {".pyc", ".pyo"} files_uploaded = False for root, dirs, files in os.walk(folder): - dirs[:] = [d for d in dirs if d not in skip_dirs] + dirs[:] = [ + d for d in dirs + if d not in skip_dirs + and not d.startswith(".") + and not (folder_exclusions_pattern and folder_exclusions_pattern.search(d)) + ] for file_name in files: - if any(file_name.endswith(ext) for ext in skip_extensions): + if file_name.startswith('.') or any(file_name.endswith(ext) for ext in skip_extensions): + continue + if file_pattern and not file_pattern.search(file_name): continue file_path = os.path.join(root, file_name) blob_name = os.path.relpath(file_path, folder).replace("\\", "/") @@ -203,6 +220,8 @@ def upload( *, folder: str, connection_name: Optional[str] = None, + file_pattern: Optional[re.Pattern] = None, + folder_exclusions_pattern: Optional[re.Pattern] = None, **kwargs: Any, ) -> EvaluatorVersion: """Upload all files in a folder to blob storage and create a code-based evaluator version @@ -225,6 +244,12 @@ def upload( should be uploaded. If not specified, the default Azure Storage Account connection will be used. Optional. :paramtype connection_name: str + :keyword file_pattern: Optional regex pattern to filter files. Only files whose name + matches this pattern will be uploaded. Optional. + :paramtype file_pattern: Optional[re.Pattern] + :keyword folder_exclusions_pattern: Optional regex pattern to exclude directories. + Directories whose name matches this pattern will be skipped during upload. Optional. + :paramtype folder_exclusions_pattern: Optional[re.Pattern] :return: The created evaluator version. :rtype: ~azure.ai.projects.models.EvaluatorVersion :raises ~azure.core.exceptions.HttpResponseError: If an error occurs during the HTTP request. @@ -246,7 +271,7 @@ def upload( ) with container_client: - self._upload_folder_to_blob(container_client, folder, **kwargs) + self._upload_folder_to_blob(container_client, folder, file_pattern, folder_exclusions_pattern, **kwargs) self._set_blob_uri(evaluator_version, blob_uri) result = self.create_version( diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_eval_upload_friendly_evaluator.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_eval_upload_friendly_evaluator.py index 67e168d3509a..e69de29bb2d1 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_eval_upload_friendly_evaluator.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_eval_upload_friendly_evaluator.py @@ -1,244 +0,0 @@ -# pylint: disable=line-too-long,useless-suppression -# ------------------------------------ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. -# ------------------------------------ - -""" -DESCRIPTION: - Given an AIProjectClient, this sample demonstrates how to: - 1. Upload a custom LLM-based evaluator (FriendlyEvaluator) with nested - folder structure (common_util/) using `evaluators.upload()`. - 2. Create an evaluation (eval) that references the uploaded evaluator. - 3. Run the evaluation with inline data and poll for results. - - The FriendlyEvaluator calls Azure OpenAI to judge the friendliness of a - response and returns score, label, reason, and explanation. - -USAGE: - python sample_eval_upload_friendly_evaluator.py - - Before running the sample: - - pip install "azure-ai-projects>=2.0.0b4" azure-storage-blob python-dotenv azure-identity openai - - Set these environment variables with your own values: - 1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint. - 2) FOUNDRY_MODEL_NAME - Optional. The name of the model deployment to use for evaluation. -""" - -import os -import time -import random -import string -from pathlib import Path -from pprint import pprint - -from dotenv import load_dotenv -from openai.types.evals.create_eval_jsonl_run_data_source_param import ( - CreateEvalJSONLRunDataSourceParam, - SourceFileContent, - SourceFileContentContent, -) -from openai.types.eval_create_params import DataSourceConfigCustom -from azure.identity import DefaultAzureCredential -from azure.ai.projects import AIProjectClient -from azure.ai.projects.models import ( - CodeBasedEvaluatorDefinition, - EvaluatorCategory, - EvaluatorMetric, - EvaluatorMetricType, - EvaluatorMetricDirection, - EvaluatorType, - EvaluatorVersion, -) - -load_dotenv() - -endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] -model_deployment_name = os.environ.get("FOUNDRY_MODEL_NAME") -azure_openai_endpoint = os.environ["AZURE_OPENAI_ENDPOINT"] -azure_openai_api_key = os.environ["AZURE_OPENAI_API_KEY"] - -# The folder containing the FriendlyEvaluator code, including common_util/ subfolder -local_upload_folder = str(Path(__file__).parent / "custom_evaluators" / "friendly_evaluator") - -with ( - DefaultAzureCredential() as credential, - AIProjectClient(endpoint=endpoint, credential=credential) as project_client, - project_client.get_openai_client() as client, -): - # --------------------------------------------------------------- - # 1. Upload evaluator code and create evaluator version - # The folder structure uploaded is: - # friendly_evaluator/ - # friendly_evaluator.py <- entry point - # common_util/ - # __init__.py - # util.py <- helper functions - # --------------------------------------------------------------- - suffix = "".join(random.choices(string.ascii_lowercase, k=5)) - evaluator_name = f"friendly_evaluator_{suffix}" - - evaluator_version = EvaluatorVersion( - evaluator_type=EvaluatorType.CUSTOM, - categories=[EvaluatorCategory.QUALITY], - display_name="Friendliness Evaluator", - description="LLM-based evaluator that scores how friendly a response is (1-5)", - definition=CodeBasedEvaluatorDefinition( - entry_point="friendly_evaluator:FriendlyEvaluator", - init_parameters={ - "type": "object", - "properties": { - "model_config": { - "type": "object", - "description": "Azure OpenAI configuration for the LLM judge", - "properties": { - "azure_endpoint": {"type": "string"}, - "api_version": {"type": "string"}, - "api_key": {"type": "string"}, - }, - "required": ["azure_endpoint", "api_key"], - }, - "threshold": {"type": "number"}, - }, - "required": ["model_config", "threshold"], - }, - data_schema={ - "type": "object", - "properties": { - "query": {"type": "string", "description": "The original user query"}, - "response": {"type": "string", "description": "The response to evaluate for friendliness"}, - }, - "required": ["query", "response"], - }, - metrics={ - "score": EvaluatorMetric( - type=EvaluatorMetricType.ORDINAL, - desirable_direction=EvaluatorMetricDirection.INCREASE, - min_value=1, - max_value=5, - ) - }, - ), - ) - - print("Uploading FriendlyEvaluator (with nested common_util folder)...") - friendly_evaluator = project_client.beta.evaluators.upload( - name=evaluator_name, - evaluator_version=evaluator_version, - folder=local_upload_folder, - ) - - print(f"\nEvaluator created: name={friendly_evaluator.name}, version={friendly_evaluator.version}") - print(f"Evaluator ID: {friendly_evaluator.id}") - pprint(friendly_evaluator) - - # --------------------------------------------------------------- - # 2. Create an evaluation referencing the uploaded evaluator - # --------------------------------------------------------------- - data_source_config = DataSourceConfigCustom( - { - "type": "custom", - "item_schema": { - "type": "object", - "properties": { - "query": {"type": "string"}, - "response": {"type": "string"}, - }, - "required": ["query", "response"], - }, - "include_sample_schema": True, - } - ) - - testing_criteria = [ - { - "type": "azure_ai_evaluator", - "name": evaluator_name, - "evaluator_name": evaluator_name, - "initialization_parameters": { - "deployment_name": f"{model_deployment_name}", # provide model_config or, deployment name passed is used to construct the model_config for the evaluator. - "threshold": 3, - }, - } - ] - - print("\nCreating evaluation...") - eval_object = client.evals.create( - name=f"Friendliness Evaluation - {suffix}", - data_source_config=data_source_config, - testing_criteria=testing_criteria, # type: ignore - ) - print(f"Evaluation created (id: {eval_object.id}, name: {eval_object.name})") - - # --------------------------------------------------------------- - # 3. Run the evaluation with inline data - # --------------------------------------------------------------- - print("\nCreating evaluation run with inline data...") - eval_run_object = client.evals.runs.create( - eval_id=eval_object.id, - name=f"Friendliness Eval Run - {suffix}", - metadata={"team": "eval-exp", "scenario": "friendliness-v1"}, - data_source=CreateEvalJSONLRunDataSourceParam( - type="jsonl", - source=SourceFileContent( - type="file_content", - content=[ - SourceFileContentContent( - item={ - "query": "How do I reset my password?", - "response": "Go to settings and click reset. That's it.", - } - ), - SourceFileContentContent( - item={ - "query": "I'm having trouble with my account", - "response": "I'm really sorry to hear you're having trouble! I'd love to help you get this sorted out. Could you tell me a bit more about what's happening so I can assist you better?", - } - ), - SourceFileContentContent( - item={ - "query": "Can you help me?", - "response": "Read the docs.", - } - ), - SourceFileContentContent( - item={ - "query": "What's the weather like today?", - "response": "Great question! While I'm not a weather service, I'd be happy to suggest some wonderful weather apps that can give you accurate forecasts. Would you like some recommendations? 😊", - } - ), - ], - ), - ), - ) - - print(f"Evaluation run created (id: {eval_run_object.id})") - pprint(eval_run_object) - - # --------------------------------------------------------------- - # 4. Poll for evaluation run completion - # --------------------------------------------------------------- - while True: - run = client.evals.runs.retrieve(run_id=eval_run_object.id, eval_id=eval_object.id) - if run.status in ("completed", "failed"): - print(f"\nEvaluation run finished with status: {run.status}") - output_items = list(client.evals.runs.output_items.list(run_id=run.id, eval_id=eval_object.id)) - pprint(output_items) - print(f"\nEvaluation run Report URL: {run.report_url}") - break - time.sleep(5) - print("Waiting for evaluation run to complete...") - - # --------------------------------------------------------------- - # 5. Cleanup (uncomment to delete) - # --------------------------------------------------------------- - # print("\nCleaning up...") - # project_client.beta.evaluators.delete_version( - # name=friendly_evaluator.name, - # version=friendly_evaluator.version, - # ) - # client.evals.delete(eval_id=eval_object.id) - # print("Cleanup done.") - print("\nDone - FriendlyEvaluator upload, eval creation, and eval run verified successfully.")