diff --git a/pyrit/analytics/result_analysis.py b/pyrit/analytics/result_analysis.py index 7db3c02e87..63a6dadcff 100644 --- a/pyrit/analytics/result_analysis.py +++ b/pyrit/analytics/result_analysis.py @@ -38,7 +38,8 @@ def analyze_results(attack_results: list[AttackResult]) -> dict[str, AttackStats Returns: A dictionary of AttackStats objects. The overall stats are accessible with the key "Overall", and the stats of any attack can be retrieved using "By_attack_identifier" - followed by the identifier of the attack. + followed by the identifier of the attack. Stats grouped by converter type can be + retrieved using "By_converter_type". Raises: ValueError: if attack_results is empty. @@ -48,7 +49,8 @@ def analyze_results(attack_results: list[AttackResult]) -> dict[str, AttackStats >>> analyze_results(attack_results) { "Overall": AttackStats, - "By_attack_identifier": dict[str, AttackStats] + "By_attack_identifier": dict[str, AttackStats], + "By_converter_type": dict[str, AttackStats] } """ if not attack_results: @@ -56,6 +58,7 @@ def analyze_results(attack_results: list[AttackResult]) -> dict[str, AttackStats overall_counts: DefaultDict[str, int] = defaultdict(int) by_type_counts: DefaultDict[str, DefaultDict[str, int]] = defaultdict(lambda: defaultdict(int)) + by_converter_counts: DefaultDict[str, DefaultDict[str, int]] = defaultdict(lambda: defaultdict(int)) for attack in attack_results: if not isinstance(attack, AttackResult): @@ -64,15 +67,30 @@ def analyze_results(attack_results: list[AttackResult]) -> dict[str, AttackStats outcome = attack.outcome attack_type = attack.attack_identifier.get("type", "unknown") + # Extract converter types from last_response + converter_types = [] + if attack.last_response is not None and attack.last_response.converter_identifiers: + converter_types = [conv.class_name for conv in attack.last_response.converter_identifiers] + + # If no converters, track as "no_converter" + if not converter_types: + converter_types = ["no_converter"] + if outcome == AttackOutcome.SUCCESS: overall_counts["successes"] += 1 by_type_counts[attack_type]["successes"] += 1 + for converter_type in converter_types: + by_converter_counts[converter_type]["successes"] += 1 elif outcome == AttackOutcome.FAILURE: overall_counts["failures"] += 1 by_type_counts[attack_type]["failures"] += 1 + for converter_type in converter_types: + by_converter_counts[converter_type]["failures"] += 1 else: overall_counts["undetermined"] += 1 by_type_counts[attack_type]["undetermined"] += 1 + for converter_type in converter_types: + by_converter_counts[converter_type]["undetermined"] += 1 overall_stats = _compute_stats( successes=overall_counts["successes"], @@ -89,7 +107,17 @@ def analyze_results(attack_results: list[AttackResult]) -> dict[str, AttackStats for attack_type, counts in by_type_counts.items() } + by_converter_stats = { + converter_type: _compute_stats( + successes=counts["successes"], + failures=counts["failures"], + undetermined=counts["undetermined"], + ) + for converter_type, counts in by_converter_counts.items() + } + return { "Overall": overall_stats, "By_attack_identifier": by_type_stats, + "By_converter_type": by_converter_stats, } diff --git a/tests/unit/analytics/test_result_analysis.py b/tests/unit/analytics/test_result_analysis.py index 44b2a56e8a..4bd9e7ca4b 100644 --- a/tests/unit/analytics/test_result_analysis.py +++ b/tests/unit/analytics/test_result_analysis.py @@ -4,7 +4,8 @@ import pytest from pyrit.analytics.result_analysis import AttackStats, analyze_results -from pyrit.models import AttackOutcome, AttackResult +from pyrit.identifiers import ConverterIdentifier +from pyrit.models import AttackOutcome, AttackResult, MessagePiece # helpers @@ -28,6 +29,24 @@ def make_attack( ) +def make_converter( + class_name: str, + class_module: str = "pyrit.prompt_converter.test_converter", +) -> ConverterIdentifier: + """ + Create a test ConverterIdentifier with minimal required fields. + """ + return ConverterIdentifier( + class_name=class_name, + class_module=class_module, + class_description="Test converter", + identifier_type="instance", + supported_input_types=("text",), + supported_output_types=("text",), + ) + + + def test_analyze_results_empty_raises(): with pytest.raises(ValueError): analyze_results([]) @@ -133,3 +152,176 @@ def test_group_by_attack_type_parametrized(items, type_key, exp_succ, exp_fail, assert stats.undetermined == exp_und assert stats.total_decided == exp_succ + exp_fail assert stats.success_rate == exp_rate + + +def test_analyze_results_returns_by_converter_type(): + """Test that analyze_results returns By_converter_type key.""" + attacks = [make_attack(AttackOutcome.SUCCESS)] + result = analyze_results(attacks) + + assert "By_converter_type" in result + assert isinstance(result["By_converter_type"], dict) + + +def test_analyze_results_no_converter_tracking(): + """Test that attacks without converters are tracked as 'no_converter'.""" + attacks = [ + AttackResult( + conversation_id="conv-1", + objective="test", + attack_identifier={"type": "test"}, + outcome=AttackOutcome.SUCCESS, + last_response=None, # No response, so no converters + ), + AttackResult( + conversation_id="conv-2", + objective="test", + attack_identifier={"type": "test"}, + outcome=AttackOutcome.FAILURE, + last_response=None, + ), + ] + result = analyze_results(attacks) + + assert "no_converter" in result["By_converter_type"] + stats = result["By_converter_type"]["no_converter"] + assert stats.successes == 1 + assert stats.failures == 1 + assert stats.total_decided == 2 + assert stats.success_rate == 0.5 + + +def test_analyze_results_with_converter_identifiers(): + """Test that attacks with converters are properly grouped by converter type.""" + # Create attacks with different converters + converter1 = make_converter("Base64Converter", "pyrit.prompt_converter.base64_converter") + converter2 = make_converter("ROT13Converter", "pyrit.prompt_converter.rot13_converter") + + message1 = MessagePiece( + role="user", + original_value="test", + converter_identifiers=[converter1], + ) + + message2 = MessagePiece( + role="user", + original_value="test", + converter_identifiers=[converter2], + ) + + message3 = MessagePiece( + role="user", + original_value="test", + converter_identifiers=[converter1], + ) + + attacks = [ + AttackResult( + conversation_id="conv-1", + objective="test", + attack_identifier={"type": "test"}, + outcome=AttackOutcome.SUCCESS, + last_response=message1, + ), + AttackResult( + conversation_id="conv-2", + objective="test", + attack_identifier={"type": "test"}, + outcome=AttackOutcome.FAILURE, + last_response=message2, + ), + AttackResult( + conversation_id="conv-3", + objective="test", + attack_identifier={"type": "test"}, + outcome=AttackOutcome.SUCCESS, + last_response=message3, + ), + ] + + result = analyze_results(attacks) + + # Check Base64Converter stats + assert "Base64Converter" in result["By_converter_type"] + base64_stats = result["By_converter_type"]["Base64Converter"] + assert base64_stats.successes == 2 + assert base64_stats.failures == 0 + assert base64_stats.total_decided == 2 + assert base64_stats.success_rate == 1.0 + + # Check ROT13Converter stats + assert "ROT13Converter" in result["By_converter_type"] + rot13_stats = result["By_converter_type"]["ROT13Converter"] + assert rot13_stats.successes == 0 + assert rot13_stats.failures == 1 + assert rot13_stats.total_decided == 1 + assert rot13_stats.success_rate == 0.0 + + +def test_analyze_results_multiple_converters_per_attack(): + """Test that attacks with multiple converters count towards each converter's stats.""" + converter1 = make_converter("Base64Converter", "pyrit.prompt_converter.base64_converter") + converter2 = make_converter("ROT13Converter", "pyrit.prompt_converter.rot13_converter") + + # Attack with multiple converters (pipeline) + message = MessagePiece( + role="user", + original_value="test", + converter_identifiers=[converter1, converter2], + ) + + attacks = [ + AttackResult( + conversation_id="conv-1", + objective="test", + attack_identifier={"type": "test"}, + outcome=AttackOutcome.SUCCESS, + last_response=message, + ), + ] + + result = analyze_results(attacks) + + # Both converters should have the success counted + assert "Base64Converter" in result["By_converter_type"] + assert result["By_converter_type"]["Base64Converter"].successes == 1 + assert "ROT13Converter" in result["By_converter_type"] + assert result["By_converter_type"]["ROT13Converter"].successes == 1 + + +def test_analyze_results_converter_with_undetermined(): + """Test that undetermined outcomes are tracked correctly for converters.""" + converter = make_converter("Base64Converter", "pyrit.prompt_converter.base64_converter") + + message = MessagePiece( + role="user", + original_value="test", + converter_identifiers=[converter], + ) + + attacks = [ + AttackResult( + conversation_id="conv-1", + objective="test", + attack_identifier={"type": "test"}, + outcome=AttackOutcome.SUCCESS, + last_response=message, + ), + AttackResult( + conversation_id="conv-2", + objective="test", + attack_identifier={"type": "test"}, + outcome=AttackOutcome.UNDETERMINED, + last_response=message, + ), + ] + + result = analyze_results(attacks) + + assert "Base64Converter" in result["By_converter_type"] + stats = result["By_converter_type"]["Base64Converter"] + assert stats.successes == 1 + assert stats.failures == 0 + assert stats.undetermined == 1 + assert stats.total_decided == 1 + assert stats.success_rate == 1.0