FEAT Add ASR reporting by converter type

pyrit/analytics/result_analysis.py

@@ -38,7 +38,8 @@ def analyze_results(attack_results: list[AttackResult]) -> dict[str, AttackStats
     Returns:
         A dictionary of AttackStats objects. The overall stats are accessible with the key
         "Overall", and the stats of any attack can be retrieved using "By_attack_identifier"
-        followed by the identifier of the attack.
+        followed by the identifier of the attack. Stats grouped by converter type can be
+        retrieved using "By_converter_type".
 
     Raises:
         ValueError: if attack_results is empty.
@@ -48,14 +49,16 @@ def analyze_results(attack_results: list[AttackResult]) -> dict[str, AttackStats
         >>> analyze_results(attack_results)
         {
             "Overall": AttackStats,
-            "By_attack_identifier": dict[str, AttackStats]
+            "By_attack_identifier": dict[str, AttackStats],
+            "By_converter_type": dict[str, AttackStats]
         }
     """
     if not attack_results:
         raise ValueError("attack_results cannot be empty")
 
     overall_counts: DefaultDict[str, int] = defaultdict(int)
     by_type_counts: DefaultDict[str, DefaultDict[str, int]] = defaultdict(lambda: defaultdict(int))
+    by_converter_counts: DefaultDict[str, DefaultDict[str, int]] = defaultdict(lambda: defaultdict(int))
 
     for attack in attack_results:
         if not isinstance(attack, AttackResult):
@@ -64,15 +67,30 @@ def analyze_results(attack_results: list[AttackResult]) -> dict[str, AttackStats
         outcome = attack.outcome
         attack_type = attack.attack_identifier.get("type", "unknown")
 
+        # Extract converter types from last_response
+        converter_types = []
+        if attack.last_response is not None and attack.last_response.converter_identifiers:
+            converter_types = [conv.class_name for conv in attack.last_response.converter_identifiers]
+
+        # If no converters, track as "no_converter"
+        if not converter_types:
+            converter_types = ["no_converter"]
+
         if outcome == AttackOutcome.SUCCESS:
             overall_counts["successes"] += 1
             by_type_counts[attack_type]["successes"] += 1
+            for converter_type in converter_types:
+                by_converter_counts[converter_type]["successes"] += 1
         elif outcome == AttackOutcome.FAILURE:
             overall_counts["failures"] += 1
             by_type_counts[attack_type]["failures"] += 1
+            for converter_type in converter_types:
+                by_converter_counts[converter_type]["failures"] += 1
         else:
             overall_counts["undetermined"] += 1
             by_type_counts[attack_type]["undetermined"] += 1
+            for converter_type in converter_types:
+                by_converter_counts[converter_type]["undetermined"] += 1
 
     overall_stats = _compute_stats(
         successes=overall_counts["successes"],
@@ -89,7 +107,17 @@ def analyze_results(attack_results: list[AttackResult]) -> dict[str, AttackStats
         for attack_type, counts in by_type_counts.items()
     }
 
+    by_converter_stats = {
+        converter_type: _compute_stats(
+            successes=counts["successes"],
+            failures=counts["failures"],
+            undetermined=counts["undetermined"],
+        )
+        for converter_type, counts in by_converter_counts.items()
+    }
+
     return {
         "Overall": overall_stats,
         "By_attack_identifier": by_type_stats,
+        "By_converter_type": by_converter_stats,
     }

tests/unit/analytics/test_result_analysis.py

@@ -4,7 +4,8 @@
 import pytest
 
 from pyrit.analytics.result_analysis import AttackStats, analyze_results
-from pyrit.models import AttackOutcome, AttackResult
+from pyrit.identifiers import ConverterIdentifier
+from pyrit.models import AttackOutcome, AttackResult, MessagePiece
 
 
 # helpers
@@ -28,6 +29,24 @@ def make_attack(
     )
 
 
+def make_converter(
+    class_name: str,
+    class_module: str = "pyrit.prompt_converter.test_converter",
+) -> ConverterIdentifier:
+    """
+    Create a test ConverterIdentifier with minimal required fields.
+    """
+    return ConverterIdentifier(
+        class_name=class_name,
+        class_module=class_module,
+        class_description="Test converter",
+        identifier_type="instance",
+        supported_input_types=("text",),
+        supported_output_types=("text",),
+    )
+
+
+
 def test_analyze_results_empty_raises():
     with pytest.raises(ValueError):
         analyze_results([])
@@ -133,3 +152,176 @@ def test_group_by_attack_type_parametrized(items, type_key, exp_succ, exp_fail,
     assert stats.undetermined == exp_und
     assert stats.total_decided == exp_succ + exp_fail
     assert stats.success_rate == exp_rate
+
+
+def test_analyze_results_returns_by_converter_type():
+    """Test that analyze_results returns By_converter_type key."""
+    attacks = [make_attack(AttackOutcome.SUCCESS)]
+    result = analyze_results(attacks)
+
+    assert "By_converter_type" in result
+    assert isinstance(result["By_converter_type"], dict)
+
+
+def test_analyze_results_no_converter_tracking():
+    """Test that attacks without converters are tracked as 'no_converter'."""
+    attacks = [
+        AttackResult(
+            conversation_id="conv-1",
+            objective="test",
+            attack_identifier={"type": "test"},
+            outcome=AttackOutcome.SUCCESS,
+            last_response=None,  # No response, so no converters
+        ),
+        AttackResult(
+            conversation_id="conv-2",
+            objective="test",
+            attack_identifier={"type": "test"},
+            outcome=AttackOutcome.FAILURE,
+            last_response=None,
+        ),
+    ]
+    result = analyze_results(attacks)
+
+    assert "no_converter" in result["By_converter_type"]
+    stats = result["By_converter_type"]["no_converter"]
+    assert stats.successes == 1
+    assert stats.failures == 1
+    assert stats.total_decided == 2
+    assert stats.success_rate == 0.5
+
+
+def test_analyze_results_with_converter_identifiers():
+    """Test that attacks with converters are properly grouped by converter type."""
+    # Create attacks with different converters
+    converter1 = make_converter("Base64Converter", "pyrit.prompt_converter.base64_converter")
+    converter2 = make_converter("ROT13Converter", "pyrit.prompt_converter.rot13_converter")
+
+    message1 = MessagePiece(
+        role="user",
+        original_value="test",
+        converter_identifiers=[converter1],
+    )
+
+    message2 = MessagePiece(
+        role="user",
+        original_value="test",
+        converter_identifiers=[converter2],
+    )
+
+    message3 = MessagePiece(
+        role="user",
+        original_value="test",
+        converter_identifiers=[converter1],
+    )
+
+    attacks = [
+        AttackResult(
+            conversation_id="conv-1",
+            objective="test",
+            attack_identifier={"type": "test"},
+            outcome=AttackOutcome.SUCCESS,
+            last_response=message1,
+        ),
+        AttackResult(
+            conversation_id="conv-2",
+            objective="test",
+            attack_identifier={"type": "test"},
+            outcome=AttackOutcome.FAILURE,
+            last_response=message2,
+        ),
+        AttackResult(
+            conversation_id="conv-3",
+            objective="test",
+            attack_identifier={"type": "test"},
+            outcome=AttackOutcome.SUCCESS,
+            last_response=message3,
+        ),
+    ]
+
+    result = analyze_results(attacks)
+
+    # Check Base64Converter stats
+    assert "Base64Converter" in result["By_converter_type"]
+    base64_stats = result["By_converter_type"]["Base64Converter"]
+    assert base64_stats.successes == 2
+    assert base64_stats.failures == 0
+    assert base64_stats.total_decided == 2
+    assert base64_stats.success_rate == 1.0
+
+    # Check ROT13Converter stats
+    assert "ROT13Converter" in result["By_converter_type"]
+    rot13_stats = result["By_converter_type"]["ROT13Converter"]
+    assert rot13_stats.successes == 0
+    assert rot13_stats.failures == 1
+    assert rot13_stats.total_decided == 1
+    assert rot13_stats.success_rate == 0.0
+
+
+def test_analyze_results_multiple_converters_per_attack():
+    """Test that attacks with multiple converters count towards each converter's stats."""
+    converter1 = make_converter("Base64Converter", "pyrit.prompt_converter.base64_converter")
+    converter2 = make_converter("ROT13Converter", "pyrit.prompt_converter.rot13_converter")
+
+    # Attack with multiple converters (pipeline)
+    message = MessagePiece(
+        role="user",
+        original_value="test",
+        converter_identifiers=[converter1, converter2],
+    )
+
+    attacks = [
+        AttackResult(
+            conversation_id="conv-1",
+            objective="test",
+            attack_identifier={"type": "test"},
+            outcome=AttackOutcome.SUCCESS,
+            last_response=message,
+        ),
+    ]
+
+    result = analyze_results(attacks)
+
+    # Both converters should have the success counted
+    assert "Base64Converter" in result["By_converter_type"]
+    assert result["By_converter_type"]["Base64Converter"].successes == 1
+    assert "ROT13Converter" in result["By_converter_type"]
+    assert result["By_converter_type"]["ROT13Converter"].successes == 1
+
+
+def test_analyze_results_converter_with_undetermined():
+    """Test that undetermined outcomes are tracked correctly for converters."""
+    converter = make_converter("Base64Converter", "pyrit.prompt_converter.base64_converter")
+
+    message = MessagePiece(
+        role="user",
+        original_value="test",
+        converter_identifiers=[converter],
+    )
+
+    attacks = [
+        AttackResult(
+            conversation_id="conv-1",
+            objective="test",
+            attack_identifier={"type": "test"},
+            outcome=AttackOutcome.SUCCESS,
+            last_response=message,
+        ),
+        AttackResult(
+            conversation_id="conv-2",
+            objective="test",
+            attack_identifier={"type": "test"},
+            outcome=AttackOutcome.UNDETERMINED,
+            last_response=message,
+        ),
+    ]
+
+    result = analyze_results(attacks)
+
+    assert "Base64Converter" in result["By_converter_type"]
+    stats = result["By_converter_type"]["Base64Converter"]
+    assert stats.successes == 1
+    assert stats.failures == 0
+    assert stats.undetermined == 1
+    assert stats.total_decided == 1
+    assert stats.success_rate == 1.0