Skip to content

MAINT: Standardize scenario VERSION constant and replace asserts with proper lifecycle checks #1355

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Copilot wants to merge 5 commits into
base: main
Choose a base branch
from
+59 −24
Open

MAINT: Standardize scenario VERSION constant and replace asserts with proper lifecycle checks #1355

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
78130f4
Initial plan
Copilot Feb 5, 2026
e1c86b2
Standardize version to VERSION and replace assert with proper error c…
Copilot Feb 5, 2026
ac78b46
Add version alias for backward compatibility with tests
Copilot Feb 5, 2026
3cd2d55
Use explicit None checks instead of falsy checks for better type safety
Copilot Feb 5, 2026
4290950
Use VERSION as reference for version alias and add ValueError to docs…
Copilot Feb 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions pyrit/scenario/scenarios/airt/content_harms.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,8 @@ class ContentHarms(Scenario):
with respect to certain harm categories.
"""

version: int = 1
VERSION: int = 1
version: int = VERSION # Alias for backward compatibility

@classmethod
def get_strategy_class(cls) -> Type[ScenarioStrategy]:
Expand Down Expand Up @@ -179,7 +180,7 @@ def __init__(

super().__init__(
name="Content Harms",
version=self.version,
version=self.VERSION,
objective_scorer=self._objective_scorer,
strategy_class=ContentHarmsStrategy,
scenario_result_id=scenario_result_id,
Expand Down Expand Up @@ -239,9 +240,15 @@ def _get_strategy_attacks(

Returns:
List[AtomicAttack]: The constructed AtomicAttack instances for each attack type.

Raises:
ValueError: If scenario is not properly initialized.
"""
# objective_target is guaranteed to be non-None by parent class validation
assert self._objective_target is not None
if self._objective_target is None:
raise ValueError(
"Scenario not properly initialized. Call await scenario.initialize_async() before running."
)
Comment on lines +249 to +251
Copy link
Contributor

@ValbuenaVC ValbuenaVC Feb 5, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@copilot Make sure that for each scenario you add the ValueError each scenario raises to the function's docstring(s) to enforce ruff compliance.

Copy link
Contributor Author

Copilot AI Feb 5, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added ValueError documentation to all 6 affected function docstrings across the scenario files for ruff compliance. (4290950)


prompt_sending_attack = PromptSendingAttack(
objective_target=self._objective_target,
Expand Down
15 changes: 10 additions & 5 deletions pyrit/scenario/scenarios/airt/cyber.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,8 @@ class Cyber(Scenario):
techniques.
"""

version: int = 1
VERSION: int = 1
version: int = VERSION # Alias for backward compatibility
Copy link
Contributor

@rlundeen2 rlundeen2 Feb 5, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we should either print_deprecation_message when lowercase version is used, or we can likely just migrate. I think just migrating is fine since it's likely not used anywhere

If we print_deprecation_message, we should also check the unit test warnings to ensure everything is using the new VERSION

Copy link
Contributor

@rlundeen2 rlundeen2 Feb 5, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would probably just get rid of version

Copy link
Contributor

@romanlutz romanlutz Feb 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree since it's really really new.


@classmethod
def get_strategy_class(cls) -> type[ScenarioStrategy]:
Expand Down Expand Up @@ -141,7 +142,7 @@ def __init__(

super().__init__(
name="Cyber",
version=self.version,
version=self.VERSION,
strategy_class=CyberStrategy,
objective_scorer=objective_scorer,
include_default_baseline=include_baseline,
Expand Down Expand Up @@ -241,10 +242,13 @@ def _get_atomic_attack_from_strategy(self, strategy: str) -> AtomicAttack:
AtomicAttack: configured for the specified strategy.

Raises:
ValueError: if an unknown CyberStrategy is passed.
ValueError: If scenario is not properly initialized or an unknown CyberStrategy is passed.
"""
# objective_target is guaranteed to be non-None by parent class validation
assert self._objective_target is not None
if self._objective_target is None:
raise ValueError(
"Scenario not properly initialized. Call await scenario.initialize_async() before running."
)
attack_strategy: Optional[AttackStrategy[Any, Any]] = None
if strategy == "single_turn":
attack_strategy = PromptSendingAttack(
Expand All @@ -261,7 +265,8 @@ def _get_atomic_attack_from_strategy(self, strategy: str) -> AtomicAttack:
raise ValueError(f"Unknown CyberStrategy: {strategy}")

# _seed_groups is guaranteed to be set by _get_atomic_attacks_async before this method is called
assert self._seed_groups is not None, "_seed_groups must be resolved before creating atomic attacks"
if self._seed_groups is None:
raise ValueError("_seed_groups must be resolved before creating atomic attacks")

return AtomicAttack(
atomic_attack_name=f"cyber_{strategy}",
Expand Down
13 changes: 10 additions & 3 deletions pyrit/scenario/scenarios/airt/jailbreak.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,8 @@ class Jailbreak(Scenario):
scored to determine if the jailbreak was successful.
"""

version: int = 1
VERSION: int = 1
version: int = VERSION # Alias for backward compatibility

@classmethod
def get_strategy_class(cls) -> type[ScenarioStrategy]:
Expand Down Expand Up @@ -114,7 +115,7 @@ def __init__(

super().__init__(
name="Jailbreak",
version=self.version,
version=self.VERSION,
strategy_class=JailbreakStrategy,
objective_scorer=objective_scorer,
include_default_baseline=include_baseline,
Expand Down Expand Up @@ -182,9 +183,15 @@ async def _get_atomic_attack_from_jailbreak_async(self, *, jailbreak_template_na

Returns:
AtomicAttack: An atomic attack using the specified jailbreak template.

Raises:
ValueError: If scenario is not properly initialized.
"""
# objective_target is guaranteed to be non-None by parent class validation
assert self._objective_target is not None
if self._objective_target is None:
raise ValueError(
"Scenario not properly initialized. Call await scenario.initialize_async() before running."
)

# Create the jailbreak converter
jailbreak_converter = TextJailbreakConverter(
Expand Down
12 changes: 8 additions & 4 deletions pyrit/scenario/scenarios/airt/leakage_scenario.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,8 @@ class LeakageScenario(Scenario):
attack variations designed to extract sensitive information from models.
"""

version: int = 1
VERSION: int = 1
version: int = VERSION # Alias for backward compatibility

@classmethod
def get_strategy_class(cls) -> type[ScenarioStrategy]:
Expand Down Expand Up @@ -161,7 +162,7 @@ def __init__(

super().__init__(
name="Leakage Scenario",
version=self.version,
version=self.VERSION,
strategy_class=LeakageStrategy,
objective_scorer=objective_scorer,
include_default_baseline=include_baseline,
Expand Down Expand Up @@ -261,10 +262,13 @@ async def _get_atomic_attack_from_strategy_async(self, strategy: str) -> AtomicA
AtomicAttack: Configured for the specified strategy.

Raises:
ValueError: If an unknown LeakageStrategy is passed.
ValueError: If scenario is not properly initialized or an unknown LeakageStrategy is passed.
"""
# objective_target is guaranteed to be non-None by parent class validation
assert self._objective_target is not None
if self._objective_target is None:
raise ValueError(
"Scenario not properly initialized. Call await scenario.initialize_async() before running."
)

strategy_factories = {
"first_letter": self._create_first_letter_attack,
Expand Down
12 changes: 8 additions & 4 deletions pyrit/scenario/scenarios/airt/scam.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,8 @@ class Scam(Scenario):
(e.g., phishing emails, fraudulent messages) with primarily persuasion-oriented techniques.
"""

version: int = 1
VERSION: int = 1
version: int = VERSION # Alias for backward compatibility

@classmethod
def get_strategy_class(cls) -> type[ScenarioStrategy]:
Expand Down Expand Up @@ -167,7 +168,7 @@ def __init__(

super().__init__(
name="Scam",
version=self.version,
version=self.VERSION,
strategy_class=ScamStrategy,
objective_scorer=objective_scorer,
include_default_baseline=include_baseline,
Expand Down Expand Up @@ -266,10 +267,13 @@ def _get_atomic_attack_from_strategy(self, strategy: str) -> AtomicAttack:
AtomicAttack: Configured for the specified strategy.

Raises:
ValueError: If an unknown ScamStrategy is provided.
ValueError: If scenario is not properly initialized or an unknown ScamStrategy is provided.
"""
# objective_target is guaranteed to be non-None by parent class validation
assert self._objective_target is not None
if self._objective_target is None:
raise ValueError(
"Scenario not properly initialized. Call await scenario.initialize_async() before running."
)
attack_strategy: Optional[AttackStrategy[Any, Any]] = None

if strategy == "persuasive_rta":
Expand Down
5 changes: 3 additions & 2 deletions pyrit/scenario/scenarios/foundry/red_team_agent.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,8 @@ class RedTeamAgent(Scenario):
providing a consistent PyRIT contract for their integration.
"""

version: int = 1
VERSION: int = 1
version: int = VERSION # Alias for backward compatibility

@classmethod
def get_strategy_class(cls) -> Type[ScenarioStrategy]:
Expand Down Expand Up @@ -297,7 +298,7 @@ def __init__(
# Call super().__init__() first to initialize self._memory
super().__init__(
name="RedTeamAgent",
version=self.version,
version=self.VERSION,
strategy_class=FoundryStrategy,
objective_scorer=objective_scorer,
include_default_baseline=include_baseline,
Expand Down
13 changes: 10 additions & 3 deletions pyrit/scenario/scenarios/garak/encoding.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,8 @@ class Encoding(Scenario):
By default, this uses the same dataset as Garak: slur terms and web XSS payloads.
"""

version: int = 1
VERSION: int = 1
version: int = VERSION # Alias for backward compatibility

@classmethod
def get_strategy_class(cls) -> type[ScenarioStrategy]:
Expand Down Expand Up @@ -209,7 +210,7 @@ def __init__(

super().__init__(
name="Encoding",
version=self.version,
version=self.VERSION,
strategy_class=EncodingStrategy,
objective_scorer=objective_scorer,
include_default_baseline=include_baseline,
Expand Down Expand Up @@ -332,6 +333,9 @@ def _get_prompt_attacks(self, *, converters: list[PromptConverter], encoding_nam

Returns:
list[AtomicAttack]: List of atomic attacks for this encoding scheme.

Raises:
ValueError: If scenario is not properly initialized.
"""
converter_configs = [
AttackConverterConfig(
Expand All @@ -351,7 +355,10 @@ def _get_prompt_attacks(self, *, converters: list[PromptConverter], encoding_nam
atomic_attacks = []
for attack_converter_config in converter_configs:
# objective_target is guaranteed to be non-None by parent class validation
assert self._objective_target is not None
if self._objective_target is None:
raise ValueError(
"Scenario not properly initialized. Call await scenario.initialize_async() before running."
)
attack = PromptSendingAttack(
objective_target=self._objective_target,
attack_converter_config=attack_converter_config,
Expand Down