diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3f48429..6cefb4b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,7 @@ # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks +default_language_version: + python: python3.10 repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index 4001205..050ad1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,15 @@ ## [Unreleased] ### Added +- Attribute key length and number truncation, by @HardNorth +- Binary character replacement in basic text fields, by @HardNorth + +## [5.7.0] +### Added - Official `Python 3.14` support, by @HardNorth - Custom log level support in `RPLogHandler` class, by @HardNorth ### Removed -- `Python 3.7` support, by @HardNorth +- `Python 3.8` support, by @HardNorth - Deprecated `log_manager.py` module, by @HardNorth ## [5.6.7] diff --git a/reportportal_client/__init__.py b/reportportal_client/__init__.py index bf99d38..4b344e5 100644 --- a/reportportal_client/__init__.py +++ b/reportportal_client/__init__.py @@ -62,6 +62,12 @@ class _ClientOptions(TypedDict, total=False): launch_uuid_print: bool print_output: OutputType truncate_attributes: bool + truncate_fields: bool + replace_binary_chars: bool + launch_name_length_limit: int + item_name_length_limit: int + launch_description_length_limit: int + item_description_length_limit: int log_batch_size: int log_batch_payload_limit: int # Async client specific parameters @@ -80,42 +86,48 @@ def create_client( ) -> Optional[RP]: """Create and ReportPortal Client based on the type and arguments provided. - :param client_type: Type of the Client to create. - :param endpoint: Endpoint of the ReportPortal service. - :param project: Project name to report to. - :param api_key: Authorization API key. - :param oauth_uri: OAuth 2.0 token endpoint URI (for OAuth authentication). - :param oauth_username: Username for OAuth 2.0 authentication. - :param oauth_password: Password for OAuth 2.0 authentication. - :param oauth_client_id: OAuth 2.0 client ID. - :param oauth_client_secret: OAuth 2.0 client secret (optional). - :param oauth_scope: OAuth 2.0 scope (optional). - :param launch_uuid: A launch UUID to use instead of starting own one. - :param is_skipped_an_issue: Option to mark skipped tests as not 'To Investigate' items on the server - side. - :param verify_ssl: Option to skip ssl verification. - :param retries: Number of retry attempts to make in case of connection / server - errors. - :param max_pool_size: Option to set the maximum number of connections to save the pool. - :param http_timeout : A float in seconds for connect and read timeout. Use a Tuple to - specific connect and read separately. - :param mode: Launch mode, all Launches started by the client will be in that mode. - :param launch_uuid_print: Print Launch UUID into passed TextIO or by default to stdout. - :param print_output: Set output stream for Launch UUID printing. - :param truncate_attributes: Truncate test item attributes to default maximum length. - :param log_batch_size: Option to set the maximum number of logs that can be processed in one - batch. - :param log_batch_payload_limit: Maximum size in bytes of logs that can be processed in one batch. - :param keepalive_timeout: For Async Clients only. Maximum amount of idle time in seconds before - force connection closing. - :param task_timeout: For Async Threaded and Batched Clients only. Time limit in seconds for a - Task processing. - :param shutdown_timeout: For Async Threaded and Batched Clients only. Time limit in seconds for - shutting down internal Tasks. - :param trigger_num: For Async Batched Client only. Number of tasks which triggers Task batch - execution. - :param trigger_interval: For Async Batched Client only. Time limit which triggers Task batch - execution. + :param client_type: Type of the Client to create. + :param endpoint: Endpoint of the ReportPortal service. + :param project: Project name to report to. + :param api_key: Authorization API key. + :param oauth_uri: OAuth 2.0 token endpoint URI (for OAuth authentication). + :param oauth_username: Username for OAuth 2.0 authentication. + :param oauth_password: Password for OAuth 2.0 authentication. + :param oauth_client_id: OAuth 2.0 client ID. + :param oauth_client_secret: OAuth 2.0 client secret (optional). + :param oauth_scope: OAuth 2.0 scope (optional). + :param launch_uuid: A launch UUID to use instead of starting own one. + :param is_skipped_an_issue: Option to mark skipped tests as not 'To Investigate' items on the server + side. + :param verify_ssl: Option to skip ssl verification. + :param retries: Number of retry attempts to make in case of connection / server + errors. + :param max_pool_size: Option to set the maximum number of connections to save the pool. + :param http_timeout: A float in seconds for connect and read timeout. Use a Tuple to + specific connect and read separately. + :param mode: Launch mode, all Launches started by the client will be in that mode. + :param launch_uuid_print: Print Launch UUID into passed TextIO or by default to stdout. + :param print_output: Set output stream for Launch UUID printing. + :param truncate_attributes: Truncate test item attributes to default maximum length. + :param truncate_fields: Truncate request fields to configured limits. + :param replace_binary_chars: Toggle replacement of basic binary characters with \ufffd char. + :param launch_name_length_limit: Maximum allowed launch name length. + :param item_name_length_limit: Maximum allowed test item name length. + :param launch_description_length_limit: Maximum allowed launch description length. + :param item_description_length_limit: Maximum allowed test item description length. + :param log_batch_size: Option to set the maximum number of logs that can be processed in one + batch. + :param log_batch_payload_limit: Maximum size in bytes of logs that can be processed in one batch. + :param keepalive_timeout: For Async Clients only. Maximum amount of idle time in seconds before + force connection closing. + :param task_timeout: For Async Threaded and Batched Clients only. Time limit in seconds for a + Task processing. + :param shutdown_timeout: For Async Threaded and Batched Clients only. Time limit in seconds for + shutting down internal Tasks. + :param trigger_num: For Async Batched Client only. Number of tasks which triggers Task batch + execution. + :param trigger_interval: For Async Batched Client only. Time limit which triggers Task batch + execution. :return: ReportPortal Client instance. """ my_kwargs = kwargs.copy() diff --git a/reportportal_client/_internal/aio/tasks.py b/reportportal_client/_internal/aio/tasks.py index 7b75d26..263a4fd 100644 --- a/reportportal_client/_internal/aio/tasks.py +++ b/reportportal_client/_internal/aio/tasks.py @@ -17,15 +17,25 @@ import sys import time from asyncio import Future -from typing import Any, Awaitable, Coroutine, Generator, Generic, Optional, TypeVar, Union +from typing import Any, Coroutine, Generator, Generic, Optional, TypeVar, Union from reportportal_client.aio.tasks import BlockingOperationError, Task +if sys.version_info >= (3, 10): + from typing import TypeAlias +else: + from typing_extensions import TypeAlias + _T = TypeVar("_T") DEFAULT_TASK_TRIGGER_NUM: int = 10 DEFAULT_TASK_TRIGGER_INTERVAL: float = 1.0 +if sys.version_info >= (3, 12): + _TaskCompatibleCoro: TypeAlias = Coroutine[Any, Any, Any] +else: + _TaskCompatibleCoro: TypeAlias = Union[Generator[Optional[Future[object]], None, Any], Coroutine[Any, Any, Any]] + class BatchedTask(Generic[_T], Task[_T]): """Represents a Task which uses the current Thread to execute itself.""" @@ -34,7 +44,7 @@ class BatchedTask(Generic[_T], Task[_T]): def __init__( self, - coro: Union[Generator[Future, None, _T], Awaitable[_T]], + coro: _TaskCompatibleCoro, *, loop: asyncio.AbstractEventLoop, name: Optional[str] = None, @@ -68,7 +78,7 @@ class ThreadedTask(Generic[_T], Task[_T]): def __init__( self, - coro: Union[Generator[Future, None, _T], Awaitable[_T]], + coro: _TaskCompatibleCoro, wait_timeout: float, *, loop: asyncio.AbstractEventLoop, diff --git a/reportportal_client/_internal/logs/batcher.py b/reportportal_client/_internal/logs/batcher.py index 00fdf9f..7beec88 100644 --- a/reportportal_client/_internal/logs/batcher.py +++ b/reportportal_client/_internal/logs/batcher.py @@ -22,10 +22,10 @@ logger = logging.getLogger(__name__) -T_co = TypeVar("T_co", bound="RPRequestLog", covariant=True) +LogRequestType = TypeVar("LogRequestType", bound=RPRequestLog) -class LogBatcher(Generic[T_co]): +class LogBatcher(Generic[LogRequestType]): """Log packaging class to automate compiling separate Log entries into log batches. The class accepts the maximum number of log entries in desired batches and maximum batch size to conform @@ -35,7 +35,7 @@ class LogBatcher(Generic[T_co]): entry_num: int payload_limit: int _lock: threading.Lock - _batch: list[T_co] + _batch: list[LogRequestType] _payload_size: int def __init__(self, entry_num=MAX_LOG_BATCH_SIZE, payload_limit=MAX_LOG_BATCH_PAYLOAD_SIZE) -> None: @@ -50,7 +50,7 @@ def __init__(self, entry_num=MAX_LOG_BATCH_SIZE, payload_limit=MAX_LOG_BATCH_PAY self._batch = [] self._payload_size = 0 - def _append(self, size: int, log_req: RPRequestLog) -> Optional[list[RPRequestLog]]: + def _append(self, size: int, log_req: LogRequestType) -> Optional[list[LogRequestType]]: with self._lock: if self._payload_size + size >= self.payload_limit: if len(self._batch) > 0: @@ -74,7 +74,7 @@ def append(self, log_req: RPRequestLog) -> Optional[list[RPRequestLog]]: :param log_req: log request object :return: a batch or None """ - return self._append(log_req.multipart_size, log_req) + return self._append(log_req.multipart_size, log_req) # type: ignore async def append_async(self, log_req: AsyncRPRequestLog) -> Optional[list[AsyncRPRequestLog]]: """Add a log request object to internal batch and return the batch if it's full. @@ -82,9 +82,9 @@ async def append_async(self, log_req: AsyncRPRequestLog) -> Optional[list[AsyncR :param log_req: log request object :return: a batch or None """ - return self._append(await log_req.multipart_size, log_req) + return self._append(await log_req.multipart_size, log_req) # type: ignore - def flush(self) -> Optional[list[T_co]]: + def flush(self) -> Optional[list[LogRequestType]]: """Immediately return everything what's left in the internal batch. :return: a batch or None diff --git a/reportportal_client/_internal/services/statistics.py b/reportportal_client/_internal/services/statistics.py index 6b8cd67..240f600 100644 --- a/reportportal_client/_internal/services/statistics.py +++ b/reportportal_client/_internal/services/statistics.py @@ -37,7 +37,7 @@ def _get_client_info() -> tuple[str, str]: :return: ('reportportal-client', '5.0.4') """ name, version = get_package_parameters("reportportal-client", ["name", "version"]) - return name, version + return name or "None", version or "None" def _get_platform_info() -> str: diff --git a/reportportal_client/_internal/static/defines.py b/reportportal_client/_internal/static/defines.py index 20c6f0f..ea574f8 100644 --- a/reportportal_client/_internal/static/defines.py +++ b/reportportal_client/_internal/static/defines.py @@ -26,6 +26,7 @@ 10000: "DEBUG", 5000: "TRACE", } +DEFAULT_LOG_LEVEL = RP_LOG_LEVELS[40000] class _PresenceSentinel: diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 420da55..8061b43 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -13,6 +13,8 @@ """This module contains asynchronous implementations of ReportPortal Client.""" +# mypy: ignore-errors + import asyncio import logging import ssl @@ -52,9 +54,9 @@ # noinspection PyProtectedMember from reportportal_client._internal.static.abstract import AbstractBaseClass, abstractmethod +from reportportal_client._internal.static.defines import DEFAULT_LOG_LEVEL # noinspection PyProtectedMember -from reportportal_client._internal.static.defines import NOT_SET from reportportal_client.aio.tasks import Task from reportportal_client.client import RP, OutputType from reportportal_client.core.rp_issues import Issue @@ -65,17 +67,21 @@ AsyncRPLogBatch, AsyncRPRequestLog, ErrorPrintingAsyncHttpRequest, + ItemUpdateRequest, LaunchFinishRequest, LaunchStartRequest, RPFile, ) from reportportal_client.helpers import ( + ITEM_DESCRIPTION_LENGTH_LIMIT, + ITEM_NAME_LENGTH_LIMIT, + LAUNCH_DESCRIPTION_LENGTH_LIMIT, + LAUNCH_NAME_LENGTH_LIMIT, LifoQueue, agent_name_version, await_if_necessary, root_uri_join, uri_join, - verify_value_length, ) from reportportal_client.logs import MAX_LOG_BATCH_PAYLOAD_SIZE from reportportal_client.steps import StepReporter @@ -121,7 +127,13 @@ class Client: launch_uuid_print: bool print_output: OutputType truncate_attributes: bool - _skip_analytics: str + truncate_fields: bool + replace_binary_chars: bool + launch_name_length_limit: int + item_name_length_limit: int + launch_description_length_limit: int + item_description_length_limit: int + _skip_analytics: Optional[str] _session: Optional[ClientSession] __stat_task: Optional[asyncio.Task] @@ -133,7 +145,7 @@ def __init__( api_key: Optional[str] = None, is_skipped_an_issue: bool = True, verify_ssl: Union[bool, str] = True, - retries: int = NOT_SET, + retries: Optional[int] = -1, max_pool_size: int = 50, http_timeout: Optional[Union[float, tuple[float, float]]] = (10, 10), keepalive_timeout: Optional[float] = None, @@ -141,6 +153,12 @@ def __init__( launch_uuid_print: bool = False, print_output: OutputType = OutputType.STDOUT, truncate_attributes: bool = True, + truncate_fields: bool = True, + replace_binary_chars: bool = True, + launch_name_length_limit: int = LAUNCH_NAME_LENGTH_LIMIT, + item_name_length_limit: int = ITEM_NAME_LENGTH_LIMIT, + launch_description_length_limit: int = LAUNCH_DESCRIPTION_LENGTH_LIMIT, + item_description_length_limit: int = ITEM_DESCRIPTION_LENGTH_LIMIT, # OAuth 2.0 Password Grant parameters oauth_uri: Optional[str] = None, oauth_username: Optional[str] = None, @@ -152,27 +170,35 @@ def __init__( ) -> None: """Initialize the class instance with arguments. - :param endpoint: Endpoint of the ReportPortal service. - :param project: Project name to report to. - :param api_key: Authorization API key. - :param oauth_uri: OAuth 2.0 token endpoint URI (for OAuth authentication). - :param oauth_username: Username for OAuth 2.0 authentication. - :param oauth_password: Password for OAuth 2.0 authentication. - :param oauth_client_id: OAuth 2.0 client ID. - :param oauth_client_secret: OAuth 2.0 client secret (optional). - :param oauth_scope: OAuth 2.0 scope (optional). - :param is_skipped_an_issue: Option to mark skipped tests as not 'To Investigate' items on the - server side. - :param verify_ssl: Option to skip ssl verification. - :param retries: Number of retry attempts to make in case of connection / server errors. - :param max_pool_size: Option to set the maximum number of connections to save the pool. - :param http_timeout: A float in seconds for connect and read timeout. Use a Tuple to - specific connect and read separately. - :param keepalive_timeout: Maximum amount of idle time in seconds before force connection closing. - :param mode: Launch mode, all Launches started by the client will be in that mode. - :param launch_uuid_print: Print Launch UUID into passed TextIO or by default to stdout. - :param print_output: Set output stream for Launch UUID printing. - :param truncate_attributes: Truncate test item attributes to default maximum length. + :param endpoint: Endpoint of the ReportPortal service. + :param project: Project name to report to. + :param api_key: Authorization API key. + :param oauth_uri: OAuth 2.0 token endpoint URI (for OAuth authentication). + :param oauth_username: Username for OAuth 2.0 authentication. + :param oauth_password: Password for OAuth 2.0 authentication. + :param oauth_client_id: OAuth 2.0 client ID. + :param oauth_client_secret: OAuth 2.0 client secret (optional). + :param oauth_scope: OAuth 2.0 scope (optional). + :param is_skipped_an_issue: Option to mark skipped tests as not 'To Investigate' items on the + server side. + :param verify_ssl: Option to skip ssl verification. + :param retries: Number of retry attempts to make in case of connection / server + errors. + :param max_pool_size: Option to set the maximum number of connections to save the pool. + :param http_timeout: A float in seconds for connect and read timeout. Use a Tuple to + specific connect and read separately. + :param keepalive_timeout: Maximum amount of idle time in seconds before force connection + closing. + :param mode: Launch mode, all Launches started by the client will be in that mode. + :param launch_uuid_print: Print Launch UUID into passed TextIO or by default to stdout. + :param print_output: Set output stream for Launch UUID printing. + :param truncate_attributes: Truncate test item attributes to default maximum length. + :param truncate_fields: Truncate request fields to configured limits. + :param replace_binary_chars: Toggle replacement of basic binary characters with \ufffd char. + :param launch_name_length_limit: Maximum allowed launch name length. + :param item_name_length_limit: Maximum allowed test item name length. + :param launch_description_length_limit: Maximum allowed launch description length. + :param item_description_length_limit: Maximum allowed test item description length. """ self.api_v1, self.api_v2 = "v1", "v2" self.endpoint = endpoint @@ -192,6 +218,12 @@ def __init__( self._session = None self.__stat_task = None self.truncate_attributes = truncate_attributes + self.truncate_fields = truncate_fields + self.replace_binary_chars = replace_binary_chars + self.launch_name_length_limit = launch_name_length_limit + self.item_name_length_limit = item_name_length_limit + self.launch_description_length_limit = launch_description_length_limit + self.item_description_length_limit = item_description_length_limit self.api_key = api_key # Handle deprecated token argument @@ -217,16 +249,19 @@ def __init__( if oauth_provided: # Use OAuth 2.0 Password Grant authentication + # These params will be defined, since we checked them with "all" keyword above + # These 'or ""' just to mute dump type checkers self.auth = OAuthPasswordGrantAsync( - oauth_uri=oauth_uri, - username=oauth_username, - password=oauth_password, - client_id=oauth_client_id, + oauth_uri=oauth_uri or "", + username=oauth_username or "", + password=oauth_password or "", + client_id=oauth_client_id or "", client_secret=oauth_client_secret, scope=oauth_scope, ) elif self.api_key: - self.auth = ApiKeyAuthAsync(api_key) + # Use API key authentication + self.auth = ApiKeyAuthAsync(self.api_key) else: # Neither OAuth nor API key provided raise ValueError( @@ -273,20 +308,23 @@ async def session(self) -> ClientSession: connect_timeout, read_timeout = self.http_timeout, self.http_timeout session_params["timeout"] = aiohttp.ClientTimeout(connect=connect_timeout, sock_read=read_timeout) - retries_set = self.retries is not NOT_SET and self.retries and self.retries > 0 - use_retries = self.retries is NOT_SET or (self.retries and self.retries > 0) + retries_set = self.retries is not None and self.retries > 0 + # Use retries with default parameters if not set, but don't use retries if it's `None` + use_retries = self.retries == -1 or retries_set if retries_set: session_params["max_retry_number"] = self.retries + wrapped_session: Union[aiohttp.ClientSession, RetryingClientSession] if use_retries: wrapped_session = RetryingClientSession(self.endpoint, **session_params) else: # noinspection PyTypeChecker wrapped_session = aiohttp.ClientSession(self.endpoint, **session_params) + my_session = ClientSession(wrapped=wrapped_session, auth=self.auth) - self._session = ClientSession(wrapped=wrapped_session, auth=self.auth) - return self._session + self._session = my_session + return my_session async def close(self) -> None: """Gracefully close internal aiohttp.ClientSession class instance and reset it.""" @@ -334,7 +372,10 @@ async def start_launch( request_payload = LaunchStartRequest( name=name, start_time=start_time, - attributes=verify_value_length(attributes) if self.truncate_attributes else attributes, + attributes=attributes, + truncate_attributes_enabled=self.truncate_attributes, + truncate_fields_enabled=self.truncate_fields, + replace_binary_characters=self.replace_binary_chars, description=description, mode=self.mode, rerun=rerun, @@ -403,11 +444,14 @@ async def start_test_item( else: url = root_uri_join(self.base_url_v2, "item") request_payload = AsyncItemStartRequest( - name, - start_time, - item_type, - launch_uuid, - attributes=verify_value_length(attributes) if self.truncate_attributes else attributes, + name=name, + start_time=start_time, + type_=item_type, + launch_uuid=launch_uuid, + attributes=attributes, + truncate_attributes_enabled=self.truncate_attributes, + truncate_fields_enabled=self.truncate_fields, + replace_binary_characters=self.replace_binary_chars, code_ref=code_ref, description=description, has_stats=has_stats, @@ -464,10 +508,13 @@ async def finish_test_item( """ url = self.__get_item_url(item_id) request_payload = AsyncItemFinishRequest( - end_time, - launch_uuid, - status, - attributes=verify_value_length(attributes) if self.truncate_attributes else attributes, + end_time=end_time, + launch_uuid=launch_uuid, + status=status, + attributes=attributes, + truncate_attributes_enabled=self.truncate_attributes, + truncate_fields_enabled=self.truncate_fields, + replace_binary_characters=self.replace_binary_chars, description=description, test_case_id=test_case_id, is_skipped_an_issue=self.is_skipped_an_issue, @@ -505,9 +552,12 @@ async def finish_launch( """ url = self.__get_launch_url(launch_uuid) request_payload = LaunchFinishRequest( - end_time, + end_time=end_time, status=status, - attributes=verify_value_length(attributes) if self.truncate_attributes else attributes, + attributes=attributes, + truncate_attributes_enabled=self.truncate_attributes, + truncate_fields_enabled=self.truncate_fields, + replace_binary_characters=self.replace_binary_chars, description=kwargs.get("description"), ).payload response = await AsyncHttpRequest( @@ -532,10 +582,13 @@ async def update_test_item( :param description: Test Item description. :return: Response message or None. """ - data = { - "description": description, - "attributes": verify_value_length(attributes) if self.truncate_attributes else attributes, - } + data = ItemUpdateRequest( + description=description, + attributes=attributes, + truncate_attributes_enabled=self.truncate_attributes, + truncate_fields_enabled=self.truncate_fields, + replace_binary_characters=self.replace_binary_chars, + ).payload item_id = await self.get_item_id_by_uuid(item_uuid) url = root_uri_join(self.base_url_v1, "item", item_id, "update") response = await AsyncHttpRequest( @@ -640,7 +693,15 @@ async def log_batch(self, log_batch: Optional[list[AsyncRPRequestLog]]) -> Optio url = root_uri_join(self.base_url_v2, "log") response = await ErrorPrintingAsyncHttpRequest( - (await self.session()).post, url=url, data=AsyncRPLogBatch(log_batch).payload, name="log" + (await self.session()).post, + url=url, + data=AsyncRPLogBatch( + truncate_attributes_enabled=None, + truncate_fields_enabled=None, + replace_binary_characters=None, + log_reqs=log_batch, + ).payload, + name="log", ).make() return await response.messages if response else None @@ -663,6 +724,13 @@ def clone(self) -> "Client": mode=self.mode, launch_uuid_print=self.launch_uuid_print, print_output=self.print_output, + truncate_fields=self.truncate_fields, + truncate_attributes=self.truncate_attributes, + replace_binary_chars=self.replace_binary_chars, + launch_name_length_limit=self.launch_name_length_limit, + item_name_length_limit=self.item_name_length_limit, + launch_description_length_limit=self.launch_description_length_limit, + item_description_length_limit=self.item_description_length_limit, oauth_uri=self.oauth_uri, oauth_username=self.oauth_username, oauth_password=self.oauth_password, @@ -1060,8 +1128,19 @@ async def log( :param item_id: UUID of the ReportPortal Item the message belongs to. :return: Response message Tuple if Log message batch was sent or None. """ + rp_level = str(level) if level else DEFAULT_LOG_LEVEL rp_file = RPFile(**attachment) if attachment else None - rp_log = AsyncRPRequestLog(self.__launch_uuid, time, rp_file, item_id, level, message) + rp_log = AsyncRPRequestLog( + truncate_attributes_enabled=None, + truncate_fields_enabled=None, + replace_binary_characters=None, + launch_uuid=self.__launch_uuid, + time=time, + file=rp_file, + item_uuid=item_id, + level=rp_level, + message=message, + ) return await self.__client.log_batch(await self._log_batcher.append_async(rp_log)) def clone(self) -> "AsyncRPClient": @@ -1511,8 +1590,19 @@ def log( :param item_id: UUID of the ReportPortal Item the message belongs to. :return: Response message Tuple if Log message batch was sent or None. """ + rp_level = str(level) if level else DEFAULT_LOG_LEVEL rp_file = RPFile(**attachment) if attachment else None - rp_log = AsyncRPRequestLog(self.launch_uuid, time, rp_file, item_id, level, message) + rp_log = AsyncRPRequestLog( + truncate_attributes_enabled=None, + truncate_fields_enabled=None, + replace_binary_characters=None, + launch_uuid=self.launch_uuid, + time=time, + file=rp_file, + item_uuid=item_id, + level=rp_level, + message=message, + ) return self.create_task(self._log(rp_log)) def close(self) -> None: diff --git a/reportportal_client/aio/tasks.py b/reportportal_client/aio/tasks.py index e6dccba..4989537 100644 --- a/reportportal_client/aio/tasks.py +++ b/reportportal_client/aio/tasks.py @@ -14,9 +14,15 @@ """This module contains customized asynchronous Tasks and Task Factories for the ReportPortal client.""" import asyncio +import sys from abc import abstractmethod from asyncio import Future -from typing import Awaitable, Generator, Generic, Optional, TypeVar, Union +from typing import Any, Coroutine, Generator, Generic, Optional, TypeVar, Union + +if sys.version_info >= (3, 10): + from typing import TypeAlias +else: + from typing_extensions import TypeAlias # noinspection PyProtectedMember from reportportal_client._internal.static.abstract import AbstractBaseClass @@ -28,6 +34,12 @@ class BlockingOperationError(RuntimeError): """An issue with task blocking execution.""" +if sys.version_info >= (3, 12): + _TaskCompatibleCoro: TypeAlias = Coroutine[Any, Any, Any] +else: + _TaskCompatibleCoro: TypeAlias = Union[Generator[Optional[Future[object]], None, Any], Coroutine[Any, Any, Any]] + + class Task(Generic[_T], asyncio.Task, metaclass=AbstractBaseClass): """Base class for ReportPortal client tasks. @@ -41,7 +53,7 @@ class Task(Generic[_T], asyncio.Task, metaclass=AbstractBaseClass): def __init__( self, - coro: Union[Generator[Future, None, _T], Awaitable[_T]], + coro: _TaskCompatibleCoro, *, loop: asyncio.AbstractEventLoop, name: Optional[str] = None, diff --git a/reportportal_client/client.py b/reportportal_client/client.py index 8078010..5200fc6 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -48,13 +48,20 @@ HttpRequest, ItemFinishRequest, ItemStartRequest, + ItemUpdateRequest, LaunchFinishRequest, LaunchStartRequest, RPFile, RPLogBatch, RPRequestLog, ) -from reportportal_client.helpers import LifoQueue, agent_name_version, uri_join, verify_value_length +from reportportal_client.helpers import LifoQueue, agent_name_version, uri_join +from reportportal_client.helpers.common_helpers import ( + ITEM_DESCRIPTION_LENGTH_LIMIT, + ITEM_NAME_LENGTH_LIMIT, + LAUNCH_DESCRIPTION_LENGTH_LIMIT, + LAUNCH_NAME_LENGTH_LIMIT, +) from reportportal_client.logs import MAX_LOG_BATCH_PAYLOAD_SIZE from reportportal_client.steps import StepReporter @@ -315,7 +322,7 @@ def log( message: str, level: Optional[Union[int, str]] = None, attachment: Optional[dict] = None, - item_id: Optional[str] = None, + item_id: Optional[Any] = None, ) -> Optional[tuple[str, ...]]: """Send Log message to the ReportPortal and attach it to a Test Item or Launch. @@ -401,7 +408,7 @@ class RPClient(RP): oauth_scope: Optional[str] auth: Auth verify_ssl: Union[bool, str] - retries: int + retries: Optional[int] max_pool_size: int http_timeout: Union[float, tuple[float, float]] session: ClientSession @@ -410,7 +417,13 @@ class RPClient(RP): launch_uuid_print: Optional[bool] print_output: OutputType truncate_attributes: bool - _skip_analytics: str + truncate_fields: bool + replace_binary_chars: bool + launch_name_length_limit: int + item_name_length_limit: int + launch_description_length_limit: int + item_description_length_limit: int + _skip_analytics: Optional[str] _item_stack: LifoQueue _log_batcher: LogBatcher[RPRequestLog] @@ -466,7 +479,7 @@ def __init__( log_batch_size: int = 20, is_skipped_an_issue: bool = True, verify_ssl: Union[bool, str] = True, - retries: int = None, + retries: Optional[int] = None, max_pool_size: int = 50, launch_uuid: Optional[str] = None, http_timeout: Union[float, tuple[float, float]] = (10, 10), @@ -476,6 +489,12 @@ def __init__( print_output: OutputType = OutputType.STDOUT, log_batcher: Optional[LogBatcher[RPRequestLog]] = None, truncate_attributes: bool = True, + truncate_fields: bool = True, + replace_binary_chars: bool = True, + launch_name_length_limit: int = LAUNCH_NAME_LENGTH_LIMIT, + item_name_length_limit: int = ITEM_NAME_LENGTH_LIMIT, + launch_description_length_limit: int = LAUNCH_DESCRIPTION_LENGTH_LIMIT, + item_description_length_limit: int = ITEM_DESCRIPTION_LENGTH_LIMIT, # OAuth 2.0 Password Grant parameters oauth_uri: Optional[str] = None, oauth_username: Optional[str] = None, @@ -487,31 +506,38 @@ def __init__( ) -> None: """Initialize the class instance with arguments. - :param endpoint: Endpoint of the ReportPortal service. - :param project: Project name to report to. - :param api_key: Authorization API key. - :param oauth_uri: OAuth 2.0 token endpoint URI (for OAuth authentication). - :param oauth_username: Username for OAuth 2.0 authentication. - :param oauth_password: Password for OAuth 2.0 authentication. - :param oauth_client_id: OAuth 2.0 client ID. - :param oauth_client_secret: OAuth 2.0 client secret (optional). - :param oauth_scope: OAuth 2.0 scope (optional). - :param log_batch_size: Option to set the maximum number of logs that can be processed in one - batch. - :param is_skipped_an_issue: Option to mark skipped tests as not 'To Investigate' items on the - server side. - :param verify_ssl: Option to skip ssl verification. - :param retries: Number of retry attempts to make in case of connection / server errors. - :param max_pool_size: Option to set the maximum number of connections to save the pool. - :param launch_uuid: A launch UUID to use instead of starting own one. - :param http_timeout: A float in seconds for connect and read timeout. Use a Tuple to - specific connect and read separately. - :param log_batch_payload_limit: Maximum size in bytes of logs that can be processed in one batch. - :param mode: Launch mode, all Launches started by the client will be in that mode. - :param launch_uuid_print: Print Launch UUID into passed TextIO or by default to stdout. - :param print_output: Set output stream for Launch UUID printing. - :param log_batcher: Use existing LogBatcher instance instead of creation of own one. - :param truncate_attributes: Truncate test item attributes to default maximum length. + :param endpoint: Endpoint of the ReportPortal service. + :param project: Project name to report to. + :param api_key: Authorization API key. + :param oauth_uri: OAuth 2.0 token endpoint URI (for OAuth authentication). + :param oauth_username: Username for OAuth 2.0 authentication. + :param oauth_password: Password for OAuth 2.0 authentication. + :param oauth_client_id: OAuth 2.0 client ID. + :param oauth_client_secret: OAuth 2.0 client secret (optional). + :param oauth_scope: OAuth 2.0 scope (optional). + :param log_batch_size: Option to set the maximum number of logs that can be processed in one + batch. + :param is_skipped_an_issue: Option to mark skipped tests as not 'To Investigate' items on the + server side. + :param verify_ssl: Option to skip ssl verification. + :param retries: Number of retry attempts to make in case of connection / server + errors. + :param max_pool_size: Option to set the maximum number of connections to save the pool. + :param launch_uuid: A launch UUID to use instead of starting own one. + :param http_timeout: A float in seconds for connect and read timeout. Use a Tuple to + specific connect and read separately. + :param log_batch_payload_limit: Maximum size in bytes of logs that can be processed in one batch. + :param mode: Launch mode, all Launches started by the client will be in that mode. + :param launch_uuid_print: Print Launch UUID into passed TextIO or by default to stdout. + :param print_output: Set output stream for Launch UUID printing. + :param log_batcher: Use existing LogBatcher instance instead of creation of own one. + :param truncate_attributes: Truncate test item attributes to default maximum length. + :param truncate_fields: Truncate request fields to configured limits. + :param replace_binary_chars: Toggle replacement of basic binary characters with \ufffd char. + :param launch_name_length_limit: Maximum allowed launch name length. + :param item_name_length_limit: Maximum allowed test item name length. + :param launch_description_length_limit: Maximum allowed launch description length. + :param item_description_length_limit: Maximum allowed test item description length. """ set_current(self) self.api_v1, self.api_v2 = "v1", "v2" @@ -546,6 +572,12 @@ def __init__( self.launch_uuid_print = launch_uuid_print self.print_output = print_output self.truncate_attributes = truncate_attributes + self.truncate_fields = truncate_fields + self.replace_binary_chars = replace_binary_chars + self.launch_name_length_limit = launch_name_length_limit + self.item_name_length_limit = item_name_length_limit + self.launch_description_length_limit = launch_description_length_limit + self.item_description_length_limit = item_description_length_limit self.api_key = api_key # Handle deprecated token argument @@ -571,16 +603,19 @@ def __init__( if oauth_provided: # Use OAuth 2.0 Password Grant authentication + # These params will be defined, since we checked them with "all" keyword above + # These 'or ""' just to mute dump type checkers self.auth = OAuthPasswordGrantSync( - oauth_uri=oauth_uri, - username=oauth_username, - password=oauth_password, - client_id=oauth_client_id, + oauth_uri=oauth_uri or "", + username=oauth_username or "", + password=oauth_password or "", + client_id=oauth_client_id or "", client_secret=oauth_client_secret, scope=oauth_scope, ) elif self.api_key: - self.auth = ApiKeyAuthSync(api_key) + # Use API key authentication + self.auth = ApiKeyAuthSync(self.api_key) else: # Neither OAuth nor API key provided raise ValueError( @@ -626,7 +661,10 @@ def start_launch( request_payload = LaunchStartRequest( name=name, start_time=start_time, - attributes=verify_value_length(attributes) if self.truncate_attributes else attributes, + attributes=attributes, + truncate_attributes_enabled=self.truncate_attributes, + truncate_fields_enabled=self.truncate_fields, + replace_binary_characters=self.replace_binary_chars, description=description, mode=self.mode, rerun=rerun, @@ -661,9 +699,9 @@ def start_test_item( attributes: Optional[Union[list[dict], dict]] = None, parameters: Optional[dict] = None, parent_item_id: Optional[str] = None, - has_stats: bool = True, + has_stats: Optional[bool] = True, code_ref: Optional[str] = None, - retry: bool = False, + retry: Optional[bool] = False, test_case_id: Optional[str] = None, retry_of: Optional[str] = None, uuid: Optional[str] = None, @@ -695,11 +733,14 @@ def start_test_item( else: url = uri_join(self.base_url_v2, "item") request_payload = ItemStartRequest( - name, - start_time, - item_type, - self.__launch_uuid, - attributes=verify_value_length(attributes) if self.truncate_attributes else attributes, + name=name, + start_time=start_time, + type_=item_type, + launch_uuid=self.__launch_uuid, + attributes=attributes, + truncate_attributes_enabled=self.truncate_attributes, + truncate_fields_enabled=self.truncate_fields, + replace_binary_characters=self.replace_binary_chars, code_ref=code_ref, description=description, has_stats=has_stats, @@ -730,7 +771,7 @@ def start_test_item( def finish_test_item( self, - item_id: str, + item_id: Any, end_time: str, status: Optional[str] = None, issue: Optional[Issue] = None, @@ -762,10 +803,13 @@ def finish_test_item( return None url = uri_join(self.base_url_v2, "item", item_id) request_payload = ItemFinishRequest( - end_time, - self.__launch_uuid, - status, - attributes=verify_value_length(attributes) if self.truncate_attributes else attributes, + end_time=end_time, + launch_uuid=self.__launch_uuid, + status=status, + attributes=attributes, + truncate_attributes_enabled=self.truncate_attributes, + truncate_fields_enabled=self.truncate_fields, + replace_binary_characters=self.replace_binary_chars, description=description, is_skipped_an_issue=self.is_skipped_an_issue, issue=issue, @@ -808,9 +852,12 @@ def finish_launch( return None url = uri_join(self.base_url_v2, "launch", self.__launch_uuid, "finish") request_payload = LaunchFinishRequest( - end_time, + end_time=end_time, status=status, - attributes=verify_value_length(attributes) if self.truncate_attributes else attributes, + attributes=attributes, + truncate_attributes_enabled=self.truncate_attributes, + truncate_fields_enabled=self.truncate_fields, + replace_binary_characters=self.replace_binary_chars, description=kwargs.get("description"), ).payload response = HttpRequest( @@ -828,7 +875,10 @@ def finish_launch( return None def update_test_item( - self, item_uuid: str, attributes: Optional[Union[list, dict]] = None, description: Optional[str] = None + self, + item_uuid: Optional[str], + attributes: Optional[Union[list, dict]] = None, + description: Optional[str] = None, ) -> Optional[str]: """Update existing Test Item at the ReportPortal. @@ -837,11 +887,19 @@ def update_test_item( :param description: Test Item description. :return: Response message or None. """ - data = { - "description": description, - "attributes": verify_value_length(attributes) if self.truncate_attributes else attributes, - } + if not item_uuid: + logger.warning("Attempt to update non-existent item") + return None + data = ItemUpdateRequest( + description=description, + attributes=attributes, + truncate_attributes_enabled=self.truncate_attributes, + truncate_fields_enabled=self.truncate_fields, + replace_binary_characters=self.replace_binary_chars, + ).payload item_id = self.get_item_id_by_uuid(item_uuid) + if not item_id: + return None url = uri_join(self.base_url_v1, "item", item_id, "update") response = HttpRequest( self.session.put, @@ -864,7 +922,12 @@ def _log(self, batch: Optional[list[RPRequestLog]]) -> Optional[tuple[str, ...]] response = ErrorPrintingHttpRequest( self.session.post, url, - files=RPLogBatch(batch).payload, + files=RPLogBatch( + truncate_attributes_enabled=None, + truncate_fields_enabled=None, + replace_binary_characters=None, + log_reqs=batch, + ).payload, verify_ssl=self.verify_ssl, http_timeout=self.http_timeout, name="log", @@ -877,7 +940,7 @@ def log( message: str, level: Optional[Union[int, str]] = None, attachment: Optional[dict] = None, - item_id: Optional[str] = None, + item_id: Optional[Any] = None, ) -> Optional[tuple[str, ...]]: """Send Log message to the ReportPortal and attach it to a Test Item or Launch. @@ -892,7 +955,17 @@ def log( :return: Response message Tuple if Log message batch was sent or None. """ rp_file = RPFile(**attachment) if attachment else None - rp_log = RPRequestLog(self.__launch_uuid, time, rp_file, item_id, level, message) + rp_log = RPRequestLog( + truncate_attributes_enabled=None, + truncate_fields_enabled=None, + replace_binary_characters=None, + launch_uuid=self.__launch_uuid, + time=time, + file=rp_file, + item_uuid=item_id, + level=str(level), + message=message, + ) return self._log(self._log_batcher.append(rp_log)) def get_item_id_by_uuid(self, item_uuid: str) -> Optional[str]: @@ -914,7 +987,10 @@ def get_launch_info(self) -> Optional[dict]: """ if self.launch_uuid is None: return {} - url = uri_join(self.base_url_v1, "launch", "uuid", self.__launch_uuid) + launch_uuid = self.__launch_uuid + if launch_uuid is None: + return {} + url = uri_join(self.base_url_v1, "launch", "uuid", launch_uuid) logger.debug("get_launch_info - ID: %s", self.__launch_uuid) response = HttpRequest( self.session.get, @@ -1018,6 +1094,13 @@ def clone(self) -> "RPClient": http_timeout=self.http_timeout, log_batch_payload_limit=self.log_batch_payload_limit, mode=self.mode, + truncate_fields=self.truncate_fields, + truncate_attributes=self.truncate_attributes, + replace_binary_chars=self.replace_binary_chars, + launch_name_length_limit=self.launch_name_length_limit, + item_name_length_limit=self.item_name_length_limit, + launch_description_length_limit=self.launch_description_length_limit, + item_description_length_limit=self.item_description_length_limit, log_batcher=self._log_batcher, oauth_uri=self.oauth_uri, oauth_username=self.oauth_username, diff --git a/reportportal_client/core/rp_requests.py b/reportportal_client/core/rp_requests.py index ec68681..8b5ada0 100644 --- a/reportportal_client/core/rp_requests.py +++ b/reportportal_client/core/rp_requests.py @@ -18,13 +18,15 @@ https://github.com/reportportal/documentation/blob/master/src/md/src/DevGuides/reporting.md """ +# mypy: disable_error_code=override + import asyncio import logging import sys import traceback from dataclasses import dataclass from datetime import datetime -from typing import Any, Callable, Optional, TypeVar, Union +from typing import Any, Awaitable, Callable, Optional, TypeVar, Union, cast import aiohttp @@ -34,17 +36,18 @@ from reportportal_client._internal.static.abstract import AbstractBaseClass, abstractmethod # noinspection PyProtectedMember -from reportportal_client._internal.static.defines import DEFAULT_PRIORITY, LOW_PRIORITY, RP_LOG_LEVELS, Priority +from reportportal_client._internal.static.defines import DEFAULT_LOG_LEVEL, DEFAULT_PRIORITY, LOW_PRIORITY, Priority from reportportal_client.core.rp_file import RPFile from reportportal_client.core.rp_issues import Issue from reportportal_client.core.rp_responses import AsyncRPResponse, RPResponse from reportportal_client.helpers import await_if_necessary, dict_to_payload +from reportportal_client.helpers.common_helpers import clean_binary_characters, verify_value_length try: # noinspection PyPackageRequirements - import simplejson as json_converter + import simplejson as _json_converter except ImportError: - import json as json_converter + import json as _json_converter # type: ignore[no-redef] logger = logging.getLogger(__name__) T = TypeVar("T") @@ -95,7 +98,7 @@ def __init__( self.verify_ssl = verify_ssl self.http_timeout = http_timeout self.name = name - self._priority = DEFAULT_PRIORITY + self._priority = cast(Priority, DEFAULT_PRIORITY) def __lt__(self, other: "HttpRequest") -> bool: """Priority protocol for the PriorityQueue. @@ -121,7 +124,7 @@ def priority(self, value: Priority) -> None: """ self._priority = value - def make(self) -> Optional[RPResponse]: + def make(self) -> Any: """Make HTTP request to the ReportPortal API. The method catches any request error to not fail reporting. Since we are reporting tool and should not fail @@ -142,6 +145,7 @@ def make(self) -> Optional[RPResponse]: ) except (KeyError, IOError, ValueError, TypeError) as exc: logger.warning("ReportPortal %s request failed", self.name, exc_info=exc) + return None class ErrorPrintingHttpRequest(HttpRequest): @@ -179,6 +183,7 @@ def make(self) -> Optional[RPResponse]: f"{datetime.now().isoformat()} - [ERROR] - ReportPortal request error:\n{traceback.format_exc()}", file=sys.stderr, ) + return None class AsyncHttpRequest(HttpRequest): @@ -220,6 +225,7 @@ async def make(self) -> Optional[AsyncRPResponse]: return AsyncRPResponse(await self.session_method(url, data=data, json=json)) except (KeyError, IOError, ValueError, TypeError) as exc: logger.warning("ReportPortal %s request failed", self.name, exc_info=exc) + return None class ErrorPrintingAsyncHttpRequest(AsyncHttpRequest): @@ -253,8 +259,10 @@ async def make(self) -> Optional[AsyncRPResponse]: f"{datetime.now().isoformat()} - [ERROR] - ReportPortal request error:\n{traceback.format_exc()}", file=sys.stderr, ) + return None +@dataclass(frozen=True) class RPRequestBase(metaclass=AbstractBaseClass): """Base class for specific ReportPortal request models. @@ -262,10 +270,81 @@ class RPRequestBase(metaclass=AbstractBaseClass): """ __metaclass__ = AbstractBaseClass + truncate_attributes_enabled: Optional[bool] + truncate_fields_enabled: Optional[bool] + replace_binary_characters: Optional[bool] + + @property + def _truncate_attributes_enabled(self) -> bool: + return self.truncate_attributes_enabled is not False + + @property + def _truncate_fields_enabled(self) -> bool: + return self.truncate_fields_enabled is not False + + @property + def _replace_binary_characters_enabled(self) -> bool: + return self.replace_binary_characters is not False + + def _sanitize_field(self, value: Optional[str], limit: int) -> Optional[str]: + if not value: + return value + + sanitized_value = value + if self._replace_binary_characters_enabled: + sanitized_value = clean_binary_characters(sanitized_value) + if not sanitized_value: + return value + + if not self._truncate_fields_enabled: + return sanitized_value + + effective_limit = max(0, limit) + if len(sanitized_value) <= effective_limit: + return sanitized_value + if effective_limit == 0: + return "" + if effective_limit <= len(helpers.TRUNCATE_REPLACEMENT): + return sanitized_value[:effective_limit] + return sanitized_value[: effective_limit - len(helpers.TRUNCATE_REPLACEMENT)] + helpers.TRUNCATE_REPLACEMENT + + def _truncate_attributes(self, attributes: Optional[Union[list, dict]]) -> Optional[list[dict[str, Any]]]: + if attributes is None: + return None + + my_attributes = attributes + if isinstance(my_attributes, dict): + converted_attributes = dict_to_payload(my_attributes) + if not converted_attributes: + return None + my_attributes = converted_attributes + + normalized_attributes = [dict(attribute) for attribute in my_attributes if isinstance(attribute, dict)] + if len(normalized_attributes) == 0: + return [] + + if len(normalized_attributes) > helpers.ATTRIBUTE_NUMBER_LIMIT: + normalized_attributes = sorted(normalized_attributes, key=lambda attr: str(attr.get("key", "")))[ + : helpers.ATTRIBUTE_NUMBER_LIMIT + ] + + if self._replace_binary_characters_enabled: + for attribute in normalized_attributes: + key = attribute.get("key") + value = attribute.get("value") + if key is not None: + attribute["key"] = clean_binary_characters(str(key)) + if value is not None: + attribute["value"] = clean_binary_characters(str(value)) + + if not self._truncate_attributes_enabled: + return normalized_attributes + + return verify_value_length(normalized_attributes) @property @abstractmethod - def payload(self) -> dict: + def payload(self) -> Any: """Abstract interface for getting HTTP request payload. :return: JSON representation in the form of a Dictionary @@ -286,8 +365,8 @@ class LaunchStartRequest(RPRequestBase): description: Optional[str] = None mode: str = "default" rerun: bool = False - rerun_of: str = None - uuid: str = None + rerun_of: Optional[str] = None + uuid: Optional[str] = None @property def payload(self) -> dict: @@ -295,14 +374,13 @@ def payload(self) -> dict: :return: JSON representation in the form of a Dictionary """ - my_attributes = self.attributes - if my_attributes and isinstance(self.attributes, dict): - my_attributes = dict_to_payload(self.attributes) + my_name = self._sanitize_field(self.name, helpers.LAUNCH_NAME_LENGTH_LIMIT) + my_attributes = self._truncate_attributes(self.attributes) result = { "attributes": my_attributes, - "description": self.description, + "description": self._sanitize_field(self.description, helpers.LAUNCH_DESCRIPTION_LENGTH_LIMIT), "mode": self.mode, - "name": self.name, + "name": my_name, "rerun": self.rerun, "rerunOf": self.rerun_of, "startTime": self.start_time, @@ -330,12 +408,10 @@ def payload(self) -> dict: :return: JSON representation in the form of a Dictionary """ - my_attributes = self.attributes - if my_attributes and isinstance(self.attributes, dict): - my_attributes = dict_to_payload(self.attributes) + my_attributes = self._truncate_attributes(self.attributes) return { "attributes": my_attributes, - "description": self.description, + "description": self._sanitize_field(self.description, helpers.LAUNCH_DESCRIPTION_LENGTH_LIMIT), "endTime": self.end_time, "status": self.status, } @@ -362,13 +438,13 @@ class ItemStartRequest(RPRequestBase): test_case_id: Optional[str] uuid: Optional[str] - @staticmethod - def _create_request(**kwargs) -> dict: + def _create_request(self, **kwargs) -> dict: + name = self._sanitize_field(kwargs.get("name"), helpers.ITEM_NAME_LENGTH_LIMIT) request = { "codeRef": kwargs.get("code_ref"), - "description": kwargs.get("description"), + "description": self._sanitize_field(kwargs.get("description"), helpers.ITEM_DESCRIPTION_LENGTH_LIMIT), "hasStats": kwargs.get("has_stats"), - "name": kwargs["name"], + "name": name, "retry": kwargs.get("retry"), "retryOf": kwargs.get("retry_of"), "startTime": kwargs["start_time"], @@ -377,9 +453,7 @@ def _create_request(**kwargs) -> dict: "launchUuid": kwargs["launch_uuid"], } attributes = kwargs.get("attributes") - if attributes and isinstance(attributes, dict): - attributes = dict_to_payload(kwargs["attributes"]) - request["attributes"] = attributes + request["attributes"] = self._truncate_attributes(attributes) parameters = kwargs.get("parameters") if parameters is not None and isinstance(parameters, dict): parameters = dict_to_payload(kwargs["parameters"]) @@ -397,7 +471,7 @@ def payload(self) -> dict: """ data = self.__dict__.copy() data["type"] = data.pop("type_") - return ItemStartRequest._create_request(**data) + return self._create_request(**data) class AsyncItemStartRequest(ItemStartRequest): @@ -419,7 +493,7 @@ async def payload(self) -> dict: data = self.__dict__.copy() data["type"] = data.pop("type_") data["launch_uuid"] = await await_if_necessary(data.pop("launch_uuid")) - return ItemStartRequest._create_request(**data) + return self._create_request(**data) @dataclass(frozen=True) @@ -440,10 +514,9 @@ class ItemFinishRequest(RPRequestBase): retry_of: Optional[str] test_case_id: Optional[str] - @staticmethod - def _create_request(**kwargs) -> dict: + def _create_request(self, **kwargs) -> dict: request = { - "description": kwargs.get("description"), + "description": self._sanitize_field(kwargs.get("description"), helpers.ITEM_DESCRIPTION_LENGTH_LIMIT), "endTime": kwargs["end_time"], "launchUuid": kwargs["launch_uuid"], "status": kwargs.get("status"), @@ -452,19 +525,19 @@ def _create_request(**kwargs) -> dict: "testCaseId": kwargs.get("test_case_id"), } attributes = kwargs.get("attributes") - if attributes and isinstance(attributes, dict): - attributes = dict_to_payload(kwargs["attributes"]) - request["attributes"] = attributes + request["attributes"] = self._truncate_attributes(attributes) - issue_payload = None + issue_payload: Any = None + status = kwargs.get("status") + issue = kwargs.get("issue") if ( - kwargs.get("issue") is None - and (kwargs.get("status") is not None and kwargs.get("status").lower() == "skipped") + issue is None + and (status is not None and str(status).lower() == "skipped") and not kwargs.get("is_skipped_an_issue") ): issue_payload = {"issue_type": "NOT_ISSUE"} - elif kwargs.get("issue") is not None: - issue_payload = kwargs.get("issue").payload + elif issue is not None: + issue_payload = cast(Issue, issue).payload request["issue"] = issue_payload return request @@ -474,7 +547,7 @@ def payload(self) -> dict: :return: JSON representation in the form of a Dictionary """ - return ItemFinishRequest._create_request(**self.__dict__) + return self._create_request(**self.__dict__) class AsyncItemFinishRequest(ItemFinishRequest): @@ -495,7 +568,26 @@ async def payload(self) -> dict: """ data = self.__dict__.copy() data["launch_uuid"] = await await_if_necessary(data.pop("launch_uuid")) - return ItemFinishRequest._create_request(**data) + return self._create_request(**data) + + +@dataclass(frozen=True) +class ItemUpdateRequest(RPRequestBase): + """ReportPortal update test item request model.""" + + attributes: Optional[Union[list, dict]] = None + description: Optional[str] = None + + @property + def payload(self) -> dict: + """Get HTTP payload for the request. + + :return: JSON representation in the form of a Dictionary + """ + return { + "description": self._sanitize_field(self.description, helpers.ITEM_DESCRIPTION_LENGTH_LIMIT), + "attributes": self._truncate_attributes(self.attributes), + } @dataclass(frozen=True) @@ -509,7 +601,7 @@ class RPRequestLog(RPRequestBase): time: str file: Optional[RPFile] = None item_uuid: Optional[Any] = None - level: str = RP_LOG_LEVELS[40000] + level: str = DEFAULT_LOG_LEVEL message: Optional[str] = None @staticmethod @@ -541,7 +633,7 @@ def _multipart_size(payload: dict, file: Optional[RPFile]): return size @property - def multipart_size(self) -> int: + def multipart_size(self) -> Any: """Calculate request size how it would be transfer in Multipart HTTP. :return: estimate request size @@ -582,25 +674,16 @@ async def multipart_size(self) -> int: return RPRequestLog._multipart_size(await self.payload, self.file) +@dataclass(frozen=True) class RPLogBatch(RPRequestBase): """ReportPortal log save batches with attachments request model. https://github.com/reportportal/documentation/blob/master/src/md/src/DevGuides/reporting.md#batch-save-logs """ - default_content: str log_reqs: list[Union[RPRequestLog, AsyncRPRequestLog]] - priority: Priority - - def __init__(self, log_reqs: list[Union[RPRequestLog, AsyncRPRequestLog]]) -> None: - """Initialize instance attributes. - - :param log_reqs: - """ - super().__init__() - self.default_content = "application/octet-stream" - self.log_reqs = log_reqs - self.priority = LOW_PRIORITY + default_content: str = "application/octet-stream" + priority: Priority = cast(Priority, LOW_PRIORITY) def __get_file(self, rp_file) -> tuple[str, tuple]: """Form a tuple for the single file.""" @@ -618,7 +701,7 @@ def __get_request_part(self) -> list[tuple[str, tuple]]: body = [ ( "json_request_part", - (None, json_converter.dumps([log.payload for log in self.log_reqs]), "application/json"), + (None, _json_converter.dumps([log.payload for log in self.log_reqs]), "application/json"), ) ] return body @@ -658,7 +741,7 @@ def __int__(self, *args, **kwargs) -> None: super.__init__(*args, **kwargs) async def __get_request_part(self) -> list[dict]: - coroutines = [log.payload for log in self.log_reqs] + coroutines = [cast(Awaitable[dict], log.payload) for log in self.log_reqs] return list(await asyncio.gather(*coroutines)) @property diff --git a/reportportal_client/core/worker.py b/reportportal_client/core/worker.py index a9c0a48..d22fbe4 100644 --- a/reportportal_client/core/worker.py +++ b/reportportal_client/core/worker.py @@ -51,8 +51,8 @@ def is_stop_cmd(self) -> bool: def priority(self) -> Priority: """Get the priority of the command.""" if self is ControlCommand.STOP_IMMEDIATE: - return Priority.PRIORITY_IMMEDIATE - return Priority.PRIORITY_LOW + return Priority(Priority.PRIORITY_IMMEDIATE) + return Priority(Priority.PRIORITY_LOW) def __lt__(self, other: Union["ControlCommand", "HttpRequest"]) -> bool: """Priority protocol for the PriorityQueue.""" @@ -88,7 +88,7 @@ def _command_get(self) -> Optional[ControlCommand]: except queue.Empty: return None - def _command_process(self, cmd: Optional[ControlCommand]) -> None: + def _command_process(self, cmd: ControlCommand) -> None: """Process control command sent to the worker. :param cmd: a command to be processed @@ -157,7 +157,7 @@ def _stop_immediately(self) -> None: may be some records still left on the queue, which won't be processed. """ self._stop_lock.acquire() - if self._thread.is_alive() and self._thread is not current_thread(): + if self._thread is not None and self._thread.is_alive() and self._thread is not current_thread(): self._thread.join(timeout=THREAD_TIMEOUT) self._thread = None self._stop_lock.notify_all() @@ -168,7 +168,7 @@ def is_alive(self) -> bool: :return: True is self._thread is not None, False otherwise """ - return bool(self._thread) and self._thread.is_alive() + return self._thread is not None and self._thread.is_alive() def send(self, entity: Union[ControlCommand, HttpRequest]) -> None: """Send control command or a request to the worker queue.""" diff --git a/reportportal_client/helpers/__init__.py b/reportportal_client/helpers/__init__.py index 287ae7a..92f9a0e 100644 --- a/reportportal_client/helpers/__init__.py +++ b/reportportal_client/helpers/__init__.py @@ -3,7 +3,12 @@ from reportportal_client.helpers import markdown_helpers from reportportal_client.helpers.common_helpers import ( ATTRIBUTE_LENGTH_LIMIT, + ATTRIBUTE_NUMBER_LIMIT, CONTENT_TYPE_TO_EXTENSIONS, + ITEM_DESCRIPTION_LENGTH_LIMIT, + ITEM_NAME_LENGTH_LIMIT, + LAUNCH_DESCRIPTION_LENGTH_LIMIT, + LAUNCH_NAME_LENGTH_LIMIT, TRUNCATE_REPLACEMENT, TYPICAL_FILE_PART_HEADER, TYPICAL_MULTIPART_FOOTER_LENGTH, @@ -36,6 +41,11 @@ __all__ = [ "markdown_helpers", "ATTRIBUTE_LENGTH_LIMIT", + "ATTRIBUTE_NUMBER_LIMIT", + "LAUNCH_DESCRIPTION_LENGTH_LIMIT", + "ITEM_DESCRIPTION_LENGTH_LIMIT", + "LAUNCH_NAME_LENGTH_LIMIT", + "ITEM_NAME_LENGTH_LIMIT", "TRUNCATE_REPLACEMENT", "CONTENT_TYPE_TO_EXTENSIONS", "TYPICAL_MULTIPART_FOOTER_LENGTH", diff --git a/reportportal_client/helpers/common_helpers.py b/reportportal_client/helpers/common_helpers.py index 330c822..9457b3e 100644 --- a/reportportal_client/helpers/common_helpers.py +++ b/reportportal_client/helpers/common_helpers.py @@ -24,7 +24,7 @@ import uuid from platform import machine, processor, system from types import MappingProxyType -from typing import Any, Callable, Generic, Iterable, Optional, TypeVar, Union +from typing import Any, Callable, Generic, Iterable, Optional, Sized, TypeVar, Union from reportportal_client.core.rp_file import RPFile @@ -32,12 +32,17 @@ # noinspection PyPackageRequirements import simplejson as json except ImportError: - import json + import json # type: ignore logger: logging.Logger = logging.getLogger(__name__) _T = TypeVar("_T") ATTRIBUTE_LENGTH_LIMIT: int = 128 +ATTRIBUTE_NUMBER_LIMIT: int = 256 TRUNCATE_REPLACEMENT: str = "..." +LAUNCH_NAME_LENGTH_LIMIT: int = 256 +ITEM_NAME_LENGTH_LIMIT: int = 1024 +LAUNCH_DESCRIPTION_LENGTH_LIMIT: int = 2048 +ITEM_DESCRIPTION_LENGTH_LIMIT: int = 65536 BYTES_TO_READ_FOR_DETECTION = 128 ATTRIBUTE_DELIMITER = ":" @@ -93,7 +98,7 @@ def get(self) -> Optional[_T]: self.__items = self.__items[:-1] return result - def last(self) -> _T: + def last(self) -> Optional[_T]: """Return the last element from the queue, but does not remove it. :return: The last element in the queue. @@ -186,7 +191,7 @@ def gen_attributes(rp_attributes: Iterable[str]) -> list[dict[str, str]]: return attributes -def get_launch_sys_attrs() -> dict[str, str]: +def get_launch_sys_attrs() -> dict[str, Any]: """Generate attributes for the launch containing system information. :return: dict {'os': 'Windows', @@ -201,14 +206,14 @@ def get_launch_sys_attrs() -> dict[str, str]: } -def get_package_parameters(package_name: str, parameters: list[str] = None) -> list[Optional[str]]: +def get_package_parameters(package_name: str, parameters: Optional[list[str]] = None) -> list[Optional[str]]: """Get parameters of the given package. :param package_name: Name of the package. :param parameters: Wanted parameters. :return: Parameter List. """ - result = [] + result: list[Optional[str]] = [] if not parameters: return result @@ -244,7 +249,7 @@ def truncate_attribute_string(text: str) -> str: return text -def verify_value_length(attributes: Optional[Union[list[dict], dict]]) -> Optional[list[dict]]: +def verify_value_length(attributes: list[dict]) -> Optional[list[dict]]: """Verify length of the attribute value. The length of the attribute value should have size from '1' to '128'. @@ -255,15 +260,8 @@ def verify_value_length(attributes: Optional[Union[list[dict], dict]]) -> Option :param attributes: List of attributes(tags) :return: List of attributes with corrected value length """ - if attributes is None: - return attributes - - my_attributes = attributes - if isinstance(my_attributes, dict): - my_attributes = dict_to_payload(my_attributes) - result = [] - for pair in my_attributes: + for pair in attributes: if not isinstance(pair, dict): continue attr_value = pair.get("value") @@ -312,7 +310,7 @@ def root_uri_join(*uri_parts: str) -> str: return "/" + uri_join(*uri_parts) -def get_function_params(func: Callable, args: tuple, kwargs: dict[str, Any]) -> dict[str, Any]: +def get_function_params(func: Callable, args: tuple, kwargs: dict[str, Any]) -> Optional[dict[str, Any]]: """Extract argument names from the function and combine them with values. :param func: the function to get arg names @@ -379,7 +377,9 @@ def calculate_file_part_size(file: Optional[RPFile]) -> int: if file is None: return 0 size = len(TYPICAL_FILE_PART_HEADER.format(file.name, file.content_type)) - size += len(file.content) + content = file.content + if isinstance(content, Sized): + size += len(content) return size @@ -435,9 +435,10 @@ def guess_content_type_from_bytes(data: Union[bytes, bytearray, list[int]]) -> s :param data: bytes or bytearray :return: content type """ - my_data = data if isinstance(data, list): - my_data = bytes(my_data) + my_data: Union[bytes, bytearray] = bytes(data) + else: + my_data = data if len(my_data) >= BYTES_TO_READ_FOR_DETECTION: my_data = my_data[:BYTES_TO_READ_FOR_DETECTION] @@ -497,9 +498,9 @@ def to_bool(value: Optional[Any]) -> Optional[bool]: """ if value is None: return None - if value in {"TRUE", "True", "true", "1", "Y", "y", 1, True}: + if value in {"TRUE", "True", "true", "1", "Y", "y", True}: return True - if value in {"FALSE", "False", "false", "0", "N", "n", 0, False}: + if value in {"FALSE", "False", "false", "0", "N", "n", False}: return False raise ValueError(f"Invalid boolean value {value}.") @@ -549,3 +550,39 @@ def caseless_equal(left: str, right: str) -> bool: :return: True if strings are equal ignoring case, False otherwise """ return normalize_caseless(left) == normalize_caseless(right) + + +# System/Data Controls +NUL = 0x00 # Null +SOH = 0x01 # Start of Heading +STX = 0x02 # Start of Text +ETX = 0x03 # End of Text +EOT = 0x04 # End of Transmission +ENQ = 0x05 # Enquiry +ACK = 0x06 # Acknowledge +NAK = 0x15 # Negative Acknowledge +SYN = 0x16 # Synchronous Idle +ETB = 0x17 # End of Trans. Block + +# Legacy Device Controls +BEL = 0x07 # Bell/Beep +DC1 = 0x11 # Device Control 1 (XON) +DC2 = 0x12 # Device Control 2 +DC3 = 0x13 # Device Control 3 (XOFF) +DC4 = 0x14 # Device Control 4 +CAN = 0x18 # Cancel +EM = 0x19 # End of Medium +SUB = 0x1A # Substitute +ESC = 0x1B # Escape + +PURELY_BINARY_CODES = {NUL, SOH, STX, ETX, EOT, ENQ, ACK, NAK, SYN, ETB, BEL, DC1, DC2, DC3, DC4, CAN, EM, SUB, ESC} + +REPLACEMENT = chr(0xFFFD) +CLEANUP_TABLE = str.maketrans({code: REPLACEMENT for code in PURELY_BINARY_CODES}) + + +def clean_binary_characters(text: str) -> str: + """Clean a string from binary characters, replace them with a question mark inside a diamond.""" + if not text: + return "" + return text.translate(CLEANUP_TABLE) diff --git a/reportportal_client/helpers/markdown_helpers.py b/reportportal_client/helpers/markdown_helpers.py index 9730003..3802cc8 100644 --- a/reportportal_client/helpers/markdown_helpers.py +++ b/reportportal_client/helpers/markdown_helpers.py @@ -1,7 +1,7 @@ """A set of utility methods for reporting to ReportPortal.""" from itertools import zip_longest -from typing import Any, Optional +from typing import Any, Optional, Sequence MARKDOWN_MODE = "!!!MARKDOWN_MODE!!!" NEW_LINE = "\n" @@ -36,7 +36,7 @@ def as_code(language: Optional[str], script: Optional[str]) -> str: return as_markdown(f"```{lang}\n{script}\n```") -def calculate_col_sizes(table: list[list[str]]) -> list[int]: +def calculate_col_sizes(table: list[Sequence[str]]) -> list[int]: """Calculate maximum width for each column in the table. :param table: Table data as list of rows @@ -66,7 +66,7 @@ def calculate_table_size(col_sizes: list[int]) -> int: return col_table_size -def transpose_table(table: list[list[Any]]) -> list[list[Any]]: +def transpose_table(table: list[Sequence[Any]]) -> list[Sequence[Any]]: """Transpose table rows into columns. :param table: Table data as list of rows @@ -78,7 +78,7 @@ def transpose_table(table: list[list[Any]]) -> list[list[Any]]: transposed = table else: # noinspection PyArgumentList - transposed = zip_longest(*table) + transposed = list(zip_longest(*table)) return [list(filter(None, col)) for col in transposed] @@ -109,7 +109,7 @@ def adjust_col_sizes(col_sizes: list[int], max_table_size: int) -> list[int]: return [size for size, _ in sorted(cols_by_size, key=lambda x: x[1])] -def format_data_table(table: list[list[str]], max_table_size: int = MAX_TABLE_SIZE) -> str: +def format_data_table(table: list[Sequence[str]], max_table_size: int = MAX_TABLE_SIZE) -> str: """Convert a table represented as List of Lists to a formatted table string. :param table: Table data as list of rows diff --git a/reportportal_client/steps/__init__.py b/reportportal_client/steps/__init__.py index 2e4ddad..2f9a046 100644 --- a/reportportal_client/steps/__init__.py +++ b/reportportal_client/steps/__init__.py @@ -104,7 +104,11 @@ def start_nested_step( ) def finish_nested_step( - self, item_id: str, end_time: str, status: str = None, **_: dict[str, Any] + self, + item_id: Union[str, Task[Optional[str]]], + end_time: str, + status: Optional[str] = None, + **_: dict[str, Any], ) -> Union[Optional[str], Task[Optional[str]]]: """Finish a Nested Step on ReportPortal. @@ -112,19 +116,19 @@ def finish_nested_step( :param end_time: Nested Step finish time :param status: Nested Step finish status """ - return self.client.finish_test_item(item_id, end_time, status=status) + return self.client.finish_test_item(item_id, end_time, status=status) # type: ignore -class Step(Callable[[_Param], _Return]): +class Step: """Step context handling class.""" name: str - params: dict + params: Optional[dict] status: str client: Optional["rp.RP"] __item_id: Union[Optional[str], Task[Optional[str]]] - def __init__(self, name: str, params: dict, status: str, rp_client: Optional["rp.RP"]) -> None: + def __init__(self, name: str, params: Optional[dict], status: str, rp_client: Optional["rp.RP"]) -> None: """Initialize required attributes. :param name: Nested Step name diff --git a/requirements-dev.txt b/requirements-dev.txt index 1b16115..c385c8a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,4 +4,5 @@ pytest-asyncio black isort types-requests -mypy +mypy==1.19.1 +types-simplejson==3.20.0.20250822 diff --git a/requirements.txt b/requirements.txt index 1eec00f..87b3923 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -aenum -typing-extensions>=4.13.2 -requests>=2.32.4 -aiohttp>=3.11.18 -certifi>=2025.11.12 +aenum==3.1.17 +typing-extensions==4.13.2 +requests==2.32.5 +aiohttp==3.11.18 +certifi==2026.2.25 diff --git a/setup.py b/setup.py index 9a8fd8c..6a80726 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages, setup -__version__ = "5.7.0" +__version__ = "5.7.1" TYPE_STUBS = ["*.pyi"] diff --git a/tests/_internal/logs/test_log_batcher.py b/tests/_internal/logs/test_log_batcher.py index cf73026..4308e88 100644 --- a/tests/_internal/logs/test_log_batcher.py +++ b/tests/_internal/logs/test_log_batcher.py @@ -37,6 +37,9 @@ def test_log_batch_send_by_length(): for i in range(TEST_BATCH_SIZE): result = log_batcher.append( RPRequestLog( + truncate_attributes_enabled=None, + truncate_fields_enabled=None, + replace_binary_characters=None, launch_uuid=TEST_LAUNCH_ID, time=helpers.timestamp(), message=TEST_MASSAGE, @@ -60,6 +63,9 @@ def test_log_batch_send_by_flush(): for _ in range(TEST_BATCH_SIZE - 1): log_batcher.append( RPRequestLog( + truncate_attributes_enabled=None, + truncate_fields_enabled=None, + replace_binary_characters=None, launch_uuid=TEST_LAUNCH_ID, time=helpers.timestamp(), message=TEST_MASSAGE, @@ -80,6 +86,9 @@ def test_log_batch_send_by_size(): random_byte_array = bytearray(os.urandom(MAX_LOG_BATCH_PAYLOAD_SIZE)) binary_result = log_batcher.append( RPRequestLog( + truncate_attributes_enabled=None, + truncate_fields_enabled=None, + replace_binary_characters=None, launch_uuid=TEST_LAUNCH_ID, time=helpers.timestamp(), message=TEST_MASSAGE, @@ -90,6 +99,9 @@ def test_log_batch_send_by_size(): ) message_result = log_batcher.append( RPRequestLog( + truncate_attributes_enabled=None, + truncate_fields_enabled=None, + replace_binary_characters=None, launch_uuid=TEST_LAUNCH_ID, time=helpers.timestamp(), message=TEST_MASSAGE, @@ -113,6 +125,9 @@ def test_log_batch_triggers_previous_request_to_send(): message_result = log_batcher.append( RPRequestLog( + truncate_attributes_enabled=None, + truncate_fields_enabled=None, + replace_binary_characters=None, launch_uuid=TEST_LAUNCH_ID, time=helpers.timestamp(), message=TEST_MASSAGE, @@ -123,6 +138,9 @@ def test_log_batch_triggers_previous_request_to_send(): binary_result = log_batcher.append( RPRequestLog( + truncate_attributes_enabled=None, + truncate_fields_enabled=None, + replace_binary_characters=None, launch_uuid=TEST_LAUNCH_ID, time=helpers.timestamp(), message=TEST_MASSAGE, diff --git a/tests/aio/test_aio_client.py b/tests/aio/test_aio_client.py index 888c02e..86b8188 100644 --- a/tests/aio/test_aio_client.py +++ b/tests/aio/test_aio_client.py @@ -30,7 +30,6 @@ from reportportal_client._internal.aio.http import DEFAULT_RETRY_NUMBER, ClientSession, RetryingClientSession # noinspection PyProtectedMember -from reportportal_client._internal.static.defines import NOT_SET from reportportal_client.aio.client import Client from reportportal_client.core.rp_issues import Issue from reportportal_client.core.rp_requests import AsyncRPRequestLog @@ -58,10 +57,10 @@ def test_client_pickling(): "retry_num, expected_wrapped_class, expected_param", [ (1, RetryingClientSession, 1), - (0, aiohttp.ClientSession, NOT_SET), - (-1, aiohttp.ClientSession, NOT_SET), - (None, aiohttp.ClientSession, NOT_SET), - (NOT_SET, RetryingClientSession, DEFAULT_RETRY_NUMBER), + (0, aiohttp.ClientSession, -1), + (-2, aiohttp.ClientSession, -1), + (None, aiohttp.ClientSession, -1), + (-1, RetryingClientSession, DEFAULT_RETRY_NUMBER), ], ) @pytest.mark.asyncio @@ -73,7 +72,7 @@ async def test_retries_param(retry_num, expected_wrapped_class, expected_param): # Check the wrapped session type # noinspection PyProtectedMember assert isinstance(session._client, expected_wrapped_class) - if expected_param is not NOT_SET: + if expected_param != -1: # noinspection PyProtectedMember assert getattr(session._client, "_RetryingClientSession__retry_number") == expected_param @@ -566,7 +565,22 @@ def request_error(*_, **__): ("get", "get_launch_ui_id", ["launch_uuid"]), ("get", "get_launch_ui_url", ["launch_uuid"]), ("get", "get_project_settings", []), - ("post", "log_batch", [[AsyncRPRequestLog("launch_uuid", timestamp(), item_uuid="test_item_uuid")]]), + ( + "post", + "log_batch", + [ + [ + AsyncRPRequestLog( + truncate_attributes_enabled=None, + truncate_fields_enabled=None, + replace_binary_characters=None, + launch_uuid="launch_uuid", + time=timestamp(), + item_uuid="test_item_uuid", + ) + ] + ], + ), ], ) @pytest.mark.asyncio diff --git a/tests/aio/test_async_client.py b/tests/aio/test_async_client.py index d6ec4bf..d0a5c0c 100644 --- a/tests/aio/test_async_client.py +++ b/tests/aio/test_async_client.py @@ -176,7 +176,16 @@ async def test_logs_flush_on_close(async_client: AsyncRPClient): # noinspection PyTypeChecker client: mock.Mock = async_client.client batcher: mock.Mock = mock.Mock() - batcher.flush.return_value = [AsyncRPRequestLog("test_launch_uuid", timestamp(), message="test_message")] + batcher.flush.return_value = [ + AsyncRPRequestLog( + truncate_attributes_enabled=None, + truncate_fields_enabled=None, + replace_binary_characters=None, + launch_uuid="test_launch_uuid", + time=timestamp(), + message="test_message", + ) + ] async_client._log_batcher = batcher await async_client.close() diff --git a/tests/aio/test_batched_client.py b/tests/aio/test_batched_client.py index b1cdac8..e080e3f 100644 --- a/tests/aio/test_batched_client.py +++ b/tests/aio/test_batched_client.py @@ -150,7 +150,16 @@ def test_logs_flush_on_close(batched_client: BatchedRPClient): # noinspection PyTypeChecker client: mock.Mock = batched_client.client batcher: mock.Mock = mock.Mock() - batcher.flush.return_value = [AsyncRPRequestLog("test_launch_uuid", timestamp(), message="test_message")] + batcher.flush.return_value = [ + AsyncRPRequestLog( + truncate_attributes_enabled=None, + truncate_fields_enabled=None, + replace_binary_characters=None, + launch_uuid="test_launch_uuid", + time=timestamp(), + message="test_message", + ) + ] batched_client._log_batcher = batcher batched_client.close() diff --git a/tests/aio/test_threaded_client.py b/tests/aio/test_threaded_client.py index a6ce46a..ee5f6b0 100644 --- a/tests/aio/test_threaded_client.py +++ b/tests/aio/test_threaded_client.py @@ -148,7 +148,16 @@ def test_logs_flush_on_close(batched_client: ThreadedRPClient): # noinspection PyTypeChecker client: mock.Mock = batched_client.client batcher: mock.Mock = mock.Mock() - batcher.flush.return_value = [AsyncRPRequestLog("test_launch_uuid", timestamp(), message="test_message")] + batcher.flush.return_value = [ + AsyncRPRequestLog( + truncate_attributes_enabled=None, + truncate_fields_enabled=None, + replace_binary_characters=None, + launch_uuid="test_launch_uuid", + time=timestamp(), + message="test_message", + ) + ] batched_client._log_batcher = batcher batched_client.close() diff --git a/tests/core/test_rp_requests.py b/tests/core/test_rp_requests.py new file mode 100644 index 0000000..a1f3e58 --- /dev/null +++ b/tests/core/test_rp_requests.py @@ -0,0 +1,119 @@ +from reportportal_client.core.rp_requests import ( + ItemFinishRequest, + ItemStartRequest, + LaunchFinishRequest, + LaunchStartRequest, +) +from reportportal_client.helpers import ( + ITEM_DESCRIPTION_LENGTH_LIMIT, + ITEM_NAME_LENGTH_LIMIT, + LAUNCH_DESCRIPTION_LENGTH_LIMIT, + LAUNCH_NAME_LENGTH_LIMIT, +) + + +def test_launch_name_truncated_in_payload(): + launch_name = "n" * (LAUNCH_NAME_LENGTH_LIMIT + 20) + payload = LaunchStartRequest( + truncate_attributes_enabled=None, + truncate_fields_enabled=None, + replace_binary_characters=None, + name=launch_name, + start_time="0", + ).payload + assert len(payload["name"]) == LAUNCH_NAME_LENGTH_LIMIT + assert payload["name"].endswith("...") + + +def test_item_name_cleaned_and_truncated_in_payload(): + item_name = "bad\x00name" + ("n" * ITEM_NAME_LENGTH_LIMIT) + payload = ItemStartRequest( + truncate_attributes_enabled=None, + truncate_fields_enabled=None, + replace_binary_characters=None, + name=item_name, + start_time="0", + type_="SUITE", + launch_uuid="launch_uuid", + attributes=None, + code_ref=None, + description=None, + has_stats=True, + parameters=None, + retry=False, + retry_of=None, + test_case_id=None, + uuid=None, + ).payload + assert "\x00" not in payload["name"] + assert len(payload["name"]) == ITEM_NAME_LENGTH_LIMIT + + +def test_launch_description_cleaned_and_truncated_in_payload(): + launch_description = "bad\x00description" + ("d" * LAUNCH_DESCRIPTION_LENGTH_LIMIT) + payload = LaunchStartRequest( + truncate_attributes_enabled=None, + truncate_fields_enabled=None, + replace_binary_characters=None, + name="launch", + start_time="0", + description=launch_description, + ).payload + assert "\x00" not in payload["description"] + assert len(payload["description"]) == LAUNCH_DESCRIPTION_LENGTH_LIMIT + assert payload["description"].endswith("...") + + +def test_item_description_cleaned_and_truncated_in_payload(): + item_description = "bad\x00description" + ("d" * ITEM_DESCRIPTION_LENGTH_LIMIT) + payload = ItemStartRequest( + truncate_attributes_enabled=None, + truncate_fields_enabled=None, + replace_binary_characters=None, + name="item", + start_time="0", + type_="SUITE", + launch_uuid="launch_uuid", + attributes=None, + code_ref=None, + description=item_description, + has_stats=True, + parameters=None, + retry=False, + retry_of=None, + test_case_id=None, + uuid=None, + ).payload + assert "\x00" not in payload["description"] + assert len(payload["description"]) == ITEM_DESCRIPTION_LENGTH_LIMIT + assert payload["description"].endswith("...") + + +def test_finish_requests_truncate_description(): + launch_finish_payload = LaunchFinishRequest( + truncate_attributes_enabled=None, + truncate_fields_enabled=None, + replace_binary_characters=None, + end_time="0", + description="d" * (LAUNCH_DESCRIPTION_LENGTH_LIMIT + 10), + ).payload + item_finish_payload = ItemFinishRequest( + truncate_attributes_enabled=None, + truncate_fields_enabled=None, + replace_binary_characters=None, + end_time="0", + launch_uuid="launch_uuid", + status="PASSED", + attributes=None, + description="d" * (ITEM_DESCRIPTION_LENGTH_LIMIT + 10), + is_skipped_an_issue=True, + issue=None, + retry=False, + retry_of=None, + test_case_id=None, + ).payload + + assert len(launch_finish_payload["description"]) == LAUNCH_DESCRIPTION_LENGTH_LIMIT + assert launch_finish_payload["description"].endswith("...") + assert len(item_finish_payload["description"]) == ITEM_DESCRIPTION_LENGTH_LIMIT + assert item_finish_payload["description"].endswith("...") diff --git a/tests/core/test_worker.py b/tests/core/test_worker.py index a442d1b..66260ff 100644 --- a/tests/core/test_worker.py +++ b/tests/core/test_worker.py @@ -31,8 +31,20 @@ def test_worker_continue_working_on_request_error(): worker = APIWorker(test_queue) worker.start() - log_request = RPRequestLog(TEST_LAUNCH_UUID, timestamp(), message=TEST_MASSAGE) - log_batch = RPLogBatch([log_request]) + log_request = RPRequestLog( + truncate_attributes_enabled=None, + truncate_fields_enabled=None, + replace_binary_characters=None, + launch_uuid=TEST_LAUNCH_UUID, + time=timestamp(), + message=TEST_MASSAGE, + ) + log_batch = RPLogBatch( + truncate_attributes_enabled=None, + truncate_fields_enabled=None, + replace_binary_characters=None, + log_reqs=[log_request], + ) fail_session = mock.Mock() fail_session.side_effect = Exception() diff --git a/tests/helpers/test_helpers.py b/tests/helpers/test_helpers.py index 51f614a..d2f7539 100644 --- a/tests/helpers/test_helpers.py +++ b/tests/helpers/test_helpers.py @@ -66,7 +66,7 @@ def test_get_launch_sys_attrs_docker(): "attributes, expected_attributes", [ ( - {"tn": "v" * 129}, + [{"key": "tn", "value": "v" * 129}], [ { "key": "tn", @@ -74,25 +74,12 @@ def test_get_launch_sys_attrs_docker(): } ], ), - ({"tn": "v" * 128}, [{"key": "tn", "value": "v" * 128}]), ( - {"k" * 129: "v"}, + [{"key": "k" * 129, "value": "v"}], [{"key": "k" * (ATTRIBUTE_LENGTH_LIMIT - len(TRUNCATE_REPLACEMENT)) + TRUNCATE_REPLACEMENT, "value": "v"}], ), - ({"k" * 128: "v"}, [{"key": "k" * 128, "value": "v"}]), - ({"tn": "v" * 128, "system": True}, [{"key": "tn", "value": "v" * 128, "system": True}]), ( - {"tn": "v" * 129, "system": True}, - [ - { - "key": "tn", - "value": "v" * (ATTRIBUTE_LENGTH_LIMIT - len(TRUNCATE_REPLACEMENT)) + TRUNCATE_REPLACEMENT, - "system": True, - } - ], - ), - ( - {"k" * 129: "v", "system": False}, + [{"key": "k" * 129, "value": "v", "system": False}], [ { "key": "k" * (ATTRIBUTE_LENGTH_LIMIT - len(TRUNCATE_REPLACEMENT)) + TRUNCATE_REPLACEMENT, @@ -102,23 +89,21 @@ def test_get_launch_sys_attrs_docker(): ], ), ( - [{"key": "tn", "value": "v" * 129}], + [{"system": True, "key": "tn", "value": "v" * 129}], [ { "key": "tn", "value": "v" * (ATTRIBUTE_LENGTH_LIMIT - len(TRUNCATE_REPLACEMENT)) + TRUNCATE_REPLACEMENT, + "system": True, } ], ), - ( - [{"key": "k" * 129, "value": "v"}], - [{"key": "k" * (ATTRIBUTE_LENGTH_LIMIT - len(TRUNCATE_REPLACEMENT)) + TRUNCATE_REPLACEMENT, "value": "v"}], - ), ], ) def test_verify_value_length(attributes, expected_attributes): """Test for validate verify_value_length() function.""" result = verify_value_length(attributes) + assert result is not None assert len(result) == len(expected_attributes) for i, element in enumerate(result): expected = expected_attributes[i] diff --git a/tests/test_client.py b/tests/test_client.py index b899cdd..19b3b80 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -262,14 +262,31 @@ def test_attribute_truncation(rp_client: RPClient, method, call_method, argument if method != "start_launch": rp_client._RPClient__launch_uuid = "test_launch_id" - getattr(rp_client, method)(*arguments, **{"attributes": {"key": "value" * 26}}) + getattr(rp_client, method)(*arguments, **{"attributes": {"k" * 140: "v" * 140}}) getattr(session, call_method).assert_called_once() kwargs = getattr(session, call_method).call_args_list[0][1] assert "attributes" in kwargs["json"] assert kwargs["json"]["attributes"] + assert len(kwargs["json"]["attributes"][0]["key"]) == 128 assert len(kwargs["json"]["attributes"][0]["value"]) == 128 +def test_attribute_sanitization_binary_and_number_limit(rp_client: RPClient): + # noinspection PyTypeChecker + session: mock.Mock = rp_client.session + rp_client._RPClient__launch_uuid = "test_launch_id" + attributes = [{"key": f"k{index:03d}", "value": "value"} for index in range(300)] + attributes[0] = {"key": "a\x00key", "value": "v\x1balue"} + + rp_client.start_test_item("Test Item", timestamp(), "SUITE", attributes=attributes) + + kwargs = session.post.call_args_list[0][1] + sanitized_attributes = kwargs["json"]["attributes"] + assert len(sanitized_attributes) == 256 + assert all("\x00" not in attr["key"] for attr in sanitized_attributes if attr.get("key")) + assert all("\x1b" not in attr["value"] for attr in sanitized_attributes if attr.get("value")) + + @pytest.mark.parametrize( "method, call_method, arguments", [ @@ -308,7 +325,16 @@ def test_logs_flush_on_close(rp_client: RPClient): # noinspection PyTypeChecker session: mock.Mock = rp_client.session batcher: mock.Mock = mock.Mock() - batcher.flush.return_value = [RPRequestLog("test_launch_uuid", timestamp(), message="test_message")] + batcher.flush.return_value = [ + RPRequestLog( + truncate_attributes_enabled=None, + truncate_fields_enabled=None, + replace_binary_characters=None, + launch_uuid="test_launch_uuid", + time=timestamp(), + message="test_message", + ) + ] rp_client._log_batcher = batcher rp_client.close()