[do not merge] feat: Span streaming & new span API

CHANGELOG.md

@@ -1,5 +1,60 @@
 # Changelog
 
+## 2.52.0a2
+
+### New Features ✨
+
+#### Span Streaming
+
+- feat(span-streaming): Add spans to telemetry pipeline, add span name and attributes (3) by @sentrivana in [#5399](https://github.com/getsentry/sentry-python/pull/5399)
+- feat(span-streaming): Add span batcher (2) by @sentrivana in [#5398](https://github.com/getsentry/sentry-python/pull/5398)
+
+### Internal Changes 🔧
+
+- test(mcp): Simulate stdio transport with memory streams by @alexander-alderman-webb in [#5329](https://github.com/getsentry/sentry-python/pull/5329)
+
+### Other
+
+- [do not merge] feat: Span streaming & new span API by @sentrivana in [#5317](https://github.com/getsentry/sentry-python/pull/5317)
+
+## 2.52.0a1
+
+### New Features ✨
+
+#### Openai
+
+- feat(openai): Set system instruction attribute for Responses API by @alexander-alderman-webb in [#5376](https://github.com/getsentry/sentry-python/pull/5376)
+- feat(openai): Set system instruction attribute for Completions API by @alexander-alderman-webb in [#5358](https://github.com/getsentry/sentry-python/pull/5358)
+
+#### Other
+
+- feat(ai): Add original input length meta attribute by @alexander-alderman-webb in [#5375](https://github.com/getsentry/sentry-python/pull/5375)
+- feat(openai-agents): Set system instruction attribute on `gen_ai.chat` spans by @alexander-alderman-webb in [#5370](https://github.com/getsentry/sentry-python/pull/5370)
+- feat(span-streaming): Add experimental `trace_lifecycle` switch (1) by @sentrivana in [#5397](https://github.com/getsentry/sentry-python/pull/5397)
+- feat(transport): Report 413 responses for oversized envelopes by @alexander-alderman-webb in [#5380](https://github.com/getsentry/sentry-python/pull/5380)
+
+### Bug Fixes 🐛
+
+- fix(ai): Keep single content input message by @alexander-alderman-webb in [#5345](https://github.com/getsentry/sentry-python/pull/5345)
+- fix(arq): handle settings_cls passed as keyword argument by @nc9 in [#5393](https://github.com/getsentry/sentry-python/pull/5393)
+- fix(dramatiq): cleanup isolated scope and transaction when message is skipped by @frankie567 in [#5346](https://github.com/getsentry/sentry-python/pull/5346)
+- fix(google-genai): deactivate google genai when langchain is used by @shellmayr in [#5389](https://github.com/getsentry/sentry-python/pull/5389)
+
+### Internal Changes 🔧
+
+- ci: migration to the new codecov action by @MathurAditya724 in [#5392](https://github.com/getsentry/sentry-python/pull/5392)
+- ref: Replace `set_data_normalized()` with `Span.set_data()` for system instructions by @alexander-alderman-webb in [#5374](https://github.com/getsentry/sentry-python/pull/5374)
+
+### Other
+
+- [do not merge] feat: Span streaming & new span API by @sentrivana in [#5317](https://github.com/getsentry/sentry-python/pull/5317)
+- Update CHANGELOG.md by @alexander-alderman-webb in [8517eb0a](https://github.com/getsentry/sentry-python/commit/8517eb0a0750796ae73d0e4c020b0a71c7724d0a)
+- release: 2.51.0 by @alexander-alderman-webb in [93e89e4c](https://github.com/getsentry/sentry-python/commit/93e89e4c1b7e837c03dd62a81951e56634d4a9c0)
+
+## 2.51.0a1
+
+This is an alpha release for internal testing of span streaming.
+
 ## 2.51.0
 
 ### New Features ✨

docs/conf.py

@@ -31,7 +31,7 @@
 copyright = "2019-{}, Sentry Team and Contributors".format(datetime.now().year)
 author = "Sentry Team and Contributors"
 
-release = "2.51.0"
+release = "2.52.0a2"
 version = ".".join(release.split(".")[:2])  # The short X.Y version.
 
 

sentry_sdk/_span_batcher.py

@@ -4,14 +4,12 @@
 from typing import TYPE_CHECKING
 
 from sentry_sdk._batcher import Batcher
-from sentry_sdk.consts import SPANSTATUS
 from sentry_sdk.envelope import Envelope, Item, PayloadRef
-from sentry_sdk.utils import format_timestamp, serialize_attribute, safe_repr
+from sentry_sdk.utils import format_timestamp, serialize_attribute
 
 if TYPE_CHECKING:
     from typing import Any, Callable, Optional
     from sentry_sdk.traces import StreamedSpan
-    from sentry_sdk._types import SerializedAttributeValue
 
 
 class SpanBatcher(Batcher["StreamedSpan"]):
@@ -69,11 +67,25 @@ def add(self, span: "StreamedSpan") -> None:
 
     @staticmethod
     def _to_transport_format(item: "StreamedSpan") -> "Any":
-        # TODO[span-first]
         res: "dict[str, Any]" = {
-            "name": item.name,
+            "trace_id": item.trace_id,
+            "span_id": item.span_id,
+            "name": item.get_name(),
+            "status": item.status,
+            "is_segment": item.is_segment(),
+            "start_timestamp": item.start_timestamp.timestamp(),  # TODO[span-first]
         }
 
+        if item.timestamp:
+            res["end_timestamp"] = item.timestamp.timestamp()
+
+        if item._last_valid_parent_id:
+            # This span needs to be reparented because its parent is ignored
+            res["parent_span_id"] = item._last_valid_parent_id
+
+        elif item.parent_span_id:
+            res["parent_span_id"] = item.parent_span_id
+
         if item._attributes:
             res["attributes"] = {
                 k: serialize_attribute(v) for (k, v) in item._attributes.items()
@@ -87,11 +99,9 @@ def _flush(self) -> None:
                 return None
 
             envelopes = []
-            for trace_id, spans in self._span_buffer.items():
+            for spans in self._span_buffer.values():
                 if spans:
-                    # TODO[span-first]
-                    # dsc = spans[0].dynamic_sampling_context()
-                    dsc = None
+                    dsc = spans[0].dynamic_sampling_context()
 
                     envelope = Envelope(
                         headers={

sentry_sdk/_types.py

@@ -1,4 +1,4 @@
-from typing import TYPE_CHECKING, TypeVar, Union
+from typing import TYPE_CHECKING, Pattern, TypeVar, Union
 
 
 # Re-exported for compat, since code out there in the wild might use this variable.
@@ -363,3 +363,11 @@ class SDKInfo(TypedDict):
     class TextPart(TypedDict):
         type: Literal["text"]
         content: str
+
+    IgnoreSpansName = Union[str, Pattern[str]]
+    IgnoreSpansContext = TypedDict(
+        "IgnoreSpansContext",
+        {"name": IgnoreSpansName, "attributes": Attributes},
+        total=False,
+    )
+    IgnoreSpansConfig = list[Union[IgnoreSpansName, IgnoreSpansContext]]

sentry_sdk/ai/monitoring.py

@@ -6,6 +6,7 @@
 import sentry_sdk.utils
 from sentry_sdk import start_span
 from sentry_sdk.tracing import Span
+from sentry_sdk.traces import StreamedSpan
 from sentry_sdk.utils import ContextVar, reraise, capture_internal_exceptions
 
 from typing import TYPE_CHECKING
@@ -97,7 +98,7 @@ async def async_wrapped(*args: "Any", **kwargs: "Any") -> "Any":
 
 
 def record_token_usage(
-    span: "Span",
+    span: "Union[Span, StreamedSpan]",
     input_tokens: "Optional[int]" = None,
     input_tokens_cached: "Optional[int]" = None,
     input_tokens_cache_write: "Optional[int]" = None,
@@ -106,30 +107,35 @@ def record_token_usage(
     total_tokens: "Optional[int]" = None,
 ) -> None:
     # TODO: move pipeline name elsewhere
+    if isinstance(span, StreamedSpan):
+        set_on_span = span.set_attribute
+    else:
+        set_on_span = span.set_data
+
     ai_pipeline_name = get_ai_pipeline_name()
     if ai_pipeline_name:
-        span.set_data(SPANDATA.GEN_AI_PIPELINE_NAME, ai_pipeline_name)
+        set_on_span(SPANDATA.GEN_AI_PIPELINE_NAME, ai_pipeline_name)
 
     if input_tokens is not None:
-        span.set_data(SPANDATA.GEN_AI_USAGE_INPUT_TOKENS, input_tokens)
+        set_on_span(SPANDATA.GEN_AI_USAGE_INPUT_TOKENS, input_tokens)
 
     if input_tokens_cached is not None:
-        span.set_data(
+        set_on_span(
             SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED,
             input_tokens_cached,
         )
 
     if input_tokens_cache_write is not None:
-        span.set_data(
+        set_on_span(
             SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE,
             input_tokens_cache_write,
         )
 
     if output_tokens is not None:
-        span.set_data(SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS, output_tokens)
+        set_on_span(SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS, output_tokens)
 
     if output_tokens_reasoning is not None:
-        span.set_data(
+        set_on_span(
             SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS_REASONING,
             output_tokens_reasoning,
         )
@@ -138,4 +144,4 @@ def record_token_usage(
         total_tokens = input_tokens + output_tokens
 
     if total_tokens is not None:
-        span.set_data(SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS, total_tokens)
+        set_on_span(SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS, total_tokens)

sentry_sdk/ai/utils.py

@@ -8,12 +8,14 @@
 from sentry_sdk._types import BLOB_DATA_SUBSTITUTE
 
 if TYPE_CHECKING:
-    from typing import Any, Callable, Dict, List, Optional, Tuple
+    from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
     from sentry_sdk.tracing import Span
 
 import sentry_sdk
 from sentry_sdk.utils import logger
+from sentry_sdk.traces import StreamedSpan
+from sentry_sdk.tracing_utils import has_span_streaming_enabled
 from sentry_sdk.consts import SPANDATA
 
 MAX_GEN_AI_MESSAGE_BYTES = 20_000  # 20KB
@@ -490,13 +492,19 @@ def _normalize_data(data: "Any", unpack: bool = True) -> "Any":
 
 
 def set_data_normalized(
-    span: "Span", key: str, value: "Any", unpack: bool = True
+    span: "Union[Span, StreamedSpan]", key: str, value: "Any", unpack: bool = True
 ) -> None:
     normalized = _normalize_data(value, unpack=unpack)
+
+    if isinstance(span, StreamedSpan):
+        set_on_span = span.set_attribute
+    else:
+        set_on_span = span.set_data
+
     if isinstance(normalized, (int, float, bool, str)):
-        span.set_data(key, normalized)
+        set_on_span(key, normalized)
     else:
-        span.set_data(key, json.dumps(normalized))
+        set_on_span(key, json.dumps(normalized))
 
 
 def normalize_message_role(role: str) -> str:
@@ -526,7 +534,14 @@ def normalize_message_roles(messages: "list[dict[str, Any]]") -> "list[dict[str,
 
 
 def get_start_span_function() -> "Callable[..., Any]":
+    client = sentry_sdk.get_client()
+
     current_span = sentry_sdk.get_current_span()
+    if isinstance(current_span, StreamedSpan) or has_span_streaming_enabled(
+        client.options
+    ):
+        return sentry_sdk.traces.start_span
+
     transaction_exists = (
         current_span is not None and current_span.containing_transaction is not None
     )

sentry_sdk/api.py

@@ -7,6 +7,7 @@
 from sentry_sdk.consts import INSTRUMENTER
 from sentry_sdk.scope import Scope, _ScopeManager, new_scope, isolation_scope
 from sentry_sdk.tracing import NoOpSpan, Transaction, trace
+from sentry_sdk.traces import StreamedSpan
 from sentry_sdk.crons import monitor
 
 from typing import TYPE_CHECKING
@@ -385,7 +386,9 @@ def set_measurement(name: str, value: float, unit: "MeasurementUnit" = "") -> No
         transaction.set_measurement(name, value, unit)
 
 
-def get_current_span(scope: "Optional[Scope]" = None) -> "Optional[Span]":
+def get_current_span(
+    scope: "Optional[Scope]" = None,
+) -> "Optional[Union[Span, StreamedSpan]]":
     """
     Returns the currently active span if there is one running, otherwise `None`
     """
@@ -501,6 +504,16 @@ def update_current_span(
     if current_span is None:
         return
 
+    if isinstance(current_span, StreamedSpan):
+        warnings.warn(
+            "The `update_current_span` API isn't available in streaming mode. "
+            "Retrieve the current span with get_current_span() and use its API "
+            "directly.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return
+
     if op is not None:
         current_span.op = op
 

sentry_sdk/client.py

@@ -941,6 +941,7 @@ def _capture_telemetry(
             before_send = get_before_send_log(self.options)
         elif ty == "metric":
             before_send = get_before_send_metric(self.options)  # type: ignore
+        # no before_send for spans
 
         if before_send is not None:
             telemetry = before_send(telemetry, {})  # type: ignore

sentry_sdk/consts.py

@@ -37,6 +37,7 @@ class CompressionAlgo(Enum):
         Dict,
         List,
         Optional,
+        Pattern,
         Sequence,
         Tuple,
         Type,
@@ -52,6 +53,7 @@ class CompressionAlgo(Enum):
         Event,
         EventProcessor,
         Hint,
+        IgnoreSpansConfig,
         Log,
         MeasurementUnit,
         Metric,
@@ -83,6 +85,7 @@ class CompressionAlgo(Enum):
             "enable_metrics": Optional[bool],
             "before_send_metric": Optional[Callable[[Metric, Hint], Optional[Metric]]],
             "trace_lifecycle": Optional[Literal["static", "stream"]],
+            "ignore_spans": Optional[IgnoreSpansConfig],
         },
         total=False,
     )
@@ -1492,4 +1495,4 @@ def _get_default_options() -> "dict[str, Any]":
 del _get_default_options
 
 
-VERSION = "2.51.0"
+VERSION = "2.52.0a2"

sentry_sdk/envelope.py

@@ -253,6 +253,8 @@ def data_category(self) -> "EventDataCategory":
             return "session"
         elif ty == "attachment":
             return "attachment"
+        elif ty == "span":
+            return "span"
         elif ty == "transaction":
             return "transaction"
         elif ty == "event":