From 1bad483ffa172b47493443d33bde179060a368ac Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Mon, 7 Jul 2025 18:48:45 +0200 Subject: [PATCH 01/16] feat: Barely functional version of Rust map and Rust filter --- .pre-commit-config.yaml | 22 +- sentry_streams/Cargo.lock | 9 + sentry_streams/Cargo.toml | 5 +- .../adapters/arroyo/rust_arroyo.py | 34 +- .../rust_simple_map_filter/pipeline.py | 41 + .../rust_transforms/Cargo.lock | 2428 ++++++++++++++++ .../rust_transforms/Cargo.toml | 23 + .../metrics_rust_transforms.pyi | 39 + .../rust_transforms/py.typed | 0 .../rust_transforms/src/lib.rs | 65 + .../examples/simple_map_filter.py | 9 - .../sentry_streams/pipeline/pipeline.py | 65 + .../pipeline/rust_function_protocol.py | 28 + sentry_streams/src/ffi.rs | 137 + sentry_streams/src/lib.rs | 5 + sentry_streams/src/macros.rs | 203 ++ sentry_streams/src/operators.rs | 54 +- sentry_streams/src/transformer.rs | 228 ++ .../tests/rust_test_functions/Cargo.lock | 2438 +++++++++++++++++ .../tests/rust_test_functions/Cargo.toml | 19 + .../tests/rust_test_functions/py.typed | 0 .../rust_test_functions.pyi | 19 + .../tests/rust_test_functions/src/lib.rs | 77 + sentry_streams/tests/test_mypy_integration.py | 148 + sentry_streams/tests/test_true.py | 5 - 25 files changed, 6061 insertions(+), 40 deletions(-) create mode 100644 sentry_streams/sentry_streams/examples/rust_simple_map_filter/pipeline.py create mode 100644 sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/Cargo.lock create mode 100644 sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/Cargo.toml create mode 100644 sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/metrics_rust_transforms.pyi create mode 100644 sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/py.typed create mode 100644 sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/src/lib.rs create mode 100644 sentry_streams/sentry_streams/pipeline/rust_function_protocol.py create mode 100644 sentry_streams/src/ffi.rs create mode 100644 sentry_streams/src/macros.rs create mode 100644 sentry_streams/tests/rust_test_functions/Cargo.lock create mode 100644 sentry_streams/tests/rust_test_functions/Cargo.toml create mode 100644 sentry_streams/tests/rust_test_functions/py.typed create mode 100644 sentry_streams/tests/rust_test_functions/rust_test_functions.pyi create mode 100644 sentry_streams/tests/rust_test_functions/src/lib.rs create mode 100644 sentry_streams/tests/test_mypy_integration.py delete mode 100644 sentry_streams/tests/test_true.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b3a2b25e..084d408e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,22 +31,6 @@ repos: hooks: - id: flake8 language_version: python3.11 - - repo: https://github.com/pre-commit/mirrors-mypy - rev: 'v1.14.1' - hooks: - - id: mypy - args: [--config-file, sentry_streams/mypy.ini, --strict] - additional_dependencies: [ - pytest==7.1.2, - types-requests, - responses, - "sentry-arroyo>=2.18.2", - types-pyYAML, - types-jsonschema, - "sentry-kafka-schemas>=1.2.0", - "polars==1.30.0", - ] - files: ^sentry_streams/.+ - repo: https://github.com/pycqa/isort rev: 6.0.0 hooks: @@ -67,6 +51,12 @@ repos: # https://github.com/rust-lang/rustfmt/issues/4485 entry: rustfmt --edition 2021 files: ^sentry_streams/.*\.rs$ + - id: mypy + name: mypy + entry: mypy + files: ^sentry_streams/.*\.py$ + types: [python] + language: system default_language_version: python: python3.11 diff --git a/sentry_streams/Cargo.lock b/sentry_streams/Cargo.lock index ffbdfa71..d76d7fda 100644 --- a/sentry_streams/Cargo.lock +++ b/sentry_streams/Cargo.lock @@ -995,6 +995,12 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "percent-encoding" version = "2.3.1" @@ -1300,16 +1306,19 @@ name = "rust_streams" version = "0.1.0" dependencies = [ "anyhow", + "base64", "chrono", "ctrlc", "log", "parking_lot", + "paste", "pyo3", "pyo3-build-config", "rdkafka", "reqwest", "sentry_arroyo", "serde", + "serde_json", "tokio", "tracing", "tracing-subscriber", diff --git a/sentry_streams/Cargo.toml b/sentry_streams/Cargo.toml index ed27a32b..def82196 100644 --- a/sentry_streams/Cargo.toml +++ b/sentry_streams/Cargo.toml @@ -6,6 +6,9 @@ edition = "2021" [dependencies] pyo3 = { version = "0.24.0"} serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +base64 = "0.22" +paste = "1.0" sentry_arroyo = "2.19.5" chrono = "0.4.40" tracing = "0.1.40" @@ -19,7 +22,7 @@ log = "0.4.27" [lib] name = "rust_streams" -crate-type = ["cdylib"] +crate-type = ["cdylib", "rlib"] [features] extension-module = ["pyo3/extension-module"] diff --git a/sentry_streams/sentry_streams/adapters/arroyo/rust_arroyo.py b/sentry_streams/sentry_streams/adapters/arroyo/rust_arroyo.py index d5e0cd87..8b109031 100644 --- a/sentry_streams/sentry_streams/adapters/arroyo/rust_arroyo.py +++ b/sentry_streams/sentry_streams/adapters/arroyo/rust_arroyo.py @@ -231,6 +231,22 @@ def map(self, step: Map, stream: Route) -> Route: stream.source in self.__consumers ), f"Stream starting at source {stream.source} not found when adding a map" + # Check if this is a Rust function that should be handled directly + if step.has_rust_function(): + # Handle Rust functions directly without going through the chain system + self.__close_chain(stream) + + route = RustRoute(stream.source, stream.waypoints) + logger.info(f"Adding Rust map: {step.name} to pipeline") + + # For Rust functions, pass the function directly - the Rust runtime will handle it + self.__consumers[stream.source].add_step( + RuntimeOperator.Map(route, step.resolved_function) + ) + + return stream + + # Handle Python functions with the existing chain system step_config: Mapping[str, Any] = self.steps_config.get(step.name, {}) parallelism_config = step_config.get("parallelism") @@ -270,12 +286,22 @@ def filter(self, step: Filter, stream: Route) -> Route: stream.source in self.__consumers ), f"Stream starting at source {stream.source} not found when adding a map" - def filter_msg(msg: Message[Any]) -> bool: - return step.resolved_function(msg) - route = RustRoute(stream.source, stream.waypoints) logger.info(f"Adding filter: {step.name} to pipeline") - self.__consumers[stream.source].add_step(RuntimeOperator.Filter(route, filter_msg)) + + if step.has_rust_function(): + self.__consumers[stream.source].add_step( + RuntimeOperator.Filter(route, step.resolved_function) + ) + else: + # XXX(markus): I don't know what this lambda is for, but i had to + # disable it for the rust path since we need access to methods on + # the callable. This seems like a useless indirection though? + def filter_msg(msg: Message[Any]) -> bool: + return step.resolved_function(msg) + + self.__consumers[stream.source].add_step(RuntimeOperator.Filter(route, filter_msg)) + return stream def reduce( diff --git a/sentry_streams/sentry_streams/examples/rust_simple_map_filter/pipeline.py b/sentry_streams/sentry_streams/examples/rust_simple_map_filter/pipeline.py new file mode 100644 index 00000000..1f4a01ef --- /dev/null +++ b/sentry_streams/sentry_streams/examples/rust_simple_map_filter/pipeline.py @@ -0,0 +1,41 @@ +""" +Rust version of simple_map_filter.py example +""" + +from sentry_kafka_schemas.schema_types.ingest_metrics_v1 import IngestMetric + +from sentry_streams.pipeline import Filter, Map, Parser, Serializer, streaming_source +from sentry_streams.pipeline.chain import StreamSink + +# Import the compiled Rust functions +try: + from metrics_rust_transforms import RustFilterEvents, RustTransformMsg +except ImportError as e: + raise ImportError( + "Rust extension 'metrics_rust_transforms' not found. " + "You must build it first:\n" + " cd rust_transforms\n" + " maturin develop\n" + f"Original error: {e}" + ) from e + +# Same pipeline structure as simple_map_filter.py, but with Rust functions +# that will be called directly without Python overhead +pipeline = ( + streaming_source( + name="myinput", + stream_name="ingest-metrics", + ) + .apply( + "parser", + Parser( + msg_type=IngestMetric, + ), + ) + # This filter will run in native Rust with zero Python overhead + .apply("filter", Filter(function=RustFilterEvents())) + # This transform will run in native Rust with zero Python overhead + .apply("transform", Map(function=RustTransformMsg())) + .apply("serializer", Serializer()) + .sink("mysink", StreamSink(stream_name="transformed-events")) +) diff --git a/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/Cargo.lock b/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/Cargo.lock new file mode 100644 index 00000000..2547b6ce --- /dev/null +++ b/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/Cargo.lock @@ -0,0 +1,2428 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "backtrace" +version = "0.3.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets 0.52.6", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" + +[[package]] +name = "bumpalo" +version = "3.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" + +[[package]] +name = "bytes" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" + +[[package]] +name = "cc" +version = "1.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1599538de2394445747c8cf7935946e3cc27e9625f889d979bfb2aaf569362" +dependencies = [ + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chrono" +version = "0.4.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "cmake" +version = "0.1.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" +dependencies = [ + "cc", +] + +[[package]] +name = "coarsetime" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91849686042de1b41cd81490edc83afbcb0abe5a9b6f2c4114f23ce8cca1bcf4" +dependencies = [ + "libc", + "wasix", + "wasm-bindgen", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "ctrlc" +version = "3.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46f93780a459b7d656ef7f071fe699c4d3d2cb201c4b24d085b6ddc505276e73" +dependencies = [ + "nix", + "windows-sys 0.59.0", +] + +[[package]] +name = "debugid" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" +dependencies = [ + "serde", + "uuid", +] + +[[package]] +name = "deranged" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "pin-utils", +] + +[[package]] +name = "getrandom" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", +] + +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + +[[package]] +name = "h2" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17da50a276f1e01e0ba6c029e47b7100754904ee8a278f886546e98575380785" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hyper" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc2fdfdbff08affe55bb779f33b053aa1fe5dd5b54c257343c17edfa55711bdb" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" + +[[package]] +name = "icu_properties" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "potential_utf", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" + +[[package]] +name = "icu_provider" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" +dependencies = [ + "displaydoc", + "icu_locale_core", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe4cd85333e22411419a0bcae1297d25e58c9443848b11dc6a86fefe8c78a661" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "indoc" +version = "2.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" + +[[package]] +name = "io-uring" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b86e202f00093dcba4275d4636b93ef9dd75d025ae560d2521b45ea28ab49013" +dependencies = [ + "bitflags", + "cfg-if", + "libc", +] + +[[package]] +name = "ipnet" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" + +[[package]] +name = "iri-string" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "js-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.174" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" + +[[package]] +name = "libz-sys" +version = "1.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" + +[[package]] +name = "litemap" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" + +[[package]] +name = "lock_api" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + +[[package]] +name = "memchr" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "metrics_rust_transforms" +version = "0.1.0" +dependencies = [ + "paste", + "pyo3", + "rust_streams", + "serde", + "serde_json", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + +[[package]] +name = "mio" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" +dependencies = [ + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys 0.59.0", +] + +[[package]] +name = "native-tls" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "nix" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_enum" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a973b4e44ce6cad84ce69d797acf9a044532e4184c4f267913d1b546a0727b7a" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77e878c846a8abae00dd069496dbe8751b16ac1c3d6bd2a7283a938e8228f90d" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "openssl" +version = "0.10.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-sys" +version = "0.9.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90096e2e47630d78b7d1c20952dc621f957103f8bc2c8359ec81290d75238571" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + +[[package]] +name = "parking_lot" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets 0.52.6", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "portable-atomic" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" + +[[package]] +name = "potential_utf" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro-crate" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" +dependencies = [ + "toml_edit", +] + +[[package]] +name = "proc-macro2" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5203598f366b11a02b13aa20cab591229ff0a89fd121a308a5df751d5fc9219" +dependencies = [ + "cfg-if", + "indoc", + "libc", + "memoffset", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99636d423fa2ca130fa5acde3059308006d46f98caac629418e53f7ebb1e9999" +dependencies = [ + "once_cell", + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78f9cf92ba9c409279bc3305b5409d90db2d2c22392d443a87df3a1adad59e33" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b999cb1a6ce21f9a6b147dcf1be9ffedf02e0043aec74dc390f3007047cecd9" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "822ece1c7e1012745607d5cf0bcb2874769f0f7cb34c4cde03b9358eb9ef911a" +dependencies = [ + "heck", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.16", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.3", +] + +[[package]] +name = "rdkafka" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14b52c81ac3cac39c9639b95c20452076e74b8d9a71bc6fc4d83407af2ea6fff" +dependencies = [ + "futures-channel", + "futures-util", + "libc", + "log", + "rdkafka-sys", + "serde", + "serde_derive", + "serde_json", + "slab", + "tokio", + "tracing", +] + +[[package]] +name = "rdkafka-sys" +version = "4.9.0+2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5230dca48bc354d718269f3e4353280e188b610f7af7e2fcf54b7a79d5802872" +dependencies = [ + "cmake", + "libc", + "libz-sys", + "num_enum", + "pkg-config", +] + +[[package]] +name = "redox_syscall" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d04b7d0ee6b4a0207a0a7adb104d23ecb0b47d6beae7152d0fa34b692b29fd6" +dependencies = [ + "bitflags", +] + +[[package]] +name = "reqwest" +version = "0.12.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbc931937e6ca3a06e3b6c0aa7841849b160a90351d6ab467a8b9b9959767531" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-tls", + "hyper-util", + "js-sys", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.16", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rust_streams" +version = "0.1.0" +dependencies = [ + "anyhow", + "base64", + "chrono", + "ctrlc", + "log", + "paste", + "pyo3", + "pyo3-build-config", + "rdkafka", + "reqwest", + "sentry_arroyo", + "serde", + "serde_json", + "tokio", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f" + +[[package]] +name = "rustix" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustls" +version = "0.23.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7160e3e10bf4535308537f3c4e1641468cd0e485175d6163087c0393c7d46643" +dependencies = [ + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "schannel" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "sentry-core" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00e9bd2cadaeda3af41e9fa5d14645127d6f6a4aec73da3ae38e477ecafd3682" +dependencies = [ + "rand 0.9.1", + "sentry-types", + "serde", + "serde_json", +] + +[[package]] +name = "sentry-types" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a08e7154abe2cd557f26fd70038452810748aefdf39bc973f674421224b147c1" +dependencies = [ + "debugid", + "hex", + "rand 0.9.1", + "serde", + "serde_json", + "thiserror 2.0.12", + "time", + "url", + "uuid", +] + +[[package]] +name = "sentry_arroyo" +version = "2.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9556aa8820b4e81f28ca6490a9e31d87699f945eee380202d9c88a896205078" +dependencies = [ + "chrono", + "coarsetime", + "once_cell", + "parking_lot", + "rand 0.8.5", + "rdkafka", + "sentry-core", + "serde", + "serde_json", + "thiserror 1.0.69", + "tokio", + "tracing", + "uuid", +] + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04dc19736151f35336d325007ac991178d504a119863a2fcb3758cdb5e52c50d" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "target-lexicon" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" + +[[package]] +name = "tempfile" +version = "3.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" +dependencies = [ + "fastrand", + "getrandom 0.3.3", + "once_cell", + "rustix", + "windows-sys 0.59.0", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +dependencies = [ + "thiserror-impl 2.0.12", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" + +[[package]] +name = "time-macros" +version = "0.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokio" +version = "1.46.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17" +dependencies = [ + "backtrace", + "bytes", + "io-uring", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "slab", + "socket2", + "tokio-macros", + "windows-sys 0.52.0", +] + +[[package]] +name = "tokio-macros" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "toml_datetime", + "winnow", +] + +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "iri-string", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" +dependencies = [ + "nu-ansi-term", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" + +[[package]] +name = "unindent" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d" +dependencies = [ + "getrandom 0.3.3", + "js-sys", + "serde", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + +[[package]] +name = "wasix" +version = "0.12.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1fbb4ef9bbca0c1170e0b00dd28abc9e3b68669821600cad1caaed606583c6d" +dependencies = [ + "wasi 0.11.1+wasi-snapshot-preview1", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" +dependencies = [ + "cfg-if", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + +[[package]] +name = "windows-registry" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b8a9ed28765efc97bbc954883f4e6796c33a06546ebafacbabee9696967499e" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.2", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c66f69fcc9ce11da9966ddb31a40968cad001c5bedeb5c2b82ede4253ab48aef" +dependencies = [ + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + +[[package]] +name = "winnow" +version = "0.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74c7b26e3480b707944fc872477815d29a8e429d2f93a1ce000f5fa84a15cbcd" +dependencies = [ + "memchr", +] + +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags", +] + +[[package]] +name = "writeable" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" + +[[package]] +name = "yoke" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + +[[package]] +name = "zerotrie" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/Cargo.toml b/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/Cargo.toml new file mode 100644 index 00000000..12e8324b --- /dev/null +++ b/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "metrics_rust_transforms" +version = "0.1.0" +edition = "2021" + +[lib] +name = "metrics_rust_transforms" +crate-type = ["cdylib"] + +[dependencies] +pyo3 = { version = "0.24" } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +paste = "1.0" + +# In practice, users would depend on the published sentry_streams crate +# which would re-export the macros +# sentry_streams = "0.1" + +# For this example, we'll include the dependencies directly +[dependencies.rust_streams] +path = "../../../../" # Point to the main rust_streams crate +default-features = false # Disable default features to avoid conflicts diff --git a/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/metrics_rust_transforms.pyi b/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/metrics_rust_transforms.pyi new file mode 100644 index 00000000..0da90090 --- /dev/null +++ b/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/metrics_rust_transforms.pyi @@ -0,0 +1,39 @@ +""" +Type stubs for metrics_rust_transforms Rust extension + +This file provides type information for mypy and other static type checkers. +""" + +from typing import Any + +from sentry_kafka_schemas.schema_types.ingest_metrics_v1 import IngestMetric + +from sentry_streams.pipeline.message import Message +from sentry_streams.pipeline.rust_function_protocol import ( + RustFilterFunction, + RustMapFunction, +) + +class RustFilterEvents(RustFilterFunction[IngestMetric]): + """Rust filter function that accepts IngestMetric and returns bool""" + + def __init__(self) -> None: ... + def __call__(self, msg: Message[IngestMetric]) -> bool: ... + + # Rust-specific methods for runtime detection + def get_rust_function_pointer(self) -> int: ... + def input_type(self) -> str: ... + def output_type(self) -> str: ... + def callback_type(self) -> str: ... + +class RustTransformMsg(RustMapFunction[IngestMetric, Any]): + """Rust map function that accepts IngestMetric and returns JSON Value""" + + def __init__(self) -> None: ... + def __call__(self, msg: Message[IngestMetric]) -> Message[Any]: ... + + # Rust-specific methods for runtime detection + def get_rust_function_pointer(self) -> int: ... + def input_type(self) -> str: ... + def output_type(self) -> str: ... + def callback_type(self) -> str: ... diff --git a/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/py.typed b/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/src/lib.rs b/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/src/lib.rs new file mode 100644 index 00000000..e31cb09b --- /dev/null +++ b/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/src/lib.rs @@ -0,0 +1,65 @@ +use pyo3::prelude::*; +use serde::{Deserialize, Serialize}; + +// Import the macros from rust_streams (the actual crate name) +// In practice: use sentry_streams::{rust_map_function, rust_filter_function}; (when published) +use rust_streams::ffi::Message; +use rust_streams::rust_filter_function; +use rust_streams::rust_map_function; + +/// IngestMetric structure matching the schema from simple_map_filter.py +/// This would normally be imported from sentry_kafka_schemas in a real implementation +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct IngestMetric { + #[serde(rename = "type")] + pub metric_type: String, + pub name: String, + pub value: f64, + pub tags: std::collections::HashMap, + pub timestamp: u64, +} + +// Rust equivalent of filter_events() from simple_map_filter.py +rust_filter_function!(RustFilterEvents, serde_json::Value, |msg: Message< + serde_json::Value, +>| + -> bool { + // Deserialize JSON payload to IngestMetric struct + let payload: IngestMetric = serde_json::from_value(msg.payload).unwrap(); + + // TODO: Fix segfault in Debug formatting - this should not crash even with binary data + // println!("Seen in filter: {:?}", msg); + // Same logic as Python version: return bool(msg.payload["type"] == "c") + payload.metric_type == "c" +}); + +// Rust equivalent of transform_msg() from simple_map_filter.py +rust_map_function!( + RustTransformMsg, + serde_json::Value, + serde_json::Value, // Output as a flexible JSON value like the Python version + |msg: Message| -> Message { + // Deserialize JSON payload to IngestMetric struct + let payload: IngestMetric = serde_json::from_value(msg.payload).unwrap(); + + // TODO: Fix segfault in Debug formatting - this should not crash even with binary data + // println!("Seen in map: {:?}", msg); + // Convert IngestMetric to JSON, then add "transformed": true + // This matches the Python logic: {**msg.payload, "transformed": True} + let mut result = serde_json::to_value(&payload).unwrap(); + + if let Some(obj) = result.as_object_mut() { + obj.insert("transformed".to_string(), serde_json::Value::Bool(true)); + } + + Message::new(result, msg.headers, msg.timestamp, msg.schema) + } +); + +// this makes the Rust functions available to Python +#[pymodule] +fn metrics_rust_transforms(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_class::()?; + m.add_class::()?; + Ok(()) +} diff --git a/sentry_streams/sentry_streams/examples/simple_map_filter.py b/sentry_streams/sentry_streams/examples/simple_map_filter.py index b666669d..4c418ae1 100644 --- a/sentry_streams/sentry_streams/examples/simple_map_filter.py +++ b/sentry_streams/sentry_streams/examples/simple_map_filter.py @@ -1,5 +1,3 @@ -from datetime import datetime - from sentry_kafka_schemas.schema_types.ingest_metrics_v1 import IngestMetric from sentry_streams.examples.transform_metrics import transform_msg @@ -18,13 +16,6 @@ def filter_events(msg: Message[IngestMetric]) -> bool: return bool(msg.payload["type"] == "c") -def generate_files() -> str: - now = datetime.now() - cur_time = now.strftime("%H:%M:%S") - - return f"file_{cur_time}.txt" - - pipeline = streaming_source(name="myinput", stream_name="ingest-metrics") ( diff --git a/sentry_streams/sentry_streams/pipeline/pipeline.py b/sentry_streams/sentry_streams/pipeline/pipeline.py index dc4dcb61..8cde9355 100644 --- a/sentry_streams/sentry_streams/pipeline/pipeline.py +++ b/sentry_streams/sentry_streams/pipeline/pipeline.py @@ -241,6 +241,29 @@ class TransformFunction(ABC, Generic[TransformFuncReturnType]): def resolved_function(self) -> Callable[..., TransformFuncReturnType]: raise NotImplementedError() + def has_rust_function(self) -> bool: + """Check if this function provides a Rust implementation""" + func = self.resolved_function + return ( + hasattr(func, "get_rust_function_pointer") + and hasattr(func, "input_type") + and hasattr(func, "output_type") + and hasattr(func, "callback_type") + ) + + def get_rust_callback_info(self) -> Mapping[str, Any]: + """Get information about the Rust callback function""" + if not self.has_rust_function(): + raise ValueError("Function does not have Rust implementation") + + func = self.resolved_function + return { + "function_ptr": func.get_rust_function_pointer(), # type: ignore[attr-defined] + "input_type": func.input_type(), # type: ignore[attr-defined] + "output_type": func.output_type(), # type: ignore[attr-defined] + "callback_type": func.callback_type(), # type: ignore[attr-defined] + } + @dataclass class TransformStep(WithInput, TransformFunction[TransformFuncReturnType]): @@ -289,6 +312,24 @@ class Map(TransformStep[Any]): # TODO: Allow product to both enable and access # configuration (e.g. a DB that is used as part of Map) + def __post_init__(self) -> None: + """Validate the function after initialization""" + super().__post_init__() + if self.has_rust_function(): + self._validate_rust_function_type("map") + + def _validate_rust_function_type(self, expected_callback_type: str) -> None: + """Validate that the Rust function has the correct type""" + try: + info = self.get_rust_callback_info() + if info["callback_type"] != expected_callback_type: + raise TypeError( + f"Function {self.function} is a {info['callback_type']} function, " + f"but expected {expected_callback_type}" + ) + except Exception as e: + raise TypeError(f"Invalid Rust function {self.function}: {e}") + @dataclass class Filter(TransformStep[bool]): @@ -298,6 +339,30 @@ class Filter(TransformStep[bool]): step_type: StepType = StepType.FILTER + def __post_init__(self) -> None: + """Validate the function after initialization""" + super().__post_init__() + if self.has_rust_function(): + self._validate_rust_function_type("filter") + + def _validate_rust_function_type(self, expected_callback_type: str) -> None: + """Validate that the Rust function has the correct type""" + try: + info = self.get_rust_callback_info() + if info["callback_type"] != expected_callback_type: + raise TypeError( + f"Function {self.function} is a {info['callback_type']} function, " + f"but expected {expected_callback_type}" + ) + # Additional validation for filter: output type should be bool + if expected_callback_type == "filter" and info["output_type"] != "bool": + raise TypeError( + f"Filter function {self.function} should return bool, " + f"but returns {info['output_type']}" + ) + except Exception as e: + raise TypeError(f"Invalid Rust function {self.function}: {e}") + @dataclass class Branch(Step): diff --git a/sentry_streams/sentry_streams/pipeline/rust_function_protocol.py b/sentry_streams/sentry_streams/pipeline/rust_function_protocol.py new file mode 100644 index 00000000..75d00839 --- /dev/null +++ b/sentry_streams/sentry_streams/pipeline/rust_function_protocol.py @@ -0,0 +1,28 @@ +""" +Protocol definitions for Rust functions to integrate with Python's type system +""" + +from typing import Protocol, TypeVar, runtime_checkable + +from sentry_streams.pipeline.message import Message + +TInput = TypeVar("TInput") +TOutput = TypeVar("TOutput") + + +@runtime_checkable +class RustFilterFunction(Protocol[TInput]): + """Protocol for Rust filter functions""" + + def __call__(self, msg: Message[TInput]) -> bool: + """Filter function that returns True to keep the message""" + ... + + +@runtime_checkable +class RustMapFunction(Protocol[TInput, TOutput]): + """Protocol for Rust map functions""" + + def __call__(self, msg: Message[TInput]) -> Message[TOutput]: + """Map function that transforms the message""" + ... diff --git a/sentry_streams/src/ffi.rs b/sentry_streams/src/ffi.rs new file mode 100644 index 00000000..4b48fcaf --- /dev/null +++ b/sentry_streams/src/ffi.rs @@ -0,0 +1,137 @@ +#[allow(unused_imports)] +use serde::{Deserialize, Serialize}; + +/// Generic Message struct for use with Rust callbacks +/// This matches the PyMessage structure but with a generic payload +#[derive(Debug, Clone)] +pub struct Message { + pub payload: T, + pub headers: Vec<(String, Vec)>, + pub timestamp: f64, + pub schema: Option, +} + +impl Message { + pub fn new( + payload: T, + headers: Vec<(String, Vec)>, + timestamp: f64, + schema: Option, + ) -> Self { + Self { + payload, + headers, + timestamp, + schema, + } + } + + /// Transform the message payload while keeping metadata + pub fn map(self, f: F) -> Message + where + F: FnOnce(T) -> U, + { + Message { + payload: f(self.payload), + headers: self.headers, + timestamp: self.timestamp, + schema: self.schema, + } + } +} + +/// Convert a boxed message to an opaque pointer for FFI +pub fn message_to_ptr(msg: Message) -> *const Message { + Box::into_raw(Box::new(msg)) +} + +/// Convert an opaque pointer back to a boxed message for FFI +pub unsafe fn ptr_to_message(ptr: *const Message) -> Message { + *Box::from_raw(ptr as *mut Message) +} + +/// Free a message pointer (used for cleanup) +pub unsafe fn free_message_ptr(ptr: *const Message) { + if !ptr.is_null() { + let _ = Box::from_raw(ptr as *mut Message); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde::{Deserialize, Serialize}; + + #[derive(Serialize, Deserialize, Debug, PartialEq, Clone)] + struct TestPayload { + id: u64, + name: String, + } + + #[test] + fn test_message_creation() { + let payload = TestPayload { + id: 42, + name: "test".to_string(), + }; + + let headers = vec![("content-type".to_string(), b"application/json".to_vec())]; + + let msg = Message::new( + payload.clone(), + headers.clone(), + 1234567890.5, + Some("test_schema".to_string()), + ); + + assert_eq!(msg.payload.id, 42); + assert_eq!(msg.payload.name, "test"); + assert_eq!(msg.headers, headers); + assert_eq!(msg.timestamp, 1234567890.5); + assert_eq!(msg.schema, Some("test_schema".to_string())); + } + + #[test] + fn test_message_map() { + let original = Message::new( + TestPayload { + id: 1, + name: "input".to_string(), + }, + vec![], + 0.0, + None, + ); + + let transformed = original.map(|payload| TestPayload { + id: payload.id * 2, + name: format!("transformed_{}", payload.name), + }); + + assert_eq!(transformed.payload.id, 2); + assert_eq!(transformed.payload.name, "transformed_input"); + } + + #[test] + fn test_ptr_roundtrip() { + let original = Message::new( + TestPayload { + id: 99, + name: "ptr_test".to_string(), + }, + vec![("header".to_string(), b"value".to_vec())], + 123.456, + Some("schema".to_string()), + ); + + // Convert to pointer and back + let ptr = message_to_ptr(original.clone()); + let recovered = unsafe { ptr_to_message(ptr) }; + + assert_eq!(original.payload.id, recovered.payload.id); + assert_eq!(original.payload.name, recovered.payload.name); + assert_eq!(original.headers, recovered.headers); + assert_eq!(original.timestamp, recovered.timestamp); + assert_eq!(original.schema, recovered.schema); + } +} diff --git a/sentry_streams/src/lib.rs b/sentry_streams/src/lib.rs index 88088bcd..be555cc4 100644 --- a/sentry_streams/src/lib.rs +++ b/sentry_streams/src/lib.rs @@ -3,9 +3,11 @@ mod broadcaster; mod callers; mod committable; mod consumer; +pub mod ffi; mod filter_step; mod gcs_writer; mod kafka_config; +pub mod macros; mod messages; mod operators; mod python_operator; @@ -17,6 +19,9 @@ mod transformer; mod utils; mod watermark; +// Re-export macros so they can be used by external crates +pub use macros::*; + #[cfg(test)] mod fake_strategy; #[cfg(test)] diff --git a/sentry_streams/src/macros.rs b/sentry_streams/src/macros.rs new file mode 100644 index 00000000..25066190 --- /dev/null +++ b/sentry_streams/src/macros.rs @@ -0,0 +1,203 @@ +/// Base trait for all Rust callback functions that can be called from the streaming pipeline +pub trait RustCallback: Send + Sync { + /// Get the raw function pointer for direct Rust calls + fn get_rust_function_pointer(&self) -> usize; + + /// Get the input type name for type validation + fn input_type(&self) -> &'static str; + + /// Get the output type name for type validation + fn output_type(&self) -> &'static str; + + /// Get the callback type ("map", "filter", "reduce", etc.) + fn callback_type(&self) -> &'static str; +} + +/// Macro to create a Rust map function that can be called from Python +/// Usage: rust_map_function!(MyFunction, InputType, OutputType, |msg: Message| -> Message { ... }); +#[macro_export] +macro_rules! rust_map_function { + ($name:ident, $input_type:ty, $output_type:ty, $transform_fn:expr) => { + #[pyo3::pyclass] + pub struct $name { + rust_fn_ptr: usize, + } + + #[pyo3::pymethods] + impl $name { + #[new] + pub fn new() -> Self { + Self { + rust_fn_ptr: Self::get_rust_fn_ptr(), + } + } + + #[pyo3(name = "__call__")] + pub fn call(&self, _msg: pyo3::Py) -> pyo3::PyResult> { + // Python fallback - not implemented for performance reasons + Err(pyo3::PyErr::new::( + "This function should be called directly from Rust for performance" + )) + } + + pub fn get_rust_function_pointer(&self) -> usize { + self.rust_fn_ptr + } + + pub fn input_type(&self) -> &'static str { + std::any::type_name::<$input_type>() + } + + pub fn output_type(&self) -> &'static str { + std::any::type_name::<$output_type>() + } + + pub fn callback_type(&self) -> &'static str { + "map" + } + } + + impl $crate::macros::RustCallback for $name { + fn get_rust_function_pointer(&self) -> usize { + self.rust_fn_ptr + } + + fn input_type(&self) -> &'static str { + std::any::type_name::<$input_type>() + } + + fn output_type(&self) -> &'static str { + std::any::type_name::<$output_type>() + } + + fn callback_type(&self) -> &'static str { + "map" + } + } + + // The actual Rust implementation function (unique name per macro invocation) + paste::paste! { + extern "C" fn []( + input: *const $crate::ffi::Message<$input_type>, + ) -> *const $crate::ffi::Message<$output_type> { + let input_msg = unsafe { + $crate::ffi::ptr_to_message(input) + }; + + let transform_fn: fn($crate::ffi::Message<$input_type>) -> $crate::ffi::Message<$output_type> = $transform_fn; + let output_msg = transform_fn(input_msg); + + $crate::ffi::message_to_ptr(output_msg) + } + } + + impl $name { + fn get_rust_fn_ptr() -> usize { + paste::paste! { + [] as usize + } + } + } + }; +} + +/// Macro to create a Rust filter function that can be called from Python +/// Usage: rust_filter_function!(MyFilter, InputType, |msg: Message| -> bool { ... }); +#[macro_export] +macro_rules! rust_filter_function { + ($name:ident, $input_type:ty, $filter_fn:expr) => { + #[pyo3::pyclass] + pub struct $name { + rust_fn_ptr: usize, + } + + #[pyo3::pymethods] + impl $name { + #[new] + pub fn new() -> Self { + Self { + rust_fn_ptr: Self::get_rust_fn_ptr(), + } + } + + #[pyo3(name = "__call__")] + pub fn call(&self, _msg: pyo3::Py) -> pyo3::PyResult { + // Python fallback - not implemented for performance reasons + Err( + pyo3::PyErr::new::( + "This function should be called directly from Rust for performance", + ), + ) + } + + pub fn get_rust_function_pointer(&self) -> usize { + self.rust_fn_ptr + } + + pub fn input_type(&self) -> &'static str { + std::any::type_name::<$input_type>() + } + + pub fn output_type(&self) -> &'static str { + "bool" + } + + pub fn callback_type(&self) -> &'static str { + "filter" + } + } + + impl $crate::macros::RustCallback for $name { + fn get_rust_function_pointer(&self) -> usize { + self.rust_fn_ptr + } + + fn input_type(&self) -> &'static str { + std::any::type_name::<$input_type>() + } + + fn output_type(&self) -> &'static str { + "bool" + } + + fn callback_type(&self) -> &'static str { + "filter" + } + } + + // The actual Rust implementation function (unique name per macro invocation) + paste::paste! { + extern "C" fn []( + input: *const $crate::ffi::Message<$input_type>, + ) -> bool { + let input_msg = unsafe { + $crate::ffi::ptr_to_message(input) + }; + + let filter_fn: fn($crate::ffi::Message<$input_type>) -> bool = $filter_fn; + filter_fn(input_msg) + } + } + + impl $name { + fn get_rust_fn_ptr() -> usize { + paste::paste! { + [] as usize + } + } + } + }; +} + +/// Generic macro for creating any type of Rust callback function +/// Usage: rust_callback_function!(map, MyFunction, InputType, OutputType, |msg| { ... }); +#[macro_export] +macro_rules! rust_callback_function { + (map, $name:ident, $input_type:ty, $output_type:ty, $transform_fn:expr) => { + $crate::rust_map_function!($name, $input_type, $output_type, $transform_fn); + }; + + (filter, $name:ident, $input_type:ty, $filter_fn:expr) => { + $crate::rust_filter_function!($name, $input_type, $filter_fn); + }; +} diff --git a/sentry_streams/src/operators.rs b/sentry_streams/src/operators.rs index 0dc61cf7..5d430e8b 100644 --- a/sentry_streams/src/operators.rs +++ b/sentry_streams/src/operators.rs @@ -5,7 +5,7 @@ use crate::routers::build_router; use crate::routes::{Route, RoutedValue}; use crate::sinks::StreamSink; use crate::store_sinks::GCSSink; -use crate::transformer::{build_filter, build_map}; +use crate::transformer::{build_filter, build_map, build_rust_filter, build_rust_map}; use crate::utils::traced_with_gil; use pyo3::prelude::*; use sentry_arroyo::backends::kafka::producer::KafkaProducer; @@ -86,12 +86,56 @@ pub fn build( ) -> Box> { match step.get() { RuntimeOperator::Map { function, route } => { - let func_ref = traced_with_gil!(|py| function.clone_ref(py)); - build_map(route, func_ref, next) + // Check if this is a Rust function + let is_rust_function = traced_with_gil!(|py| { + function + .bind(py) + .hasattr("get_rust_function_pointer") + .unwrap_or(false) + }); + + if is_rust_function { + // Get the raw function pointer and call Rust directly + let rust_fn_ptr = traced_with_gil!(|py| { + function + .bind(py) + .call_method0("get_rust_function_pointer") + .unwrap() + .extract::() + .unwrap() + }); + build_rust_map(route, rust_fn_ptr, next) + } else { + // Use existing Python path + let func_ref = traced_with_gil!(|py| function.clone_ref(py)); + build_map(route, func_ref, next) + } } RuntimeOperator::Filter { function, route } => { - let func_ref = traced_with_gil!(|py| { function.clone_ref(py) }); - build_filter(route, func_ref, next) + // Check if this is a Rust function + let is_rust_function = traced_with_gil!(|py| { + function + .bind(py) + .hasattr("get_rust_function_pointer") + .unwrap_or(false) + }); + + if is_rust_function { + // Get the raw function pointer and call Rust directly + let rust_fn_ptr = traced_with_gil!(|py| { + function + .bind(py) + .call_method0("get_rust_function_pointer") + .unwrap() + .extract::() + .unwrap() + }); + build_rust_filter(route, rust_fn_ptr, next) + } else { + // Use existing Python path + let func_ref = traced_with_gil!(|py| { function.clone_ref(py) }); + build_filter(route, func_ref, next) + } } RuntimeOperator::StreamSink { route, diff --git a/sentry_streams/src/transformer.rs b/sentry_streams/src/transformer.rs index 42102a65..370c2946 100644 --- a/sentry_streams/src/transformer.rs +++ b/sentry_streams/src/transformer.rs @@ -65,6 +65,234 @@ pub fn build_filter( Box::new(Filter::new(callable, next, copied_route)) } +/// Creates a Rust-native map transformation strategy that calls a Rust function directly +/// without Python GIL overhead. The function pointer should point to a function with +/// signature: extern "C" fn(*const Message) -> *const Message +pub fn build_rust_map( + route: &Route, + function_ptr: usize, + next: Box>, +) -> Box> { + let copied_route = route.clone(); + + let mapper = move |message: Message| { + if message.payload().route != copied_route { + return Ok(message); + } + + let RoutedValuePayload::PyStreamingMessage(ref py_streaming_msg) = + message.payload().payload + else { + return Ok(message); + }; + + let route = message.payload().route.clone(); + + // Convert Python message to our FFI Message format + let rust_msg = traced_with_gil!(|py| { + let py_any: Py = py_streaming_msg.into(); + convert_py_message_to_rust(py, &py_any) + }); + + let input_ptr = crate::ffi::message_to_ptr(rust_msg); + + // Cast function pointer and call it + let rust_fn: extern "C" fn( + *const crate::ffi::Message, + ) -> *const crate::ffi::Message = + unsafe { std::mem::transmute(function_ptr) }; + + let output_ptr = rust_fn(input_ptr); + let output_msg = unsafe { crate::ffi::ptr_to_message(output_ptr) }; + + // Convert result back to Python message + let py_result = traced_with_gil!(|py| { convert_rust_message_to_py(py, output_msg) }); + + Ok(message.replace(RoutedValue { + route, + payload: RoutedValuePayload::PyStreamingMessage(py_result.into()), + })) + }; + + Box::new(RunTask::new(mapper, next)) +} + +/// Creates a Rust-native filter strategy that calls a Rust function directly +/// without Python GIL overhead. The function pointer should point to a function with +/// signature: extern "C" fn(*const Message) -> bool +pub fn build_rust_filter( + route: &Route, + function_ptr: usize, + next: Box>, +) -> Box> { + let copied_route = route.clone(); + + // Create a custom Rust filter that follows the same pattern as the Python one + Box::new(RustFilter::new(function_ptr, next, copied_route)) +} + +/// A custom filter strategy that uses Rust functions directly +pub struct RustFilter { + pub function_ptr: usize, + pub next_step: Box>, + pub route: Route, +} + +impl RustFilter { + pub fn new( + function_ptr: usize, + next_step: Box>, + route: Route, + ) -> Self { + Self { + function_ptr, + next_step, + route, + } + } +} + +impl ProcessingStrategy for RustFilter { + fn poll( + &mut self, + ) -> Result< + Option, + sentry_arroyo::processing::strategies::StrategyError, + > { + self.next_step.poll() + } + + fn submit(&mut self, message: Message) -> Result<(), SubmitError> { + // Handle messages for different routes and watermarks like the Python filter + if self.route != message.payload().route || message.payload().payload.is_watermark_msg() { + return self.next_step.submit(message); + } + + let RoutedValuePayload::PyStreamingMessage(ref py_streaming_msg) = + message.payload().payload + else { + unreachable!("Watermark message trying to be passed to rust filter function.") + }; + + // Convert Python message to our FFI Message format + let rust_msg = traced_with_gil!(|py| { + let py_any: Py = py_streaming_msg.into(); + convert_py_message_to_rust(py, &py_any) + }); + + let input_ptr = crate::ffi::message_to_ptr(rust_msg); + + // Cast function pointer and call it + let rust_fn: extern "C" fn(*const crate::ffi::Message) -> bool = + unsafe { std::mem::transmute(self.function_ptr) }; + + let result = rust_fn(input_ptr); + + // Follow the same pattern as Python filter: submit if true, drop if false + if result { + self.next_step.submit(message) + } else { + Ok(()) // Drop the message + } + } + + fn terminate(&mut self) { + self.next_step.terminate() + } + + fn join( + &mut self, + timeout: Option, + ) -> Result< + Option, + sentry_arroyo::processing::strategies::StrategyError, + > { + self.next_step.join(timeout)?; + Ok(None) + } +} + +/// Convert a Python streaming message to our generic Rust Message format +fn convert_py_message_to_rust( + py: Python, + py_msg: &Py, +) -> crate::ffi::Message { + // Extract payload as serde_json::Value for safe deserialization + let payload_obj = py_msg.bind(py).getattr("payload").unwrap(); + let payload_json = py + .import("json") + .unwrap() + .getattr("dumps") + .unwrap() + .call1((payload_obj,)) + .unwrap() + .extract::() + .unwrap(); + + // Parse JSON into serde_json::Value for safe handling + let payload_value: serde_json::Value = serde_json::from_str(&payload_json).unwrap(); + + // Extract headers + let headers_py: Vec<(String, Vec)> = py_msg + .bind(py) + .getattr("headers") + .unwrap() + .extract() + .unwrap(); + + // Extract timestamp + let timestamp: f64 = py_msg + .bind(py) + .getattr("timestamp") + .unwrap() + .extract() + .unwrap(); + + // Extract schema + let schema: Option = py_msg + .bind(py) + .getattr("schema") + .unwrap() + .extract() + .unwrap(); + + crate::ffi::Message::new(payload_value, headers_py, timestamp, schema) +} + +/// Convert our generic Rust Message format back to a Python streaming message +fn convert_rust_message_to_py( + py: Python, + rust_msg: crate::ffi::Message, +) -> Py { + // Convert serde_json::Value back to JSON string, then to Python object + let payload_json = serde_json::to_string(&rust_msg.payload).unwrap(); + let payload_obj = py + .import("json") + .unwrap() + .getattr("loads") + .unwrap() + .call1((payload_json,)) + .unwrap(); + + // Create a new Python message with the transformed payload + // Use PyAnyMessage directly since that's what PyStreamingMessage conversion expects + let py_msg_class = py + .import("sentry_streams.rust_streams") + .unwrap() + .getattr("PyAnyMessage") + .unwrap(); + + py_msg_class + .call1(( + payload_obj, + rust_msg.headers, + rust_msg.timestamp, + rust_msg.schema, + )) + .unwrap() + .unbind() +} + #[cfg(test)] mod tests { use super::*; diff --git a/sentry_streams/tests/rust_test_functions/Cargo.lock b/sentry_streams/tests/rust_test_functions/Cargo.lock new file mode 100644 index 00000000..75bc7dfa --- /dev/null +++ b/sentry_streams/tests/rust_test_functions/Cargo.lock @@ -0,0 +1,2438 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "backtrace" +version = "0.3.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets 0.52.6", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" + +[[package]] +name = "bumpalo" +version = "3.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" + +[[package]] +name = "bytes" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" + +[[package]] +name = "cc" +version = "1.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1599538de2394445747c8cf7935946e3cc27e9625f889d979bfb2aaf569362" +dependencies = [ + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chrono" +version = "0.4.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "cmake" +version = "0.1.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" +dependencies = [ + "cc", +] + +[[package]] +name = "coarsetime" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91849686042de1b41cd81490edc83afbcb0abe5a9b6f2c4114f23ce8cca1bcf4" +dependencies = [ + "libc", + "wasix", + "wasm-bindgen", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "ctrlc" +version = "3.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46f93780a459b7d656ef7f071fe699c4d3d2cb201c4b24d085b6ddc505276e73" +dependencies = [ + "nix", + "windows-sys 0.59.0", +] + +[[package]] +name = "debugid" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" +dependencies = [ + "serde", + "uuid", +] + +[[package]] +name = "deranged" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "pin-utils", +] + +[[package]] +name = "getrandom" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", +] + +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + +[[package]] +name = "h2" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17da50a276f1e01e0ba6c029e47b7100754904ee8a278f886546e98575380785" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hyper" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc2fdfdbff08affe55bb779f33b053aa1fe5dd5b54c257343c17edfa55711bdb" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" + +[[package]] +name = "icu_properties" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "potential_utf", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" + +[[package]] +name = "icu_provider" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" +dependencies = [ + "displaydoc", + "icu_locale_core", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe4cd85333e22411419a0bcae1297d25e58c9443848b11dc6a86fefe8c78a661" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "indoc" +version = "2.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" + +[[package]] +name = "io-uring" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b86e202f00093dcba4275d4636b93ef9dd75d025ae560d2521b45ea28ab49013" +dependencies = [ + "bitflags", + "cfg-if", + "libc", +] + +[[package]] +name = "ipnet" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" + +[[package]] +name = "iri-string" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "js-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.174" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" + +[[package]] +name = "libz-sys" +version = "1.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" + +[[package]] +name = "litemap" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" + +[[package]] +name = "lock_api" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + +[[package]] +name = "memchr" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + +[[package]] +name = "mio" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" +dependencies = [ + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys 0.59.0", +] + +[[package]] +name = "native-tls" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "nix" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_enum" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a973b4e44ce6cad84ce69d797acf9a044532e4184c4f267913d1b546a0727b7a" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77e878c846a8abae00dd069496dbe8751b16ac1c3d6bd2a7283a938e8228f90d" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "openssl" +version = "0.10.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-sys" +version = "0.9.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90096e2e47630d78b7d1c20952dc621f957103f8bc2c8359ec81290d75238571" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + +[[package]] +name = "parking_lot" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets 0.52.6", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "portable-atomic" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" + +[[package]] +name = "potential_utf" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro-crate" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" +dependencies = [ + "toml_edit", +] + +[[package]] +name = "proc-macro2" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5203598f366b11a02b13aa20cab591229ff0a89fd121a308a5df751d5fc9219" +dependencies = [ + "cfg-if", + "indoc", + "libc", + "memoffset", + "once_cell", + "portable-atomic", + "pyo3-build-config 0.24.2", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99636d423fa2ca130fa5acde3059308006d46f98caac629418e53f7ebb1e9999" +dependencies = [ + "once_cell", + "target-lexicon", +] + +[[package]] +name = "pyo3-build-config" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "458eb0c55e7ece017adeba38f2248ff3ac615e53660d7c71a238d7d2a01c7598" +dependencies = [ + "once_cell", + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78f9cf92ba9c409279bc3305b5409d90db2d2c22392d443a87df3a1adad59e33" +dependencies = [ + "libc", + "pyo3-build-config 0.24.2", +] + +[[package]] +name = "pyo3-macros" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b999cb1a6ce21f9a6b147dcf1be9ffedf02e0043aec74dc390f3007047cecd9" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "822ece1c7e1012745607d5cf0bcb2874769f0f7cb34c4cde03b9358eb9ef911a" +dependencies = [ + "heck", + "proc-macro2", + "pyo3-build-config 0.24.2", + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.16", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.3", +] + +[[package]] +name = "rdkafka" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14b52c81ac3cac39c9639b95c20452076e74b8d9a71bc6fc4d83407af2ea6fff" +dependencies = [ + "futures-channel", + "futures-util", + "libc", + "log", + "rdkafka-sys", + "serde", + "serde_derive", + "serde_json", + "slab", + "tokio", + "tracing", +] + +[[package]] +name = "rdkafka-sys" +version = "4.9.0+2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5230dca48bc354d718269f3e4353280e188b610f7af7e2fcf54b7a79d5802872" +dependencies = [ + "cmake", + "libc", + "libz-sys", + "num_enum", + "pkg-config", +] + +[[package]] +name = "redox_syscall" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d04b7d0ee6b4a0207a0a7adb104d23ecb0b47d6beae7152d0fa34b692b29fd6" +dependencies = [ + "bitflags", +] + +[[package]] +name = "reqwest" +version = "0.12.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbc931937e6ca3a06e3b6c0aa7841849b160a90351d6ab467a8b9b9959767531" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-tls", + "hyper-util", + "js-sys", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.16", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rust_streams" +version = "0.1.0" +dependencies = [ + "anyhow", + "base64", + "chrono", + "ctrlc", + "log", + "paste", + "pyo3", + "pyo3-build-config 0.25.1", + "rdkafka", + "reqwest", + "sentry_arroyo", + "serde", + "serde_json", + "tokio", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "rust_test_functions" +version = "0.1.0" +dependencies = [ + "paste", + "pyo3", + "rust_streams", + "serde", + "serde_json", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f" + +[[package]] +name = "rustix" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustls" +version = "0.23.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7160e3e10bf4535308537f3c4e1641468cd0e485175d6163087c0393c7d46643" +dependencies = [ + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "schannel" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "sentry-core" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00e9bd2cadaeda3af41e9fa5d14645127d6f6a4aec73da3ae38e477ecafd3682" +dependencies = [ + "rand 0.9.1", + "sentry-types", + "serde", + "serde_json", +] + +[[package]] +name = "sentry-types" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a08e7154abe2cd557f26fd70038452810748aefdf39bc973f674421224b147c1" +dependencies = [ + "debugid", + "hex", + "rand 0.9.1", + "serde", + "serde_json", + "thiserror 2.0.12", + "time", + "url", + "uuid", +] + +[[package]] +name = "sentry_arroyo" +version = "2.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9556aa8820b4e81f28ca6490a9e31d87699f945eee380202d9c88a896205078" +dependencies = [ + "chrono", + "coarsetime", + "once_cell", + "parking_lot", + "rand 0.8.5", + "rdkafka", + "sentry-core", + "serde", + "serde_json", + "thiserror 1.0.69", + "tokio", + "tracing", + "uuid", +] + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04dc19736151f35336d325007ac991178d504a119863a2fcb3758cdb5e52c50d" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "target-lexicon" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" + +[[package]] +name = "tempfile" +version = "3.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" +dependencies = [ + "fastrand", + "getrandom 0.3.3", + "once_cell", + "rustix", + "windows-sys 0.59.0", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +dependencies = [ + "thiserror-impl 2.0.12", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" + +[[package]] +name = "time-macros" +version = "0.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokio" +version = "1.46.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17" +dependencies = [ + "backtrace", + "bytes", + "io-uring", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "slab", + "socket2", + "tokio-macros", + "windows-sys 0.52.0", +] + +[[package]] +name = "tokio-macros" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "toml_datetime", + "winnow", +] + +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "iri-string", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" +dependencies = [ + "nu-ansi-term", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" + +[[package]] +name = "unindent" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d" +dependencies = [ + "getrandom 0.3.3", + "js-sys", + "serde", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + +[[package]] +name = "wasix" +version = "0.12.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1fbb4ef9bbca0c1170e0b00dd28abc9e3b68669821600cad1caaed606583c6d" +dependencies = [ + "wasi 0.11.1+wasi-snapshot-preview1", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" +dependencies = [ + "cfg-if", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + +[[package]] +name = "windows-registry" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b8a9ed28765efc97bbc954883f4e6796c33a06546ebafacbabee9696967499e" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.2", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c66f69fcc9ce11da9966ddb31a40968cad001c5bedeb5c2b82ede4253ab48aef" +dependencies = [ + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + +[[package]] +name = "winnow" +version = "0.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74c7b26e3480b707944fc872477815d29a8e429d2f93a1ce000f5fa84a15cbcd" +dependencies = [ + "memchr", +] + +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags", +] + +[[package]] +name = "writeable" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" + +[[package]] +name = "yoke" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + +[[package]] +name = "zerotrie" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/sentry_streams/tests/rust_test_functions/Cargo.toml b/sentry_streams/tests/rust_test_functions/Cargo.toml new file mode 100644 index 00000000..1beca45c --- /dev/null +++ b/sentry_streams/tests/rust_test_functions/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "rust_test_functions" +version = "0.1.0" +edition = "2021" + +[lib] +name = "rust_test_functions" +crate-type = ["cdylib"] + +[dependencies] +pyo3 = { version = "0.24" } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +paste = "1.0" + +# Depend on the main rust_streams crate for macros +[dependencies.rust_streams] +path = "../../" +default-features = false diff --git a/sentry_streams/tests/rust_test_functions/py.typed b/sentry_streams/tests/rust_test_functions/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/sentry_streams/tests/rust_test_functions/rust_test_functions.pyi b/sentry_streams/tests/rust_test_functions/rust_test_functions.pyi new file mode 100644 index 00000000..9af9e58a --- /dev/null +++ b/sentry_streams/tests/rust_test_functions/rust_test_functions.pyi @@ -0,0 +1,19 @@ +from sentry_streams.pipeline.message import Message +from sentry_streams.pipeline.rust_function_protocol import ( + RustFilterFunction, + RustMapFunction, +) + +class TestMessage: ... + +class TestFilterCorrect(RustFilterFunction[TestMessage]): + def __call__(self, msg: Message[TestMessage]) -> bool: ... + +class TestMapCorrect(RustMapFunction[TestMessage, str]): + def __call__(self, msg: Message[TestMessage]) -> Message[str]: ... + +class TestMapWrongType(RustMapFunction[bool, str]): + def __call__(self, msg: Message[bool]) -> Message[str]: ... + +class TestMapString(RustMapFunction[str, int]): + def __call__(self, msg: Message[str]) -> Message[int]: ... diff --git a/sentry_streams/tests/rust_test_functions/src/lib.rs b/sentry_streams/tests/rust_test_functions/src/lib.rs new file mode 100644 index 00000000..205ab53d --- /dev/null +++ b/sentry_streams/tests/rust_test_functions/src/lib.rs @@ -0,0 +1,77 @@ +use pyo3::prelude::*; +use serde::{Deserialize, Serialize}; + +// Import the macros from rust_streams +use rust_streams::ffi::Message; +use rust_streams::rust_filter_function; +use rust_streams::rust_map_function; + +/// Test data structure for type validation tests +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct TestMessage { + pub id: u64, + pub content: String, +} + +rust_filter_function!(TestFilterCorrect, TestMessage, |msg: Message< + TestMessage, +>| + -> bool { + msg.payload.id > 0 +}); + +rust_map_function!(TestMapCorrect, TestMessage, String, |msg: Message< + TestMessage, +>| + -> Message { + Message::new( + format!("Processed: {}", msg.payload.content), + msg.headers, + msg.timestamp, + msg.schema, + ) +}); + +// Wrong type map function - accepts bool instead of TestMessage +rust_map_function!( + TestMapWrongType, + bool, // This expects bool, but will get TestMessage in the test + String, + |msg: Message| -> Message { + Message::new( + if msg.payload { + "true".to_string() + } else { + "false".to_string() + }, + msg.headers, + msg.timestamp, + msg.schema, + ) + } +); + +// Map that accepts String (for testing chained operations) +rust_map_function!( + TestMapString, + String, + u64, + |msg: Message| -> Message { + Message::new( + msg.payload.len() as u64, + msg.headers, + msg.timestamp, + msg.schema, + ) + } +); + +/// PyO3 module definition +#[pymodule] +fn rust_test_functions(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + Ok(()) +} diff --git a/sentry_streams/tests/test_mypy_integration.py b/sentry_streams/tests/test_mypy_integration.py new file mode 100644 index 00000000..b1b73b74 --- /dev/null +++ b/sentry_streams/tests/test_mypy_integration.py @@ -0,0 +1,148 @@ +""" +Test mypy integration for Rust functions + +This test verifies that mypy can detect type mismatches in pipelines using Rust functions. +""" + +import subprocess +import sys +import tempfile +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="module") +def test_rust_extension(): # type: ignore[no-untyped-def] + """Build the test Rust extension before running tests""" + test_crate_dir = Path(__file__).parent / "rust_test_functions" + + # Build the extension + result = subprocess.run( + ["maturin", "develop"], cwd=test_crate_dir, capture_output=True, text=True + ) + + if result.returncode != 0: + pytest.fail(f"Failed to build test Rust extension: {result.stderr}") + + # Import and return the module + try: + import rust_test_functions # type: ignore[import-not-found] + except ImportError as e: + pytest.fail(f"Failed to import test Rust extension: {e}") + + yield rust_test_functions + + # Try to uninstall the test extension (best effort) + try: + subprocess.run( + ["uv", "pip", "uninstall", "rust-test-functions"], + capture_output=True, + ) + except Exception: + pass + + +def test_mypy_detects_correct_pipeline(test_rust_extension) -> None: # type: ignore[no-untyped-def] + """Test that mypy accepts a correctly typed pipeline""" + + # Create a test file with correct types + correct_code = """ +from sentry_streams.pipeline import Filter, Map, Parser, streaming_source +from sentry_streams.pipeline.chain import StreamSink +from rust_test_functions import TestFilterCorrect, TestMapCorrect, TestMapString, TestMessage + +def create_correct_pipeline(): + return ( + streaming_source("input", "test-stream") + .apply("parser", Parser(msg_type=TestMessage)) # bytes -> TestMessage + .apply("filter", Filter(function=TestFilterCorrect())) # TestMessage -> TestMessage + .apply("transform", Map(function=TestMapCorrect())) # TestMessage -> String + .apply("length", Map(function=TestMapString())) # String -> u64 + .sink("output", StreamSink("output-stream")) + ) +""" + + # Write to temp file and run mypy + with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f: + f.write(correct_code) + temp_file = f.name + + result = subprocess.run( + [sys.executable, "-m", "mypy", temp_file, "--show-error-codes", "--check-untyped-defs"], + capture_output=True, + text=True, + ) + + assert result.stdout == "Success: no issues found in 1 source file\n" + assert not result.stderr + assert result.returncode == 0 + + +def test_mypy_detects_type_mismatch(test_rust_extension) -> None: # type: ignore[no-untyped-def] + """Test that mypy detects type mismatches in pipeline definitions""" + + wrong_code = """ +from sentry_streams.pipeline import Filter, Map, Parser, streaming_source +from sentry_streams.pipeline.chain import StreamSink +from rust_test_functions import TestFilterCorrect, TestMapCorrect, TestMapWrongType, TestMessage + +# expect failure: TestMapWrongType expects Message[bool] but gets Message[TestMessage] +def create_wrong_pipeline(): + return ( + streaming_source("input", "test-stream") + .apply("parser", Parser(msg_type=TestMessage)) # bytes -> TestMessage + .apply("filter", Filter(function=TestFilterCorrect())) # TestMessage -> TestMessage + .apply("wrong", Map(function=TestMapWrongType())) # Expects Message[bool], gets Message[TestMessage]! + .sink("output", StreamSink("output-stream")) + ) +""" + + # Write to temp file and run mypy + with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f: + f.write(wrong_code) + temp_file = f.name + + try: + result = subprocess.run( + [sys.executable, "-m", "mypy", temp_file, "--show-error-codes", "--check-untyped-defs"], + capture_output=True, + text=True, + ) + + # Should detect type mismatch + assert result.returncode > 0 + assert ( + 'Argument "function" to "Map" has incompatible type "TestMapWrongType"' in result.stdout + ) + + finally: + Path(temp_file).unlink() + + +def test_rust_functions_have_proper_types(test_rust_extension) -> None: # type: ignore[no-untyped-def] + """Test that Rust functions expose proper type information""" + + filter_func = test_rust_extension.TestFilterCorrect() + map_func = test_rust_extension.TestMapCorrect() + wrong_func = test_rust_extension.TestMapWrongType() + + # Check that they have the required methods for type checking + assert hasattr(filter_func, "__call__") + assert hasattr(map_func, "__call__") + assert hasattr(wrong_func, "__call__") + + # Check that they're callable + assert callable(filter_func) + assert callable(map_func) + assert callable(wrong_func) + + # Check protocol compliance + from sentry_streams.pipeline.rust_function_protocol import ( + RustFilterFunction, + RustMapFunction, + ) + + assert isinstance(filter_func, RustFilterFunction) + assert isinstance(map_func, RustMapFunction) + assert isinstance(wrong_func, RustMapFunction) diff --git a/sentry_streams/tests/test_true.py b/sentry_streams/tests/test_true.py deleted file mode 100644 index 6170ef12..00000000 --- a/sentry_streams/tests/test_true.py +++ /dev/null @@ -1,5 +0,0 @@ -# TODO: This is here to make CI not fail as there are -# no tests at the time of writing. -# Actually write tests and remove this. -def test_always_passes() -> None: - assert True From 0314b6e7cf960d2fe967764bd537f7d43ca734a5 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Wed, 9 Jul 2025 19:27:23 +0200 Subject: [PATCH 02/16] fix typing???? --- .../rust_simple_map_filter/pipeline.py | 25 ++++++++++--------- .../sentry_streams/pipeline/pipeline.py | 2 -- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/sentry_streams/sentry_streams/examples/rust_simple_map_filter/pipeline.py b/sentry_streams/sentry_streams/examples/rust_simple_map_filter/pipeline.py index 1f4a01ef..ba322891 100644 --- a/sentry_streams/sentry_streams/examples/rust_simple_map_filter/pipeline.py +++ b/sentry_streams/sentry_streams/examples/rust_simple_map_filter/pipeline.py @@ -4,8 +4,14 @@ from sentry_kafka_schemas.schema_types.ingest_metrics_v1 import IngestMetric -from sentry_streams.pipeline import Filter, Map, Parser, Serializer, streaming_source -from sentry_streams.pipeline.chain import StreamSink +from sentry_streams.pipeline.pipeline import ( + Filter, + Map, + Parser, + Serializer, + StreamSink, + streaming_source, +) # Import the compiled Rust functions try: @@ -26,16 +32,11 @@ name="myinput", stream_name="ingest-metrics", ) - .apply( - "parser", - Parser( - msg_type=IngestMetric, - ), - ) + .apply(Parser("parser", msg_type=IngestMetric)) # This filter will run in native Rust with zero Python overhead - .apply("filter", Filter(function=RustFilterEvents())) + .apply(Filter("filter", function=RustFilterEvents())) # This transform will run in native Rust with zero Python overhead - .apply("transform", Map(function=RustTransformMsg())) - .apply("serializer", Serializer()) - .sink("mysink", StreamSink(stream_name="transformed-events")) + .apply(Map("transform", function=RustTransformMsg())) + .apply(Serializer("serializer")) + .sink(StreamSink("mysink", stream_name="transformed-events")) ) diff --git a/sentry_streams/sentry_streams/pipeline/pipeline.py b/sentry_streams/sentry_streams/pipeline/pipeline.py index 8cde9355..4ae88f28 100644 --- a/sentry_streams/sentry_streams/pipeline/pipeline.py +++ b/sentry_streams/sentry_streams/pipeline/pipeline.py @@ -314,7 +314,6 @@ class Map(TransformStep[Any]): def __post_init__(self) -> None: """Validate the function after initialization""" - super().__post_init__() if self.has_rust_function(): self._validate_rust_function_type("map") @@ -341,7 +340,6 @@ class Filter(TransformStep[bool]): def __post_init__(self) -> None: """Validate the function after initialization""" - super().__post_init__() if self.has_rust_function(): self._validate_rust_function_type("filter") From 7115b95b57583e79b3314aff1ff493d5f77f101d Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Wed, 9 Jul 2025 19:39:49 +0200 Subject: [PATCH 03/16] try to fix some tests after rebase --- .../adapters/arroyo/rust_arroyo.py | 15 +++--------- .../rust_simple_map_filter/pipeline.py | 13 +++++----- sentry_streams/tests/test_mypy_integration.py | 24 +++++++++---------- 3 files changed, 21 insertions(+), 31 deletions(-) diff --git a/sentry_streams/sentry_streams/adapters/arroyo/rust_arroyo.py b/sentry_streams/sentry_streams/adapters/arroyo/rust_arroyo.py index 8b109031..d53fdd53 100644 --- a/sentry_streams/sentry_streams/adapters/arroyo/rust_arroyo.py +++ b/sentry_streams/sentry_streams/adapters/arroyo/rust_arroyo.py @@ -289,18 +289,9 @@ def filter(self, step: Filter, stream: Route) -> Route: route = RustRoute(stream.source, stream.waypoints) logger.info(f"Adding filter: {step.name} to pipeline") - if step.has_rust_function(): - self.__consumers[stream.source].add_step( - RuntimeOperator.Filter(route, step.resolved_function) - ) - else: - # XXX(markus): I don't know what this lambda is for, but i had to - # disable it for the rust path since we need access to methods on - # the callable. This seems like a useless indirection though? - def filter_msg(msg: Message[Any]) -> bool: - return step.resolved_function(msg) - - self.__consumers[stream.source].add_step(RuntimeOperator.Filter(route, filter_msg)) + self.__consumers[stream.source].add_step( + RuntimeOperator.Filter(route, step.resolved_function) + ) return stream diff --git a/sentry_streams/sentry_streams/examples/rust_simple_map_filter/pipeline.py b/sentry_streams/sentry_streams/examples/rust_simple_map_filter/pipeline.py index ba322891..6ea55b45 100644 --- a/sentry_streams/sentry_streams/examples/rust_simple_map_filter/pipeline.py +++ b/sentry_streams/sentry_streams/examples/rust_simple_map_filter/pipeline.py @@ -27,12 +27,13 @@ # Same pipeline structure as simple_map_filter.py, but with Rust functions # that will be called directly without Python overhead -pipeline = ( - streaming_source( - name="myinput", - stream_name="ingest-metrics", - ) - .apply(Parser("parser", msg_type=IngestMetric)) +pipeline = streaming_source( + name="myinput", + stream_name="ingest-metrics", +) + +( + pipeline.apply(Parser("parser", msg_type=IngestMetric)) # This filter will run in native Rust with zero Python overhead .apply(Filter("filter", function=RustFilterEvents())) # This transform will run in native Rust with zero Python overhead diff --git a/sentry_streams/tests/test_mypy_integration.py b/sentry_streams/tests/test_mypy_integration.py index b1b73b74..2273a22b 100644 --- a/sentry_streams/tests/test_mypy_integration.py +++ b/sentry_streams/tests/test_mypy_integration.py @@ -48,18 +48,17 @@ def test_mypy_detects_correct_pipeline(test_rust_extension) -> None: # type: ig # Create a test file with correct types correct_code = """ -from sentry_streams.pipeline import Filter, Map, Parser, streaming_source -from sentry_streams.pipeline.chain import StreamSink +from sentry_streams.pipeline.pipeline import Filter, Map, Parser, streaming_source, StreamSink from rust_test_functions import TestFilterCorrect, TestMapCorrect, TestMapString, TestMessage def create_correct_pipeline(): return ( streaming_source("input", "test-stream") - .apply("parser", Parser(msg_type=TestMessage)) # bytes -> TestMessage - .apply("filter", Filter(function=TestFilterCorrect())) # TestMessage -> TestMessage - .apply("transform", Map(function=TestMapCorrect())) # TestMessage -> String - .apply("length", Map(function=TestMapString())) # String -> u64 - .sink("output", StreamSink("output-stream")) + .apply(Parser("parser", msg_type=TestMessage)) # bytes -> TestMessage + .apply(Filter("filter", function=TestFilterCorrect())) # TestMessage -> TestMessage + .apply(Map("transform", function=TestMapCorrect())) # TestMessage -> String + .apply(Map("length", function=TestMapString())) # String -> u64 + .sink(StreamSink("output", "output-stream")) ) """ @@ -83,18 +82,17 @@ def test_mypy_detects_type_mismatch(test_rust_extension) -> None: # type: ignor """Test that mypy detects type mismatches in pipeline definitions""" wrong_code = """ -from sentry_streams.pipeline import Filter, Map, Parser, streaming_source -from sentry_streams.pipeline.chain import StreamSink +from sentry_streams.pipeline.pipeline import Filter, Map, Parser, streaming_source, StreamSink from rust_test_functions import TestFilterCorrect, TestMapCorrect, TestMapWrongType, TestMessage # expect failure: TestMapWrongType expects Message[bool] but gets Message[TestMessage] def create_wrong_pipeline(): return ( streaming_source("input", "test-stream") - .apply("parser", Parser(msg_type=TestMessage)) # bytes -> TestMessage - .apply("filter", Filter(function=TestFilterCorrect())) # TestMessage -> TestMessage - .apply("wrong", Map(function=TestMapWrongType())) # Expects Message[bool], gets Message[TestMessage]! - .sink("output", StreamSink("output-stream")) + .apply(Parser("parser", msg_type=TestMessage)) # bytes -> TestMessage + .apply(Filter("filter", function=TestFilterCorrect())) # TestMessage -> TestMessage + .apply(Map("wrong", function=TestMapWrongType())) # Expects Message[bool], gets Message[TestMessage]! + .sink(StreamSink("output", "output-stream")) ) """ From dd6a7d58400f869b43f2b3fda4bbc1e23519129b Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Thu, 10 Jul 2025 16:36:11 +0200 Subject: [PATCH 04/16] remove useless trait --- sentry_streams/src/macros.rs | 50 ------------------------------------ 1 file changed, 50 deletions(-) diff --git a/sentry_streams/src/macros.rs b/sentry_streams/src/macros.rs index 25066190..0d1540c0 100644 --- a/sentry_streams/src/macros.rs +++ b/sentry_streams/src/macros.rs @@ -1,18 +1,4 @@ /// Base trait for all Rust callback functions that can be called from the streaming pipeline -pub trait RustCallback: Send + Sync { - /// Get the raw function pointer for direct Rust calls - fn get_rust_function_pointer(&self) -> usize; - - /// Get the input type name for type validation - fn input_type(&self) -> &'static str; - - /// Get the output type name for type validation - fn output_type(&self) -> &'static str; - - /// Get the callback type ("map", "filter", "reduce", etc.) - fn callback_type(&self) -> &'static str; -} - /// Macro to create a Rust map function that can be called from Python /// Usage: rust_map_function!(MyFunction, InputType, OutputType, |msg: Message| -> Message { ... }); #[macro_export] @@ -57,24 +43,6 @@ macro_rules! rust_map_function { } } - impl $crate::macros::RustCallback for $name { - fn get_rust_function_pointer(&self) -> usize { - self.rust_fn_ptr - } - - fn input_type(&self) -> &'static str { - std::any::type_name::<$input_type>() - } - - fn output_type(&self) -> &'static str { - std::any::type_name::<$output_type>() - } - - fn callback_type(&self) -> &'static str { - "map" - } - } - // The actual Rust implementation function (unique name per macro invocation) paste::paste! { extern "C" fn []( @@ -147,24 +115,6 @@ macro_rules! rust_filter_function { } } - impl $crate::macros::RustCallback for $name { - fn get_rust_function_pointer(&self) -> usize { - self.rust_fn_ptr - } - - fn input_type(&self) -> &'static str { - std::any::type_name::<$input_type>() - } - - fn output_type(&self) -> &'static str { - "bool" - } - - fn callback_type(&self) -> &'static str { - "filter" - } - } - // The actual Rust implementation function (unique name per macro invocation) paste::paste! { extern "C" fn []( From d9f393bdd1436573611688afefe04834632c6c8e Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Mon, 14 Jul 2025 11:10:36 +0200 Subject: [PATCH 05/16] wip --- .../rust_transforms/src/lib.rs | 60 ++--- .../sentry_streams/pipeline/pipeline.py | 8 +- sentry_streams/src/lib.rs | 89 ++++++- sentry_streams/src/macros.rs | 127 +++------- sentry_streams/src/{ffi.rs => message.rs} | 43 ---- sentry_streams/src/operators.rs | 58 +---- sentry_streams/src/transformer.rs | 228 ------------------ .../tests/rust_test_functions/src/lib.rs | 4 +- 8 files changed, 167 insertions(+), 450 deletions(-) rename sentry_streams/src/{ffi.rs => message.rs} (62%) diff --git a/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/src/lib.rs b/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/src/lib.rs index e31cb09b..d1328e20 100644 --- a/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/src/lib.rs +++ b/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/src/lib.rs @@ -2,10 +2,8 @@ use pyo3::prelude::*; use serde::{Deserialize, Serialize}; // Import the macros from rust_streams (the actual crate name) -// In practice: use sentry_streams::{rust_map_function, rust_filter_function}; (when published) -use rust_streams::ffi::Message; -use rust_streams::rust_filter_function; -use rust_streams::rust_map_function; +// In practice: use sentry_streams::{rust_map_function, rust_filter_function, Message}; (when published) +use rust_streams::{rust_filter_function, rust_map_function, Message}; /// IngestMetric structure matching the schema from simple_map_filter.py /// This would normally be imported from sentry_kafka_schemas in a real implementation @@ -20,39 +18,43 @@ pub struct IngestMetric { } // Rust equivalent of filter_events() from simple_map_filter.py -rust_filter_function!(RustFilterEvents, serde_json::Value, |msg: Message< - serde_json::Value, +rust_filter_function!(RustFilterEvents, IngestMetric, |msg: Message< + IngestMetric, >| -> bool { - // Deserialize JSON payload to IngestMetric struct - let payload: IngestMetric = serde_json::from_value(msg.payload).unwrap(); - - // TODO: Fix segfault in Debug formatting - this should not crash even with binary data - // println!("Seen in filter: {:?}", msg); - // Same logic as Python version: return bool(msg.payload["type"] == "c") - payload.metric_type == "c" + // Direct access to strongly typed payload - no JSON conversion needed! + msg.payload.metric_type == "c" }); +// Enhanced IngestMetric with transform flag for output +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct TransformedIngestMetric { + #[serde(rename = "type")] + pub metric_type: String, + pub name: String, + pub value: f64, + pub tags: std::collections::HashMap, + pub timestamp: u64, + pub transformed: bool, +} + // Rust equivalent of transform_msg() from simple_map_filter.py rust_map_function!( RustTransformMsg, - serde_json::Value, - serde_json::Value, // Output as a flexible JSON value like the Python version - |msg: Message| -> Message { - // Deserialize JSON payload to IngestMetric struct - let payload: IngestMetric = serde_json::from_value(msg.payload).unwrap(); - - // TODO: Fix segfault in Debug formatting - this should not crash even with binary data - // println!("Seen in map: {:?}", msg); - // Convert IngestMetric to JSON, then add "transformed": true - // This matches the Python logic: {**msg.payload, "transformed": True} - let mut result = serde_json::to_value(&payload).unwrap(); - - if let Some(obj) = result.as_object_mut() { - obj.insert("transformed".to_string(), serde_json::Value::Bool(true)); - } + IngestMetric, + TransformedIngestMetric, + |msg: Message| -> Message { + // Direct access to strongly typed payload - no JSON conversion needed! + let transformed_payload = TransformedIngestMetric { + metric_type: msg.payload.metric_type, + name: msg.payload.name, + value: msg.payload.value, + tags: msg.payload.tags, + timestamp: msg.payload.timestamp, + transformed: true, + }; - Message::new(result, msg.headers, msg.timestamp, msg.schema) + Message::new(transformed_payload, msg.headers, msg.timestamp, msg.schema) } ); diff --git a/sentry_streams/sentry_streams/pipeline/pipeline.py b/sentry_streams/sentry_streams/pipeline/pipeline.py index b725a10b..796e3642 100644 --- a/sentry_streams/sentry_streams/pipeline/pipeline.py +++ b/sentry_streams/sentry_streams/pipeline/pipeline.py @@ -278,12 +278,7 @@ def resolved_function(self) -> Callable[..., TransformFuncReturnType]: def has_rust_function(self) -> bool: """Check if this function provides a Rust implementation""" func = self.resolved_function - return ( - hasattr(func, "get_rust_function_pointer") - and hasattr(func, "input_type") - and hasattr(func, "output_type") - and hasattr(func, "callback_type") - ) + return hasattr(func, "is_rust_function") and func.is_rust_function() def get_rust_callback_info(self) -> Mapping[str, Any]: """Get information about the Rust callback function""" @@ -292,7 +287,6 @@ def get_rust_callback_info(self) -> Mapping[str, Any]: func = self.resolved_function return { - "function_ptr": func.get_rust_function_pointer(), # type: ignore[attr-defined] "input_type": func.input_type(), # type: ignore[attr-defined] "output_type": func.output_type(), # type: ignore[attr-defined] "callback_type": func.callback_type(), # type: ignore[attr-defined] diff --git a/sentry_streams/src/lib.rs b/sentry_streams/src/lib.rs index be555cc4..cd717006 100644 --- a/sentry_streams/src/lib.rs +++ b/sentry_streams/src/lib.rs @@ -3,11 +3,11 @@ mod broadcaster; mod callers; mod committable; mod consumer; -pub mod ffi; mod filter_step; mod gcs_writer; mod kafka_config; pub mod macros; +pub mod message; mod messages; mod operators; mod python_operator; @@ -15,12 +15,93 @@ mod routers; mod routes; mod sinks; mod store_sinks; -mod transformer; +pub mod transformer; mod utils; mod watermark; -// Re-export macros so they can be used by external crates -pub use macros::*; +// Re-export types so they can be used by external crates +pub use message::Message; +// Macros are automatically exported at crate root due to #[macro_export] + +/// Convert a Python streaming message to a typed Rust Message format +/// This function handles the conversion for any type that implements serde::Deserialize +pub fn convert_py_message_to_rust( + py: pyo3::Python, + py_msg: &pyo3::Py, +) -> pyo3::PyResult> +where + T: serde::de::DeserializeOwned, +{ + // Extract payload and serialize to JSON for safe deserialization + let payload_obj = py_msg.bind(py).getattr("payload")?; + let payload_json = py + .import("json")? + .getattr("dumps")? + .call1((payload_obj,))? + .extract::()?; + + // Parse JSON into the desired type + let payload_value: T = serde_json::from_str(&payload_json).map_err(|e| { + pyo3::PyErr::new::(format!( + "Failed to parse JSON: {}", + e + )) + })?; + + // Extract headers + let headers_py: Vec<(String, Vec)> = py_msg.bind(py).getattr("headers")?.extract()?; + + // Extract timestamp + let timestamp: f64 = py_msg.bind(py).getattr("timestamp")?.extract()?; + + // Extract schema + let schema: Option = py_msg.bind(py).getattr("schema")?.extract()?; + + Ok(crate::message::Message::new( + payload_value, + headers_py, + timestamp, + schema, + )) +} + +/// Convert a typed Rust Message back to a Python streaming message +/// This function handles the conversion for any type that implements serde::Serialize +pub fn convert_rust_message_to_py( + py: pyo3::Python, + rust_msg: crate::message::Message, +) -> pyo3::PyResult> +where + T: serde::Serialize, +{ + // Convert payload to JSON string, then to Python object + let payload_json = serde_json::to_string(&rust_msg.payload).map_err(|e| { + pyo3::PyErr::new::(format!( + "Failed to serialize JSON: {}", + e + )) + })?; + let payload_obj = py + .import("json")? + .getattr("loads")? + .call1((payload_json,))?; + + // Create a new Python message with the transformed payload + let py_msg_class = py + .import("sentry_streams.rust_streams")? + .getattr("PyAnyMessage")?; + + let result = py_msg_class + .call1(( + payload_obj, + rust_msg.headers, + rust_msg.timestamp, + rust_msg.schema, + ))? + .unbind(); + + Ok(result) +} #[cfg(test)] mod fake_strategy; diff --git a/sentry_streams/src/macros.rs b/sentry_streams/src/macros.rs index 0d1540c0..08a42237 100644 --- a/sentry_streams/src/macros.rs +++ b/sentry_streams/src/macros.rs @@ -1,33 +1,37 @@ -/// Base trait for all Rust callback functions that can be called from the streaming pipeline /// Macro to create a Rust map function that can be called from Python /// Usage: rust_map_function!(MyFunction, InputType, OutputType, |msg: Message| -> Message { ... }); #[macro_export] macro_rules! rust_map_function { ($name:ident, $input_type:ty, $output_type:ty, $transform_fn:expr) => { #[pyo3::pyclass] - pub struct $name { - rust_fn_ptr: usize, - } + pub struct $name; #[pyo3::pymethods] impl $name { #[new] pub fn new() -> Self { - Self { - rust_fn_ptr: Self::get_rust_fn_ptr(), - } + Self } #[pyo3(name = "__call__")] - pub fn call(&self, _msg: pyo3::Py) -> pyo3::PyResult> { - // Python fallback - not implemented for performance reasons - Err(pyo3::PyErr::new::( - "This function should be called directly from Rust for performance" - )) - } - - pub fn get_rust_function_pointer(&self) -> usize { - self.rust_fn_ptr + pub fn call( + &self, + py: pyo3::Python<'_>, + py_msg: pyo3::Py, + ) -> pyo3::PyResult> { + // Convert Python message to typed Rust message + let rust_msg = $crate::convert_py_message_to_rust::<$input_type>(py, &py_msg)?; + + // Release GIL and call Rust function + let result_msg = py.allow_threads(|| { + let transform_fn: fn( + $crate::message::Message<$input_type>, + ) -> $crate::message::Message<$output_type> = $transform_fn; + transform_fn(rust_msg) + }); + + // Convert result back to Python message + $crate::convert_rust_message_to_py(py, result_msg) } pub fn input_type(&self) -> &'static str { @@ -41,29 +45,9 @@ macro_rules! rust_map_function { pub fn callback_type(&self) -> &'static str { "map" } - } - - // The actual Rust implementation function (unique name per macro invocation) - paste::paste! { - extern "C" fn []( - input: *const $crate::ffi::Message<$input_type>, - ) -> *const $crate::ffi::Message<$output_type> { - let input_msg = unsafe { - $crate::ffi::ptr_to_message(input) - }; - - let transform_fn: fn($crate::ffi::Message<$input_type>) -> $crate::ffi::Message<$output_type> = $transform_fn; - let output_msg = transform_fn(input_msg); - - $crate::ffi::message_to_ptr(output_msg) - } - } - impl $name { - fn get_rust_fn_ptr() -> usize { - paste::paste! { - [] as usize - } + pub fn is_rust_function(&self) -> bool { + true } } }; @@ -75,31 +59,31 @@ macro_rules! rust_map_function { macro_rules! rust_filter_function { ($name:ident, $input_type:ty, $filter_fn:expr) => { #[pyo3::pyclass] - pub struct $name { - rust_fn_ptr: usize, - } + pub struct $name; #[pyo3::pymethods] impl $name { #[new] pub fn new() -> Self { - Self { - rust_fn_ptr: Self::get_rust_fn_ptr(), - } + Self } #[pyo3(name = "__call__")] - pub fn call(&self, _msg: pyo3::Py) -> pyo3::PyResult { - // Python fallback - not implemented for performance reasons - Err( - pyo3::PyErr::new::( - "This function should be called directly from Rust for performance", - ), - ) - } + pub fn call( + &self, + py: pyo3::Python<'_>, + py_msg: pyo3::Py, + ) -> pyo3::PyResult { + // Convert Python message to typed Rust message + let rust_msg = $crate::convert_py_message_to_rust::<$input_type>(py, &py_msg)?; - pub fn get_rust_function_pointer(&self) -> usize { - self.rust_fn_ptr + // Release GIL and call Rust function + let result = py.allow_threads(|| { + let filter_fn: fn($crate::message::Message<$input_type>) -> bool = $filter_fn; + filter_fn(rust_msg) + }); + + Ok(result) } pub fn input_type(&self) -> &'static str { @@ -113,41 +97,10 @@ macro_rules! rust_filter_function { pub fn callback_type(&self) -> &'static str { "filter" } - } - - // The actual Rust implementation function (unique name per macro invocation) - paste::paste! { - extern "C" fn []( - input: *const $crate::ffi::Message<$input_type>, - ) -> bool { - let input_msg = unsafe { - $crate::ffi::ptr_to_message(input) - }; - - let filter_fn: fn($crate::ffi::Message<$input_type>) -> bool = $filter_fn; - filter_fn(input_msg) - } - } - impl $name { - fn get_rust_fn_ptr() -> usize { - paste::paste! { - [] as usize - } + pub fn is_rust_function(&self) -> bool { + true } } }; } - -/// Generic macro for creating any type of Rust callback function -/// Usage: rust_callback_function!(map, MyFunction, InputType, OutputType, |msg| { ... }); -#[macro_export] -macro_rules! rust_callback_function { - (map, $name:ident, $input_type:ty, $output_type:ty, $transform_fn:expr) => { - $crate::rust_map_function!($name, $input_type, $output_type, $transform_fn); - }; - - (filter, $name:ident, $input_type:ty, $filter_fn:expr) => { - $crate::rust_filter_function!($name, $input_type, $filter_fn); - }; -} diff --git a/sentry_streams/src/ffi.rs b/sentry_streams/src/message.rs similarity index 62% rename from sentry_streams/src/ffi.rs rename to sentry_streams/src/message.rs index 4b48fcaf..8de98422 100644 --- a/sentry_streams/src/ffi.rs +++ b/sentry_streams/src/message.rs @@ -1,6 +1,3 @@ -#[allow(unused_imports)] -use serde::{Deserialize, Serialize}; - /// Generic Message struct for use with Rust callbacks /// This matches the PyMessage structure but with a generic payload #[derive(Debug, Clone)] @@ -40,23 +37,6 @@ impl Message { } } -/// Convert a boxed message to an opaque pointer for FFI -pub fn message_to_ptr(msg: Message) -> *const Message { - Box::into_raw(Box::new(msg)) -} - -/// Convert an opaque pointer back to a boxed message for FFI -pub unsafe fn ptr_to_message(ptr: *const Message) -> Message { - *Box::from_raw(ptr as *mut Message) -} - -/// Free a message pointer (used for cleanup) -pub unsafe fn free_message_ptr(ptr: *const Message) { - if !ptr.is_null() { - let _ = Box::from_raw(ptr as *mut Message); - } -} - #[cfg(test)] mod tests { use super::*; @@ -111,27 +91,4 @@ mod tests { assert_eq!(transformed.payload.id, 2); assert_eq!(transformed.payload.name, "transformed_input"); } - - #[test] - fn test_ptr_roundtrip() { - let original = Message::new( - TestPayload { - id: 99, - name: "ptr_test".to_string(), - }, - vec![("header".to_string(), b"value".to_vec())], - 123.456, - Some("schema".to_string()), - ); - - // Convert to pointer and back - let ptr = message_to_ptr(original.clone()); - let recovered = unsafe { ptr_to_message(ptr) }; - - assert_eq!(original.payload.id, recovered.payload.id); - assert_eq!(original.payload.name, recovered.payload.name); - assert_eq!(original.headers, recovered.headers); - assert_eq!(original.timestamp, recovered.timestamp); - assert_eq!(original.schema, recovered.schema); - } } diff --git a/sentry_streams/src/operators.rs b/sentry_streams/src/operators.rs index 5d430e8b..20488577 100644 --- a/sentry_streams/src/operators.rs +++ b/sentry_streams/src/operators.rs @@ -5,7 +5,7 @@ use crate::routers::build_router; use crate::routes::{Route, RoutedValue}; use crate::sinks::StreamSink; use crate::store_sinks::GCSSink; -use crate::transformer::{build_filter, build_map, build_rust_filter, build_rust_map}; +use crate::transformer::{build_filter, build_map}; use crate::utils::traced_with_gil; use pyo3::prelude::*; use sentry_arroyo::backends::kafka::producer::KafkaProducer; @@ -86,56 +86,16 @@ pub fn build( ) -> Box> { match step.get() { RuntimeOperator::Map { function, route } => { - // Check if this is a Rust function - let is_rust_function = traced_with_gil!(|py| { - function - .bind(py) - .hasattr("get_rust_function_pointer") - .unwrap_or(false) - }); - - if is_rust_function { - // Get the raw function pointer and call Rust directly - let rust_fn_ptr = traced_with_gil!(|py| { - function - .bind(py) - .call_method0("get_rust_function_pointer") - .unwrap() - .extract::() - .unwrap() - }); - build_rust_map(route, rust_fn_ptr, next) - } else { - // Use existing Python path - let func_ref = traced_with_gil!(|py| function.clone_ref(py)); - build_map(route, func_ref, next) - } + // All functions (Python and Rust) are called the same way now + // Rust functions automatically release the GIL internally + let func_ref = traced_with_gil!(|py| function.clone_ref(py)); + build_map(route, func_ref, next) } RuntimeOperator::Filter { function, route } => { - // Check if this is a Rust function - let is_rust_function = traced_with_gil!(|py| { - function - .bind(py) - .hasattr("get_rust_function_pointer") - .unwrap_or(false) - }); - - if is_rust_function { - // Get the raw function pointer and call Rust directly - let rust_fn_ptr = traced_with_gil!(|py| { - function - .bind(py) - .call_method0("get_rust_function_pointer") - .unwrap() - .extract::() - .unwrap() - }); - build_rust_filter(route, rust_fn_ptr, next) - } else { - // Use existing Python path - let func_ref = traced_with_gil!(|py| { function.clone_ref(py) }); - build_filter(route, func_ref, next) - } + // All functions (Python and Rust) are called the same way now + // Rust functions automatically release the GIL internally + let func_ref = traced_with_gil!(|py| function.clone_ref(py)); + build_filter(route, func_ref, next) } RuntimeOperator::StreamSink { route, diff --git a/sentry_streams/src/transformer.rs b/sentry_streams/src/transformer.rs index 370c2946..42102a65 100644 --- a/sentry_streams/src/transformer.rs +++ b/sentry_streams/src/transformer.rs @@ -65,234 +65,6 @@ pub fn build_filter( Box::new(Filter::new(callable, next, copied_route)) } -/// Creates a Rust-native map transformation strategy that calls a Rust function directly -/// without Python GIL overhead. The function pointer should point to a function with -/// signature: extern "C" fn(*const Message) -> *const Message -pub fn build_rust_map( - route: &Route, - function_ptr: usize, - next: Box>, -) -> Box> { - let copied_route = route.clone(); - - let mapper = move |message: Message| { - if message.payload().route != copied_route { - return Ok(message); - } - - let RoutedValuePayload::PyStreamingMessage(ref py_streaming_msg) = - message.payload().payload - else { - return Ok(message); - }; - - let route = message.payload().route.clone(); - - // Convert Python message to our FFI Message format - let rust_msg = traced_with_gil!(|py| { - let py_any: Py = py_streaming_msg.into(); - convert_py_message_to_rust(py, &py_any) - }); - - let input_ptr = crate::ffi::message_to_ptr(rust_msg); - - // Cast function pointer and call it - let rust_fn: extern "C" fn( - *const crate::ffi::Message, - ) -> *const crate::ffi::Message = - unsafe { std::mem::transmute(function_ptr) }; - - let output_ptr = rust_fn(input_ptr); - let output_msg = unsafe { crate::ffi::ptr_to_message(output_ptr) }; - - // Convert result back to Python message - let py_result = traced_with_gil!(|py| { convert_rust_message_to_py(py, output_msg) }); - - Ok(message.replace(RoutedValue { - route, - payload: RoutedValuePayload::PyStreamingMessage(py_result.into()), - })) - }; - - Box::new(RunTask::new(mapper, next)) -} - -/// Creates a Rust-native filter strategy that calls a Rust function directly -/// without Python GIL overhead. The function pointer should point to a function with -/// signature: extern "C" fn(*const Message) -> bool -pub fn build_rust_filter( - route: &Route, - function_ptr: usize, - next: Box>, -) -> Box> { - let copied_route = route.clone(); - - // Create a custom Rust filter that follows the same pattern as the Python one - Box::new(RustFilter::new(function_ptr, next, copied_route)) -} - -/// A custom filter strategy that uses Rust functions directly -pub struct RustFilter { - pub function_ptr: usize, - pub next_step: Box>, - pub route: Route, -} - -impl RustFilter { - pub fn new( - function_ptr: usize, - next_step: Box>, - route: Route, - ) -> Self { - Self { - function_ptr, - next_step, - route, - } - } -} - -impl ProcessingStrategy for RustFilter { - fn poll( - &mut self, - ) -> Result< - Option, - sentry_arroyo::processing::strategies::StrategyError, - > { - self.next_step.poll() - } - - fn submit(&mut self, message: Message) -> Result<(), SubmitError> { - // Handle messages for different routes and watermarks like the Python filter - if self.route != message.payload().route || message.payload().payload.is_watermark_msg() { - return self.next_step.submit(message); - } - - let RoutedValuePayload::PyStreamingMessage(ref py_streaming_msg) = - message.payload().payload - else { - unreachable!("Watermark message trying to be passed to rust filter function.") - }; - - // Convert Python message to our FFI Message format - let rust_msg = traced_with_gil!(|py| { - let py_any: Py = py_streaming_msg.into(); - convert_py_message_to_rust(py, &py_any) - }); - - let input_ptr = crate::ffi::message_to_ptr(rust_msg); - - // Cast function pointer and call it - let rust_fn: extern "C" fn(*const crate::ffi::Message) -> bool = - unsafe { std::mem::transmute(self.function_ptr) }; - - let result = rust_fn(input_ptr); - - // Follow the same pattern as Python filter: submit if true, drop if false - if result { - self.next_step.submit(message) - } else { - Ok(()) // Drop the message - } - } - - fn terminate(&mut self) { - self.next_step.terminate() - } - - fn join( - &mut self, - timeout: Option, - ) -> Result< - Option, - sentry_arroyo::processing::strategies::StrategyError, - > { - self.next_step.join(timeout)?; - Ok(None) - } -} - -/// Convert a Python streaming message to our generic Rust Message format -fn convert_py_message_to_rust( - py: Python, - py_msg: &Py, -) -> crate::ffi::Message { - // Extract payload as serde_json::Value for safe deserialization - let payload_obj = py_msg.bind(py).getattr("payload").unwrap(); - let payload_json = py - .import("json") - .unwrap() - .getattr("dumps") - .unwrap() - .call1((payload_obj,)) - .unwrap() - .extract::() - .unwrap(); - - // Parse JSON into serde_json::Value for safe handling - let payload_value: serde_json::Value = serde_json::from_str(&payload_json).unwrap(); - - // Extract headers - let headers_py: Vec<(String, Vec)> = py_msg - .bind(py) - .getattr("headers") - .unwrap() - .extract() - .unwrap(); - - // Extract timestamp - let timestamp: f64 = py_msg - .bind(py) - .getattr("timestamp") - .unwrap() - .extract() - .unwrap(); - - // Extract schema - let schema: Option = py_msg - .bind(py) - .getattr("schema") - .unwrap() - .extract() - .unwrap(); - - crate::ffi::Message::new(payload_value, headers_py, timestamp, schema) -} - -/// Convert our generic Rust Message format back to a Python streaming message -fn convert_rust_message_to_py( - py: Python, - rust_msg: crate::ffi::Message, -) -> Py { - // Convert serde_json::Value back to JSON string, then to Python object - let payload_json = serde_json::to_string(&rust_msg.payload).unwrap(); - let payload_obj = py - .import("json") - .unwrap() - .getattr("loads") - .unwrap() - .call1((payload_json,)) - .unwrap(); - - // Create a new Python message with the transformed payload - // Use PyAnyMessage directly since that's what PyStreamingMessage conversion expects - let py_msg_class = py - .import("sentry_streams.rust_streams") - .unwrap() - .getattr("PyAnyMessage") - .unwrap(); - - py_msg_class - .call1(( - payload_obj, - rust_msg.headers, - rust_msg.timestamp, - rust_msg.schema, - )) - .unwrap() - .unbind() -} - #[cfg(test)] mod tests { use super::*; diff --git a/sentry_streams/tests/rust_test_functions/src/lib.rs b/sentry_streams/tests/rust_test_functions/src/lib.rs index 205ab53d..8616df54 100644 --- a/sentry_streams/tests/rust_test_functions/src/lib.rs +++ b/sentry_streams/tests/rust_test_functions/src/lib.rs @@ -2,9 +2,7 @@ use pyo3::prelude::*; use serde::{Deserialize, Serialize}; // Import the macros from rust_streams -use rust_streams::ffi::Message; -use rust_streams::rust_filter_function; -use rust_streams::rust_map_function; +use rust_streams::{rust_filter_function, rust_map_function, Message}; /// Test data structure for type validation tests #[derive(Serialize, Deserialize, Debug, Clone)] From 35535f16cee80178dbf599db4fbf297f6a4ef4bc Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Tue, 15 Jul 2025 13:27:08 +0200 Subject: [PATCH 06/16] wip --- sentry_streams/Cargo.lock | 8 - sentry_streams/Cargo.toml | 2 - .../metrics_rust_transforms.pyi | 23 +- .../rust_transforms/src/lib.rs | 25 ++- .../sentry_streams/pipeline/pipeline.py | 86 +++----- .../pipeline/rust_function_protocol.py | 26 +-- sentry_streams/src/ffi.rs | 202 ++++++++++++++++++ sentry_streams/src/lib.rs | 90 +------- sentry_streams/src/macros.rs | 106 --------- sentry_streams/src/message.rs | 94 -------- .../tests/rust_test_functions/Cargo.lock | 2 - 11 files changed, 269 insertions(+), 395 deletions(-) create mode 100644 sentry_streams/src/ffi.rs delete mode 100644 sentry_streams/src/macros.rs delete mode 100644 sentry_streams/src/message.rs diff --git a/sentry_streams/Cargo.lock b/sentry_streams/Cargo.lock index d76d7fda..3643f50b 100644 --- a/sentry_streams/Cargo.lock +++ b/sentry_streams/Cargo.lock @@ -995,12 +995,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - [[package]] name = "percent-encoding" version = "2.3.1" @@ -1306,12 +1300,10 @@ name = "rust_streams" version = "0.1.0" dependencies = [ "anyhow", - "base64", "chrono", "ctrlc", "log", "parking_lot", - "paste", "pyo3", "pyo3-build-config", "rdkafka", diff --git a/sentry_streams/Cargo.toml b/sentry_streams/Cargo.toml index def82196..1ddda223 100644 --- a/sentry_streams/Cargo.toml +++ b/sentry_streams/Cargo.toml @@ -7,8 +7,6 @@ edition = "2021" pyo3 = { version = "0.24.0"} serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -base64 = "0.22" -paste = "1.0" sentry_arroyo = "2.19.5" chrono = "0.4.40" tracing = "0.1.40" diff --git a/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/metrics_rust_transforms.pyi b/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/metrics_rust_transforms.pyi index 0da90090..795d9c2b 100644 --- a/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/metrics_rust_transforms.pyi +++ b/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/metrics_rust_transforms.pyi @@ -1,7 +1,10 @@ """ -Type stubs for metrics_rust_transforms Rust extension +Manually written type stubs for Rust functions. These type stubs are not used +by the streaming runtime but are mainly there to maintain type safety within +the pipeline definition. -This file provides type information for mypy and other static type checkers. +We hope that in a future version the runtime can generate those out of the rust +macros. """ from typing import Any @@ -15,25 +18,9 @@ from sentry_streams.pipeline.rust_function_protocol import ( ) class RustFilterEvents(RustFilterFunction[IngestMetric]): - """Rust filter function that accepts IngestMetric and returns bool""" - def __init__(self) -> None: ... def __call__(self, msg: Message[IngestMetric]) -> bool: ... - # Rust-specific methods for runtime detection - def get_rust_function_pointer(self) -> int: ... - def input_type(self) -> str: ... - def output_type(self) -> str: ... - def callback_type(self) -> str: ... - class RustTransformMsg(RustMapFunction[IngestMetric, Any]): - """Rust map function that accepts IngestMetric and returns JSON Value""" - def __init__(self) -> None: ... def __call__(self, msg: Message[IngestMetric]) -> Message[Any]: ... - - # Rust-specific methods for runtime detection - def get_rust_function_pointer(self) -> int: ... - def input_type(self) -> str: ... - def output_type(self) -> str: ... - def callback_type(self) -> str: ... diff --git a/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/src/lib.rs b/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/src/lib.rs index d1328e20..e48f9bb5 100644 --- a/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/src/lib.rs +++ b/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/src/lib.rs @@ -3,10 +3,13 @@ use serde::{Deserialize, Serialize}; // Import the macros from rust_streams (the actual crate name) // In practice: use sentry_streams::{rust_map_function, rust_filter_function, Message}; (when published) -use rust_streams::{rust_filter_function, rust_map_function, Message}; +use rust_streams::{convert_via_json, rust_filter_function, rust_map_function, Message}; /// IngestMetric structure matching the schema from simple_map_filter.py /// This would normally be imported from sentry_kafka_schemas in a real implementation +/// +/// Types are converted from/to Rust using JSON serialization. The input type must be +/// JSON-serializable and be able to deserialize into this type. #[derive(Serialize, Deserialize, Debug, Clone)] pub struct IngestMetric { #[serde(rename = "type")] @@ -17,16 +20,23 @@ pub struct IngestMetric { pub timestamp: u64, } +// Implement the FromPythonPayload and IntoPythonPayload traits. This decides how IngestMetric is +// going to be converted from the previous step's Python value. +// +// Currently, all values passed between steps are still Python objects, even between two Rust +// steps. +// +// This macro implements these traits by roundtripping values via JSON. +convert_via_json!(IngestMetric); + // Rust equivalent of filter_events() from simple_map_filter.py -rust_filter_function!(RustFilterEvents, IngestMetric, |msg: Message< +rust_function!(RustFilterEvents, IngestMetric, bool, |msg: Message< IngestMetric, >| -> bool { - // Direct access to strongly typed payload - no JSON conversion needed! msg.payload.metric_type == "c" }); -// Enhanced IngestMetric with transform flag for output #[derive(Serialize, Deserialize, Debug, Clone)] pub struct TransformedIngestMetric { #[serde(rename = "type")] @@ -38,13 +48,14 @@ pub struct TransformedIngestMetric { pub transformed: bool, } +convert_via_json!(TransformedIngestMetric); + // Rust equivalent of transform_msg() from simple_map_filter.py -rust_map_function!( +rust_function!( RustTransformMsg, IngestMetric, TransformedIngestMetric, |msg: Message| -> Message { - // Direct access to strongly typed payload - no JSON conversion needed! let transformed_payload = TransformedIngestMetric { metric_type: msg.payload.metric_type, name: msg.payload.name, @@ -54,7 +65,7 @@ rust_map_function!( transformed: true, }; - Message::new(transformed_payload, msg.headers, msg.timestamp, msg.schema) + transformed_payload } ); diff --git a/sentry_streams/sentry_streams/pipeline/pipeline.py b/sentry_streams/sentry_streams/pipeline/pipeline.py index 796e3642..ca4b8c24 100644 --- a/sentry_streams/sentry_streams/pipeline/pipeline.py +++ b/sentry_streams/sentry_streams/pipeline/pipeline.py @@ -41,6 +41,7 @@ resolve_polars_schema, serialize_to_parquet, ) +from sentry_streams.pipeline.rust_function_protocol import InternalRustFunction from sentry_streams.pipeline.window import MeasurementUnit, TumblingWindow, Window @@ -268,6 +269,8 @@ class StreamSink(Sink): RoutingFuncReturnType = TypeVar("RoutingFuncReturnType") TransformFuncReturnType = TypeVar("TransformFuncReturnType") +RUST_FUNCTION_VERSION = 1 + class TransformFunction(ABC, Generic[TransformFuncReturnType]): @property @@ -275,23 +278,6 @@ class TransformFunction(ABC, Generic[TransformFuncReturnType]): def resolved_function(self) -> Callable[..., TransformFuncReturnType]: raise NotImplementedError() - def has_rust_function(self) -> bool: - """Check if this function provides a Rust implementation""" - func = self.resolved_function - return hasattr(func, "is_rust_function") and func.is_rust_function() - - def get_rust_callback_info(self) -> Mapping[str, Any]: - """Get information about the Rust callback function""" - if not self.has_rust_function(): - raise ValueError("Function does not have Rust implementation") - - func = self.resolved_function - return { - "input_type": func.input_type(), # type: ignore[attr-defined] - "output_type": func.output_type(), # type: ignore[attr-defined] - "callback_type": func.callback_type(), # type: ignore[attr-defined] - } - @dataclass class TransformStep(WithInput, TransformFunction[TransformFuncReturnType]): @@ -323,6 +309,27 @@ def resolved_function(self) -> Callable[..., TransformFuncReturnType]: function_callable = imported_func return function_callable + def _validate_rust_function(self): + func = self.resolved_function + if not hasattr(func, "rust_function_version"): + # not a rust function + return None + + rust_function_version = func.rust_function_version() # type: ignore[attr-defined] + if rust_function_version != 1: + raise TypeError( + r"Invalid rust function version: {rust_function_version} -- if you are defining your own rust functions, maybe the version is out of date?" + ) + + return func + + def post_rust_function_validation(self, func: InternalRustFunction) -> None: + # Overridden in Filter step + pass + + def __post_init__(self) -> None: + self._validate_rust_function() + @dataclass class Map(TransformStep[Any]): @@ -340,23 +347,6 @@ class Map(TransformStep[Any]): # TODO: Allow product to both enable and access # configuration (e.g. a DB that is used as part of Map) - def __post_init__(self) -> None: - """Validate the function after initialization""" - if self.has_rust_function(): - self._validate_rust_function_type("map") - - def _validate_rust_function_type(self, expected_callback_type: str) -> None: - """Validate that the Rust function has the correct type""" - try: - info = self.get_rust_callback_info() - if info["callback_type"] != expected_callback_type: - raise TypeError( - f"Function {self.function} is a {info['callback_type']} function, " - f"but expected {expected_callback_type}" - ) - except Exception as e: - raise TypeError(f"Invalid Rust function {self.function}: {e}") - @dataclass class Filter(TransformStep[bool]): @@ -366,28 +356,12 @@ class Filter(TransformStep[bool]): step_type: StepType = StepType.FILTER - def __post_init__(self) -> None: - """Validate the function after initialization""" - if self.has_rust_function(): - self._validate_rust_function_type("filter") - - def _validate_rust_function_type(self, expected_callback_type: str) -> None: - """Validate that the Rust function has the correct type""" - try: - info = self.get_rust_callback_info() - if info["callback_type"] != expected_callback_type: - raise TypeError( - f"Function {self.function} is a {info['callback_type']} function, " - f"but expected {expected_callback_type}" - ) - # Additional validation for filter: output type should be bool - if expected_callback_type == "filter" and info["output_type"] != "bool": - raise TypeError( - f"Filter function {self.function} should return bool, " - f"but returns {info['output_type']}" - ) - except Exception as e: - raise TypeError(f"Invalid Rust function {self.function}: {e}") + def post_rust_function_validation(self, func: InternalRustFunction) -> None: + output_type = func.output_type() + if output_type != "bool": + raise TypeError( + f"Filter function {func} should return bool, " f"but returns {output_type}" + ) @dataclass diff --git a/sentry_streams/sentry_streams/pipeline/rust_function_protocol.py b/sentry_streams/sentry_streams/pipeline/rust_function_protocol.py index 75d00839..eaa02603 100644 --- a/sentry_streams/sentry_streams/pipeline/rust_function_protocol.py +++ b/sentry_streams/sentry_streams/pipeline/rust_function_protocol.py @@ -2,27 +2,21 @@ Protocol definitions for Rust functions to integrate with Python's type system """ -from typing import Protocol, TypeVar, runtime_checkable +from typing import Protocol, TypeVar from sentry_streams.pipeline.message import Message TInput = TypeVar("TInput") -TOutput = TypeVar("TOutput") +TOutput = TypeVar("TOutput", covariant=True) -@runtime_checkable -class RustFilterFunction(Protocol[TInput]): - """Protocol for Rust filter functions""" +# External interface that we need users to use in their stubs for type-safety +class RustFunction(Protocol[TInput, TOutput]): + def __call__(self, msg: Message[TInput]) -> TOutput: ... - def __call__(self, msg: Message[TInput]) -> bool: - """Filter function that returns True to keep the message""" - ... - -@runtime_checkable -class RustMapFunction(Protocol[TInput, TOutput]): - """Protocol for Rust map functions""" - - def __call__(self, msg: Message[TInput]) -> Message[TOutput]: - """Map function that transforms the message""" - ... +# Methods that we use internally, but don't want the user to see (or have to write out in their stubfiles) +class InternalRustFunction(RustFunction, Protocol): + def input_type(self) -> str: ... + def output_type(self) -> str: ... + def rust_function_version(self) -> int: ... diff --git a/sentry_streams/src/ffi.rs b/sentry_streams/src/ffi.rs new file mode 100644 index 00000000..ace6cf47 --- /dev/null +++ b/sentry_streams/src/ffi.rs @@ -0,0 +1,202 @@ +use pyo3::prelude::*; + +pub const RUST_FUNCTION_VERSION: usize = 1; + +/// The message type exposed to Rust functions, with typed payload. +#[derive(Debug, Clone)] +pub struct Message { + payload: T, + headers: Vec<(String, Vec)>, + timestamp: f64, + schema: Option, +} + +impl Message { + /// Split up the message into payload and metadata + pub fn take(self) -> (T, Message<()>) { + ( + self.payload, + Message { + payload: (), + headers: self.headers, + timestamp: self.timestamp, + schema: self.schema, + }, + ) + } +} + +/// Convert a Python payload into a given Rust type +/// +/// You can implement this trait easiest by calling `convert_via_json!(MyType)`, provided your type +/// is JSON-serializable and deserializable on both sides. +pub trait FromPythonPayload { + fn from_python_payload( + py: pyo3::Python<'_>, + value: pyo3::Py, + ) -> pyo3::PyResult; +} + +/// Convert a Rust type back into a Python payload +/// +/// You can implement this trait easiest by calling `convert_via_json!(MyType)`, provided your type +/// is JSON-serializable and deserializable with serde. +pub trait IntoPythonPayload { + fn into_python_payload(self, py: pyo3::Python<'_>) -> pyo3::Py; +} + +/// Implement type conversion from/to Python by roundtripping with `serde_json` and `json.loads`. +/// +/// You need `serde_json` and `pyo3` in your crate's dependencies. +macro_rules! convert_via_json { + ($ty:ty) => { + impl FromPythonPayload for $ty { + fn from_python_payload( + py: pyo3::Python<'_>, + value: ::pyo3::Py, + ) -> ::pyo3::PyResult { + use pyo3::prelude::*; + + let payload_json = py + .import("json")? + .getattr("dumps")? + .call1((value,))? + .extract::()?; + + let payload_value: Self = ::serde_json::from_str(&payload_json).map_err(|e| { + ::pyo3::PyErr::new::<::pyo3::exceptions::PyValueError, _>(format!( + "Failed to parse JSON: {}", + e + )) + })?; + + Ok(payload_value) + } + } + + impl IntoPythonPayload for $ty { + fn into_python_payload(self, py: ::pyo3::Python<'_>) -> ::pyo3::Py { + use pyo3::prelude::*; + + let payload_json = ::serde_json::to_string(&rust_msg.payload).map_err(|e| { + ::pyo3::PyErr::new::(format!( + "Failed to serialize JSON: {}", + e + )) + })?; + let payload_obj = py + .import("json")? + .getattr("loads")? + .call1((payload_json,))?; + + Ok(payload_obj) + } + } + }; +} + +pub use convert_via_json; + +/// Convert a Python streaming message to a typed Rust Message format +/// This function handles the conversion for any type that implements serde::Deserialize +pub fn convert_py_message_to_rust( + py: pyo3::Python, + py_msg: &pyo3::Py, +) -> pyo3::PyResult> +where + T: FromPythonPayload, +{ + let payload_obj = py_msg.bind(py).getattr("payload")?; + let payload_value = T::from_python_payload(payload_obj); + + let headers_py: Vec<(String, Vec)> = py_msg.bind(py).getattr("headers")?.extract()?; + let timestamp: f64 = py_msg.bind(py).getattr("timestamp")?.extract()?; + let schema: Option = py_msg.bind(py).getattr("schema")?.extract()?; + + Ok(crate::message::Message::new( + payload_value, + headers_py, + timestamp, + schema, + )) +} + +pub fn convert_rust_message_to_py( + py: pyo3::Python, + rust_msg: crate::message::Message, +) -> pyo3::PyResult> +where + T: IntoPythonPayload, +{ + let payload_obj = rust_msg.payload.into_python_payload()?; + + // Create a new Python message with the transformed payload + let py_msg_class = py + .import("sentry_streams.rust_streams")? + .getattr("PyAnyMessage")?; + + let result = py_msg_class + .call1(( + payload_obj, + rust_msg.headers, + rust_msg.timestamp, + rust_msg.schema, + ))? + .unbind(); + + Ok(result) +} + +/// Macro to create a Rust map function that can be called from Python +/// Usage: rust_map_function!(MyFunction, InputType, OutputType, |msg: Message| -> OutputType { ... }); +macro_rules! rust_function { + ($name:ident, $input_type:ty, $output_type:ty, $transform_fn:expr) => { + #[pyo3::pyclass] + pub struct $name; + + #[pyo3::pymethods] + impl $name { + #[new] + pub fn new() -> Self { + Self + } + + #[pyo3(name = "__call__")] + pub fn call( + &self, + py: pyo3::Python<'_>, + py_msg: pyo3::Py, + ) -> pyo3::PyResult> { + // If this cast fails, the user is not providing the right types + let transform_fn: fn( + $crate::ffi::Message<$input_type>, + ) -> $crate::ffi::Message<$output_type> = $transform_fn; + + // Convert Python message to typed Rust message + let rust_msg = $crate::ffi::convert_py_message_to_rust::<$input_type>(py, &py_msg)?; + + // Release GIL and call Rust function + let result_msg = py.allow_threads(|| transform_fn(rust_msg)); + + // Convert result back to Python message + $crate::ffi::convert_rust_message_to_py(py, result_msg) + } + + pub fn input_type(&self) -> &'static str { + std::any::type_name::<$input_type>() + } + + pub fn output_type(&self) -> &'static str { + std::any::type_name::<$output_type>() + } + + pub fn rust_function_version(&self) -> usize { + $crate::ffi::RUST_FUNCTION_VERSION + } + } + }; +} + +pub use rust_function; + +use crate::python_operator; diff --git a/sentry_streams/src/lib.rs b/sentry_streams/src/lib.rs index cd717006..df2997fb 100644 --- a/sentry_streams/src/lib.rs +++ b/sentry_streams/src/lib.rs @@ -6,8 +6,6 @@ mod consumer; mod filter_step; mod gcs_writer; mod kafka_config; -pub mod macros; -pub mod message; mod messages; mod operators; mod python_operator; @@ -15,93 +13,13 @@ mod routers; mod routes; mod sinks; mod store_sinks; -pub mod transformer; +mod transformer; mod utils; mod watermark; -// Re-export types so they can be used by external crates -pub use message::Message; -// Macros are automatically exported at crate root due to #[macro_export] - -/// Convert a Python streaming message to a typed Rust Message format -/// This function handles the conversion for any type that implements serde::Deserialize -pub fn convert_py_message_to_rust( - py: pyo3::Python, - py_msg: &pyo3::Py, -) -> pyo3::PyResult> -where - T: serde::de::DeserializeOwned, -{ - // Extract payload and serialize to JSON for safe deserialization - let payload_obj = py_msg.bind(py).getattr("payload")?; - let payload_json = py - .import("json")? - .getattr("dumps")? - .call1((payload_obj,))? - .extract::()?; - - // Parse JSON into the desired type - let payload_value: T = serde_json::from_str(&payload_json).map_err(|e| { - pyo3::PyErr::new::(format!( - "Failed to parse JSON: {}", - e - )) - })?; - - // Extract headers - let headers_py: Vec<(String, Vec)> = py_msg.bind(py).getattr("headers")?.extract()?; - - // Extract timestamp - let timestamp: f64 = py_msg.bind(py).getattr("timestamp")?.extract()?; - - // Extract schema - let schema: Option = py_msg.bind(py).getattr("schema")?.extract()?; - - Ok(crate::message::Message::new( - payload_value, - headers_py, - timestamp, - schema, - )) -} - -/// Convert a typed Rust Message back to a Python streaming message -/// This function handles the conversion for any type that implements serde::Serialize -pub fn convert_rust_message_to_py( - py: pyo3::Python, - rust_msg: crate::message::Message, -) -> pyo3::PyResult> -where - T: serde::Serialize, -{ - // Convert payload to JSON string, then to Python object - let payload_json = serde_json::to_string(&rust_msg.payload).map_err(|e| { - pyo3::PyErr::new::(format!( - "Failed to serialize JSON: {}", - e - )) - })?; - let payload_obj = py - .import("json")? - .getattr("loads")? - .call1((payload_json,))?; - - // Create a new Python message with the transformed payload - let py_msg_class = py - .import("sentry_streams.rust_streams")? - .getattr("PyAnyMessage")?; - - let result = py_msg_class - .call1(( - payload_obj, - rust_msg.headers, - rust_msg.timestamp, - rust_msg.schema, - ))? - .unbind(); - - Ok(result) -} +#[doc(hidden)] +pub mod ffi; +pub use ffi::{rust_map_function, Message}; #[cfg(test)] mod fake_strategy; diff --git a/sentry_streams/src/macros.rs b/sentry_streams/src/macros.rs deleted file mode 100644 index 08a42237..00000000 --- a/sentry_streams/src/macros.rs +++ /dev/null @@ -1,106 +0,0 @@ -/// Macro to create a Rust map function that can be called from Python -/// Usage: rust_map_function!(MyFunction, InputType, OutputType, |msg: Message| -> Message { ... }); -#[macro_export] -macro_rules! rust_map_function { - ($name:ident, $input_type:ty, $output_type:ty, $transform_fn:expr) => { - #[pyo3::pyclass] - pub struct $name; - - #[pyo3::pymethods] - impl $name { - #[new] - pub fn new() -> Self { - Self - } - - #[pyo3(name = "__call__")] - pub fn call( - &self, - py: pyo3::Python<'_>, - py_msg: pyo3::Py, - ) -> pyo3::PyResult> { - // Convert Python message to typed Rust message - let rust_msg = $crate::convert_py_message_to_rust::<$input_type>(py, &py_msg)?; - - // Release GIL and call Rust function - let result_msg = py.allow_threads(|| { - let transform_fn: fn( - $crate::message::Message<$input_type>, - ) -> $crate::message::Message<$output_type> = $transform_fn; - transform_fn(rust_msg) - }); - - // Convert result back to Python message - $crate::convert_rust_message_to_py(py, result_msg) - } - - pub fn input_type(&self) -> &'static str { - std::any::type_name::<$input_type>() - } - - pub fn output_type(&self) -> &'static str { - std::any::type_name::<$output_type>() - } - - pub fn callback_type(&self) -> &'static str { - "map" - } - - pub fn is_rust_function(&self) -> bool { - true - } - } - }; -} - -/// Macro to create a Rust filter function that can be called from Python -/// Usage: rust_filter_function!(MyFilter, InputType, |msg: Message| -> bool { ... }); -#[macro_export] -macro_rules! rust_filter_function { - ($name:ident, $input_type:ty, $filter_fn:expr) => { - #[pyo3::pyclass] - pub struct $name; - - #[pyo3::pymethods] - impl $name { - #[new] - pub fn new() -> Self { - Self - } - - #[pyo3(name = "__call__")] - pub fn call( - &self, - py: pyo3::Python<'_>, - py_msg: pyo3::Py, - ) -> pyo3::PyResult { - // Convert Python message to typed Rust message - let rust_msg = $crate::convert_py_message_to_rust::<$input_type>(py, &py_msg)?; - - // Release GIL and call Rust function - let result = py.allow_threads(|| { - let filter_fn: fn($crate::message::Message<$input_type>) -> bool = $filter_fn; - filter_fn(rust_msg) - }); - - Ok(result) - } - - pub fn input_type(&self) -> &'static str { - std::any::type_name::<$input_type>() - } - - pub fn output_type(&self) -> &'static str { - "bool" - } - - pub fn callback_type(&self) -> &'static str { - "filter" - } - - pub fn is_rust_function(&self) -> bool { - true - } - } - }; -} diff --git a/sentry_streams/src/message.rs b/sentry_streams/src/message.rs deleted file mode 100644 index 8de98422..00000000 --- a/sentry_streams/src/message.rs +++ /dev/null @@ -1,94 +0,0 @@ -/// Generic Message struct for use with Rust callbacks -/// This matches the PyMessage structure but with a generic payload -#[derive(Debug, Clone)] -pub struct Message { - pub payload: T, - pub headers: Vec<(String, Vec)>, - pub timestamp: f64, - pub schema: Option, -} - -impl Message { - pub fn new( - payload: T, - headers: Vec<(String, Vec)>, - timestamp: f64, - schema: Option, - ) -> Self { - Self { - payload, - headers, - timestamp, - schema, - } - } - - /// Transform the message payload while keeping metadata - pub fn map(self, f: F) -> Message - where - F: FnOnce(T) -> U, - { - Message { - payload: f(self.payload), - headers: self.headers, - timestamp: self.timestamp, - schema: self.schema, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use serde::{Deserialize, Serialize}; - - #[derive(Serialize, Deserialize, Debug, PartialEq, Clone)] - struct TestPayload { - id: u64, - name: String, - } - - #[test] - fn test_message_creation() { - let payload = TestPayload { - id: 42, - name: "test".to_string(), - }; - - let headers = vec![("content-type".to_string(), b"application/json".to_vec())]; - - let msg = Message::new( - payload.clone(), - headers.clone(), - 1234567890.5, - Some("test_schema".to_string()), - ); - - assert_eq!(msg.payload.id, 42); - assert_eq!(msg.payload.name, "test"); - assert_eq!(msg.headers, headers); - assert_eq!(msg.timestamp, 1234567890.5); - assert_eq!(msg.schema, Some("test_schema".to_string())); - } - - #[test] - fn test_message_map() { - let original = Message::new( - TestPayload { - id: 1, - name: "input".to_string(), - }, - vec![], - 0.0, - None, - ); - - let transformed = original.map(|payload| TestPayload { - id: payload.id * 2, - name: format!("transformed_{}", payload.name), - }); - - assert_eq!(transformed.payload.id, 2); - assert_eq!(transformed.payload.name, "transformed_input"); - } -} diff --git a/sentry_streams/tests/rust_test_functions/Cargo.lock b/sentry_streams/tests/rust_test_functions/Cargo.lock index 75bc7dfa..07054e4f 100644 --- a/sentry_streams/tests/rust_test_functions/Cargo.lock +++ b/sentry_streams/tests/rust_test_functions/Cargo.lock @@ -1256,11 +1256,9 @@ name = "rust_streams" version = "0.1.0" dependencies = [ "anyhow", - "base64", "chrono", "ctrlc", "log", - "paste", "pyo3", "pyo3-build-config 0.25.1", "rdkafka", From ad8894b704ad3ebed17222f52b9485b990725752 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Tue, 15 Jul 2025 14:09:58 +0200 Subject: [PATCH 07/16] wip --- .../adapters/arroyo/rust_arroyo.py | 15 ---- .../rust_transforms/Cargo.lock | 2 - .../rust_transforms/src/lib.rs | 24 +++--- sentry_streams/src/ffi.rs | 79 +++++++++++-------- sentry_streams/src/lib.rs | 2 +- 5 files changed, 61 insertions(+), 61 deletions(-) diff --git a/sentry_streams/sentry_streams/adapters/arroyo/rust_arroyo.py b/sentry_streams/sentry_streams/adapters/arroyo/rust_arroyo.py index d53fdd53..a369ecec 100644 --- a/sentry_streams/sentry_streams/adapters/arroyo/rust_arroyo.py +++ b/sentry_streams/sentry_streams/adapters/arroyo/rust_arroyo.py @@ -231,21 +231,6 @@ def map(self, step: Map, stream: Route) -> Route: stream.source in self.__consumers ), f"Stream starting at source {stream.source} not found when adding a map" - # Check if this is a Rust function that should be handled directly - if step.has_rust_function(): - # Handle Rust functions directly without going through the chain system - self.__close_chain(stream) - - route = RustRoute(stream.source, stream.waypoints) - logger.info(f"Adding Rust map: {step.name} to pipeline") - - # For Rust functions, pass the function directly - the Rust runtime will handle it - self.__consumers[stream.source].add_step( - RuntimeOperator.Map(route, step.resolved_function) - ) - - return stream - # Handle Python functions with the existing chain system step_config: Mapping[str, Any] = self.steps_config.get(step.name, {}) parallelism_config = step_config.get("parallelism") diff --git a/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/Cargo.lock b/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/Cargo.lock index 2547b6ce..395d7158 100644 --- a/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/Cargo.lock +++ b/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/Cargo.lock @@ -1257,11 +1257,9 @@ name = "rust_streams" version = "0.1.0" dependencies = [ "anyhow", - "base64", "chrono", "ctrlc", "log", - "paste", "pyo3", "pyo3-build-config", "rdkafka", diff --git a/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/src/lib.rs b/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/src/lib.rs index e48f9bb5..3c3ea552 100644 --- a/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/src/lib.rs +++ b/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/src/lib.rs @@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize}; // Import the macros from rust_streams (the actual crate name) // In practice: use sentry_streams::{rust_map_function, rust_filter_function, Message}; (when published) -use rust_streams::{convert_via_json, rust_filter_function, rust_map_function, Message}; +use rust_streams::{convert_via_json, rust_function, Message}; /// IngestMetric structure matching the schema from simple_map_filter.py /// This would normally be imported from sentry_kafka_schemas in a real implementation @@ -34,7 +34,8 @@ rust_function!(RustFilterEvents, IngestMetric, bool, |msg: Message< IngestMetric, >| -> bool { - msg.payload.metric_type == "c" + let (payload, _) = msg.take(); + payload.metric_type == "c" }); #[derive(Serialize, Deserialize, Debug, Clone)] @@ -55,17 +56,16 @@ rust_function!( RustTransformMsg, IngestMetric, TransformedIngestMetric, - |msg: Message| -> Message { - let transformed_payload = TransformedIngestMetric { - metric_type: msg.payload.metric_type, - name: msg.payload.name, - value: msg.payload.value, - tags: msg.payload.tags, - timestamp: msg.payload.timestamp, + |msg: Message| -> TransformedIngestMetric { + let (payload, _) = msg.take(); + TransformedIngestMetric { + metric_type: payload.metric_type, + name: payload.name, + value: payload.value, + tags: payload.tags, + timestamp: payload.timestamp, transformed: true, - }; - - transformed_payload + } } ); diff --git a/sentry_streams/src/ffi.rs b/sentry_streams/src/ffi.rs index ace6cf47..ce6e1fe2 100644 --- a/sentry_streams/src/ffi.rs +++ b/sentry_streams/src/ffi.rs @@ -1,4 +1,5 @@ use pyo3::prelude::*; +use pyo3::IntoPyObjectExt; pub const RUST_FUNCTION_VERSION: usize = 1; @@ -24,17 +25,24 @@ impl Message { }, ) } + + /// Map the payload to a new type while preserving metadata + pub fn map(self, f: impl FnOnce(T) -> U) -> Message { + Message { + payload: f(self.payload), + headers: self.headers, + timestamp: self.timestamp, + schema: self.schema, + } + } } /// Convert a Python payload into a given Rust type /// /// You can implement this trait easiest by calling `convert_via_json!(MyType)`, provided your type /// is JSON-serializable and deserializable on both sides. -pub trait FromPythonPayload { - fn from_python_payload( - py: pyo3::Python<'_>, - value: pyo3::Py, - ) -> pyo3::PyResult; +pub trait FromPythonPayload: Sized { + fn from_python_payload(value: pyo3::Bound<'_, pyo3::PyAny>) -> pyo3::PyResult; } /// Convert a Rust type back into a Python payload @@ -42,21 +50,20 @@ pub trait FromPythonPayload { /// You can implement this trait easiest by calling `convert_via_json!(MyType)`, provided your type /// is JSON-serializable and deserializable with serde. pub trait IntoPythonPayload { - fn into_python_payload(self, py: pyo3::Python<'_>) -> pyo3::Py; + fn into_python_payload(self, py: pyo3::Python<'_>) -> pyo3::PyResult>; } /// Implement type conversion from/to Python by roundtripping with `serde_json` and `json.loads`. /// /// You need `serde_json` and `pyo3` in your crate's dependencies. +#[macro_export] macro_rules! convert_via_json { ($ty:ty) => { - impl FromPythonPayload for $ty { - fn from_python_payload( - py: pyo3::Python<'_>, - value: ::pyo3::Py, - ) -> ::pyo3::PyResult { + impl $crate::ffi::FromPythonPayload for $ty { + fn from_python_payload(value: pyo3::Bound<'_, pyo3::PyAny>) -> ::pyo3::PyResult { use pyo3::prelude::*; + let py = value.py(); let payload_json = py .import("json")? .getattr("dumps")? @@ -74,11 +81,14 @@ macro_rules! convert_via_json { } } - impl IntoPythonPayload for $ty { - fn into_python_payload(self, py: ::pyo3::Python<'_>) -> ::pyo3::Py { + impl $crate::ffi::IntoPythonPayload for $ty { + fn into_python_payload( + self, + py: ::pyo3::Python<'_>, + ) -> ::pyo3::PyResult<::pyo3::Py> { use pyo3::prelude::*; - let payload_json = ::serde_json::to_string(&rust_msg.payload).map_err(|e| { + let payload_json = ::serde_json::to_string(&self).map_err(|e| { ::pyo3::PyErr::new::(format!( "Failed to serialize JSON: {}", e @@ -89,20 +99,18 @@ macro_rules! convert_via_json { .getattr("loads")? .call1((payload_json,))?; - Ok(payload_obj) + Ok(payload_obj.unbind()) } } }; } -pub use convert_via_json; - /// Convert a Python streaming message to a typed Rust Message format /// This function handles the conversion for any type that implements serde::Deserialize pub fn convert_py_message_to_rust( py: pyo3::Python, py_msg: &pyo3::Py, -) -> pyo3::PyResult> +) -> pyo3::PyResult> where T: FromPythonPayload, { @@ -113,22 +121,22 @@ where let timestamp: f64 = py_msg.bind(py).getattr("timestamp")?.extract()?; let schema: Option = py_msg.bind(py).getattr("schema")?.extract()?; - Ok(crate::message::Message::new( - payload_value, - headers_py, + Ok(Message { + payload: payload_value?, + headers: headers_py, timestamp, schema, - )) + }) } pub fn convert_rust_message_to_py( py: pyo3::Python, - rust_msg: crate::message::Message, + rust_msg: Message, ) -> pyo3::PyResult> where T: IntoPythonPayload, { - let payload_obj = rust_msg.payload.into_python_payload()?; + let payload_obj = rust_msg.payload.into_python_payload(py)?; // Create a new Python message with the transformed payload let py_msg_class = py @@ -149,6 +157,7 @@ where /// Macro to create a Rust map function that can be called from Python /// Usage: rust_map_function!(MyFunction, InputType, OutputType, |msg: Message| -> OutputType { ... }); +#[macro_export] macro_rules! rust_function { ($name:ident, $input_type:ty, $output_type:ty, $transform_fn:expr) => { #[pyo3::pyclass] @@ -168,15 +177,20 @@ macro_rules! rust_function { py_msg: pyo3::Py, ) -> pyo3::PyResult> { // If this cast fails, the user is not providing the right types - let transform_fn: fn( - $crate::ffi::Message<$input_type>, - ) -> $crate::ffi::Message<$output_type> = $transform_fn; + let transform_fn: fn($crate::ffi::Message<$input_type>) -> $output_type = + $transform_fn; // Convert Python message to typed Rust message let rust_msg = $crate::ffi::convert_py_message_to_rust::<$input_type>(py, &py_msg)?; // Release GIL and call Rust function - let result_msg = py.allow_threads(|| transform_fn(rust_msg)); + let result_msg = py.allow_threads(|| { + // clone metadata, but try very hard to avoid cloning the payload + let (payload, metadata) = rust_msg.take(); + let metadata_clone = metadata.clone(); + let result_payload = transform_fn(metadata.map(|()| payload)); + metadata_clone.map(|()| result_payload) + }); // Convert result back to Python message $crate::ffi::convert_rust_message_to_py(py, result_msg) @@ -197,6 +211,9 @@ macro_rules! rust_function { }; } -pub use rust_function; - -use crate::python_operator; +// Built-in implementations for common types +impl IntoPythonPayload for bool { + fn into_python_payload(self, py: pyo3::Python<'_>) -> pyo3::PyResult> { + Ok(pyo3::types::PyBool::new(py, self).into_py_any(py)?) + } +} diff --git a/sentry_streams/src/lib.rs b/sentry_streams/src/lib.rs index df2997fb..6375ebf1 100644 --- a/sentry_streams/src/lib.rs +++ b/sentry_streams/src/lib.rs @@ -19,7 +19,7 @@ mod watermark; #[doc(hidden)] pub mod ffi; -pub use ffi::{rust_map_function, Message}; +pub use ffi::Message; #[cfg(test)] mod fake_strategy; From eb17e6e0d082bb086124c96c8a3f29a03a19d8be Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Tue, 15 Jul 2025 14:21:49 +0200 Subject: [PATCH 08/16] fix tests --- rust-streams/src/lib.rs | 14 ----- sentry_streams/Cargo.lock | 1 - sentry_streams/Cargo.toml | 1 - .../adapters/arroyo/rust_arroyo.py | 1 - .../rust_transforms/Cargo.toml | 3 +- .../metrics_rust_transforms.pyi | 11 ++-- .../pipeline/rust_function_protocol.py | 3 +- sentry_streams/src/ffi.rs | 30 ++++++++++ .../tests/rust_test_functions/Cargo.lock | 1 - .../rust_test_functions.pyi | 19 +++--- .../tests/rust_test_functions/src/lib.rs | 59 ++++++++----------- sentry_streams/tests/test_mypy_integration.py | 11 ++-- 12 files changed, 72 insertions(+), 82 deletions(-) delete mode 100644 rust-streams/src/lib.rs diff --git a/rust-streams/src/lib.rs b/rust-streams/src/lib.rs deleted file mode 100644 index b93cf3ff..00000000 --- a/rust-streams/src/lib.rs +++ /dev/null @@ -1,14 +0,0 @@ -pub fn add(left: u64, right: u64) -> u64 { - left + right -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn it_works() { - let result = add(2, 2); - assert_eq!(result, 4); - } -} diff --git a/sentry_streams/Cargo.lock b/sentry_streams/Cargo.lock index 3643f50b..ffbdfa71 100644 --- a/sentry_streams/Cargo.lock +++ b/sentry_streams/Cargo.lock @@ -1310,7 +1310,6 @@ dependencies = [ "reqwest", "sentry_arroyo", "serde", - "serde_json", "tokio", "tracing", "tracing-subscriber", diff --git a/sentry_streams/Cargo.toml b/sentry_streams/Cargo.toml index 1ddda223..d1e40d5c 100644 --- a/sentry_streams/Cargo.toml +++ b/sentry_streams/Cargo.toml @@ -6,7 +6,6 @@ edition = "2021" [dependencies] pyo3 = { version = "0.24.0"} serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" sentry_arroyo = "2.19.5" chrono = "0.4.40" tracing = "0.1.40" diff --git a/sentry_streams/sentry_streams/adapters/arroyo/rust_arroyo.py b/sentry_streams/sentry_streams/adapters/arroyo/rust_arroyo.py index a369ecec..03072a9e 100644 --- a/sentry_streams/sentry_streams/adapters/arroyo/rust_arroyo.py +++ b/sentry_streams/sentry_streams/adapters/arroyo/rust_arroyo.py @@ -231,7 +231,6 @@ def map(self, step: Map, stream: Route) -> Route: stream.source in self.__consumers ), f"Stream starting at source {stream.source} not found when adding a map" - # Handle Python functions with the existing chain system step_config: Mapping[str, Any] = self.steps_config.get(step.name, {}) parallelism_config = step_config.get("parallelism") diff --git a/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/Cargo.toml b/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/Cargo.toml index 12e8324b..0befee44 100644 --- a/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/Cargo.toml +++ b/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/Cargo.toml @@ -11,7 +11,6 @@ crate-type = ["cdylib"] pyo3 = { version = "0.24" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -paste = "1.0" # In practice, users would depend on the published sentry_streams crate # which would re-export the macros @@ -20,4 +19,4 @@ paste = "1.0" # For this example, we'll include the dependencies directly [dependencies.rust_streams] path = "../../../../" # Point to the main rust_streams crate -default-features = false # Disable default features to avoid conflicts +default-features = false # Disable default features to avoid loading pyo3 a second time diff --git a/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/metrics_rust_transforms.pyi b/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/metrics_rust_transforms.pyi index 795d9c2b..b8b0de12 100644 --- a/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/metrics_rust_transforms.pyi +++ b/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/metrics_rust_transforms.pyi @@ -12,15 +12,12 @@ from typing import Any from sentry_kafka_schemas.schema_types.ingest_metrics_v1 import IngestMetric from sentry_streams.pipeline.message import Message -from sentry_streams.pipeline.rust_function_protocol import ( - RustFilterFunction, - RustMapFunction, -) +from sentry_streams.pipeline.rust_function_protocol import RustFunction -class RustFilterEvents(RustFilterFunction[IngestMetric]): +class RustFilterEvents(RustFunction[IngestMetric, bool]): def __init__(self) -> None: ... def __call__(self, msg: Message[IngestMetric]) -> bool: ... -class RustTransformMsg(RustMapFunction[IngestMetric, Any]): +class RustTransformMsg(RustFunction[IngestMetric, Any]): def __init__(self) -> None: ... - def __call__(self, msg: Message[IngestMetric]) -> Message[Any]: ... + def __call__(self, msg: Message[IngestMetric]) -> Any: ... diff --git a/sentry_streams/sentry_streams/pipeline/rust_function_protocol.py b/sentry_streams/sentry_streams/pipeline/rust_function_protocol.py index eaa02603..523606af 100644 --- a/sentry_streams/sentry_streams/pipeline/rust_function_protocol.py +++ b/sentry_streams/sentry_streams/pipeline/rust_function_protocol.py @@ -2,7 +2,7 @@ Protocol definitions for Rust functions to integrate with Python's type system """ -from typing import Protocol, TypeVar +from typing import Protocol, TypeVar, runtime_checkable from sentry_streams.pipeline.message import Message @@ -11,6 +11,7 @@ # External interface that we need users to use in their stubs for type-safety +@runtime_checkable class RustFunction(Protocol[TInput, TOutput]): def __call__(self, msg: Message[TInput]) -> TOutput: ... diff --git a/sentry_streams/src/ffi.rs b/sentry_streams/src/ffi.rs index ce6e1fe2..cc3349fd 100644 --- a/sentry_streams/src/ffi.rs +++ b/sentry_streams/src/ffi.rs @@ -217,3 +217,33 @@ impl IntoPythonPayload for bool { Ok(pyo3::types::PyBool::new(py, self).into_py_any(py)?) } } + +impl FromPythonPayload for bool { + fn from_python_payload(value: pyo3::Bound<'_, pyo3::PyAny>) -> pyo3::PyResult { + value.extract::() + } +} + +impl IntoPythonPayload for String { + fn into_python_payload(self, py: pyo3::Python<'_>) -> pyo3::PyResult> { + Ok(pyo3::types::PyString::new(py, &self).into_py_any(py)?) + } +} + +impl FromPythonPayload for String { + fn from_python_payload(value: pyo3::Bound<'_, pyo3::PyAny>) -> pyo3::PyResult { + value.extract::() + } +} + +impl IntoPythonPayload for u64 { + fn into_python_payload(self, py: pyo3::Python<'_>) -> pyo3::PyResult> { + Ok(self.into_py_any(py)?) + } +} + +impl FromPythonPayload for u64 { + fn from_python_payload(value: pyo3::Bound<'_, pyo3::PyAny>) -> pyo3::PyResult { + value.extract::() + } +} diff --git a/sentry_streams/tests/rust_test_functions/Cargo.lock b/sentry_streams/tests/rust_test_functions/Cargo.lock index 07054e4f..bd0e90f7 100644 --- a/sentry_streams/tests/rust_test_functions/Cargo.lock +++ b/sentry_streams/tests/rust_test_functions/Cargo.lock @@ -1265,7 +1265,6 @@ dependencies = [ "reqwest", "sentry_arroyo", "serde", - "serde_json", "tokio", "tracing", "tracing-subscriber", diff --git a/sentry_streams/tests/rust_test_functions/rust_test_functions.pyi b/sentry_streams/tests/rust_test_functions/rust_test_functions.pyi index 9af9e58a..895cad0b 100644 --- a/sentry_streams/tests/rust_test_functions/rust_test_functions.pyi +++ b/sentry_streams/tests/rust_test_functions/rust_test_functions.pyi @@ -1,19 +1,16 @@ from sentry_streams.pipeline.message import Message -from sentry_streams.pipeline.rust_function_protocol import ( - RustFilterFunction, - RustMapFunction, -) +from sentry_streams.pipeline.rust_function_protocol import RustFunction class TestMessage: ... -class TestFilterCorrect(RustFilterFunction[TestMessage]): +class TestFilterCorrect(RustFunction[TestMessage, bool]): def __call__(self, msg: Message[TestMessage]) -> bool: ... -class TestMapCorrect(RustMapFunction[TestMessage, str]): - def __call__(self, msg: Message[TestMessage]) -> Message[str]: ... +class TestMapCorrect(RustFunction[TestMessage, str]): + def __call__(self, msg: Message[TestMessage]) -> str: ... -class TestMapWrongType(RustMapFunction[bool, str]): - def __call__(self, msg: Message[bool]) -> Message[str]: ... +class TestMapWrongType(RustFunction[bool, str]): + def __call__(self, msg: Message[bool]) -> str: ... -class TestMapString(RustMapFunction[str, int]): - def __call__(self, msg: Message[str]) -> Message[int]: ... +class TestMapString(RustFunction[str, int]): + def __call__(self, msg: Message[str]) -> int: ... diff --git a/sentry_streams/tests/rust_test_functions/src/lib.rs b/sentry_streams/tests/rust_test_functions/src/lib.rs index 8616df54..9b382568 100644 --- a/sentry_streams/tests/rust_test_functions/src/lib.rs +++ b/sentry_streams/tests/rust_test_functions/src/lib.rs @@ -2,7 +2,7 @@ use pyo3::prelude::*; use serde::{Deserialize, Serialize}; // Import the macros from rust_streams -use rust_streams::{rust_filter_function, rust_map_function, Message}; +use rust_streams::{convert_via_json, rust_function, Message}; /// Test data structure for type validation tests #[derive(Serialize, Deserialize, Debug, Clone)] @@ -11,58 +11,45 @@ pub struct TestMessage { pub content: String, } -rust_filter_function!(TestFilterCorrect, TestMessage, |msg: Message< +// Implement type conversion for TestMessage +convert_via_json!(TestMessage); + +rust_function!(TestFilterCorrect, TestMessage, bool, |msg: Message< TestMessage, >| -> bool { - msg.payload.id > 0 + let (payload, _) = msg.take(); + payload.id > 0 }); -rust_map_function!(TestMapCorrect, TestMessage, String, |msg: Message< +rust_function!(TestMapCorrect, TestMessage, String, |msg: Message< TestMessage, >| - -> Message { - Message::new( - format!("Processed: {}", msg.payload.content), - msg.headers, - msg.timestamp, - msg.schema, - ) + -> String { + let (payload, _) = msg.take(); + format!("Processed: {}", payload.content) }); // Wrong type map function - accepts bool instead of TestMessage -rust_map_function!( +rust_function!( TestMapWrongType, bool, // This expects bool, but will get TestMessage in the test String, - |msg: Message| -> Message { - Message::new( - if msg.payload { - "true".to_string() - } else { - "false".to_string() - }, - msg.headers, - msg.timestamp, - msg.schema, - ) + |msg: Message| -> String { + let (payload, _) = msg.take(); + if payload { + "true".to_string() + } else { + "false".to_string() + } } ); // Map that accepts String (for testing chained operations) -rust_map_function!( - TestMapString, - String, - u64, - |msg: Message| -> Message { - Message::new( - msg.payload.len() as u64, - msg.headers, - msg.timestamp, - msg.schema, - ) - } -); +rust_function!(TestMapString, String, u64, |msg: Message| -> u64 { + let (payload, _) = msg.take(); + payload.len() as u64 +}); /// PyO3 module definition #[pymodule] diff --git a/sentry_streams/tests/test_mypy_integration.py b/sentry_streams/tests/test_mypy_integration.py index 2273a22b..5aef75b6 100644 --- a/sentry_streams/tests/test_mypy_integration.py +++ b/sentry_streams/tests/test_mypy_integration.py @@ -136,11 +136,8 @@ def test_rust_functions_have_proper_types(test_rust_extension) -> None: # type: assert callable(wrong_func) # Check protocol compliance - from sentry_streams.pipeline.rust_function_protocol import ( - RustFilterFunction, - RustMapFunction, - ) + from sentry_streams.pipeline.rust_function_protocol import RustFunction - assert isinstance(filter_func, RustFilterFunction) - assert isinstance(map_func, RustMapFunction) - assert isinstance(wrong_func, RustMapFunction) + assert isinstance(filter_func, RustFunction) + assert isinstance(map_func, RustFunction) + assert isinstance(wrong_func, RustFunction) From 70e4f44dce2292b221141c3f08ad9e9bdfc8a3f5 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Tue, 15 Jul 2025 14:29:56 +0200 Subject: [PATCH 09/16] add another testcase --- sentry_streams/tests/test_mypy_integration.py | 64 +++++++++++++------ 1 file changed, 44 insertions(+), 20 deletions(-) diff --git a/sentry_streams/tests/test_mypy_integration.py b/sentry_streams/tests/test_mypy_integration.py index 5aef75b6..1ba416e3 100644 --- a/sentry_streams/tests/test_mypy_integration.py +++ b/sentry_streams/tests/test_mypy_integration.py @@ -43,7 +43,7 @@ def test_rust_extension(): # type: ignore[no-untyped-def] pass -def test_mypy_detects_correct_pipeline(test_rust_extension) -> None: # type: ignore[no-untyped-def] +def test_mypy_detects_correct_pipeline_rust(test_rust_extension) -> None: # type: ignore[no-untyped-def] """Test that mypy accepts a correctly typed pipeline""" # Create a test file with correct types @@ -78,7 +78,8 @@ def create_correct_pipeline(): assert result.returncode == 0 -def test_mypy_detects_type_mismatch(test_rust_extension) -> None: # type: ignore[no-untyped-def] +@pytest.mark.xfail(reason="Type checking not working for both Rust and Python, apparently") +def test_mypy_detects_type_mismatch_rust(test_rust_extension) -> None: # type: ignore[no-untyped-def] """Test that mypy detects type mismatches in pipeline definitions""" wrong_code = """ @@ -118,26 +119,49 @@ def create_wrong_pipeline(): Path(temp_file).unlink() -def test_rust_functions_have_proper_types(test_rust_extension) -> None: # type: ignore[no-untyped-def] - """Test that Rust functions expose proper type information""" +@pytest.mark.xfail(reason="Type checking not working for both Rust and Python, apparently") +def test_mypy_detects_type_mismatch_python() -> None: # type: ignore[no-untyped-def] + """Test that mypy detects type mismatches in pipeline definitions with Python functions""" - filter_func = test_rust_extension.TestFilterCorrect() - map_func = test_rust_extension.TestMapCorrect() - wrong_func = test_rust_extension.TestMapWrongType() + wrong_code = """ +from sentry_streams.pipeline.pipeline import Filter, Map, Parser, streaming_source, StreamSink +from sentry_streams.pipeline.message import Message - # Check that they have the required methods for type checking - assert hasattr(filter_func, "__call__") - assert hasattr(map_func, "__call__") - assert hasattr(wrong_func, "__call__") +class TestMessage: + pass - # Check that they're callable - assert callable(filter_func) - assert callable(map_func) - assert callable(wrong_func) +def filter_correct(msg: Message[TestMessage]) -> bool: + return True - # Check protocol compliance - from sentry_streams.pipeline.rust_function_protocol import RustFunction +def map_wrong_type(msg: Message[bool]) -> str: # This expects bool but will get TestMessage + return "test" + +# expect failure: map_wrong_type expects Message[bool] but gets Message[TestMessage] +def create_wrong_pipeline(): + return ( + streaming_source("input", "test-stream") + .apply(Parser("parser", msg_type=TestMessage)) # bytes -> TestMessage + .apply(Filter("filter", function=filter_correct)) # TestMessage -> TestMessage + .apply(Map("wrong", function=map_wrong_type)) # Expects Message[bool], gets Message[TestMessage]! + .sink(StreamSink("output", "output-stream")) + ) +""" - assert isinstance(filter_func, RustFunction) - assert isinstance(map_func, RustFunction) - assert isinstance(wrong_func, RustFunction) + # Write to temp file and run mypy + with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f: + f.write(wrong_code) + temp_file = f.name + + try: + result = subprocess.run( + [sys.executable, "-m", "mypy", temp_file, "--show-error-codes", "--check-untyped-defs"], + capture_output=True, + text=True, + ) + + # Should detect type mismatch + assert result.returncode > 0 + assert 'Argument "function" to "Map" has incompatible type' in result.stdout + + finally: + Path(temp_file).unlink() From 4ecfd9df9d21e2b3e228284143491bb0d50dbb5d Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Wed, 16 Jul 2025 18:32:28 +0200 Subject: [PATCH 10/16] fix additional typing issues --- Makefile | 1 + .../rust_simple_map_filter/rust_transforms/Cargo.lock | 8 -------- sentry_streams/sentry_streams/pipeline/pipeline.py | 10 ++++++---- .../sentry_streams/pipeline/rust_function_protocol.py | 2 +- sentry_streams/tests/test_mypy_integration.py | 6 +++--- 5 files changed, 11 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index 2d24cd9b..8b63d8a7 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,7 @@ tests-flink: .PHONY: tests-flink typecheck: + cd ./sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/ && maturin develop ./sentry_streams/.venv/bin/mypy --config-file sentry_streams/mypy.ini --strict sentry_streams/ ./sentry_flink/.venv/bin/mypy --config-file sentry_flink/mypy.ini --strict sentry_flink/sentry_flink/ .PHONY: typecheck diff --git a/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/Cargo.lock b/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/Cargo.lock index 395d7158..1077014d 100644 --- a/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/Cargo.lock +++ b/sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/Cargo.lock @@ -745,7 +745,6 @@ dependencies = [ name = "metrics_rust_transforms" version = "0.1.0" dependencies = [ - "paste", "pyo3", "rust_streams", "serde", @@ -942,12 +941,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - [[package]] name = "percent-encoding" version = "2.3.1" @@ -1266,7 +1259,6 @@ dependencies = [ "reqwest", "sentry_arroyo", "serde", - "serde_json", "tokio", "tracing", "tracing-subscriber", diff --git a/sentry_streams/sentry_streams/pipeline/pipeline.py b/sentry_streams/sentry_streams/pipeline/pipeline.py index 7f85003e..cadd4c0c 100644 --- a/sentry_streams/sentry_streams/pipeline/pipeline.py +++ b/sentry_streams/sentry_streams/pipeline/pipeline.py @@ -319,13 +319,15 @@ def resolved_function(self) -> Callable[[Message[TIn]], TOut]: function_callable = imported_func return function_callable - def _validate_rust_function(self): + def _validate_rust_function(self) -> Callable[[Message[TIn]], TOut] | None: func = self.resolved_function if not hasattr(func, "rust_function_version"): # not a rust function return None - rust_function_version = func.rust_function_version() # type: ignore[attr-defined] + func = cast(InternalRustFunction[TIn, TOut], func) + + rust_function_version = func.rust_function_version() if rust_function_version != 1: raise TypeError( r"Invalid rust function version: {rust_function_version} -- if you are defining your own rust functions, maybe the version is out of date?" @@ -333,7 +335,7 @@ def _validate_rust_function(self): return func - def post_rust_function_validation(self, func: InternalRustFunction) -> None: + def post_rust_function_validation(self, func: InternalRustFunction[TIn, TOut]) -> None: # Overridden in Filter step pass @@ -372,7 +374,7 @@ class Filter(Transform[TIn, TIn], Generic[TIn]): function: Union[Callable[[Message[TIn]], bool], str] step_type: StepType = StepType.FILTER - def post_rust_function_validation(self, func: InternalRustFunction) -> None: + def post_rust_function_validation(self, func: InternalRustFunction[TIn, TOut]) -> None: output_type = func.output_type() if output_type != "bool": raise TypeError( diff --git a/sentry_streams/sentry_streams/pipeline/rust_function_protocol.py b/sentry_streams/sentry_streams/pipeline/rust_function_protocol.py index 523606af..471d4925 100644 --- a/sentry_streams/sentry_streams/pipeline/rust_function_protocol.py +++ b/sentry_streams/sentry_streams/pipeline/rust_function_protocol.py @@ -17,7 +17,7 @@ def __call__(self, msg: Message[TInput]) -> TOutput: ... # Methods that we use internally, but don't want the user to see (or have to write out in their stubfiles) -class InternalRustFunction(RustFunction, Protocol): +class InternalRustFunction(RustFunction[TInput, TOutput], Protocol): def input_type(self) -> str: ... def output_type(self) -> str: ... def rust_function_version(self) -> int: ... diff --git a/sentry_streams/tests/test_mypy_integration.py b/sentry_streams/tests/test_mypy_integration.py index bedf9eed..7c73f377 100644 --- a/sentry_streams/tests/test_mypy_integration.py +++ b/sentry_streams/tests/test_mypy_integration.py @@ -12,7 +12,7 @@ @pytest.fixture(scope="module") -def test_rust_extension(): # type: ignore[no-untyped-def] +def test_rust_extension() -> object: """Build the test Rust extension before running tests""" test_crate_dir = Path(__file__).parent / "rust_test_functions" @@ -135,7 +135,7 @@ def create_wrong_pipeline(): assert "expected" in result.stdout and "Mapping" in result.stdout -def test_mypy_detects_correct_pipeline_rust(tmp_path: Path, test_rust_extension) -> None: +def test_mypy_detects_correct_pipeline_rust(tmp_path: Path, test_rust_extension: object) -> None: """Test that mypy accepts a correctly typed pipeline""" # Create a test file with correct types @@ -182,7 +182,7 @@ def create_correct_pipeline(): assert "Success: no issues found in 1 source file" in result.stdout -def test_mypy_detects_type_mismatch_rust(tmp_path: Path, test_rust_extension) -> None: +def test_mypy_detects_type_mismatch_rust(tmp_path: Path, test_rust_extension: object) -> None: """Test that mypy detects type mismatches in pipeline definitions""" wrong_code = """ From 9fe2d699398a4fcfc83131850d6c5d40b340be98 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Wed, 16 Jul 2025 18:46:39 +0200 Subject: [PATCH 11/16] fix make typecheck --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8b63d8a7..09b7ad0c 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ tests-flink: .PHONY: tests-flink typecheck: - cd ./sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/ && maturin develop + . ./sentry_streams/.venv/bin/activate && cd ./sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/ && maturin develop ./sentry_streams/.venv/bin/mypy --config-file sentry_streams/mypy.ini --strict sentry_streams/ ./sentry_flink/.venv/bin/mypy --config-file sentry_flink/mypy.ini --strict sentry_flink/sentry_flink/ .PHONY: typecheck From 760af3591bb871476485312c80ba396f53e8e737 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Wed, 16 Jul 2025 18:48:10 +0200 Subject: [PATCH 12/16] disable broken doctest --- sentry_streams/src/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_streams/src/utils.rs b/sentry_streams/src/utils.rs index 59e20e37..25e49683 100644 --- a/sentry_streams/src/utils.rs +++ b/sentry_streams/src/utils.rs @@ -8,7 +8,7 @@ /// /// # Examples /// -/// ``` +/// ```nocheck /// let py_err = ...; /// traced_with_gil!(|py| { py_err.print(py) }).unwrap(); /// From 12f34a6ce9786bce2da9510489b456737e923b57 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Mon, 4 Aug 2025 12:30:55 +0200 Subject: [PATCH 13/16] try to fix ci --- Makefile | 5 ++++- sentry_streams/tests/test_mypy_integration.py | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 09b7ad0c..a69eb389 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,10 @@ tests-integration: ./sentry_streams/.venv/bin/pytest -vv sentry_streams/integration_tests .PHONY: tests-integration -test-rust-streams: +test-rust-streams: tests-rust-streams +.PHONY: test-rust-streams + +tests-rust-streams: . sentry_streams/.venv/bin/activate && . scripts/rust-envvars && cd ./sentry_streams/ && cargo test .PHONY: tests-rust-streams diff --git a/sentry_streams/tests/test_mypy_integration.py b/sentry_streams/tests/test_mypy_integration.py index 7c73f377..cfe37df1 100644 --- a/sentry_streams/tests/test_mypy_integration.py +++ b/sentry_streams/tests/test_mypy_integration.py @@ -16,9 +16,11 @@ def test_rust_extension() -> object: """Build the test Rust extension before running tests""" test_crate_dir = Path(__file__).parent / "rust_test_functions" + maturin_path = Path(sys.exec_prefix) / "bin/maturin" + # Build the extension result = subprocess.run( - ["maturin", "develop"], cwd=test_crate_dir, capture_output=True, text=True + [maturin_path, "develop"], cwd=test_crate_dir, capture_output=True, text=True ) if result.returncode != 0: From 710e9a4a3717a55fd85ce072aa039fbaa06d05f7 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Mon, 4 Aug 2025 12:34:05 +0200 Subject: [PATCH 14/16] add some basic rust design docs --- sentry_streams/docs/source/index.rst | 1 + sentry_streams/docs/source/rust.rst | 199 +++++++++++++++++++++++++++ 2 files changed, 200 insertions(+) create mode 100644 sentry_streams/docs/source/rust.rst diff --git a/sentry_streams/docs/source/index.rst b/sentry_streams/docs/source/index.rst index 73e97daa..08a0f19c 100644 --- a/sentry_streams/docs/source/index.rst +++ b/sentry_streams/docs/source/index.rst @@ -12,3 +12,4 @@ configure_pipeline runtime/arroyo deployment + rust diff --git a/sentry_streams/docs/source/rust.rst b/sentry_streams/docs/source/rust.rst new file mode 100644 index 00000000..d1f3da56 --- /dev/null +++ b/sentry_streams/docs/source/rust.rst @@ -0,0 +1,199 @@ +Rust applications +================= + +Hybrid applications +------------------- + +PR: https://github.com/getsentry/streams/pull/177 + +**User story:** I want to rewrite a pipeline step in getsentry monolith +in Rust. + +Currently Rust-ification within the monolith is being done by adding new +pyo3-based Python dependencies to getsentry’s requirements. We’ll go the +same path, users can define pipeline steps using pyo3, but using our +helper functions/”framework.” + +Here is how a function definition works: + +.. code:: rust + + // mypackage/src/lib.rs + sentry_streams::rust_function!( + RustTransformMsg, + IngestMetric, + TransformedIngestMetric, + |msg: Message| -> TransformedIngestMetric { + let (payload, _) = msg.take(); + TransformedIngestMetric { + metric_type: payload.metric_type, + name: payload.name, + value: payload.value, + tags: payload.tags, + timestamp: payload.timestamp, + transformed: true, + } + } + ); + +This would be packaged up in a pyo3-based crate, and then can be +referenced from the regular pipeline definition like this: + +.. code:: python + + .apply(Map("transform", function=my_package.RustTransformMsg())) + +Message payloads +~~~~~~~~~~~~~~~~ + +``IngestMetric`` and ``TransformedIngestMetric`` types have to be +defined by the user in both Rust and Python. + +.. code:: rust + + // mypackage/src/lib.rs + #[derive(Serialize, Deserialize) + struct IngestMetric { ... } + +.. code:: python + + class IngestMetric(TypedDict): ... + +The user has to write their own Python ``.pyi`` stub file to declare +that ``RustTransformMsg`` takes ``IngestMetric`` and returns +``TransformedIngestMetric``: + +.. code:: python + + # mypackage/mypackage.pyi + class RustTransformMsg(RustFunction[IngestMetric, Any]): + def __init__(self) -> None: ... + def __call__(self, msg: Message[IngestMetric]) -> Any: ... + +Then, the user has to define how conversion works between these types. +They can implement this function manually, or use a builtin conversion +method provided by us. We currently only provide one builtin conversion +by round-tripping via JSON: + +.. code:: rust + + // mypackage/src/lib.rs + sentry_streams::convert_via_json!(IngestMetric); + +…and the same procedure has to be repeated for the output type +``TransformedIngestMetric``. + +What happens at runtime +~~~~~~~~~~~~~~~~~~~~~~~ + +The ``rust_function`` macro currently just generates a simple Python +function for the given Rust function. The GIL *is* released while the +user’s Rust code is running, but there is still some GIL overhead when +entering and exiting the function. + +In the future we can transparently optimize this without users having to +change their applications. For example, batching function calls to +amortize GIL overhead. We would then only hold the GIL while entering +and exiting the batch. + +What we want to improve in the future +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- improve performance of calling convention/reduce overhead + + - take inspiration from + https://github.com/ealmloff/sledgehammer_bindgen + +- automatically generate type stubs for user’s Rust code — pyo3 does + have something like that, but it doesn’t work perfectly (exposes + internals of our Rust macro) +- improve ergonomics of message types and their conversion, add protobuf + or msgpack as a way to roundtrip +- each team at sentry would have to maintain a new python package for + their Rust functions, set up pyo3 and CI from scratch, etc. we can + streamline this. + + - we already have: ``sentry_relay`` (relay integration), ``ophio`` + (grouping engine), ``vroomrs`` (profiles), ``symbolic`` (stacktrace + processing) + - easiest: we provide a “monorepo” and “monopackage” where all rust + functions for getsentry go. we maintain CI for this monorepo. + - medium: repository template + - also, ideally this is aligned with devinfra’s “golden path” for + python devenv + - in practice some team will have to provide support for questions + about pyo3, since its entire API surface is exposed to product teams + (although we can templatize and abstract a lot) + +Pure-Rust pipelines +------------------- + +A lot of the complexity mentioned above is only really necessary for +when you want to mix Python and Rust code. For pure-Rust applications, +we could do something entirely different: + +- The runner does not have to be started from Python at all. If we + started it from Rust, we would have a much easier time optimizing + function calls. +- The pipeline definition does not have to be Python. We could have it + be YAML or even Rust as well. +- Type stubs are not really necessary. We can easily validate that the + types match during startup, or if the pipeline definition is in Rust, + let the compiler do that job for us. + +Any of these will however split the ecosystem. I think we have plenty of +ergonomic improvements we can make even for hybrid applications, that +would benefit pure-Rust users as well. We should focus on those first. + +Meeting notes July 24, 2025 +=========================== + +- a better pure-rust story + + - we have too much boilerplate, and now especially for pure rust apps + - build a rust runner, and try to get rid of as much pyo3 junk as + possible + + - reference: `The rust arroyo + runtime `__ + + - maybe hybrid will get better through this rearchitecture + - maybe denormalize Parse steps into Map (@Filippo Pacifici) + + .. code:: rust + + + // mypackage/src/lib.rs as pyo3 + use sentry_streams; + + sentry_streams::rust_function!(...); + + sentry_streams::main_function!(); + + // or, in bin target: + pub use sentry_streams::main; + + .. code:: python + + + mypackage.run_streams() + + concerns: + + - user can freely downgrade/upgrade verison, since they “own” the + runtime (as they are statically linking it) + - ability to opt out of message conversion trait requirements + +- message type conversion + + - boilerplate is an issue + + - integration with existing schema repos, or copy schema-to-type + generation into streams for “inline schemas” + + - better performance + +- better runtime semantics for rust functions + + - map chains, but in rust? + - no multiprocessing! From 901405659596aaf41f5718dabc4c9403027a775b Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Mon, 4 Aug 2025 15:55:27 +0200 Subject: [PATCH 15/16] add some rust tests, incl full pipeline test with localbroker --- Makefile | 1 + sentry_streams/src/ffi.rs | 31 +-- sentry_streams/tests/conftest.py | 48 ++++ sentry_streams/tests/test_mypy_integration.py | 39 +-- .../tests/test_rust_functions_integration.py | 263 ++++++++++++++++++ 5 files changed, 317 insertions(+), 65 deletions(-) create mode 100644 sentry_streams/tests/conftest.py create mode 100644 sentry_streams/tests/test_rust_functions_integration.py diff --git a/Makefile b/Makefile index a69eb389..662774f2 100644 --- a/Makefile +++ b/Makefile @@ -35,6 +35,7 @@ tests-flink: typecheck: . ./sentry_streams/.venv/bin/activate && cd ./sentry_streams/sentry_streams/examples/rust_simple_map_filter/rust_transforms/ && maturin develop + . ./sentry_streams/.venv/bin/activate && cd ./sentry_streams/tests/rust_test_functions/ && maturin develop ./sentry_streams/.venv/bin/mypy --config-file sentry_streams/mypy.ini --strict sentry_streams/ ./sentry_flink/.venv/bin/mypy --config-file sentry_flink/mypy.ini --strict sentry_flink/sentry_flink/ .PHONY: typecheck diff --git a/sentry_streams/src/ffi.rs b/sentry_streams/src/ffi.rs index cc3349fd..b6420dbe 100644 --- a/sentry_streams/src/ffi.rs +++ b/sentry_streams/src/ffi.rs @@ -129,32 +129,6 @@ where }) } -pub fn convert_rust_message_to_py( - py: pyo3::Python, - rust_msg: Message, -) -> pyo3::PyResult> -where - T: IntoPythonPayload, -{ - let payload_obj = rust_msg.payload.into_python_payload(py)?; - - // Create a new Python message with the transformed payload - let py_msg_class = py - .import("sentry_streams.rust_streams")? - .getattr("PyAnyMessage")?; - - let result = py_msg_class - .call1(( - payload_obj, - rust_msg.headers, - rust_msg.timestamp, - rust_msg.schema, - ))? - .unbind(); - - Ok(result) -} - /// Macro to create a Rust map function that can be called from Python /// Usage: rust_map_function!(MyFunction, InputType, OutputType, |msg: Message| -> OutputType { ... }); #[macro_export] @@ -192,8 +166,9 @@ macro_rules! rust_function { metadata_clone.map(|()| result_payload) }); - // Convert result back to Python message - $crate::ffi::convert_rust_message_to_py(py, result_msg) + // Return the raw payload directly (not wrapped in a message) + let (payload, _) = result_msg.take(); + $crate::ffi::IntoPythonPayload::into_python_payload(payload, py) } pub fn input_type(&self) -> &'static str { diff --git a/sentry_streams/tests/conftest.py b/sentry_streams/tests/conftest.py new file mode 100644 index 00000000..a1a06310 --- /dev/null +++ b/sentry_streams/tests/conftest.py @@ -0,0 +1,48 @@ +"""Shared test fixtures for sentry_streams tests""" + +import subprocess +import sys +from pathlib import Path +from typing import Any + +import pytest + + +@pytest.fixture(scope="module") +def rust_test_functions() -> Any: + """Build and import the test Rust functions + + This fixture builds the rust_test_functions crate using maturin + and makes it available for import in tests. The build happens + once per test module for efficiency. + + Returns: + module: The imported rust_test_functions module + """ + test_crate_dir = Path(__file__).parent / "rust_test_functions" + maturin_path = Path(sys.exec_prefix) / "bin/maturin" + + # Build the extension + result = subprocess.run( + [maturin_path, "develop"], cwd=test_crate_dir, capture_output=True, text=True + ) + + if result.returncode != 0: + pytest.fail(f"Failed to build test Rust extension: {result.stderr}") + + # Import and return the module + try: + import rust_test_functions + except ImportError as e: + pytest.fail(f"Failed to import test Rust extension: {e}") + + yield rust_test_functions + + # Clean up - try to uninstall the test extension (best effort) + try: + subprocess.run( + ["uv", "pip", "uninstall", "rust-test-functions", "-y"], + capture_output=True, + ) + except Exception: + pass diff --git a/sentry_streams/tests/test_mypy_integration.py b/sentry_streams/tests/test_mypy_integration.py index cfe37df1..5b820bc5 100644 --- a/sentry_streams/tests/test_mypy_integration.py +++ b/sentry_streams/tests/test_mypy_integration.py @@ -8,41 +8,6 @@ import sys from pathlib import Path -import pytest - - -@pytest.fixture(scope="module") -def test_rust_extension() -> object: - """Build the test Rust extension before running tests""" - test_crate_dir = Path(__file__).parent / "rust_test_functions" - - maturin_path = Path(sys.exec_prefix) / "bin/maturin" - - # Build the extension - result = subprocess.run( - [maturin_path, "develop"], cwd=test_crate_dir, capture_output=True, text=True - ) - - if result.returncode != 0: - pytest.fail(f"Failed to build test Rust extension: {result.stderr}") - - # Import and return the module - try: - import rust_test_functions # type: ignore[import-not-found] - except ImportError as e: - pytest.fail(f"Failed to import test Rust extension: {e}") - - yield rust_test_functions - - # Try to uninstall the test extension (best effort) - try: - subprocess.run( - ["uv", "pip", "uninstall", "rust-test-functions"], - capture_output=True, - ) - except Exception: - pass - def test_mypy_detects_correct_pipeline(tmp_path: Path) -> None: """Test that mypy accepts a correctly typed pipeline""" @@ -137,7 +102,7 @@ def create_wrong_pipeline(): assert "expected" in result.stdout and "Mapping" in result.stdout -def test_mypy_detects_correct_pipeline_rust(tmp_path: Path, test_rust_extension: object) -> None: +def test_mypy_detects_correct_pipeline_rust(tmp_path: Path, rust_test_functions: object) -> None: """Test that mypy accepts a correctly typed pipeline""" # Create a test file with correct types @@ -184,7 +149,7 @@ def create_correct_pipeline(): assert "Success: no issues found in 1 source file" in result.stdout -def test_mypy_detects_type_mismatch_rust(tmp_path: Path, test_rust_extension: object) -> None: +def test_mypy_detects_type_mismatch_rust(tmp_path: Path, rust_test_functions: object) -> None: """Test that mypy detects type mismatches in pipeline definitions""" wrong_code = """ diff --git a/sentry_streams/tests/test_rust_functions_integration.py b/sentry_streams/tests/test_rust_functions_integration.py new file mode 100644 index 00000000..02848ebe --- /dev/null +++ b/sentry_streams/tests/test_rust_functions_integration.py @@ -0,0 +1,263 @@ +"""Integration tests for Rust functions in streaming pipelines""" + +import json +from typing import Any, cast + +import pytest +from arroyo.backends.kafka import KafkaConsumer, KafkaProducer +from arroyo.backends.kafka.consumer import KafkaPayload +from arroyo.backends.local.backend import LocalBroker +from arroyo.backends.local.storages.memory import MemoryMessageStorage +from arroyo.types import Partition, Topic +from arroyo.utils.clock import MockedClock + +from sentry_streams.adapters.arroyo.adapter import ArroyoAdapter +from sentry_streams.adapters.loader import load_adapter +from sentry_streams.adapters.stream_adapter import PipelineConfig, RuntimeTranslator +from sentry_streams.dummy.dummy_adapter import DummyAdapter +from sentry_streams.pipeline.message import PyMessage as Message +from sentry_streams.pipeline.pipeline import ( + Filter, + Map, + Serializer, + StreamSink, + streaming_source, +) +from sentry_streams.runner import iterate_edges + + +def test_basic_rust_function_execution(rust_test_functions: Any) -> None: + """Test that Rust functions execute correctly in a pipeline""" + from rust_test_functions import TestFilterCorrect, TestMapCorrect + + # TestMessage in Rust corresponds to dicts with id/content in Python + test_messages = [ + Message(payload=cast(Any, {"id": 1, "content": "Hello"}), headers=[], timestamp=0.0), + Message( + payload=cast(Any, {"id": 0, "content": "Should be filtered"}), headers=[], timestamp=0.0 + ), + Message(payload=cast(Any, {"id": 2, "content": "World"}), headers=[], timestamp=0.0), + ] + + rust_filter = TestFilterCorrect() + rust_map = TestMapCorrect() + + filtered_messages = [] + for msg in test_messages: + if rust_filter(msg): + filtered_messages.append(msg) + + mapped_messages = [] + for msg in filtered_messages: + mapped_messages.append(rust_map(msg)) + + assert len(filtered_messages) == 2 + # Type ignore because the payload is actually a dict at runtime but typed differently + assert filtered_messages[0].payload["id"] == 1 + assert filtered_messages[1].payload["id"] == 2 + + assert mapped_messages[0] == "Processed: Hello" + assert mapped_messages[1] == "Processed: World" + + +def test_rust_python_interoperability(rust_test_functions: Any) -> None: + """Test that Rust and Python functions work together in pipelines""" + from rust_test_functions import TestFilterCorrect, TestMapCorrect + + def python_uppercase(msg: Message[str]) -> str: + return msg.payload.upper() + + def python_length(msg: Message[str]) -> int: + return len(msg.payload) + + test_msg = Message(payload=cast(Any, {"id": 5, "content": "test"}), headers=[], timestamp=0.0) + + rust_filter = TestFilterCorrect() + rust_map = TestMapCorrect() + + if rust_filter(test_msg): + rust_output = rust_map(test_msg) + py_msg = Message(payload=rust_output, headers=[], timestamp=0.0) + uppercase_output = python_uppercase(py_msg) + length_msg = Message(payload=uppercase_output, headers=[], timestamp=0.0) + final_output = python_length(length_msg) + + assert rust_output == "Processed: test" + assert uppercase_output == "PROCESSED: TEST" + assert final_output == 15 + + +def test_error_handling_in_rust_functions(rust_test_functions: Any) -> None: + """Test error handling when Rust functions fail""" + from rust_test_functions import TestMapWrongType + + wrong_type_map = TestMapWrongType() + + # Note: We're deliberately passing wrong type (dict instead of bool) to test error handling + test_msg = Message(payload=cast(Any, {"id": 1, "content": "test"}), headers=[], timestamp=0.0) + + with pytest.raises(TypeError, match="'dict' object cannot be converted to 'PyBool'"): + wrong_type_map(test_msg) # Intentionally wrong type for error testing + + +def test_complex_data_serialization(rust_test_functions: Any) -> None: + """Test that complex data structures survive Rust function roundtrips""" + from rust_test_functions import TestMapCorrect + + complex_msg = {"id": 12345, "content": 'Test with special chars: 你好 🚀 \n\t"quotes"'} + + rust_map = TestMapCorrect() + msg = Message(payload=cast(Any, complex_msg), headers=[], timestamp=123456789.0) + + result = rust_map(msg) + + assert result == 'Processed: Test with special chars: 你好 🚀 \n\t"quotes"' + + empty_msg = Message(payload=cast(Any, {"id": 1, "content": ""}), headers=[], timestamp=0.0) + assert rust_map(empty_msg) == "Processed: " + + +def test_chained_rust_functions(rust_test_functions: Any) -> None: + """Test multiple Rust functions chained together""" + from rust_test_functions import ( + TestFilterCorrect, + TestMapCorrect, + TestMapString, + ) + + rust_filter = TestFilterCorrect() + rust_map_to_string = TestMapCorrect() + rust_map_to_length = TestMapString() + + test_messages = [ + Message(payload=cast(Any, {"id": 3, "content": "Hello World!"}), headers=[], timestamp=0.0), + Message(payload=cast(Any, {"id": 0, "content": "Filtered"}), headers=[], timestamp=0.0), + Message(payload=cast(Any, {"id": 10, "content": "Short"}), headers=[], timestamp=0.0), + ] + + results = [] + for msg in test_messages: + if rust_filter(msg): + string_result = rust_map_to_string(msg) + string_msg = Message(payload=string_result, headers=[], timestamp=0.0) + length_result = rust_map_to_length(string_msg) + results.append(length_result) + + assert len(results) == 2 + assert results[0] == len("Processed: Hello World!") + assert results[1] == len("Processed: Short") + + +def test_rust_functions_in_pipeline_structure(rust_test_functions: Any) -> None: + """Test that Rust functions work in actual pipeline infrastructure""" + from rust_test_functions import TestFilterCorrect, TestMapCorrect + + # Create a pipeline using the same structure as the examples + # The pipeline will handle type conversion from bytes to TestMessage + pipeline = ( + streaming_source("input", stream_name="test-messages") + .apply(Filter("rust_filter", function=cast(Any, TestFilterCorrect()))) + .apply(Map("rust_map", function=cast(Any, TestMapCorrect()))) + .apply(Serializer("serializer")) + .sink(StreamSink("output", stream_name="processed-messages")) + ) + + # Use the dummy adapter to verify pipeline structure + dummy_config: PipelineConfig = {} + adapter: DummyAdapter[Any, Any] = load_adapter("dummy", dummy_config, None) # type: ignore + translator: RuntimeTranslator[Any, Any] = RuntimeTranslator(adapter) + + # Execute pipeline structure creation + iterate_edges(pipeline, translator) + + # Verify that the pipeline was built with Rust functions + assert "input" in adapter.input_streams + assert "rust_filter" in adapter.input_streams + assert "rust_map" in adapter.input_streams + assert "output" in adapter.input_streams + + # Verify the pipeline structure includes our Rust function steps + expected_streams = ["input", "rust_filter", "rust_map", "serializer", "output"] + for stream in expected_streams: + assert stream in adapter.input_streams, f"Missing stream: {stream}" + + +def test_rust_functions_with_message_flow(rust_test_functions: Any) -> None: + """Test that Rust functions process actual messages through a pipeline""" + from rust_test_functions import TestFilterCorrect, TestMapCorrect + + # This test demonstrates that Rust functions work in the pipeline infrastructure + # by creating a pipeline and showing that messages flow through Rust functions + # Create in-memory broker + storage = MemoryMessageStorage[KafkaPayload]() + broker = LocalBroker(storage, MockedClock()) + broker.create_topic(Topic("ingest-metrics"), 1) + broker.create_topic(Topic("transformed-events"), 1) + + # Create pipeline that uses Rust functions + def parse_json_bytes(msg: Message[bytes]) -> Any: + """Parse JSON bytes and cast to dict for testing""" + parsed_dict = json.loads(msg.payload.decode("utf-8")) + return parsed_dict + + # Track processed messages to verify Rust functions executed + processed_messages = [] + + def capture_result(msg: Message[str]) -> str: + """Capture the result from Rust map function""" + processed_messages.append(msg.payload) + return msg.payload + + pipeline = ( + streaming_source("input", stream_name="ingest-metrics") + .apply(Map("json_parser", function=cast(Any, parse_json_bytes))) + .apply(Filter("rust_filter", function=cast(Any, TestFilterCorrect()))) + .apply(Map("rust_map", function=cast(Any, TestMapCorrect()))) + .apply(Map("capture", function=cast(Any, capture_result))) + .apply(Serializer("serializer")) + .sink(StreamSink("output", stream_name="transformed-events")) + ) + + # Setup ArroyoAdapter with LocalBroker + adapter = ArroyoAdapter.build( + { + "env": {}, + "steps_config": { + "input": {"input": {}}, + "output": {"output": {}}, + }, + }, + {"input": cast(KafkaConsumer, broker.get_consumer("ingest-metrics"))}, + {"output": cast(KafkaProducer, broker.get_producer())}, + ) + + # Configure and create pipeline processors + iterate_edges(pipeline, RuntimeTranslator(adapter)) + adapter.create_processors() + processor = adapter.get_processor("input") + + # Send test messages + test_messages = [ + {"id": 1, "content": "Hello"}, # Should pass filter (id > 0) + {"id": 0, "content": "Filtered"}, # Should be filtered out (id = 0) + {"id": 2, "content": "World"}, # Should pass filter (id > 0) + ] + + for msg in test_messages: + broker.produce( + Partition(Topic("ingest-metrics"), 0), + KafkaPayload(None, json.dumps(msg).encode("utf-8"), []), + ) + + # Process messages through pipeline + processor._run_once() # Process first message + processor._run_once() # Process second message + processor._run_once() # Process third message + + # Verify that Rust functions processed the messages correctly + # This demonstrates that Rust functions execute within pipeline infrastructure + assert len(processed_messages) == 2 # Only messages with id > 0 passed the filter + assert processed_messages[0] == "Processed: Hello" + assert processed_messages[1] == "Processed: World" + + # The key success: Rust functions executed and transformed data within the pipeline! From 96a7494a2744e6182f16a9a94db67dd9fa8cc1e2 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Mon, 4 Aug 2025 16:04:35 +0200 Subject: [PATCH 16/16] remove some useless tests --- .../tests/test_rust_functions_integration.py | 127 +----------------- 1 file changed, 1 insertion(+), 126 deletions(-) diff --git a/sentry_streams/tests/test_rust_functions_integration.py b/sentry_streams/tests/test_rust_functions_integration.py index 02848ebe..819b0f10 100644 --- a/sentry_streams/tests/test_rust_functions_integration.py +++ b/sentry_streams/tests/test_rust_functions_integration.py @@ -3,7 +3,6 @@ import json from typing import Any, cast -import pytest from arroyo.backends.kafka import KafkaConsumer, KafkaProducer from arroyo.backends.kafka.consumer import KafkaPayload from arroyo.backends.local.backend import LocalBroker @@ -12,9 +11,7 @@ from arroyo.utils.clock import MockedClock from sentry_streams.adapters.arroyo.adapter import ArroyoAdapter -from sentry_streams.adapters.loader import load_adapter -from sentry_streams.adapters.stream_adapter import PipelineConfig, RuntimeTranslator -from sentry_streams.dummy.dummy_adapter import DummyAdapter +from sentry_streams.adapters.stream_adapter import RuntimeTranslator from sentry_streams.pipeline.message import PyMessage as Message from sentry_streams.pipeline.pipeline import ( Filter, @@ -60,128 +57,6 @@ def test_basic_rust_function_execution(rust_test_functions: Any) -> None: assert mapped_messages[1] == "Processed: World" -def test_rust_python_interoperability(rust_test_functions: Any) -> None: - """Test that Rust and Python functions work together in pipelines""" - from rust_test_functions import TestFilterCorrect, TestMapCorrect - - def python_uppercase(msg: Message[str]) -> str: - return msg.payload.upper() - - def python_length(msg: Message[str]) -> int: - return len(msg.payload) - - test_msg = Message(payload=cast(Any, {"id": 5, "content": "test"}), headers=[], timestamp=0.0) - - rust_filter = TestFilterCorrect() - rust_map = TestMapCorrect() - - if rust_filter(test_msg): - rust_output = rust_map(test_msg) - py_msg = Message(payload=rust_output, headers=[], timestamp=0.0) - uppercase_output = python_uppercase(py_msg) - length_msg = Message(payload=uppercase_output, headers=[], timestamp=0.0) - final_output = python_length(length_msg) - - assert rust_output == "Processed: test" - assert uppercase_output == "PROCESSED: TEST" - assert final_output == 15 - - -def test_error_handling_in_rust_functions(rust_test_functions: Any) -> None: - """Test error handling when Rust functions fail""" - from rust_test_functions import TestMapWrongType - - wrong_type_map = TestMapWrongType() - - # Note: We're deliberately passing wrong type (dict instead of bool) to test error handling - test_msg = Message(payload=cast(Any, {"id": 1, "content": "test"}), headers=[], timestamp=0.0) - - with pytest.raises(TypeError, match="'dict' object cannot be converted to 'PyBool'"): - wrong_type_map(test_msg) # Intentionally wrong type for error testing - - -def test_complex_data_serialization(rust_test_functions: Any) -> None: - """Test that complex data structures survive Rust function roundtrips""" - from rust_test_functions import TestMapCorrect - - complex_msg = {"id": 12345, "content": 'Test with special chars: 你好 🚀 \n\t"quotes"'} - - rust_map = TestMapCorrect() - msg = Message(payload=cast(Any, complex_msg), headers=[], timestamp=123456789.0) - - result = rust_map(msg) - - assert result == 'Processed: Test with special chars: 你好 🚀 \n\t"quotes"' - - empty_msg = Message(payload=cast(Any, {"id": 1, "content": ""}), headers=[], timestamp=0.0) - assert rust_map(empty_msg) == "Processed: " - - -def test_chained_rust_functions(rust_test_functions: Any) -> None: - """Test multiple Rust functions chained together""" - from rust_test_functions import ( - TestFilterCorrect, - TestMapCorrect, - TestMapString, - ) - - rust_filter = TestFilterCorrect() - rust_map_to_string = TestMapCorrect() - rust_map_to_length = TestMapString() - - test_messages = [ - Message(payload=cast(Any, {"id": 3, "content": "Hello World!"}), headers=[], timestamp=0.0), - Message(payload=cast(Any, {"id": 0, "content": "Filtered"}), headers=[], timestamp=0.0), - Message(payload=cast(Any, {"id": 10, "content": "Short"}), headers=[], timestamp=0.0), - ] - - results = [] - for msg in test_messages: - if rust_filter(msg): - string_result = rust_map_to_string(msg) - string_msg = Message(payload=string_result, headers=[], timestamp=0.0) - length_result = rust_map_to_length(string_msg) - results.append(length_result) - - assert len(results) == 2 - assert results[0] == len("Processed: Hello World!") - assert results[1] == len("Processed: Short") - - -def test_rust_functions_in_pipeline_structure(rust_test_functions: Any) -> None: - """Test that Rust functions work in actual pipeline infrastructure""" - from rust_test_functions import TestFilterCorrect, TestMapCorrect - - # Create a pipeline using the same structure as the examples - # The pipeline will handle type conversion from bytes to TestMessage - pipeline = ( - streaming_source("input", stream_name="test-messages") - .apply(Filter("rust_filter", function=cast(Any, TestFilterCorrect()))) - .apply(Map("rust_map", function=cast(Any, TestMapCorrect()))) - .apply(Serializer("serializer")) - .sink(StreamSink("output", stream_name="processed-messages")) - ) - - # Use the dummy adapter to verify pipeline structure - dummy_config: PipelineConfig = {} - adapter: DummyAdapter[Any, Any] = load_adapter("dummy", dummy_config, None) # type: ignore - translator: RuntimeTranslator[Any, Any] = RuntimeTranslator(adapter) - - # Execute pipeline structure creation - iterate_edges(pipeline, translator) - - # Verify that the pipeline was built with Rust functions - assert "input" in adapter.input_streams - assert "rust_filter" in adapter.input_streams - assert "rust_map" in adapter.input_streams - assert "output" in adapter.input_streams - - # Verify the pipeline structure includes our Rust function steps - expected_streams = ["input", "rust_filter", "rust_map", "serializer", "output"] - for stream in expected_streams: - assert stream in adapter.input_streams, f"Missing stream: {stream}" - - def test_rust_functions_with_message_flow(rust_test_functions: Any) -> None: """Test that Rust functions process actual messages through a pipeline""" from rust_test_functions import TestFilterCorrect, TestMapCorrect