-
Notifications
You must be signed in to change notification settings - Fork 10
Fix BinaryParameter truncation in xarray dataset creation #244
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
42f9f04
9da8d16
14af896
3d51fc1
4dac600
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -26,6 +26,7 @@ build | |
| dist | ||
| space_packet_parser/_version.py | ||
| uv.lock | ||
| node_modules | ||
|
|
||
| # Packages # | ||
| ############ | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -16,6 +16,7 @@ | |
| from pathlib import Path | ||
| from typing import BinaryIO | ||
|
|
||
| from space_packet_parser import common | ||
| from space_packet_parser.exceptions import UnrecognizedPacketTypeError | ||
| from space_packet_parser.generators import ccsds_generator | ||
| from space_packet_parser.generators.utils import _read_packet_file | ||
|
|
@@ -236,6 +237,10 @@ def _process_generator(generator): | |
| else: | ||
| val = value | ||
|
|
||
| # Convert BinaryParameter to plain bytes to prevent numpy truncation | ||
| if isinstance(val, common.BinaryParameter): | ||
| val = bytes(val) | ||
|
|
||
| data_dict[apid][key].append(val) | ||
|
Comment on lines
+240
to
244
|
||
| if key not in datatype_mapping[apid]: | ||
| # Add this datatype to the mapping | ||
|
|
||
| Original file line number | Diff line number | Diff line change | ||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -178,6 +178,41 @@ def test_create_dataset_with_custom_generator(tmp_path, fixed_length_packet_defi | |||||||||||||||
| assert list(dataset["INT32_FIELD"].values) == [12345, 67890, -99999] | ||||||||||||||||
|
|
||||||||||||||||
|
|
||||||||||||||||
| def test_create_dataset_preserves_binary_parameter_width(tmp_path): | ||||||||||||||||
| """Test that binary parameters keep their full byte width in the resulting dataset.""" | ||||||||||||||||
| packet_definition = definitions.XtcePacketDefinition( | ||||||||||||||||
| container_set=[ | ||||||||||||||||
| containers.SequenceContainer( | ||||||||||||||||
| "BINARY_CONTAINER", | ||||||||||||||||
| entry_list=[ | ||||||||||||||||
| parameters.Parameter( | ||||||||||||||||
| "BIN_FIELD", | ||||||||||||||||
| parameter_type=parameter_types.BinaryParameterType( | ||||||||||||||||
| "BIN_TYPE", encoding=encodings.BinaryDataEncoding(fixed_size_in_bits=64) | ||||||||||||||||
| ), | ||||||||||||||||
| ) | ||||||||||||||||
| ], | ||||||||||||||||
| ) | ||||||||||||||||
| ] | ||||||||||||||||
| ) | ||||||||||||||||
| packet_data = b"ABCDEFGH" | ||||||||||||||||
| test_file = tmp_path / "binary_packets.bin" | ||||||||||||||||
| test_file.write_bytes(packet_data) | ||||||||||||||||
|
|
||||||||||||||||
| datasets = xarr.create_dataset( | ||||||||||||||||
| test_file, | ||||||||||||||||
| packet_definition, | ||||||||||||||||
| packet_bytes_generator=fixed_length_generator, | ||||||||||||||||
| generator_kwargs={"packet_length_bytes": 8}, | ||||||||||||||||
| parse_bytes_kwargs={"root_container_name": "BINARY_CONTAINER"}, | ||||||||||||||||
| ) | ||||||||||||||||
|
|
||||||||||||||||
| dataset = list(datasets.values())[0] | ||||||||||||||||
|
|
||||||||||||||||
| assert dataset["BIN_FIELD"].values.dtype.itemsize == 8 | ||||||||||||||||
| assert dataset["BIN_FIELD"].values.tolist() == [packet_data] | ||||||||||||||||
|
Comment on lines
+212
to
+213
|
||||||||||||||||
| assert dataset["BIN_FIELD"].values.dtype.itemsize == 8 | |
| assert dataset["BIN_FIELD"].values.tolist() == [packet_data] | |
| values = dataset["BIN_FIELD"].values | |
| # Ensure we truly have a fixed-width bytes dtype (not an object array whose itemsize matches pointer size). | |
| assert values.dtype.kind == "S" | |
| assert values.dtype.itemsize == 8 | |
| assert values.tolist() == [packet_data] |
Uh oh!
There was an error while loading. Please reload this page.