From b29b4c25b6f8c4afe73440d1a25b1c46fa0b349a Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Wed, 25 Mar 2026 16:21:26 -0600 Subject: [PATCH 1/5] bugfix --- imap_processing/codice/codice_l1a_ialirt_hi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imap_processing/codice/codice_l1a_ialirt_hi.py b/imap_processing/codice/codice_l1a_ialirt_hi.py index 09c8c07db..a5dfcc1f3 100644 --- a/imap_processing/codice/codice_l1a_ialirt_hi.py +++ b/imap_processing/codice/codice_l1a_ialirt_hi.py @@ -152,7 +152,7 @@ def l1a_ialirt_hi(unpacked_dataset: xr.Dataset, lut_file: Path) -> xr.Dataset: # -> (epoch, n_spins, energy, spin_sector, inst_az) -> # finally (epoch * n_spins, energy, # spin_sector, inst_az) - decompressed_data = decompressed_data.transpose(0, 2, 1, 3, 4).reshape( + decompressed_data = decompressed_data.transpose(0, 2, 1, 4, 3).reshape( -1, chunk_size, *collapse_shape ) From 2b540198e4b5dfd3645b226d91b562317bfc4121 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Wed, 1 Apr 2026 13:39:24 -0600 Subject: [PATCH 2/5] add test --- .../tests/external_test_data_config.py | 1 + .../tests/ialirt/unit/test_process_codice.py | 111 ++++++++++++++---- 2 files changed, 87 insertions(+), 25 deletions(-) diff --git a/imap_processing/tests/external_test_data_config.py b/imap_processing/tests/external_test_data_config.py index 416f8dfee..ff54bdaec 100644 --- a/imap_processing/tests/external_test_data_config.py +++ b/imap_processing/tests/external_test_data_config.py @@ -53,6 +53,7 @@ (f"imap_codice_l1a_lo-counters-singles_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1a_validation"), (f"imap_codice_l1a_lo-direct-events_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1a_validation"), (f"imap_codice_l1a_lo-ialirt_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1a_validation"), + ("imap_codice_l1a_hi-ialirt_20260331_v0.0.22.cdf", "codice/data/l1a_validation"), (f"imap_codice_l1a_lo-nsw-priority_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1a_validation"), (f"imap_codice_l1a_lo-nsw-angular_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1a_validation"), (f"imap_codice_l1a_lo-sw-angular_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1a_validation"), diff --git a/imap_processing/tests/ialirt/unit/test_process_codice.py b/imap_processing/tests/ialirt/unit/test_process_codice.py index 12a6be1d7..7c2e61f98 100644 --- a/imap_processing/tests/ialirt/unit/test_process_codice.py +++ b/imap_processing/tests/ialirt/unit/test_process_codice.py @@ -27,6 +27,7 @@ ) from imap_processing.codice.decompress import decompress from imap_processing.ialirt.l0.process_codice import ( + COD_HI_COUNTER, COD_LO_COUNTER, concatenate_bytes, convert_to_intensities, @@ -153,6 +154,59 @@ def cod_hi_test_dataset(cod_hi_test_file): return datasets +@pytest.fixture(scope="session") +def cod_hi_l1a_test_data_transposed(): + """Returns the test data directory.""" + data_path = ( + imap_module_directory + / "tests" + / "codice" + / "data" + / "l1a_validation" + / "imap_codice_l1a_hi-ialirt_20260331_v0.0.22.cdf" + ) + + data = load_cdf(data_path) + + return data + + +@pytest.fixture(scope="session") +def postlaunch_packet_path(): + """Returns the paths to the binary packets.""" + directory = imap_module_directory / "tests" / "ialirt" / "data" / "l0" + filenames = [ + "iois_1_packets_2026_090_05_02_04", + "iois_1_packets_2026_090_05_03_05", + "iois_1_packets_2026_090_05_04_06", + "iois_1_packets_2026_090_05_05_07", + "iois_1_packets_2026_090_05_06_08", + "iois_1_packets_2026_090_05_07_09", + "iois_1_packets_2026_090_05_08_10", + "iois_1_packets_2026_090_05_09_11", + "iois_1_packets_2026_090_05_10_12", + "iois_1_packets_2026_090_05_11_13", + "iois_1_packets_2026_090_05_12_14", + "iois_1_packets_2026_090_05_13_15", + "iois_1_packets_2026_090_05_14_16", + ] + return tuple(directory / fname for fname in filenames) + +@pytest.fixture +def postlaunch_xarray_data(postlaunch_packet_path, sc_packet_path): + """Create xarray data for multiple packets.""" + apid = 478 + _, xtce_ialirt_path = sc_packet_path + + xarray_data = tuple( + packet_file_to_datasets(packet, xtce_ialirt_path, use_derived_value=False)[apid] + for packet in postlaunch_packet_path + ) + + merged_xarray_data = xr.concat(xarray_data, dim="epoch") + return merged_xarray_data + + @pytest.fixture def codice_test_data(test_datasets): return test_datasets[478] @@ -372,7 +426,7 @@ def l1a_lut_path(): / "codice" / "data" / "l1a_lut" - / "imap_codice_l1a-sci-lut_20251007_v005.json" + / "imap_codice_l1a-sci-lut_20260129_v002.json" ) return lut_path @@ -804,34 +858,41 @@ def test_process_codice_lo( @pytest.mark.external_test_data -@patch("imap_processing.ialirt.l0.process_codice.COD_HI_COUNTER", 197) -@patch( - "imap_processing.codice.constants.IAL_BIT_STRUCTURE", - OLD_IAL_BIT_STRUCTURE, -) def test_process_codice_hi( - cod_hi_test_dataset, l1a_lut_path, l2_lut_path, cod_hi_l2_test_data + postlaunch_xarray_data, cod_hi_l1a_test_data_transposed, l1a_lut_path ): """Test process_codice for hi.""" - test_data = cod_hi_l2_test_data["h"] - - n = cod_hi_test_dataset.dims["epoch"] - cod_hi_test_dataset = cod_hi_test_dataset.assign( - sc_sclk_sec=("epoch", np.zeros(n, dtype=np.int64)), - sc_sclk_sub_sec=("epoch", np.zeros(n, dtype=np.int64)), + grouped_cod_hi_data = find_groups( + postlaunch_xarray_data, (0, COD_HI_COUNTER), "cod_hi_counter", "cod_hi_acq" ) + unique_cod_hi_groups = np.unique(grouped_cod_hi_data["group"]) - _, cod_hi_data = process_codice( - cod_hi_test_dataset, l1a_lut_path, l2_lut_path, "codice_hi" - ) - samples_per_group = test_data.shape[0] // len(cod_hi_data) - grouped_test_data = test_data.reshape( - len(cod_hi_data), - samples_per_group, - *test_data.shape[1:], - ) + l1a_results = [] + for group in unique_cod_hi_groups: + cod_hi_data_stream = concatenate_bytes(grouped_cod_hi_data, group, "hi") + cod_hi_science_values, cod_hi_metadata_values = process_ialirt_data_streams( + [cod_hi_data_stream] + ) + if not cod_hi_science_values: + continue + cod_hi_dataset = create_xarray_dataset( + cod_hi_science_values, cod_hi_metadata_values, "hi" + ) + l1a_hi = l1a_ialirt_hi(cod_hi_dataset, l1a_lut_path) + l1a_results.append(l1a_hi) - for i, group in enumerate(cod_hi_data): - arr = np.array(group["codice_hi_h"], dtype=float) + result = xr.concat(l1a_results, dim="epoch") - np.testing.assert_allclose(arr, grouped_test_data[i], atol=3e-2, rtol=1e-5) + np.testing.assert_array_equal( + result["h"].values, + cod_hi_l1a_test_data_transposed["h"].data, + ) + np.testing.assert_array_equal( + result["data_quality"].values, + cod_hi_l1a_test_data_transposed["data_quality"].data, + ) + np.testing.assert_allclose( + result["spin_period"].values, + cod_hi_l1a_test_data_transposed["spin_period"].data, + atol=1e-6, + ) From d2b30def46de4f1ea68300ef84475ab665803b6b Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Wed, 1 Apr 2026 13:52:31 -0600 Subject: [PATCH 3/5] update lut --- .../tests/ialirt/unit/test_process_codice.py | 43 ++++++++----------- 1 file changed, 17 insertions(+), 26 deletions(-) diff --git a/imap_processing/tests/ialirt/unit/test_process_codice.py b/imap_processing/tests/ialirt/unit/test_process_codice.py index 7c2e61f98..48c7cf2bd 100644 --- a/imap_processing/tests/ialirt/unit/test_process_codice.py +++ b/imap_processing/tests/ialirt/unit/test_process_codice.py @@ -176,19 +176,11 @@ def postlaunch_packet_path(): """Returns the paths to the binary packets.""" directory = imap_module_directory / "tests" / "ialirt" / "data" / "l0" filenames = [ - "iois_1_packets_2026_090_05_02_04", "iois_1_packets_2026_090_05_03_05", "iois_1_packets_2026_090_05_04_06", "iois_1_packets_2026_090_05_05_07", "iois_1_packets_2026_090_05_06_08", "iois_1_packets_2026_090_05_07_09", - "iois_1_packets_2026_090_05_08_10", - "iois_1_packets_2026_090_05_09_11", - "iois_1_packets_2026_090_05_10_12", - "iois_1_packets_2026_090_05_11_13", - "iois_1_packets_2026_090_05_12_14", - "iois_1_packets_2026_090_05_13_15", - "iois_1_packets_2026_090_05_14_16", ] return tuple(directory / fname for fname in filenames) @@ -426,7 +418,7 @@ def l1a_lut_path(): / "codice" / "data" / "l1a_lut" - / "imap_codice_l1a-sci-lut_20260129_v002.json" + / "imap_codice_l1a-sci-lut_20251007_v005.json" ) return lut_path @@ -859,7 +851,7 @@ def test_process_codice_lo( @pytest.mark.external_test_data def test_process_codice_hi( - postlaunch_xarray_data, cod_hi_l1a_test_data_transposed, l1a_lut_path + postlaunch_xarray_data, cod_hi_l1a_test_data_transposed ): """Test process_codice for hi.""" grouped_cod_hi_data = find_groups( @@ -867,7 +859,6 @@ def test_process_codice_hi( ) unique_cod_hi_groups = np.unique(grouped_cod_hi_data["group"]) - l1a_results = [] for group in unique_cod_hi_groups: cod_hi_data_stream = concatenate_bytes(grouped_cod_hi_data, group, "hi") cod_hi_science_values, cod_hi_metadata_values = process_ialirt_data_streams( @@ -878,21 +869,21 @@ def test_process_codice_hi( cod_hi_dataset = create_xarray_dataset( cod_hi_science_values, cod_hi_metadata_values, "hi" ) + l1a_lut_path = ( + imap_module_directory + / "tests" + / "codice" + / "data" + / "l1a_lut" + / "imap_codice_l1a-sci-lut_20260129_v002.json" + ) l1a_hi = l1a_ialirt_hi(cod_hi_dataset, l1a_lut_path) - l1a_results.append(l1a_hi) - result = xr.concat(l1a_results, dim="epoch") + expected = cod_hi_l1a_test_data_transposed.sel( + epoch=l1a_hi["epoch"], method="nearest" + ) - np.testing.assert_array_equal( - result["h"].values, - cod_hi_l1a_test_data_transposed["h"].data, - ) - np.testing.assert_array_equal( - result["data_quality"].values, - cod_hi_l1a_test_data_transposed["data_quality"].data, - ) - np.testing.assert_allclose( - result["spin_period"].values, - cod_hi_l1a_test_data_transposed["spin_period"].data, - atol=1e-6, - ) + np.testing.assert_array_equal( + l1a_hi["h"].values, + expected["h"].data, + ) From 2e94545748c69439b71ac0ff8b1dddd132b07dcb Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Wed, 1 Apr 2026 13:59:24 -0600 Subject: [PATCH 4/5] add external test data --- imap_processing/tests/external_test_data_config.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/imap_processing/tests/external_test_data_config.py b/imap_processing/tests/external_test_data_config.py index ff54bdaec..d485eb234 100644 --- a/imap_processing/tests/external_test_data_config.py +++ b/imap_processing/tests/external_test_data_config.py @@ -131,6 +131,11 @@ ("iois_1_packets_2025_284_05_54_39", "ialirt/data/l0/"), ("iois_1_packets_2025_344_05_57_56", "ialirt/data/l0/"), ("iois_1_packets_2025_344_05_59_58", "ialirt/data/l0/"), + ("iois_1_packets_2026_090_05_03_05", "ialirt/data/l0/"), + ("iois_1_packets_2026_090_05_04_06", "ialirt/data/l0/"), + ("iois_1_packets_2026_090_05_05_07", "ialirt/data/l0/"), + ("iois_1_packets_2026_090_05_06_08", "ialirt/data/l0/"), + ("iois_1_packets_2026_090_05_07_09", "ialirt/data/l0/"), ("imap_recon_od005_20250925_20251014_v01.bsp", "spice/test_data/"), ("imap_2025_283_2025_284_001.ah.bc", "spice/test_data/"), From f6765124604b8999bde6accb23ee11aff8565d42 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 1 Apr 2026 20:36:13 +0000 Subject: [PATCH 5/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../tests/ialirt/unit/test_process_codice.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/imap_processing/tests/ialirt/unit/test_process_codice.py b/imap_processing/tests/ialirt/unit/test_process_codice.py index 48c7cf2bd..28ac8109f 100644 --- a/imap_processing/tests/ialirt/unit/test_process_codice.py +++ b/imap_processing/tests/ialirt/unit/test_process_codice.py @@ -184,6 +184,7 @@ def postlaunch_packet_path(): ] return tuple(directory / fname for fname in filenames) + @pytest.fixture def postlaunch_xarray_data(postlaunch_packet_path, sc_packet_path): """Create xarray data for multiple packets.""" @@ -850,9 +851,7 @@ def test_process_codice_lo( @pytest.mark.external_test_data -def test_process_codice_hi( - postlaunch_xarray_data, cod_hi_l1a_test_data_transposed -): +def test_process_codice_hi(postlaunch_xarray_data, cod_hi_l1a_test_data_transposed): """Test process_codice for hi.""" grouped_cod_hi_data = find_groups( postlaunch_xarray_data, (0, COD_HI_COUNTER), "cod_hi_counter", "cod_hi_acq" @@ -870,12 +869,12 @@ def test_process_codice_hi( cod_hi_science_values, cod_hi_metadata_values, "hi" ) l1a_lut_path = ( - imap_module_directory - / "tests" - / "codice" - / "data" - / "l1a_lut" - / "imap_codice_l1a-sci-lut_20260129_v002.json" + imap_module_directory + / "tests" + / "codice" + / "data" + / "l1a_lut" + / "imap_codice_l1a-sci-lut_20260129_v002.json" ) l1a_hi = l1a_ialirt_hi(cod_hi_dataset, l1a_lut_path)