I-ALiRT - bugfix for CoDICE data structure

imap_processing/codice/codice_l1a_ialirt_hi.py

@@ -152,7 +152,7 @@ def l1a_ialirt_hi(unpacked_dataset: xr.Dataset, lut_file: Path) -> xr.Dataset:
     # -> (epoch, n_spins, energy, spin_sector, inst_az) ->
     # finally (epoch * n_spins, energy,
     # spin_sector, inst_az)
-    decompressed_data = decompressed_data.transpose(0, 2, 1, 3, 4).reshape(
+    decompressed_data = decompressed_data.transpose(0, 2, 1, 4, 3).reshape(
         -1, chunk_size, *collapse_shape
     )
 

imap_processing/tests/external_test_data_config.py

@@ -53,6 +53,7 @@
     (f"imap_codice_l1a_lo-counters-singles_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1a_validation"),
     (f"imap_codice_l1a_lo-direct-events_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1a_validation"),
     (f"imap_codice_l1a_lo-ialirt_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1a_validation"),
+    ("imap_codice_l1a_hi-ialirt_20260331_v0.0.22.cdf", "codice/data/l1a_validation"),
     (f"imap_codice_l1a_lo-nsw-priority_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1a_validation"),
     (f"imap_codice_l1a_lo-nsw-angular_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1a_validation"),
     (f"imap_codice_l1a_lo-sw-angular_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1a_validation"),
@@ -130,6 +131,11 @@
     ("iois_1_packets_2025_284_05_54_39", "ialirt/data/l0/"),
     ("iois_1_packets_2025_344_05_57_56", "ialirt/data/l0/"),
     ("iois_1_packets_2025_344_05_59_58", "ialirt/data/l0/"),
+    ("iois_1_packets_2026_090_05_03_05", "ialirt/data/l0/"),
+    ("iois_1_packets_2026_090_05_04_06", "ialirt/data/l0/"),
+    ("iois_1_packets_2026_090_05_05_07", "ialirt/data/l0/"),
+    ("iois_1_packets_2026_090_05_06_08", "ialirt/data/l0/"),
+    ("iois_1_packets_2026_090_05_07_09", "ialirt/data/l0/"),
     ("imap_recon_od005_20250925_20251014_v01.bsp", "spice/test_data/"),
     ("imap_2025_283_2025_284_001.ah.bc", "spice/test_data/"),
 

imap_processing/tests/ialirt/unit/test_process_codice.py

@@ -27,6 +27,7 @@
 )
 from imap_processing.codice.decompress import decompress
 from imap_processing.ialirt.l0.process_codice import (
+    COD_HI_COUNTER,
     COD_LO_COUNTER,
     concatenate_bytes,
     convert_to_intensities,
@@ -153,6 +154,52 @@ def cod_hi_test_dataset(cod_hi_test_file):
     return datasets
 
 
+@pytest.fixture(scope="session")
+def cod_hi_l1a_test_data_transposed():
+    """Returns the test data directory."""
+    data_path = (
+        imap_module_directory
+        / "tests"
+        / "codice"
+        / "data"
+        / "l1a_validation"
+        / "imap_codice_l1a_hi-ialirt_20260331_v0.0.22.cdf"
+    )
+
+    data = load_cdf(data_path)
+
+    return data
+
+
+@pytest.fixture(scope="session")
+def postlaunch_packet_path():
+    """Returns the paths to the binary packets."""
+    directory = imap_module_directory / "tests" / "ialirt" / "data" / "l0"
+    filenames = [
+        "iois_1_packets_2026_090_05_03_05",
+        "iois_1_packets_2026_090_05_04_06",
+        "iois_1_packets_2026_090_05_05_07",
+        "iois_1_packets_2026_090_05_06_08",
+        "iois_1_packets_2026_090_05_07_09",
+    ]
+    return tuple(directory / fname for fname in filenames)
+
+
+@pytest.fixture
+def postlaunch_xarray_data(postlaunch_packet_path, sc_packet_path):
+    """Create xarray data for multiple packets."""
+    apid = 478
+    _, xtce_ialirt_path = sc_packet_path
+
+    xarray_data = tuple(
+        packet_file_to_datasets(packet, xtce_ialirt_path, use_derived_value=False)[apid]
+        for packet in postlaunch_packet_path
+    )
+
+    merged_xarray_data = xr.concat(xarray_data, dim="epoch")
+    return merged_xarray_data
+
+
 @pytest.fixture
 def codice_test_data(test_datasets):
     return test_datasets[478]
@@ -804,34 +851,38 @@ def test_process_codice_lo(
 
 
 @pytest.mark.external_test_data
-@patch("imap_processing.ialirt.l0.process_codice.COD_HI_COUNTER", 197)
-@patch(
-    "imap_processing.codice.constants.IAL_BIT_STRUCTURE",
-    OLD_IAL_BIT_STRUCTURE,
-)
-def test_process_codice_hi(
-    cod_hi_test_dataset, l1a_lut_path, l2_lut_path, cod_hi_l2_test_data
-):
+def test_process_codice_hi(postlaunch_xarray_data, cod_hi_l1a_test_data_transposed):
     """Test process_codice for hi."""
-    test_data = cod_hi_l2_test_data["h"]
-
-    n = cod_hi_test_dataset.dims["epoch"]
-    cod_hi_test_dataset = cod_hi_test_dataset.assign(
-        sc_sclk_sec=("epoch", np.zeros(n, dtype=np.int64)),
-        sc_sclk_sub_sec=("epoch", np.zeros(n, dtype=np.int64)),
+    grouped_cod_hi_data = find_groups(
+        postlaunch_xarray_data, (0, COD_HI_COUNTER), "cod_hi_counter", "cod_hi_acq"
     )
+    unique_cod_hi_groups = np.unique(grouped_cod_hi_data["group"])
 
-    _, cod_hi_data = process_codice(
-        cod_hi_test_dataset, l1a_lut_path, l2_lut_path, "codice_hi"
-    )
-    samples_per_group = test_data.shape[0] // len(cod_hi_data)
-    grouped_test_data = test_data.reshape(
-        len(cod_hi_data),
-        samples_per_group,
-        *test_data.shape[1:],
-    )
+    for group in unique_cod_hi_groups:
+        cod_hi_data_stream = concatenate_bytes(grouped_cod_hi_data, group, "hi")
+        cod_hi_science_values, cod_hi_metadata_values = process_ialirt_data_streams(
+            [cod_hi_data_stream]
+        )
+        if not cod_hi_science_values:
+            continue
+        cod_hi_dataset = create_xarray_dataset(
+            cod_hi_science_values, cod_hi_metadata_values, "hi"
+        )
+        l1a_lut_path = (
+            imap_module_directory
+            / "tests"
+            / "codice"
+            / "data"
+            / "l1a_lut"
+            / "imap_codice_l1a-sci-lut_20260129_v002.json"
+        )
+        l1a_hi = l1a_ialirt_hi(cod_hi_dataset, l1a_lut_path)
 
-    for i, group in enumerate(cod_hi_data):
-        arr = np.array(group["codice_hi_h"], dtype=float)
+        expected = cod_hi_l1a_test_data_transposed.sel(
+            epoch=l1a_hi["epoch"], method="nearest"
+        )
 
-        np.testing.assert_allclose(arr, grouped_test_data[i], atol=3e-2, rtol=1e-5)
+        np.testing.assert_array_equal(
+            l1a_hi["h"].values,
+            expected["h"].data,
+        )