From e90ecbd74dd80f2b6a9cf785b5af8d30012e3e54 Mon Sep 17 00:00:00 2001 From: Maxine Hartnett Date: Mon, 30 Mar 2026 13:48:32 -0600 Subject: [PATCH 1/5] rename GLOWS anc files --- imap_processing/cli.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/imap_processing/cli.py b/imap_processing/cli.py index 68da35fdd..c1ae56293 100644 --- a/imap_processing/cli.py +++ b/imap_processing/cli.py @@ -676,7 +676,7 @@ def do_processing( # Load conversion table (needed for both hist and DE) conversion_table_file = dependencies.get_processing_inputs( - descriptor="conversion-table-for-anc-data" + descriptor="l1b-conversion-table-for-anc-data" )[0] with open(conversion_table_file.imap_file_paths[0].construct_path()) as f: @@ -691,16 +691,16 @@ def do_processing( if "hist" in self.descriptor: # Create file lists for each ancillary type excluded_regions_files = dependencies.get_processing_inputs( - descriptor="map-of-excluded-regions" + descriptor="l1b-map-of-excluded-regions" )[0] uv_sources_files = dependencies.get_processing_inputs( - descriptor="map-of-uv-sources" + descriptor="l1b-map-of-uv-sources" )[0] suspected_transients_files = dependencies.get_processing_inputs( - descriptor="suspected-transients" + descriptor="l1b-suspected-transients" )[0] exclusions_by_instr_team_files = dependencies.get_processing_inputs( - descriptor="exclusions-by-instr-team" + descriptor="l1b-exclusions-by-instr-team" )[0] pipeline_settings = dependencies.get_processing_inputs( descriptor="pipeline-settings" From c1529196bcbb71549f5f14026f395106874da596 Mon Sep 17 00:00:00 2001 From: Maxine Hartnett Date: Mon, 30 Mar 2026 13:59:06 -0600 Subject: [PATCH 2/5] Add L2 calibration file --- imap_processing/cli.py | 5 +++++ imap_processing/glows/l2/glows_l2.py | 4 ++++ imap_processing/tests/glows/test_glows_l2.py | 4 ++-- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/imap_processing/cli.py b/imap_processing/cli.py index c1ae56293..bdb815371 100644 --- a/imap_processing/cli.py +++ b/imap_processing/cli.py @@ -758,10 +758,15 @@ def do_processing( pipeline_settings_combiner = GlowsAncillaryCombiner( pipeline_settings_input, day_buffer ) + calibration_input = dependencies.get_processing_inputs( + descriptor="l2-calibration" + )[0] + calibration_combiner = GlowsAncillaryCombiner(calibration_input, day_buffer) datasets = glows_l2( input_dataset, pipeline_settings_combiner.combined_dataset, + calibration_combiner.combined_dataset, ) return datasets diff --git a/imap_processing/glows/l2/glows_l2.py b/imap_processing/glows/l2/glows_l2.py index 5027a07cd..a7fd02577 100644 --- a/imap_processing/glows/l2/glows_l2.py +++ b/imap_processing/glows/l2/glows_l2.py @@ -26,6 +26,7 @@ def glows_l2( input_dataset: xr.Dataset, pipeline_settings_dataset: xr.Dataset, + calibration_dataset: xr.Dataset, ) -> list[xr.Dataset]: """ Will process GLOWS L2 data from L1 data. @@ -37,6 +38,9 @@ def glows_l2( pipeline_settings_dataset : xarray.Dataset Dataset containing pipeline settings from GlowsAncillaryCombiner. + calibration_dataset : xarray.Dataset + Dataset containing calibration data from + GlowsAncillaryCombiner. Returns ------- diff --git a/imap_processing/tests/glows/test_glows_l2.py b/imap_processing/tests/glows/test_glows_l2.py index ab7ab336b..28340952a 100644 --- a/imap_processing/tests/glows/test_glows_l2.py +++ b/imap_processing/tests/glows/test_glows_l2.py @@ -66,14 +66,14 @@ def test_glows_l2( ) # Test case 1: L1B dataset has good times - l2 = glows_l2(l1b_hist_dataset, mock_pipeline_settings)[0] + l2 = glows_l2(l1b_hist_dataset, mock_pipeline_settings, None)[0] assert l2.attrs["Logical_source"] == "imap_glows_l2_hist" assert np.allclose(l2["filter_temperature_average"].values, [57.6], rtol=0.1) # Test case 2: L1B dataset has no good times (all flags 0) l1b_hist_dataset["flags"].values = np.zeros(l1b_hist_dataset.flags.shape) caplog.set_level("WARNING") - result = glows_l2(l1b_hist_dataset, mock_pipeline_settings) + result = glows_l2(l1b_hist_dataset, mock_pipeline_settings, None) assert result == [] assert any(record.levelname == "WARNING" for record in caplog.records) From 46ae47a0c1cb97bc24575cb04f8bba0fa2b88007 Mon Sep 17 00:00:00 2001 From: Maxine Hartnett Date: Tue, 31 Mar 2026 09:14:14 -0600 Subject: [PATCH 3/5] Add in a fallback case for an empty file. --- .../ancillary/ancillary_dataset_combiner.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/imap_processing/ancillary/ancillary_dataset_combiner.py b/imap_processing/ancillary/ancillary_dataset_combiner.py index 01819a9d1..123e142d0 100644 --- a/imap_processing/ancillary/ancillary_dataset_combiner.py +++ b/imap_processing/ancillary/ancillary_dataset_combiner.py @@ -364,6 +364,19 @@ def convert_file_to_dataset(self, filepath: str | Path) -> xr.Dataset: if "excluded-regions" in filename: # Handle excluded regions (2 columns: longitude, latitude) data = np.loadtxt(filepath, comments="#") + if data.size == 0: + return xr.Dataset( + { + "ecliptic_longitude_deg": ( + ["region"], + np.array([], dtype=float), + ), + "ecliptic_latitude_deg": ( + ["region"], + np.array([], dtype=float), + ), + } + ) return xr.Dataset( { "ecliptic_longitude_deg": (["region"], data[:, 0]), From 9b036157b7a2e6ab3820d51649fb02651e3eae99 Mon Sep 17 00:00:00 2001 From: Maxine Hartnett Date: Tue, 31 Mar 2026 09:31:23 -0600 Subject: [PATCH 4/5] Add unit tests and fix for L2 ancillary file --- .../ancillary/ancillary_dataset_combiner.py | 15 ++++++++++++++- .../ancillary/test_ancillary_dataset_combiner.py | 13 +++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/imap_processing/ancillary/ancillary_dataset_combiner.py b/imap_processing/ancillary/ancillary_dataset_combiner.py index 123e142d0..104255f87 100644 --- a/imap_processing/ancillary/ancillary_dataset_combiner.py +++ b/imap_processing/ancillary/ancillary_dataset_combiner.py @@ -338,7 +338,7 @@ def __init__( ): super().__init__(ancillary_input, expected_end_date) - def convert_file_to_dataset(self, filepath: str | Path) -> xr.Dataset: + def convert_file_to_dataset(self, filepath: str | Path) -> xr.Dataset: # noqa: PLR0911 """ Convert GLOWS ancillary .dat files to xarray datasets. @@ -425,6 +425,19 @@ def convert_file_to_dataset(self, filepath: str | Path) -> xr.Dataset: } ) + elif "l2-calibration" in filename: + # Handle calibration file (timestamp + cps_per_R float value) + with open(filepath) as f: + lines = [line.strip() for line in f if not line.startswith("#")] + identifiers = [line.split(" ", 1)[0] for line in lines] + values = [float(line.split(" ", 1)[1]) for line in lines] + return xr.Dataset( + { + "start_time_utc": (["time_block"], identifiers), + "cps_per_r": (["time_block"], values), + } + ) + elif filename.endswith(".json"): # Handle pipeline settings JSON file using the generic read_json method return self.convert_json_to_dataset(filepath) diff --git a/imap_processing/tests/ancillary/test_ancillary_dataset_combiner.py b/imap_processing/tests/ancillary/test_ancillary_dataset_combiner.py index fb45059fa..06733e79b 100644 --- a/imap_processing/tests/ancillary/test_ancillary_dataset_combiner.py +++ b/imap_processing/tests/ancillary/test_ancillary_dataset_combiner.py @@ -236,6 +236,19 @@ def test_glows_excluded_regions_combiner(glows_ancillary_filepath): assert dataset["ecliptic_latitude_deg"].dims == ("region",) +def test_glows_excluded_regions_combiner_empty_file(tmp_path): + file_path = tmp_path / "imap_glows_l1b-map-of-excluded-regions_20251112_v001.dat" + file_path.write_text("# header only\n") + + combiner = GlowsAncillaryCombiner([], "20251115") + dataset = combiner.convert_file_to_dataset(file_path) + + assert "ecliptic_longitude_deg" in dataset.data_vars + assert "ecliptic_latitude_deg" in dataset.data_vars + assert len(dataset["ecliptic_longitude_deg"]) == 0 + assert len(dataset["ecliptic_latitude_deg"]) == 0 + + def test_glows_uv_sources_combiner(glows_ancillary_filepath): file_path = ( glows_ancillary_filepath / "imap_glows_map-of-uv-sources_20250923_v002.dat" From 49128fb1dd9be27f460e9939e008d3cd83526d3a Mon Sep 17 00:00:00 2001 From: Maxine Hartnett Date: Tue, 31 Mar 2026 10:08:37 -0600 Subject: [PATCH 5/5] adding test --- .../ancillary/test_ancillary_dataset_combiner.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/imap_processing/tests/ancillary/test_ancillary_dataset_combiner.py b/imap_processing/tests/ancillary/test_ancillary_dataset_combiner.py index 06733e79b..f7d3f0020 100644 --- a/imap_processing/tests/ancillary/test_ancillary_dataset_combiner.py +++ b/imap_processing/tests/ancillary/test_ancillary_dataset_combiner.py @@ -313,6 +313,21 @@ def test_glows_exclusions_by_instr_team_combiner(glows_ancillary_filepath): assert combiner.timestamped_data[0].version == "v002" +def test_glows_l2_calibration_combiner(tmp_path): + file_path = tmp_path / "imap_glows_l2-calibration_20251112_v001.dat" + file_path.write_text( + "# header\n2025-11-13T18:12:48 1.020\n2025-11-14T09:58:04 0.849\n" + ) + + combiner = GlowsAncillaryCombiner([], "20251115") + dataset = combiner.convert_file_to_dataset(file_path) + + assert "start_time_utc" in dataset.data_vars + assert "cps_per_r" in dataset.data_vars + assert len(dataset["cps_per_r"]) == 2 + assert dataset["cps_per_r"].values[0] == pytest.approx(1.020) + + def test_ancillary_combiner_empty_input(): """Test AncillaryCombiner with empty input list.""" combiner = AncillaryCombiner([], "20251031")