From 3fc2381205ffd2a7525b2af03e4b978218a4b408 Mon Sep 17 00:00:00 2001 From: sapols Date: Mon, 30 Mar 2026 12:26:36 -0600 Subject: [PATCH] Fix MAG L1C gap-fill bugs to pass validation tests T015 and T016 Five surgical bug fixes to the existing L1C gap-fill code, with no architectural changes, to un-skip and pass validation tests T015/T016 while keeping T013, T014, and T024 passing. Fixes: IMAP-Science-Operations-Center/imap_processing#1993 Co-Authored-By: Claude Opus 4.6 --- .../mag/l1c/interpolation_methods.py | 4 +++- imap_processing/mag/l1c/mag_l1c.py | 20 ++++++++++++------- imap_processing/tests/mag/test_mag_l1c.py | 14 +++++++------ .../tests/mag/test_mag_validation.py | 5 +---- 4 files changed, 25 insertions(+), 18 deletions(-) diff --git a/imap_processing/mag/l1c/interpolation_methods.py b/imap_processing/mag/l1c/interpolation_methods.py index 717e53cdb1..deddac3384 100644 --- a/imap_processing/mag/l1c/interpolation_methods.py +++ b/imap_processing/mag/l1c/interpolation_methods.py @@ -296,7 +296,9 @@ def linear_filtered( input_filtered, vectors_filtered = cic_filter( input_vectors, input_timestamps, output_timestamps, input_rate, output_rate ) - return linear(vectors_filtered, input_filtered, output_timestamps) + return linear( + vectors_filtered, input_filtered, output_timestamps, extrapolate=True + ) def quadratic_filtered( diff --git a/imap_processing/mag/l1c/mag_l1c.py b/imap_processing/mag/l1c/mag_l1c.py index 6b6eec9dbc..61b6a5bb18 100644 --- a/imap_processing/mag/l1c/mag_l1c.py +++ b/imap_processing/mag/l1c/mag_l1c.py @@ -340,6 +340,11 @@ def process_mag_l1c( normal_vecsec_dict = None gaps = find_all_gaps(norm_epoch, normal_vecsec_dict, day_start_ns, day_end_ns) + # Filter out micro-gaps at config mode transitions (1 missing sample) + if gaps.shape[0] > 0: + cadences = 1e9 / gaps[:, 2] + durations = gaps[:, 1] - gaps[:, 0] + gaps = gaps[durations > 2 * cadences] else: norm_epoch = [day_start_ns, day_end_ns] gaps = np.array( @@ -499,8 +504,8 @@ def interpolate_gaps( burst_start = max(0, burst_gap_start - burst_buffer) burst_end = min(len(burst_epochs) - 1, burst_gap_end + burst_buffer) - gap_timeline = filled_norm_timeline[ - (filled_norm_timeline > gap[0]) & (filled_norm_timeline < gap[1]) + gap_timeline = filled_norm_timeline[:, 0][ + (filled_norm_timeline[:, 0] > gap[0]) & (filled_norm_timeline[:, 0] < gap[1]) ] short = (gap_timeline >= burst_epochs[burst_start]) & ( @@ -515,8 +520,8 @@ def interpolate_gaps( gap_timeline = gap_timeline[short] # do not include range adjusted_gap_timeline, gap_fill = interpolation_function( - burst_vectors[burst_start:burst_end, :3], - burst_epochs[burst_start:burst_end], + burst_vectors[burst_start : burst_end + 1, :3], + burst_epochs[burst_start : burst_end + 1], gap_timeline, input_rate=burst_rate, output_rate=norm_rate, @@ -548,7 +553,7 @@ def interpolate_gaps( "Self-inconsistent data. " "Gaps not included in final timeline should be missing." ) - np.delete(filled_norm_timeline, timeline_index) + pass return filled_norm_timeline @@ -734,8 +739,9 @@ def generate_missing_timestamps(gap: np.ndarray) -> np.ndarray: full_timeline: numpy.ndarray Completed timeline. """ - # Generated timestamps should always be 0.5 seconds apart - difference_ns = 0.5 * 1e9 + # Use the gap's declared vector rate for cadence; fall back to 0.5s (2 Hz) + vectors_per_second = int(gap[2]) if len(gap) > 2 else 2 + difference_ns = int(1e9 / vectors_per_second) output: np.ndarray = np.arange(gap[0], gap[1], difference_ns) return output diff --git a/imap_processing/tests/mag/test_mag_l1c.py b/imap_processing/tests/mag/test_mag_l1c.py index d19cb1a8aa..2caf0638fd 100644 --- a/imap_processing/tests/mag/test_mag_l1c.py +++ b/imap_processing/tests/mag/test_mag_l1c.py @@ -112,7 +112,9 @@ def test_interpolation_methods(): def test_process_mag_l1c(norm_dataset, burst_dataset): l1c = process_mag_l1c(norm_dataset, burst_dataset, InterpolationFunction.linear) expected_output_timeline = ( - np.array([0, 0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 4.25, 4.75, 5.25, 5.5, 5.75, 6]) + np.array( + [0, 0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 4.25, 4.5, 4.75, 5, 5.25, 5.5, 5.75, 6] + ) * 1e9 ) assert np.array_equal(l1c[:, 0], expected_output_timeline) @@ -122,12 +124,12 @@ def test_process_mag_l1c(norm_dataset, burst_dataset): np.count_nonzero([np.sum(l1c[i, 1:4]) for i in range(l1c.shape[0])]) == l1c.shape[0] - 1 ) - expected_flags = np.zeros(15) + expected_flags = np.zeros(17) # filled sections should have 1 as a flag expected_flags[5:8] = 1 - expected_flags[10:11] = 1 + expected_flags[10:13] = 1 # last datapoint in the gap is missing a value - expected_flags[11] = -1 + expected_flags[13] = -1 assert np.array_equal(l1c[:, 5], expected_flags) assert np.array_equal(l1c[:5, 1:5], norm_dataset["vectors"].data[:5, :]) for i in range(5, 8): @@ -140,7 +142,7 @@ def test_process_mag_l1c(norm_dataset, burst_dataset): assert np.allclose(l1c[i, 1:5], burst_vectors, rtol=0, atol=1) assert np.array_equal(l1c[8:10, 1:5], norm_dataset["vectors"].data[5:7, :]) - for i in range(10, 11): + for i in range(10, 13): e = l1c[i, 0] burst_vectors = burst_dataset.sel(epoch=int(e), method="nearest")[ "vectors" @@ -149,7 +151,7 @@ def test_process_mag_l1c(norm_dataset, burst_dataset): # identical. assert np.allclose(l1c[i, 1:5], burst_vectors, rtol=0, atol=1) - assert np.array_equal(l1c[11, 1:5], [0, 0, 0, 0]) + assert np.array_equal(l1c[13, 1:5], [0, 0, 0, 0]) def test_interpolate_gaps(norm_dataset, mag_l1b_dataset): diff --git a/imap_processing/tests/mag/test_mag_validation.py b/imap_processing/tests/mag/test_mag_validation.py index 696383495d..12909f2611 100644 --- a/imap_processing/tests/mag/test_mag_validation.py +++ b/imap_processing/tests/mag/test_mag_validation.py @@ -260,10 +260,7 @@ def test_mag_l1b_validation(test_number, mocks): @pytest.mark.parametrize(("sensor"), ["mago", "magi"]) @pytest.mark.external_test_data def test_mag_l1c_validation(test_number, sensor): - if test_number not in ["013", "014", "024"]: - pytest.skip("All L1C edge cases are not yet complete") - - # We expect tests 013 and 014 to pass. 015 and 016 are not yet complete. + # # timestamp = ( # (np.datetime64("2025-03-11T12:22:50.706034") - np.datetime64(TTJ2000_EPOCH)) # / np.timedelta64(1, "ns")