From 91753614f88205ad3b9e04ac56142c1637d24782 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 14 May 2026 11:39:44 -0400 Subject: [PATCH 1/2] fix(codice): clean up direct-events l2 metadata for display_type and labels Fix the remaining CoDICE direct-events metadata issues in imap_codice_l2_lo-direct-events and imap_codice_l2_hi-direct-events that were causing SPDF DISPLAY_TYPE failures and stale label-metadata conflicts in the written L2 CDFs. The main problem was that many direct-event variables were missing an explicit DISPLAY_TYPE in the L2 attr templates, and the direct-event L2 builders were merging L2 attrs onto inherited L1A attrs with `attrs.update(...)`. Because `update()` does not remove keys, stale L1A metadata such as LABLAXIS survived into the L2 output even when the L2 template intentionally omitted it. That created two classes of issues: - SPDF failures for missing or invalid DISPLAY_TYPE metadata - write-time failures with newer cdflib when a variable ended up with both LABLAXIS and LABL_PTR_* in the final L2 attrs Update the direct-event L2 attr templates to add explicit DISPLAY_TYPE values using a mixed policy: - time_series for num_events - spectrogram for plottable multidimensional physical variables such as tof, spin_angle, elevation_angle, energy_per_charge, apd_energy, ssd_energy, and energy_per_nuc - no_plot for categorical/index/support variables such as data_quality, gain, multi_flag, type, apd_id, position, energy_step, and ssd_id Also remove LABLAXIS from the direct-event data variables that already use LABL_PTR_* label metadata. This matches the CoDICE/CAVA guidance captured in issue #2765 and avoids the LABLAXIS/LABL_PTR conflict during CDF writing. In the direct-event L2 builders, replace inherited variable attrs with the L2 template attrs instead of merging them. This keeps the L2 attr template as the source of truth for normal direct-event data variables and prevents stale L1A-only keys from surviving in memory. The existing skip logic for nso*/rgfo* variables is preserved. Add a parameterized written-CDF regression test that: - generates fresh lo-direct-events and hi-direct-events L2 files - inspects the written CDF with cdflib - verifies the affected variables have DISPLAY_TYPE present - verifies DISPLAY_TYPE is stored as a string - verifies the emitted DISPLAY_TYPE values match the intended policy This also restores the normal L2 write_cdf() path in the direct-events tests after removing the temporary workaround that had disabled strict ISTP writing for these products. Validated locally with: - focused CoDICE direct-events pytest runs - pre-commit on the touched files - SPDF checks on regenerated lo-direct-events and hi-direct-events files After this change, the regenerated direct-events files no longer show the old DISPLAY_TYPE / ClassCastException failures, and the LABLAXIS / LABL_PTR conflict is cleared. Other unrelated CoDICE ISTP issues remain out of scope for this commit. --- ...ce_l2-hi-direct-events_variable_attrs.yaml | 25 +++--- ...ce_l2-lo-direct-events_variable_attrs.yaml | 24 ++--- imap_processing/codice/codice_l2.py | 4 +- .../tests/codice/test_codice_l2.py | 87 +++++++++++++++++++ 4 files changed, 115 insertions(+), 25 deletions(-) diff --git a/imap_processing/cdf/config/imap_codice_l2-hi-direct-events_variable_attrs.yaml b/imap_processing/cdf/config/imap_codice_l2-hi-direct-events_variable_attrs.yaml index e7e2814579..25062e1081 100644 --- a/imap_processing/cdf/config/imap_codice_l2-hi-direct-events_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_codice_l2-hi-direct-events_variable_attrs.yaml @@ -67,10 +67,10 @@ num_events: DEPEND_0: epoch DEPEND_1: priority DICT_KEY: SPASE>Support>SupportQuantity:Other + DISPLAY_TYPE: time_series FIELDNAM: Number of Events FILLVAL: *uint16_fillval FORMAT: I5 - LABLAXIS: Number of Events LABL_PTR_1: priority_label SCALETYP: linear UNITS: " " @@ -83,10 +83,10 @@ data_quality: DEPEND_0: epoch DEPEND_1: priority DICT_KEY: SPASE>Support>SupportQuantity:DataQuality + DISPLAY_TYPE: no_plot FIELDNAM: Data Quality FILLVAL: *uint8_fillval FORMAT: I3 - LABLAXIS: Data Quality LABL_PTR_1: priority_label SCALETYP: linear UNITS: " " @@ -100,10 +100,10 @@ energy_step: DEPEND_1: priority DEPEND_2: event_num DICT_KEY: SPASE>Support>SupportQuantity:Other + DISPLAY_TYPE: no_plot FIELDNAM: Energy Step FILLVAL: *uint8_fillval FORMAT: I3 - LABLAXIS: Energy Step LABL_PTR_1: priority_label LABL_PTR_2: event_num_label SCALETYP: linear @@ -118,10 +118,10 @@ energy_per_charge: DEPEND_1: priority DEPEND_2: event_num DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:EnergyPerCharge + DISPLAY_TYPE: spectrogram FIELDNAM: Energy per Charge FILLVAL: *real_fillval FORMAT: F12.4 - LABLAXIS: Energy per Charge LABL_PTR_1: priority_label LABL_PTR_2: event_num_label SCALETYP: linear @@ -136,10 +136,10 @@ gain: DEPEND_1: priority DEPEND_2: event_num DICT_KEY: SPASE>Support>SupportQuantity:Other + DISPLAY_TYPE: no_plot FIELDNAM: Gain FILLVAL: *uint8_fillval FORMAT: I1 - LABLAXIS: Gain LABL_PTR_1: priority_label LABL_PTR_2: event_num_label SCALETYP: linear @@ -154,10 +154,10 @@ multi_flag: DEPEND_1: priority DEPEND_2: event_num DICT_KEY: SPASE>Support>SupportQuantity:DataQuality + DISPLAY_TYPE: no_plot FIELDNAM: Multi Flag FILLVAL: *uint8_fillval FORMAT: I1 - LABLAXIS: Multi Flag LABL_PTR_1: priority_label LABL_PTR_2: event_num_label SCALETYP: linear @@ -172,10 +172,10 @@ type: DEPEND_1: priority DEPEND_2: event_num DICT_KEY: SPASE>Support>SupportQuantity:Other + DISPLAY_TYPE: no_plot FIELDNAM: Type FILLVAL: *uint8_fillval FORMAT: I1 - LABLAXIS: Type LABL_PTR_1: priority_label LABL_PTR_2: event_num_label SCALETYP: linear @@ -190,10 +190,10 @@ tof: DEPEND_1: priority DEPEND_2: event_num DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:TimeOfFlight + DISPLAY_TYPE: spectrogram FIELDNAM: Time of Flight FILLVAL: *real_fillval FORMAT: F12.4 - LABLAXIS: TOF LABL_PTR_1: priority_label LABL_PTR_2: event_num_label SCALETYP: linear @@ -226,6 +226,7 @@ spin_angle: DEPEND_1: priority DEPEND_2: event_num DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:ArrivalDirection,Qualifier:DirectionAngle.AzimuthAngle + DISPLAY_TYPE: spectrogram FIELDNAM: Spin Angle FILLVAL: *real_fillval FORMAT: F8.2 @@ -243,10 +244,10 @@ elevation_angle: DEPEND_1: priority DEPEND_2: event_num DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:ArrivalDirection,Qualifier:DirectionAngle.ElevationAngle + DISPLAY_TYPE: spectrogram FIELDNAM: Elevation Angle FILLVAL: *real_fillval FORMAT: F8.2 - LABLAXIS: Elevation Angle LABL_PTR_1: priority_label LABL_PTR_2: event_num_label SCALETYP: linear @@ -261,10 +262,10 @@ ssd_energy: DEPEND_1: priority DEPEND_2: event_num DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:Energy + DISPLAY_TYPE: spectrogram FIELDNAM: SSD Energy FILLVAL: *real_fillval FORMAT: F12.4 - LABLAXIS: SSD Energy LABL_PTR_1: priority_label LABL_PTR_2: event_num_label SCALETYP: linear @@ -279,10 +280,10 @@ energy_per_nuc: DEPEND_1: priority_label DEPEND_2: event_num DICT_KEY: SPASE>Support>SupportQuantity:Other + DISPLAY_TYPE: spectrogram FIELDNAM: Energy per Nucleon FILLVAL: *real_fillval FORMAT: F12.4 - LABLAXIS: Energy per Nucleon LABL_PTR_1: priority_label LABL_PTR_2: event_num_label SCALETYP: linear @@ -297,10 +298,10 @@ ssd_id: DEPEND_1: priority DEPEND_2: event_num DICT_KEY: SPASE>Support>SupportQuantity:Other + DISPLAY_TYPE: no_plot FIELDNAM: SSD ID FILLVAL: *uint8_fillval FORMAT: I2 - LABLAXIS: SSD ID LABL_PTR_1: priority_label LABL_PTR_2: event_num_label SCALETYP: linear diff --git a/imap_processing/cdf/config/imap_codice_l2-lo-direct-events_variable_attrs.yaml b/imap_processing/cdf/config/imap_codice_l2-lo-direct-events_variable_attrs.yaml index 35ffbdf766..f2ffb4ea39 100644 --- a/imap_processing/cdf/config/imap_codice_l2-lo-direct-events_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_codice_l2-lo-direct-events_variable_attrs.yaml @@ -65,10 +65,10 @@ num_events: DEPEND_0: epoch DEPEND_1: priority DICT_KEY: SPASE>Support>SupportQuantity:Other + DISPLAY_TYPE: time_series FIELDNAM: Number of Events FILLVAL: *uint16_fillval FORMAT: I5 - LABLAXIS: Number of Events LABL_PTR_1: priority_label SCALETYP: linear UNITS: " " @@ -81,6 +81,7 @@ data_quality: DEPEND_0: epoch DEPEND_1: priority DICT_KEY: SPASE>Support>SupportQuantity:DataQuality + DISPLAY_TYPE: no_plot FIELDNAM: Data Quality FILLVAL: *uint8_fillval FORMAT: I3 @@ -97,10 +98,10 @@ energy_step: DEPEND_1: priority DEPEND_2: event_num DICT_KEY: SPASE>Support>SupportQuantity:Other + DISPLAY_TYPE: no_plot FIELDNAM: Energy Step FILLVAL: *uint8_fillval FORMAT: I3 - LABLAXIS: Energy Step LABL_PTR_1: priority_label LABL_PTR_2: event_num_label SCALETYP: linear @@ -115,10 +116,10 @@ energy_per_charge: DEPEND_1: priority DEPEND_2: event_num DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:EnergyPerCharge + DISPLAY_TYPE: spectrogram FIELDNAM: Energy per Charge FILLVAL: *real_fillval FORMAT: F12.4 - LABLAXIS: Energy per Charge LABL_PTR_1: priority_label LABL_PTR_2: event_num_label SCALETYP: linear @@ -133,10 +134,10 @@ apd_energy: DEPEND_1: priority DEPEND_2: event_num DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:Energy + DISPLAY_TYPE: spectrogram FIELDNAM: APD Energy FILLVAL: *real_fillval FORMAT: F12.4 - LABLAXIS: APD Energy LABL_PTR_1: priority_label LABL_PTR_2: event_num_label SCALETYP: linear @@ -151,10 +152,10 @@ gain: DEPEND_1: priority DEPEND_2: event_num DICT_KEY: SPASE>Support>SupportQuantity:Other + DISPLAY_TYPE: no_plot FIELDNAM: Gain FILLVAL: *uint8_fillval FORMAT: I1 - LABLAXIS: Gain LABL_PTR_1: priority_label LABL_PTR_2: event_num_label SCALETYP: linear @@ -169,10 +170,10 @@ apd_id: DEPEND_1: priority DEPEND_2: event_num DICT_KEY: SPASE>Support>SupportQuantity:Other + DISPLAY_TYPE: no_plot FIELDNAM: APD ID FILLVAL: *uint8_fillval FORMAT: I2 - LABLAXIS: APD ID LABL_PTR_1: priority_label LABL_PTR_2: event_num_label SCALETYP: linear @@ -187,10 +188,10 @@ position: DEPEND_1: priority DEPEND_2: event_num DICT_KEY: SPASE>Support>SupportQuantity:Other + DISPLAY_TYPE: no_plot FIELDNAM: Position FILLVAL: *uint8_fillval FORMAT: I2 - LABLAXIS: Position LABL_PTR_1: priority_label LABL_PTR_2: event_num_label SCALETYP: linear @@ -205,10 +206,10 @@ multi_flag: DEPEND_1: priority DEPEND_2: event_num DICT_KEY: SPASE>Support>SupportQuantity:DataQuality + DISPLAY_TYPE: no_plot FIELDNAM: Multi Flag FILLVAL: *uint8_fillval FORMAT: I1 - LABLAXIS: Multi Flag LABL_PTR_1: priority_label LABL_PTR_2: event_num_label SCALETYP: linear @@ -223,10 +224,10 @@ type: DEPEND_1: priority DEPEND_2: event_num DICT_KEY: SPASE>Support>SupportQuantity:Other + DISPLAY_TYPE: no_plot FIELDNAM: Type FILLVAL: *uint8_fillval FORMAT: I1 - LABLAXIS: Type LABL_PTR_1: priority_label LABL_PTR_2: event_num_label SCALETYP: linear @@ -241,10 +242,10 @@ tof: DEPEND_1: priority DEPEND_2: event_num DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:TimeOfFlight + DISPLAY_TYPE: spectrogram FIELDNAM: Time of Flight FILLVAL: *real_fillval FORMAT: F12.4 - LABLAXIS: TOF LABL_PTR_1: priority_label LABL_PTR_2: event_num_label SCALETYP: linear @@ -277,6 +278,7 @@ spin_angle: DEPEND_1: priority DEPEND_2: event_num DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:ArrivalDirection,Qualifier:DirectionAngle.AzimuthAngle + DISPLAY_TYPE: spectrogram FIELDNAM: Spin Angle FILLVAL: *real_fillval FORMAT: F8.2 @@ -294,10 +296,10 @@ elevation_angle: DEPEND_1: priority DEPEND_2: event_num DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:ArrivalDirection,Qualifier:DirectionAngle.ElevationAngle + DISPLAY_TYPE: spectrogram FIELDNAM: Elevation Angle FILLVAL: *real_fillval FORMAT: F8.2 - LABLAXIS: Elevation Angle LABL_PTR_1: priority_label LABL_PTR_2: event_num_label SCALETYP: linear diff --git a/imap_processing/codice/codice_l2.py b/imap_processing/codice/codice_l2.py index 6567278073..650613ab12 100644 --- a/imap_processing/codice/codice_l2.py +++ b/imap_processing/codice/codice_l2.py @@ -1332,7 +1332,7 @@ def process_lo_direct_events(dependencies: ProcessingInputCollection) -> xr.Data # skip adding attributes for these variables. They should already # have attrs carried over from l1a. continue - l2_dataset[var].attrs.update(cdf_attrs.get_variable_attributes(var)) + l2_dataset[var].attrs = cdf_attrs.get_variable_attributes(var) # Update coord attributes l2_dataset["priority"].attrs.update( cdf_attrs.get_variable_attributes("priority", check_schema=False) @@ -1461,7 +1461,7 @@ def process_hi_direct_events(dependencies: ProcessingInputCollection) -> xr.Data cdf_attrs.get_global_attributes("imap_codice_l2_hi-direct-events") ) for var in l2_dataset.data_vars: - l2_dataset[var].attrs.update(cdf_attrs.get_variable_attributes(var)) + l2_dataset[var].attrs = cdf_attrs.get_variable_attributes(var) # Update coord attributes l2_dataset["priority"].attrs.update( cdf_attrs.get_variable_attributes("priority", check_schema=False) diff --git a/imap_processing/tests/codice/test_codice_l2.py b/imap_processing/tests/codice/test_codice_l2.py index ae738e00b0..9ad0db6f55 100644 --- a/imap_processing/tests/codice/test_codice_l2.py +++ b/imap_processing/tests/codice/test_codice_l2.py @@ -3,6 +3,7 @@ from unittest import mock from unittest.mock import MagicMock, patch +import cdflib import numpy as np import pandas as pd import pytest @@ -43,6 +44,50 @@ "imap_codice_l2_lo-direct-events", ] +DIRECT_EVENT_LUTS = { + "lo-direct-events": [ + "l2-lo-onboard-energy-table", + "l2-lo-onboard-energy-bins", + "l2-lo-onboard-mpq-cal", + "l2-lo-onboard-mpq-cal", + ], + "hi-direct-events": [ + "l2-hi-energy-table", + "l2-hi-tof-table", + ], +} + +DIRECT_EVENT_DISPLAY_TYPES = { + "lo-direct-events": { + "num_events": "time_series", + "data_quality": "no_plot", + "gain": "no_plot", + "multi_flag": "no_plot", + "spin_angle": "spectrogram", + "elevation_angle": "spectrogram", + "tof": "spectrogram", + "type": "no_plot", + "apd_energy": "spectrogram", + "apd_id": "no_plot", + "energy_per_charge": "spectrogram", + "energy_step": "no_plot", + "position": "no_plot", + }, + "hi-direct-events": { + "num_events": "time_series", + "data_quality": "no_plot", + "gain": "no_plot", + "multi_flag": "no_plot", + "spin_angle": "spectrogram", + "elevation_angle": "spectrogram", + "tof": "spectrogram", + "type": "no_plot", + "ssd_energy": "spectrogram", + "energy_per_nuc": "spectrogram", + "ssd_id": "no_plot", + }, +} + @pytest.fixture def processing_dependencies(codice_lut_path): @@ -80,6 +125,27 @@ def mock_cdf_attrs(): return cdf_attrs +def _generate_direct_events_l2_file(mock_get_file_paths, codice_lut_path, descriptor): + """Generate a fresh CoDICE direct-events L2 CDF for metadata assertions.""" + mock_get_file_paths.side_effect = [ + codice_lut_path(descriptor=descriptor, data_type="l0") + ] + l1a_cdf = process_l1a(ProcessingInputCollection())[0] + processed_l1a_file = write_cdf(l1a_cdf) + file_path = processed_l1a_file.as_posix() + mock_get_file_paths.side_effect = [ + [file_path], + [file_path], + *[ + codice_lut_path(descriptor=lut_descriptor) + for lut_descriptor in DIRECT_EVENT_LUTS[descriptor] + ], + ] + processed_l2_ds = process_codice_l2(descriptor, ProcessingInputCollection()) + processed_l2_ds.attrs["Data_version"] = "001" + return write_cdf(processed_l2_ds) + + @pytest.fixture def mock_half_spin_per_esa_step(): """ @@ -720,3 +786,24 @@ def test_codice_l2_hi_de(mock_get_file_paths, codice_lut_path): errors = CDFValidator().validate(file) assert not errors load_cdf(file) + + +@pytest.mark.parametrize("descriptor", ["lo-direct-events", "hi-direct-events"]) +@patch("imap_data_access.processing_input.ProcessingInputCollection.get_file_paths") +def test_codice_l2_direct_events_display_type_cdf_metadata( + mock_get_file_paths, codice_lut_path, descriptor +): + file = _generate_direct_events_l2_file( + mock_get_file_paths, codice_lut_path, descriptor + ) + cdf_file = cdflib.CDF(str(file)) + + for variable, expected_display_type in DIRECT_EVENT_DISPLAY_TYPES[ + descriptor + ].items(): + attrs = cdf_file.varattsget(variable) + assert "DISPLAY_TYPE" in attrs, f"{variable} is missing DISPLAY_TYPE" + assert isinstance(attrs["DISPLAY_TYPE"], str), ( + f"{variable} DISPLAY_TYPE must be stored as a string" + ) + assert attrs["DISPLAY_TYPE"] == expected_display_type From 533e64e776bce2aecaf7e82a4011cdb650887990 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 14 May 2026 18:23:16 -0400 Subject: [PATCH 2/2] fix(codice): correct hi direct-events energy_per_nuc dependency metadata Fix the remaining CoDICE direct-events metadata mismatch in `imap_codice_l2_hi-direct-events`. `energy_per_nuc` is a multidimensional data variable whose second axis is the numeric `priority` coordinate. Its L2 attr template was incorrectly setting: DEPEND_1 = priority_label That made the dependency point at a character label variable rather than the actual coordinate variable, which caused SPDF to report: DEPEND_1 is a character type Update the L2 attr template so `energy_per_nuc` follows the same ISTP pattern already used by neighboring direct-events variables: - `DEPEND_1 = priority` - keep `LABL_PTR_1 = priority_label` This preserves the label mapping for display purposes while making the dimension dependency point to the correct numeric support variable. Also extend the existing direct-events written-CDF metadata regression to verify that `hi-direct-events` writes: - `energy_per_nuc DEPEND_1 = priority` - `energy_per_nuc LABL_PTR_1 = priority_label` Validated locally with: - focused CoDICE direct-events pytest runs - pre-commit on the touched files This commit intentionally leaves the separate `epoch_delta_*` metadata question untouched. --- .../imap_codice_l2-hi-direct-events_variable_attrs.yaml | 2 +- imap_processing/tests/codice/test_codice_l2.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/imap_processing/cdf/config/imap_codice_l2-hi-direct-events_variable_attrs.yaml b/imap_processing/cdf/config/imap_codice_l2-hi-direct-events_variable_attrs.yaml index 25062e1081..93604c66d9 100644 --- a/imap_processing/cdf/config/imap_codice_l2-hi-direct-events_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_codice_l2-hi-direct-events_variable_attrs.yaml @@ -277,7 +277,7 @@ ssd_energy: energy_per_nuc: CATDESC: Energy per Nucleon DEPEND_0: epoch - DEPEND_1: priority_label + DEPEND_1: priority DEPEND_2: event_num DICT_KEY: SPASE>Support>SupportQuantity:Other DISPLAY_TYPE: spectrogram diff --git a/imap_processing/tests/codice/test_codice_l2.py b/imap_processing/tests/codice/test_codice_l2.py index 9ad0db6f55..b4f9791abc 100644 --- a/imap_processing/tests/codice/test_codice_l2.py +++ b/imap_processing/tests/codice/test_codice_l2.py @@ -807,3 +807,8 @@ def test_codice_l2_direct_events_display_type_cdf_metadata( f"{variable} DISPLAY_TYPE must be stored as a string" ) assert attrs["DISPLAY_TYPE"] == expected_display_type + + if descriptor == "hi-direct-events": + attrs = cdf_file.varattsget("energy_per_nuc") + assert attrs["DEPEND_1"] == "priority" + assert attrs["LABL_PTR_1"] == "priority_label"