Fix ISTP validation FILLVAL errors

imap_processing/cdf/config/imap_codice_l2-hi-direct-events_variable_attrs.yaml

@@ -84,7 +84,7 @@ data_quality:
   DEPEND_1: priority
   DICT_KEY: SPASE>Support>SupportQuantity:DataQuality
   FIELDNAM: Data Quality
-  FILLVAL: *uint8_fillval
+  FILLVAL: *uint16_fillval
   FORMAT: I3
   LABLAXIS: Data Quality
   LABL_PTR_1: priority_label

imap_processing/cdf/config/imap_codice_l2-lo-direct-events_variable_attrs.yaml

@@ -82,7 +82,7 @@ data_quality:
   DEPEND_1: priority
   DICT_KEY: SPASE>Support>SupportQuantity:DataQuality
   FIELDNAM: Data Quality
-  FILLVAL: *uint8_fillval
+  FILLVAL: *uint16_fillval
   FORMAT: I3
   LABL_PTR_1: priority_label
   SCALETYP: linear
@@ -261,7 +261,7 @@ spin_sector:
   DICT_KEY: SPASE>Support>SupportQuantity:Positional
   FIELDNAM: Spin Sector Index
   FILLVAL: *uint8_fillval
-  FORMAT: I2
+  FORMAT: I3
   LABLAXIS: Spin Sector
   LABL_PTR_1: priority_label
   LABL_PTR_2: event_num_label

imap_processing/cdf/config/imap_glows_l2_variable_attrs.yaml

@@ -483,6 +483,7 @@ flux_uncertainties:
 histogram_flag_array:
   <<: *lightcurve_defaults
   CATDESC: Bad-angle flags for histogram bins
+  CDF_DATA_TYPE: CDF_UINT1
   DICT_KEY: SPASE>Support>SupportQuantity:DataQuality
   FIELDNAM: Bad-angle flags for histogram
   FILLVAL: 255
@@ -522,9 +523,10 @@ ecliptic_lat:
 number_of_bins:
   <<: *support_data_defaults
   CATDESC: Number of bins in histogram
+  CDF_DATA_TYPE: CDF_UINT2
   DICT_KEY: SPASE>Support>SupportQuantity:Other
   FIELDNAM: Number of bins in histogram
-  FILLVAL: -9223372036854775808
+  FILLVAL: *max_uint16
   FORMAT: I4
   LABLAXIS: No. of bins
   UNITS: ' '

imap_processing/cdf/config/imap_hit_l2_variable_attrs.yaml

@@ -331,7 +331,7 @@ dynamic_threshold_state:
   DICT_KEY: SPASE>Support>SupportQuantity:InstrumentMode
   DISPLAY_TYPE: time_series
   FIELDNAM: Dynamic threshold state
-  FILLVAL: -128
+  FILLVAL: 255
   FORMAT: I1
   LABLAXIS: State
   SCALEMAX: 1000
@@ -1997,4 +1997,3 @@ fe_total_uncert_minus_macropixel:
 
 
 
-

imap_processing/cdf/config/imap_idex_l1b_variable_attrs.yaml

@@ -483,8 +483,8 @@ spin_phase:
   CATDESC: IMAP Spin Phase
   DICT_KEY: SPASE>Support>SupportQuantity:SpinPhase
   FIELDNAM: Spin Phase
-  FILLVAL: *int_fillval
-  FORMAT: I3
+  FILLVAL: *double_fillval
+  FORMAT: F8.3
   LABLAXIS: Spin Phase
   UNITS: Degrees
   VALIDMAX: 360
@@ -499,4 +499,3 @@ solar_longitude:
   UNITS: Degrees
   VALIDMAX: 180
   VALIDMIN: -180
-

imap_processing/cdf/config/imap_mag_l2_variable_attrs.yaml

@@ -49,7 +49,7 @@ vector_attrs: &vectors_default
     DEPEND_1: direction
     DICT_KEY: "SPASE>Field>FieldQuantity:Magnetic,Qualifier:Vector,CoordinateSystemName:DSRF,CoordinateRepresentation:Cartesian"
     FIELDNAM: Magnetic Field Vector
-    FILLVAL: 9223372036854775807
+    FILLVAL: -1.0e31
     FORMAT: F12.5
     LABL_PTR_1: direction_label
 

imap_processing/cdf/config/imap_swapi_variable_attrs.yaml

@@ -107,14 +107,14 @@ esa_energy:
   DEPEND_1: esa_step
   DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:EnergyPerCharge
   FIELDNAM: ESA Energy
-  FILLVAL: -9223372036854775808
-  FORMAT: I5
+  FILLVAL: -1.0000000E+31
+  FORMAT: F8.1
   LABLAXIS: Energy(eV)
   LABL_PTR_1: esa_step_label
   SCALETYP: linear
   UNITS: eV / q
-  VALIDMAX: 65535
-  VALIDMIN: 0
+  VALIDMAX: 21000.0
+  VALIDMIN: 0.0
   VAR_TYPE: support_data
 
 metadata_default: &metadata_default

imap_processing/cdf/config/imap_swe_l2_variable_attrs.yaml

@@ -291,7 +291,7 @@ acq_duration:
   DICT_KEY: SPASE>Support>SupportQuantity:Temporal,Qualifier:Array
   DISPLAY_TYPE: spectrogram
   FIELDNAM: Acquisition Duration
-  FILLVAL: -9223372036854775808
+  FILLVAL: 4294967295
   FORMAT: I10
   LABL_PTR_1: esa_step_label
   LABL_PTR_2: spin_sector_label

imap_processing/codice/codice_l2.py

@@ -563,6 +563,20 @@ def process_lo_species_intensity(
     for species in species_list:
         dataset[species].data[half_spin_boundary] = np.nan
 
+    for var in ["nso_esa_step", "nso_spin_sector"]:
+        if var in dataset:
+            fillval = dataset[var].attrs["FILLVAL"]
+            restored_values = dataset[var].data.astype(np.float64, copy=True)
+            restored_values = np.nan_to_num(
+                restored_values, nan=fillval, posinf=fillval, neginf=fillval
+            )
+            restored_values = np.clip(np.rint(restored_values), 0, 255).astype(np.uint8)
+            dataset[var] = xr.DataArray(
+                restored_values,
+                dims=dataset[var].dims,
+                attrs=dataset[var].attrs,
+            )
+
     return dataset
 
 
@@ -1093,6 +1107,10 @@ def process_lo_direct_events(dependencies: ProcessingInputCollection) -> xr.Data
         l2_dataset["position"].dims,
         elevation_angle.astype(np.float32),
     )
+    spin_sector_attrs = cdf_attrs.get_variable_attributes(
+        "spin_sector", check_schema=False
+    )
+    spin_sector_fillval = np.uint8(spin_sector_attrs["FILLVAL"])
     # Convert spin_sector to spin_angle in degrees
     # Use equation from section 11.2.2 of algorithm document
     # Shift all spin sectors for all positions 13 - 24 adding 12 and mod 24
@@ -1104,13 +1122,16 @@ def process_lo_direct_events(dependencies: ProcessingInputCollection) -> xr.Data
     )
     l2_dataset["spin_angle"] = l2_dataset["spin_sector"].astype(np.float32) * 15.0 + 7.5
 
-    # Set spin angle and sector to NaN for invalid positions (>23)
+    # Preserve spin_sector as an integer index while marking invalid sectors.
+    invalid_spin_sector = ~np.isfinite(original_spin_sector) | (
+        original_spin_sector > 23
+    )
     l2_dataset["spin_angle"] = xr.where(
-        (original_spin_sector > 23), np.nan, l2_dataset["spin_angle"]
+        invalid_spin_sector, np.nan, l2_dataset["spin_angle"]
     )
     l2_dataset["spin_sector"] = xr.where(
-        (original_spin_sector > 23), np.nan, l2_dataset["spin_sector"]
-    )
+        invalid_spin_sector, spin_sector_fillval, l2_dataset["spin_sector"]
+    ).astype(np.uint8)
     # convert apd energy to physical units
     # Set the gain labels based on gain values
     gains = l2_dataset["gain"].values.ravel()

imap_processing/glows/l2/glows_l2.py

@@ -23,6 +23,40 @@
 logger = logging.getLogger(__name__)
 
 
+def _pad_daily_lightcurve_bins(value: object, fillval: object) -> np.ndarray:
+    """
+    Pad chopped daily-lightcurve bin data back to the standard bin count.
+
+    Parameters
+    ----------
+    value : object
+        Chopped daily-lightcurve bin data.
+    fillval : object
+        CDF fill value used for padded bins.
+
+    Returns
+    -------
+    numpy.ndarray
+        Bin data padded to the standard bin count.
+    """
+    value_array = np.asarray(value)
+    padded_dtype = value_array.dtype
+    fillval_dtype = np.asarray(fillval).dtype
+
+    if np.issubdtype(padded_dtype, np.integer):
+        try:
+            cast_fillval = np.array(fillval, dtype=padded_dtype).item()
+        except (OverflowError, TypeError, ValueError):
+            padded_dtype = np.result_type(padded_dtype, fillval_dtype)
+        else:
+            if cast_fillval != fillval:
+                padded_dtype = np.result_type(padded_dtype, fillval_dtype)
+
+    padded = np.full(GlowsConstants.STANDARD_BIN_COUNT, fillval, dtype=padded_dtype)
+    padded[: len(value_array)] = value_array
+    return padded
+
+
 def glows_l2(
     input_dataset: xr.Dataset,
     pipeline_settings_dataset: xr.Dataset,
@@ -193,7 +227,9 @@ def create_l2_dataset(
             # Convert time to UTC
             utc_string = [met_to_utc(ttj2000ns_to_met(value))]
             output[key] = xr.DataArray(
-                utc_string, dims=["epoch"], attrs=attrs.get_variable_attributes(key)
+                utc_string,
+                dims=["epoch"],
+                attrs=attrs.get_variable_attributes(key),
             )
         elif key != "daily_lightcurve":
             val = value
@@ -205,12 +241,11 @@ def create_l2_dataset(
                 attrs=attrs.get_variable_attributes(key),
             )
 
-    n_bins = histogram_l2.daily_lightcurve.number_of_bins
     for key, value in dataclasses.asdict(histogram_l2.daily_lightcurve).items():
         if key == "number_of_bins":
             # number_of_bins does not have a bins dimension.
             output[key] = xr.DataArray(
-                np.array([value]),
+                np.array([value], dtype=np.uint16),
                 dims=["epoch"],
                 attrs=attrs.get_variable_attributes(key),
             )
@@ -219,9 +254,7 @@ def create_l2_dataset(
             # avoid operating on FILLVAL data. Re-expand to STANDARD_BIN_COUNT
             # here, filling unused bins with the variable's CDF FILLVAL.
             var_attrs = attrs.get_variable_attributes(key)
-            fillval = var_attrs["FILLVAL"]
-            padded = np.full(GlowsConstants.STANDARD_BIN_COUNT, fillval)
-            padded[:n_bins] = value
+            padded = _pad_daily_lightcurve_bins(value, var_attrs["FILLVAL"])
             output[key] = xr.DataArray(
                 np.array([padded]),
                 dims=["epoch", "bins"],

imap_processing/tests/codice/test_codice_l2.py

@@ -3,6 +3,7 @@
 from unittest import mock
 from unittest.mock import MagicMock, patch
 
+import cdflib
 import numpy as np
 import pandas as pd
 import pytest
@@ -379,7 +380,13 @@ def test_codice_l2_sw_species_intensity(mock_get_file_paths, codice_lut_path):
         )
     processed_2_ds.attrs["Data_version"] = "001"
     assert processed_2_ds.attrs["Logical_source"] == "imap_codice_l2_lo-sw-species"
-    write_cdf(processed_2_ds)
+    cdf_file_path = write_cdf(processed_2_ds)
+    cdf_file = cdflib.CDF(cdf_file_path)
+    for var in ["nso_esa_step", "nso_spin_sector"]:
+        var_info = cdf_file.varinq(var)
+        var_attrs = cdf_file.varattsget(var)
+        assert var_info.Data_Type_Description == "CDF_UINT1"
+        assert var_attrs["FILLVAL"] == np.uint8(255)
 
 
 @patch("imap_data_access.processing_input.ProcessingInputCollection.get_file_paths")
@@ -402,6 +409,30 @@ def test_codice_l2_lo_de(mock_get_file_paths, codice_lut_path):
     ]
 
     processed_l2_ds = process_codice_l2("lo-direct-events", ProcessingInputCollection())
+    l1a_input_ds = load_cdf(processed_l1a_file)
+    original_spin_sector = l1a_input_ds["spin_sector"].values
+    # Mirror the LO direct-event spin-sector remapping so this test catches any
+    # unintended changes to valid sector values while still checking that only
+    # invalid sectors are replaced with the uint8 fill value.
+    expected_spin_sector = np.where(
+        (l1a_input_ds["position"].values >= 13)
+        & (l1a_input_ds["position"].values <= 24),
+        (original_spin_sector + 12) % 24,
+        original_spin_sector,
+    )
+    invalid_spin_sector = ~np.isfinite(original_spin_sector) | (
+        original_spin_sector > 23
+    )
+    expected_spin_sector = np.where(
+        invalid_spin_sector, np.uint8(255), expected_spin_sector
+    ).astype(np.uint8)
+    assert processed_l2_ds["spin_sector"].dtype == np.uint8
+    np.testing.assert_array_equal(
+        processed_l2_ds["spin_sector"].values,
+        expected_spin_sector,
+        err_msg="LO direct-event spin_sector values changed unexpectedly",
+    )
+
     l2_val_data = (
         imap_module_directory
         / "tests"
@@ -441,6 +472,15 @@ def test_codice_l2_lo_de(mock_get_file_paths, codice_lut_path):
     file = write_cdf(processed_l2_ds)
     errors = CDFValidator().validate(file)
     assert not errors
+    cdf_file = cdflib.CDF(file)
+    spin_sector_info = cdf_file.varinq("spin_sector")
+    spin_sector_attrs = cdf_file.varattsget("spin_sector")
+    data_quality_info = cdf_file.varinq("data_quality")
+    data_quality_attrs = cdf_file.varattsget("data_quality")
+    assert spin_sector_info.Data_Type_Description == "CDF_UINT1"
+    assert spin_sector_attrs["FILLVAL"] == np.uint8(255)
+    assert data_quality_info.Data_Type_Description == "CDF_UINT2"
+    assert data_quality_attrs["FILLVAL"] == np.uint16(65535)
     load_cdf(file)
 
 
@@ -494,4 +534,9 @@ def test_codice_l2_hi_de(mock_get_file_paths, codice_lut_path):
     file = write_cdf(processed_l2_ds)
     errors = CDFValidator().validate(file)
     assert not errors
+    cdf_file = cdflib.CDF(file)
+    data_quality_info = cdf_file.varinq("data_quality")
+    data_quality_attrs = cdf_file.varattsget("data_quality")
+    assert data_quality_info.Data_Type_Description == "CDF_UINT2"
+    assert data_quality_attrs["FILLVAL"] == np.uint16(65535)
     load_cdf(file)