diff --git a/imap_processing/cdf/config/imap_codice_l2-hi-direct-events_variable_attrs.yaml b/imap_processing/cdf/config/imap_codice_l2-hi-direct-events_variable_attrs.yaml index e7e2814579..65f43bba35 100644 --- a/imap_processing/cdf/config/imap_codice_l2-hi-direct-events_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_codice_l2-hi-direct-events_variable_attrs.yaml @@ -84,7 +84,7 @@ data_quality: DEPEND_1: priority DICT_KEY: SPASE>Support>SupportQuantity:DataQuality FIELDNAM: Data Quality - FILLVAL: *uint8_fillval + FILLVAL: *uint16_fillval FORMAT: I3 LABLAXIS: Data Quality LABL_PTR_1: priority_label diff --git a/imap_processing/cdf/config/imap_codice_l2-lo-direct-events_variable_attrs.yaml b/imap_processing/cdf/config/imap_codice_l2-lo-direct-events_variable_attrs.yaml index 35ffbdf766..80774344a0 100644 --- a/imap_processing/cdf/config/imap_codice_l2-lo-direct-events_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_codice_l2-lo-direct-events_variable_attrs.yaml @@ -82,7 +82,7 @@ data_quality: DEPEND_1: priority DICT_KEY: SPASE>Support>SupportQuantity:DataQuality FIELDNAM: Data Quality - FILLVAL: *uint8_fillval + FILLVAL: *uint16_fillval FORMAT: I3 LABL_PTR_1: priority_label SCALETYP: linear @@ -261,7 +261,7 @@ spin_sector: DICT_KEY: SPASE>Support>SupportQuantity:Positional FIELDNAM: Spin Sector Index FILLVAL: *uint8_fillval - FORMAT: I2 + FORMAT: I3 LABLAXIS: Spin Sector LABL_PTR_1: priority_label LABL_PTR_2: event_num_label diff --git a/imap_processing/cdf/config/imap_glows_l2_variable_attrs.yaml b/imap_processing/cdf/config/imap_glows_l2_variable_attrs.yaml index 209b93e097..564895922e 100644 --- a/imap_processing/cdf/config/imap_glows_l2_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_glows_l2_variable_attrs.yaml @@ -483,6 +483,7 @@ flux_uncertainties: histogram_flag_array: <<: *lightcurve_defaults CATDESC: Bad-angle flags for histogram bins + CDF_DATA_TYPE: CDF_UINT1 DICT_KEY: SPASE>Support>SupportQuantity:DataQuality FIELDNAM: Bad-angle flags for histogram FILLVAL: 255 @@ -522,9 +523,10 @@ ecliptic_lat: number_of_bins: <<: *support_data_defaults CATDESC: Number of bins in histogram + CDF_DATA_TYPE: CDF_UINT2 DICT_KEY: SPASE>Support>SupportQuantity:Other FIELDNAM: Number of bins in histogram - FILLVAL: -9223372036854775808 + FILLVAL: *max_uint16 FORMAT: I4 LABLAXIS: No. of bins UNITS: ' ' diff --git a/imap_processing/cdf/config/imap_hit_l2_variable_attrs.yaml b/imap_processing/cdf/config/imap_hit_l2_variable_attrs.yaml index 0fa85ae795..b782d93ee7 100644 --- a/imap_processing/cdf/config/imap_hit_l2_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_hit_l2_variable_attrs.yaml @@ -331,7 +331,7 @@ dynamic_threshold_state: DICT_KEY: SPASE>Support>SupportQuantity:InstrumentMode DISPLAY_TYPE: time_series FIELDNAM: Dynamic threshold state - FILLVAL: -128 + FILLVAL: 255 FORMAT: I1 LABLAXIS: State SCALEMAX: 1000 @@ -1997,4 +1997,3 @@ fe_total_uncert_minus_macropixel: - diff --git a/imap_processing/cdf/config/imap_idex_l1b_variable_attrs.yaml b/imap_processing/cdf/config/imap_idex_l1b_variable_attrs.yaml index 0739e5b1f0..b18f6c54d8 100644 --- a/imap_processing/cdf/config/imap_idex_l1b_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_idex_l1b_variable_attrs.yaml @@ -483,8 +483,8 @@ spin_phase: CATDESC: IMAP Spin Phase DICT_KEY: SPASE>Support>SupportQuantity:SpinPhase FIELDNAM: Spin Phase - FILLVAL: *int_fillval - FORMAT: I3 + FILLVAL: *double_fillval + FORMAT: F8.3 LABLAXIS: Spin Phase UNITS: Degrees VALIDMAX: 360 @@ -499,4 +499,3 @@ solar_longitude: UNITS: Degrees VALIDMAX: 180 VALIDMIN: -180 - diff --git a/imap_processing/cdf/config/imap_mag_l2_variable_attrs.yaml b/imap_processing/cdf/config/imap_mag_l2_variable_attrs.yaml index bb193c8280..6d376c8e4d 100644 --- a/imap_processing/cdf/config/imap_mag_l2_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_mag_l2_variable_attrs.yaml @@ -49,7 +49,7 @@ vector_attrs: &vectors_default DEPEND_1: direction DICT_KEY: "SPASE>Field>FieldQuantity:Magnetic,Qualifier:Vector,CoordinateSystemName:DSRF,CoordinateRepresentation:Cartesian" FIELDNAM: Magnetic Field Vector - FILLVAL: 9223372036854775807 + FILLVAL: -1.0e31 FORMAT: F12.5 LABL_PTR_1: direction_label diff --git a/imap_processing/cdf/config/imap_swapi_variable_attrs.yaml b/imap_processing/cdf/config/imap_swapi_variable_attrs.yaml index 30b0a30378..a74cb91600 100644 --- a/imap_processing/cdf/config/imap_swapi_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_swapi_variable_attrs.yaml @@ -107,14 +107,14 @@ esa_energy: DEPEND_1: esa_step DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:EnergyPerCharge FIELDNAM: ESA Energy - FILLVAL: -9223372036854775808 - FORMAT: I5 + FILLVAL: -1.0000000E+31 + FORMAT: F8.1 LABLAXIS: Energy(eV) LABL_PTR_1: esa_step_label SCALETYP: linear UNITS: eV / q - VALIDMAX: 65535 - VALIDMIN: 0 + VALIDMAX: 21000.0 + VALIDMIN: 0.0 VAR_TYPE: support_data metadata_default: &metadata_default diff --git a/imap_processing/cdf/config/imap_swe_l2_variable_attrs.yaml b/imap_processing/cdf/config/imap_swe_l2_variable_attrs.yaml index c61e8494bc..5cc3fd2207 100644 --- a/imap_processing/cdf/config/imap_swe_l2_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_swe_l2_variable_attrs.yaml @@ -291,7 +291,7 @@ acq_duration: DICT_KEY: SPASE>Support>SupportQuantity:Temporal,Qualifier:Array DISPLAY_TYPE: spectrogram FIELDNAM: Acquisition Duration - FILLVAL: -9223372036854775808 + FILLVAL: 4294967295 FORMAT: I10 LABL_PTR_1: esa_step_label LABL_PTR_2: spin_sector_label diff --git a/imap_processing/codice/codice_l2.py b/imap_processing/codice/codice_l2.py index 0a7998ee91..ea41c68654 100644 --- a/imap_processing/codice/codice_l2.py +++ b/imap_processing/codice/codice_l2.py @@ -563,6 +563,20 @@ def process_lo_species_intensity( for species in species_list: dataset[species].data[half_spin_boundary] = np.nan + for var in ["nso_esa_step", "nso_spin_sector"]: + if var in dataset: + fillval = dataset[var].attrs["FILLVAL"] + restored_values = dataset[var].data.astype(np.float64, copy=True) + restored_values = np.nan_to_num( + restored_values, nan=fillval, posinf=fillval, neginf=fillval + ) + restored_values = np.clip(np.rint(restored_values), 0, 255).astype(np.uint8) + dataset[var] = xr.DataArray( + restored_values, + dims=dataset[var].dims, + attrs=dataset[var].attrs, + ) + return dataset @@ -1093,6 +1107,10 @@ def process_lo_direct_events(dependencies: ProcessingInputCollection) -> xr.Data l2_dataset["position"].dims, elevation_angle.astype(np.float32), ) + spin_sector_attrs = cdf_attrs.get_variable_attributes( + "spin_sector", check_schema=False + ) + spin_sector_fillval = np.uint8(spin_sector_attrs["FILLVAL"]) # Convert spin_sector to spin_angle in degrees # Use equation from section 11.2.2 of algorithm document # Shift all spin sectors for all positions 13 - 24 adding 12 and mod 24 @@ -1104,13 +1122,16 @@ def process_lo_direct_events(dependencies: ProcessingInputCollection) -> xr.Data ) l2_dataset["spin_angle"] = l2_dataset["spin_sector"].astype(np.float32) * 15.0 + 7.5 - # Set spin angle and sector to NaN for invalid positions (>23) + # Preserve spin_sector as an integer index while marking invalid sectors. + invalid_spin_sector = ~np.isfinite(original_spin_sector) | ( + original_spin_sector > 23 + ) l2_dataset["spin_angle"] = xr.where( - (original_spin_sector > 23), np.nan, l2_dataset["spin_angle"] + invalid_spin_sector, np.nan, l2_dataset["spin_angle"] ) l2_dataset["spin_sector"] = xr.where( - (original_spin_sector > 23), np.nan, l2_dataset["spin_sector"] - ) + invalid_spin_sector, spin_sector_fillval, l2_dataset["spin_sector"] + ).astype(np.uint8) # convert apd energy to physical units # Set the gain labels based on gain values gains = l2_dataset["gain"].values.ravel() diff --git a/imap_processing/glows/l2/glows_l2.py b/imap_processing/glows/l2/glows_l2.py index e6f08e0e9b..ef5927e2c7 100644 --- a/imap_processing/glows/l2/glows_l2.py +++ b/imap_processing/glows/l2/glows_l2.py @@ -23,6 +23,40 @@ logger = logging.getLogger(__name__) +def _pad_daily_lightcurve_bins(value: object, fillval: object) -> np.ndarray: + """ + Pad chopped daily-lightcurve bin data back to the standard bin count. + + Parameters + ---------- + value : object + Chopped daily-lightcurve bin data. + fillval : object + CDF fill value used for padded bins. + + Returns + ------- + numpy.ndarray + Bin data padded to the standard bin count. + """ + value_array = np.asarray(value) + padded_dtype = value_array.dtype + fillval_dtype = np.asarray(fillval).dtype + + if np.issubdtype(padded_dtype, np.integer): + try: + cast_fillval = np.array(fillval, dtype=padded_dtype).item() + except (OverflowError, TypeError, ValueError): + padded_dtype = np.result_type(padded_dtype, fillval_dtype) + else: + if cast_fillval != fillval: + padded_dtype = np.result_type(padded_dtype, fillval_dtype) + + padded = np.full(GlowsConstants.STANDARD_BIN_COUNT, fillval, dtype=padded_dtype) + padded[: len(value_array)] = value_array + return padded + + def glows_l2( input_dataset: xr.Dataset, pipeline_settings_dataset: xr.Dataset, @@ -193,7 +227,9 @@ def create_l2_dataset( # Convert time to UTC utc_string = [met_to_utc(ttj2000ns_to_met(value))] output[key] = xr.DataArray( - utc_string, dims=["epoch"], attrs=attrs.get_variable_attributes(key) + utc_string, + dims=["epoch"], + attrs=attrs.get_variable_attributes(key), ) elif key != "daily_lightcurve": val = value @@ -205,12 +241,11 @@ def create_l2_dataset( attrs=attrs.get_variable_attributes(key), ) - n_bins = histogram_l2.daily_lightcurve.number_of_bins for key, value in dataclasses.asdict(histogram_l2.daily_lightcurve).items(): if key == "number_of_bins": # number_of_bins does not have a bins dimension. output[key] = xr.DataArray( - np.array([value]), + np.array([value], dtype=np.uint16), dims=["epoch"], attrs=attrs.get_variable_attributes(key), ) @@ -219,9 +254,7 @@ def create_l2_dataset( # avoid operating on FILLVAL data. Re-expand to STANDARD_BIN_COUNT # here, filling unused bins with the variable's CDF FILLVAL. var_attrs = attrs.get_variable_attributes(key) - fillval = var_attrs["FILLVAL"] - padded = np.full(GlowsConstants.STANDARD_BIN_COUNT, fillval) - padded[:n_bins] = value + padded = _pad_daily_lightcurve_bins(value, var_attrs["FILLVAL"]) output[key] = xr.DataArray( np.array([padded]), dims=["epoch", "bins"], diff --git a/imap_processing/tests/codice/test_codice_l2.py b/imap_processing/tests/codice/test_codice_l2.py index 86e356e4d3..86362c782c 100644 --- a/imap_processing/tests/codice/test_codice_l2.py +++ b/imap_processing/tests/codice/test_codice_l2.py @@ -3,6 +3,7 @@ from unittest import mock from unittest.mock import MagicMock, patch +import cdflib import numpy as np import pandas as pd import pytest @@ -379,7 +380,13 @@ def test_codice_l2_sw_species_intensity(mock_get_file_paths, codice_lut_path): ) processed_2_ds.attrs["Data_version"] = "001" assert processed_2_ds.attrs["Logical_source"] == "imap_codice_l2_lo-sw-species" - write_cdf(processed_2_ds) + cdf_file_path = write_cdf(processed_2_ds) + cdf_file = cdflib.CDF(cdf_file_path) + for var in ["nso_esa_step", "nso_spin_sector"]: + var_info = cdf_file.varinq(var) + var_attrs = cdf_file.varattsget(var) + assert var_info.Data_Type_Description == "CDF_UINT1" + assert var_attrs["FILLVAL"] == np.uint8(255) @patch("imap_data_access.processing_input.ProcessingInputCollection.get_file_paths") @@ -402,6 +409,30 @@ def test_codice_l2_lo_de(mock_get_file_paths, codice_lut_path): ] processed_l2_ds = process_codice_l2("lo-direct-events", ProcessingInputCollection()) + l1a_input_ds = load_cdf(processed_l1a_file) + original_spin_sector = l1a_input_ds["spin_sector"].values + # Mirror the LO direct-event spin-sector remapping so this test catches any + # unintended changes to valid sector values while still checking that only + # invalid sectors are replaced with the uint8 fill value. + expected_spin_sector = np.where( + (l1a_input_ds["position"].values >= 13) + & (l1a_input_ds["position"].values <= 24), + (original_spin_sector + 12) % 24, + original_spin_sector, + ) + invalid_spin_sector = ~np.isfinite(original_spin_sector) | ( + original_spin_sector > 23 + ) + expected_spin_sector = np.where( + invalid_spin_sector, np.uint8(255), expected_spin_sector + ).astype(np.uint8) + assert processed_l2_ds["spin_sector"].dtype == np.uint8 + np.testing.assert_array_equal( + processed_l2_ds["spin_sector"].values, + expected_spin_sector, + err_msg="LO direct-event spin_sector values changed unexpectedly", + ) + l2_val_data = ( imap_module_directory / "tests" @@ -441,6 +472,15 @@ def test_codice_l2_lo_de(mock_get_file_paths, codice_lut_path): file = write_cdf(processed_l2_ds) errors = CDFValidator().validate(file) assert not errors + cdf_file = cdflib.CDF(file) + spin_sector_info = cdf_file.varinq("spin_sector") + spin_sector_attrs = cdf_file.varattsget("spin_sector") + data_quality_info = cdf_file.varinq("data_quality") + data_quality_attrs = cdf_file.varattsget("data_quality") + assert spin_sector_info.Data_Type_Description == "CDF_UINT1" + assert spin_sector_attrs["FILLVAL"] == np.uint8(255) + assert data_quality_info.Data_Type_Description == "CDF_UINT2" + assert data_quality_attrs["FILLVAL"] == np.uint16(65535) load_cdf(file) @@ -494,4 +534,9 @@ def test_codice_l2_hi_de(mock_get_file_paths, codice_lut_path): file = write_cdf(processed_l2_ds) errors = CDFValidator().validate(file) assert not errors + cdf_file = cdflib.CDF(file) + data_quality_info = cdf_file.varinq("data_quality") + data_quality_attrs = cdf_file.varattsget("data_quality") + assert data_quality_info.Data_Type_Description == "CDF_UINT2" + assert data_quality_attrs["FILLVAL"] == np.uint16(65535) load_cdf(file) diff --git a/imap_processing/tests/glows/test_glows_l2.py b/imap_processing/tests/glows/test_glows_l2.py index b4e40de25f..29ecff9ad0 100644 --- a/imap_processing/tests/glows/test_glows_l2.py +++ b/imap_processing/tests/glows/test_glows_l2.py @@ -1,10 +1,12 @@ from unittest.mock import patch +import cdflib import numpy as np import pytest import xarray as xr from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes +from imap_processing.cdf.utils import write_cdf from imap_processing.glows.l1b.glows_l1b import glows_l1b from imap_processing.glows.l1b.glows_l1b_data import ( HistogramL1B, @@ -169,10 +171,10 @@ def test_generate_l2( cdf_attrs = ImapCdfAttributes() cdf_attrs.add_instrument_global_attrs("glows") cdf_attrs.add_instrument_variable_attrs("glows", "l2") - assert ( - create_l2_dataset(l2, cdf_attrs, l1b_hist_dataset.attrs)["epoch"].data[0] - == (l2.start_time + l2.end_time) / 2 - ) + output = create_l2_dataset(l2, cdf_attrs, l1b_hist_dataset.attrs) + assert output["epoch"].data[0] == (l2.start_time + l2.end_time) / 2 + for var in ["photon_flux", "ecliptic_lon"]: + assert np.issubdtype(output[var].dtype, np.floating) # Test case 2: L1B dataset has no good times (all flags 0) l1b_hist_dataset["flags"].values = np.zeros(l1b_hist_dataset.flags.shape) @@ -182,6 +184,63 @@ def test_generate_l2( assert ds.bad_time_flag_occurrences.dtype == np.uint16 +@patch.object(HistogramL2, "compute_position_angle", return_value=42.0) +@patch.object( + HistogramL1B, + "flag_uv_and_excluded", + return_value=(np.zeros(3600, dtype=bool), np.zeros(3600, dtype=bool)), +) +@patch.object(HistogramL1B, "update_spice_parameters", autospec=True) +def test_glows_l2_cdf_fillvals( + mock_spice_function, + mock_flag_uv_and_excluded, + mock_compute_position_angle, + l1a_dataset, + mock_ancillary_exclusions, + mock_pipeline_settings, + mock_conversion_table_dict, + mock_ecliptic_bin_centers, + mock_calibration_dataset, +): + mock_spice_function.side_effect = mock_update_spice_parameters + + l1b_hist_dataset = glows_l1b( + l1a_dataset[0], + mock_ancillary_exclusions.excluded_regions, + mock_ancillary_exclusions.uv_sources, + mock_ancillary_exclusions.suspected_transients, + mock_ancillary_exclusions.exclusions_by_instr_team, + mock_pipeline_settings, + mock_conversion_table_dict, + ) + l1b_hist_dataset.attrs["Repointing"] = "repoint00047" + + l2_dataset = glows_l2( + l1b_hist_dataset, mock_pipeline_settings, mock_calibration_dataset + )[0] + l2_dataset.attrs["Data_version"] = "001" + cdf_file_path = write_cdf(l2_dataset) + cdf_file = cdflib.CDF(cdf_file_path) + + histogram_flag_info = cdf_file.varinq("histogram_flag_array") + histogram_flag_attrs = cdf_file.varattsget("histogram_flag_array") + number_of_bins_info = cdf_file.varinq("number_of_bins") + number_of_bins_attrs = cdf_file.varattsget("number_of_bins") + photon_flux_info = cdf_file.varinq("photon_flux") + photon_flux_attrs = cdf_file.varattsget("photon_flux") + ecliptic_lon_info = cdf_file.varinq("ecliptic_lon") + ecliptic_lon_attrs = cdf_file.varattsget("ecliptic_lon") + + assert histogram_flag_info.Data_Type_Description == "CDF_UINT1" + assert histogram_flag_attrs["FILLVAL"] == np.uint8(255) + assert number_of_bins_info.Data_Type_Description == "CDF_UINT2" + assert number_of_bins_attrs["FILLVAL"] == np.uint16(65535) + assert photon_flux_info.Data_Type_Description == "CDF_DOUBLE" + assert np.isclose(photon_flux_attrs["FILLVAL"], np.float64(-1.0e31)) + assert ecliptic_lon_info.Data_Type_Description == "CDF_DOUBLE" + assert np.isclose(ecliptic_lon_attrs["FILLVAL"], np.float64(-1.0e31)) + + def test_bin_exclusions(l1b_hists): # TODO test excluding bins as well diff --git a/imap_processing/tests/hit/test_hit_l2.py b/imap_processing/tests/hit/test_hit_l2.py index 85164726b0..5121415e3d 100644 --- a/imap_processing/tests/hit/test_hit_l2.py +++ b/imap_processing/tests/hit/test_hit_l2.py @@ -1,11 +1,13 @@ from unittest.mock import Mock, patch +import cdflib import numpy as np import pandas as pd import pytest import xarray as xr from imap_processing import imap_module_directory +from imap_processing.cdf.utils import write_cdf from imap_processing.hit.l1a import hit_l1a from imap_processing.hit.l1b.hit_l1b import ( SUMMED_PARTICLE_ENERGY_RANGE_MAPPING, @@ -883,3 +885,10 @@ def test_hit_l2( dependencies[dataset_key], ancillary_dependencies[ancillary_key] ) assert l2_dataset.attrs["Logical_source"] == expected_logical_source + l2_dataset.attrs["Data_version"] = "001" + l2_cdf_filepath = write_cdf(l2_dataset) + cdf_file = cdflib.CDF(l2_cdf_filepath) + dynamic_threshold_info = cdf_file.varinq("dynamic_threshold_state") + dynamic_threshold_attrs = cdf_file.varattsget("dynamic_threshold_state") + assert dynamic_threshold_info.Data_Type_Description == "CDF_UINT1" + assert dynamic_threshold_attrs["FILLVAL"] == np.uint8(255) diff --git a/imap_processing/tests/idex/conftest.py b/imap_processing/tests/idex/conftest.py index add6dd9ad9..c309f3f9d9 100644 --- a/imap_processing/tests/idex/conftest.py +++ b/imap_processing/tests/idex/conftest.py @@ -143,7 +143,7 @@ def get_spice_data_side_effect_func(l1a_ds, idex_attrs): spin_phase_angles = xr.DataArray( name="spin_phase", dims=["epoch"], - data=np.random.randint(0, 360, len(l1a_ds.epoch)), + data=np.random.uniform(0.0, 360.0, len(l1a_ds.epoch)), attrs=idex_attrs.get_variable_attributes("spin_phase"), ) longitude = xr.DataArray( diff --git a/imap_processing/tests/idex/test_idex_l2a.py b/imap_processing/tests/idex/test_idex_l2a.py index 09a7abc2a0..70b6d40ef4 100644 --- a/imap_processing/tests/idex/test_idex_l2a.py +++ b/imap_processing/tests/idex/test_idex_l2a.py @@ -2,6 +2,7 @@ from unittest import mock +import cdflib import numpy as np import pandas as pd import pytest @@ -44,7 +45,7 @@ def l2a_dataset( """ idex_attrs = get_idex_attrs("l1b") spin_phase_angles = xr.DataArray( - np.random.randint(0, 360, len(l1b_dataset.epoch)), + np.random.uniform(0.0, 360.0, len(l1b_dataset.epoch)), dims="epoch", attrs=idex_attrs.get_variable_attributes("spin_phase"), ) @@ -93,6 +94,11 @@ def test_l2a_logical_source_and_cdf(l2a_dataset: xr.Dataset): file_name = write_cdf(l2a_dataset) assert file_name.exists() assert file_name.name == "imap_idex_l2a_sci-1week_20231218_v999.cdf" + cdf_file = cdflib.CDF(file_name) + spin_phase_info = cdf_file.varinq("spin_phase") + spin_phase_attrs = cdf_file.varattsget("spin_phase") + assert spin_phase_info.Data_Type_Description == "CDF_DOUBLE" + assert np.isclose(spin_phase_attrs["FILLVAL"], np.float64(-1.0e31)) expected_vars = [ "tof_snr", diff --git a/imap_processing/tests/mag/test_mag_l2.py b/imap_processing/tests/mag/test_mag_l2.py index 12f26ffd12..7cc18574bd 100644 --- a/imap_processing/tests/mag/test_mag_l2.py +++ b/imap_processing/tests/mag/test_mag_l2.py @@ -1,10 +1,12 @@ from unittest.mock import patch +import cdflib import numpy as np import pytest import xarray as xr from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes +from imap_processing.cdf.utils import write_cdf from imap_processing.mag.constants import FILLVAL, DataMode from imap_processing.mag.l2.mag_l2 import mag_l2, retrieve_matrix_from_l2_calibration from imap_processing.mag.l2.mag_l2_data import MagL2, ValidFrames @@ -140,6 +142,13 @@ def test_mag_l2(norm_dataset, mag_test_l2_data): for i, dataset in enumerate(l2): assert expected_frames[i].var_name in dataset.data_vars assert expected_frames[i].name in dataset.attrs["Data_type"] + dataset.attrs["Data_version"] = "001" + cdf_filepath = write_cdf(dataset) + cdf_file = cdflib.CDF(cdf_filepath) + vector_info = cdf_file.varinq(expected_frames[i].var_name) + vector_attrs = cdf_file.varattsget(expected_frames[i].var_name) + assert vector_info.Data_Type_Description == "CDF_FLOAT" + assert np.isclose(vector_attrs["FILLVAL"], np.float32(-1.0e31)) def test_mag_l2_some_epochs_not_in_spice(norm_dataset, mag_test_l2_data): diff --git a/imap_processing/tests/swapi/test_swapi_l2.py b/imap_processing/tests/swapi/test_swapi_l2.py index 07a10cf380..8189eaa0cd 100644 --- a/imap_processing/tests/swapi/test_swapi_l2.py +++ b/imap_processing/tests/swapi/test_swapi_l2.py @@ -1,6 +1,7 @@ import json from unittest.mock import patch +import cdflib import numpy as np import pandas as pd import pytest @@ -125,6 +126,13 @@ def second_get_file_paths_side_effect(descriptor): ) l2_cdf = write_cdf(l2_dataset) assert l2_cdf.name == "imap_swapi_l2_sci_20240924_v999.cdf" + cdf_file = cdflib.CDF(l2_cdf) + esa_energy_info = cdf_file.varinq("esa_energy") + esa_energy_attrs = cdf_file.varattsget("esa_energy") + assert esa_energy_info.Data_Type_Description == "CDF_DOUBLE" + assert np.isclose(esa_energy_attrs["FILLVAL"], np.float64(-1.0e31)) + assert esa_energy_attrs["VALIDMAX"] == np.float64(21000.0) + assert esa_energy_attrs["VALIDMIN"] == np.float64(0.0) # Test uncertainty variables are as expected np.testing.assert_array_equal( diff --git a/imap_processing/tests/swe/test_swe_l2.py b/imap_processing/tests/swe/test_swe_l2.py index f86c1c380d..983ac316ef 100644 --- a/imap_processing/tests/swe/test_swe_l2.py +++ b/imap_processing/tests/swe/test_swe_l2.py @@ -1,5 +1,6 @@ from unittest.mock import patch +import cdflib import numpy as np import pytest import xarray as xr @@ -378,6 +379,11 @@ def get_file_paths_side_effect(descriptor): l2_dataset.attrs["Data_version"] = "002" l2_cdf_filepath = write_cdf(l2_dataset) assert l2_cdf_filepath.name == "imap_swe_l2_sci_20240510_v002.cdf" + cdf_file = cdflib.CDF(l2_cdf_filepath) + acq_duration_info = cdf_file.varinq("acq_duration") + acq_duration_attrs = cdf_file.varattsget("acq_duration") + assert acq_duration_info.Data_Type_Description == "CDF_UINT4" + assert acq_duration_attrs["FILLVAL"] == np.uint32(4294967295) # --------- sector validation-------- sector_psd_data = l2_dataset["phase_space_density_spin_sector"].data