From de79f6f8cb7c3a4303ac9833d9ae71e1639b9b30 Mon Sep 17 00:00:00 2001 From: Luisa Date: Mon, 11 May 2026 13:42:14 -0600 Subject: [PATCH 01/11] idex 10 day window --- imap_processing/cli.py | 6 +- .../idex/idex_10_day_CDF_names.csv | 445 ++++++++++++++++++ imap_processing/idex/idex_constants.py | 7 + imap_processing/idex/idex_l1a.py | 76 ++- imap_processing/idex/idex_utils.py | 41 ++ imap_processing/tests/idex/conftest.py | 9 +- imap_processing/tests/idex/test_idex_l1a.py | 46 +- 7 files changed, 612 insertions(+), 18 deletions(-) create mode 100644 imap_processing/idex/idex_10_day_CDF_names.csv diff --git a/imap_processing/cli.py b/imap_processing/cli.py index 9f992e3391..296cc39ddb 100644 --- a/imap_processing/cli.py +++ b/imap_processing/cli.py @@ -60,7 +60,7 @@ from imap_processing.hit.l1a.hit_l1a import hit_l1a from imap_processing.hit.l1b.hit_l1b import hit_l1b from imap_processing.hit.l2.hit_l2 import hit_l2 -from imap_processing.idex.idex_l1a import PacketParser +from imap_processing.idex.idex_l1a import idex_l1a from imap_processing.idex.idex_l1b import idex_l1b from imap_processing.idex.idex_l2a import idex_l2a from imap_processing.idex.idex_l2b import idex_l2b @@ -1069,9 +1069,9 @@ def do_processing( f"Unexpected dependencies found for IDEX L1A:" f"{dependency_list}. Expected only two dependency." ) - # get l0 file + # get l0 files science_files = dependencies.get_file_paths(source="idex") - datasets = PacketParser(science_files[0]).data + datasets = idex_l1a(science_files, self.start_date) elif self.data_level == "l1b": n_expected_deps = 3 if self.descriptor == "sci-1week" else 1 if len(dependency_list) != n_expected_deps: diff --git a/imap_processing/idex/idex_10_day_CDF_names.csv b/imap_processing/idex/idex_10_day_CDF_names.csv new file mode 100644 index 0000000000..922fc8720b --- /dev/null +++ b/imap_processing/idex/idex_10_day_CDF_names.csv @@ -0,0 +1,445 @@ +start_date,end_date,doy +20250101,20250110,1 +20250110,20250120,10 +20250120,20250130,20 +20250130,20250209,30 +20250209,20250219,40 +20250219,20250301,50 +20250301,20250311,60 +20250311,20250321,70 +20250321,20250331,80 +20250331,20250410,90 +20250410,20250420,100 +20250420,20250430,110 +20250430,20250510,120 +20250510,20250520,130 +20250520,20250530,140 +20250530,20250609,150 +20250609,20250619,160 +20250619,20250629,170 +20250629,20250709,180 +20250709,20250719,190 +20250719,20250729,200 +20250729,20250808,210 +20250808,20250818,220 +20250818,20250828,230 +20250828,20250907,240 +20250907,20250917,250 +20250917,20250927,260 +20250927,20251007,270 +20251007,20251017,280 +20251017,20251027,290 +20251027,20251106,300 +20251106,20251116,310 +20251116,20251126,320 +20251126,20251206,330 +20251206,20251216,340 +20251216,20251226,350 +20251226,20260101,360 +20260101,20260110,1 +20260110,20260120,10 +20260120,20260130,20 +20260130,20260209,30 +20260209,20260219,40 +20260219,20260301,50 +20260301,20260311,60 +20260311,20260321,70 +20260321,20260331,80 +20260331,20260410,90 +20260410,20260420,100 +20260420,20260430,110 +20260430,20260510,120 +20260510,20260520,130 +20260520,20260530,140 +20260530,20260609,150 +20260609,20260619,160 +20260619,20260629,170 +20260629,20260709,180 +20260709,20260719,190 +20260719,20260729,200 +20260729,20260808,210 +20260808,20260818,220 +20260818,20260828,230 +20260828,20260907,240 +20260907,20260917,250 +20260917,20260927,260 +20260927,20261007,270 +20261007,20261017,280 +20261017,20261027,290 +20261027,20261106,300 +20261106,20261116,310 +20261116,20261126,320 +20261126,20261206,330 +20261206,20261216,340 +20261216,20261226,350 +20261226,20270101,360 +20270101,20270110,1 +20270110,20270120,10 +20270120,20270130,20 +20270130,20270209,30 +20270209,20270219,40 +20270219,20270301,50 +20270301,20270311,60 +20270311,20270321,70 +20270321,20270331,80 +20270331,20270410,90 +20270410,20270420,100 +20270420,20270430,110 +20270430,20270510,120 +20270510,20270520,130 +20270520,20270530,140 +20270530,20270609,150 +20270609,20270619,160 +20270619,20270629,170 +20270629,20270709,180 +20270709,20270719,190 +20270719,20270729,200 +20270729,20270808,210 +20270808,20270818,220 +20270818,20270828,230 +20270828,20270907,240 +20270907,20270917,250 +20270917,20270927,260 +20270927,20271007,270 +20271007,20271017,280 +20271017,20271027,290 +20271027,20271106,300 +20271106,20271116,310 +20271116,20271126,320 +20271126,20271206,330 +20271206,20271216,340 +20271216,20271226,350 +20271226,20280101,360 +20280101,20280110,1 +20280110,20280120,10 +20280120,20280130,20 +20280130,20280209,30 +20280209,20280219,40 +20280219,20280229,50 +20280229,20280310,60 +20280310,20280320,70 +20280320,20280330,80 +20280330,20280409,90 +20280409,20280419,100 +20280419,20280429,110 +20280429,20280509,120 +20280509,20280519,130 +20280519,20280529,140 +20280529,20280608,150 +20280608,20280618,160 +20280618,20280628,170 +20280628,20280708,180 +20280708,20280718,190 +20280718,20280728,200 +20280728,20280807,210 +20280807,20280817,220 +20280817,20280827,230 +20280827,20280906,240 +20280906,20280916,250 +20280916,20280926,260 +20280926,20281006,270 +20281006,20281016,280 +20281016,20281026,290 +20281026,20281105,300 +20281105,20281115,310 +20281115,20281125,320 +20281125,20281205,330 +20281205,20281215,340 +20281215,20281225,350 +20281225,20290101,360 +20290101,20290110,1 +20290110,20290120,10 +20290120,20290130,20 +20290130,20290209,30 +20290209,20290219,40 +20290219,20290301,50 +20290301,20290311,60 +20290311,20290321,70 +20290321,20290331,80 +20290331,20290410,90 +20290410,20290420,100 +20290420,20290430,110 +20290430,20290510,120 +20290510,20290520,130 +20290520,20290530,140 +20290530,20290609,150 +20290609,20290619,160 +20290619,20290629,170 +20290629,20290709,180 +20290709,20290719,190 +20290719,20290729,200 +20290729,20290808,210 +20290808,20290818,220 +20290818,20290828,230 +20290828,20290907,240 +20290907,20290917,250 +20290917,20290927,260 +20290927,20291007,270 +20291007,20291017,280 +20291017,20291027,290 +20291027,20291106,300 +20291106,20291116,310 +20291116,20291126,320 +20291126,20291206,330 +20291206,20291216,340 +20291216,20291226,350 +20291226,20300101,360 +20300101,20300110,1 +20300110,20300120,10 +20300120,20300130,20 +20300130,20300209,30 +20300209,20300219,40 +20300219,20300301,50 +20300301,20300311,60 +20300311,20300321,70 +20300321,20300331,80 +20300331,20300410,90 +20300410,20300420,100 +20300420,20300430,110 +20300430,20300510,120 +20300510,20300520,130 +20300520,20300530,140 +20300530,20300609,150 +20300609,20300619,160 +20300619,20300629,170 +20300629,20300709,180 +20300709,20300719,190 +20300719,20300729,200 +20300729,20300808,210 +20300808,20300818,220 +20300818,20300828,230 +20300828,20300907,240 +20300907,20300917,250 +20300917,20300927,260 +20300927,20301007,270 +20301007,20301017,280 +20301017,20301027,290 +20301027,20301106,300 +20301106,20301116,310 +20301116,20301126,320 +20301126,20301206,330 +20301206,20301216,340 +20301216,20301226,350 +20301226,20310101,360 +20310101,20310110,1 +20310110,20310120,10 +20310120,20310130,20 +20310130,20310209,30 +20310209,20310219,40 +20310219,20310301,50 +20310301,20310311,60 +20310311,20310321,70 +20310321,20310331,80 +20310331,20310410,90 +20310410,20310420,100 +20310420,20310430,110 +20310430,20310510,120 +20310510,20310520,130 +20310520,20310530,140 +20310530,20310609,150 +20310609,20310619,160 +20310619,20310629,170 +20310629,20310709,180 +20310709,20310719,190 +20310719,20310729,200 +20310729,20310808,210 +20310808,20310818,220 +20310818,20310828,230 +20310828,20310907,240 +20310907,20310917,250 +20310917,20310927,260 +20310927,20311007,270 +20311007,20311017,280 +20311017,20311027,290 +20311027,20311106,300 +20311106,20311116,310 +20311116,20311126,320 +20311126,20311206,330 +20311206,20311216,340 +20311216,20311226,350 +20311226,20320101,360 +20320101,20320110,1 +20320110,20320120,10 +20320120,20320130,20 +20320130,20320209,30 +20320209,20320219,40 +20320219,20320229,50 +20320229,20320310,60 +20320310,20320320,70 +20320320,20320330,80 +20320330,20320409,90 +20320409,20320419,100 +20320419,20320429,110 +20320429,20320509,120 +20320509,20320519,130 +20320519,20320529,140 +20320529,20320608,150 +20320608,20320618,160 +20320618,20320628,170 +20320628,20320708,180 +20320708,20320718,190 +20320718,20320728,200 +20320728,20320807,210 +20320807,20320817,220 +20320817,20320827,230 +20320827,20320906,240 +20320906,20320916,250 +20320916,20320926,260 +20320926,20321006,270 +20321006,20321016,280 +20321016,20321026,290 +20321026,20321105,300 +20321105,20321115,310 +20321115,20321125,320 +20321125,20321205,330 +20321205,20321215,340 +20321215,20321225,350 +20321225,20330101,360 +20330101,20330110,1 +20330110,20330120,10 +20330120,20330130,20 +20330130,20330209,30 +20330209,20330219,40 +20330219,20330301,50 +20330301,20330311,60 +20330311,20330321,70 +20330321,20330331,80 +20330331,20330410,90 +20330410,20330420,100 +20330420,20330430,110 +20330430,20330510,120 +20330510,20330520,130 +20330520,20330530,140 +20330530,20330609,150 +20330609,20330619,160 +20330619,20330629,170 +20330629,20330709,180 +20330709,20330719,190 +20330719,20330729,200 +20330729,20330808,210 +20330808,20330818,220 +20330818,20330828,230 +20330828,20330907,240 +20330907,20330917,250 +20330917,20330927,260 +20330927,20331007,270 +20331007,20331017,280 +20331017,20331027,290 +20331027,20331106,300 +20331106,20331116,310 +20331116,20331126,320 +20331126,20331206,330 +20331206,20331216,340 +20331216,20331226,350 +20331226,20340101,360 +20340101,20340110,1 +20340110,20340120,10 +20340120,20340130,20 +20340130,20340209,30 +20340209,20340219,40 +20340219,20340301,50 +20340301,20340311,60 +20340311,20340321,70 +20340321,20340331,80 +20340331,20340410,90 +20340410,20340420,100 +20340420,20340430,110 +20340430,20340510,120 +20340510,20340520,130 +20340520,20340530,140 +20340530,20340609,150 +20340609,20340619,160 +20340619,20340629,170 +20340629,20340709,180 +20340709,20340719,190 +20340719,20340729,200 +20340729,20340808,210 +20340808,20340818,220 +20340818,20340828,230 +20340828,20340907,240 +20340907,20340917,250 +20340917,20340927,260 +20340927,20341007,270 +20341007,20341017,280 +20341017,20341027,290 +20341027,20341106,300 +20341106,20341116,310 +20341116,20341126,320 +20341126,20341206,330 +20341206,20341216,340 +20341216,20341226,350 +20341226,20350101,360 +20350101,20350110,1 +20350110,20350120,10 +20350120,20350130,20 +20350130,20350209,30 +20350209,20350219,40 +20350219,20350301,50 +20350301,20350311,60 +20350311,20350321,70 +20350321,20350331,80 +20350331,20350410,90 +20350410,20350420,100 +20350420,20350430,110 +20350430,20350510,120 +20350510,20350520,130 +20350520,20350530,140 +20350530,20350609,150 +20350609,20350619,160 +20350619,20350629,170 +20350629,20350709,180 +20350709,20350719,190 +20350719,20350729,200 +20350729,20350808,210 +20350808,20350818,220 +20350818,20350828,230 +20350828,20350907,240 +20350907,20350917,250 +20350917,20350927,260 +20350927,20351007,270 +20351007,20351017,280 +20351017,20351027,290 +20351027,20351106,300 +20351106,20351116,310 +20351116,20351126,320 +20351126,20351206,330 +20351206,20351216,340 +20351216,20351226,350 +20351226,20360101,360 +20360101,20360110,1 +20360110,20360120,10 +20360120,20360130,20 +20360130,20360209,30 +20360209,20360219,40 +20360219,20360229,50 +20360229,20360310,60 +20360310,20360320,70 +20360320,20360330,80 +20360330,20360409,90 +20360409,20360419,100 +20360419,20360429,110 +20360429,20360509,120 +20360509,20360519,130 +20360519,20360529,140 +20360529,20360608,150 +20360608,20360618,160 +20360618,20360628,170 +20360628,20360708,180 +20360708,20360718,190 +20360718,20360728,200 +20360728,20360807,210 +20360807,20360817,220 +20360817,20360827,230 +20360827,20360906,240 +20360906,20360916,250 +20360916,20360926,260 +20360926,20361006,270 +20361006,20361016,280 +20361016,20361026,290 +20361026,20361105,300 +20361105,20361115,310 +20361115,20361125,320 +20361125,20361205,330 +20361205,20361215,340 +20361215,20361225,350 +20361225,20370101,360 diff --git a/imap_processing/idex/idex_constants.py b/imap_processing/idex/idex_constants.py index 8d50ddcd99..fa4583d8dd 100644 --- a/imap_processing/idex/idex_constants.py +++ b/imap_processing/idex/idex_constants.py @@ -3,6 +3,7 @@ from dataclasses import dataclass from enum import Enum, IntEnum +from imap_processing import imap_module_directory from imap_processing.spice.geometry import SpiceFrame @@ -64,6 +65,12 @@ class IdexConstants: TARGET_NOISE_FREQUENCY = 7000 +# This CSV was provided by the IDEX team. +# It defines the start and stop date of each 10-day window for IDEX l1a processing. +# All IDEX data will be grouped into these 10-day windows from l1a-l2a. +# the last window of each year may be less than 10 days. That is expected. +IDEX_10_DAY_RANGES_PATH = f"{imap_module_directory}/idex/idex_10_day_CDF_names.csv" + class ConversionFactors(float, Enum): """Conversion factor values (DN to picocoulombs) for each of the six waveforms.""" diff --git a/imap_processing/idex/idex_l1a.py b/imap_processing/idex/idex_l1a.py index 2e68afc0be..11e6c5a9b6 100644 --- a/imap_processing/idex/idex_l1a.py +++ b/imap_processing/idex/idex_l1a.py @@ -16,6 +16,8 @@ import json import logging +from collections import defaultdict +from datetime import datetime from enum import IntEnum from pathlib import Path @@ -30,13 +32,57 @@ from imap_processing.idex.evt_msg_decode_utils import render_event_template from imap_processing.idex.idex_constants import IDEXAPID from imap_processing.idex.idex_l0 import decom_packets -from imap_processing.idex.idex_utils import get_idex_attrs -from imap_processing.spice.time import met_to_ttj2000ns +from imap_processing.idex.idex_utils import get_10_day_window_end_date, get_idex_attrs +from imap_processing.spice.time import et_to_ttj2000ns, met_to_ttj2000ns, str_to_et from imap_processing.utils import convert_to_binary_string logger = logging.getLogger(__name__) +def idex_l1a(packet_files: list[Path], start_date: str) -> list[xr.Dataset]: + """ + Process a list of IDEX L0 packet files into a single xarray.Dataset. + + Parameters + ---------- + packet_files : list[Path] + List of paths to IDEX L0 packet files to process. These l0 files should all + contain data that belongs in the same 10-day window specified by start_date. + start_date : str + The start date of the 10-day window in YYYYMMDD format. Used to filter the + data for the 10-day window. + + Returns + ------- + list[xr.Dataset] + A list of xarray Datasets containing the processed IDEX L1a data products. + """ + idex_products = [] + # decom each idex l0 file and gather the data for each product type + # (science, event message, catlst) into separate lists + data_dicts = defaultdict(list) + for packet_file in packet_files: + data = PacketParser(packet_file).data + for product, dataset in data.items(): + data_dicts[product].append(dataset) + # Get the end date of the data window. This will be used to filter events. + end_date = get_10_day_window_end_date(start_date) + # Convert from strings to ttj2000ns for easier epoch comparison + start_epoch_ns = _yyyymmdd_to_ttj2000ns(start_date) + end_epoch_ns = _yyyymmdd_to_ttj2000ns(end_date) + # combine the data for each product type into a single dataset. + # filter each dataset for epochs that are within the 10-day window range. + for datasets in data_dicts.values(): + concat_ds = xr.concat(datasets, dim="epoch").sortby("epoch") + in_window_mask = (concat_ds["epoch"] >= start_epoch_ns) & ( + concat_ds["epoch"] < end_epoch_ns + ) + filtered_ds = concat_ds.where(in_window_mask, drop=True) + idex_products.append(filtered_ds) + + return idex_products + + class Scitype(IntEnum): """Define parameters for IDEX Science Type.""" @@ -76,7 +122,7 @@ def __init__(self, packet_file: str | Path) -> None: ----- Currently assumes one L0 file will generate exactly one L1a file. """ - self.data = [] + self.data = {} self.idex_attrs = get_idex_attrs("l1a") epoch_attrs = self.idex_attrs.get_variable_attributes( "epoch", check_schema=False @@ -88,7 +134,7 @@ def __init__(self, packet_file: str | Path) -> None: if science_packets: logger.info("Processing IDEX L1A Science data.") - self.data.append(self._create_science_dataset(science_packets)) + self.data["l1a_sci-1week"] = self._create_science_dataset(science_packets) datasets_by_level = {"l1a": raw_datset_by_apid, "l1b": derived_datasets_by_apid} for level, dataset in datasets_by_level.items(): # Only produce l1a products for event messages. L1b will be processed in a @@ -98,7 +144,7 @@ def __init__(self, packet_file: str | Path) -> None: data = dataset[IDEXAPID.IDEX_EVT] processed_data = self._create_evt_msg_data(data) processed_data["epoch"].attrs = epoch_attrs - self.data.append(processed_data) + self.data["l1a_msg"] = processed_data if IDEXAPID.IDEX_CATLST in dataset: logger.info(f"Processing IDEX {level} CATLST data") @@ -110,7 +156,7 @@ def __init__(self, packet_file: str | Path) -> None: data["shcoarse"].data, data["shfine"].data ) data["epoch"].attrs = epoch_attrs - self.data.append(data) + self.data[f"{level}_catlst"] = data logger.info("IDEX L1A data processing completed.") @@ -358,6 +404,24 @@ def _read_waveform_bits(waveform_raw: str, high_sample: bool = True) -> list[int return ints +def _yyyymmdd_to_ttj2000ns(date_str: str) -> np.int64: + """ + Convert a YYYYMMDD date to TTJ2000 nanoseconds. + + Parameters + ---------- + date_str : str + The date string in YYYYMMDD format. + + Returns + ------- + int + The corresponding time in TTJ2000 nanoseconds. + """ + date_string = datetime.strptime(date_str, "%Y%m%d").strftime("%Y-%m-%dT00:00:00") + return np.int64(et_to_ttj2000ns(str_to_et(date_string))) + + def calculate_idex_event_time( coarse_time_sec: np.ndarray, fine_time_subs: np.ndarray, diff --git a/imap_processing/idex/idex_utils.py b/imap_processing/idex/idex_utils.py index 2d33029690..924cbfa74f 100644 --- a/imap_processing/idex/idex_utils.py +++ b/imap_processing/idex/idex_utils.py @@ -1,8 +1,10 @@ """Contains helper functions to support IDEX processing.""" +import pandas as pd import xarray as xr from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes +from imap_processing.idex.idex_constants import IDEX_10_DAY_RANGES_PATH def get_idex_attrs(data_level: str) -> ImapCdfAttributes: @@ -68,3 +70,42 @@ def setup_dataset( new_dataset[var] = dataset[var].copy() return new_dataset + + +def get_10_day_window_end_date(start_date: str) -> str: + """ + Use the start date to find the end date of the 10-day window. + + IDEX l1a data is processed in 10-day windows, so this function will be used to + determine the end date of the window to process based on the start + date passed into the job. + + Parameters + ---------- + start_date : str + Start date of the window to process. + + Returns + ------- + end_date : str + End date of the window to process. + """ + # This CSV was provided by the IDEX team. + idex_10_day_ranges = pd.read_csv(IDEX_10_DAY_RANGES_PATH, header=0, dtype=str) + # Find the row where the input start date is equal to the start date in the df. + matching_row = idex_10_day_ranges[idex_10_day_ranges["start_date"] == start_date] + # if there is no match, raise an error. We expect that the start date passed into + # the job will always be the start date of a 10-day window, so there should always + # be a match in the csv. + if matching_row.empty: + raise ValueError( + f"Start date {start_date} is not an IDEX defined start date" + f" for a 10-day window." + ) + if len(matching_row["end_date"]) > 1: + raise ValueError( + f"There should only be one row where start_date is equal " + f"to {start_date}. Please check lookup table: " + f"{IDEX_10_DAY_RANGES_PATH}." + ) + return matching_row["end_date"].values[0] diff --git a/imap_processing/tests/idex/conftest.py b/imap_processing/tests/idex/conftest.py index add6dd9ad9..48166fde57 100644 --- a/imap_processing/tests/idex/conftest.py +++ b/imap_processing/tests/idex/conftest.py @@ -34,11 +34,11 @@ def decom_test_data_sci() -> xr.Dataset: dataset : xarray.Dataset A ``xarray`` dataset containing the science test data """ - return PacketParser(TEST_L0_FILE_SCI).data[0] + return PacketParser(TEST_L0_FILE_SCI).data["l1a_sci-1week"] @pytest.fixture -def decom_test_data_catlst() -> xr.Dataset: +def decom_test_data_catlst() -> list[xr.Dataset]: """List of ``xarray`` datasets containing the raw and derived catalog list data. Returns @@ -46,7 +46,8 @@ def decom_test_data_catlst() -> xr.Dataset: dataset : list[xarray.Dataset] A list of ``xarray`` dataset containing the catalog list summary datasets. """ - return PacketParser(TEST_L0_FILE_CATLST).data + data = PacketParser(TEST_L0_FILE_CATLST).data + return [data["l1a_catlst"], data["l1b_catlst"]] @pytest.fixture @@ -58,7 +59,7 @@ def decom_test_data_msg() -> xr.Dataset: dataset : xarray.Dataset ``xarray`` dataset containing the event log data. """ - return PacketParser(TEST_L0_FILE_MSG).data[0] + return PacketParser(TEST_L0_FILE_MSG).data["l1a_msg"] @pytest.fixture diff --git a/imap_processing/tests/idex/test_idex_l1a.py b/imap_processing/tests/idex/test_idex_l1a.py index 1c96499537..89a109ab24 100644 --- a/imap_processing/tests/idex/test_idex_l1a.py +++ b/imap_processing/tests/idex/test_idex_l1a.py @@ -2,6 +2,7 @@ from pathlib import Path from unittest import mock +from unittest.mock import patch import numpy as np import pandas as pd @@ -12,8 +13,9 @@ from imap_processing import imap_module_directory from imap_processing.cdf.utils import load_cdf, write_cdf from imap_processing.idex.decode import _decode_sub_frame, read_bits, rice_decode -from imap_processing.idex.idex_l1a import PacketParser -from imap_processing.spice.time import met_to_ttj2000ns +from imap_processing.idex.idex_l1a import PacketParser, _yyyymmdd_to_ttj2000ns +from imap_processing.idex.idex_utils import get_10_day_window_end_date +from imap_processing.spice.time import et_to_ttj2000ns, met_to_ttj2000ns, str_to_et from imap_processing.tests.idex.conftest import TEST_L0_FILE_SCI from imap_processing.utils import packet_generator @@ -107,7 +109,7 @@ def test_incomplete_event(caplog): "imap_processing.idex.idex_l1a.decom_packets", return_value=(packets, xr.Dataset(), xr.Dataset()), ): - l1a_dataset = PacketParser(TEST_L0_FILE_SCI).data[0] + l1a_dataset = PacketParser(TEST_L0_FILE_SCI).data["l1a_sci-1week"] # Assert that all the events are present except for one. assert len(l1a_dataset["epoch"]) == 13 assert "Missing packet for event number 1" in caplog.text @@ -198,8 +200,8 @@ def test_compressed_packet(): compressed = Path(f"{TEST_DATA_DIR}/compressed_2023_102_14_24_55.pkts") non_compressed = Path(f"{TEST_DATA_DIR}/non_compressed_2023_102_14_22_26.pkts") - decompressed = PacketParser(compressed).data[0] - expected = PacketParser(non_compressed).data[0] + decompressed = PacketParser(compressed).data["l1a_sci-1week"] + expected = PacketParser(non_compressed).data["l1a_sci-1week"] waveforms = [ "TOF_High", @@ -381,3 +383,37 @@ def test_msg_dataset(decom_test_data_msg: xr.Dataset): messages = example_data.iloc[:, 1].tolist() np.testing.assert_array_equal(decom_test_data_msg["messages"].data, messages) + + +def test_get_window_end_date(): + """Verify that the end date is returned for a 10-day window.""" + assert get_10_day_window_end_date("20260101") == "20260110" + assert get_10_day_window_end_date("20261226") == "20270101" + + with pytest.raises( + ValueError, + match="Start date 20260102 is not an IDEX defined " + "start date for a 10-day window.", + ): + # This invalid start date should raise an error. + get_10_day_window_end_date("20260102") + + +def test_get_window_invalid_lookup(): + """Verify that an invalid lookup table raises an error.""" + with patch( + "imap_processing.idex.idex_utils.IDEX_10_DAY_RANGES_PATH", + "imap_processing/tests/idex/test_data/test_idex_10_day_window.csv", + ): + message = ( + "There should only be one row where start_date is equal to 20250101. " + "Please check lookup table" + ) + with pytest.raises(ValueError, match=message): + get_10_day_window_end_date("20250101") + + +def test_yyyymmdd_to_ttj2000ns(): + """Verify YYYYMMDD dates convert to TTJ2000ns using UTC.""" + expected = np.int64(et_to_ttj2000ns(str_to_et("2026-01-01T00:00:00"))) + assert _yyyymmdd_to_ttj2000ns("20260101") == expected From da4df830b3eb609f52272d057ab3544c70143cd3 Mon Sep 17 00:00:00 2001 From: Luisa Date: Mon, 11 May 2026 13:54:09 -0600 Subject: [PATCH 02/11] comment update --- imap_processing/idex/idex_l1a.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imap_processing/idex/idex_l1a.py b/imap_processing/idex/idex_l1a.py index 11e6c5a9b6..b611d8f199 100644 --- a/imap_processing/idex/idex_l1a.py +++ b/imap_processing/idex/idex_l1a.py @@ -41,7 +41,7 @@ def idex_l1a(packet_files: list[Path], start_date: str) -> list[xr.Dataset]: """ - Process a list of IDEX L0 packet files into a single xarray.Dataset. + Process a list of IDEX L0 packet files into a list of xarray Datasets. Parameters ---------- From d04a1b27a2bf2b6a02efede331cf6a41c3b62328 Mon Sep 17 00:00:00 2001 From: Luisa Date: Mon, 11 May 2026 14:15:53 -0600 Subject: [PATCH 03/11] update descriptors --- .../config/imap_idex_global_cdf_attrs.yaml | 42 +++++++++--------- imap_processing/cli.py | 14 +++--- imap_processing/idex/idex_l1a.py | 14 +++--- imap_processing/idex/idex_l1b.py | 8 ++-- imap_processing/idex/idex_l2b.py | 2 +- imap_processing/tests/idex/conftest.py | 14 +++--- .../imap_idex_l2a_sci-1week_20251017_v001.cdf | Bin 28088 -> 0 bytes imap_processing/tests/idex/test_idex_l1a.py | 14 +++--- imap_processing/tests/idex/test_idex_l1b.py | 8 ++-- imap_processing/tests/idex/test_idex_l2a.py | 6 +-- 10 files changed, 63 insertions(+), 59 deletions(-) delete mode 100644 imap_processing/tests/idex/test_data/imap_idex_l2a_sci-1week_20251017_v001.cdf diff --git a/imap_processing/cdf/config/imap_idex_global_cdf_attrs.yaml b/imap_processing/cdf/config/imap_idex_global_cdf_attrs.yaml index 14cdb25c0f..9b0dcd86ad 100644 --- a/imap_processing/cdf/config/imap_idex_global_cdf_attrs.yaml +++ b/imap_processing/cdf/config/imap_idex_global_cdf_attrs.yaml @@ -11,45 +11,45 @@ instrument_base: &instrument_base imap_idex_l1a_sci: <<: *instrument_base - Data_type: L1A_SCI-1WEEK>Level-1A Science Weekly Data - Logical_source: imap_idex_l1a_sci-1week - Logical_source_description: IMAP Mission IDEX Instrument Level-1A Weekly Data. + Data_type: L1A_SCI-10DAYS>Level-1A Science Data + Logical_source: imap_idex_l1a_sci-10days + Logical_source_description: IMAP Mission IDEX Instrument Level-1A 10-Day Data. -imap_idex_l1a_msg: +imap_idex_l1a_msg-10days: <<: *instrument_base - Data_type: L1A_MSG>Level-1A Event Message Data - Logical_source: imap_idex_l1a_msg + Data_type: L1A_MSG-10DAYS>Level-1A Event Message Data + Logical_source: imap_idex_l1a_msg-10days Logical_source_description: IMAP Mission IDEX Instrument Level-1A Event Message Data. -imap_idex_l1a_catlst: +imap_idex_l1a_catlst-10days: <<: *instrument_base - Data_type: L1A_CATLST>Level-1A Packet Catalog Summary Data - Logical_source: imap_idex_l1a_catlst + Data_type: L1A_CATLST-10DAYS>Level-1A Packet Catalog Summary Data + Logical_source: imap_idex_l1a_catlst-10days Logical_source_description: IMAP Mission IDEX Instrument Level-1A Packet Catalog Summary Data. imap_idex_l1b_sci: <<: *instrument_base - Data_type: L1B_SCI-1WEEK>Level-1B Science Weekly Data - Logical_source: imap_idex_l1b_sci-1week - Logical_source_description: IMAP Mission IDEX Instrument Level-1B Weekly Data. + Data_type: L1B_SCI-10DAYS>Level-1B Science 10-Day Data + Logical_source: imap_idex_l1b_sci-10days + Logical_source_description: IMAP Mission IDEX Instrument Level-1B 10-Day Data. -imap_idex_l1b_msg: +imap_idex_l1b_msg-10days: <<: *instrument_base - Data_type: L1B_MSG>Level-1B Event Message Data - Logical_source: imap_idex_l1b_msg + Data_type: L1B_MSG-10DAYS>Level-1B Event Message Data + Logical_source: imap_idex_l1b_msg-10days Logical_source_description: IMAP Mission IDEX Instrument Level-1B Event Message Data. -imap_idex_l1b_catlst: +imap_idex_l1b_catlst-10days: <<: *instrument_base - Data_type: L1B_CATLST>Level-1B Packet Catalog Summary Data - Logical_source: imap_idex_l1b_catlst + Data_type: L1B_CATLST-10DAYS>Level-1B Packet Catalog Summary Data + Logical_source: imap_idex_l1b_catlst-10days Logical_source_description: IMAP Mission IDEX Instrument Level-1B Packet Catalog Summary Data. imap_idex_l2a_sci: <<: *instrument_base - Data_type: L2A_SCI-1WEEK>Level-2A Science Weekly Data - Logical_source: imap_idex_l2a_sci-1week - Logical_source_description: IMAP Mission IDEX Instrument Level-2A Weekly Data + Data_type: L2A_SCI-10DAYS>Level-2A Science 10-Day Data + Logical_source: imap_idex_l2a_sci-10days + Logical_source_description: IMAP Mission IDEX Instrument Level-2A 10-Day Data imap_idex_l2b_sci: <<: *instrument_base diff --git a/imap_processing/cli.py b/imap_processing/cli.py index 296cc39ddb..5252cf4dc6 100644 --- a/imap_processing/cli.py +++ b/imap_processing/cli.py @@ -419,12 +419,12 @@ def upload_products(self, products: list[Path]) -> None: logger.info(f"Uploading file: {filename}") imap_data_access.upload(filename) except IMAPDataAccessError as e: - msg = str(e) - if "FileAlreadyExists" in msg and "409" in msg: + message = str(e) + if "FileAlreadyExists" in message and "409" in message: logger.warning("Skipping upload of existing file, %s", filename) continue else: - logger.error(f"Upload failed with error: {msg}") + logger.error(f"Upload failed with error: {message}") except Exception as e: logger.error(f"Upload failed unknown error: {e}") @@ -1073,7 +1073,7 @@ def do_processing( science_files = dependencies.get_file_paths(source="idex") datasets = idex_l1a(science_files, self.start_date) elif self.data_level == "l1b": - n_expected_deps = 3 if self.descriptor == "sci-1week" else 1 + n_expected_deps = 3 if self.descriptor == "sci-10days" else 1 if len(dependency_list) != n_expected_deps: raise ValueError( f"Unexpected dependencies found for IDEX L1B {self.descriptor}:" @@ -1117,12 +1117,14 @@ def do_processing( f"{dependency_list}. Expected three or four dependencies." ) sci_files = dependencies.get_file_paths( - source="idex", descriptor="sci-1week" + source="idex", descriptor="sci-10days" ) sci_dependencies = [load_cdf(f) for f in sci_files] # sort science files by the first epoch value sci_dependencies.sort(key=lambda ds: ds["epoch"].values[0]) - hk_files = dependencies.get_file_paths(source="idex", descriptor="msg") + hk_files = dependencies.get_file_paths( + source="idex", descriptor="msg-10days" + ) # Remove duplicate housekeeping files hk_dependencies = [load_cdf(dep) for dep in list(set(hk_files))] # sort housekeeping files by the first epoch value diff --git a/imap_processing/idex/idex_l1a.py b/imap_processing/idex/idex_l1a.py index b611d8f199..d1e75ac2cf 100644 --- a/imap_processing/idex/idex_l1a.py +++ b/imap_processing/idex/idex_l1a.py @@ -134,7 +134,7 @@ def __init__(self, packet_file: str | Path) -> None: if science_packets: logger.info("Processing IDEX L1A Science data.") - self.data["l1a_sci-1week"] = self._create_science_dataset(science_packets) + self.data["l1a_sci-10days"] = self._create_science_dataset(science_packets) datasets_by_level = {"l1a": raw_datset_by_apid, "l1b": derived_datasets_by_apid} for level, dataset in datasets_by_level.items(): # Only produce l1a products for event messages. L1b will be processed in a @@ -144,19 +144,19 @@ def __init__(self, packet_file: str | Path) -> None: data = dataset[IDEXAPID.IDEX_EVT] processed_data = self._create_evt_msg_data(data) processed_data["epoch"].attrs = epoch_attrs - self.data["l1a_msg"] = processed_data + self.data["l1a_msg-10days"] = processed_data if IDEXAPID.IDEX_CATLST in dataset: logger.info(f"Processing IDEX {level} CATLST data") data = dataset[IDEXAPID.IDEX_CATLST] data.attrs = self.idex_attrs.get_global_attributes( - f"imap_idex_{level}_catlst" + f"imap_idex_{level}_catlst-10days" ) data["epoch"] = calculate_idex_event_time( data["shcoarse"].data, data["shfine"].data ) data["epoch"].attrs = epoch_attrs - self.data[f"{level}_catlst"] = data + self.data[f"{level}_catlst-10days"] = data logger.info("IDEX L1A data processing completed.") @@ -196,7 +196,7 @@ def _create_evt_msg_data(self, data: xr.Dataset) -> xr.Dataset: attrs=self.idex_attrs.get_variable_attributes("elssec_evtpkt"), ), }, - attrs=self.idex_attrs.get_global_attributes("imap_idex_l1a_msg"), + attrs=self.idex_attrs.get_global_attributes("imap_idex_l1a_msg-10days"), ) # Load the event decoding dictionaries with open( @@ -268,7 +268,9 @@ def _create_evt_msg_data(self, data: xr.Dataset) -> xr.Dataset: "messages", check_schema=False ), ) - l1a_msg_ds.attrs = self.idex_attrs.get_global_attributes("imap_idex_l1a_msg") + l1a_msg_ds.attrs = self.idex_attrs.get_global_attributes( + "imap_idex_l1a_msg-10days" + ) return l1a_msg_ds def _create_science_dataset(self, science_decom_packet_list: list) -> xr.Dataset: diff --git a/imap_processing/idex/idex_l1b.py b/imap_processing/idex/idex_l1b.py index 2b767dd675..ba0fa4ed56 100644 --- a/imap_processing/idex/idex_l1b.py +++ b/imap_processing/idex/idex_l1b.py @@ -127,7 +127,7 @@ def idex_l1b(l1a_dataset: xr.Dataset, descriptor: str) -> xr.Dataset | None: l1a_dataset : xarray.Dataset IDEX L1a dataset to process. descriptor : str - Descriptor to determine the type of l1b processing to perform. E.g. "sci-1week" + Descriptor to determine the type of l1b processing to perform. E.g. "sci-10days" or "msg". Returns @@ -135,9 +135,9 @@ def idex_l1b(l1a_dataset: xr.Dataset, descriptor: str) -> xr.Dataset | None: l1b_dataset : xarray.Dataset The``xarray`` dataset containing the processed data and supporting metadata. """ - if descriptor.startswith("sci"): + if descriptor.startswith("sci-10days"): return idex_l1b_science(l1a_dataset) - elif descriptor.startswith("msg"): + elif descriptor.startswith("msg-10days"): return idex_l1b_msg(l1a_dataset) else: raise ValueError(f"Unsupported descriptor: {descriptor}") @@ -166,7 +166,7 @@ def idex_l1b_msg(l1a_dataset: xr.Dataset) -> xr.Dataset | None: idex_attrs = get_idex_attrs("l1b") # set up a dataset with only epoch. l1b_dataset = setup_dataset(l1a_dataset, [], idex_attrs, data_vars=None) - l1b_dataset.attrs = idex_attrs.get_global_attributes("imap_idex_l1b_msg") + l1b_dataset.attrs = idex_attrs.get_global_attributes("imap_idex_l1b_msg-10days") # Compute science_on and pulser_on variables based on the event message. The # "science_on" variable indicates when the science data collection is turned on or # off and the "pulser_on" variable indicates when the pulser is turned on or off. diff --git a/imap_processing/idex/idex_l2b.py b/imap_processing/idex/idex_l2b.py index d16aff0660..c470d31897 100644 --- a/imap_processing/idex/idex_l2b.py +++ b/imap_processing/idex/idex_l2b.py @@ -15,7 +15,7 @@ l1a_data, _ = PacketParser(l0_file) _, l1a_msg_data = PacketParser(l0_file_hk) msg_data_l1b = idex_l1b(msg_data_l1a, "msg") - l1b_data = idex_l1b(l1a_data, "sci-1week") + l1b_data = idex_l1b(l1a_data, "sci-10days") l1a_data = idex_l2a(l1b_data) l2b_and_l2c_datasets = idex_l2b(l2a_data, [msg_data_l1b]) diff --git a/imap_processing/tests/idex/conftest.py b/imap_processing/tests/idex/conftest.py index 48166fde57..21640e19a1 100644 --- a/imap_processing/tests/idex/conftest.py +++ b/imap_processing/tests/idex/conftest.py @@ -19,8 +19,8 @@ L1A_EXAMPLE_FILE = TEST_DATA_PATH / "idex_l1a_validation_file.h5" L1B_EXAMPLE_FILE = TEST_DATA_PATH / "imap_idex_l1b_sci_20231218_v004.h5" -L2A_CDF = TEST_DATA_PATH / "imap_idex_l2a_sci-1week_20251017_v001.cdf" -L1B_MSG_CDF = TEST_DATA_PATH / "imap_idex_l1b_msg_20250108_v001.cdf" +L2A_CDF = TEST_DATA_PATH / "imap_idex_l2a_sci-10days_20251017_v001.cdf" +L1B_MSG_CDF = TEST_DATA_PATH / "imap_idex_l1b_msg-10days_20250108_v001.cdf" pytestmark = pytest.mark.external_test_data @@ -34,7 +34,7 @@ def decom_test_data_sci() -> xr.Dataset: dataset : xarray.Dataset A ``xarray`` dataset containing the science test data """ - return PacketParser(TEST_L0_FILE_SCI).data["l1a_sci-1week"] + return PacketParser(TEST_L0_FILE_SCI).data["l1a_sci-10days"] @pytest.fixture @@ -47,7 +47,7 @@ def decom_test_data_catlst() -> list[xr.Dataset]: A list of ``xarray`` dataset containing the catalog list summary datasets. """ data = PacketParser(TEST_L0_FILE_CATLST).data - return [data["l1a_catlst"], data["l1b_catlst"]] + return [data["l1a_catlst-10days"], data["l1b_catlst-10days"]] @pytest.fixture @@ -59,7 +59,7 @@ def decom_test_data_msg() -> xr.Dataset: dataset : xarray.Dataset ``xarray`` dataset containing the event log data. """ - return PacketParser(TEST_L0_FILE_MSG).data["l1a_msg"] + return PacketParser(TEST_L0_FILE_MSG).data["l1a_msg-10days"] @pytest.fixture @@ -71,7 +71,7 @@ def test_l1b_msg(decom_test_data_msg) -> xr.Dataset: dataset : xarray.Dataset ``xarray`` dataset containing the event log data. """ - return idex_l1b(decom_test_data_msg, "msg") + return idex_l1b(decom_test_data_msg, "msg-10days") @pytest.fixture @@ -125,7 +125,7 @@ def l1b_dataset(mock_get_spice_data, decom_test_data_sci: xr.Dataset) -> xr.Data """ mock_get_spice_data.side_effect = get_spice_data_side_effect_func - dataset = idex_l1b(decom_test_data_sci, "sci-1week") + dataset = idex_l1b(decom_test_data_sci, "sci-10days") return dataset diff --git a/imap_processing/tests/idex/test_data/imap_idex_l2a_sci-1week_20251017_v001.cdf b/imap_processing/tests/idex/test_data/imap_idex_l2a_sci-1week_20251017_v001.cdf deleted file mode 100644 index aeeaadac88e4dcac4f333d7a65e6222b69a5c7c7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 28088 zcmd^HeQ;dWb(d`X$#?jMkUSX-5?G5Q6C2xDdTVKg*VdP{YiwiidbRsjda&Bv?Aw(Q zrHupW6bNLd4Q-iWn2@$%I!#;J(oEB4(t4VZq$%kiBn?fI5}FCqhLo0Rni7Tq_xHQ^ zo~OIowJd1)qj_iMy>s4u_uO~y`Q3Z&=k7Ot($?PAcJgGKqV}7>@JwoVaa)ON9j?_z z1`BD$NznSi^6YH6;i5l1Q*#ErQq8NbzoqJV zr75pi+vJR9tMkQdDev_4Y~9-H>`%qlckU0osuR>Jm2$PV-Qjn;k%4X#;uig&7KAzJ zS|1kXe<`G)zoIdXYd@f$+aJ|@_Vecb^|YIj*BY)^`{rls3-8*85O{`uzw%zqSNsFG z!6)SUmAO-&6wlpNDjzF)`Dt$!hHHoup@-i=SywG8?`HE{w8OR4PM(}7dfC8pvJ2*O zYBQdb9F0#nqka(hsOu9|zm)SU*`kvy&3Qr1pU&3G)lCR57hr2NH?B{UhF-SM2VKJE@%SmEfEXc-~#bdF7k5wvE#2PwTMN z1#o_eEuXxJ`CE86;Vowv2*sZ6hSBKf=U)!(4*1QZ(vr~0x z^Ls(A>Q~T6wPr-`hfr?Y{Ir{xzr(hPm_Ip`IM|_Fzkc;|Fl2MZe@}&4}J_L%D5mXg2|8gKZN5CsigOW)#)G+bjCz zpfUqsauTSuY;?{%uEWlBz=;aKxFkBme$mU6va^0^I+H7-yDQbu1wt!YbQHz4(V;Bp zblRp6bb@ldn)5RG(DP*S=&9LJXXZhpqxpPhZftB!>)IHdeTfb`>&xK8hGXv~G1{F@ zPh>`tV|%q?RPQMi+{TD@6O68~jUpJy48@<#R$>(n>NrG;dHMPcI=T(ZIUGuY&N+-5 z;aFohSR+aD$x=|O*6AiAJwQvsIyNk{o4|6uZ8@>qM7CPxDgD_qn2zVY#y0(GCh(&%;*y5JT@4#n}BhnZ59DTO?=Xv0olWV(QKuH9>L+{ zJD$sVMX#D=zrZQbl&90N@jCyfI_&(%z=;#r6qd%TJniSQ#Y}?Iz=7YbjI5v+=m>1s!(TgW$yD9K&_6bq~``a+>K` zZz3sjLTKYby9pi_+ZGA!)^gEV$Fvc-qO8NN;8ci+oNKraw(eHScY3+VZXJk4IMuKb zq1^uz_tsTE)BC0MUh&g7n2=R9mTm8Lf-<$92v_PCV4mDO``#HC$gWMao7AQr+jwF()}+R@@pn2Y-f%>x>-+&Y zF?yfjI@r3)H`9rO^hOb<_CJOJY^-QEgH=1n-ywx|TfBa;XdA=nj#K`&V?&wAJ=VKWZ*i)YP#acu^N zBG90*FN{SkzJ{0Kx*)oRTzR%q#&)o3^hn9e>-Mfv&f~N~)t{>Kt=dQS?L`=sC$7M# z&wE>H%N@RP=2t0oTJqAG;noRIb%-B^PDBt zNRfsH=XU(Yflkz**@pFkA;)cL6VjLe;o>9^uXoW^r-Z zm7aDJ2gOyqki>4X;-g>KLynEVqQf>`15OM$&|k8Xz>+MgYoOJld(WciHU_ktV9;fo zMKG{d<9eq~u^P7}h674k*#@cemvq=Qe+y1wR}xD3Oi(V=jsdSsM9&+f zU>X}F+D$-exee44>%B476N4MB^=*yiy}6$2dp3K$n>TMY%g*gbk%VbuJN^@LH?DQK zKsyX{igQC_Z7ISNiXfU z5Z?9Ikcr0{h7ZQmLy6Sj!s}>C_oLXRPVSt9OuI?7=f-g8-)S?_h}1(ftI&Nv4nmb` z4o|I3;Uwi8WkYK(1z}Wc z;ry%$QMchs`rriVvhBvj8CE7yo8&D7;2x_F(2tx0lBb=v4l)D;Ej*Q$I=k`UU z$wlb}1P)DUHx5rpApm@!ElF4Z(b@<8$s3QP-4B#@JXy9qc zw3~oHTZP_;wj>RC)A%h{fLDBL4UBGL{T3nwALIj7B(SY*D;-z}0Xl-@&}fpIr%@&s zg%=3^9Ww2-c;Vz6xwf)-&AKgf6yCKqKl7`U7mJGf4d!P`%} zhzGJU*w4;Gh!}M6p%H`ae}|y3L%uaW8TH8lO~gY*f3l#0R6w#I>?u3}2 z+riiPum8<~6aQ6nzdWA${=D zkGr4THsEgnzxPL1z46N%Cftr&H+3C-A?Y5w;_b8luydnp(ta4+_HChC)VXNBBmTPP zP#vPv+_C;t?@tx3_1ceuv)+K>zlIy@sIghx5(RTEkO?7a@*zo^qIj+qbmJV*i7iB4lUv4=pwU(0Fjv$iB{_d|q zmSqqsUxTYD`=?n3X{;dD41RD9smTnUsV;8@-{vB!$>1O-hBEtP60Fxv85H~C9m&Nd z`foy(i4*lgUC3d*Z=5*U)*ae`Xfo)hZcFQ0gMRztLw7?#2CP(GPUfw#*3e%-N5)#z z51k6+F!K##t%lFPfmkw5pgOORtTrBoL2z*y+a*}D99;AtOY4Vyob8_z^U&+SMgL6w zW%=h1@7(_6ebo&oKKt@p>mC`{cH%o=YBl06a4!xJ@PGQP2$8ipZdYC z7veo9hAUIW~SGec1NI_g7Q+Q{taL>z75m!j{mZe9yO)egc_4aOr?BHTkU0T1TE8Ocr zuLGAK=Aa+cb)zLL&H@!csj+0qu>ivjdiZb>jWEr+n znI5)DSr1#~C#U@)r13-T?K~wIhJ-tMgLNG0F4g)GQ1lDFSKU6Oi-qJ#gSbo<#vRDw z)R~24I`s)Fi5z5c>P&(qYAS_g>!d!UIA~XD7dn!qFF+O-&-B8yATPtkkH3j%;^L{_ z5|ZJ*o*1{xNLFDcZo;#;_;*{F?MTBEX5taZ;_#VXgV&YXS{cL}l8Z|;zXn-+JvCdJ z0pXyRzNERWzmNXN!9`zmI(_;6&4_Ehzi&6>4qWv0)L)jbf9Ufk{`scl*9YcaAG-7( zM>n{SAO7k`55I4<>)+NtG`;ye_u1EacE8=#=l=eco%daJ9R4C-e-G?p>R{@-kj2-t{_Ud8a48bI zCXBiA`h1W&m#mHiSw1K5r%O1?Jf(g)0oQL1O6!ibgP*7H)l`H9%d;cGzun=i zdFO)FyP!jIafxZZ3R#>x(+$&vybR}_{y3tEbEkewTx{pQACb(*Z+9V!bEh)pFauJZ z7V@wEOGuSUN(u$i4qTGxFG3a%&qN!1tt6VAl`|Z!rTS-(#l@e}7>Hw&iwF7i>yRB> ztZ%2$f2oCwPd^L24qSBc)Ne>bvc7ob`TviL-}f;o!*V%n0BYwBExS+9p%iYkkD-IC zz#zp}XnCRH;+Zq^Rq{Q^;^Hxo%-|;vIE;uNVPuL~{6SOb$CZA{t3X(UtbJHSnq zzU>*v;_=zGH{xn)q9e+B>en0`qw7wTLv<&#a3D^d_?g5HbH`Ae8WX1@YD(O+|6!zt zv9`v`(O*P#k~O}&ofCNRM@*7cqAAHZy(!7XWioTeR>&Q=Sh=Xl3fH8G+LB#pfcDng zt&bzV5Wl*C53G0+h^bK$GSgJXH|JM-7v4uwnnbA$2CDqDn_$4o(S(7nUihCEy8-(p z{GLeZH*ARB@e4Xk-HFOolp0Yv@O=b@sBlAcY&@M{J&iE^D#~u7Lc0knhafkh61L3% zhrV+~{8~Z%E{?xC)`PoE{3%&b!zovU6#@@`$D5tSzxZ(%aFbK@124l$3@bu2-k@otjP0nxz=kx3U0c& z>H8sf;4&B#IJ5vMzEaB#-5l{y2fX-A$nEx6SmH;Gn@W1zl=~iVW6zl%LzaGxo7cN= zHPz{~s*k1m>=b{eS#V0(QaQlSB1?JvhLeB(=IGxv9P{y02mB1Y;>cUm1oB9gWc8^2 zPCQ?Vi>--YZrP)@-WxS1#M2&r0b0#+vP)uurda-Ckfldu;_S$pie*;~7ZF7H70B#@ zJ8)4sRD>MXdCH*U>fYA24czG8c^zcwdqqD?L)wC46s=sVkSlQ)lbkl!R*!!tZ7kz` zewALn>8qsxp(GcVf=_>z_8jO!5@N)q3GODZ}1vfY- z6VPsggVbI1Lh3F5ZZ=z0K@A8T{iqI8cOt;{ya<7#aRh}3jE-Y`F+RpT;4 z!xL>e8w-z}p%Ag?7;FaCtjnjQw@_&|v3O)4Wg3(a@DKh1&M z!sE?aONZ!>x6gpD)v~%<^6VxZt~e{5B1shw%(MDc9K-m0x>XgGz|Z%MHc=~cre+;C z#a+BYCG$pN-*Sb@ELk=zfMQi7T%l?^3{BhB$b66~a;3#&V8PY2o)5}ThR3n$YKEq5 zi!&>;!P8~7aeXyuW!7DkSxymI)xxR|S&>z}_AINlcQMO@)&kLF zoM|uWLW-Y35DlVQHGk%pnMhA&qP@DDf<0^;Xg9%u)wrnwwe*4wn?tOdgi(>`i-(n! Zxkc6C-E Date: Mon, 11 May 2026 14:28:49 -0600 Subject: [PATCH 04/11] failing tests --- imap_processing/idex/idex_l1a.py | 4 ++-- .../tests/idex/test_data/test_idex_10_day_window.csv | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 imap_processing/tests/idex/test_data/test_idex_10_day_window.csv diff --git a/imap_processing/idex/idex_l1a.py b/imap_processing/idex/idex_l1a.py index d1e75ac2cf..26210977ed 100644 --- a/imap_processing/idex/idex_l1a.py +++ b/imap_processing/idex/idex_l1a.py @@ -45,7 +45,7 @@ def idex_l1a(packet_files: list[Path], start_date: str) -> list[xr.Dataset]: Parameters ---------- - packet_files : list[Path] + packet_files : list[pathlib.Path] List of paths to IDEX L0 packet files to process. These l0 files should all contain data that belongs in the same 10-day window specified by start_date. start_date : str @@ -54,7 +54,7 @@ def idex_l1a(packet_files: list[Path], start_date: str) -> list[xr.Dataset]: Returns ------- - list[xr.Dataset] + list[xarray.Dataset] A list of xarray Datasets containing the processed IDEX L1a data products. """ idex_products = [] diff --git a/imap_processing/tests/idex/test_data/test_idex_10_day_window.csv b/imap_processing/tests/idex/test_data/test_idex_10_day_window.csv new file mode 100644 index 0000000000..285d531035 --- /dev/null +++ b/imap_processing/tests/idex/test_data/test_idex_10_day_window.csv @@ -0,0 +1,8 @@ +start_date,end_date,doy +20250101,20250110,1 +20250101,20250111,1 +20250110,20250120,10 +20250120,20250130,20 +20250130,20250209,30 +20250209,20250219,40 +20250219,20250301,50 \ No newline at end of file From fdde77a4e27bb13d1c9217a2ce7e559cddf0952c Mon Sep 17 00:00:00 2001 From: Luisa Date: Tue, 12 May 2026 09:32:22 -0600 Subject: [PATCH 05/11] pr feedback --- imap_processing/idex/idex_l1a.py | 55 ++++++++++----------- imap_processing/idex/idex_l1b.py | 2 +- imap_processing/idex/idex_l2b.py | 2 +- imap_processing/spice/time.py | 20 ++++++++ imap_processing/tests/idex/conftest.py | 2 +- imap_processing/tests/idex/test_idex_l1a.py | 48 +++++++++++++++--- imap_processing/tests/spice/test_time.py | 15 ++++++ 7 files changed, 103 insertions(+), 41 deletions(-) diff --git a/imap_processing/idex/idex_l1a.py b/imap_processing/idex/idex_l1a.py index 26210977ed..09e1261f99 100644 --- a/imap_processing/idex/idex_l1a.py +++ b/imap_processing/idex/idex_l1a.py @@ -17,8 +17,8 @@ import json import logging from collections import defaultdict -from datetime import datetime from enum import IntEnum +from os import path from pathlib import Path import numpy as np @@ -33,13 +33,18 @@ from imap_processing.idex.idex_constants import IDEXAPID from imap_processing.idex.idex_l0 import decom_packets from imap_processing.idex.idex_utils import get_10_day_window_end_date, get_idex_attrs -from imap_processing.spice.time import et_to_ttj2000ns, met_to_ttj2000ns, str_to_et +from imap_processing.spice.time import ( + met_to_ttj2000ns, + str_yyyymmdd_to_ttj2000ns, +) from imap_processing.utils import convert_to_binary_string logger = logging.getLogger(__name__) -def idex_l1a(packet_files: list[Path], start_date: str) -> list[xr.Dataset]: +def idex_l1a( + packet_files: list[Path], window_start_date: str +) -> list[xr.Dataset | None]: """ Process a list of IDEX L0 packet files into a list of xarray Datasets. @@ -48,14 +53,15 @@ def idex_l1a(packet_files: list[Path], start_date: str) -> list[xr.Dataset]: packet_files : list[pathlib.Path] List of paths to IDEX L0 packet files to process. These l0 files should all contain data that belongs in the same 10-day window specified by start_date. - start_date : str + window_start_date : str The start date of the 10-day window in YYYYMMDD format. Used to filter the data for the 10-day window. Returns ------- - list[xarray.Dataset] - A list of xarray Datasets containing the processed IDEX L1a data products. + list[xarray.Dataset|None] + A list of xarray Datasets containing the processed IDEX L1a data products. If + There is no Data found for the 10-day window, None is returned. """ idex_products = [] # decom each idex l0 file and gather the data for each product type @@ -66,18 +72,25 @@ def idex_l1a(packet_files: list[Path], start_date: str) -> list[xr.Dataset]: for product, dataset in data.items(): data_dicts[product].append(dataset) # Get the end date of the data window. This will be used to filter events. - end_date = get_10_day_window_end_date(start_date) + window_end_date = get_10_day_window_end_date(window_start_date) # Convert from strings to ttj2000ns for easier epoch comparison - start_epoch_ns = _yyyymmdd_to_ttj2000ns(start_date) - end_epoch_ns = _yyyymmdd_to_ttj2000ns(end_date) + window_start_date_ns = str_yyyymmdd_to_ttj2000ns(window_start_date) + window_end_date_ns = str_yyyymmdd_to_ttj2000ns(window_end_date) # combine the data for each product type into a single dataset. # filter each dataset for epochs that are within the 10-day window range. - for datasets in data_dicts.values(): + for product, datasets in data_dicts.items(): concat_ds = xr.concat(datasets, dim="epoch").sortby("epoch") - in_window_mask = (concat_ds["epoch"] >= start_epoch_ns) & ( - concat_ds["epoch"] < end_epoch_ns + in_window_mask = (concat_ds["epoch"] >= window_start_date_ns) & ( + concat_ds["epoch"] < window_end_date_ns ) filtered_ds = concat_ds.where(in_window_mask, drop=True) + if len(filtered_ds.epoch) == 0: + logger.warning( + f"No data found for dates {window_start_date_ns} - {window_end_date_ns}" + f" for {product} in packet files: " + f"{[path.basename(f) for f in packet_files]}" + ) + continue idex_products.append(filtered_ds) return idex_products @@ -406,24 +419,6 @@ def _read_waveform_bits(waveform_raw: str, high_sample: bool = True) -> list[int return ints -def _yyyymmdd_to_ttj2000ns(date_str: str) -> np.int64: - """ - Convert a YYYYMMDD date to TTJ2000 nanoseconds. - - Parameters - ---------- - date_str : str - The date string in YYYYMMDD format. - - Returns - ------- - int - The corresponding time in TTJ2000 nanoseconds. - """ - date_string = datetime.strptime(date_str, "%Y%m%d").strftime("%Y-%m-%dT00:00:00") - return np.int64(et_to_ttj2000ns(str_to_et(date_string))) - - def calculate_idex_event_time( coarse_time_sec: np.ndarray, fine_time_subs: np.ndarray, diff --git a/imap_processing/idex/idex_l1b.py b/imap_processing/idex/idex_l1b.py index ba0fa4ed56..a2a5caa4d7 100644 --- a/imap_processing/idex/idex_l1b.py +++ b/imap_processing/idex/idex_l1b.py @@ -128,7 +128,7 @@ def idex_l1b(l1a_dataset: xr.Dataset, descriptor: str) -> xr.Dataset | None: IDEX L1a dataset to process. descriptor : str Descriptor to determine the type of l1b processing to perform. E.g. "sci-10days" - or "msg". + or "msg-10days". Returns ------- diff --git a/imap_processing/idex/idex_l2b.py b/imap_processing/idex/idex_l2b.py index c470d31897..f9b8e6ed5b 100644 --- a/imap_processing/idex/idex_l2b.py +++ b/imap_processing/idex/idex_l2b.py @@ -14,7 +14,7 @@ l0_file_hk = "imap_processing/tests/idex/imap_idex_l0_raw_20250108_v001.pkts" l1a_data, _ = PacketParser(l0_file) _, l1a_msg_data = PacketParser(l0_file_hk) - msg_data_l1b = idex_l1b(msg_data_l1a, "msg") + msg_data_l1b = idex_l1b(msg_data_l1a, "msg-10days") l1b_data = idex_l1b(l1a_data, "sci-10days") l1a_data = idex_l2a(l1b_data) diff --git a/imap_processing/spice/time.py b/imap_processing/spice/time.py index 32af85676a..6c499027b2 100644 --- a/imap_processing/spice/time.py +++ b/imap_processing/spice/time.py @@ -445,3 +445,23 @@ def single_et_to_fractional_doy(et: float) -> float: # numpydoc ignore=GL08 vectorized_et_to_frac_doy = _vectorize(single_et_to_fractional_doy, otypes=[float]) return vectorized_et_to_frac_doy(et) + + +def str_yyyymmdd_to_ttj2000ns(date_str: str) -> np.int64: + """ + Convert a YYYYMMDD date string to TTJ2000 nanoseconds. + + Parameters + ---------- + date_str : str + The date string in YYYYMMDD format. + + Returns + ------- + int + The corresponding time in TTJ2000 nanoseconds. + """ + if len(date_str) != 8: + raise ValueError("Date string must be 8 characters long in yyyymmdd format.") + date_string = datetime.strptime(date_str, "%Y%m%d").strftime("%Y-%m-%dT00:00:00") + return np.int64(et_to_ttj2000ns(str_to_et(date_string))) diff --git a/imap_processing/tests/idex/conftest.py b/imap_processing/tests/idex/conftest.py index 21640e19a1..c3cfcca8a0 100644 --- a/imap_processing/tests/idex/conftest.py +++ b/imap_processing/tests/idex/conftest.py @@ -19,7 +19,7 @@ L1A_EXAMPLE_FILE = TEST_DATA_PATH / "idex_l1a_validation_file.h5" L1B_EXAMPLE_FILE = TEST_DATA_PATH / "imap_idex_l1b_sci_20231218_v004.h5" -L2A_CDF = TEST_DATA_PATH / "imap_idex_l2a_sci-10days_20251017_v001.cdf" +L2A_CDF = TEST_DATA_PATH / "imap_idex_l2a_sci-1week_20251017_v001.cdf" L1B_MSG_CDF = TEST_DATA_PATH / "imap_idex_l1b_msg-10days_20250108_v001.cdf" pytestmark = pytest.mark.external_test_data diff --git a/imap_processing/tests/idex/test_idex_l1a.py b/imap_processing/tests/idex/test_idex_l1a.py index bfb58d8848..7fba5036cf 100644 --- a/imap_processing/tests/idex/test_idex_l1a.py +++ b/imap_processing/tests/idex/test_idex_l1a.py @@ -13,9 +13,12 @@ from imap_processing import imap_module_directory from imap_processing.cdf.utils import load_cdf, write_cdf from imap_processing.idex.decode import _decode_sub_frame, read_bits, rice_decode -from imap_processing.idex.idex_l1a import PacketParser, _yyyymmdd_to_ttj2000ns +from imap_processing.idex.idex_l1a import ( + PacketParser, + idex_l1a, +) from imap_processing.idex.idex_utils import get_10_day_window_end_date -from imap_processing.spice.time import et_to_ttj2000ns, met_to_ttj2000ns, str_to_et +from imap_processing.spice.time import met_to_ttj2000ns from imap_processing.tests.idex.conftest import TEST_L0_FILE_SCI from imap_processing.utils import packet_generator @@ -91,6 +94,41 @@ def test_bad_cdf_file_data(decom_test_data_sci: xr.Dataset): del decom_test_data_sci["Bad_data"] +def test_idex_l1a_decom(): + """Verify idex_l1a function returns the correct datasets.""" + with mock.patch( + "imap_processing.idex.idex_l1a.get_10_day_window_end_date" + ) as mock_get_window_end_date: + mock_get_window_end_date.return_value = "20231228" + datasets = idex_l1a([TEST_L0_FILE_SCI, TEST_L0_FILE_SCI], "20231218") + + assert len(datasets) == 2 + # We should have 28 science events + assert len(datasets[0].epoch) == 28 + + +def test_idex_l1a_decom_no_data(caplog): + """Verify idex_l1a function returns None if there is no data for the window.""" + datasets = idex_l1a([TEST_L0_FILE_SCI], "20260101") + # If there is no data in the window we expect an empty list + assert datasets == [] + # We also expect a warning to be logged that no data was found for the window + message = ( + "No data found for dates 820497669184000000 - 821275269184000000 for" + " l1a_msg-10days in packet files: ['imap_idex_l0_raw_20231218_v001.pkts'" + ", 'imap_idex_l0_raw_20231218_v001.pkts']" + ) + assert message in caplog.text + + +def test_idex_l1a_invalid_window_start(): + """Verify that the idex_l1a function raises an error with an invalid start date.""" + with pytest.raises( + ValueError, match="Start date 20231218 is not an IDEX defined start date" + ): + idex_l1a([TEST_L0_FILE_SCI], "20231218") + + def test_incomplete_event(caplog): """Verify that a CDF is still produced if a packet is dropped. @@ -411,9 +449,3 @@ def test_get_window_invalid_lookup(): ) with pytest.raises(ValueError, match=message): get_10_day_window_end_date("20250101") - - -def test_yyyymmdd_to_ttj2000ns(): - """Verify YYYYMMDD dates convert to TTJ2000ns using UTC.""" - expected = np.int64(et_to_ttj2000ns(str_to_et("2026-01-01T00:00:00"))) - assert _yyyymmdd_to_ttj2000ns("20260101") == expected diff --git a/imap_processing/tests/spice/test_time.py b/imap_processing/tests/spice/test_time.py index 2dd62d0bf5..afd675e7d8 100644 --- a/imap_processing/tests/spice/test_time.py +++ b/imap_processing/tests/spice/test_time.py @@ -20,6 +20,7 @@ sct_to_et, sct_to_ttj2000s, str_to_et, + str_yyyymmdd_to_ttj2000ns, ttj2000ns_to_et, ttj2000ns_to_met, ) @@ -403,3 +404,17 @@ def test_array_input(self): doys = epoch_to_fractional_doy(epochs) assert np.all(doys >= 1.0) assert np.all(doys < 367.0) + + +def test_yyyymmdd_to_ttj2000ns(): + """Verify YYYYMMDD dates convert to TTJ2000ns using UTC.""" + expected = np.int64(et_to_ttj2000ns(str_to_et("2026-01-01T00:00:00"))) + assert str_yyyymmdd_to_ttj2000ns("20260101") == expected + + +def test_yyyymmdd_to_ttj2000ns_invalid_date(): + """Verify a value error is raised when a date is invalid.""" + with pytest.raises( + ValueError, match="Date string must be 8 characters long in yyyymmdd format." + ): + str_yyyymmdd_to_ttj2000ns("202601012") From f75f71defffb93380e45e80f19f23f19a7822eab Mon Sep 17 00:00:00 2001 From: Luisa Date: Tue, 12 May 2026 10:04:46 -0600 Subject: [PATCH 06/11] improve logs --- imap_processing/idex/idex_l1a.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/imap_processing/idex/idex_l1a.py b/imap_processing/idex/idex_l1a.py index 09e1261f99..4319c914ac 100644 --- a/imap_processing/idex/idex_l1a.py +++ b/imap_processing/idex/idex_l1a.py @@ -16,6 +16,7 @@ import json import logging +import os from collections import defaultdict from enum import IntEnum from os import path @@ -144,7 +145,8 @@ def __init__(self, packet_file: str | Path) -> None: science_packets, raw_datset_by_apid, derived_datasets_by_apid = decom_packets( packet_file ) - + filename = os.path.basename(packet_file) + logger.info(f"Processing IDEX L1A Packet {filename}") if science_packets: logger.info("Processing IDEX L1A Science data.") self.data["l1a_sci-10days"] = self._create_science_dataset(science_packets) From 598e0df8d59ae23a66a06fdecf91429c63f55102 Mon Sep 17 00:00:00 2001 From: Luisa Date: Tue, 12 May 2026 11:11:32 -0600 Subject: [PATCH 07/11] update lookup table --- .../idex/idex_10_day_CDF_names.csv | 28 +------------------ 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/imap_processing/idex/idex_10_day_CDF_names.csv b/imap_processing/idex/idex_10_day_CDF_names.csv index 922fc8720b..1e13a34e58 100644 --- a/imap_processing/idex/idex_10_day_CDF_names.csv +++ b/imap_processing/idex/idex_10_day_CDF_names.csv @@ -1,31 +1,5 @@ start_date,end_date,doy -20250101,20250110,1 -20250110,20250120,10 -20250120,20250130,20 -20250130,20250209,30 -20250209,20250219,40 -20250219,20250301,50 -20250301,20250311,60 -20250311,20250321,70 -20250321,20250331,80 -20250331,20250410,90 -20250410,20250420,100 -20250420,20250430,110 -20250430,20250510,120 -20250510,20250520,130 -20250520,20250530,140 -20250530,20250609,150 -20250609,20250619,160 -20250619,20250629,170 -20250629,20250709,180 -20250709,20250719,190 -20250719,20250729,200 -20250729,20250808,210 -20250808,20250818,220 -20250818,20250828,230 -20250828,20250907,240 -20250907,20250917,250 -20250917,20250927,260 +20250924,20250927,260 20250927,20251007,270 20251007,20251017,280 20251017,20251027,290 From 9da203dc1bf707e9252bad32a9a13ace0c3c1ae6 Mon Sep 17 00:00:00 2001 From: Luisa Date: Tue, 12 May 2026 13:38:36 -0600 Subject: [PATCH 08/11] fix the concat --- imap_processing/idex/idex_l1a.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/imap_processing/idex/idex_l1a.py b/imap_processing/idex/idex_l1a.py index 4319c914ac..73005a163b 100644 --- a/imap_processing/idex/idex_l1a.py +++ b/imap_processing/idex/idex_l1a.py @@ -80,7 +80,15 @@ def idex_l1a( # combine the data for each product type into a single dataset. # filter each dataset for epochs that are within the 10-day window range. for product, datasets in data_dicts.items(): - concat_ds = xr.concat(datasets, dim="epoch").sortby("epoch") + concat_ds = xr.concat( + datasets, + dim="epoch", + # Keep non-epoch support variables (e.g. label/index vectors) from a + # single dataset instead of broadcasting them across epochs. + data_vars="minimal", + coords="minimal", + compat="override", + ).sortby("epoch") in_window_mask = (concat_ds["epoch"] >= window_start_date_ns) & ( concat_ds["epoch"] < window_end_date_ns ) From f0d0828d9419cbb8594016b1c26798b6cb330840 Mon Sep 17 00:00:00 2001 From: Luisa Date: Tue, 12 May 2026 14:53:41 -0600 Subject: [PATCH 09/11] fix tests --- .../imap_idex_l2a_sci-1week_20251017_v001.cdf | Bin 0 -> 28088 bytes imap_processing/tests/idex/test_idex_l1a.py | 3 +-- imap_processing/tests/test_cli.py | 10 +++++----- 3 files changed, 6 insertions(+), 7 deletions(-) create mode 100644 imap_processing/tests/idex/test_data/imap_idex_l2a_sci-1week_20251017_v001.cdf diff --git a/imap_processing/tests/idex/test_data/imap_idex_l2a_sci-1week_20251017_v001.cdf b/imap_processing/tests/idex/test_data/imap_idex_l2a_sci-1week_20251017_v001.cdf new file mode 100644 index 0000000000000000000000000000000000000000..aeeaadac88e4dcac4f333d7a65e6222b69a5c7c7 GIT binary patch literal 28088 zcmd^HeQ;dWb(d`X$#?jMkUSX-5?G5Q6C2xDdTVKg*VdP{YiwiidbRsjda&Bv?Aw(Q zrHupW6bNLd4Q-iWn2@$%I!#;J(oEB4(t4VZq$%kiBn?fI5}FCqhLo0Rni7Tq_xHQ^ zo~OIowJd1)qj_iMy>s4u_uO~y`Q3Z&=k7Ot($?PAcJgGKqV}7>@JwoVaa)ON9j?_z z1`BD$NznSi^6YH6;i5l1Q*#ErQq8NbzoqJV zr75pi+vJR9tMkQdDev_4Y~9-H>`%qlckU0osuR>Jm2$PV-Qjn;k%4X#;uig&7KAzJ zS|1kXe<`G)zoIdXYd@f$+aJ|@_Vecb^|YIj*BY)^`{rls3-8*85O{`uzw%zqSNsFG z!6)SUmAO-&6wlpNDjzF)`Dt$!hHHoup@-i=SywG8?`HE{w8OR4PM(}7dfC8pvJ2*O zYBQdb9F0#nqka(hsOu9|zm)SU*`kvy&3Qr1pU&3G)lCR57hr2NH?B{UhF-SM2VKJE@%SmEfEXc-~#bdF7k5wvE#2PwTMN z1#o_eEuXxJ`CE86;Vowv2*sZ6hSBKf=U)!(4*1QZ(vr~0x z^Ls(A>Q~T6wPr-`hfr?Y{Ir{xzr(hPm_Ip`IM|_Fzkc;|Fl2MZe@}&4}J_L%D5mXg2|8gKZN5CsigOW)#)G+bjCz zpfUqsauTSuY;?{%uEWlBz=;aKxFkBme$mU6va^0^I+H7-yDQbu1wt!YbQHz4(V;Bp zblRp6bb@ldn)5RG(DP*S=&9LJXXZhpqxpPhZftB!>)IHdeTfb`>&xK8hGXv~G1{F@ zPh>`tV|%q?RPQMi+{TD@6O68~jUpJy48@<#R$>(n>NrG;dHMPcI=T(ZIUGuY&N+-5 z;aFohSR+aD$x=|O*6AiAJwQvsIyNk{o4|6uZ8@>qM7CPxDgD_qn2zVY#y0(GCh(&%;*y5JT@4#n}BhnZ59DTO?=Xv0olWV(QKuH9>L+{ zJD$sVMX#D=zrZQbl&90N@jCyfI_&(%z=;#r6qd%TJniSQ#Y}?Iz=7YbjI5v+=m>1s!(TgW$yD9K&_6bq~``a+>K` zZz3sjLTKYby9pi_+ZGA!)^gEV$Fvc-qO8NN;8ci+oNKraw(eHScY3+VZXJk4IMuKb zq1^uz_tsTE)BC0MUh&g7n2=R9mTm8Lf-<$92v_PCV4mDO``#HC$gWMao7AQr+jwF()}+R@@pn2Y-f%>x>-+&Y zF?yfjI@r3)H`9rO^hOb<_CJOJY^-QEgH=1n-ywx|TfBa;XdA=nj#K`&V?&wAJ=VKWZ*i)YP#acu^N zBG90*FN{SkzJ{0Kx*)oRTzR%q#&)o3^hn9e>-Mfv&f~N~)t{>Kt=dQS?L`=sC$7M# z&wE>H%N@RP=2t0oTJqAG;noRIb%-B^PDBt zNRfsH=XU(Yflkz**@pFkA;)cL6VjLe;o>9^uXoW^r-Z zm7aDJ2gOyqki>4X;-g>KLynEVqQf>`15OM$&|k8Xz>+MgYoOJld(WciHU_ktV9;fo zMKG{d<9eq~u^P7}h674k*#@cemvq=Qe+y1wR}xD3Oi(V=jsdSsM9&+f zU>X}F+D$-exee44>%B476N4MB^=*yiy}6$2dp3K$n>TMY%g*gbk%VbuJN^@LH?DQK zKsyX{igQC_Z7ISNiXfU z5Z?9Ikcr0{h7ZQmLy6Sj!s}>C_oLXRPVSt9OuI?7=f-g8-)S?_h}1(ftI&Nv4nmb` z4o|I3;Uwi8WkYK(1z}Wc z;ry%$QMchs`rriVvhBvj8CE7yo8&D7;2x_F(2tx0lBb=v4l)D;Ej*Q$I=k`UU z$wlb}1P)DUHx5rpApm@!ElF4Z(b@<8$s3QP-4B#@JXy9qc zw3~oHTZP_;wj>RC)A%h{fLDBL4UBGL{T3nwALIj7B(SY*D;-z}0Xl-@&}fpIr%@&s zg%=3^9Ww2-c;Vz6xwf)-&AKgf6yCKqKl7`U7mJGf4d!P`%} zhzGJU*w4;Gh!}M6p%H`ae}|y3L%uaW8TH8lO~gY*f3l#0R6w#I>?u3}2 z+riiPum8<~6aQ6nzdWA${=D zkGr4THsEgnzxPL1z46N%Cftr&H+3C-A?Y5w;_b8luydnp(ta4+_HChC)VXNBBmTPP zP#vPv+_C;t?@tx3_1ceuv)+K>zlIy@sIghx5(RTEkO?7a@*zo^qIj+qbmJV*i7iB4lUv4=pwU(0Fjv$iB{_d|q zmSqqsUxTYD`=?n3X{;dD41RD9smTnUsV;8@-{vB!$>1O-hBEtP60Fxv85H~C9m&Nd z`foy(i4*lgUC3d*Z=5*U)*ae`Xfo)hZcFQ0gMRztLw7?#2CP(GPUfw#*3e%-N5)#z z51k6+F!K##t%lFPfmkw5pgOORtTrBoL2z*y+a*}D99;AtOY4Vyob8_z^U&+SMgL6w zW%=h1@7(_6ebo&oKKt@p>mC`{cH%o=YBl06a4!xJ@PGQP2$8ipZdYC z7veo9hAUIW~SGec1NI_g7Q+Q{taL>z75m!j{mZe9yO)egc_4aOr?BHTkU0T1TE8Ocr zuLGAK=Aa+cb)zLL&H@!csj+0qu>ivjdiZb>jWEr+n znI5)DSr1#~C#U@)r13-T?K~wIhJ-tMgLNG0F4g)GQ1lDFSKU6Oi-qJ#gSbo<#vRDw z)R~24I`s)Fi5z5c>P&(qYAS_g>!d!UIA~XD7dn!qFF+O-&-B8yATPtkkH3j%;^L{_ z5|ZJ*o*1{xNLFDcZo;#;_;*{F?MTBEX5taZ;_#VXgV&YXS{cL}l8Z|;zXn-+JvCdJ z0pXyRzNERWzmNXN!9`zmI(_;6&4_Ehzi&6>4qWv0)L)jbf9Ufk{`scl*9YcaAG-7( zM>n{SAO7k`55I4<>)+NtG`;ye_u1EacE8=#=l=eco%daJ9R4C-e-G?p>R{@-kj2-t{_Ud8a48bI zCXBiA`h1W&m#mHiSw1K5r%O1?Jf(g)0oQL1O6!ibgP*7H)l`H9%d;cGzun=i zdFO)FyP!jIafxZZ3R#>x(+$&vybR}_{y3tEbEkewTx{pQACb(*Z+9V!bEh)pFauJZ z7V@wEOGuSUN(u$i4qTGxFG3a%&qN!1tt6VAl`|Z!rTS-(#l@e}7>Hw&iwF7i>yRB> ztZ%2$f2oCwPd^L24qSBc)Ne>bvc7ob`TviL-}f;o!*V%n0BYwBExS+9p%iYkkD-IC zz#zp}XnCRH;+Zq^Rq{Q^;^Hxo%-|;vIE;uNVPuL~{6SOb$CZA{t3X(UtbJHSnq zzU>*v;_=zGH{xn)q9e+B>en0`qw7wTLv<&#a3D^d_?g5HbH`Ae8WX1@YD(O+|6!zt zv9`v`(O*P#k~O}&ofCNRM@*7cqAAHZy(!7XWioTeR>&Q=Sh=Xl3fH8G+LB#pfcDng zt&bzV5Wl*C53G0+h^bK$GSgJXH|JM-7v4uwnnbA$2CDqDn_$4o(S(7nUihCEy8-(p z{GLeZH*ARB@e4Xk-HFOolp0Yv@O=b@sBlAcY&@M{J&iE^D#~u7Lc0knhafkh61L3% zhrV+~{8~Z%E{?xC)`PoE{3%&b!zovU6#@@`$D5tSzxZ(%aFbK@124l$3@bu2-k@otjP0nxz=kx3U0c& z>H8sf;4&B#IJ5vMzEaB#-5l{y2fX-A$nEx6SmH;Gn@W1zl=~iVW6zl%LzaGxo7cN= zHPz{~s*k1m>=b{eS#V0(QaQlSB1?JvhLeB(=IGxv9P{y02mB1Y;>cUm1oB9gWc8^2 zPCQ?Vi>--YZrP)@-WxS1#M2&r0b0#+vP)uurda-Ckfldu;_S$pie*;~7ZF7H70B#@ zJ8)4sRD>MXdCH*U>fYA24czG8c^zcwdqqD?L)wC46s=sVkSlQ)lbkl!R*!!tZ7kz` zewALn>8qsxp(GcVf=_>z_8jO!5@N)q3GODZ}1vfY- z6VPsggVbI1Lh3F5ZZ=z0K@A8T{iqI8cOt;{ya<7#aRh}3jE-Y`F+RpT;4 z!xL>e8w-z}p%Ag?7;FaCtjnjQw@_&|v3O)4Wg3(a@DKh1&M z!sE?aONZ!>x6gpD)v~%<^6VxZt~e{5B1shw%(MDc9K-m0x>XgGz|Z%MHc=~cre+;C z#a+BYCG$pN-*Sb@ELk=zfMQi7T%l?^3{BhB$b66~a;3#&V8PY2o)5}ThR3n$YKEq5 zi!&>;!P8~7aeXyuW!7DkSxymI)xxR|S&>z}_AINlcQMO@)&kLF zoM|uWLW-Y35DlVQHGk%pnMhA&qP@DDf<0^;Xg9%u)wrnwwe*4wn?tOdgi(>`i-(n! Zxkc6C-E Date: Wed, 13 May 2026 11:48:40 -0600 Subject: [PATCH 10/11] updates to the windows --- .../idex/idex_10_day_CDF_names.csv | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/imap_processing/idex/idex_10_day_CDF_names.csv b/imap_processing/idex/idex_10_day_CDF_names.csv index 1e13a34e58..922fc8720b 100644 --- a/imap_processing/idex/idex_10_day_CDF_names.csv +++ b/imap_processing/idex/idex_10_day_CDF_names.csv @@ -1,5 +1,31 @@ start_date,end_date,doy -20250924,20250927,260 +20250101,20250110,1 +20250110,20250120,10 +20250120,20250130,20 +20250130,20250209,30 +20250209,20250219,40 +20250219,20250301,50 +20250301,20250311,60 +20250311,20250321,70 +20250321,20250331,80 +20250331,20250410,90 +20250410,20250420,100 +20250420,20250430,110 +20250430,20250510,120 +20250510,20250520,130 +20250520,20250530,140 +20250530,20250609,150 +20250609,20250619,160 +20250619,20250629,170 +20250629,20250709,180 +20250709,20250719,190 +20250719,20250729,200 +20250729,20250808,210 +20250808,20250818,220 +20250818,20250828,230 +20250828,20250907,240 +20250907,20250917,250 +20250917,20250927,260 20250927,20251007,270 20251007,20251017,280 20251017,20251027,290 From f4027b074cd98a1854109d02592b4ae8ded702b3 Mon Sep 17 00:00:00 2001 From: Luisa Date: Thu, 14 May 2026 14:50:49 -0600 Subject: [PATCH 11/11] fov theta for u45 --- imap_processing/ultra/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imap_processing/ultra/constants.py b/imap_processing/ultra/constants.py index 5112204267..b1b66d014a 100644 --- a/imap_processing/ultra/constants.py +++ b/imap_processing/ultra/constants.py @@ -184,7 +184,7 @@ class UltraConstants: # Restricted FOV theta/phi acceptance limits (degrees). # Samples outside these bounds are excluded from GF, efficiency, exposure, # and counts maps at L1C (fine energy bin maps only). - RESTRICTED_FOV_THETA_LOW_DEG_45: float = -43.0 + RESTRICTED_FOV_THETA_LOW_DEG_45: float = -46.0 RESTRICTED_FOV_THETA_HIGH_DEG_45: float = 43.0 RESTRICTED_FOV_THETA_LOW_DEG_90: float = -43.0 RESTRICTED_FOV_THETA_HIGH_DEG_90: float = 43.0