Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 21 additions & 21 deletions imap_processing/cdf/config/imap_idex_global_cdf_attrs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,45 +11,45 @@ instrument_base: &instrument_base

imap_idex_l1a_sci:
<<: *instrument_base
Data_type: L1A_SCI-1WEEK>Level-1A Science Weekly Data
Logical_source: imap_idex_l1a_sci-1week
Logical_source_description: IMAP Mission IDEX Instrument Level-1A Weekly Data.
Data_type: L1A_SCI-10DAYS>Level-1A Science Data
Logical_source: imap_idex_l1a_sci-10days
Logical_source_description: IMAP Mission IDEX Instrument Level-1A 10-Day Data.

imap_idex_l1a_msg:
imap_idex_l1a_msg-10days:
<<: *instrument_base
Data_type: L1A_MSG>Level-1A Event Message Data
Logical_source: imap_idex_l1a_msg
Data_type: L1A_MSG-10DAYS>Level-1A Event Message Data
Logical_source: imap_idex_l1a_msg-10days
Logical_source_description: IMAP Mission IDEX Instrument Level-1A Event Message Data.

imap_idex_l1a_catlst:
imap_idex_l1a_catlst-10days:
<<: *instrument_base
Data_type: L1A_CATLST>Level-1A Packet Catalog Summary Data
Logical_source: imap_idex_l1a_catlst
Data_type: L1A_CATLST-10DAYS>Level-1A Packet Catalog Summary Data
Logical_source: imap_idex_l1a_catlst-10days
Logical_source_description: IMAP Mission IDEX Instrument Level-1A Packet Catalog Summary Data.

imap_idex_l1b_sci:
<<: *instrument_base
Data_type: L1B_SCI-1WEEK>Level-1B Science Weekly Data
Logical_source: imap_idex_l1b_sci-1week
Logical_source_description: IMAP Mission IDEX Instrument Level-1B Weekly Data.
Data_type: L1B_SCI-10DAYS>Level-1B Science 10-Day Data
Logical_source: imap_idex_l1b_sci-10days
Logical_source_description: IMAP Mission IDEX Instrument Level-1B 10-Day Data.

imap_idex_l1b_msg:
imap_idex_l1b_msg-10days:
<<: *instrument_base
Data_type: L1B_MSG>Level-1B Event Message Data
Logical_source: imap_idex_l1b_msg
Data_type: L1B_MSG-10DAYS>Level-1B Event Message Data
Logical_source: imap_idex_l1b_msg-10days
Logical_source_description: IMAP Mission IDEX Instrument Level-1B Event Message Data.

imap_idex_l1b_catlst:
imap_idex_l1b_catlst-10days:
<<: *instrument_base
Data_type: L1B_CATLST>Level-1B Packet Catalog Summary Data
Logical_source: imap_idex_l1b_catlst
Data_type: L1B_CATLST-10DAYS>Level-1B Packet Catalog Summary Data
Logical_source: imap_idex_l1b_catlst-10days
Logical_source_description: IMAP Mission IDEX Instrument Level-1B Packet Catalog Summary Data.

imap_idex_l2a_sci:
<<: *instrument_base
Data_type: L2A_SCI-1WEEK>Level-2A Science Weekly Data
Logical_source: imap_idex_l2a_sci-1week
Logical_source_description: IMAP Mission IDEX Instrument Level-2A Weekly Data
Data_type: L2A_SCI-10DAYS>Level-2A Science 10-Day Data
Logical_source: imap_idex_l2a_sci-10days
Logical_source_description: IMAP Mission IDEX Instrument Level-2A 10-Day Data

imap_idex_l2b_sci:
<<: *instrument_base
Expand Down
20 changes: 11 additions & 9 deletions imap_processing/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@
from imap_processing.hit.l1a.hit_l1a import hit_l1a
from imap_processing.hit.l1b.hit_l1b import hit_l1b
from imap_processing.hit.l2.hit_l2 import hit_l2
from imap_processing.idex.idex_l1a import PacketParser
from imap_processing.idex.idex_l1a import idex_l1a
from imap_processing.idex.idex_l1b import idex_l1b
from imap_processing.idex.idex_l2a import idex_l2a
from imap_processing.idex.idex_l2b import idex_l2b
Expand Down Expand Up @@ -419,12 +419,12 @@ def upload_products(self, products: list[Path]) -> None:
logger.info(f"Uploading file: {filename}")
imap_data_access.upload(filename)
except IMAPDataAccessError as e:
msg = str(e)
if "FileAlreadyExists" in msg and "409" in msg:
message = str(e)
if "FileAlreadyExists" in message and "409" in message:
logger.warning("Skipping upload of existing file, %s", filename)
continue
else:
logger.error(f"Upload failed with error: {msg}")
logger.error(f"Upload failed with error: {message}")
except Exception as e:
logger.error(f"Upload failed unknown error: {e}")

Expand Down Expand Up @@ -1069,11 +1069,11 @@ def do_processing(
f"Unexpected dependencies found for IDEX L1A:"
f"{dependency_list}. Expected only two dependency."
)
# get l0 file
# get l0 files
science_files = dependencies.get_file_paths(source="idex")
datasets = PacketParser(science_files[0]).data
datasets = idex_l1a(science_files, self.start_date)
elif self.data_level == "l1b":
n_expected_deps = 3 if self.descriptor == "sci-1week" else 1
n_expected_deps = 3 if self.descriptor == "sci-10days" else 1
if len(dependency_list) != n_expected_deps:
raise ValueError(
f"Unexpected dependencies found for IDEX L1B {self.descriptor}:"
Expand Down Expand Up @@ -1117,12 +1117,14 @@ def do_processing(
f"{dependency_list}. Expected three or four dependencies."
)
sci_files = dependencies.get_file_paths(
source="idex", descriptor="sci-1week"
source="idex", descriptor="sci-10days"
)
sci_dependencies = [load_cdf(f) for f in sci_files]
# sort science files by the first epoch value
sci_dependencies.sort(key=lambda ds: ds["epoch"].values[0])
hk_files = dependencies.get_file_paths(source="idex", descriptor="msg")
hk_files = dependencies.get_file_paths(
source="idex", descriptor="msg-10days"
)
# Remove duplicate housekeeping files
hk_dependencies = [load_cdf(dep) for dep in list(set(hk_files))]
# sort housekeeping files by the first epoch value
Expand Down
Loading
Loading