22
33"""Script create_final_cat.py
44
5- Create and update hdf5 file of all final ShapePipe output FITS files, runs of
6- ShapePipe module ``make_catalogue_runner``. Supercedes `merge_final_cat.py`.
5+ Create and update hdf5 file of all per-tile final catalogues, the output of
6+ ShapePipe module ``make_cat_runner`` (hdf5, one dataset per column; FITS for
7+ catalogues written before that format). Supercedes `merge_final_cat.py`.
78
89Usage: in parent dir of patches:
910create_final_cat.py -p ~/shapepipe/workflow/config/cfis/final_cat.param -i . -P 7 -v -m final_cat_P7.hdf5
2122import tqdm
2223import glob
2324import h5py
24- from astropy .io import fits
2525
2626from cs_util import args as cs_args
2727from cs_util import logging
2828
29+ from shapepipe .utilities .final_cat import read_final_cat
30+
2931
3032def params_from_run_config (params , defaults ):
3133 """Fill unset paths from a workflow run config.
@@ -374,9 +376,14 @@ def get_patch_group(hdf5_file, patch, verbose=False):
374376 return patch_group
375377
376378
377- def read_data (fits_file , params ):
379+ def read_data (cat_file , params ):
378380 """Read the parameter list's columns out of one catalogue.
379381
382+ The catalogue is HDF5 or FITS, decided per file
383+ (``shapepipe.utilities.final_cat.read_final_cat``); a FITS table is read
384+ at HDU ``params["hdu_num"]`` (default 1). Returns the requested columns
385+ and a structured dtype describing them.
386+
380387 @sc [label:schema] read-data-raises-on-missing-column
381388 A requested column the catalogue lacks raises `KeyError` naming it; it is
382389 never skipped or filled. `copy_data` keeps only columns present in the
@@ -385,33 +392,17 @@ def read_data(fits_file, params):
385392 silently narrower, with that slot's exposure identity gone. Enforced by
386393 tests/unit/test_final_cat_merge_invariants.py.
387394 """
388- with fits .open (fits_file ) as hdu_list :
389- try :
390- data = hdu_list [params ["hdu_num" ]].data
391- except :
392- print (f"Error with ID { id } { fits_file } )
393- raise
395+ data = read_final_cat (
396+ cat_file , params ["param_list" ], hdu = params .get ("hdu_num" , 1 )
397+ )
394398
395399 # If columns not given on input: read all column names
396400 if params ["param_list" ] is None :
397- params ["param_list" ] = [col for col in data .keys ()]
398-
399- # RAISE, do not print and fall through. The bare `except:` this replaces
400- # left extracted_data and dtype unbound, so the caller's own error was an
401- # UnboundLocalError from the return statement below, naming neither the
402- # file nor the column that was actually missing.
403- present = set (data .dtype .names or ())
404- missing = [col for col in params ["param_list" ] if col not in present ]
405- if missing :
406- raise KeyError (
407- f"{ fits_file } { len (missing )}
408- f"{ len (params ['param_list' ])}
409- f"{ ' ' .join (missing )}
410- )
401+ params ["param_list" ] = list (data .dtype .names )
402+
411403 extracted_data = {col : data [col ] for col in params ["param_list" ]}
412- dtype = data .dtype
413404
414- return extracted_data , dtype
405+ return extracted_data , data . dtype
415406
416407
417408def copy_data (param_list , extracted_data , dtype ):
@@ -496,26 +487,36 @@ def collect_tile_ids_image_sims(patch_path):
496487 return result
497488
498489
490+ # make_cat_runner writes hdf5; catalogues made before that are FITS.
491+ FINAL_CAT_SUFFIXES = (".hdf5" , ".fits" )
492+
493+
499494def find_final_cat (id , id_path , run_prefix ):
500495 """Path of this tile's final catalogue, or None.
501496
502497 Flat layout first: the unified workflow copies one catalogue per tile to
503- <products>/tiles/<shard>/<tile>/final_cat-<tile>.fits, in DOT form and with
504- no run sub-tree. Then the legacy layout, where the catalogue sits under the
505- tile's own shapepipe run dir in DASH form, newest run wins.
498+ <products>/tiles/<shard>/<tile>/final_cat-<tile>.hdf5, in DOT form and
499+ with no run sub-tree. Then the run-dir layout, where the catalogue sits
500+ under the tile's own shapepipe run dir in DASH form, newest run wins.
501+ Either layout may hold an hdf5 or a FITS catalogue; hdf5 is preferred.
506502 """
507- flat = os .path .join (id_path , f"final_cat-{ id } )
508- if os .path .exists (flat ):
509- return flat
503+ for suffix in FINAL_CAT_SUFFIXES :
504+ flat = os .path .join (id_path , f"final_cat-{ id } { suffix } )
505+ if os .path .exists (flat ):
506+ return flat
510507
511508 base_pattern = os .path .join (id_path , "output" , run_prefix )
512509 all_matches = [d for d in glob .glob (base_pattern ) if os .path .isdir (d )]
513510 if not all_matches :
514511 return None
515512 newest_dir = max (all_matches , key = os .path .getmtime )
516513 id_dash = re .sub (r"\." , "-" , id )
517- legacy = f"{ newest_dir } { id_dash }
518- return legacy if os .path .exists (legacy ) else None
514+ for suffix in FINAL_CAT_SUFFIXES :
515+ in_run = (f"{ newest_dir }
516+ f"final_cat-{ id_dash } { suffix } )
517+ if os .path .exists (in_run ):
518+ return in_run
519+ return None
519520
520521
521522def process (params ):
@@ -577,13 +578,13 @@ def process(params):
577578 print (f"Skipping { id } )
578579 continue
579580
580- fits_file = find_final_cat (id , id_path , run_prefix )
581- if fits_file is None :
581+ cat_file = find_final_cat (id , id_path , run_prefix )
582+ if cat_file is None :
582583 if params ["verbose" ]:
583584 print (f"Final cat for { id } )
584585 continue
585586
586- extracted_data , dtype = read_data (fits_file , params )
587+ extracted_data , dtype = read_data (cat_file , params )
587588
588589 structured_data = copy_data (params ["param_list" ], extracted_data , dtype )
589590
0 commit comments