From de69f51677469d57648094c2209e71c72c5abe04 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Sun, 1 Jun 2025 14:43:20 +0100 Subject: [PATCH 01/69] refresh --- .gitignore | 4 +- DRUID/__init__.py | 2 - DRUID/main.py | 951 ------------------ DRUID/src/background.py | 167 --- .../src/{homology/__init__.py => homology.py} | 0 DRUID/src/homology/homology.py | 561 ----------- DRUID/src/homology_new.py | 602 ----------- DRUID/src/source.py | 752 -------------- DRUID/src/utils.py | 586 ----------- Examples/Resolved_Galaxies.ipynb | 4 +- environment.yml | 20 + requirements.txt | 2 +- 12 files changed, 25 insertions(+), 3626 deletions(-) rename DRUID/src/{homology/__init__.py => homology.py} (100%) delete mode 100644 DRUID/src/homology/homology.py delete mode 100644 DRUID/src/homology_new.py create mode 100644 environment.yml diff --git a/.gitignore b/.gitignore index 4b7fd4e..803c8f2 100644 --- a/.gitignore +++ b/.gitignore @@ -16,8 +16,8 @@ ignore/ - - +dev_plan.md +*.drawio DRUID.egg-info build backup diff --git a/DRUID/__init__.py b/DRUID/__init__.py index 0e72604..e69de29 100644 --- a/DRUID/__init__.py +++ b/DRUID/__init__.py @@ -1,2 +0,0 @@ -from .main import sf -from .src import * \ No newline at end of file diff --git a/DRUID/main.py b/DRUID/main.py index 38af491..e69de29 100644 --- a/DRUID/main.py +++ b/DRUID/main.py @@ -1,951 +0,0 @@ -""" -File: main.py -Author: Rhys Shaw -Date: 23/12/2023 -Version: 0.0 -Description: Main file for DRUID -""" - -version = "0.0-test" -import setproctitle - -setproctitle.setproctitle("DRUID") - -from .src import utils -from .src import homology_new as homology -from .src import background -from matplotlib import colors -from .src import source -import numpy as np -from skimage import measure -from tqdm import tqdm -import matplotlib.pyplot as plt -from astropy.io import fits -from astropy.wcs import WCS -import astropy -import pandas as pd -from multiprocessing import Pool -import time -import os -from scipy import ndimage -import logging - - -DRUID_MESSAGE = """ - - -############################################# - -_______ _______ _________ ______ -( __ \ ( ____ )|\ /|\__ __/( __ \ -| ( \ )| ( )|| ) ( | ) ( | ( \ ) -| | ) || (____)|| | | | | | | | ) | -| | | || __)| | | | | | | | | | -| | ) || (\ ( | | | | | | | | ) | -| (__/ )| ) \ \__| (___) |___) (___| (__/ ) -(______/ |/ \__/(_______)\_______/(______/ - - -############################################# - -Detector of astRonomical soUrces in optIcal and raDio images - -Version: {} - -For more information see: -https://github.com/RhysAlfShaw/DRUID - """.format( - version -) - - -class sf: - - def __init__( - self, - image: np.ndarray = None, - image_path: str = None, - mode: str = None, - pb_path: str = None, - cutup: bool = False, - cutup_size: int = 500, - cutup_buff: int = None, - output: bool = True, - area_limit: int = 5, - smooth_sigma=1, - nproc: int = 1, - GPU: bool = False, - header: astropy.io.fits.header.Header = None, - Xoff: int = None, - Yoff: int = None, - debug_mode=False, - remove_edge=True, - ) -> None: - """Initialise the DRUID, here general parameters can be set.. - - Args: - image (np.ndarray, optional): _description_. Defaults to None. - image_path (str, optional): _description_. Defaults to None. - mode (str, optional): _description_. Defaults to None. - pb_path (str, optional): _description_. Defaults to None. - cutup (bool, optional): _description_. Defaults to False. - cutup_size (int, optional): _description_. Defaults to 500. - output (bool, optional): _description_. Defaults to True. - area_limit (int, optional): _description_. Defaults to 5. - smooth_sigma (int, optional): _description_. Defaults to 1. - nproc (int, optional): _description_. Defaults to 1. - GPU (bool, optional): _description_. Defaults to False. - header (astropy.io.fits.header.Header, optional): _description_. Defaults to None. - Xoff (int, optional): _description_. Defaults to None. - Yoff (int, optional): _description_. Defaults to None. - - Raises: - ValueError: _description_ - """ - # start up message! - print(DRUID_MESSAGE) - - if debug_mode: - logging.basicConfig(format="%(asctime)s - %(message)s", level=logging.DEBUG) - logging.debug("Debug mode enabled") - else: - logging.basicConfig(format="%(asctime)s - %(message)s", level=logging.INFO) - - self.cutup = cutup - self.output = output - self.image_path = image_path - self.area_limit = area_limit - self.smooth_sigma = smooth_sigma - self.GPU = GPU - self.Xoff = Xoff - self.Yoff = Yoff - self.cutup_buff = cutup_buff - self.remove_edge = remove_edge - - if self.GPU: - - # Lets try importing the GPU stuff, if it fails then we can just use the CPU. - - try: - import cupy as cp - from cupyx.scipy.ndimage import label as cp_label - - # num_gpus = cp.cuda.runtime.getDeviceCount() - # print(f'Found {num_gpus} GPUs') - - # if num_gpus > 0: - # #print('GPUs are avalible, GPU functions will now be avalible.') - # GPU_AVALIBLE = True - # else: - # print('No GPUs avalible, using CPU') - # GPU_AVALIBLE = False - except: - - # print('Could not import cupy. DRUID GPU functions will not be avalible') - GPU_AVALIBLE = False - - self.nproc = nproc - - self.header = header - - if self.image_path is None: - self.image = image - - else: - self.image, self.header = utils.open_image(self.image_path) - - if mode == "Radio": - pass - # self.image = np.pad( - # self.image, ((1, 1), (1, 1)), mode="constant", constant_values=0 - # ) - - if self.smooth_sigma != 0: - self.image_smooth = utils.smoothing(self.image, self.smooth_sigma) - logging.info("Image smoothed with sigma = {}".format(self.smooth_sigma)) - else: - self.image_smooth = self.image - self.mode = mode - - if self.mode not in ["Radio", "optical", "other"]: - raise ValueError("Mode must be either radio, optical or other.") - - self.pb_path = pb_path - - if self.pb_path is not None: - self.pb_image, self.pb_header = utils.open_image(self.pb_path) - - if self.cutup: - self.cutup_size = cutup_size - if self.cutup_buff is not None: - self.cutouts_smooth, self.coords = utils.cut_image_buff( - self.image_smooth, cutup_size, buffer_size=self.cutup_buff - ) - self.cutouts, self.coords = utils.cut_image_buff( - self.image, cutup_size, buffer_size=self.cutup_buff - ) - else: - self.cutouts_smooth, self.coords = utils.cut_image( - cutup_size, self.image_smooth - ) - self.cutouts, self.coords = utils.cut_image(cutup_size, self.image) - - if self.pb_path is not None: - self.pb_cutouts, self.pb_coords = utils.cut_image( - cutup_size, self.pb_image - ) - - else: - self.pb_cutouts = None - self.pb_coords = None - else: - self.cutup_size = None - self.cutouts = None - self.cutouts_smooth = None - self.coords = None - self.pb_cutouts = None - self.pb_coords = None - - def phsf(self, lifetime_limit: float = 0, lifetime_limit_fraction: float = 2): - """Performs the persistent homology source finding algorithm. - - Args: - - lifetime_limit (float): The lifetime limit for the persistent homology algorithm. - - Returns: - - None. The catalogue is stored in the self.catalogue attribute. - - """ - - if self.cutup == True: - - catalogue_list = [] - IDoffset = 0 - for i, cutout in tqdm( - enumerate(self.cutouts_smooth), - total=len(self.cutouts_smooth), - desc="Processing Cutouts", - disable=not self.output, - ): - - print( - "Computing for Cutout number :{}/{}".format( - i + 1, len(self.cutouts_smooth) - ) - ) - - catalogue = homology.compute_ph_components( - cutout, - self.local_bg, - analysis_threshold_val=self.analysis_threshold_val, - lifetime_limit=lifetime_limit, - output=self.output, - bg_map=self.bg_map, - area_limit=self.area_limit, - GPU=self.GPU, - lifetime_limit_fraction=lifetime_limit_fraction, - mean_bg=self.mean_bg, - IDoffset=IDoffset, - box_size=self.box_size, - detection_threshold=self.sigma, - Cutout_X_offset=self.coords[i][1], - Cutout_Y_offset=self.coords[i][0], - ) - if len(catalogue) == 0: - continue - - IDoffset += len(catalogue) - - catalogue["Y0_cutout"] = self.coords[i][0] - catalogue["X0_cutout"] = self.coords[i][1] - catalogue["x1"] = catalogue["x1"] + self.coords[i][0] - catalogue["x2"] = catalogue["x2"] + self.coords[i][0] - catalogue["y1"] = catalogue["y1"] + self.coords[i][1] - catalogue["y2"] = catalogue["y2"] + self.coords[i][1] - catalogue["bbox1"] = catalogue["bbox1"] + self.coords[i][0] - catalogue["bbox2"] = catalogue["bbox2"] + self.coords[i][1] - catalogue["bbox3"] = catalogue["bbox3"] + self.coords[i][0] - catalogue["bbox4"] = catalogue["bbox4"] + self.coords[i][1] - catalogue["distance_from_center"] = ( - (catalogue["x1"] - cutout.shape[0] / 2) ** 2 - + (catalogue["y1"] - cutout.shape[1] / 2) ** 2 - ) ** 0.5 - catalogue["cutup_number"] = i - catalogue_list.append(catalogue) - self.catalogue = pd.concat(catalogue_list) - # print(self.catalogue) - # remove duplicates and keep the one closest to its cutout centre. - # print('before duplicated removal :',len(self.catalogue)) - # self.catalogue = utils.remove_duplicates(self.catalogue) - # drop any with edge_flag == 1 - # set edge flag False to 0 - self.catalogue["edge_flag"] = self.catalogue["edge_flag"].astype(int) - - if self.remove_edge: - self.catalogue = self.catalogue[self.catalogue.edge_flag != 1] - self.catalogue = self.catalogue.sort_values( - by=["distance_from_center"], ascending=True - ) - self.catalogue = self.catalogue.drop_duplicates( - subset=["x1", "y1", "Birth"], keep="first" - ) - else: - - for i, row in catalogue.iterrows(): - if row.edge_flag == 1: - if row.bbox1 == 0: - row.bbox1 = 1 - if row.bbox2 == 0: - row.bbox2 = 1 - if row.bbox3 == self.image.shape[0]: - row.bbox3 = self.image.shape[0] - 1 - if row.bbox4 == self.image.shape[1]: - row.bbox4 = self.image.shape[1] - 1 - - self.catalogue.at[i, "bbox1"] = row.bbox1 - self.catalogue.at[i, "bbox2"] = row.bbox2 - self.catalogue.at[i, "bbox3"] = row.bbox3 - self.catalogue.at[i, "bbox4"] = row.bbox4 - - else: - IDoffset = 0 - catalogue = homology.compute_ph_components( - self.image_smooth, - self.local_bg, - analysis_threshold_val=self.analysis_threshold_val, - lifetime_limit=lifetime_limit, - output=self.output, - bg_map=self.bg_map, - area_limit=self.area_limit, - GPU=self.GPU, - lifetime_limit_fraction=lifetime_limit_fraction, - mean_bg=self.mean_bg, - IDoffset=IDoffset, - box_size=self.cutup_size, - detection_threshold=self.sigma, - ) - - self.catalogue = catalogue - - self.catalogue["Y0_cutout"] = 0 - self.catalogue["X0_cutout"] = 0 - self.catalogue["edge_flag"] = self.catalogue["edge_flag"].astype(int) - - if self.remove_edge: - self.catalogue = self.catalogue[self.catalogue.edge_flag != 1] - else: - for i, row in catalogue.iterrows(): - if row.edge_flag == 1: - if row.bbox1 == 0: - row.bbox1 = 1 - if row.bbox2 == 0: - row.bbox2 = 1 - if row.bbox3 == self.image_smooth.shape[0]: - row.bbox3 = self.image_smooth.shape[0] - 1 - if row.bbox4 == self.image_smooth.shape[1]: - row.bbox4 = self.image_smooth.shape[1] - 1 - - self.catalogue.at[i, "bbox1"] = row.bbox1 - self.catalogue.at[i, "bbox2"] = row.bbox2 - self.catalogue.at[i, "bbox3"] = row.bbox3 - self.catalogue.at[i, "bbox4"] = row.bbox4 - - # print(self.catalogue) - - self.catalogue = self.catalogue.sort_values(by=["lifetime"], ascending=False) - # print('after duplicate removal :',len(self.catalogue)) - - # do enclosed_i evaluation with the bounding box to ensure we dont use the whole image. - # plt.figure(figsize=(20,20)) - # plt.imshow(self.image,cmap='gray_r',norm=colors.LogNorm(clip=True)) - # plt.scatter(self.catalogue.y1,self.catalogue.x1,c='r',marker='x') - # # # # plot the bounding boxes - # for i, row in self.catalogue.iterrows(): - # ymin = row.bbox1 - # ymax = row.bbox3 - # xmin = row.bbox2 - # xmax = row.bbox4 - # plt.plot([xmin,xmax,xmax,xmin,xmin],[ymin,ymin,ymax,ymax,ymin],c='r') - # plt.savefig('test.png') - # time.sleep(3) - enclosed_i_list = [] - t0 = time.time() - for i in tqdm( - range(0, len(self.catalogue)), - total=len(self.catalogue), - desc="Calculating enclosed_i", - disable=not self.output, - ): - row = self.catalogue.iloc[i] - x1 = row.x1 - row.bbox1 + 1 - y1 = row.y1 - row.bbox2 + 1 - Birth = row.Birth - Death = row.Death - # is this a new row? - # if row.new_row == 1: - - img = self.image_smooth[ - int(row.bbox1) - 1 : int(row.bbox3) + 1, - int(row.bbox2) - 1 : int(row.bbox4) + 1, - ] - # reduce the cat to just the sources in the bounding box. - - cat = self.catalogue[self.catalogue["x1"] > int(row.bbox1)] - cat = cat[cat["x1"] < int(row.bbox3)] - cat = cat[cat["y1"] > int(row.bbox2)] - cat = cat[cat["y1"] < int(row.bbox4)] - cat["x1"] = cat["x1"] - int(row.bbox1) + 1 - cat["y1"] = cat["y1"] - int(row.bbox2) + 1 - - # plt.imshow(img,cmap='gray_r',norm=colors.LogNorm(clip=True,vmin=1E-13,vmax=1E-9)) - # plt.scatter(cat.y1,cat.x1,c='r',marker='x') - # plt.savefig('test.png') - # plt.close() - # # sleep - # time.sleep(2) - - if self.GPU == True: - import cupy as cp - - # this is not the best way to deal with this. We should crop the gpu version of the image. - img_gpu = cp.asarray(img, dtype=cp.float64) - enclosed_i = homology.make_point_enclosure_assoc_GPU( - 0, x1, y1, Birth, Death, cat, img_gpu - ) - enclosed_i_list.append(enclosed_i) - else: - # print(self.catalogue) - enclosed_i = homology.make_point_enclosure_assoc_CPU( - 0, x1, y1, Birth, Death, cat, img - ) - enclosed_i_list.append(enclosed_i) - - # print('enclosed_i calculated! t='+str(time.time()-t0)+' s') - self.catalogue["enclosed_i"] = enclosed_i_list - # print(self.catalogue) - # print("Enclosed_i calculated! TESTESTEST") - # correct for first destruction - # t0_parent_tag = time.time() - # self.catalogue = homology.parent_tag_func_vectorized_new(self.catalogue) - # t1_parent_tag = time.time() - # print("Time to assign parent tags: ", t1_parent_tag - t0_parent_tag) - # print(self.catalogue) - # print(len(self.catalogue)) - t0_correct_firs = time.time() - # print('BEfore',len(self.catalogue)) - self.catalogue = homology.correct_first_destruction( - self.catalogue, output=not self.output - ) - t1_correct_firs = time.time() - print("Time to correct first destruction: ", t1_correct_firs - t0_correct_firs) - # print('Time to correct first destruction: ',t1_correct_firs-t0_correct_firs) - # print('After',len(self.catalogue)) - # parent tag - # print("Assigning parent tags..") - t0_parent_tag = time.time() - self.catalogue = homology.parent_tag_func_optimized(self.catalogue) - t1_parent_tag = time.time() - - print("Time to assign parent tags: ", t1_parent_tag - t0_parent_tag) - - t0_classify = time.time() - self.catalogue["Class"] = self.catalogue.apply(homology.classify_single, axis=1) - t1_classify = time.time() - print("Time to classify sources: ", t1_classify - t0_classify) - - def set_background( - self, - detection_threshold, - analysis_threshold, - set_bg=None, - bg_map_bool=False, - box_size=None, - mode="mad_std", - smooth=True, - ): - - self.sigma = detection_threshold - self.analysis_threshold = analysis_threshold - self.bg_map = bg_map_bool - self.box_size = box_size - self.bgmode = mode - self.set_bg = set_bg - self.bg_map_bool = bg_map_bool - # mode should be MAD_Std, RMS or other. - - if smooth == True: - img = self.image_smooth - else: - img = self.image - - if mode == "Radio": - # old verion was called radio. - bgmode = "mad_std" - - # need to account dor the cutputs if not usinh bg_map. - - # bg_map and cutup are require only the same code. - - # bg_map and no cuput is the same as bg_map and cutup. - - # no cutup and no bg_map is just one estimation for the whole image. - - if self.cutup == True: - - # we want to do the bg_map but for the cutout dims. as the box size is in pixels. - self.bg_map = True - bg_map_bool = True - # which is smaller the cutout size or the box size? - if box_size is None: - box_size = self.cutup_size - else: - if box_size > self.cutup_size: - box_size = self.cutup_size - else: - pass - - self.box_size = box_size - - if bg_map_bool == True: - # print('Creating a background map. Inputed Box size = ',box_size) - # these will be returned as arrays like a map. - std, mean_bg = background.calculate_background_map( - img, box_size, mode=self.bgmode - ) - # print('Background map created.') - # print('Mean Background across cutouts: ', np.nanmean(std)) - # print('Median of bg distribution: ', np.nanmean(mean_bg)) - - else: - # print('Not creating a background map.') - std, mean_bg = background.calculate_background(img, mode=self.bgmode) - # print('Background set to: ',std) - # print('Background mean set to: ',mean_bg) - - if set_bg is not None: - # set bg should be a tuple of (std,mean_bg) - # print('User has set the background.') - std = set_bg[0] - mean_bg = set_bg[1] - - self.local_bg = std * self.sigma - self.analysis_threshold_val = std * self.analysis_threshold - self.mean_bg = mean_bg - - def set_background_old( - self, - detection_threshold: float, - analysis_threshold, - set_bg: float = None, - bg_map: bool = None, - box_size: int = 10, - mode: str = "Radio", - ): - """Sets the background for the source finding algorithm. - - Args: - detection_threshold (int): _description_ - analysis_threshold (int): _description_ - set_bg (float, optional): _description_. Defaults to None. - bg_map (bool, optional): _description_. Defaults to None. - box_size (int, optional): _description_. Defaults to 10. - mode (str, optional): _description_. Defaults to 'Radio'. - """ - - self.bg_map = bg_map - self.sigma = detection_threshold - self.analysis_threshold = analysis_threshold - if mode == "Radio": - if self.cutup: - - # loop though each cutout and calculate the local background. - - if bg_map is not None: - - # users wants to use background map so lets make it - local_bg_list = [] - analysis_threshold_list = [] - mean_bg_list = [] - for i, cutout in enumerate(self.cutouts): - local_bg_map, mean_bg = background.radio_background_map( - cutout, box_size - ) - analysis_threshold_list.append( - local_bg_map * self.analysis_threshold - ) - local_bg_list.append(local_bg_map * self.sigma) - mean_bg_list.append(mean_bg) - else: - local_bg_list = [] - analysis_threshold_list = [] - mean_bg_list = [] - for cutout in self.cutouts: - local_bg, mean_bg = background.radio_background(cutout) - analysis_threshold_list.append( - local_bg * self.analysis_threshold - ) - local_bg_list.append(local_bg * self.sigma) - mean_bg_list.append(mean_bg) - local_bg = local_bg_list - analysis_threshold = analysis_threshold_list - mean_bg = mean_bg_list - - else: - - # Radio background is calculated using the median absolute deviation of the total image. - if bg_map is not None: - local_bg_o, mean_bg = background.radio_background_map( - self.image, box_size - ) - local_bg = local_bg_o * self.sigma - analysis_threshold = local_bg_o * self.analysis_threshold - else: - local_bg_o, mean_bg = background.radio_background(self.image) - local_bg = local_bg_o * self.sigma - analysis_threshold = local_bg_o * self.analysis_threshold - - if mode == "Optical": - # Optical background is calculated using a random sample of pixels - mean_bg, std_bg = background.optical_background(nsamples=1000) - local_bg = mean_bg + self.sigma * std_bg - analysis_threshold = mean_bg + std_bg * self.analysis_threshold - - if mode == "other": - # If the user has a custom background function, they can pass it in here. - local_bg = set_bg * self.sigma - analysis_threshold = local_bg * self.analysis_threshold - # print('Background set to: ',local_bg) - # print('Analysis threshold set to: ',analysis_threshold) - - self.analysis_threshold_val = analysis_threshold - self.local_bg = local_bg - self.mean_bg = mean_bg - # print(self.mean_bg) - - # if bg_map: - # print('Using bg_map for analysis.') - # else: - # if self.cutup: - - # print('Mean Background across cutouts: ', np.nanmean(self.local_bg)) - # print('Median of bg distribution: ', np.nanmean(self.mean_bg)) - - # else: - # print('Background set to: ',self.local_bg) - - def source_characterising(self, use_gpu: bool = False): - """Source Characterising function. This function takes the catalogue and the image and calculates the source properties. - - Args: - use_gpu (bool, optional): Option to use the GPU True to use and False to not, - requires cupy module and a avalible GPU. Defaults to False. - """ - self.set_background( - detection_threshold=self.sigma, - analysis_threshold=self.analysis_threshold, - set_bg=self.set_bg, - bg_map_bool=self.bg_map_bool, - box_size=self.box_size, - mode=self.bgmode, - smooth=False, - ) - - self.catalogue, self.polygons = source.measure_source_properties( - use_gpu=use_gpu, - catalogue=self.catalogue, - cutout=self.image, - smooth_cutout=self.image_smooth, - background_map=self.local_bg, - output=self.output, - cutupts=self.cutouts, - mode=self.mode, - header=self.header, - sigma=self.sigma, - ) - - if self.Xoff is not None: - # correct for the poistion of the cutout. when using cutout from a larger image. - self.catalogue["Xc"] = self.catalogue["Xc"] + self.Xoff - self.catalogue["bbox1"] = self.catalogue["bbox1"] + self.Xoff - self.catalogue["bbox3"] = self.catalogue["bbox3"] + self.Xoff - - if self.Yoff is not None: - # correct for the poistion of the cutout. when using cutout from a larger image. - self.catalogue["Yc"] = self.catalogue["Yc"] + self.Yoff - self.catalogue["bbox2"] = self.catalogue["bbox2"] + self.Yoff - self.catalogue["bbox4"] = self.catalogue["bbox4"] + self.Yoff - - if self.header is not None: - # try: - # print('Converting Xc and Yc to RA and DEC') - print(self.catalogue["Xc"], self.catalogue["Yc"]) - Ra, Dec = utils.xy_to_RaDec( - self.catalogue["Xc"], self.catalogue["Yc"], self.header, mode=self.mode - ) - self.catalogue["RA"] = Ra - self.catalogue["DEC"] = Dec - self.catalogue["RA"] = self.catalogue["RA"].astype(float) - self.catalogue["DEC"] = self.catalogue["DEC"].astype(float) - # except: - # pass - - self._set_types_of_dataframe() - - if self.mode == "optical": - - def ABmag(flux): - return -2.5 * np.log10(flux) - - def RONoise(EFFRON, EFFGAIN, EXPTIME, Area): - return np.sqrt(Area) * (EFFRON / EFFGAIN) * EXPTIME - - def SkyNoise(sky): - return np.sqrt(sky) - - def SourceNoise(Flux): - return np.sqrt(Flux) - - def Flux_err(EFFRON, EFFGAIN, EXPTIME, Area, sky, Flux): - return np.sqrt( - RONoise(EFFRON, EFFGAIN, EXPTIME, Area) ** 2 - + SkyNoise(sky) - + SourceNoise(Flux) - ) - - def NOISE(row, local_ng): - return np.sum( - np.random.normal(row["mean_bg"], local_ng, int(row["Area"])) - ) - - EFFGAIN = utils.get_EFFGAIN(self.header) - EXPTIME = utils.get_EXPTIME(self.header) - EFFRON = utils.get_EFFRON(self.header) - # print('EFFGAIN: ',EFFGAIN) - # print('EXPTIME: ',EXPTIME) - # print('EFFRON: ',EFFRON) - # print(self.catalogue['Noise']) - self.catalogue["Flux_total_new"] = ( - self.catalogue["Flux_total"] * EFFGAIN * EXPTIME - - self.catalogue["Noise"] * EFFGAIN * EXPTIME - - RONoise(EFFRON, EFFGAIN, EXPTIME, self.catalogue["Area"]) - ) - # print('Flux_total_new: ',self.catalogue['Flux_total_new']) - self.catalogue["Flux_total_err"] = Flux_err( - EFFRON, - EFFGAIN, - EXPTIME, - self.catalogue["Area"], - self.catalogue["Noise"] * EFFGAIN * EXPTIME, - self.catalogue["Flux_total_new"], - ) - # print('Flux_total_err: ',self.catalogue['Flux_total_err']) - self.catalogue["SNR"] = ( - self.catalogue["Flux_total_new"] - / self.catalogue["Flux_total_err"] - / self.catalogue["Area"] - ) - # print('SNR: ',self.catalogue['SNR']) - self.catalogue["MAG_err"] = ( - 1 / self.catalogue["SNR"] - ) # use the approximate error for the magnitude. - self.catalogue["Flux_total_new"] = self.catalogue["Flux_total_new"] / ( - EFFGAIN * EXPTIME - ) - self.catalogue["MAG_flux"] = ABmag(self.catalogue["Flux_total_new"]) - - def create_polygons(self, use_gpu=False): - """ - Creates Polygons/contours best when you just want segmentations and not source charateristics. - """ - - self.catalogue = source.create_polygons( - use_gpu=use_gpu, - catalogue=self.catalogue, - cutout=self.image, - output=self.output, - cutupts=self.cutouts, - ) - - def create_polygons_fast(self): - - # since we have a bounding box, we can just create a polygon in the bounding box. - - polygons = [] - for index, row in tqdm( - self.catalogue.iterrows(), - total=len(self.catalogue), - desc="Creating polygons", - ): - contour = utils._get_polygons_in_bbox( - row.bbox2 - 2, - row.bbox4 + 2, - row.bbox1 - 2, - row.bbox3 + 2, - row.x1, - row.y1, - row.Birth, - row.Death, - ) - polygons.append(contour) - - self.polygons = polygons - - def create_polygons_gpu(self): - """ - Recommended when using GPU acceleration. and not using the bounding box to simplify the polygon creation. - """ - polygons = [] - self.image_gpu = cp.asarray(self.image, dtype=cp.float64) - - for index, row in self.catalogue.iterrows(): - t0 = time.time() - contour = utils._get_polygons_gpu(row.x1, row.y1, row.Birth, row.Death) - t1 = time.time() - # print('Time to create polygon: ',t1-t0) - polygons.append(contour) - self.polygons = polygons - - def _set_types_of_dataframe(self): - """ - Sets the catalogue to the correct data types. This is important to allow for writing data. - Otherwise the datatypes will remain object which will try to be pickled. - - """ - self.catalogue["ID"] = self.catalogue["ID"].astype(int) - self.catalogue["Birth"] = self.catalogue["Birth"].astype(float) - self.catalogue["Death"] = self.catalogue["Death"].astype(float) - self.catalogue["x1"] = self.catalogue["x1"].astype(float) - self.catalogue["y1"] = self.catalogue["y1"].astype(float) - self.catalogue["x2"] = self.catalogue["x2"].astype(float) - self.catalogue["y2"] = self.catalogue["y2"].astype(float) - self.catalogue["Flux_total"] = self.catalogue["Flux_total"].astype(float) - self.catalogue["Flux_peak"] = self.catalogue["Flux_peak"].astype(float) - self.catalogue["Area"] = self.catalogue["Area"].astype(float) - self.catalogue["Xc"] = self.catalogue["Xc"].astype(float) - self.catalogue["Yc"] = self.catalogue["Yc"].astype(float) - self.catalogue["bbox1"] = self.catalogue["bbox1"].astype(float) - self.catalogue["bbox2"] = self.catalogue["bbox2"].astype(float) - self.catalogue["bbox3"] = self.catalogue["bbox3"].astype(float) - self.catalogue["bbox4"] = self.catalogue["bbox4"].astype(float) - self.catalogue["Maj"] = self.catalogue["Maj"].astype(float) - self.catalogue["Min"] = self.catalogue["Min"].astype(float) - self.catalogue["Pa"] = self.catalogue["Pa"].astype(float) - self.catalogue["parent_tag"] = self.catalogue["parent_tag"].astype(float) - self.catalogue["Class"] = self.catalogue["Class"].astype(float) - if self.cutup: - self.catalogue["Y0_cutout"] = self.catalogue["Y0_cutout"].astype(float) - self.catalogue["X0_cutout"] = self.catalogue["X0_cutout"].astype(float) - self.catalogue["SNR"] = self.catalogue["SNR"].astype(float) - self.catalogue["Noise"] = self.catalogue["Noise"].astype(float) - - def plot_sources(self, cmap, figsize=(10, 10), norm="linear", save_path=None): - """Plots the source polygons on the image. - - Args: - cmap (str): matplotlib cmap to use, e.g. 'gray'. See https://matplotlib.org/stable/tutorials/colors/colormaps.html for more info. - figsize (tuple, optional): Desired figure size. Defaults to (10,10). - norm (str, optional): _description_. Defaults to 'linear'. - save_path (str, optional): Save path if you desire to save the figure. Defaults to None. - - """ - plt.figure(figsize=figsize) - plt.imshow(self.image, cmap=cmap, origin="lower", norm=norm) - # plt.scatter(self.catalogue['Xc'],self.catalogue['Yc'],s=10,c='r') - - for i, poly in enumerate(self.polygons): - if poly is not None: - plt.plot(poly[:, 1], poly[:, 0]) - if save_path is not None: - plt.savefig(save_path) - plt.show() - - def save_catalogue(self, save_path, filetype=None, overwrite=False): - """Save Catalogue to a file. - - Args: - save_path (str): Desired path to save the catalogue. - filetype (str, optional): Specify the file type or include the approprate file extention. Defaults to None. - overwrite (bool, optional): Overwrite the save file if the name is the same. Defaults to False. - - """ - - # get the extension from the save_path - # print('Saving Catalogue to file: ',save_path) - fileextention = save_path.split(".")[-1] - - if filetype is None: - filetype = fileextention - - if filetype == "csv": - self.catalogue.to_csv(save_path, index=False, overwrite=overwrite) - # print('Catalogue saved to: ',save_path) - - if filetype == "fits": - from astropy.table import Table - - # print('Saving to fits with astropy') - enclosed_i = self.catalogue["enclosed_i"] - - for i in range(len(enclosed_i)): - for j in range(len(enclosed_i[i])): - enclosed_i[i][j] = int(enclosed_i[i][j]) - if len(enclosed_i[i]) == 0: - enclosed_i[i] = [0] - self.catalogue["enclosed_i"] = enclosed_i - # print(self.catalogue) - t = Table.from_pandas(self.catalogue) - t.write(save_path, overwrite=overwrite) - - if filetype == "hdf": - self.catalogue.to_hdf(save_path, key="catalogue", mode="w") - # print('Catalogue saved to: ',save_path) - - if filetype == ("txt" or "ascii"): - self.catalogue.to_csv(save_path, index=False, overwrite=overwrite) - # print('Catalogue saved to: ',save_path) - - def open_catalogue(self, file_path, filetype=None): - from astropy.table import Table - - self.catalogue = Table.read(file_path) - - for i in range(len(self.catalogue)): - self.catalogue["contour"][i] = np.array( - self.catalogue["contour"][i] - ).reshape(-1, 2) - - self.catalogue = self.catalogue.to_pandas() - - def save_polygons_to_ds9(self, filename): - """ - Saves the polygons to a ds9 region file. - """ - - with open(filename, "w") as f: - f.write("# Region file format: DS9 version 4.1\n") - f.write( - 'global color=green dashlist=8 3 width=1 font="helvetica 10 normal roman" select=1 highlite=1 dash=0 fixed=0 edit=1 move=1 delete=1 include=1 source=1\n' - ) - for polygon in self.polygons: - f.write("polygon(") - for i, point in enumerate(polygon): - f.write( - "{:.2f},{:.2f}".format(point[1], point[0]) - ) # note this transformation as the index in some CARTA inmages start at -1. - if i < len(polygon) - 1: - f.write(",") - f.write(")\n") - - def save_polygons_to_hdf5(self, filename): - """ - Saves the polygons to a hdf5 file. - """ - import h5py - - hf = h5py.File(filename, "w") - for i in range(len(self.catalogue)): - key = self.catalogue["ID"][i] - hf.create_dataset(str(key), data=self.catalogue["contour"][i]) - hf.close() diff --git a/DRUID/src/background.py b/DRUID/src/background.py index 0b80660..e69de29 100644 --- a/DRUID/src/background.py +++ b/DRUID/src/background.py @@ -1,167 +0,0 @@ -""" - -File: background.py -Author: Rhys Shaw -Date: 23/12/2023 -Version: v1.0 -Description: Functions for calculating the background of an image. - -""" - -import numpy as np -from astropy.stats import mad_std, sigma_clipped_stats -import matplotlib.pyplot as plt -from photutils.background import SExtractorBackground - - -def radio_background_map(cutout: np.ndarray, box_size: int): - """ - - This function takes an image and a box size and calculates the radio_background() for each box to create a map of local background. - - """ - - step_size = box_size // 2 - - # initialize the map - map_shape = (cutout.shape[0] // step_size, cutout.shape[1] // step_size) - bg_map = np.zeros(map_shape) - mean_bg_map = np.zeros(map_shape) - - # box - box = np.ones((box_size, box_size)) - - for i in range(0, cutout.shape[0], step_size): - for j in range(0, cutout.shape[1], step_size): - # get the box - box_image = cutout[i : i + box_size, j : j + box_size] - # calculate the radio background - local_bg = mad_std(box_image, ignore_nan=True) - mean_bg = np.nanmedian(box_image) - # set the value in the map - bg_map[i // step_size, j // step_size] = local_bg - mean_bg_map[i // step_size, j // step_size] = mean_bg - - # now upsample the map to the original image size - bg_map = np.repeat(bg_map, step_size, axis=0) - bg_map = np.repeat(bg_map, step_size, axis=1) - - mean_bg_map = np.repeat(mean_bg_map, step_size, axis=0) - mean_bg_map = np.repeat(mean_bg_map, step_size, axis=1) - # shift the map to the correct position - - return bg_map, mean_bg_map - - -def calculate_background_map(image, box_size, mode="mad_std"): - - image_height, image_width = len(image), len(image[0]) - box_sum = 0 - box_mean_bg = np.zeros((image_height // box_size + 1, image_width // box_size + 1)) - box_std_bg = np.zeros((image_height // box_size + 1, image_width // box_size + 1)) - for i in range(image_height // box_size + 1): - for j in range(image_width // box_size + 1): - xmin = i * box_size - ymin = j * box_size - subarray = image[xmin : xmin + box_size, ymin : ymin + box_size] - box_mean_bg[i, j], box_std_bg[i, j] = calculate_background( - subarray, mode=mode - ) - - return box_mean_bg, box_std_bg - - -def get_bg_value_from_result_image(original_location_in_full_image, box_size, bg_map): - i, j = original_location_in_full_image - x = i // box_size - y = j // box_size - if x >= bg_map.shape[0]: - x = bg_map.shape[0] - 1 - if y >= bg_map.shape[1]: - y = bg_map.shape[1] - 1 - - result_value = bg_map[x, y] - return result_value - - -def calculate_background(image, mode="mad_std"): - """ - This function calculates the background of an image. - - Args: - image (np.ndarray): The image. - mode (str, optional): Can choose from mad_std or rms. - - Returns: - background (float): The background of the image. - """ - - if mode == "mad_std" or mode == "rms": - - local_bg, mean_bg = radio_background(image, metric=mode) - - elif mode == "SEX" or mode == "sigma_clip": - - local_bg, mean_bg = get_optical_background_estimate(image, mode) - - else: - # some other method can be added here. - raise ValueError("mode not recognised. Please use mad_std or rms") - - return local_bg, mean_bg - - -def get_optical_background_estimate(image, mode): - """Returns the background estimates from the image. using the SExtractorBackground or Sigma Clipping. - - Args: - image (_type_): _description_ - mode (_type_): _description_ - - Returns: - _type_: _description_ - """ - if mode == "sigma_clip": - mean, median, std = sigma_clipped_stats(image, sigma=3, maxiters=5) - - return std, median - - if mode == "SEX": - bkg = SExtractorBackground() - bkg_sigma = bkg.sigma_clip(image).std() - bkg_meadian = np.median(bkg.sigma_clip(image)) - return bkg_sigma, bkg_meadian - - -def radio_background(image: np.ndarray, metric: str = "mad_std"): - """ - - Returns the local background of an image. - - Args: - image (np.ndarray): The image. - metric (str, optional): Can Choose from mad_std or rms. - - Raises: - ValueError: If metric is not recognised. - - Returns: - - local_bg (float): The local background of the image./ - - """ - - if metric == "mad_std": - - local_bg = mad_std(image, ignore_nan=True) - - elif metric == "rms": # rmse (tends to rms when mean is 0.) - predicted = np.nanmean(image) - local_bg = np.sqrt(np.nanmean((image - predicted) ** 2)) - - else: - raise ValueError("metric not recognised. Please use mad_std or rms") - - mean_bg = np.nanmedian(image) - - return local_bg, mean_bg diff --git a/DRUID/src/homology/__init__.py b/DRUID/src/homology.py similarity index 100% rename from DRUID/src/homology/__init__.py rename to DRUID/src/homology.py diff --git a/DRUID/src/homology/homology.py b/DRUID/src/homology/homology.py deleted file mode 100644 index a8cc616..0000000 --- a/DRUID/src/homology/homology.py +++ /dev/null @@ -1,561 +0,0 @@ -""" -File: src/homology.py -author: Rhys Shaw -date: 27-12-2023 -Description: This file contains the functions that deal with calculating - persistence diagrams from a given image. -""" - - -import cripser -import numpy as np -from .src import utils -import time -from functools import partial -import pandas -from tqdm import tqdm -from multiprocessing import Pool, freeze_support -import pdb - -from collections import deque - -try: - import cupy as cp - from cupyx.scipy.ndimage import label as cupy_label - GPU_AVAILABLE = True -except: - GPU_AVAILABLE = False - - - - - - -def parent_tag_func_vectorized(df): - """ - - Vectorised implenetation of parent tag function. - - - Args: - - df: pd.Dataframe - data frame for which we calculate the parent tags. - - - Returns: - - df: pd.Dataframe - Pandas data frame with addition parent tag column. - - """ - - enclosed_i_dict = {row['ID']: set(row['enclosed_i']) for idx, row in df.iterrows()} - - def find_parent_tag(row): - #if row.new_row == 0: # only set it if it is not a new row. - - for ID, enclosed_i_set in enclosed_i_dict.items(): - if row.ID in enclosed_i_set: - return ID - return np.nan - - #else: # if it is a new row then we already know the parent tag. - # return row.parent_tags - - return df.apply(find_parent_tag, axis=1) - - - - - - -def classify_single(row): - - """Classifiying the Rows based on orgin. - - Args: - row: pd.series - The row that is being classified. - - Returns: - Class: int - the Class integer that indiceates the class the row belongs too. - - """ - if row.new_row == 0: - if len(row.enclosed_i) == 0: # no children - if np.isnan(row.parent_tag): # no parent - return 0 # no children, no parent. - else: - return 1 # no child has parent. - else: - if np.isnan(row.parent_tag): - return 2 # has children, no parent. - else: - return 3 # has children, has parent. - else: - return 4 # new row has children. - - - - - - -def make_point_enclosure_assoc(row,pd,img): - """Returns a list of the indices of the points that are enclosed by the mask pd point. - - Args: - row (pd.Series): _description_ - pd (pd.DataFrame): _description_ - img (np.ndarray): _description_ - - Returns: - enclosed_list (list): _description_ - - """ - - mask = utils.get_mask_CPU(row,img) - - encloses = [] - for i in range(len(pd)): - point = pd.iloc[i] - # we dont want to include ourselves - if point['ID'] == row['ID']: - continue - if mask[int(point.x1),int(point.y1)]: - encloses.append(point['ID']) - - return encloses - - - - - -def make_point_enclosure_assoc_GPU(Birth,Death,row,pd,img,img_gpu): - """Returns a list of the ID of the points that are enclosed by the mask pd point. - Uses GPU for computation. - - Args: - Birth (float): _description_ - Death (float): _description_ - row (pd.Series): _description_ - pd (pd.DataFrame): _description_ - img (np.ndarray): _description_ - img_gpu (cp.ndarray): _description_ - - Returns: - enclosed_list (list): _description_ - """ - - mask = utils.get_mask_GPU(Birth,Death,row,img_gpu) - #pdb.set_trace() - mask_coords = np.column_stack((pd['x1'], pd['y1'])) - points_inside_mask = mask[mask_coords[:, 0].astype(int), mask_coords[:, 1].astype(int)] - encloses_vectorized = pd.iloc[points_inside_mask]['ID'].tolist() - # remove self from list - #print(row.ID) - #print(encloses_vectorized) - #encloses_vectorized.remove(row.ID) - #print(encloses_vectorized) - #pdb.set_trace() - return encloses_vectorized - - - -def make_point_enclosure_assoc_GPU_second(Birth,Death,row,pd,img,img_gpu): - """Returns a list of the ID of the points that are enclosed by the mask pd point. - Uses GPU for computation. - - Args: - Birth (float): _description_ - Death (float): _description_ - row (pd.Series): _description_ - pd (pd.DataFrame): _description_ - img (np.ndarray): _description_ - img_gpu (cp.ndarray): _description_ - - Returns: - enclosed_list (list): _description_ - """ - - mask = utils.get_mask_GPU(Birth,Death,row,img_gpu) - #pdb.set_trace() - mask_coords = np.column_stack((pd['x1'], pd['y1'])) - points_inside_mask = mask[mask_coords[:, 0].astype(int), mask_coords[:, 1].astype(int)] - encloses_vectorized = pd.iloc[points_inside_mask]['ID'].tolist() - # remove self from list - #print(row.ID) - #print(encloses_vectorized) - encloses_vectorized.remove(row.ID) - #print(encloses_vectorized) - #pdb.set_trace() - return encloses_vectorized - - - - -def correct_first_destruction(pd,output,img=None,img_gpu=None,GPU=False): - """ - Function for correcting for the First destruction of a parent Island. - - Args: - pd (pd.DataFrame): Input catalogue of sources to correct. - output (bool): True if you want interation logginf with tqdm. - - Returns: - pd (pd.DataFrame): The new Catalogue. - - """ - - pd['new_row'] = 0 - - for i in tqdm(range(0,len(pd)),total=len(pd),desc='Correcting first destruction',disable=output): - - row = pd.iloc[i] - #print(row) - enlosed_i = row['enclosed_i'] - - if len(enlosed_i) >= 1: - new_row = row.copy() - - new_row['Death'] = pd.loc[pd['ID'] == enlosed_i[0]]['Death'] - new_row['parent_tag'] = pd.loc[pd['ID'] == enlosed_i[0]]['ID'] - ## this accounts for a bug were the entire series is placed in the death column. - # not sure on the origin of this but the following corrects for it. It only occationally happends so this is not - # computationally expensive. - if type(new_row['Death']) == pandas.core.series.Series: - # get the Death value from the first item in the Series. - new_row['Death'] = new_row['Death'].iloc[0] - - new_row['new_row'] = 1 - new_row['ID'] = pd['ID'].max() + 1 - new_row['enclosed_i'] = [] - - - pd = pandas.concat((pd,new_row.to_frame().T), ignore_index=False) - - return pd - - - - - - - - - -def calculate_area_CPU(row, img): - """Calculates area of source mask (for CPU) - - Args: - row (pd.series): _description_ - img (np.ndarray): Image - - Returns: - _type_: _description_ - """ - mask = utils.get_mask_CPU(row,img) - area = np.sum(mask) - return area - - - - - - - -def calculate_area_GPU(Birth,Death,row, img_gpu): - """Calcualtes are of source mask (for GPU) - - Args: - Birth (float): - Death (float): - row (pd.series): - img_gpu (cp.ndarray): - Returns: - area (float): the calculated area of the source mask. - """ - mask = utils.get_mask_GPU(Birth,Death,row,img_gpu) - # evalute if mask is True on an edge. - edge = utils.check_edge(mask) - if edge: - edge = 1 - - area = np.sum(mask) - return area, edge - - -def process_area(i,pd,img): - # handles worker function - return calculate_area_CPU(pd.iloc[i], img) - - -def process_assoc(i): - # hanldes worker function - return make_point_enclosure_assoc_CPU(pd.iloc[i], pd, img) - - - - - - - - - -def compute_ph_components(img,local_bg,analysis_threshold_val,lifetime_limit=0,output=True,bg_map=False,area_limit=3,nproc=1,GPU=False,lifetime_limit_fraction=2,mean_bg=None,IDoffset=None): - - - global GPU_Option - GPU_Option = GPU - t0_compute_ph = time.time() - pd = cripser.computePH(-img,maxdim=0) - t1_compute_ph = time.time() - print('PH computed! t='+str(t1_compute_ph-t0_compute_ph)+' s') - pd = pandas.DataFrame(pd,columns=['dim','Birth','Death','x1','y1','z1','x2','y2','z2'],index=range(1,len(pd)+1)) - pd.drop(columns=['dim','z1','z2'],inplace=True) - pd['lifetime'] = pd['Death'] - pd['Birth'] - pd['Birth'] = -pd['Birth'] - pd['Death'] = -pd['Death'] - print("mean_bg: ",mean_bg) - pd['mean_bg'] = mean_bg - pd['bg'] = 0 - pd['edge_flag'] = 0 - - if bg_map: - - list_of_index_to_drop = [] - - - for index, row in pd.iterrows(): - # check if local_bg is a map or a value - if row['Birth'] < local_bg[int(row.x1),int(row.y1)]: - list_of_index_to_drop.append(index) - - pd.drop(list_of_index_to_drop,inplace=True) - - - # for each row evaluate if death is below analysis thresholdval map value at its birth point. if its below then set Death to bg map value. - for index, row in pd.iterrows(): - Analy_val = analysis_threshold_val[int(row.y1),int(row.x1)] - if row['Death'] < Analy_val: - row['Death'] = Analy_val - # assign each row the local bg value - row['bg'] = local_bg[int(row.y1),int(row.x1)] - - - else: - - pd = pd[pd['Birth']>local_bg] # maybe this should be at the beginning. - pd['Death'] = np.where(pd['Death'] < analysis_threshold_val, analysis_threshold_val, pd['Death']) - pd['bg'] = local_bg - - pd['lifetime'] = abs(pd['Death'] - pd['Birth']) - - pd['lifetimeFrac'] = pd['Birth']/pd['Death'] - - # fiter by lifetimeFrac - - pd = pd[pd['lifetimeFrac']>lifetime_limit_fraction] - - - print('Persis Diagram computed. Length: ',len(pd)) - - if lifetime_limit > 0: - pd = pd[pd['lifetime'] > lifetime_limit] - - - pd.sort_values(by='lifetime',ascending=False,inplace=True,ignore_index=True) - - pd['ID'] = pd.index + IDoffset - - if len(pd) > 0: - - - - area_list = [] - edge_list = [] - - if nproc == 1: - - if GPU_Option == True: - - if GPU_AVAILABLE == True: - - # convert img to cupy array and define type so it does not have to be converted each time. - img_gpu = cp.asarray(img,dtype=cp.float64) - # Calculate area and enforce area limit Single Process. - print('Calculating area with GPU...') - t0 = time.time() - - for i in tqdm(range(0,len(pd)),total=len(pd),desc='Calculating area',disable=not output): - - row = pd.iloc[i] - Birth = row.Birth - Death = row.Death - area, edge = calculate_area_GPU(Birth,Death,row,img_gpu) - area_list.append(area) - edge_list.append(edge) - #percentage_completed = (i/len(pd))*100 - - #if percentage_completed % 10 == 0: - # print(percentage_completed,'%') - - print('Area calculated! t='+str(time.time()-t0)+' s') - - pd['area'] = area_list - pd['edge_flag'] = edge_list - pd = pd[pd['area'] > area_limit] - - enclosed_i_list = [] - print('Calculating enclosed_i with GPU...') - t0 = time.time() - for i in tqdm(range(0,len(pd)),total=len(pd),desc='Calculating enclosed_i',disable=not output): - row = pd.iloc[i] - Birth = row.Birth - Death = row.Death - enclosed_i = make_point_enclosure_assoc_GPU(Birth,Death,row,pd,img,img_gpu) - enclosed_i_list.append(enclosed_i) - - print('enclosed_i calculated! t='+str(time.time()-t0)+' s') - - pd['enclosed_i'] = enclosed_i_list - - - elif GPU_Option == False: - # 1 Core no GPU. - - # Calculate area and enforce area limit Single Process. - - t0 = time.time() - print("No GPU") - - for i in tqdm(range(0,len(pd)),total=len(pd),desc='Calculating area',disable=not output): - area = calculate_area_CPU(pd.iloc[i],img) - #print(area) - area_list.append(area) - - print('Area calculated! t='+str(time.time()-t0)+' s') - - pd['area'] = area_list - pd = pd[pd['area'] > area_limit] - - - # Parent Associations Single Process - - enclosed_i_list = [] - - t0 = time.time() - for i in tqdm(range(0,len(pd)),total=len(pd),desc='Calculating enclosed_i',disable=not output): - row = pd.iloc[i] - enclosed_i = make_point_enclosure_assoc(row,pd,img) - enclosed_i_list.append(enclosed_i) - - print('enclosed_i calculated! t='+str(time.time()-t0)+' s') - - pd['enclosed_i'] = enclosed_i_list - - - else: - # Multiple CPUs *** currently not working - - print('Calculating area with ',nproc,' processes') - - # Calculate area and enforce area limit Multi Process. - - t0 = time.time() - print("Chunksize: ",len(pd)//nproc) - index_and_args_list = [(i, pd, img) for i in range(len(pd))] - - with Pool(nproc) as p: - area_list = list(p.starmap(process_area, range(len(pd)),chunksize=len(pd)//nproc)) - print('Area calculated! t='+str(time.time()-t0)+' s') - - pd['area'] = area_list - pd = pd[pd['area'] > area_limit] # remove 1 pixel points - - print(len(pd)) - - ## Parent Associations Multi Process. - print('Calculating enclosed_i with ',nproc,' processes') - t0 = time.time() - - print("Chunksize: ",len(pd)//nproc) - - with Pool(nproc) as pool: - enclosed_i_list = list(pool.imap(process_assoc, range(len(pd)),chunksize=len(pd)//nproc)) - - print('enclosed_i calculated! t='+str(time.time()-t0)+' s') - - pd['enclosed_i'] = enclosed_i_list - - - #pd['parent_tag'] = 0 - pd = correct_first_destruction(pd,output=not output,img=img,img_gpu=img_gpu,GPU=GPU) - - # what if we do this all together after the first destruction correction? - - pd['lifetime'] = pd['Birth'] - pd['Death'] - #print(pd) - pd.sort_values(by='lifetime',ascending=False,inplace=True,ignore_index=False) - #print(pd) - #pdb.set_trace() - # corrected for first destruction. points need to have enlosed_i updated. - # update enclosed_i - enclosed_i_list = [] - print('Updating enclosed_i') - if GPU_Option: - enclosed_i_list = [] - t0 = time.time() - for i in tqdm(range(0,len(pd)),total=len(pd),desc='Calculating enclosed_i',disable=not output): - row = pd.iloc[i] - # is this a new row? - #if row.new_row == 1: - Birth = row.Birth - Death = row.Death - enclosed_i = make_point_enclosure_assoc_GPU_second(Birth,Death,row,pd,img,img_gpu) - enclosed_i_list.append(enclosed_i) - #else: - # enclosed_i_list.append(row.enclosed_i) - print('enclosed_i calculated! t='+str(time.time()-t0)+' s') - - pd['enclosed_i'] = enclosed_i_list - - else: - t0 = time.time() - for i in tqdm(range(0,len(pd)),total=len(pd),desc='Calculating enclosed_i',disable=not output): - row = pd.iloc[i] - # is this a new row? - #if row.new_row == 1: - enclosed_i = make_point_enclosure_assoc(row,pd,img) - enclosed_i_list.append(enclosed_i) - #else: - enclosed_i_list.append(row.enclosed_i) - print('enclosed_i calculated! t='+str(time.time()-t0)+' s') - - pd['enclosed_i'] = enclosed_i_list - - - print('Calculating parent_tags... ') - - t0_parent_tag = time.time() - parent_tag_list = parent_tag_func_vectorized(pd) - pd['parent_tag'] = parent_tag_list - t1_parent_tag = time.time() - print('parent_tag calculated! t='+str(t1_parent_tag-t0_parent_tag)+' s') - - #print(pd) - #pd['parent_tag'] = pd.apply(lambda row: parent_tag_func(row,pd), axis=1) - - print('Assigning Class ...') - t0_CLass = time.time() - pd['Class'] = pd.apply(classify_single,axis=1) - t1_Class = time.time() - print('Class assigned! t='+str(t1_Class-t0_CLass)+' s') - # drop the enclosed_i column - #pd.drop(columns=['enclosed_i'],inplace=True) - # distance from center of each image - - pd['distance_from_center'] = ((pd['x1'] - img.shape[0]/2)**2 + (pd['y1'] - img.shape[1]/2)**2)**0.5 - - return pd - - else: - - return pd \ No newline at end of file diff --git a/DRUID/src/homology_new.py b/DRUID/src/homology_new.py deleted file mode 100644 index b28201f..0000000 --- a/DRUID/src/homology_new.py +++ /dev/null @@ -1,602 +0,0 @@ -""" -File: src/homology_new.py -Author: Rhys Shaw -Date: 27/12/2025 -Version: v1.0 -Description: Functions for calculating source properties for sources. - -""" - -import cripser -import numpy as np -from ..src import utils -from ..src import background - -import time -import pandas -from tqdm import tqdm - - -try: - import cupy as cp - from cupyx.scipy.ndimage import label as cupy_label - - GPU_AVAILABLE = True -except: - GPU_AVAILABLE = False - -# used for debugging -# import matplotlib.pyplot as plt -# import pdb - - -def make_point_enclosure_assoc_GPU(id, x1, y1, Birth, Death, pd, img_gpu): - """Returns a list of the ID of the points that are enclosed by the mask pd point. - Uses GPU for computation. - - Args: - Birth (float): _description_ - Death (float): _description_ - row (pd.Series): _description_ - pd (pd.DataFrame): _description_ - img (np.ndarray): _description_ - img_gpu (cp.ndarray): _description_ - - Returns: - enclosed_list (list): _description_ - """ - # print(pd) - mask = utils.get_mask_GPU(Birth, Death, x1, y1, img_gpu).get() - # pdb.set_trace() - mask_coords = np.column_stack((pd["x1"], pd["y1"])) - points_inside_mask = mask[ - mask_coords[:, 0].astype(int), mask_coords[:, 1].astype(int) - ] - encloses_vectorized = pd.iloc[points_inside_mask]["ID"].tolist() - # remove self from list - # print(row.ID) - # print(encloses_vectorized) - # encloses_vectorized.remove(row.ID) - # print(encloses_vectorized) - # pdb.set_trace() - return encloses_vectorized - - -def make_point_enclosure_assoc_CPU(ID, x1, y1, Birth, Death, pd, img): - """Returns a list of the ID of the points that are enclosed by the mask pd point. - Uses GPU for computation. - - Args: - Birth (float): _description_ - Death (float): _description_ - row (pd.Series): _description_ - pd (pd.DataFrame): _description_ - img (np.ndarray): _description_ - img_gpu (cp.ndarray): _description_ - - Returns: - enclosed_list (list): _description_ - """ - - mask = utils.get_mask_CPU(x1, y1, Birth, Death, img) - # print(mask) - # plt.imshow(mask) - # plt.savefig('test.png') - # pdb.set_trace() - mask_coords = np.column_stack((pd["x1"], pd["y1"])) - points_inside_mask = mask[ - mask_coords[:, 0].astype(int), mask_coords[:, 1].astype(int) - ] - encloses_vectorized = pd.iloc[points_inside_mask]["ID"].tolist() - # remove self from list - # print(row.ID) - # print(encloses_vectorized) - # encloses_vectorized.remove(ID) - # print(encloses_vectorized) - # pdb.set_trace() - return encloses_vectorized - - -def classify_single(row): - """Classifiying the Rows based on orgin. - - Args: - row: pd.series - The row that is being classified. - - Returns: - Class: int - the Class integer that indiceates the class the row belongs too. - - """ - if row.new_row == 0: - if len(row.enclosed_i) <= 1: # no children - if row.parent_tag == row.ID: # no parent - return 0 # no children, no parent. - else: - return 2 # no child has parent. - else: - if row.parent_tag == row.ID: # no parent - return 4 # has children, no parent. - else: - return 3 # has children, has parent. - else: - return 1 # new row has children. - - -def parent_tag_func_vectorized(df): - """ - - Vectorised implenetation of parent tag function. - - - Args: - - df: pd.Dataframe - data frame for which we calculate the parent tags. - - - Returns: - - df: pd.Dataframe - Pandas data frame with addition parent tag column. - - """ - - enclosed_i_dict = {row["ID"]: set(row["enclosed_i"]) for idx, row in df.iterrows()} - - def find_parent_tag(row): - # if row.new_row == 0: # only set it if it is not a new row. - - for ID, enclosed_i_set in enclosed_i_dict.items(): - if row.ID in enclosed_i_set: - return ID - - return np.nan - - # else: # if it is a new row then we already know the parent tag. - # return row.parent_tags - - return df.apply(find_parent_tag, axis=1) - - -def parent_tag_func_vectorized_new(df): - """ - Vectorised implementation of parent tag function. - - Args: - df: pd.Dataframe - data frame for which we calculate the parent tags. - - Returns: - df: pd.Dataframe - Pandas data frame with addition parent tag column. - """ - - # Create a dictionary that maps each enclosed ID to its parent ID - def find_parent(row, df): - potential_parents = [] - for idx, other_row in df.iterrows(): - if row["ID"] in other_row["enclosed_i"] and row["ID"] != other_row["ID"]: - potential_parents.append( - (other_row["ID"], len(other_row["enclosed_i"])) - ) - - if potential_parents: - # Sort by length of enclosed_i (descending) and return the ID of the longest - return sorted(potential_parents, key=lambda x: x[1], reverse=True)[0][0] - - # If no parent found, check if this row has the largest enclosed_i - if len(row["enclosed_i"]) == df["enclosed_i"].apply(len).max(): - return row["ID"] # Set its own ID as parent - - return row["ID"] - - # Apply the function to each row in the DataFrame - df["parent_tag"] = df.apply(find_parent, axis=1, args=(df,)) - return df - - -# def parent_tag_func_optimized(df): -# """ -# Optimized version of the parent tag function. - -# Args: -# df: pd.DataFrame - DataFrame for which we calculate the parent tags. - -# Returns: -# df: pd.DataFrame - DataFrame with additional parent_tag column. -# """ -# # Convert enclosed_i lists to sets for faster lookups and map IDs to their indices -# enclosed_sets = df['enclosed_i'].apply(set) -# id_to_index = {id_: idx for idx, id_ in enumerate(df['ID'])} - -# # Create an array of parent indices, initialized to self (i.e., no parent) -# parent_indices = np.arange(len(df)) - -# for row_idx in range(len(df)): -# potential_parents = [id_to_index[id_] for id_ in df['enclosed_i'].iloc[row_idx] if id_ in id_to_index] - -# if potential_parents: -# # Find the parent with the smallest enclosed set -# min_parent = min(potential_parents, key=lambda idx: len(enclosed_sets.iloc[idx])) -# parent_indices[row_idx] = min_parent - -# # Map indices back to IDs -# df['parent_tag'] = df['ID'].iloc[parent_indices].values - -# return df - - -# def parent_tag_func_optimized(df): -# """ -# Corrected and optimized vectorized implementation of parent tag function. - -# Args: -# df: pd.DataFrame - data frame for which we calculate the parent tags. - -# Returns: -# df: pd.DataFrame - Pandas data frame with additional parent tag column. -# """ -# # Create a dictionary mapping IDs to their index in the dataframe -# id_to_index = {id: idx for idx, id in enumerate(df["ID"])} - -# # Convert enclosed_i lists to sets for faster lookup -# enclosed_sets = df["enclosed_i"].apply(set) - -# # Create a matrix of containment relationships -# containment_matrix = enclosed_sets.apply( -# lambda x: [1 if id in x else 0 for id in df["ID"]] -# ) -# containment_matrix = np.array(containment_matrix.tolist()) - -# # Remove self-containment -# np.fill_diagonal(containment_matrix, 0) - -# def find_immediate_parent(row_idx): -# potential_parents = np.where(containment_matrix[:, row_idx] == 1)[0] -# if len(potential_parents) == 0: -# return row_idx # No parent found, return self - -# # Among potential parents, find the one with the smallest enclosed_i set -# parent_sizes = [len(enclosed_sets.iloc[i]) for i in potential_parents] -# immediate_parent_idx = potential_parents[np.argmin(parent_sizes)] -# return immediate_parent_idx - -# # parent_indices = [ -# # find_immediate_parent(i) -# # for i in tqdm(range(len(df)), desc="Finding immediate parents", total=len(df)) -# # ] - -# # rewrite the linline function to a normal function - -# parent_indices = [] -# for i in tqdm(range(len(df)), desc="Finding immediate parents", total=len(df)): -# immediate_parent_idx = find_immediate_parent(i) -# parent_indices.append(immediate_parent_idx) - -# # Map indices back to IDs -# df["parent_tag"] = df["ID"].iloc[parent_indices].values - -# return df - - -def parent_tag_func_optimized(df): - # Create a DataFrame with only IDs that have enclosed_i of length greater than 1 - enclosed_i = df[["ID", "enclosed_i"]] - enclosed_i = enclosed_i[enclosed_i["enclosed_i"].apply(len) > 1] - - # Initialize parent_tag with ID - df["parent_tag"] = df["ID"] - - # Create a mapping from ID to parent ID - mapping = {} - - for idx, row in enclosed_i.iterrows(): - parent_id = row["ID"] - for child_id in row["enclosed_i"]: - if child_id != parent_id: - mapping[child_id] = parent_id - - # Update parent_tag using the mapping - df["parent_tag"] = df["ID"].map(mapping).combine_first(df["parent_tag"]) - - return df - - -def correct_first_destruction(pd, output): - """ - Function for correcting for the First destruction of a parent Island. - - Args: - pd (pd.DataFrame): Input catalogue of sources to correct. - output (bool): True if you want interation logginf with tqdm. - - Returns: - pd (pd.DataFrame): The new Catalogue. - - """ - - pd["new_row"] = 0 - - for i in tqdm( - range(0, len(pd)), - total=len(pd), - desc="Correcting first destruction", - disable=output, - ): - - row = pd.iloc[i] - # print(row) - enlosed_i = row["enclosed_i"] - if len(enlosed_i) > 1: - new_row = row.copy() - # print(i) - new_row["Death"] = pd.loc[pd["ID"] == enlosed_i[1]]["Death"] - new_row["parent_tag"] = pd.loc[pd["ID"] == enlosed_i[1]]["ID"] - - ## this accounts for a bug were the entire series is placed in the death column. - # not sure on the origin of this but the following corrects for it. It only occationally happends so this is not - # computationally expensive - if type(new_row["Death"]) == pandas.core.series.Series: - # get the Death value from the first item in the Series. - # print(new_row['Death']) - new_row["Death"] = new_row["Death"].iloc[0] # - - new_row["new_row"] = 1 - new_row["ID"] = pd["ID"].max() + 1 - new_row["enclosed_i"] = [] - - # concat the new row to the dataframe. - - pd = pandas.concat([pd, new_row.to_frame().T], ignore_index=True) - - # pd = pd.append(new_row,ignore_index=True) - # pd.loc[len(pd)+1] = new_row - - return pd - - -def calculate_area_GPU(Birth, Death, row, img_gpu): - """Calcualtes are of source mask (for GPU) - - Args: - Birth (float): - Death (float): - row (pd.series): - img_gpu (cp.ndarray): - Returns: - area (float): the calculated area of the source mask. - """ - mask = utils.get_mask_GPU(Birth, Death, row.x1, row.y1, img_gpu) - # get bounding box here - # evalute if mask is True on an edge. - bounding_box = utils.bounding_box_gpu(mask) - mask = mask.get() - edge = utils.check_edge(mask) - if edge: - edge = True - - area = np.sum(mask) - return area, edge, bounding_box - - -def calculate_area_CPU(Birth, Death, row, img): - - mask = utils.get_mask_CPU(row.x1, row.y1, Birth, Death, img) - # get bounding box here - bounding_box = utils.bounding_box_cpu(mask) - edge = utils.check_edge(mask) - if edge: - edge = 1 - area = np.sum(mask) - return area, edge, bounding_box - - -def compute_ph_components( - img, - local_bg, - analysis_threshold_val, - lifetime_limit, - output=False, - bg_map=False, - area_limit=3, - GPU=False, - lifetime_limit_fraction=2, - mean_bg=None, - IDoffset=0, - box_size=None, - detection_threshold=None, - Cutout_X_offset=0, - Cutout_Y_offset=0, -): - - global GPU_Option - GPU_Option = GPU - print("Computing PH components...") - t0_compute_ph = time.time() - pd = cripser.computePH(-img, maxdim=0) - t1_compute_ph = time.time() - print("Time to compute PH: {}".format(t1_compute_ph - t0_compute_ph)) - pd = pandas.DataFrame( - pd, - columns=["dim", "Birth", "Death", "x1", "y1", "z1", "x2", "y2", "z2"], - index=range(1, len(pd) + 1), - ) - pd.drop(columns=["dim", "z1", "z2"], inplace=True) - pd["lifetime"] = pd["Death"] - pd["Birth"] - pd["Birth"] = -pd["Birth"] - pd["Death"] = -pd["Death"] - - # print("mean_bg: ",mean_bg) - # get rid of birth less than 0, helps speed up the code alittle. - # mean_bg_temp = np.nanmean(local_bg)/detection_threshold - # print('mean_bg: ',mean_bg_temp) - # get rid of alot of defintly not sources. - # pd = pd[pd['Birth']>mean_bg_temp] - - pd["bg"] = 0 - pd["edge_flag"] = 0 - pd["mean_bg"] = 0 # this is the mean of the background - # assign each row the local bg valuw from the map. - if bg_map: - - # this is a slow function. Can it be improved? - # for each row we need to assign the local bg value. - pd["bg"] = pd["bg"].astype(float) - pd["mean_bg"] = pd["mean_bg"].astype(float) - for index, row in pd.iterrows(): - - pd.loc[index, "bg"] = float( - background.get_bg_value_from_result_image( - (int(row.x1 + Cutout_Y_offset), int(row.y1 + Cutout_X_offset)), - box_size, - local_bg, - ) - ) - pd.loc[index, "mean_bg"] = float( - background.get_bg_value_from_result_image( - (int(row.x1 + Cutout_Y_offset), int(row.y1 + Cutout_X_offset)), - box_size, - mean_bg, - ) - ) - - # evaluate if the death value is below the analysis threshold value at the birth point. - # if it is then set the death value to the analysis threshold value. - - Anal_val = background.get_bg_value_from_result_image( - (int(row.x1 + Cutout_Y_offset), int(row.y1 + Cutout_X_offset)), - box_size, - analysis_threshold_val, - ) - - if row["Death"] < Anal_val: - pd.loc[index, "Death"] = Anal_val - - else: - # no bg map so just assign the local bg value. asn this should be single value. - pd["bg"] = local_bg - pd["mean_bg"] = mean_bg - - for index, row in pd.iterrows(): - if row["Death"] < analysis_threshold_val: - pd.loc[index, "Death"] = analysis_threshold_val - - # print('Before Cull',len(pd)) - # print(pd['bg']) - # print('mean_bg: ',np.mean(pd['bg'])) - pd = pd[pd["Birth"] > pd["bg"]] # maybe this should be at the beginning. - # also make sure Death is less than Birth - # pd = pd[pd['Death'] < pd['Birth']] - # print('After Cull',len(pd)) - # if bg_map: - - # list_of_index_to_drop = [] - - # for index, row in pd.iterrows(): - # # check if local_bg is a map or a value - # if row['Birth'] < local_bg[int(row.x1),int(row.y1)]: - # list_of_index_to_drop.append(index) - - # pd.drop(list_of_index_to_drop,inplace=True) - - # # for each row evaluate if death is below analysis thresholdval map value at its birth point. if its below then set Death to bg map value. - # for index, row in pd.iterrows(): - # Analy_val = analysis_threshold_val[int(row.y1),int(row.x1)] - # if row['Death'] < Analy_val: - # row['Death'] = Analy_val - # # assign each row the local bg value - # row['bg'] = local_bg[int(row.y1),int(row.x1)] - # row['mean_bg'] = mean_bg[int(row.y1),int(row.x1)] - - # else: - # print(local_bg) - # pd = pd[pd['Birth']>local_bg] # maybe this should be at the beginning. - # pd['Death'] = np.where(pd['Death'] < analysis_threshold_val, analysis_threshold_val, pd['Death']) - # pd['bg'] = local_bg - # pd['mean_bg'] = mean_bg - - pd["lifetime"] = abs(pd["Death"] - pd["Birth"]) - - pd["lifetimeFrac"] = pd["Birth"] / pd["Death"] - pd = pd[pd["lifetimeFrac"] > lifetime_limit_fraction] - pd = pd[pd["lifetime"] > lifetime_limit] - pd.sort_values(by="lifetime", ascending=False, inplace=True, ignore_index=True) - - pd["ID"] = pd.index + IDoffset - - # print('PD: ',pd) - # begins here - - if len(pd) > 0: - - area_list = [] - edge_list = [] - bbox1 = [] - bbox2 = [] - bbox3 = [] - bbox4 = [] - - if GPU_Option == True: - if GPU_AVAILABLE == True: - img_gpu = cp.asarray(img, dtype=cp.float64) - # Calculate area and enforce area limit Single Process. - # print('Calculating area with GPU...') - t0 = time.time() - - for i in tqdm( - range(0, len(pd)), - total=len(pd), - desc="Calculating area", - disable=not output, - ): - - row = pd.iloc[i] - Birth = row.Birth - Death = row.Death - area, edge, bbox = calculate_area_GPU(Birth, Death, row, img_gpu) - area_list.append(area) - edge_list.append(edge) - bbox1.append(bbox[0].get()) - bbox2.append(bbox[1].get()) - bbox3.append(bbox[2].get()) - bbox4.append(bbox[3].get()) - - t1 = time.time() - # print('Time to calculate area and inital bbox: {}'.format(t1-t0)) - pd["area"] = area_list - pd["edge_flag"] = edge_list - pd["bbox1"] = bbox1 - pd["bbox2"] = bbox2 - pd["bbox3"] = bbox3 - pd["bbox4"] = bbox4 - pd = pd[pd["area"] > area_limit] - - else: - - # print('Calculating area with CPU...') - t0 = time.time() - for i in tqdm( - range(0, len(pd)), - total=len(pd), - desc="Calculating area", - disable=not output, - ): - - row = pd.iloc[i] - Birth = row.Birth - Death = row.Death - area, edge, bbox = calculate_area_CPU(Birth, Death, row, img) - area_list.append(area) - edge_list.append(edge) - bbox1.append(bbox[0]) - bbox2.append(bbox[1]) - bbox3.append(bbox[2]) - bbox4.append(bbox[3]) - - t1 = time.time() - # print('Time to calculate area and inital bbox: {}'.format(t1-t0)) - pd["area"] = area_list - pd["edge_flag"] = edge_list - pd["bbox1"] = bbox1 - pd["bbox2"] = bbox2 - pd["bbox3"] = bbox3 - pd["bbox4"] = bbox4 - pd = pd[pd["area"] > area_limit] - - return pd diff --git a/DRUID/src/source.py b/DRUID/src/source.py index bfb46a7..e69de29 100644 --- a/DRUID/src/source.py +++ b/DRUID/src/source.py @@ -1,752 +0,0 @@ -""" -File: src/source.py -Author: Rhys Shaw -Date: 27/12/2023 -Version: v1.0 -Description: Functions for calculating source properties for sources. - -""" - -from ..src import utils -import pandas as pd -import numpy as np -from tqdm import tqdm -from scipy.ndimage import label -import pdb -from scipy.ndimage import binary_dilation - -try: - import cupy as cp - from cupyx.scipy.ndimage import label as cupy_label - -except: - - pass - - -def create_params_df(cutup: bool, params: list): - """ - - Creates a pandas dataframe from the parameters. - - """ - - params = pd.DataFrame( - params, - columns=[ - "ID", - "Birth", - "Death", - "x1", - "y1", - "x2", - "y2", - "Flux_total", - "Flux_total_err", - "Flux_peak", - "Flux_correction_factor", - "Area", - "Xc", - "Yc", - "bbox1", - "bbox2", - "bbox3", - "bbox4", - "Maj", - "Min", - "Pa", - "parent_tag", - "Class", - "SNR", - "Noise", - "X0_cutout", - "Y0_cutout", - "mean_bg", - "bg_rms", - "Edge_flag", - "contour", - "enclosed_i", - ], - ) - - return params - - -def large_mask_red_image_procc_GPU(Birth, Death, x1, y1, image, X0, Y0): - """ - Does all gpu processing for the large mask. - - return the red_image and red_mask and the bounding box. - - """ - - mask = cp.zeros(image.shape, dtype=cp.bool_) - mask = cp.logical_or(mask, cp.logical_and(image <= Birth, image > Death)) - - # mask_enclosed = self.get_enclosing_mask_gpu(y1,x1,mask) - labeled_mask, num_features = cupy_label(mask) - - # Check if the specified pixel is within the mask - if 0 <= y1 < mask.shape[1] and 0 <= x1 < mask.shape[0]: - label_at_pixel = labeled_mask[x1, y1] - # print(x1) - # print(y1) - # print(label_at_pixel) - if label_at_pixel != 0: - # Extract the connected component containing the specified pixel - component_mask = labeled_mask == label_at_pixel - # plt.imshow(component_mask.get()) - # plt.show() - # pdb.set_trace() - non_zero_indices = cp.nonzero(component_mask) - - # Extract minimum and maximum coordinates - xmin = cp.min(non_zero_indices[1]) - ymin = cp.min(non_zero_indices[0]) - xmax = cp.max(non_zero_indices[1]) - ymax = cp.max(non_zero_indices[0]) - - # correct the bounding box for the cutout. - xmin = xmin # + Y0 - xmax = xmax # + Y0 - ymin = ymin # + X0 - ymax = ymax # + X0 - - # images are not being cropped? - - red_image = image[ymin : ymax + 1, xmin : xmax + 1] - red_mask = component_mask[ymin : ymax + 1, xmin : xmax + 1] - - return red_image, red_mask, xmin, xmax, ymin, ymax - - -def large_mask_red_image_procc_CPU(Birth, Death, x1, y1, image, image_smooth): - """ - Does all gpu processing for the large mask. - - return the red_image and red_mask and the bounding box. - - """ - mask = np.zeros(image.shape) - mask = np.logical_or( - mask, np.logical_and(image_smooth <= Birth, image_smooth > Death) - ) - # print(mask.shape) - # mask_enclosed = self.get_enclosing_mask_gpu(y1,x1,mask) - labeled_mask, num_features = label(mask) - - # Check if the specified pixel is within the mask - if 0 <= y1 < mask.shape[1] and 0 <= x1 < mask.shape[0]: - - label_at_pixel = labeled_mask[x1, y1] - - if label_at_pixel != 0: - # Extract the connected component containing the specified pixel - component_mask = labeled_mask == label_at_pixel - # plt.imshow(component_mask.get()) - # plt.show() - # pdb.set_trace() - non_zero_indices = np.nonzero(component_mask) - - # Extract minimum and maximum coordinates - xmin = np.min(non_zero_indices[1]) - ymin = np.min(non_zero_indices[0]) - xmax = np.max(non_zero_indices[1]) - ymax = np.max(non_zero_indices[0]) - - # images are not being cropped? - - red_image = image[ymin : ymax + 1, xmin : xmax + 1] - red_mask = component_mask[ymin : ymax + 1, xmin : xmax + 1] - - return red_image, red_mask, xmin, xmax, ymin, ymax - - -# import cv2 -# import numpy as np - -# def dilate_mask_circular(mask, radius): -# Create a circular structuring element using cv2.getStructuringElement -# circular_kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (2*radius+1, 2*radius+1)) - -# Perform dilation using cv2.dilate -# dilated_mask = cv2.dilate(mask.astype(np.uint8), circular_kernel)# - -# return dilated_mask - - -# def curve_of_growth_dilation(mask,image): -# converged = False -# i = 0 -# dilated_mask = np.zeros(image.shape, dtype=bool) -# mask_before = mask -# while converged == False: -# if i == 50: -# #print('max iterations reached') -# return dilated_mask.astype(int) -# # print(i) -# i += 1 - -# dilated_mask = dilate_mask_circular(mask_before,1) - -# # check if the mask has converged -# flux = np.sum(image*dilated_mask) -# flux_old = np.sum(image*mask_before) -# diff = (flux - flux_old) -# #print(diff) -# if diff<0: -# converged = True -# dilated_mask = mask_before - -# else: -# mask_before = dilated_mask -# #print('converged {}'.format(i)) -# return dilated_mask.astype(int) - - -def measure_source_properties( - use_gpu, - catalogue=None, - cutout=None, - smooth_cutout=None, - background_map=None, - output=None, - cutupts=None, - mode="optical", - header=None, - sigma=5, -): - """ - - Characterising the Source Assuming the input image of of the format of a optical astronomical image. - - # needs to work on cutup images. - # work on the whole image. - - """ - if header == None: - mode = "other" - - if mode == "Radio": - print("Radio mode selected") - Beam, BMAJ, BMIN, BPA = utils.calculate_beam(header=header) - - if mode == "optical": - print("Optical mode selected, modeling the PSF as a gaussian.") - psf_fwhm_p = utils.get_psf_FWHM(header) - # EFFRON = utils.get_EFFRON(header) - # EFFGAIN = utils.get_EFFGAIN(header) - # EXPTIME = utils.get_EXPTIME(header) - - if use_gpu: - - try: - - import cupy as cp - - except: - - raise ImportError("cupy not installed. GPU acceleration not possible.") - - image = cutout - if use_gpu: - image_gpu = cp.asarray(image, dtype=cp.float64) - smooth_image_gpu = cp.asarray(smooth_cutout, dtype=cp.float64) - - Birth = catalogue["Birth"].to_numpy() - Death = catalogue["Death"].to_numpy() - parent_tag = catalogue["parent_tag"].to_numpy() - Class = catalogue["Class"].to_numpy() - bg = catalogue["bg"].to_numpy() - X0 = catalogue["X0_cutout"].to_numpy() - Y0 = catalogue["Y0_cutout"].to_numpy() - mean_bg = catalogue["mean_bg"].to_numpy() - enclosed_i = catalogue["enclosed_i"].to_numpy() - IDs = catalogue["ID"].to_numpy() - Edge_flags = catalogue["edge_flag"].to_numpy() - bbox1_og = catalogue["bbox1"].to_list() - bbox2_og = catalogue["bbox2"].to_list() - bbox3_og = catalogue["bbox3"].to_list() - bbox4_og = catalogue["bbox4"].to_list() - - x1 = catalogue["x1"].to_numpy() # - 1 #- X0 - y1 = catalogue["y1"].to_numpy() # - 1#- X0 - x2 = catalogue["x2"].to_numpy() # - 1#- Y0 - y2 = catalogue["y2"].to_numpy() # - 1#- X0 - - # map X0 and Y0 to the cutout number - params = [] - polygons = [] - - # import matplotlib.pylab as plt - - for i, source in tqdm( - enumerate(Birth), - total=len(Birth), - desc="Calculating Source Properties..", - disable=not output, - ): - - if use_gpu == True: - - cropped_image_gpu = image_gpu[ - bbox1_og[i] - 1 : bbox3_og[i] + 1, bbox2_og[i] - 1 : bbox4_og[i] + 1 - ] - - cropped_smooth_image_gpu = smooth_image_gpu[ - bbox1_og[i] - 1 : bbox3_og[i] + 1, bbox2_og[i] - 1 : bbox4_og[i] + 1 - ] - - try: - red_image, red_mask, xmin, xmax, ymin, ymax = ( - large_mask_red_image_procc_GPU( - Birth[i], - Death[i], - x1[i] - bbox1_og[i] + 1, - y1[i] - bbox2_og[i] + 1, - cropped_image_gpu, - X0[i], - Y0[i], - ) - ) - except: - - print("Error in GPU processing!") - print("Source ID: ", IDs[i]) - print("x1:", x1[i] - bbox1_og[i] + 1) - print("y1:", y1[i] - bbox2_og[i] + 1) - print("Birth:", Birth[i]) - print("Death:", Death[i]) - # print('Cropped_image',cropped_image_gpu.get()) - - red_mask = red_mask.astype(int) - - red_image = red_image.get() - red_mask = red_mask.get() - xmin = xmin.get() + bbox2_og[i] - xmax = xmax.get() + bbox2_og[i] - ymin = ymin.get() + bbox1_og[i] - ymax = ymax.get() + bbox1_og[i] - - else: - cropped_image = image[ - bbox1_og[i] - 1 : bbox3_og[i] + 1, bbox2_og[i] - 1 : bbox4_og[i] + 1 - ] - cropped_smooth_image = smooth_cutout[ - bbox1_og[i] - 1 : bbox3_og[i] + 1, bbox2_og[i] - 1 : bbox4_og[i] + 1 - ] - try: - red_image, red_mask, xmin, xmax, ymin, ymax = ( - large_mask_red_image_procc_CPU( - Birth[i], - Death[i], - int(x1[i]) - int(bbox1_og[i]) + 1, - int(y1[i]) - int(bbox2_og[i]) + 1, - cropped_image, - cropped_smooth_image, - ) - ) - except: - print("Error in CPU processing!") - print("Source ID: ", IDs[i]) - print("x1:", x1[i] - bbox1_og[i] + 1) - print("y1:", y1[i] - bbox2_og[i] + 1) - print("Birth:", Birth[i]) - print("Death:", Death[i]) - - red_mask = red_mask.astype(int) - - xmin = xmin + bbox2_og[i] - xmax = xmax + bbox2_og[i] - ymin = ymin + bbox1_og[i] - ymax = ymax + bbox1_og[i] - - # DILATION TEST Not Working. - # print(red_mask) - - # #try: - # if Class[i] == 0: - # if xmin > 50 and ymin > 50: - # expanx = 50 - # expany = 50 - # # increase mask and image size by 100 pixels in each direction. - # # this is to ensure that the mask is large enough to capture the entire source. - # # if the source is less than 50px away from the image edge then we padd with the distance to the edge -1 - # red_mask = np.pad(red_mask,pad_width=((expanx,expanx-1),(expany,expany-1)),mode='constant',constant_values=0) - # #,((expanx,expanx),(expany,expany)),mode='constant',constant_values=0) - # #plt.imshow(red_mask) - # #plt.savefig('red_mask.png') - # red_mask = red_mask#[0:-1,0:-1] - # #print('red_mask shape',red_mask.shape) - # red_image = image[ymin-expanx:ymax+expanx,xmin-expany:xmax+expany] - # #c - # plt.figure(figsize=(10,10)) - # plt.imshow(red_image,cmap='gray',origin='lower',vmin=1E-12,vmax=1E-10) - # plt.contour(red_mask) - # plt.savefig('red_image.png') - - # #print('red_image shape',red_image.shape) - # # dilate the mask until the flux converges. - # # check that the shapes are the same - # if red_mask.shape != red_image.shape: - # print('red_mask and red_image shapes are not the same.') - # print('Skipping source...') - # continue - # red_mask = curve_of_growth_dilation(red_mask,red_image) - # plt.figure(figsize=(10,10)) - # plt.imshow(red_image,cmap='gray',origin='lower',vmin=1E-12,vmax=1E-10) - # plt.contour(red_mask) - # plt.savefig('red_image_after.png') - - # pdb.set_trace() - - # #print('red_mask shape',red_mask.shape) - # # reduce the mask and recalculate the boundig box. - # # plt.imshow(red_mask) - # # plt.savefig('red_mask.png') - # xminn,yminn,xmaxn,ymaxn = utils.bounding_box_cpu(red_mask) - # xmax = xmaxn - expany + xmax - # xmin = xminn - expany + xmin - # ymin = yminn - expanx + ymin - # ymax = ymaxn - expanx + ymax - # red_mask = red_mask[yminn:ymaxn,xminn:xmaxn] - # red_image = red_image[yminn:ymaxn,xminn:xmaxn] - - contour = utils._get_polygons_in_bbox( - xmin, xmax, ymin, ymax, x1[i], y1[i], Birth[i], Death[i], red_mask, 0, 0 - ) - - # TEST # - # from matplotlib.pylab import plt - - # plt.imshow(red_image, cmap="gray", origin="lower") - # plt.imshow(red_mask, alpha=0.5, origin="lower") - # # plot the countour of the red_mask - # plt.plot(contour[:, 1] - xmin, contour[:, 0] - ymin, color="red") - # plt.show() - - # # # # - - source_props = utils.get_region_props(red_mask, image=red_image) - source_props = utils.props_to_dict(source_props[0]) - # print(background_map) - background_map = bg[i] / sigma # background_map - bg_rms = background_map - # print('Mean bg',mean_bg) - mean_bg_s = mean_bg[i] - - background_map = np.random.normal(mean_bg_s, background_map, red_mask.shape) - - red_background_mask = np.where(red_mask == 0, np.nan, red_mask * background_map) - - Noise = np.nansum(red_background_mask) - - # print('Noise',Noise) - peak_coords = np.where(red_image == source_props["max_intensity"]) - - y_peak_loc = peak_coords[0][0] - x_peak_loc = peak_coords[1][0] - # print('x_peak_loc',x_peak_loc) - # print('y_peak_loc',y_peak_loc) - shape = red_image.shape - - Area = source_props["area"] - # print('Class',Class[i]) - SNR = 0 # default value gets overwritten if calculated - if mode == "Radio" or mode == "optical": - shape = red_image.shape - # if shape is smaller than 100 in any direction then we add padding evenly to each side. - if shape[0] < 100: - pad = int((100 - shape[0]) / 2) - red_image = np.pad( - red_image, ((pad, pad), (0, 0)), mode="constant", constant_values=0 - ) - red_background_mask = np.pad( - red_background_mask, - ((pad, pad), (0, 0)), - mode="constant", - constant_values=0, - ) - red_mask = np.pad( - red_mask, ((pad, pad), (0, 0)), mode="constant", constant_values=0 - ) - shape = red_image.shape - y = y_peak_loc + pad - - else: - x = y_peak_loc - y = x_peak_loc - - if shape[1] < 100: - pad = int((100 - shape[1]) / 2) - red_image = np.pad( - red_image, ((0, 0), (pad, pad)), mode="constant", constant_values=0 - ) - red_background_mask = np.pad( - red_background_mask, - ((0, 0), (pad, pad)), - mode="constant", - constant_values=0, - ) - red_mask = np.pad( - red_mask, ((0, 0), (pad, pad)), mode="constant", constant_values=0 - ) - shape = red_image.shape - x = x_peak_loc + pad - - else: - x = y_peak_loc - y = x_peak_loc - - # print(shape) - - if mode == "optical": - - MAJ = psf_fwhm_p / 2.355 # maybe use full expression in future. - MIN = MAJ - BPA = 0 - - Model_Beam = utils.model_beam_func( - source_props["max_intensity"], shape, x, y, MAJ, MIN, BPA - ) - # plt.figure(figsize=(10,10)) - # plt.imshow(Model_Beam) - # plt.scatter(x,y,color='red',marker='x',s=10) - # plot the contour of the red_mask - # plt.contour(red_mask) - # plt.savefig('Model_Beam_test.png') - # pdb.set_trace() - - else: - BMAJ = BMAJ - BMIN = BMIN - - Model_Beam = utils.model_beam_func( - source_props["max_intensity"], shape, x, y, BMAJ / 2, BMIN / 2, BPA - ) - - if mode == "Radio": - Flux_total = ( - np.nansum(red_mask * red_image - red_background_mask) / Beam - ) - else: - - Flux_total = np.nansum(red_mask * red_image - red_background_mask) - - Flux_peak = ( - np.nanmax(red_mask * red_image) - - red_background_mask[y_peak_loc, x_peak_loc] - ) - - Flux_correction_factor = utils.flux_correction_factor(red_mask, Model_Beam) - Flux_total_err = 0 - SNR = 0 - if Area < 100: - # we assume that the flux cannot be corrected - Flux_total = Flux_total * Flux_correction_factor - - # Calcualte Radio Flux error - if mode == "Radio": - # adapted from https://github.com/mhardcastle/radioflux/blob/master/radioflux/radioflux.py - rms = bg[i] / sigma - gfactor = 2 * np.sqrt(2 * np.log(2)) - Beam_area = 2 * np.pi * (BMAJ * BMIN) / gfactor - - Flux_total_err = rms * np.sqrt(Area / Beam_area) - SNR = Flux_total / Flux_total_err - - elif mode == "optical": - SNR = 0 - - padding = 0 - - else: - - Flux_total = np.nansum(red_mask * red_image - red_background_mask) - Flux_peak = ( - np.nanmax(red_mask * red_image) - - red_background_mask[y_peak_loc, x_peak_loc] - ) - Flux_correction_factor = np.nan - padding = 0 - Flux_total_err = 0 - - # SNR = Flux_total / Flux_total_err - # Noise = np.std(red_background_mask) - - Xc = source_props["centroid"][1] + xmin - padding - Yc = source_props["centroid"][0] + ymin - padding - - Xc = Xc # + X0[i] - Yc = Yc # + Y0[i] - bbox1 = source_props["bbox"][0] + xmin - padding - bbox2 = source_props["bbox"][1] + ymin - padding - bbox3 = source_props["bbox"][2] + xmin - padding - bbox4 = source_props["bbox"][3] + ymin - padding - - Maj = source_props["major_axis_length"] - Min = source_props["minor_axis_length"] - Pa = source_props["orientation"] - - # print(Flux_total_err) - if Edge_flags[i] != 1: - - params.append( - [ - IDs[i], - Birth[i], - Death[i], - x1[i], - y1[i], - x2[i], - y2[i], - Flux_total, - Flux_total_err, - Flux_peak, # this should be the peak flux. - Flux_correction_factor, - Area, - Xc, - Yc, - bbox1, - bbox2, - bbox3, - bbox4, - Maj, - Min, - Pa, - parent_tag[i], - Class[i], - SNR, - Noise, - X0[i], - Y0[i], - mean_bg_s, - bg_rms, - Edge_flags[i], - contour, - enclosed_i[i], - ] - ) - - polygons.append(contour) - # except: - # print('Error in optical characteristing!') - # print('Source ID: ',IDs[i]) - # print('Skipping source...') - # continue - # print(params) - # print(len(params[0])) - - return create_params_df(False, params), polygons - - -def create_polygons(use_gpu, catalogue=None, cutout=None, output=None, cutupts=None): - - if use_gpu: - - try: - - import cupy as cp - - except: - - raise ImportError("cupy not installed. GPU acceleration not possible.") - - image = cutout - if use_gpu: - image_gpu = cp.asarray(image, dtype=cp.float64) - - Birth = catalogue["Birth"].to_numpy() - Death = catalogue["Death"].to_numpy() - - X0 = catalogue["X0_cutout"].to_numpy() - Y0 = catalogue["Y0_cutout"].to_numpy() - IDs = catalogue["ID"].to_numpy() - bbox1_og = catalogue["bbox1"].to_list() - bbox2_og = catalogue["bbox2"].to_list() - bbox3_og = catalogue["bbox3"].to_list() - bbox4_og = catalogue["bbox4"].to_list() - - x1 = catalogue["x1"].to_numpy() # - 1 #- X0 - y1 = catalogue["y1"].to_numpy() # - 1#- X0 - polygons = [] - for i, source in tqdm( - enumerate(Birth), - total=len(Birth), - desc="Creating contours..", - disable=not output, - ): - - if use_gpu == True: - - cropped_image_gpu = image_gpu[ - bbox1_og[i] - 1 : bbox3_og[i] + 1, bbox2_og[i] - 1 : bbox4_og[i] + 1 - ] - - try: - red_image, red_mask, xmin, xmax, ymin, ymax = ( - large_mask_red_image_procc_GPU( - Birth[i], - Death[i], - x1[i] - bbox1_og[i] + 1, - y1[i] - bbox2_og[i] + 1, - cropped_image_gpu, - X0[i], - Y0[i], - ) - ) - except: - - print("Error in GPU processing!") - print("Source ID: ", IDs[i]) - print("x1:", x1[i] - bbox1_og[i] + 1) - print("y1:", y1[i] - bbox2_og[i] + 1) - print("Birth:", Birth[i]) - print("Death:", Death[i]) - # print('Cropped_image',cropped_image_gpu.get()) - - red_mask = red_mask.astype(int) - - red_image = red_image.get() - red_mask = red_mask.get() - xmin = xmin.get() + bbox2_og[i] - xmax = xmax.get() + bbox2_og[i] - ymin = ymin.get() + bbox1_og[i] - ymax = ymax.get() + bbox1_og[i] - - else: - cropped_image = image[ - bbox1_og[i] - 1 : bbox3_og[i] + 1, bbox2_og[i] - 1 : bbox4_og[i] + 1 - ] - try: - red_image, red_mask, xmin, xmax, ymin, ymax = ( - large_mask_red_image_procc_CPU( - Birth[i], - Death[i], - int(x1[i]) - int(bbox1_og[i]) + 1, - int(y1[i]) - int(bbox2_og[i]) + 1, - cropped_image, - ) - ) - except: - print("Error in CPU processing!") - print("Source ID: ", IDs[i]) - print("x1:", x1[i] - bbox1_og[i] + 1) - print("y1:", y1[i] - bbox2_og[i] + 1) - print("Birth:", Birth[i]) - print("Death:", Death[i]) - - red_mask = red_mask.astype(int) - - xmin = xmin + bbox2_og[i] - xmax = xmax + bbox2_og[i] - ymin = ymin + bbox1_og[i] - ymax = ymax + bbox1_og[i] - - contour = utils._get_polygons_in_bbox( - xmin, xmax, ymin, ymax, x1[i], y1[i], Birth[i], Death[i], red_mask, 0, 0 - ) - polygons.append(contour) - - catalogue["contour"] = polygons - return catalogue diff --git a/DRUID/src/utils.py b/DRUID/src/utils.py index ba157d6..e69de29 100644 --- a/DRUID/src/utils.py +++ b/DRUID/src/utils.py @@ -1,586 +0,0 @@ -""" -File: utils.py -Author: Rhys Shaw -Date: 23/12/2023 -Version: v1.0 -Description: Utility functions for DRUID - -""" - -from astropy.io import fits -import numpy as np -from scipy.ndimage import gaussian_filter -from skimage import measure -from scipy.ndimage import label -from astropy.wcs import WCS - - -try: - import cupy as cp - from cupyx.scipy.ndimage import label as cupy_label - -except: - - pass - - -def open_image(PATH: str): - """ - - Function to open fits image and return image and header. - - - Args: - PATH (str): Path to the fits image. - - Returns: - image (np.ndarray): The image data. - header (astropy.io.fits.header.Header): The image header. - - - """ - - hdul = fits.open(PATH) - image = hdul[0].data - header = hdul[0].header - hdul.close() - - image_shape = image.shape - - if len(image_shape) == 4: - - image = np.squeeze(image, axis=(0, 1)) - - if len(image_shape) == 3: - - image = np.squeeze(image, axis=(0)) - - return image, header - - -def get_psf_FWHM(header: fits.header.Header): - """Return the psf FWHM in pixels. - - Args: - header (fits.header.Header): _description_ - - Returns: - _type_: _description_ - """ - psf_FWHM_arcsec = header["PSF_FWHM"] - ## convert to pixels assumes square pixels and uniform pixel scale - arcseconds_per_pixel = abs(header["CD1_1"]) * 3600 * -1 - psf_FWHM_pixels = psf_FWHM_arcsec / arcseconds_per_pixel - - return psf_FWHM_pixels - - -def get_EFFGAIN(header: fits.header.Header): - """Return the effective gain of the image. - - Args: - header (fits.header.Header): _description_ - - Returns: - _type_: _description_ - """ - EFFGAIN = header["EFFGAIN"] - - return EFFGAIN - - -def get_EFFRON(header: fits.header.Header): - """Return the effective read noise of the image. - - Args: - header (fits.header.Header): _description_ - - Returns: - _type_: _description_ - """ - EFFRON = header["EFFRON"] - - return EFFRON - - -def get_EXPTIME(header: fits.header.Header): - """Return the exposure time of the image. - - Args: - header (fits.header.Header): _description_ - - Returns: - _type_: _description_ - """ - EXPTIME = header["EXPTIME"] - - return EXPTIME - - -def cut_image(size: int, image: np.ndarray): - """Cuts an image into smaller images of the specified size (square). - - Args: - size (int): The size of the cutouts. - image (np.ndarray): The image to be cut. - - Returns: - cutouts (list): A list of the cutout images. - coords (list): A list of the coordinates of the cutouts. - - """ - - cutouts = [] - coords = [] - - for i in range(0, image.shape[0], size): - - for j in range(0, image.shape[1], size): - - cutout = image[i : i + size, j : j + size] - - cutouts.append(cutout) - - coords.append([i, j]) - - return cutouts, coords - - -import numpy as np - - -def cut_image_buff(image, cutout_size, buffer_size): - """ - Cuts an image into smaller images of the specified size (square) with optional overlap. - - Args: - image (np.ndarray): The image to be cut. - size (int): The size of the cutouts.s - buffer (int): The size of the buffer for overlap. - - Returns: - cutouts (list): A list of the cutout images. - coords (list): A list of the coordinates of the cutouts. - """ - - # Initialize lists to store cut-up images and their locations - height, width = image.shape - cutup_images = [] - cutup_locations = [] - - # Iterate through the image with the specified cutout size and buffer - for y in range(0, height, cutout_size - buffer_size): - for x in range(0, width, cutout_size - buffer_size): - # coordinates for the current cutout - left = max(0, x - buffer_size) - upper = max(0, y - buffer_size) - right = min(width, x + cutout_size + buffer_size) - lower = min(height, y + cutout_size + buffer_size) - - # Crop the image array to get the cutout - cutout = image[upper:lower, left:right] - - # Append the cutout and its location to the lists - cutup_images.append(cutout) - cutup_locations.append([upper, left]) - - return cutup_images, cutup_locations - - -def remove_duplicates(catalogue): - """ - - Removes duplicate sources from the catalogue. - - Args: - catalogue (pd.DataFrame): The catalogue to be cleaned. - - Returns: - catalogue (pd.DataFrame): The cleaned catalogue. - - """ - # sort by area - catalogue = catalogue.sort_values(by=["distance_from_center"], ascending=False) - catalogue = catalogue.drop_duplicates( - subset=["x1", "y1", "Birth", "Class"], keep="first" - ) - # sort by ID - # sort by ID - # choose the row with the highest area - - return catalogue - - -def smoothing(image: np.ndarray, sigma: float): - """Smooths an image using a gaussian filter. - - Args: - image (np.ndarray): The image to be smoothed. - sigma (float): The sigma value for the gaussian filter. - - Returns: - smoothed_image (np.ndarray): The smoothed image. - - """ - - smoothed_image = gaussian_filter(image, sigma=sigma) - - return smoothed_image - - -def calculate_beam(header: fits.header.Header): - """ - - Calculates the beam of the image in pixels. - - Args: - header (astropy.io.fits.header.Header): The header of the image. - - returns: - beam (float): The beam correction factor. - BMAJ (float): The BMAJ value in pixels. - BMIN (float): The BMIN value in pixels. - BPA (float): The BPA value in pixels. - - """ - - # get beam info from header - - BMAJ = header["BMAJ"] - BMIN = header["BMIN"] - # convert to pixels - arcseconds_per_pixel = header["CDELT1"] * 3600 * -1 - beam_size_arcseconds = header["BMAJ"] * 3600 - BMAJ_oversampled_spacial_width = (BMAJ**2 + beam_size_arcseconds**2) ** 0.5 - BMAJ = BMAJ_oversampled_spacial_width / arcseconds_per_pixel - - beam_size_arcseconds = header["BMIN"] * 3600 - BMIN_oversampled_spacial_width = (BMIN**2 + beam_size_arcseconds**2) ** 0.5 - BMIN = BMIN_oversampled_spacial_width / arcseconds_per_pixel - - try: - BPA = header["BPA"] - - except KeyError: - BPA = 0 - - return np.pi * (BMAJ) * (BMIN) / (4 * np.log(2)), BMAJ, BMIN, BPA - - -def check_edge(mask): - - return 1 in mask[0, :] or 1 in mask[-1, :] or 1 in mask[:, 0] or 1 in mask[:, -1] - - -def props_to_dict(regionprops): - """ - - Converts the regionprops to a dictionary. - # slow function when all are called. - """ - - dict = { - "area": regionprops.area, - "bbox": regionprops.bbox, - #'bbox_area': regionprops.bbox_area, - "centroid": regionprops.centroid, - #'convex_area': regionprops.convex_area, - "eccentricity": regionprops.eccentricity, - #'equivalent_diameter': regionprops.equivalent_diameter, - #'euler_number': regionprops.euler_number, - #'extent': regionprops.extent, - #'filled_area': regionprops.filled_area, - "major_axis_length": regionprops.major_axis_length, - "minor_axis_length": regionprops.minor_axis_length, - #'moments': regionprops.moments, - #'perimeter': regionprops.perimeter, - #'solidity': regionprops.solidity, - "orientation": regionprops.orientation, - "max_intensity": regionprops.max_intensity, - } - - return dict - - -def bounding_box_cpu(mask): - # Get the indices of elements that are True - rows, cols = np.where(mask) - # Get the minimum and maximum x and y coordinates - min_y, max_y = np.min(rows), np.max(rows) - min_x, max_x = np.min(cols), np.max(cols) - # Return the bounding box as a tuple of tuples - return min_y, min_x, max_y, max_x - - -def bounding_box_gpu(binary_mask_gpu): - """ - Calculate the bounding box of a binary mask using cupy. - - Parameters: - - binary_mask_gpu: cupy array, binary mask. - - Returns: - - tuple: (min_row, min_col, max_row, max_col), representing the bounding box. - """ - - # Get the indices of elements that are True - - rows, cols = cp.where(binary_mask_gpu) - - # Get the minimum and maximum x and y coordinates - min_y, max_y = cp.min(rows), cp.max(rows) - min_x, max_x = cp.min(cols), cp.max(cols) - - # Return the bounding box as a tuple of tuples - return min_y, min_x, max_y, max_x - - -def get_region_props(mask, image): - - region = measure.regionprops(mask, image) - - return region - - -def model_beam_func(peak_flux, shape, x, y, bmaj, bmin, bpa): - model_beam = np.zeros(shape) - model_beam = generate_2d_gaussian( - peak_flux, shape, (x, y), bmaj, bmin, bpa, norm=False - ) - return model_beam - - -def flux_correction_factor(mask, Model_Beam): - - # calculate the correction factor - - model_beam_flux = np.sum(Model_Beam) - masked_beam_flux = np.sum(mask * Model_Beam) - - correction_factor = model_beam_flux / masked_beam_flux - - return correction_factor - - -def generate_2d_gaussian(A, shape, center, sigma_x, sigma_y, angle_deg=0, norm=True): - """ - - Generate a 2D elliptical Gaussian distribution on a 2D array. - - Parameters: - - shape (tuple): Shape of the output array (height, width). - center (tuple): Center of the Gaussian distribution (x, y). - sigma_x (float): Standard deviation along the x-axis. - sigma_y (float): Standard deviation along the y-axis. - angle_deg (float): Rotation angle in degrees (default is 0). - - Returns: - - ndarray: 2D array containing the Gaussian distribution. - - """ - x, y = np.meshgrid(np.arange(shape[1]), np.arange(shape[0])) - x_c, y_c = center - angle_rad = np.radians(angle_deg) - - # Rotate coordinates - - x_rot = (x - x_c) * np.cos(angle_rad) - (y - y_c) * np.sin(angle_rad) - y_rot = (x - x_c) * np.sin(angle_rad) + (y - y_c) * np.cos(angle_rad) - - # Calculate Gaussian values - - gaussian = A * np.exp(-(x_rot**2 / (2 * sigma_x**2) + y_rot**2 / (2 * sigma_y**2))) - - if norm: - return gaussian / (2 * np.pi * sigma_x * sigma_y) - else: - return gaussian - - -def get_enclosing_mask_gpu(x, y, mask): - """Returns the connected components inside the mask starting from the point (x, y) using the GPU. - - - Args: - x (int): x location of the source in the mask. - y (int): y location of the source in the mask. - mask (cp.ndarray): unfiltered mask of source (on GPU memory). - - Returns: - component_mask(np.ndarray): Mask for the source in the provided coordinates. - """ - - labeled_mask, num_features = cupy_label(mask) - - # Check if the specified pixel is within the mask - if 0 <= x < mask.shape[1] and 0 <= y < mask.shape[0]: - label_at_pixel = labeled_mask[y, x] - - if label_at_pixel != 0: - # Extract the connected component containing the specified pixel - component_mask = labeled_mask == label_at_pixel - return component_mask # still in GPU memory - else: - return None - else: - return None - - -def get_mask_CPU(x1, y1, Birth, Death, img): - """Get mask for a single row uses the CPU - - Args: - row (pd.Series): row we want to get the mask for. - img (np.ndarray): image that the source is in. - - Returns: - mask_enclosed(np.ndarray): Array of the mask - - """ - - mask = np.zeros(img.shape) - mask = np.logical_or(mask, np.logical_and(img <= Birth, img > Death)) - mask_enclosed = get_enclosing_mask_CPU(int(y1), int(x1), mask) - - return mask_enclosed - - -def get_enclosing_mask_CPU(x, y, mask): - """ - Returns the connected components inside the mask starting from the point (x, y). - """ - labeled_mask, num_features = label(mask) - - # Check if the specified pixel is within the mask - if 0 <= x < mask.shape[1] and 0 <= y < mask.shape[0]: - label_at_pixel = labeled_mask[y, x] - - if label_at_pixel != 0: - # Extract the connected component containing the specified pixel - component_mask = labeled_mask == label_at_pixel - return component_mask - else: - # The specified pixel is not part of any connected component - return None - else: - # The specified pixel is outside the mask - return None - - -def get_mask_GPU(Birth, Death, x1, y1, img): - """Gets mask for a single row using the GPU (requires cupy) - - Args: - Birth (float): Birth value of the row of interest. - Death (float): Death value of the row of interest. - row (pd.series): row of interest. - img (cp.ndarray): Image containing the source (in the GPUs memory) - - Returns: - mask_enclosed (np.ndarray): The mask returned in normal memory. - - """ - - mask = cp.zeros(img.shape, dtype=cp.float64) - mask = cp.logical_or(mask, cp.logical_and(img <= Birth, img > Death)) - mask_enclosed = get_enclosing_mask_gpu(int(y1), int(x1), mask) - - return mask_enclosed - - -def _get_polygons_gpu(x1: int, y1: int, birth: float, death: float, image): - """ - Returns the polygon of the enclosed area of the point (x,y) in the mask. - """ - # is the image on the GPU memory? - # if not cp.is_cuda_array(self.image): - # self.image = cp.asarray(self.image, dtype=cp.float64) - - mask = cp.zeros(image.shape) - mask = cp.logical_or(mask, cp.logical_and(image <= birth, image > death)) - mask = get_enclosing_mask_GPU(int(y1), int(x1), mask) - contour = measure.find_contours(mask, 0)[0] - - return contour - - -def _get_polygons_CPU(x1, y1, birth, death, image: np.ndarray): - """ - Returns the polygon of the enclosed area of the point (x,y) in the mask. - """ - - mask = np.zeros(image.shape) - mask = np.logical_or(mask, np.logical_and(image <= birth, image > death)) - mask = get_enclosing_mask_CPU(int(y1), int(x1), mask) - contour = measure.find_contours(mask, 0)[0] - - return contour - - -import matplotlib.pyplot as plt - - -def _get_polygons_in_bbox( - Xmin, Xmax, Ymin, Ymax, x1, y1, birth, death, mask, X0, Y0, pad=1 -): - - mask = np.pad(mask, pad, mode="constant", constant_values=0) - # print(mask) - # try: - contour = measure.find_contours(mask, 0)[0] - # except: - # plt.imshow(mask) - # plt.savefig('mask.png') - contour = measure.find_contours(mask, 0)[0] - # print(contour) - # remove the border - contour[:, 0] -= pad - contour[:, 1] -= pad - - # correct the coordinates to the original image - contour[:, 0] += Ymin + Y0 - contour[:, 1] += Xmin + X0 - - return contour - - -def xy_to_RaDec(x, y, header, mode): - """ - - Convert an X and Y coordinate to RA and Dec using a header file with astropy. - - Parameters: - x (float): The X coordinate. - y (float): The Y coordinate. - header_file (str): The path to the FITS header file. - stokes (int): The Stokes dimension. - freq (int): The frequency dimension. - - Returns: - tuple: A tuple containing the RA and Dec in degrees. - - """ - wcs = WCS(header) - print("-----------------") - print("Mode :", mode) - if mode == "Radio": - - stokes = 0 # stokes and freq are not used in this function. - freq = 0 # stokes and freq are not used in this function. - ra, dec, _, _ = wcs.all_pix2world(x, y, stokes, freq, 0) - - elif mode == "optical": - print("Optical") - # try: - # image is 2d so no stokes or freq - ra, dec = wcs.all_pix2world(x, y, 0) - - # except: - # try radio - # stokes = 0 - # freq = 0 - # ra, dec, _, _ = wcs.all_pix2world(x, y, stokes, freq, 0) - - return ra, dec diff --git a/Examples/Resolved_Galaxies.ipynb b/Examples/Resolved_Galaxies.ipynb index f3f6837..1f49d76 100644 --- a/Examples/Resolved_Galaxies.ipynb +++ b/Examples/Resolved_Galaxies.ipynb @@ -101,7 +101,7 @@ ], "metadata": { "kernelspec": { - "display_name": "DRUID", + "display_name": "base", "language": "python", "name": "python3" }, @@ -115,7 +115,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.13" + "version": "3.12.7" } }, "nbformat": 4, diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..a432ffd --- /dev/null +++ b/environment.yml @@ -0,0 +1,20 @@ +name: DRUID +channels: + - conda-forge + - defaults +dependencies: + - python=3.12 + - pip + - pip: + - numpy + - pandas + - pytest + - numpy + - scikit-image + - astropy + - tqdm + - matplotlib + - setproctitle + - scipy + - photutils + - cripser \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index b36f66b..77faf8b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,4 +9,4 @@ matplotlib setproctitle scipy photutils -cripser +cripser \ No newline at end of file From 94ed475dbcdd1e26ab8e71af73336be8477fb32b Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Sun, 1 Jun 2025 15:22:23 +0100 Subject: [PATCH 02/69] Improved Background map calculating and management --- DRUID/src/background.py | 288 ++++++++++++++++++++++++++++++++++++++++ environment.yml | 3 +- 2 files changed, 290 insertions(+), 1 deletion(-) diff --git a/DRUID/src/background.py b/DRUID/src/background.py index e69de29..1705cef 100644 --- a/DRUID/src/background.py +++ b/DRUID/src/background.py @@ -0,0 +1,288 @@ +import numpy as np +from astropy.io import fits +from astropy.stats import sigma_clipped_stats +from photutils.background import ( + Background2D, + MedianBackground, + StdBackgroundRMS, + MADStdBackgroundRMS, + BiweightLocationBackground, + BiweightScaleBackgroundRMS, + MMMBackground, + MeanBackground, + ModeEstimatorBackground, + SExtractorBackground, + BackgroundBase, +) + +from photutils.segmentation import detect_sources + + +def make_source_mask(data, nsigma=3.0, kernel_size=3): + """ + Create a mask for sources in the image data using sigma clipping. + + Parameters + ---------- + data : numpy.ndarray + The 2D image data. + nsigma : float, optional + The number of standard deviations to use for sigma clipping. + The default is 3.0. + kernel_size : int, optional + The size of the convolution kernel for source detection. + The default is 3. + + Returns + ------- + mask : numpy.ndarray + A boolean mask where True indicates a source pixel. + """ + mean, median, std = sigma_clipped_stats(data, sigma=nsigma) + threshold = median + nsigma * std + + # Detect sources using a simple thresholding method can add masked pixels e.g. known bad areas of image. + segm = detect_sources(data, threshold, npixels=kernel_size**2) + + # Create a mask from the segmentation map + mask = segm.data > 0 + + return mask + + +def calculate_background_maps( + image_path, + bg_estimator="median", + box_size=(50, 50), + filter_size=(3, 3), + nsigma=3.0, + kernel_size=3, +): + """ + Calculates background and background RMS maps from a FITS image + similar to the style of PyBDSF. + + Parameters + ---------- + image_path : str + Path to the FITS image file. + box_size : tuple of int, optional + The size of the box to use for background estimation. + The default is (50, 50). + filter_size : tuple of int, optional + The size of the median filter to apply to the background map. + The default is (3, 3). + nsigma : float, optional + The number of standard deviations to use for sigma clipping + when detecting sources to mask. The default is 3.0. + bg_estimator : str or photutils.background.BackgroundBase, optional + The background estimator to use. Options are 'median', 'std', 'mad_std', + or a custom photutils background estimator object. The default is 'median'. + kernel_size : int, optional + The size of the convolution kernel for source detection. + The default is 3. + + Returns + ------- + background_map : numpy.ndarray + The calculated background map. + background_rms_map : numpy.ndarray + The calculated background RMS map. + """ + with fits.open(image_path) as hdul: + data = hdul[0].data + + # Mask sources + mask = make_source_mask(data, nsigma=nsigma, kernel_size=kernel_size) + + # calculate background and RMS Avalible background estimators + available_estimators = { + "median": MedianBackground, + "std": StdBackgroundRMS, + "mad_std": MADStdBackgroundRMS, + "rms": StdBackgroundRMS, + "BiweightLocation": BiweightLocationBackground, + "BiweightScale": BiweightScaleBackgroundRMS, + "MM": MMMBackground, + "Mean": MeanBackground, + "mode": ModeEstimatorBackground, + "SEx": SExtractorBackground, + } + + if isinstance(bg_estimator, str): + bkg_estimator = available_estimators.get( + bg_estimator.lower(), MedianBackground + )() + elif isinstance(bg_estimator, BackgroundBase): + bkg_estimator = bg_estimator + else: + bkg_estimator = MedianBackground() + + bkg = Background2D( + data, + box_size, + filter_size=filter_size, + mask=mask, + bkg_estimator=bkg_estimator, + ) + + return bkg.background, bkg.background_rms + + +def make_gaussian_sources_image(image_size, sources): + """ + Create a 2D image with Gaussian sources. + Parameters + ---------- + image_size : tuple of int + Size of the image (height, width). + sources : list of dict + List of sources, each defined by a dictionary with keys: + 'amplitude', 'x_mean', 'y_mean', 'x_stddev', 'y_stddev', 'theta'. + Returns + ------- + numpy.ndarray + 2D array representing the image with Gaussian sources. + """ + image = np.zeros(image_size) + y, x = np.indices(image_size) + for source in sources: + amplitude = source["amplitude"] + x_mean = source["x_mean"] + y_mean = source["y_mean"] + x_stddev = source["x_stddev"] + y_stddev = source["y_stddev"] + theta = source["theta"] + a = (np.cos(theta) ** 2) / (2 * x_stddev**2) + (np.sin(theta) ** 2) / ( + 2 * y_stddev**2 + ) + b = -np.sin(2 * theta) / (4 * x_stddev**2) + np.sin(2 * theta) / ( + 4 * y_stddev**2 + ) + c = (np.sin(theta) ** 2) / (2 * x_stddev**2) + (np.cos(theta) ** 2) / ( + 2 * y_stddev**2 + ) + gaussian = amplitude * np.exp( + -( + a * (x - x_mean) ** 2 + + 2 * b * (x - x_mean) * (y - y_mean) + + c * (y - y_mean) ** 2 + ) + ) + image += gaussian + return image + + +if __name__ == "__main__": + # Example usage: + # Create a dummy FITS file for demonstration + from astropy.wcs import WCS + from astropy.coordinates import SkyCoord + + # Define image parameters + image_size = (1000, 1000) + pixel_scale = 0.1 # degrees per pixel + center_coord = SkyCoord(ra=180, dec=30, unit="deg") + + # Create a dummy WCS + wcs = WCS(naxis=2) + wcs.wcs.crpix = [image_size[0] / 2, image_size[1] / 2] + wcs.wcs.cdelt = np.array([-pixel_scale, pixel_scale]) + wcs.wcs.crval = [center_coord.ra.deg, center_coord.dec.deg] + wcs.wcs.ctype = ["RA---TAN", "DEC--TAN"] + + # Create dummy sources + sources = [ + { + "amplitude": 100, + "x_mean": 500, + "y_mean": 500, + "x_stddev": 50, + "y_stddev": 50, + "theta": 0, + }, + { + "amplitude": 150, + "x_mean": 150, + "y_mean": 150, + "x_stddev": 7, + "y_stddev": 7, + "theta": np.pi / 4, + }, + { + "amplitude": 200, + "x_mean": 300, + "y_mean": 300, + "x_stddev": 10, + "y_stddev": 10, + "theta": np.pi / 2, + }, + { + "amplitude": 80, + "x_mean": 700, + "y_mean": 800, + "x_stddev": 6, + "y_stddev": 6, + "theta": np.pi / 3, + }, + { + "amplitude": 120, + "x_mean": 900, + "y_mean": 200, + "x_stddev": 8, + "y_stddev": 8, + "theta": np.pi / 6, + }, + { + "amplitude": 90, + "x_mean": 400, + "y_mean": 600, + "x_stddev": 4, + "y_stddev": 4, + "theta": np.pi / 8, + }, + ] + + # Create a dummy image with sources and background noise + dummy_data = make_gaussian_sources_image(image_size, sources) + dummy_data += np.random.normal(0, 5, size=image_size) # Add some noise + + # Create a dummy FITS file + hdu = fits.PrimaryHDU(dummy_data, header=wcs.to_header()) + dummy_fits_path = "DRUID/temp/dummy_image.fits" + hdu.writeto(dummy_fits_path, overwrite=True) + + print(f"Dummy FITS file created at: {dummy_fits_path}") + + # Calculate background maps + background_map, background_rms_map = calculate_background_maps(dummy_fits_path) + # save the background maps to a FITS file + background_hdu = fits.PrimaryHDU(background_map) + background_rms_hdu = fits.PrimaryHDU(background_rms_map) + background_hdu.writeto("DRUID/temp/background_map.fits", overwrite=True) + background_rms_hdu.writeto("DRUID/temp/background_rms_map.fits", overwrite=True) + + print("Background map and background RMS map saved to FITS files.") + print("Background map calculated.") + print("Background RMS map calculated.") + + # You can optionally save the background maps to + # plot the results with matplotlib or any other visualization library. + from matplotlib import pyplot as plt + + # plot dummy data, background map, and background RMS map + plt.figure(figsize=(12, 6)) + plt.subplot(1, 3, 1) + plt.imshow(dummy_data, origin="lower", cmap="gray", interpolation="nearest") + plt.title("Dummy Image with Sources") + plt.colorbar() + plt.subplot(1, 3, 2) + plt.imshow(background_map, origin="lower", cmap="gray", interpolation="nearest") + plt.title("Background Map") + plt.colorbar() + plt.subplot(1, 3, 3) + plt.imshow(background_rms_map, origin="lower", cmap="gray", interpolation="nearest") + plt.title("Background RMS Map") + plt.colorbar() + plt.tight_layout() + plt.show() diff --git a/environment.yml b/environment.yml index a432ffd..be41c32 100644 --- a/environment.yml +++ b/environment.yml @@ -17,4 +17,5 @@ dependencies: - setproctitle - scipy - photutils - - cripser \ No newline at end of file + - cripser + - bottleneck \ No newline at end of file From 5819f61e7fa8af3663e0ff87664ce92167ee4dea Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Mon, 9 Jun 2025 11:37:15 +0100 Subject: [PATCH 03/69] starting homology section --- DRUID/src/homology.py | 202 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 202 insertions(+) diff --git a/DRUID/src/homology.py b/DRUID/src/homology.py index e69de29..04ce3f0 100644 --- a/DRUID/src/homology.py +++ b/DRUID/src/homology.py @@ -0,0 +1,202 @@ +import cripser +import numpy as np +import polars as pl + +from tqdm import tqdm + + +def get_enclosing_mask_CPU(x, y, mask): + """ + Returns the connected components inside the mask starting from the point (x, y). + """ + labeled_mask, num_features = label(mask) + + # Check if the specified pixel is within the mask + if 0 <= x < mask.shape[1] and 0 <= y < mask.shape[0]: + label_at_pixel = labeled_mask[y, x] + + if label_at_pixel != 0: + # Extract the connected component containing the specified pixel + component_mask = labeled_mask == label_at_pixel + return component_mask + else: + # The specified pixel is not part of any connected component + return None + else: + # The specified pixel is outside the mask + return None + + +def bounding_box_cpu(mask): + # Get the indices of elements that are True + rows, cols = np.where(mask) + # Get the minimum and maximum x and y coordinates + min_y, max_y = np.min(rows), np.max(rows) + min_x, max_x = np.min(cols), np.max(cols) + # Return the bounding box as a tuple of tuples + return min_y, min_x, max_y, max_x + + +def get_mask_CPU(x1, y1, Birth, Death, img): + """Get mask for a single row uses the CPU + + Args: + row (pd.Series): row we want to get the mask for. + img (np.ndarray): image that the source is in. + + Returns: + mask_enclosed(np.ndarray): Array of the mask + + """ + + mask = np.zeros(img.shape) + mask = np.logical_or(mask, np.logical_and(img <= Birth, img > Death)) + mask_enclosed = get_enclosing_mask_CPU(int(y1), int(x1), mask) + + return mask_enclosed + + +def compute_ph_components( + threholded_image, + local_bg, + analysis_threshold_val, +): + print("Computing persistent homology components...") + + +if __name__ == "__main__": + import pandas as pd + import matplotlib.pyplot as plt + from astropy.io import fits + + print("DRUID - Homology.py test script") + # get example tresholded image. + dummy_data_path = "DRUID/temp/dummy_image.fits" + + background_map_path = "DRUID/temp/background_map.fits" + background_rms_map_path = "DRUID/temp/background_rms_map.fits" + + # load the images + dummy_data = fits.open(dummy_data_path)[0].data + background_map = fits.open(background_map_path)[0].data + background_rms_map = fits.open(background_rms_map_path)[0].data + + # set anything in the mask to 0 + thresholded_image = np.where( + dummy_data > background_map + 10 * background_rms_map, dummy_data, 0 + ) + + # sort by area and remove components smaller than 5 pixels + + from skimage.measure import regionprops + from skimage.measure import label + + labeled_image = label(thresholded_image > 0, connectivity=2) + properties = regionprops(labeled_image, intensity_image=thresholded_image) + + # filter out components smaller than 5 pixels + min_area = 5 + filtered_labels = [prop.label for prop in properties if prop.area >= min_area] + + # create a new labeled image with only the filtered labels + filtered_labeled_image = np.zeros_like(labeled_image) + for label_value in filtered_labels: + filtered_labeled_image[labeled_image == label_value] = label_value + + # use the filtered labeled image for further processing + labeled_image = filtered_labeled_image + + # for each label crop around it. + unique_labels = np.unique(labeled_image) + components = [] + for label_value in tqdm(unique_labels): + if label_value == 0: + continue # Skip the background label + component_mask = labeled_image == label_value + component = np.where(component_mask, thresholded_image, 0) + # crop around the component + y_indices, x_indices = np.where(component_mask) + + if len(x_indices) == 0 or len(y_indices) == 0: + continue + + x_min, x_max = np.min(x_indices), np.max(x_indices) + y_min, y_max = np.min(y_indices), np.max(y_indices) + component = component[y_min : y_max + 1, x_min : x_max + 1] + components.append(component) + + print(f"Found {len(components)} components.") + # plt.figure(figsize=(8, 6)) + # plt.imshow( + # components[0], origin="lower", cmap="nipy_spectral", interpolation="nearest" + # ) + # plt.title("Labeled Image") + # plt.colorbar() + # plt.show() + import time + + img = components[0] + t0_compute_ph = time.time() + pd = cripser.computePH(-img, maxdim=0) + t1_compute_ph = time.time() + print( + f"Computed persistent homology in {t1_compute_ph - t0_compute_ph:.2f} seconds." + ) + # create polar dataframe to handle the data + + columns = ["dim", "birth", "death", "x1", "y1", "z1", "x2", "y2", "z2"] + polar_df = pl.DataFrame(pd, schema=columns) + # drop cols dim, z1, z2 + polar_df = polar_df.drop(["dim", "z1", "z2"]) + # create ne column lifetime death - birth + polar_df = polar_df.with_columns( + (polar_df["death"] - polar_df["birth"]).alias("lifetime") + ) + # make column birth and death - birth and death. + polar_df = polar_df.with_columns( + [(-polar_df["birth"]).alias("birth"), (-polar_df["death"]).alias("death")] + ) + + # lifetime_threshold. this is setby the user. + + polar_df = polar_df.with_columns( + (polar_df["birth"] - polar_df["death"]).alias("lifetimeFrac") + ) + print(len(polar_df)) + liftetime_limit_fraction = 3.0 # set the lifetime limit fraction + # filter out components with lifetime less than 3 + polar_df = polar_df.filter(polar_df["lifetime"] > liftetime_limit_fraction) + print( + f"Filtered polar dataframe to {len(polar_df)} components with lifetime > {liftetime_limit_fraction}." + ) + print(polar_df) + + # filter by pixel size. get bounding box of the component, and contour?. + + # for each of the components compute the area left between the birth and death. + + # compute the area of the component + + for row in polar_df.iter_rows(): + # get the component + index = row.index + component = components[0] + + # get mask of the component using brith and death values. + birth = row["birth"] + death = row["death"] + + mask = get_mask_CPU( + row["x1"], + row["y1"], + birth, + death, + component, + ) + bounding_box = bounding_box_cpu(mask) + + # compute the area of the component + area = np.sum(mask) + polar_df.at[index, "area"] = area + + print(polar_df) From de89806a84519a2f478eb5df746f627ff2dae7cf Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Mon, 9 Jun 2025 11:57:14 +0100 Subject: [PATCH 04/69] ph on smaller component in principle --- DRUID/src/homology.py | 107 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 88 insertions(+), 19 deletions(-) diff --git a/DRUID/src/homology.py b/DRUID/src/homology.py index 04ce3f0..8ad89d1 100644 --- a/DRUID/src/homology.py +++ b/DRUID/src/homology.py @@ -1,7 +1,7 @@ import cripser import numpy as np import polars as pl - +from scipy.ndimage import label as scipy_label from tqdm import tqdm @@ -9,7 +9,9 @@ def get_enclosing_mask_CPU(x, y, mask): """ Returns the connected components inside the mask starting from the point (x, y). """ - labeled_mask, num_features = label(mask) + from skimage.measure import label + + labeled_mask, num_features = scipy_label(mask) # Check if the specified pixel is within the mask if 0 <= x < mask.shape[1] and 0 <= y < mask.shape[0]: @@ -135,7 +137,7 @@ def compute_ph_components( # plt.show() import time - img = components[0] + img = components[1] t0_compute_ph = time.time() pd = cripser.computePH(-img, maxdim=0) t1_compute_ph = time.time() @@ -176,27 +178,94 @@ def compute_ph_components( # for each of the components compute the area left between the birth and death. # compute the area of the component + # for each of the components compute the area left between the birth and death. - for row in polar_df.iter_rows(): - # get the component - index = row.index - component = components[0] - - # get mask of the component using brith and death values. - birth = row["birth"] - death = row["death"] + # compute the area of the component + areas = [] + bbox_min_y_list = [] + bbox_min_x_list = [] + bbox_max_y_list = [] + bbox_max_x_list = [] + + # Assuming 'components[0]' is the correct component for all rows in polar_df + # If each row corresponds to a different component, you'll need to adjust this. + # For now, let's stick to the logic in your snippet. + component_img = components[0] + + for row_tuple in polar_df.iter_rows( + named=True + ): # named=True gives you a dictionary per row + # get mask of the component using birth and death values. + birth = row_tuple["birth"] + death = row_tuple["death"] + x1 = row_tuple["x1"] + y1 = row_tuple["y1"] mask = get_mask_CPU( - row["x1"], - row["y1"], + x1, # Note: Your get_mask_CPU expects x1, y1, Birth, Death, img + y1, birth, death, - component, + component_img, # Use the pre-selected component ) - bounding_box = bounding_box_cpu(mask) - # compute the area of the component - area = np.sum(mask) - polar_df.at[index, "area"] = area + if mask is not None: + bounding_box = bounding_box_cpu(mask) + area = np.sum(mask) - print(polar_df) + areas.append(area) + bbox_min_y_list.append(bounding_box[0]) + bbox_min_x_list.append(bounding_box[1]) + bbox_max_y_list.append(bounding_box[2]) + bbox_max_x_list.append(bounding_box[3]) + else: + # Handle cases where mask is None (e.g., point outside, no component) + # Append NaN or a placeholder, or filter these rows out later + areas.append(np.nan) + bbox_min_y_list.append(np.nan) + bbox_min_x_list.append(np.nan) + bbox_max_y_list.append(np.nan) + bbox_max_x_list.append(np.nan) + + # Add the new columns to the DataFrame + polar_df = polar_df.with_columns( + [ + pl.Series("area", areas), + pl.Series("bbox_min_y", bbox_min_y_list), + pl.Series("bbox_min_x", bbox_min_x_list), + pl.Series("bbox_max_y", bbox_max_y_list), + pl.Series("bbox_max_x", bbox_max_x_list), + ] + ) + + # remove those with area < 5 pixels + polar_df = polar_df.filter(polar_df["area"] > 5) + + # plot the components with bounding boxes + plt.figure(figsize=(10, 8)) + plt.imshow( + component_img, origin="lower", cmap="nipy_spectral", interpolation="nearest" + ) + for row_tuple in polar_df.iter_rows(named=True): + + bbox_min_y = row_tuple["bbox_min_y"] + bbox_min_x = row_tuple["bbox_min_x"] + bbox_max_y = row_tuple["bbox_max_y"] + bbox_max_x = row_tuple["bbox_max_x"] + + if not np.isnan(bbox_min_y) and not np.isnan(bbox_min_x): + # Draw the bounding box + plt.gca().add_patch( + plt.Rectangle( + (bbox_min_x, bbox_min_y), + bbox_max_x - bbox_min_x, + bbox_max_y - bbox_min_y, + edgecolor="blue", + facecolor="none", + linewidth=2, + ) + ) + + plt.title("Component with Bounding Boxes") + plt.colorbar() + plt.show() From 694d4786ca8a9398ca031bf1faac91e121c6d295 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Wed, 11 Jun 2025 11:04:20 +0100 Subject: [PATCH 05/69] working progress --- DRUID/main.py | 33 ++++++ DRUID/src/background.py | 92 ++++++-------- DRUID/src/homology.py | 257 ++++++++++++++++++++++++++++++++++++---- 3 files changed, 306 insertions(+), 76 deletions(-) diff --git a/DRUID/main.py b/DRUID/main.py index e69de29..a168a87 100644 --- a/DRUID/main.py +++ b/DRUID/main.py @@ -0,0 +1,33 @@ +version = "1.0" + +import setproctitle + +setproctitle.setproctitle("DRUID") + + +DRUID_MESSAGE = """ + + +############################################# + +_______ _______ _________ ______ +( __ \ ( ____ )|\ /|\__ __/( __ \ +| ( \ )| ( )|| ) ( | ) ( | ( \ ) +| | ) || (____)|| | | | | | | | ) | +| | | || __)| | | | | | | | | | +| | ) || (\ ( | | | | | | | | ) | +| (__/ )| ) \ \__| (___) |___) (___| (__/ ) +(______/ |/ \__/(_______)\_______/(______/ + + +############################################# + +Detector of astRonomical soUrces in optIcal and raDio images + +Version: {} + +For more information see: +https://github.com/RhysAlfShaw/DRUID + """.format( + version +) diff --git a/DRUID/src/background.py b/DRUID/src/background.py index 1705cef..651779a 100644 --- a/DRUID/src/background.py +++ b/DRUID/src/background.py @@ -1,6 +1,7 @@ import numpy as np from astropy.io import fits from astropy.stats import sigma_clipped_stats + from photutils.background import ( Background2D, MedianBackground, @@ -92,7 +93,7 @@ def calculate_background_maps( with fits.open(image_path) as hdul: data = hdul[0].data - # Mask sources + # mask sources mask = make_source_mask(data, nsigma=nsigma, kernel_size=kernel_size) # calculate background and RMS Avalible background estimators @@ -174,73 +175,53 @@ def make_gaussian_sources_image(image_size, sources): if __name__ == "__main__": - # Example usage: - # Create a dummy FITS file for demonstration + from astropy.wcs import WCS from astropy.coordinates import SkyCoord - # Define image parameters image_size = (1000, 1000) pixel_scale = 0.1 # degrees per pixel center_coord = SkyCoord(ra=180, dec=30, unit="deg") - # Create a dummy WCS wcs = WCS(naxis=2) wcs.wcs.crpix = [image_size[0] / 2, image_size[1] / 2] wcs.wcs.cdelt = np.array([-pixel_scale, pixel_scale]) wcs.wcs.crval = [center_coord.ra.deg, center_coord.dec.deg] wcs.wcs.ctype = ["RA---TAN", "DEC--TAN"] + num_sources = 6 + + # Define some dummy sources with different parameters + amplitude = np.random.uniform(50, 200, num_sources) + x_means = np.random.uniform(100, 900, num_sources) + y_means = np.random.uniform(100, 900, num_sources) + x_stds = np.random.uniform(5, 20, num_sources) + y_stds = np.random.uniform(5, 20, num_sources) + thetas = np.random.uniform(0, 2 * np.pi, num_sources) - # Create dummy sources + # put tow gaussians close together + + amplitude[0] = 200 + amplitude[1] = 200 + x_means[0] = 500 + y_means[0] = 500 + x_means[1] = 520 + y_means[1] = 520 + x_stds[0] = 10 + x_stds[1] = 10 + y_stds[0] = 10 + y_stds[1] = 10 + thetas[0] = 0 + thetas[1] = 0 sources = [ { - "amplitude": 100, - "x_mean": 500, - "y_mean": 500, - "x_stddev": 50, - "y_stddev": 50, - "theta": 0, - }, - { - "amplitude": 150, - "x_mean": 150, - "y_mean": 150, - "x_stddev": 7, - "y_stddev": 7, - "theta": np.pi / 4, - }, - { - "amplitude": 200, - "x_mean": 300, - "y_mean": 300, - "x_stddev": 10, - "y_stddev": 10, - "theta": np.pi / 2, - }, - { - "amplitude": 80, - "x_mean": 700, - "y_mean": 800, - "x_stddev": 6, - "y_stddev": 6, - "theta": np.pi / 3, - }, - { - "amplitude": 120, - "x_mean": 900, - "y_mean": 200, - "x_stddev": 8, - "y_stddev": 8, - "theta": np.pi / 6, - }, - { - "amplitude": 90, - "x_mean": 400, - "y_mean": 600, - "x_stddev": 4, - "y_stddev": 4, - "theta": np.pi / 8, - }, + "amplitude": amplitude[i], + "x_mean": x_means[i], + "y_mean": y_means[i], + "x_stddev": x_stds[i], + "y_stddev": y_stds[i], + "theta": thetas[i], + } + for i in range(num_sources) ] # Create a dummy image with sources and background noise @@ -254,9 +235,9 @@ def make_gaussian_sources_image(image_size, sources): print(f"Dummy FITS file created at: {dummy_fits_path}") - # Calculate background maps + # calculate background maps background_map, background_rms_map = calculate_background_maps(dummy_fits_path) - # save the background maps to a FITS file + # save the background maps to a FITS file for testing purposes with other functions. background_hdu = fits.PrimaryHDU(background_map) background_rms_hdu = fits.PrimaryHDU(background_rms_map) background_hdu.writeto("DRUID/temp/background_map.fits", overwrite=True) @@ -266,7 +247,6 @@ def make_gaussian_sources_image(image_size, sources): print("Background map calculated.") print("Background RMS map calculated.") - # You can optionally save the background maps to # plot the results with matplotlib or any other visualization library. from matplotlib import pyplot as plt diff --git a/DRUID/src/homology.py b/DRUID/src/homology.py index 8ad89d1..4d13847 100644 --- a/DRUID/src/homology.py +++ b/DRUID/src/homology.py @@ -5,6 +5,186 @@ from tqdm import tqdm +def classify_single(row): + """Classifiying the Rows based on orgin. + + Args: + row: pd.series - The row that is being classified. + + Returns: + Class: int - the Class integer that indiceates the class the row belongs too. + + """ + print(row) + if row["new_row"] == 0: + if len(row["encloses"]) == 0: # no children + if np.isnan(row["parent_tag"]): # no parent + return 0 # no children, no parent. + else: + return 1 # no child has parent. + else: + if np.isnan(row["parent_tag"]): + return 2 # has children, no parent. + else: + return 3 # has children, has parent. + else: + return 4 # new row has children. + + +def correct_first_destruction_pl(df: pl.DataFrame) -> pl.DataFrame: + """ + Function for correcting for the First destruction of a parent Island, adapted for Polars. + + This function identifies rows with "enclosed" islands, creates a new row for each, + and inherits properties from the first enclosed island. + + Args: + df (pl.DataFrame): Input catalogue of sources to correct. + + Returns: + pl.DataFrame: The new catalogue with added rows. + """ + # Ensure the 'new_row' column exists, initializing to 0 + if "new_row" not in df.columns: + df = df.with_columns(pl.lit(0, dtype=pl.Int8).alias("new_row")) + + # 1. Filter the DataFrame to find all rows that have enclosed islands. + islands_to_split = df.filter(pl.col("encloses").list.len() > 0) + + # If no such rows exist, return the original DataFrame. + if islands_to_split.is_empty(): + return df + + # 2. Perform a self-join to fetch the 'Death' attribute from the parent island. + # The parent is identified by the first ID in the 'enclosed_i' list. + new_rows_base = islands_to_split.join( + # Select only the necessary columns from the right side of the join + df.select(["ID", "death"]), + # Join condition: first element of 'enclosed_i' matches 'ID' + left_on=pl.col("encloses").list.get(0), + right_on="ID", + how="inner", + # Suffix prevents column name collisions ('Death' becomes 'Death_parent') + suffix="_parent", + ) + + # If the join results in an empty DataFrame, return the original. + if new_rows_base.is_empty(): + return df + + # 3. Generate a range of new, unique IDs for the rows to be added. + # This correctly assigns a different ID to each new row. + max_id = df["ID"].max() + num_new_rows = len(new_rows_base) + id_dtype = df.schema["ID"] # Match the original ID data type + new_ids = pl.int_range( + start=max_id + 1, + end=max_id + num_new_rows + 1, + dtype=id_dtype, + eager=True, # Generate the series of new IDs immediately + ) + + # 4. Construct the new rows with updated and new values. + new_rows = ( + new_rows_base.with_columns( + # Overwrite the original ID with the new unique ID + ID=new_ids, + # Update 'Death' with the value from the joined parent + Death=pl.col("death_parent"), + # Set 'parent_tag' to the ID of the parent island + parent_tag=pl.col("ID_parent"), + # Mark this as a newly generated row + new_row=pl.lit(1, dtype=pl.Int8), + # Set 'enclosed_i' to an empty list + enclosed_i=pl.lit(None, dtype=df.schema["encloses"]), + ) + # Remove temporary columns created by the join + .drop(["ID_parent", "death_parent"]) + # Ensure the column order matches the original DataFrame + .select(df.columns) + ) + + # 5. Concatenate the original DataFrame with the newly created rows. + return pl.concat([df, new_rows], how="vertical") + + +def parent_tag_func_pl(df: pl.DataFrame) -> pl.DataFrame: + """ + Sets the 'parent_tag' for each row based on 'enclosed_i' lists. + + This function identifies parent-child relationships where a parent's + 'enclosed_i' list contains child IDs. It then creates a 'parent_tag' + column where each child's tag is set to its parent's ID. If an ID is + not a child, its 'parent_tag' is set to its own ID. + + Args: + df: The input Polars DataFrame. Must contain 'ID' and 'enclosed_i' + (list of IDs) columns. + + Returns: + The DataFrame with an added 'parent_tag' column. + """ + # 1. Filter to get only the rows that are parents (i.e., they enclose other islands). + # We also select only the necessary columns for creating the mapping. + parents = df.filter(pl.col("encloses").list.len() > 1).select( + pl.col("ID").alias("parent_id"), pl.col("encloses") + ) + + # 2. Create the parent-child mapping. + # We "explode" the 'enclosed_i' list so that each child ID gets its own row + # next to its parent's ID. This is the Polars way to create a lookup table. + mapping = ( + parents.explode("encloses") + .rename({"encloses": "child_id"}) + .filter(pl.col("child_id") != pl.col("parent_id")) # Exclude self-references + ) + + # 3. Join the original DataFrame with the mapping. + # This will add a 'parent_id' column to our DataFrame, but it will only + # have values for rows that are children. Other rows will have null. + df_with_parent_info = df.join( + mapping, left_on="ID", right_on="child_id", how="left" + ) + + # 4. Create the final 'parent_tag' column. + # We use coalesce() to fill in the missing values. It takes the first + # non-null value it finds. So, if 'parent_id' exists, we use it; + # otherwise, we fall back to the row's own 'ID'. + df_final = df_with_parent_info.with_columns( + parent_tag=pl.when(pl.col("parent_id").is_not_null()) + .then(pl.col("parent_id")) + .otherwise(pl.col("ID")) + ).drop( + "parent_id" + ) # Clean up the temporary column + return df_final + + +def make_point_enclosure_assoc_CPU(x1, y1, Birth, Death, pd, img): + """Returns a list of the ID of the points that are enclosed by the mask pd point. + Uses GPU for computation. + + Args: + Birth (float): _description_ + Death (float): _description_ + row (pd.Series): _description_ + pd (pl.DataFrame): polars DataFrame with point data. + img (np.ndarray): _description_ + img_gpu (cp.ndarray): _description_ + + Returns: + enclosed_list (list): _description_ + """ + mask = get_mask_CPU(x1, y1, Birth, Death, img) + mask_coords = np.column_stack((pd["x1"], pd["y1"])) + + points_inside_mask = mask[ + mask_coords[:, 0].astype(int), mask_coords[:, 1].astype(int) + ] + encloses_vectorized = pd.filter(points_inside_mask)["ID"].to_list() + return encloses_vectorized + + def get_enclosing_mask_CPU(x, y, mask): """ Returns the connected components inside the mask starting from the point (x, y). @@ -128,23 +308,17 @@ def compute_ph_components( components.append(component) print(f"Found {len(components)} components.") - # plt.figure(figsize=(8, 6)) - # plt.imshow( - # components[0], origin="lower", cmap="nipy_spectral", interpolation="nearest" - # ) - # plt.title("Labeled Image") - # plt.colorbar() - # plt.show() + import time - img = components[1] + img = components[2] t0_compute_ph = time.time() pd = cripser.computePH(-img, maxdim=0) t1_compute_ph = time.time() + print( f"Computed persistent homology in {t1_compute_ph - t0_compute_ph:.2f} seconds." ) - # create polar dataframe to handle the data columns = ["dim", "birth", "death", "x1", "y1", "z1", "x2", "y2", "z2"] polar_df = pl.DataFrame(pd, schema=columns) @@ -165,7 +339,7 @@ def compute_ph_components( (polar_df["birth"] - polar_df["death"]).alias("lifetimeFrac") ) print(len(polar_df)) - liftetime_limit_fraction = 3.0 # set the lifetime limit fraction + liftetime_limit_fraction = 1.0 # set the lifetime limit fraction # filter out components with lifetime less than 3 polar_df = polar_df.filter(polar_df["lifetime"] > liftetime_limit_fraction) print( @@ -190,7 +364,7 @@ def compute_ph_components( # Assuming 'components[0]' is the correct component for all rows in polar_df # If each row corresponds to a different component, you'll need to adjust this. # For now, let's stick to the logic in your snippet. - component_img = components[0] + component_img = components[2] for row_tuple in polar_df.iter_rows( named=True @@ -221,7 +395,7 @@ def compute_ph_components( else: # Handle cases where mask is None (e.g., point outside, no component) # Append NaN or a placeholder, or filter these rows out later - areas.append(np.nan) + areas.append(0) bbox_min_y_list.append(np.nan) bbox_min_x_list.append(np.nan) bbox_max_y_list.append(np.nan) @@ -238,21 +412,19 @@ def compute_ph_components( ] ) + plt.figure(figsize=(10, 10)) + plt.imshow(component_img, cmap="gray", origin="lower") + plt.title("Component Image with Bounding Boxes") + # remove those with area < 5 pixels - polar_df = polar_df.filter(polar_df["area"] > 5) + polar_df = polar_df.filter(polar_df["area"] > 2) - # plot the components with bounding boxes - plt.figure(figsize=(10, 8)) - plt.imshow( - component_img, origin="lower", cmap="nipy_spectral", interpolation="nearest" - ) for row_tuple in polar_df.iter_rows(named=True): bbox_min_y = row_tuple["bbox_min_y"] bbox_min_x = row_tuple["bbox_min_x"] bbox_max_y = row_tuple["bbox_max_y"] bbox_max_x = row_tuple["bbox_max_x"] - if not np.isnan(bbox_min_y) and not np.isnan(bbox_min_x): # Draw the bounding box plt.gca().add_patch( @@ -266,6 +438,51 @@ def compute_ph_components( ) ) - plt.title("Component with Bounding Boxes") plt.colorbar() plt.show() + # assign an ID to each point in the polar_df + polar_df = polar_df.with_columns(pl.Series("ID", range(len(polar_df)))) + + polar_df = polar_df.with_columns( + pl.Series( + "encloses", + [ + make_point_enclosure_assoc_CPU( + row["x1"], + row["y1"], + row["birth"], + row["death"], + polar_df, + component_img, + ) + for row in polar_df.iter_rows(named=True) + ], + ) + ) + + print("Enclosure associations computed.") + print(polar_df) + print(len(polar_df)) + # correct first destruction + polar_df = correct_first_destruction_pl(polar_df) + print("First destruction corrected.") + print(polar_df) + print(len(polar_df)) + + # assign parent tags + polar_df = parent_tag_func_pl(polar_df) + print("Parent tags assigned.") + print(polar_df) + print(len(polar_df)) + + # calculate contours + + # # Classify the components by iterating over each row and applying the classify_single function + # polar_df = polar_df.with_columns( + # pl.col( + # "Class", [classify_single(row) for row in polar_df.iter_rows(named=True)] + # ) # Apply classification function + # ) + # print("Components classified.") + # print(polar_df) + # print(len(polar_df)) From 9dca250fdfd9bb39a287775f4cd1cc21ee15723f Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Tue, 1 Jul 2025 12:57:44 +0100 Subject: [PATCH 06/69] working processing on island fixed --- DRUID/main.py | 6 +- DRUID/src/background.py | 2 +- DRUID/src/homology.py | 493 +++++++++++++++++++++++++++++----------- 3 files changed, 371 insertions(+), 130 deletions(-) diff --git a/DRUID/main.py b/DRUID/main.py index a168a87..b865abd 100644 --- a/DRUID/main.py +++ b/DRUID/main.py @@ -1,6 +1,7 @@ version = "1.0" import setproctitle + setproctitle.setproctitle("DRUID") @@ -9,7 +10,6 @@ ############################################# - _______ _______ _________ ______ ( __ \ ( ____ )|\ /|\__ __/( __ \ | ( \ )| ( )|| ) ( | ) ( | ( \ ) @@ -31,3 +31,7 @@ """.format( version ) + +def main(): + + diff --git a/DRUID/src/background.py b/DRUID/src/background.py index 651779a..fd91dc1 100644 --- a/DRUID/src/background.py +++ b/DRUID/src/background.py @@ -226,7 +226,7 @@ def make_gaussian_sources_image(image_size, sources): # Create a dummy image with sources and background noise dummy_data = make_gaussian_sources_image(image_size, sources) - dummy_data += np.random.normal(0, 5, size=image_size) # Add some noise + dummy_data += np.random.normal(0, 1, size=image_size) # Add some noise # Create a dummy FITS file hdu = fits.PrimaryHDU(dummy_data, header=wcs.to_header()) diff --git a/DRUID/src/homology.py b/DRUID/src/homology.py index 4d13847..b112a49 100644 --- a/DRUID/src/homology.py +++ b/DRUID/src/homology.py @@ -1,8 +1,36 @@ +""" +Author: Rhys Shaw +Date: 11-06-2025 +""" + import cripser import numpy as np import polars as pl from scipy.ndimage import label as scipy_label from tqdm import tqdm +from skimage import measure + +# For testing and development purposes, we import the following libraries: +import pandas as pd +import matplotlib.pyplot as plt +from astropy.io import fits + + +def _get_polygons_CPU(x1, y1, birth, death, image: np.ndarray): + """ + Returns the polygon of the enclosed area of the point (x,y) in the mask. + """ + + # add a 1 pxl padding to the image to avoid index errors + image = np.pad(image, pad_width=1, mode="constant", constant_values=0) + mask = np.zeros(image.shape) + mask = np.logical_or(mask, np.logical_and(image <= birth, image > death)) + mask = get_enclosing_mask_CPU(int(y1) + 1, int(x1) + 1, mask) + contour = measure.find_contours(mask, 0)[0] + # Adjust the contour coordinates to account for the padding + contour[:, 0] -= 1 # Adjust y-coordinates + contour[:, 1] -= 1 # Adjust x-coordinates + return contour def classify_single(row): @@ -15,7 +43,7 @@ def classify_single(row): Class: int - the Class integer that indiceates the class the row belongs too. """ - print(row) + # print(row) if row["new_row"] == 0: if len(row["encloses"]) == 0: # no children if np.isnan(row["parent_tag"]): # no parent @@ -49,7 +77,7 @@ def correct_first_destruction_pl(df: pl.DataFrame) -> pl.DataFrame: df = df.with_columns(pl.lit(0, dtype=pl.Int8).alias("new_row")) # 1. Filter the DataFrame to find all rows that have enclosed islands. - islands_to_split = df.filter(pl.col("encloses").list.len() > 0) + islands_to_split = df.filter(pl.col("encloses").list.len() > 1) # If no such rows exist, return the original DataFrame. if islands_to_split.is_empty(): @@ -238,87 +266,40 @@ def get_mask_CPU(x1, y1, Birth, Death, img): return mask_enclosed -def compute_ph_components( - threholded_image, - local_bg, - analysis_threshold_val, -): - print("Computing persistent homology components...") - - -if __name__ == "__main__": - import pandas as pd - import matplotlib.pyplot as plt - from astropy.io import fits - - print("DRUID - Homology.py test script") - # get example tresholded image. - dummy_data_path = "DRUID/temp/dummy_image.fits" - - background_map_path = "DRUID/temp/background_map.fits" - background_rms_map_path = "DRUID/temp/background_rms_map.fits" - - # load the images - dummy_data = fits.open(dummy_data_path)[0].data - background_map = fits.open(background_map_path)[0].data - background_rms_map = fits.open(background_rms_map_path)[0].data - - # set anything in the mask to 0 - thresholded_image = np.where( - dummy_data > background_map + 10 * background_rms_map, dummy_data, 0 - ) - - # sort by area and remove components smaller than 5 pixels - - from skimage.measure import regionprops - from skimage.measure import label - - labeled_image = label(thresholded_image > 0, connectivity=2) - properties = regionprops(labeled_image, intensity_image=thresholded_image) - - # filter out components smaller than 5 pixels - min_area = 5 - filtered_labels = [prop.label for prop in properties if prop.area >= min_area] - - # create a new labeled image with only the filtered labels - filtered_labeled_image = np.zeros_like(labeled_image) - for label_value in filtered_labels: - filtered_labeled_image[labeled_image == label_value] = label_value +def compute_homology( + img: np.ndarray, + liftetime_limit_fraction: float = 1.0, + area_size_threshold: int = 2, +) -> pl.DataFrame: + """ + Computed the persistent homology of the image using the cripser library. + This function then also calculates contours and some basic properties of the components + in the image, such as area, bounding box, and enclosure associations. - # use the filtered labeled image for further processing - labeled_image = filtered_labeled_image + Some basic cuts are made here e.g. lifetime limit fraction and area size threshold. - # for each label crop around it. - unique_labels = np.unique(labeled_image) - components = [] - for label_value in tqdm(unique_labels): - if label_value == 0: - continue # Skip the background label - component_mask = labeled_image == label_value - component = np.where(component_mask, thresholded_image, 0) - # crop around the component - y_indices, x_indices = np.where(component_mask) + We assume that the image is already an island from a thresholded image. - if len(x_indices) == 0 or len(y_indices) == 0: - continue + Args: + img (np.ndarray): The input image, which is a 2D numpy array. + liftetime_limit_fraction (float): The lifetime limit fraction to filter components. + Defaults to 1.0, meaning all components with lifetime greater than 1.0 + will be included. + area_size_threshold (int): The minimum area size for components to be included. + Defaults to 2, meaning only components with area greater than 2 pixels will be included + in the final DataFrame. + Returns: + pl.DataFrame: A Polars DataFrame containing the computed persistent homology, + contours, and other properties of the components in the image. + This DataFrame includes columns for birth, death, lifetime, area, bounding box coordinates, + and enclosure associations. + It also includes a contour column with the computed contours of each component. - x_min, x_max = np.min(x_indices), np.max(x_indices) - y_min, y_max = np.min(y_indices), np.max(y_indices) - component = component[y_min : y_max + 1, x_min : x_max + 1] - components.append(component) - print(f"Found {len(components)} components.") - import time + """ - img = components[2] - t0_compute_ph = time.time() pd = cripser.computePH(-img, maxdim=0) - t1_compute_ph = time.time() - - print( - f"Computed persistent homology in {t1_compute_ph - t0_compute_ph:.2f} seconds." - ) columns = ["dim", "birth", "death", "x1", "y1", "z1", "x2", "y2", "z2"] polar_df = pl.DataFrame(pd, schema=columns) @@ -338,34 +319,31 @@ def compute_ph_components( polar_df = polar_df.with_columns( (polar_df["birth"] - polar_df["death"]).alias("lifetimeFrac") ) - print(len(polar_df)) - liftetime_limit_fraction = 1.0 # set the lifetime limit fraction + # liftetime_limit_fraction = 1.0 # set the lifetime limit fraction + # filter out components with lifetime less than 3 + polar_df = polar_df.filter(polar_df["lifetime"] > liftetime_limit_fraction) print( f"Filtered polar dataframe to {len(polar_df)} components with lifetime > {liftetime_limit_fraction}." ) - print(polar_df) - - # filter by pixel size. get bounding box of the component, and contour?. - # for each of the components compute the area left between the birth and death. + # set the longest lifetime rows death to 0. + polar_df = polar_df.with_columns( + pl.when(pl.col("lifetime") == pl.col("lifetime").max()) + .then(pl.lit(0)) # If lifetime is max, set death to 0 + .otherwise(pl.col("death")) # Otherwise, keep the original death value + .alias("death") # Assign this result to the 'death' column + ) - # compute the area of the component - # for each of the components compute the area left between the birth and death. + # compute the area of the component and bbox. - # compute the area of the component areas = [] bbox_min_y_list = [] bbox_min_x_list = [] bbox_max_y_list = [] bbox_max_x_list = [] - # Assuming 'components[0]' is the correct component for all rows in polar_df - # If each row corresponds to a different component, you'll need to adjust this. - # For now, let's stick to the logic in your snippet. - component_img = components[2] - for row_tuple in polar_df.iter_rows( named=True ): # named=True gives you a dictionary per row @@ -380,7 +358,7 @@ def compute_ph_components( y1, birth, death, - component_img, # Use the pre-selected component + img, # Use the pre-selected component ) if mask is not None: @@ -411,35 +389,10 @@ def compute_ph_components( pl.Series("bbox_max_x", bbox_max_x_list), ] ) + # area size filter. + area_size_threshold = 2 # replace with argument #### TODO #### + polar_df = polar_df.filter(polar_df["area"] > area_size_threshold) - plt.figure(figsize=(10, 10)) - plt.imshow(component_img, cmap="gray", origin="lower") - plt.title("Component Image with Bounding Boxes") - - # remove those with area < 5 pixels - polar_df = polar_df.filter(polar_df["area"] > 2) - - for row_tuple in polar_df.iter_rows(named=True): - - bbox_min_y = row_tuple["bbox_min_y"] - bbox_min_x = row_tuple["bbox_min_x"] - bbox_max_y = row_tuple["bbox_max_y"] - bbox_max_x = row_tuple["bbox_max_x"] - if not np.isnan(bbox_min_y) and not np.isnan(bbox_min_x): - # Draw the bounding box - plt.gca().add_patch( - plt.Rectangle( - (bbox_min_x, bbox_min_y), - bbox_max_x - bbox_min_x, - bbox_max_y - bbox_min_y, - edgecolor="blue", - facecolor="none", - linewidth=2, - ) - ) - - plt.colorbar() - plt.show() # assign an ID to each point in the polar_df polar_df = polar_df.with_columns(pl.Series("ID", range(len(polar_df)))) @@ -453,29 +406,313 @@ def compute_ph_components( row["birth"], row["death"], polar_df, - component_img, + img, ) for row in polar_df.iter_rows(named=True) ], ) ) - print("Enclosure associations computed.") - print(polar_df) - print(len(polar_df)) # correct first destruction polar_df = correct_first_destruction_pl(polar_df) - print("First destruction corrected.") - print(polar_df) - print(len(polar_df)) - # assign parent tags polar_df = parent_tag_func_pl(polar_df) - print("Parent tags assigned.") + contours = [] + + for row in polar_df.iter_rows(named=True): + try: + contour = _get_polygons_CPU( + row["x1"], row["y1"], row["birth"], row["death"], img + ) + contours.append(contour) + except Exception as e: + print(f"Error computing contour for row {row['ID']}: {e}") + contours.append([0]) + + # change countours from list of arrays of tuples to list of lists of tuples + contours = [ + list(map(tuple, contour)) if isinstance(contour, np.ndarray) else [0] + for contour in contours + ] + polar_df = polar_df.with_columns(pl.Series("contour", contours)) + + return polar_df + + +if __name__ == "__main__": + + ############################################################ + # This is a test script for the DRUID Homology module. + + print("DRUID - Homology.py test script") + + # get example tresholded image. + dummy_data_path = "DRUID/temp/dummy_image.fits" + background_map_path = "DRUID/temp/background_map.fits" + background_rms_map_path = "DRUID/temp/background_rms_map.fits" + + # load the images + dummy_data = fits.open(dummy_data_path)[0].data + background_map = fits.open(background_map_path)[0].data + background_rms_map = fits.open(background_rms_map_path)[0].data + + # set anything in the mask to 0 + thresholded_image = np.where( + dummy_data > background_map + 10 * background_rms_map, dummy_data, 0 + ) + + # sort by area and remove components smaller than 5 pixels + + from skimage.measure import regionprops + from skimage.measure import label + + labeled_image = label(thresholded_image > 0, connectivity=2) + properties = regionprops(labeled_image, intensity_image=thresholded_image) + + # filter out components smaller than 5 pixels + min_area = 2 + filtered_labels = [prop.label for prop in properties if prop.area >= min_area] + + # create a new labeled image with only the filtered labels + + filtered_labeled_image = np.zeros_like(labeled_image) + for label_value in filtered_labels: + filtered_labeled_image[labeled_image == label_value] = label_value + + # use the filtered labeled image for further processing + labeled_image = filtered_labeled_image + + # for each label crop around it. + unique_labels = np.unique(labeled_image) + components = [] + for label_value in tqdm(unique_labels): + if label_value == 0: + continue # Skip the background label + component_mask = labeled_image == label_value + component = np.where(component_mask, thresholded_image, 0) + # crop around the component + y_indices, x_indices = np.where(component_mask) + + if len(x_indices) == 0 or len(y_indices) == 0: + continue + + x_min, x_max = np.min(x_indices), np.max(x_indices) + y_min, y_max = np.min(y_indices), np.max(y_indices) + component = component[y_min : y_max + 1, x_min : x_max + 1] + components.append(component) + + print(f"Found {len(components)} components.") + + ################################################################ + # Where the img cut out is used for the computation of the persistent homology. + # We will use the first component for now. + + img = components[0] + polar_df = compute_homology(img) + print("Contours computed.") print(polar_df) - print(len(polar_df)) + print("-------------------------") + contours = polar_df["contour"].to_list() + # plot the contours + plt.figure(figsize=(10, 10)) + plt.imshow(img, cmap="gray", origin="lower") + plt.title("Component Image with Contours") + for contour in contours: + if contour != [0]: # Check if contour is not empty + contour = np.array(contour) + plt.plot( + contour[:, 1], contour[:, 0], color="red", alpha=0.5, linewidth=5 + ) # Plot y, x for correct orientation + plt.colorbar() + plt.show() + + # pd = cripser.computePH(-img, maxdim=0) + + # columns = ["dim", "birth", "death", "x1", "y1", "z1", "x2", "y2", "z2"] + # polar_df = pl.DataFrame(pd, schema=columns) + # # drop cols dim, z1, z2 + # polar_df = polar_df.drop(["dim", "z1", "z2"]) + # # create ne column lifetime death - birth + # polar_df = polar_df.with_columns( + # (polar_df["death"] - polar_df["birth"]).alias("lifetime") + # ) + # # make column birth and death - birth and death. + # polar_df = polar_df.with_columns( + # [(-polar_df["birth"]).alias("birth"), (-polar_df["death"]).alias("death")] + # ) + + # # lifetime_threshold. this is setby the user. + + # polar_df = polar_df.with_columns( + # (polar_df["birth"] - polar_df["death"]).alias("lifetimeFrac") + # ) + # liftetime_limit_fraction = 1.0 # set the lifetime limit fraction + # # filter out components with lifetime less than 3 + # polar_df = polar_df.filter(polar_df["lifetime"] > liftetime_limit_fraction) + # print( + # f"Filtered polar dataframe to {len(polar_df)} components with lifetime > {liftetime_limit_fraction}." + # ) + + # # set the longest lifetime rows death to 0. + # polar_df = polar_df.with_columns( + # pl.when(pl.col("lifetime") == pl.col("lifetime").max()) + # .then(pl.lit(0)) # If lifetime is max, set death to 0 + # .otherwise(pl.col("death")) # Otherwise, keep the original death value + # .alias("death") # Assign this result to the 'death' column + # ) + # filter by pixel size. get bounding box of the component, and contour?. + + # for each of the components compute the area left between the birth and death. + + # compute the area of the component + # for each of the components compute the area left between the birth and death. + + # compute the area of the component + # areas = [] + # bbox_min_y_list = [] + # bbox_min_x_list = [] + # bbox_max_y_list = [] + # bbox_max_x_list = [] + + # Assuming 'components[0]' is the correct component for all rows in polar_df + # If each row corresponds to a different component, you'll need to adjust this. + # For now, let's stick to the logic in your snippet. + # component_img = components[0] + + # for row_tuple in polar_df.iter_rows( + # named=True + # ): # named=True gives you a dictionary per row + # # get mask of the component using birth and death values. + # birth = row_tuple["birth"] + # death = row_tuple["death"] + # x1 = row_tuple["x1"] + # y1 = row_tuple["y1"] + + # mask = get_mask_CPU( + # x1, # Note: Your get_mask_CPU expects x1, y1, Birth, Death, img + # y1, + # birth, + # death, + # component_img, # Use the pre-selected component + # ) + + # if mask is not None: + # bounding_box = bounding_box_cpu(mask) + # area = np.sum(mask) + + # areas.append(area) + # bbox_min_y_list.append(bounding_box[0]) + # bbox_min_x_list.append(bounding_box[1]) + # bbox_max_y_list.append(bounding_box[2]) + # bbox_max_x_list.append(bounding_box[3]) + # else: + # # Handle cases where mask is None (e.g., point outside, no component) + # # Append NaN or a placeholder, or filter these rows out later + # areas.append(0) + # bbox_min_y_list.append(np.nan) + # bbox_min_x_list.append(np.nan) + # bbox_max_y_list.append(np.nan) + # bbox_max_x_list.append(np.nan) + + # # Add the new columns to the DataFrame + # polar_df = polar_df.with_columns( + # [ + # pl.Series("area", areas), + # pl.Series("bbox_min_y", bbox_min_y_list), + # pl.Series("bbox_min_x", bbox_min_x_list), + # pl.Series("bbox_max_y", bbox_max_y_list), + # pl.Series("bbox_max_x", bbox_max_x_list), + # ] + # ) + + # plt.figure(figsize=(10, 10)) + # plt.imshow(component_img, cmap="gray", origin="lower") + # plt.title("Component Image with Bounding Boxes") + + # remove those with area < 5 pixels + # polar_df = polar_df.filter(polar_df["area"] > 2) + + # for row_tuple in polar_df.iter_rows(named=True): + + # bbox_min_y = row_tuple["bbox_min_y"] + # bbox_min_x = row_tuple["bbox_min_x"] + # bbox_max_y = row_tuple["bbox_max_y"] + # bbox_max_x = row_tuple["bbox_max_x"] + # if not np.isnan(bbox_min_y) and not np.isnan(bbox_min_x): + # # Draw the bounding box + # plt.gca().add_patch( + # plt.Rectangle( + # (bbox_min_x, bbox_min_y), + # bbox_max_x - bbox_min_x, + # bbox_max_y - bbox_min_y, + # edgecolor="blue", + # facecolor="none", + # linewidth=2, + # ) + # ) + + # # plt.colorbar() + # plt.show() + # assign an ID to each point in the polar_df + # polar_df = polar_df.with_columns(pl.Series("ID", range(len(polar_df)))) + + # polar_df = polar_df.with_columns( + # pl.Series( + # "encloses", + # [ + # make_point_enclosure_assoc_CPU( + # row["x1"], + # row["y1"], + # row["birth"], + # row["death"], + # polar_df, + # component_img, + # ) + # for row in polar_df.iter_rows(named=True) + # ], + # ) + # ) + + # print("Enclosure associations computed.") + # print(polar_df) + # print(len(polar_df)) + # print("------------------------") + # # correct first destruction + # polar_df = correct_first_destruction_pl(polar_df) + # print("First destruction corrected.") + # print(polar_df) + # print(len(polar_df)) + # print("------------------------") + + # # assign parent tags + # polar_df = parent_tag_func_pl(polar_df) + # print("Parent tags assigned.") + # print(polar_df) + # print(len(polar_df)) + # print("------------------------") + # print(polar_df) + # print(len(polar_df)) # calculate contours + # contours = [] + + # for row in polar_df.iter_rows(named=True): + # try: + # contour = _get_polygons_CPU( + # row["x1"], row["y1"], row["birth"], row["death"], component_img + # ) + # contours.append(contour) + # except Exception as e: + # print(f"Error computing contour for row {row['ID']}: {e}") + # contours.append([0]) + # # print(contours) + + # # change countours from list of arrays of tuples to list of lists of tuples + # contours = [ + # list(map(tuple, contour)) if isinstance(contour, np.ndarray) else [0] + # for contour in contours + # ] + # polar_df = polar_df.with_columns(pl.Series("contour", contours)) # # Classify the components by iterating over each row and applying the classify_single function # polar_df = polar_df.with_columns( From 322b46be389d442814b9b51422d156bb0ba05077 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Tue, 1 Jul 2025 12:58:16 +0100 Subject: [PATCH 07/69] tests --- DRUID/tests/test_background.py | 172 ++++++++++++++++++++++++++------- DRUID/tests/test_homology.py | 4 - DRUID/tests/test_main.py | 16 --- DRUID/tests/test_source.py | 4 - DRUID/tests/test_utils.py | 93 ------------------ 5 files changed, 135 insertions(+), 154 deletions(-) diff --git a/DRUID/tests/test_background.py b/DRUID/tests/test_background.py index 73e5dcc..d0feeeb 100644 --- a/DRUID/tests/test_background.py +++ b/DRUID/tests/test_background.py @@ -1,39 +1,137 @@ -from DRUID.src.background import calculate_background import pytest import numpy as np -# test the use of the calculate_background function - -# prehaps use a sample more realistic array with cahracteristics of Radio and Optical data. -test_array = np.array([[1,2,3,4,5], - [6,7,8,9,10], - [11,12,13,14,15], - [16,17,18,19,20], - [21,22,23,24,25]]) -def test_calculate_background_not_valid(): - # test that a parameter not valid will fail - with pytest.raises(ValueError): - calculate_background(np.ones((10,10)), mode='not_valid') - -def test_mad_std_value(): - # test that the mad_std value is correct - # assert has to be approximately equal to the value because of float - - assert calculate_background(test_array, mode='mad_std')[0]== pytest.approx(8.8956, 0.001) - -def test_rms_std_value(): - # test that the rms_std value is correct - # assert has to be approximately equal to the value because of float - assert calculate_background(test_array, mode='rms')[0]== pytest.approx(14.8660, 0.001) - -def test_mean_value(): - # test that the mean value is correct - # assert has to be approximately equal to the value because of float - assert calculate_background(test_array, mode='mad_std')[1]== pytest.approx(13.0, 0.001) - -def test_sigma_clipping_value(): - # test that the sigma_clipping value is correct - # assert has to be approximately equal to the value because of float - assert calculate_background(test_array, mode='sigma_clip')[0]== pytest.approx(7.21110, 0.001) - -def test_SEX_background_value(): - assert calculate_background(test_array, mode='SEX')[0]== pytest.approx(7.21110, 0.001) \ No newline at end of file +from astropy.io import fits +from photutils.background import MedianBackground, StdBackgroundRMS +from DRUID.src.background import ( + make_source_mask, + calculate_background_maps, + make_gaussian_sources_image, +) +import os + + +@pytest.fixture +def dummy_fits_file(tmp_path): + """Creates a dummy FITS file with a simple image.""" + data = np.random.rand(100, 100) * 10 + 5 # Random data with some offset + hdu = fits.PrimaryHDU(data) + file_path = tmp_path / "dummy_image.fits" + hdu.writeto(file_path) + return file_path + + +@pytest.fixture +def dummy_fits_file_with_source(tmp_path): + """Creates a dummy FITS file with a simple image and a source.""" + image_size = (100, 100) + sources = [ + { + "amplitude": 100, + "x_mean": 50, + "y_mean": 50, + "x_stddev": 5, + "y_stddev": 5, + "theta": 0, + } + ] + data = make_gaussian_sources_image(image_size, sources) + data += np.random.normal(0, 1, size=image_size) # Add noise + hdu = fits.PrimaryHDU(data) + file_path = tmp_path / "dummy_image_with_source.fits" + hdu.writeto(file_path) + return file_path + + +def test_make_source_mask_with_sources(dummy_fits_file_with_source): + """Test make_source_mask with an image containing a known source.""" + with fits.open(dummy_fits_file_with_source) as hdul: + data = hdul[0].data + mask = make_source_mask(data, nsigma=3.0, kernel_size=3) + assert mask.shape == data.shape + assert np.any(mask) # Expect some sources to be masked + + +def test_calculate_background_maps_defaults(dummy_fits_file_with_source): + """Test calculate_background_maps with default parameters.""" + background_map, background_rms_map = calculate_background_maps( + dummy_fits_file_with_source + ) + with fits.open(dummy_fits_file_with_source) as hdul: + data_shape = hdul[0].data.shape + + assert background_map.shape == data_shape + assert background_rms_map.shape == data_shape + assert isinstance(background_map, np.ndarray) + assert isinstance(background_rms_map, np.ndarray) + + +def test_calculate_background_maps_custom_estimator_str(dummy_fits_file_with_source): + """Test calculate_background_maps with a string-specified background estimator.""" + background_map, background_rms_map = calculate_background_maps( + dummy_fits_file_with_source, bg_estimator="mean" + ) + with fits.open(dummy_fits_file_with_source) as hdul: + data_shape = hdul[0].data.shape + assert background_map.shape == data_shape + assert background_rms_map.shape == data_shape + + +def test_calculate_background_maps_custom_estimator_obj(dummy_fits_file_with_source): + """Test calculate_background_maps with a BackgroundBase object estimator.""" + custom_estimator = MedianBackground() + background_map, background_rms_map = calculate_background_maps( + dummy_fits_file_with_source, bg_estimator=custom_estimator + ) + with fits.open(dummy_fits_file_with_source) as hdul: + data_shape = hdul[0].data.shape + assert background_map.shape == data_shape + assert background_rms_map.shape == data_shape + + +def test_calculate_background_maps_invalid_estimator_str(dummy_fits_file_with_source): + """Test calculate_background_maps with an invalid string-specified background estimator, + expecting it to default to MedianBackground.""" + background_map, background_rms_map = calculate_background_maps( + dummy_fits_file_with_source, bg_estimator="not_an_estimator" + ) + with fits.open(dummy_fits_file_with_source) as hdul: + data_shape = hdul[0].data.shape + assert background_map.shape == data_shape + assert background_rms_map.shape == data_shape + # Further checks could involve inspecting the bkg_estimator used if it were returned or logged + + +def test_calculate_background_maps_file_not_found(tmp_path): + """Test calculate_background_maps with a non-existent FITS file.""" + non_existent_file = tmp_path / "non_existent.fits" + with pytest.raises(FileNotFoundError): + calculate_background_maps(str(non_existent_file)) + + +def test_make_gaussian_sources_image_no_sources(): + """Test make_gaussian_sources_image with an empty list of sources.""" + image_size = (50, 50) + sources = [] + image = make_gaussian_sources_image(image_size, sources) + assert image.shape == image_size + assert np.all(image == 0) + + +def test_make_gaussian_sources_image_single_source(): + """Test make_gaussian_sources_image with a single source.""" + image_size = (100, 100) + sources = [ + { + "amplitude": 50, + "x_mean": 25, + "y_mean": 25, + "x_stddev": 3, + "y_stddev": 3, + "theta": 0, + } + ] + image = make_gaussian_sources_image(image_size, sources) + assert image.shape == image_size + assert np.sum(image) > 0 # Check that the source contributes to the image + # Check peak value is close to amplitude (could be affected by pixel grid) + assert np.isclose(np.max(image), sources[0]["amplitude"], atol=1) diff --git a/DRUID/tests/test_homology.py b/DRUID/tests/test_homology.py index a5adc1c..e69de29 100644 --- a/DRUID/tests/test_homology.py +++ b/DRUID/tests/test_homology.py @@ -1,4 +0,0 @@ -import pytest - -def test_homology(): - pass \ No newline at end of file diff --git a/DRUID/tests/test_main.py b/DRUID/tests/test_main.py index 79362f8..e69de29 100644 --- a/DRUID/tests/test_main.py +++ b/DRUID/tests/test_main.py @@ -1,16 +0,0 @@ -from DRUID.main import sf -import pytest -import numpy as np - -def test_sf(): - ''' - Test initalisation of the main Class - ''' - arr2d = np.random.rand(100,100) - assert sf(image=arr2d, image_path=None, - mode="Radio",pb_path=None,cutup=False, - cutup_size=None,cutup_buff=None,output=False, - area_limit=5,smooth_sigma=1,nproc=1,GPU=True, - header=None,Xoff=None,Yoff=None) != None - - \ No newline at end of file diff --git a/DRUID/tests/test_source.py b/DRUID/tests/test_source.py index 2b1d6fb..e69de29 100644 --- a/DRUID/tests/test_source.py +++ b/DRUID/tests/test_source.py @@ -1,4 +0,0 @@ -import pytest - -def test_source(): - pass \ No newline at end of file diff --git a/DRUID/tests/test_utils.py b/DRUID/tests/test_utils.py index 9db6172..e69de29 100644 --- a/DRUID/tests/test_utils.py +++ b/DRUID/tests/test_utils.py @@ -1,93 +0,0 @@ -import pytest -import numpy as np -from astropy.io import fits -from DRUID.src.utils import smoothing, get_region_props, model_beam_func, flux_correction_factor, bounding_box_cpu, open_image, calculate_beam, xy_to_RaDec, generate_2d_gaussian -import pandas - -PATH_test_image_file = 'https://drive.google.com/uc?id=10a6goXcr6wEHX9U5LQ07cCEo2nGQ9QK5' - - - -test_image = np.array([[1,2,3,4,5], - [6,7,8,9,10], - [11,12,13,14,15], - [16,17,18,19,20], - [21,22,23,24,25]]) - -def test_smoothing_works(): - # test that the smoothing function returns the correct shape image that went in. - - assert smoothing(test_image, 3).shape == (5,5) - - -mask = np.array([[0,0,0,0,0], - [0,1,1,1,0], - [0,1,1,1,0], - [0,1,1,1,0], - [0,0,0,0,0]]) -def test_regionprops_function_params(): - # test that the regionprops function returns the correct shape image that went in. - - region = get_region_props(mask, test_image)[0] - - assert region.area == 9 - assert region.centroid == (2.0,2.0) - assert region.max_intensity == 19 - assert region.major_axis_length == pytest.approx(3.2659, 0.0001) - assert region.minor_axis_length == pytest.approx(3.2659, 0.0001) - assert abs(region.orientation) == pytest.approx(0.7853, 0.001) - -def test_model_beam_function(): - test_beam = np.array([[0.00193045, 0.01426423, 0.03877421, 0.03877421, 0.01426423], - [0.01426423, 0.10539922, 0.2865048 , 0.2865048 , 0.10539922], - [0.03877421, 0.2865048 , 0.77880078, 0.77880078, 0.2865048 ], - [0.03877421, 0.2865048 , 0.77880078, 0.77880078, 0.2865048 ], - [0.01426423, 0.10539922, 0.2865048 , 0.2865048 , 0.10539922]]) - model_beam = model_beam_func(1,(5,5),2.5,2.5,1,1,0) - # make sure the model beam and the test beam are the same. - assert np.allclose(model_beam, test_beam, rtol=1e-05, atol=1e-08) - -def test_flux_correction_factor(): - - # test if the correction value returned is correct. - mask = np.array([[0,0,0,0,0], - [0,1,1,1,0], - [0,1,1,1,0], - [0,1,1,1,0], - [0,0,0,0,0]]) - model_beam = model_beam_func(1,(5,5),2.5,2.5,1,1,0)# gaussian beam model of sixe 5x5. - assert flux_correction_factor(mask, model_beam) == pytest.approx(1.38389, 0.0001) - -def test_bouding_box_cpu(): - # test if the bounding box function returns the correct shape image that went in. - mask = np.array([[0,0,0,0,0], - [0,1,1,1,0], - [0,1,1,1,0], - [0,1,1,1,0], - [0,0,0,0,0]]) - bounding_box = bounding_box_cpu(mask) - assert bounding_box == (1,1,3,3) - -def test_open_image(): - image, header = open_image(PATH_test_image_file) - assert image.shape == (256,256) - assert type(header) == fits.header.Header - -def test_calculate_beam(): - image, header = open_image(PATH_test_image_file) - header['BMAJ'] = 0.0001388888888888889 - header['BMIN'] = 0.0001388888888888889 - header['BPA'] = 0.0 - beam,bmaj,bmin,bpa = calculate_beam(header) - assert beam == pytest.approx(0.28327, 0.0001) - assert bmaj == pytest.approx(0.5000, 0.0001) - assert bmin == pytest.approx(0.5000, 0.0001) - assert bpa == pytest.approx(0.0, 0.0001) - -def test_xy_to_RaDec(): - x,y = 128,128 - image, header = open_image(PATH_test_image_file) - ra, dec = xy_to_RaDec(128,128,header,mode='Radio') - assert ra == pytest.approx(230.68, 0.0001) - assert dec == pytest.approx(54.64416, 0.0001) - From 6d2f3b3dfa1ff38f0d469296f4dc2706c19fc797 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Tue, 1 Jul 2025 12:58:47 +0100 Subject: [PATCH 08/69] polars instead of pandas --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index be41c32..bb5be2e 100644 --- a/environment.yml +++ b/environment.yml @@ -7,7 +7,7 @@ dependencies: - pip - pip: - numpy - - pandas + - polars - pytest - numpy - scikit-image From 5f1a16d43c6533f76ccd96717a8b3b15d21eaa59 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Tue, 1 Jul 2025 16:21:04 +0100 Subject: [PATCH 09/69] parallel homology computed --- DRUID/__init__.py | 1 + DRUID/main.py | 155 ++++++++++++++++++++++++++++-- DRUID/src/background.py | 14 ++- DRUID/src/homology.py | 204 +--------------------------------------- DRUID/src/source.py | 85 +++++++++++++++++ DRUID/src/utils.py | 42 +++++++++ test.py | 19 ++++ 7 files changed, 309 insertions(+), 211 deletions(-) create mode 100644 test.py diff --git a/DRUID/__init__.py b/DRUID/__init__.py index e69de29..4e241be 100644 --- a/DRUID/__init__.py +++ b/DRUID/__init__.py @@ -0,0 +1 @@ +from .main import sf diff --git a/DRUID/main.py b/DRUID/main.py index b865abd..53134a9 100644 --- a/DRUID/main.py +++ b/DRUID/main.py @@ -1,14 +1,22 @@ version = "1.0" import setproctitle - + setproctitle.setproctitle("DRUID") +import numpy as np +import astropy +from multiprocessing import Pool + + +from .src import utils +from .src import homology +from .src import background +from .src import source + -DRUID_MESSAGE = """ - - +DRUID_MESSAGE = """ ############################################# _______ _______ _________ ______ ( __ \ ( ____ )|\ /|\__ __/( __ \ @@ -32,6 +40,141 @@ version ) -def main(): - +def _worker(image: np.ndarray) -> "pl.DataFrame": + """ + Worker function to compute homology for a single source island. + """ + return homology.compute_homology(image) + + +class sf: + def __init__( + self, + image: str | np.ndarray = None, + mode: str = None, + verbose: bool = True, + area_limit: int = 0, + smooth_sigma: float = 0, + num_threads: int = 1, + header: astropy.io.fits.header.Header = None, + ): + """ + + Initialise DRUID and preform some basic checks. + + """ + + print(DRUID_MESSAGE) + + self.mode = mode + self.verbose = verbose + self.area_limit = area_limit + self.smooth_sigma = smooth_sigma + self.num_threads = num_threads + self.header = header + + if image is None: + raise ValueError( + "No image provided. Please provide a file path or a NumPy array." + ) + + if isinstance(image, str): + try: + self.image = utils.get_image_from_path(image) + except Exception as e: + raise ValueError(f"Could not load image from path: {image}") from e + elif isinstance(image, np.ndarray): + self.image = image + else: + raise TypeError( + "Image must be a file path (str) or a NumPy array (np.ndarray)." + ) + + def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 2): + """ + Runs the source findin algorithm on the image. + + Requires that the background has first been calculated. + + """ + if self.background_map is None or self.background_rms_map is None: + raise ValueError( + "Background map and RMS map must be set before running source finding." + "Please call set_background() first. or assign them manually." + ) + + source_islands = source.create_source_islands( + self.image, + self.background_map, + self.background_rms_map, + detection_threshold=self.detection_threshold, + analysis_threshold=self.analysis_threshold, + area_limit=self.area_limit, + verbose=self.verbose, + ) + + if self.verbose: + print( + f"Found {len(source_islands['positions'])} source islands in the image with area limit {self.area_limit}." + ) + + images_to_process = source_islands["island_image"] + + if not images_to_process: + if self.verbose: + print("No source islands to process.") + # Create an empty catalog if no islands are found + import polars as pl + + self.catalog = pl.DataFrame() + return + + if self.num_threads > 1: + if self.verbose: + print( + f"Processing {len(images_to_process)} source islands in parallel. with {self.num_threads} threads." + ) + + with Pool(self.num_threads) as p: + results = p.map(_worker, images_to_process) + else: + results = [] + for image in images_to_process: + results.append(_worker(image)) + + # combine the results catalogs to a single catalog + if results: + self.catalog = utils.combine_polars_catalogs(results) + + def set_background( + self, + method: str = "rms", + detection_threshold: int = 5, + analysis_threshold: int = 3, + box_size: tuple = (50, 50), # kernal size for background calculation + filter_size: tuple = (3, 3), # size of median filter for background map + kernel_size: int = 3, # size of kernel for sigma clipping. + ): + """ + Calculate the background map of the image. + This is required before running the source finding algorithm. + """ + if self.verbose: + print("Calculating background map and RMS map...") + self.detection_threshold = detection_threshold + self.analysis_threshold = analysis_threshold + + self.background_map, self.background_rms_map = ( + background.calculate_background_maps( + self.image, + bg_estimator=method, + box_size=box_size, + filter_size=(3, 3), + nsigma=detection_threshold, + kernel_size=3, + ) + ) + + if self.verbose: + print("Background map and RMS map calculated.") diff --git a/DRUID/src/background.py b/DRUID/src/background.py index fd91dc1..dfcb438 100644 --- a/DRUID/src/background.py +++ b/DRUID/src/background.py @@ -52,7 +52,7 @@ def make_source_mask(data, nsigma=3.0, kernel_size=3): def calculate_background_maps( - image_path, + image, bg_estimator="median", box_size=(50, 50), filter_size=(3, 3), @@ -90,10 +90,16 @@ def calculate_background_maps( background_rms_map : numpy.ndarray The calculated background RMS map. """ - with fits.open(image_path) as hdul: - data = hdul[0].data + # Check if the input is a FITS file path or a numpy array + # to handle both cases of test and np.ndarray input. + if isinstance(image, str): + with fits.open(image) as hdul: + data = hdul[0].data - # mask sources + elif isinstance(image, np.ndarray): + data = image + + # mask sources with sigma clipping. mask = make_source_mask(data, nsigma=nsigma, kernel_size=kernel_size) # calculate background and RMS Avalible background estimators diff --git a/DRUID/src/homology.py b/DRUID/src/homology.py index b112a49..ec33e09 100644 --- a/DRUID/src/homology.py +++ b/DRUID/src/homology.py @@ -268,7 +268,7 @@ def get_mask_CPU(x1, y1, Birth, Death, img): def compute_homology( img: np.ndarray, - liftetime_limit_fraction: float = 1.0, + lifetime_limit_fraction: float = 1.0, area_size_threshold: int = 2, ) -> pl.DataFrame: """ @@ -323,9 +323,9 @@ def compute_homology( # filter out components with lifetime less than 3 - polar_df = polar_df.filter(polar_df["lifetime"] > liftetime_limit_fraction) + polar_df = polar_df.filter(polar_df["lifetime"] > lifetime_limit_fraction) print( - f"Filtered polar dataframe to {len(polar_df)} components with lifetime > {liftetime_limit_fraction}." + f"Filtered polar dataframe to {len(polar_df)} components with lifetime > {lifetime_limit_fraction}." ) # set the longest lifetime rows death to 0. @@ -525,201 +525,3 @@ def compute_homology( ) # Plot y, x for correct orientation plt.colorbar() plt.show() - - # pd = cripser.computePH(-img, maxdim=0) - - # columns = ["dim", "birth", "death", "x1", "y1", "z1", "x2", "y2", "z2"] - # polar_df = pl.DataFrame(pd, schema=columns) - # # drop cols dim, z1, z2 - # polar_df = polar_df.drop(["dim", "z1", "z2"]) - # # create ne column lifetime death - birth - # polar_df = polar_df.with_columns( - # (polar_df["death"] - polar_df["birth"]).alias("lifetime") - # ) - # # make column birth and death - birth and death. - # polar_df = polar_df.with_columns( - # [(-polar_df["birth"]).alias("birth"), (-polar_df["death"]).alias("death")] - # ) - - # # lifetime_threshold. this is setby the user. - - # polar_df = polar_df.with_columns( - # (polar_df["birth"] - polar_df["death"]).alias("lifetimeFrac") - # ) - # liftetime_limit_fraction = 1.0 # set the lifetime limit fraction - # # filter out components with lifetime less than 3 - # polar_df = polar_df.filter(polar_df["lifetime"] > liftetime_limit_fraction) - # print( - # f"Filtered polar dataframe to {len(polar_df)} components with lifetime > {liftetime_limit_fraction}." - # ) - - # # set the longest lifetime rows death to 0. - # polar_df = polar_df.with_columns( - # pl.when(pl.col("lifetime") == pl.col("lifetime").max()) - # .then(pl.lit(0)) # If lifetime is max, set death to 0 - # .otherwise(pl.col("death")) # Otherwise, keep the original death value - # .alias("death") # Assign this result to the 'death' column - # ) - # filter by pixel size. get bounding box of the component, and contour?. - - # for each of the components compute the area left between the birth and death. - - # compute the area of the component - # for each of the components compute the area left between the birth and death. - - # compute the area of the component - # areas = [] - # bbox_min_y_list = [] - # bbox_min_x_list = [] - # bbox_max_y_list = [] - # bbox_max_x_list = [] - - # Assuming 'components[0]' is the correct component for all rows in polar_df - # If each row corresponds to a different component, you'll need to adjust this. - # For now, let's stick to the logic in your snippet. - # component_img = components[0] - - # for row_tuple in polar_df.iter_rows( - # named=True - # ): # named=True gives you a dictionary per row - # # get mask of the component using birth and death values. - # birth = row_tuple["birth"] - # death = row_tuple["death"] - # x1 = row_tuple["x1"] - # y1 = row_tuple["y1"] - - # mask = get_mask_CPU( - # x1, # Note: Your get_mask_CPU expects x1, y1, Birth, Death, img - # y1, - # birth, - # death, - # component_img, # Use the pre-selected component - # ) - - # if mask is not None: - # bounding_box = bounding_box_cpu(mask) - # area = np.sum(mask) - - # areas.append(area) - # bbox_min_y_list.append(bounding_box[0]) - # bbox_min_x_list.append(bounding_box[1]) - # bbox_max_y_list.append(bounding_box[2]) - # bbox_max_x_list.append(bounding_box[3]) - # else: - # # Handle cases where mask is None (e.g., point outside, no component) - # # Append NaN or a placeholder, or filter these rows out later - # areas.append(0) - # bbox_min_y_list.append(np.nan) - # bbox_min_x_list.append(np.nan) - # bbox_max_y_list.append(np.nan) - # bbox_max_x_list.append(np.nan) - - # # Add the new columns to the DataFrame - # polar_df = polar_df.with_columns( - # [ - # pl.Series("area", areas), - # pl.Series("bbox_min_y", bbox_min_y_list), - # pl.Series("bbox_min_x", bbox_min_x_list), - # pl.Series("bbox_max_y", bbox_max_y_list), - # pl.Series("bbox_max_x", bbox_max_x_list), - # ] - # ) - - # plt.figure(figsize=(10, 10)) - # plt.imshow(component_img, cmap="gray", origin="lower") - # plt.title("Component Image with Bounding Boxes") - - # remove those with area < 5 pixels - # polar_df = polar_df.filter(polar_df["area"] > 2) - - # for row_tuple in polar_df.iter_rows(named=True): - - # bbox_min_y = row_tuple["bbox_min_y"] - # bbox_min_x = row_tuple["bbox_min_x"] - # bbox_max_y = row_tuple["bbox_max_y"] - # bbox_max_x = row_tuple["bbox_max_x"] - # if not np.isnan(bbox_min_y) and not np.isnan(bbox_min_x): - # # Draw the bounding box - # plt.gca().add_patch( - # plt.Rectangle( - # (bbox_min_x, bbox_min_y), - # bbox_max_x - bbox_min_x, - # bbox_max_y - bbox_min_y, - # edgecolor="blue", - # facecolor="none", - # linewidth=2, - # ) - # ) - - # # plt.colorbar() - # plt.show() - # assign an ID to each point in the polar_df - # polar_df = polar_df.with_columns(pl.Series("ID", range(len(polar_df)))) - - # polar_df = polar_df.with_columns( - # pl.Series( - # "encloses", - # [ - # make_point_enclosure_assoc_CPU( - # row["x1"], - # row["y1"], - # row["birth"], - # row["death"], - # polar_df, - # component_img, - # ) - # for row in polar_df.iter_rows(named=True) - # ], - # ) - # ) - - # print("Enclosure associations computed.") - # print(polar_df) - # print(len(polar_df)) - # print("------------------------") - # # correct first destruction - # polar_df = correct_first_destruction_pl(polar_df) - # print("First destruction corrected.") - # print(polar_df) - # print(len(polar_df)) - # print("------------------------") - - # # assign parent tags - # polar_df = parent_tag_func_pl(polar_df) - # print("Parent tags assigned.") - # print(polar_df) - # print(len(polar_df)) - # print("------------------------") - # print(polar_df) - # print(len(polar_df)) - - # calculate contours - # contours = [] - - # for row in polar_df.iter_rows(named=True): - # try: - # contour = _get_polygons_CPU( - # row["x1"], row["y1"], row["birth"], row["death"], component_img - # ) - # contours.append(contour) - # except Exception as e: - # print(f"Error computing contour for row {row['ID']}: {e}") - # contours.append([0]) - # # print(contours) - - # # change countours from list of arrays of tuples to list of lists of tuples - # contours = [ - # list(map(tuple, contour)) if isinstance(contour, np.ndarray) else [0] - # for contour in contours - # ] - # polar_df = polar_df.with_columns(pl.Series("contour", contours)) - - # # Classify the components by iterating over each row and applying the classify_single function - # polar_df = polar_df.with_columns( - # pl.col( - # "Class", [classify_single(row) for row in polar_df.iter_rows(named=True)] - # ) # Apply classification function - # ) - # print("Components classified.") - # print(polar_df) - # print(len(polar_df)) diff --git a/DRUID/src/source.py b/DRUID/src/source.py index e69de29..9691b2e 100644 --- a/DRUID/src/source.py +++ b/DRUID/src/source.py @@ -0,0 +1,85 @@ +""" +Author: Rhys Shaw +Date: 01-07-2025 +""" + +import numpy as np +from skimage.measure import regionprops +from skimage.measure import label +from tqdm import tqdm + + +def create_source_islands( + image, + background_map, + background_rms_map, + detection_threshold=5, + analysis_threshold=3, + area_limit=2, + verbose=True, +): + """ + Create source islands from the background and RMS maps. And create cutouts of them. + + Parameters + ---------- + background_map : numpy.ndarray + The background map of the image. + background_rms_map : numpy.ndarray + The RMS map of the image. + detection_threshold : float, optional + Threshold for detecting sources, by default 5. + analysis_threshold : float, optional + Threshold for analyzing sources, by default 3. + + Returns + ------- + a dictionary of source islands, with keys, array (the cropped image), + poistion (the position of the source in the original image), + """ + + thresholded_image = np.where( + image > background_map + analysis_threshold * background_rms_map, image, 0 + ) + + labeled_image = label(thresholded_image > 0, connectivity=2) + properties = regionprops(labeled_image, intensity_image=thresholded_image) + + # filter out components smaller than 5 pixels + min_area = area_limit + filtered_labels = [prop.label for prop in properties if prop.area >= min_area] + + filtered_labeled_image = np.zeros_like(labeled_image) + for label_value in filtered_labels: + filtered_labeled_image[labeled_image == label_value] = label_value + + labeled_image = filtered_labeled_image + + # for each label crop around it. + unique_labels = np.unique(labeled_image) + components = [] + source_islands_positions = [] + for label_value in tqdm(unique_labels): + if label_value == 0: + continue # Skip the background label + component_mask = labeled_image == label_value + component = np.where(component_mask, thresholded_image, 0) + # crop around the component + y_indices, x_indices = np.where(component_mask) + + if len(x_indices) == 0 or len(y_indices) == 0: + continue + + x_min, x_max = np.min(x_indices), np.max(x_indices) + y_min, y_max = np.min(y_indices), np.max(y_indices) + component = component[y_min : y_max + 1, x_min : x_max + 1] + position = (y_min, x_min) + source_islands_positions.append(position) + components.append(component) + + source_islands = { + "island_image": components, + "positions": source_islands_positions, + } + + return source_islands diff --git a/DRUID/src/utils.py b/DRUID/src/utils.py index e69de29..3bffdee 100644 --- a/DRUID/src/utils.py +++ b/DRUID/src/utils.py @@ -0,0 +1,42 @@ +import polars as pl + + +def get_image_from_path(image_path): + """ + Load an image from a file path. + + Parameters + ---------- + image_path : str + Path to the image file. + + Returns + ------- + numpy.ndarray + The loaded image as a NumPy array. + """ + from astropy.io import fits + + with fits.open(image_path) as hdul: + image = hdul[0].data + + return image + + +def combine_polars_catalogs(catalogs: list): + """ + Combine multiple polar catalogs into a single catalog. + + """ + if not catalogs: + raise ValueError("No catalogs provided to combine.") + + combined_catalog = pl.concat(catalogs) + + # Ensure the 'id' column is unique + if "id" in combined_catalog.columns: + combined_catalog = combined_catalog.with_columns( + pl.col("id").cast(pl.Int64) + ).with_columns(pl.col("id").rank(method="dense").alias("id")) + + return combined_catalog diff --git a/test.py b/test.py new file mode 100644 index 0000000..4634375 --- /dev/null +++ b/test.py @@ -0,0 +1,19 @@ +from DRUID import sf + + +def main(): + image_path = "DRUID/temp/dummy_image.fits" + test_optical_image = "/Users/rs17612/Documents/Optical_IR_Data/EUCLID/EUC_MER_BGSUB-MOSAIC-VIS_TILE101158277-BB647A_20240122T115602.395130Z_00.00.fits" + findmysource = sf( + image=test_optical_image, mode="optical", area_limit=5, num_threads=2 + ) + findmysource.set_background() + findmysource.phsf() + + # pint the catalog + catalog = findmysource.catalog + print("Catalog:", catalog) + + +if __name__ == "__main__": + main() From 35f946ec4928fe682eae7061319cf3e88aaf3a87 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Wed, 2 Jul 2025 08:26:13 +0100 Subject: [PATCH 10/69] experiemental source island optimsed function --- DRUID/main.py | 8 +-- DRUID/src/source.py | 129 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 4 deletions(-) diff --git a/DRUID/main.py b/DRUID/main.py index 53134a9..65f0f62 100644 --- a/DRUID/main.py +++ b/DRUID/main.py @@ -8,6 +8,7 @@ import numpy as np import astropy from multiprocessing import Pool +import polars as pl from .src import utils @@ -103,7 +104,9 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 2): "Background map and RMS map must be set before running source finding." "Please call set_background() first. or assign them manually." ) - + if self.verbose: + print("Thresholding to find source islands...") + # this function is rather slow. source_islands = source.create_source_islands( self.image, self.background_map, @@ -124,9 +127,6 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 2): if not images_to_process: if self.verbose: print("No source islands to process.") - # Create an empty catalog if no islands are found - import polars as pl - self.catalog = pl.DataFrame() return diff --git a/DRUID/src/source.py b/DRUID/src/source.py index 9691b2e..41d506a 100644 --- a/DRUID/src/source.py +++ b/DRUID/src/source.py @@ -8,6 +8,11 @@ from skimage.measure import label from tqdm import tqdm +import numpy as np +from skimage.measure import regionprops, label, regionprops_table +from tqdm import tqdm +import pandas as pd # + def create_source_islands( image, @@ -83,3 +88,127 @@ def create_source_islands( } return source_islands + + +def create_source_islands_optimized( + image, + background_map, + background_rms_map, + detection_threshold=5, # Not used in current logic, but kept for signature + analysis_threshold=3, + area_limit=2, + verbose=True, +): + """ + Create source islands from the background and RMS maps. And create cutouts of them. + + Parameters + ---------- + image : numpy.ndarray + The input image. + background_map : numpy.ndarray + The background map of the image. + background_rms_map : numpy.ndarray + The RMS map of the image. + detection_threshold : float, optional + Threshold for detecting sources (currently not used for analysis logic), by default 5. + analysis_threshold : float, optional + Threshold for analyzing sources, by default 3. + area_limit : int, optional + Minimum area (in pixels) for a detected region to be considered a source island, by default 2. + verbose : bool, optional + If True, display progress bars, by default True. + + Returns + ------- + a dictionary of source islands, with keys, array (the cropped image), + poistion (the position of the source in the original image), + """ + + if verbose: + print( + "Step 1: Applying analysis threshold and labeling connected components..." + ) + + # Create a boolean mask directly. This avoids creating a full-size float array of zeros. + analysis_mask = image > (background_map + analysis_threshold * background_rms_map) + + # Label connected components on the boolean mask + # connectivity=2 is 8-connectivity for 2D images + labeled_image = label(analysis_mask, connectivity=2) + + # Use regionprops_table for efficiency, requesting only necessary properties + # 'bbox' for cropping, 'label' for filtering, 'area' for filtering + # 'image' would give the cropped binary mask, 'intensity_image' would give cropped intensities. + # We will slice the original image/thresholded data later for actual intensities. + if verbose: + print("Step 2: Measuring region properties...") + + # We only need 'bbox' and 'area' for filtering and cropping + # If you need other properties for analysis later, add them here. + properties_table = regionprops_table( + labeled_image, + intensity_image=image, # Pass the original image for intensity measurements + properties=("label", "bbox", "area"), + ) + + # Convert to DataFrame for easier filtering + props_df = pd.DataFrame(properties_table) + + if verbose: + print(f"Initial regions found: {len(props_df)}") + print(f"Step 3: Filtering regions by area (>{area_limit} pixels)...") + + # Filter out components smaller than area_limit pixels + # Filtering on the DataFrame is much faster than iterating a list of RegionProperties objects. + filtered_props_df = props_df[props_df["area"] >= area_limit] + + if verbose: + print(f"Regions after area filtering: {len(filtered_props_df)}") + print("Step 4: Extracting source island cutouts...") + + components = [] + source_islands_positions = [] + + # Iterate through the filtered DataFrame rows + # Using itertuples() is generally faster than iterrows() for DataFrames + for row in tqdm( + filtered_props_df.itertuples(), + total=len(filtered_props_df), + disable=not verbose, + ): + # Bounding box is (min_row, min_col, max_row, max_col) + min_row, min_col, max_row, max_col = row.bbox + + # Slice the *original* image directly to get the intensities within the bounding box + # This is more efficient than recreating a masked array for each component. + # Ensure max_row and max_col are exclusive in python slicing, so bbox_coords[2] and bbox_coords[3] work directly + component_image_cutout = image[ + min_row:max_row, min_col:max_col + ].copy() # .copy() to ensure it's a separate array + + # To get the thresholded values only within the cutout (if needed): + # component_thresholded_cutout = thresholded_image[min_row:max_row, min_col:max_col] + # Or even better, apply the threshold condition directly to the cutout: + component_analysis_cutout = component_image_cutout * ( + component_image_cutout + > ( + background_map[min_row:max_row, min_col:max_col] + + analysis_threshold + * background_rms_map[min_row:max_row, min_col:max_col] + ) + ) + + position = (min_row, min_col) + source_islands_positions.append(position) + components.append(component_analysis_cutout) # Store the thresholded cutout + + source_islands = { + "island_image": components, + "positions": source_islands_positions, + } + + if verbose: + print("Source island creation complete.") + + return source_islands From 660558d72576aa9e048f337f19741b237d425bbd Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Mon, 14 Jul 2025 13:57:42 +0100 Subject: [PATCH 11/69] improved loop performance --- DRUID/main.py | 18 ++++++++++-- DRUID/src/source.py | 68 ++++++++++++++++++++++++++++---------------- test.py | 69 ++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 124 insertions(+), 31 deletions(-) diff --git a/DRUID/main.py b/DRUID/main.py index 65f0f62..c479809 100644 --- a/DRUID/main.py +++ b/DRUID/main.py @@ -9,6 +9,7 @@ import astropy from multiprocessing import Pool import polars as pl +import time from .src import utils @@ -107,6 +108,7 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 2): if self.verbose: print("Thresholding to find source islands...") # this function is rather slow. + t0 = time.time() source_islands = source.create_source_islands( self.image, self.background_map, @@ -116,7 +118,9 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 2): area_limit=self.area_limit, verbose=self.verbose, ) - + t1 = time.time() + print(f"Thresholding took {t1 - t0:.2f} seconds.") + t0 = time.time() if self.verbose: print( f"Found {len(source_islands['positions'])} source islands in the image with area limit {self.area_limit}." @@ -135,9 +139,11 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 2): print( f"Processing {len(images_to_process)} source islands in parallel. with {self.num_threads} threads." ) - + print("images to process:", len(images_to_process)) + batch_size = len(images_to_process) // self.num_threads + print(f"Batch size: {batch_size}") with Pool(self.num_threads) as p: - results = p.map(_worker, images_to_process) + results = p.map(_worker, images_to_process, chunksize=batch_size) else: results = [] for image in images_to_process: @@ -147,6 +153,9 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 2): if results: self.catalog = utils.combine_polars_catalogs(results) + t1 = time.time() + print(f"Homology computation took {t1 - t0:.2f} seconds.") + def set_background( self, method: str = "rms", @@ -162,6 +171,7 @@ def set_background( """ if self.verbose: print("Calculating background map and RMS map...") + t0 = time.time() self.detection_threshold = detection_threshold self.analysis_threshold = analysis_threshold @@ -175,6 +185,8 @@ def set_background( kernel_size=3, ) ) + t1 = time.time() + print(f"Background calculation took {t1 - t0:.2f} seconds.") if self.verbose: print("Background map and RMS map calculated.") diff --git a/DRUID/src/source.py b/DRUID/src/source.py index 41d506a..255db4c 100644 --- a/DRUID/src/source.py +++ b/DRUID/src/source.py @@ -46,42 +46,63 @@ def create_source_islands( thresholded_image = np.where( image > background_map + analysis_threshold * background_rms_map, image, 0 ) + import time + t0 = time.time() labeled_image = label(thresholded_image > 0, connectivity=2) + t1 = time.time() + if verbose: + print( + f"Labeling connected components took {t1 - t0:.2f} seconds. Found {np.unique(labeled_image).size - 1} components." + ) + t0 = time.time() properties = regionprops(labeled_image, intensity_image=thresholded_image) - + t1 = time.time() + if verbose: + print( + f"Calculating region properties took {t1 - t0:.2f} seconds. Found {len(properties)} properties." + ) # filter out components smaller than 5 pixels min_area = area_limit + t0 = time.time() filtered_labels = [prop.label for prop in properties if prop.area >= min_area] - + t1 = time.time() + if verbose: + print( + f"Filtering components by area took {t1 - t0:.2f} seconds. Found {len(filtered_labels)} components after filtering." + ) + t0 = time.time() filtered_labeled_image = np.zeros_like(labeled_image) for label_value in filtered_labels: filtered_labeled_image[labeled_image == label_value] = label_value - + t1 = time.time() + if verbose: + print( + f"Creating filtered labeled image took {t1 - t0:.2f} seconds. Filtered image has {np.unique(filtered_labeled_image).size - 1} components." + ) labeled_image = filtered_labeled_image - - # for each label crop around it. - unique_labels = np.unique(labeled_image) components = [] source_islands_positions = [] - for label_value in tqdm(unique_labels): - if label_value == 0: - continue # Skip the background label - component_mask = labeled_image == label_value - component = np.where(component_mask, thresholded_image, 0) - # crop around the component - y_indices, x_indices = np.where(component_mask) - - if len(x_indices) == 0 or len(y_indices) == 0: - continue - - x_min, x_max = np.min(x_indices), np.max(x_indices) - y_min, y_max = np.min(y_indices), np.max(y_indices) - component = component[y_min : y_max + 1, x_min : x_max + 1] - position = (y_min, x_min) - source_islands_positions.append(position) - components.append(component) + t0 = time.time() + + # Calculate properties for all labeled regions + # We pass thresholded_image as intensity_image to get the actual pixel values + props = regionprops(labeled_image, intensity_image=thresholded_image) + + for prop in props: + # prop.intensity_image is the cropped and masked component + components.append(prop.intensity_image) + + # prop.bbox returns (min_row, min_col, max_row, max_col) + y_min, x_min, _, _ = prop.bbox + source_islands_positions.append((y_min, x_min)) + + t1 = time.time() + if verbose: + print( + f"Cropping components took {t1 - t0:.2f} seconds. Found {len(components)} source islands." + ) source_islands = { "island_image": components, "positions": source_islands_positions, @@ -177,7 +198,6 @@ def create_source_islands_optimized( total=len(filtered_props_df), disable=not verbose, ): - # Bounding box is (min_row, min_col, max_row, max_col) min_row, min_col, max_row, max_col = row.bbox # Slice the *original* image directly to get the intensities within the bounding box diff --git a/test.py b/test.py index 4634375..f3f3e12 100644 --- a/test.py +++ b/test.py @@ -1,12 +1,73 @@ from DRUID import sf +def create_dummy_image(): + """ + Create a dummy FITS image for testing purposes. + """ + from astropy.io import fits + import numpy as np + + # Create a dummy image with random data + data = np.random.normal(size=(10000, 10000)).astype(np.float32) + # add many bright sources + for _ in range(500): + x = np.random.randint(0, 10000) + y = np.random.randint(0, 10000) + data[x, y] += np.random.uniform(500, 10000) + + # convolve the image with a Gaussian kernel to simulate a more realistic image + from scipy.ndimage import gaussian_filter + + data = gaussian_filter(data, sigma=5) + + # Create a FITS file + hdu = fits.PrimaryHDU(data) + hdu.writeto("DRUID/temp/dummy_image.fits", overwrite=True) + + # # plot the image to verify + # import matplotlib.pyplot as plt + + # img_size = [2000, 3000, 5000, 10000] + # bg_time = [0.79, 1.7, 4.35, 17.36] + # thresh_time = [5.74, 16.2, 50, 60 * 7] + # # fit exponential curves to the data + # from scipy.optimize import curve_fit + + # def exp_func(x, a, b): + # return a * np.exp(b * x) + + # popt_bg, _ = curve_fit(exp_func, img_size, bg_time) + # popt_thresh, _ = curve_fit(exp_func, img_size, thresh_time) + + # plt.plot(img_size, bg_time, label="Background Calculation Time") + # plt.plot(img_size, thresh_time, label="Thresholding Time") + # extrapolated_img_size = np.linspace(0, 20000, 100) + # plt.plot( + # extrapolated_img_size, + # exp_func(np.array(extrapolated_img_size), *popt_bg), + # linestyle="--", + # color="blue", + # ) + + # plt.plot( + # extrapolated_img_size, + # exp_func(np.array(extrapolated_img_size), *popt_thresh), + # linestyle="--", + # color="orange", + # ) + # plt.legend() + # plt.xlabel("Image Size (pixels)") + # plt.ylabel("Time (seconds)") + # plt.title("Background Calculation and Thresholding Time vs Image Size") + # plt.show() + + def main(): + create_dummy_image() # Create a dummy image for testing image_path = "DRUID/temp/dummy_image.fits" - test_optical_image = "/Users/rs17612/Documents/Optical_IR_Data/EUCLID/EUC_MER_BGSUB-MOSAIC-VIS_TILE101158277-BB647A_20240122T115602.395130Z_00.00.fits" - findmysource = sf( - image=test_optical_image, mode="optical", area_limit=5, num_threads=2 - ) + # image_path = "/Users/rs17612/Documents/Optical_IR_Data/EUCLID/EUC_MER_BGSUB-MOSAIC-VIS_TILE101158277-BB647A_20240122T115602.395130Z_00.00.fits" + findmysource = sf(image=image_path, mode="optical", area_limit=5, num_threads=1) findmysource.set_background() findmysource.phsf() From e43bb5f788aae5136dce548c46a11f995a4d8216 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Tue, 15 Jul 2025 11:16:47 +0100 Subject: [PATCH 12/69] added working dir to cashe results and save by default --- DRUID/main.py | 82 +++++++++++++++++++++++++++++-------------- DRUID/src/homology.py | 6 ++-- DRUID/src/source.py | 31 +++------------- test.py | 16 ++++++--- 4 files changed, 73 insertions(+), 62 deletions(-) diff --git a/DRUID/main.py b/DRUID/main.py index c479809..d16d031 100644 --- a/DRUID/main.py +++ b/DRUID/main.py @@ -2,15 +2,15 @@ import setproctitle - setproctitle.setproctitle("DRUID") import numpy as np import astropy -from multiprocessing import Pool import polars as pl import time - +import os +from multiprocessing import Pool +from tqdm import tqdm from .src import utils from .src import homology @@ -60,6 +60,7 @@ def __init__( smooth_sigma: float = 0, num_threads: int = 1, header: astropy.io.fits.header.Header = None, + working_directory: str = "DRUID/temp", ): """ @@ -93,6 +94,11 @@ def __init__( "Image must be a file path (str) or a NumPy array (np.ndarray)." ) + # check if there are files in the working directory + if not os.path.exists(working_directory): + os.makedirs(working_directory) + self.working_directory = working_directory + def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 2): """ Runs the source findin algorithm on the image. @@ -105,6 +111,7 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 2): "Background map and RMS map must be set before running source finding." "Please call set_background() first. or assign them manually." ) + if self.verbose: print("Thresholding to find source islands...") # this function is rather slow. @@ -133,21 +140,24 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 2): print("No source islands to process.") self.catalog = pl.DataFrame() return - - if self.num_threads > 1: - if self.verbose: - print( - f"Processing {len(images_to_process)} source islands in parallel. with {self.num_threads} threads." - ) - print("images to process:", len(images_to_process)) - batch_size = len(images_to_process) // self.num_threads - print(f"Batch size: {batch_size}") - with Pool(self.num_threads) as p: - results = p.map(_worker, images_to_process, chunksize=batch_size) - else: - results = [] - for image in images_to_process: - results.append(_worker(image)) + print(self.num_threads) + + # if self.num_threads > 1: + # if self.verbose: + # print( + # f"Processing {len(images_to_process)} source islands in parallel. with {self.num_threads} threads." + # ) + # print("images to process:", len(images_to_process)) + # batch_size = len(images_to_process) // self.num_threads + # print(f"Batch size: {batch_size}") + # with Pool(self.num_threads) as p: + # results = p.map(_worker, images_to_process, chunksize=batch_size) + + # else: + print(f"Processing {len(images_to_process)} source islands sequentially.") + results = [] + for image in tqdm(images_to_process): + results.append(homology.compute_homology(image)) # combine the results catalogs to a single catalog if results: @@ -169,22 +179,40 @@ def set_background( Calculate the background map of the image. This is required before running the source finding algorithm. """ + # Check if background maps already exist in the working directory. + if self.verbose: print("Calculating background map and RMS map...") t0 = time.time() self.detection_threshold = detection_threshold self.analysis_threshold = analysis_threshold - self.background_map, self.background_rms_map = ( - background.calculate_background_maps( - self.image, - bg_estimator=method, - box_size=box_size, - filter_size=(3, 3), - nsigma=detection_threshold, - kernel_size=3, + if os.path.exists(self.working_directory + "/background_map.npy"): + if os.path.exists(self.working_directory + "/background_rms_map.npy"): + if self.verbose: + print( + "Background map and RMS map already exist. Loading from disk." + ) + self.background_map = np.load( + self.working_directory + "/background_map.npy" + ) + self.background_rms_map = np.load( + self.working_directory + "/background_rms_map.npy" + ) + + else: + if self.verbose: + print("Calculating background map and RMS map from image.") + self.background_map, self.background_rms_map = ( + background.calculate_background_maps( + self.image, + bg_estimator=method, + box_size=box_size, + filter_size=(3, 3), + nsigma=detection_threshold, + kernel_size=3, + ) ) - ) t1 = time.time() print(f"Background calculation took {t1 - t0:.2f} seconds.") diff --git a/DRUID/src/homology.py b/DRUID/src/homology.py index ec33e09..0c3c0ed 100644 --- a/DRUID/src/homology.py +++ b/DRUID/src/homology.py @@ -324,9 +324,9 @@ def compute_homology( # filter out components with lifetime less than 3 polar_df = polar_df.filter(polar_df["lifetime"] > lifetime_limit_fraction) - print( - f"Filtered polar dataframe to {len(polar_df)} components with lifetime > {lifetime_limit_fraction}." - ) + # print( + # f"Filtered polar dataframe to {len(polar_df)} components with lifetime > {lifetime_limit_fraction}." + # ) # set the longest lifetime rows death to 0. polar_df = polar_df.with_columns( diff --git a/DRUID/src/source.py b/DRUID/src/source.py index 255db4c..dd6550b 100644 --- a/DRUID/src/source.py +++ b/DRUID/src/source.py @@ -62,35 +62,12 @@ def create_source_islands( print( f"Calculating region properties took {t1 - t0:.2f} seconds. Found {len(properties)} properties." ) - # filter out components smaller than 5 pixels - min_area = area_limit - t0 = time.time() - filtered_labels = [prop.label for prop in properties if prop.area >= min_area] - t1 = time.time() - if verbose: - print( - f"Filtering components by area took {t1 - t0:.2f} seconds. Found {len(filtered_labels)} components after filtering." - ) - t0 = time.time() - filtered_labeled_image = np.zeros_like(labeled_image) - for label_value in filtered_labels: - filtered_labeled_image[labeled_image == label_value] = label_value - t1 = time.time() - if verbose: - print( - f"Creating filtered labeled image took {t1 - t0:.2f} seconds. Filtered image has {np.unique(filtered_labeled_image).size - 1} components." - ) - labeled_image = filtered_labeled_image components = [] source_islands_positions = [] - t0 = time.time() - - # Calculate properties for all labeled regions - - # We pass thresholded_image as intensity_image to get the actual pixel values - props = regionprops(labeled_image, intensity_image=thresholded_image) - - for prop in props: + min_area = area_limit + for prop in properties: + if prop.area < min_area: + continue # prop.intensity_image is the cropped and masked component components.append(prop.intensity_image) diff --git a/test.py b/test.py index f3f3e12..8827ff5 100644 --- a/test.py +++ b/test.py @@ -9,11 +9,11 @@ def create_dummy_image(): import numpy as np # Create a dummy image with random data - data = np.random.normal(size=(10000, 10000)).astype(np.float32) + data = np.random.normal(size=(20000, 20000)).astype(np.float32) # add many bright sources - for _ in range(500): - x = np.random.randint(0, 10000) - y = np.random.randint(0, 10000) + for _ in range(10000): + x = np.random.randint(0, 20000) + y = np.random.randint(0, 20000) data[x, y] += np.random.uniform(500, 10000) # convolve the image with a Gaussian kernel to simulate a more realistic image @@ -67,7 +67,13 @@ def main(): create_dummy_image() # Create a dummy image for testing image_path = "DRUID/temp/dummy_image.fits" # image_path = "/Users/rs17612/Documents/Optical_IR_Data/EUCLID/EUC_MER_BGSUB-MOSAIC-VIS_TILE101158277-BB647A_20240122T115602.395130Z_00.00.fits" - findmysource = sf(image=image_path, mode="optical", area_limit=5, num_threads=1) + findmysource = sf( + image=image_path, + mode="optical", + area_limit=5, + num_threads=1, + working_directory="DRUID/temp", + ) findmysource.set_background() findmysource.phsf() From da66b33679c990617420863d1e6c788114d01f4b Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Tue, 15 Jul 2025 11:24:46 +0100 Subject: [PATCH 13/69] save important data --- .gitignore | 1 + DRUID/main.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/.gitignore b/.gitignore index 803c8f2..d5d4326 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ DRUID.egg-info build backup notepad.ipynb +temp \ No newline at end of file diff --git a/DRUID/main.py b/DRUID/main.py index d16d031..ca878e7 100644 --- a/DRUID/main.py +++ b/DRUID/main.py @@ -213,6 +213,12 @@ def set_background( kernel_size=3, ) ) + # Save the background maps to disk for future use. + np.save(self.working_directory + "/background_map.npy", self.background_map) + np.save( + self.working_directory + "/background_rms_map.npy", + self.background_rms_map, + ) t1 = time.time() print(f"Background calculation took {t1 - t0:.2f} seconds.") From e0cf45c271317d7e9b26e7b0a772009132c23bce Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Tue, 15 Jul 2025 11:47:55 +0100 Subject: [PATCH 14/69] multi proc fix by limiting polars threads --- DRUID/main.py | 48 +++++++++++++++++++++++++++--------------------- test.py | 17 ++++++++++------- 2 files changed, 37 insertions(+), 28 deletions(-) diff --git a/DRUID/main.py b/DRUID/main.py index ca878e7..8e5190d 100644 --- a/DRUID/main.py +++ b/DRUID/main.py @@ -6,10 +6,16 @@ import numpy as np import astropy +import os + +# this prevent polars from using all available threads. +# Especially for multithreaded homology computation. otherwise we will spawn nested threads. +os.environ["POLARS_MAX_THREADS"] = "1" import polars as pl + import time -import os -from multiprocessing import Pool + +from multiprocessing import get_context from tqdm import tqdm from .src import utils @@ -17,7 +23,6 @@ from .src import background from .src import source - DRUID_MESSAGE = """ ############################################# _______ _______ _________ ______ @@ -142,24 +147,25 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 2): return print(self.num_threads) - # if self.num_threads > 1: - # if self.verbose: - # print( - # f"Processing {len(images_to_process)} source islands in parallel. with {self.num_threads} threads." - # ) - # print("images to process:", len(images_to_process)) - # batch_size = len(images_to_process) // self.num_threads - # print(f"Batch size: {batch_size}") - # with Pool(self.num_threads) as p: - # results = p.map(_worker, images_to_process, chunksize=batch_size) - - # else: - print(f"Processing {len(images_to_process)} source islands sequentially.") - results = [] - for image in tqdm(images_to_process): - results.append(homology.compute_homology(image)) - - # combine the results catalogs to a single catalog + if self.num_threads > 1: + if self.verbose: + print( + f"Processing {len(images_to_process)} source islands in parallel. with {self.num_threads} threads." + ) + print("images to process:", len(images_to_process)) + batch_size = len(images_to_process) // self.num_threads + print(f"Batch size: {batch_size}") + with get_context("spawn").Pool(self.num_threads) as p: + results = p.map(_worker, images_to_process, chunksize=batch_size) + + else: + print(f"Processing {len(images_to_process)} source islands sequentially.") + results = [] + for image in tqdm(images_to_process): + results.append(homology.compute_homology(image)) + + # combine the results catalogs to a single catalog + if results: self.catalog = utils.combine_polars_catalogs(results) diff --git a/test.py b/test.py index 8827ff5..c9939da 100644 --- a/test.py +++ b/test.py @@ -1,7 +1,7 @@ from DRUID import sf -def create_dummy_image(): +def create_dummy_image(working_directory="DRUID/temp"): """ Create a dummy FITS image for testing purposes. """ @@ -11,7 +11,7 @@ def create_dummy_image(): # Create a dummy image with random data data = np.random.normal(size=(20000, 20000)).astype(np.float32) # add many bright sources - for _ in range(10000): + for _ in range(100000): x = np.random.randint(0, 20000) y = np.random.randint(0, 20000) data[x, y] += np.random.uniform(500, 10000) @@ -23,7 +23,7 @@ def create_dummy_image(): # Create a FITS file hdu = fits.PrimaryHDU(data) - hdu.writeto("DRUID/temp/dummy_image.fits", overwrite=True) + hdu.writeto(f"{working_directory}/dummy_image.fits", overwrite=True) # # plot the image to verify # import matplotlib.pyplot as plt @@ -64,15 +64,18 @@ def create_dummy_image(): def main(): - create_dummy_image() # Create a dummy image for testing - image_path = "DRUID/temp/dummy_image.fits" + working_dir = "/data/typhon2/Rhys/data/DRUID_TEST" + create_dummy_image( + working_directory=working_dir + ) # Create a dummy image for testing + image_path = f"{working_dir}/dummy_image.fits" # image_path = "/Users/rs17612/Documents/Optical_IR_Data/EUCLID/EUC_MER_BGSUB-MOSAIC-VIS_TILE101158277-BB647A_20240122T115602.395130Z_00.00.fits" findmysource = sf( image=image_path, mode="optical", area_limit=5, - num_threads=1, - working_directory="DRUID/temp", + num_threads=10, + working_directory=working_dir, ) findmysource.set_background() findmysource.phsf() From 9696304afa15a4d9904feab397be7d524a85bae3 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Tue, 15 Jul 2025 12:12:11 +0100 Subject: [PATCH 15/69] style on DRUID message --- DRUID/main.py | 76 ++++++++++++++++++++++++++++++--------------------- test.py | 11 +++++--- 2 files changed, 52 insertions(+), 35 deletions(-) diff --git a/DRUID/main.py b/DRUID/main.py index 8e5190d..0b623d2 100644 --- a/DRUID/main.py +++ b/DRUID/main.py @@ -12,7 +12,6 @@ # Especially for multithreaded homology computation. otherwise we will spawn nested threads. os.environ["POLARS_MAX_THREADS"] = "1" import polars as pl - import time from multiprocessing import get_context @@ -23,8 +22,15 @@ from .src import background from .src import source -DRUID_MESSAGE = """ -############################################# +RED = "\033[91m" +GREEN = "\033[92m" +BLUE = "\033[94m" +RESET = "\033[0m" +BOLD = "\033[1m" +DRUID_MESSAGE = rf""" + +{RED}#############################################{RESET} +{GREEN} _______ _______ _________ ______ ( __ \ ( ____ )|\ /|\__ __/( __ \ | ( \ )| ( )|| ) ( | ) ( | ( \ ) @@ -34,18 +40,16 @@ | (__/ )| ) \ \__| (___) |___) (___| (__/ ) (______/ |/ \__/(_______)\_______/(______/ - -############################################# +{RESET} +{RED}#############################################{RESET} -Detector of astRonomical soUrces in optIcal and raDio images +{BOLD}Detector of astRonomical soUrces in optIcal and raDio images{RESET} -Version: {} +Version: {version} For more information see: -https://github.com/RhysAlfShaw/DRUID - """.format( - version -) +{BLUE}https://github.com/RhysAlfShaw/DRUID{RESET} +""" def _worker(image: np.ndarray) -> "pl.DataFrame": @@ -66,6 +70,7 @@ def __init__( num_threads: int = 1, header: astropy.io.fits.header.Header = None, working_directory: str = "DRUID/temp", + cashe: bool = False, ): """ @@ -81,6 +86,7 @@ def __init__( self.smooth_sigma = smooth_sigma self.num_threads = num_threads self.header = header + self.cashe = cashe if image is None: raise ValueError( @@ -100,9 +106,12 @@ def __init__( ) # check if there are files in the working directory - if not os.path.exists(working_directory): - os.makedirs(working_directory) - self.working_directory = working_directory + if self.cashe: + if not os.path.exists(working_directory): + os.makedirs(working_directory) + self.working_directory = working_directory + else: + self.working_directory = None def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 2): """ @@ -193,18 +202,19 @@ def set_background( self.detection_threshold = detection_threshold self.analysis_threshold = analysis_threshold - if os.path.exists(self.working_directory + "/background_map.npy"): - if os.path.exists(self.working_directory + "/background_rms_map.npy"): - if self.verbose: - print( - "Background map and RMS map already exist. Loading from disk." + if self.cashe: + if os.path.exists(self.working_directory + "/background_map.npy"): + if os.path.exists(self.working_directory + "/background_rms_map.npy"): + if self.verbose: + print( + "Background map and RMS map already exist. Loading from disk." + ) + self.background_map = np.load( + self.working_directory + "/background_map.npy" + ) + self.background_rms_map = np.load( + self.working_directory + "/background_rms_map.npy" ) - self.background_map = np.load( - self.working_directory + "/background_map.npy" - ) - self.background_rms_map = np.load( - self.working_directory + "/background_rms_map.npy" - ) else: if self.verbose: @@ -219,12 +229,16 @@ def set_background( kernel_size=3, ) ) - # Save the background maps to disk for future use. - np.save(self.working_directory + "/background_map.npy", self.background_map) - np.save( - self.working_directory + "/background_rms_map.npy", - self.background_rms_map, - ) + + if self.cashe: + # Save the background maps to disk for future use. + np.save( + self.working_directory + "/background_map.npy", self.background_map + ) + np.save( + self.working_directory + "/background_rms_map.npy", + self.background_rms_map, + ) t1 = time.time() print(f"Background calculation took {t1 - t0:.2f} seconds.") diff --git a/test.py b/test.py index c9939da..db0bc6a 100644 --- a/test.py +++ b/test.py @@ -9,11 +9,13 @@ def create_dummy_image(working_directory="DRUID/temp"): import numpy as np # Create a dummy image with random data - data = np.random.normal(size=(20000, 20000)).astype(np.float32) + dim = 10_000 # 20,000 x 20,000 pixels + n_sources = 50_000 # Number of bright sources to add + data = np.random.normal(size=(dim, dim)).astype(np.float32) # add many bright sources - for _ in range(100000): - x = np.random.randint(0, 20000) - y = np.random.randint(0, 20000) + for _ in range(n_sources): + x = np.random.randint(0, dim) + y = np.random.randint(0, dim) data[x, y] += np.random.uniform(500, 10000) # convolve the image with a Gaussian kernel to simulate a more realistic image @@ -76,6 +78,7 @@ def main(): area_limit=5, num_threads=10, working_directory=working_dir, + cashe=False, ) findmysource.set_background() findmysource.phsf() From 3a840395b48090af8a9c4249b4ede1c02a00cb8e Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Wed, 16 Jul 2025 09:55:23 +0100 Subject: [PATCH 16/69] fixed loading images when dim>2 --- DRUID/src/utils.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/DRUID/src/utils.py b/DRUID/src/utils.py index 3bffdee..16ac593 100644 --- a/DRUID/src/utils.py +++ b/DRUID/src/utils.py @@ -20,6 +20,12 @@ def get_image_from_path(image_path): with fits.open(image_path) as hdul: image = hdul[0].data + # warn if the image is not 2D + # reduce the image to 2D if it is not + if image.ndim == 3: + image = image[0, :, :] + elif image.ndim == 4: + image = image[0, 0, :, :] return image From ffa7fdd2c789bb36d59238a67e60b205a61d62dd Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Wed, 16 Jul 2025 09:55:49 +0100 Subject: [PATCH 17/69] fixed caching issue, when data no makde before --- DRUID/main.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/DRUID/main.py b/DRUID/main.py index 0b623d2..11496d6 100644 --- a/DRUID/main.py +++ b/DRUID/main.py @@ -215,6 +215,29 @@ def set_background( self.background_rms_map = np.load( self.working_directory + "/background_rms_map.npy" ) + else: + if self.verbose: + print( + "Background map and RMS map do not exist. Calculating from image." + ) + self.background_map, self.background_rms_map = ( + background.calculate_background_maps( + self.image, + bg_estimator=method, + box_size=box_size, + filter_size=filter_size, + nsigma=detection_threshold, + kernel_size=kernel_size, + ) + ) + # Save the background maps to disk for future use. + np.save( + self.working_directory + "/background_map.npy", self.background_map + ) + np.save( + self.working_directory + "/background_rms_map.npy", + self.background_rms_map, + ) else: if self.verbose: From bb046c5cd5665294b959d15e9172152412f99261 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Wed, 16 Jul 2025 12:32:51 +0100 Subject: [PATCH 18/69] added correct lifetime filtering for better nested feature extraction --- DRUID/main.py | 49 +++++++++++++++++++++++++++++++++++++------ DRUID/src/homology.py | 31 +++++++++++++++++---------- DRUID/src/source.py | 12 +++++++++++ 3 files changed, 75 insertions(+), 17 deletions(-) diff --git a/DRUID/main.py b/DRUID/main.py index 11496d6..5249b17 100644 --- a/DRUID/main.py +++ b/DRUID/main.py @@ -21,6 +21,7 @@ from .src import homology from .src import background from .src import source +from functools import partial RED = "\033[91m" GREEN = "\033[92m" @@ -52,11 +53,25 @@ """ -def _worker(image: np.ndarray) -> "pl.DataFrame": +def _worker( + iterable_image, analysis_threshold, lifetime_limit, lifetime_limit_fraction +) -> "pl.DataFrame": """ Worker function to compute homology for a single source island. """ - return homology.compute_homology(image) + image, position, background, background_rms = iterable_image + cat = homology.compute_homology( + image, + analysis_threshold=analysis_threshold * background_rms, + lifetime_limit=lifetime_limit, + lifetime_limit_fraction=lifetime_limit_fraction, + ) + # Add position to the catalog + cat = cat.with_columns( + pl.lit(position[0]).alias("Island_X"), + pl.lit(position[1]).alias("Island_Y"), + ) + return cat class sf: @@ -113,7 +128,7 @@ def __init__( else: self.working_directory = None - def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 2): + def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1): """ Runs the source findin algorithm on the image. @@ -156,6 +171,14 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 2): return print(self.num_threads) + # make the iterable images_to_process and poistions + iterable_images = zip( + images_to_process, + source_islands["positions"], + source_islands["background"], + source_islands["background_rms"], + ) + if self.num_threads > 1: if self.verbose: print( @@ -165,13 +188,27 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 2): batch_size = len(images_to_process) // self.num_threads print(f"Batch size: {batch_size}") with get_context("spawn").Pool(self.num_threads) as p: - results = p.map(_worker, images_to_process, chunksize=batch_size) + # Use functools.partial to pass additional arguments to _worker + worker_func = partial( + _worker, # analysis threshold * rms at this point. + analysis_threshold=self.analysis_threshold, + lifetime_limit=lifetime_limit, + lifetime_limit_fraction=lifetime_limit_fraction, + ) + results = p.map(worker_func, iterable_images, chunksize=batch_size) else: print(f"Processing {len(images_to_process)} source islands sequentially.") results = [] - for image in tqdm(images_to_process): - results.append(homology.compute_homology(image)) + for img, position, background, background_rms in tqdm(iterable_images): + results.append( + _worker( + (img, position, background, background_rms), + self.analysis_threshold, + lifetime_limit, + lifetime_limit_fraction, + ) + ) # combine the results catalogs to a single catalog diff --git a/DRUID/src/homology.py b/DRUID/src/homology.py index 0c3c0ed..82daee4 100644 --- a/DRUID/src/homology.py +++ b/DRUID/src/homology.py @@ -268,6 +268,8 @@ def get_mask_CPU(x1, y1, Birth, Death, img): def compute_homology( img: np.ndarray, + analysis_threshold: float, + lifetime_limit: float = None, lifetime_limit_fraction: float = 1.0, area_size_threshold: int = 2, ) -> pl.DataFrame: @@ -306,27 +308,33 @@ def compute_homology( # drop cols dim, z1, z2 polar_df = polar_df.drop(["dim", "z1", "z2"]) # create ne column lifetime death - birth - polar_df = polar_df.with_columns( - (polar_df["death"] - polar_df["birth"]).alias("lifetime") - ) # make column birth and death - birth and death. polar_df = polar_df.with_columns( [(-polar_df["birth"]).alias("birth"), (-polar_df["death"]).alias("death")] ) + # set the death column to atleast the analysis threshold value + print(analysis_threshold) + polar_df = polar_df.with_columns( + pl.when(pl.col("death") < analysis_threshold) + .then(pl.lit(analysis_threshold)) + .otherwise(pl.col("death")) + .alias("death") + ) + + polar_df = polar_df.with_columns( + (abs(polar_df["death"] - polar_df["birth"])).alias("lifetime") + ) # lifetime_threshold. this is setby the user. polar_df = polar_df.with_columns( - (polar_df["birth"] - polar_df["death"]).alias("lifetimeFrac") + (polar_df["birth"] / polar_df["death"]).alias("lifetimeFrac") ) - # liftetime_limit_fraction = 1.0 # set the lifetime limit fraction # filter out components with lifetime less than 3 - polar_df = polar_df.filter(polar_df["lifetime"] > lifetime_limit_fraction) - # print( - # f"Filtered polar dataframe to {len(polar_df)} components with lifetime > {lifetime_limit_fraction}." - # ) + polar_df = polar_df.filter(polar_df["lifetimeFrac"] > lifetime_limit_fraction) + polar_df = polar_df.filter(polar_df["lifetime"] > lifetime_limit) # set the longest lifetime rows death to 0. polar_df = polar_df.with_columns( @@ -435,7 +443,7 @@ def compute_homology( for contour in contours ] polar_df = polar_df.with_columns(pl.Series("contour", contours)) - + print(f"Computed {len(polar_df)} components with contours.") return polar_df @@ -507,7 +515,7 @@ def compute_homology( # Where the img cut out is used for the computation of the persistent homology. # We will use the first component for now. - img = components[0] + img = components[2] polar_df = compute_homology(img) print("Contours computed.") print(polar_df) @@ -524,4 +532,5 @@ def compute_homology( contour[:, 1], contour[:, 0], color="red", alpha=0.5, linewidth=5 ) # Plot y, x for correct orientation plt.colorbar() + plt.savefig("DRUID/temp/component_with_contours.png") plt.show() diff --git a/DRUID/src/source.py b/DRUID/src/source.py index dd6550b..fa936c1 100644 --- a/DRUID/src/source.py +++ b/DRUID/src/source.py @@ -64,6 +64,8 @@ def create_source_islands( ) components = [] source_islands_positions = [] + source_island_bg_rms = [] + source_island_bg = [] min_area = area_limit for prop in properties: if prop.area < min_area: @@ -74,15 +76,25 @@ def create_source_islands( # prop.bbox returns (min_row, min_col, max_row, max_col) y_min, x_min, _, _ = prop.bbox source_islands_positions.append((y_min, x_min)) + # Get the background and RMS values for the component + source_island_bg.append( + background_map[y_min : prop.bbox[2], x_min : prop.bbox[3]].mean() + ) + source_island_bg_rms.append( + background_rms_map[y_min : prop.bbox[2], x_min : prop.bbox[3]].mean() + ) t1 = time.time() if verbose: print( f"Cropping components took {t1 - t0:.2f} seconds. Found {len(components)} source islands." ) + source_islands = { "island_image": components, "positions": source_islands_positions, + "background": source_island_bg, + "background_rms": source_island_bg_rms, } return source_islands From 383c10a158e2de52a7cc0ca120264c15e03f2a52 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Wed, 16 Jul 2025 14:19:58 +0100 Subject: [PATCH 19/69] updated homology --- DRUID/src/homology.py | 4 +- test.py | 182 ++++++++++++++++++++++++++---------------- 2 files changed, 116 insertions(+), 70 deletions(-) diff --git a/DRUID/src/homology.py b/DRUID/src/homology.py index 82daee4..dcd84d3 100644 --- a/DRUID/src/homology.py +++ b/DRUID/src/homology.py @@ -313,7 +313,7 @@ def compute_homology( [(-polar_df["birth"]).alias("birth"), (-polar_df["death"]).alias("death")] ) # set the death column to atleast the analysis threshold value - print(analysis_threshold) + # print(analysis_threshold) polar_df = polar_df.with_columns( pl.when(pl.col("death") < analysis_threshold) .then(pl.lit(analysis_threshold)) @@ -443,7 +443,7 @@ def compute_homology( for contour in contours ] polar_df = polar_df.with_columns(pl.Series("contour", contours)) - print(f"Computed {len(polar_df)} components with contours.") + # print(f"Computed {len(polar_df)} components with contours.") return polar_df diff --git a/test.py b/test.py index db0bc6a..e3e734f 100644 --- a/test.py +++ b/test.py @@ -1,82 +1,82 @@ from DRUID import sf -def create_dummy_image(working_directory="DRUID/temp"): - """ - Create a dummy FITS image for testing purposes. - """ - from astropy.io import fits - import numpy as np - - # Create a dummy image with random data - dim = 10_000 # 20,000 x 20,000 pixels - n_sources = 50_000 # Number of bright sources to add - data = np.random.normal(size=(dim, dim)).astype(np.float32) - # add many bright sources - for _ in range(n_sources): - x = np.random.randint(0, dim) - y = np.random.randint(0, dim) - data[x, y] += np.random.uniform(500, 10000) - - # convolve the image with a Gaussian kernel to simulate a more realistic image - from scipy.ndimage import gaussian_filter - - data = gaussian_filter(data, sigma=5) - - # Create a FITS file - hdu = fits.PrimaryHDU(data) - hdu.writeto(f"{working_directory}/dummy_image.fits", overwrite=True) - - # # plot the image to verify - # import matplotlib.pyplot as plt - - # img_size = [2000, 3000, 5000, 10000] - # bg_time = [0.79, 1.7, 4.35, 17.36] - # thresh_time = [5.74, 16.2, 50, 60 * 7] - # # fit exponential curves to the data - # from scipy.optimize import curve_fit - - # def exp_func(x, a, b): - # return a * np.exp(b * x) - - # popt_bg, _ = curve_fit(exp_func, img_size, bg_time) - # popt_thresh, _ = curve_fit(exp_func, img_size, thresh_time) - - # plt.plot(img_size, bg_time, label="Background Calculation Time") - # plt.plot(img_size, thresh_time, label="Thresholding Time") - # extrapolated_img_size = np.linspace(0, 20000, 100) - # plt.plot( - # extrapolated_img_size, - # exp_func(np.array(extrapolated_img_size), *popt_bg), - # linestyle="--", - # color="blue", - # ) - - # plt.plot( - # extrapolated_img_size, - # exp_func(np.array(extrapolated_img_size), *popt_thresh), - # linestyle="--", - # color="orange", - # ) - # plt.legend() - # plt.xlabel("Image Size (pixels)") - # plt.ylabel("Time (seconds)") - # plt.title("Background Calculation and Thresholding Time vs Image Size") - # plt.show() +# def create_dummy_image(working_directory="DRUID/temp"): +# """ +# Create a dummy FITS image for testing purposes. +# """ +# from astropy.io import fits +# import numpy as np + +# # Create a dummy image with random data +# dim = 10_000 # 20,000 x 20,000 pixels +# n_sources = 50_000 # Number of bright sources to add +# data = np.random.normal(size=(dim, dim)).astype(np.float32) +# # add many bright sources +# for _ in range(n_sources): +# x = np.random.randint(0, dim) +# y = np.random.randint(0, dim) +# data[x, y] += np.random.uniform(500, 10000) + +# # convolve the image with a Gaussian kernel to simulate a more realistic image +# from scipy.ndimage import gaussian_filter + +# data = gaussian_filter(data, sigma=5) + +# # Create a FITS file +# hdu = fits.PrimaryHDU(data) +# hdu.writeto(f"{working_directory}/dummy_image.fits", overwrite=True) + +# # # plot the image to verify +# import matplotlib.pyplot as plt + +# img_size = [2000, 3000, 5000, 10000] +# bg_time = [0.79, 1.7, 4.35, 17.36] +# thresh_time = [5.74, 16.2, 50, 60 * 7] +# # fit exponential curves to the data +# from scipy.optimize import curve_fit + +# def exp_func(x, a, b): +# return a * np.exp(b * x) + +# popt_bg, _ = curve_fit(exp_func, img_size, bg_time) +# popt_thresh, _ = curve_fit(exp_func, img_size, thresh_time) + +# plt.plot(img_size, bg_time, label="Background Calculation Time") +# plt.plot(img_size, thresh_time, label="Thresholding Time") +# extrapolated_img_size = np.linspace(0, 20000, 100) +# plt.plot( +# extrapolated_img_size, +# exp_func(np.array(extrapolated_img_size), *popt_bg), +# linestyle="--", +# color="blue", +# ) + +# plt.plot( +# extrapolated_img_size, +# exp_func(np.array(extrapolated_img_size), *popt_thresh), +# linestyle="--", +# color="orange", +# ) +# plt.legend() +# plt.xlabel("Image Size (pixels)") +# plt.ylabel("Time (seconds)") +# plt.title("Background Calculation and Thresholding Time vs Image Size") +# plt.show() def main(): - working_dir = "/data/typhon2/Rhys/data/DRUID_TEST" - create_dummy_image( - working_directory=working_dir - ) # Create a dummy image for testing - image_path = f"{working_dir}/dummy_image.fits" + working_dir = "DRUID/temp" + # create_dummy_image( + # working_directory=working_dir + # ) # Create a dummy image for testing + image_path = "DRUID/temp/dummy_image.fits" # image_path = "/Users/rs17612/Documents/Optical_IR_Data/EUCLID/EUC_MER_BGSUB-MOSAIC-VIS_TILE101158277-BB647A_20240122T115602.395130Z_00.00.fits" findmysource = sf( image=image_path, mode="optical", area_limit=5, - num_threads=10, + num_threads=1, working_directory=working_dir, cashe=False, ) @@ -87,6 +87,52 @@ def main(): catalog = findmysource.catalog print("Catalog:", catalog) + # plot the image, background, and catalog + import matplotlib.pyplot as plt + import numpy as np + + plt.figure(figsize=(10, 10)) + plt.imshow(findmysource.image, cmap="gray", origin="lower") + plt.scatter( + catalog["y1"] + catalog["Island_Y"], + catalog["x1"] + catalog["Island_X"], + s=1, + c="red", + label="Source Islands", + ) + plt.colorbar() + plt.title("Source Islands on Image") + plt.xlabel("X Pixel") + plt.ylabel("Y Pixel") + plt.legend() + plt.savefig(f"{working_dir}/source_islands_on_image.png") + plt.show() + + # plot the contours + contours = catalog["contour"].to_list() + Island_X = catalog["Island_Y"].to_list() + Island_Y = catalog["Island_X"].to_list() + + plt.figure(figsize=(10, 10)) + plt.imshow(findmysource.image, cmap="gray", origin="lower") + for i, contour in enumerate(contours): + contour = np.array(contour) + Island_X_val = Island_X[i] + Island_Y_val = Island_Y[i] + plt.plot( + contour[:, 1] + Island_X_val, + contour[:, 0] + Island_Y_val, + color="red", + alpha=0.5, + linewidth=0.5, + ) + plt.colorbar() + plt.title("Contours of Source Islands") + plt.xlabel("X Pixel") + plt.ylabel("Y Pixel") + plt.savefig(f"{working_dir}/source_islands_contours.png") + plt.show() + if __name__ == "__main__": main() From 9a1505112d486285288cee86763b96565d16d0e7 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Mon, 8 Sep 2025 12:29:43 +0100 Subject: [PATCH 20/69] fixed corr dest function: renamed columns --- DRUID/src/homology.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/DRUID/src/homology.py b/DRUID/src/homology.py index dcd84d3..318dd69 100644 --- a/DRUID/src/homology.py +++ b/DRUID/src/homology.py @@ -76,7 +76,8 @@ def correct_first_destruction_pl(df: pl.DataFrame) -> pl.DataFrame: if "new_row" not in df.columns: df = df.with_columns(pl.lit(0, dtype=pl.Int8).alias("new_row")) - # 1. Filter the DataFrame to find all rows that have enclosed islands. + # 1. Filter the DataFrame to find all rows that have enclosed islands + islands_to_split = df.filter(pl.col("encloses").list.len() > 1) # If no such rows exist, return the original DataFrame. @@ -95,6 +96,7 @@ def correct_first_destruction_pl(df: pl.DataFrame) -> pl.DataFrame: # Suffix prevents column name collisions ('Death' becomes 'Death_parent') suffix="_parent", ) + print("New Rows Base: ", new_rows_base) # If the join results in an empty DataFrame, return the original. if new_rows_base.is_empty(): @@ -118,13 +120,13 @@ def correct_first_destruction_pl(df: pl.DataFrame) -> pl.DataFrame: # Overwrite the original ID with the new unique ID ID=new_ids, # Update 'Death' with the value from the joined parent - Death=pl.col("death_parent"), + death=pl.col("death_parent"), # Set 'parent_tag' to the ID of the parent island parent_tag=pl.col("ID_parent"), # Mark this as a newly generated row new_row=pl.lit(1, dtype=pl.Int8), # Set 'enclosed_i' to an empty list - enclosed_i=pl.lit(None, dtype=df.schema["encloses"]), + encloses=pl.lit(None, dtype=df.schema["encloses"]), ) # Remove temporary columns created by the join .drop(["ID_parent", "death_parent"]) From 949cefc9b5ffa73a450e714775aac2f498b93532 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Mon, 8 Sep 2025 12:31:15 +0100 Subject: [PATCH 21/69] adjusted commets --- DRUID/src/homology.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/DRUID/src/homology.py b/DRUID/src/homology.py index 318dd69..ee45791 100644 --- a/DRUID/src/homology.py +++ b/DRUID/src/homology.py @@ -223,29 +223,26 @@ def get_enclosing_mask_CPU(x, y, mask): labeled_mask, num_features = scipy_label(mask) - # Check if the specified pixel is within the mask + # check if the specified pixel is within the mask if 0 <= x < mask.shape[1] and 0 <= y < mask.shape[0]: label_at_pixel = labeled_mask[y, x] if label_at_pixel != 0: - # Extract the connected component containing the specified pixel + # get the connected component containing the specified pixel component_mask = labeled_mask == label_at_pixel return component_mask else: - # The specified pixel is not part of any connected component + # the specified pixel is not part of any connected component return None else: - # The specified pixel is outside the mask + # the specified pixel is outside the mask return None def bounding_box_cpu(mask): - # Get the indices of elements that are True rows, cols = np.where(mask) - # Get the minimum and maximum x and y coordinates min_y, max_y = np.min(rows), np.max(rows) min_x, max_x = np.min(cols), np.max(cols) - # Return the bounding box as a tuple of tuples return min_y, min_x, max_y, max_x From 4bf0ef6fa697e561c2fe45342bde4defbc592792 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Mon, 8 Sep 2025 12:34:59 +0100 Subject: [PATCH 22/69] temporailty removed failing tests --- DRUID/tests/test_background.py | 96 +++++++++++++++++----------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/DRUID/tests/test_background.py b/DRUID/tests/test_background.py index d0feeeb..b85edf4 100644 --- a/DRUID/tests/test_background.py +++ b/DRUID/tests/test_background.py @@ -51,54 +51,54 @@ def test_make_source_mask_with_sources(dummy_fits_file_with_source): assert np.any(mask) # Expect some sources to be masked -def test_calculate_background_maps_defaults(dummy_fits_file_with_source): - """Test calculate_background_maps with default parameters.""" - background_map, background_rms_map = calculate_background_maps( - dummy_fits_file_with_source - ) - with fits.open(dummy_fits_file_with_source) as hdul: - data_shape = hdul[0].data.shape - - assert background_map.shape == data_shape - assert background_rms_map.shape == data_shape - assert isinstance(background_map, np.ndarray) - assert isinstance(background_rms_map, np.ndarray) - - -def test_calculate_background_maps_custom_estimator_str(dummy_fits_file_with_source): - """Test calculate_background_maps with a string-specified background estimator.""" - background_map, background_rms_map = calculate_background_maps( - dummy_fits_file_with_source, bg_estimator="mean" - ) - with fits.open(dummy_fits_file_with_source) as hdul: - data_shape = hdul[0].data.shape - assert background_map.shape == data_shape - assert background_rms_map.shape == data_shape - - -def test_calculate_background_maps_custom_estimator_obj(dummy_fits_file_with_source): - """Test calculate_background_maps with a BackgroundBase object estimator.""" - custom_estimator = MedianBackground() - background_map, background_rms_map = calculate_background_maps( - dummy_fits_file_with_source, bg_estimator=custom_estimator - ) - with fits.open(dummy_fits_file_with_source) as hdul: - data_shape = hdul[0].data.shape - assert background_map.shape == data_shape - assert background_rms_map.shape == data_shape - - -def test_calculate_background_maps_invalid_estimator_str(dummy_fits_file_with_source): - """Test calculate_background_maps with an invalid string-specified background estimator, - expecting it to default to MedianBackground.""" - background_map, background_rms_map = calculate_background_maps( - dummy_fits_file_with_source, bg_estimator="not_an_estimator" - ) - with fits.open(dummy_fits_file_with_source) as hdul: - data_shape = hdul[0].data.shape - assert background_map.shape == data_shape - assert background_rms_map.shape == data_shape - # Further checks could involve inspecting the bkg_estimator used if it were returned or logged +# def test_calculate_background_maps_defaults(dummy_fits_file_with_source): +# """Test calculate_background_maps with default parameters.""" +# background_map, background_rms_map = calculate_background_maps( +# dummy_fits_file_with_source +# ) +# with fits.open(dummy_fits_file_with_source) as hdul: +# data_shape = hdul[0].data.shape + +# assert background_map.shape == data_shape +# assert background_rms_map.shape == data_shape +# assert isinstance(background_map, np.ndarray) +# assert isinstance(background_rms_map, np.ndarray) + + +# def test_calculate_background_maps_custom_estimator_str(dummy_fits_file_with_source): +# """Test calculate_background_maps with a string-specified background estimator.""" +# background_map, background_rms_map = calculate_background_maps( +# dummy_fits_file_with_source, bg_estimator="mean" +# ) +# with fits.open(dummy_fits_file_with_source) as hdul: +# data_shape = hdul[0].data.shape +# assert background_map.shape == data_shape +# assert background_rms_map.shape == data_shape + + +# def test_calculate_background_maps_custom_estimator_obj(dummy_fits_file_with_source): +# """Test calculate_background_maps with a BackgroundBase object estimator.""" +# custom_estimator = MedianBackground() +# background_map, background_rms_map = calculate_background_maps( +# dummy_fits_file_with_source, bg_estimator=custom_estimator +# ) +# with fits.open(dummy_fits_file_with_source) as hdul: +# data_shape = hdul[0].data.shape +# assert background_map.shape == data_shape +# assert background_rms_map.shape == data_shape + + +# def test_calculate_background_maps_invalid_estimator_str(dummy_fits_file_with_source): +# """Test calculate_background_maps with an invalid string-specified background estimator, +# expecting it to default to MedianBackground.""" +# background_map, background_rms_map = calculate_background_maps( +# dummy_fits_file_with_source, bg_estimator="not_an_estimator" +# ) +# with fits.open(dummy_fits_file_with_source) as hdul: +# data_shape = hdul[0].data.shape +# assert background_map.shape == data_shape +# assert background_rms_map.shape == data_shape +# # Further checks could involve inspecting the bkg_estimator used if it were returned or logged def test_calculate_background_maps_file_not_found(tmp_path): From 8b04af6d4e772371dabdddd886b5d238f1d76b78 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Mon, 8 Sep 2025 13:52:33 +0100 Subject: [PATCH 23/69] prevent a zero batchsize --- DRUID/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/DRUID/main.py b/DRUID/main.py index 5249b17..d5e64f3 100644 --- a/DRUID/main.py +++ b/DRUID/main.py @@ -186,6 +186,8 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1): ) print("images to process:", len(images_to_process)) batch_size = len(images_to_process) // self.num_threads + if batch_size < 1: # prevent batch size of 0 + batch_size = 1 print(f"Batch size: {batch_size}") with get_context("spawn").Pool(self.num_threads) as p: # Use functools.partial to pass additional arguments to _worker From c40b13155b298875313e52b1491da9c5fa9fdbcb Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Mon, 8 Sep 2025 13:53:02 +0100 Subject: [PATCH 24/69] allow for no sources to be detected --- DRUID/src/background.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/DRUID/src/background.py b/DRUID/src/background.py index dfcb438..1f3d8fd 100644 --- a/DRUID/src/background.py +++ b/DRUID/src/background.py @@ -44,7 +44,9 @@ def make_source_mask(data, nsigma=3.0, kernel_size=3): # Detect sources using a simple thresholding method can add masked pixels e.g. known bad areas of image. segm = detect_sources(data, threshold, npixels=kernel_size**2) - + if segm is None: + # No sources detected, return an empty mask + return np.zeros(data.shape, dtype=bool) # Create a mask from the segmentation map mask = segm.data > 0 From 23858548bd42deaead5415516446c5d12448c61b Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Mon, 8 Sep 2025 13:53:32 +0100 Subject: [PATCH 25/69] refined background.py tests --- DRUID/tests/test_background.py | 98 +++++++++++++++++----------------- 1 file changed, 50 insertions(+), 48 deletions(-) diff --git a/DRUID/tests/test_background.py b/DRUID/tests/test_background.py index b85edf4..bc30049 100644 --- a/DRUID/tests/test_background.py +++ b/DRUID/tests/test_background.py @@ -51,54 +51,56 @@ def test_make_source_mask_with_sources(dummy_fits_file_with_source): assert np.any(mask) # Expect some sources to be masked -# def test_calculate_background_maps_defaults(dummy_fits_file_with_source): -# """Test calculate_background_maps with default parameters.""" -# background_map, background_rms_map = calculate_background_maps( -# dummy_fits_file_with_source -# ) -# with fits.open(dummy_fits_file_with_source) as hdul: -# data_shape = hdul[0].data.shape - -# assert background_map.shape == data_shape -# assert background_rms_map.shape == data_shape -# assert isinstance(background_map, np.ndarray) -# assert isinstance(background_rms_map, np.ndarray) - - -# def test_calculate_background_maps_custom_estimator_str(dummy_fits_file_with_source): -# """Test calculate_background_maps with a string-specified background estimator.""" -# background_map, background_rms_map = calculate_background_maps( -# dummy_fits_file_with_source, bg_estimator="mean" -# ) -# with fits.open(dummy_fits_file_with_source) as hdul: -# data_shape = hdul[0].data.shape -# assert background_map.shape == data_shape -# assert background_rms_map.shape == data_shape - - -# def test_calculate_background_maps_custom_estimator_obj(dummy_fits_file_with_source): -# """Test calculate_background_maps with a BackgroundBase object estimator.""" -# custom_estimator = MedianBackground() -# background_map, background_rms_map = calculate_background_maps( -# dummy_fits_file_with_source, bg_estimator=custom_estimator -# ) -# with fits.open(dummy_fits_file_with_source) as hdul: -# data_shape = hdul[0].data.shape -# assert background_map.shape == data_shape -# assert background_rms_map.shape == data_shape - - -# def test_calculate_background_maps_invalid_estimator_str(dummy_fits_file_with_source): -# """Test calculate_background_maps with an invalid string-specified background estimator, -# expecting it to default to MedianBackground.""" -# background_map, background_rms_map = calculate_background_maps( -# dummy_fits_file_with_source, bg_estimator="not_an_estimator" -# ) -# with fits.open(dummy_fits_file_with_source) as hdul: -# data_shape = hdul[0].data.shape -# assert background_map.shape == data_shape -# assert background_rms_map.shape == data_shape -# # Further checks could involve inspecting the bkg_estimator used if it were returned or logged +def test_calculate_background_maps_defaults(dummy_fits_file_with_source): + """Test calculate_background_maps with default parameters.""" + + background_map, background_rms_map = calculate_background_maps( + str(dummy_fits_file_with_source) + ) + + with fits.open(dummy_fits_file_with_source) as hdul: + data_shape = hdul[0].data.shape + + assert background_map.shape == data_shape + assert background_rms_map.shape == data_shape + assert isinstance(background_map, np.ndarray) + assert isinstance(background_rms_map, np.ndarray) + + +def test_calculate_background_maps_custom_estimator_str(dummy_fits_file_with_source): + """Test calculate_background_maps with a string-specified background estimator.""" + background_map, background_rms_map = calculate_background_maps( + str(dummy_fits_file_with_source), bg_estimator="mean" + ) + with fits.open(dummy_fits_file_with_source) as hdul: + data_shape = hdul[0].data.shape + assert background_map.shape == data_shape + assert background_rms_map.shape == data_shape + + +def test_calculate_background_maps_custom_estimator_obj(dummy_fits_file_with_source): + """Test calculate_background_maps with a BackgroundBase object estimator.""" + custom_estimator = MedianBackground() + background_map, background_rms_map = calculate_background_maps( + str(dummy_fits_file_with_source), bg_estimator=custom_estimator + ) + with fits.open(dummy_fits_file_with_source) as hdul: + data_shape = hdul[0].data.shape + assert background_map.shape == data_shape + assert background_rms_map.shape == data_shape + + +def test_calculate_background_maps_invalid_estimator_str(dummy_fits_file_with_source): + """Test calculate_background_maps with an invalid string-specified background estimator, + expecting it to default to MedianBackground.""" + background_map, background_rms_map = calculate_background_maps( + str(dummy_fits_file_with_source), bg_estimator="not_an_estimator" + ) + with fits.open(dummy_fits_file_with_source) as hdul: + data_shape = hdul[0].data.shape + assert background_map.shape == data_shape + assert background_rms_map.shape == data_shape + # Further checks could involve inspecting the bkg_estimator used if it were returned or logged def test_calculate_background_maps_file_not_found(tmp_path): From 4fdfa93bfc08e21e9aa79ea1f1238fa1f9483f33 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Mon, 8 Sep 2025 13:53:52 +0100 Subject: [PATCH 26/69] homology.py test --- DRUID/tests/test_homology.py | 163 +++++++++++++++++++++++++++++++++++ 1 file changed, 163 insertions(+) diff --git a/DRUID/tests/test_homology.py b/DRUID/tests/test_homology.py index e69de29..31e6511 100644 --- a/DRUID/tests/test_homology.py +++ b/DRUID/tests/test_homology.py @@ -0,0 +1,163 @@ +import pytest +import numpy as np +import polars as pl +from polars.testing import assert_frame_equal + +from DRUID.src.homology import ( + compute_homology, + get_mask_CPU, + get_enclosing_mask_CPU, + bounding_box_cpu, + parent_tag_func_pl, + correct_first_destruction_pl, +) +from DRUID.src.background import make_gaussian_sources_image + + +@pytest.fixture +def simple_image(): + """Creates a simple 100x100 image with one Gaussian source.""" + image_size = (100, 100) + sources = [ + { + "amplitude": 100, + "x_mean": 50, + "y_mean": 50, + "x_stddev": 5, + "y_stddev": 5, + "theta": 0, + } + ] + return make_gaussian_sources_image(image_size, sources) + 0.1 + + +@pytest.fixture +def nested_source_image(): + """Creates an image with two nested Gaussian sources.""" + image_size = (100, 100) + sources = [ + { + "amplitude": 100, + "x_mean": 50, + "y_mean": 50, + "x_stddev": 10, + "y_stddev": 10, + "theta": 0, + }, + { + "amplitude": 50, + "x_mean": 50, + "y_mean": 50, + "x_stddev": 3, + "y_stddev": 3, + "theta": 0, + }, + ] + return make_gaussian_sources_image(image_size, sources) + 0.1 + + +def test_compute_homology_simple_source(simple_image): + """Test compute_homology on an image with a single, simple source.""" + result_df = compute_homology( + simple_image, analysis_threshold=1.0, lifetime_limit=0.1 + ) + + assert isinstance(result_df, pl.DataFrame) + assert not result_df.is_empty() + assert result_df["birth"].max() == pytest.approx(100, abs=1) + expected_cols = { + "birth", + "death", + "x1", + "y1", + "lifetime", + "lifetimeFrac", + "area", + "bbox_min_y", + "ID", + "encloses", + "parent_tag", + "contour", + } + assert expected_cols.issubset(result_df.columns) + + +def test_get_mask_cpu(): + """Test the get_mask_CPU function.""" + img = np.array([[0, 0, 0, 0], [0, 5, 5, 0], [0, 5, 5, 0], [0, 0, 0, 0]]) + mask = get_mask_CPU(x1=1, y1=1, Birth=6, Death=4, img=img) + expected_mask = np.array( + [ + [False, False, False, False], + [False, True, True, False], + [False, True, True, False], + [False, False, False, False], + ] + ) + assert np.array_equal(mask, expected_mask) + + +def test_get_enclosing_mask_cpu(): + """Test the get_enclosing_mask_CPU function.""" + mask = np.array( + [[0, 1, 1, 0], [0, 1, 1, 0], [0, 0, 0, 0], [0, 1, 0, 0]], dtype=bool + ) + component_mask = get_enclosing_mask_CPU(x=1, y=1, mask=mask) + expected = np.array( + [ + [False, True, True, False], + [False, True, True, False], + [False, False, False, False], + [False, False, False, False], + ] + ) + assert np.array_equal(component_mask, expected) + + component_mask_none = get_enclosing_mask_CPU(x=0, y=0, mask=mask) + assert component_mask_none is None + + +def test_bounding_box_cpu(): + """Test the bounding_box_cpu function.""" + mask = np.zeros((10, 10), dtype=bool) + mask[2:5, 3:7] = True + bbox = bounding_box_cpu(mask) + assert bbox == (2, 3, 4, 6) + + +def test_parent_tag_func_pl(): + """Test the parent_tag_func_pl function.""" + df = pl.DataFrame( + { + "ID": [0, 1, 2, 3], + "encloses": [[1, 2], [], [], [0]], + } + ) + result = parent_tag_func_pl(df) + expected = pl.DataFrame( + { + "ID": [0, 1, 2, 3], + "encloses": [[1, 2], [], [], [0]], + "parent_tag": [0, 0, 0, 3], + } + ) + assert_frame_equal(result, expected) + + +def test_correct_first_destruction_pl(): + """Test the correct_first_destruction_pl function.""" + df = pl.DataFrame( + { + "ID": [0, 1, 2], + "death": [10.0, 5.0, 8.0], + "encloses": [[1, 2], [], []], + "parent_tag": [0, 0, 0], + } + ) + result = correct_first_destruction_pl(df) + assert len(result) == 4 + new_row = result.filter(pl.col("ID") == 3) + assert not new_row.is_empty() + assert new_row["death"][0] == 5.0 + assert new_row["parent_tag"][0] == 1 + assert new_row["new_row"][0] == 1 From 6095478e70fc3b4569123ed9eec14c732b5d1d56 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Mon, 8 Sep 2025 13:54:10 +0100 Subject: [PATCH 27/69] tests for main.py --- DRUID/tests/test_main.py | 192 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 192 insertions(+) diff --git a/DRUID/tests/test_main.py b/DRUID/tests/test_main.py index e69de29..9bf23e4 100644 --- a/DRUID/tests/test_main.py +++ b/DRUID/tests/test_main.py @@ -0,0 +1,192 @@ +import pytest +import numpy as np +import polars as pl +from astropy.io import fits +import os +import shutil + +from DRUID.main import sf, _worker +from DRUID.src.background import make_gaussian_sources_image + + +@pytest.fixture +def simple_image_data(): + """Creates a simple 100x100 image with one Gaussian source and noise.""" + image_size = (100, 100) + sources = [ + { + "amplitude": 100, + "x_mean": 50, + "y_mean": 50, + "x_stddev": 5, + "y_stddev": 5, + "theta": 0, + } + ] + image = make_gaussian_sources_image(image_size, sources) + image += np.random.normal(5, 1, size=image_size) # Add background and noise + return image + + +@pytest.fixture +def empty_image_data(): + """Creates an empty 100x100 image with just noise.""" + return np.random.normal(5, 1, size=(100, 100)) + + +@pytest.fixture +def simple_fits_file(tmp_path, simple_image_data): + """Creates a dummy FITS file with a simple image.""" + file_path = tmp_path / "simple_image.fits" + hdu = fits.PrimaryHDU(simple_image_data) + hdu.writeto(file_path) + return str(file_path) + + +def test_sf_init_with_numpy_array(simple_image_data): + """Test sf initialization with a NumPy array.""" + finder = sf(image=simple_image_data, verbose=False) + assert isinstance(finder.image, np.ndarray) + assert np.array_equal(finder.image, simple_image_data) + + +def test_sf_init_with_fits_path(simple_fits_file, simple_image_data): + """Test sf initialization with a FITS file path.""" + finder = sf(image=simple_fits_file, verbose=False) + assert isinstance(finder.image, np.ndarray) + assert np.array_equal(finder.image, simple_image_data) + + +def test_sf_init_no_image_raises_error(): + """Test that sf initialization raises ValueError if no image is provided.""" + with pytest.raises(ValueError, match="No image provided"): + sf(verbose=False) + + +def test_sf_init_invalid_path_raises_error(): + """Test that sf initialization raises ValueError for an invalid file path.""" + with pytest.raises(ValueError, match="Could not load image from path"): + sf(image="non_existent_file.fits", verbose=False) + + +def test_sf_init_invalid_type_raises_error(): + """Test that sf initialization raises TypeError for an invalid image type.""" + with pytest.raises(TypeError, match="Image must be a file path"): + sf(image=12345, verbose=False) + + +def test_set_background(simple_image_data): + """Test the set_background method.""" + finder = sf(image=simple_image_data, verbose=False) + finder.set_background(detection_threshold=5, analysis_threshold=3) + assert hasattr(finder, "background_map") + assert hasattr(finder, "background_rms_map") + assert finder.background_map.shape == simple_image_data.shape + assert finder.background_rms_map.shape == simple_image_data.shape + assert finder.detection_threshold == 5 + assert finder.analysis_threshold == 3 + + +def test_set_background_caching(simple_image_data, tmp_path): + """Test the caching mechanism of the set_background method.""" + cache_dir = tmp_path / "druid_cache" + os.makedirs(cache_dir) + + # First run, should calculate and save + finder1 = sf( + image=simple_image_data, + verbose=False, + cashe=True, + working_directory=str(cache_dir), + ) + finder1.set_background() + + bg_map_path = cache_dir / "background_map.npy" + bg_rms_map_path = cache_dir / "background_rms_map.npy" + + assert bg_map_path.exists() + assert bg_rms_map_path.exists() + + # Second run, should load from cache + finder2 = sf( + image=simple_image_data, + verbose=False, + cashe=True, + working_directory=str(cache_dir), + ) + finder2.set_background() + + assert np.array_equal(finder1.background_map, finder2.background_map) + assert np.array_equal(finder1.background_rms_map, finder2.background_rms_map) + + shutil.rmtree(cache_dir) + + +# def test_phsf_raises_error_if_no_background(simple_image_data): +# """Test that phsf raises ValueError if background is not set.""" +# finder = sf(image=simple_image_data, verbose=False) +# with pytest.raises(ValueError, match="Background map and RMS map must be set"): +# finder.phsf() + + +def test_phsf_sequential(simple_image_data): + """Test phsf with sequential processing (num_threads=1).""" + finder = sf(image=simple_image_data, verbose=False, num_threads=1) + finder.set_background(analysis_threshold=3) + finder.phsf() + + assert hasattr(finder, "catalog") + assert isinstance(finder.catalog, pl.DataFrame) + assert not finder.catalog.is_empty() + assert "ID" in finder.catalog.columns + assert "birth" in finder.catalog.columns + + +@pytest.mark.skipif(os.cpu_count() < 2, reason="Test requires at least 2 CPU cores") +def test_phsf_parallel(simple_image_data): + """Test phsf with parallel processing (num_threads > 1).""" + finder = sf(image=simple_image_data, verbose=False, num_threads=2) + finder.set_background(analysis_threshold=3) + finder.phsf() + + assert hasattr(finder, "catalog") + assert isinstance(finder.catalog, pl.DataFrame) + assert not finder.catalog.is_empty() + assert "ID" in finder.catalog.columns + + +def test_phsf_no_sources_found(empty_image_data): + """Test phsf on an image with no sources, expecting an empty catalog.""" + finder = sf(image=empty_image_data, verbose=False) + # Set a high threshold to ensure no sources are found + finder.set_background(analysis_threshold=100) + finder.phsf() + + assert hasattr(finder, "catalog") + assert isinstance(finder.catalog, pl.DataFrame) + assert finder.catalog.is_empty() + + +def test_worker_function(simple_image_data): + """Test the internal _worker function directly.""" + # Simulate a source island cutout + island_image = simple_image_data[30:70, 30:70] + position = (30, 30) + background_rms = 1.0 # For simplicity + background = 5.0 + + iterable = (island_image, position, background, background_rms) + + result_cat = _worker( + iterable, + analysis_threshold=3.0, + lifetime_limit=0.1, + lifetime_limit_fraction=1.0, + ) + + assert isinstance(result_cat, pl.DataFrame) + assert not result_cat.is_empty() + assert "Island_X" in result_cat.columns + assert "Island_Y" in result_cat.columns + assert result_cat["Island_X"][0] == position[0] + assert result_cat["Island_Y"][0] == position[1] From 6ff31106ba5d35fd032b2561be9fc0ec715b46a8 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Mon, 8 Sep 2025 13:54:31 +0100 Subject: [PATCH 28/69] test for utils.py --- DRUID/tests/test_utils.py | 93 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/DRUID/tests/test_utils.py b/DRUID/tests/test_utils.py index e69de29..dabe40d 100644 --- a/DRUID/tests/test_utils.py +++ b/DRUID/tests/test_utils.py @@ -0,0 +1,93 @@ +import pytest +import numpy as np +import polars as pl +from polars.testing import assert_frame_equal +from astropy.io import fits + +from DRUID.src.utils import get_image_from_path, combine_polars_catalogs + + +@pytest.fixture +def create_fits_file(tmp_path): + """A fixture to create a FITS file with given data.""" + + def _create_fits(data, filename="test.fits"): + file_path = tmp_path / filename + hdu = fits.PrimaryHDU(data) + hdu.writeto(file_path, overwrite=True) + return str(file_path) + + return _create_fits + + +def test_get_image_from_path_2d(create_fits_file): + """Test loading a standard 2D FITS image.""" + image_data = np.arange(100, dtype=np.float32).reshape(10, 10) + fits_path = create_fits_file(image_data) + + loaded_image = get_image_from_path(fits_path) + + assert isinstance(loaded_image, np.ndarray) + assert loaded_image.shape == (10, 10) + assert np.array_equal(loaded_image, image_data) + + +def test_get_image_from_path_3d_squeezes(create_fits_file): + """Test that a 3D FITS image is correctly squeezed to 2D.""" + image_data = np.arange(100, dtype=np.float32).reshape(1, 10, 10) + fits_path = create_fits_file(image_data) + + loaded_image = get_image_from_path(fits_path) + + assert loaded_image.shape == (10, 10) + assert np.array_equal(loaded_image, image_data.squeeze()) + + +def test_get_image_from_path_4d_squeezes(create_fits_file): + """Test that a 4D FITS image is correctly squeezed to 2D.""" + image_data = np.arange(100, dtype=np.float32).reshape(1, 1, 10, 10) + fits_path = create_fits_file(image_data) + + loaded_image = get_image_from_path(fits_path) + + assert loaded_image.shape == (10, 10) + assert np.array_equal(loaded_image, image_data.squeeze()) + + +def test_get_image_from_path_file_not_found(): + """Test that an error is raised for a non-existent file.""" + with pytest.raises(FileNotFoundError): + get_image_from_path("non_existent_file.fits") + + +def test_combine_polars_catalogs_basic(): + """Test combining a list of simple Polars DataFrames.""" + cat1 = pl.DataFrame({"A": [1, 2], "B": ["x", "y"]}) + cat2 = pl.DataFrame({"A": [3, 4], "B": ["z", "w"]}) + catalogs = [cat1, cat2] + + combined = combine_polars_catalogs(catalogs) + + expected = pl.DataFrame({"A": [1, 2, 3, 4], "B": ["x", "y", "z", "w"]}) + + assert_frame_equal(combined, expected) + assert combined.shape == (4, 2) + + +def test_combine_polars_catalogs_with_uppercase_id(): + """Test that a column named 'ID' (uppercase) is not re-indexed.""" + cat1 = pl.DataFrame({"ID": [0, 1], "data": [10, 20]}) + cat2 = pl.DataFrame({"ID": [0, 1], "data": [30, 40]}) + catalogs = [cat1, cat2] + + combined = combine_polars_catalogs(catalogs) + + # The 'ID' column should remain as is, with duplicates + expected = pl.DataFrame({"ID": [0, 1, 0, 1], "data": [10, 20, 30, 40]}) + assert_frame_equal(combined, expected) + + +def test_combine_polars_catalogs_empty_list(): + """Test that combining an empty list of catalogs raises a ValueError.""" + with pytest.raises(ValueError, match="No catalogs provided to combine."): + combine_polars_catalogs([]) From 8c1f7bc4b6f811face540aa5060a33b62e22372e Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Mon, 8 Sep 2025 15:45:57 +0100 Subject: [PATCH 29/69] corrected for when polars table has no values --- DRUID/main.py | 6 +++++- DRUID/src/homology.py | 22 +++++++++++++++++++--- DRUID/src/properties.py | 0 3 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 DRUID/src/properties.py diff --git a/DRUID/main.py b/DRUID/main.py index d5e64f3..4c7f63b 100644 --- a/DRUID/main.py +++ b/DRUID/main.py @@ -67,6 +67,8 @@ def _worker( lifetime_limit_fraction=lifetime_limit_fraction, ) # Add position to the catalog + if cat is None or cat.is_empty(): + return None cat = cat.with_columns( pl.lit(position[0]).alias("Island_X"), pl.lit(position[1]).alias("Island_Y"), @@ -169,7 +171,6 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1): print("No source islands to process.") self.catalog = pl.DataFrame() return - print(self.num_threads) # make the iterable images_to_process and poistions iterable_images = zip( @@ -197,6 +198,7 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1): lifetime_limit=lifetime_limit, lifetime_limit_fraction=lifetime_limit_fraction, ) + # print(iterable_images) results = p.map(worker_func, iterable_images, chunksize=batch_size) else: @@ -215,6 +217,8 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1): # combine the results catalogs to a single catalog if results: + # remove any None results + results = [res for res in results if res is not None] self.catalog = utils.combine_polars_catalogs(results) t1 = time.time() diff --git a/DRUID/src/homology.py b/DRUID/src/homology.py index ee45791..fe706e6 100644 --- a/DRUID/src/homology.py +++ b/DRUID/src/homology.py @@ -77,8 +77,13 @@ def correct_first_destruction_pl(df: pl.DataFrame) -> pl.DataFrame: df = df.with_columns(pl.lit(0, dtype=pl.Int8).alias("new_row")) # 1. Filter the DataFrame to find all rows that have enclosed islands - - islands_to_split = df.filter(pl.col("encloses").list.len() > 1) + # print(df) + try: + islands_to_split = df.filter(pl.col("encloses").list.len() > 1) + except Exception as e: + print("Error filtering islands to split:", e) + print(df) + # exit(1) # If no such rows exist, return the original DataFrame. if islands_to_split.is_empty(): @@ -96,7 +101,6 @@ def correct_first_destruction_pl(df: pl.DataFrame) -> pl.DataFrame: # Suffix prevents column name collisions ('Death' becomes 'Death_parent') suffix="_parent", ) - print("New Rows Base: ", new_rows_base) # If the join results in an empty DataFrame, return the original. if new_rows_base.is_empty(): @@ -399,6 +403,10 @@ def compute_homology( # area size filter. area_size_threshold = 2 # replace with argument #### TODO #### polar_df = polar_df.filter(polar_df["area"] > area_size_threshold) + init_df = polar_df.clone() + # if after the area size filter there are no components return empty df + if polar_df.is_empty(): + return None # assign an ID to each point in the polar_df polar_df = polar_df.with_columns(pl.Series("ID", range(len(polar_df)))) @@ -421,7 +429,13 @@ def compute_homology( ) # correct first destruction + # try: polar_df = correct_first_destruction_pl(polar_df) + # except Exception as e: + # print("Error correcting first destruction:", e) + # print(init_df) + # plt.imshow(img) + # plt.show() # assign parent tags polar_df = parent_tag_func_pl(polar_df) contours = [] @@ -443,6 +457,8 @@ def compute_homology( ] polar_df = polar_df.with_columns(pl.Series("contour", contours)) # print(f"Computed {len(polar_df)} components with contours.") + # print(len(polar_df.columns)) + # print(polar_df.columns) return polar_df diff --git a/DRUID/src/properties.py b/DRUID/src/properties.py new file mode 100644 index 0000000..e69de29 From fb98589a711ae31abc9a85d4f2e70b66f1ace43a Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Mon, 8 Sep 2025 15:50:48 +0100 Subject: [PATCH 30/69] start on source properties calculations --- DRUID/src/properties.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/DRUID/src/properties.py b/DRUID/src/properties.py index e69de29..5138853 100644 --- a/DRUID/src/properties.py +++ b/DRUID/src/properties.py @@ -0,0 +1,4 @@ +""" +Author: Rhys Shaw +Date: 08-09-2025 +""" From 2558a6773d2577e4b3045933e2ac146f5dfae5c7 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Tue, 9 Sep 2025 09:43:43 +0100 Subject: [PATCH 31/69] removed debugging lines --- DRUID/src/homology.py | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/DRUID/src/homology.py b/DRUID/src/homology.py index fe706e6..77c2133 100644 --- a/DRUID/src/homology.py +++ b/DRUID/src/homology.py @@ -43,7 +43,6 @@ def classify_single(row): Class: int - the Class integer that indiceates the class the row belongs too. """ - # print(row) if row["new_row"] == 0: if len(row["encloses"]) == 0: # no children if np.isnan(row["parent_tag"]): # no parent @@ -78,12 +77,8 @@ def correct_first_destruction_pl(df: pl.DataFrame) -> pl.DataFrame: # 1. Filter the DataFrame to find all rows that have enclosed islands # print(df) - try: - islands_to_split = df.filter(pl.col("encloses").list.len() > 1) - except Exception as e: - print("Error filtering islands to split:", e) - print(df) - # exit(1) + + islands_to_split = df.filter(pl.col("encloses").list.len() > 1) # If no such rows exist, return the original DataFrame. if islands_to_split.is_empty(): @@ -429,13 +424,7 @@ def compute_homology( ) # correct first destruction - # try: polar_df = correct_first_destruction_pl(polar_df) - # except Exception as e: - # print("Error correcting first destruction:", e) - # print(init_df) - # plt.imshow(img) - # plt.show() # assign parent tags polar_df = parent_tag_func_pl(polar_df) contours = [] @@ -456,9 +445,7 @@ def compute_homology( for contour in contours ] polar_df = polar_df.with_columns(pl.Series("contour", contours)) - # print(f"Computed {len(polar_df)} components with contours.") - # print(len(polar_df.columns)) - # print(polar_df.columns) + return polar_df From de8e12b074dc6c6363271930795fba4f2c060693 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Wed, 10 Sep 2025 10:21:52 +0100 Subject: [PATCH 32/69] starting scripts calculating source properties. --- DRUID/main.py | 14 +++++++ DRUID/src/homology.py | 3 +- DRUID/src/properties.py | 84 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+), 1 deletion(-) diff --git a/DRUID/main.py b/DRUID/main.py index 4c7f63b..af287ed 100644 --- a/DRUID/main.py +++ b/DRUID/main.py @@ -21,6 +21,7 @@ from .src import homology from .src import background from .src import source +from .src import properties from functools import partial RED = "\033[91m" @@ -66,6 +67,19 @@ def _worker( lifetime_limit=lifetime_limit, lifetime_limit_fraction=lifetime_limit_fraction, ) + + # source characteristics measure here! + if cat is not None and not cat.is_empty(): + # Add source characteristics to the catalog + cat = properties.calculate_properties( + cat, + image, + background, + background_rms, + position, + analysis_threshold, + ) + # Add position to the catalog if cat is None or cat.is_empty(): return None diff --git a/DRUID/src/homology.py b/DRUID/src/homology.py index 77c2133..b270a8a 100644 --- a/DRUID/src/homology.py +++ b/DRUID/src/homology.py @@ -6,14 +6,15 @@ import cripser import numpy as np import polars as pl + from scipy.ndimage import label as scipy_label from tqdm import tqdm from skimage import measure +from astropy.io import fits # For testing and development purposes, we import the following libraries: import pandas as pd import matplotlib.pyplot as plt -from astropy.io import fits def _get_polygons_CPU(x1, y1, birth, death, image: np.ndarray): diff --git a/DRUID/src/properties.py b/DRUID/src/properties.py index 5138853..8d58536 100644 --- a/DRUID/src/properties.py +++ b/DRUID/src/properties.py @@ -2,3 +2,87 @@ Author: Rhys Shaw Date: 08-09-2025 """ + +from skimage import measure +import numpy as np +from skimage.draw import polygon +import polars as pl + + +def get_region_properties(mask, image): + labeled_mask = measure.label(mask) + + properties = measure.regionprops(labeled_mask, intensity_image=image) + + return properties + + +def calculate_properties( + cat, image, background, background_rms, position, analysis_threshold +): + + # create a mask of source based on birth and death + mask = np.zeros_like(image, dtype=bool) + mask = np.logical_or(mask, image > (analysis_threshold * background_rms)) + + # bbox_data in cat to reduce to min size.? + # Calculate using + + source_properties = get_region_properties(mask, image) + + # cat_with_properties = cat.with_columns( + # [ + # pl.Series("area", [prop.area for prop in source_properties]), + # pl.Series( + # "centroid_row", + # [prop.centroid[0] + position[0] for prop in source_properties], + # ), + # pl.Series( + # "centroid_col", + # [prop.centroid[1] + position[1] for prop in source_properties], + # ), + # ] + # ) + return cat + + +if __name__ == "__main__": + + # open dummy image parquet + dummy_image = np.load("DRUID/temp/image_3C401.npy") + dummy_background = np.load("DRUID/temp/background_3C401.npy") + dummy_background_rms = np.load("DRUID/temp/background_rms_3C401.npy") + + import source + import homology + + source_islands = source.create_source_islands( + dummy_image, dummy_background, dummy_background_rms, 5, 3, 15, False + ) + + images_to_process = source_islands["island_image"] + print(len(images_to_process)) + iterable_images = zip( + images_to_process, + source_islands["positions"], + source_islands["background"], + source_islands["background_rms"], + ) + # i = 0 + for img, pos, back, back_rms in iterable_images: + cat = homology.compute_homology( + img, + analysis_threshold=3 * back_rms, + lifetime_limit=0.0, + lifetime_limit_fraction=1.4, + ) + cat_with_props = calculate_properties( + cat, + img, + back, + back_rms, + pos, + analysis_threshold=3, + ) + print(cat_with_props) + # i += 1 From 160413990115f4eeac43ac09933ca005d5178720 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Wed, 10 Sep 2025 12:09:24 +0100 Subject: [PATCH 33/69] calculating source properties functions -- not implemented in pipeline --- DRUID/src/properties.py | 167 ++++++++++++++++++++++++++++++++-------- DRUID/src/utils.py | 46 +++++++++++ 2 files changed, 183 insertions(+), 30 deletions(-) diff --git a/DRUID/src/properties.py b/DRUID/src/properties.py index 8d58536..a283000 100644 --- a/DRUID/src/properties.py +++ b/DRUID/src/properties.py @@ -7,42 +7,141 @@ import numpy as np from skimage.draw import polygon import polars as pl +import homology -def get_region_properties(mask, image): - labeled_mask = measure.label(mask) +def calculate_radio_flux_error(background_rms, area, BMAJ, BMIN): + # adapted from https://github.com/mhardcastle/radioflux/blob/master/radioflux/radioflux.py + + gfactor = 2 * np.sqrt(2 * np.log(2)) + Beam_area = 2 * np.pi * (BMAJ * BMIN) / gfactor + return np.mean(background_rms) * np.sqrt(area / Beam_area) - properties = measure.regionprops(labeled_mask, intensity_image=image) +def get_region_properties(mask, image): + # labeled_mask = measure.label(mask) + properties = measure.regionprops(mask, intensity_image=image) return properties +def get_row_mask(row, image): + mask = np.zeros_like(image, dtype=bool) + mask = np.logical_or(mask, np.logical_and(img <= row["birth"], img > row["death"])) + mask = homology.get_enclosing_mask_CPU(int(row["y1"]), int(row["x1"]), mask) + mask = mask.astype(int) + return mask + + +def ABmag(flux): + return -2.5 * np.log10(flux) + + +def RONoise(EFFRON, EFFGAIN, EXPTIME, Area): + return np.sqrt(Area) * (EFFRON / EFFGAIN) * EXPTIME + + +def SkyNoise(sky): + return np.sqrt(sky) + + +def SourceNoise(Flux): + return np.sqrt(Flux) + + +def optical_flux_err(EFFRON, EFFGAIN, EXPTIME, Area, sky, Flux): + try: + RON_noise = RONoise(EFFRON, EFFGAIN, EXPTIME, Area) + except: + print( + "Error calculating RONoise (likely missing EFFORN, EFFGAIN or EXPTIME in header), setting to 0" + ) + RON_noise = 0 + Sky_noise = SkyNoise(sky) + Source_noise = SourceNoise(Flux) + return np.sqrt(RON_noise**2 + Sky_noise + Source_noise) + + +def NOISE(row, local_ng): + return np.sum(np.random.normal(row["mean_bg"], local_ng, int(row["Area"]))) + + def calculate_properties( - cat, image, background, background_rms, position, analysis_threshold + cat, + image, + background, + background_rms, + position, + analysis_threshold, + mode, + BMAJ=None, + BMIN=None, + EFFRON=None, + EFFGAIN=None, + EXPTIME=None, ): + from matplotlib import pyplot as plt + + plt.imshow(image, cmap="gray", origin="lower") + plt.show() + print(cat) + maj = [] + min = [] + pa = [] + centroid = [] + flux = [] + flux_peak = [] + bg = [] + flux_err = [] + snr = [] + for row in cat.iter_rows(named=True): + # print(row) + # create a mask of source based on birth and death + mask = get_row_mask(row, image) + props = get_region_properties(mask, image) + + maj.append(props[0].major_axis_length) + min.append(props[0].minor_axis_length) + pa.append(props[0].orientation) + centroid.append(props[0].centroid) + + # calculate fluxes + flux_tot = np.nansum(mask * (image - background)) + flux.append(flux_tot) + flux_peak.append(np.nanmax(mask * (image - background))) + bg.append(np.mean(background * mask)) + + if mode == "radio": + + Flux_total_err = calculate_radio_flux_error( + background_rms, row["area"], BMAJ, BMIN + ) + flux_err.append(Flux_total_err) + + elif mode == "optical": + Flux_total_err = optical_flux_err( + EFFRON=EFFRON, + EFFGAIN=EFFGAIN, + EXPTIME=EXPTIME, + Area=row["area"], + sky=np.nansum(background * mask), + Flux=np.nansum(mask * (image - background)), + ) + flux_err.append(Flux_total_err) + + snr.append(flux_tot / Flux_total_err) + + cat = cat.with_columns( + pl.Series("maj", maj), + pl.Series("min", min), + pl.Series("pa", pa), + pl.Series("centroid", centroid), + pl.Series("flux_peak", flux_peak), + pl.Series("bg", bg), + pl.Series("flux_err", flux_err), + pl.Series("flux", flux), + pl.Series("snr", snr), + ) - # create a mask of source based on birth and death - mask = np.zeros_like(image, dtype=bool) - mask = np.logical_or(mask, image > (analysis_threshold * background_rms)) - - # bbox_data in cat to reduce to min size.? - # Calculate using - - source_properties = get_region_properties(mask, image) - - # cat_with_properties = cat.with_columns( - # [ - # pl.Series("area", [prop.area for prop in source_properties]), - # pl.Series( - # "centroid_row", - # [prop.centroid[0] + position[0] for prop in source_properties], - # ), - # pl.Series( - # "centroid_col", - # [prop.centroid[1] + position[1] for prop in source_properties], - # ), - # ] - # ) return cat @@ -61,21 +160,25 @@ def calculate_properties( ) images_to_process = source_islands["island_image"] - print(len(images_to_process)) iterable_images = zip( images_to_process, source_islands["positions"], source_islands["background"], source_islands["background_rms"], ) - # i = 0 + i = 0 + for img, pos, back, back_rms in iterable_images: + if i > 0: + break cat = homology.compute_homology( img, analysis_threshold=3 * back_rms, lifetime_limit=0.0, lifetime_limit_fraction=1.4, ) + BMAJ = 0.35 # arcsec + BMIN = 0.35 # arcsec cat_with_props = calculate_properties( cat, img, @@ -83,6 +186,10 @@ def calculate_properties( back_rms, pos, analysis_threshold=3, + mode="radio", + BMAJ=BMAJ, + BMIN=BMIN, ) - print(cat_with_props) - # i += 1 + # print(cat_with_props) + + i += 1 diff --git a/DRUID/src/utils.py b/DRUID/src/utils.py index 16ac593..98e1a80 100644 --- a/DRUID/src/utils.py +++ b/DRUID/src/utils.py @@ -1,4 +1,5 @@ import polars as pl +import numpy as np def get_image_from_path(image_path): @@ -46,3 +47,48 @@ def combine_polars_catalogs(catalogs: list): ).with_columns(pl.col("id").rank(method="dense").alias("id")) return combined_catalog + + +def generate_2d_gaussian(A, shape, center, sigma_x, sigma_y, angle_deg=0, norm=True): + """ + + Generate a 2D elliptical Gaussian distribution on a 2D array. + + Parameters: + + shape (tuple): Shape of the output array (height, width). + center (tuple): Center of the Gaussian distribution (x, y). + sigma_x (float): Standard deviation along the x-axis. + sigma_y (float): Standard deviation along the y-axis. + angle_deg (float): Rotation angle in degrees (default is 0). + + Returns: + + ndarray: 2D array containing the Gaussian distribution. + + """ + x, y = np.meshgrid(np.arange(shape[1]), np.arange(shape[0])) + x_c, y_c = center + angle_rad = np.radians(angle_deg) + + # Rotate coordinates + + x_rot = (x - x_c) * np.cos(angle_rad) - (y - y_c) * np.sin(angle_rad) + y_rot = (x - x_c) * np.sin(angle_rad) + (y - y_c) * np.cos(angle_rad) + + # Calculate Gaussian values + + gaussian = A * np.exp(-(x_rot**2 / (2 * sigma_x**2) + y_rot**2 / (2 * sigma_y**2))) + + if norm: + return gaussian / (2 * np.pi * sigma_x * sigma_y) + else: + return gaussian + + +def model_beam_func(peak_flux, shape, x, y, bmaj, bmin, bpa): + model_beam = np.zeros(shape) + model_beam = generate_2d_gaussian( + peak_flux, shape, (x, y), bmaj, bmin, bpa, norm=False + ) + return model_beam From 896144c51ea34735d067af3b6d39fb1157cfbfe4 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Mon, 22 Sep 2025 08:37:51 +0100 Subject: [PATCH 34/69] update to some files when running --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index d5d4326..f00d4c8 100644 --- a/.gitignore +++ b/.gitignore @@ -22,4 +22,5 @@ DRUID.egg-info build backup notepad.ipynb -temp \ No newline at end of file +temp +_* \ No newline at end of file From c27272dc9b40044cf1d8e594b4852e9216d4133b Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Mon, 22 Sep 2025 08:38:23 +0100 Subject: [PATCH 35/69] Change of some files --- DRUID/main.py | 49 +++++++++++++++++++++++------------------ DRUID/src/properties.py | 4 ++-- DRUID/src/source.py | 34 ++++++++++++++++++++++++++-- 3 files changed, 61 insertions(+), 26 deletions(-) diff --git a/DRUID/main.py b/DRUID/main.py index af287ed..4474de9 100644 --- a/DRUID/main.py +++ b/DRUID/main.py @@ -7,6 +7,7 @@ import numpy as np import astropy import os +import random # this prevent polars from using all available threads. # Especially for multithreaded homology computation. otherwise we will spawn nested threads. @@ -68,25 +69,25 @@ def _worker( lifetime_limit_fraction=lifetime_limit_fraction, ) - # source characteristics measure here! - if cat is not None and not cat.is_empty(): - # Add source characteristics to the catalog - cat = properties.calculate_properties( - cat, - image, - background, - background_rms, - position, - analysis_threshold, - ) - - # Add position to the catalog - if cat is None or cat.is_empty(): - return None - cat = cat.with_columns( - pl.lit(position[0]).alias("Island_X"), - pl.lit(position[1]).alias("Island_Y"), - ) + # # source characteristics measure here! + # if cat is not None and not cat.is_empty(): + # # Add source characteristics to the catalog + # cat = properties.calculate_properties( + # cat, + # image, + # background, + # background_rms, + # position, + # analysis_threshold, + # ) + + # # Add position to the catalog + # if cat is None or cat.is_empty(): + # return None + # cat = cat.with_columns( + # pl.lit(position[0]).alias("Island_X"), + # pl.lit(position[1]).alias("Island_Y"), + # ) return cat @@ -170,6 +171,7 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1): area_limit=self.area_limit, verbose=self.verbose, ) + t1 = time.time() print(f"Thresholding took {t1 - t0:.2f} seconds.") t0 = time.time() @@ -179,7 +181,10 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1): ) images_to_process = source_islands["island_image"] - + print( + "Max island image shape:", + np.max([img.shape for img in images_to_process], axis=0), + ) if not images_to_process: if self.verbose: print("No source islands to process.") @@ -193,14 +198,14 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1): source_islands["background"], source_islands["background_rms"], ) - + print(iterable_images) if self.num_threads > 1: if self.verbose: print( f"Processing {len(images_to_process)} source islands in parallel. with {self.num_threads} threads." ) print("images to process:", len(images_to_process)) - batch_size = len(images_to_process) // self.num_threads + batch_size = len(images_to_process) // (self.num_threads * 10) if batch_size < 1: # prevent batch size of 0 batch_size = 1 print(f"Batch size: {batch_size}") diff --git a/DRUID/src/properties.py b/DRUID/src/properties.py index a283000..e14d6b5 100644 --- a/DRUID/src/properties.py +++ b/DRUID/src/properties.py @@ -6,8 +6,8 @@ from skimage import measure import numpy as np from skimage.draw import polygon -import polars as pl -import homology +import polars as pl +from . import homology def calculate_radio_flux_error(background_rms, area, BMAJ, BMIN): diff --git a/DRUID/src/source.py b/DRUID/src/source.py index fa936c1..70f6731 100644 --- a/DRUID/src/source.py +++ b/DRUID/src/source.py @@ -12,6 +12,7 @@ from skimage.measure import regionprops, label, regionprops_table from tqdm import tqdm import pandas as pd # +import polars as pl def create_source_islands( @@ -67,11 +68,13 @@ def create_source_islands( source_island_bg_rms = [] source_island_bg = [] min_area = area_limit + area = [] for prop in properties: if prop.area < min_area: continue + area.append(prop.area) # prop.intensity_image is the cropped and masked component - components.append(prop.intensity_image) + components.append(np.array(prop.intensity_image)) # prop.bbox returns (min_row, min_col, max_row, max_col) y_min, x_min, _, _ = prop.bbox @@ -83,6 +86,11 @@ def create_source_islands( source_island_bg_rms.append( background_rms_map[y_min : prop.bbox[2], x_min : prop.bbox[3]].mean() ) + import matplotlib.pyplot as plt + + plt.hist(area) + plt.yscale("log") + plt.savefig("area_distribution.png") t1 = time.time() if verbose: @@ -97,7 +105,29 @@ def create_source_islands( "background_rms": source_island_bg_rms, } - return source_islands + shuffled_islands = shuffle_in_unison( + [ + source_islands["island_image"], + source_islands["positions"], + source_islands["background"], + source_islands["background_rms"], + ] + ) + return shuffled_islands + + +def shuffle_in_unison(arrays): + """Shuffle multiple arrays in unison, preserving the correspondence between them.""" + assert all( + len(arr) == len(arrays[0]) for arr in arrays + ), "All arrays must have the same length." + p = np.random.permutation(len(arrays[0])) + return { + key: [array[i] for i in p] + for key, array in zip( + ["island_image", "positions", "background", "background_rms"], arrays + ) + } def create_source_islands_optimized( From 77d2fea9b6c2b88b223f30694f9f68f2f52d97a9 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Mon, 13 Jul 2026 15:36:50 +0100 Subject: [PATCH 36/69] resolved? --- DRUID/main.py | 73 ++++++++- DRUID/src/properties.py | 60 ++++++-- DRUID/src/utils.py | 3 +- Examples/Resolved_Galaxies.ipynb | 244 ++++++++++++++++++++++++++++--- test.py | 212 +++++++++++---------------- 5 files changed, 426 insertions(+), 166 deletions(-) diff --git a/DRUID/main.py b/DRUID/main.py index 4474de9..cc47fdb 100644 --- a/DRUID/main.py +++ b/DRUID/main.py @@ -56,7 +56,16 @@ def _worker( - iterable_image, analysis_threshold, lifetime_limit, lifetime_limit_fraction + iterable_image, + analysis_threshold, + lifetime_limit, + lifetime_limit_fraction, + mode=None, + BMAJ=None, + BMIN=None, + EFFRON=None, + EFFGAIN=None, + EXPTIME=None, ) -> "pl.DataFrame": """ Worker function to compute homology for a single source island. @@ -69,6 +78,7 @@ def _worker( lifetime_limit_fraction=lifetime_limit_fraction, ) +<<<<<<< HEAD # # source characteristics measure here! # if cat is not None and not cat.is_empty(): # # Add source characteristics to the catalog @@ -80,6 +90,25 @@ def _worker( # position, # analysis_threshold, # ) +======= + # source characteristics measure here! + if cat is not None and not cat.is_empty(): + # Add source characteristics to the catalog + cat = properties.calculate_properties( + cat, + image, + background, + background_rms, + position, + analysis_threshold, + mode, + BMAJ, + BMIN, + EFFRON, + EFFGAIN, + EXPTIME, + ) +>>>>>>> 1e15fb5 (update?) # # Add position to the catalog # if cat is None or cat.is_empty(): @@ -127,11 +156,12 @@ def __init__( if isinstance(image, str): try: - self.image = utils.get_image_from_path(image) + self.image, self.header = utils.get_image_from_path(image) except Exception as e: raise ValueError(f"Could not load image from path: {image}") from e elif isinstance(image, np.ndarray): self.image = image + self.header = header else: raise TypeError( "Image must be a file path (str) or a NumPy array (np.ndarray)." @@ -145,6 +175,33 @@ def __init__( else: self.working_directory = None + if self.mode == "radio": + try: + self.BMAJ = self.header["BMAJ"] + self.BMIN = self.header["BMIN"] + self.EFFGAIN = None + self.EFFRON = None + self.EXPTIME = None + + except KeyError as e: + print( + "Warning: Could not find BMAJ or BMIN in header, beam parameters will be set to None." + ) + self.BMAJ = None + self.BMIN = None + + elif self.mode == "optical": + try: + self.EFFRON = self.header["EFFRON"] + self.EFFGAIN = self.header["EFFGAIN"] + self.EXPTIME = self.header["EXPTIME"] + self.BMAJ = None + self.BMIN = None + except KeyError as e: + print( + "Warning: Could not find EFFRON, EFFGAIN, or EXPTIME, flux_err will be set to 0." + ) + def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1): """ Runs the source findin algorithm on the image. @@ -216,6 +273,12 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1): analysis_threshold=self.analysis_threshold, lifetime_limit=lifetime_limit, lifetime_limit_fraction=lifetime_limit_fraction, + mode=self.mode, + BMAJ=self.BMAJ, + BMIN=self.BMIN, + EFFRON=self.EFFRON, + EFFGAIN=self.EFFGAIN, + EXPTIME=self.EXPTIME, ) # print(iterable_images) results = p.map(worker_func, iterable_images, chunksize=batch_size) @@ -230,6 +293,12 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1): self.analysis_threshold, lifetime_limit, lifetime_limit_fraction, + self.mode, + self.BMAJ, + self.BMIN, + self.EFFRON, + self.EFFGAIN, + self.EXPTIME, ) ) diff --git a/DRUID/src/properties.py b/DRUID/src/properties.py index e14d6b5..4e455fe 100644 --- a/DRUID/src/properties.py +++ b/DRUID/src/properties.py @@ -6,13 +6,41 @@ from skimage import measure import numpy as np from skimage.draw import polygon + import polars as pl from . import homology +import polars as pl +from scipy.ndimage import label as scipy_label + + +def get_enclosing_mask_CPU(x, y, mask): + """ + Returns the connected components inside the mask starting from the point (x, y). + """ + from skimage.measure import label + + labeled_mask, num_features = scipy_label(mask) + + # check if the specified pixel is within the mask + if 0 <= x < mask.shape[1] and 0 <= y < mask.shape[0]: + label_at_pixel = labeled_mask[y, x] + + if label_at_pixel != 0: + # get the connected component containing the specified pixel + component_mask = labeled_mask == label_at_pixel + return component_mask + else: + # the specified pixel is not part of any connected component + return None + else: + # the specified pixel is outside the mask + return None +>>>>>>> 1e15fb5 (update?) + def calculate_radio_flux_error(background_rms, area, BMAJ, BMIN): # adapted from https://github.com/mhardcastle/radioflux/blob/master/radioflux/radioflux.py - gfactor = 2 * np.sqrt(2 * np.log(2)) Beam_area = 2 * np.pi * (BMAJ * BMIN) / gfactor return np.mean(background_rms) * np.sqrt(area / Beam_area) @@ -26,8 +54,10 @@ def get_region_properties(mask, image): def get_row_mask(row, image): mask = np.zeros_like(image, dtype=bool) - mask = np.logical_or(mask, np.logical_and(img <= row["birth"], img > row["death"])) - mask = homology.get_enclosing_mask_CPU(int(row["y1"]), int(row["x1"]), mask) + mask = np.logical_or( + mask, np.logical_and(image <= row["birth"], image > row["death"]) + ) + mask = get_enclosing_mask_CPU(int(row["y1"]), int(row["x1"]), mask) mask = mask.astype(int) return mask @@ -81,9 +111,9 @@ def calculate_properties( ): from matplotlib import pyplot as plt - plt.imshow(image, cmap="gray", origin="lower") - plt.show() - print(cat) + # plt.imshow(image, cmap="gray", origin="lower") + # plt.show() + # print(cat) maj = [] min = [] pa = [] @@ -116,6 +146,7 @@ def calculate_properties( background_rms, row["area"], BMAJ, BMIN ) flux_err.append(Flux_total_err) + snr.append(flux_tot / Flux_total_err) elif mode == "optical": Flux_total_err = optical_flux_err( @@ -127,10 +158,14 @@ def calculate_properties( Flux=np.nansum(mask * (image - background)), ) flux_err.append(Flux_total_err) + snr.append(flux_tot / Flux_total_err) - snr.append(flux_tot / Flux_total_err) + else: + Flux_total_err = 0 + flux_err.append(Flux_total_err) + snr.append(0) - cat = cat.with_columns( + cat_with_props = cat.with_columns( pl.Series("maj", maj), pl.Series("min", min), pl.Series("pa", pa), @@ -142,7 +177,7 @@ def calculate_properties( pl.Series("snr", snr), ) - return cat + return cat_with_props if __name__ == "__main__": @@ -166,11 +201,8 @@ def calculate_properties( source_islands["background"], source_islands["background_rms"], ) - i = 0 for img, pos, back, back_rms in iterable_images: - if i > 0: - break cat = homology.compute_homology( img, analysis_threshold=3 * back_rms, @@ -190,6 +222,4 @@ def calculate_properties( BMAJ=BMAJ, BMIN=BMIN, ) - # print(cat_with_props) - - i += 1 + print(cat_with_props) diff --git a/DRUID/src/utils.py b/DRUID/src/utils.py index 98e1a80..68acf38 100644 --- a/DRUID/src/utils.py +++ b/DRUID/src/utils.py @@ -20,6 +20,7 @@ def get_image_from_path(image_path): with fits.open(image_path) as hdul: image = hdul[0].data + header = hdul[0].header # warn if the image is not 2D # reduce the image to 2D if it is not @@ -27,7 +28,7 @@ def get_image_from_path(image_path): image = image[0, :, :] elif image.ndim == 4: image = image[0, 0, :, :] - return image + return image, header def combine_polars_catalogs(catalogs: list): diff --git a/Examples/Resolved_Galaxies.ipynb b/Examples/Resolved_Galaxies.ipynb index 1f49d76..a166891 100644 --- a/Examples/Resolved_Galaxies.ipynb +++ b/Examples/Resolved_Galaxies.ipynb @@ -14,6 +14,72 @@ "Some of DRUIDs strengths are demonstrated with its ability to handle nested image features. Like the details resovlable in face on galaxies." ] }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: astrocut in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (1.0.1)\n", + "Requirement already satisfied: asdf>=4.1.0 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from astrocut) (4.5.0)\n", + "Requirement already satisfied: astropy>=5.2 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from astrocut) (7.1.0)\n", + "Requirement already satisfied: cachetools>=5.3.2 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from astrocut) (6.2.0)\n", + "Requirement already satisfied: fsspec>=2022.8.2 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from fsspec[http]>=2022.8.2->astrocut) (2025.9.0)\n", + "Requirement already satisfied: s3fs>=2022.8.2 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from astrocut) (2025.9.0)\n", + "Requirement already satisfied: s3path>=0.5.7 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from astrocut) (0.6.4)\n", + "Requirement already satisfied: roman_datamodels>=0.19.0 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from astrocut) (0.27.0)\n", + "Requirement already satisfied: requests>=2.32.3 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from astrocut) (2.32.5)\n", + "Requirement already satisfied: spherical_geometry>=1.3.0 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from astrocut) (1.3.3)\n", + "Requirement already satisfied: gwcs>=0.21.0 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from astrocut) (0.25.2)\n", + "Requirement already satisfied: scipy in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from astrocut) (1.15.3)\n", + "Requirement already satisfied: Pillow in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from astrocut) (11.2.1)\n", + "Requirement already satisfied: asdf-standard>=1.1.0 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from asdf>=4.1.0->astrocut) (1.4.0)\n", + "Requirement already satisfied: asdf-transform-schemas>=0.3 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from asdf>=4.1.0->astrocut) (0.6.0)\n", + "Requirement already satisfied: jmespath>=0.6.2 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from asdf>=4.1.0->astrocut) (1.0.1)\n", + "Requirement already satisfied: numpy>=1.22 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from asdf>=4.1.0->astrocut) (2.2.6)\n", + "Requirement already satisfied: packaging>=19 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from asdf>=4.1.0->astrocut) (25.0)\n", + "Requirement already satisfied: pyyaml>=5.4.1 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from asdf>=4.1.0->astrocut) (6.0.2)\n", + "Requirement already satisfied: semantic_version>=2.8 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from asdf>=4.1.0->astrocut) (2.10.0)\n", + "Requirement already satisfied: attrs>=22.2.0 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from asdf>=4.1.0->astrocut) (25.3.0)\n", + "Requirement already satisfied: pyerfa>=2.0.1.1 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from astropy>=5.2->astrocut) (2.0.1.5)\n", + "Requirement already satisfied: astropy-iers-data>=0.2025.4.28.0.37.27 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from astropy>=5.2->astrocut) (0.2025.5.26.0.37.21)\n", + "Requirement already satisfied: aiohttp!=4.0.0a0,!=4.0.0a1 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from fsspec[http]>=2022.8.2->astrocut) (3.12.15)\n", + "Requirement already satisfied: aiohappyeyeballs>=2.5.0 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>=2022.8.2->astrocut) (2.6.1)\n", + "Requirement already satisfied: aiosignal>=1.4.0 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>=2022.8.2->astrocut) (1.4.0)\n", + "Requirement already satisfied: frozenlist>=1.1.1 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>=2022.8.2->astrocut) (1.7.0)\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>=2022.8.2->astrocut) (6.6.4)\n", + "Requirement already satisfied: propcache>=0.2.0 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>=2022.8.2->astrocut) (0.3.2)\n", + "Requirement already satisfied: yarl<2.0,>=1.17.0 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>=2022.8.2->astrocut) (1.20.1)\n", + "Requirement already satisfied: idna>=2.0 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from yarl<2.0,>=1.17.0->aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>=2022.8.2->astrocut) (3.10)\n", + "Requirement already satisfied: typing-extensions>=4.2 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from aiosignal>=1.4.0->aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>=2022.8.2->astrocut) (4.15.0)\n", + "Requirement already satisfied: asdf_wcs_schemas>=0.5.0 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from gwcs>=0.21.0->astrocut) (0.5.0)\n", + "Requirement already satisfied: asdf-astropy>=0.8.0 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from gwcs>=0.21.0->astrocut) (0.8.0)\n", + "Requirement already satisfied: asdf-coordinates-schemas>=0.4 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from asdf-astropy>=0.8.0->gwcs>=0.21.0->astrocut) (0.4.0)\n", + "Requirement already satisfied: charset_normalizer<4,>=2 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from requests>=2.32.3->astrocut) (3.4.3)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from requests>=2.32.3->astrocut) (2.5.0)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from requests>=2.32.3->astrocut) (2025.8.3)\n", + "Requirement already satisfied: lz4>=4.3.0 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from roman_datamodels>=0.19.0->astrocut) (4.4.4)\n", + "Requirement already satisfied: rad>=0.27.0 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from roman_datamodels>=0.19.0->astrocut) (0.27.0)\n", + "Requirement already satisfied: pyarrow>=10.0.1 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from roman_datamodels>=0.19.0->astrocut) (21.0.0)\n", + "Requirement already satisfied: aiobotocore<3.0.0,>=2.5.4 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from s3fs>=2022.8.2->astrocut) (2.24.2)\n", + "Requirement already satisfied: aioitertools<1.0.0,>=0.5.1 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from aiobotocore<3.0.0,>=2.5.4->s3fs>=2022.8.2->astrocut) (0.12.0)\n", + "Requirement already satisfied: botocore<1.40.19,>=1.40.15 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from aiobotocore<3.0.0,>=2.5.4->s3fs>=2022.8.2->astrocut) (1.40.18)\n", + "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from aiobotocore<3.0.0,>=2.5.4->s3fs>=2022.8.2->astrocut) (2.9.0.post0)\n", + "Requirement already satisfied: wrapt<2.0.0,>=1.10.10 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from aiobotocore<3.0.0,>=2.5.4->s3fs>=2022.8.2->astrocut) (1.17.3)\n", + "Requirement already satisfied: six>=1.5 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from python-dateutil<3.0.0,>=2.1->aiobotocore<3.0.0,>=2.5.4->s3fs>=2022.8.2->astrocut) (1.17.0)\n", + "Requirement already satisfied: boto3>=1.16.35 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from s3path>=0.5.7->astrocut) (1.40.18)\n", + "Requirement already satisfied: smart-open>=5.1.0 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from s3path>=0.5.7->astrocut) (7.3.1)\n", + "Requirement already satisfied: s3transfer<0.14.0,>=0.13.0 in /Users/rs17612/anaconda3/envs/DRUID/lib/python3.12/site-packages (from boto3>=1.16.35->s3path>=0.5.7->astrocut) (0.13.1)\n" + ] + } + ], + "source": [ + "!pip install astrocut" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -23,9 +89,29 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Filename: (No file associated with this HDUList)\n", + "No. Name Ver Type Cards Dimensions Format\n", + " 0 PRIMARY 1 PrimaryHDU 9 () \n", + " 1 CUTOUT 1 ImageHDU 93 (1000, 1000) float32 \n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING: FITSFixedWarning: RADECSYS= 'ICRS ' / Coordinate reference frame \n", + "the RADECSYS keyword is deprecated, use RADESYSa. [astropy]\n", + "WARNING: FITSFixedWarning: 'datfix' made the change 'Set DATE-END to '2016-08-30T09:29:55.786' from MJD-END'. [astropy]\n" + ] + } + ], "source": [ "from astrocut import fits_cut\n", "from astropy.coordinates import SkyCoord\n", @@ -54,23 +140,127 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " \n", + "\n", + "\u001b[91m#############################################\u001b[0m\n", + "\u001b[92m\n", + "_______ _______ _________ ______ \n", + "( __ \\ ( ____ )|\\ /|\\__ __/( __ \\ \n", + "| ( \\ )| ( )|| ) ( | ) ( | ( \\ )\n", + "| | ) || (____)|| | | | | | | | ) |\n", + "| | | || __)| | | | | | | | | |\n", + "| | ) || (\\ ( | | | | | | | | ) |\n", + "| (__/ )| ) \\ \\__| (___) |___) (___| (__/ )\n", + "(______/ |/ \\__/(_______)\\_______/(______/ \n", + " \n", + "\u001b[0m\n", + "\u001b[91m#############################################\u001b[0m\n", + "\n", + "\u001b[1mDetector of astRonomical soUrces in optIcal and raDio images\u001b[0m\n", + "\n", + "Version: 1.0\n", + "\n", + "For more information see:\n", + "\u001b[94mhttps://github.com/RhysAlfShaw/DRUID\u001b[0m\n", + "\n", + "Calculating background map and RMS map...\n", + "Calculating background map and RMS map from image.\n", + "Background calculation took 0.21 seconds.\n", + "Background map and RMS map calculated.\n", + "Thresholding to find source islands...\n", + "Labeling connected components took 0.00 seconds. Found 1256 components.\n", + "Calculating region properties took 0.01 seconds. Found 1256 properties.\n", + "Cropping components took 0.02 seconds. Found 143 source islands.\n", + "Thresholding took 0.03 seconds.\n", + "Found 143 source islands in the image with area limit 5.\n", + "Processing 143 source islands sequentially.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "143it [00:08, 16.56it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Homology computation took 8.64 seconds.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], "source": [ + "# import package from a local directory\n", + "import sys\n", + "sys.path.append('../') # path to the DRUID package\n", "from DRUID import sf\n", "\n", - "findmysources = sf(image=image, # Image Array\n", - " header=cutout_file[0][1].header, # Image Header\n", - " mode='optical', # Mode (optical, radio, other)\n", - " cutup=False, # Cutup the image into smaller pieces?\n", - " smooth_sigma=1.1, # Sigma for the smoothing the image with gaussian filter\n", - " area_limit=3) # Minimum area for a source to be considered as a source\n", - "findmysources.set_background(detection_threshold=2, # Detection threshold for the background\n", - " analysis_threshold=2, # Analysis threshold for the background\n", - " mode='SEX') # Mode (SEX, rms, mad_std, sigma_clip) \n", - "findmysources.phsf(lifetime_limit=findmysources.local_bg, # Lifetime limit for the sources (in absolute units)\n", - " lifetime_limit_fraction=1.001) # Lifetime limit for the sources (as fraction of brith - death)\n", - "findmysources.source_characterising() # Characterise the sources (i.e. measure the properties of the sources and calculate the contours)\n", - "catalogue = findmysources.catalogue # get the catalogue from the sf object\n" + "findmysources = sf(image=image, \n", + " header=cutout_file[0][1].header, \n", + " mode='optical', \n", + " area_limit=5,\n", + " num_threads=1) \n", + "findmysources.set_background(detection_threshold=5) \n", + "findmysources.phsf(lifetime_limit_fraction=1) \n", + "catalogue = findmysources.catalog " + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "shape: (2_255, 20)\n", + "┌────────────┬────────────┬─────┬─────┬───┬────────────┬─────────────────────┬──────────┬──────────┐\n", + "│ birth ┆ death ┆ x1 ┆ y1 ┆ … ┆ parent_tag ┆ contour ┆ Island_X ┆ Island_Y │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ f64 ┆ f64 ┆ f64 ┆ f64 ┆ ┆ i64 ┆ list[list[f64]] ┆ i32 ┆ i32 │\n", + "╞════════════╪════════════╪═════╪═════╪═══╪════════════╪═════════════════════╪══════════╪══════════╡\n", + "│ 8.7076e-12 ┆ 6.4912e-12 ┆ 1.0 ┆ 2.0 ┆ … ┆ 1 ┆ [[2.0, 2.0], [1.0, ┆ 3 ┆ 996 │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ 1.0], … [2.… ┆ ┆ │\n", + "│ 9.3193e-12 ┆ 0.0 ┆ 2.0 ┆ 3.0 ┆ … ┆ 1 ┆ [[5.0, 2.0], [5.0, ┆ 3 ┆ 996 │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ 1.0], … [5.… ┆ ┆ │\n", + "│ 9.3193e-12 ┆ 6.4912e-12 ┆ 2.0 ┆ 3.0 ┆ … ┆ 2 ┆ [[5.0, 1.0], [4.0, ┆ 3 ┆ 996 │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ 0.0], … [5.… ┆ ┆ │\n", + "│ 2.3841e-11 ┆ 0.0 ┆ 4.0 ┆ 3.0 ┆ … ┆ 0 ┆ [[9.0, 3.0], [8.0, ┆ 23 ┆ 436 │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ 2.0], … [9.… ┆ ┆ │\n", + "│ 8.2292e-12 ┆ 6.7574e-12 ┆ 7.0 ┆ 1.0 ┆ … ┆ 1 ┆ [[8.0, 1.0], [7.0, ┆ 37 ┆ 891 │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ 0.0], … [8.… ┆ ┆ │\n", + "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", + "│ 7.0367e-12 ┆ 4.6814e-12 ┆ 1.0 ┆ 5.0 ┆ … ┆ 1 ┆ [[2.0, 5.0], [2.0, ┆ 962 ┆ 448 │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ 4.0], … [2.… ┆ ┆ │\n", + "│ 1.5760e-11 ┆ 0.0 ┆ 7.0 ┆ 6.0 ┆ … ┆ 2 ┆ [[13.0, 5.0], ┆ 962 ┆ 448 │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ [13.0, 4.0], … [… ┆ ┆ │\n", + "│ 1.5760e-11 ┆ 1.2681e-11 ┆ 7.0 ┆ 6.0 ┆ … ┆ 3 ┆ [[9.0, 7.0], [9.0, ┆ 962 ┆ 448 │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ 6.0], … [9.… ┆ ┆ │\n", + "│ 1.0088e-11 ┆ 0.0 ┆ 3.0 ┆ 4.0 ┆ … ┆ 0 ┆ [[6.0, 3.0], [6.0, ┆ 971 ┆ 408 │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ 2.0], … [6.… ┆ ┆ │\n", + "│ 1.4803e-11 ┆ 0.0 ┆ 3.0 ┆ 2.0 ┆ … ┆ 0 ┆ [[5.0, 4.0], [5.0, ┆ 978 ┆ 823 │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ 3.0], … [5.… ┆ ┆ │\n", + "└────────────┴────────────┴─────┴─────┴───┴────────────┴─────────────────────┴──────────┴──────────┘\n" + ] + } + ], + "source": [ + "print(catalogue)" ] }, { @@ -79,16 +269,30 @@ "metadata": {}, "outputs": [], "source": [ + "import numpy as np\n", "from matplotlib.pylab import plt\n", "\n", "plt.figure(figsize=(10,10))\n", "plt.imshow(image, cmap='gray', vmin=0,vmax=1E-9)\n", "for con in catalogue['contour']:\n", - " plt.plot(con[:, 1], con[:, 0])\n", + " contours = catalogue[\"contour\"].to_list()\n", + " Island_X = catalogue[\"Island_X\"].to_list()\n", + " Island_Y = catalogue[\"Island_Y\"].to_list()\n", + " for i, contour in enumerate(contours):\n", + " contour = np.array(contour)\n", + " Island_X_val = Island_X[i]\n", + " Island_Y_val = Island_Y[i]\n", + " plt.plot(\n", + " contour[:, 1] + Island_Y_val,\n", + " contour[:, 0] + Island_X_val,\n", + " # color=\"red\",\n", + " alpha=1,\n", + " linewidth=1,\n", + " )\n", "plt.xlim(300,700)\n", "plt.ylim(300,700)\n", "plt.savefig('images/sample_gal.png')\n", - "#plt.show()" + "plt.show()" ] }, { @@ -101,7 +305,7 @@ ], "metadata": { "kernelspec": { - "display_name": "base", + "display_name": "DRUID", "language": "python", "name": "python3" }, @@ -115,7 +319,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.7" + "version": "3.12.10" } }, "nbformat": 4, diff --git a/test.py b/test.py index e3e734f..2161d89 100644 --- a/test.py +++ b/test.py @@ -1,137 +1,93 @@ from DRUID import sf - - -# def create_dummy_image(working_directory="DRUID/temp"): -# """ -# Create a dummy FITS image for testing purposes. -# """ -# from astropy.io import fits -# import numpy as np - -# # Create a dummy image with random data -# dim = 10_000 # 20,000 x 20,000 pixels -# n_sources = 50_000 # Number of bright sources to add -# data = np.random.normal(size=(dim, dim)).astype(np.float32) -# # add many bright sources -# for _ in range(n_sources): -# x = np.random.randint(0, dim) -# y = np.random.randint(0, dim) -# data[x, y] += np.random.uniform(500, 10000) - -# # convolve the image with a Gaussian kernel to simulate a more realistic image -# from scipy.ndimage import gaussian_filter - -# data = gaussian_filter(data, sigma=5) - -# # Create a FITS file -# hdu = fits.PrimaryHDU(data) -# hdu.writeto(f"{working_directory}/dummy_image.fits", overwrite=True) - -# # # plot the image to verify -# import matplotlib.pyplot as plt - -# img_size = [2000, 3000, 5000, 10000] -# bg_time = [0.79, 1.7, 4.35, 17.36] -# thresh_time = [5.74, 16.2, 50, 60 * 7] -# # fit exponential curves to the data -# from scipy.optimize import curve_fit - -# def exp_func(x, a, b): -# return a * np.exp(b * x) - -# popt_bg, _ = curve_fit(exp_func, img_size, bg_time) -# popt_thresh, _ = curve_fit(exp_func, img_size, thresh_time) - -# plt.plot(img_size, bg_time, label="Background Calculation Time") -# plt.plot(img_size, thresh_time, label="Thresholding Time") -# extrapolated_img_size = np.linspace(0, 20000, 100) -# plt.plot( -# extrapolated_img_size, -# exp_func(np.array(extrapolated_img_size), *popt_bg), -# linestyle="--", -# color="blue", -# ) - -# plt.plot( -# extrapolated_img_size, -# exp_func(np.array(extrapolated_img_size), *popt_thresh), -# linestyle="--", -# color="orange", -# ) -# plt.legend() -# plt.xlabel("Image Size (pixels)") -# plt.ylabel("Time (seconds)") -# plt.title("Background Calculation and Thresholding Time vs Image Size") -# plt.show() +import matplotlib.pyplot as plt +import numpy as np def main(): working_dir = "DRUID/temp" - # create_dummy_image( - # working_directory=working_dir - # ) # Create a dummy image for testing - image_path = "DRUID/temp/dummy_image.fits" - # image_path = "/Users/rs17612/Documents/Optical_IR_Data/EUCLID/EUC_MER_BGSUB-MOSAIC-VIS_TILE101158277-BB647A_20240122T115602.395130Z_00.00.fits" - findmysource = sf( - image=image_path, - mode="optical", - area_limit=5, - num_threads=1, - working_directory=working_dir, - cashe=False, - ) - findmysource.set_background() - findmysource.phsf() - - # pint the catalog - catalog = findmysource.catalog - print("Catalog:", catalog) - - # plot the image, background, and catalog - import matplotlib.pyplot as plt - import numpy as np - - plt.figure(figsize=(10, 10)) - plt.imshow(findmysource.image, cmap="gray", origin="lower") - plt.scatter( - catalog["y1"] + catalog["Island_Y"], - catalog["x1"] + catalog["Island_X"], - s=1, - c="red", - label="Source Islands", - ) - plt.colorbar() - plt.title("Source Islands on Image") - plt.xlabel("X Pixel") - plt.ylabel("Y Pixel") - plt.legend() - plt.savefig(f"{working_dir}/source_islands_on_image.png") - plt.show() - - # plot the contours - contours = catalog["contour"].to_list() - Island_X = catalog["Island_Y"].to_list() - Island_Y = catalog["Island_X"].to_list() - - plt.figure(figsize=(10, 10)) - plt.imshow(findmysource.image, cmap="gray", origin="lower") - for i, contour in enumerate(contours): - contour = np.array(contour) - Island_X_val = Island_X[i] - Island_Y_val = Island_Y[i] - plt.plot( - contour[:, 1] + Island_X_val, - contour[:, 0] + Island_Y_val, - color="red", - alpha=0.5, - linewidth=0.5, + image_paths = [ + "/Users/rs17612/Documents/Radio_Data/3CRR/3C401", + "/Users/rs17612/Documents/Radio_Data/3CRR/3C295", + "/Users/rs17612/Documents/Radio_Data/3CRR/3C438", + "/Users/rs17612/Documents/Radio_Data/3CRR/3C452", + "/Users/rs17612/Documents/Radio_Data/3CRR/3C76P1", + ] + + n_images = len(image_paths) + fig, axes = plt.subplots(1, n_images, figsize=(5 * n_images, 5)) + i = 1 + if n_images == 1: + axes = [axes] + + for ax, image_path in zip(axes, image_paths): + findmysource = sf( + image=image_path, + mode="radio", + area_limit=15, + num_threads=2, + working_directory=working_dir, + cashe=False, + ) + findmysource.set_background( + detection_threshold=5, + analysis_threshold=3, + box_size=50, + ) + findmysource.phsf(lifetime_limit_fraction=1.4) + + catalog = findmysource.catalog + # save polars catalog to temp folder + if i == 1: + catalog.write_parquet( + f"{working_dir}/catalog_{image_path.split('/')[-1]}.parquet" + ) + # save image as numpy array + np.save( + f"{working_dir}/image_{image_path.split('/')[-1]}.npy", + findmysource.image, + ) + # save background and background rms as numpy array + np.save( + f"{working_dir}/background_{image_path.split('/')[-1]}.npy", + findmysource.background_map, + ) + np.save( + f"{working_dir}/background_rms_{image_path.split('/')[-1]}.npy", + findmysource.background_rms_map, + ) + i += 1 + + print(catalog) + ax.imshow(findmysource.image, cmap="gray", origin="lower") + ax.scatter( + catalog["y1"] + catalog["Island_Y"], + catalog["x1"] + catalog["Island_X"], + s=1, + c="red", + label="Source Islands", ) - plt.colorbar() - plt.title("Contours of Source Islands") - plt.xlabel("X Pixel") - plt.ylabel("Y Pixel") - plt.savefig(f"{working_dir}/source_islands_contours.png") - plt.show() + contours = catalog["contour"].to_list() + Island_X = catalog["Island_X"].to_list() + Island_Y = catalog["Island_Y"].to_list() + for i, contour in enumerate(contours): + contour = np.array(contour) + Island_X_val = Island_X[i] + Island_Y_val = Island_Y[i] + ax.plot( + contour[:, 1] + Island_Y_val, + contour[:, 0] + Island_X_val, + # color="red", + alpha=1, + linewidth=1, + ) + ax.set_title(f"{image_path.split('/')[-1]}") + ax.set_xlabel("X Pixel") + ax.set_ylabel("Y Pixel") + ax.legend() + + plt.tight_layout() + plt.savefig(f"{working_dir}/all_source_islands_on_images.png") + # plt.show() if __name__ == "__main__": From aa048b3c4ae1bf2ef437d67434007980bae8f82f Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Mon, 13 Jul 2026 15:38:18 +0100 Subject: [PATCH 37/69] fixed rebase error --- DRUID/main.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/DRUID/main.py b/DRUID/main.py index cc47fdb..1395e6d 100644 --- a/DRUID/main.py +++ b/DRUID/main.py @@ -78,7 +78,6 @@ def _worker( lifetime_limit_fraction=lifetime_limit_fraction, ) -<<<<<<< HEAD # # source characteristics measure here! # if cat is not None and not cat.is_empty(): # # Add source characteristics to the catalog @@ -90,7 +89,7 @@ def _worker( # position, # analysis_threshold, # ) -======= + # source characteristics measure here! if cat is not None and not cat.is_empty(): # Add source characteristics to the catalog @@ -108,7 +107,6 @@ def _worker( EFFGAIN, EXPTIME, ) ->>>>>>> 1e15fb5 (update?) # # Add position to the catalog # if cat is None or cat.is_empty(): From 6612246ce407392e385271755712b07a145fd2ed Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Mon, 13 Jul 2026 17:15:34 +0100 Subject: [PATCH 38/69] some testing scripts --- large_image_test.py | 25 +++++++++++++++++++++++++ test.py | 13 ++++++++----- 2 files changed, 33 insertions(+), 5 deletions(-) create mode 100644 large_image_test.py diff --git a/large_image_test.py b/large_image_test.py new file mode 100644 index 0000000..2caea5b --- /dev/null +++ b/large_image_test.py @@ -0,0 +1,25 @@ +from DRUID import sf + + +def main(): + path = "/Users/rs17612/Documents/Optical_IR_Data/EUCLID/EUC_MER_BGSUB-MOSAIC-DES-Z_TILE102026098-321DAA_20240407T183900.016572Z_00.00.fits" + working_dir = "DRUID/temp_new" + findmysource = sf( + image=path, + mode="optical", + area_limit=15, + num_threads=2, + working_directory=working_dir, + cashe=False, + ) + findmysource.set_background( + detection_threshold=5, + analysis_threshold=3, + box_size=50, + ) + findmysource.phsf(lifetime_limit_fraction=1.4) + print(findmysource.catalog) + + +if __name__ == "__main__": + main() diff --git a/test.py b/test.py index 2161d89..9a6ead5 100644 --- a/test.py +++ b/test.py @@ -59,24 +59,27 @@ def main(): print(catalog) ax.imshow(findmysource.image, cmap="gray", origin="lower") + # Corrected Scatter Plot ax.scatter( - catalog["y1"] + catalog["Island_Y"], - catalog["x1"] + catalog["Island_X"], + catalog["y1"] + catalog["Island_X"], # X + X + catalog["x1"] + catalog["Island_Y"], # Y + Y s=1, c="red", label="Source Islands", ) + contours = catalog["contour"].to_list() Island_X = catalog["Island_X"].to_list() Island_Y = catalog["Island_Y"].to_list() + + # Corrected Contour Plot for i, contour in enumerate(contours): contour = np.array(contour) Island_X_val = Island_X[i] Island_Y_val = Island_Y[i] ax.plot( - contour[:, 1] + Island_Y_val, - contour[:, 0] + Island_X_val, - # color="red", + contour[:, 1] + Island_X_val, # X + X + contour[:, 0] + Island_Y_val, # Y + Y alpha=1, linewidth=1, ) From 9d9fb3ec51fc9cf6a6d431f7e761331a9e424bf5 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Mon, 13 Jul 2026 17:19:54 +0100 Subject: [PATCH 39/69] large update with multiprocessing improvements --- DRUID/main.py | 403 +++++++++++++++++++++++++------------------------- 1 file changed, 199 insertions(+), 204 deletions(-) diff --git a/DRUID/main.py b/DRUID/main.py index 1395e6d..83ed490 100644 --- a/DRUID/main.py +++ b/DRUID/main.py @@ -5,17 +5,14 @@ setproctitle.setproctitle("DRUID") import numpy as np -import astropy +import astropy.io.fits import os -import random - -# this prevent polars from using all available threads. -# Especially for multithreaded homology computation. otherwise we will spawn nested threads. -os.environ["POLARS_MAX_THREADS"] = "1" -import polars as pl +import sys import time - +import polars as pl +from functools import partial from multiprocessing import get_context +import multiprocessing from tqdm import tqdm from .src import utils @@ -23,7 +20,9 @@ from .src import background from .src import source from .src import properties -from functools import partial + +# Prevent Polars from thread oversubscription during multiprocessing +os.environ["POLARS_MAX_THREADS"] = "1" RED = "\033[91m" GREEN = "\033[92m" @@ -31,7 +30,6 @@ RESET = "\033[0m" BOLD = "\033[1m" DRUID_MESSAGE = rf""" - {RED}#############################################{RESET} {GREEN} _______ _______ _________ ______ @@ -49,14 +47,30 @@ {BOLD}Detector of astRonomical soUrces in optIcal and raDio images{RESET} Version: {version} - For more information see: {BLUE}https://github.com/RhysAlfShaw/DRUID{RESET} """ +# Global variables for worker processes to avoid IPC memory overhead +global_image = None +global_background_map = None +global_background_rms_map = None + + +def _worker_init(img, bg, bg_rms): + """ + Initializer for multiprocessing pool. + Loads the main arrays into the global namespace of each worker process, + preventing massive IPC data transfers. + """ + global global_image, global_background_map, global_background_rms_map + global_image = img + global_background_map = bg + global_background_rms_map = bg_rms + def _worker( - iterable_image, + island_info, analysis_threshold, lifetime_limit, lifetime_limit_fraction, @@ -66,38 +80,41 @@ def _worker( EFFRON=None, EFFGAIN=None, EXPTIME=None, -) -> "pl.DataFrame": +) -> pl.DataFrame: """ Worker function to compute homology for a single source island. + Reads from global arrays to minimize memory serialization. """ - image, position, background, background_rms = iterable_image + bbox, position = island_info + min_row, min_col, max_row, max_col = bbox + + # Slice the global arrays natively in the worker + raw_image_cutout = global_image[min_row:max_row, min_col:max_col] + bg_cutout = global_background_map[min_row:max_row, min_col:max_col] + bg_rms_cutout = global_background_rms_map[min_row:max_row, min_col:max_col] + + # ---> FIX: Re-mask the cutout to remove bounding box corners <--- + # We must zero out pixels below the threshold so the homology algorithm + # doesn't trace the artificial rectangular boundary of the cutout. + local_threshold = bg_cutout + (analysis_threshold * bg_rms_cutout) + island_mask = raw_image_cutout > local_threshold + + # Create a new array to avoid mutating the global shared memory + image_cutout = np.where(island_mask, raw_image_cutout, 0) + cat = homology.compute_homology( - image, - analysis_threshold=analysis_threshold * background_rms, + image_cutout, + analysis_threshold=analysis_threshold * np.mean(bg_rms_cutout), lifetime_limit=lifetime_limit, lifetime_limit_fraction=lifetime_limit_fraction, ) - # # source characteristics measure here! - # if cat is not None and not cat.is_empty(): - # # Add source characteristics to the catalog - # cat = properties.calculate_properties( - # cat, - # image, - # background, - # background_rms, - # position, - # analysis_threshold, - # ) - - # source characteristics measure here! if cat is not None and not cat.is_empty(): - # Add source characteristics to the catalog cat = properties.calculate_properties( cat, - image, - background, - background_rms, + image_cutout, + bg_cutout, + bg_rms_cutout, position, analysis_threshold, mode, @@ -108,13 +125,14 @@ def _worker( EXPTIME, ) - # # Add position to the catalog - # if cat is None or cat.is_empty(): - # return None - # cat = cat.with_columns( - # pl.lit(position[0]).alias("Island_X"), - # pl.lit(position[1]).alias("Island_Y"), - # ) + # Append global offsets to the catalog for plotting + cat = cat.with_columns( + [ + pl.lit(position[0]).alias("Island_Y"), + pl.lit(position[1]).alias("Island_X"), + ] + ) + return cat @@ -131,12 +149,57 @@ def __init__( working_directory: str = "DRUID/temp", cashe: bool = False, ): - """ + print(multiprocessing.current_process().name) + error_msg = f""" + {RED}===================================================================={RESET} + {BOLD}DRUID MULTIPROCESSING ERROR{RESET} + + It looks like you are running DRUID with `num_threads > 1` without + protecting your execution code. + + Because DRUID uses Python's robust multiprocessing, you must wrap your + top-level code in the `if __name__ == '__main__':` block. + + {BLUE}Please update your script to look like this:{RESET} - Initialise DRUID and preform some basic checks. + from DRUID import sf + def main(): + findmysource = sf(num_threads={num_threads}, ...) + findmysource.set_background(...) + findmysource.phsf(...) + + if __name__ == "__main__": + main() + {RED}===================================================================={RESET} """ + # CHILD PROCESS TRAP (Catches the fork bomb during spawn) + if multiprocessing.current_process().name != "MainProcess": + raise RuntimeError(error_msg) + + # PRE-FLIGHT FAST FAIL (Saves time in the MainProcess) + if num_threads > 1 and multiprocessing.current_process().name == "MainProcess": + try: + import __main__ + + # Ensure we are running from a script file, not an interactive REPL/Jupyter + if hasattr(__main__, "__file__") and os.path.exists(__main__.__file__): + with open(__main__.__file__, "r") as f: + script_content = f.read() + + # Remove spaces and normalize quotes to catch all syntax variations + clean_script = script_content.replace(" ", "").replace("'", '"') + + # If the guard is missing, blow up immediately! + if 'if__name__=="__main__":' not in clean_script: + raise RuntimeError(error_msg) + except Exception as e: + # If we can't read the file (e.g. running in Jupyter), + # we silently pass and let Trap #1 catch it if a failure happens later. + if isinstance(e, RuntimeError): + raise e + print(DRUID_MESSAGE) self.mode = mode @@ -165,7 +228,6 @@ def __init__( "Image must be a file path (str) or a NumPy array (np.ndarray)." ) - # check if there are files in the working directory if self.cashe: if not os.path.exists(working_directory): os.makedirs(working_directory) @@ -173,49 +235,35 @@ def __init__( else: self.working_directory = None - if self.mode == "radio": - try: - self.BMAJ = self.header["BMAJ"] - self.BMIN = self.header["BMIN"] - self.EFFGAIN = None - self.EFFRON = None - self.EXPTIME = None - - except KeyError as e: - print( - "Warning: Could not find BMAJ or BMIN in header, beam parameters will be set to None." - ) - self.BMAJ = None - self.BMIN = None + self.BMAJ, self.BMIN = None, None + self.EFFRON, self.EFFGAIN, self.EXPTIME = None, None, None - elif self.mode == "optical": + if self.mode == "radio" and self.header: try: - self.EFFRON = self.header["EFFRON"] - self.EFFGAIN = self.header["EFFGAIN"] - self.EXPTIME = self.header["EXPTIME"] - self.BMAJ = None - self.BMIN = None - except KeyError as e: - print( - "Warning: Could not find EFFRON, EFFGAIN, or EXPTIME, flux_err will be set to 0." - ) - - def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1): - """ - Runs the source findin algorithm on the image. - - Requires that the background has first been calculated. - - """ - if self.background_map is None or self.background_rms_map is None: + self.BMAJ = self.header.get("BMAJ") + self.BMIN = self.header.get("BMIN") + except KeyError: + print("Warning: Could not find BMAJ or BMIN in header.") + elif self.mode == "optical" and self.header: + try: + self.EFFRON = self.header.get("EFFRON") + self.EFFGAIN = self.header.get("EFFGAIN") + self.EXPTIME = self.header.get("EXPTIME") + except KeyError: + print("Warning: Could not find EFFRON, EFFGAIN, or EXPTIME.") + + def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0): + if ( + getattr(self, "background_map", None) is None + or getattr(self, "background_rms_map", None) is None + ): raise ValueError( - "Background map and RMS map must be set before running source finding." - "Please call set_background() first. or assign them manually." + "Background maps must be set before running source finding." ) if self.verbose: print("Thresholding to find source islands...") - # this function is rather slow. + t0 = time.time() source_islands = source.create_source_islands( self.image, @@ -226,173 +274,120 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1): area_limit=self.area_limit, verbose=self.verbose, ) - t1 = time.time() - print(f"Thresholding took {t1 - t0:.2f} seconds.") - t0 = time.time() + if self.verbose: - print( - f"Found {len(source_islands['positions'])} source islands in the image with area limit {self.area_limit}." - ) + print(f"Thresholding took {t1 - t0:.2f} seconds.") + print(f"Found {len(source_islands['positions'])} source islands.") - images_to_process = source_islands["island_image"] - print( - "Max island image shape:", - np.max([img.shape for img in images_to_process], axis=0), + # Zipping bounding boxes and positions (lightweight metadata) + iterable_islands = list( + zip(source_islands["bboxes"], source_islands["positions"]) + ) + # ---> FIX: Strategy 1 - LPT Scheduling <--- + # Sort the islands by bounding box area (proxy for complexity) in DESCENDING order. + # bbox is (min_row, min_col, max_row, max_col) + # Area = (max_row - min_row) * (max_col - min_col) + iterable_islands.sort( + key=lambda item: (item[0][2] - item[0][0]) * (item[0][3] - item[0][1]), + reverse=True, ) - if not images_to_process: + + if not iterable_islands: if self.verbose: print("No source islands to process.") self.catalog = pl.DataFrame() return - # make the iterable images_to_process and poistions - iterable_images = zip( - images_to_process, - source_islands["positions"], - source_islands["background"], - source_islands["background_rms"], + t0 = time.time() + + worker_func = partial( + _worker, + analysis_threshold=self.analysis_threshold, + lifetime_limit=lifetime_limit, + lifetime_limit_fraction=lifetime_limit_fraction, + mode=self.mode, + BMAJ=self.BMAJ, + BMIN=self.BMIN, + EFFRON=self.EFFRON, + EFFGAIN=self.EFFGAIN, + EXPTIME=self.EXPTIME, ) - print(iterable_images) + + results = [] if self.num_threads > 1: if self.verbose: - print( - f"Processing {len(images_to_process)} source islands in parallel. with {self.num_threads} threads." - ) - print("images to process:", len(images_to_process)) - batch_size = len(images_to_process) // (self.num_threads * 10) - if batch_size < 1: # prevent batch size of 0 - batch_size = 1 - print(f"Batch size: {batch_size}") - with get_context("spawn").Pool(self.num_threads) as p: - # Use functools.partial to pass additional arguments to _worker - worker_func = partial( - _worker, # analysis threshold * rms at this point. - analysis_threshold=self.analysis_threshold, - lifetime_limit=lifetime_limit, - lifetime_limit_fraction=lifetime_limit_fraction, - mode=self.mode, - BMAJ=self.BMAJ, - BMIN=self.BMIN, - EFFRON=self.EFFRON, - EFFGAIN=self.EFFGAIN, - EXPTIME=self.EXPTIME, - ) - # print(iterable_images) - results = p.map(worker_func, iterable_images, chunksize=batch_size) - - else: - print(f"Processing {len(images_to_process)} source islands sequentially.") - results = [] - for img, position, background, background_rms in tqdm(iterable_images): - results.append( - _worker( - (img, position, background, background_rms), - self.analysis_threshold, - lifetime_limit, - lifetime_limit_fraction, - self.mode, - self.BMAJ, - self.BMIN, - self.EFFRON, - self.EFFGAIN, - self.EXPTIME, + print(f"Processing in parallel with {self.num_threads} threads.") + + optimal_chunksize = 1 + # Using initializer to set memory on workers safely + with get_context("spawn").Pool( + self.num_threads, + initializer=_worker_init, + initargs=(self.image, self.background_map, self.background_rms_map), + ) as p: + # imap_unordered will now instantly yield massive sources as they finish, + # while dynamically feeding tiny sources to whatever worker is free. + results = list( + p.imap_unordered( + worker_func, iterable_islands, chunksize=optimal_chunksize ) ) + else: + if self.verbose: + print("Processing sequentially.") + _worker_init(self.image, self.background_map, self.background_rms_map) + for island in tqdm(iterable_islands, disable=not self.verbose): + results.append(worker_func(island)) - # combine the results catalogs to a single catalog - + results = [res for res in results if res is not None and not res.is_empty()] if results: - # remove any None results - results = [res for res in results if res is not None] self.catalog = utils.combine_polars_catalogs(results) + else: + self.catalog = pl.DataFrame() t1 = time.time() - print(f"Homology computation took {t1 - t0:.2f} seconds.") + if self.verbose: + print(f"Homology computation took {t1 - t0:.2f} seconds.") def set_background( self, method: str = "rms", detection_threshold: int = 5, analysis_threshold: int = 3, - box_size: tuple = (50, 50), # kernal size for background calculation - filter_size: tuple = (3, 3), # size of median filter for background map - kernel_size: int = 3, # size of kernel for sigma clipping. + box_size: tuple = (50, 50), + filter_size: tuple = (3, 3), + kernel_size: int = 3, ): - """ - Calculate the background map of the image. - This is required before running the source finding algorithm. - """ - # Check if background maps already exist in the working directory. - if self.verbose: print("Calculating background map and RMS map...") t0 = time.time() self.detection_threshold = detection_threshold self.analysis_threshold = analysis_threshold - if self.cashe: - if os.path.exists(self.working_directory + "/background_map.npy"): - if os.path.exists(self.working_directory + "/background_rms_map.npy"): - if self.verbose: - print( - "Background map and RMS map already exist. Loading from disk." - ) - self.background_map = np.load( - self.working_directory + "/background_map.npy" - ) - self.background_rms_map = np.load( - self.working_directory + "/background_rms_map.npy" - ) - else: - if self.verbose: - print( - "Background map and RMS map do not exist. Calculating from image." - ) - self.background_map, self.background_rms_map = ( - background.calculate_background_maps( - self.image, - bg_estimator=method, - box_size=box_size, - filter_size=filter_size, - nsigma=detection_threshold, - kernel_size=kernel_size, - ) - ) - # Save the background maps to disk for future use. - np.save( - self.working_directory + "/background_map.npy", self.background_map - ) - np.save( - self.working_directory + "/background_rms_map.npy", - self.background_rms_map, - ) + bg_file = os.path.join(self.working_directory or "", "background_map.npy") + rms_file = os.path.join(self.working_directory or "", "background_rms_map.npy") - else: + if self.cashe and os.path.exists(bg_file) and os.path.exists(rms_file): if self.verbose: - print("Calculating background map and RMS map from image.") + print("Background maps exist. Loading from disk.") + self.background_map = np.load(bg_file) + self.background_rms_map = np.load(rms_file) + else: self.background_map, self.background_rms_map = ( background.calculate_background_maps( self.image, bg_estimator=method, box_size=box_size, - filter_size=(3, 3), + filter_size=filter_size, nsigma=detection_threshold, - kernel_size=3, + kernel_size=kernel_size, ) ) - if self.cashe: - # Save the background maps to disk for future use. - np.save( - self.working_directory + "/background_map.npy", self.background_map - ) - np.save( - self.working_directory + "/background_rms_map.npy", - self.background_rms_map, - ) - t1 = time.time() - print(f"Background calculation took {t1 - t0:.2f} seconds.") + np.save(bg_file, self.background_map) + np.save(rms_file, self.background_rms_map) + t1 = time.time() if self.verbose: - print("Background map and RMS map calculated.") + print(f"Background calculation took {t1 - t0:.2f} seconds.") From cc0b90260dc288ffce7bcc62dd8189704ed46fa2 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Mon, 13 Jul 2026 17:21:56 +0100 Subject: [PATCH 40/69] fixes and parallel update --- DRUID/src/background.py | 214 +---------------- DRUID/src/homology.py | 507 ++++++++-------------------------------- DRUID/src/properties.py | 271 ++++++++------------- DRUID/src/source.py | 217 ++--------------- DRUID/src/utils.py | 54 +---- 5 files changed, 231 insertions(+), 1032 deletions(-) diff --git a/DRUID/src/background.py b/DRUID/src/background.py index 1f3d8fd..9669ca6 100644 --- a/DRUID/src/background.py +++ b/DRUID/src/background.py @@ -15,42 +15,16 @@ SExtractorBackground, BackgroundBase, ) - from photutils.segmentation import detect_sources def make_source_mask(data, nsigma=3.0, kernel_size=3): - """ - Create a mask for sources in the image data using sigma clipping. - - Parameters - ---------- - data : numpy.ndarray - The 2D image data. - nsigma : float, optional - The number of standard deviations to use for sigma clipping. - The default is 3.0. - kernel_size : int, optional - The size of the convolution kernel for source detection. - The default is 3. - - Returns - ------- - mask : numpy.ndarray - A boolean mask where True indicates a source pixel. - """ mean, median, std = sigma_clipped_stats(data, sigma=nsigma) threshold = median + nsigma * std - - # Detect sources using a simple thresholding method can add masked pixels e.g. known bad areas of image. segm = detect_sources(data, threshold, npixels=kernel_size**2) if segm is None: - # No sources detected, return an empty mask return np.zeros(data.shape, dtype=bool) - # Create a mask from the segmentation map - mask = segm.data > 0 - - return mask + return segm.data > 0 def calculate_background_maps( @@ -61,61 +35,27 @@ def calculate_background_maps( nsigma=3.0, kernel_size=3, ): - """ - Calculates background and background RMS maps from a FITS image - similar to the style of PyBDSF. - - Parameters - ---------- - image_path : str - Path to the FITS image file. - box_size : tuple of int, optional - The size of the box to use for background estimation. - The default is (50, 50). - filter_size : tuple of int, optional - The size of the median filter to apply to the background map. - The default is (3, 3). - nsigma : float, optional - The number of standard deviations to use for sigma clipping - when detecting sources to mask. The default is 3.0. - bg_estimator : str or photutils.background.BackgroundBase, optional - The background estimator to use. Options are 'median', 'std', 'mad_std', - or a custom photutils background estimator object. The default is 'median'. - kernel_size : int, optional - The size of the convolution kernel for source detection. - The default is 3. - - Returns - ------- - background_map : numpy.ndarray - The calculated background map. - background_rms_map : numpy.ndarray - The calculated background RMS map. - """ - # Check if the input is a FITS file path or a numpy array - # to handle both cases of test and np.ndarray input. if isinstance(image, str): with fits.open(image) as hdul: data = hdul[0].data - elif isinstance(image, np.ndarray): data = image + else: + raise TypeError("Image must be a path or a numpy array.") - # mask sources with sigma clipping. mask = make_source_mask(data, nsigma=nsigma, kernel_size=kernel_size) - # calculate background and RMS Avalible background estimators available_estimators = { "median": MedianBackground, "std": StdBackgroundRMS, "mad_std": MADStdBackgroundRMS, "rms": StdBackgroundRMS, - "BiweightLocation": BiweightLocationBackground, - "BiweightScale": BiweightScaleBackgroundRMS, - "MM": MMMBackground, - "Mean": MeanBackground, + "biweightlocation": BiweightLocationBackground, + "biweightscale": BiweightScaleBackgroundRMS, + "mm": MMMBackground, + "mean": MeanBackground, "mode": ModeEstimatorBackground, - "SEx": SExtractorBackground, + "sex": SExtractorBackground, } if isinstance(bg_estimator, str): @@ -136,141 +76,3 @@ def calculate_background_maps( ) return bkg.background, bkg.background_rms - - -def make_gaussian_sources_image(image_size, sources): - """ - Create a 2D image with Gaussian sources. - Parameters - ---------- - image_size : tuple of int - Size of the image (height, width). - sources : list of dict - List of sources, each defined by a dictionary with keys: - 'amplitude', 'x_mean', 'y_mean', 'x_stddev', 'y_stddev', 'theta'. - Returns - ------- - numpy.ndarray - 2D array representing the image with Gaussian sources. - """ - image = np.zeros(image_size) - y, x = np.indices(image_size) - for source in sources: - amplitude = source["amplitude"] - x_mean = source["x_mean"] - y_mean = source["y_mean"] - x_stddev = source["x_stddev"] - y_stddev = source["y_stddev"] - theta = source["theta"] - a = (np.cos(theta) ** 2) / (2 * x_stddev**2) + (np.sin(theta) ** 2) / ( - 2 * y_stddev**2 - ) - b = -np.sin(2 * theta) / (4 * x_stddev**2) + np.sin(2 * theta) / ( - 4 * y_stddev**2 - ) - c = (np.sin(theta) ** 2) / (2 * x_stddev**2) + (np.cos(theta) ** 2) / ( - 2 * y_stddev**2 - ) - gaussian = amplitude * np.exp( - -( - a * (x - x_mean) ** 2 - + 2 * b * (x - x_mean) * (y - y_mean) - + c * (y - y_mean) ** 2 - ) - ) - image += gaussian - return image - - -if __name__ == "__main__": - - from astropy.wcs import WCS - from astropy.coordinates import SkyCoord - - image_size = (1000, 1000) - pixel_scale = 0.1 # degrees per pixel - center_coord = SkyCoord(ra=180, dec=30, unit="deg") - - wcs = WCS(naxis=2) - wcs.wcs.crpix = [image_size[0] / 2, image_size[1] / 2] - wcs.wcs.cdelt = np.array([-pixel_scale, pixel_scale]) - wcs.wcs.crval = [center_coord.ra.deg, center_coord.dec.deg] - wcs.wcs.ctype = ["RA---TAN", "DEC--TAN"] - num_sources = 6 - - # Define some dummy sources with different parameters - amplitude = np.random.uniform(50, 200, num_sources) - x_means = np.random.uniform(100, 900, num_sources) - y_means = np.random.uniform(100, 900, num_sources) - x_stds = np.random.uniform(5, 20, num_sources) - y_stds = np.random.uniform(5, 20, num_sources) - thetas = np.random.uniform(0, 2 * np.pi, num_sources) - - # put tow gaussians close together - - amplitude[0] = 200 - amplitude[1] = 200 - x_means[0] = 500 - y_means[0] = 500 - x_means[1] = 520 - y_means[1] = 520 - x_stds[0] = 10 - x_stds[1] = 10 - y_stds[0] = 10 - y_stds[1] = 10 - thetas[0] = 0 - thetas[1] = 0 - sources = [ - { - "amplitude": amplitude[i], - "x_mean": x_means[i], - "y_mean": y_means[i], - "x_stddev": x_stds[i], - "y_stddev": y_stds[i], - "theta": thetas[i], - } - for i in range(num_sources) - ] - - # Create a dummy image with sources and background noise - dummy_data = make_gaussian_sources_image(image_size, sources) - dummy_data += np.random.normal(0, 1, size=image_size) # Add some noise - - # Create a dummy FITS file - hdu = fits.PrimaryHDU(dummy_data, header=wcs.to_header()) - dummy_fits_path = "DRUID/temp/dummy_image.fits" - hdu.writeto(dummy_fits_path, overwrite=True) - - print(f"Dummy FITS file created at: {dummy_fits_path}") - - # calculate background maps - background_map, background_rms_map = calculate_background_maps(dummy_fits_path) - # save the background maps to a FITS file for testing purposes with other functions. - background_hdu = fits.PrimaryHDU(background_map) - background_rms_hdu = fits.PrimaryHDU(background_rms_map) - background_hdu.writeto("DRUID/temp/background_map.fits", overwrite=True) - background_rms_hdu.writeto("DRUID/temp/background_rms_map.fits", overwrite=True) - - print("Background map and background RMS map saved to FITS files.") - print("Background map calculated.") - print("Background RMS map calculated.") - - # plot the results with matplotlib or any other visualization library. - from matplotlib import pyplot as plt - - # plot dummy data, background map, and background RMS map - plt.figure(figsize=(12, 6)) - plt.subplot(1, 3, 1) - plt.imshow(dummy_data, origin="lower", cmap="gray", interpolation="nearest") - plt.title("Dummy Image with Sources") - plt.colorbar() - plt.subplot(1, 3, 2) - plt.imshow(background_map, origin="lower", cmap="gray", interpolation="nearest") - plt.title("Background Map") - plt.colorbar() - plt.subplot(1, 3, 3) - plt.imshow(background_rms_map, origin="lower", cmap="gray", interpolation="nearest") - plt.title("Background RMS Map") - plt.colorbar() - plt.tight_layout() - plt.show() diff --git a/DRUID/src/homology.py b/DRUID/src/homology.py index b270a8a..dced62f 100644 --- a/DRUID/src/homology.py +++ b/DRUID/src/homology.py @@ -6,313 +6,140 @@ import cripser import numpy as np import polars as pl - from scipy.ndimage import label as scipy_label -from tqdm import tqdm from skimage import measure -from astropy.io import fits -# For testing and development purposes, we import the following libraries: -import pandas as pd -import matplotlib.pyplot as plt + +def get_enclosing_mask_CPU(x, y, mask): + """ + Returns the connected components inside the mask starting from the point (x, y). + """ + labeled_mask, _ = scipy_label(mask) + if 0 <= x < mask.shape[1] and 0 <= y < mask.shape[0]: + label_at_pixel = labeled_mask[y, x] + if label_at_pixel != 0: + return labeled_mask == label_at_pixel + return None def _get_polygons_CPU(x1, y1, birth, death, image: np.ndarray): """ Returns the polygon of the enclosed area of the point (x,y) in the mask. """ + image_padded = np.pad(image, pad_width=1, mode="constant", constant_values=0) + mask = (image_padded <= birth) & (image_padded > death) + enclosed_mask = get_enclosing_mask_CPU(int(y1) + 1, int(x1) + 1, mask) - # add a 1 pxl padding to the image to avoid index errors - image = np.pad(image, pad_width=1, mode="constant", constant_values=0) - mask = np.zeros(image.shape) - mask = np.logical_or(mask, np.logical_and(image <= birth, image > death)) - mask = get_enclosing_mask_CPU(int(y1) + 1, int(x1) + 1, mask) - contour = measure.find_contours(mask, 0)[0] - # Adjust the contour coordinates to account for the padding - contour[:, 0] -= 1 # Adjust y-coordinates - contour[:, 1] -= 1 # Adjust x-coordinates - return contour - + if enclosed_mask is None: + return [0] -def classify_single(row): - """Classifiying the Rows based on orgin. + contours = measure.find_contours(enclosed_mask, 0) + if not contours: + return [0] - Args: - row: pd.series - The row that is being classified. + contour = contours[0] + contour[:, 0] -= 1 + contour[:, 1] -= 1 + return contour - Returns: - Class: int - the Class integer that indiceates the class the row belongs too. - """ - if row["new_row"] == 0: - if len(row["encloses"]) == 0: # no children - if np.isnan(row["parent_tag"]): # no parent - return 0 # no children, no parent. - else: - return 1 # no child has parent. - else: - if np.isnan(row["parent_tag"]): - return 2 # has children, no parent. - else: - return 3 # has children, has parent. - else: - return 4 # new row has children. +def get_mask_CPU(x1, y1, Birth, Death, img): + mask = (img <= Birth) & (img > Death) + return get_enclosing_mask_CPU(int(y1), int(x1), mask) -def correct_first_destruction_pl(df: pl.DataFrame) -> pl.DataFrame: - """ - Function for correcting for the First destruction of a parent Island, adapted for Polars. +def make_point_enclosure_assoc_CPU(x1, y1, Birth, Death, x1_arr, y1_arr, ids, img): + mask = get_mask_CPU(x1, y1, Birth, Death, img) + if mask is None: + return [] - This function identifies rows with "enclosed" islands, creates a new row for each, - and inherits properties from the first enclosed island. + # FIX: cripser returns x1 as the row (axis 0) and y1 as the col (axis 1). + # Therefore we index the numpy array using mask[x1, y1]. + valid_coords = mask[x1_arr.astype(int), y1_arr.astype(int)] + return ids[valid_coords].tolist() - Args: - df (pl.DataFrame): Input catalogue of sources to correct. - Returns: - pl.DataFrame: The new catalogue with added rows. - """ - # Ensure the 'new_row' column exists, initializing to 0 +def correct_first_destruction_pl(df: pl.DataFrame) -> pl.DataFrame: if "new_row" not in df.columns: df = df.with_columns(pl.lit(0, dtype=pl.Int8).alias("new_row")) - # 1. Filter the DataFrame to find all rows that have enclosed islands - # print(df) - islands_to_split = df.filter(pl.col("encloses").list.len() > 1) - - # If no such rows exist, return the original DataFrame. if islands_to_split.is_empty(): return df - # 2. Perform a self-join to fetch the 'Death' attribute from the parent island. - # The parent is identified by the first ID in the 'enclosed_i' list. new_rows_base = islands_to_split.join( - # Select only the necessary columns from the right side of the join df.select(["ID", "death"]), - # Join condition: first element of 'enclosed_i' matches 'ID' left_on=pl.col("encloses").list.get(0), right_on="ID", how="inner", - # Suffix prevents column name collisions ('Death' becomes 'Death_parent') suffix="_parent", ) - # If the join results in an empty DataFrame, return the original. if new_rows_base.is_empty(): return df - # 3. Generate a range of new, unique IDs for the rows to be added. - # This correctly assigns a different ID to each new row. max_id = df["ID"].max() num_new_rows = len(new_rows_base) - id_dtype = df.schema["ID"] # Match the original ID data type new_ids = pl.int_range( start=max_id + 1, end=max_id + num_new_rows + 1, - dtype=id_dtype, - eager=True, # Generate the series of new IDs immediately + dtype=df.schema["ID"], + eager=True, ) - # 4. Construct the new rows with updated and new values. new_rows = ( new_rows_base.with_columns( - # Overwrite the original ID with the new unique ID ID=new_ids, - # Update 'Death' with the value from the joined parent death=pl.col("death_parent"), - # Set 'parent_tag' to the ID of the parent island parent_tag=pl.col("ID_parent"), - # Mark this as a newly generated row new_row=pl.lit(1, dtype=pl.Int8), - # Set 'enclosed_i' to an empty list encloses=pl.lit(None, dtype=df.schema["encloses"]), ) - # Remove temporary columns created by the join .drop(["ID_parent", "death_parent"]) - # Ensure the column order matches the original DataFrame .select(df.columns) ) - # 5. Concatenate the original DataFrame with the newly created rows. return pl.concat([df, new_rows], how="vertical") def parent_tag_func_pl(df: pl.DataFrame) -> pl.DataFrame: - """ - Sets the 'parent_tag' for each row based on 'enclosed_i' lists. - - This function identifies parent-child relationships where a parent's - 'enclosed_i' list contains child IDs. It then creates a 'parent_tag' - column where each child's tag is set to its parent's ID. If an ID is - not a child, its 'parent_tag' is set to its own ID. - - Args: - df: The input Polars DataFrame. Must contain 'ID' and 'enclosed_i' - (list of IDs) columns. - - Returns: - The DataFrame with an added 'parent_tag' column. - """ - # 1. Filter to get only the rows that are parents (i.e., they enclose other islands). - # We also select only the necessary columns for creating the mapping. parents = df.filter(pl.col("encloses").list.len() > 1).select( pl.col("ID").alias("parent_id"), pl.col("encloses") ) - # 2. Create the parent-child mapping. - # We "explode" the 'enclosed_i' list so that each child ID gets its own row - # next to its parent's ID. This is the Polars way to create a lookup table. mapping = ( parents.explode("encloses") .rename({"encloses": "child_id"}) - .filter(pl.col("child_id") != pl.col("parent_id")) # Exclude self-references + .filter(pl.col("child_id") != pl.col("parent_id")) ) - # 3. Join the original DataFrame with the mapping. - # This will add a 'parent_id' column to our DataFrame, but it will only - # have values for rows that are children. Other rows will have null. df_with_parent_info = df.join( mapping, left_on="ID", right_on="child_id", how="left" ) - # 4. Create the final 'parent_tag' column. - # We use coalesce() to fill in the missing values. It takes the first - # non-null value it finds. So, if 'parent_id' exists, we use it; - # otherwise, we fall back to the row's own 'ID'. - df_final = df_with_parent_info.with_columns( + return df_with_parent_info.with_columns( parent_tag=pl.when(pl.col("parent_id").is_not_null()) .then(pl.col("parent_id")) .otherwise(pl.col("ID")) - ).drop( - "parent_id" - ) # Clean up the temporary column - return df_final - - -def make_point_enclosure_assoc_CPU(x1, y1, Birth, Death, pd, img): - """Returns a list of the ID of the points that are enclosed by the mask pd point. - Uses GPU for computation. - - Args: - Birth (float): _description_ - Death (float): _description_ - row (pd.Series): _description_ - pd (pl.DataFrame): polars DataFrame with point data. - img (np.ndarray): _description_ - img_gpu (cp.ndarray): _description_ - - Returns: - enclosed_list (list): _description_ - """ - mask = get_mask_CPU(x1, y1, Birth, Death, img) - mask_coords = np.column_stack((pd["x1"], pd["y1"])) - - points_inside_mask = mask[ - mask_coords[:, 0].astype(int), mask_coords[:, 1].astype(int) - ] - encloses_vectorized = pd.filter(points_inside_mask)["ID"].to_list() - return encloses_vectorized - - -def get_enclosing_mask_CPU(x, y, mask): - """ - Returns the connected components inside the mask starting from the point (x, y). - """ - from skimage.measure import label - - labeled_mask, num_features = scipy_label(mask) - - # check if the specified pixel is within the mask - if 0 <= x < mask.shape[1] and 0 <= y < mask.shape[0]: - label_at_pixel = labeled_mask[y, x] - - if label_at_pixel != 0: - # get the connected component containing the specified pixel - component_mask = labeled_mask == label_at_pixel - return component_mask - else: - # the specified pixel is not part of any connected component - return None - else: - # the specified pixel is outside the mask - return None - - -def bounding_box_cpu(mask): - rows, cols = np.where(mask) - min_y, max_y = np.min(rows), np.max(rows) - min_x, max_x = np.min(cols), np.max(cols) - return min_y, min_x, max_y, max_x - - -def get_mask_CPU(x1, y1, Birth, Death, img): - """Get mask for a single row uses the CPU - - Args: - row (pd.Series): row we want to get the mask for. - img (np.ndarray): image that the source is in. - - Returns: - mask_enclosed(np.ndarray): Array of the mask - - """ - - mask = np.zeros(img.shape) - mask = np.logical_or(mask, np.logical_and(img <= Birth, img > Death)) - mask_enclosed = get_enclosing_mask_CPU(int(y1), int(x1), mask) - - return mask_enclosed + ).drop("parent_id") def compute_homology( img: np.ndarray, analysis_threshold: float, - lifetime_limit: float = None, + lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0, area_size_threshold: int = 2, ) -> pl.DataFrame: - """ - Computed the persistent homology of the image using the cripser library. - This function then also calculates contours and some basic properties of the components - in the image, such as area, bounding box, and enclosure associations. - - Some basic cuts are made here e.g. lifetime limit fraction and area size threshold. - - We assume that the image is already an island from a thresholded image. - - Args: - img (np.ndarray): The input image, which is a 2D numpy array. - liftetime_limit_fraction (float): The lifetime limit fraction to filter components. - Defaults to 1.0, meaning all components with lifetime greater than 1.0 - will be included. - area_size_threshold (int): The minimum area size for components to be included. - Defaults to 2, meaning only components with area greater than 2 pixels will be included - in the final DataFrame. - Returns: - pl.DataFrame: A Polars DataFrame containing the computed persistent homology, - contours, and other properties of the components in the image. - This DataFrame includes columns for birth, death, lifetime, area, bounding box coordinates, - and enclosure associations. - It also includes a contour column with the computed contours of each component. - - - - """ - - pd = cripser.computePH(-img, maxdim=0) + pd_data = cripser.computePH(-img, maxdim=0) columns = ["dim", "birth", "death", "x1", "y1", "z1", "x2", "y2", "z2"] - polar_df = pl.DataFrame(pd, schema=columns) - # drop cols dim, z1, z2 - polar_df = polar_df.drop(["dim", "z1", "z2"]) - # create ne column lifetime death - birth - # make column birth and death - birth and death. + polar_df = pl.DataFrame(pd_data, schema=columns).drop(["dim", "z1", "z2"]) + polar_df = polar_df.with_columns( - [(-polar_df["birth"]).alias("birth"), (-polar_df["death"]).alias("death")] + [(-pl.col("birth")).alias("birth"), (-pl.col("death")).alias("death")] ) - # set the death column to atleast the analysis threshold value - # print(analysis_threshold) + polar_df = polar_df.with_columns( pl.when(pl.col("death") < analysis_threshold) .then(pl.lit(analysis_threshold)) @@ -321,219 +148,91 @@ def compute_homology( ) polar_df = polar_df.with_columns( - (abs(polar_df["death"] - polar_df["birth"])).alias("lifetime") + (abs(pl.col("death") - pl.col("birth"))).alias("lifetime"), + (pl.col("birth") / pl.col("death")).alias("lifetimeFrac"), ) - # lifetime_threshold. this is setby the user. - - polar_df = polar_df.with_columns( - (polar_df["birth"] / polar_df["death"]).alias("lifetimeFrac") + polar_df = polar_df.filter( + (pl.col("lifetimeFrac") > lifetime_limit_fraction) + & (pl.col("lifetime") > lifetime_limit) ) - # filter out components with lifetime less than 3 - - polar_df = polar_df.filter(polar_df["lifetimeFrac"] > lifetime_limit_fraction) - polar_df = polar_df.filter(polar_df["lifetime"] > lifetime_limit) + if polar_df.is_empty(): + return None - # set the longest lifetime rows death to 0. polar_df = polar_df.with_columns( pl.when(pl.col("lifetime") == pl.col("lifetime").max()) - .then(pl.lit(0)) # If lifetime is max, set death to 0 - .otherwise(pl.col("death")) # Otherwise, keep the original death value - .alias("death") # Assign this result to the 'death' column + .then(pl.lit(0)) + .otherwise(pl.col("death")) + .alias("death") ) - # compute the area of the component and bbox. - - areas = [] - bbox_min_y_list = [] - bbox_min_x_list = [] - bbox_max_y_list = [] - bbox_max_x_list = [] - - for row_tuple in polar_df.iter_rows( - named=True - ): # named=True gives you a dictionary per row - # get mask of the component using birth and death values. - birth = row_tuple["birth"] - death = row_tuple["death"] - x1 = row_tuple["x1"] - y1 = row_tuple["y1"] - - mask = get_mask_CPU( - x1, # Note: Your get_mask_CPU expects x1, y1, Birth, Death, img - y1, - birth, - death, - img, # Use the pre-selected component - ) + # Fast NumPy extraction to avoid iter_rows bottleneck + births = polar_df["birth"].to_numpy() + deaths = polar_df["death"].to_numpy() + x1s = polar_df["x1"].to_numpy() + y1s = polar_df["y1"].to_numpy() + + areas, min_ys, min_xs, max_ys, max_xs = [], [], [], [], [] + for b, d, x, y in zip(births, deaths, x1s, y1s): + mask = get_mask_CPU(x, y, b, d, img) if mask is not None: - bounding_box = bounding_box_cpu(mask) - area = np.sum(mask) - - areas.append(area) - bbox_min_y_list.append(bounding_box[0]) - bbox_min_x_list.append(bounding_box[1]) - bbox_max_y_list.append(bounding_box[2]) - bbox_max_x_list.append(bounding_box[3]) + rows, cols = np.where(mask) + areas.append(mask.sum()) + min_ys.append(rows.min()) + min_xs.append(cols.min()) + max_ys.append(rows.max()) + max_xs.append(cols.max()) else: - # Handle cases where mask is None (e.g., point outside, no component) - # Append NaN or a placeholder, or filter these rows out later areas.append(0) - bbox_min_y_list.append(np.nan) - bbox_min_x_list.append(np.nan) - bbox_max_y_list.append(np.nan) - bbox_max_x_list.append(np.nan) + min_ys.append(np.nan) + min_xs.append(np.nan) + max_ys.append(np.nan) + max_xs.append(np.nan) - # Add the new columns to the DataFrame polar_df = polar_df.with_columns( [ pl.Series("area", areas), - pl.Series("bbox_min_y", bbox_min_y_list), - pl.Series("bbox_min_x", bbox_min_x_list), - pl.Series("bbox_max_y", bbox_max_y_list), - pl.Series("bbox_max_x", bbox_max_x_list), + pl.Series("bbox_min_y", min_ys), + pl.Series("bbox_min_x", min_xs), + pl.Series("bbox_max_y", max_ys), + pl.Series("bbox_max_x", max_xs), ] ) - # area size filter. - area_size_threshold = 2 # replace with argument #### TODO #### - polar_df = polar_df.filter(polar_df["area"] > area_size_threshold) - init_df = polar_df.clone() - # if after the area size filter there are no components return empty df + + polar_df = polar_df.filter(pl.col("area") > area_size_threshold) if polar_df.is_empty(): return None - # assign an ID to each point in the polar_df polar_df = polar_df.with_columns(pl.Series("ID", range(len(polar_df)))) - polar_df = polar_df.with_columns( - pl.Series( - "encloses", - [ - make_point_enclosure_assoc_CPU( - row["x1"], - row["y1"], - row["birth"], - row["death"], - polar_df, - img, - ) - for row in polar_df.iter_rows(named=True) - ], + # ---> FIX: Re-extract arrays from the FILTERED DataFrame <--- + # This ensures ids, x1s, and y1s all have the exact same length + ids = polar_df["ID"].to_numpy() + filtered_x1s = polar_df["x1"].to_numpy() + filtered_y1s = polar_df["y1"].to_numpy() + + encloses = [ + make_point_enclosure_assoc_CPU(x, y, b, d, filtered_x1s, filtered_y1s, ids, img) + for b, d, x, y in zip( + polar_df["birth"], polar_df["death"], polar_df["x1"], polar_df["y1"] ) - ) + ] + polar_df = polar_df.with_columns(pl.Series("encloses", encloses)) - # correct first destruction polar_df = correct_first_destruction_pl(polar_df) - # assign parent tags polar_df = parent_tag_func_pl(polar_df) - contours = [] - - for row in polar_df.iter_rows(named=True): - try: - contour = _get_polygons_CPU( - row["x1"], row["y1"], row["birth"], row["death"], img - ) - contours.append(contour) - except Exception as e: - print(f"Error computing contour for row {row['ID']}: {e}") - contours.append([0]) - - # change countours from list of arrays of tuples to list of lists of tuples + contours = [ - list(map(tuple, contour)) if isinstance(contour, np.ndarray) else [0] - for contour in contours + ( + list(map(tuple, _get_polygons_CPU(x, y, b, d, img))) + if isinstance(_get_polygons_CPU(x, y, b, d, img), np.ndarray) + else [0] + ) + for b, d, x, y in zip( + polar_df["birth"], polar_df["death"], polar_df["x1"], polar_df["y1"] + ) ] - polar_df = polar_df.with_columns(pl.Series("contour", contours)) - - return polar_df - - -if __name__ == "__main__": - - ############################################################ - # This is a test script for the DRUID Homology module. - - print("DRUID - Homology.py test script") - - # get example tresholded image. - dummy_data_path = "DRUID/temp/dummy_image.fits" - background_map_path = "DRUID/temp/background_map.fits" - background_rms_map_path = "DRUID/temp/background_rms_map.fits" - - # load the images - dummy_data = fits.open(dummy_data_path)[0].data - background_map = fits.open(background_map_path)[0].data - background_rms_map = fits.open(background_rms_map_path)[0].data - - # set anything in the mask to 0 - thresholded_image = np.where( - dummy_data > background_map + 10 * background_rms_map, dummy_data, 0 - ) - # sort by area and remove components smaller than 5 pixels - - from skimage.measure import regionprops - from skimage.measure import label - - labeled_image = label(thresholded_image > 0, connectivity=2) - properties = regionprops(labeled_image, intensity_image=thresholded_image) - - # filter out components smaller than 5 pixels - min_area = 2 - filtered_labels = [prop.label for prop in properties if prop.area >= min_area] - - # create a new labeled image with only the filtered labels - - filtered_labeled_image = np.zeros_like(labeled_image) - for label_value in filtered_labels: - filtered_labeled_image[labeled_image == label_value] = label_value - - # use the filtered labeled image for further processing - labeled_image = filtered_labeled_image - - # for each label crop around it. - unique_labels = np.unique(labeled_image) - components = [] - for label_value in tqdm(unique_labels): - if label_value == 0: - continue # Skip the background label - component_mask = labeled_image == label_value - component = np.where(component_mask, thresholded_image, 0) - # crop around the component - y_indices, x_indices = np.where(component_mask) - - if len(x_indices) == 0 or len(y_indices) == 0: - continue - - x_min, x_max = np.min(x_indices), np.max(x_indices) - y_min, y_max = np.min(y_indices), np.max(y_indices) - component = component[y_min : y_max + 1, x_min : x_max + 1] - components.append(component) - - print(f"Found {len(components)} components.") - - ################################################################ - # Where the img cut out is used for the computation of the persistent homology. - # We will use the first component for now. - - img = components[2] - polar_df = compute_homology(img) - print("Contours computed.") - print(polar_df) - print("-------------------------") - contours = polar_df["contour"].to_list() - # plot the contours - plt.figure(figsize=(10, 10)) - plt.imshow(img, cmap="gray", origin="lower") - plt.title("Component Image with Contours") - for contour in contours: - if contour != [0]: # Check if contour is not empty - contour = np.array(contour) - plt.plot( - contour[:, 1], contour[:, 0], color="red", alpha=0.5, linewidth=5 - ) # Plot y, x for correct orientation - plt.colorbar() - plt.savefig("DRUID/temp/component_with_contours.png") - plt.show() + return polar_df.with_columns(pl.Series("contour", contours)) diff --git a/DRUID/src/properties.py b/DRUID/src/properties.py index 4e455fe..256522d 100644 --- a/DRUID/src/properties.py +++ b/DRUID/src/properties.py @@ -3,96 +3,36 @@ Date: 08-09-2025 """ -from skimage import measure import numpy as np -from skimage.draw import polygon - -import polars as pl -from . import homology - import polars as pl +from skimage import measure from scipy.ndimage import label as scipy_label def get_enclosing_mask_CPU(x, y, mask): - """ - Returns the connected components inside the mask starting from the point (x, y). - """ - from skimage.measure import label - - labeled_mask, num_features = scipy_label(mask) - - # check if the specified pixel is within the mask + labeled_mask, _ = scipy_label(mask) if 0 <= x < mask.shape[1] and 0 <= y < mask.shape[0]: label_at_pixel = labeled_mask[y, x] - if label_at_pixel != 0: - # get the connected component containing the specified pixel - component_mask = labeled_mask == label_at_pixel - return component_mask - else: - # the specified pixel is not part of any connected component - return None - else: - # the specified pixel is outside the mask - return None ->>>>>>> 1e15fb5 (update?) + return labeled_mask == label_at_pixel + return None def calculate_radio_flux_error(background_rms, area, BMAJ, BMIN): - # adapted from https://github.com/mhardcastle/radioflux/blob/master/radioflux/radioflux.py + if BMAJ is None or BMIN is None: + return np.nan + gfactor = 2 * np.sqrt(2 * np.log(2)) Beam_area = 2 * np.pi * (BMAJ * BMIN) / gfactor return np.mean(background_rms) * np.sqrt(area / Beam_area) -def get_region_properties(mask, image): - # labeled_mask = measure.label(mask) - properties = measure.regionprops(mask, intensity_image=image) - return properties - - -def get_row_mask(row, image): - mask = np.zeros_like(image, dtype=bool) - mask = np.logical_or( - mask, np.logical_and(image <= row["birth"], image > row["death"]) - ) - mask = get_enclosing_mask_CPU(int(row["y1"]), int(row["x1"]), mask) - mask = mask.astype(int) - return mask - - -def ABmag(flux): - return -2.5 * np.log10(flux) - - -def RONoise(EFFRON, EFFGAIN, EXPTIME, Area): - return np.sqrt(Area) * (EFFRON / EFFGAIN) * EXPTIME - - -def SkyNoise(sky): - return np.sqrt(sky) - - -def SourceNoise(Flux): - return np.sqrt(Flux) - - def optical_flux_err(EFFRON, EFFGAIN, EXPTIME, Area, sky, Flux): try: - RON_noise = RONoise(EFFRON, EFFGAIN, EXPTIME, Area) - except: - print( - "Error calculating RONoise (likely missing EFFORN, EFFGAIN or EXPTIME in header), setting to 0" - ) + RON_noise = np.sqrt(Area) * (EFFRON / EFFGAIN) * EXPTIME + except Exception: RON_noise = 0 - Sky_noise = SkyNoise(sky) - Source_noise = SourceNoise(Flux) - return np.sqrt(RON_noise**2 + Sky_noise + Source_noise) - - -def NOISE(row, local_ng): - return np.sum(np.random.normal(row["mean_bg"], local_ng, int(row["Area"]))) + return np.sqrt(RON_noise**2 + np.sqrt(sky) + np.sqrt(Flux)) def calculate_properties( @@ -109,117 +49,100 @@ def calculate_properties( EFFGAIN=None, EXPTIME=None, ): - from matplotlib import pyplot as plt - - # plt.imshow(image, cmap="gray", origin="lower") - # plt.show() - # print(cat) - maj = [] - min = [] - pa = [] - centroid = [] - flux = [] - flux_peak = [] - bg = [] - flux_err = [] - snr = [] - for row in cat.iter_rows(named=True): - # print(row) - # create a mask of source based on birth and death - mask = get_row_mask(row, image) - props = get_region_properties(mask, image) - - maj.append(props[0].major_axis_length) - min.append(props[0].minor_axis_length) - pa.append(props[0].orientation) - centroid.append(props[0].centroid) - - # calculate fluxes - flux_tot = np.nansum(mask * (image - background)) + # Vectorized extraction to avoid Polars iter_rows + births = cat["birth"].to_numpy() + deaths = ( + cat["deaths"].to_numpy() if "deaths" in cat.columns else cat["death"].to_numpy() + ) + x1s = cat["x1"].to_numpy() + y1s = cat["y1"].to_numpy() + areas = cat["area"].to_numpy() + + maj, min_ax, pa, centroid_lst, flux, flux_peak, bg, flux_err, snr = ( + [], + [], + [], + [], + [], + [], + [], + [], + [], + ) + + for b, d, x, y, area in zip(births, deaths, x1s, y1s, areas): + mask = (image <= b) & (image > d) + enclosed_mask = get_enclosing_mask_CPU(int(y), int(x), mask) + + if enclosed_mask is None: + # Fallback for empty/invalid properties + maj.append(np.nan) + min_ax.append(np.nan) + pa.append(np.nan) + centroid_lst.append((np.nan, np.nan)) + flux.append(np.nan) + flux_peak.append(np.nan) + bg.append(np.nan) + flux_err.append(np.nan) + snr.append(np.nan) + continue + + enclosed_mask_int = enclosed_mask.astype(int) + props = measure.regionprops(enclosed_mask_int, intensity_image=image) + + if props: + p = props[0] + maj.append(p.major_axis_length) + min_ax.append(p.minor_axis_length) + pa.append(p.orientation) + centroid_lst.append(p.centroid) + else: + maj.append(np.nan) + min_ax.append(np.nan) + pa.append(np.nan) + centroid_lst.append((np.nan, np.nan)) + + flux_tot = np.nansum(enclosed_mask_int * (image - background)) flux.append(flux_tot) - flux_peak.append(np.nanmax(mask * (image - background))) - bg.append(np.mean(background * mask)) + flux_peak.append(np.nanmax(enclosed_mask_int * (image - background))) - if mode == "radio": + bg_mean = np.mean(background * enclosed_mask_int) + bg.append(bg_mean) - Flux_total_err = calculate_radio_flux_error( - background_rms, row["area"], BMAJ, BMIN - ) - flux_err.append(Flux_total_err) - snr.append(flux_tot / Flux_total_err) + if mode == "radio": + f_err = calculate_radio_flux_error(background_rms, area, BMAJ, BMIN) + flux_err.append(f_err) + # Safely calculate SNR, handling NaN and zero division + if f_err and not np.isnan(f_err): + snr.append(flux_tot / f_err) + else: + snr.append(np.nan) elif mode == "optical": - Flux_total_err = optical_flux_err( - EFFRON=EFFRON, - EFFGAIN=EFFGAIN, - EXPTIME=EXPTIME, - Area=row["area"], - sky=np.nansum(background * mask), - Flux=np.nansum(mask * (image - background)), + f_err = optical_flux_err( + EFFRON, + EFFGAIN, + EXPTIME, + area, + np.nansum(background * enclosed_mask_int), + flux_tot, ) - flux_err.append(Flux_total_err) - snr.append(flux_tot / Flux_total_err) - + flux_err.append(f_err) + snr.append(flux_tot / f_err if f_err else 0) else: - Flux_total_err = 0 - flux_err.append(Flux_total_err) + flux_err.append(0) snr.append(0) - cat_with_props = cat.with_columns( - pl.Series("maj", maj), - pl.Series("min", min), - pl.Series("pa", pa), - pl.Series("centroid", centroid), - pl.Series("flux_peak", flux_peak), - pl.Series("bg", bg), - pl.Series("flux_err", flux_err), - pl.Series("flux", flux), - pl.Series("snr", snr), + return cat.with_columns( + [ + pl.Series("maj", maj), + pl.Series("min", min_ax), + pl.Series("pa", pa), + pl.Series("centroid", centroid_lst), + pl.Series("flux_peak", flux_peak), + pl.Series("bg", bg), + pl.Series("flux_err", flux_err), + pl.Series("flux", flux), + pl.Series("snr", snr), + ] ) - - return cat_with_props - - -if __name__ == "__main__": - - # open dummy image parquet - dummy_image = np.load("DRUID/temp/image_3C401.npy") - dummy_background = np.load("DRUID/temp/background_3C401.npy") - dummy_background_rms = np.load("DRUID/temp/background_rms_3C401.npy") - - import source - import homology - - source_islands = source.create_source_islands( - dummy_image, dummy_background, dummy_background_rms, 5, 3, 15, False - ) - - images_to_process = source_islands["island_image"] - iterable_images = zip( - images_to_process, - source_islands["positions"], - source_islands["background"], - source_islands["background_rms"], - ) - - for img, pos, back, back_rms in iterable_images: - cat = homology.compute_homology( - img, - analysis_threshold=3 * back_rms, - lifetime_limit=0.0, - lifetime_limit_fraction=1.4, - ) - BMAJ = 0.35 # arcsec - BMIN = 0.35 # arcsec - cat_with_props = calculate_properties( - cat, - img, - back, - back_rms, - pos, - analysis_threshold=3, - mode="radio", - BMAJ=BMAJ, - BMIN=BMIN, - ) - print(cat_with_props) diff --git a/DRUID/src/source.py b/DRUID/src/source.py index 70f6731..e571c6e 100644 --- a/DRUID/src/source.py +++ b/DRUID/src/source.py @@ -4,15 +4,9 @@ """ import numpy as np -from skimage.measure import regionprops -from skimage.measure import label +from skimage.measure import regionprops_table, label from tqdm import tqdm - -import numpy as np -from skimage.measure import regionprops, label, regionprops_table -from tqdm import tqdm -import pandas as pd # -import polars as pl +import pandas as pd def create_source_islands( @@ -25,226 +19,55 @@ def create_source_islands( verbose=True, ): """ - Create source islands from the background and RMS maps. And create cutouts of them. - - Parameters - ---------- - background_map : numpy.ndarray - The background map of the image. - background_rms_map : numpy.ndarray - The RMS map of the image. - detection_threshold : float, optional - Threshold for detecting sources, by default 5. - analysis_threshold : float, optional - Threshold for analyzing sources, by default 3. - - Returns - ------- - a dictionary of source islands, with keys, array (the cropped image), - poistion (the position of the source in the original image), - """ - - thresholded_image = np.where( - image > background_map + analysis_threshold * background_rms_map, image, 0 - ) - import time - - t0 = time.time() - labeled_image = label(thresholded_image > 0, connectivity=2) - t1 = time.time() - if verbose: - print( - f"Labeling connected components took {t1 - t0:.2f} seconds. Found {np.unique(labeled_image).size - 1} components." - ) - t0 = time.time() - properties = regionprops(labeled_image, intensity_image=thresholded_image) - t1 = time.time() - if verbose: - print( - f"Calculating region properties took {t1 - t0:.2f} seconds. Found {len(properties)} properties." - ) - components = [] - source_islands_positions = [] - source_island_bg_rms = [] - source_island_bg = [] - min_area = area_limit - area = [] - for prop in properties: - if prop.area < min_area: - continue - area.append(prop.area) - # prop.intensity_image is the cropped and masked component - components.append(np.array(prop.intensity_image)) - - # prop.bbox returns (min_row, min_col, max_row, max_col) - y_min, x_min, _, _ = prop.bbox - source_islands_positions.append((y_min, x_min)) - # Get the background and RMS values for the component - source_island_bg.append( - background_map[y_min : prop.bbox[2], x_min : prop.bbox[3]].mean() - ) - source_island_bg_rms.append( - background_rms_map[y_min : prop.bbox[2], x_min : prop.bbox[3]].mean() - ) - import matplotlib.pyplot as plt - - plt.hist(area) - plt.yscale("log") - plt.savefig("area_distribution.png") - - t1 = time.time() - if verbose: - print( - f"Cropping components took {t1 - t0:.2f} seconds. Found {len(components)} source islands." - ) - - source_islands = { - "island_image": components, - "positions": source_islands_positions, - "background": source_island_bg, - "background_rms": source_island_bg_rms, - } - - shuffled_islands = shuffle_in_unison( - [ - source_islands["island_image"], - source_islands["positions"], - source_islands["background"], - source_islands["background_rms"], - ] - ) - return shuffled_islands - - -def shuffle_in_unison(arrays): - """Shuffle multiple arrays in unison, preserving the correspondence between them.""" - assert all( - len(arr) == len(arrays[0]) for arr in arrays - ), "All arrays must have the same length." - p = np.random.permutation(len(arrays[0])) - return { - key: [array[i] for i in p] - for key, array in zip( - ["island_image", "positions", "background", "background_rms"], arrays - ) - } - - -def create_source_islands_optimized( - image, - background_map, - background_rms_map, - detection_threshold=5, # Not used in current logic, but kept for signature - analysis_threshold=3, - area_limit=2, - verbose=True, -): - """ - Create source islands from the background and RMS maps. And create cutouts of them. - - Parameters - ---------- - image : numpy.ndarray - The input image. - background_map : numpy.ndarray - The background map of the image. - background_rms_map : numpy.ndarray - The RMS map of the image. - detection_threshold : float, optional - Threshold for detecting sources (currently not used for analysis logic), by default 5. - analysis_threshold : float, optional - Threshold for analyzing sources, by default 3. - area_limit : int, optional - Minimum area (in pixels) for a detected region to be considered a source island, by default 2. - verbose : bool, optional - If True, display progress bars, by default True. - - Returns - ------- - a dictionary of source islands, with keys, array (the cropped image), - poistion (the position of the source in the original image), + Create source islands using optimized vectorization. + Returns bounding boxes instead of full arrays to save IPC overhead. """ - if verbose: print( "Step 1: Applying analysis threshold and labeling connected components..." ) - # Create a boolean mask directly. This avoids creating a full-size float array of zeros. + # Vectorized boolean mask creation analysis_mask = image > (background_map + analysis_threshold * background_rms_map) - - # Label connected components on the boolean mask - # connectivity=2 is 8-connectivity for 2D images labeled_image = label(analysis_mask, connectivity=2) - # Use regionprops_table for efficiency, requesting only necessary properties - # 'bbox' for cropping, 'label' for filtering, 'area' for filtering - # 'image' would give the cropped binary mask, 'intensity_image' would give cropped intensities. - # We will slice the original image/thresholded data later for actual intensities. if verbose: print("Step 2: Measuring region properties...") - # We only need 'bbox' and 'area' for filtering and cropping - # If you need other properties for analysis later, add them here. + # Use regionprops_table for C-level fast property extraction properties_table = regionprops_table( labeled_image, - intensity_image=image, # Pass the original image for intensity measurements properties=("label", "bbox", "area"), ) - # Convert to DataFrame for easier filtering props_df = pd.DataFrame(properties_table) if verbose: print(f"Initial regions found: {len(props_df)}") - print(f"Step 3: Filtering regions by area (>{area_limit} pixels)...") + print(f"Step 3: Filtering regions by area (>={area_limit} pixels)...") - # Filter out components smaller than area_limit pixels - # Filtering on the DataFrame is much faster than iterating a list of RegionProperties objects. + # Fast pandas filtering filtered_props_df = props_df[props_df["area"] >= area_limit] if verbose: print(f"Regions after area filtering: {len(filtered_props_df)}") - print("Step 4: Extracting source island cutouts...") - - components = [] - source_islands_positions = [] - # Iterate through the filtered DataFrame rows - # Using itertuples() is generally faster than iterrows() for DataFrames - for row in tqdm( - filtered_props_df.itertuples(), - total=len(filtered_props_df), - disable=not verbose, - ): - min_row, min_col, max_row, max_col = row.bbox - - # Slice the *original* image directly to get the intensities within the bounding box - # This is more efficient than recreating a masked array for each component. - # Ensure max_row and max_col are exclusive in python slicing, so bbox_coords[2] and bbox_coords[3] work directly - component_image_cutout = image[ - min_row:max_row, min_col:max_col - ].copy() # .copy() to ensure it's a separate array - - # To get the thresholded values only within the cutout (if needed): - # component_thresholded_cutout = thresholded_image[min_row:max_row, min_col:max_col] - # Or even better, apply the threshold condition directly to the cutout: - component_analysis_cutout = component_image_cutout * ( - component_image_cutout - > ( - background_map[min_row:max_row, min_col:max_col] - + analysis_threshold - * background_rms_map[min_row:max_row, min_col:max_col] - ) + # Extract metadata arrays + # bbox columns from regionprops_table are bbox-0, bbox-1, bbox-2, bbox-3 + bboxes = list( + zip( + filtered_props_df["bbox-0"], + filtered_props_df["bbox-1"], + filtered_props_df["bbox-2"], + filtered_props_df["bbox-3"], ) + ) - position = (min_row, min_col) - source_islands_positions.append(position) - components.append(component_analysis_cutout) # Store the thresholded cutout + positions = list(zip(filtered_props_df["bbox-0"], filtered_props_df["bbox-1"])) source_islands = { - "island_image": components, - "positions": source_islands_positions, + "bboxes": bboxes, + "positions": positions, } if verbose: diff --git a/DRUID/src/utils.py b/DRUID/src/utils.py index 68acf38..8129eee 100644 --- a/DRUID/src/utils.py +++ b/DRUID/src/utils.py @@ -1,29 +1,13 @@ import polars as pl import numpy as np +from astropy.io import fits def get_image_from_path(image_path): - """ - Load an image from a file path. - - Parameters - ---------- - image_path : str - Path to the image file. - - Returns - ------- - numpy.ndarray - The loaded image as a NumPy array. - """ - from astropy.io import fits - with fits.open(image_path) as hdul: image = hdul[0].data header = hdul[0].header - # warn if the image is not 2D - # reduce the image to 2D if it is not if image.ndim == 3: image = image[0, :, :] elif image.ndim == 4: @@ -32,16 +16,10 @@ def get_image_from_path(image_path): def combine_polars_catalogs(catalogs: list): - """ - Combine multiple polar catalogs into a single catalog. - - """ if not catalogs: raise ValueError("No catalogs provided to combine.") combined_catalog = pl.concat(catalogs) - - # Ensure the 'id' column is unique if "id" in combined_catalog.columns: combined_catalog = combined_catalog.with_columns( pl.col("id").cast(pl.Int64) @@ -51,45 +29,19 @@ def combine_polars_catalogs(catalogs: list): def generate_2d_gaussian(A, shape, center, sigma_x, sigma_y, angle_deg=0, norm=True): - """ - - Generate a 2D elliptical Gaussian distribution on a 2D array. - - Parameters: - - shape (tuple): Shape of the output array (height, width). - center (tuple): Center of the Gaussian distribution (x, y). - sigma_x (float): Standard deviation along the x-axis. - sigma_y (float): Standard deviation along the y-axis. - angle_deg (float): Rotation angle in degrees (default is 0). - - Returns: - - ndarray: 2D array containing the Gaussian distribution. - - """ x, y = np.meshgrid(np.arange(shape[1]), np.arange(shape[0])) x_c, y_c = center angle_rad = np.radians(angle_deg) - # Rotate coordinates - x_rot = (x - x_c) * np.cos(angle_rad) - (y - y_c) * np.sin(angle_rad) y_rot = (x - x_c) * np.sin(angle_rad) + (y - y_c) * np.cos(angle_rad) - # Calculate Gaussian values - gaussian = A * np.exp(-(x_rot**2 / (2 * sigma_x**2) + y_rot**2 / (2 * sigma_y**2))) if norm: return gaussian / (2 * np.pi * sigma_x * sigma_y) - else: - return gaussian + return gaussian def model_beam_func(peak_flux, shape, x, y, bmaj, bmin, bpa): - model_beam = np.zeros(shape) - model_beam = generate_2d_gaussian( - peak_flux, shape, (x, y), bmaj, bmin, bpa, norm=False - ) - return model_beam + return generate_2d_gaussian(peak_flux, shape, (x, y), bmaj, bmin, bpa, norm=False) From 4029661e7a6fdff433da5374fe2930f6312c2fb6 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Tue, 14 Jul 2026 11:00:29 +0100 Subject: [PATCH 41/69] update to script and implemented max_area limit --- DRUID/main.py | 40 +++++++++++++++++----- DRUID/src/homology.py | 1 + DRUID/src/properties.py | 3 +- DRUID/src/source.py | 76 +++++++++++++++++++++++++++-------------- 4 files changed, 85 insertions(+), 35 deletions(-) diff --git a/DRUID/main.py b/DRUID/main.py index 83ed490..a7cb74f 100644 --- a/DRUID/main.py +++ b/DRUID/main.py @@ -93,7 +93,7 @@ def _worker( bg_cutout = global_background_map[min_row:max_row, min_col:max_col] bg_rms_cutout = global_background_rms_map[min_row:max_row, min_col:max_col] - # ---> FIX: Re-mask the cutout to remove bounding box corners <--- + # Re-mask the cutout to remove bounding box corners # We must zero out pixels below the threshold so the homology algorithm # doesn't trace the artificial rectangular boundary of the cutout. local_threshold = bg_cutout + (analysis_threshold * bg_rms_cutout) @@ -143,8 +143,10 @@ def __init__( mode: str = None, verbose: bool = True, area_limit: int = 0, + max_area_limit: int = 10000, smooth_sigma: float = 0, num_threads: int = 1, + chunksize: int = 10, header: astropy.io.fits.header.Header = None, working_directory: str = "DRUID/temp", cashe: bool = False, @@ -205,8 +207,10 @@ def main(): self.mode = mode self.verbose = verbose self.area_limit = area_limit + self.max_area_limit = max_area_limit self.smooth_sigma = smooth_sigma self.num_threads = num_threads + self.chunksize = chunksize self.header = header self.cashe = cashe @@ -272,6 +276,7 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0 detection_threshold=self.detection_threshold, analysis_threshold=self.analysis_threshold, area_limit=self.area_limit, + max_area_limit=self.max_area_limit, verbose=self.verbose, ) t1 = time.time() @@ -319,25 +324,34 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0 if self.verbose: print(f"Processing in parallel with {self.num_threads} threads.") - optimal_chunksize = 1 - # Using initializer to set memory on workers safely + optimal_chunksize = self.chunksize + with get_context("spawn").Pool( self.num_threads, initializer=_worker_init, initargs=(self.image, self.background_map, self.background_rms_map), ) as p: - # imap_unordered will now instantly yield massive sources as they finish, - # while dynamically feeding tiny sources to whatever worker is free. + + # Wrap the imap_unordered generator with tqdm + # list() will pull from tqdm, which in turn pulls from imap_unordered results = list( - p.imap_unordered( - worker_func, iterable_islands, chunksize=optimal_chunksize + tqdm( + p.imap_unordered( + worker_func, + iterable_islands, + chunksize=optimal_chunksize + ), + total=len(iterable_islands), + disable=not self.verbose, + desc="Computing Homology", + dynamic_ncols=True ) ) else: if self.verbose: print("Processing sequentially.") _worker_init(self.image, self.background_map, self.background_rms_map) - for island in tqdm(iterable_islands, disable=not self.verbose): + for island in tqdm(iterable_islands, disable=not self.verbose, desc="Computing Homology", dynamic_ncols=True): results.append(worker_func(island)) results = [res for res in results if res is not None and not res.is_empty()] @@ -348,7 +362,15 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0 t1 = time.time() if self.verbose: - print(f"Homology computation took {t1 - t0:.2f} seconds.") + print(f"Homology computation took {t1 - t0:.2f} seconds.") + # print some basic stats about the catalog + print("---------------CATALOG SUMMARY---------------------") + print(f"Total sources detected: {self.catalog.height}") + print(f"Number of large sources (area > {self.max_area_limit}): {self.catalog.filter(pl.col('area') > self.max_area_limit).height}") + print("Average Background: ", self.background_map.mean()) + print("Average Background RMS: ", self.background_rms_map.mean()) + print("---------------------------------------------------") + def set_background( self, diff --git a/DRUID/src/homology.py b/DRUID/src/homology.py index dced62f..7a7a0e0 100644 --- a/DRUID/src/homology.py +++ b/DRUID/src/homology.py @@ -202,6 +202,7 @@ def compute_homology( ) polar_df = polar_df.filter(pl.col("area") > area_size_threshold) + if polar_df.is_empty(): return None diff --git a/DRUID/src/properties.py b/DRUID/src/properties.py index 256522d..a78045e 100644 --- a/DRUID/src/properties.py +++ b/DRUID/src/properties.py @@ -138,7 +138,8 @@ def calculate_properties( pl.Series("maj", maj), pl.Series("min", min_ax), pl.Series("pa", pa), - pl.Series("centroid", centroid_lst), + pl.Series("centroid_x", [c[1] for c in centroid_lst]), + pl.Series("centroid_y", [c[0] for c in centroid_lst]), pl.Series("flux_peak", flux_peak), pl.Series("bg", bg), pl.Series("flux_err", flux_err), diff --git a/DRUID/src/source.py b/DRUID/src/source.py index e571c6e..c799ef4 100644 --- a/DRUID/src/source.py +++ b/DRUID/src/source.py @@ -1,13 +1,6 @@ -""" -Author: Rhys Shaw -Date: 01-07-2025 -""" - import numpy as np from skimage.measure import regionprops_table, label -from tqdm import tqdm -import pandas as pd - +import polars as pl def create_source_islands( image, @@ -16,16 +9,16 @@ def create_source_islands( detection_threshold=5, analysis_threshold=3, area_limit=2, + max_area_limit=10000, verbose=True, ): """ Create source islands using optimized vectorization. Returns bounding boxes instead of full arrays to save IPC overhead. + Separates massive islands for cataloging without computing homology. """ if verbose: - print( - "Step 1: Applying analysis threshold and labeling connected components..." - ) + print("Step 1: Applying analysis threshold and labeling connected components...") # Vectorized boolean mask creation analysis_mask = image > (background_map + analysis_threshold * background_rms_map) @@ -40,37 +33,70 @@ def create_source_islands( properties=("label", "bbox", "area"), ) - props_df = pd.DataFrame(properties_table) + # Initialize Polars DataFrame directly from the dictionary of arrays + props_df = pl.DataFrame(properties_table) if verbose: - print(f"Initial regions found: {len(props_df)}") - print(f"Step 3: Filtering regions by area (>={area_limit} pixels)...") + print(f"Initial regions found: {props_df.height}") + print(f"Step 3: Filtering regions by area ({area_limit} <= area <= {max_area_limit} pixels)...") - # Fast pandas filtering - filtered_props_df = props_df[props_df["area"] >= area_limit] + # ---> FAST POLARS FILTERING <--- + # Standard processing queue + filtered_props_df = props_df.filter( + (pl.col("area") >= area_limit) & + (pl.col("area") <= max_area_limit) + ) + + # Flagged massive islands + massive_props_df = props_df.filter(pl.col("area") > max_area_limit) if verbose: - print(f"Regions after area filtering: {len(filtered_props_df)}") + if massive_props_df.height > 0: + print(f"Flagged {massive_props_df.height} massive region(s) to retain for the final catalog.") + print(f"Regions queued for homology processing: {filtered_props_df.height}") - # Extract metadata arrays - # bbox columns from regionprops_table are bbox-0, bbox-1, bbox-2, bbox-3 + # Extract standard metadata (for the multiprocessing pool) bboxes = list( zip( - filtered_props_df["bbox-0"], - filtered_props_df["bbox-1"], - filtered_props_df["bbox-2"], - filtered_props_df["bbox-3"], + filtered_props_df["bbox-0"].to_numpy(), + filtered_props_df["bbox-1"].to_numpy(), + filtered_props_df["bbox-2"].to_numpy(), + filtered_props_df["bbox-3"].to_numpy(), + ) + ) + + positions = list( + zip( + filtered_props_df["bbox-0"].to_numpy(), + filtered_props_df["bbox-1"].to_numpy(), ) ) - positions = list(zip(filtered_props_df["bbox-0"], filtered_props_df["bbox-1"])) + # Extract massive metadata (to bypass pool but append to catalog) + massive_bboxes = list( + zip( + massive_props_df["bbox-0"].to_numpy(), + massive_props_df["bbox-1"].to_numpy(), + massive_props_df["bbox-2"].to_numpy(), + massive_props_df["bbox-3"].to_numpy(), + ) + ) + + massive_positions = list( + zip( + massive_props_df["bbox-0"].to_numpy(), + massive_props_df["bbox-1"].to_numpy(), + ) + ) source_islands = { "bboxes": bboxes, "positions": positions, + "massive_bboxes": massive_bboxes, # <-- New: Saved massive bounding boxes + "massive_positions": massive_positions, # <-- New: Saved massive coordinates } if verbose: print("Source island creation complete.") - return source_islands + return source_islands \ No newline at end of file From 3fb00ff5fa588bce852602bc984374cd7153f6b1 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Tue, 14 Jul 2026 11:34:59 +0100 Subject: [PATCH 42/69] no message --- DRUID/main.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/DRUID/main.py b/DRUID/main.py index a7cb74f..b0a61c3 100644 --- a/DRUID/main.py +++ b/DRUID/main.py @@ -148,10 +148,10 @@ def __init__( num_threads: int = 1, chunksize: int = 10, header: astropy.io.fits.header.Header = None, - working_directory: str = "DRUID/temp", + working_directory: str = "./druid-working-dir", cashe: bool = False, + no_message: bool = False, ): - print(multiprocessing.current_process().name) error_msg = f""" {RED}===================================================================={RESET} {BOLD}DRUID MULTIPROCESSING ERROR{RESET} @@ -197,12 +197,13 @@ def main(): if 'if__name__=="__main__":' not in clean_script: raise RuntimeError(error_msg) except Exception as e: - # If we can't read the file (e.g. running in Jupyter), - # we silently pass and let Trap #1 catch it if a failure happens later. if isinstance(e, RuntimeError): raise e - print(DRUID_MESSAGE) + self.no_message = no_message + + if not self.no_message: + print(DRUID_MESSAGE) self.mode = mode self.verbose = verbose From 756b54c631685e5c02c72b234d915bbc75907548 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Wed, 15 Jul 2026 09:13:35 +0100 Subject: [PATCH 43/69] update --- DRUID/main.py | 78 ++++++++++++++++++++++++++++--------------- DRUID/src/homology.py | 25 ++++++++------ 2 files changed, 66 insertions(+), 37 deletions(-) diff --git a/DRUID/main.py b/DRUID/main.py index b0a61c3..112ccc3 100644 --- a/DRUID/main.py +++ b/DRUID/main.py @@ -12,6 +12,7 @@ import polars as pl from functools import partial from multiprocessing import get_context +from multiprocessing import shared_memory import multiprocessing from tqdm import tqdm @@ -56,18 +57,37 @@ global_background_map = None global_background_rms_map = None +# Keep shared memory objects alive in the worker +shm_img = None +shm_bg = None +shm_rms = None -def _worker_init(img, bg, bg_rms): +def _worker_init( + shm_img_name, img_shape, img_dtype, + shm_bg_name, bg_shape, bg_dtype, + shm_rms_name, rms_shape, rms_dtype +): """ Initializer for multiprocessing pool. - Loads the main arrays into the global namespace of each worker process, - preventing massive IPC data transfers. + Attaches to shared memory blocks created by the main process. """ global global_image, global_background_map, global_background_rms_map - global_image = img - global_background_map = bg - global_background_rms_map = bg_rms - + global shm_img, shm_bg, shm_rms + + from multiprocessing import shared_memory + import numpy as np + + # 1. Attach and map the main image + shm_img = shared_memory.SharedMemory(name=shm_img_name) + global_image = np.ndarray(shape=img_shape, dtype=img_dtype, buffer=shm_img.buf) + + # 2. Attach and map the background map + shm_bg = shared_memory.SharedMemory(name=shm_bg_name) + global_background_map = np.ndarray(shape=bg_shape, dtype=bg_dtype, buffer=shm_bg.buf) + + # 3. Attach and map the background RMS map + shm_rms = shared_memory.SharedMemory(name=shm_rms_name) + global_background_rms_map = np.ndarray(shape=rms_shape, dtype=rms_dtype, buffer=shm_rms.buf) def _worker( island_info, @@ -88,18 +108,13 @@ def _worker( bbox, position = island_info min_row, min_col, max_row, max_col = bbox - # Slice the global arrays natively in the worker raw_image_cutout = global_image[min_row:max_row, min_col:max_col] bg_cutout = global_background_map[min_row:max_row, min_col:max_col] bg_rms_cutout = global_background_rms_map[min_row:max_row, min_col:max_col] - # Re-mask the cutout to remove bounding box corners - # We must zero out pixels below the threshold so the homology algorithm - # doesn't trace the artificial rectangular boundary of the cutout. local_threshold = bg_cutout + (analysis_threshold * bg_rms_cutout) island_mask = raw_image_cutout > local_threshold - # Create a new array to avoid mutating the global shared memory image_cutout = np.where(island_mask, raw_image_cutout, 0) cat = homology.compute_homology( @@ -125,7 +140,6 @@ def _worker( EXPTIME, ) - # Append global offsets to the catalog for plotting cat = cat.with_columns( [ pl.lit(position[0]).alias("Island_Y"), @@ -176,24 +190,19 @@ def main(): {RED}===================================================================={RESET} """ - # CHILD PROCESS TRAP (Catches the fork bomb during spawn) if multiprocessing.current_process().name != "MainProcess": raise RuntimeError(error_msg) - # PRE-FLIGHT FAST FAIL (Saves time in the MainProcess) if num_threads > 1 and multiprocessing.current_process().name == "MainProcess": try: import __main__ - # Ensure we are running from a script file, not an interactive REPL/Jupyter if hasattr(__main__, "__file__") and os.path.exists(__main__.__file__): with open(__main__.__file__, "r") as f: script_content = f.read() - # Remove spaces and normalize quotes to catch all syntax variations clean_script = script_content.replace(" ", "").replace("'", '"') - # If the guard is missing, blow up immediately! if 'if__name__=="__main__":' not in clean_script: raise RuntimeError(error_msg) except Exception as e: @@ -286,14 +295,10 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0 print(f"Thresholding took {t1 - t0:.2f} seconds.") print(f"Found {len(source_islands['positions'])} source islands.") - # Zipping bounding boxes and positions (lightweight metadata) iterable_islands = list( zip(source_islands["bboxes"], source_islands["positions"]) ) - # ---> FIX: Strategy 1 - LPT Scheduling <--- - # Sort the islands by bounding box area (proxy for complexity) in DESCENDING order. - # bbox is (min_row, min_col, max_row, max_col) - # Area = (max_row - min_row) * (max_col - min_col) + iterable_islands.sort( key=lambda item: (item[0][2] - item[0][0]) * (item[0][3] - item[0][1]), reverse=True, @@ -327,14 +332,26 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0 optimal_chunksize = self.chunksize + # 1. Create shared memory blocks for all three arrays + shm_img = shared_memory.SharedMemory(create=True, size=self.image.nbytes) + shm_bg = shared_memory.SharedMemory(create=True, size=self.background_map.nbytes) + shm_rms = shared_memory.SharedMemory(create=True, size=self.background_rms_map.nbytes) + + # 2. Copy the data into the shared memory buffers + np.ndarray(self.image.shape, dtype=self.image.dtype, buffer=shm_img.buf)[:] = self.image[:] + np.ndarray(self.background_map.shape, dtype=self.background_map.dtype, buffer=shm_bg.buf)[:] = self.background_map[:] + np.ndarray(self.background_rms_map.shape, dtype=self.background_rms_map.dtype, buffer=shm_rms.buf)[:] = self.background_rms_map[:] + with get_context("spawn").Pool( self.num_threads, initializer=_worker_init, - initargs=(self.image, self.background_map, self.background_rms_map), + initargs=( + shm_img.name, self.image.shape, self.image.dtype, + shm_bg.name, self.background_map.shape, self.background_map.dtype, + shm_rms.name, self.background_rms_map.shape, self.background_rms_map.dtype + ) ) as p: - # Wrap the imap_unordered generator with tqdm - # list() will pull from tqdm, which in turn pulls from imap_unordered results = list( tqdm( p.imap_unordered( @@ -348,6 +365,14 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0 dynamic_ncols=True ) ) + + # 3. Clean up shared memory in the main process + shm_img.close() + shm_img.unlink() + shm_bg.close() + shm_bg.unlink() + shm_rms.close() + shm_rms.unlink() else: if self.verbose: print("Processing sequentially.") @@ -364,7 +389,6 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0 t1 = time.time() if self.verbose: print(f"Homology computation took {t1 - t0:.2f} seconds.") - # print some basic stats about the catalog print("---------------CATALOG SUMMARY---------------------") print(f"Total sources detected: {self.catalog.height}") print(f"Number of large sources (area > {self.max_area_limit}): {self.catalog.filter(pl.col('area') > self.max_area_limit).height}") diff --git a/DRUID/src/homology.py b/DRUID/src/homology.py index 7a7a0e0..a442165 100644 --- a/DRUID/src/homology.py +++ b/DRUID/src/homology.py @@ -30,17 +30,24 @@ def _get_polygons_CPU(x1, y1, birth, death, image: np.ndarray): mask = (image_padded <= birth) & (image_padded > death) enclosed_mask = get_enclosing_mask_CPU(int(y1) + 1, int(x1) + 1, mask) + # Return empty list instead of [0] to maintain consistent Polars schema if enclosed_mask is None: - return [0] + return [] - contours = measure.find_contours(enclosed_mask, 0) + # 0.5 is mathematically correct to find the boundary of a boolean (0/1) mask + contours = measure.find_contours(enclosed_mask, 0.5) if not contours: - return [0] + return [] contour = contours[0] + # Shift coordinates back due to padding contour[:, 0] -= 1 contour[:, 1] -= 1 - return contour + + # skimage returns (row, col). Convert to standard (x, y) for plotting + contour_xy = np.column_stack((contour[:, 1], contour[:, 0])) + + return contour_xy.tolist() def get_mask_CPU(x1, y1, Birth, Death, img): @@ -226,14 +233,12 @@ def compute_homology( polar_df = parent_tag_func_pl(polar_df) contours = [ - ( - list(map(tuple, _get_polygons_CPU(x, y, b, d, img))) - if isinstance(_get_polygons_CPU(x, y, b, d, img), np.ndarray) - else [0] - ) + _get_polygons_CPU(x, y, b, d, img) for b, d, x, y in zip( polar_df["birth"], polar_df["death"], polar_df["x1"], polar_df["y1"] ) ] - return polar_df.with_columns(pl.Series("contour", contours)) + return polar_df.with_columns( + pl.Series("contour", contours, dtype=pl.List(pl.List(pl.Float64))) + ) \ No newline at end of file From 7bda750536fa60b6ad2dbacb51115f86857f616e Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Wed, 15 Jul 2026 19:28:22 +0100 Subject: [PATCH 44/69] docs folder with execution time graph --- docs/assets/druid_performance_scaling.png | Bin 0 -> 115100 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 docs/assets/druid_performance_scaling.png diff --git a/docs/assets/druid_performance_scaling.png b/docs/assets/druid_performance_scaling.png new file mode 100644 index 0000000000000000000000000000000000000000..58affac1df00d03f48e2f1c259a03685dfd4654f GIT binary patch literal 115100 zcmeFZcR1JY-#@IaO;S-BHVs)RGiC2hGD1p(?2%DY$*PQ!LfIo^kBlNjLu3mf+1Y#a zdz_!^`hKtPbszU}|8@U!AIJSZj?e1D`~7~Muk(C9AM5#gprRzbeGB~-3JQwtvN95C z6cigPC@40lQf|bbw4A#yj(?o7m(;OWx4vTUWMpefp=e}(?18DyXpTI-l`=^?F!F;|Aghl3`3swRBq#_|ne`}iyOy=9%H zziGTd{urg>hZCH>^*8S?td83mUcI_EH$%R_vpMJK*VZ%rXFk2jDi^-__3>t}!?%3? zqO=4-P&T zus-FOCsWI*opwyHvi$x|0mr94KGWuZeiQ`N@-m83jgOW7HC~>dFl%{pq$Y%8-PWyJ ztH){r_6zuTXcmu8Oi&k}ICSWciPHmri3uk+{95IoUDR}so@EG~{r5JxXE*mpf7?Of zeFD~DgQ?E-(R^mqZ zlX-0~PyPO)J10AFF(COMD=X`p%uKJC7^WXTetgTfF_|7|rTHGOSpNIVbDO`b?i~h2 zksM3U28QcWw7E=5g&Aw^^iUo@FX|)1uQ{g3#>V!cvT~>)_t`T#X>%&N){eJMb+0e* z-eLaWKXb;gz_!oL(^5i05t$Tmfe!9%# zeDh|EVX4RbuV7`B5;vDSGsoH3Hf`R#Sw=?Yo!?%r8!L+r%M1NcVSLwi>E>AloU!}; z(y~2whAkHn_?V>o1_L;C-tv+vaPuzCsKS4;M@BNwu*yADRV&(dXPX8N~9MPuaz zbh55&Z_B=RXXAGIpSTg^u2WOf?mDWS_5qjU-?5#szP>t`jfLq$IKRcc;pTL*An)J5 z&$sElH=tVV>cBMf#K%YG?`buEpG2QG85!cCoSGq`Ps74|aG4*fs`|q()qMW^;mPrf zZ=A*qj_Ku71+yyF%XbNjh!|N|9ObC3t|r&nS>$rZ-+x>6l-J=)GeeCsii#ESiqB7X zgnHp~%YOXOQ;U53_;yQ+4&IW9pP#?^RGMLF;6V}BmlyM_TC%Uv$UQms%&$nSxl@;WLQ_MXgvR)R(>Km$ zTUuHMoWH;0l}3`BxA7*|8kDe4_}Zed$U9ldQu{~+(7*5M z602PyU;oxEiuLQ)|NK=QoNwFbdpWA1K|>)@m~PFQHS@#irI)V6w^vmx&J5e>b1{mD zFleV6kpIds{j}%TuU|fqf2V52>Ixk!{12Q_OYqpuUlqvk$u4aw&oH12qwGaV%v2aFxuj6>|C`p`){J&vD(@>78AB zxeUkwDLSKo9b8aTUHx95%gtoOit6#6-3c`UvwM3;nF)~PW{fsYGb+E=l4-hi2ZP}H zy?ggoPYF4WysM9s-`*58JKEm+P;B*4$o;!_-`CdGURq&K*3;7)|02-!6WiRZJva5& z!ua@&>EUMOLHj58EDd27XXkhA?H4s1M_RaFh}>+F5n*Jstf;E;%eB0I-M@i`k&&_Y z61KGu1w}`p!}&{>Ue;4fWL^1EmUXpuThr55=kIgs9~#A_ZX7I#>^d!@;ei5t=L6RX1qFpS zM&%pVZ`#!RH9{!5f6peMNA&E(za z4*u87^y<~# zwj2xR#c2*kCMN1bhjxPHYFss4P2;XP#i=;$`9 ztE-0`yRZv4ynNFkH;RiF+ui(DzM^H@O#N2$zI*ow2gl~$zkjo<$NP)N2Ya$HF_F_x zxpnJ#2?_7zqKk$0*Ba9Fg}T1ZF<<#?!7NVo%QFh!a|;It$D8bI36$S&=zuxxE>pj^ z<6PhS_)!MCklcqjxlk{hV;sbdxQWHHLRdM{+lp__QDF~wd#@*Zpt^c9i~JLbmBpDU zy`2~L;-V^Xq$ei(gjBz`|yAAU8YzgMQ+~1ugAv6m(|zrYHDi2 zx8Fu~n|Wh0-LC^u67*C)K8fF47N?Ua&Q6kE}F+p-@wxZ@p_ z(t9J`y*n%0PT_6nu^`Dc+AsAq>Ot2oWvin3_ zvu^+y3AZ;kb8EV#7Zen5bfDI7)vor>s5ItgPNfQtIhm#z6mxDna9ZVtt!+&X7j836 zr9T}LIld{Uefzgo)NT?J6MJP+73hUlUL)F&tSR48=#cU#I5QK#>FNzJ3 z>FL&y4wH)=J^=x=S3ZXw!J(7ut1WO_m@LaQ{p9`mvqGp1jvb>vJ-^(WoE**8z^6~2 z-Wd3HB*Jrt*I`O*=(0kGVQM0*;n8$AT99B4 z-JU(HE?m>2?E1na_Vo5T3}qc*FJVLT-XKop z{qp5uaiGu06s=U(T_0N+4nh?&s*9HNx_|$E&HO#m zAYVy8JXjDpyoU1N!GmQux($nO6{x1|0xp}?bdDvbedja%Y!O@Z)kIZQwalN6*PF+* zhCLwVYwclHR;e%{r?j7=qsPx^m;OD;%}tHZWqxf?to!lf$8Q4;MZv+rOyo<4ow5x7 z7As5h{`2Q!wOw&rXgMjd730Q7T6t5AIfu)!%6EaV68*24n{&F&ci#^xHL?}r`o(hU zl$Oq$OSe^0w2tR=o{x`@KZm32jpDa??+HEjN6pO=7263hyQ|{CR)M=6J-xHX&~t^q zws29-H&y?wR=pAqd_!7oY}l2yT+5@^y3ooDD-8w)~vHFKJ#isHCkeGw;xvt6#y-tZ^F<$WnJCyldjWd0gK;~)Y(wM zWMk$R6fcN5#Er$ap#0s%j!j;%v+5|!p`xK-dXV8bs{6jCMsCy*?I_`Uz?0Iys}~d1 z;(RN|4&A(Y(+H{COJB=zT8_nrvJc0s2OT@t%Ft6+wAQtE$>hC1AE z^}I^xZkqMt`}oWr{o+s6&5;|;_0v_m-c{mm?q2Y#gW&a`DOV+>z?0UUMS3)9V`7(i zp1ryBerW9fmZjbHW54;yi50v+inQ zk2KytmL`9~u;dsD0!MAw!r(-271^hJt7lqTS}3RbYxibmW!YIq$#Qd>7%=rY9u#t$ zfAe!>$tDYveFNWK ztpf&6)4cw3G+QY)l1(}CC3;Rvj>TSl&qS*$S6H=P>)ZxDWW+8{@Q96zdu7-ERj1UW zSZYS;#p&B8^b39B8{&RMoUz-;AZUNq**U+>0dVGiXYoxYUS4&B*_F92laZDTkO9rN z5sfL@y};2;iKP^vQolbxyMv#za4?Vs^YY$IGd`FQ^?P_s4#h$x&q`mlHP^CTLQFu> zva{$ywm$kXpq1=k$*Whd%vv*9@G%JvMbSU+AM&_`I-Yoab$LNXN$Hf@L;tJ8O{oLz zd0eP=jcI^-2mOJ_OO8{nMH#CL&LX5D%CaA%~LXPUZDA?21W z=fHMzbOp|wIYWn!t;Attw(x#@#os+3+oJgdp}XAN+`5g*3JMYr4~m>LsoWOldA`37 zXJDWuBSzP=yvVS>!f!8ql`GrPqg%z6$Je>ejy!n%`dGlQn9Jn))QfqQV6lA3bM4lp z-NDMDb@SoN5i`ThChu;o@$vU>NEW50J)ox!z#C%TRU)$E!0E%BZ-MRBi=X^x{5j96 zBPgV=UvOiEetWK^k+HGYix&q}7vkif`i=-5F{$)_-e&AYg`SU24T_g*L4>~Vd6vPO~@0d*kJ1m3O}KMMqS zP}unv-~gzY*^G3`k(RbL-zM_{m#O#x7xa46f7a$xxN`|}9+DCgF9Fl#f|+PJHO@wf zx~pVeQ9Laq#Kg$B781vYF3-Qm4e$Ssz^RS_ZX^Wk*s){TL!WTKy8HXty3pzc1)ct& z1Dn5VZKa!-n8^275d;^xk6-s0iWG9%@#)F&@`fb!9AEtx!Z&BPf%^t^z0A(dl|s?@ z2D(R}BwDre@;o>By`MiXsdnJ*8{>e@FAk@Z1^pZ;a&h$qAz+T9zUU$Z{L0A5l>vKY zU}IEIfOXLU$NF9Q5}wpCRx7%+&7$e`;}9Vsp&0BnqWH)?J+U2-_(Mg-k<+L5{asz+ z`qNd))#f?cmUF}2{zFfNUt?fu(qkrR+e<)=ah=m8pv&IJ2P z((AYU_3^Qs_1dO%!$YE?qKR6m^thv>AuzMD;-K%D`_oa5twyzPpU&l=PiVRO1gu^R zd}2vAEUiQtVq7dfKD2h-I&Qn)3JYH}Q3$~oHh^%P2kLtq9`5_-5!Fa%@qV1s=)(3q zYw2`@;&17OVnjIDf9m=pvHst`@1c*p|MY3=i4!M${QS_JgaenNw6y_)vAo(;rJyhJM@I46tm0 z50|z--UJjtNSu|G)tjuW^Ultv|8y4niQJqu`%xlz?%cVby}dEhpLDS0CC568RpaDn zTiV-uaS93h06Sq`8tZtw7xm&a`bPa}-)5Oxj?J=?&$!bc1A5*8JFEG-k(Zxe5*w?o z#Qmm5qRJXAEiLDTNfuI!r?*;y2SCqzpJUO?Tm&8&7iVb1R;Gj7GK@`3>T)eF;=syArl`ejS)3kZavjqva;6683iY(X{$s!fd{vn~ zvAVJpcGAd;`PCIu)4M>WuRvID*xB7lO+Aj*_UO^0Tj1w}(GE4H1l6u#bu3d}-TD)uLXZM)z-gJbYU-Oy}O3gW( z4CYbN)kd33Y|5eOo+ zxqdZR+|n*Kr5A~|y1fvi;=a>pr5orI&1>1s+S2m)@Dan-|R> z#~?4$qIXX0kd*(vSCjmguU_3M|KymeU`IDp$41|%h=}9pa!P8YfDmPUpPt&Kf@dF9 zc*Y~ivpoYq3H#&G)$0{uV}Ezx8)~B#N7ob z?Zq>R)vmd|4%=NuQ+J$KiZc6X!EBm7ywW9!o{TD}W9n{N>Q}RRB2_cx$qgV~y##e7 zSy^i6Qp^j&4z{+t42ql|_gl1PUkm=(-F;+vynH7qxl=z3sXCP=K{o?Q^3Jh zN!hNM-unK%c*`53J2irU%F#)lp^AzT%}bXW5>20cXHDt3P|Xm^pW{F6rofdU)fzy8H~`U&FA>p*rW$-3T^x3 z(q?c2nFYAkP-1H|I~Q|`JHI`sYc=hlu+>iOI(|6F^~sYbWl8D@4Zd%$e4&?hf2P2S zVga&I^L6jL_WkoN3_{}0OE=?>t$E%aAb4JFw#_~+J+|d-6=7ArP~&uDJDg-$@!8!! z-`{b`eHZ}!rvpvS z{hf#Qz_VbUsf`qQezgP8Nq)$Eb)la=;%ErV1cb>KS4A|_-{LErb8{)UjmwPKq~KuEj8d0+`|sK)^YpP;}nk4_IY z%F8G#YrhTXt$HZG`up!lR_)~Lr;i@JoatXiw`aCkfZ8OJT6ncKGPLN*#fulaQBceq z|MvKCX&k&$)Vaymu+6eLEkaA|4tRe+AT@R0e}R5EY>@Y@tryhNu7|I62QI~-dS7;T zD@$#_A+du1H(&Bv)`y(X*HwEGRdq2G7&-=QUQh(mTAX7Mnwxx6y^Os4c|d$g^%;tH z(A>xSqda|Upvg3Z-}#GLJdgjDF!kGWybSlW@lT2ZYdI7o0+m!3Ai0KF6ysxL4dfvL zyfLZTks99%P3L>Qk_{=&vaQNuE8gfIDVm-@@&pScDp0MD8%>Upc(4n2Fi*Eii8Yub zz;1coCe?Va0O(s;t=Qk!O(j6X!3vFYR_3ps@m`HaJ8{W1va&kyEb|0*bSl5O$Lhk> z*x1;Qc~EPKyA^%z?v^3@Lli0*#*!mH3e+|B960dWmXpif*;9n|=V%)PJ6R&qM>q>yuumY{tolQF{_Zqsr zYAOL7ERG%NSXyZ|^pu42G&Fhk@ zp>4jV8G^=eh6_!P7RE}0neHqO^QGy5x@hp?Cr6l=&X1nzw7GHP9d=7Yq?fKC8gppc zhCG+4E7jo-l8J?O$bI^8;xQ2s{ob7Jf#W8IOqywWbfj3ooOtEPAWn7CbJb1t0@Q;y zH|MOX!ygt5HYCviDKl}lqk>%yKQHO%=%~w;q>`k@@){gjBNx;rr|HTvROZkpNkXR6 zlYO7+id^hw1Q#Pee1H=w`Je{=MqFdWuncbDHcnIG?t&cx~S zwJQ2iV7CcE6xHaI6$AICP4|UO*wZr9fr?1wkMUS*{r5%`PFms!JTTRQkFIKJLBCiZ z6cn7vz23T(hQ?sa@B5#gaxt;9%V=mNC^5K^uhS_d(61ZQ_(C@1N~NJbWZCTk7-Ef3maS8GT^tyz}DiMPSxA@m=60Daq;l%g3`Dv z)O}^9`G%8|w$L9T{L&+4H{fUMfOgep=|k?!3(=o(b^PY%Cvw_bS}vi1bIxfm8{hWx z^OKtJWkeIJ{?dT;`y9^yNk)X{7;0lquq*{JFo~w91|@t(k0#SMBy}ddc}YaTg(Z8p zVw9+U&}lK}@t5^8yj_s#V+QVJ!UphI8Ws2=xcK)5uV+H=)J^3>^@0)yOH=d_iNl{CbOdHuUo|!D24yN}+PrgT6+o(lDQ@w%fn|>1 z>TJO|c7t^kMLQ02jySV<`}k<6713>_fUH2D_Z`Ua<7J-e@`i@pN=ix)>_u6XUa()7 zrlTOR8=}U$%1Wt}=CecJn zj#5vtW4&~QX6UcYf;vgTi~;PI&`n$Q`j zsm;%Wd<9C=^a{Md!ufCN*#{gHp@(ro6N(#>qQhJ#WIynduR8QO^g;O=f{g0S)+sA1 zR}aJ&L7L>VXxip5)KHfg7#A1!YvpNZ=w0;$rKHzcz>;;2ZyiUo<=(BcmQLWDz3_c1 zK!G*b|1|$@(`HIEndW>3ybn8oPMLUSJs~3xtTUCvCxR_KhNdp1(dZFe@vI`74nU z%Ja;m20hjb?>y}pg}1(J-}VohNf~*lTk-rttl{64xv1C?*n3UMsUQ(4 zjUHC7U%zG+ux@zeoW&?4v|mh8;JQYlk%O!Wtx5Lv0I ztzIwdWU9()!p58`9JAfi!6xKWTwyvq4oNC}6#;ldwGT3riAiL-H)F+cKkTY_va^|CxHnybBa>aMg3Pk zW5d%-u{SvdX3ZhR#UdMb9`wgD?M;dG&!8K2J{rz@b;s|o5f;n@E;{)Efh;P{Q3PmzR@1e`i6jq51XBRqk1Hs^T&5aWy(a2{>nLZzsH4x zlT&VxeUAgjTe^L;v~l+42JVTfF&jnaf3B-LV+DGs14Ap#f@O=>;S2lgzklbgU0;P$ zd=mOV-9RA#cW+CE$v#oH)1a1jz~c9Qx^_)$b>0MrL&3g$5>@-O$8!E|PU$uBKjQRN@1m2lwavUN z@i;FpuZedPpVbc?waUlF;oP(QL8nUnz_8AlnI$HUp!GKn@-AavMq4^`sSgeg&ZD=# zx%Qp)NAXSV;rX+j)$SE`3J`6&hlY|hk6C_oXOi-N7b_e5s~tsavR2H{(9nLk>5Eyl zBV+vmqUgt}UBCW1R^fR?qKJqHi|1iVuUlZ-MXob)Jnrgn5rM?+fyeZM9+{^$OB9k4 zIdX3soflDu;77|v<-K|H?oY>C{>4^8ziS{aWdDibsjI7#`$XFum8@5gS^wNN_K??M znLtUfi5#oirlvi#N3}lS9e$aHcP`sGIyQ0UWW7 zP;hXllF4g7RF71v`4o!}KGe~%I?&gUK}fWp{vZVXvOJhgS;7?GvUW=TG>}XMkSDjx z5>q_2V?ON(Q*B(UWKFSgoxBr7b7i$6M~S0=+X#p*s}+vDK@LiF4zFu>FDTm zOLwS6Ws1dSpXro*4-uYu*spl%6IZF(a;lP|b(7io#zkweada4~lE;g%F`3mL4+*MC zO@whA{9a@>wcV^fu*9Qj!TqzEy88CaE0;a0_MN;$@j5kC(iETmH7BrV3TdhFnHkNa zo!EloNS4T_X?H&G^DECVsXB>F25AiNC(jYgZrfMG=otfvqbbOt&+141Z3sqmNzx1H zZ;HC?Vm|Gy-94O4-)HpM;;^RwOdOY~=fy$WfJ?^4j4sFjW4Y_7#yBeKB7B#ZmvKYc zidjC~Dhl@X?ce7fS=wM^WOO@5`e8Kc4(FW9^Z@lK>rVg8G^{VtE7&h&GLxLx0U<|) zLA4DMmxUEbAVLj9P`mI}5o|nz-mviSXviiyIp(xVQKH8S&KBS3UQ57cK)a@6IcK`x zO$Uyp@{5hzGs`3xm}JX>E+>i9l+W8>&p z1)B&#MUE!zJ$GbiDuvkwgP|^8E-nEq`H8kKH>V7Z6^f9ADa?s)16vQH_f|v*Ii0-x zkxEBb_jjSSmKM@k`L`a>u$3WbKx{c2n#MrQmVeS7gG{AMe}p^=k|@6SP04oD#(O!J za5p&!rBlm1Ms2C|P_C$~GzMS{O8Nmok*wM=-E2xC&U)RsL%u$&Oz0#C^wgcVv9~eU z?U^^@vZ|(C9mHbO^I`KyTaJ7x$37Ed<2TSE?7=N^%p1!}+?TR_oo`K$5iW`92s-!N*u%>^1EwY`8LEVj7a8-Xh(FfLsq6N06;aYQZH1 zSXKK@-*{PXZM4Pf*6nqh%$JYrnObBaB_OkUx}ootxxIIffXd8_=;6K)Ma86H#Tyli zFII!><3?UI;HVcz2Co;t_3)u`N$C@i9iLou_6ZOtjt(c#?V!0}V|NFQbOQ#EvI?9X zqI3HL9Pyhse01=Q4GGyzY!jeND#iP{AwY#jkXE8I;aGyZ)X0cL7Y$4<#QwP&oH_O7 zkmvHNX1jo%MY#>MSlmM)#R>7sQE79vH#A9l4*3Vw3o%jwH^w?eAnP4~xV4=GYGAw* z1sRbw)xysBcqU=mf7Z9&TY;6(VQOS}-a3sLW3}O`2_Z~mo9cv|Azbpk$k1I#6EkGw z+4c|kK><^et%0=@Ic8_CpG6I{FKOtHVq>VqY|h6l;{Mgz+Z9R2z8 z76g0~Gz}7xvA5?7JaEPfwowc)n%uCk^Eie6TgT&>F$uqGqqykA=gN1A$w(dFuWqz_ zy_(AUMeML=56CD5i;Al(J$za5L-G#Vg-Vhk? zt|eSGFtr?)3Z0-QCB38!lz{|HHtjrksr=q{61xaF$VQ<9X$dzd@#(xotjIqL*#mY+ z>6sCYiw z)_IHXL-^)J*5GR9{yXp=QrqQ~lqw-CNt)hWaXVkh7F?|YyH~Xp%^_6jx!yMfy0#q> z)zbh`5vlf>gY)?lP%SzWf$h0k;e)>*tb+~WdO_4alQOEAJh5eG`{^42;YT97rcDi6a^c#nU1+6Cx@Tx z3XUl3?zXPiv$L~LLOOtXV(PBc5U?_tdJ2X;94W~J#{iDQhbfR-cogC}S6UPCyJn~cp-+uG$SQI5=7MD zzAqH)X&({qU}iFY!ey9dn=0VI?vX~(=AL``$`!d1 zGnE(DmLg%%K4Ioa8p=vijgbnPOL&!bdkZ~`GgK;l4aeTf?Wuo73tVTCWBr>cB@Lk) zF*B{@GAK%%>M9s=p6Iz7`x?BJKFNSAgSxmu21kNej6JHJj7ufq~8;fDO1l9U#GdnIWE0p<>@ZbS&0Po= zJRZvk4X~=eQu7t1?V6r+>zUsuCQbPa@7^>EA!9-O}nr`Hgnokio`b#L~c{jGQ9Aj>@dhDaK4J5HbQ!fYr?aVxkFP@25Z=(_bJMF|Up)5TL4U?J_Wj~U3iMSf%V7(VU@vo6 z9VBa#cpc|oSNQ@9id(8Kzv_jvXKz)G4otSH?ba`}-;6{heWR?h@>9K1P+=7C9h}36 zAgHU&f{2^%&P!Br40z)@V;;~>LVe87v@$QtR+~9+hNJt2uoz5#fwH3(Kgf3r6ubor zR%Hh40rrI;5;)`6?Wc1V668I3Ql)pkP^D-ih3~2Zg{t^Jm}Si^w-O7};bX@PiuS>l zKj|C=LVK*BU==a4IHuN+h2^PQ7E?IG&5MlAi1-9u+nAb~%5C1jp4!G6kDTT!+;KVS zbixla56Y;o6Hp)R6jG9I6!yu(yhx{ zp=fF*t(m3^^xCaUdf%zdp}l?^+Df>aBr+uurYNrTsX`|xy;N7fYW}YqqsmQE%@DiR zi?~j!2yTCop3cRp5O&EsvN&R`VyjsOV3=xZk~O&Zorj`}G8#`XLjl~_gEgd|VPR*d z#C{77dAN^zEl?MIj;R9N)p=Njs-+(8al>bbk$rP^gyWQ_3Ibl+P|TxwAeMrd>0j%4 zk5nnZSd+PX201qINBsfhc&M*#xUy~t*Cnxm7hdoH@?$?zD;3#ghZ-;ZF4NADu zRj>F?{f17~o5ZVs83YzliIneVp$LQeBr+Jeh6im(-I$bpE zy?e-m4aa#(ylYo|zh}>$Xi}Cg!OGB$*iUhok8d~V!UwQ|={m_AKxQKpkmyX*=bwnT z1YHX%bR12d_>!1jvM4t^*h%8JxC4GQuvMN~L;%eac{JuE-Kp#Qii(OPOi?brxyoHM zjXF-gNXj4la9ydV=N*NMdMz(R-E{}|I)xyr<~fdJlzdI;e2<^r@R`%6e*y;W3Ek{< z3nYm+Z$nO6vYv!`rVj5v|6mtONN30)%z?D^hoPAD4%Ed&^B@)Ie@M)L%RSbs7I0xu z;RGBF^`)(`HN?D!V6L@L1V#V4rKO~4>FQz%vuaumFb=`#sY?kXdQ%m94cYY#m7f9p za_Y+qal!uh03q2I)F5Ql{Be9Z zJDi~864UhdP4-haex4;^SdBdg4+cWvIiI?CRO|JlV_#R1V|N>Wzlr?Zul-Ma!Fne^ zm&w>bP|^|P%$d8qPM9HzNV2=jv+dU4Er$uVqT{`Cn1@F#6G1zn<&~8H9?oz3I#W0A zfw9aP#d|fd-XXuU&eS~vGC>UT+y8hHBzR7mgYI`TC^AqTj3<9plSBlhr>Ffg?C{vh zd{5B+WbDUEAoyFjQ+;#-NWLJRLIUI9*Dv(u5IZ|#-t?M-6EZD5q^scJ+K;uhI}f>y z-(KE%QC|L&@KPM->Jma`+?U=_qy|51Lwu2{(0-7SQ(jR~^63eE5OcZEm}y)WiFyIK zNb%@iMLZ|y@h*u%;zzhnU57kQmJ@hBx+yUOOHXtRe1~&@Cgk^#x$BdDY8Q9Wk#t_8+`Q|fG1EpOJKr%0iJXIOOzBIA*$Ifhizz>k8gUn156kIgH=@S~eKCP|=! zZnGOeG`_(`pcc;I(XP@%(365awak~$EsB>e5Q7S$x~dBU2=E;MyBNeugFf^?yTVAz zu&@clPeq;zWAtX%xjz9AA(BOsmjL#4@^2i$#{U*4zgM=b;3SFmGV|_)_4K}~imf^4 zNl#A?iIv7Zs3v+*8h4TzfW(_$c3cR!p`4T>g!6hfToWyI9yS&k=*l*$TZ`_iR$z^N zPlE8O7jWd;@~opQiqIU4#(%z#E+Kty13PRZrRAtBzVXe0m`ZM$jHCTKc}F``tDJ)ZRyZ^h)) zM$*N_(mG|8l%z1$L_&mx&RH25?~$i=4T!uQpj&4C@VR z1EC>_B!QkL*@vL&dz@2}l7ck92NIN*pWoJzkrApxhaQ5_l5qn>q~)Yv>*(lQBB<>6 zvDaK^+_#Ztj1|Vr08jw%g7!^&1^i#aII6#}nh==Kw6*jonGUMizy5oIaupIqP&kAf zKLC49UU&NW?lw%4y&Vc3c_l4ijhY_MOYvc+oyQNmxw%PHv0^%bpm(ngTfZSK%V*tr z0L|kr;2e%6#v|@yk-OpUAkU~K_~n{`>-4~F;5VdfJg7pEh-AP=NuadWLGB>@pUfm2 zrbLXZ95si`tr1JhV-*SHue_a*6YJhOS*x}pnf4PxHN=f*k{SZDg+hwqFK;N;y+9O` z?iX{fZx#>`NG(wmyxwxsZD}+wI(7~^oDB#enFMKU)Z`J2Cs#rSqcUu>(2g6@4GsBB zp#by1mLgMjbWrxiv9(DI5kV<3c;)$bC9gh*4$}yi9_275&QHwDNXp1i?%^^x##xH_ z88Q?F)&CVqS!t&qgiJupj`?}#g@q&Nc}NXQfyF&Gc>kC0TK#&Qxjm_mW5c#NIXGwv z_Q>dC=aIn0Nl1_w>e%1e*_o)BvIp<-6JG(tG9NyCILyqv9?(>y@SYarI_746NlgsY zkWsN5>FVE==!|y1Kfj00C(%blNL+JWp0y#a?GOxK3=dQQe~{$bk5W&=lCgKU*HNRs z0znM@h>E_1*ekN@^tbZFszAA8aP!Q5E;uTi-JqH$= z5dwZlYZeT+AE8_e>~SBxG8$|NiEMt7;2at8hZjVWhVV}znVwTop*nZ|{LExqK=qFw z2S}ukIB&y|SQlgLHj=Pkmfwi)>P!Y!;FSR$aWEv+W37z&XU?qDqFGK1)G^80IzyIe zfft8AxDOcc5}fvijm>R1qo@;9IK3DxJB&pqDG{@_Ut=XpJ5YUhb!a<}|6BuOZ=z26 z5J88DDzDZeaz=IV;0{kuPfTse@vU`7tdWSrTKzrP&ln+7wpa4Su0#pTKvEnw!zNI> zy9le1Q9r~{yHPHq=i0#G(r0CQ=UAATFJb2pB!W{46tE7HqUDgW0|NueRHUvheOXx< z+2GKi^ah`9e{$?Xw2g6~&$Pl}#^`hXs3~hHip4&E`EnO^aVtpWw?b>eXdPFUm+f7P zA-Z^Bz9W{zR8l%B5q;hO3%7XrMa`4rN6836Z*vB=wCqJ}2GtZR)V9gKFoEkWD*x>F zbVqjBzE^=L2P^xaxI*O*?i0V%;U~W7`6{+SKctOS6`*;FG)FjKFy)Xj?HT zv`-?fKwTIPMvig7KJIfVvxt{+V6sf^D;vYD8}U-R(6YHtq#OYTBA zH~M?+Tdd|e%>K4<)7I|A$d2mUKT9FSVw%MX+hkj7Wwh}I^nU|~7vD}m)BajmWVl|O z3e$%iVb7lrj~3a_3~2@!-VueQ!`$ql7$K;c;>d8|!1Scu#N^~IgA%uASA{j3UGGKD zM~+WVHx#A<-}f)_*8MsVN&;opT?Q#p?#OfBg`k-F^)beaf@ZS|N+Qtw4FZ!+rHfw} zAu&GBzpJ_{!bdd{50fF}-jXV0$gocu+Y92sW?d>ZUdQ~Iaq@Tz=C1;`D#E~%1bh%w z4(Qp_5LG2q^VKulBREj_;?HMEPe6LRb&c^SKM&WU(XIRM=zHT(mcQ{FYqTa7~SY|5@u%X*z_7d z<7k707TU~xph+fh zt_17Pd!>-C4N8*2zz&-6j<{i;r07qJ#s&rk`K~jZfb}n@_>^$4rBG=aJ3Rq`{xR7) zjw@m|FFB_RP5tDR&-((}{3&@e%y62dRV0ir_%m~>`^hi#Rbs6@~ z-0Dmfg1}L|mfJVjHx~h)1IQjqLu_&|ud1op0!%G|PB7iT`(+!0;O;9|lG!Bct_(^1 z9KD)w;xZ4No4dPBciCNIfr<-&sS{ug!*RzEraU0(rqg>Z0i&qC=mb;G7q;Mskz%|@ zICqRIuKyhnZ#NQ-HVPO4$B*yB4$)q6Jx7aXQO&P!RG+BIL|P6{YOLV(>(?1jyor)QPS@qijfx1`_TcyE|0;@l zEIVKCDAeq=An+if>ng6R49VI_y{+Ls+em^Yn(^z059^T{C4CG)?OU|u)@^h=>v4NX zxPmC9$bcFHd9h#2eVy?(MGdWwr2_(5yNU}ZGDAUl-akJYe=I44cP*S@c|v-mqq5FYj4)cnvWmPqO7tE`+SKI zA_ufCUqruYZ&qGjm38ULJx~w*_z5TwbgfB)o1eb||7rt`3y?v$S?ugEia00$)Cd~aXHOci zA_P|8?qmSBBIZbe%?NKO_PzB%tkD(}@^c7!12#Rn>XmZ6HFF>Dm2E%)Z?GtxBaC;h z)yHoB8YT7+i)ixk(GhbtYorS96S|6Fv%O{I<@~Ebw?zvI3+0-oZYB`bab`!O1N7VV z8#Z(=jg?9R3O5>T?d#ZH37O0YhUJYLH@TWZQ+PZTwhsVdq-+_7(r^kM*lL2u8mp>TexQszU zN_20n;kL)V_m7;v2fDYZCD(F4<`9l8^h9zJ69{>=+elt_1BuTg1k;c_*LpRIpfchU zlE9d)7&ZraTVmp3bJ{EU6!&&_-zKXEnX*7>)CyGuT2n8)iFuGs%+w~^yvE5Q)5Pc> z?=aFqh#C**?r14)w zM5sb6;=A5Lg`>gaD3BBelp+bU6!BEj#W6^#bMYbuGItPJ0$ZD$-^N3B1(2hv;LK6u zT>*jdAb@g|4h(2wsm>xdXn&ApO_Bc5UZ6(Gojaug`|%=aK!Uu;(*T@V?w)saJca24 zYI=HKyagF(t*fKP$(F*h*%nvJUp9mCeii@^k~{{113-;&hR9qr-hi0Qb-ONAhJfVY zE}~cLgMM%xP!z+ZmyC?oKnV{yesK@1RL!NWr(=u3iGRYW#svFWZ~+*!-GBZBB4JNE zz{NwW0sVuFP9GHW@Q2g4ForWHfTB5d+{)Ee5MPpv+)7AGGl5z{u_V(3@S;?q)d4pb ze`FKj<>4WWmuT!r9Fx@rMF1Av0M%6^Me9q8e(X6$5G6RNRmAf591af)L%?eX$zti| zaGEzHt^tOKFkxY1>jgc*)Y4{SW8=fY)tFcFhnGXJw(En(PocthqklNh4AGMD>Z?~b z?cBMO=xhCS>>(gJl}G?g{HmrQ#sUH(NSb*ACy_B|IOsSmc`Y7v_Yz;4kXicwdHD(l zX0hUy|LYaa_>ja4|L4Uy8MYhw|9`*!-}r&r&n*n>lo%93H$@;Z;LV$pfU57y%GUn( zqClD025qFc(hU`v?gI*Eb`~RNGmXg{CzlTI?P}%g- zZHN$ozDH_tMn=X-0Rd@@iGPD7>g?=1aC}GK2}~EEfdpA&vicyl-YMgUK8Yv0&v^UT zS4>e-4|?@eG1hOWJt7f%G*a4E^cj*N8A3+e@Ls2H{G?c2nKOi>zY&{Z7Fi$CHap(-`^OWcbn^Q8feW*XtI$Jfhs(XK@00H#5RG?|DV5liSZft z9zc{p_-&WaE0OkfO54m$f&K9Tzz35s>wqb72F(sk0beKTzdeOyV}!-IZ4^jWkm()5 zE)g-2a&*j<%0M=V+kQZ08ZxITegqSXMByMVE{fQ9_DUh}VM3lrj4%`*Lni8QP<{>% z`yq;*YK-ZXf4{D!=&)9RYee&Ln;^o6dEJ+g&d8Vyb}jP~$;4u&j)_b~20Fp7CGI-j zYgZ_q00XaUgJL+Jc!Z$CB~w!tS5ZvXqit1@IflScl!i6yHeMoXbMtSUc7m-9v@c&~ zCUat-4M_}}CT*wq*%( zR@?v=2yO1hW`Nza0Y!p5Gyp}S9NH2};I78am_`s%4aMVENbf^XX_82NWO@oU!g+rD z2w_+--7!1527asr>Yjup!7D+()cSCeNS z=k_7CLyHD7;ckG2M=eI#e0RaMTrN)g7_QPrP68-XO3i zx=slQRKYwY+87=P;WNw6%X=Qc8gpkpSaZL~=VG27kr~8U1$|R(#nkOMQ1nYE0FyjG zj@U-~QJy96`F(xRk4OqF(&koPb>5tGNx|EC?~OoSQGS>*SZ=1 zQ^tIJ>@|-C8v`(uV_Ap~Q5|x(28k6SAmbqyj}zUX(z!Hg9?0Q`lvVwamjtol42wya zqSw5G>|vWa_gz`+2sgKC37W5k0^+}YtLWWNi|(NV9uX82Y(9;pG>wm5YK~}z(Zn@SBqiT`Q=%z&j4G9U+8^)gS$%UAk=vXw@yu9`nw1v0YMsl$q7SLU?CR` z@3LanTt~jm`RoqF=W}`}&f=j7Dh0Nt3y$aw#I3aN!dxe8=9;{Yjt*5M6LL&)%<9f% zr?y!C$EQUvzl#mq2oTHF=4fa4$<8tpmBc5PJS;&~O*WKl1xyN*bU@TRjTw|5e^=Ji zQ=tAGt=(5Jn%)xmsu#`b->*R$tPXMB%-*8xxHLhyx(flpZ3)H%5y)|4HJdIV_d)Fa zu3X@Ka5#-WQLvYRSN&@EF1%G}U7Z{2(k_fLE=aQ>kNCi}S3nzqY5#iFhaiBeqC>J< znNONf0jFMHwXzDtJ$#2e3XT=UEgXto7`*#n0EhEkTZ`xRpjt3O)55N*K%la32KVX# zPC;2mhrX#lWH(B$!(AuP*`PwD)(s!U!&}UR7#YtEdieH1i~YtPh*4O~?K}z%-9YgX zPmi*<&t@)($5*1Fqf1ha0DMFey9|&G)fG-Bnc1};k`^C_o~oDkz{j@S4qeu_pZdSM z?AFNhJ~G!i5p%o?mfzPp3l&Y$0TKnDM?mTFq+cSjz&Jb7p)f?Rz+#WLA(M2f`yrxz z&)+Ud#G2v$8rTT%H~#PcW;Tnqf9vbq#MZS+QDOHXSAZdFh}LYSK&v9`C!mc3v(I3u zzU1hfDa6yZwju8H4+((r`McE-Llb80_4R5oC2MYlhH9o+(lhP(_dY=(E;-5Z_F#6k!@tI#d0s_s6))dU& zH=7kMNCq+!w>(16!0bGhI)(&0*Ew*gMq<@T7J za9|H?QoH_rUk0Bl`U;u9N;pM^)zQ3$f9(C&>0xAyj+U_5m|y1mosH85bG7q}i|44; zyIcU@e}kqNC(EIUr%8|zT^Asp)%o6?vgZ_3QC)7}he<-r;5WW2lJ)EzhTKVH$Y|)T zAS#@lrMhN;ad8o)*CVd~g{h-|prE}2RzOQSkLO+#y#`H0zIqdcAb#^?Tc>!7c%&%)FQnK!|{3-!S_$N>^CR`gq9I+tO|Zp##Uhd7D#gu!mUhb^l)%N!`Nc3G{Bl9JFbA#$>){gQLv*^2&fa-(iQ6M5G`x+94`R=1Ypv|+}81Dbi+xf^uh3#HP zGFJ9T^M%|JwL)vCev&I&wr`gtPq)ZeC6||LT#5&){rfd}K110T_YlY0o+|ZyA3!58Z!}=0 zV9sby4`Z9h%uuM*3~lp5Q1#`YQ(c2isn5-p@Js=!_2Q|gF^5ZKnTI}^o0}^)mjjZB z4lQX)41=SAhbhUkt`Ze&d}aP;&oM?XtGo_22B{`W4n*l+rob9i5TI2tlp4{Vj^Mc2 z5_&s*qqbf!0EA+nSr$fyVh}!?9K{>I!_!JqPs8n`vc%L4Z}ZAc%u!poYa;q9p)Oa5 zhj#&N{9jtNWQL+ICUak6yOAAM^ct6U+GR4f(!w*T9; z1-Wygz-j;F>=5WG5|;Lh5Ot?VZ+dFo4p|EowV;U(301R*9Q5+OXX7;M*8qjlghHcU z%gS0YtNrNTYm7(8CF)#ZM$aQ202#tH_zi;v-u+8JAXmTd=%QpCOXCz$19�aI-7+ z`91{Z$a4}nVV>)x%^*~IB77Fl^vTjGa?Yq(F)VR=0Jp`jret|izRri&l-MRnHT%>^ zJv8Cf_xaY4DK2!s>TqorqN#vW zhKQ18N-;*F{H0< za0>7k*K?rDW9c}`=_b-J-$P^^$MX?C?po?;w26yaWg zY-iD_js#azP!Nw5F5Vjq2CJ^X(IZo*sXG6@8vILP&}Nq}>DudyTl`>P2DDzatkW3+&GK4hRrICt4WF8~4XfRY1nMG10!%iZz zaMn|8_5S>>>s;r*bB^o!e&2n{-uty*ujjd+`(A6^>jpN{hfwU}R2Pbn5eaO33 zKzv{TYsYHFT-nvC9j8&urN~eE(%@K%LZo7NdZk4Mb1`8NGdx+ODyQXyQx_2GaX0SO zq>=v5kJd{_VZitxuycW~M(Xm**0g;l5`A=$KgE*5j^u-RSZbbD?7uJEhZlnOCmy2` z5={-wYq=vld6{MajzDhd2gDFg3{PPrf?HVfO)vbg>DPtRIrsH1uwBGXhQU?gS)zFf zym#+1S&ZAZRW}&&v;G5S2^{&bz6gySmz&wRxwodFIS?S8ukkgtnK7z7^d}>51b@P+ zL)RC9+;EaXhzGm1mEo>(asixpTGxVt{E+REuZfoENdx4s0J_!Dr?qbA;m3dAwpbQ) zhC>kWOV0$b<0%A=`||PjdvFY2@b_N;)bJd7&q;kTX@I;ISqA_K>!LNwj@^^KPJdvr zED)JqKXAR6e7!!RjYRi~(%xBMRyg7m&dvPCLHgXq)^tA*C z`jv4z|MwRcs3Ij^H;CTR%gTNG7ayRuU-_G~MC?XJbi~};uN3Uey9QAma%irS>{HSFvHT?qwd9OEO`CWBVy&HQ`cdf$75`0Uwf^y%oqRo@;T zkoD}IvJ;Rv80e$x=24UI@nYy;*#q~L66=Q2zqp(RiU_Uj6x-{%+C(4=vqc*h3O}I0VDnIeeJ3 zyMXFg8i?gfY?L6IJvsDsCGNvnm|3A@rVcg1t`sCuD(VpwW|nMIQ2h}bDLHdIA!vOR zKwA)lU_iTmp+s7Qz%2}xHpz4uZ;7;Yqy*`)`1A9UZ};=1TRb%>A9x~ZHRaDZQjU)* zZl7HBZk9-C11-)~z6F38p?DGX+4D9IOtJ4c7OV4|2acgo+B^dmvlzZKLRYEO-H@t% z2cb)R&)8q20Du)oQ*-|)cvmx^pesbIS*xn2h6_SOY?i^iO6#-t?aQY;_qqNPS1vLO z@<=kwtT(C?TRT*K_pB51LC2NDMjJY#@+0Em2h%^L09+c1s}dTs)f;d=$RdjYe(^k* zfar)cG@re2p=1A7uwf)~j7USEhNCl=aVy*25=rTR|9tWLrZ>0Yi~AgTv&FDD=IC=* zGYbA@pwUjfAAK*fXadQ)4LS=>)9Z#VIuXkxVDtjY8pH))bQ!vPQu%a~@5O#a**dx` z<>A)4zO}Xg#31f7!H#?f?bj8t<}2=d@xJicXqAgP93=f=m2t3OHPl zB4m#V@3WxNi0A=v{Q2D?^Xl<-ZDQI0&DAY?lten)CywlD1*=Ia%hG;F_`-ZXylWzd zmGwM!!>=aXKauMQ3sxH!*Y*;w)ES{AA17&YXNm+a6N^El3oohE`(B)&MzR; z7+6NKUqCOi;BX(429-O~r^KO%BnN_<8%}%laQQ*Gp&}h<28x|DqZ9e9)i@Hb+bL#) zkiUa;-WR~<5f&eL$mrGf;>bggg&{L-2;N~Uz@G=tJAnTK?8J5uBuqpa80Bt}z3N9| z+xEo&GiO5Uz}Xiu;kBw>CEw0JJC^g4S)cjd#hE@osv9FB9%Vdhv+jm7A+de`teAi! z$_)N%;3?zFZQ(y*0;yQUS)d0$zBjwJ@P{0&0FepJ=`MsH(%&NKIl@9A4GEW5ODy~F z;lmAaqPIzU<%9|;uOtrNg{;zS0<=d-zL zXs5S8CAdDK(ATPi06YH$zcsV7=e`Wz31>!ng2v5Vx*VWvF?NVxOL=MObI@o5RYQU7Djse}&NnT@E|PuUx%0EPZV)bb5l+U%gZU@!&g+}_Tu0mlzkS2y;81(1 z5W&FcO@uGGMkpm#6Y`XYSIHeK1@*U$-BGf zK32hM@`_Ox&2;D7{WiS326XZe~{9R_ZtsT=ohDE`nh<&_WPbYOvdG&P|h|9&vq>aZzuV~^D( z9@5K-x$_hoiyhR{-m7?&)W(Z`e(LNAIAFJkhy>bi)aQH+ed6Sl>w|ni4x5AB)l)wJ zW+!@qQ5TaS@x>I#j^&%E>uwGZ4;36M#X#@|Ke(t{T*Qw2Ps(;=1>%Q;m!AHE;;?ku zY~Rpg_Dx3Xc5c|jq1JM^?*l*Wu+{fFxxrgZW&piDkA`Z>Q|MJ;Te}E0YW_`NFzXz9 zBmK_05^E8##iK7XfyS&wBshB1-u@JB*-PW4T%j{k5gW4|I~Mw6DW>8LO2af)Ip`cD z4qjX^h|r3D#gv8bXY$J$2&EgrED#)tW27(sJA(rD_Y;ja9|{ZDChh4cSUx9r4QJKA z8=&}XP~2Praea=>UYf2JXLyT)-9m zER+md()EC#+%z`9;2C*cF_3tQtmKSDgog;Ll~Al+g=+$ZLgYmV^TZfiLzVYtxUGKM z`kNvS&=BD)x>Sm6%h|db=@NRaWFgqz0zetYU-OZn&GBu18DgE8kiM0Igt7GjCOQ%vlwbpL5e|S)yh{5~N$B#J4rT(WW7*U5AUMet(GaSnnFh{EY@K) zz)OD@xLAmHkxRCZlOsI+i{7s2W1B@GSIW<@H#0kf6%!7W|0Fu~@x0#RAuNHc?A4Hy6=>zA$jp8=oy*r{J z1)3}t<$M$fR>YWLr`>xNWrjlsqg7`4K+;CdOaR1=^0rgtJbX~FmDY{9;p|^dcwQib zj!6l3vX!9Ahu|B<&UEk-tqOy{P(F?!};dPEKxQ?nM8`Z()(MZ-%adiMsh z)eROAdf%@L z5&rzY38s~9A)8cpd;ZWhLH9bL3z;HH9X6LEN&yq$#9toxCRB<8<*{{E_#ID;16$iK z4BynpavMR0Ddzxi_!4Yt6ua}ozUM-G1Sn2!5M&yxXlb6^8NVd9=|jPgRD*WEYu`tr zd*u)e*s$h`4QQkl5DOz>t8HD9rcQ5jKoDbK_|--HzdHBm^;XT}H5s3*?XL^j<~8ac zwi|9l3hY2M>6PLN#2praq$U~QsTc3%e~P8eU6rWRnPE65VabT&k)@*V z?$9!tTlezb{N1^LbYu|{aOpy4yk>jY=%<5y_4D2v{UHf51p0w?G zcJr@;RYnf=XsJOj%ft!9^C}=&A2g#w3K2K zwl{%lSHK4Gc^o6_DHGK1#jV_z_$<$@f&{BjoO;vIVs5(KuT2jY6*k7PG11 zi?p=}>6?A~ba>U{aIZ8u^@_3fEm^avOjQXMjRbU^xx1}_wnMTpKqg; zIlxRU1{qTm4|=GU;M{GHl{Be7%hn5_qAcKX z8_^aT+`YA+yM%07&&vS^&2#SHW_$nSvWaFu@DLLc6WaH%*7mEH( z3Ln@Yp!Ha1x1#P^l{>m*r*To+P}#|%m`VH98?hf_H9s&63W(oV$nTFe+B?|ES#=c) z!ZL()rVldXR*(y%b&ibPa2FI1n1^qfQ+NB-5z^$Wmx`uyGO-2%l%)x7>x(}qr{Gjm zS9gPatM3%t@kc=U_Cct;Z?EpkD)LwrQ$a?zf$u^)PiUz)@uNh|KtP7~1*h}}4A9H) zSyW&QFGH3aZJ@xwuD?r6dO+r%giigmdj93r1%jG0iBtWJZE4r92hMcdwzT2SU6j%* zb1oF`hrp^5Iu0^-inarp!(tW|AABxpE6awr4FG4YZ}yPwAiCT~K1J3yapa`J?PUfM zudgjnjE#to@*?RGu8Ld~YC;bcg`w#sI>q8*M)T5uUC6^ffvtyB`#yA>ztQfC@IpXq z-5gYD6j&btoxX~V(XK~`RWc0-yW#?Qc=VU~0So8>g~k|64s1 zQGDC2yU3U+-W6R)24&PaOd|>SL%KOwixq{aTK^ShhF%f_Zr+Oa zFe1AHI1HdQ{mMjl?MC8ej$KZA;T@GPGCW121(E2#+I=R6I7$%)4(C?ffr|{Xu$Cbf z)sXxy2XAMcHC2FWu@Fw}DDAYx7Cq!*LT}*(3_U0q{lam)MCETK&Am0vDpN*tb80R->Q_w=A(G|RXA*8-=xWKwf`ii+y$ zrO55h1O@fY-Z*>C94xB+1sI{R8eNl3mJ@XCW9fc;@EU1PK7j-PN9w()EQ_!AeXj4= zt)A~~v`K+V(l|)kUO>s;j|FQVqiq1R45>Yj0OUjYdVM4tQs143Wrm)LyLzAr(AA-7 z6JH|c_~qgf(=g9|ztXcSD5O^tvbDv5fAYedlDP{PmS9JJZbFqafHM5~P&2Kt-OoPoCPoH{YI_!1FM3(}gzy*6j#h#cr;Y3U97@sSa*M(Zlv zLE+!%TC!-B=rx-V;5~ks3h)Qk;ly+2VqfiwUcE|oGKg~eXY`z+@B(z-DwYC}a7=oH zF67D~dKR4=TOpwW6oOr78UQd9DM+sNb_*)si} z_yZ(G!V56TMnmIEYiq=bDF^@nz3naN`TY4T4zl+BEwHg$+o-csosoj|2zSJDo}|!4 z@}sqp_`Yt}dzgX*Ehm#yfb^dL&O5BtPvIOP--K9^ldMeqetwBITU0N~D=3%&)J3pf zfX+P%hSb3>0(8dqr+~p6zP%UA7!M2z;%Coiq!8Ut9-P4$ajeUmLuaitti7q8=Stdy zb}@aOlE7EBfwy=&;g%|0fMK3!o*a;jJzaN z7myN?kKrJ=X7L2ICAXgHYZRjJ>eU)HXQ06QfUmK@d#a#ng0+l|*0pir%F|qwxpQT; zYMQ(mpEY34;q*@x$6!bP#xKwk@UF}kL7%??jql0_7>&I^Eh7wXlMGYXqOwDq$bkBs zC&=adDMY-89sp^?h}D_3NlV)ZjB`k9Y;d%?CjN~*E9#^tHL5A=^lAx=#QfMO!nI?_ zyFyHiVF5cuIhb;0&P*et6558p5%l_%JFJ>}4jf>Xo*2+nKLnK}W@+6tpcNMv`w@qZ zje2_$@LY+66iEOhZoVYu*+9m;e0>Yz{af@EkXCJAbQ*IR_f!TSZ6xS84(}LEj^fhF2TxmzeVzy0=T1$<@n+9?X&FP39s^L+&%(QJ z{5>;cxlA=SEdt55Lfnb(mtYDc=|MCWg*vm{G)2L5d&(PR0W5>opta$`N0?I-p&kNW zMLG{CWGKz~MVwbaQ}FpexD%U}p8oogJr$Y630qC$RMxh~Hk$R3E8_UY+3$$<^>l7g z4w0D}5% z3`(e)sIID#Baz8%Ncp;{h5+WL8k_R(_ll#+ySvbqsbwX*%uOA(ZJEprY04U`7Mfli z+}txi?)yJQoN5ULY3f?Yo{^3?ggT@DkBWUpbYVuBnCwM5{3=9y_ujF>pi`;(9fNH9 zg;nfQx0CJ#GVtwzI27lkH^b?QA>aF4#*vYW^~}}93A(a(cc5QWHbi{P>K~N|AEz*X z^mC-9!d6v0>hGX3?n9{)-+UgvKat09GFhh-ieUDDy2e)|@YcoZtKLK{EUwfm9ZSA? zuypUaIBJw1y09K#(2RJNCXc+r%n%8h(ApuDW1GHLD%dZzyQefQaeyK~R3$97h!_q4 zZp3g!bHYHQqX4{mjh(`wv>0qY>w4YyJAuSO8thhr1}Ghg^YP-{*v|e@*-roy811F1 z3-YRXN~CR+dF;;U*(X1lu1t?JIA0YGQ2Bvr>;b6_74v!_8hT9#Kn%%~f8z;f=fD8u z3d%iWp!09?gyU)OjPqZ)2Ff_9QCO>1U8GwnRT6|zj40sys#s~O&QA5_dX}=Xp&t8x zHQ;QA4^Z0HRG%=b(Y1 zV!8|)g!$~nZj(QAEQuCWX*dBs<;jBsY!?RP{0C_RtubNDOaBo}?%RKWI`ITq9+`-O z*+e&PyLosJ<6i3&hG=-O>D@@f{_}5CGk8=6y>yx?%sF7_0CyCR;p9ePfV-sCL_H|} zNT6vBuLA|12XaZGk#n-_XLeL@@YdxxLV|VP4{osZpGZdDVhV^+PN<1l{h$9wp-Cos z9LY}v@fK?+gN*H$@&);F)HtRV8K<8De45b)J+a>nXTV6>Vy7ZZ?-2=Jbnxpk_&waV z@rYCym9mXUZTn~ES>12I1U`u1Nvl9z_X$_m7iiCPHtH=!<&9 zr*vS}hr0`FxJ=DX58nu-uL1m*Q8D!Z6&ObAgR9cJj~{>d_%$>m(8S6jVH#vFLlXd* zco{f4Hbz~dP6gVz1p-e<(_WPkgCGb7KC{Ev&TWhEXv^AWj23qJoK{|qqD2_aVCwHM zAbO6Y;}3(jUm2_uK_X%Qv3s{~)db@%;_gg}T(_VL7v=!x39?yLwQKKrht`6u9jpJj zce_k)4GYSvBIF%MhNS)ui34fzM#gMaql4z#PoF;r(4=9+^%PNtOfsjCLE@wz;4m=P z&Xy-KP=GN|aQvWndI7cOIplaRX;3gF8s;qnZue3|1tUYW8ZYYNx>|I!AobxE6@$NrLxkkZS>Yp_XADf%c<1)R^nU$mEh4}tgU?a(sOO*# z25NB_l^2RQ#n3rEz-tcovDtF;ObC_>(9_a+f|A8o(g>#svLiL{LVkjHbBvibRfS;%Lx1g0Ub1^otFC{yB zh08qo4I8+`+26K}U;4AJiU`v8sE;BGctYOh%~CcV03u{Yw5Lb;Ju&Oe;}Hu0vfPvM z&?zuvwm+pQ959VGEQmxj!-F7QmBf|A%}q&Z9wu`=NWX%q6r}eMHnw^_q5w!?nIMWs zdI9JD%Q7_UeqLA0N~U(;xUAw^JDY-I`Yaa{5y+rUZEr`&B?+pY1sv2!*Yre`loZ&kf(fn>4r;cNXF2o}6!5LfP zKp{mj8R|?JTY~!Gj8hu6fxZ>ZLsx+`Sh8J6e;es%a??~fc!l6}7l@TI%dA!6PE5p? z2Ep=rXqk8)_|Q%?rjb++UGn8V`J3(#1s4`bd+t?a&FI!4Z6N3wCH^j?DO4}r@-%R; z&VVCYs}{suKEQhhUE~B5g0urE_(!t!aZia$iMnL*buVP^ObZs&M$_>B&tY^_?53vO2XafCWWf0`lF13!e_QGul=&b*C&pJ~ zz9CRH2mJE?2@>_s9!9f1ut?Hj_z`(C>9a)rZenHShxa2`Fdw>@${qIYIWwE#(z2C1 z;5YgEhs}}<$Hqaqn3XlMZW5D?8ZaXfMYBIt%4C=qhN3D-52qd9ud1p5eJ4;EBAq4& z#^gN!4yfYKpbIhr{Rg8sC>`jd~f$i1du}Xx1(!bAZCJ_do?T?87=PYqm$(GqvMB+4z2E~$B#04Q>yy2 zN67OI+}X0L>zFF2t6y-9fdOfj21R`2$dQXfD43av=op4^IByEonguofXz7l<&t6P8 zX_OEPcHAc(dSoI5P#u!ivGW`?F?-#%=lrwC73%6cPUV{1Fs+X1@MflmovKD)%S1w; zj_Fi|*VgVK4ZgVd&tba{r%_#!5hy3K1qDF7SvA6ut97g&~EIe zZQGUtnFH8gh*R-6$OMA)BNM_)Z*$NB}!f zdEh6MP@mvUg~1V)Y>&1GlT~&>Q8*$Z(WXADn*CeDb#LL>i)M@NrgtOspW&kT`N{5` zeb{bc!#jqpL8%_vov2@D|C%I}i04ga+#`l2{R$}{=E%H5GA51mBSb_*P$;1H(4Ipq z10j&|_w4oFRR`)lV+k)9Nb+JB++KkM+s#c&ixnNUzTns95Ni)nFB}x8zvl#2U^#H$ zU?>CKhyKefsf^@&4$y}0N*~tlzQ~9KTbp@goIZ! zh()so)f%zkMhVrQhG)jSM)Kg+Vs3tDe9pm3qZLYpJCbf`ojLe7BHuv78eQT6?g|~c zaj~1==eK>v!|ZL)To#&Ypk2_>fnjoD`SBA~_IWBQ_D#+XmG*gm5(F*D)ly?F&HcH~ zr{ympik@yb*e9KuYG1Z|vwL2sJC<~+SxJ}uT2kf`IHAB6h&ynNXN84(9*_a#J`fpF zOEFiGRNb@(cZ)HPZ6!i!{C34b@PHlq?{UQu_nt5BfSN1BzHi<}v(=niS3#CXJ%hdT z$YOA(DZYAEPi@?Tu%SPPPwWWwidUZGo-DiI6o0SDpH1!S&NI6;E%ZYhcN>J1tbt?* z85x?W(}QDq0XD~yzzGN#fW=g0Ub1$BJeDZ7V0Fm|oz^$ zKrd?nelKY6cgW>Nmn86Q8NfX=!9B~4)trMvp+`PiaP}gvriDvkKm6+_GC@DbOc+TPnrFVdV`Y0Eyg&3x=06*gp)Kcg|-#Pq8a z1ZhicAro8;5TqvV4GCxPRmmx-ynt(cfUNk}#K2RaD~Nkg8yQ}m6r^(yo(%Vql1HTd zH0aphXTfvxRsUuk^~0Ti0+2&Z2Cb=UQHyc5j>8+_4KY)j4d9T~G?C{dtl~}I36_3W zW{bPtN?sYA&~Ol^k!T>d58@7*I5f3}epwaS7Q2wnul)4!a)PR$!vpyA#ZnAJ{{&?y zk{mJ%+h`K%Hxrcb?mcJ15wsN2ClKi|jC7N+DG)5bO!VUYSg)dP`jUQeU)mWr9i-fo z933(hUW*aFp5yjT15N#m^FREOHjqaeDYSG$xBb~W&@1~06=3_$zYcO`U)@-h%VuGD zY7Y)G(v{+FHI6VwR5J2pw383IcMxdE(TIVZ>p-fZ+smhHs=^Yb{o~y^dvyRfuQvzs zQSWgN-3ue*;}{D$IIb43vU}v(C(==2|LTZO$ZEB;&rRN9p22^Xet%)*gWQz0w{Bx~ zmGGB9o3WuW6Xs}u%m%8^f}9n(358#Wi{ptC5KJAc-0qE&=o$9jr?$3r!QaS0CYWB! zfvN>3a+S1AVApm?ufQFwJ#1sMQNRO(mp`;6V#E!?MnoDiSelfIx*2qc=L`Q8sg$=C zpZEJwt$N;1QxrE{LLU$W|-kZc^2JDqUo0S>Z!H6Crl7UlzTb&K&RRjJyhkUW~{Tws& z+^{BT1_lH~x}Z?$ZAB+w1n4+o42hV(O7bKNIO&KEunJT_661=4`zy?#(?GRoP0q{m zeGPn94pd!0xlKkm;Js@-n&d;)URktq;Xk(~U~0)yoD+p5m;D*3U1Ks-#dtIXF}W+T zz^@QWqz&~agWbIa_EO+HSed=RwtK~0E~blA>XJa{byRxB@Lk|%89^iz#GML68DzO- z!)O{M9T&h3BC{r!J43rBaPl7xJmRNCqVvhBYX7D6FCiloY*fMQ5%9r-RYS<;*PDYM zMNs28_c;p9TqU-h9?jeLV2*L<5dkG6t09f)7$^mtE{M5eA9eGaSBJ%T}z2y`72iNpeky zpKSj7s?MA4-Uul~+m3;JyWq96_zn%1WBjpw51D+2^a{Bu_3lI6t(=1YIO ziMUI>x2}A9|FntOj$pJE7?L8TrEJq-}$~a{T+7+)ARmwr$ z0J1LJQHHq-MGWcgi%rBygCiB1#!jF+v0rg2LU9x{W!76z_#nPt=8CsnX(WSUr*ga# zOQavSvV7|41Pv{n#fe|_mhUQ_ad5Hgu1$%ttn%uWY-~ztXBD>wHS})lo6#Cg)f|6F z_+y}py=A4Mh`C+HNH6o#LpmpnLSi`5#c4{OefgCNY=A#f0 z0>J?bH}d+BiFO+@nkS>)h~{GZbEMDW)VJ49mMg9w^AH1vVn&8by-yxQy;}5RWIxxW z*GJB?rho3s&j*GQaSDs35^EpD)MO@k*QKR`5I1t}u1PI%b8};dTRNoRO4}O?_W4lwIA6jCR6Shes%g>^m#paAOFXyXZko&_tn$^lGT6NXurSr{nR`iQ->3=4a( zPT#)k@sINACzFFhS;OEN2VZRn%g(sENeIEXE>PE~%f%KgBE{0xkc#Z?C_$g~bK2?) z_W6UijJzkDvuqYg4P4WIyUfyMbbQz7k2eL-%>MfMUQ(+YW^E(tYB!qX@Z(LDwr*g{ z&t1{>Ed{}%4iRXUwl!?RwLlx0q;Hoa4@^uG=nC7KHFo)7H*Y?`Bhcx6{6i&t#|fP3 z!59Y$lC-T_9Qo$}9x>vj(N<;68w0hv#$C_n3)O&rNd#Pd=#vo~4y4CA*J0|S9!N!P zIk~fd6rykfgw;9i0PLlMr`42!HMUyD%K&^iYVo{uSZYjwwFRszFyM)LX#jU27CGXi zz#&zs!2mLL%H}6N%L?oGT=_zYYMYe&X!Hnmqp-RW3HCo8zf9&Xfb&VtWOOnA^7(To z7FQj)uOzj%>os)mipuQ>_3#|=lbm1jgfl?s?}i=hPN>5Co$=vuuWz}|QE{55WbChb ze7>&0x_o)eoHZw(&YOFnx(4;)cBGNWU1Cuj#)Dx7pBfKEKwd_k#AYz-z4^nTll`}! z1nduic;hYX`kEVmCK=`HA-1Vw1=XNnG`Fy@fp^vk@sK(xq#V^~`TDtrirpSxQ?U@M z1ba;fx~)?OE8n3IQNC^H5<~zXTI1QdOXr=_wSeO87U9xJnuQOz5eadCa;W4X@;WRx zZRGw?W-5{!i;DUZa}b`}1KFiC%`A=pxpsOhk-;iyb zSRs)GfKwp(tp|!8@(!@0jYA>0Y~{t~xaK_rM-ukW`d-FobbUZ=tk7i!+*x-*h zS5kAGRT)j?Jx#94ICoXaqt2;>bN(QbYZ_4;mbLev^oEI4a&EUkvtxx+?m*T437rny zK2A2z&e_V0}x_5QtqZA(FOj~>u7*2Rm9 zrdoyrSi2_2qZUc?O{_u#O1wF@M3K_#E+=bjkH&&idBju+>jDHiGEzNj3|TFgner4{ zb;YL+{QxSsnB8DO;;OQG$;tPGpw~m;IW6z zRM2$D`FnozFWGYn@_SvWWn5K3p|GC#GMJ|<@Lb|mXI z%i_)%e?I&qIs!FNR>2i1Jvqlfu;+b_CQv^t{u?;qaK@<2-%Mdzi}a@S%^Pi4wF5p> z=VJkJuZ16<`1yZ7`C1&}9R^sm)$ORAL_F$%(Y4Sx2Luh9eOLh`1i*$ zZG{RI+|^0!=TO+_xCeUX6*x#fBN9uzi*;} z<1XI#NkM;qe*mU7K}%qMNz#P)hv@KdFHr)ngv5PJUQMnWerYC+zq+D(?E0)L(|fzQ2T<> zy)Y))=rjRCUR`Yc{gPo;!Q{Y)KYuJPhRsbCm+FMGts6Ej+UO_GsToQqM$|#hlIs%`AOufgOT1Yfqz$tA2w$UN= z^c@~}eptd2zZm)w!IQ?k2;P$ETS$~!# z)_f)hY@j%D99B+F8{F6H}>ir{e z^h=?XM&nv6l=kg=i&n&pEb$3e zNPb@l$Bjr(cU!{@hrSmsvcb9kt4@G{RwS7ei-WTp;&ff~o*pf~7l#ogv;tr_aHQU4 z#QA&-iEAegMqMwj0_P-rXFWLU0B%Uc;bJmT~H|2Aqg2o+D3~{NY=o%X#R#-#%z=;UaRZ;VnCEtv;$X!_*4&92X|pwt|X*8DWBHm>+rq!O3;IBqfG((?8ZHOW+8f zjviyhGE96^2W8~f@#@W+T0q2!L#KDXKYj*E0psXK%!BN#y8>AG1BPt9Iur)2S2ga- zHy2y%%3^6Ubtv0k;qr=4+Wu&yAodm_ldv*M_nz`f37tMf`7T|QUCwf#T=F(89ZV_a ztlqK=#eF-3T_R3j-i8QZG22j03B4HV7>bV^+X@XW!dGZ%HD2w`^e zqkr>}5W@nQ4Xhha1M}Tok2|CjAqKuG8iG*$HaK;wqv+&!v_;10%;CYIrHIrbgcXyr zku)h*VO2ZB>#a6cy)BVOT<8`r{xDTf$`Kttof=PA9`+B=_v%`zk;;`R})MThcJGJ4LT(m*Hcd=L0 zK$lOxmo!>3fgRSO6+6)e%^`wYS(_lnjH~DGp7i~}sDZ$=1EVE6mk$omP)%h$s!=k< zc`Tj0jQ*vkm;!q%<`yd*F1ftYt}P)sH(stn_3n;0^XgEL5+kj`Eu{wa$o87cmtI15 zE0V?i+h2%v+-qA#wS^lkgxH&$TJRj^OX*fQQ}3oc{@mW)h2zTZNi3r)N!F9z*R?#d zP|bO>;Lm5uT4_g;&wTu=5JX^c%ELu&NeOGob4^(TUwRu!OdMR&o7VpsT$)`QYKOp+ zS9~j2?^Rb1f~UW26@tJN(CMa%+Jc;*t3{7L9eel0`w59-1bjt-XZI}p$ztm76NN&V z0mGcaq$rbT%|_rK61KF2Ev34x6%ol$yIV+0&Qk$CH0XW>75$6u+i;+r@4s52&SDUU zc(#9AYpar^!ZV9`CvjsAjTfu|JulxH>dd@)cN$Hz# zd&NQxkw?IU5-A=b^H4ofV-e_8QaUhL=t{v@_eRAVCM(>3yxDna!D*%V!)owhHEb1w zQj!`{t!u+(&@pgNe!}hXP@_#Sn6{yB%?U6YTavWY@=v}2=IcI)^;iJerv>N!xxFmh zBa@Gvn2O6=Nri$*zCA8IZ+5l}jBQE+970an01mbjWtUL zq^8pLtQ1Iq<4`A`^HQ(yga-%bwMP0v{W$)kYAx%Jk7pK1A8-1!UJ>9!HOQgP!Bxhd zedkC{VIv8$84{zhc)9)1+r2Mgbaixl@X;>GmwZZu%Q-i_gvVZSc%X^KZzHO@LD&jx z-I``l_q%4IQ0^!GzH`)va3XGYX|mm_T%GFzI}sWp+bxq7MsHv?6YbHPU(asr)*i?X zZ_x^yzC&S>fpC3P#BPpO%&jq6#1p4G|^D1{l2tmg!@6Y52rJ8sL=og+Ec?o6#IrGq7^Uy@SSCEf&(CKV`m82q+1Pfqpnel=NR5uF_rb`>2u0^hKJfit zqf@^PJ_5Yt(qwuF7|7M0rSp@?fcNZ-V9C1#N|!*@_xnfIE%v)aG4viuWrBGJ(6yME zEMoWmAb@WH_3>xEWQvSUa!zzn#e&u1WU6Qy>4;B*d568f)ap>zHzM_&IBL^M@PaC{ zt5RQTgYl;m9L9iF*Yw)HON&CQWIVx4s^aRdK?b6WgQ_A|1@*Mo{I*nR8fF(DVzyVzYX+Xu;du?6A zc9-M^Ko99iMUg;6Hgo``0ulE#sC zVg>z1Qv$Tl*@mGLA|8uSPa>2j4hbIC!F3q(uX!sQzDJeq~ z9o0y0l5F5UHHLH)DPaOdM`s86Fmw+#gn361G!4o{THhoVkrjNZ?dn54`xTKvnbun& z-AA-M;%_K;`%ldf@p4?bx&|R77FC%3uT_rGAK;M0Q7TOiXuZ5i8ER*6T0#)ks8r;FcBJ8udjJ*1}E5$51y+*?A3Pz8z{Y+6q7^%THESTpU#38wHV3GuH zv^BZsPu~vNLm{X3v-m0j6(gNtxDYkaIho0M27*WiGG1ChrxYnvFcKk$5{=;3sAGVk z6X0?XXLcvb7J2DyuwKsxmbPAR4*fRYPb76-crw-uSwhG=k^#77HH@!8xtD~Y%0M?a z*6NYm8mYqlyY>Z#pu$_zOT)MWQn%p%9CeCWLw`G;odQS3i>ODjT!4T1FI**h|M6o< zBGtyttYutopmywG|MCA&yMo+;qw^0mN1;3nk@fUe;hP;m3?{6|lpe$rte%yfC+Oey z;W=3^@+%{1nE7CI@)INRNykv3>tqjOKa0Laf6MP57B!cV$@g27s|RRQ*^4CB50G;O z^~$*qTSw7@5{aTXh43u9TVV zTLhM}KEVRTRLp###}Dsq{KJ7!xQ>6FoDh>J_E|!#*#+|lGyn~~Ofc?*n*bSah5|($ zA$me>E&Yy6E@vibC4&*1rKyZp8erTflIdH}=P6SONuDW-$nKs%#IFZc zm7n_r(bpaQ(J1-0oI95~clrZ@eFyOYIdkTT0?a`?7If}uti|_${I3}rdU8>S!YW+! zB#24z+3zntbsHrMj!aEe$-c92W%yRRjiA<0G1LON>fbzlMU=U}N(hP~F1WU3YSMKA zLC~lfeR`(|pM@m#8gn6EM>qXOFD+G9MmqqbwMq7g20!}#49fX2KDdllLwH$@`e{NDcxk+lD zehxpGs-3w=l5~%{IeQ~<5J8e?==esY9q8AepVLdbTf(ZP7qjh!0Xy&`agv`X zd*!i6)tJ|oZC57gmV0OTg>(Q#DWT{~_a|BNe!-l`dkND^Z;L?pa7VhT+&+S;2tN6F zg3uX8Q@g===mFf&SDV^R|6R3v)SnYMCc)7wJGm*wVh*im$xJwcDgf44@9u)k(;8n| zOq>3AWj}r=%bJB7GVnpH#&`mTbjVC~)e2!kYSM(--@Afz&6}Hb&s$`c{y>z%o9yzG zF^NQyHi2We8p0MRH91cf~Om$jC@EZs6u4ef)rr5(EoIOjEqV0d0!F zI}!PpJXX5nxzTg<`=qtW)BBs03-wu@f#7ZOjNli|CneRKV1WuW25T@DH|~EfX}tp9 z>?U%4eS0$Hh4(B@7za0#h+pn(v`I-)d$r@wlDKdB=~p##W;3CC2u`FsyjMYiMvnP) zS0y9hlzSq1I&eKDzua~}xIqJXr4!1Lu)zWjyX;*s`m5Abl`0mZgI8RiNh!9N`YdyD zw8C3JTs-&XuwLe|I9zEL;+;Ac!^A(U{&zY)y9xRJ^)((}g|lo>8q^U|7FBy!TAAQ{ zHS_5;ys#zg_QLQL$9hmyBH8ihQ6#sKjNJ47*~W58}{IL}Fp{)P=R@P@1KhAGehoff1AbV?Jx?2YjMhexahmEjssP;}houLkX( zgQ(0qwfF^gpEsG6yNUiWp1x|#u}?EsNrns6eXGws^&(^K=-H}!Vs##U!q=_!Q>cyg zs&9Q34%S4tv{Q8E{dt+}I~FPf+uPR77Y2IQOJrE<#=r2ui}W(r$(OlAcKWaPjM}9#HQ+~m?sUfLUlbteUB1cqO5BZwH9=$>9!@?J}44Z z5Yu9!f6ge=d?yCVPqQBgLBpk2qvjQr4~#ZWkUuw`_+^NfQnSS$pXNRv~X}w~eyZzV~fhaLYPjUO;Z3HMTD!FMS%`o*g)!M(evm(`o?UEg%~OZ_1IHGFe-3v_4AzLNT4>Q%o1S}@iL)$#1Ovoig4kLB??uEH%f znMQ0+lHVm8J!W3FMWTlO&!QZ-T)B*tu#YW+<$l97cltjB%p%`?%r%zQGT2!EQ#_#Oaw@K>`^lmK9v1ovniUA#m-5tmN!u!S^OvLSZ^M%( zCkj=Pl#~X?-r@Uqe=t(6jWK_7B9Z0K$NTi};+w4{dpkWbYQJ7xRo$7cF;l}d=`r(^ zD@U{EHR~oQ1=xMnGM;Fv*SypiI8%+Ee%S@&b%!@w>(;Hy(C@a)-Oo9ZoAq{l`S#>! zg=EW>KBe?u{Jy!Ib>{ts*zepa7CHSt4d~|+r5)6#kLl{r-KhpLy7a9sJBnv=|DZx; zh}JuP`psry+vOO2r>wmSH7c?;K_Y>(va&*|62OBVX@0n45P}imvj*d%P?S=1nP*+S z^D@GfDu$Hq&(GTDnc)p6!84A$S~5ZM7kJOcqJ$IJS)Zq42W$x^y1)C-*SL+z*}I8K z#L3A?3~wX^{KC;Pj2>K=7G%yZEZagZsnu3&NJc*1FIOnIz9 zq)RZtNxWaSAA zI1qIkDAq~J2rZ&fY;%NFaAS^R4AFl{{dm329deuj+(JVz$zBuheVR{NhjqSwrRh}= z_i)RDm&5cR!nBNGelg^fQ9z>&aLR%{u-kQ~@Y64R1!4pAp%__1SH`_=-Tv)Q@PL6G z4#0)j5!TCE#hJ)pDtK>a?-Vp2XeSFia(}iI)dK}rt zb(0m)VF7liC)s*4Fo<|OpPZzXU{N^iAFJ>DaPk3x(t%&laBcPIvBrn6RWY!sU%s%d zn1_F(glhcf7pC-Y!Z*j{v|cu#9kP-L1u8XIquI`Wku75g`AHOL7eeC<(i(kMyNPQa zxRG|^gjB(F+W3%ZAKws|7*Y_9HMLAecXoEl25>17$I6z;p`C;obB7vhrp_D|%i;`$ z*C*OXQ>RPSQU%$@4Odof<7FvxXI;J8HZPdsO})Z=(`4V2cTpI}hUxm8Yu2npq!r^^ znD?ZvDQ7Et9X*|7aE%}WUgIe~X&K)6+9ZZv(bNB0oce*kB#8cPebi+6X3OG){PXR( z(%G7YHeVzxK3LdXV%5GHmXDY=MzoehS_YF5nYZl>neF_nj=7?m^fZa(=dsn)B!e6c z@-h*x?zkAIZ#fTIyxYMFhAs?_+4lY^5KdhkPS(p%0^0P*8)myF6@z;Zrl5SNK`+$RVJ|fM-B1Zx3-Nh0P^R6t z4Nmx>{Q;^3seEi0QfX$`e1#NC6Xw)z-m|!3Pm2Hh#pWI3paMzSUlsT08R1Sa#pUX7 zsNExwrlicu0Q2xx>Z;Vxok4L7SU@x6uphC4Y4Pe(U=iJE&Y^7@cPl= zVlg#+@Xx}^#n%k63$hF*L($E9W^L?*qBYQ|TQU`G@yjGE;Ql+9o84?7D_``_vOe0R zMiT!z8srGL?TK%tfYZK(Y%u9Z5fq<>U+)pr+i1C~%1X7xZVQVW{vLZ?nG1>mb*I3y zYvXrz886Rpk5IQSD;2#%H@HA5=W(wWqf|f}T*L~ zrgJObb9mu*eBR{WV`6`!7^w93BU#Y+6SSuL%I(SllK9Pe^2G6HNymwx+S#UIS;oGIE zlImX#$PsB2=2$L9f}vXg^Q@$7zKZaJIxfiq^6$pN=*bv`YTguLdZPvwLCRs$hBp#- zP@}E%@?NMI=YdJA0flq*hiHfj(T~<8r$OHm=-4j%-Ou6V!hBKP>An9Idtb52cxp_S zobcMX`lKxf#M!Q(?f6iI)bKqJx;kH1p&?ld8sWj8BJ~EE$s#kK@$E>v6Jko2SW$eq zma|U09Rs6@^uQWtB|&qrySnR=66OnL%R<{^aY}mUPTfV4-+e?1utmxq8F29xvwB1y zOy=B3zeP`neYsgBV8)ri^yNTQycDyHnyyNh&&&cKK)H2rQTU4gi?25U%dzdcfQ=6ip(rvZ zQJP4F=A@8P5t>JmG>}HkJcbmRr8J|`s8T6SXc8*YKq`vnBu$$CwNvW-zyJ6h-{(CZ z-QD+nUDtW;bMLj+TDxu0_{*MqA)i{8yJJ9hQgqH|8JcG=;)oHp)8nGdW{+k09j7G?S zzfzd4L({r8kSN|ia*G7Y)?A(bjPOF-s}11qlEPYE{vGn57<7!9;NWp+TiH{~!$d6s zq%1iiE*<{!gXHyzK&`*zno4U@rA23dHG1+oHl+utb)Z;s0QgHcOtD)4cpS2~f2a3F zVZW|!z+ztYAWBQn-Y*02MojAC5T8Xrk+V_qAI4L);^z!~_>sGgefb&5kENHWklIVf zGKWPIZdn{nv93dr_~l5j6qYP^m*0;WXBf~PM0Gy>2pu2<6>D&o@|&#t?>n(IX8$F( zO<@edTHOl;F}iVRSA(h*hUvf2?(R3tEuhmc25r9-{e2W@)N%nb_x*qX<7Y#Yi9~cC z!UG1?2%CQrG97%cu<^D@MoAGw5rDs5eLr>EM3Rm9^qiddNvZ28{00qSgcphYe(+;J zp7q}Rwj8eN!}zNvl-Urt0rdZ#EnVVpKo@7%g|Bym4)G`A!D=jpk7otNx}HU+5D-r}6OgylH(rxdK!?1?Tw{s3gcT$JARLJr|tr zbqShhRG5|@>{1etFDO_)@zgg~ULfO-)bDq+5aAHjpf>-2@e6Pm&6MpV2SREIQL3I| zk@9^p21VzdEamM8HX;eq49%@)NLVv`EFzZu|NkPuD{;Ivvzqm!TEXkKM2EhB}9MPHccL z-}{!IT)rDLHYB5uponk6#UiSsS%eSl2Th1W9zAf6t5?-<(XfDk%}8iknyOxPprll5 zmlblWe#j@#QfCamI-^2UQ9vxm&u`*5GEAz*AwXIP^f^Dkf;DpJYtuELub2hF4T>B{ zIS2Rb@i=8=72R8ymW*AX8GqywxLqq6@pgCCvMXNyy7PLw(NSQ+W1EUG&1|)Y{C)tK zi4ZZ-rP2ONoUV9-#mT?V3e^hzH}ydXiUdJghpk}N**!Srp5^#W3Wow($;#rY&^Hi8 z0ytU|X-%jbd%P6iY+VzY@aT%GfG!0p+lL48i--$Kuse|%A3qMtAD*J+ZX?^6{p!WD z7n6?sH)q!~i}6ABgO(SD-wsXFnZtMkQH|#gO1AOP$CkU8dy0i2;)T+ZNilN?qV`rx zuaK?RjvfW9{r8H&lzvKy;l%!8!yd}Ik|&rmX{#E_^~xutAlhI!J$%ylDutwn%WIYTo{)ELa^f5sS2roqXC)i|4#V zzRU4KAjpquzDe~aV}cD=4?-*j(a}^QUuZ9aS7)J^GDJ!$h&Cv^mNafYbFJ3oe-0lX zKr>h3(Zi9c?UrDpQ45z2iSUiufKQdd;LJlc-vx%(@Uh1P>)2HgB-g+ zdyVwWMT<17xz@*NO+0dsA2*LqhYSUj;JauLkaiV}5I`3*Q=@{cetnbY)638%+!>q- z6cEIKBrY;2fR~iZ`{4>f>zGO*JTL6h^t+Zq5N@+Z;`h5*^sHXO9~Xw-jiHM#0iiPpHp~WS=VHURHPc_Xwvp zRW{{KILRRu;qDM6x~iykIEi^iImaj=V$*wk$shvU)41%GtV~0Xe<$yqLrL> zAI+7)PAGwT>k*nAGgEm3Vf0**-do*rGlhyu$`go>UD=IEazvUT*#5=KsvTE}g-k6E zq&*J)SfDWIR3})bvKxICs<~)cV+wqwBWCr!qYr>4IOSa6oMC3IH$H|MG1^jx#uuYo zdxa?T%?E@6(2Ed+o?ui9Zn_dcsHaFFv_cs33Rj)}#)R#zu-~U}FR09S(XQhSD>2bA zg))&6b%+vSMZFV4=#VIR|Hv4yNH!kn3>-NIt0Uc5qOFd~tIARDu4?<3u#3#!Nxz5R z8$vH?AIM-)j)?o1qRIB~;lta%R729_&Ru-Pw69BQ-o(Ew*~t;@$18BU%9Mig83mx4`_V7<{7`Sh*UO5!kiw zwH|u5Y9wDN`*@5VDhbN4Tr-4LFLGgQneEN9HM7{}FIQ0dal8#z65zt67uJ*r8L?iV zz-4-oa72*k6vW$`PEUK=WZmo~_T#Z+5s@+g2HiRyLxc+8DmKr0AP8ik)cOy7L`%&@<|3KyjmS^C-`8n*Nv3BBi?=`f}JT53foC_)?J6^Fd2 z{m-C7QR>0e&(9x#K{nke^*R_6g(JPvjTSb{p3SF|`e9j7pp+hPv{<#G7nz1D8KgnM*NYAF!zJz z>=i(fMJBR<2eWeqx$yL0UwiHK_Zu7}UIR1C*rx%=ab)FhKdZtG?>c9;*Pw&}3JQD~ zP%cQpzpbOsCca!{DBj9!75~3)cMQOoCIqz{uRw-JN~;@~chC=v)Mr-^+&Jg;2g zvkX^zoD@>F;R-QkL0Vepa1`jIq=pzXK*GSr2NTfY?wLAVcTkSN-(4EqtoyX`kgsuX zY%ki_{PN|?QY>HG;JsjDLWEXz#51TB3kK;B zW5@Dr;q8{86(|b{dLwj=*qSdkz1$=|ygN)L0mTq0NJ-X>*9__o0Gz>eDK1l|yp?w0 zcwmw`_rG}YdA~kbD}vh3)4wFJh3)76$vtnj*^Sc3NwNylKMwynFBD6joQ(sg1;)_F zQ4Fc#R^ca~KTQe9a78d}u1t=l(hz8UU-WS1P}>Pnm%*nTj|hHldg$$xvGaM)wV8I- zVE%eVhiEGGoBF?pAJc<>=C4o>F@||(FkqTO&X$Y5+ZGsm7)@TAsB@8fgo7N5ir%tv z2wjJ*q&f1|b$Eo6LAx;pZZUkiY3lIIDTegZOae2G_I0<{=waci!S2|#3quf_fIlYW z5-A0zr&ET~bNwc3NBI-zy+n*YHlRhW8P#0s_hMz@aQH{yLL`0W=Cm=do!G6MC=;B} z=}*K^LDOV4$Yi>qQ$LQO1F3ahV$joaJOcho;uyLWG_er+-MDI$C-+F1VPEJi|9kBXe{)1GG31BJmr zlYNwb;)Vs5E;}~!zlTVvZ#VYAcwvc{UQn6Lmu%qW=sg?@{@Wmv0Sw?W(6&`RtneA~ zg%2Q|76{d>wEG;4X(XoFCzB6uGDWMc(zp^?3`m4ux;p{XSuM9~EPhV0R||UQ@)n|$ zJhN?!&qWfBc3R?S>}_Mdsg$TPj)DP}te(G6KddMM1qYEa;W*+`udsVJq4ZB6G{QG{ z$9;k_F33CfbiKir8r@9Vk%#!uzRPP1K%!A@4gsPOY!XyQpwEjkB`Ps;WQVeD;5SGD zk%)k1(rrHiwD3OCAk8|O6&Pxg>MvaXD!Im`Wy=pd56A$m%NoA83gn;+oP@StLv;h; zVv7yv{!W{XvE-u5fo)0pBy2;Pf;%M~ z1K}nA(=XGm_Uc)`lS8tdO77wE6qk@b?{DON5<-E*tk1IVNZ)@8quEquV*^K{M?+KZg3x{V0nF^@MDjU>*7 z1_?f9pJV9XMPuLIBtkUF!h|ukCmy7%1C(am&9D8$xo`qgyow9Q{nm_yU3w6fY8({W8N}EZc($Y)@*Mu3gM~M0^-ek0Eba6IEPBXdAxeVMf|r9VA@IP!CGuD$a*O!L&wR)w;x>; zsO!${f7|`ps;CL&*ljco8Ft`g@A@cWWV;^(}q3 zY6-D(Pw$$qTLUz?L?;#bxNUbisF0&_H>xw1*m~3;)Ro8tpIygwsM`UJv?vjdCRhw` zC;Lv12DgynQSeGV(u7i#fgt|x8LVz*d-)k{LEA%h_!X0A0yC3iDO= z;&<#EV{sA@Z557qy*He>X2?d}Z&aBCi|8L|QHjFmQ1yq?OC}oB|NWSbrUK5rFeHkA z;Pj}dpjd;iuG|lhm;SvQb7K*o%Fzw7IBx)wD0SbV@pF^X=p$nGs4N22bP#X0=bUJtMl2@w$a9wF6Tr#tHmkV5N$Y{kj@ z+#uj|#jSZ%WJ7`Zeq^{;%9OLD=~}B$%8|WDc%grodF` zltnKZRc{!z7GR1Iqg?(e?(ju&UBG^YU}zl&M>who0?4%#c0D8FEKOIrvK5qVI7iKR z%Dj@ELSz!W=5+RW1&~W>WH5oQn_AmAg$S}kMS=-5;oyn_3A8}{cn@l?={j2&Z%ry_)@^6;@L#-o zbu2*7MgE7As( zz4Gs`Y^@H5PVCn407TYiaJ;=ud_g$?T=UD|w)8QN?qbR7|F2+(J{NN5H7_@D$Rjpz zmpJXXz!0Z9g7HmI4;c9_m~FGl4(XYO!CyVmbK=P$t+?&o+u+?!Lc|g$998ktwtyv! z(0M0OmhC}~{p=h?RZ*Xtlaq<%luRUEnQ!G$wWN4p*wQcKtwi-3Df*fp1PX7O!cs zflID({S29dmSX?Y8rh7bh1{RgB~N~FLr+09v=4vZ#|~V2Vc#*ADd**OJ)WqyM9~T+ zgl;p#%QPoJ|L= zb)#crg46z?FAq*O?2`CtNtjjDh{~@+BO1$Sg~z)IoLn0pDF0r^u4>&Vlz}eGhgdzS zm6A?RWv$>ayqoZ8JZj-`yyTz~%k*G00=epsOpq?}0YE)NjzEPI$s<2KHuw)wB$6;P zPR_6jV|Wem`Pecx;*fvbDp)!0|UdieoGO7N^@ zflQ)aN1JXLz7qjiI;UQ(BdRhT^{5s%tX;bUiEtT7>d_-c0h;q^C4vuxRK#}%X-nEw zh>a_I=cJ2XHrmYv3KWh3jIif2D0uZMZdwcwYwA~%5bJu8Pv{2uvLSS2eI*=|2o!a_ zvtWjBk+H|5)~dEym1M^gn4KTPbaWDG(EpXu)s3-&vqUWqtRaN@ozVwwKDs62VI1UvAKlY}vsBTio`|~J z*K>X1A zqBYM{tn=0mA-#ct!9$)tB=@TJzXSDtp7`|S6(_p0Z*ceIPo+*e0#_k^VdCq3AS+tN z7tmJ@R_p{loZ5h8+cs=?q}Vx?WxFM~A%Bf}rfknfp!X3=fjx>15VAX`3ju+**uWss zrvZ`LEUsBTPBi0AwR8SSSPJeT}YxnVltWuI-z}T9_;Jit8z^urF}7|RN+PH3?TIS=uu0Z z3^Xtv)l;XGx+(yd(GVU0SPfJ+)R)$)=RIDBUa7yxM8@%jsq57~NTLq_^OyFX zdX9IZK#FO-7^X842PQq1x#{JA8sFX(*a4_&qc_41OWhh~ ztMBzZTQ}-4eKp%Qb1eK+1V-2$7yv&Wf9OUf=vuK{aPa^dEfXM2cPa=>w0QFz zQNmcYbW4}nn_^Otat;9(jDZ##)MFCWz3A$5 z!&39pOql&;vbTG}KM!eUlqJUQT;-lAbI*Wvm7S-^q<+*woFOk>4D<5gx9Xg!wC)zg zUI62x$NmIbC*7hF66t0$r?I$)S>;Ub^3pHDQH))F^WKCC9UW+w?^qw-`WlMhpr0p; z8dl@>)-3+_0Ark&JA3RTVjIyCapyE*$$=Iqz~WR+ssaap{>M=F;q)&23%_k4x|c0B zmV8U6KJ~oR^<*+{;)_1GJ;*z59GR($Nr4-!i^ST~)_T7-^5?XDwa(k#uLw4LI3CKJ zP*rr=ypqoM@SK!2JjK(4m%<&W&Sm7oG`TV$o^}9JE_#X?I1E6h{j~B+;nsh(6DCed z2)AXOU@`DCC{H=(#kv4He`4da;R!0!1qdEdqTrbm+mUPsv%pDw^7%+30IP=+pgt!91UB?sh*o8K;{!ZY zNV=O-2pd3g@Sp^2Tgu)#EYA9vBRX-eKYv>WG>2?siM14!bkP{W9G6gr>K~2UHcKxc zLHodaOFH`)T8EI&%`;{8MS*^DYG^{8@ldsE`}QrhU^7=cwl|%bwF=V+J1GW;_$!fY z0L}DL>{}M5eS>*-C7n}Il}C5rLJOXmu{|J6`0@lXN#9^VrS%ESZX%&wc<5k@88ebG z|Aea>f=fS{k@7DX4l@|-R~0e8o{*}sZG&iR8_(5X^~o9M85M>sjY(&11*&+wNGwNt z;s~8_(e0cejv|Ka_?-B|d);)hQMCGrOoHd=1(a|S)QPipmf`8GIP}VhB@XWYj8x{V zB+4{j34Skl+iUf=LcT?GNGRWlfrj$NXSlySlT5yP9cf5-32bso(@tBFKBTYDDQIN}3?=~z8q`Qz zxH~lgds2!{b`&rbXhie7*n@>9%4S^?LJUcA3(^)dM&PP%J(zTQopozCk@|Wc5XNrc zrR2coPm5Y7WNMoKv_j0IWVbiHN4} zI@Kglovf52DQCwh;z)A(IvAt<(V^Y??z*9m$2-iVRt;5u}!$|G}V$yv`KXw5A`*3#V+%7D(L) zd4CrjWS=^4;YtM>Y6D9XE^h7!C#flnFAP_FJtfjPS0`+u)M2gQ1|tprRG`MElQo?D zdu*&kut`WBj%WNHW_#qyg@ZtTKX`sXe2KG(IVU^GVGp$xC($ylt)4!6L!8%1=o#ne z6iqlW%$TZm!Eh#F%{1>CJAAs%92J8c`)z7(nri>5i`<4j(tq?UX+fWO63Q?ed-dlY z;J8;FPTGglasKLd{7>lChuR~HCqQPhFkvdNlGtq7_YMorx$k%Ch4LsN_8LH$~4H{__DI6R-LLx^fnez+~DZ zy?bqkVw|5i6q1vAXG=#fi`nD*_j?6}{}Ue=Mz2KF!y=IDoy<7bQQvj+jZDJsT1T8` z9}5LjY>E5FlXKosNo^8!Jm}l#7YlvrZ#Z!uEBqREoHaX6dLFb9o>({SH^)pnpk!dB z+~jZu5JBkn90@C!Tmy&ReFJpb1NV2%%PeC!ceb{W`Sm0$v_h-+oFVap$gIH`Wiz72 zji`}Zfd9oM#V~R>9YyYq2M7_=clr7?z%G3X7q9pml>@SOEq*pOM(pjluf zqDurS?Pj<~skmeeEcuml-XLtKZuQ7DYi?OFs1(4sR^dklURaVc`KtWecEq8hx(B$% zI~3bdrYJYyl-vCNp=YIqfHEoK5CygB$o=s9ejdhV@kEsg8e@e&l2)SZgx<3tS~rqP z7~^a^83ayQ80pxM3*AVoQpG5^*!KL6TaYDNaenSDmN5fcE zPEA^ghmdALk7q$8>(NJ=>lfd;c1=HBNkM_9(@9#c4)n*riLTpQlGNa)Q}_dtSHo$w zQ+6=r^*R`0aa!RD%pQcPT(039TZV4wc=_D90|`$+bPIgrLu6wmMa8aHe$sLamrMe) zPsW>b9m1L-CqOWQ^nDs$V=51x5BOD;gHzH*3(mirP~QsQ7;ETAx+K^`sOM2+XN(Lx zf-qbFJe!2~1wyM;m?mLZWE5G>wi&qoW zHMHVTN2k_$@a$P}IUT8YKU%`@>chcHl6#z{zc#SWwzS+gDT&}jp-#i4z$t`+688a* zh-vjrjtXY!_n!M6PWmuv8 zF1^_72pTnHB29WzEGv*ki6%?3J&qi3A-x1GoNpDAyWezWPajMV!_n_AJ`Ed88f zMcM>8*$S7#EA&kR0G2)A)#MuIM4Iu2?+?LkF~tO18dX5Hg(d)K$(;k#mS{&zJR^fV zGIj;LT>|hl>VE}9>zED%SD2`2Tb_@3T06(o4-VR6MLW{) zGdv8lEk_#(f}BNHZ;wY`mCf6idaywI8d*_MK^1#ig`rFwFHyJu{K?8_&~4kDE`mkiKS4nG?>+GVaEp<0nZ8GcFW=?#vUkuV5KYbhs6E6kBm%q z9F%VKv^=Nab3Rr~XHt+?Ltq9BTsmvDQxL#%awfS45AprYDd>(^@auh<{_v|w*#F?K zJw^j6FtsH5yaU>G->gNWO{Yv-TLxO^a;1rb(~PLx2$9s8bB2ZMAx2o~F}!4BojxtQ z&S!E+zKfU&U#viCcX4QJzY7{ZH7=0Qkva!MabVH zy-E7FgcvIF*xs8Mh&46C&~0A!PL`D(A`_%K{liIneSl!EY5JgHhz774sC3h{nWkKt zco@%L0S5%I)SZf!#MG!6Sw19Dnvk$O76jY?d?!H{gdQ%p+fz`-K$<&AL-SYwufQ9t zqD6(MK&UiYWHVAZrraQD@I>d%&7VT*f(7O9EIM*!1Z~oP9xs~`HcE8+-#Z~7k05Xh zYGeT2B>A9GAakwdrOz*EiF0P3v=nhd0W55cjLo3PdCS{74BMv#9eMCj9q7(QK{y|L zSCCV=uP-z(a$Me(B_`LqJ>fbYoe7vu4k$N9M_mA;AdrNlbwn2s22&i0_e%}w`049l zO|~xT@jXy|uC9HnZA4@I%??O!?`X3@i<~M5*qm+06DA2d@sq)d@ODsw$CS0l-1K*AxgIcAHgngn*PLnjhiN}3&se##-#$0&{tzz7^|^1TiIb^%}(JOuLae24tE`g&?l zQ@aavH)go~q9YKF9t)}cmCLsAA-$otGY%&{bmhs-8)P$LiyVBke_deA`K%6WQu2KN z)<2<+PBX?gBO6eFn@3nLB);ZZvt~Q_%}m-N#YO?8VDyF%SqSu|5_FAj-P3Y9i8)oV z_bgK7NjnJ4S|dUjjL9y@E{~JJ&>=eh%iehGg+%Et*lav6bt>&|AI&46KfYBFs@%P)* zFud*Or(N5CrhPZM!7Wg7^6xvA2%0Pm3#Cye>Q=czxkmPi`!N>xQZw*eFmm2u>Tg=A z6cl=xVO2J7>t~e^n}XMP^U@@9Ez>lbme3wBR}MMct2(_$;jSX}m!C>6*&6v{BxF43 zN-hE+;o{8HDayqC$Mg}NxpCl>@*_k zr3Kib&K^y6XBr|=cu#hJ!00w?-u%8To%&bkoIE7jY)H{bApJ|`WCMVQx$b=gkq8Hy z-jf{gQJ~vs9O!%~#72K3G%{r`Qw%luC!%F5LPZ6V2x4|FgdH)NuRWf5dmr|`4VG9l zj(dtpWK547Z&YE04ipkbej2_3{w+DzS0)gr@r{pQQR+^JXb9B-GOJlm{XXivq60{X z0kQ;Zq?cr9Xkg$v+>JB7Xqkk=lk&g@knhUSrl_}VyRh7%7)dP|0ni*15*+}g4nt43 zD3x>LM&;xx+k4%;o?n>jg3pHtqRl>jza0^p7q4>}VWq|Eg zieME9x_41qA9RN%Kwgc9#L)~U^42q3Xl{?;6A2`#>?TqPae1K#Gt`%`Tz@cxd_Rp*wnt2bG7b|Og7Vzim5fJbrw3i)|a& z;uy~wEg@+I%U@HFeaf~$n+p(I%+whbuUDl~EEb0eL=vl{4nwd3nI+%Fmj*hHSQ<35 zDIsR2<1c1NZYPN2`pQULl~z}vK}{BZUycvP5(5X$1C@z5Ssc?wz{0W|up)9eaDtwM z7(vub$(v}FHmWh69v=_`E6oSNwIK#_g)ymB5UjQ802{?>%De~w8FLa%(5wJQGh#Z2 zJUv#>SEp&IOc65w^ugSzUc+GKsH~$7xolNrl?kA3I8CNU(I_i2JHFc6UYT6f)*b@$ zbHLxl8s*o?wSrko@a(QAI=m*DEc&XrERH5{UXygCIi}eghw@ z%B3(?-hvTvVAsR1H@3V9ydzAmP16j6z|X??B^)VBBL&~+O9@P5J~Ngg3C85mQea=` z)sl&&L|5^SGyJbkfJ6!2R;_KczF1BPIciXMvKgfHZeEre?4InK?ndndQ# zbh;9Be5tl=&E1-hH%=tPGnV5cZV*M`3bG*}TzDGI;MGBNz5;TlDoiOR7j)ClXynVB zd2ceaXqgyXO| zBJ@y`b+=+7Jou;YF{QT@MhuP}il*+)f&6 zuzjd=n0WY&&|9~QM5`%*^l$W`BkF`ACN@%c8V68RFX#ni?TP*Mq0R{;3)|%&{AXqh z6Hbd7-}+ucxJoOE4kOJ-U~?9AXA`~4&_kJ9YP$t8qo1!nv0TsdH~gR2hvm3WyzS1M z^+cH0g>Ro>JG3X1h7(Z}+KVNQCvfUlZsX^V$rfgD3jN}D19|6d?}m&;;SWqtCn%Yu zLA46gf;8p&7au8=m}nb5=Bd1B7-o0jpb$Cxp|j&!CauW5g7@|l{|D4}VipSvGhL_i z`ksxxYut6Yi-tZEsl+`fNT3{MZd^F6o?;_-6j7@r90=eP#p9N!cI@YU3}V`@OXZ#( z9;cRz`4=X{sQM%s>tD~k5cwXWaRW2%gt7f5_F_A-lX`P*876CRa zJ<-ZUtW%cLC!Vj|k9g11;eY`~8*K|q5DoW*CgNk#HD~qMZma6F$o4AYeb09ywpd|g z@oiCb;j)J0Ww#%w3W=UoGNjHj9`OU1JU&(sS9R$HgDbu5HL(EWm!#4_35z;p*SC)O zV@p0u)VAKX0-8wq>%Dhuu!qPaTefKJV>qfk*gi6>vB2qRS)Bx@QvLCdUd-un-xlZy zeBU*%zqhF0<+Z^G%eW3(nhBS}i-Zjk-R7Ass!y3-7KS@yi;6NWw)~HJ2Zx!5J6EcN z?(;-Y3chuV%0kP6 zalrc5zCstUhh@rPE7er$oU#9d!*e!xqZcQF1QrR5;E+DmOlT^%z^SL}cp*-slq+J^ zjbA#d&$dGp`Z-dVV*?H|bc%jC@KwAdqdJ(-!Kh9r^t7bjGd&*L&||O^JFfA}<7&h> zzb8u!J`3;`X)9Y?yomwnUqH8{UdNMwss~|VJi?{UYD(x)d!+S}fEK6y;VVn{G`~SB zhs->=gQf{~RHMfDWRB^wr~as`enhqKY9%Sc0J-cb(qm*1DQd#(o3^ap7QCiJjq9yX z-Cy6l#{18nG5z(<8{`gZTeP?2iA!2~k($(VeG+gbmWA)zL9Ogx)i6$YcEz~Ruc@?Q z5WAVYpVaFB<&Y|KRl5|ZgVw-0wCcd1r^zzLI=gOY9pjV-EBa@%$c)@LMehr5wZA7w zw>}Xv_OiGa=W*yS?KN|bI#<2xD5VID6^yF6WYV~I1V`E z>0Cg8`GCBo@@>7zdl}jVZts>N|LsJ?&P!#q_e@^#eo0G@cgwQC*s54Ejk|G2E_udy z+ds>jAtA&f=hsaf&c~45(l9jog2d_ruC)(s=SpXsw@9%F4+or6x2GoAcNkLkFxvAsrTm^;Ku}^hCLTKd@&B@)2Uxq>1K}pw{P6_ueR8?0 zdzLOXFz%!KWp!}>kVvq)h}cRGe@NrPaRkCg{~D%8p>8C@q^_+`upRnm5+j@EO;)jHz z6qs3Z3oJP37(-=oh>P(Vv?X15Ah);}Z$2PCnu@}%5}~n5B2|TLF#s@31N_9!_@A*} z6kko!rSoN4-mMd6ku0jxI|NFEq61ZktV!)QQkJ3Rf2wQ%DqyO=;1zCy)gCTRE z&V6Py9?B!MN7O-|u}q1jnN+rCXQser6bNS3OV4T1ysZ7OY=y_Bi;ke=`u&z( zCo^Nl&%bcwVpG%sCRREe985j>%%NlR<*Zhf@xXOp1(L1LPyUb5rklqleB8a^mCgNj z={?oos`dG z)5wqsEDi>)NXS=B83=)pS-NN~n7yU9%Od1pLV*{%6U z73Mum+3Y%QGLaAYime|idyv1)pL4K4u<8L_G#-&}0T zz}M}0sCEt$hM|1ggZJ>S>-WHvLsiH+SY5g{@o|rgrtm^Esoxpx+KR*)aco66*`YS1nQ~ zxA6SvdED^b>%@+Vcr|vP`j1=Hu6pqlMR>i;oB#T>|L96B4p3oejK@EC^5h=cC7WHD zDY!L2?^*Oq!IJqAjDNmJu_=nw#yb&A|CM;f##H&#?HT=5Ho;)lV!qofOJ(BtxUTc5iGCzPs9YN6n$z1_Rlt z7wRig6E@_Pj=r(9gIu5XAV4>ZA#Du}pS(S8L&E$!xE3?Tg(gWB%v#a_sFwch-WVHP zfdkdI==&L7R>3HtISZNiOnfEPPCUBW9QV>nk9qWEPj2?MEjp7C9sZatmqFPzM*Zy_ ze~~fcqWAQ55BV6(%@NjWwi+YYJfJVlr^=P2Id&Br!Ol*Nju#*5xfE$?{>S4kk{sD*P_1CAAk={^|J?U#Hhc zFa)d1%3k4P9({a`Gx7M64>D}<+Md%iVjF+-b>aWKivq{OrO7JV(SL*vZn|jFACWUU zeL*Ga(Af7J#)n0PN3q_r)gJ#|pyK3vys6|A9s8|8T(HJj?#N?iX02t~ua0b9cuDpS z_w=6pC(eel+0A_2W6jZD5#<*RU4Oq^T?>@$DvqgS+jIXW2K6r0-p{)B0&`yd%9a1M z`0^F9jNV?hOb2Af_8qnMz0YcJhP1l7R~e{p_G$fPdr8{jz##^uM1+&aH((z$Hk&@^D7ohxf+oOy7I=I)_Sw^dh2mXP&P; zK8D_Y(}-JMXw#A#1X?${P*0(PjHh zHFjQ3b8|Dhp`CZdnu{wI!ja~5%Bx&L78)c|~{c9bd<~C8j&#rqACN)NQqI zUu@NPrekYMdM+_B2`Bv_YqDLX?}D`6-F^4^n{fa{#nq-;=QbMr7xI|xdQY{l=2$1A zkKPCp<~*FhYuOEfDa7ISimT0NRWR7jo;zQ8^6l7%Z;0pUR4*84l0N*YJmU^s#4@!2 zN#@9p@q&4`M=vDH6ra7AspH*M7ooy!b(ir4I&KCIasZ|*bMl_ix+Byo2f>hDD=pdG ztipSJcQ!2*`;vRr%vdRsUBL@-!X&#NRe-mZp)h@OKC{QiKQt6KC+-5fsg9lCF>GUJ zAMVy05r+BaO0bQtwmUV5Sz^*;j#l-rJKG>+HAEgFa7?AY?Nd*^-blAzf(}0;(`^=? zLgs`q0#9E_x%1<^{=!95}5}aB(uh9ZcdQ#fjeRC%v#TEjD?ap z?$gba9KFh}2fFFbwX0qoiSM?;!uq%1#;YS?u8Dzj^D`(c_-fsVGpLLi-_Xyx`&FXM z@7~_f*xalwUG%5`q76g>t3NPn{!Mx>ifru*-T1jzYp>X`XU^eWJFG4`MkasAi(9nw zTQK+Nr~Z?wyV8TQ;XeMUjUm{Nbp^9><}Pm*{j8-LIlko^yDK@TIV&4o;A2)h@o_wS z>u(Kay=HU!yN`^(`0~bRzWh|Q4aY(Qn=%4Y*}+2$d&DfpoEUyRJmH0UDF-TKLrnY-fP!`#eprXn>8+j5CN`sH=E?)*jAG009agb4@ZuRwp7euS zR7Ztub?D){4F0?*br~>VTW236{y7)%$baYWl3a+}|91O%Y1w)eUc=rL!k{*=Vo%DDGsx?k>yz&Ou5?I;p|u$mXRG;9tdM z3Q1zqH{5q(J^2sIlH93A>rC29%^nqv$Mac9%7dW5tVUU9B`a&1e;aED4G~MY?c{;g zbr|3a=1MNMv9akW;EBx!>|+STyaR0a-|FFyS^cS;8jVFBz zvm?$YEveP^U(Q&*_?Oe#Gy@)|0`Q#kw{>9~z{6jV(~LFv+ADW#7iLM9BiMnMvP)JL zPyN67qFg*yN&d!g+|uw_v^$o2N0|D#@hzFIyWC^mvaV03WpT{p7x;B|#}2V&K!3$! zHLYpowkRQwISWnNqpNEbrAEKesmSsRhK@<1iU_x-z86@`I!wnC3(q0zyQ>?Qo}YP% zRnST0lcTBiA5s3b!?*ykU-@g-8us>F0xo!a-FWX4X2shrt@PEZ7j14b$`YqUBK4 zI)v)8$r)WxiW3$c3srFlK+Zi0fmbkaW)r*ZiTl4Hoy-AMik0dpG}kzNeSP0&lz;fZ zY*qi}5~{NtOi?+Eu5YxCA)pGcBA1>7k%9S}t{KhAIW%|tU6Fh|-in^Z6)a<(>fxT` zJaqdIS7osR2fuMjT?)kC`?R%Jl$MqX0NUi@;qgpOO@;jWFt0{D$FgP1*x5Coy~JfP zEED3wR*1nXUUzQ&?@aJ$9@mo8<`G(dhl{(lG(~*$fer1Bou9+bt|Zp>=GqsSn|s+wAe7eLJ?wR z;s+QQR%?szS|2~fO<`vb2@YNZhqL7r z#F1^~p+$6xZvAl^-vl+qrl^N$JOpivCmrRa$|{QN=buE+~^m;a7r%>liODF zaLUrluNgg}LH4GrzqZ2}EYzU(hzM-(w9#aCH0ESUL-g(N;pwmECVAx_n#Xb_I~+Av zk!`K^7^951X+t|95i$C(?%4_Lblzd*^@lW}AH-dd7Xlk>`2rh!IrREY2NoGAgm$~N zBGE>bkeVbpU*zPm`QtAyP~gH`%VZqu?(XhKHv0%XI6k1A)M_q)sBpBA*|Q@3e8^ag z)qd>EUVNV-e0VS-d^BY-e^S{^D&HTf@*3LZ9oiB&-6584NkPwr`E{rpI#vE~ucq(Q zm5IX+KhiA{WVc*Rk}E*>VC6*oUhztSYYPs?D9Kq}7Jq{0qr|b)Bjqb@Ro+1(3`rlm zdpuxj(uEXQEgsr(WSREZM=EaM5LT)P# zfUe57McK7qwZKXG#}QFg-WQ8K=&4NkwiC-7+oC15dfVz^ky-QkuGGGe{p2{7f1e6g zyNHVRl5GF#2h*K*ZAZ>`C#*(h-&lpg-_?)8Wi_3oGM0PWP@s%GQz3nsxC<99locU; zJ{>jVpU?<~29MIlv8OxhA4RSn#DzqcRiI6*;yH81MEgS9BAc;mojZL!GZ{&<8}=?W z0z6r0CA{h90X!P3F(zP&%B}Wkpt4myY5@I$Sx(M><5nNVT;%YEpwe4{HJXRX*qZGt zT?adK_!c#WDpqpFRHh30`f7!QpUgVa_2<&JSMP{}HyeU|zP;4w=@N$8CMvqGx;JC1 zsnt|}!5DVbczmziZ7OYz7~YQl7Ta=%y{0=+Q~Wg5>lGV_jK<7muIk^pTkC&CPQ8A5 zAbYmfAJ$xPr-K|3Is!tM0O3VZNNa^nyoGtbS{l$E%H^0!OZ-jS}gJ?AOHV z3Q7Wbl0wHz#v&}73IO|``%j}~izkJyWT97fyQop5I0V2#=ZQ6gtUJfd; zlB#BC`b%s+fo-`&*FJbx{Xd&|__!}mzj!Lw&#?fC!f{R!j*@<&4 zs&JOIu2El5SFn2Y7_r|$DX>CP;QDUsf%1UUZiC1y{Qefg5|7mEJXSjWe6BM;9)4I% z6z@VqL*4E9W-k{olx0{gDc&MHO-JxAa`6L|h#5|)fo@$KIf%HMT)?@@Tj28ZzY36P zn3!wk z813ZZR9}MdublDvzp2s3H&${U9T=1z{mqK|dKV1MArW4)3W*FWeC36Ng>#T35C6tr z0qybnSC{F)F(2Q`lLbZZ;2OWr}m~ z90wYfPG@KKbl(Od%YC@lBNz-KkMu(R{rwpS>$RP?Z|}4;U~fzJC0J+F8qMvDVb(dK z5L`jG1=-SOpUb;eI|~)DJkL6ya?RcHdlP^ORi3f6ZOm|2g_k`fEiJ9P6M*X7ygmB* zub)hm(OuW3vUmX64eqv5RaK>E5WRUvSVgnum^`^ykirF0*KCMi*soVO-+2c%)h}%( z25mx{#>*AF4U@;Zo3DOnb4tGJ7hyL99k-A7-+!K1E=@vhJ$_P2+^@N4IPyDBs&-|& z{VuvvrKW{qU(YT2rM;9v+lLUF%B5G{^8LNkAS!X*F3J3zGvpAPks(Qq9!y?$r?lEe z{7;22(jPl=d|tlW=g0y^dNl498#2Y(IJRBI&bYYJR^1P?_Bx&~aM|P~{wtsN&_C9@jQYN) zr&?(tW+aY<`1n}tCV0!9^sbTTSSSD}pH3({E3$I*34XCq`kBN7L97 zoz>M0jaZ4&YrB7?7c5m1*w*&DnNMsSFPnbmWujYJcmeD9>)8fyn(G37*(1h3_70r0 zbKKHc)dDh9%~}dWD>5R-!M@@eM2ECLFwn9p!_hr(L}OpZ+=_d^`g0p3dsE6BbTmC~S06m~WD}?~$2`A}{e57Idw|j1RxS>8#3hQ~ltp7fHYvOMMlU>L z5&q^eR{eObWrPer^3kJ9=(S~Z_XiF)1*QtuNR4z#CA;Wilyn-1vsJze8Z!z!y0-T= z^NlCiG04%WKN=nA>FHVbw9c!gjO+%rF+2LtPDCT###AzBfcy3MdAb~*q!nifIx4RB zE|lRQx1bz?Op*Shr5h}f;^;oNAIrS~1dby=9}nV;-TV<&Ff9!p9v)F|L{QC_6(uGn zYWO5X;>x15Q#O;cqB?A_(A-}KDix)Q9jWE%{9d4r4M~de-6SC3@KuS$NS=fcMk_mcPBzv{N{#?!GX*X$IC0lw(zHX zyYUoBPtP*c*+N^kM{QhW_@kts8NP8Qy97B+4IzAS; z*k+!60h(=gWpz)|?=Q2Y?g}t^x>Nn*1TS7MI&lcOBXqj zqaq?Ay3sVhM>SrR?+N@J2UbN{3-A{k(~QD?6pX%GZgKPSUhb&N%fpby=N{%6h{7<#I!SAJ=!MCGqJX>ftK+CHr0} z+nV_I_xIaZR#q=wAo*Lrz#c@_6}A-ymM~SbYn%gX!P9=QeU8KG;*XXav6j}5zs3A; zq_Vh~Z#;H|p{hRNRJZ*8g9i_U)A~>q)kODMJpGEylMdVW@|^hgNU@PKo2l>;QHl_PuHpUj&2F+96um<=0Jt6V+*DEoaZ3ecBumNhI|+ z>!$Z~qbdU&$b ze!Hpcb!78LhodCb=rxVmwdynnCU__91~9psqZa6!Vx>>1>Ut$VR;m7XR}Hb$I?~&n zHx_~%#fmJuKoBHrY?RJG>PaP;Pvi#`5g8*+$8p6P0 z%XUD>-yuLb94#ZLH>%wC6}T?TVRdtbQZ51ell7f9d$_P1F$HuDgt1R=bSx`YAdF9HL0d3FW=y>)ay(N+UV;S#aq=T((CtMw=+y^kc7U#LSxw)TB znMTK%1x{>0%`(gF8i1X_@5>{hP?YDmsp(onyV;0)oH>9&toFIc%ECTMWZ(Y%?;qOD znLD=*H&}7$0KQ0OQ7^_*+_`1GXd`Bx1*+tXhdwEe%@guo%Qf{Ki3PeE%9GfO}o zQR{0O^U>IO_3x}%hA{MrF2_hHL+>3*{ed_(Q1m2?C7>D~fMvV814I(UPDc;nH*W62 z(Txi}`Bg$$#i~2Jo{A#Uz+fB@l_ZbM~ddf|l=OiY-fXNH*{ zNxGm9i(=q=aZ_)TYs6-2N-tZIKK)Uuy0tbgqMjlA4hCC|r92_%;V1KP+}cyfJT7O> z2RG+;(a2ztdi)XjL1evN;F2~B@L0csRKWL?1a4`1z}_A-*TS^zgr}H|I1Tjmped)= zw7aEQ6UDn_{+f%{$NB!bV2Qr2yZf@S;`Z7){}bz)*Ug9h1A^^xc6N$!9h9SkenSLU z0kew7upBE#XH{5rm;oQt>5vN1{TDM}7JaTN`> zFh9;MNDwF`^DZZK)c!J2Adk<4W)0;PzYk%R4ChMvVb*-|#NOyJfTp$4Dp*|$E2K@( z_?%773D<$jyJB`01rqKe?gY$jC@XGkK^JaUl+Z z(_5XH@o`G2cAtt;q+s1q^F86!|HIx__(i#WUys+T+-ra+iYQ?NdO_(9FDN2mAYDpG z3?b4CWfCHwQc4OEN_T@wjDRu-(lH8>LpXHv?(+cmz29H_1MlbM^D$@UnJ3QKXYaMv zUZ=x)tNbctyZ$H5WHyTvBd@C|jIa$RmKK5vrdd=?PmgsC7XMPIbxDD#7>o)wx86q)WLQQB6ihCbWeN>AU3M=W6)Yj%X{sMc^%g?oNG=k@7H( z`ChjHK2Hx)B~1j6($hm!RYB>P-$W0T$J8&@?tI6N!Cr!_u*wXpp8pJJR?|F;=qO%6 z_@DU^_41Bkk;k+wukLo2#o_fE?eV$Me1|D-w9j;l>ds(nBlHe&mu5pc;WQ{lZXe4X zF00-3x-BnKDbv!#aM$4*5q;~R*pT(2EQbShx)EWg2qq;ZC3G&oWdIL!^ay?e#YY$v zWFJ@!!5kdl1wEM5vul%QC3GDUvC4aBhW42kM>%GD5FKJ7oYk(52@h+Sow8dCA%Gg zok`wtm+GsiytwNIDXD|``c(M%S9J)TuOic(F9$ZUD>tZ;kbQ#dbR64Lxb8)Y@0m1j zRDpr=H?#m6J&qepXtR$~udGd9q0c0ARv}e@gqEYpG7N{O@cLQfr4JBWIf1#=zX0Ph zbWm*hSA|NNJ~)(fmy%Z(fM3fYLAS7oudg2dTuOR+0O;hWKWXh^u6neM$3?@egOI#& zCydSmNQv@vSr|%`5Fhkii1vB{O;ZleX)4Mh@mTWO2Ub*{Zok!?1ScjUq<6Dm=trQm z@erB!A(M10RUfcm`%40_7G0M$P?nJ?t^}x@h(M@kk?|12&cJT%J#TXlYVby$$-Ax4 zN=n5*E6-UVty>xz_8^o7@73ldwfqC+ZrS817RXq6j6ManP-c%_P5Vo?h#%c|L{3xM z=;fH{k##cU$*$#tk%E?5wKK4_<;yItzA|wzDJDc(ThFlqN~xSa1shRT0jZ&tP6Q|K zCD~)gY&k&hgBBT2iy=q?Q5glCk*NgVC_Iay7T0atwjo@;g|Z^DGr3p~SYM|)?Vi2h z!*0zI&Tn>I9L#5!NWVr>;7$k8rU4zI?4^nrN1#XJXGDQkM@v4o-T`RuP*gul0^vhp zqh2g@C^7)KAay{fTx316 z!F!MnB=+6A(`WDzV2{n&ig?h(9^`4Ml6O}e==WG4srsg%rOTnV&;XdA;f#|8qENe` zlr|_6EF>M?U0f{x{15pYvjh(SWz61f-MQcLD$Zzw^<9>v$te`6VVfJie&%dMIf1Ru zhm?nI1eCx)UaXalT4N_{!#{yBQnh;dVslU?N+1btCLjf+);pGE&xpdx4`wXcq(zi< zX~3wb%s|UtFPF7*og~@tCcqRw6f-*(P0i#tSu2%>tcB@=3AAe>kC$T8c>t-dO;s{Eb>7n{<$yANy`#r;8OSm5)NMDQOMzq z5ZJJhezOopGkKuYFO{5v4JbumSCmy~D+6djeBtK{8wmW>C^rw|F}z5zMdff``6{Ix zDQ4uI*OK*D%waZ7WQTwgB=71Nd6f(9HCv9qE;yl6A;}=JLnLbvJ$(2ubOw%lqe>|g zWb`d2Ku5MBQn`d=e^B`JnZ{_&1DS-KRbMN3$?M#i+b~Q4sJ}WEouOGQ6H1Vzw6vay z1$Q`we5;oz6^2B1!r^Ho4@lH$4fW?gGS1@DP$S^Uw#$PcK6nbukO-8ea~F6E33Y`4 zlT-nN#|i+!O=cK0us8A|f-)#8601{EYkc*vSSYtnL5?qm(l3u%iE^UnbsPlI86U9umM4QlA9b*4#9LO7+10%CrQ3?eMhr>ueyN`?Lu_9WHfP zs)Wnu2C%BVqkRSjM~3#4Ti;=_>56OHEw7nRBF3dv5XlZA9pNt4h$5D5(Af4u>##5C z_mVj>3xJX+*vTw%r}9DInTL{r%_PF+VyzQ|M-+gfplYVQ*H-DX4=Y(g^^l@~|GIW^ zuMdKt48DXcZcyIR9!TEOL3;sf8rD3{a69M-@zVwl?Y!A9>K&6+LDf z=%traRaJ!+XHrqfWLB4ww3BRx>hsfeSh}ciPyFaKv#RT~i;?fP;vy7=Q6su4O3QGNLpO35# z*tMfv+0{KY;xOQ@2(cB>q(JXN%b(Tm8}@4rDxhlv$rN0WB(x?UKy{n;gY@Xt=|pN% z^eTUUA(TWWk@jj{nck*jdzSzKr)Si)yH;;bQs02VYeOYVGa?=kv=V zQ|2U1Lm%1LnbxSxyt!izUIzLLwdz>;TUPd4!=HeM3tDvUFMjpPc7xDak&phPYR8Tp z7(xa*^lEMLVvFCP-PoVRSr|T;!UYhDKECJ$#kAUOvns|H7L}oLu+J^NyRp6Eic&gR zGBXur2Z+?bN)x02K^y`!7Azpt3L%P*cQNe+V-{bSY1mXnS6K36xAiB-c4_vfXReTe z-My9_FG(wT^U?;p8zVIgIwDq30bJA7Wr+qZU{ZHS;05FO3)dtIgv}|35osMSNw**4 zq#N|7eLxF(?>Ly;;LSIZjnQ%d>jn7vr$3dTj5px(PCI?o$F^1Gd840gPOc-5?2sNI z&Y+iVZCdGPXgH+bvD0T0&n|Ud+@N%X6A1v2e}JkDN#X(Rnhr_F4QW&9xQNtK<^-XL1GN~-2#jk2X_4s4IOu0E%ty!R$0ID_Qa?Osa*V|-gu=L9N`pK$n`eXB zhMNM|1|T<1C?K5JOHYrmD@?*w3?;WD*$oOa6l?<^pcdK#D1~UruRPmJH4?7=!z*KV z5o!UlFSJe*p`TIvL8@-CLlyM@vQD_P7GEza??5i<*nv>ldUDTFgO?-&IR6?kJO6xE zxN=N7@hOwqp>00Xb1iKftFCJB(SKx& zOUa~1a29AH1-OkG0^YQj{%cWuZ`93E=u-Daxf4nCx1}>65dFV%AR4N`+?fZo+9CDKo314or3b+t;wfa~bPL>UL-!Q{4XHFATo)2#umLvH($;p6 zZK;je+dbj#2+Yn~d)J1T^OzJAg>M9nq6G`(IrEUhGJ&i%bwEjPA@A+1V8rs^2C)_3oge;3AJFA1~ zE{l~_Z^-dRy&y7PkcF@ShJevY!0ZC-jA(xfMdwE{fLy zvGd~CKFXn2hIluXe_n!Iqf{*cE!EANqO+JOAa2tIpcZF1@P=$K^Ju2ia)UHV+}~e} z^HfA(^k-miTSn>Yiqf_D_WSb_gy^Sn?;aDsQTOxMYN;|yI%u%ITW>8m3wou}Mj3^c zs;9OgiMuOjlTxkyh#{8Pufqc+dz#q&{pxNrqb8R6Tk-2l?KTiTskiI2?olV{F=sQT zj!za(sag3t=Ue)s{+L&`iIv`2&sme!B~fLcUcwnumtUpYw9MGfo!Nt@ls!uBOQ8%_ zKhM?9zQKeY)D5*~?2ppV7T{j~urJe!l9uv0AAT?Rp1}UQA@&wo*jZ#~n_#OzWs3IT z1xq(WwZT+4+b7m98N(^O(F&Lj;{AJQ`j{LaX@8R4a^x=XZf4w zNII)lR$8Zt_gJ0VPIV$&`^?a~P-oie0yG~4KpAr3+Si^sO`v>hXAs^7lJTagt)(Qm zM}xN`!5H!9;DUpXYJl=S+t>_Ntc;?H_**`22xG_0>FDYvf6 zbLNKjgNmT|L8ea0)@jx!YL+!)E@xW@>^6=Q07DkKTr+O&+@1tWKdHsy%kV4Gw{Lk4 z5x=2tI+yWN0E3PsLun>WA+>;q{CW6Q?#xrrA2Q1*5R_j0g`JT|;~6lF*dA3qupBYf z+VJgps6CdHeL4$5bjGSj3+*2>-h3j@VsGAlOp0auq=Bi~wIDj~`r%jo51z!Oyg8SM z$G)>c49N)~SFyR#{3{~O*-q8@#NLJB^)|U9S@F;92RF-4$qFWl^);fQkq(!XTd*^Z zaRhCmv;D=HFkx$Rl2x3rt(I|wo2);^GZOkmyx|5_O3-%Xsdqa5Ca|7T*50uC&s^=55~p+(h+zH2G2qH-7>DihgE`C<^A zjr+O@8|rV;8-rWc>zj$fpW>;wwAUheV-|i*<&x|Ug1L51?)8l`Hp`XI3c2^83-i>g zhbbEkff7AkzcXyi)-K&iIVABCBN_i>?D58+z~R9U5l z3Y+my2p*DK%W(5BX@6&4Hs?Tj!Dj1ILpWTNB&ERTxWOg)TOEYWg@OwmDfqzt>KZTq z%LMEC>l?bAbXX?bI{Yet$(@Bjl$q!751X4017+hSDCa(rTL&}tVo%ovHP0ogs_ycy zsNSVSAZ@5KN1@Yfu|%=kz+;to=4)Hm^SB;V zS%hvG1qCxCl+OLSzL5(-pjy25JodiuLc=w)35nhf;T>4OPwz`qPn1%(sOhjo4RxDU zKqpKo_oGl4(bL=7J;8?8E+>e^`Ulk|g=yLN{u(QllFSzrY!;}@i+lfsyosT(F>L7JH4YR#4zs6}pr|%d0u#W7$~9$Rt1F{6h$*VEXcKKW zGg_rd*@L(4Zqc>*vhsvrWQD#3Hz@=6TZE2FvvrKGa<(5WO|MTzREQre#kz&!_t;FT5$%sDVIg0KZ(g&WUgfJ{@5J}|EZZevF9u4$7`yTk21q`I+k3Tl$gjKnb3tEjvN}wGJb@UP;5lN}Xddj8U(cMbkX#TyOYnWni z$42pxYDn(F`nj>7qo_$ZIHA}KiIPM8L2cb)K5&hAqWKa#gl7*LEY8PkFZV9)4Divu z&BEBk{&aNKb!{opqY{cG;feKmyj+G*@)_Yo$+}ceYVNXasAMpwizw zp6YHRsa{nXjj=g%<1QopPI(D}n=gYR$y5F5A+1Afgegj6kHV?rWlrESj>6)8>;&5( z0-N^rzBpA~x%KgzQX%X9g!wT(ye)WMq95$#IJm{DOA*KUde)-Jl5D81(fHTPgIscc z{z5FGr!g(A%~Z7drogmFfn$4vG60RqJvK%RCmP9XJr#8Z-7S%YT-crUw1R2+#%8|l z7WFK<;w6GLG!MFt-TOzSHV$FI=$8*4-IE^=d>ea;39IikrYA$X4q%D`||U~tsjMi?Y-VIqn}-*sL|-?`~Y$5{%;vNa(iN zG>gdHE?H}aOPksKMI>ciA-4i+T$>k@7u&%^R)?EF`hj+qZh)J#q4KkX=W>$`a#kJt zdirY`ftLtt)pB~?0=XbYUca_9IC;B|pz(Y%7cG2rtaPz@V7_Ck!seaTh2sys2`KNk zzsB=Mc&t>PE6Hc!tY!V?@k6feig|JRaZn|D&s~k`CH<1MLd5iB9K)WAD7v%TCd7Vx z7q**j$eBA(R&sJ&`^Jk+rWoAAr5PC*4^*m72=n%>P~7wR9r*eCH$M%R-f$t2#o=be zH3`wN(p+sv)U&^NZKKe-byI42PDD=j+Idh|hqF2#1-x1g+XOa+i+12F(S7xX#vU|b{N~DM-#<_4 zY!0gM#cs5Y{2= z2`Pv6DGMKk-iCnZXeGkw6an;XBN zqqxi{O#pYmPJby5l->bUfU7RSU=DcDLZOaQNjDtXjXW|g|F zwE4|7W9q#Z>{T#AC+?)bUVk9eQV(Yt*7Z!{$j4b95h`H+RR}FjdiTy32`16sQHND8 z5cJA_R%RJPx2Mg|XL1-~npbv}={GoM*J3SlZB?aI?-K#s{qFyZx-SnUg zp#qF0gw)(f<`UX^udG+9UYMVj&Q_f8e+j#zMG8`dW5uUmX+6Z6pt(u$H zLTqRsc>Vf=LgS=yNo?#f3{3_ZFtiFKbS(i5$&ApS<1a@G#H8)GHx2mOp! zPebD%aKF9)e_jB=QEe&k^_7TapRdzjZLG51=2r3Rd=>R=K5G6uI1@GXa@OfTq49J6qnQ6^r!{=J{-I6>!OggXPR<5^_G;+x zMQw*vFqS2v=GS@Py9SYjx}^p7jFMUQSR=26^3DQ_c<$sI^9hWN1u21LXS5`b%21nx;VInWN_Xnt4}Xz?O&Pq%JcpA%yL)kf^@T>hM!9<~=4#OyRRVkX=;G zvAQ4KSlx~=)ZLSz6*+;%9S6ryA$c5OxS8O#(?Wr9E6}q%AT%)(iB?yAH~ZQ)uKcd7 z*6b=^=zCkVt(BOH2VUW?i+*i%2Js&zv!%>{*AYT1J1Alep>E@&$7A1|ThX7&Uv|Dw zQjp2L@x8k&{?2rdU0Kj$TmDWXHq(RnS{Q-mrQktnDGp8LOwn*#iH6Xjo7xJEV*b$0 zWoMo`D}Z(kj&z}0jD0|NaBpad+4T;>?k`-_U^2H^sWNong51aA9p2kg?C{(nMXUhG z@IVo(J0GrTT|kCtE#=a1!0$i@pny8kEm)R7+ronGX_MqyIC=S0|00l@T_&L#us!-K zJ`?3{&CM6Ub}g~Tt84&GOZSCu@0^U6(rD*cHA)1z*B0)he>vsd#(!%mcc|A`_}+_h zV8ABm!OaDOYqBiReDQRbPt=gg00Duih3gp22%2`8hE35XLG@r%A91%7$PYhKUb!O@ zIJ-oG#u6J__CT77gF+?Joo(?IBwRNAowOvUHB@FmvZnGIV0#^l)lL@e)djcecT)2v zK0lr7nh$kys#%QU#`5_X{e8YOTS-!DuC+7ZncN;f*C|DIFLO=Zw(>!WMNBf`2JJ)( zGXmh~x@L`ZR;RXQ7N#9lSdYWI{({GK6I|grTWMJAu7>tluPuWV>L9c)kN(neb8{FG&){%vE-Ji+sxEJq7nihguaY6tjE6i=_h z6pc+9%I)iiA!PvkKT8w==u}{U^Yj=&y)*PRt~f^x#&oUyQpf9ZXZJ5H^OM$Dt@MTe z3F_(I09C}$Se`XG>aH#{x5Uo@1G|e=To^`Uh)unnA!^(JwNrdod3NJ1GWyrbAD^f!rpU$!6-YsxB|zDkcYw z6)I^I3S}yfcA5riM@EE#eiazkE)YfF;s(kzboQaaiu96yDq1^8nRs<`A%!CD44u@i zD<3wX#Jky;+mN)r$x@~JP4M33Y#kFT5{(0i-jaxIVg2engoH!TkK(?5!6o2Q^0{`g zp#`8*!=73t5lwuLuTrMnSe~%!XhuW|RxmWE|5>$oZ}!7=P>+Z9Tsie(X}|ASTjp^< zDWSwuPH!&Dvv`Dcb^}pmV$401*1nVA&{;`K;d-P^2`atSbKcdy#T%74NPCXd9?4xM~u;5tsH++*H2q#zj zq*jj`kCH)Yg!&zr$e>*)+Y8+bmZjX_#RL1L8ARie4T608>@NssKB)7gZwk2~Sg*}m za~gu4#`!XlX_{SysQ%NT^g*rdx>K68{tf7U+U^T9+h35Ui20q>%HimDfpnIoe!c@4 zETE;(505xoOo9#;q0nTBnvapbA1St~A$RHQkk10i<@{u5IkQl=ol5Nz(#W!asbbfl zBNM2cUB_*spCENisv$Dl*biITT*ovn5Ya(K4ys%g9?Qtf>hy#8x;b1Jr;7_Y+i1AU7&Qg41HVQ`gR) zKMzd@<=)V-cFwp2cv(nA=JMXtpAsfw8|J|UWi93=<#03Av!%U%MWwDU@ zq`b6*G_`_84{rNuh!1wvOeU2uix`pKco)<2LffCH7Z)|eAc7S3w;!7n1yQyyYVo2% ze#;A4W(c(q+?lW-37tMVFk|BCDAM|N>%hNpeM>Ul)c=h8f}QKM{{mVUO*SWU-j&fo z7)O0bJpkcEGS8xYFEq?t1EnN;#GSv9NM2JHkJLfXY#?B6ZjO8ykha?UxW6|DMn%vj zB+a>5ugmj18VMEveP*XcABLr)cDTs!acH$ree|KklJWNuX&7`gO}8>LIQTcu_LS8{ zMuE~gh~Itp!=upRACc}>Q}d#?Hk&r!eAOcn{ln;%3>B|GhCdn}yd%;Eh)LZw@MG$k z;kw+!#8<+e>K!`DvT5u%X4#kK>GwxA*<~vaa*rWO{o%q(t)}~fI zXV(T6oYuHJo&qeew6YHKEoLED>|7vJ-v7-2Pf^LT_N5_^NPR=^Cc|8=I!6fQo~C6J z`>W4Y<#b)&>YA&H>x`p9Zyr&7+C zwcjxL_%WnN)0MI1=g*auzjhu%Q`Y>f~)d7Ih&rVh0((Xb*`x92Qu#cjpXv_5R<8_yN6Xw_sx2`mO<$3ewPC9rW>#zZs z9Xek<5qc5v9;Y#E=Mykq(o!!V&As18=TCMY4GMF(TBR3lDSy#*?uRk;3fV(E#JVji z{5R}rPlC%_!xbfF)dS@v(@$*{!u1&+Z6lDMQ-@V^ZR*!nMcfx#L*YIQB>jyBmS-0_ z8^a5awowtdR4h4JE$s9&uvAi;+{PTe)XYbH&9=s+*Isr;Za=o}{d^3F2u|4Hqxbw+ ze_|!U|8P0UWNm(VQJ5lTi?H&$S231B+6vH%?cR{%Qgc0`4hiW^KTbM$y`G|nL{Fg&utk(eV0Ljd zPgjZRBgR-obT2z+?#N#ae8_wo=|JVl{FPJ%1Z0#1hxcyHmtYEAb3G4{YN%cv-4$Hh z8PF#*jJ%PN!xHA`;<=aeUz+f6#FZy%MCf$etHf2)RuX5EP8!F!asYp;$3hj6bz`vr zR-8zSLKF0*Ym#q-%a(c!dpj8wpWccmtXZ=QT38P`9LXah0v^S=f0Yt>0Atw&!tmB5 z`UQ~n+%Gdr*~r$nY;gG|XE5~m2nF6{WCxFeIz?B`ljhX<JEhW}JMx0D*iXu`l4rtddXwebvpXoc(+sZ~z@ncnc z^!d;p1L&It~Va4MbaBHqSrgWJd=Eqr32-F=)2I@1v4t zfjeH+(3aP|+(~!YomzBIQ*G~{QmUV0`RmjEX{W(P;aPJo(p6+Lrukn;&a2E8(2mP1 z%bd~s@1?u3y(PL2F$6T?YU24$5lcEPo{TzV4k1+R{H>fcR4yD*`WI6u%8dp#dj-Hb!NaCwDQdOw`u>hYf+r)QsMg zE?m>m*kdhepjfit&2z0dj>moF^GJb-yOQ`!?*R9gP{oVlivjArUp-3pg6|Y{HjY>& zReH-c;+&Zg(kl7Y^{<)Z7TpG@34gFB&v>5zlzOXi;LS*l>~ z4N8|_CNEbsYEUohdJS$0el&NkY*4$hh*#YZPw0k%qN>J~amxb2UH)q(d%w)2rXv-| z-tLJ6-?r|fEx}grLPCr_YZr2kyPK;n&%P7@eM<=$U;g8}uhZP&YGPq}x?hM9FObew z0rnpxfaYWQ8}ou2QbO6*BZFNUD>G-!KjnV0-;FrpK7eWF2>u(Qrcp3Ct0kksZDfzEV1@&gTVxarz-wtGJZ2w|=zf&5;t9k7G?zTbN7ALF9Kj^1n-}=W zwDMnJkdN`A$n#k{^KH|Fax_0Lx7b*qaH)%3zWG7$P@ND@IE^M{3dlI;xKn(JqqX+0 zX_S{LO!4#QPn#Y&kh!$(L+m%gBTs za!xh_nM=zrj5{21hVZvN2n0IRRUu`dLTn8F(k_>9&%LR-Z0@dv62)9Gq~s5D6&H-53-5Ow`oTqhZqeOK#~9b zY6Iu?_n!GcvX3|j0B;UyKoZ`xr?k{4iVFtn?Xx;8pvUq7?*1Y$^pw*7lvCWLJ%r|t zo7$?!UK*S;bOc3PB?YV+xktDTZLwK?^UUUkv2zeZf8M<%NThnJMe0xlWcuIq7XMP= zVGF?a5-$~;Hmk(~R!8=ePT31{c_fsV9!)5=5Nh z;=H{+-mv)h&HEhh1@R5~(?Dn8yH)xWsA_m9EzU@|XZ6LF{IQu@Ir$V*PXXa>)ru{` z5=yXEdxpX&fgB2jQxxiX3&q~MAaszbuLp7z>hbOQ4$oPnY@ki~djlZbuA!Zq5)k|k z@@6^#^-wIP0rAuWg#?ZsWn}3)sv0$O3nkSo(|i!4xUOyKde|yjgbhP4H`nc?Y|8V+tnl;B@FF=;!xT>^3p9KB5s z+$_Lo7r5gT5YBi-p@kS7k`mzU4H_Ph&xrd+3-h91J?C!82GRAnaks+_cO=wRDx%QV z+Ep%$ra$XTl#<3^@{xayo`hwZs{bLoBwnsTnMvjNR51uDn2?t6gH}cFnB3DR?CMs& zK$`lq?Ea$pG9PfND4jwP^y`=&*rWFT;;AsOScnPBwv$n7`ScSa8Hmqnh2vZl1JY1N z8MqtamY{4YYmMI4+#*zMSSN`6LW}+IKDlaRK3X5pU<_tl&w{iBmtF*_jW9=T=x~k} zNiLevj{@tq-UWfq-Wtd*tL%(TP!DwHaiWNcuaE5I@R0;%8!lZG&Jl{gyXC9#N!jqC;zGI)h*~#8 zOTka27f{YrD}*~#zBnU9Bm?q|?ggXZw;2tIcK-FuEj$%P7*D<7&2`PKvJ%1C|4L!A zZ?F`C0>D!c2}_pktEyQ=Jn$PrK!V#TXvTcFwbtO{7Go-j=ns$iDKIc(y{>)FE&WaK z=5cRNzMNUix&=Y0z`(-3tS*K|#10g9o@GPum3<%C#Gl>0t>Ku`S#*ZGVAuy4&hMyR z+yf6Y2c{dQRMEMkGP`<`(_rpH3M{E^d2pU_+%6=2#Cw>ggAEm9) z>tdeL?_vfWka23u_l&#>hrvElzi<#sf!d_JY|Bxwb9F@oiH^AylNa1^3A-V8!Bm09&Zd$|K@*+wwvKlr(7| zKH~ItM!9RRt#T~x4xQ^h zIvpd?od6b#O-T0p-`3LZL>0d+$>pcIrDr8z&FY%}G#Qp3yR(}YbMo)c*S@WyD^v1A zfo{NZMMwUG!`fAQqjr3^31P`zTGMP7H|J-^_ENPxS{DDI1#Wp4y4N(rLxS5t$>s!* z+i+v~i}7@{DS$4UvtdB`i!%rjg7WDm8xlUD#1RREM=uCu4)u39B_!lk(76Ttvt|>B zbZ$>}(p)sT$jg*zEC=0KUk}#BrKwEpI?Zj`on~XCX?A(8-J7;kG(;9MVThb-<rCa2sUq}dlpQU7ej&u5olWpBN)#&b= z&KObqdCh$D{Z|q0B-klI%fmr-0ycm3TnW51D#^}%c}iFQHn2nf?+&pBaCL@d3+&vv z_iI<7VP02i~W? zHpFKb&D69)51Vo^JExevE`dI9qA}^0^Zqn@PWUZMeQS8Dq+IwOb~oG$v*p}EG5`GC z_n+68GLFBgZ_P?f5cKHc>(P?wqy~NGWcpHLIv1(8^M$DEgoADFiiEk@65;t|_w2W? zHfi_ndN;*};-_?fTSCFd-C|_kH{!8DXq2FcP+ zA2Au9_~-4$?kh}WKC)y*^tbo7^?#70|Fjr#201x$*t9SHlZwpnrVR&`lKr4TAg~y; z+81nK?y(ATT%H0 z+mlR(!Q*##rzqJCoR+V>Nz;ejxIZY*mc7xUE+m03QR!xO{vGy{ zKh4&b8h^}<<^Gbp36`mls6qA$yQzEY)vWR28Uj>7!VQfMpKD%c0le9jUSWhtNVXfy z6*T*al^m3BV$f}=E%vq1;mw#StONQKS>%_#az~wrHpKt~6yM;(j&>spkO>@I7-=^p z6p{AQ!{k$W$Z8bZ$uRV#!|_#)L;0KCoQ?J|qb4TGk8w%yUs{Zp#8ndQ{V!#6X&1+^ z$HQrh7pGgccXV5K-fNnjJFAqx7xxx)el9aMhqpq51a7eqlTZU%bRJK-@TTTINaWO!DmJB7!Q$b@vpVJ1 zdI7>E%H2UGx?|_wypW;(hpeMdtyJ2iYK#^GV`XqJUT-x{kK|lxdii3~;#IbxR;+Hp zi$iU19*YeV=Sty(b35%jM=iDUXvs=9w@OC^F&GafZFK>#5Y7?_)16h$5T~1NF7MnKJKr3WQjig&bf@KZis_t_O}L4Ib~LBylwxmgW~_v5xnPLZ zc#E@MWd0c6g{|TR`&cmb z=O3=A@)W3$gaq;{Ll&!B^X|CL1P~|}@upV8KdiqBop@1P+y+nln9FU)vJ}(K?vO+8 z}!dcB_$8&z@8Q+usMy4Rt`}xO@-BS$qyCyGpe9JINVZSd1vxJXui{!8e z>mGqptw`o(cIz0C_aOrd$=}C6QpB=a%t|-pwss!>J*7d?H~FTQO{YBN+YhC7&#%Ii zSzKzeNfx-b8!q-3RqK*USR2bX28I(I4~2`@G&C~A8aOY#@t<&am}7>hE3nLYY1DnS zIkS6C%%3sG*@WaoS!p2^PmJN0b6t-2x68)%Idx9t4bHt@aXMdoBzc98#r;ahBmJcg z=b!rS)3V}u7KZh!bCN~xT&34&g56*z6x?>tjjoGLb|mGCHmo!VivYHvrX#{l^%?{l zP<6&V4ymSyQeA<|M4Qv(l-_cb)9OxZyU}_ue8eI5vAN~e90>ma1givi(D9->Jahgw zUPEnKOHRk8_vDNpUi#&G}%-S7uN{XJU@jB2X zaYi3+A9tT@HaYe@bIEJDSWLL{^UTO#(4e7X;$@0PZK=~tcXuCkgiTxPIp0^N`GOtf)9! zIX75xBx8%`UuJA=3}qdd1YJ9akyE~8@t01Hq;#L+!PcL}yIkJ3-WS7dzc2Aa(VSh;PGqP} z;sEEZi=EV#n7Eaxr9V8bKL~_nGiUexV-T1R6Eqmjw0&xAil~%nMdO z8e#=C^;tiU^@YK;symBN;6s?MMDZ`(`8gyuwLO zG^~COF15PQlD3syCFWV747(CdeUyOc?=0!t_&5h^j1H1=zGx-TM$Qg4L!n87ah_f zYQ<)9G8Kbocjgxj8C%|&mjfZDrk5|ugb!XHloxV@@t~zeqRHNUSbF^AUdso*{vQ;* z;ITV}b5-)%zsjN0YTR~c+nCs$>!NUL&%JoN#2&vjajavwx|8PI%Oa0PyMa-{lRVRX zy?y$su`0+=Amy+>zGJsxl=av!~~ub>J)pCgfaR=MiK1 z$-#kVwk|aM0A0m}M?M^<{SGoeTHt1n>;9mtfS1#$NRG7I&AFuq|5}&$o~wQQ<+nX) zhKahJN76+Ol-xM{r8Fg;n>wO-(nnp2y~EwVwf=~_tn8?>ApJ9MNLNqjMVics5yshP z#ly2MF6QpgTlwIAEYswRplv*~2R?2cxRTS70m-Xv?gI1I%vC#l{7R3dhyD&xYH4SW z_0o!VG{Le5@EY|g^{TpX3Sk~}=8A8bS);|4ylFouCZu-FcC5UxdLlU33VMG9)EI%9 zF186tJZnDwJ7B~_po<&_T-c6*$?(DV%%WXOW2?8^Z|yo=f91~HSt*xeiQNy`S?)@A z@lxi4j!<$N2Gyh}9Hh&$? zPmX|)|7-<}f6$1-C@0ph)8y}@Q&8grZr2?GEz7t({3?d@=u4DUlTL%>4xF?gV!i?8J!SRM zUPm$dTdqfFq@n$L?uJL7|0<36}AyksbQbGde5?9gq>t&giD$+RM!dkF^pXiRVMu^JF^Wnb(+>PMu1?JBBFC7?P zV=_dEp}?RreFz*+^CTSaW;(N|qBdNmF}x3TjrAP|IMi1u(%o?*kP7Pq+&zTmKTgz= z^t$3JpPa&1xj!U^cA?uLZG9Y$f}4HP@Sq+&CEWA@^v)RVhpcuXP|V?uL_>atlWzG# z0-=4WI}j&~{Y>+(vuZHTpb-iCFHd)bE21Cia_CdR{uV=qn2PCFuqaElWE|8W1$-#a~Msf_V({_%*>09+H$er=}UAVz=uqAd8iAunxY+wK~Q5vsUd zSO4b8>P6<)XTiaV2X^Yqne(D$IfR^Ok=rfoc^lj}f(;|jd-bzEc6DGo*NfcC9ZM;? zOj|)EjJAFOGu3Wk*n!A!ysJ=Ty&D)O&2do(96c(xG3sDOO|?$oN2vIUpA*zk2m1T_ zIp@HsbPw26boQO~SWw_rz@T{CcH!3tm&U+C+NBFTJwSG+=VFGmz?rTKlNuQ}&m;9e zXx*cPK~AO}-41qRpa7ik0b@&@(DfODu4%Ro%xL@8R?3myJviO-0YwF^m0X6qc1yAwGRBN|;w!C4BK zcDuZ}xPCuDZeZ6An`5&c7PK%ejRM0Dq!Lq^KsIQKg=Ns{kgb6USa8xVfAbWLB17uh z8QArwki?ytN6Hfb^RlGmkOUB`b|ELU0xgOc6RzaxVlDF-}~(MXM2{O zdtcSY`^1E3X*%n;bW+zu;RFeJ+z#5T-vk`)ySZ|-WNJf0 z!+q9^Q8uo}mCW>Azq#@DdP2@jJr6-+Vpp~mpU~+WNijS2pBC0h2Mayt@#2hR z)#Q*v;B(ecfx9ZptrL3G>cw=GvkM$W1x8VC9aGSgX z0s?;9x>d>BcAssP?cd+>Mq)BS_2kwr-s*y5QS><(0aK3V{)8h zyO)4fS6dM5c(e4tAHE-MfhL?UUdXBo&U*A+ z@vVyODn-yzbXxUYP*e5#5&&-vWP;PuNLEX|k)|V}un7GF9a+XtbnU^;&sL>KE`YV3 z8KX(jDM%-v!;X)S9};)U;IwXhBQG+3pfktph5Is9PNAcz1zpw5pybh;P3VX05lt6EcD%wvS zB)weDXv^UzG4}m? z=|>MAa`5xt*s^7d3pB6UjeO%&U}9rqEMU0q#W5Z_1;aJXNNswpuvp0Rb^-MZkL z-V7%4Akj5YaEG1L{n@0V%pyHKaiHW?gbS%pzqL9RbXH#vg!1R&x4hT;d=xk7>-@X4 z-Z>aXI|ugQoU6op7rS;FOUF(`~3&Pj}$m9xA_Yz;ez~`x&wHm^oClo&1JlmmN1Bs6G7Y`_nMGnDSbQQ9 zVRNnXuxbUvM(w@d%5ViC&`xFwit|OVtp@|Pll#x==|JQ_8+q-Q z>%X@>di?kuocUc_+d5k*u(Jdqd+6*0IlZ7j*xAV`3BrImcxJ)l{r>RGma7au2PL-Q z^B;L`d4Cf2|7iSU_ilj`#)~TNZgBR2ueSiVPQld%nQh-?sZ?rpiSv}7q}$>_*fpu{ zE^VNZlZ(rt0LjH9F?{d25TS`Q>e;hrKYk|v9{2(T@drWhehv~36X*v0@M~tqzSe{uzug2-vS>yhgwhFEbgkn*LmicHUge&RN=kUGF0W&A)MgCf?+-yu3}p`1^ZX z(Jl+g@K9-6b+e3oz~?Zwva*u7uBfokY_u)&6|b?cN?FaHM_{Mf@-@}D@ovk>2lwwo zZ0x_P4JUdHrky>t_d;{Y$dN%>XOSz zuBec?ckkZh)KrttK6bl>N#mvog&>aCu%Nz!C%+(~gHt{L&8GSuE6u0YTAs1N7I`51t@6^-N+S)WO$jQ|$&h~;u zY6I+~Fg8vj{o@XNjeu!`^w+On1c#=3r);=E-t%U`6SaXtfWPqQiyoPK=m$dM!c zeSKtH7Kgk~r&%DMVbKe0qo+4HJ$TD~)^o|Fh*`I{3S0@}*HprRAB{6-f-w{$Br?)s z&Kt-n$9=*qP;A9$EY_(yMm9V)Fr8pytxor3ZnjyQxV-b}s$D_xryg2lHBIBpquApo zPXtXHXtV%4s_l5T@dQGMY}&Mm*=~|)oNNf{Om<00iI7A8$v&K6z}T|<3;bbJsu_Ck zZO1sgnA!*A9Ck;S5n^%ez2yOHqQYXf-B$cFSo}NiEVAjAShT0|ZJ1^+qFo|&=K0io z&pmGCPkns>NU*JqMo3ollaB#Pc52Xoxcr0eC=fPAV#9gy6)Kh@tO-b7@ysm&P;*Lm_ty%g1FP#)!`a!k&bvCRb21y!jpQlCIH$BD?0`{SwiR}bs?qM>ZR8-XNN;_9q@owLx1%2NQI4G{A>U{O;RaLIvQj2kBC4St;$47F%(=uKOAP>=^ z;$p!u9W^yIWAt!zX~`Q}s3j{4Z>(d|=a02(MS!8H)(=&q#C9Nzi!^!S)n%nlUxk^O`Q^MU(e+_V|Vx?RYc9<*lXa?zLn|+H*MY= zaQE(pYEM@*oz7p}#kO;&7);dew2=j zoXhlE(BIVRW%f+U9PEJlXX_pi}Tg*0>Y{o(EXu9MDoB{@BpUO<_8yQK(`TeMI zVb4=ces6|w?l~clL;5d8=5^~DF<`{8s~H@@d8A!bxd2bq16}p;OM`>)XgUx+jvkN~ zC)RCbT4@W!bNkk<=W(7hqp97+p3*rigBxEtbI5tMhNPWMjRYnz)9M0y?FO>b=_}r&uE4dFJrt7#kq@2|4?4IxO z9_^G|KwIy8&eeX>#(+Bbw8!^E4hAu$?UR%14oNdCF6eSz0e(QgcI}!#wQfB(PS{r3 z+v^tOpX^C9kXDuw71ab1%<}YSVh;PYQ0;)mSP%Dj&phPoN~G*9v>CJsE$+gZFsFq% z=Y)Vi6neGdRdz%beR(m<5qEIq3wa zz{`ozEZ8!F{2hZMhluvt3&sNnd&g=cTLxE2aiZGPpnZtH1G=%?K~_LyQaO+HJ#6T>{B+6 zX5yLS2sh8;C}++%ef@ZNZ@@Z*(;GNAtnY8zu;H$$J3}%7O)8Lr!@Cx8Y`Ua6rkPkd zNgG`xJ03*lx)7;IVbD@2Nu5Q$y5 zn^J&qyuK-Kzik&MQ_D2Oi^v8*(Kfj#f63L~J+a?oF@S?Zc@AX9@$=COX5sQoExDdE zHk1NjOsek^g1!eD?lwI;%^k`|Ik876L>*xjSZc3ojGQd(0Nvz<`)_vEBxMP1-Me>h z$JEmY{rx61`}#A@<*_;?W0p-x1MjmUQitwURGh@jnKlF50x)8sEBXjveR60~mU zIU-6E&lM@fS|IX`+UV$#-eWH$ktwvRH_?&Xz56gCg|Ofzd1z%GXz3v=7}Me&hg1e_ zA@}F||Md3vo9#^MU;jl+ATR1~^^4q2-#D+`5FH(zr2gp2QELqhg$W4^2tfM;CRQJ^ zCnS%0e*O5mtvMr_8*q0}J*L^QZiFRjrMd)~LRXK+en#Z6#HMqJagTXeR7Y^sA&!gf8tU0ad;OeS{Rd4;$?|wvF9FJ;*@NwLI6OeC~KsmW9AeGPPzaKWx z%`GUXdUZ9+SwJvR=V6Uh{roc+@N@O$;+!QJnZf+~??STG323;hi*A+Fks)W!dp4XG zsRmI@%q9P3nXIMDyO@}-V@pLeU?teOeMka5guL)d^j&$mCHlL>7m+^S)mbyPxUrO> zyAQO()F3iAIAmg&9J|HpR}by+z>2?>?W9gZu1qR$W=K69!Ts6T*nr>&35GlNEN5L0 z0xkAB1&1jO(Yd!UE^4gDrmG+k-wQeunPyay*5kWmVB|d&S{E&AnV4nYcj6CD8Ey0o zsn6yb-1u><7oywghf>ayh+hmq@?e|M6FJ@YJPNV%EC9mU78InGYrWh~7FWY|@SGpshH>S@#6%XOXtAhHy4mrRne}pB z3CuSP8Mr(RZKR1L!0g<7TVy;8JW}nsLXGS3nO6;F-LN4RMBD-)*#l$Znn3ww(cmId z$~lRkNo3)u;Z*I6BiP=i@e7EszFcTiQsRPmjaDwlk;U4me;_AeH2Y)x<%>Hv+=r^v z8DiwP%<)t&I4$pml=>v+{@&+#+GBMmm6WPz;aF!F!}S55V_P+kktrimEmG+dr_7|^ zguD8^=|#$I0rA<>^<%IPg9aU(ZSQRv)njT2ZTjSY^XAQ$^P|$FMKc5d@QEz3I|9JQV{j5))Jb4SlI@2a6C#|ZZrJR+ZfoLE=ILKs%y8q@E;`<-2 zu*BeQaPTL)ot2f9WXa@Up(}HFeZ9MB!InPw0C_1B5?qVd~% zWYNV%U|WXco%yY=oK2?;*~P5Fu|)4#8@Rf6+}7OrNDZgu=}6TwI}F)8%8&h;dX6NUtX*3Eba5Ls@JjVVf%@g8}f1=Y8EEj*wM&#DZELW&1$@h~?_)4;> zQ1ktB7cN|gFZVYYYIZ!DX|dVQQX7;Jr;AM@xjm-8BoZwA=w^{_{l;d;lc!Rux9vOY zZ!5WMr4{4?-F%0qdloRlp|Yo^hrZD7ym)ZkRhDZCy9fIEj3;WijEsy#%^Tj1iXkRn zR2pl`Dvw`BvJ(zFKSoE9&fqizIt-U549iq69Z)Dz)-#vS7LnG)dw2Z@cb+V%8g7LT z_kRHz9|CCN_46I#-+gIp)Oc4`Ce?TX8Jf=kvPFVFZ~!3qBY>%Wf`YZdEl4+lIHX^G z^Q>N0G&eS4pSm})EaLF`rqv(4^0M;-c9Jvxj=34@mT)Y@z#uPJ<{&-?sFVG zIg^j%sQsk}vq3H?-B=M9T)naWWifUsz6jv7Eu@+uvk9<#&Po#pN1IY~Xph90Sd0FR zYd>LE^X6W(0q02aMIN>4SfTDjG9yl>-*h4n*AP4{8)zjILX4KR@DNjl;2_JZ96-^Y zkX#>LuTv>88K)l~`#nmSki7yYdhmXumsVA4*ROXW+HiB>KAw)51t9&9;NT0cT+I_F z5gRbndGHnQGS*c@R9R5&h0w=)&&N(76=ImVTMj-oUE*@wOq||-bfM;Pd${1 zcxh^oZYYaR?S>#(`8F*cb#29-dGrdu2M8A2KvXZ%u&^s=G+J=r?b~NmRQ$W9BINzv z2p&4r7$WDpSaftMHE0v_nl;aVKDo4-<@CT+Q8qDLJbbb!+*Wz26sl6ss|@KZy=2kC zmFqu)r=PN{2hgI6{BwKwBJobpx81b4KQ1R20xThNYuJ^mSGRC-Mh{(dLqyjCd8AHa zfau7}M}W8!YIoBI!P3U!3HY6uDfB>7{yqjH84YS8 z$KRBej<7vBza5(J6TbR;+@VaJ?hp{3(|26hd+LF@mse5ORO+^;sX?N_i2Yhron4)C zZ3dDVGqdlS%LUq{gG|m+-GAea9H029+ynrJ{Du`@HMy%CUD+(bxLleDoL8lD%6dpR zUSeWaz(d1`j%Bo?7ocw@ftV)Zl1|Z|kmdW6IoOsgG&_YDYioxS-7-XISw%6l-bC=s z^$Ulr+Z!8ix-gy!_U+2HjWpY)z&b*}&W7H_!`01CV zC@M7`5iN3@sHf&Sh<>6(`NWUJ26W_&1@=PT6MkyDB-}HSGIS3-uOmmn`%9~kbyq=C z@bu)QMoXFj2OGEF8}5=(KJ7xmPRPE;?-mvou43V|sv>i-0>z#184W@5ir^8TRfdS0RBWIHvNBySg;T3ShuqLMWtD-EDZ`kL zLWTPGvtw}8dyOk51g-@GLkO0g|B@A7go#|Ho-z%XnVN0gA*S43U#|vCk#B7L+g(pEY8r5( zu_BkG!!f8v>L*T|pgI#P03jf9oDAz8I3j`+QLt*=_H#;OJtaoK1~o40@5O-#OpW#Q zglM7B7m-qFH7eTY#XSF!A>$4avmSzR>MtarlA?M10P z#B0fE$QY-e9Q$We{34sEg$Z=D@G3&rKeuHi)V5LubR2FjZbCobW5=%1WjS!F)wK(> z`pKE6N@7!Z#@65>+EowbyxX@6UAR{J;~Pj)&*go~KMD}hrL(hBWvl}cf(_G+ghG{1 zo;(OS?R*F%PmO7GMH7a87i>)e4rBQJqpMKNu=C1*#R8BBDQ$PQ9bkhHC;sEd536A! z3Uv6KIdjGz!7h9_L{P8!J6gY}EWCa9Zij8rw;R2qqc;4~4*eC<4hu|+7tup7B27mn zMy_kp2c3T|P#HS`DJpVRZiEV0C1%3qKuErP&NF)qoOr&r{bF*h@2n>A2^>N3~?^!;_on&qr^RWbvB-8}($9_bF2KL7WD85;Z9CF#w&XC@I~h&%hqTV&W2~Y?kqFWr56VE(lWoH+_N2lGDy$#E_p)U=4^Z zK(ycm^(4U+dIEh0l#?{4f%4n;v1yszF`!r{Ol*_MLHWflL5$<*5T14Z=?ol068nVV ze2;b?y>SU)!zGku&6@WRoWJP)Sf&1A%Q1D8Ua=mPs-Yr!Z+-0d=l=A= zLW=E0?pg4Xt+O_Sh62!Ag3-gzad^hXw6YHzy%|CoCICMO#nbtW5rgR#Eos)ja+H+! zT=uH*5D}wuF`d_PW+(u2*^kP4Og{JQEk>9laxCm=4z`_4;L*stqiT$CIMr=M9+}L{+am>5aAWja)$7|w`wQUL;$I)Ml4rJJf!Jqd= zWRE~nlAHU?O(4hmm2Y_v`ZYDv4Gl=JMg!4CsGIqP7fVd&ScMm(-A`a8)8LM>&lP=u z^njWJwe**BNmU_y_f7KwK`l|Y(*FGXie*BtRt~iMw5?dA-!X05ykkcq5E!47lrG%6 z@tGMd@OP4c+&w+FO@3v15l+l|e7KayU3-{?MPWBJBqR<4 z1+h7g64?MRSQEZd&ZQ4WkWcwP_b#h=LOuj)9QhD{pcDj#M5J`cFq$Bp4;ogJu!m1i zg82R!ix4049% z4VJyQ@oU?D{J;E682tao&;0-Di2j#Zum7!Iv^Wh4^Cu8Mbv?50T91=iiCiD!hiee_eYm8r=reE5y}L05`-e*@`}}D{qmpv z;>+c_6V1mq{IuTHD>nSJ!kH8EdVKWI@+vql&-F0NN?xbMImUsJ?&6MM8yK??iFjOH z*B>KyaaZ!E$*!<_3kOFk8q6MpL382fuMk8_@qmGJYV9o&$$NWyT|zPTG!ii5oFY1v zF2o=*UD7gec}_lP_RdcqU;P#lrNDf(5Sy7jcoiNYTWzEi0Ee8iEQAG}q@SN@U|Iqh zic$bSX1F!u39W27KH!zRgckXe*93QFIy-|(sY&SiTRYGB9pOio3IUgQdcBU+Zz$Y5 z{L|RyH9_ZLfuC>=0UsRY%(B2Wyk3aa=aqUEf408xtFiO?6ThNfamc*k7@*b*1i`-) z3%q7CQRfy9%_Gn6Znkm|+SFBzif+a|sud1kcf1u>Y<0pRy@r#L_rPJv| zTm%V$E|9E0Wxo9etz4%SI0mf$rtcxMBW+oK!yShsPXs#xlA%mPz}5vkb7#{!a^woRExbwwDKmf&3y4YTl1qdXg<8#NSrZfQ&&^>uS zieDt29>A;VEWrg-VEAqa-xdX*p_+3A z`=e^756y`WVvXHPU?{4pisEsY9s*{d`DA()_m|-U1E8hv0BX@OjKX6Og3-s#&tLV( zGIM4xT)e~<6KhN@i&#wMaE3Tak;XpF{9yxteL`DXo7$ocB5VxN$Ymz9Y5p73%o8O( z3;vt-D78)s!Pd7bU+llH#s>Gj*1;dD>_LLJ*2t0T!CI`@i6c@DT?e;&mO@&%G%6P8IR%tZ4DBG=;8 z(6EnGXQ4ARV5VJcjPIf(&Ev-TOSmM9509^d`aeki2I-6dM}q!PaB|l?0&aq!VUaeN zbqmzAG4=V3j0^&t3tY$2@FMuo#=1Ha20{Z0AmdOUfdFE>e#)^L@=4>PqpL~Tq!gZb zTD#p_01b`gvx2qJvU>LR_Bzx;kEx`WKgN}t5-ERUPrU5wF-dxPX>uq@D{Nmx z%fF}Z->VnHNP&T58(6p^h)X^V-BR}n2&iDQxfosLc_?9j1ZcAXYc8uO5cgaAP#WU~ zL>u52v0E4gLmJS`voXMBccgLKA22L>hF#E4Um~*y%zjou`5m!92gi8h0l9@q3W!Mo z)8G9vm?y0sBj=6UMjl17)$x)@A8@Sds47f1w?KUk0Wdt~!;OiN684WxFL#3Afl_Kf z_CQqQqjZ6xCZhC1NmbSPk{6QAa93d-0?7HwwQJU><2}N@l?Aejsxr!VI1aF*0mfu7 z!X@jW0RW$l#CoC16-FHHg?uyS8P_JE)uajbhmd5md|O{^nIz}!MA z7o#LGt1Gc@Gs-XomuwI;GEgbXEh^e{Mh*`E64XId>scmgAsJKhYqU$aY>VZa57%M3CQK?gi{>cQc_aYQZPQmR^}lyl7bDrP$5CKF3EXS5JdG+l{_!rJq{)&Sn9v zwp&@h=WK-ry?GrFn`W-J>9kLZMq;o%Xm?>*0Se8O?Q{!fza)1Rx}xj^4=dh-C|J2a zz=5roQ`-K)OOF(@wkSxex#E;Jk193Q0c;O{uufCGp{W+5`pG5!=1q% zd`eBN1#uX)+JfE~+1th`B!;AoR8Stg2o8?=u)g=(LG)7I-Pv}rLk;O`3cY$G$1%yk z*6~=oAgaxpG2-4#QQWFk9T+7?@~;n(01L!WgPdWbd-DKl?3t9{0`=EfoSsabcv z5UL*kgYA$ywAe3f_njtAzK1+p*|p%S%z^g06}f>^LV;sN6M8NewJE(^DU> z_n@qZ9eiq7vl)dii|JL@WAObSP46)YcF8gM+g#eR!-=jdk9UEg@&!eh9_1ZKBaY zH=kj_XUC~QZAW5h)zm83QDNukm{kZg&-wL!+%_Yx({NxUo8idPh;HhRCf4?Y?{CND zb$e8&4Jl92Y2_h{BY8dS!wFbG0K8cSrWleM2PC1_peFO}8le@gMo4xPbR!m^!?Nv= zSC?)~Gth$;loyAV$`Ph*z#?=Q(YbL%BgemVr}^0UHtU_Tk(K_oHF^hXi`DIZ6(3*M zNBS`Ipzl$ZLM$@7c=9w28NaZq3j{4b8@vx5oVivP$$}wrkiERG(rS-0kmK78e1QMr zCy^C}1@mCRRF~|7cZdMg-gT3Q)U(7-5G#^OAO8FHoY$Nc1fB z1}gqPXJ>2?gM#s0EzS5Y4XGOeiITeCJ=SBEts7sz zeoghgxNTjYPYlW`TSomf5j`4lP^Pd-ckC9a;X;kdND}EkJA^7T{K2{m{i=hPFJCS| z2e)A$D)PPBI8Zxs=Tlq$Lgdq^J6)z#A(fh4*gLJ_5?wOQ&oM&RCCnB?lkZ{MQZ0a4 zw(pXUvJ_+vrqY!z3pT*=47^LBI&zaYl(IxIRm z4S_@2+5h-IPtt=rf_Bslf3HjFe3`|+xB0MuXP|1f%RJ*d?4029+a;SN;Q{@u{y=^3 zdkED}@a(GK+M&=uOg?X{QAI^1yDicA&eVM4ocDvxp4IreM9A0FNkr;Ke5I9Z2I9)c zx?TU?G=9_!MM}jT-{7$<;IAhViD6)G(c{l&xhV#>Gbk_hM0!lOcP&$NLqd zt}lWvMU4Y~gwe0~xX2Uwl7o_d+m`P(J(7+I4`kdMLUM=PR4jp!gr9P>bq!gtyHlw= zG|SR^Ed~9pj8dr$dJyDq8kLh6(};{!FLurw)sM2X{r9O){@Au({~46JePD!;R-aqe zKvazwjFGT^6XG)IZCkukRJytRDr$ywK+MiUg^Tu?Gl!X4F(^xz^U$yIc-Ds%r=sB)my2Qp_l;&bYOKFH! zi0hp#X3d2)U(OfbVIb4RJDn}m9RCxWFU?_EZZF?~l=POlvGXhpDTwy9wD5CWe;abm zPD!0oR6LDD>^xYrgW`j@jyW+05g}BG14se!ThAvYrZ{X1qa}r0s!<<_Ttosv zJ)kDAT?AFqeLQZk?qTw3PB#UDnqe;-^~RO%@_fhq>vY2xS4$xYetxB3!ug5s>w(Dy zYhBLm|4T^dL$4IUQ(#D>yx_nKGz>VLUOXMyceOB}xL&K8rMbm&VdnWwXAUEy9+`mO z>$Kp#()L%N(%`MJx)m^JnUx5_tI&|RGae#hZa zJ1{8P{rLD=atXJQU=XCUA?AWTsbb(w9$z80BT;lPa`ruX%73;q!fiZ1{!3Cha(ml> zG=Lfl#e3Ted<2)JfR`YJhI8RYd8CwtN<3e^mTo2XN}(*l<4%5>2aS8hC$v z_LH%B=s&ba<86?;Zi9-)f|%1W!^8q8e83fK7eg0LF%V8aj`{x*$lkiSxqSwW5gsU> z9gI>^Q*(XB!FZ?w44GFrI@j=P-nhkR1qZ1Y}~dj0h70t3t+gr zw8%R2-$E_=2*0Q2N5xEUuA#Rd#;2K?nHvB{RNXuBY)iPk5fFcL{gM=K#~K!UBtlrB zP6Ez9g*spLNG!#*_Xo@{mzD)!H;&=tXV61(Bwq?j8+m=Z&euhdceRc@3!PM?^H#GtyPqGWwF8fH=xsN%6l6phvvg`U{|}L6rp!WxT5HV5$z6?8$A$#~#Htg$n18 zl7}is6u~u&`rJ-}2E@S5m!#@}LYe3JTW|(;!K^gCEFN}St3XSAdzqqrUFr~}m* zK4c>h4a7_Bae}ge+Iv1^{5jL@GjE+=xMVBY3mVu0L~5Erk zkq0zMQgo0Ht6Uvy-xaPeUCd13Xd#Iv)nfKfQbRuFRBP zYnPO+0}dFHAd_p;Wuf~O3R(e)-AFIUoXx>T85fQdQ@Jfy)^SjU_8|<~mj#O2q2&h; z1csyX&>Vc4c+^CRk^TB=1>C2jh6iYL*u94!M7n!;=#CRYha-=q5K)}tY8Z+&QDt!b z`t>&bYDh%=BO?!O`)(NP=Gj<_%&3GPJd7AdgG4Fv$0GPKHs(=MaF0_xDbQU}UTc5^ zQJ>AiI5$B`B~c0ZZqnzz4d*@yaMqdc$!du|Y0Z zF=gSDd4_$e<3eu+B}tRJU~D{vA$f^_zF6;JM9Ct-LU$U>&;w^Z$WfW7@is{u82!lp z&S5{1_hO&Jq5>lr?th2 zNT(5co4&`;8*(NJ^@jUoWc2Z_li|pxV?$tg8VR6M_MfhqZ@b9@+)u}0xVmrQB5-e0 zb+S$%%`{;N^C*JIk<3QE2%ngQiUyoNqFlJ}m=yD&3gV_qFKZkW8KUc{$uDFNh;GKK z$8A^QkSBuTT^QV`+KDj#YJyXsV)dykPCbw~-=0s(Bl+Ptf{f=DrbF*d z2AZHMXu_;USa^ouLd`}Hri+Xp{H-48?op9UyCsZd7B z%(k`Tp@bUm1ekT1(%f)^g=;@*kH{ST-qe(j}ZYkkN30 zEvhl!@ge~a!7(_fGCWTOG5!qE`8|Od!{xfL&jRc1*%51x)+-`LDop=N{Xe0)mzrSk z8yyL9r!sgCPW%{|133(1BT`>q?jU7ea9^4ccFE`5oU69Q8RFA8c7>B1Px0W* zbn39=00$cJ_>L>2tYMR|i$(dvB+?-|35-@{ctr@CHbS2Bo*f7y=l4iek7*lB1ekGp zz~EN7<>uX}@6kmfLfOb}aq+xP{TsODNn{U;&KLGNqtlb);|);ir*Nd?GtR!`7LK4< z#3g9%&@RKtDN|#2NLI4bt;TaWp@psk>|6Gg%|u}RJa|{j%YV@ho0{oGAHw-0kj*f_ z2^PeClyc!~zIKMv zN+G-z?;(~rA5{qXR-3!ZI+Ntq;S#!2cy$q%HeoS5^8_`_t9RwxHYJr{s-UDysxr+S9Rd!rfBjc6px=emUiUa GmH!1ToA(m{ literal 0 HcmV?d00001 From e2463ad4e52ba9cede1f588a3f00657dccd25956 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Wed, 15 Jul 2026 19:29:00 +0100 Subject: [PATCH 45/69] fix to single threaded run --- DRUID/main.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/DRUID/main.py b/DRUID/main.py index 112ccc3..aadc79f 100644 --- a/DRUID/main.py +++ b/DRUID/main.py @@ -376,8 +376,19 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0 else: if self.verbose: print("Processing sequentially.") - _worker_init(self.image, self.background_map, self.background_rms_map) - for island in tqdm(iterable_islands, disable=not self.verbose, desc="Computing Homology", dynamic_ncols=True): + + # Safely bind module-level globals for single-threaded execution + global global_image, global_background_map, global_background_rms_map + global_image = self.image + global_background_map = self.background_map + global_background_rms_map = self.background_rms_map + + for island in tqdm( + iterable_islands, + disable=not self.verbose, + desc="Computing Homology", + dynamic_ncols=True + ): results.append(worker_func(island)) results = [res for res in results if res is not None and not res.is_empty()] From a8b58f61aa4786b2ba7ca06a8237576e707f1105 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Wed, 15 Jul 2026 19:50:09 +0100 Subject: [PATCH 46/69] new tests, fixed utils func --- DRUID/src/utils.py | 13 +- DRUID/tests/test_background.py | 163 +++++-------------------- DRUID/tests/test_homology.py | 186 +++++----------------------- DRUID/tests/test_main.py | 216 +++++++-------------------------- DRUID/tests/test_properties.py | 46 +++++++ DRUID/tests/test_source.py | 33 +++++ DRUID/tests/test_utils.py | 128 ++++++------------- 7 files changed, 230 insertions(+), 555 deletions(-) create mode 100644 DRUID/tests/test_properties.py diff --git a/DRUID/src/utils.py b/DRUID/src/utils.py index 8129eee..4907ae3 100644 --- a/DRUID/src/utils.py +++ b/DRUID/src/utils.py @@ -15,19 +15,24 @@ def get_image_from_path(image_path): return image, header -def combine_polars_catalogs(catalogs: list): +def combine_polars_catalogs(catalogs: list) -> pl.DataFrame: if not catalogs: raise ValueError("No catalogs provided to combine.") combined_catalog = pl.concat(catalogs) + + # Check for 'id' or 'ID' depending on your upstream schema if "id" in combined_catalog.columns: combined_catalog = combined_catalog.with_columns( - pl.col("id").cast(pl.Int64) - ).with_columns(pl.col("id").rank(method="dense").alias("id")) + pl.int_range(1, pl.len() + 1, dtype=pl.Int64).alias("id") + ) + elif "ID" in combined_catalog.columns: + combined_catalog = combined_catalog.with_columns( + pl.int_range(1, pl.len() + 1, dtype=pl.Int64).alias("ID") + ) return combined_catalog - def generate_2d_gaussian(A, shape, center, sigma_x, sigma_y, angle_deg=0, norm=True): x, y = np.meshgrid(np.arange(shape[1]), np.arange(shape[0])) x_c, y_c = center diff --git a/DRUID/tests/test_background.py b/DRUID/tests/test_background.py index bc30049..a1aba44 100644 --- a/DRUID/tests/test_background.py +++ b/DRUID/tests/test_background.py @@ -1,139 +1,36 @@ +""" +Unit tests for background and RMS map estimation. +""" import pytest import numpy as np from astropy.io import fits -from photutils.background import MedianBackground, StdBackgroundRMS -from DRUID.src.background import ( - make_source_mask, - calculate_background_maps, - make_gaussian_sources_image, -) -import os - +from photutils.background import MedianBackground +from DRUID.src.background import make_source_mask, calculate_background_maps @pytest.fixture -def dummy_fits_file(tmp_path): - """Creates a dummy FITS file with a simple image.""" - data = np.random.rand(100, 100) * 10 + 5 # Random data with some offset - hdu = fits.PrimaryHDU(data) +def dummy_fits_file(tmp_path) -> str: + """Fixture to generate a dummy FITS file with a noise floor.""" + rng = np.random.default_rng(42) # Reproducible seed + data = rng.uniform(5, 15, size=(100, 100)) file_path = tmp_path / "dummy_image.fits" - hdu.writeto(file_path) - return file_path - - -@pytest.fixture -def dummy_fits_file_with_source(tmp_path): - """Creates a dummy FITS file with a simple image and a source.""" - image_size = (100, 100) - sources = [ - { - "amplitude": 100, - "x_mean": 50, - "y_mean": 50, - "x_stddev": 5, - "y_stddev": 5, - "theta": 0, - } - ] - data = make_gaussian_sources_image(image_size, sources) - data += np.random.normal(0, 1, size=image_size) # Add noise - hdu = fits.PrimaryHDU(data) - file_path = tmp_path / "dummy_image_with_source.fits" - hdu.writeto(file_path) - return file_path - - -def test_make_source_mask_with_sources(dummy_fits_file_with_source): - """Test make_source_mask with an image containing a known source.""" - with fits.open(dummy_fits_file_with_source) as hdul: - data = hdul[0].data - mask = make_source_mask(data, nsigma=3.0, kernel_size=3) - assert mask.shape == data.shape - assert np.any(mask) # Expect some sources to be masked - - -def test_calculate_background_maps_defaults(dummy_fits_file_with_source): - """Test calculate_background_maps with default parameters.""" - - background_map, background_rms_map = calculate_background_maps( - str(dummy_fits_file_with_source) - ) - - with fits.open(dummy_fits_file_with_source) as hdul: - data_shape = hdul[0].data.shape - - assert background_map.shape == data_shape - assert background_rms_map.shape == data_shape - assert isinstance(background_map, np.ndarray) - assert isinstance(background_rms_map, np.ndarray) - - -def test_calculate_background_maps_custom_estimator_str(dummy_fits_file_with_source): - """Test calculate_background_maps with a string-specified background estimator.""" - background_map, background_rms_map = calculate_background_maps( - str(dummy_fits_file_with_source), bg_estimator="mean" - ) - with fits.open(dummy_fits_file_with_source) as hdul: - data_shape = hdul[0].data.shape - assert background_map.shape == data_shape - assert background_rms_map.shape == data_shape - - -def test_calculate_background_maps_custom_estimator_obj(dummy_fits_file_with_source): - """Test calculate_background_maps with a BackgroundBase object estimator.""" - custom_estimator = MedianBackground() - background_map, background_rms_map = calculate_background_maps( - str(dummy_fits_file_with_source), bg_estimator=custom_estimator - ) - with fits.open(dummy_fits_file_with_source) as hdul: - data_shape = hdul[0].data.shape - assert background_map.shape == data_shape - assert background_rms_map.shape == data_shape - - -def test_calculate_background_maps_invalid_estimator_str(dummy_fits_file_with_source): - """Test calculate_background_maps with an invalid string-specified background estimator, - expecting it to default to MedianBackground.""" - background_map, background_rms_map = calculate_background_maps( - str(dummy_fits_file_with_source), bg_estimator="not_an_estimator" - ) - with fits.open(dummy_fits_file_with_source) as hdul: - data_shape = hdul[0].data.shape - assert background_map.shape == data_shape - assert background_rms_map.shape == data_shape - # Further checks could involve inspecting the bkg_estimator used if it were returned or logged - - -def test_calculate_background_maps_file_not_found(tmp_path): - """Test calculate_background_maps with a non-existent FITS file.""" - non_existent_file = tmp_path / "non_existent.fits" - with pytest.raises(FileNotFoundError): - calculate_background_maps(str(non_existent_file)) - - -def test_make_gaussian_sources_image_no_sources(): - """Test make_gaussian_sources_image with an empty list of sources.""" - image_size = (50, 50) - sources = [] - image = make_gaussian_sources_image(image_size, sources) - assert image.shape == image_size - assert np.all(image == 0) - - -def test_make_gaussian_sources_image_single_source(): - """Test make_gaussian_sources_image with a single source.""" - image_size = (100, 100) - sources = [ - { - "amplitude": 50, - "x_mean": 25, - "y_mean": 25, - "x_stddev": 3, - "y_stddev": 3, - "theta": 0, - } - ] - image = make_gaussian_sources_image(image_size, sources) - assert image.shape == image_size - assert np.sum(image) > 0 # Check that the source contributes to the image - # Check peak value is close to amplitude (could be affected by pixel grid) - assert np.isclose(np.max(image), sources[0]["amplitude"], atol=1) + fits.PrimaryHDU(data).writeto(file_path) + return str(file_path) + +def test_calculate_background_maps_defaults(dummy_fits_file): + """Test background map generation with default estimators.""" + bg_map, rms_map = calculate_background_maps(dummy_fits_file) + + assert bg_map.shape == (100, 100) + assert rms_map.shape == (100, 100) + assert bg_map.dtype == np.float64 + assert np.all(bg_map > 0) + assert np.all(rms_map > 0) + +def test_calculate_background_maps_array_input(): + """Test background map generation passing a NumPy array directly.""" + rng = np.random.default_rng(42) + data = rng.normal(10, 1, size=(50, 50)) + bg_map, rms_map = calculate_background_maps(data, bg_estimator=MedianBackground()) + + assert bg_map.shape == (50, 50) + np.testing.assert_allclose(np.median(bg_map), 10.0, rtol=0.1) \ No newline at end of file diff --git a/DRUID/tests/test_homology.py b/DRUID/tests/test_homology.py index 31e6511..8535394 100644 --- a/DRUID/tests/test_homology.py +++ b/DRUID/tests/test_homology.py @@ -1,163 +1,35 @@ +""" +Unit tests for Persistent Homology computation (Cripser & Polars). +""" import pytest import numpy as np import polars as pl -from polars.testing import assert_frame_equal - -from DRUID.src.homology import ( - compute_homology, - get_mask_CPU, - get_enclosing_mask_CPU, - bounding_box_cpu, - parent_tag_func_pl, - correct_first_destruction_pl, -) -from DRUID.src.background import make_gaussian_sources_image - - -@pytest.fixture -def simple_image(): - """Creates a simple 100x100 image with one Gaussian source.""" - image_size = (100, 100) - sources = [ - { - "amplitude": 100, - "x_mean": 50, - "y_mean": 50, - "x_stddev": 5, - "y_stddev": 5, - "theta": 0, - } - ] - return make_gaussian_sources_image(image_size, sources) + 0.1 - - -@pytest.fixture -def nested_source_image(): - """Creates an image with two nested Gaussian sources.""" - image_size = (100, 100) - sources = [ - { - "amplitude": 100, - "x_mean": 50, - "y_mean": 50, - "x_stddev": 10, - "y_stddev": 10, - "theta": 0, - }, - { - "amplitude": 50, - "x_mean": 50, - "y_mean": 50, - "x_stddev": 3, - "y_stddev": 3, - "theta": 0, - }, - ] - return make_gaussian_sources_image(image_size, sources) + 0.1 - - -def test_compute_homology_simple_source(simple_image): - """Test compute_homology on an image with a single, simple source.""" - result_df = compute_homology( - simple_image, analysis_threshold=1.0, lifetime_limit=0.1 +from DRUID.src.homology import compute_homology + +def test_compute_homology_schema(): + """ + Test that compute_homology returns the mathematically expected schema. + """ + image = np.zeros((50, 50)) + # Add a topological maximum (birth) + image[20:30, 20:30] = 10.0 + image[25, 25] = 20.0 + + df = compute_homology( + image, + analysis_threshold=1.0, + lifetime_limit=0.1, + area_size_threshold=2 ) - - assert isinstance(result_df, pl.DataFrame) - assert not result_df.is_empty() - assert result_df["birth"].max() == pytest.approx(100, abs=1) + + assert isinstance(df, pl.DataFrame) + assert not df.is_empty() + expected_cols = { - "birth", - "death", - "x1", - "y1", - "lifetime", - "lifetimeFrac", - "area", - "bbox_min_y", - "ID", - "encloses", - "parent_tag", - "contour", + "birth", "death", "x1", "y1", "lifetime", "lifetimeFrac", + "area", "bbox_min_y", "ID", "encloses", "parent_tag", "contour" } - assert expected_cols.issubset(result_df.columns) - - -def test_get_mask_cpu(): - """Test the get_mask_CPU function.""" - img = np.array([[0, 0, 0, 0], [0, 5, 5, 0], [0, 5, 5, 0], [0, 0, 0, 0]]) - mask = get_mask_CPU(x1=1, y1=1, Birth=6, Death=4, img=img) - expected_mask = np.array( - [ - [False, False, False, False], - [False, True, True, False], - [False, True, True, False], - [False, False, False, False], - ] - ) - assert np.array_equal(mask, expected_mask) - - -def test_get_enclosing_mask_cpu(): - """Test the get_enclosing_mask_CPU function.""" - mask = np.array( - [[0, 1, 1, 0], [0, 1, 1, 0], [0, 0, 0, 0], [0, 1, 0, 0]], dtype=bool - ) - component_mask = get_enclosing_mask_CPU(x=1, y=1, mask=mask) - expected = np.array( - [ - [False, True, True, False], - [False, True, True, False], - [False, False, False, False], - [False, False, False, False], - ] - ) - assert np.array_equal(component_mask, expected) - - component_mask_none = get_enclosing_mask_CPU(x=0, y=0, mask=mask) - assert component_mask_none is None - - -def test_bounding_box_cpu(): - """Test the bounding_box_cpu function.""" - mask = np.zeros((10, 10), dtype=bool) - mask[2:5, 3:7] = True - bbox = bounding_box_cpu(mask) - assert bbox == (2, 3, 4, 6) - - -def test_parent_tag_func_pl(): - """Test the parent_tag_func_pl function.""" - df = pl.DataFrame( - { - "ID": [0, 1, 2, 3], - "encloses": [[1, 2], [], [], [0]], - } - ) - result = parent_tag_func_pl(df) - expected = pl.DataFrame( - { - "ID": [0, 1, 2, 3], - "encloses": [[1, 2], [], [], [0]], - "parent_tag": [0, 0, 0, 3], - } - ) - assert_frame_equal(result, expected) - - -def test_correct_first_destruction_pl(): - """Test the correct_first_destruction_pl function.""" - df = pl.DataFrame( - { - "ID": [0, 1, 2], - "death": [10.0, 5.0, 8.0], - "encloses": [[1, 2], [], []], - "parent_tag": [0, 0, 0], - } - ) - result = correct_first_destruction_pl(df) - assert len(result) == 4 - new_row = result.filter(pl.col("ID") == 3) - assert not new_row.is_empty() - assert new_row["death"][0] == 5.0 - assert new_row["parent_tag"][0] == 1 - assert new_row["new_row"][0] == 1 + assert expected_cols.issubset(set(df.columns)) + + # Check that birth is strictly greater than death + assert df.filter(pl.col("birth") <= pl.col("death")).is_empty() \ No newline at end of file diff --git a/DRUID/tests/test_main.py b/DRUID/tests/test_main.py index 9bf23e4..fe4238b 100644 --- a/DRUID/tests/test_main.py +++ b/DRUID/tests/test_main.py @@ -1,192 +1,64 @@ +""" +Integration tests for the DRUID main pipeline. +""" import pytest import numpy as np import polars as pl -from astropy.io import fits -import os -import shutil - from DRUID.main import sf, _worker -from DRUID.src.background import make_gaussian_sources_image - - -@pytest.fixture -def simple_image_data(): - """Creates a simple 100x100 image with one Gaussian source and noise.""" - image_size = (100, 100) - sources = [ - { - "amplitude": 100, - "x_mean": 50, - "y_mean": 50, - "x_stddev": 5, - "y_stddev": 5, - "theta": 0, - } - ] - image = make_gaussian_sources_image(image_size, sources) - image += np.random.normal(5, 1, size=image_size) # Add background and noise - return image - - -@pytest.fixture -def empty_image_data(): - """Creates an empty 100x100 image with just noise.""" - return np.random.normal(5, 1, size=(100, 100)) - +import DRUID.main as main_module # Imported to mock globals @pytest.fixture -def simple_fits_file(tmp_path, simple_image_data): - """Creates a dummy FITS file with a simple image.""" - file_path = tmp_path / "simple_image.fits" - hdu = fits.PrimaryHDU(simple_image_data) - hdu.writeto(file_path) - return str(file_path) - - -def test_sf_init_with_numpy_array(simple_image_data): - """Test sf initialization with a NumPy array.""" - finder = sf(image=simple_image_data, verbose=False) - assert isinstance(finder.image, np.ndarray) - assert np.array_equal(finder.image, simple_image_data) - - -def test_sf_init_with_fits_path(simple_fits_file, simple_image_data): - """Test sf initialization with a FITS file path.""" - finder = sf(image=simple_fits_file, verbose=False) - assert isinstance(finder.image, np.ndarray) - assert np.array_equal(finder.image, simple_image_data) - - -def test_sf_init_no_image_raises_error(): - """Test that sf initialization raises ValueError if no image is provided.""" - with pytest.raises(ValueError, match="No image provided"): - sf(verbose=False) - - -def test_sf_init_invalid_path_raises_error(): - """Test that sf initialization raises ValueError for an invalid file path.""" - with pytest.raises(ValueError, match="Could not load image from path"): - sf(image="non_existent_file.fits", verbose=False) - - -def test_sf_init_invalid_type_raises_error(): - """Test that sf initialization raises TypeError for an invalid image type.""" - with pytest.raises(TypeError, match="Image must be a file path"): - sf(image=12345, verbose=False) - - -def test_set_background(simple_image_data): - """Test the set_background method.""" - finder = sf(image=simple_image_data, verbose=False) - finder.set_background(detection_threshold=5, analysis_threshold=3) - assert hasattr(finder, "background_map") - assert hasattr(finder, "background_rms_map") - assert finder.background_map.shape == simple_image_data.shape - assert finder.background_rms_map.shape == simple_image_data.shape - assert finder.detection_threshold == 5 - assert finder.analysis_threshold == 3 - - -def test_set_background_caching(simple_image_data, tmp_path): - """Test the caching mechanism of the set_background method.""" - cache_dir = tmp_path / "druid_cache" - os.makedirs(cache_dir) - - # First run, should calculate and save - finder1 = sf( - image=simple_image_data, - verbose=False, - cashe=True, - working_directory=str(cache_dir), - ) - finder1.set_background() - - bg_map_path = cache_dir / "background_map.npy" - bg_rms_map_path = cache_dir / "background_rms_map.npy" - - assert bg_map_path.exists() - assert bg_rms_map_path.exists() - - # Second run, should load from cache - finder2 = sf( - image=simple_image_data, - verbose=False, - cashe=True, - working_directory=str(cache_dir), - ) - finder2.set_background() - - assert np.array_equal(finder1.background_map, finder2.background_map) - assert np.array_equal(finder1.background_rms_map, finder2.background_rms_map) - - shutil.rmtree(cache_dir) - - -# def test_phsf_raises_error_if_no_background(simple_image_data): -# """Test that phsf raises ValueError if background is not set.""" -# finder = sf(image=simple_image_data, verbose=False) -# with pytest.raises(ValueError, match="Background map and RMS map must be set"): -# finder.phsf() - - -def test_phsf_sequential(simple_image_data): - """Test phsf with sequential processing (num_threads=1).""" - finder = sf(image=simple_image_data, verbose=False, num_threads=1) +def pipeline_image(): + """Provides a baseline image with a single synthetic source.""" + img = np.random.normal(5, 0.5, size=(100, 100)) + img[45:55, 45:55] += 15.0 # Bright source + return img + +def test_pipeline_sequential(pipeline_image): + """Test end-to-end pipeline executing sequentially.""" + finder = sf(image=pipeline_image, verbose=False, num_threads=1, cashe=False) finder.set_background(analysis_threshold=3) finder.phsf() - + assert hasattr(finder, "catalog") assert isinstance(finder.catalog, pl.DataFrame) assert not finder.catalog.is_empty() assert "ID" in finder.catalog.columns - assert "birth" in finder.catalog.columns - - -@pytest.mark.skipif(os.cpu_count() < 2, reason="Test requires at least 2 CPU cores") -def test_phsf_parallel(simple_image_data): - """Test phsf with parallel processing (num_threads > 1).""" - finder = sf(image=simple_image_data, verbose=False, num_threads=2) - finder.set_background(analysis_threshold=3) - finder.phsf() - - assert hasattr(finder, "catalog") - assert isinstance(finder.catalog, pl.DataFrame) - assert not finder.catalog.is_empty() - assert "ID" in finder.catalog.columns - - -def test_phsf_no_sources_found(empty_image_data): - """Test phsf on an image with no sources, expecting an empty catalog.""" - finder = sf(image=empty_image_data, verbose=False) - # Set a high threshold to ensure no sources are found - finder.set_background(analysis_threshold=100) - finder.phsf() - - assert hasattr(finder, "catalog") - assert isinstance(finder.catalog, pl.DataFrame) - assert finder.catalog.is_empty() - - -def test_worker_function(simple_image_data): - """Test the internal _worker function directly.""" - # Simulate a source island cutout - island_image = simple_image_data[30:70, 30:70] - position = (30, 30) - background_rms = 1.0 # For simplicity - background = 5.0 - - iterable = (island_image, position, background, background_rms) - + assert finder.catalog["flux_peak"].max() > 10.0 + +def test_worker_function(pipeline_image): + """ + Test the inner multiprocessing worker. + Requires binding module-level globals to simulate shared memory attachment. + """ + # 1. Setup mock data + bg = np.ones((100, 100)) * 5.0 + rms = np.ones((100, 100)) * 0.5 + + # 2. Inject into the main module namespace (simulating _worker_init) + main_module.global_image = pipeline_image + main_module.global_background_map = bg + main_module.global_background_rms_map = rms + + bbox = (40, 40, 60, 60) # min_row, min_col, max_row, max_col + position = (40, 40) + island_info = (bbox, position) + + # 3. Execute worker result_cat = _worker( - iterable, + island_info, analysis_threshold=3.0, - lifetime_limit=0.1, + lifetime_limit=0.0, lifetime_limit_fraction=1.0, + mode="radio", BMAJ=2.0, BMIN=2.0 ) - + assert isinstance(result_cat, pl.DataFrame) assert not result_cat.is_empty() assert "Island_X" in result_cat.columns assert "Island_Y" in result_cat.columns - assert result_cat["Island_X"][0] == position[0] - assert result_cat["Island_Y"][0] == position[1] + + # 4. Cleanup namespace + main_module.global_image = None + main_module.global_background_map = None + main_module.global_background_rms_map = None \ No newline at end of file diff --git a/DRUID/tests/test_properties.py b/DRUID/tests/test_properties.py new file mode 100644 index 0000000..b3fe66c --- /dev/null +++ b/DRUID/tests/test_properties.py @@ -0,0 +1,46 @@ +""" +Unit tests for source property calculations (flux, SNR, geometry). +""" +import pytest +import numpy as np +import polars as pl +from DRUID.src.properties import calculate_properties, calculate_radio_flux_error + +def test_calculate_radio_flux_error(): + """Test flux error analytic derivation.""" + bg_rms = np.array([1.0, 1.0, 1.0]) + area = 100 + bmaj, bmin = 2.0, 2.0 + + err = calculate_radio_flux_error(bg_rms, area, bmaj, bmin) + assert not np.isnan(err) + assert err > 0 + +def test_calculate_properties(): + """ + Test integration of skimage.measure with Polars catalogs. + """ + image = np.zeros((30, 30)) + bg = np.zeros((30, 30)) + rms = np.ones((30, 30)) + + image[10:20, 10:20] = 5.0 + + # Mock homology dataframe row + cat = pl.DataFrame({ + "birth": [5.1], + "death": [0.0], + "x1": [15], + "y1": [15], + "area": [100] + }) + + result = calculate_properties( + cat, image, bg, rms, position=(0,0), + analysis_threshold=1.0, mode="radio", BMAJ=2.0, BMIN=2.0 + ) + + assert "flux" in result.columns + assert "maj" in result.columns + assert "snr" in result.columns + np.testing.assert_allclose(result["flux"][0], 500.0, atol=1e-3) \ No newline at end of file diff --git a/DRUID/tests/test_source.py b/DRUID/tests/test_source.py index e69de29..02eacef 100644 --- a/DRUID/tests/test_source.py +++ b/DRUID/tests/test_source.py @@ -0,0 +1,33 @@ +""" +Unit tests for initial source island thresholding and bounding box generation. +""" +import pytest +import numpy as np +from DRUID.src.source import create_source_islands + +def test_create_source_islands(): + """ + Test extraction of standard and massive source islands based on thresholds. + """ + image = np.zeros((100, 100)) + bg = np.ones((100, 100)) + rms = np.ones((100, 100)) * 0.5 + + # Create one standard source (area ~ 25) + image[20:25, 20:25] = 10 + # Create one massive source (area = 400) + image[50:70, 50:70] = 10 + + # threshold = 1 + (3 * 0.5) = 2.5 + islands = create_source_islands( + image, bg, rms, + analysis_threshold=3, area_limit=10, max_area_limit=200, verbose=False + ) + + # Check standard queue + assert len(islands["bboxes"]) == 1 + assert len(islands["positions"]) == 1 + + # Check massive queue + assert len(islands["massive_bboxes"]) == 1 + assert len(islands["massive_positions"]) == 1 \ No newline at end of file diff --git a/DRUID/tests/test_utils.py b/DRUID/tests/test_utils.py index dabe40d..1747cb2 100644 --- a/DRUID/tests/test_utils.py +++ b/DRUID/tests/test_utils.py @@ -1,93 +1,43 @@ +""" +Unit tests for DRUID utilities. +""" import pytest import numpy as np import polars as pl -from polars.testing import assert_frame_equal -from astropy.io import fits - -from DRUID.src.utils import get_image_from_path, combine_polars_catalogs - - -@pytest.fixture -def create_fits_file(tmp_path): - """A fixture to create a FITS file with given data.""" - - def _create_fits(data, filename="test.fits"): - file_path = tmp_path / filename - hdu = fits.PrimaryHDU(data) - hdu.writeto(file_path, overwrite=True) - return str(file_path) - - return _create_fits - - -def test_get_image_from_path_2d(create_fits_file): - """Test loading a standard 2D FITS image.""" - image_data = np.arange(100, dtype=np.float32).reshape(10, 10) - fits_path = create_fits_file(image_data) - - loaded_image = get_image_from_path(fits_path) - - assert isinstance(loaded_image, np.ndarray) - assert loaded_image.shape == (10, 10) - assert np.array_equal(loaded_image, image_data) - - -def test_get_image_from_path_3d_squeezes(create_fits_file): - """Test that a 3D FITS image is correctly squeezed to 2D.""" - image_data = np.arange(100, dtype=np.float32).reshape(1, 10, 10) - fits_path = create_fits_file(image_data) - - loaded_image = get_image_from_path(fits_path) - - assert loaded_image.shape == (10, 10) - assert np.array_equal(loaded_image, image_data.squeeze()) - - -def test_get_image_from_path_4d_squeezes(create_fits_file): - """Test that a 4D FITS image is correctly squeezed to 2D.""" - image_data = np.arange(100, dtype=np.float32).reshape(1, 1, 10, 10) - fits_path = create_fits_file(image_data) - - loaded_image = get_image_from_path(fits_path) - - assert loaded_image.shape == (10, 10) - assert np.array_equal(loaded_image, image_data.squeeze()) - - -def test_get_image_from_path_file_not_found(): - """Test that an error is raised for a non-existent file.""" - with pytest.raises(FileNotFoundError): - get_image_from_path("non_existent_file.fits") - - -def test_combine_polars_catalogs_basic(): - """Test combining a list of simple Polars DataFrames.""" - cat1 = pl.DataFrame({"A": [1, 2], "B": ["x", "y"]}) - cat2 = pl.DataFrame({"A": [3, 4], "B": ["z", "w"]}) - catalogs = [cat1, cat2] - - combined = combine_polars_catalogs(catalogs) - - expected = pl.DataFrame({"A": [1, 2, 3, 4], "B": ["x", "y", "z", "w"]}) - - assert_frame_equal(combined, expected) - assert combined.shape == (4, 2) - - -def test_combine_polars_catalogs_with_uppercase_id(): - """Test that a column named 'ID' (uppercase) is not re-indexed.""" - cat1 = pl.DataFrame({"ID": [0, 1], "data": [10, 20]}) - cat2 = pl.DataFrame({"ID": [0, 1], "data": [30, 40]}) - catalogs = [cat1, cat2] - - combined = combine_polars_catalogs(catalogs) - - # The 'ID' column should remain as is, with duplicates - expected = pl.DataFrame({"ID": [0, 1, 0, 1], "data": [10, 20, 30, 40]}) - assert_frame_equal(combined, expected) - - -def test_combine_polars_catalogs_empty_list(): - """Test that combining an empty list of catalogs raises a ValueError.""" - with pytest.raises(ValueError, match="No catalogs provided to combine."): +from DRUID.src.utils import combine_polars_catalogs, generate_2d_gaussian + +def test_combine_polars_catalogs(): + """ + Test the concatenation and dense ranking of catalog IDs. + """ + df1 = pl.DataFrame({"id": [1, 2], "flux": [10.5, 20.1]}) + df2 = pl.DataFrame({"id": [1, 2], "flux": [30.4, 40.2]}) + + combined = combine_polars_catalogs([df1, df2]) + + assert combined.height == 4 + # Ensure dense ranking re-indexes the IDs sequentially + assert combined["id"].to_list() == [1, 2, 3, 4] + +def test_combine_polars_catalogs_empty(): + """Ensure a ValueError is raised for empty catalog lists.""" + with pytest.raises(ValueError, match="No catalogs provided"): combine_polars_catalogs([]) + +def test_generate_2d_gaussian(): + """ + Test the 2D Gaussian generation for mathematical correctness. + """ + shape = (50, 50) + center = (25, 25) + sigma = 5 + + gauss = generate_2d_gaussian( + A=1.0, shape=shape, center=center, + sigma_x=sigma, sigma_y=sigma, norm=False + ) + + assert gauss.shape == shape + np.testing.assert_allclose(gauss[25, 25], 1.0, atol=1e-5) + # Check symmetric decay + np.testing.assert_allclose(gauss[20, 25], gauss[30, 25], atol=1e-5) \ No newline at end of file From a2e0011028f4e7bae0e5ae016a36df5cbb4753a0 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Wed, 15 Jul 2026 20:45:06 +0100 Subject: [PATCH 47/69] filtering by analysis thres --- DRUID/src/homology.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/DRUID/src/homology.py b/DRUID/src/homology.py index a442165..dc90f24 100644 --- a/DRUID/src/homology.py +++ b/DRUID/src/homology.py @@ -164,6 +164,10 @@ def compute_homology( & (pl.col("lifetime") > lifetime_limit) ) + polar_df = polar_df.filter( + pl.col("lifetime") > analysis_threshold + ) + if polar_df.is_empty(): return None From 471468209efac7898ba1a7dc4e39ef509f3d3977 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Wed, 15 Jul 2026 21:00:40 +0100 Subject: [PATCH 48/69] update to readme --- README.md | 83 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 46 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index b3b6ea4..a497169 100644 --- a/README.md +++ b/README.md @@ -2,77 +2,86 @@ [![Run tests](https://github.com/RhysAlfShaw/DRUID/actions/workflows/pytest.yaml/badge.svg)](https://github.com/RhysAlfShaw/DRUID/actions/workflows/pytest.yaml) [![codecov](https://codecov.io/gh/RhysAlfShaw/DRUID/graph/badge.svg?token=C4KD4C6IXA)](https://codecov.io/gh/RhysAlfShaw/DRUID) -DRUID is a general purpose source finder for optical and radio images written in `python`. That can be appied in a broad range of situations. It comes some GPU acceleration of key compute functions, making processing of large sources faster. +DRUID is a general-purpose source finder for optical and radio images written in Python, applicable to a broad range of scenarios. -DRUID relies on the use of Persistent homology to find sources and nested components from within the image. This information is then processed as described in Shaw et al (in prep). +DRUID relies on the use of persistent homology to find sources and nested components within an image. This information is then processed as described in Shaw et al. (in prep). -Currently DRUID uses the `criper` (https://github.com/shizuo-kaji/CubicalRipser_3dim) library to calculate the persistence of homology groups within the 2d data. +Currently, DRUID uses the [`cripser`](https://github.com/shizuo-kaji/CubicalRipser_3dim) library to calculate the persistence of homology groups within 2D data. -# Installation +## Versions -To use DRUID currently the best way is to clone this repository and install it with its dependencies. +This is the newly parallelized version of DRUID, featuring improved background data handling. The version of DRUID used in Shaw et al. (2025) can be found in the [releases](). + +### Notes on the new version +- DRUID's architecture has been significantly updated to allow for highly efficient parallelization. +- Pandas DataFrames have been replaced with Polars, drastically improving compute speed and memory management. +- GPU usage has been removed, as it was incompatible with the new parallel strategy. + +These changes have increased DRUID's speed by roughly 10x. This improvement stems mostly from the new processing architecture and the switch to Polars, with further gains driven by parallelization. To get an idea of its new performance, check out the scaling plot below. This benchmark reflects the processing of an optical image with 0.1" resolution over a 0.57 deg² field of view, containing around 100,000 sources. + +![DRUID Performance Scaling](docs/assets/DRUID_performance_scaling.png) + +## Installation + +Currently, the best way to use DRUID is to clone this repository and install it along with its dependencies: ```bash pip install . ``` -You can then test if it is working with. +You can then verify the installation by running: ```python from DRUID import sf ``` -## Note for Apple Silicon -For Apple Silcon Users! cripser does not provide compiled binaries for apple silicon. So you need to compile the library locally. This should be simple with the command: +### Note for Apple Silicon Users +`cripser` does not provide compiled binaries for Apple Silicon, so you will need to compile the library locally. This can typically be done with the following command: ```bash -pip install -U git+https://github.com/shizuo-kaji/CubicalRipser_3dim +pip install -U git+[https://github.com/shizuo-kaji/CubicalRipser_3dim](https://github.com/shizuo-kaji/CubicalRipser_3dim) ``` -Any installation error from here will likely be from the the version of CMAKE or the C compilers you have installed. See https://github.com/shizuo-kaji/CubicalRipser_3dim for further details on required compilers. - -## Using the GPU functionality +Any installation errors at this stage will likely stem from the version of CMake or the C compilers you have installed. See the [CubicalRipser_3dim repository](https://github.com/shizuo-kaji/CubicalRipser_3dim) for further details on required compilers. -To use the GPU functions that DRUID offers you need to install `cupy` [https://cupy.dev/]. -This is not done in the normal requirements as it requires access to a Nvidia GPU with Cuda. And hence requires cuda tool kit to be installed on you machine. +## Using DRUID -If you have sucessfully install cupy then you can use `GPU=True`. +To run DRUID, follow these steps: -# Using DRUID - -To use DRUID you need to do the following steps. - -1. Initailise the sf (source finding) object. +1. **Initialize the `sf` (source finding) object:** ```python -findmysource = sf(image=image,image_path=None, mode='optical',area_limit=5,GPU=True, header=header) +findmysource = sf(image=image, image_path=None, mode='optical', area_limit=5, header=header) ``` -2. Define the background. + +2. **Define the background:** ```python -findmysource.set_background(detection_threshold=5,analysis_threshold=2,mode='rms') +findmysource.set_background(detection_threshold=5, analysis_threshold=2, mode='rms') ``` -3. Find and Deblend sources with Persistent Homology. + +3. **Find and deblend sources using Persistent Homology:** ```python -findmysources.phsf() +findmysource.phsf() ``` -4. Now we have a list of sources and a hierachy of nested components, we can charaterise them and measure some properties. -```pythons -findmysources.source_charaterising(use_gpu=False) +4. **Characterize the sources:** Now that we have a list of sources and a hierarchy of nested components, we can characterize them and measure their properties. +```python +findmysource.source_characterising(use_gpu=False) ``` -To explore how DRUID can be used Check out the example notebooks where we demonstraight some of DRUIDs functions. (This is incoming, will be based on analysis in Shaw et al in prep) +To explore how DRUID can be used in practice, check out the example notebooks where we demonstrate several of DRUID's functions. *(Coming soon: based on the analysis in Shaw et al., in prep).* -## Saving the catalogue. -To save the output catalogue with the contours you should use the ```save_catalogue()``` function. As this will properly save the object. To correctly open the catalogue again use ```open_catalogue()``` after initlising the sf class. +### Saving the Catalogue +To save the output catalogue along with the contours, you should use the `save_catalogue()` function, as this will properly serialize the object. To correctly open the catalogue again, use `open_catalogue()` after initializing the `sf` class. -# Bugs/issues +## Bugs & Issues -Please report any bug or issues using DRUID to this repositories issue page. Thank you. +Please report any bugs or issues you encounter while using DRUID on this repository's [Issues](#) page. Thank you! -# Further application/developement +## Further Application & Development -If you want to increase the functionality, whether thats adding additional functionality to improving what is already implemented, feel free to submit a pull request or email me (rhys.shaw@bristol.ac.uk) to discuss. +If you want to extend DRUID's capabilities—whether that means adding new functionality or improving what is already implemented—feel free to submit a pull request or email me at [rhys.shaw@bristol.ac.uk](mailto:rhys.shaw@bristol.ac.uk) to discuss. -# Acknowledgements +## Acknowledgements -If you use DRUID for your research please cite: ([Shaw et al.2025](https://doi.org/10.1093/rasti/rzaf006)) \ No newline at end of file +If you use DRUID for your research, please cite: +> [Shaw et al. 2025](https://doi.org/10.1093/rasti/rzaf006) \ No newline at end of file From a2d0470ba98b6362ec46aca2f7ac319526006643 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Wed, 15 Jul 2026 21:05:58 +0100 Subject: [PATCH 49/69] update to readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a497169..8d6c011 100644 --- a/README.md +++ b/README.md @@ -17,9 +17,9 @@ This is the newly parallelized version of DRUID, featuring improved background d - Pandas DataFrames have been replaced with Polars, drastically improving compute speed and memory management. - GPU usage has been removed, as it was incompatible with the new parallel strategy. -These changes have increased DRUID's speed by roughly 10x. This improvement stems mostly from the new processing architecture and the switch to Polars, with further gains driven by parallelization. To get an idea of its new performance, check out the scaling plot below. This benchmark reflects the processing of an optical image with 0.1" resolution over a 0.57 deg² field of view, containing around 100,000 sources. +These changes have increased DRUID's speed by roughly 5-60x. This improvement stems from the new processing architecture and the switch to Polars, with further gains driven by parallelization. To get an idea of its new performance, check out the scaling plot below. This benchmark reflects the processing of an optical image with 0.1" resolution over a 0.57 deg² field of view, containing around 100,000 sources. -![DRUID Performance Scaling](docs/assets/DRUID_performance_scaling.png) +![DRUID Performance Scaling](./docs/assets/druid_performance_scaling.png) ## Installation From 4cf3e0a38a45effe9d3adbade050e52afc8e3a57 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Wed, 15 Jul 2026 21:10:38 +0100 Subject: [PATCH 50/69] update to requirement and setup version --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 77faf8b..bbb1806 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ numpy -pandas +polars pytest numpy scikit-image diff --git a/setup.py b/setup.py index 8facd69..6c58bf7 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='DRUID', - version='0.0.0', + version='1.0', author='Rhys Shaw', author_email='rhys.shaw@bristol.ac.uk', url='https://github.com/RhysAlfShaw/DRUID', From 12389c1a0c59fdcbd5099cd1a44fdbbbe44a1bb9 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Thu, 16 Jul 2026 14:48:07 +0100 Subject: [PATCH 51/69] added initial smoothing to begining of process --- DRUID/main.py | 108 ++++++++++++++++++--------------- DRUID/src/properties.py | 28 ++++----- DRUID/tests/test_main.py | 28 ++++----- DRUID/tests/test_properties.py | 27 ++++++--- 4 files changed, 102 insertions(+), 89 deletions(-) diff --git a/DRUID/main.py b/DRUID/main.py index aadc79f..5dd2773 100644 --- a/DRUID/main.py +++ b/DRUID/main.py @@ -15,6 +15,7 @@ from multiprocessing import shared_memory import multiprocessing from tqdm import tqdm +from scipy.ndimage import gaussian_filter from .src import utils from .src import homology @@ -54,16 +55,19 @@ # Global variables for worker processes to avoid IPC memory overhead global_image = None +global_smoothed_image = None global_background_map = None global_background_rms_map = None # Keep shared memory objects alive in the worker shm_img = None +shm_smooth = None shm_bg = None shm_rms = None def _worker_init( shm_img_name, img_shape, img_dtype, + shm_smooth_name, smooth_shape, smooth_dtype, shm_bg_name, bg_shape, bg_dtype, shm_rms_name, rms_shape, rms_dtype ): @@ -71,21 +75,21 @@ def _worker_init( Initializer for multiprocessing pool. Attaches to shared memory blocks created by the main process. """ - global global_image, global_background_map, global_background_rms_map - global shm_img, shm_bg, shm_rms + global global_image, global_smoothed_image, global_background_map, global_background_rms_map + global shm_img, shm_smooth, shm_bg, shm_rms from multiprocessing import shared_memory import numpy as np - # 1. Attach and map the main image shm_img = shared_memory.SharedMemory(name=shm_img_name) global_image = np.ndarray(shape=img_shape, dtype=img_dtype, buffer=shm_img.buf) - # 2. Attach and map the background map + shm_smooth = shared_memory.SharedMemory(name=shm_smooth_name) + global_smoothed_image = np.ndarray(shape=smooth_shape, dtype=smooth_dtype, buffer=shm_smooth.buf) + shm_bg = shared_memory.SharedMemory(name=shm_bg_name) global_background_map = np.ndarray(shape=bg_shape, dtype=bg_dtype, buffer=shm_bg.buf) - # 3. Attach and map the background RMS map shm_rms = shared_memory.SharedMemory(name=shm_rms_name) global_background_rms_map = np.ndarray(shape=rms_shape, dtype=rms_dtype, buffer=shm_rms.buf) @@ -103,41 +107,46 @@ def _worker( ) -> pl.DataFrame: """ Worker function to compute homology for a single source island. - Reads from global arrays to minimize memory serialization. """ bbox, position = island_info min_row, min_col, max_row, max_col = bbox + # Cutouts extraction raw_image_cutout = global_image[min_row:max_row, min_col:max_col] + smoothed_image_cutout = global_smoothed_image[min_row:max_row, min_col:max_col] bg_cutout = global_background_map[min_row:max_row, min_col:max_col] bg_rms_cutout = global_background_rms_map[min_row:max_row, min_col:max_col] local_threshold = bg_cutout + (analysis_threshold * bg_rms_cutout) - island_mask = raw_image_cutout > local_threshold - - image_cutout = np.where(island_mask, raw_image_cutout, 0) + + # Island mask derived strictly from the smoothed cutout + island_mask = smoothed_image_cutout > local_threshold + smoothed_cutout_masked = np.where(island_mask, smoothed_image_cutout, 0) + # Topology computed on smoothed data cat = homology.compute_homology( - image_cutout, + smoothed_cutout_masked, analysis_threshold=analysis_threshold * np.mean(bg_rms_cutout), lifetime_limit=lifetime_limit, lifetime_limit_fraction=lifetime_limit_fraction, ) if cat is not None and not cat.is_empty(): + # Properties utilize BOTH raw and smoothed arrays cat = properties.calculate_properties( cat, - image_cutout, - bg_cutout, - bg_rms_cutout, - position, - analysis_threshold, - mode, - BMAJ, - BMIN, - EFFRON, - EFFGAIN, - EXPTIME, + raw_image=raw_image_cutout, + smoothed_image=smoothed_cutout_masked, + background=bg_cutout, + background_rms=bg_rms_cutout, + position=position, + analysis_threshold=analysis_threshold, + mode=mode, + BMAJ=BMAJ, + BMIN=BMIN, + EFFRON=EFFRON, + EFFGAIN=EFFGAIN, + EXPTIME=EXPTIME, ) cat = cat.with_columns( @@ -196,13 +205,10 @@ def main(): if num_threads > 1 and multiprocessing.current_process().name == "MainProcess": try: import __main__ - if hasattr(__main__, "__file__") and os.path.exists(__main__.__file__): with open(__main__.__file__, "r") as f: script_content = f.read() - clean_script = script_content.replace(" ", "").replace("'", '"') - if 'if__name__=="__main__":' not in clean_script: raise RuntimeError(error_msg) except Exception as e: @@ -210,7 +216,6 @@ def main(): raise e self.no_message = no_message - if not self.no_message: print(DRUID_MESSAGE) @@ -223,6 +228,7 @@ def main(): self.chunksize = chunksize self.header = header self.cashe = cashe + self.smoothed_image = None if image is None: raise ValueError( @@ -273,14 +279,24 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0 ): raise ValueError( "Background maps must be set before running source finding." - ) + ) + + t0 = time.time() + + # Apply structural smoothing before thresholding + if self.smooth_sigma > 0: + if self.verbose: + print(f"Applying Gaussian smoothing with sigma={self.smooth_sigma}...") + self.smoothed_image = gaussian_filter(self.image, sigma=self.smooth_sigma) + else: + self.smoothed_image = self.image if self.verbose: print("Thresholding to find source islands...") t0 = time.time() source_islands = source.create_source_islands( - self.image, + self.smoothed_image, self.background_map, self.background_rms_map, detection_threshold=self.detection_threshold, @@ -293,12 +309,11 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0 if self.verbose: print(f"Thresholding took {t1 - t0:.2f} seconds.") - print(f"Found {len(source_islands['positions'])} source islands.") + print(f"Found {len(source_islands['bboxes'])} source islands.") iterable_islands = list( zip(source_islands["bboxes"], source_islands["positions"]) ) - iterable_islands.sort( key=lambda item: (item[0][2] - item[0][0]) * (item[0][3] - item[0][1]), reverse=True, @@ -306,7 +321,7 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0 if not iterable_islands: if self.verbose: - print("No source islands to process.") + print("No source islands found. Returning empty catalog.") self.catalog = pl.DataFrame() return @@ -329,16 +344,16 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0 if self.num_threads > 1: if self.verbose: print(f"Processing in parallel with {self.num_threads} threads.") - optimal_chunksize = self.chunksize - # 1. Create shared memory blocks for all three arrays + # Shared memory allocations shm_img = shared_memory.SharedMemory(create=True, size=self.image.nbytes) + shm_smooth = shared_memory.SharedMemory(create=True, size=self.smoothed_image.nbytes) shm_bg = shared_memory.SharedMemory(create=True, size=self.background_map.nbytes) shm_rms = shared_memory.SharedMemory(create=True, size=self.background_rms_map.nbytes) - # 2. Copy the data into the shared memory buffers np.ndarray(self.image.shape, dtype=self.image.dtype, buffer=shm_img.buf)[:] = self.image[:] + np.ndarray(self.smoothed_image.shape, dtype=self.smoothed_image.dtype, buffer=shm_smooth.buf)[:] = self.smoothed_image[:] np.ndarray(self.background_map.shape, dtype=self.background_map.dtype, buffer=shm_bg.buf)[:] = self.background_map[:] np.ndarray(self.background_rms_map.shape, dtype=self.background_rms_map.dtype, buffer=shm_rms.buf)[:] = self.background_rms_map[:] @@ -347,11 +362,11 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0 initializer=_worker_init, initargs=( shm_img.name, self.image.shape, self.image.dtype, + shm_smooth.name, self.smoothed_image.shape, self.smoothed_image.dtype, shm_bg.name, self.background_map.shape, self.background_map.dtype, shm_rms.name, self.background_rms_map.shape, self.background_rms_map.dtype ) ) as p: - results = list( tqdm( p.imap_unordered( @@ -361,32 +376,31 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0 ), total=len(iterable_islands), disable=not self.verbose, - desc="Computing Homology", + desc="Computing", dynamic_ncols=True ) ) - # 3. Clean up shared memory in the main process + # Flush memory shm_img.close() shm_img.unlink() + shm_smooth.close() + shm_smooth.unlink() shm_bg.close() shm_bg.unlink() shm_rms.close() shm_rms.unlink() else: - if self.verbose: - print("Processing sequentially.") - - # Safely bind module-level globals for single-threaded execution - global global_image, global_background_map, global_background_rms_map + global global_image, global_smoothed_image, global_background_map, global_background_rms_map global_image = self.image + global_smoothed_image = self.smoothed_image global_background_map = self.background_map global_background_rms_map = self.background_rms_map for island in tqdm( iterable_islands, disable=not self.verbose, - desc="Computing Homology", + desc="Computing", dynamic_ncols=True ): results.append(worker_func(island)) @@ -409,13 +423,9 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0 def set_background( - self, - method: str = "rms", - detection_threshold: int = 5, - analysis_threshold: int = 3, - box_size: tuple = (50, 50), - filter_size: tuple = (3, 3), - kernel_size: int = 3, + self, method: str = "rms", detection_threshold: int = 5, + analysis_threshold: int = 3, box_size: tuple = (50, 50), + filter_size: tuple = (3, 3), kernel_size: int = 3, ): if self.verbose: print("Calculating background map and RMS map...") diff --git a/DRUID/src/properties.py b/DRUID/src/properties.py index a78045e..bf60412 100644 --- a/DRUID/src/properties.py +++ b/DRUID/src/properties.py @@ -37,7 +37,8 @@ def optical_flux_err(EFFRON, EFFGAIN, EXPTIME, Area, sky, Flux): def calculate_properties( cat, - image, + raw_image, + smoothed_image, background, background_rms, position, @@ -59,19 +60,12 @@ def calculate_properties( areas = cat["area"].to_numpy() maj, min_ax, pa, centroid_lst, flux, flux_peak, bg, flux_err, snr = ( - [], - [], - [], - [], - [], - [], - [], - [], - [], + [], [], [], [], [], [], [], [], [] ) for b, d, x, y, area in zip(births, deaths, x1s, y1s, areas): - mask = (image <= b) & (image > d) + # mask boundaries determined by the SMOOTHED image + mask = (smoothed_image <= b) & (smoothed_image > d) enclosed_mask = get_enclosing_mask_CPU(int(y), int(x), mask) if enclosed_mask is None: @@ -88,7 +82,9 @@ def calculate_properties( continue enclosed_mask_int = enclosed_mask.astype(int) - props = measure.regionprops(enclosed_mask_int, intensity_image=image) + + # Geometries and intensities extracted from the RAW image + props = measure.regionprops(enclosed_mask_int, intensity_image=raw_image) if props: p = props[0] @@ -102,9 +98,10 @@ def calculate_properties( pa.append(np.nan) centroid_lst.append((np.nan, np.nan)) - flux_tot = np.nansum(enclosed_mask_int * (image - background)) + # Flux summations computed on RAW data + flux_tot = np.nansum(enclosed_mask_int * (raw_image - background)) flux.append(flux_tot) - flux_peak.append(np.nanmax(enclosed_mask_int * (image - background))) + flux_peak.append(np.nanmax(enclosed_mask_int * (raw_image - background))) bg_mean = np.mean(background * enclosed_mask_int) bg.append(bg_mean) @@ -112,7 +109,6 @@ def calculate_properties( if mode == "radio": f_err = calculate_radio_flux_error(background_rms, area, BMAJ, BMIN) flux_err.append(f_err) - # Safely calculate SNR, handling NaN and zero division if f_err and not np.isnan(f_err): snr.append(flux_tot / f_err) else: @@ -146,4 +142,4 @@ def calculate_properties( pl.Series("flux", flux), pl.Series("snr", snr), ] - ) + ) \ No newline at end of file diff --git a/DRUID/tests/test_main.py b/DRUID/tests/test_main.py index fe4238b..9504284 100644 --- a/DRUID/tests/test_main.py +++ b/DRUID/tests/test_main.py @@ -4,8 +4,9 @@ import pytest import numpy as np import polars as pl +from scipy.ndimage import gaussian_filter from DRUID.main import sf, _worker -import DRUID.main as main_module # Imported to mock globals +import DRUID.main as main_module @pytest.fixture def pipeline_image(): @@ -14,9 +15,9 @@ def pipeline_image(): img[45:55, 45:55] += 15.0 # Bright source return img -def test_pipeline_sequential(pipeline_image): - """Test end-to-end pipeline executing sequentially.""" - finder = sf(image=pipeline_image, verbose=False, num_threads=1, cashe=False) +def test_pipeline_sequential_with_smoothing(pipeline_image): + """Test end-to-end pipeline with smooth_sigma > 0.""" + finder = sf(image=pipeline_image, verbose=False, num_threads=1, smooth_sigma=1.0) finder.set_background(analysis_threshold=3) finder.phsf() @@ -24,27 +25,27 @@ def test_pipeline_sequential(pipeline_image): assert isinstance(finder.catalog, pl.DataFrame) assert not finder.catalog.is_empty() assert "ID" in finder.catalog.columns - assert finder.catalog["flux_peak"].max() > 10.0 + # Ensure smoothed_image was allocated + assert getattr(finder, "smoothed_image", None) is not None def test_worker_function(pipeline_image): """ - Test the inner multiprocessing worker. - Requires binding module-level globals to simulate shared memory attachment. + Test the inner multiprocessing worker with raw and smoothed global arrays. """ - # 1. Setup mock data bg = np.ones((100, 100)) * 5.0 rms = np.ones((100, 100)) * 0.5 + smoothed_image = gaussian_filter(pipeline_image, sigma=1.0) - # 2. Inject into the main module namespace (simulating _worker_init) + # Inject into the main module namespace main_module.global_image = pipeline_image + main_module.global_smoothed_image = smoothed_image main_module.global_background_map = bg main_module.global_background_rms_map = rms - bbox = (40, 40, 60, 60) # min_row, min_col, max_row, max_col + bbox = (40, 40, 60, 60) position = (40, 40) island_info = (bbox, position) - # 3. Execute worker result_cat = _worker( island_info, analysis_threshold=3.0, @@ -55,10 +56,9 @@ def test_worker_function(pipeline_image): assert isinstance(result_cat, pl.DataFrame) assert not result_cat.is_empty() - assert "Island_X" in result_cat.columns - assert "Island_Y" in result_cat.columns - # 4. Cleanup namespace + # Cleanup namespace main_module.global_image = None + main_module.global_smoothed_image = None main_module.global_background_map = None main_module.global_background_rms_map = None \ No newline at end of file diff --git a/DRUID/tests/test_properties.py b/DRUID/tests/test_properties.py index b3fe66c..5cf2a42 100644 --- a/DRUID/tests/test_properties.py +++ b/DRUID/tests/test_properties.py @@ -18,29 +18,36 @@ def test_calculate_radio_flux_error(): def test_calculate_properties(): """ - Test integration of skimage.measure with Polars catalogs. + Test properties calculated on raw image while mask bounds dictate via smoothed image. """ - image = np.zeros((30, 30)) + raw_image = np.zeros((30, 30)) + smoothed_image = np.zeros((30, 30)) bg = np.zeros((30, 30)) rms = np.ones((30, 30)) - image[10:20, 10:20] = 5.0 + # Simulate a bright sharp core in raw image + raw_image[10:20, 10:20] = 5.0 + + # Simulate a wider, dimmer dispersion in smoothed image + smoothed_image[8:22, 8:22] = 2.0 - # Mock homology dataframe row cat = pl.DataFrame({ - "birth": [5.1], + "birth": [2.1], # Mask encompasses smoothed_image's 2.0 block "death": [0.0], "x1": [15], "y1": [15], - "area": [100] + "area": [196] # (14 x 14 block area) }) result = calculate_properties( - cat, image, bg, rms, position=(0,0), + cat, raw_image=raw_image, smoothed_image=smoothed_image, + background=bg, background_rms=rms, position=(0,0), analysis_threshold=1.0, mode="radio", BMAJ=2.0, BMIN=2.0 ) - assert "flux" in result.columns - assert "maj" in result.columns - assert "snr" in result.columns + # Peak flux should pull from the 5.0 raw array, NOT the 2.0 smoothed one + assert result["flux_peak"][0] == 5.0 + + # Total flux is the 10x10 core inside the 14x14 bounds + # (100 pixels * 5.0) + (96 pixels * 0.0) = 500 np.testing.assert_allclose(result["flux"][0], 500.0, atol=1e-3) \ No newline at end of file From 18ddf0a96e60e376c05e8b82d78e521bf8dbc6f9 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Tue, 28 Jul 2026 14:34:28 +0100 Subject: [PATCH 52/69] pyproject update --- pyproject.toml | 46 ++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 15 --------------- 2 files changed, 46 insertions(+), 15 deletions(-) create mode 100644 pyproject.toml delete mode 100644 setup.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..5257498 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,46 @@ +[build-system] +requires = ["setuptools>=61.0.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "DRUID" +version = "1.0" +description = "Astronomical Source finder build with Persistent Homology." +readme = "README.md" +requires-python = ">=3.12" +authors = [ + { name = "Rhys Shaw", email = "rhys.shaw@bristol.ac.uk" } +] +classifiers = [ + "Intended Audience :: Science/Research", + "Programming Language :: Python :: 3.12", + "Topic :: Scientific/Engineering :: Astronomy", + "Topic :: Scientific/Engineering :: Physics", +] + +dependencies = [ + "numpy", + "polars", + "scipy", + "astropy", + "scikit-image", + "photutils", + "cripser", + "bottleneck", + "matplotlib", + "tqdm", + "setproctitle" +] + +[project.urls] +Homepage = "https://github.com/RhysAlfShaw/DRUID" +Repository = "https://github.com/RhysAlfShaw/DRUID.git" + +[project.optional-dependencies] +test = [ + "pytest>=7.0", + "pytest-cov" +] + +[tool.setuptools.packages.find] +include = ["DRUID*"] \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100644 index 6c58bf7..0000000 --- a/setup.py +++ /dev/null @@ -1,15 +0,0 @@ -from setuptools import setup, find_packages - -with open('requirements.txt') as f: - requiremets = f.read().splitlines() - -setup( - name='DRUID', - version='1.0', - author='Rhys Shaw', - author_email='rhys.shaw@bristol.ac.uk', - url='https://github.com/RhysAlfShaw/DRUID', - description='descriptions', - install_requires=requiremets, - packages=find_packages(), -) \ No newline at end of file From 470787eec4685c7246d60ada2cc5854295ef7a65 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Tue, 28 Jul 2026 14:58:34 +0100 Subject: [PATCH 53/69] update to readme installation --- README.md | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8d6c011..9be38de 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ DRUID is a general-purpose source finder for optical and radio images written in Python, applicable to a broad range of scenarios. -DRUID relies on the use of persistent homology to find sources and nested components within an image. This information is then processed as described in Shaw et al. (in prep). +DRUID relies on the use of persistent homology to find sources and nested components within an image. This information is then processed as described in Shaw et al. (2025). Currently, DRUID uses the [`cripser`](https://github.com/shizuo-kaji/CubicalRipser_3dim) library to calculate the persistence of homology groups within 2D data. @@ -25,6 +25,18 @@ These changes have increased DRUID's speed by roughly 5-60x. This improvement st Currently, the best way to use DRUID is to clone this repository and install it along with its dependencies: +```bash +git clone https://github.com/RhysAlfShaw/DRUID.git +cd DRUID +``` + +### Conda + +Create conda environement: +```bash +conda env create -f enviroment.yml +``` + ```bash pip install . ``` @@ -34,6 +46,22 @@ You can then verify the installation by running: ```python from DRUID import sf ``` +### UV + +For a faster install with a single command using uv, simply. + +```bash +uv sync --python 3.12 +``` + +uv will automatically detect the requirements and install DRUID. Test as above or with + +```bash +uv run python -c "from DRUID import sf" +``` + +No errors indicates a successful install. + ### Note for Apple Silicon Users `cripser` does not provide compiled binaries for Apple Silicon, so you will need to compile the library locally. This can typically be done with the following command: @@ -55,7 +83,7 @@ findmysource = sf(image=image, image_path=None, mode='optical', area_limit=5, he 2. **Define the background:** ```python -findmysource.set_background(detection_threshold=5, analysis_threshold=2, mode='rms') +findmysource.set_background(detection_threshold=5, analysis_threshold=2) ``` 3. **Find and deblend sources using Persistent Homology:** From 693bba7a54ba5e095674fc62319ff7760fbdbb33 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Tue, 28 Jul 2026 16:06:02 +0100 Subject: [PATCH 54/69] cashe -> cache --- DRUID/main.py | 148 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 98 insertions(+), 50 deletions(-) diff --git a/DRUID/main.py b/DRUID/main.py index 5dd2773..7e6ee9d 100644 --- a/DRUID/main.py +++ b/DRUID/main.py @@ -65,11 +65,20 @@ shm_bg = None shm_rms = None + def _worker_init( - shm_img_name, img_shape, img_dtype, - shm_smooth_name, smooth_shape, smooth_dtype, - shm_bg_name, bg_shape, bg_dtype, - shm_rms_name, rms_shape, rms_dtype + shm_img_name, + img_shape, + img_dtype, + shm_smooth_name, + smooth_shape, + smooth_dtype, + shm_bg_name, + bg_shape, + bg_dtype, + shm_rms_name, + rms_shape, + rms_dtype, ): """ Initializer for multiprocessing pool. @@ -77,21 +86,28 @@ def _worker_init( """ global global_image, global_smoothed_image, global_background_map, global_background_rms_map global shm_img, shm_smooth, shm_bg, shm_rms - + from multiprocessing import shared_memory import numpy as np - + shm_img = shared_memory.SharedMemory(name=shm_img_name) global_image = np.ndarray(shape=img_shape, dtype=img_dtype, buffer=shm_img.buf) - + shm_smooth = shared_memory.SharedMemory(name=shm_smooth_name) - global_smoothed_image = np.ndarray(shape=smooth_shape, dtype=smooth_dtype, buffer=shm_smooth.buf) - + global_smoothed_image = np.ndarray( + shape=smooth_shape, dtype=smooth_dtype, buffer=shm_smooth.buf + ) + shm_bg = shared_memory.SharedMemory(name=shm_bg_name) - global_background_map = np.ndarray(shape=bg_shape, dtype=bg_dtype, buffer=shm_bg.buf) - + global_background_map = np.ndarray( + shape=bg_shape, dtype=bg_dtype, buffer=shm_bg.buf + ) + shm_rms = shared_memory.SharedMemory(name=shm_rms_name) - global_background_rms_map = np.ndarray(shape=rms_shape, dtype=rms_dtype, buffer=shm_rms.buf) + global_background_rms_map = np.ndarray( + shape=rms_shape, dtype=rms_dtype, buffer=shm_rms.buf + ) + def _worker( island_info, @@ -118,7 +134,7 @@ def _worker( bg_rms_cutout = global_background_rms_map[min_row:max_row, min_col:max_col] local_threshold = bg_cutout + (analysis_threshold * bg_rms_cutout) - + # Island mask derived strictly from the smoothed cutout island_mask = smoothed_image_cutout > local_threshold smoothed_cutout_masked = np.where(island_mask, smoothed_image_cutout, 0) @@ -172,7 +188,7 @@ def __init__( chunksize: int = 10, header: astropy.io.fits.header.Header = None, working_directory: str = "./druid-working-dir", - cashe: bool = False, + cache: bool = False, no_message: bool = False, ): error_msg = f""" @@ -205,6 +221,7 @@ def main(): if num_threads > 1 and multiprocessing.current_process().name == "MainProcess": try: import __main__ + if hasattr(__main__, "__file__") and os.path.exists(__main__.__file__): with open(__main__.__file__, "r") as f: script_content = f.read() @@ -227,7 +244,7 @@ def main(): self.num_threads = num_threads self.chunksize = chunksize self.header = header - self.cashe = cashe + self.cache = cache self.smoothed_image = None if image is None: @@ -248,7 +265,7 @@ def main(): "Image must be a file path (str) or a NumPy array (np.ndarray)." ) - if self.cashe: + if self.cache: if not os.path.exists(working_directory): os.makedirs(working_directory) self.working_directory = working_directory @@ -279,10 +296,10 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0 ): raise ValueError( "Background maps must be set before running source finding." - ) + ) t0 = time.time() - + # Apply structural smoothing before thresholding if self.smooth_sigma > 0: if self.verbose: @@ -344,43 +361,69 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0 if self.num_threads > 1: if self.verbose: print(f"Processing in parallel with {self.num_threads} threads.") - optimal_chunksize = self.chunksize - + optimal_chunksize = self.chunksize + # Shared memory allocations shm_img = shared_memory.SharedMemory(create=True, size=self.image.nbytes) - shm_smooth = shared_memory.SharedMemory(create=True, size=self.smoothed_image.nbytes) - shm_bg = shared_memory.SharedMemory(create=True, size=self.background_map.nbytes) - shm_rms = shared_memory.SharedMemory(create=True, size=self.background_rms_map.nbytes) + shm_smooth = shared_memory.SharedMemory( + create=True, size=self.smoothed_image.nbytes + ) + shm_bg = shared_memory.SharedMemory( + create=True, size=self.background_map.nbytes + ) + shm_rms = shared_memory.SharedMemory( + create=True, size=self.background_rms_map.nbytes + ) - np.ndarray(self.image.shape, dtype=self.image.dtype, buffer=shm_img.buf)[:] = self.image[:] - np.ndarray(self.smoothed_image.shape, dtype=self.smoothed_image.dtype, buffer=shm_smooth.buf)[:] = self.smoothed_image[:] - np.ndarray(self.background_map.shape, dtype=self.background_map.dtype, buffer=shm_bg.buf)[:] = self.background_map[:] - np.ndarray(self.background_rms_map.shape, dtype=self.background_rms_map.dtype, buffer=shm_rms.buf)[:] = self.background_rms_map[:] + np.ndarray(self.image.shape, dtype=self.image.dtype, buffer=shm_img.buf)[ + : + ] = self.image[:] + np.ndarray( + self.smoothed_image.shape, + dtype=self.smoothed_image.dtype, + buffer=shm_smooth.buf, + )[:] = self.smoothed_image[:] + np.ndarray( + self.background_map.shape, + dtype=self.background_map.dtype, + buffer=shm_bg.buf, + )[:] = self.background_map[:] + np.ndarray( + self.background_rms_map.shape, + dtype=self.background_rms_map.dtype, + buffer=shm_rms.buf, + )[:] = self.background_rms_map[:] with get_context("spawn").Pool( self.num_threads, initializer=_worker_init, initargs=( - shm_img.name, self.image.shape, self.image.dtype, - shm_smooth.name, self.smoothed_image.shape, self.smoothed_image.dtype, - shm_bg.name, self.background_map.shape, self.background_map.dtype, - shm_rms.name, self.background_rms_map.shape, self.background_rms_map.dtype - ) + shm_img.name, + self.image.shape, + self.image.dtype, + shm_smooth.name, + self.smoothed_image.shape, + self.smoothed_image.dtype, + shm_bg.name, + self.background_map.shape, + self.background_map.dtype, + shm_rms.name, + self.background_rms_map.shape, + self.background_rms_map.dtype, + ), ) as p: results = list( tqdm( p.imap_unordered( - worker_func, - iterable_islands, - chunksize=optimal_chunksize + worker_func, iterable_islands, chunksize=optimal_chunksize ), total=len(iterable_islands), disable=not self.verbose, desc="Computing", - dynamic_ncols=True + dynamic_ncols=True, ) ) - + # Flush memory shm_img.close() shm_img.unlink() @@ -396,12 +439,12 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0 global_smoothed_image = self.smoothed_image global_background_map = self.background_map global_background_rms_map = self.background_rms_map - + for island in tqdm( - iterable_islands, - disable=not self.verbose, - desc="Computing", - dynamic_ncols=True + iterable_islands, + disable=not self.verbose, + desc="Computing", + dynamic_ncols=True, ): results.append(worker_func(island)) @@ -413,19 +456,24 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0 t1 = time.time() if self.verbose: - print(f"Homology computation took {t1 - t0:.2f} seconds.") + print(f"Homology computation took {t1 - t0:.2f} seconds.") print("---------------CATALOG SUMMARY---------------------") print(f"Total sources detected: {self.catalog.height}") - print(f"Number of large sources (area > {self.max_area_limit}): {self.catalog.filter(pl.col('area') > self.max_area_limit).height}") + print( + f"Number of large sources (area > {self.max_area_limit}): {self.catalog.filter(pl.col('area') > self.max_area_limit).height}" + ) print("Average Background: ", self.background_map.mean()) print("Average Background RMS: ", self.background_rms_map.mean()) print("---------------------------------------------------") - def set_background( - self, method: str = "rms", detection_threshold: int = 5, - analysis_threshold: int = 3, box_size: tuple = (50, 50), - filter_size: tuple = (3, 3), kernel_size: int = 3, + self, + method: str = "rms", + detection_threshold: int = 5, + analysis_threshold: int = 3, + box_size: tuple = (50, 50), + filter_size: tuple = (3, 3), + kernel_size: int = 3, ): if self.verbose: print("Calculating background map and RMS map...") @@ -436,7 +484,7 @@ def set_background( bg_file = os.path.join(self.working_directory or "", "background_map.npy") rms_file = os.path.join(self.working_directory or "", "background_rms_map.npy") - if self.cashe and os.path.exists(bg_file) and os.path.exists(rms_file): + if self.cache and os.path.exists(bg_file) and os.path.exists(rms_file): if self.verbose: print("Background maps exist. Loading from disk.") self.background_map = np.load(bg_file) @@ -452,7 +500,7 @@ def set_background( kernel_size=kernel_size, ) ) - if self.cashe: + if self.cache: np.save(bg_file, self.background_map) np.save(rms_file, self.background_rms_map) From 34dd99fe79f40cc0cf1f8c5f93e70e9be1bad8e8 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Wed, 29 Jul 2026 11:21:02 +0100 Subject: [PATCH 55/69] update to readme --- DRUID/main.py | 10 +++++++++ README.md | 57 +++++++++++++++++++++++++++++++++++++++------------ 2 files changed, 54 insertions(+), 13 deletions(-) diff --git a/DRUID/main.py b/DRUID/main.py index 7e6ee9d..67b71a8 100644 --- a/DRUID/main.py +++ b/DRUID/main.py @@ -189,6 +189,7 @@ def __init__( header: astropy.io.fits.header.Header = None, working_directory: str = "./druid-working-dir", cache: bool = False, + output_arg: str = "", no_message: bool = False, ): error_msg = f""" @@ -245,6 +246,7 @@ def main(): self.chunksize = chunksize self.header = header self.cache = cache + self.output_arg = output_arg self.smoothed_image = None if image is None: @@ -451,6 +453,14 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0 results = [res for res in results if res is not None and not res.is_empty()] if results: self.catalog = utils.combine_polars_catalogs(results) + # save catalog to working directory + if self.cache and self.working_directory: + catalog_file = os.path.join( + self.working_directory, + f"druid_source_catalog_{self.output_arg}.fits", + ) + self.catalog.write_parquet(catalog_file) + else: self.catalog = pl.DataFrame() diff --git a/README.md b/README.md index 9be38de..c20f6e7 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ cd DRUID Create conda environement: ```bash -conda env create -f enviroment.yml +conda env create -f environment.yml ``` ```bash @@ -78,36 +78,67 @@ To run DRUID, follow these steps: 1. **Initialize the `sf` (source finding) object:** ```python -findmysource = sf(image=image, image_path=None, mode='optical', area_limit=5, header=header) +findmysource = sf( + image=image, # image, either a 2d np.array, or path to fits file. + mode="optical", + area_limit=5, # Helps remove noise sources. + smooth_sigma=1, # smooth image before ph analysis, fluxes measured on original image. + num_threads=2, # number of threads, as num_threads increases speed gains decrease. + chunksize=20, # chuncking size for multithreading, only provides minor speedup. + max_area_limit = 1E5, # there are size limits on ph analysis this prevent unintentional infinate compute time. + working_directory=".", # where to save outputs and cache results. + cache=False, # Cache/save results to working directory. + ) ``` 2. **Define the background:** ```python -findmysource.set_background(detection_threshold=5, analysis_threshold=2) +findmysource.set_background( + method='mad_std', # background statistic (sex,rms,mad_std...) + detection_threshold=5, # how many sigmas above the background should we call a source. + analysis_threshold=3, # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ we analyse for a source. + box_size=10 # boz size for creating a background map. + ) ``` 3. **Find and deblend sources using Persistent Homology:** ```python -findmysource.phsf() +findmysource.phsf(, + lifetime_limit = 0, # float value for this limit + lifetime_limit_fraction=1.2 # fraction based on birth and death. + ) ``` -4. **Characterize the sources:** Now that we have a list of sources and a hierarchy of nested components, we can characterize them and measure their properties. +This function also calculates source properties. + + +## Runnig in parallel. + +To prevent issues with pythons multiprocessing functionality. DRUID will not run unless wrapped in a __main__. If you run this inside a jupyter notebook __main__ is not necessary. + ```python -findmysource.source_characterising(use_gpu=False) -``` +from DRUID import sf + +def main(): + findmysource = sf( + image=image, + mode="optical", + area_limit=5, + num_threads=2, + ) + findmysource.set_background() + findmysource.phsf() -To explore how DRUID can be used in practice, check out the example notebooks where we demonstrate several of DRUID's functions. *(Coming soon: based on the analysis in Shaw et al., in prep).* +if __name__ == "__main__": + main() -### Saving the Catalogue -To save the output catalogue along with the contours, you should use the `save_catalogue()` function, as this will properly serialize the object. To correctly open the catalogue again, use `open_catalogue()` after initializing the `sf` class. +``` ## Bugs & Issues Please report any bugs or issues you encounter while using DRUID on this repository's [Issues](#) page. Thank you! -## Further Application & Development - -If you want to extend DRUID's capabilities—whether that means adding new functionality or improving what is already implemented—feel free to submit a pull request or email me at [rhys.shaw@bristol.ac.uk](mailto:rhys.shaw@bristol.ac.uk) to discuss. +or email me at [rhys.shaw@bristol.ac.uk](mailto:rhys.shaw@bristol.ac.uk). ## Acknowledgements From 77be78dc2c9eb527cf223922e02be1fd8560b995 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Wed, 29 Jul 2026 11:51:39 +0100 Subject: [PATCH 56/69] function for calculating ra and dec coords --- DRUID/src/utils.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/DRUID/src/utils.py b/DRUID/src/utils.py index 4907ae3..03e2999 100644 --- a/DRUID/src/utils.py +++ b/DRUID/src/utils.py @@ -20,7 +20,7 @@ def combine_polars_catalogs(catalogs: list) -> pl.DataFrame: raise ValueError("No catalogs provided to combine.") combined_catalog = pl.concat(catalogs) - + # Check for 'id' or 'ID' depending on your upstream schema if "id" in combined_catalog.columns: combined_catalog = combined_catalog.with_columns( @@ -33,6 +33,7 @@ def combine_polars_catalogs(catalogs: list) -> pl.DataFrame: return combined_catalog + def generate_2d_gaussian(A, shape, center, sigma_x, sigma_y, angle_deg=0, norm=True): x, y = np.meshgrid(np.arange(shape[1]), np.arange(shape[0])) x_c, y_c = center @@ -50,3 +51,16 @@ def generate_2d_gaussian(A, shape, center, sigma_x, sigma_y, angle_deg=0, norm=T def model_beam_func(peak_flux, shape, x, y, bmaj, bmin, bpa): return generate_2d_gaussian(peak_flux, shape, (x, y), bmaj, bmin, bpa, norm=False) + + +def calculate_radec(catalog: pl.DataFrame, header) -> pl.DataFrame: + # Convert pixel coordinates to RA/Dec using WCS. + from astropy.wcs import WCS + + wcs = WCS(header) + ra_dec = wcs.all_pix2world( + catalog["centroid_x"].to_numpy(), catalog["centroid_y"].to_numpy(), 0 + ) + ra, dec = ra_dec + catalog = catalog.with_columns(pl.Series("ra", ra), pl.Series("dec", dec)) + return catalog From 26e8abb81626e3eb9a22e1e0486e4d646d3db0de Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Wed, 29 Jul 2026 12:07:16 +0100 Subject: [PATCH 57/69] catalog order and island offset --- DRUID/main.py | 76 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/DRUID/main.py b/DRUID/main.py index 67b71a8..32cacb3 100644 --- a/DRUID/main.py +++ b/DRUID/main.py @@ -10,6 +10,7 @@ import sys import time import polars as pl +import polars.selectors as cs from functools import partial from multiprocessing import get_context from multiprocessing import shared_memory @@ -453,6 +454,81 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0 results = [res for res in results if res is not None and not res.is_empty()] if results: self.catalog = utils.combine_polars_catalogs(results) + # calculate the ra and dec columns if the header is available + # add island offsets to the centroid and contour coordinates + self.catalog = self.catalog.with_columns( + [ + (pl.col("centroid_x") + pl.col("Island_X")).alias("centroid_x"), + (pl.col("centroid_y") + pl.col("Island_Y")).alias("centroid_y"), + ] + ) + # add island offsets to the contour coordinates + self.catalog = ( + self.catalog.with_row_index("__row_id") + .explode("contour") + .with_columns( + pl.concat_list( + [ + pl.col("contour").list.get(0) + pl.col("Island_X"), + pl.col("contour").list.get(1) + pl.col("Island_Y"), + ] + ).alias("contour") + ) + .group_by("__row_id", maintain_order=True) + .agg( + pl.all().exclude("contour").first(), + pl.col("contour"), + ) + .drop("__row_id") + ) + if self.header is not None: + self.catalog = utils.calculate_radec(self.catalog, self.header) + + else: + print( + "Warning: No FITS header provided. RA and Dec columns will not be calculated." + ) + self.catalog = self.catalog.with_columns( + pl.lit(None).alias("ra"), pl.lit(None).alias("dec") + ) + desired_order = [ + "ID", + "ra", + "dec", + "centroid_x", + "centroid_y", + "flux", + "flux_peak", + "flux_err", + "bg", + "snr", + "maj", + "min", + "pa", + "area", + "contour", + "lifetime", + "birth", + "death", + "x1", + "y1", + "x2", + "y2", + "encloses", + "new_row", + "parent_tag", + "lifetimeFrac", + "bbox_min_y", + "bbox_min_x", + "bbox_max_y", + "bbox_max_x", + "Island_X", + "Island_Y", + ] + self.catalog = self.catalog.select( + *desired_order, cs.all().exclude(desired_order) + ) + # save catalog to working directory if self.cache and self.working_directory: catalog_file = os.path.join( From 40c4db729ff08bd7d9434bbe9d747816c18203c5 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Wed, 29 Jul 2026 12:37:05 +0100 Subject: [PATCH 58/69] saving file as parquet automatically --- DRUID/main.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/DRUID/main.py b/DRUID/main.py index 32cacb3..c527366 100644 --- a/DRUID/main.py +++ b/DRUID/main.py @@ -6,6 +6,7 @@ import numpy as np import astropy.io.fits +from astropy.table import Table import os import sys import time @@ -267,13 +268,10 @@ def main(): raise TypeError( "Image must be a file path (str) or a NumPy array (np.ndarray)." ) - - if self.cache: + self.working_directory = working_directory + if self.working_directory: if not os.path.exists(working_directory): os.makedirs(working_directory) - self.working_directory = working_directory - else: - self.working_directory = None self.BMAJ, self.BMIN = None, None self.EFFRON, self.EFFGAIN, self.EXPTIME = None, None, None @@ -517,6 +515,7 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0 "encloses", "new_row", "parent_tag", + "class", "lifetimeFrac", "bbox_min_y", "bbox_min_x", @@ -530,13 +529,14 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0 ) # save catalog to working directory - if self.cache and self.working_directory: - catalog_file = os.path.join( - self.working_directory, - f"druid_source_catalog_{self.output_arg}.fits", - ) - self.catalog.write_parquet(catalog_file) + catalog_file = os.path.join( + self.working_directory, + f"druid_source_catalog_{self.output_arg}", + ) + print(catalog_file) + self.catalog.write_parquet(f"{catalog_file}.parquet") + print(f"Catalog saved to {catalog_file}.parquet") else: self.catalog = pl.DataFrame() From 81ecf7790e56df992d418a2c99bdf069e06d566f Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Wed, 29 Jul 2026 12:37:24 +0100 Subject: [PATCH 59/69] adding missing ph class --- DRUID/src/homology.py | 36 ++++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/DRUID/src/homology.py b/DRUID/src/homology.py index dc90f24..864bed7 100644 --- a/DRUID/src/homology.py +++ b/DRUID/src/homology.py @@ -43,10 +43,10 @@ def _get_polygons_CPU(x1, y1, birth, death, image: np.ndarray): # Shift coordinates back due to padding contour[:, 0] -= 1 contour[:, 1] -= 1 - + # skimage returns (row, col). Convert to standard (x, y) for plotting contour_xy = np.column_stack((contour[:, 1], contour[:, 0])) - + return contour_xy.tolist() @@ -131,6 +131,30 @@ def parent_tag_func_pl(df: pl.DataFrame) -> pl.DataFrame: ).drop("parent_id") +def assign_ph_class(df: pl.DataFrame) -> pl.DataFrame: + # assign a class based on parent_tag, new_row, and encloses length + + is_new_row = pl.col("new_row") != 0 + has_children = pl.col("encloses").list.len() > 1 + has_parent = pl.col("parent_tag") != pl.col("ID") + + # 2. Chain the logic + df = df.with_columns( + pl.when(is_new_row) + .then(1) + .when(~has_children & ~has_parent) + .then(0) + .when(~has_children & has_parent) + .then(2) + .when(has_children & ~has_parent) + .then(4) + .otherwise(3) + .alias("class") # Replace with your desired column name + ) + + return df + + def compute_homology( img: np.ndarray, analysis_threshold: float, @@ -164,9 +188,7 @@ def compute_homology( & (pl.col("lifetime") > lifetime_limit) ) - polar_df = polar_df.filter( - pl.col("lifetime") > analysis_threshold - ) + polar_df = polar_df.filter(pl.col("lifetime") > analysis_threshold) if polar_df.is_empty(): return None @@ -243,6 +265,8 @@ def compute_homology( ) ] + polar_df = assign_ph_class(polar_df) + return polar_df.with_columns( pl.Series("contour", contours, dtype=pl.List(pl.List(pl.Float64))) - ) \ No newline at end of file + ) From 047612da81b951bd4b9fddec1ebee667de886936 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Wed, 29 Jul 2026 15:33:45 +0100 Subject: [PATCH 60/69] fix from merger --- DRUID/main.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/DRUID/main.py b/DRUID/main.py index eecadd3..109ad5b 100644 --- a/DRUID/main.py +++ b/DRUID/main.py @@ -21,11 +21,6 @@ from scipy import ndimage import logging -DRUID_MESSAGE = """ - - -############################################# - from .src import utils from .src import homology from .src import background @@ -40,7 +35,7 @@ BLUE = "\033[94m" RESET = "\033[0m" BOLD = "\033[1m" -DRUID_MESSAGE = rf""" +DRUID_MESSAGE = f""" {RED}#############################################{RESET} {GREEN} _______ _______ _________ ______ From eccc6a281378ed5af447273ea14b6570f2a7bcf9 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Wed, 29 Jul 2026 16:42:11 +0100 Subject: [PATCH 61/69] fix depreciated arguments --- DRUID/src/background.py | 2 +- DRUID/src/properties.py | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/DRUID/src/background.py b/DRUID/src/background.py index 9669ca6..a4fc93f 100644 --- a/DRUID/src/background.py +++ b/DRUID/src/background.py @@ -21,7 +21,7 @@ def make_source_mask(data, nsigma=3.0, kernel_size=3): mean, median, std = sigma_clipped_stats(data, sigma=nsigma) threshold = median + nsigma * std - segm = detect_sources(data, threshold, npixels=kernel_size**2) + segm = detect_sources(data, threshold, n_pixels=kernel_size**2) if segm is None: return np.zeros(data.shape, dtype=bool) return segm.data > 0 diff --git a/DRUID/src/properties.py b/DRUID/src/properties.py index bf60412..12e9b08 100644 --- a/DRUID/src/properties.py +++ b/DRUID/src/properties.py @@ -60,7 +60,15 @@ def calculate_properties( areas = cat["area"].to_numpy() maj, min_ax, pa, centroid_lst, flux, flux_peak, bg, flux_err, snr = ( - [], [], [], [], [], [], [], [], [] + [], + [], + [], + [], + [], + [], + [], + [], + [], ) for b, d, x, y, area in zip(births, deaths, x1s, y1s, areas): @@ -82,14 +90,14 @@ def calculate_properties( continue enclosed_mask_int = enclosed_mask.astype(int) - + # Geometries and intensities extracted from the RAW image props = measure.regionprops(enclosed_mask_int, intensity_image=raw_image) if props: p = props[0] - maj.append(p.major_axis_length) - min_ax.append(p.minor_axis_length) + maj.append(p.axis_major_length) + min_ax.append(p.axis_minor_length) pa.append(p.orientation) centroid_lst.append(p.centroid) else: @@ -142,4 +150,4 @@ def calculate_properties( pl.Series("flux", flux), pl.Series("snr", snr), ] - ) \ No newline at end of file + ) From 9f6fd1829eb7d0e694c10a5305980e06c282842b Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Wed, 29 Jul 2026 17:15:17 +0100 Subject: [PATCH 62/69] increase beautiy factor --- DRUID/main.py | 183 +++++++++++++++++++++++++++----------------- DRUID/src/source.py | 47 ++++++++---- DRUID/src/utils.py | 13 ++++ 3 files changed, 158 insertions(+), 85 deletions(-) diff --git a/DRUID/main.py b/DRUID/main.py index 109ad5b..328afc8 100644 --- a/DRUID/main.py +++ b/DRUID/main.py @@ -8,6 +8,7 @@ import astropy.io.fits from astropy.table import Table import os +import ast import sys import time import polars as pl @@ -15,6 +16,7 @@ from functools import partial from multiprocessing import get_context from multiprocessing import shared_memory +from rich.progress import Progress import multiprocessing from tqdm import tqdm from scipy.ndimage import gaussian_filter @@ -22,6 +24,19 @@ import logging from .src import utils +from .src.utils import ( + TITLE, + LINK, + GOLD, + RESET, + BOLD, + NOTICE, + ERROR, + WARNING, + CODEBLOCK, + GREEN, + BLACK, +) from .src import homology from .src import background from .src import source @@ -30,34 +45,25 @@ # Prevent Polars from thread oversubscription during multiprocessing os.environ["POLARS_MAX_THREADS"] = "1" -RED = "\033[91m" -GREEN = "\033[92m" -BLUE = "\033[94m" -RESET = "\033[0m" -BOLD = "\033[1m" -DRUID_MESSAGE = f""" -{RED}#############################################{RESET} -{GREEN} -_______ _______ _________ ______ -( __ \ ( ____ )|\ /|\__ __/( __ \ -| ( \ )| ( )|| ) ( | ) ( | ( \ ) -| | ) || (____)|| | | | | | | | ) | -| | | || __)| | | | | | | | | | -| | ) || (\ ( | | | | | | | | ) | -| (__/ )| ) \ \__| (___) |___) (___| (__/ ) -(______/ |/ \__/(_______)\_______/(______/ + +DRUID_MESSAGE = rf""" +{TITLE} + _____ _____ _ _ _____ _____ + | __ \| __ \| | | |_ _| __ \ + | | | | |__) | | | | | | | | | | + | | | | _ /| | | | | | | | | | + | |__| | | \ \| |__| |_| |_| |__| | + |_____/|_| \_\\____/|_____|_____/ {RESET} -{RED}#############################################{RESET} {BOLD}Detector of astRonomical soUrces in optIcal and raDio images{RESET} -Version: {version} +{GOLD}Version{RESET}: {version} For more information see: -{BLUE}https://github.com/RhysAlfShaw/DRUID{RESET} +{LINK}https://github.com/RhysAlfShaw/DRUID{RESET} """ -# Global variables for worker processes to avoid IPC memory overhead global_image = None global_smoothed_image = None global_background_map = None @@ -197,8 +203,8 @@ def __init__( no_message: bool = False, ): error_msg = f""" - {RED}===================================================================={RESET} - {BOLD}DRUID MULTIPROCESSING ERROR{RESET} + {ERROR}===================================================================={RESET} + {BOLD}DRUID MULTIPROCESSING {ERROR}ERROR!{RESET} It looks like you are running DRUID with `num_threads > 1` without protecting your execution code. @@ -206,9 +212,9 @@ def __init__( Because DRUID uses Python's robust multiprocessing, you must wrap your top-level code in the `if __name__ == '__main__':` block. - {BLUE}Please update your script to look like this:{RESET} + {BOLD}Please update your script to look like this:{RESET} - from DRUID import sf + {BLACK}from DRUID import sf def main(): findmysource = sf(num_threads={num_threads}, ...) @@ -217,25 +223,43 @@ def main(): if __name__ == "__main__": main() - {RED}===================================================================={RESET} + {ERROR}===================================================================={RESET} """ if multiprocessing.current_process().name != "MainProcess": raise RuntimeError(error_msg) - if num_threads > 1 and multiprocessing.current_process().name == "MainProcess": try: import __main__ - if hasattr(__main__, "__file__") and os.path.exists(__main__.__file__): - with open(__main__.__file__, "r") as f: - script_content = f.read() - clean_script = script_content.replace(" ", "").replace("'", '"') - if 'if__name__=="__main__":' not in clean_script: - raise RuntimeError(error_msg) + if not hasattr(__main__, "__file__") or not os.path.exists( + __main__.__file__ + ): + raise RuntimeError( + f"{error_msg} (Cannot verify script safety in interactive environments)" + ) + + with open(__main__.__file__, "r", encoding="utf-8") as f: + source_code = f.read() + + tree = ast.parse(source_code) + + is_protected = False + for node in tree.body: + if isinstance(node, ast.If): + if isinstance(node.test, ast.Compare): + left = node.test.left + if isinstance(left, ast.Name) and left.id == "__name__": + is_protected = True + break + + if not is_protected: + raise RuntimeError(error_msg) + except Exception as e: - if isinstance(e, RuntimeError): - raise e + raise RuntimeError( + f"Failed to validate safe multiprocessing execution: {e}" + ) self.no_message = no_message if not self.no_message: @@ -255,20 +279,22 @@ def main(): if image is None: raise ValueError( - "No image provided. Please provide a file path or a NumPy array." + f"{ERROR}No image provided. Please provide a file path or a NumPy array.{RESET}" ) if isinstance(image, str): try: self.image, self.header = utils.get_image_from_path(image) except Exception as e: - raise ValueError(f"Could not load image from path: {image}") from e + raise ValueError( + f"{ERROR}Could not load image from path{RESET}: {image}" + ) from e elif isinstance(image, np.ndarray): self.image = image self.header = header else: raise TypeError( - "Image must be a file path (str) or a NumPy array (np.ndarray)." + f"{ERROR}Image must be a file path (str) or a NumPy array (np.ndarray).{RESET}" ) self.working_directory = working_directory if self.working_directory: @@ -283,14 +309,18 @@ def main(): self.BMAJ = self.header.get("BMAJ") self.BMIN = self.header.get("BMIN") except KeyError: - print("Warning: Could not find BMAJ or BMIN in header.") + print( + f"{WARNING}Warning: Could not find BMAJ or BMIN in header.{RESET}" + ) elif self.mode == "optical" and self.header: try: self.EFFRON = self.header.get("EFFRON") self.EFFGAIN = self.header.get("EFFGAIN") self.EXPTIME = self.header.get("EXPTIME") except KeyError: - print("Warning: Could not find EFFRON, EFFGAIN, or EXPTIME.") + print( + f"{WARNING}Warning: Could not find EFFRON, EFFGAIN, or EXPTIME.{RESET}" + ) def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0): if ( @@ -298,7 +328,7 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0 or getattr(self, "background_rms_map", None) is None ): raise ValueError( - "Background maps must be set before running source finding." + f"{ERROR}Background maps must be set before running source finding.{RESET}" ) t0 = time.time() @@ -306,13 +336,15 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0 # Apply structural smoothing before thresholding if self.smooth_sigma > 0: if self.verbose: - print(f"Applying Gaussian smoothing with sigma={self.smooth_sigma}...") + print( + f"{NOTICE}Applying Gaussian smoothing with sigma={self.smooth_sigma}...{RESET}" + ) self.smoothed_image = gaussian_filter(self.image, sigma=self.smooth_sigma) else: self.smoothed_image = self.image if self.verbose: - print("Thresholding to find source islands...") + print(f"{NOTICE}Thresholding to find source islands...{RESET}") t0 = time.time() source_islands = source.create_source_islands( @@ -328,8 +360,10 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0 t1 = time.time() if self.verbose: - print(f"Thresholding took {t1 - t0:.2f} seconds.") - print(f"Found {len(source_islands['bboxes'])} source islands.") + print(f"{NOTICE}Thresholding took {t1 - t0:.2f} seconds.{RESET}") + print( + f"{NOTICE}Found {len(source_islands['bboxes'])} source islands.{RESET}" + ) iterable_islands = list( zip(source_islands["bboxes"], source_islands["positions"]) @@ -341,7 +375,9 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0 if not iterable_islands: if self.verbose: - print("No source islands found. Returning empty catalog.") + print( + "{WARNING}Warning: No source islands found. Returning empty catalog.{RESET}" + ) self.catalog = pl.DataFrame() return @@ -363,7 +399,9 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0 results = [] if self.num_threads > 1: if self.verbose: - print(f"Processing in parallel with {self.num_threads} threads.") + print( + f"{NOTICE}Processing in parallel with {self.num_threads} threads.{RESET}" + ) optimal_chunksize = self.chunksize # Shared memory allocations @@ -415,17 +453,17 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0 self.background_rms_map.dtype, ), ) as p: - results = list( - tqdm( - p.imap_unordered( - worker_func, iterable_islands, chunksize=optimal_chunksize - ), - total=len(iterable_islands), - disable=not self.verbose, - desc="Computing", - dynamic_ncols=True, + with Progress(disable=not self.verbose) as progress: + task = progress.add_task( + "[magenta]:mage: Computing...", total=len(iterable_islands) ) - ) + + results = [] + for result in p.imap_unordered( + worker_func, iterable_islands, chunksize=optimal_chunksize + ): + results.append(result) + progress.advance(task) # Update the progress bar incrementally # Flush memory shm_img.close() @@ -443,13 +481,15 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0 global_background_map = self.background_map global_background_rms_map = self.background_rms_map - for island in tqdm( - iterable_islands, - disable=not self.verbose, - desc="Computing", - dynamic_ncols=True, - ): - results.append(worker_func(island)) + # Single-threaded Rich progress bar implementation + with Progress(disable=not self.verbose) as progress: + task = progress.add_task( + f"[magenta]:mage: Computing...", total=len(iterable_islands) + ) + + for island in iterable_islands: + results.append(worker_func(island)) + progress.advance(task) results = [res for res in results if res is not None and not res.is_empty()] if results: @@ -486,7 +526,7 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0 else: print( - "Warning: No FITS header provided. RA and Dec columns will not be calculated." + f"{WARNING}Warning{RESET}: No FITS header provided. RA and Dec columns will not be calculated." ) self.catalog = self.catalog.with_columns( pl.lit(None).alias("ra"), pl.lit(None).alias("dec") @@ -535,24 +575,23 @@ def phsf(self, lifetime_limit: float = 0.0, lifetime_limit_fraction: float = 1.0 self.working_directory, f"druid_source_catalog_{self.output_arg}", ) - print(catalog_file) self.catalog.write_parquet(f"{catalog_file}.parquet") - print(f"Catalog saved to {catalog_file}.parquet") + print(f"{NOTICE}Catalog saved to {catalog_file}.parquet{RESET}") else: self.catalog = pl.DataFrame() t1 = time.time() if self.verbose: - print(f"Homology computation took {t1 - t0:.2f} seconds.") - print("---------------CATALOG SUMMARY---------------------") + print(f"{NOTICE}Homology computation took {t1 - t0:.2f} seconds.{RESET}") + print(f"{GREEN}---------------CATALOG SUMMARY---------------------{RESET}") print(f"Total sources detected: {self.catalog.height}") print( f"Number of large sources (area > {self.max_area_limit}): {self.catalog.filter(pl.col('area') > self.max_area_limit).height}" ) print("Average Background: ", self.background_map.mean()) print("Average Background RMS: ", self.background_rms_map.mean()) - print("---------------------------------------------------") + print(f"{GREEN}---------------------------------------------------{RESET}") def set_background( self, @@ -564,7 +603,7 @@ def set_background( kernel_size: int = 3, ): if self.verbose: - print("Calculating background map and RMS map...") + print(f"{NOTICE}Calculating background map and RMS map...{RESET}") t0 = time.time() self.detection_threshold = detection_threshold self.analysis_threshold = analysis_threshold @@ -574,7 +613,7 @@ def set_background( if self.cache and os.path.exists(bg_file) and os.path.exists(rms_file): if self.verbose: - print("Background maps exist. Loading from disk.") + print(f"{NOTICE}Background maps exist. Loading from disk.{RESET}") self.background_map = np.load(bg_file) self.background_rms_map = np.load(rms_file) else: @@ -594,4 +633,4 @@ def set_background( t1 = time.time() if self.verbose: - print(f"Background calculation took {t1 - t0:.2f} seconds.") + print(f"{NOTICE}Background calculation took {t1 - t0:.2f} seconds.{RESET}") diff --git a/DRUID/src/source.py b/DRUID/src/source.py index c799ef4..4889829 100644 --- a/DRUID/src/source.py +++ b/DRUID/src/source.py @@ -2,6 +2,20 @@ from skimage.measure import regionprops_table, label import polars as pl +from ..src.utils import ( + TITLE, + LINK, + GOLD, + RESET, + BOLD, + NOTICE, + ERROR, + WARNING, + CODEBLOCK, + BLACK, +) + + def create_source_islands( image, background_map, @@ -18,14 +32,16 @@ def create_source_islands( Separates massive islands for cataloging without computing homology. """ if verbose: - print("Step 1: Applying analysis threshold and labeling connected components...") + print( + f"{NOTICE}Applying analysis threshold and labeling connected components...{RESET}" + ) # Vectorized boolean mask creation analysis_mask = image > (background_map + analysis_threshold * background_rms_map) labeled_image = label(analysis_mask, connectivity=2) if verbose: - print("Step 2: Measuring region properties...") + print(f"{NOTICE}Measuring region properties...{RESET}") # Use regionprops_table for C-level fast property extraction properties_table = regionprops_table( @@ -37,23 +53,28 @@ def create_source_islands( props_df = pl.DataFrame(properties_table) if verbose: - print(f"Initial regions found: {props_df.height}") - print(f"Step 3: Filtering regions by area ({area_limit} <= area <= {max_area_limit} pixels)...") + print(f"{NOTICE}Initial regions found: {props_df.height}{RESET}") + print( + f"{NOTICE}Filtering regions by area ({area_limit} <= area <= {max_area_limit} pixels)...{RESET}" + ) # ---> FAST POLARS FILTERING <--- # Standard processing queue filtered_props_df = props_df.filter( - (pl.col("area") >= area_limit) & - (pl.col("area") <= max_area_limit) + (pl.col("area") >= area_limit) & (pl.col("area") <= max_area_limit) ) - + # Flagged massive islands massive_props_df = props_df.filter(pl.col("area") > max_area_limit) if verbose: if massive_props_df.height > 0: - print(f"Flagged {massive_props_df.height} massive region(s) to retain for the final catalog.") - print(f"Regions queued for homology processing: {filtered_props_df.height}") + print( + f"{WARNING}WARNING{RESET}: Flagged {massive_props_df.height} massive region(s) to retain for the final catalog.{RESET}" + ) + print( + f"{NOTICE}Regions queued for homology processing: {filtered_props_df.height}{RESET}" + ) # Extract standard metadata (for the multiprocessing pool) bboxes = list( @@ -92,11 +113,11 @@ def create_source_islands( source_islands = { "bboxes": bboxes, "positions": positions, - "massive_bboxes": massive_bboxes, # <-- New: Saved massive bounding boxes - "massive_positions": massive_positions, # <-- New: Saved massive coordinates + "massive_bboxes": massive_bboxes, # <-- New: Saved massive bounding boxes + "massive_positions": massive_positions, # <-- New: Saved massive coordinates } if verbose: - print("Source island creation complete.") + print(f"{NOTICE}Source island creation complete.{RESET}") - return source_islands \ No newline at end of file + return source_islands diff --git a/DRUID/src/utils.py b/DRUID/src/utils.py index 03e2999..dd7595c 100644 --- a/DRUID/src/utils.py +++ b/DRUID/src/utils.py @@ -2,6 +2,19 @@ import numpy as np from astropy.io import fits +RED = "\033[38;5;34m" +TITLE = "\033[38;5;75m" +LINK = "\033[38;5;21m" +GOLD = "\033[38;5;184m" +RESET = "\033[0m" +BOLD = "\033[1m" +NOTICE = f"{BOLD}{GOLD}Info{RESET}: " +ERROR = "\033[38;5;160m" +WARNING = "\033[38;5;220m" +CODEBLOCK = "\033[48;5;231m" +BLACK = "\033[38;5;16m" +GREEN = "\033[38;5;34m" + def get_image_from_path(image_path): with fits.open(image_path) as hdul: From b431b243b77595c887d71de44c98d47cb860a7a3 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Wed, 29 Jul 2026 17:16:05 +0100 Subject: [PATCH 63/69] update to required packages --- pyproject.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5257498..cb4bc23 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,8 @@ dependencies = [ "bottleneck", "matplotlib", "tqdm", - "setproctitle" + "setproctitle", + "rich>=15.0.0", ] [project.urls] @@ -43,4 +44,4 @@ test = [ ] [tool.setuptools.packages.find] -include = ["DRUID*"] \ No newline at end of file +include = ["DRUID*"] From 92fb0fe12060a8ce633004ca4126cb6b8e19eac3 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Wed, 29 Jul 2026 17:16:34 +0100 Subject: [PATCH 64/69] update ignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index f00d4c8..fe64310 100644 --- a/.gitignore +++ b/.gitignore @@ -23,4 +23,5 @@ build backup notepad.ipynb temp -_* \ No newline at end of file +_* +*.lock \ No newline at end of file From 9431dafbfe30242276f2e57405250276544e8b01 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Wed, 29 Jul 2026 17:17:00 +0100 Subject: [PATCH 65/69] later python version --- .github/workflows/pytest.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index f36b1ce..fd2d71a 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Set up Python 3.9 + - name: Set up Python 3.12 uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} From 7d7e3dad9b327eb66078442bd7f6e5ea8d27b7cc Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Wed, 29 Jul 2026 17:36:51 +0100 Subject: [PATCH 66/69] update for tests --- DRUID/src/background.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DRUID/src/background.py b/DRUID/src/background.py index a4fc93f..9669ca6 100644 --- a/DRUID/src/background.py +++ b/DRUID/src/background.py @@ -21,7 +21,7 @@ def make_source_mask(data, nsigma=3.0, kernel_size=3): mean, median, std = sigma_clipped_stats(data, sigma=nsigma) threshold = median + nsigma * std - segm = detect_sources(data, threshold, n_pixels=kernel_size**2) + segm = detect_sources(data, threshold, npixels=kernel_size**2) if segm is None: return np.zeros(data.shape, dtype=bool) return segm.data > 0 From da737ff30e4642dfef5bd99fb740672a19005348 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Wed, 29 Jul 2026 17:40:17 +0100 Subject: [PATCH 67/69] update fix --- DRUID/main.py | 4 ---- requirements.txt | 13 ++++++------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/DRUID/main.py b/DRUID/main.py index 328afc8..a20bc17 100644 --- a/DRUID/main.py +++ b/DRUID/main.py @@ -9,7 +9,6 @@ from astropy.table import Table import os import ast -import sys import time import polars as pl import polars.selectors as cs @@ -18,10 +17,7 @@ from multiprocessing import shared_memory from rich.progress import Progress import multiprocessing -from tqdm import tqdm from scipy.ndimage import gaussian_filter -from scipy import ndimage -import logging from .src import utils from .src.utils import ( diff --git a/requirements.txt b/requirements.txt index bbb1806..951f4d1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,11 @@ numpy polars -pytest -numpy -scikit-image +scipy astropy -tqdm +scikit-image +photutils +cripser +bottleneck matplotlib setproctitle -scipy -photutils -cripser \ No newline at end of file +rich>=15.0.0 \ No newline at end of file From 82fc5fd2405d89d4785dfebe1b430ac21b93d4a2 Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Wed, 29 Jul 2026 17:41:49 +0100 Subject: [PATCH 68/69] update to general type --- DRUID/tests/test_background.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/DRUID/tests/test_background.py b/DRUID/tests/test_background.py index a1aba44..e9137a9 100644 --- a/DRUID/tests/test_background.py +++ b/DRUID/tests/test_background.py @@ -1,36 +1,40 @@ """ Unit tests for background and RMS map estimation. """ + import pytest import numpy as np from astropy.io import fits from photutils.background import MedianBackground from DRUID.src.background import make_source_mask, calculate_background_maps + @pytest.fixture def dummy_fits_file(tmp_path) -> str: """Fixture to generate a dummy FITS file with a noise floor.""" - rng = np.random.default_rng(42) # Reproducible seed + rng = np.random.default_rng(42) # Reproducible seed data = rng.uniform(5, 15, size=(100, 100)) file_path = tmp_path / "dummy_image.fits" fits.PrimaryHDU(data).writeto(file_path) return str(file_path) + def test_calculate_background_maps_defaults(dummy_fits_file): """Test background map generation with default estimators.""" bg_map, rms_map = calculate_background_maps(dummy_fits_file) - + assert bg_map.shape == (100, 100) assert rms_map.shape == (100, 100) - assert bg_map.dtype == np.float64 + assert bg_map.dtype.type == np.float64 assert np.all(bg_map > 0) assert np.all(rms_map > 0) + def test_calculate_background_maps_array_input(): """Test background map generation passing a NumPy array directly.""" rng = np.random.default_rng(42) data = rng.normal(10, 1, size=(50, 50)) bg_map, rms_map = calculate_background_maps(data, bg_estimator=MedianBackground()) - + assert bg_map.shape == (50, 50) - np.testing.assert_allclose(np.median(bg_map), 10.0, rtol=0.1) \ No newline at end of file + np.testing.assert_allclose(np.median(bg_map), 10.0, rtol=0.1) From 1a107e6e71c590f6952bcff0a2fe2a5178f3e94f Mon Sep 17 00:00:00 2001 From: Rhys Shaw Date: Thu, 30 Jul 2026 10:19:29 +0100 Subject: [PATCH 69/69] change to photoutils optical background estimate --- DRUID/src/properties.py | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/DRUID/src/properties.py b/DRUID/src/properties.py index 12e9b08..e017b45 100644 --- a/DRUID/src/properties.py +++ b/DRUID/src/properties.py @@ -27,12 +27,14 @@ def calculate_radio_flux_error(background_rms, area, BMAJ, BMIN): return np.mean(background_rms) * np.sqrt(area / Beam_area) -def optical_flux_err(EFFRON, EFFGAIN, EXPTIME, Area, sky, Flux): - try: - RON_noise = np.sqrt(Area) * (EFFRON / EFFGAIN) * EXPTIME - except Exception: - RON_noise = 0 - return np.sqrt(RON_noise**2 + np.sqrt(sky) + np.sqrt(Flux)) +def optical_flux_err(data, bkg, eff_gain): + # https://photutils.readthedocs.io/en/stable/api/photutils.utils.calc_total_error.html + # photoutils implentation of optical flux error calculations. + # f_err = sqrt(obk^2 + (I/g_eff)^2 + (obk/rms_median)^2) + + from photutils.utils import calc_total_error + + return calc_total_error(data, bkg, eff_gain) def calculate_properties( @@ -77,7 +79,6 @@ def calculate_properties( enclosed_mask = get_enclosing_mask_CPU(int(y), int(x), mask) if enclosed_mask is None: - # Fallback for empty/invalid properties maj.append(np.nan) min_ax.append(np.nan) pa.append(np.nan) @@ -91,7 +92,7 @@ def calculate_properties( enclosed_mask_int = enclosed_mask.astype(int) - # Geometries and intensities extracted from the RAW image + # properties extracted from the RAW image props = measure.regionprops(enclosed_mask_int, intensity_image=raw_image) if props: @@ -123,14 +124,9 @@ def calculate_properties( snr.append(np.nan) elif mode == "optical": - f_err = optical_flux_err( - EFFRON, - EFFGAIN, - EXPTIME, - area, - np.nansum(background * enclosed_mask_int), - flux_tot, - ) + if EFFGAIN is None: + EFFGAIN = 0 # Default to 1 if not provided + f_err = optical_flux_err(raw_image, background, EFFGAIN).mean() flux_err.append(f_err) snr.append(flux_tot / f_err if f_err else 0) else: