diff --git a/pipeline/bundling/coadder.py b/pipeline/bundling/coadder.py index 8334788..68a079f 100644 --- a/pipeline/bundling/coadder.py +++ b/pipeline/bundling/coadder.py @@ -218,7 +218,7 @@ def bundle(self, bundle_id, map_dir, split_label=None, null_prop_val=None, indicating the inter-observation null split that observations belong to. abscal: dict - Nested dict in format {'ws0': {'f090': 1, 'f150': 1}, ...}. + Nested dict in format {'f090': {'ws0': 1, 'ws1': 1, ...}, ...}. Maps will be multiplied by the abscal factor. parallelizor: tuple (MPICommExecutor or ProcessPoolExecutor, as_completed_callable, num_workers) @@ -276,7 +276,7 @@ def __init__(self, bundle_db, freq_channel, car_map_template, map_dir, Optional; label indicating the telescope wafer to include. If no wafer is provided, coadd maps made for all the wafers. abscal: dict - Nested dict in format {'ws0': {'f090': 1, 'f150': 1}, ...}. + Nested dict in format {'f090': {'ws0': 1, 'ws1': 1, ...}, ...}. Maps will be multiplied by the abscal factor. bundle_id: int Optional; ID corresponding to the bundle that observations belong @@ -295,7 +295,8 @@ def __init__(self, bundle_db, freq_channel, car_map_template, map_dir, bundle_info = self._get_bundle_info(bundle_id, map_dir, null_prop_val=null_prop_val, split_label=split_label) self.fnames = list(bundle_info['filename']) - self.ws = bundle_info['weight'].to_numpy() + # copy(): pandas >= 3.0 returns a read-only view, but ws is scaled in place below + self.ws = bundle_info['weight'].to_numpy().copy() self.full_abscal = utils.get_abscal(abscal, bundle_info['wafer'], bundle_info['freq_channel']) self.ws *= self.full_abscal**-2 # ivar gets -2 powers of abscal diff --git a/pipeline/filtering/coadd_filtered_ext.py b/pipeline/filtering/coadd_filtered_ext.py new file mode 100644 index 0000000..518f407 --- /dev/null +++ b/pipeline/filtering/coadd_filtered_ext.py @@ -0,0 +1,394 @@ +""" +Based on coadd_filtered_ext.py, +adapted to run on the external data. +""" +import numpy as np +import argparse +import sqlite3 +import os +import sys +import time +import tracemalloc +from itertools import product + +import sotodlib.preprocess.preprocess_util as pp_util + +# TODO: Make it an actual module +sys.path.append( + os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'bundling')) +) +sys.path.append( + os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'filtering')) +) +sys.path.append( + os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'misc')) +) +import mpi_utils as mpi # noqa +import bundling_utils as bu # noqa +import filtering_utils as fu # noqa +import coordinator as coord # noqa + + +def main(args): + """ + """ + if args.pix_type not in ["hp", "car"]: + raise ValueError("Unknown pixel type, must be 'car' or 'hp'.") + if "{patch" not in args.bundle_db: + raise ValueError("bundle_db does not have \ + required placeholder 'patch'") + for required_tag in ["{sim_type"]: + if required_tag not in args.sim_string_format: + raise ValueError(f"sim_string_format does not have \ + required placeholder '{required_tag}'") + for required_tag in ["{patch", "{freq_channel"]: + if required_tag not in args.output_dir: + raise ValueError(f"output_dir does not have \ + required placeholder '{required_tag}'") + for required_tag in ["{patch", "{freq_channel"]: + if required_tag not in args.atomic_sim_dir: + raise ValueError(f"atomic_sim_dir does not have \ + required placeholder '{required_tag}'") + + # MPI related initialization + rank, size, comm = mpi.init(True) + + # Initialize the logger + logger = pp_util.init_logger("benchmark", verbosity=3) + if rank == 0: + start = time.time() + + # Ensure that freq_channels for the metadata follow the "f090" convention. + # We keep the original labels in a dict called freq_labels. + freq_labels = {} + for f in args.freq_channels: # If these don't contain the "f", add + freq_channel = f"f{f}" if "f" not in f else f + freq_labels[freq_channel] = f # dict values are the original labels + freq_channels = list(freq_labels.keys()) + + # Sim related arguments + sim_types = args.sim_types + sim_string_format = args.sim_string_format + + # Input directory + atomic_sim_dir = args.atomic_sim_dir + + # Output directories + patches = args.patches + out_dirs = { + (patch, freq_channel, sim_type): + args.output_dir.format( + patch=patch, freq_channel=freq_labels[freq_channel], sim_type=sim_type + ) + for patch, freq_channel, sim_type in product(patches, freq_channels, sim_types) + } + coadded_dirs = {key: f"{out_dir}/coadded_maps" + for key, out_dir in out_dirs.items()} + plot_dirs = {key: f"{out_dir}/plots" for key, out_dir in out_dirs.items()} + + for key in out_dirs: + os.makedirs(coadded_dirs[key], exist_ok=True) + os.makedirs(plot_dirs[key], exist_ok=True) + + # Pixelization arguments + pix_type = args.pix_type + if pix_type == "hp": + mfmt = ".fits" # TODO: test fits.gz for HEALPix + car_map_template = None + elif pix_type == "car": + mfmt = ".fits" + car_map_template = args.car_map_template + + # Databases + atom_db = args.atomic_db + bundle_dbs = {patch: args.bundle_db.format(patch=patch) + for patch in patches} + + # Bundle query arguments + freq_channel = args.freq_channels + inter_obs_splits = args.inter_obs_splits + if inter_obs_splits is None: + inter_obs_splits = [] + if isinstance(inter_obs_splits, str): + if "," in inter_obs_splits: + inter_obs_splits = inter_obs_splits.split(",") + else: + inter_obs_splits = [inter_obs_splits] + + # Load all data from bundle dbs without filtering + bundle_id = args.bundle_id + bundles = { + patch: + coord.BundleCoordinator.from_dbfile( + bundle_dbs[patch], bundle_id=bundle_id + ) for patch in patches + } + + # Gather all split labels of atomics to be coadded + intra_obs_splits = args.intra_obs_splits + intra_obs_pair = args.intra_obs_pair + if intra_obs_splits is not None: + if isinstance(intra_obs_splits, str): + if "," not in intra_obs_splits: + intra_obs_splits = [intra_obs_splits] + else: + intra_obs_splits = (intra_obs_splits).split(",") + if intra_obs_pair is not None: + if isinstance(intra_obs_pair, str): + if "," not in intra_obs_pair: + raise ValueError("You must pass a comma-separated string list " + "to 'intra_obs_pair'.") + else: + intra_obs_pair = intra_obs_pair.split(",") + if ((intra_obs_splits, intra_obs_pair) == (None, None)): + raise ValueError("You must pass at least one of the two: " + "'intra_obs_pair' or 'intra_obs_splits'.") + + logger.info(f"Split labels to coadd individually: {intra_obs_splits}") + logger.info(f"Split labels to coadd together: {intra_obs_pair}") + + # Extract list of ctimes from bundle database for the given + # bundle_id and without atomic batches - inter obs null label + ctimes = { + (patch, inter_obs_split, None): + bundles[patch].get_ctimes( + bundle_id=bundle_id, null_prop_val=inter_obs_split + ) for inter_obs_split in inter_obs_splits + for patch in patches + } + # Add the science split without atomic batches + for patch in patches: + ctimes[patch, "science", None] = bundles[patch].get_ctimes( + bundle_id=bundle_id + ) + + # Randomly split science ctimes into batches (optional) + nbatches = args.nbatch_atomics + if nbatches is not None: + nbatches_dict = {} + for patch in patches: + # Limit number of batches to one half the number of ctimes + if nbatches > len(ctimes[patch, "science", None]) // 2: + nbatches_dict[patch] = len(ctimes[patch, "science", None]) // 2 + # Must have at least two batches + if nbatches < 2: + nbatches_dict[patch] = None + batches = {} + for patch in patches: + batches[patch] = [None] + if nbatches_dict[patch] is None: + pass + elif nbatches_dict[patch] > 1: + nbatch = nbatches_dict[patch] + batches[patch] = range(nbatch) + nctimes = len(ctimes[patch, 'science', None]) + logger.info( + f"{patch}: splitting atomics into {nbatch} random " + f"batches with {nctimes // nbatch} ctimes in each." + ) + idx_rand = np.random.permutation(range(nctimes)) + for ib in batches[patch]: + ctimes[patch, "science", ib] = [ + ctimes[patch, "science", None][i] + for i in idx_rand if (i+ib) % nbatch + ] + + # Restrict the inter-obs null splits to the ctimes of the "science" split + for patch in patches: + for inter_obs_split, ib in product(inter_obs_splits, batches[patch]): + ctimes[patch, inter_obs_split, ib] = [ + ct + for ct in ctimes[patch, inter_obs_split, None] + if ct in ctimes[patch, "science", ib] + ] + + # Connect the the atomic map DB + db_con = sqlite3.connect(atom_db) + db_cur = db_con.cursor() + + # TODO: check if query_restrict is channel- or patch-specific + query_restrict = args.query_restrict + + relevant_splits = list(set(["science"] + intra_obs_splits + intra_obs_pair)) # noqa + queries = { + (patch, freq_channel, split_label, ib): fu.get_query_atomics( + freq_channel, ctimes[patch, "science", ib], + split_label=split_label, query_restrict=query_restrict + ) + for ib in batches[patch] + for patch, freq_channel, split_label in product(patches, + freq_channels, + relevant_splits) + } + atomic_metadata = {key: [] for key in queries} + + # Query all atomics used for science, filtering ctimes + for (patch, freq_channel, split_label, ib), query in queries.items(): + if split_label == "science": + res = db_cur.execute(query) + res = res.fetchall() + atomic_metadata[patch, freq_channel, "science", ib] = [ + (obs_id, wafer) for obs_id, wafer in res + ] + + # Query all atomics used for intra-obs splits + # filtering ctimes and split labels + for (patch, freq_channel, split_label, ib), query in queries.items(): + if split_label != "science": + res = db_cur.execute(query) + res = res.fetchall() + atomic_metadata[patch, + freq_channel, + split_label, + ib] = [ + (obs_id, wafer) for obs_id, wafer in res + if (obs_id, wafer) in atomic_metadata[patch, + freq_channel, + "science", + ib] + ] + + # Query all atomics used for inter-obs splits + # filtering ctimes w.r.t to the null prop considered + # for the two intra-obs splits to be coadded + if len(intra_obs_pair) != 0: + for patch, freq_channel in product(patches, freq_channels): + for inter_obs_split, intra_obs_split, ib in product(inter_obs_splits, # noqa + intra_obs_pair, # noqa + batches[patch]): # noqa + query = fu.get_query_atomics( + freq_channel, ctimes[patch, inter_obs_split, ib], + split_label=intra_obs_split + ) + res = db_cur.execute(query) + res = res.fetchall() + atomic_metadata[patch, + freq_channel, + inter_obs_split, + intra_obs_split, + ib] = [ + (obs_id, wafer) for obs_id, wafer in res + if (obs_id, wafer) in atomic_metadata[patch, + freq_channel, + "science", + ib] + ] + db_con.close() + + split_labels_all = ["science"] + intra_obs_splits + inter_obs_splits + split_labels_all = list(dict.fromkeys(split_labels_all)) # no duplicates + mpi_shared_list = [(patch, freq_channel, split_label, sim_type) + for patch in patches + for freq_channel in freq_channels + for split_label in split_labels_all + for sim_type in sim_types] + + # Every rank must have the same shared list + mpi_shared_list = comm.bcast(mpi_shared_list, root=0) + task_ids = mpi.distribute_tasks(size, rank, len(mpi_shared_list), + logger=logger) + local_mpi_list = [mpi_shared_list[i] for i in task_ids] + loop_over = [(patch, freq, split, sim_type, ib) + for ib in batches[patch] + for patch, freq, split, sim_type in local_mpi_list] + + + for patch, freq_channel, split_label, sim_type, ib in loop_over: + map_dir = atomic_sim_dir.format( + patch=patch, + freq_channel=freq_labels[freq_channel], + sim_type=sim_type + ) + assert os.path.isdir(map_dir), map_dir + + if not ib: + logger.info(f"Loading atomics for ({patch}, {freq_channel}, " + f"{split_label})" + f" to filter {sim_type}, {split_label}") + + w_list, wmap_list = ([], []) + + if split_label == "science": + tracemalloc.start() + for coadd in intra_obs_pair: + wmap_l, w_l = fu.get_atomics_maps_list( + None, sim_type, + atomic_metadata[patch, freq_channel, coadd, ib], + freq_labels[freq_channel], map_dir, coadd, + sim_string_format, mfmt=mfmt, pix_type=pix_type, + logger=logger + ) + wmap_list += wmap_l + w_list += w_l + current_gb, peak_gb = [1024**(-3) * c + for c in tracemalloc.get_traced_memory()] + logger.info("Traced Memory for 'science' (Current, Peak): " + f"{current_gb:.2f} GB, {peak_gb:.2f} GB") + tracemalloc.stop() + elif split_label in inter_obs_splits: + for coadd in intra_obs_pair: + wmap_l, w_l = fu.get_atomics_maps_list( + None, sim_type, + atomic_metadata[patch, freq_channel, split_label, coadd, ib], # noqa + freq_channel, map_dir, coadd, + sim_string_format, mfmt=mfmt, pix_type=pix_type, + logger=logger + ) + wmap_list += wmap_l + w_list += w_l + else: + wmap_list, w_list = fu.get_atomics_maps_list( + None, sim_type, + atomic_metadata[patch, freq_channel, split_label, ib], + freq_channel, map_dir, split_label, + sim_string_format, mfmt=mfmt, pix_type=pix_type, + logger=logger + ) + + if not ib: + logger.info(f"Coadding atomics for ({patch}, {freq_channel}, " + f"{split_label})" + f" to filter {sim_type}") + + map_filtered, weights = bu.coadd_maps( + wmap_list, w_list, pix_type=pix_type, + car_template_map=car_map_template + ) + out_fname = sim_string_format.format( + sim_type=sim_type, + freq_channel=freq_labels[freq_channel] + ).split("/")[-1] + batch_label = "" if ib is None else f"_batch{ib}of{nbatches}" + out_fname = out_fname.replace( + ".fits", + f"_bundle{bundle_id}_{freq_channel}_{split_label}{batch_label}_filtered.fits" # noqa + ) + fu.save_and_plot_map( + map_filtered, out_fname, + coadded_dirs[patch, freq_channel, sim_type], + plot_dirs[patch, freq_channel, sim_type], + pix_type=pix_type + ) + fu.save_and_plot_map( + weights, out_fname.replace(".fits", "_weights.fits"), + coadded_dirs[patch, freq_channel, sim_type], + plot_dirs[patch, freq_channel, sim_type], + pix_type=pix_type, do_plot=False + ) + comm.Barrier() + if rank == 0: + end = time.time() + print(f"Coadding completed in (wall time) {int(end - start)}s.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--config_file", type=str, help="yaml file with configuration." + ) + + args = parser.parse_args() + config = fu.Cfg.from_yaml(args.config_file) + config.update(vars(args)) + + main(config) diff --git a/pipeline/filtering/coadd_filtered_sims.py b/pipeline/filtering/coadd_filtered_sims.py index c762002..2c09d9f 100644 --- a/pipeline/filtering/coadd_filtered_sims.py +++ b/pipeline/filtering/coadd_filtered_sims.py @@ -334,6 +334,7 @@ def main(args): sim_id=sim_id ) assert os.path.isdir(map_dir), map_dir + tracemalloc.start() if not ib: logger.debug(f"Loading atomics for ({patch}, {freq_channel}, " @@ -343,7 +344,6 @@ def main(args): w_list, wmap_list = ([], []) if split_label == "science": - tracemalloc.start() for coadd in intra_obs_pair: wmap_l, w_l = fu.get_atomics_maps_list( sim_id, sim_type, diff --git a/pipeline/filtering/filter_ext_sotodlib.py b/pipeline/filtering/filter_ext_sotodlib.py new file mode 100755 index 0000000..f775a5a --- /dev/null +++ b/pipeline/filtering/filter_ext_sotodlib.py @@ -0,0 +1,386 @@ +""" +Based on filter_sims_sotodlib.py, +adapted to run on the external data. +""" +import numpy as np +import healpy as hp +import argparse +import sqlite3 +import os +import sys +import time +from itertools import product + + +import sotodlib.preprocess.preprocess_util as pp_util +from sotodlib.core.metadata import loader +from pixell import enmap + +# TODO: Make it an actual module +sys.path.append( + os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'bundling')) +) +sys.path.append( + os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'misc')) +) +import coordinator as coord # noqa +import filtering_utils as fu # noqa +import mpi_utils as mpi # noqa + + +def main(args): + """ + """ + if args.pix_type not in ["hp", "car"]: + raise ValueError( + "Unknown pixel type, must be 'car' or 'hp'." + ) + for required_tag in ["{sim_type"]: + if required_tag not in args.sim_string_format: + raise ValueError(f"sim_string_format does not have \ + required placeholder {required_tag}") + + # MPI related initialization + rank, size, comm = mpi.init(True) + + # Initialize the logger + logger = pp_util.init_logger("benchmark", verbosity=2) + if rank == 0: + start = time.time() + + # Path to databases + bundle_dbs = {patch: args.bundle_db.format(patch=patch) + for patch in args.patches} + atom_db = args.atomic_db + + # Pre-processing configuration files + preprocess_config_init = args.preprocess_config_init + preprocess_config_proc = args.preprocess_config_proc + + logger.debug(f"Using atomic DB from {atom_db}") + + # Sim related arguments + sim_dir = args.sim_dir + sim_string_format = args.sim_string_format + sim_types = args.sim_types + + # Ensure that freq_channels for the metadata follow the "f090" convention. + # We keep the original labels in a dict called freq_labels. + freq_labels = {} + for f in args.freq_channels: # If these don't contain the "f", add + freq_channel = f"f{f}" if "f" not in f else f + freq_labels[freq_channel] = f # dict values are the original labels + freq_channels = list(freq_labels.keys()) + + # Create output directories + atomics_dir = {} + for freq_channel, patch in product(freq_channels, args.patches): + atomics_dir[(patch, freq_channel)] = {} + for isim_type, sim_type in enumerate(sim_types): + out_dir = args.output_dir.format( + patch=patch, freq_channel=freq_labels[freq_channel], sim_type=sim_type + ) + atomics_dir[patch, freq_channel][isim_type] = f"{out_dir}/atomic_maps" # noqa + os.makedirs(atomics_dir[patch, freq_channel][isim_type], + exist_ok=True) + + # Arguments related to pixellization + pix_type = args.pix_type + if pix_type == "hp": + nside = args.nside + mfmt = ".fits" # TODO: fits.gz for HEALPix + elif pix_type == "car": + if args.car_map_template is not None: + _, wcs = enmap.read_map_geometry(args.car_map_template) + else: + _, wcs = fu.get_fullsky_geometry() # Could be problematic if using all default values! # noqa + nside = None + mfmt = ".fits" + + # Bundle query arguments + bundle_id = args.bundle_id + + # Gather all intra obs split labels in a list + intra_obs_splits = args.intra_obs_splits + if isinstance(intra_obs_splits, str): + if "," not in intra_obs_splits: + intra_obs_splits = [intra_obs_splits] + else: + intra_obs_splits = intra_obs_splits.split(",") + if not len(intra_obs_splits): + # If no intra-obs splits are given to coadd, use the default ones. + intra_obs_splits = ["scan_left", "scan_right"] + + # Extract list of ctimes from bundle database for the given + # bundle_id - null split combination + ctimes = {patch: None for patch in args.patches} + for patch in args.patches: + if os.path.isfile(bundle_dbs[patch]): + logger.info(f"Loading from {bundle_dbs[patch]}.") + bundle_coordinator = coord.BundleCoordinator.from_dbfile( + bundle_dbs[patch], bundle_id=bundle_id + ) + else: + raise ValueError(f"DB file does not exist: {bundle_dbs[patch]}") + + # Extract all ctimes for the given bundle_id + ctimes[patch] = bundle_coordinator.get_ctimes(bundle_id=bundle_id) + + # TODO: check if query_restrict is channel- or patch-specific + query_restrict = args.query_restrict + queries = { + (patch, freq_channel): fu.get_query_atomics( + freq_channel, ctimes[patch], query_restrict=query_restrict + ) + for patch, freq_channel in product(args.patches, freq_channels) + } + + atomic_metadata = {split_label: [] for split_label in intra_obs_splits} + atomic_metadata["science"] = [] + for (patch, freq_channel), query in queries.items(): + + db_cur = sqlite3.connect(atom_db).cursor() + res_science = db_cur.execute(query) + res_science = res_science.fetchall() + db_cur.close() + + for obs_id, wafer in res_science: + atomic_metadata["science"] += [(patch, freq_channel, + obs_id, wafer)] + for split_label in intra_obs_splits: + query = fu.get_query_atomics(freq_channel, ctimes[patch], + split_label=split_label, + query_restrict=query_restrict) + db_cur = sqlite3.connect(atom_db).cursor() + res_split = db_cur.execute(query) + res_split = res_split.fetchall() + db_cur.close() + for obs_id, wafer in res_split: + if (obs_id, wafer) in res_science: + atomic_metadata[split_label] += [ + (patch, freq_channel, obs_id, wafer) + ] + logger.info( + f"{patch}, {freq_channel}, 'science': " + f"{len(res_science)} atomic maps to filter." + ) + + # Load preprocessing pipeline and extract from it list of preprocessing + # metadata (detectors, samples, etc.) corresponding to each atomic map + configs_init, _ = pp_util.get_preprocess_context( + preprocess_config_init + ) + configs_proc, ctx_proc = pp_util.get_preprocess_context( + preprocess_config_proc + ) + + # Initialize tasks for MPI sharing + mpi_shared_list = atomic_metadata["science"] + + # Every rank must have the same shared list + mpi_shared_list = comm.bcast(mpi_shared_list, root=0) + task_ids = mpi.distribute_tasks(size, rank, len(mpi_shared_list), + logger=logger) + local_mpi_list = [mpi_shared_list[i] for i in task_ids] + + # Loop over set of local tasks (patch, freq_channel, obs_id, wafer). + # For each of these, loop over (sim_type) and do: + # * read simulated map + # * load map into timestreams, apply preprocessing + # * apply mapmaking + for patch, freq_channel, obs_id, wafer in local_mpi_list: + start = time.time() + + # First, check if atomic maps already exist. + maps_exist = True + for isim_type, sim_type in enumerate(args.sim_types): + # Path to unfiltered simulation + map_fname = sim_string_format.format( + sim_type=sim_type, + ) + + for split_label in intra_obs_splits: + if (patch, freq_channel, obs_id, wafer) in atomic_metadata[split_label]: # noqa + + # Saving filtered atomics to disk + atomic_fname = map_fname.split("/")[-1].replace( + mfmt, + f"_{obs_id}_{wafer}_{split_label}{mfmt}" + ) + + f_wmap = atomics_dir[patch, freq_channel][isim_type] + f_wmap += f"/{atomic_fname.replace(mfmt, '_wmap' + mfmt)}" + f_w = f_wmap.replace('_wmap' + mfmt, '_weights' + mfmt) + + if not (os.path.isfile(f_wmap) and os.path.isfile(f_w)): + maps_exist = False + else: + maps_exist = False + + # If they exist and we don't overwrite, skip this atomic. + if maps_exist and not args.overwrite_atomics: + logger.info( + f"Map exists: ({patch}, {freq_channel}, {obs_id}, {wafer})" + f" to filter sims {args.sim_types}" + ) + continue + + # Get axis manager metadata for the given obs + dets = {"wafer_slot": wafer, "wafer.bandpass": freq_channel} + + try: + meta = ctx_proc.get_meta(obs_id=obs_id, dets=dets) + except loader.LoaderError: + logger.warning(f"NO METADATA: " + f"({patch}, {freq_channel}, {obs_id}, {wafer})") + continue + except OSError as err: + logger.warning(f"{err}: " + f"({patch}, {freq_channel}, {obs_id}, {wafer})") + continue + + # Focal plane thinning + if args.fp_thin is not None: + fp_thin = int(args.fp_thin) + thinned = [ + m for im, m in enumerate(meta.dets.vals) + if im % fp_thin == 0 + ] + meta.restrict("dets", thinned) + + try: + print(pp_util.__file__) + data_aman = pp_util.multilayer_load_and_preprocess( + obs_id, + configs_init, + configs_proc, + meta=meta, + logger=logger, + stop_for_sims=True, + ignore_cfg_check=True, + ) + except loader.LoaderError: + logger.warning(f"NO METADATA IN DATA_AMAN: " + f"({patch}, {freq_channel}, {obs_id}, {wafer})") + continue + + for isim_type, sim_type in enumerate(args.sim_types): + + # Path to unfiltered simulation + map_fname = sim_string_format.format( + sim_type=sim_type + ) + map_file = f"{sim_dir}/{map_fname}" + + logger.info(f"Loading ({patch}, {freq_channel}, {obs_id}, {wafer})" + f" to filter {sim_type}") + start0 = time.time() + + # Handling pixellization + if args.pix_type == "car": + logger.debug(f"Loading CAR map: {map_file}") + sim = enmap.read_map(map_file) + elif args.pix_type == "hp": + logger.debug(f"Loading HP map: {map_file}") + sim = hp.read_map(map_file, field=[0, 1, 2]) + else: + raise ValueError("pix_type must be hp or car") + + try: + aman = pp_util.multilayer_load_and_preprocess_sim( + obs_id, + configs_init=configs_init, + configs_proc=configs_proc, + sim_map=sim, + meta=meta, + logger=logger, + ignore_cfg_check=True, + data_amans=data_aman + ) + except loader.LoaderError: + logger.warning( + "METADATA MISSING: " + f"({patch}, {freq_channel}, {obs_id}, {wafer}) " + "SKIPPING." + ) + continue + except (OSError, KeyError) as err: + logger.warning( + f"{err} " + f"({patch}, {freq_channel}, {obs_id}, {wafer}) " + "SKIPPING." + ) + continue + + if aman is None: + logger.warning( + "No detectors left in this atomic." + f"({patch}, {freq_channel}, {obs_id}, {wafer}) " + ) + continue + if aman.dets.count <= 1: + logger.warning( + "No detectors left in this atomic." + f"({patch}, {freq_channel}, {obs_id}, {wafer}) " + ) + continue + + # Run the mapmaker + wmap_dict, weights_dict = fu.make_map_wrapper( + aman, intra_obs_splits, pix_type, shape=None, wcs=wcs, + nside=nside, logger=logger + ) + + for split_label in intra_obs_splits: + # We save only files for which we actually have data for a + # given null split + if (patch, freq_channel, obs_id, wafer) in atomic_metadata[split_label]: # noqa + wmap = wmap_dict[split_label] + w = weights_dict[split_label] + + # Saving filtered atomics to disk + atomic_fname = map_fname.split("/")[-1].replace( + mfmt, + f"_{obs_id}_{wafer}_{split_label}{mfmt}" + ) + + f_wmap = atomics_dir[patch, freq_channel][isim_type] + f_wmap += f"/{atomic_fname.replace(mfmt, '_wmap' + mfmt)}" + f_w = f_wmap.replace('_wmap' + mfmt, '_weights' + mfmt) + + if pix_type == "car": + enmap.write_map(f_wmap, wmap) + enmap.write_map(f_w, w) + + elif pix_type == "hp": + hp.write_map( + f_wmap, wmap, dtype=np.float32, overwrite=True, + nest=True + ) + hp.write_map( + f_w, w, dtype=np.float32, overwrite=True, nest=True + ) + end0 = time.time() + logger.info(f"Filtered in {end0 - start0:.1f} seconds: " + f"{sim_type}" + f"({patch}, {freq_channel}, {obs_id}, {wafer})") + + logger.info(f"Processed {len(sim_types)} simulations for " + f"({patch}, {freq_channel}, {obs_id}, {wafer}) in " + f"{time.time() - start:.1f} seconds.") + comm.Barrier() + if rank == 0: + end = time.time() + print(f"Filtering completed in (wall time) {int(end - start)}s.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--config_file", type=str, help="yaml file with configuration." + ) + args = parser.parse_args() + config = fu.Cfg.from_yaml(args.config_file) + config.update(vars(args)) + + main(config) diff --git a/pipeline/filtering/filter_sims_sotodlib.py b/pipeline/filtering/filter_sims_sotodlib.py index 7b236e6..1b9f7c8 100644 --- a/pipeline/filtering/filter_sims_sotodlib.py +++ b/pipeline/filtering/filter_sims_sotodlib.py @@ -47,7 +47,8 @@ def main(args): preprocess_config_init = args.preprocess_config_init preprocess_config_proc = args.preprocess_config_proc - logger.debug(f"Using atomic DB from {atom_db}") + if rank == 0: + logger.debug(f"Using atomic DB from {atom_db}") # Sim related arguments if args.sim_types is None: @@ -72,7 +73,8 @@ def main(args): sim_ids = np.array([int(sim_ids)]) elif not isinstance(sim_ids, list): raise ValueError("Argument 'sim_ids' has the wrong format") - logger.debug(f"Processing sim_ids {sim_ids} in parallel.") + if rank == 0: + logger.debug(f"Processing sim_ids {sim_ids} in parallel.") # Ensure that freq_channels for the metadata follow the "f090" convention. # We keep the original labels in a dict called freq_labels. @@ -134,7 +136,8 @@ def main(args): ctimes = {patch: None for patch in args.patches} for patch in args.patches: if os.path.isfile(bundle_dbs[patch]): - logger.info(f"Loading from {bundle_dbs[patch]}.") + if rank == 0: + logger.info(f"Loading from {bundle_dbs[patch]}.") bundle_coordinator = coord.BundleCoordinator.from_dbfile( bundle_dbs[patch], bundle_id=bundle_id ) @@ -144,7 +147,6 @@ def main(args): # Extract all ctimes for the given bundle_id ctimes[patch] = bundle_coordinator.get_ctimes(bundle_id=bundle_id) - # TODO: check if query_restrict is channel- or patch-specific query_restrict = args.query_restrict queries = { (patch, freq_channel): fu.get_query_atomics( @@ -178,10 +180,11 @@ def main(args): atomic_metadata[split_label] += [ (patch, freq_channel, obs_id, wafer) ] - logger.info( - f"{patch}, {freq_channel}, 'science': " - f"{len(res_science)} atomic maps to filter." - ) + if rank == 0: + logger.info( + f"{patch}, {freq_channel}, 'science': " + f"{len(res_science)} atomic maps to filter." + ) # Load preprocessing pipeline and extract from it list of preprocessing # metadata (detectors, samples, etc.) corresponding to each atomic map @@ -297,11 +300,15 @@ def main(args): # detectors left, resulting in one of several errors caught below. # TODO: We should account for those directly in sotodlib. except loader.LoaderError: - logger.warning(f"NO METADATA: " + logger.warning(f"NO METADATA: ", f"({patch}, {freq_channel}, {obs_id}, {wafer})") continue except OSError as err: - logger.warning(f"{err}: " + logger.warning(f"{err}: ", + ignore_cfg_check=True, + ) + except loader.LoaderError: + logger.warning(f"NO METADATA IN DATA_AMAN: ", f"({patch}, {freq_channel}, {obs_id}, {wafer})") continue except IndexError: diff --git a/pipeline/filtering/filtering_utils.py b/pipeline/filtering/filtering_utils.py index 562b291..936fdc6 100644 --- a/pipeline/filtering/filtering_utils.py +++ b/pipeline/filtering/filtering_utils.py @@ -345,4 +345,4 @@ def update(self, dict): @classmethod def from_yaml(cls, path) -> "Cfg": d = yaml_loader(path) - return cls(**d) + return cls(**d) \ No newline at end of file diff --git a/towards_r/v4_paper/external-data/filtering_configs/satp3/coadd_atomics_planck_bundle0_20260425-noazss.slurm b/towards_r/v4_paper/external-data/filtering_configs/satp3/coadd_atomics_planck_bundle0_20260425-noazss.slurm new file mode 100644 index 0000000..dfc049e --- /dev/null +++ b/towards_r/v4_paper/external-data/filtering_configs/satp3/coadd_atomics_planck_bundle0_20260425-noazss.slurm @@ -0,0 +1,36 @@ +#!/bin/bash -l + +#SBATCH --account=simonsobs +#SBATCH --nodes=4 +#SBATCH --ntasks=16 +#SBATCH --cpus-per-task=28 +#SBATCH --time=8:00:00 +#SBATCH --job-name=satp3-coadd-pk-0 +#SBATCH --mail-user sazzoni@princeton.edu + +set -e + +module use --append /scratch/gpfs/SIMONSOBS/modules +module load soconda/3.12/v0.6.8 + +export PYTHONPATH="${PYTHONPATH}:/home/sa5705/software/sotodlib/" + +# Log file +tel_ext_data="planck" # planck # wmap +tel_filter="satp3" +bundle="1" # 0 # 1 +log="./logs/log_${tel_ext_data}_${bundle}_${tel_filter}_ext_coadd_20260417-noazss.log" + +export OMP_NUM_THREADS=28 + +bb_awg_scripts_dir=/home/sa5705/software/bb-awg-scripts ## YOUR bb_awg_scripts DIRECTORY + +filtering_config=${bb_awg_scripts_dir}/towards_r/v4_paper/external-data/filtering_configs/${tel_filter}/filtering_config_${tel_ext_data}_bundle${bundle}_20260425-noazss.yaml + +echo "Launching pipeline at $(date)" + +srun -n 16 -c 28 --cpu_bind=cores \ + python -u ${bb_awg_scripts_dir}/pipeline/filtering/coadd_filtered_ext.py --config_file $filtering_config \ + > ${log} 2>&1 + +echo "Ending batch script at $(date)" diff --git a/towards_r/v4_paper/external-data/filtering_configs/satp3/coadd_atomics_planck_bundle0_20260520-noazss.slurm b/towards_r/v4_paper/external-data/filtering_configs/satp3/coadd_atomics_planck_bundle0_20260520-noazss.slurm new file mode 100644 index 0000000..419426b --- /dev/null +++ b/towards_r/v4_paper/external-data/filtering_configs/satp3/coadd_atomics_planck_bundle0_20260520-noazss.slurm @@ -0,0 +1,36 @@ +#!/bin/bash -l + +#SBATCH --account=simonsobs +#SBATCH --nodes=4 +#SBATCH --ntasks=16 +#SBATCH --cpus-per-task=28 +#SBATCH --time=8:00:00 +#SBATCH --job-name=1-satp3-coadd-pk +#SBATCH --mail-user sazzoni@princeton.edu + +set -e + +module use --append /scratch/gpfs/SIMONSOBS/modules +module load soconda/3.12/v0.6.8 + +export PYTHONPATH="${PYTHONPATH}:/home/sa5705/software/sotodlib/" + +# Log file +tel_ext_data="planck" # planck # wmap +tel_filter="satp3" +bundle="1" # 0 # 1 +log="./logs/log_${tel_ext_data}_${bundle}_${tel_filter}_ext_coadd_20260520-noazss.log" + +export OMP_NUM_THREADS=28 + +bb_awg_scripts_dir=/home/sa5705/software/bb-awg-scripts ## YOUR bb_awg_scripts DIRECTORY + +filtering_config=${bb_awg_scripts_dir}/towards_r/v4_paper/external-data/filtering_configs/${tel_filter}/filtering_config_${tel_ext_data}_bundle${bundle}_20260520.yaml + +echo "Launching pipeline at $(date)" + +srun -n 16 -c 28 --cpu_bind=cores \ + python -u ${bb_awg_scripts_dir}/pipeline/filtering/coadd_filtered_ext.py --config_file $filtering_config \ + > ${log} 2>&1 + +echo "Ending batch script at $(date)" diff --git a/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_planck_bundle0_20260425-noazss.yaml b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_planck_bundle0_20260425-noazss.yaml new file mode 100644 index 0000000..463a737 --- /dev/null +++ b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_planck_bundle0_20260425-noazss.yaml @@ -0,0 +1,36 @@ +## Required shared arguments ## +bundle_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4_paper/bundling/satp3_20260417_noazss/bundles_{patch}_seed66_20260417.db' +atomic_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4_paper/mapmaking/satp3_20260417_noazss/atomic_db.sqlite' +#output_dir: &output_dir /scratch/gpfs/SIMONSOBS/external_data_e2e/v4/filtered_ext_data_20260304/planck/f{sim_type}/satp3/{patch}/{freq_channel} +output_dir: &output_dir /scratch/gpfs/SIMONSOBS/users/sa5705/SO/ISO_v4_paper/filtered_ext_data_20260417_noazss/planck/f{sim_type}/satp3/{patch}/{freq_channel} +sim_dir: /scratch/gpfs/SIMONSOBS/external_data_e2e/unfiltered_data/planck/maps +atomic_sim_dir: !path [*output_dir, atomic_maps] +sim_string_format: "planck_car_{sim_type}_bundle0_res1.0amin_coords_c.fits" + +# For filtering on 4 tiger nodes, 2 sims take <~ 3 hours. +#sim_types: ["030", "100", "143", "217", "353"] # ext data freq channel +sim_types: ["100", "143", "353"] # ext data freq channel +sim_ids: [0] # should be there, but not used in the filter_ext_sotodib.py script +patches: [south] +freq_channels: [f090, f150] # freq channel defining the filtering + +# (Required for coadd_filtered_sims) +intra_obs_splits: [det_in, det_out] + +## Optional shared arguments ## +pix_type: car +car_map_template: "/scratch/gpfs/SIMONSOBS/so/science-readiness/geometry/sat_f090.fits" +bundle_id: 0 +query_restrict: "pwv < 2 AND number_dets > 10 AND ((f_hwp > 1.9 AND f_hwp < 2.3) OR (f_hwp > -2.3 AND f_hwp < -1.9))" + +## Required arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +preprocess_config_init: /home/sa5705/software/bb-awg-scripts/towards_r/v4_paper/external-data/filtering_configs/satp3/preprocessing_config_20260417_init.yaml +preprocess_config_proc: /home/sa5705/software/bb-awg-scripts/towards_r/v4_paper/external-data/filtering_configs/satp3/preprocessing_config_20260425_noazss_proc.yaml + +## Optional arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +fp_thin: 8 + +## Optional arguments for coadd_filtered_sims ## +intra_obs_pair: [det_in, det_out] +inter_obs_splits: [] +nbatch_atomics: 1 diff --git a/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_planck_bundle0_20260520.yaml b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_planck_bundle0_20260520.yaml new file mode 100644 index 0000000..2678651 --- /dev/null +++ b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_planck_bundle0_20260520.yaml @@ -0,0 +1,36 @@ +## Required shared arguments ## +bundle_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4_paper/bundling/satp3_20260520/bundles_{patch}_seed66_20260520.db' +atomic_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4_paper/mapmaking/satp3_20260520/atomic_db.sqlite' +#output_dir: &output_dir /scratch/gpfs/SIMONSOBS/external_data_e2e/v4/filtered_ext_data_20260304/planck/f{sim_type}/satp3/{patch}/{freq_channel} +output_dir: &output_dir /scratch/gpfs/SIMONSOBS/users/sa5705/SO/ISO_v4_paper/filtered_ext_data_20260520/planck/f{sim_type}/satp3/{patch}/{freq_channel} +sim_dir: /scratch/gpfs/SIMONSOBS/external_data_e2e/unfiltered_data/planck/maps +atomic_sim_dir: !path [*output_dir, atomic_maps] +sim_string_format: "planck_car_{sim_type}_bundle0_res1.0amin_coords_c.fits" + +# For filtering on 4 tiger nodes, 2 sims take <~ 3 hours. +#sim_types: ["030", "100", "143", "217", "353"] # ext data freq channel +sim_types: ["100", "143", "353"] # ext data freq channel +sim_ids: [0] # should be there, but not used in the filter_ext_sotodib.py script +patches: [south] +freq_channels: [f090, f150] # freq channel defining the filtering + +# (Required for coadd_filtered_sims) +intra_obs_splits: [det_in, det_out] + +## Optional shared arguments ## +pix_type: car +car_map_template: "/scratch/gpfs/SIMONSOBS/so/science-readiness/geometry/sat_f090.fits" +bundle_id: 0 +query_restrict: "pwv < 2 AND number_dets > 10 AND ((f_hwp > 1.9 AND f_hwp < 2.3) OR (f_hwp > -2.3 AND f_hwp < -1.9))" + +## Required arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +preprocess_config_init: /home/sa5705/software/iso-sat/v4_paper/preprocessing/satp3/preprocessing_config_20260520_init.yaml +preprocess_config_proc: /home/sa5705/software/iso-sat/v4_paper/preprocessing/satp3/preprocessing_config_20260520_proc.yaml + +## Optional arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +fp_thin: 8 + +## Optional arguments for coadd_filtered_sims ## +intra_obs_pair: [det_in, det_out] +inter_obs_splits: [] +nbatch_atomics: 1 diff --git a/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_planck_bundle1_20260425-noazss.yaml b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_planck_bundle1_20260425-noazss.yaml new file mode 100644 index 0000000..1611199 --- /dev/null +++ b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_planck_bundle1_20260425-noazss.yaml @@ -0,0 +1,36 @@ +## Required shared arguments ## +bundle_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4_paper/bundling/satp3_20260417_noazss/bundles_{patch}_seed66_20260417.db' +atomic_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4_paper/mapmaking/satp3_20260417_noazss/atomic_db.sqlite' +#output_dir: &output_dir /scratch/gpfs/SIMONSOBS/external_data_e2e/v4/filtered_ext_data_20260304/planck/f{sim_type}/satp3/{patch}/{freq_channel} +output_dir: &output_dir /scratch/gpfs/SIMONSOBS/users/sa5705/SO/ISO_v4_paper/filtered_ext_data_20260417_noazss/planck/f{sim_type}/satp3/{patch}/{freq_channel} +sim_dir: /scratch/gpfs/SIMONSOBS/external_data_e2e/unfiltered_data/planck/maps +atomic_sim_dir: !path [*output_dir, atomic_maps] +sim_string_format: "planck_car_{sim_type}_bundle1_res1.0amin_coords_c.fits" + +# For filtering on 4 tiger nodes, 2 sims take <~ 3 hours. +#sim_types: ["030", "100", "143", "217", "353"] # ext data freq channel +sim_types: ["100", "143", "353"] # ext data freq channel +sim_ids: [0] # should be there, but not used in the filter_ext_sotodib.py script +patches: [south] +freq_channels: [f090, f150] # freq channel defining the filtering + +# (Required for coadd_filtered_sims) +intra_obs_splits: [det_in, det_out] + +## Optional shared arguments ## +pix_type: car +car_map_template: "/scratch/gpfs/SIMONSOBS/so/science-readiness/geometry/sat_f090.fits" +bundle_id: 1 +query_restrict: "pwv < 2 AND number_dets > 10 AND ((f_hwp > 1.9 AND f_hwp < 2.3) OR (f_hwp > -2.3 AND f_hwp < -1.9))" + +## Required arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +preprocess_config_init: /home/sa5705/software/bb-awg-scripts/towards_r/v4_paper/external-data/filtering_configs/satp3/preprocessing_config_20260417_init.yaml +preprocess_config_proc: /home/sa5705/software/bb-awg-scripts/towards_r/v4_paper/external-data/filtering_configs/satp3/preprocessing_config_20260425_noazss_proc.yaml + +## Optional arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +fp_thin: 8 + +## Optional arguments for coadd_filtered_sims ## +intra_obs_pair: [det_in, det_out] +inter_obs_splits: [] +nbatch_atomics: 1 diff --git a/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_planck_bundle1_20260520.yaml b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_planck_bundle1_20260520.yaml new file mode 100644 index 0000000..d2a3569 --- /dev/null +++ b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_planck_bundle1_20260520.yaml @@ -0,0 +1,36 @@ +## Required shared arguments ## +bundle_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4_paper/bundling/satp3_20260520/bundles_{patch}_seed66_20260520.db' +atomic_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4_paper/mapmaking/satp3_20260520/atomic_db.sqlite' +#output_dir: &output_dir /scratch/gpfs/SIMONSOBS/external_data_e2e/v4/filtered_ext_data_20260304/planck/f{sim_type}/satp3/{patch}/{freq_channel} +output_dir: &output_dir /scratch/gpfs/SIMONSOBS/users/sa5705/SO/ISO_v4_paper/filtered_ext_data_20260520/planck/f{sim_type}/satp3/{patch}/{freq_channel} +sim_dir: /scratch/gpfs/SIMONSOBS/external_data_e2e/unfiltered_data/planck/maps +atomic_sim_dir: !path [*output_dir, atomic_maps] +sim_string_format: "planck_car_{sim_type}_bundle1_res1.0amin_coords_c.fits" + +# For filtering on 4 tiger nodes, 2 sims take <~ 3 hours. +#sim_types: ["030", "100", "143", "217", "353"] # ext data freq channel +sim_types: ["100", "143", "353"] # ext data freq channel +sim_ids: [0] # should be there, but not used in the filter_ext_sotodib.py script +patches: [south] +freq_channels: [f090, f150] # freq channel defining the filtering + +# (Required for coadd_filtered_sims) +intra_obs_splits: [det_in, det_out] + +## Optional shared arguments ## +pix_type: car +car_map_template: "/scratch/gpfs/SIMONSOBS/so/science-readiness/geometry/sat_f090.fits" +bundle_id: 1 +query_restrict: "pwv < 2 AND number_dets > 10 AND ((f_hwp > 1.9 AND f_hwp < 2.3) OR (f_hwp > -2.3 AND f_hwp < -1.9))" + +## Required arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +preprocess_config_init: /home/sa5705/software/iso-sat/v4_paper/preprocessing/satp3/preprocessing_config_20260520_init.yaml +preprocess_config_proc: /home/sa5705/software/iso-sat/v4_paper/preprocessing/satp3/preprocessing_config_20260520_proc.yaml + +## Optional arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +fp_thin: 8 + +## Optional arguments for coadd_filtered_sims ## +intra_obs_pair: [det_in, det_out] +inter_obs_splits: [] +nbatch_atomics: 1 diff --git a/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_wmap_bundle0.yaml b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_wmap_bundle0.yaml new file mode 100644 index 0000000..7d5f6ab --- /dev/null +++ b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_wmap_bundle0.yaml @@ -0,0 +1,35 @@ +## Required shared arguments ## +bundle_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4/bundling/satp3_20251216/bundles_{patch}_seed66_20251216.db' +atomic_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4/mapmaking/satp3_20251216/atomic_db.sqlite' +output_dir: &output_dir /scratch/gpfs/SIMONSOBS/external_data_e2e/v4/filtered_ext_data/wmap/f{sim_type}/satp3/{patch}/{freq_channel} +sim_dir: /scratch/gpfs/SIMONSOBS/external_data_e2e/unfiltered_data/wmap/maps +atomic_sim_dir: !path [*output_dir, atomic_maps] +sim_string_format: "wmap_car_{sim_type}_bundle0_res1.0amin_coords_c.fits" + +# For filtering on 4 tiger nodes, 2 sims take <~ 3 hours. +sim_types: ["K", "Ka"] # ext data freq channel +sim_ids: [0] # should be there, but not used in the filter_ext_sotodib.py script +patches: [south] +freq_channels: [f090, f150] # freq channel defining the filtering +t2p_template: False + +# (Required for coadd_filtered_sims) +intra_obs_splits: [det_lower, det_upper] + +## Optional shared arguments ## +pix_type: car +car_map_template: "/scratch/gpfs/SIMONSOBS/so/science-readiness/geometry/sat_f090.fits" +bundle_id: 0 +query_restrict: " pwv < 2 AND number_dets > 10 " + +## Required arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +preprocess_config_init: preprocessing_config_20251216_init.yaml +preprocess_config_proc: preprocessing_config_20251216_proc.yaml + +## Optional arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +fp_thin: 8 + +## Optional arguments for coadd_filtered_sims ## +intra_obs_pair: [det_lower, det_upper] +inter_obs_splits: [] +nbatch_atomics: 1 diff --git a/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_planck_bundle0_20250425-noazss.slurm b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_planck_bundle0_20250425-noazss.slurm new file mode 100644 index 0000000..ae04bce --- /dev/null +++ b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_planck_bundle0_20250425-noazss.slurm @@ -0,0 +1,36 @@ +#!/bin/bash -l + +#SBATCH --account=simonsobs +#SBATCH --nodes=20 +#SBATCH --ntasks=1120 +#SBATCH --cpus-per-task=2 +#SBATCH --time=10:00:00 +#SBATCH --job-name=satp3-filtering-pk-b1 +#SBATCH --mail-user sazzoni@princeton.edu + +set -e + +module use --append /scratch/gpfs/SIMONSOBS/modules +module load soconda/3.12/v0.6.8 + +export PYTHONPATH="${PYTHONPATH}:/home/sa5705/software/sotodlib/" + +# Log file +tel_ext_data="planck" # planck # wmap +tel_filter="satp3" +bundle="1" #"0" +log="./logs/log_${tel_ext_data}_${bundle}_${tel_filter}_ext_filtering_20260425-noazss.log" + +export OMP_NUM_THREADS=2 + +bb_awg_scripts_dir=/home/sa5705/software/bb-awg-scripts ## YOUR bb_awg_scripts DIRECTORY + +filtering_config=${bb_awg_scripts_dir}/towards_r/v4_paper/external-data/filtering_configs/${tel_filter}/filtering_config_${tel_ext_data}_bundle${bundle}_20260425-noazss.yaml + +echo "Launching pipeline at $(date)" + +srun -n 1120 -c 2 --cpu_bind=cores \ + python -u ${bb_awg_scripts_dir}/pipeline/filtering/filter_ext_sotodlib.py --config_file $filtering_config \ + > ${log} 2>&1 + +echo "Ending batch script at $(date)" diff --git a/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_planck_bundle0_20250520.slurm b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_planck_bundle0_20250520.slurm new file mode 100644 index 0000000..ff0843a --- /dev/null +++ b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_planck_bundle0_20250520.slurm @@ -0,0 +1,36 @@ +#!/bin/bash -l + +#SBATCH --account=simonsobs +#SBATCH --nodes=20 +#SBATCH --ntasks=1120 +#SBATCH --cpus-per-task=2 +#SBATCH --time=10:00:00 +#SBATCH --job-name=satp3-filtering-pk-b0 +#SBATCH --mail-user sazzoni@princeton.edu + +set -e + +module use --append /scratch/gpfs/SIMONSOBS/modules +module load soconda/3.12/v0.6.8 + +export PYTHONPATH="${PYTHONPATH}:/home/sa5705/software/sotodlib/" + +# Log file +tel_ext_data="planck" # planck # wmap +tel_filter="satp3" +bundle="0" #"1" +log="./logs/log_${tel_ext_data}_${bundle}_${tel_filter}_ext_filtering_20260520.log" + +export OMP_NUM_THREADS=2 + +bb_awg_scripts_dir=/home/sa5705/software/bb-awg-scripts ## YOUR bb_awg_scripts DIRECTORY + +filtering_config=${bb_awg_scripts_dir}/towards_r/v4_paper/external-data/filtering_configs/${tel_filter}/filtering_config_${tel_ext_data}_bundle${bundle}_20260520.yaml + +echo "Launching pipeline at $(date)" + +srun -n 1120 -c 2 --cpu_bind=cores \ + python -u ${bb_awg_scripts_dir}/pipeline/filtering/filter_ext_sotodlib.py --config_file $filtering_config \ + > ${log} 2>&1 + +echo "Ending batch script at $(date)" diff --git a/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_planck_bundle1_20250520.slurm b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_planck_bundle1_20250520.slurm new file mode 100644 index 0000000..5a83f05 --- /dev/null +++ b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_planck_bundle1_20250520.slurm @@ -0,0 +1,36 @@ +#!/bin/bash -l + +#SBATCH --account=simonsobs +#SBATCH --nodes=20 +#SBATCH --ntasks=1120 +#SBATCH --cpus-per-task=2 +#SBATCH --time=10:00:00 +#SBATCH --job-name=satp3-filtering-pk-b1 +#SBATCH --mail-user sazzoni@princeton.edu + +set -e + +module use --append /scratch/gpfs/SIMONSOBS/modules +module load soconda/3.12/v0.6.8 + +export PYTHONPATH="${PYTHONPATH}:/home/sa5705/software/sotodlib/" + +# Log file +tel_ext_data="planck" # planck # wmap +tel_filter="satp3" +bundle="1" #"0" +log="./logs/log_${tel_ext_data}_${bundle}_${tel_filter}_ext_filtering_20260520.log" + +export OMP_NUM_THREADS=2 + +bb_awg_scripts_dir=/home/sa5705/software/bb-awg-scripts ## YOUR bb_awg_scripts DIRECTORY + +filtering_config=${bb_awg_scripts_dir}/towards_r/v4_paper/external-data/filtering_configs/${tel_filter}/filtering_config_${tel_ext_data}_bundle${bundle}_20260520.yaml + +echo "Launching pipeline at $(date)" + +srun -n 1120 -c 2 --cpu_bind=cores \ + python -u ${bb_awg_scripts_dir}/pipeline/filtering/filter_ext_sotodlib.py --config_file $filtering_config \ + > ${log} 2>&1 + +echo "Ending batch script at $(date)" diff --git a/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_wmap_bundle0.slurm b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_wmap_bundle0.slurm new file mode 100644 index 0000000..ad56d12 --- /dev/null +++ b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_wmap_bundle0.slurm @@ -0,0 +1,32 @@ +#!/bin/bash -l + +#SBATCH --account=simonsobs +#SBATCH --nodes=20 +#SBATCH --ntasks=1120 +#SBATCH --cpus-per-task=2 +#SBATCH --time=10:00:00 +#SBATCH --job-name=wmap-satp3-ext-filtering-bundle0 +#SBATCH --mail-user arianna.rizzieri@physics.ox.ac.uk + +set -e + +# Log file +tel_ext_data="wmap" # planck # wmap +tel_filter="satp3" +bundle="0" # 0 # 1 +log="./logs/log_${tel_ext_data}_${bundle}_${tel_filter}_ext_filtering" + +export OMP_NUM_THREADS=2 + +bb_awg_scripts_dir=/home/ar3186/awg-clean/bb-awg-scripts/ ## YOUR bb_awg_scripts DIRECTORY + +filtering_config=${bb_awg_scripts_dir}/towards_r/v4/external-data/filtering_configs/${tel_filter}/filtering_config_${tel_ext_data}_bundle${bundle}.yaml + +srun -n 1120 -c 2 --cpu_bind=cores \ + python -u ${bb_awg_scripts_dir}/pipeline/filtering/filter_ext_sotodlib.py --config_file $filtering_config \ + > ${log} 2>&1 + +echo ${com} +echo "Launching pipeline at $(date)" +eval ${com} > ${log} 2>&1 +echo "Ending batch script at $(date)" diff --git a/towards_r/v4_paper/external-data/filtering_configs/satp3/preprocessing_config_20260417_init.yaml b/towards_r/v4_paper/external-data/filtering_configs/satp3/preprocessing_config_20260417_init.yaml new file mode 120000 index 0000000..4353a4a --- /dev/null +++ b/towards_r/v4_paper/external-data/filtering_configs/satp3/preprocessing_config_20260417_init.yaml @@ -0,0 +1 @@ +/home/sa5705/software/iso-sat/v4_paper/preprocessing/satp3/preprocessing_config_20260417_init.yaml \ No newline at end of file diff --git a/towards_r/v4_paper/external-data/filtering_configs/satp3/preprocessing_config_20260425_noazss_proc.yaml b/towards_r/v4_paper/external-data/filtering_configs/satp3/preprocessing_config_20260425_noazss_proc.yaml new file mode 120000 index 0000000..9c8f973 --- /dev/null +++ b/towards_r/v4_paper/external-data/filtering_configs/satp3/preprocessing_config_20260425_noazss_proc.yaml @@ -0,0 +1 @@ +/home/sa5705/software/iso-sat/v4_paper/preprocessing/satp3/preprocessing_config_20260425_noazss_proc.yaml \ No newline at end of file diff --git a/towards_r/v4_paper/external-data/soopercool_configs/satp3/paramfile_south_science_filtered_satp3_f090.yaml b/towards_r/v4_paper/external-data/soopercool_configs/satp3/paramfile_south_science_filtered_satp3_f090.yaml new file mode 100644 index 0000000..8f65d83 --- /dev/null +++ b/towards_r/v4_paper/external-data/soopercool_configs/satp3/paramfile_south_science_filtered_satp3_f090.yaml @@ -0,0 +1,142 @@ +ext_data_directory: &ext_data_dir /scratch/gpfs/SIMONSOBS/users/sa5705/SO/ISO_v4_paper/ +iso_directory: &iso_dir /scratch/gpfs/SIMONSOBS/sat-iso/v4_paper +output_directory: &out_dir !path [*ext_data_dir, /spectra/filtered_satp3_f090/south/full-dataset_20260417_noazss/science] +transfer_dir: &transfer_dir !path [*iso_dir, /scratch/gpfs/SIMONSOBS/sat-iso/v4_paper/spectra/satp3/south/full-dataset_20260417_noazss/science] +inputs_dir: &inputs_dir !path [*iso_dir, soopercool_inputs] +signflip_dir: &sf_dir !path [*iso_dir, noise/satp3_20260417_noazss/signflip/south/] + +map_sets: + satp3_f090_south_science: + map_dir: !path [*iso_dir, bundling/satp3_20260417_noazss/south] + beam_dir: !path [*inputs_dir, beams] + map_template: "satp3_f090_science_bundle{id_bundle}_{map|hits}.fits" + beam_file: "beam_unity.dat" + n_bundles: 4 # Number of bundles + freq_tag: 090 # Freq. tag (e.g. useful to coadd freqs) + exp_tag: "satp3" # Experiment tag (useful to get noise-bias free cross-split spectra) + hits_tag: "satp3" + filtering_tag: "satp3_f090_south_science" + planck_f100_filtered_satp3_f090_south_science: + map_dir: !path [*ext_data_dir, /filtered_ext_data_20260417_noazss/planck/f100/satp3/south/f090/coadded_maps] + beam_dir: !path [*ext_data_dir, unfiltered_data/planck/beam_window_functions/] + map_template: "planck_car_100_bundle{id_bundle}_res1.0amin_coords_c_bundle{id_bundle}_f090_science_{filtered|filtered_weights}.fits" + beam_file: "beam_pol_100_long.txt" + n_bundles: 2 + freq_tag: 100 + exp_tag: "planck" + hits_tag: "flat" + filtering_tag: "satp3_f090_south_science" # tag only used for couplings and transfer function + planck_f143_filtered_satp3_f090_south_science: + map_dir: !path [*ext_data_dir, /filtered_ext_data_20260417_noazss/planck/f143/satp3/south/f090/coadded_maps] + beam_dir: !path [*ext_data_dir, unfiltered_data/planck/beam_window_functions/] + map_template: "planck_car_143_bundle{id_bundle}_res1.0amin_coords_c_bundle{id_bundle}_f090_science_{filtered|filtered_weights}.fits" + beam_file: "beam_pol_143_long.txt" + n_bundles: 2 + freq_tag: 143 + exp_tag: "planck" + hits_tag: "flat" + filtering_tag: "satp3_f090_south_science" # tag only used for couplings and transfer function + planck_f353_filtered_satp3_f090_south_science: + map_dir: !path [*ext_data_dir, /filtered_ext_data_20260417_noazss/planck/f353/satp3/south/f090/coadded_maps] + beam_dir: !path [*ext_data_dir, unfiltered_data/planck/beam_window_functions/] + map_template: "planck_car_353_bundle{id_bundle}_res1.0amin_coords_c_bundle{id_bundle}_f090_science_{filtered|filtered_weights}.fits" + beam_file: "beam_pol_353_long.txt" + n_bundles: 2 + freq_tag: 353 + exp_tag: "planck" + hits_tag: "flat" + filtering_tag: "satp3_f090_south_science" # tag only used for couplings and transfer function + satp3_f150_south_science: + map_dir: !path [*iso_dir, bundling/satp3_20260417_noazss/south] + beam_dir: !path [*inputs_dir, beams] + map_template: "satp3_f150_science_bundle{id_bundle}_{map|hits}.fits" + beam_file: "beam_unity.dat" + n_bundles: 4 + freq_tag: 150 + exp_tag: "satp3" + hits_tag: "satp3" + filtering_tag: "satp3_f150_south_science" + + +#################### +# Masking metadata # +#################### +masks: + # Specify coordinates if using a rectangular mask + box_mask: null + # Set to True if mask is calculated from the weights, + # otherwise set to False to use the hits + use_weights: False + analysis_mask: !path [*out_dir, masks/analysis_mask.fits] + # When specifying this file, the mapset/bundle-specific hits files will + # be ignored and instead a global hits file will be used (testing puroposes) + global_hits: !path [*iso_dir, spectra/satp3/south/full-dataset_20260417_noazss/science/masks/normalized_hits.fits] + # Path to products (binary) + galactic_mask: !path [*inputs_dir, masks/mask_GAL070_coords_c_sat_f090_geometry.fits] + point_source_catalog: null + point_source_mask: null + external_mask: null + # Apodization details + apod_radius: 10.0 + smoothing_radius: 10.0 + apod_radius_point_source: 1.0 + apod_type: "C1" + +#################################### +# Metadata related to the analysis # +#################################### +## General parameters +general_pars: + pix_type: car + car_template: /scratch/gpfs/SIMONSOBS/so/science-readiness/geometry/sat_f090.fits + nside: 512 + lmin: 2 + lmax: 803 + binning_file: /scratch/gpfs/SIMONSOBS/sat-iso/v4_paper/spectra/satp3/south/full-dataset_20260417_noazss/science/binning/binning_car_lmax2700_deltal15.npz + pure_B: False + # Where the beam window is lower than beam_floor, set it to beam_floor + beam_floor: 1.e-2 + compute_Dl: True + +covariance: + hits_files: + satp3: !path [*iso_dir, spectra/satp3/south/full-dataset_20260417_noazss/science/masks/normalized_hits.fits] + flat: /scratch/gpfs/SIMONSOBS/users/sa5705/SO/ISO_v4_paper/homogeneous_hits_for_planck.fits + cov_num_sims: 200 + fiducial_cmb: null + fiducial_dust: null + fiducial_synch: null + +transfer_settings: + transfer_directory: !path [*transfer_dir, transfer_functions] + + external_couplings_dir: !path [*transfer_dir, couplings] + tf_est_num_sims: 19 + sim_id_start: 1 + # # For estimation + # ## Number of sims for tf estimation and validation + # ## Parameters of the PL sims used for TF estimation + # power_law_pars_tf_est: + # amp: 1.0 + # delta_ell: 15 + # power_law_index: 2. + # ## Optional beams applied on PL sims + # # If true, beams will be applied only on the validation simulations. By default (false) + # # beam are applied to both the estimation and validation sims, + # # to account for potential effect of the beam on the TF (e.g. commutativity) + # do_not_beam_est_sims: True + # beams_list: [] + + ## Path to the sims for TF estimation + unfiltered_map_dir: + satp3_f090_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4/transfer_function/input_sims/ + satp3_f150_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4/transfer_function/input_sims/ + unfiltered_map_template: + satp3_f090_south_science: "{pure_type}_4.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR.fits" + satp3_f150_south_science: "{pure_type}_4.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR.fits" + filtered_map_dir: + satp3_f090_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4_paper/transfer_function/satp3_20260417_noazss/south/f090/full-dataset/coadded_sims + satp3_f150_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4_paper/transfer_function/satp3_20260417_noazss/south/f150/full-dataset/coadded_sims + filtered_map_template: + satp3_f090_south_science: "{pure_type}_1.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR_bundle0_f090_science_filtered.fits" + satp3_f150_south_science: "{pure_type}_1.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR_bundle0_f150_science_filtered.fits" \ No newline at end of file diff --git a/towards_r/v4_paper/external-data/soopercool_configs/satp3/paramfile_south_science_filtered_satp3_f150.yaml b/towards_r/v4_paper/external-data/soopercool_configs/satp3/paramfile_south_science_filtered_satp3_f150.yaml new file mode 100644 index 0000000..6fb8c6b --- /dev/null +++ b/towards_r/v4_paper/external-data/soopercool_configs/satp3/paramfile_south_science_filtered_satp3_f150.yaml @@ -0,0 +1,142 @@ +ext_data_directory: &ext_data_dir /scratch/gpfs/SIMONSOBS/users/sa5705/SO/ISO_v4_paper/ +iso_directory: &iso_dir /scratch/gpfs/SIMONSOBS/sat-iso/v4_paper +output_directory: &out_dir !path [*ext_data_dir, /spectra/filtered_satp3_f150/south/full-dataset_20260417_noazss/science] +transfer_dir: &transfer_dir !path [*iso_dir, /scratch/gpfs/SIMONSOBS/sat-iso/v4_paper/spectra/satp3/south/full-dataset_20260417_noazss/science] +inputs_dir: &inputs_dir !path [*iso_dir, soopercool_inputs] +signflip_dir: &sf_dir !path [*iso_dir, noise/satp3_20260417_noazss/signflip/south/] + +map_sets: + satp3_f090_south_science: + map_dir: !path [*iso_dir, bundling/satp3_20260417_noazss/south] + beam_dir: !path [*inputs_dir, beams] + map_template: "satp3_f090_science_bundle{id_bundle}_{map|hits}.fits" + beam_file: "beam_unity.dat" + n_bundles: 4 # Number of bundles + freq_tag: 090 # Freq. tag (e.g. useful to coadd freqs) + exp_tag: "satp3" # Experiment tag (useful to get noise-bias free cross-split spectra) + hits_tag: "satp3" + filtering_tag: "satp3_f090_south_science" + planck_f100_filtered_satp3_f150_south_science: + map_dir: !path [*ext_data_dir, /filtered_ext_data_20260417_noazss/planck/f100/satp3/south/f150/coadded_maps] + beam_dir: !path [*ext_data_dir, unfiltered_data/planck/beam_window_functions/] + map_template: "planck_car_100_bundle{id_bundle}_res1.0amin_coords_c_bundle{id_bundle}_f150_science_{filtered|filtered_weights}.fits" + beam_file: "beam_pol_100_long.txt" + n_bundles: 2 + freq_tag: 100 + exp_tag: "planck" + hits_tag: "flat" + filtering_tag: "satp3_f150_south_science" # tag only used for couplings and transfer function + planck_f143_filtered_satp3_f150_south_science: + map_dir: !path [*ext_data_dir, /filtered_ext_data_20260417_noazss/planck/f143/satp3/south/f150/coadded_maps] + beam_dir: !path [*ext_data_dir, unfiltered_data/planck/beam_window_functions/] + map_template: "planck_car_143_bundle{id_bundle}_res1.0amin_coords_c_bundle{id_bundle}_f150_science_{filtered|filtered_weights}.fits" + beam_file: "beam_pol_143_long.txt" + n_bundles: 2 + freq_tag: 143 + exp_tag: "planck" + hits_tag: "flat" + filtering_tag: "satp3_f150_south_science" # tag only used for couplings and transfer function + planck_f353_filtered_satp3_f150_south_science: + map_dir: !path [*ext_data_dir, /filtered_ext_data_20260417_noazss/planck/f353/satp3/south/f150/coadded_maps] + beam_dir: !path [*ext_data_dir, unfiltered_data/planck/beam_window_functions/] + map_template: "planck_car_353_bundle{id_bundle}_res1.0amin_coords_c_bundle{id_bundle}_f150_science_{filtered|filtered_weights}.fits" + beam_file: "beam_pol_353_long.txt" + n_bundles: 2 + freq_tag: 353 + exp_tag: "planck" + hits_tag: "flat" + filtering_tag: "satp3_f150_south_science" # tag only used for couplings and transfer function + satp3_f150_south_science: + map_dir: !path [*iso_dir, bundling/satp3_20260417_noazss/south] + beam_dir: !path [*inputs_dir, beams] + map_template: "satp3_f150_science_bundle{id_bundle}_{map|hits}.fits" + beam_file: "beam_unity.dat" + n_bundles: 4 + freq_tag: 150 + exp_tag: "satp3" + hits_tag: "satp3" + filtering_tag: "satp3_f150_south_science" + + +#################### +# Masking metadata # +#################### +masks: + # Specify coordinates if using a rectangular mask + box_mask: null + # Set to True if mask is calculated from the weights, + # otherwise set to False to use the hits + use_weights: False + analysis_mask: !path [*out_dir, masks/analysis_mask.fits] + # When specifying this file, the mapset/bundle-specific hits files will + # be ignored and instead a global hits file will be used (testing puroposes) + global_hits: !path [*iso_dir, spectra/satp3/south/full-dataset_20260417_noazss/science/masks/normalized_hits.fits] + # Path to products (binary) + galactic_mask: !path [*inputs_dir, masks/mask_GAL070_coords_c_sat_f090_geometry.fits] + point_source_catalog: null + point_source_mask: null + external_mask: null + # Apodization details + apod_radius: 10.0 + smoothing_radius: 10.0 + apod_radius_point_source: 1.0 + apod_type: "C1" + +#################################### +# Metadata related to the analysis # +#################################### +## General parameters +general_pars: + pix_type: car + car_template: /scratch/gpfs/SIMONSOBS/so/science-readiness/geometry/sat_f090.fits + nside: 512 + lmin: 2 + lmax: 803 + binning_file: /scratch/gpfs/SIMONSOBS/sat-iso/v4_paper/spectra/satp3/south/full-dataset_20260417_noazss/science/binning/binning_car_lmax2700_deltal15.npz + pure_B: False + # Where the beam window is lower than beam_floor, set it to beam_floor + beam_floor: 1.e-2 + compute_Dl: True + +covariance: + hits_files: + satp3: !path [*iso_dir, spectra/satp3/south/full-dataset_20260417_noazss/science/masks/normalized_hits.fits] + flat: /scratch/gpfs/SIMONSOBS/users/sa5705/SO/ISO_v4_paper/homogeneous_hits_for_planck.fits + cov_num_sims: 200 + fiducial_cmb: null + fiducial_dust: null + fiducial_synch: null + +transfer_settings: + transfer_directory: !path [*transfer_dir, transfer_functions] + + external_couplings_dir: !path [*transfer_dir, couplings] + tf_est_num_sims: 19 + sim_id_start: 1 + # # For estimation + # ## Number of sims for tf estimation and validation + # ## Parameters of the PL sims used for TF estimation + # power_law_pars_tf_est: + # amp: 1.0 + # delta_ell: 15 + # power_law_index: 2. + # ## Optional beams applied on PL sims + # # If true, beams will be applied only on the validation simulations. By default (false) + # # beam are applied to both the estimation and validation sims, + # # to account for potential effect of the beam on the TF (e.g. commutativity) + # do_not_beam_est_sims: True + # beams_list: [] + + ## Path to the sims for TF estimation + unfiltered_map_dir: + satp3_f090_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4/transfer_function/input_sims/ + satp3_f150_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4/transfer_function/input_sims/ + unfiltered_map_template: + satp3_f090_south_science: "{pure_type}_4.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR.fits" + satp3_f150_south_science: "{pure_type}_4.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR.fits" + filtered_map_dir: + satp3_f090_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4_paper/transfer_function/satp3_20260417_noazss/south/f090/full-dataset/coadded_sims + satp3_f150_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4_paper/transfer_function/satp3_20260417_noazss/south/f150/full-dataset/coadded_sims + filtered_map_template: + satp3_f090_south_science: "{pure_type}_1.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR_bundle0_f090_science_filtered.fits" + satp3_f150_south_science: "{pure_type}_1.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR_bundle0_f150_science_filtered.fits" \ No newline at end of file diff --git a/towards_r/v4_paper/external-data/soopercool_configs/satp3/spectra_computation_steps.sh b/towards_r/v4_paper/external-data/soopercool_configs/satp3/spectra_computation_steps.sh new file mode 100755 index 0000000..9510980 --- /dev/null +++ b/towards_r/v4_paper/external-data/soopercool_configs/satp3/spectra_computation_steps.sh @@ -0,0 +1,69 @@ +#!/bin/bash -l + +set -e + +# General +instruments=("satp3") # ("satp1" "satp3") +sat_freqs=("f090" "f150") + +# For now, we only consider south patch. +patches=("south") + +soopercool_dir=/home/sa5705/software/SOOPERCOOL # Insert path to your local SOOPERCOOL directory + +# You can add more/take away splits here +splits=("science") + +# Run each step for each instrument and split +for sat in "${instruments[@]}"; do + for sat_freq in "${sat_freqs[@]}"; do + for patch in "${patches[@]}"; do + for split in "${splits[@]}"; do + echo "=== Running for sat: $sat | split: $split ===" + + paramfile="towards_r/v4_paper/external-data/soopercool_configs/${sat}/paramfile_${patch}_${split}_filtered_${sat}_${sat_freq}.yaml" + + echo "Using paramfile: $paramfile" + + # Uncomment the steps you want to run + # This following script was taken from soopercool's "sa_dev" branch + #srun -n 1 -c 112 --cpu_bind=cores python ${soopercool_dir}/pipeline/get_analysis_mask.py --globals ${paramfile} + #echo "=== Done with get_analysis_mask.py ===" + #srun -n 1 -c 112 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/get_mode_coupling.py --globals ${paramfile} + #echo "=== Done with get_mode_coupling.py ===" + #srun -n 12 -c 8 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/transfer/compute_pseudo_cells_tf_estimation.py --globals ${paramfile} + #echo "=== Done with compute_pseudo_cells_tf_estimation.py ===" + #srun -n 1 -c 112 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/transfer/compute_transfer_function.py --globals ${paramfile} + #echo "=== Done with compute_transfer_function.py ===" + #srun -n 1 -c 112 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/get_full_couplings.py --globals ${paramfile} + #echo "=== Done with get_full_couplings.py ===" + #srun -n 1 -c 112 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/compute_pseudo_cells.py --globals ${paramfile} --verbose + #echo "=== Done with compute_pseudo_cells.py ===" + #srun -n 1 -c 112 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/coadd_pseudo_cells.py --globals ${paramfile} --no_plots + #echo "=== Done with coadd_pseudo_cells.py ===" + + #srun -n 16 -c 7 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/precompute_cov_couplings.py --globals ${paramfile} + #echo "=== Done with precompute_cov_couplings.py ===" + + #srun -n 1 -c 112 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/prepare_cov_inputs.py --globals ${paramfile} + #echo "=== Done with prepare_cov_inputs.py ===" + #srun -n 112 -c 1 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/compute_covariance.py --globals ${paramfile} + #echo "=== Done with compute_covariance.py ===" + srun -n 1 -c 112 python -u ${soopercool_dir}/pipeline/create_sacc_file_analytic.py --globals ${paramfile} --data + #echo "=== Done with create_sacc_file_analytic.py ===" + # + #srun -n 1 -c 112 python ${soopercool_dir}/pipeline/postproc/fit_cals_and_beams.py --sacc-file /scratch/gpfs/SIMONSOBS/external_data_e2e/v4/spectra/filtered_${sat}_${sat_freq}_20260304_fixed_exp_tag/south/20251216_full-dataset/science/saccs/cl_and_cov_sacc.fits --lmin-fit 150 --lmax-fit 600 --map-sets-ref planck_f100_filtered_${sat}_${sat_freq}_south_science,planck_f143_filtered_${sat}_${sat_freq}_south_science --map-sets-to-fit ${sat}_f090_south_science,${sat}_f150_south_science --lmax-sacc 650 + #echo "=== Done with fit_cals_and_beams.py ===" + + #srun -n 1 -c 112 python ${soopercool_dir}/pipeline/postproc/fit_EB_angles.py --sacc-file /scratch/gpfs/SIMONSOBS/external_data_e2e/v4/spectra/filtered_${sat}_${sat_freq}_20260304_fixed_exp_tag/south/20251216_full-dataset/science/saccs/cl_and_cov_sacc_calibrated.fits \ + # --lmin-fit 150 --lmax-fit 600 --map-sets-to-fit ${sat}_f090_south_science,${sat}_f150_south_science --lmax-sacc 650 + #echo "=== Done with fit_EB_angles.py ===" + # + #srun -n 1 -c 112 python ${soopercool_dir}/pipeline/postproc/fit_cals_and_beams.py --sacc-file /scratch/gpfs/SIMONSOBS/external_data_e2e/v4/spectra/filtered_${sat}_${sat_freq}_20260304_fixed_exp_tag/south/20251216_full-dataset/science/saccs/cl_and_cov_sacc_calibrated_rotated.fits --lmin-fit 150 --lmax-fit 600 --map-sets-ref planck_f100_filtered_${sat}_${sat_freq}_south_science,planck_f143_filtered_${sat}_${sat_freq}_south_science --map-sets-to-fit ${sat}_f090_south_science,${sat}_f150_south_science --lmax-sacc 650 + #echo "=== Done with fit_cals_and_beams.py (round 2) ===" + + echo "=== Done with sat: $sat | split: $split | filtered as: $sat $sat_freq ===" + done + done + done +done \ No newline at end of file