From 7818ddfa1a90261847b8fd77bab60fcec39bc75a Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Sat, 23 May 2026 12:26:39 +0100 Subject: [PATCH 001/120] Initial DAG for LINC+delay --- flocs_processing/dags/linc.py | 246 ++++++++++++++++++++++++++++++++++ 1 file changed, 246 insertions(+) create mode 100644 flocs_processing/dags/linc.py diff --git a/flocs_processing/dags/linc.py b/flocs_processing/dags/linc.py new file mode 100644 index 0000000..264cf70 --- /dev/null +++ b/flocs_processing/dags/linc.py @@ -0,0 +1,246 @@ +from enum import Enum +import functools +import os +import pathlib +import sqlite3 +import subprocess + +from airflow.exceptions import AirflowFailException, AirflowSkipException +from airflow.sdk import dag, task +from airflow.task.trigger_rule import TriggerRule + +# Need to replace this with a config file +TABLE_NAME = "" +DATABASE = "" +SLURM_ACCOUNT = "" +SLURM_QUEUE = "" +DATA_DIR = "" + + +@functools.total_ordering +class PIPELINE_STATUS(Enum): + nothing = 0 + downloaded = 1 + finished = 2 + running = 3 + processing = 98 + error = 99 + + def __eq__(self, other): + if other.__class__ is int: + return self.value == other + elif other.__class__ is self.__class__: + return self.value == other.value + else: + raise NotImplementedError + + def __lt__(self, other): + if self.__class__ is not other.__class__: + raise NotImplementedError + return self.value < other.value + + +def get_db_columns(): + with sqlite3.connect(DATABASE) as db: + db.row_factory = sqlite3.Row + cursor = db.cursor() + columns = "target_name,priority,finished,sas_id_calibrator1,sas_id_calibrator2,sas_id_calibrator_final,sas_id_target,status_calibrator1,status_calibrator2,status_target,status_vlbi_delay" + field = cursor.execute( + f"select {columns} from {TABLE_NAME} where finished==0 order by priority" + ).fetchall() + print(field) + return field + + +def set_status_processing(name, identifier, target): + with sqlite3.connect(DATABASE) as db: + cursor = db.cursor() + cursor.execute( + f"update {TABLE_NAME} set status_{identifier}={PIPELINE_STATUS.processing.value} where target_name=='{name}' and sas_id_target=='{target}'" + ) + + +def set_status_finished(name, identifier, target): + with sqlite3.connect(DATABASE) as db: + cursor = db.cursor() + cursor.execute( + f"update {TABLE_NAME} set status_{identifier}={PIPELINE_STATUS.finished.value} where target_name=='{name}' and sas_id_target=='{target}'" + ) + + +def set_final_calibrator(name, target, final_cal): + with sqlite3.connect(DATABASE) as db: + cursor = db.cursor() + cursor.execute( + f"update {TABLE_NAME} set sas_id_calibrator_final={final_cal} where target_name=='{name}' and sas_id_target=='{target}'" + ) + + +def get_most_recent_run(searchpath: str, sas_id: str) -> pathlib.Path: + rundirs = pathlib.Path(searchpath) + rundirs_sorted = sorted(rundirs.iterdir(), key=os.path.getctime) + rundirs_sorted_filtered = [d for d in rundirs_sorted if sas_id in d.parts[-1]] + rundir_final = rundirs_sorted_filtered[-1].absolute() + return rundir_final + + +@dag +def linc(): + @task + def get_unprocessed_target(): + field = dict(get_db_columns()[0]) + print(field["target_name"]) + return field + + @task.short_circuit + def check_fields(): + fields = get_db_columns() + return bool(fields) + + @task + def run_linc_calibrator1(field): + if (field["status_calibrator1"] == PIPELINE_STATUS.finished) or ( + field["status_calibrator1"] == PIPELINE_STATUS.running + ): + print( + f"Flux density calibrator {field['sas_id_calibrator1']} for observation {field['target_name']} {field['sas_id_target']} already processed." + ) + return field + else: + print( + f"Processing flux density calibrator {field['sas_id_calibrator1']} for observation {field['target_name']} {field['sas_id_target']}" + ) + ms_folder = f"L{field['sas_id_calibrator1']}" + set_status_processing( + field["target_name"], "calibrator1", field["sas_id_target"] + ) + outdir = os.path.join( + "/snap8/scratch/do011/dc-swei1/airflow/output", field["target_name"] + ) + cmd = f"flocs-run linc calibrator --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir /snap8/scratch/do011/dc-swei1/airflow/rundir --outdir {outdir} {os.path.join(DATA_DIR, field['target_name'], 'calibrator', ms_folder)}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with open( + f"log_LINC_calibrator_{field['target_name']}_{field['sas_id_calibrator1']}.txt", + "w", + ) as f_out, open( + f"log_LINC_calibrator_{field['target_name']}_{field['sas_id_calibrator1']}_err.txt", + "w", + ) as f_err: + proc = subprocess.run( + cmd, shell=True, text=True, stdout=f_out, stderr=f_err + ) + if not proc.returncode: + set_status_finished( + field["target_name"], "calibrator1", field["sas_id_target"] + ) + else: + raise RuntimeError + return field + + @task + def run_linc_calibrator2(field): + raise AirflowSkipException() + + @task(trigger_rule=TriggerRule.ONE_DONE) + def select_best_calibrator(result1, result2): + if result1 and result2: + print("Selecting between cal1 and cal2") + set_final_calibrator( + result1["target_name"], + result1["sas_id_target"], + result1["sas_id_calibrator1"], + ) + return result1 + elif result1 and (not result2): + print("Only cal 1 succeeded, continuing with that") + set_final_calibrator( + result1["target_name"], + result1["sas_id_target"], + result1["sas_id_calibrator1"], + ) + return result1 + elif (not result1) and result2: + print("Only cal 2 succeeded, continuing with that") + set_final_calibrator( + result1["target_name"], + result1["sas_id_target"], + result1["sas_id_calibrator2"], + ) + return result2 + else: + raise AirflowFailException("No calibrators succeeded; stopping processing.") + + @task + def run_linc_target(field): + if (field["status_target"] == PIPELINE_STATUS.finished) or ( + field["status_target"] == PIPELINE_STATUS.running + ): + return field + else: + print( + f"Processing target observation {field['target_name']} {field['sas_id_target']} with calibrator {field['sas_id_calibrator_final']}" + ) + ms_folder = f"L{field['sas_id_target']}" + outdir = os.path.join( + "/snap8/scratch/do011/dc-swei1/airflow/output", field["target_name"] + ) + calibrator_path = get_most_recent_run( + outdir, field["sas_id_calibrator_final"] + ) + calibrator_solutions = ( + calibrator_path / "results_LINC_calibrator" / "cal_solutions.h5" + ) + set_status_processing( + field["target_name"], "target", field["sas_id_target"] + ) + cmd = f"flocs-run linc target --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir /snap8/scratch/do011/dc-swei1/airflow/rundir --outdir {outdir} --cal-solutions {calibrator_solutions} {os.path.join(DATA_DIR, field['target_name'], 'target', ms_folder)}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with open( + f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}.txt", + "w", + ) as f_out, open( + f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w", + ) as f_err: + proc = subprocess.run( + cmd, shell=True, text=True, stdout=f_out, stderr=f_err + ) + if not proc.returncode: + return True + set_status_finished( + field["target_name"], "target", field["sas_id_target"] + ) + else: + raise RuntimeError + return field + + @task + def validate_linc_target(field): + return True + + @task + def run_vlbi_delay(field): + return True + + @task + def run_ddf_pipeline(field): + return True + + proceed = check_fields() + field = get_unprocessed_target() + result_cal1 = run_linc_calibrator1(field) + result_cal2 = run_linc_calibrator2(field) + best_cal = select_best_calibrator(result_cal1, result_cal2) + result_targ = run_linc_target(best_cal) + linc_is_valid = validate_linc_target(result_targ) + result_vlbi_delay = run_vlbi_delay(linc_is_valid) + # run_ddf_pipeline(vlbi_delay_is_valid) + + proceed >> field + + +linc() From da28933b772839630453e7489e7b2e1e2fd5a34a Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Thu, 28 May 2026 15:18:51 +0100 Subject: [PATCH 002/120] Add download step --- flocs_processing/dags/linc.py | 132 ++++++++++++++++++++++++++- flocs_processing/flocs_processing.py | 32 +++++-- 2 files changed, 153 insertions(+), 11 deletions(-) diff --git a/flocs_processing/dags/linc.py b/flocs_processing/dags/linc.py index 264cf70..bd56493 100644 --- a/flocs_processing/dags/linc.py +++ b/flocs_processing/dags/linc.py @@ -4,10 +4,13 @@ import pathlib import sqlite3 import subprocess +import time from airflow.exceptions import AirflowFailException, AirflowSkipException from airflow.sdk import dag, task from airflow.task.trigger_rule import TriggerRule +from flocs_lta.lta_search import ObservationStager +from stager_access import get_surls_requested, get_surls_online # Need to replace this with a config file TABLE_NAME = "" @@ -44,7 +47,7 @@ def get_db_columns(): with sqlite3.connect(DATABASE) as db: db.row_factory = sqlite3.Row cursor = db.cursor() - columns = "target_name,priority,finished,sas_id_calibrator1,sas_id_calibrator2,sas_id_calibrator_final,sas_id_target,status_calibrator1,status_calibrator2,status_target,status_vlbi_delay" + columns = "target_name,priority,finished,downloaded,sas_id_calibrator1,sas_id_calibrator2,sas_id_calibrator_final,sas_id_target,status_calibrator1,status_calibrator2,status_target,status_vlbi_delay" field = cursor.execute( f"select {columns} from {TABLE_NAME} where finished==0 order by priority" ).fetchall() @@ -68,6 +71,14 @@ def set_status_finished(name, identifier, target): ) +def set_status_downloaded(name, target): + with sqlite3.connect(DATABASE) as db: + cursor = db.cursor() + cursor.execute( + f"update {TABLE_NAME} set downloaded=1 where target_name=='{name}' and sas_id_target=='{target}'" + ) + + def set_final_calibrator(name, target, final_cal): with sqlite3.connect(DATABASE) as db: cursor = db.cursor() @@ -97,6 +108,120 @@ def check_fields(): fields = get_db_columns() return bool(fields) + @task + def download_field(field): + if field["downloaded"]: + return field + else: + has_cal1 = False + stage_calibrators = False + if field["sas_id_calibrator1"]: + ms_folder = f"L{field['sas_id_calibrator1']}" + cal1_full_path = os.path.join( + DATA_DIR, field["target_name"], "calibrator", ms_folder + ) + if os.path.exists(cal1_full_path): + has_cal1 = True + else: + stage_calibrators = True + + has_cal2 = False + if field["sas_id_calibrator2"]: + ms_folder = f"L{field['sas_id_calibrator2']}" + cal2_full_path = os.path.join( + DATA_DIR, field["target_name"], "calibrator", ms_folder + ) + if os.path.exists(cal2_full_path): + has_cal2 = True + else: + stage_calibrators = True + if field["sas_id_target"]: + ms_folder = f"L{field['sas_id_target']}" + target_full_path = os.path.join( + DATA_DIR, field["target_name"], "target", ms_folder + ) + if os.path.exists(target_full_path): + stage_target = False + else: + stage_target = True + else: + raise AirflowFailException( + f"No target SAS ID in database for field {field['target_name']}" + ) + + print(f"Field {field['sas_id_target']} is not downloaded.") + stager = ObservationStager(get_surls=True) + stager.find_observation_by_sasid( + "ALL", + field["sas_id_target"], + None, + 120e6, + 168e6, + ) + if stage_calibrators: + stager.find_nearest_calibrators(2, 120e6, 168e6) + stage_id_calibrators = stager.stage_calibrators() + if stage_target: + stage_id_target = stager.stage_target() + + calibrator_staged = False + target_staged = False + calibrator_downloaded = has_cal1 or has_cal2 + target_downloaded = not stage_target + while True: + if len(get_surls_online(stage_id_calibrators)) == len( + get_surls_requested(stage_id_calibrators) + ): + calibrator_staged = True + if calibrator_staged and not calibrator_downloaded: + dl_path = os.path.join(DATA_DIR, field["target_name"], "calibrator") + cmd = ( + f"flocs-lta download --outdir {dl_path} {stage_id_calibrators}" + ) + with open( + f"log_download_calibrators_{field['target_name']}.txt", + "w", + ) as f_out, open( + f"log_download_calibrators_{field['target_name']}.txt", + "w", + ) as f_err: + proc = subprocess.run( + cmd, shell=True, text=True, stdout=f_out, stderr=f_err + ) + if not proc.returncode: + calibrator_downloaded = True + else: + raise RuntimeError + + if len(get_surls_online(stage_id_target)) == len( + get_surls_requested(stage_id_target) + ): + calibrator_staged = True + if target_staged and not target_downloaded: + dl_path = os.path.join(DATA_DIR, field["target_name"], "target") + cmd = f"flocs-lta download --outdir {dl_path} {stage_id_target}" + with open( + f"log_download_calibrators_{field['target_name']}.txt", + "w", + ) as f_out, open( + f"log_download_calibrators_{field['target_name']}.txt", + "w", + ) as f_err: + proc = subprocess.run( + cmd, shell=True, text=True, stdout=f_out, stderr=f_err + ) + if not proc.returncode: + set_status_downloaded( + field["target_name"], + field["sas_id_target"], + ) + target_downloaded = True + else: + raise RuntimeError + if calibrator_downloaded and target_downloaded: + break + time.sleep(60) + @task def run_linc_calibrator1(field): if (field["status_calibrator1"] == PIPELINE_STATUS.finished) or ( @@ -231,7 +356,8 @@ def run_ddf_pipeline(field): return True proceed = check_fields() - field = get_unprocessed_target() + get_field = get_unprocessed_target() + field = download_field(get_field) result_cal1 = run_linc_calibrator1(field) result_cal2 = run_linc_calibrator2(field) best_cal = select_best_calibrator(result_cal1, result_cal2) @@ -240,7 +366,7 @@ def run_ddf_pipeline(field): result_vlbi_delay = run_vlbi_delay(linc_is_valid) # run_ddf_pipeline(vlbi_delay_is_valid) - proceed >> field + proceed >> get_field linc() diff --git a/flocs_processing/flocs_processing.py b/flocs_processing/flocs_processing.py index f23ef2e..517d3f2 100644 --- a/flocs_processing/flocs_processing.py +++ b/flocs_processing/flocs_processing.py @@ -98,18 +98,25 @@ def create_database( "linc" ], ): + pipeline_str = ",".join(pipelines) pipelines = list(map(str.lower, pipelines)) - dbstr = f"create table {table_name}(source_name text default NULL" + dbstr = f"create table {table_name}(target_name text default NULL, pipelines text default '{pipeline_str}', priority int default 0, finished bit default 0, downloaded bit default 0" if "linc" in pipelines: dbstr += ", sas_id_calibrator1 text default NULL, sas_id_calibrator2 text default NULL, sas_id_calibrator_final text default NULL, sas_id_target text primary key default NULL, status_calibrator1 smallint default 0, status_calibrator2 smallint default 0, status_target smallint default 0" if "ddf-pipeline" in pipelines: - dbstr += ", status_ddf smallint default 0" + dbstr += f", status_ddf smallint default {PIPELINE_STATUS.nothing.value}" if "vlbi-delay-widefield" in pipelines: - dbstr += ", status_ddf smallint default 0" - dbstr += ", status_delay smallint default 0" + dbstr += f", status_vlbi_delay smallint default {PIPELINE_STATUS.nothing.value}" + dbstr += f", status_vlbi_dd smallint default {PIPELINE_STATUS.nothing.value}" + dbstr += f", status_vlbi_intermediate_img smallint default {PIPELINE_STATUS.nothing.value}" + dbstr += f", status_vlbi_facet_subtract smallint default {PIPELINE_STATUS.nothing.value}" + dbstr += ( + f", status_vlbi_facet_img smallint default {PIPELINE_STATUS.nothing.value}" + ) if "vlbi-delay-single-target" in pipelines: - dbstr += ", status_delay smallint default 0" + dbstr += f", status_vlbi_delay smallint default {PIPELINE_STATUS.nothing.value}" + dbstr += f", status_vlbi_dd smallint default {PIPELINE_STATUS.nothing.value}" dbstr += ");" cmd = ["sqlite3", dbname, dbstr] @@ -137,8 +144,11 @@ def add_field( table_name: Annotated[ str, Parameter(help="Database table that will be processed.") ] = "processing_flocs", + pipelines: Annotated[ + str, Parameter(help="Pipelines this field needs to be processed with.") + ] = "", ): - dbstr = f"insert into {table_name} (source_name" + dbstr = f"insert into {table_name} (target_name" if len(sas_id_calibrators) == 1: dbstr += ", sas_id_calibrator1" if len(sas_id_calibrators) == 2: @@ -316,7 +326,10 @@ def launch_vlbi_delay(self, field_name, sas_id, restart: bool = False): rundirs_sorted_filtered = [ d for d in rundirs_sorted - if ((sas_id in d.parts[-1]) and (d.parts[-1].startswith(("LINC_target")))) + if ( + (sas_id in d.parts[-1]) + and (d.parts[-1].startswith(("LINC_target"))) + ) ] # Last LINC target reduction for this source linc_target_dir = rundirs_sorted_filtered[-1] @@ -373,7 +386,10 @@ def launch_vlbi_delay(self, field_name, sas_id, restart: bool = False): rundirs_sorted_filtered = [ d for d in rundirs_sorted - if ((sas_id in d.parts[-1]) and (d.parts[-1].startswith(("LINC_target")))) + if ( + (sas_id in d.parts[-1]) + and (d.parts[-1].startswith(("LINC_target"))) + ) ] # Last LINC target reduction for this source linc_target_dir = rundirs_sorted_filtered[-1] From e3c5ae1c21a1a7778aac560ce71fc14909460130 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Thu, 28 May 2026 15:33:36 +0100 Subject: [PATCH 003/120] Fix small errors --- flocs_processing/dags/linc.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/flocs_processing/dags/linc.py b/flocs_processing/dags/linc.py index bd56493..1d89fa0 100644 --- a/flocs_processing/dags/linc.py +++ b/flocs_processing/dags/linc.py @@ -149,20 +149,23 @@ def download_field(field): f"No target SAS ID in database for field {field['target_name']}" ) - print(f"Field {field['sas_id_target']} is not downloaded.") - stager = ObservationStager(get_surls=True) - stager.find_observation_by_sasid( - "ALL", - field["sas_id_target"], - None, - 120e6, - 168e6, - ) - if stage_calibrators: - stager.find_nearest_calibrators(2, 120e6, 168e6) - stage_id_calibrators = stager.stage_calibrators() - if stage_target: - stage_id_target = stager.stage_target() + if stage_calibrators or stage_target: + print(f"Field {field['sas_id_target']} is not downloaded.") + stager = ObservationStager(get_surls=True) + stager.find_observation_by_sasid( + "ALL", + field["sas_id_target"], + None, + 120e6, + 168e6, + ) + if stage_calibrators: + stager.find_nearest_calibrators(2, 120e6, 168e6) + stage_id_calibrators = stager.stage_calibrators() + if stage_target: + stage_id_target = stager.stage_target() + else: + return field calibrator_staged = False target_staged = False From e9612d6cbea0a3d5ed640b60167bef4a553b4294 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Sun, 31 May 2026 13:56:13 +0100 Subject: [PATCH 004/120] delay cal, rundir and outdir --- flocs_processing/dags/linc.py | 67 ++++++++++++++++++++++++++++------- 1 file changed, 55 insertions(+), 12 deletions(-) diff --git a/flocs_processing/dags/linc.py b/flocs_processing/dags/linc.py index 1d89fa0..1238f17 100644 --- a/flocs_processing/dags/linc.py +++ b/flocs_processing/dags/linc.py @@ -18,6 +18,8 @@ SLURM_ACCOUNT = "" SLURM_QUEUE = "" DATA_DIR = "" +OUTPUT_DIR = "" +PROCESSING_DIR = "" @functools.total_ordering @@ -87,10 +89,17 @@ def set_final_calibrator(name, target, final_cal): ) -def get_most_recent_run(searchpath: str, sas_id: str) -> pathlib.Path: +def get_most_recent_run(searchpath: str, sas_id: str, pipeline: str) -> pathlib.Path: rundirs = pathlib.Path(searchpath) rundirs_sorted = sorted(rundirs.iterdir(), key=os.path.getctime) - rundirs_sorted_filtered = [d for d in rundirs_sorted if sas_id in d.parts[-1]] + if pipeline: + rundirs_sorted_filtered = [ + d + for d in rundirs_sorted + if ((sas_id in d.parts[-1]) and (pipeline in d.parts[-1])) + ] + else: + rundirs_sorted_filtered = [d for d in rundirs_sorted if sas_id in d.parts[-1]] rundir_final = rundirs_sorted_filtered[-1].absolute() return rundir_final @@ -242,10 +251,8 @@ def run_linc_calibrator1(field): set_status_processing( field["target_name"], "calibrator1", field["sas_id_target"] ) - outdir = os.path.join( - "/snap8/scratch/do011/dc-swei1/airflow/output", field["target_name"] - ) - cmd = f"flocs-run linc calibrator --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir /snap8/scratch/do011/dc-swei1/airflow/rundir --outdir {outdir} {os.path.join(DATA_DIR, field['target_name'], 'calibrator', ms_folder)}" + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + cmd = f"flocs-run linc calibrator --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} {os.path.join(DATA_DIR, field['target_name'], 'calibrator', ms_folder)}" if not os.path.isdir(outdir): os.mkdir(outdir) print(cmd) @@ -311,11 +318,9 @@ def run_linc_target(field): f"Processing target observation {field['target_name']} {field['sas_id_target']} with calibrator {field['sas_id_calibrator_final']}" ) ms_folder = f"L{field['sas_id_target']}" - outdir = os.path.join( - "/snap8/scratch/do011/dc-swei1/airflow/output", field["target_name"] - ) + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) calibrator_path = get_most_recent_run( - outdir, field["sas_id_calibrator_final"] + outdir, field["sas_id_calibrator_final"], "LINC_calibrator" ) calibrator_solutions = ( calibrator_path / "results_LINC_calibrator" / "cal_solutions.h5" @@ -323,7 +328,7 @@ def run_linc_target(field): set_status_processing( field["target_name"], "target", field["sas_id_target"] ) - cmd = f"flocs-run linc target --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir /snap8/scratch/do011/dc-swei1/airflow/rundir --outdir {outdir} --cal-solutions {calibrator_solutions} {os.path.join(DATA_DIR, field['target_name'], 'target', ms_folder)}" + cmd = f"flocs-run linc target --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --cal-solutions {calibrator_solutions} {os.path.join(DATA_DIR, field['target_name'], 'target', ms_folder)}" if not os.path.isdir(outdir): os.mkdir(outdir) print(cmd) @@ -352,7 +357,45 @@ def validate_linc_target(field): @task def run_vlbi_delay(field): - return True + if (field["status_vlbi_delay"] == PIPELINE_STATUS.finished) or ( + field["status_vlbi_delay"] == PIPELINE_STATUS.running + ): + return field + else: + print( + f"Processing delay calibration for {field['target_name']} {field['sas_id_target']}" + ) + ms_folder = f"L{field['sas_id_target']}" + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + target_path = get_most_recent_run( + outdir, field["sas_id_calibrator_final"], "LINC_target" + ) + target_ms_path = target_path / "results_LINC_target" / "results" + set_status_processing( + field["target_name"], "vlbi_delay", field["sas_id_target"] + ) + cmd = f"flocs-run vlbi delay-calibration --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} {target_ms_path}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with open( + f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt", + "w", + ) as f_out, open( + f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w", + ) as f_err: + proc = subprocess.run( + cmd, shell=True, text=True, stdout=f_out, stderr=f_err + ) + if not proc.returncode: + return True + set_status_finished( + field["target_name"], "vlbi_delay", field["sas_id_target"] + ) + else: + raise RuntimeError + return field @task def run_ddf_pipeline(field): From 5bd78a3b4355b0f8611044b69856b03d51850348 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 5 Jun 2026 16:06:35 +0100 Subject: [PATCH 005/120] dd cal dag entry --- flocs_processing/dags/vlbi_single_target.py | 471 ++++++++++++++++++++ 1 file changed, 471 insertions(+) create mode 100644 flocs_processing/dags/vlbi_single_target.py diff --git a/flocs_processing/dags/vlbi_single_target.py b/flocs_processing/dags/vlbi_single_target.py new file mode 100644 index 0000000..d611a2c --- /dev/null +++ b/flocs_processing/dags/vlbi_single_target.py @@ -0,0 +1,471 @@ +from enum import Enum +import functools +import os +import pathlib +import sqlite3 +import subprocess +import time + +from airflow.exceptions import AirflowFailException, AirflowSkipException +from airflow.sdk import dag, task +from airflow.task.trigger_rule import TriggerRule +from flocs_lta.lta_search import ObservationStager +from stager_access import get_surls_requested, get_surls_online + +# Need to replace this with a config file +TABLE_NAME = "" +DATABASE = "" +SLURM_ACCOUNT = "" +SLURM_QUEUE = "" +DATA_DIR = "" +OUTPUT_DIR = "" +PROCESSING_DIR = "" + + +@functools.total_ordering +class PIPELINE_STATUS(Enum): + nothing = 0 + downloaded = 1 + finished = 2 + running = 3 + processing = 98 + error = 99 + + def __eq__(self, other): + if other.__class__ is int: + return self.value == other + elif other.__class__ is self.__class__: + return self.value == other.value + else: + raise NotImplementedError + + def __lt__(self, other): + if self.__class__ is not other.__class__: + raise NotImplementedError + return self.value < other.value + + +def get_db_columns(): + with sqlite3.connect(DATABASE) as db: + db.row_factory = sqlite3.Row + cursor = db.cursor() + columns = "target_name,priority,finished,downloaded,sas_id_calibrator1,sas_id_calibrator2,sas_id_calibrator_final,sas_id_target,status_calibrator1,status_calibrator2,status_target,status_vlbi_delay" + field = cursor.execute( + f"select {columns} from {TABLE_NAME} where finished==0 order by priority" + ).fetchall() + print(field) + return field + + +def set_status_processing(name, identifier, target): + with sqlite3.connect(DATABASE) as db: + cursor = db.cursor() + cursor.execute( + f"update {TABLE_NAME} set status_{identifier}={PIPELINE_STATUS.processing.value} where target_name=='{name}' and sas_id_target=='{target}'" + ) + + +def set_status_finished(name, identifier, target): + with sqlite3.connect(DATABASE) as db: + cursor = db.cursor() + cursor.execute( + f"update {TABLE_NAME} set status_{identifier}={PIPELINE_STATUS.finished.value} where target_name=='{name}' and sas_id_target=='{target}'" + ) + + +def set_status_downloaded(name, target): + with sqlite3.connect(DATABASE) as db: + cursor = db.cursor() + cursor.execute( + f"update {TABLE_NAME} set downloaded=1 where target_name=='{name}' and sas_id_target=='{target}'" + ) + + +def set_final_calibrator(name, target, final_cal): + with sqlite3.connect(DATABASE) as db: + cursor = db.cursor() + cursor.execute( + f"update {TABLE_NAME} set sas_id_calibrator_final={final_cal} where target_name=='{name}' and sas_id_target=='{target}'" + ) + + +def get_most_recent_run(searchpath: str, sas_id: str, pipeline: str) -> pathlib.Path: + rundirs = pathlib.Path(searchpath) + rundirs_sorted = sorted(rundirs.iterdir(), key=os.path.getctime) + if pipeline: + rundirs_sorted_filtered = [ + d + for d in rundirs_sorted + if ((sas_id in d.parts[-1]) and (pipeline in d.parts[-1])) + ] + else: + rundirs_sorted_filtered = [d for d in rundirs_sorted if sas_id in d.parts[-1]] + rundir_final = rundirs_sorted_filtered[-1].absolute() + return rundir_final + + +@dag +def linc(): + @task + def get_unprocessed_target(): + field = dict(get_db_columns()[0]) + print(field["target_name"]) + return field + + @task.short_circuit + def check_fields(): + fields = get_db_columns() + return bool(fields) + + @task + def download_field(field): + if field["downloaded"]: + return field + else: + has_cal1 = False + stage_calibrators = False + if field["sas_id_calibrator1"]: + ms_folder = f"L{field['sas_id_calibrator1']}" + cal1_full_path = os.path.join( + DATA_DIR, field["target_name"], "calibrator", ms_folder + ) + if os.path.exists(cal1_full_path): + has_cal1 = True + else: + stage_calibrators = True + + has_cal2 = False + if field["sas_id_calibrator2"]: + ms_folder = f"L{field['sas_id_calibrator2']}" + cal2_full_path = os.path.join( + DATA_DIR, field["target_name"], "calibrator", ms_folder + ) + if os.path.exists(cal2_full_path): + has_cal2 = True + else: + stage_calibrators = True + if field["sas_id_target"]: + ms_folder = f"L{field['sas_id_target']}" + target_full_path = os.path.join( + DATA_DIR, field["target_name"], "target", ms_folder + ) + if os.path.exists(target_full_path): + stage_target = False + else: + stage_target = True + else: + raise AirflowFailException( + f"No target SAS ID in database for field {field['target_name']}" + ) + + if stage_calibrators or stage_target: + print(f"Field {field['sas_id_target']} is not downloaded.") + stager = ObservationStager(get_surls=True) + stager.find_observation_by_sasid( + "ALL", + field["sas_id_target"], + None, + 120e6, + 168e6, + ) + if stage_calibrators: + stager.find_nearest_calibrators(2, 120e6, 168e6) + stage_id_calibrators = stager.stage_calibrators() + if stage_target: + stage_id_target = stager.stage_target() + else: + return field + + calibrator_staged = False + target_staged = False + calibrator_downloaded = has_cal1 or has_cal2 + target_downloaded = not stage_target + while True: + if len(get_surls_online(stage_id_calibrators)) == len( + get_surls_requested(stage_id_calibrators) + ): + calibrator_staged = True + if calibrator_staged and not calibrator_downloaded: + dl_path = os.path.join(DATA_DIR, field["target_name"], "calibrator") + cmd = ( + f"flocs-lta download --outdir {dl_path} {stage_id_calibrators}" + ) + with open( + f"log_download_calibrators_{field['target_name']}.txt", + "w", + ) as f_out, open( + f"log_download_calibrators_{field['target_name']}.txt", + "w", + ) as f_err: + proc = subprocess.run( + cmd, shell=True, text=True, stdout=f_out, stderr=f_err + ) + if not proc.returncode: + calibrator_downloaded = True + else: + raise RuntimeError + + if len(get_surls_online(stage_id_target)) == len( + get_surls_requested(stage_id_target) + ): + calibrator_staged = True + if target_staged and not target_downloaded: + dl_path = os.path.join(DATA_DIR, field["target_name"], "target") + cmd = f"flocs-lta download --outdir {dl_path} {stage_id_target}" + with open( + f"log_download_calibrators_{field['target_name']}.txt", + "w", + ) as f_out, open( + f"log_download_calibrators_{field['target_name']}.txt", + "w", + ) as f_err: + proc = subprocess.run( + cmd, shell=True, text=True, stdout=f_out, stderr=f_err + ) + if not proc.returncode: + set_status_downloaded( + field["target_name"], + field["sas_id_target"], + ) + target_downloaded = True + else: + raise RuntimeError + if calibrator_downloaded and target_downloaded: + break + time.sleep(60) + + @task + def run_linc_calibrator1(field): + if (field["status_calibrator1"] == PIPELINE_STATUS.finished) or ( + field["status_calibrator1"] == PIPELINE_STATUS.running + ): + print( + f"Flux density calibrator {field['sas_id_calibrator1']} for observation {field['target_name']} {field['sas_id_target']} already processed." + ) + return field + else: + print( + f"Processing flux density calibrator {field['sas_id_calibrator1']} for observation {field['target_name']} {field['sas_id_target']}" + ) + ms_folder = f"L{field['sas_id_calibrator1']}" + set_status_processing( + field["target_name"], "calibrator1", field["sas_id_target"] + ) + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + cmd = f"flocs-run linc calibrator --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} {os.path.join(DATA_DIR, field['target_name'], 'calibrator', ms_folder)}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with open( + f"log_LINC_calibrator_{field['target_name']}_{field['sas_id_calibrator1']}.txt", + "w", + ) as f_out, open( + f"log_LINC_calibrator_{field['target_name']}_{field['sas_id_calibrator1']}_err.txt", + "w", + ) as f_err: + proc = subprocess.run( + cmd, shell=True, text=True, stdout=f_out, stderr=f_err + ) + if not proc.returncode: + set_status_finished( + field["target_name"], "calibrator1", field["sas_id_target"] + ) + else: + raise RuntimeError + return field + + @task + def run_linc_calibrator2(field): + raise AirflowSkipException() + + @task(trigger_rule=TriggerRule.ONE_DONE) + def select_best_calibrator(result1, result2): + if result1 and result2: + print("Selecting between cal1 and cal2") + set_final_calibrator( + result1["target_name"], + result1["sas_id_target"], + result1["sas_id_calibrator1"], + ) + return result1 + elif result1 and (not result2): + print("Only cal 1 succeeded, continuing with that") + set_final_calibrator( + result1["target_name"], + result1["sas_id_target"], + result1["sas_id_calibrator1"], + ) + return result1 + elif (not result1) and result2: + print("Only cal 2 succeeded, continuing with that") + set_final_calibrator( + result1["target_name"], + result1["sas_id_target"], + result1["sas_id_calibrator2"], + ) + return result2 + else: + raise AirflowFailException("No calibrators succeeded; stopping processing.") + + @task + def run_linc_target(field): + if (field["status_target"] == PIPELINE_STATUS.finished) or ( + field["status_target"] == PIPELINE_STATUS.running + ): + return field + else: + print( + f"Processing target observation {field['target_name']} {field['sas_id_target']} with calibrator {field['sas_id_calibrator_final']}" + ) + ms_folder = f"L{field['sas_id_target']}" + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + calibrator_path = get_most_recent_run( + outdir, field["sas_id_calibrator_final"], "LINC_calibrator" + ) + calibrator_solutions = ( + calibrator_path / "results_LINC_calibrator" / "cal_solutions.h5" + ) + set_status_processing( + field["target_name"], "target", field["sas_id_target"] + ) + cmd = f"flocs-run linc target --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --cal-solutions {calibrator_solutions} {os.path.join(DATA_DIR, field['target_name'], 'target', ms_folder)}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with open( + f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}.txt", + "w", + ) as f_out, open( + f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w", + ) as f_err: + proc = subprocess.run( + cmd, shell=True, text=True, stdout=f_out, stderr=f_err + ) + if not proc.returncode: + return True + set_status_finished( + field["target_name"], "target", field["sas_id_target"] + ) + else: + raise RuntimeError + return field + + @task + def validate_linc_target(field): + return True + + @task + def run_vlbi_delay(field): + if (field["status_vlbi_delay"] == PIPELINE_STATUS.finished) or ( + field["status_vlbi_delay"] == PIPELINE_STATUS.running + ): + return field + else: + print( + f"Processing delay calibration for {field['target_name']} {field['sas_id_target']}" + ) + ms_folder = f"L{field['sas_id_target']}" + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + target_path = get_most_recent_run( + outdir, field["sas_id_calibrator_final"], "LINC_target" + ) + target_ms_path = target_path / "results_LINC_target" / "results" + set_status_processing( + field["target_name"], "vlbi_delay", field["sas_id_target"] + ) + cmd = f"flocs-run vlbi delay-calibration --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} {target_ms_path}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with open( + f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt", + "w", + ) as f_out, open( + f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w", + ) as f_err: + proc = subprocess.run( + cmd, shell=True, text=True, stdout=f_out, stderr=f_err + ) + if not proc.returncode: + return True + set_status_finished( + field["target_name"], "vlbi_delay", field["sas_id_target"] + ) + else: + raise RuntimeError + return field + + @task + def run_vlbi_ddcal(field): + if (field["status_vlbi_dd"] == PIPELINE_STATUS.finished) or ( + field["status_vlbi_dd"] == PIPELINE_STATUS.running + ): + return field + else: + print( + f"Processing ILT dd calibration for {field['target_name']} {field['sas_id_target']}" + ) + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + target_path = get_most_recent_run( + outdir, field["sas_id_calibrator_final"], "LINC_target" + ) + target_ms_path = target_path / "results_LINC_target" / "results" + + sols_path = get_most_recent_run( + outdir, field["sas_id_target"], "VLBI_delay" + ) + sols_path = sols_path / "results_VLBI_delay-calibration" / "results" + sols = sols_path.glob("merged_*_selfcalcycle???_linearfulljones*.h5") + + source_cat = os.path.join( + DATA_DIR, field["target_name"], "target", "vlbi_target.csv" + ) + + set_status_processing( + field["target_name"], "vlbi_dd", field["sas_id_target"] + ) + cmd = f"flocs-run vlbi dd-calibration --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --delay-solset {sols} --phasediff-score 10.0 --source-catalogue {source_cat} {target_ms_path}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with open( + f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt", + "w", + ) as f_out, open( + f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w", + ) as f_err: + proc = subprocess.run( + cmd, shell=True, text=True, stdout=f_out, stderr=f_err + ) + if not proc.returncode: + return True + set_status_finished( + field["target_name"], "vlbi_dd", field["sas_id_target"] + ) + else: + raise RuntimeError + return field + + @task + def run_ddf_pipeline(field): + return True + + proceed = check_fields() + get_field = get_unprocessed_target() + field = download_field(get_field) + result_cal1 = run_linc_calibrator1(field) + result_cal2 = run_linc_calibrator2(field) + best_cal = select_best_calibrator(result_cal1, result_cal2) + result_targ = run_linc_target(best_cal) + linc_is_valid = validate_linc_target(result_targ) + result_vlbi_delay = run_vlbi_delay(linc_is_valid) + result_vlbi_dd = run_vlbi_ddcal(result_vlbi_delay) + # run_ddf_pipeline(vlbi_delay_is_valid) + + proceed >> get_field + + +linc() From 9910c17b6c2ea96289346897ed9b7c07c92a23df Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 5 Jun 2026 16:15:43 +0100 Subject: [PATCH 006/120] Tweak dd cal --- flocs_processing/dags/vlbi_single_target.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flocs_processing/dags/vlbi_single_target.py b/flocs_processing/dags/vlbi_single_target.py index d611a2c..ca4958c 100644 --- a/flocs_processing/dags/vlbi_single_target.py +++ b/flocs_processing/dags/vlbi_single_target.py @@ -422,6 +422,8 @@ def run_vlbi_ddcal(field): source_cat = os.path.join( DATA_DIR, field["target_name"], "target", "vlbi_target.csv" ) + if not os.path.isfile(source_cat): + raise AirflowFailException(f"{source_cat} not found.") set_status_processing( field["target_name"], "vlbi_dd", field["sas_id_target"] From 834d888e936bf649f67b422549e4550f8f27fe18 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Tue, 9 Jun 2026 15:15:50 +0100 Subject: [PATCH 007/120] Return field from linc target validation --- flocs_processing/dags/vlbi_single_target.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flocs_processing/dags/vlbi_single_target.py b/flocs_processing/dags/vlbi_single_target.py index ca4958c..f07b743 100644 --- a/flocs_processing/dags/vlbi_single_target.py +++ b/flocs_processing/dags/vlbi_single_target.py @@ -353,7 +353,7 @@ def run_linc_target(field): @task def validate_linc_target(field): - return True + return field @task def run_vlbi_delay(field): From 0b698a582dffdbe1b0c5e0c028fb181aac5b9c07 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Tue, 9 Jun 2026 16:07:38 +0100 Subject: [PATCH 008/120] Get dd status column --- flocs_processing/dags/vlbi_single_target.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flocs_processing/dags/vlbi_single_target.py b/flocs_processing/dags/vlbi_single_target.py index f07b743..288e66c 100644 --- a/flocs_processing/dags/vlbi_single_target.py +++ b/flocs_processing/dags/vlbi_single_target.py @@ -49,7 +49,7 @@ def get_db_columns(): with sqlite3.connect(DATABASE) as db: db.row_factory = sqlite3.Row cursor = db.cursor() - columns = "target_name,priority,finished,downloaded,sas_id_calibrator1,sas_id_calibrator2,sas_id_calibrator_final,sas_id_target,status_calibrator1,status_calibrator2,status_target,status_vlbi_delay" + columns = "target_name,priority,finished,downloaded,sas_id_calibrator1,sas_id_calibrator2,sas_id_calibrator_final,sas_id_target,status_calibrator1,status_calibrator2,status_target,status_vlbi_delay,status_vlbi_dd" field = cursor.execute( f"select {columns} from {TABLE_NAME} where finished==0 order by priority" ).fetchall() From 125de55acc6659c1e92fffbddeee48d9ae6382a6 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Tue, 9 Jun 2026 16:15:31 +0100 Subject: [PATCH 009/120] Fix target retrieval in ddcal --- flocs_processing/dags/vlbi_single_target.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flocs_processing/dags/vlbi_single_target.py b/flocs_processing/dags/vlbi_single_target.py index 288e66c..6a1704b 100644 --- a/flocs_processing/dags/vlbi_single_target.py +++ b/flocs_processing/dags/vlbi_single_target.py @@ -409,7 +409,7 @@ def run_vlbi_ddcal(field): ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) target_path = get_most_recent_run( - outdir, field["sas_id_calibrator_final"], "LINC_target" + outdir, field["sas_id_target"], "LINC_target" ) target_ms_path = target_path / "results_LINC_target" / "results" From abd97e8b289085ecb296159d5f071e3dc6437e39 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Tue, 9 Jun 2026 16:44:06 +0100 Subject: [PATCH 010/120] Set whole field to finished once ddcal finishes successfully --- flocs_processing/dags/vlbi_single_target.py | 30 ++++++++++++++------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/flocs_processing/dags/vlbi_single_target.py b/flocs_processing/dags/vlbi_single_target.py index 6a1704b..11e21a7 100644 --- a/flocs_processing/dags/vlbi_single_target.py +++ b/flocs_processing/dags/vlbi_single_target.py @@ -13,13 +13,13 @@ from stager_access import get_surls_requested, get_surls_online # Need to replace this with a config file -TABLE_NAME = "" -DATABASE = "" -SLURM_ACCOUNT = "" -SLURM_QUEUE = "" -DATA_DIR = "" -OUTPUT_DIR = "" -PROCESSING_DIR = "" +TABLE_NAME = "processing_banados" +DATABASE = "/project/lofarvlbi/Data/fsweijen/banados-high-z/banados_airflow.sqlite" +SLURM_ACCOUNT = "lofarvlbi-fsweijen" +SLURM_QUEUE = "normal,long" +DATA_DIR = "/project/lofarvlbi/Data/fsweijen/banados-high-z" +OUTPUT_DIR = "/project/lofarvlbi/Data/fsweijen/banados-high-z" +PROCESSING_DIR = "/project/lofarvlbi/Data/fsweijen/banados-high-z/processing" @functools.total_ordering @@ -81,6 +81,15 @@ def set_status_downloaded(name, target): ) +def set_field_finished(name, target): + # name = str(field_dict["target_name"]) + # target = str(field_dict["sas_id_target"]) + query = f"update {TABLE_NAME} set downloaded=1 where target_name=='{name}' and sas_id_target=='{target}'" + with sqlite3.connect(DATABASE) as db: + cursor = db.cursor() + cursor.execute(query) + + def set_final_calibrator(name, target, final_cal): with sqlite3.connect(DATABASE) as db: cursor = db.cursor() @@ -105,7 +114,7 @@ def get_most_recent_run(searchpath: str, sas_id: str, pipeline: str) -> pathlib. @dag -def linc(): +def single_target_vlbi(max_active_runs=1): @task def get_unprocessed_target(): field = dict(get_db_columns()[0]) @@ -412,12 +421,14 @@ def run_vlbi_ddcal(field): outdir, field["sas_id_target"], "LINC_target" ) target_ms_path = target_path / "results_LINC_target" / "results" + print(f"Using LINC target run: {target_path}") sols_path = get_most_recent_run( outdir, field["sas_id_target"], "VLBI_delay" ) sols_path = sols_path / "results_VLBI_delay-calibration" / "results" sols = sols_path.glob("merged_*_selfcalcycle???_linearfulljones*.h5") + print(f"Using PILOT delay calibration run: {sols_path}") source_cat = os.path.join( DATA_DIR, field["target_name"], "target", "vlbi_target.csv" @@ -447,6 +458,7 @@ def run_vlbi_ddcal(field): set_status_finished( field["target_name"], "vlbi_dd", field["sas_id_target"] ) + set_field_finished(field["target_name"], field["sas_id_target"]) else: raise RuntimeError return field @@ -470,4 +482,4 @@ def run_ddf_pipeline(field): proceed >> get_field -linc() +single_target_vlbi() From 245b360507dfb5f0b3f3c0d916995f64ac1f7f93 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Tue, 9 Jun 2026 16:49:13 +0100 Subject: [PATCH 011/120] One target list per field --- flocs_processing/dags/vlbi_single_target.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flocs_processing/dags/vlbi_single_target.py b/flocs_processing/dags/vlbi_single_target.py index 11e21a7..e789230 100644 --- a/flocs_processing/dags/vlbi_single_target.py +++ b/flocs_processing/dags/vlbi_single_target.py @@ -431,7 +431,7 @@ def run_vlbi_ddcal(field): print(f"Using PILOT delay calibration run: {sols_path}") source_cat = os.path.join( - DATA_DIR, field["target_name"], "target", "vlbi_target.csv" + DATA_DIR, field["target_name"], "vlbi_target.csv" ) if not os.path.isfile(source_cat): raise AirflowFailException(f"{source_cat} not found.") From 137c4560257f2b04ec7f586dcfb26770959a9e6e Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Tue, 9 Jun 2026 16:55:16 +0100 Subject: [PATCH 012/120] Sort priority descending --- flocs_processing/dags/vlbi_single_target.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flocs_processing/dags/vlbi_single_target.py b/flocs_processing/dags/vlbi_single_target.py index e789230..437790e 100644 --- a/flocs_processing/dags/vlbi_single_target.py +++ b/flocs_processing/dags/vlbi_single_target.py @@ -51,7 +51,7 @@ def get_db_columns(): cursor = db.cursor() columns = "target_name,priority,finished,downloaded,sas_id_calibrator1,sas_id_calibrator2,sas_id_calibrator_final,sas_id_target,status_calibrator1,status_calibrator2,status_target,status_vlbi_delay,status_vlbi_dd" field = cursor.execute( - f"select {columns} from {TABLE_NAME} where finished==0 order by priority" + f"select {columns} from {TABLE_NAME} where finished==0 order by priority desc" ).fetchall() print(field) return field From f1966cab2ed2e4eece787f4adf38ef016fb496dc Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 10 Jun 2026 09:07:27 +0100 Subject: [PATCH 013/120] Empty explicit paths --- flocs_processing/dags/vlbi_single_target.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/flocs_processing/dags/vlbi_single_target.py b/flocs_processing/dags/vlbi_single_target.py index 437790e..6976cd0 100644 --- a/flocs_processing/dags/vlbi_single_target.py +++ b/flocs_processing/dags/vlbi_single_target.py @@ -13,13 +13,13 @@ from stager_access import get_surls_requested, get_surls_online # Need to replace this with a config file -TABLE_NAME = "processing_banados" -DATABASE = "/project/lofarvlbi/Data/fsweijen/banados-high-z/banados_airflow.sqlite" -SLURM_ACCOUNT = "lofarvlbi-fsweijen" -SLURM_QUEUE = "normal,long" -DATA_DIR = "/project/lofarvlbi/Data/fsweijen/banados-high-z" -OUTPUT_DIR = "/project/lofarvlbi/Data/fsweijen/banados-high-z" -PROCESSING_DIR = "/project/lofarvlbi/Data/fsweijen/banados-high-z/processing" +TABLE_NAME = "" +DATABASE = "" +SLURM_ACCOUNT = "" +SLURM_QUEUE = "" +DATA_DIR = "" +OUTPUT_DIR = "" +PROCESSING_DIR = "" @functools.total_ordering @@ -428,11 +428,9 @@ def run_vlbi_ddcal(field): ) sols_path = sols_path / "results_VLBI_delay-calibration" / "results" sols = sols_path.glob("merged_*_selfcalcycle???_linearfulljones*.h5") - print(f"Using PILOT delay calibration run: {sols_path}") + print(f"Using PILOT delay calibration solutions: {sols}") - source_cat = os.path.join( - DATA_DIR, field["target_name"], "vlbi_target.csv" - ) + source_cat = os.path.join(DATA_DIR, field["target_name"], "vlbi_target.csv") if not os.path.isfile(source_cat): raise AirflowFailException(f"{source_cat} not found.") From cc5421b241670d26913bde5bd1b10c185f548944 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 10 Jun 2026 09:14:24 +0100 Subject: [PATCH 014/120] Fix delay sols search path --- flocs_processing/dags/vlbi_single_target.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flocs_processing/dags/vlbi_single_target.py b/flocs_processing/dags/vlbi_single_target.py index 6976cd0..72810cd 100644 --- a/flocs_processing/dags/vlbi_single_target.py +++ b/flocs_processing/dags/vlbi_single_target.py @@ -426,7 +426,7 @@ def run_vlbi_ddcal(field): sols_path = get_most_recent_run( outdir, field["sas_id_target"], "VLBI_delay" ) - sols_path = sols_path / "results_VLBI_delay-calibration" / "results" + sols_path = sols_path / "results_VLBI_delay-calibration" sols = sols_path.glob("merged_*_selfcalcycle???_linearfulljones*.h5") print(f"Using PILOT delay calibration solutions: {sols}") From b72eec108577a5b935e778fc34a808b97e9b1c8c Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 10 Jun 2026 09:20:43 +0100 Subject: [PATCH 015/120] Fix suffix search --- flocs_processing/dags/vlbi_single_target.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flocs_processing/dags/vlbi_single_target.py b/flocs_processing/dags/vlbi_single_target.py index 72810cd..907d59b 100644 --- a/flocs_processing/dags/vlbi_single_target.py +++ b/flocs_processing/dags/vlbi_single_target.py @@ -437,7 +437,7 @@ def run_vlbi_ddcal(field): set_status_processing( field["target_name"], "vlbi_dd", field["sas_id_target"] ) - cmd = f"flocs-run vlbi dd-calibration --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --delay-solset {sols} --phasediff-score 10.0 --source-catalogue {source_cat} {target_ms_path}" + cmd = f"flocs-run vlbi dd-calibration --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --delay-solset {sols} --phasediff-score 10.0 --source-catalogue {source_cat} --ms-suffix .dp3concat {target_ms_path}" if not os.path.isdir(outdir): os.mkdir(outdir) print(cmd) From ee81117dc58529aaab1f2b7725c8ddf4be41f667 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 10 Jun 2026 15:01:18 +0100 Subject: [PATCH 016/120] Tweak most recent dir --- flocs_processing/dags/vlbi_single_target.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flocs_processing/dags/vlbi_single_target.py b/flocs_processing/dags/vlbi_single_target.py index 907d59b..493ab5e 100644 --- a/flocs_processing/dags/vlbi_single_target.py +++ b/flocs_processing/dags/vlbi_single_target.py @@ -100,12 +100,12 @@ def set_final_calibrator(name, target, final_cal): def get_most_recent_run(searchpath: str, sas_id: str, pipeline: str) -> pathlib.Path: rundirs = pathlib.Path(searchpath) - rundirs_sorted = sorted(rundirs.iterdir(), key=os.path.getctime) + rundirs_sorted = sorted(rundirs.iterdir()) if pipeline: rundirs_sorted_filtered = [ d for d in rundirs_sorted - if ((sas_id in d.parts[-1]) and (pipeline in d.parts[-1])) + if ((sas_id in d.parts[-1]) and (pipeline in d.parts[-1])) and d.is_dir() ] else: rundirs_sorted_filtered = [d for d in rundirs_sorted if sas_id in d.parts[-1]] From e36d32734ead5d4b547e218cfdfb65b590eb2516 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Mon, 15 Jun 2026 13:55:06 +0100 Subject: [PATCH 017/120] Fix target search path in delay --- flocs_processing/dags/vlbi_single_target.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flocs_processing/dags/vlbi_single_target.py b/flocs_processing/dags/vlbi_single_target.py index 493ab5e..31f967d 100644 --- a/flocs_processing/dags/vlbi_single_target.py +++ b/flocs_processing/dags/vlbi_single_target.py @@ -113,8 +113,8 @@ def get_most_recent_run(searchpath: str, sas_id: str, pipeline: str) -> pathlib. return rundir_final -@dag -def single_target_vlbi(max_active_runs=1): +@dag(max_active_runs=1) +def single_target_vlbi(): @task def get_unprocessed_target(): field = dict(get_db_columns()[0]) @@ -377,7 +377,7 @@ def run_vlbi_delay(field): ms_folder = f"L{field['sas_id_target']}" outdir = os.path.join(OUTPUT_DIR, field["target_name"]) target_path = get_most_recent_run( - outdir, field["sas_id_calibrator_final"], "LINC_target" + outdir, field["sas_id_target"], "LINC_target" ) target_ms_path = target_path / "results_LINC_target" / "results" set_status_processing( From 28037f255f0ca1f4b0dd230531e0835d7438f25a Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Mon, 15 Jun 2026 14:13:25 +0100 Subject: [PATCH 018/120] Fix suffix for delay calibration --- flocs_processing/dags/vlbi_single_target.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flocs_processing/dags/vlbi_single_target.py b/flocs_processing/dags/vlbi_single_target.py index 31f967d..9831325 100644 --- a/flocs_processing/dags/vlbi_single_target.py +++ b/flocs_processing/dags/vlbi_single_target.py @@ -383,7 +383,7 @@ def run_vlbi_delay(field): set_status_processing( field["target_name"], "vlbi_delay", field["sas_id_target"] ) - cmd = f"flocs-run vlbi delay-calibration --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} {target_ms_path}" + cmd = f"flocs-run vlbi delay-calibration --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --ms-suffix dp3concat {target_ms_path}" if not os.path.isdir(outdir): os.mkdir(outdir) print(cmd) From 16781d25d76f498fca987a04478ea67d932f54f3 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Mon, 15 Jun 2026 17:24:50 +0100 Subject: [PATCH 019/120] More power: restarts and bad nodes --- flocs_processing/dags/vlbi_single_target.py | 50 ++++++++++++++++++--- 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/flocs_processing/dags/vlbi_single_target.py b/flocs_processing/dags/vlbi_single_target.py index 9831325..d60512e 100644 --- a/flocs_processing/dags/vlbi_single_target.py +++ b/flocs_processing/dags/vlbi_single_target.py @@ -1,13 +1,15 @@ from enum import Enum +import datetime import functools import os import pathlib +import re import sqlite3 import subprocess import time from airflow.exceptions import AirflowFailException, AirflowSkipException -from airflow.sdk import dag, task +from airflow.sdk import dag, get_current_context, task from airflow.task.trigger_rule import TriggerRule from flocs_lta.lta_search import ObservationStager from stager_access import get_surls_requested, get_surls_online @@ -20,6 +22,7 @@ DATA_DIR = "" OUTPUT_DIR = "" PROCESSING_DIR = "" +NN_MODEL_CACHE = "" @functools.total_ordering @@ -364,7 +367,7 @@ def run_linc_target(field): def validate_linc_target(field): return field - @task + @task(retries=2, retry_delay=datetime.timedelta(seconds=5)) def run_vlbi_delay(field): if (field["status_vlbi_delay"] == PIPELINE_STATUS.finished) or ( field["status_vlbi_delay"] == PIPELINE_STATUS.running @@ -374,7 +377,6 @@ def run_vlbi_delay(field): print( f"Processing delay calibration for {field['target_name']} {field['sas_id_target']}" ) - ms_folder = f"L{field['sas_id_target']}" outdir = os.path.join(OUTPUT_DIR, field["target_name"]) target_path = get_most_recent_run( outdir, field["sas_id_target"], "LINC_target" @@ -383,26 +385,60 @@ def run_vlbi_delay(field): set_status_processing( field["target_name"], "vlbi_delay", field["sas_id_target"] ) - cmd = f"flocs-run vlbi delay-calibration --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --ms-suffix dp3concat {target_ms_path}" + + delay_cat = os.path.join(outdir, "delay_calibrators.csv") + + proc = subprocess.run("detect_bad_slurm_nodes.sh", shell=True, text=True) + bad_nodes = proc.stdout + os.environ["TOIL_SLURM_ARGS"] = f"--exclude={bad_nodes}" + + context = get_current_context() + if context["ti"].try_number == 1: + cmd = f"flocs-run vlbi delay-calibration --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --ms-suffix dp3concat --delay-calibrator {delay_cat} {target_ms_path}" + else: + flocs_workdir = context["ti"].xcom_pull( + task_ids=context["ti"].task_id, key="flocs_workdir" + ) + print(f"Resuming failed PILOT run in {flocs_workdir}") + cmd = f"flocs-run vlbi delay-calibration --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --ms-suffix dp3concat --delay-calibrators {delay_cat} {target_ms_path}" if not os.path.isdir(outdir): os.mkdir(outdir) print(cmd) with open( f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt", - "w", + "w+", ) as f_out, open( f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}_err.txt", - "w", + "w+", ) as f_err: proc = subprocess.run( cmd, shell=True, text=True, stdout=f_out, stderr=f_err ) + success = False + pattern = re.compile(r"Workflow.* stopped. Success: False") if not proc.returncode: - return True + if not pattern.search(proc.stderr): + success = True + + if success: set_status_finished( field["target_name"], "vlbi_delay", field["sas_id_target"] ) else: + f_out.seek(0) + flocs_workdir = "" + for line in f_out.readlines(): + print(line) + if "Running workflow with" in line: + flocs_workdir = line.split(" ")[-1] + break + if not flocs_workdir: + raise RuntimeError("Could not retrieve PILOT workdir. Flocs probably crashed before launching.") + if context["ti"].try_number == 1: + print(f"PILOT failed, storing rundir {flocs_workdir} for retries in `flocs_workdir`") + context["ti"].xcom_push( + key="flocs_workdir", value=flocs_workdir + ) raise RuntimeError return field From 2135952b0e4c7a7bbc7c396bd0025bf05ebad51d Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Mon, 15 Jun 2026 17:25:13 +0100 Subject: [PATCH 020/120] Remove outdated LINC dag --- flocs_processing/dags/linc.py | 418 ---------------------------------- 1 file changed, 418 deletions(-) delete mode 100644 flocs_processing/dags/linc.py diff --git a/flocs_processing/dags/linc.py b/flocs_processing/dags/linc.py deleted file mode 100644 index 1238f17..0000000 --- a/flocs_processing/dags/linc.py +++ /dev/null @@ -1,418 +0,0 @@ -from enum import Enum -import functools -import os -import pathlib -import sqlite3 -import subprocess -import time - -from airflow.exceptions import AirflowFailException, AirflowSkipException -from airflow.sdk import dag, task -from airflow.task.trigger_rule import TriggerRule -from flocs_lta.lta_search import ObservationStager -from stager_access import get_surls_requested, get_surls_online - -# Need to replace this with a config file -TABLE_NAME = "" -DATABASE = "" -SLURM_ACCOUNT = "" -SLURM_QUEUE = "" -DATA_DIR = "" -OUTPUT_DIR = "" -PROCESSING_DIR = "" - - -@functools.total_ordering -class PIPELINE_STATUS(Enum): - nothing = 0 - downloaded = 1 - finished = 2 - running = 3 - processing = 98 - error = 99 - - def __eq__(self, other): - if other.__class__ is int: - return self.value == other - elif other.__class__ is self.__class__: - return self.value == other.value - else: - raise NotImplementedError - - def __lt__(self, other): - if self.__class__ is not other.__class__: - raise NotImplementedError - return self.value < other.value - - -def get_db_columns(): - with sqlite3.connect(DATABASE) as db: - db.row_factory = sqlite3.Row - cursor = db.cursor() - columns = "target_name,priority,finished,downloaded,sas_id_calibrator1,sas_id_calibrator2,sas_id_calibrator_final,sas_id_target,status_calibrator1,status_calibrator2,status_target,status_vlbi_delay" - field = cursor.execute( - f"select {columns} from {TABLE_NAME} where finished==0 order by priority" - ).fetchall() - print(field) - return field - - -def set_status_processing(name, identifier, target): - with sqlite3.connect(DATABASE) as db: - cursor = db.cursor() - cursor.execute( - f"update {TABLE_NAME} set status_{identifier}={PIPELINE_STATUS.processing.value} where target_name=='{name}' and sas_id_target=='{target}'" - ) - - -def set_status_finished(name, identifier, target): - with sqlite3.connect(DATABASE) as db: - cursor = db.cursor() - cursor.execute( - f"update {TABLE_NAME} set status_{identifier}={PIPELINE_STATUS.finished.value} where target_name=='{name}' and sas_id_target=='{target}'" - ) - - -def set_status_downloaded(name, target): - with sqlite3.connect(DATABASE) as db: - cursor = db.cursor() - cursor.execute( - f"update {TABLE_NAME} set downloaded=1 where target_name=='{name}' and sas_id_target=='{target}'" - ) - - -def set_final_calibrator(name, target, final_cal): - with sqlite3.connect(DATABASE) as db: - cursor = db.cursor() - cursor.execute( - f"update {TABLE_NAME} set sas_id_calibrator_final={final_cal} where target_name=='{name}' and sas_id_target=='{target}'" - ) - - -def get_most_recent_run(searchpath: str, sas_id: str, pipeline: str) -> pathlib.Path: - rundirs = pathlib.Path(searchpath) - rundirs_sorted = sorted(rundirs.iterdir(), key=os.path.getctime) - if pipeline: - rundirs_sorted_filtered = [ - d - for d in rundirs_sorted - if ((sas_id in d.parts[-1]) and (pipeline in d.parts[-1])) - ] - else: - rundirs_sorted_filtered = [d for d in rundirs_sorted if sas_id in d.parts[-1]] - rundir_final = rundirs_sorted_filtered[-1].absolute() - return rundir_final - - -@dag -def linc(): - @task - def get_unprocessed_target(): - field = dict(get_db_columns()[0]) - print(field["target_name"]) - return field - - @task.short_circuit - def check_fields(): - fields = get_db_columns() - return bool(fields) - - @task - def download_field(field): - if field["downloaded"]: - return field - else: - has_cal1 = False - stage_calibrators = False - if field["sas_id_calibrator1"]: - ms_folder = f"L{field['sas_id_calibrator1']}" - cal1_full_path = os.path.join( - DATA_DIR, field["target_name"], "calibrator", ms_folder - ) - if os.path.exists(cal1_full_path): - has_cal1 = True - else: - stage_calibrators = True - - has_cal2 = False - if field["sas_id_calibrator2"]: - ms_folder = f"L{field['sas_id_calibrator2']}" - cal2_full_path = os.path.join( - DATA_DIR, field["target_name"], "calibrator", ms_folder - ) - if os.path.exists(cal2_full_path): - has_cal2 = True - else: - stage_calibrators = True - if field["sas_id_target"]: - ms_folder = f"L{field['sas_id_target']}" - target_full_path = os.path.join( - DATA_DIR, field["target_name"], "target", ms_folder - ) - if os.path.exists(target_full_path): - stage_target = False - else: - stage_target = True - else: - raise AirflowFailException( - f"No target SAS ID in database for field {field['target_name']}" - ) - - if stage_calibrators or stage_target: - print(f"Field {field['sas_id_target']} is not downloaded.") - stager = ObservationStager(get_surls=True) - stager.find_observation_by_sasid( - "ALL", - field["sas_id_target"], - None, - 120e6, - 168e6, - ) - if stage_calibrators: - stager.find_nearest_calibrators(2, 120e6, 168e6) - stage_id_calibrators = stager.stage_calibrators() - if stage_target: - stage_id_target = stager.stage_target() - else: - return field - - calibrator_staged = False - target_staged = False - calibrator_downloaded = has_cal1 or has_cal2 - target_downloaded = not stage_target - while True: - if len(get_surls_online(stage_id_calibrators)) == len( - get_surls_requested(stage_id_calibrators) - ): - calibrator_staged = True - if calibrator_staged and not calibrator_downloaded: - dl_path = os.path.join(DATA_DIR, field["target_name"], "calibrator") - cmd = ( - f"flocs-lta download --outdir {dl_path} {stage_id_calibrators}" - ) - with open( - f"log_download_calibrators_{field['target_name']}.txt", - "w", - ) as f_out, open( - f"log_download_calibrators_{field['target_name']}.txt", - "w", - ) as f_err: - proc = subprocess.run( - cmd, shell=True, text=True, stdout=f_out, stderr=f_err - ) - if not proc.returncode: - calibrator_downloaded = True - else: - raise RuntimeError - - if len(get_surls_online(stage_id_target)) == len( - get_surls_requested(stage_id_target) - ): - calibrator_staged = True - if target_staged and not target_downloaded: - dl_path = os.path.join(DATA_DIR, field["target_name"], "target") - cmd = f"flocs-lta download --outdir {dl_path} {stage_id_target}" - with open( - f"log_download_calibrators_{field['target_name']}.txt", - "w", - ) as f_out, open( - f"log_download_calibrators_{field['target_name']}.txt", - "w", - ) as f_err: - proc = subprocess.run( - cmd, shell=True, text=True, stdout=f_out, stderr=f_err - ) - if not proc.returncode: - set_status_downloaded( - field["target_name"], - field["sas_id_target"], - ) - target_downloaded = True - else: - raise RuntimeError - if calibrator_downloaded and target_downloaded: - break - time.sleep(60) - - @task - def run_linc_calibrator1(field): - if (field["status_calibrator1"] == PIPELINE_STATUS.finished) or ( - field["status_calibrator1"] == PIPELINE_STATUS.running - ): - print( - f"Flux density calibrator {field['sas_id_calibrator1']} for observation {field['target_name']} {field['sas_id_target']} already processed." - ) - return field - else: - print( - f"Processing flux density calibrator {field['sas_id_calibrator1']} for observation {field['target_name']} {field['sas_id_target']}" - ) - ms_folder = f"L{field['sas_id_calibrator1']}" - set_status_processing( - field["target_name"], "calibrator1", field["sas_id_target"] - ) - outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - cmd = f"flocs-run linc calibrator --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} {os.path.join(DATA_DIR, field['target_name'], 'calibrator', ms_folder)}" - if not os.path.isdir(outdir): - os.mkdir(outdir) - print(cmd) - with open( - f"log_LINC_calibrator_{field['target_name']}_{field['sas_id_calibrator1']}.txt", - "w", - ) as f_out, open( - f"log_LINC_calibrator_{field['target_name']}_{field['sas_id_calibrator1']}_err.txt", - "w", - ) as f_err: - proc = subprocess.run( - cmd, shell=True, text=True, stdout=f_out, stderr=f_err - ) - if not proc.returncode: - set_status_finished( - field["target_name"], "calibrator1", field["sas_id_target"] - ) - else: - raise RuntimeError - return field - - @task - def run_linc_calibrator2(field): - raise AirflowSkipException() - - @task(trigger_rule=TriggerRule.ONE_DONE) - def select_best_calibrator(result1, result2): - if result1 and result2: - print("Selecting between cal1 and cal2") - set_final_calibrator( - result1["target_name"], - result1["sas_id_target"], - result1["sas_id_calibrator1"], - ) - return result1 - elif result1 and (not result2): - print("Only cal 1 succeeded, continuing with that") - set_final_calibrator( - result1["target_name"], - result1["sas_id_target"], - result1["sas_id_calibrator1"], - ) - return result1 - elif (not result1) and result2: - print("Only cal 2 succeeded, continuing with that") - set_final_calibrator( - result1["target_name"], - result1["sas_id_target"], - result1["sas_id_calibrator2"], - ) - return result2 - else: - raise AirflowFailException("No calibrators succeeded; stopping processing.") - - @task - def run_linc_target(field): - if (field["status_target"] == PIPELINE_STATUS.finished) or ( - field["status_target"] == PIPELINE_STATUS.running - ): - return field - else: - print( - f"Processing target observation {field['target_name']} {field['sas_id_target']} with calibrator {field['sas_id_calibrator_final']}" - ) - ms_folder = f"L{field['sas_id_target']}" - outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - calibrator_path = get_most_recent_run( - outdir, field["sas_id_calibrator_final"], "LINC_calibrator" - ) - calibrator_solutions = ( - calibrator_path / "results_LINC_calibrator" / "cal_solutions.h5" - ) - set_status_processing( - field["target_name"], "target", field["sas_id_target"] - ) - cmd = f"flocs-run linc target --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --cal-solutions {calibrator_solutions} {os.path.join(DATA_DIR, field['target_name'], 'target', ms_folder)}" - if not os.path.isdir(outdir): - os.mkdir(outdir) - print(cmd) - with open( - f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}.txt", - "w", - ) as f_out, open( - f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}_err.txt", - "w", - ) as f_err: - proc = subprocess.run( - cmd, shell=True, text=True, stdout=f_out, stderr=f_err - ) - if not proc.returncode: - return True - set_status_finished( - field["target_name"], "target", field["sas_id_target"] - ) - else: - raise RuntimeError - return field - - @task - def validate_linc_target(field): - return True - - @task - def run_vlbi_delay(field): - if (field["status_vlbi_delay"] == PIPELINE_STATUS.finished) or ( - field["status_vlbi_delay"] == PIPELINE_STATUS.running - ): - return field - else: - print( - f"Processing delay calibration for {field['target_name']} {field['sas_id_target']}" - ) - ms_folder = f"L{field['sas_id_target']}" - outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - target_path = get_most_recent_run( - outdir, field["sas_id_calibrator_final"], "LINC_target" - ) - target_ms_path = target_path / "results_LINC_target" / "results" - set_status_processing( - field["target_name"], "vlbi_delay", field["sas_id_target"] - ) - cmd = f"flocs-run vlbi delay-calibration --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} {target_ms_path}" - if not os.path.isdir(outdir): - os.mkdir(outdir) - print(cmd) - with open( - f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt", - "w", - ) as f_out, open( - f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}_err.txt", - "w", - ) as f_err: - proc = subprocess.run( - cmd, shell=True, text=True, stdout=f_out, stderr=f_err - ) - if not proc.returncode: - return True - set_status_finished( - field["target_name"], "vlbi_delay", field["sas_id_target"] - ) - else: - raise RuntimeError - return field - - @task - def run_ddf_pipeline(field): - return True - - proceed = check_fields() - get_field = get_unprocessed_target() - field = download_field(get_field) - result_cal1 = run_linc_calibrator1(field) - result_cal2 = run_linc_calibrator2(field) - best_cal = select_best_calibrator(result_cal1, result_cal2) - result_targ = run_linc_target(best_cal) - linc_is_valid = validate_linc_target(result_targ) - result_vlbi_delay = run_vlbi_delay(linc_is_valid) - # run_ddf_pipeline(vlbi_delay_is_valid) - - proceed >> get_field - - -linc() From e9741f49dd2b19db5f5831fbaf2dd2671eb1a00e Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Tue, 16 Jun 2026 09:32:28 +0100 Subject: [PATCH 021/120] Attempt to retrieve rundir on the fly --- flocs_processing/dags/vlbi_single_target.py | 55 ++++++++++----------- 1 file changed, 25 insertions(+), 30 deletions(-) diff --git a/flocs_processing/dags/vlbi_single_target.py b/flocs_processing/dags/vlbi_single_target.py index d60512e..06840dd 100644 --- a/flocs_processing/dags/vlbi_single_target.py +++ b/flocs_processing/dags/vlbi_single_target.py @@ -15,14 +15,14 @@ from stager_access import get_surls_requested, get_surls_online # Need to replace this with a config file -TABLE_NAME = "" -DATABASE = "" -SLURM_ACCOUNT = "" -SLURM_QUEUE = "" -DATA_DIR = "" -OUTPUT_DIR = "" -PROCESSING_DIR = "" -NN_MODEL_CACHE = "" +TABLE_NAME = "processing_banados" +DATABASE = "/project/lofarvlbi/Data/fsweijen/banados-high-z/banados_airflow.sqlite" +SLURM_ACCOUNT = "lofarvlbi" +SLURM_QUEUE = "normal" +DATA_DIR = "/project/lofarvlbi/Data/fsweijen/banados-high-z" +OUTPUT_DIR = "/project/lofarvlbi/Data/fsweijen/banados-high-z" +PROCESSING_DIR = "/project/lofarvlbi/Data/fsweijen/banados-high-z/processing" +NN_MODEL_CACHE = "/project/lofarvlbi/Software/fsweijen/nn_cache" @functools.total_ordering @@ -367,7 +367,7 @@ def run_linc_target(field): def validate_linc_target(field): return field - @task(retries=2, retry_delay=datetime.timedelta(seconds=5)) + @task(retries=0, retry_delay=datetime.timedelta(seconds=5)) def run_vlbi_delay(field): if (field["status_vlbi_delay"] == PIPELINE_STATUS.finished) or ( field["status_vlbi_delay"] == PIPELINE_STATUS.running @@ -388,19 +388,28 @@ def run_vlbi_delay(field): delay_cat = os.path.join(outdir, "delay_calibrators.csv") - proc = subprocess.run("detect_bad_slurm_nodes.sh", shell=True, text=True) - bad_nodes = proc.stdout - os.environ["TOIL_SLURM_ARGS"] = f"--exclude={bad_nodes}" + proc = subprocess.run("detect_bad_slurm_nodes.sh", shell=True, text=True, stdout=subprocess.PIPE) + bad_nodes = proc.stdout.strip() + if bad_nodes: + os.environ["TOIL_SLURM_ARGS"] = f"--exclude={bad_nodes}" context = get_current_context() if context["ti"].try_number == 1: cmd = f"flocs-run vlbi delay-calibration --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --ms-suffix dp3concat --delay-calibrator {delay_cat} {target_ms_path}" else: - flocs_workdir = context["ti"].xcom_pull( - task_ids=context["ti"].task_id, key="flocs_workdir" - ) + # Extract the previous working directory + flocs_workdir = "" + print(f"Scanning log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt for workdir.") + with open(f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt") as f_out: + for line in f_out.readlines(): + print(line) + if "Running workflow with" in line: + flocs_workdir = line.split(" ")[-1] + break + if not flocs_workdir: + raise RuntimeError("Could not retrieve PILOT workdir. Flocs probably crashed before launching.") print(f"Resuming failed PILOT run in {flocs_workdir}") - cmd = f"flocs-run vlbi delay-calibration --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --ms-suffix dp3concat --delay-calibrators {delay_cat} {target_ms_path}" + cmd = f"flocs-run vlbi delay-calibration --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --ms-suffix dp3concat --delay-calibrator {delay_cat} {target_ms_path}" if not os.path.isdir(outdir): os.mkdir(outdir) print(cmd) @@ -425,20 +434,6 @@ def run_vlbi_delay(field): field["target_name"], "vlbi_delay", field["sas_id_target"] ) else: - f_out.seek(0) - flocs_workdir = "" - for line in f_out.readlines(): - print(line) - if "Running workflow with" in line: - flocs_workdir = line.split(" ")[-1] - break - if not flocs_workdir: - raise RuntimeError("Could not retrieve PILOT workdir. Flocs probably crashed before launching.") - if context["ti"].try_number == 1: - print(f"PILOT failed, storing rundir {flocs_workdir} for retries in `flocs_workdir`") - context["ti"].xcom_push( - key="flocs_workdir", value=flocs_workdir - ) raise RuntimeError return field From 0644f6eddd8623ee40634bd117dd351e2e79cbc3 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Tue, 16 Jun 2026 13:55:11 +0100 Subject: [PATCH 022/120] Small updates --- flocs_processing/dags/vlbi_single_target.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/flocs_processing/dags/vlbi_single_target.py b/flocs_processing/dags/vlbi_single_target.py index 06840dd..04948e6 100644 --- a/flocs_processing/dags/vlbi_single_target.py +++ b/flocs_processing/dags/vlbi_single_target.py @@ -220,7 +220,7 @@ def download_field(field): if len(get_surls_online(stage_id_target)) == len( get_surls_requested(stage_id_target) ): - calibrator_staged = True + target_staged = True if target_staged and not target_downloaded: dl_path = os.path.join(DATA_DIR, field["target_name"], "target") cmd = f"flocs-lta download --outdir {dl_path} {stage_id_target}" @@ -388,7 +388,12 @@ def run_vlbi_delay(field): delay_cat = os.path.join(outdir, "delay_calibrators.csv") - proc = subprocess.run("detect_bad_slurm_nodes.sh", shell=True, text=True, stdout=subprocess.PIPE) + proc = subprocess.run( + "detect_bad_slurm_nodes.sh", + shell=True, + text=True, + stdout=subprocess.PIPE, + ) bad_nodes = proc.stdout.strip() if bad_nodes: os.environ["TOIL_SLURM_ARGS"] = f"--exclude={bad_nodes}" @@ -399,15 +404,21 @@ def run_vlbi_delay(field): else: # Extract the previous working directory flocs_workdir = "" - print(f"Scanning log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt for workdir.") - with open(f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt") as f_out: + print( + f"Scanning log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt for workdir." + ) + with open( + f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt" + ) as f_out: for line in f_out.readlines(): print(line) if "Running workflow with" in line: flocs_workdir = line.split(" ")[-1] break if not flocs_workdir: - raise RuntimeError("Could not retrieve PILOT workdir. Flocs probably crashed before launching.") + raise RuntimeError( + "Could not retrieve PILOT workdir. Flocs probably crashed before launching." + ) print(f"Resuming failed PILOT run in {flocs_workdir}") cmd = f"flocs-run vlbi delay-calibration --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --ms-suffix dp3concat --delay-calibrator {delay_cat} {target_ms_path}" if not os.path.isdir(outdir): From 74dd7ef0b07b113bfbe04f33831500b15accf669 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Thu, 18 Jun 2026 13:47:32 +0100 Subject: [PATCH 023/120] Remove explicit paths --- flocs_processing/dags/vlbi_single_target.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/flocs_processing/dags/vlbi_single_target.py b/flocs_processing/dags/vlbi_single_target.py index 04948e6..4c645ef 100644 --- a/flocs_processing/dags/vlbi_single_target.py +++ b/flocs_processing/dags/vlbi_single_target.py @@ -15,14 +15,14 @@ from stager_access import get_surls_requested, get_surls_online # Need to replace this with a config file -TABLE_NAME = "processing_banados" -DATABASE = "/project/lofarvlbi/Data/fsweijen/banados-high-z/banados_airflow.sqlite" -SLURM_ACCOUNT = "lofarvlbi" -SLURM_QUEUE = "normal" -DATA_DIR = "/project/lofarvlbi/Data/fsweijen/banados-high-z" -OUTPUT_DIR = "/project/lofarvlbi/Data/fsweijen/banados-high-z" -PROCESSING_DIR = "/project/lofarvlbi/Data/fsweijen/banados-high-z/processing" -NN_MODEL_CACHE = "/project/lofarvlbi/Software/fsweijen/nn_cache" +TABLE_NAME = "" +DATABASE = "" +SLURM_ACCOUNT = "" +SLURM_QUEUE = "" +DATA_DIR = "" +OUTPUT_DIR = "" +PROCESSING_DIR = "" +NN_MODEL_CACHE = "" @functools.total_ordering From b761683906a90a59f8a6f90198e1e1c6aa879f79 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Sun, 21 Jun 2026 14:04:27 +0100 Subject: [PATCH 024/120] Fix dd cal and update success check --- flocs_processing/dags/vlbi_single_target.py | 49 ++++++++++----------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/flocs_processing/dags/vlbi_single_target.py b/flocs_processing/dags/vlbi_single_target.py index 4c645ef..97b8ff4 100644 --- a/flocs_processing/dags/vlbi_single_target.py +++ b/flocs_processing/dags/vlbi_single_target.py @@ -220,7 +220,7 @@ def download_field(field): if len(get_surls_online(stage_id_target)) == len( get_surls_requested(stage_id_target) ): - target_staged = True + calibrator_staged = True if target_staged and not target_downloaded: dl_path = os.path.join(DATA_DIR, field["target_name"], "target") cmd = f"flocs-lta download --outdir {dl_path} {stage_id_target}" @@ -387,40 +387,31 @@ def run_vlbi_delay(field): ) delay_cat = os.path.join(outdir, "delay_calibrators.csv") + image_cat = os.path.join(outdir, "image_catalogue.csv") - proc = subprocess.run( - "detect_bad_slurm_nodes.sh", - shell=True, - text=True, - stdout=subprocess.PIPE, - ) + proc = subprocess.run("detect_bad_slurm_nodes.sh", shell=True, text=True, stdout=subprocess.PIPE) bad_nodes = proc.stdout.strip() if bad_nodes: + print(f"Excluding the following bad nodes from scheduling: {bad_nodes}") os.environ["TOIL_SLURM_ARGS"] = f"--exclude={bad_nodes}" context = get_current_context() if context["ti"].try_number == 1: - cmd = f"flocs-run vlbi delay-calibration --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --ms-suffix dp3concat --delay-calibrator {delay_cat} {target_ms_path}" + cmd = f"flocs-run vlbi delay-calibration --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --ms-suffix dp3concat --delay-calibrator {delay_cat} --image-catalogue {image_cat} {target_ms_path}" else: # Extract the previous working directory flocs_workdir = "" - print( - f"Scanning log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt for workdir." - ) - with open( - f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt" - ) as f_out: + print(f"Scanning log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt for workdir.") + with open(f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt") as f_out: for line in f_out.readlines(): print(line) if "Running workflow with" in line: - flocs_workdir = line.split(" ")[-1] + flocs_workdir = line.split(" ")[-1].strip() break if not flocs_workdir: - raise RuntimeError( - "Could not retrieve PILOT workdir. Flocs probably crashed before launching." - ) + raise RuntimeError("Could not retrieve PILOT workdir. Flocs probably crashed before launching.") print(f"Resuming failed PILOT run in {flocs_workdir}") - cmd = f"flocs-run vlbi delay-calibration --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --ms-suffix dp3concat --delay-calibrator {delay_cat} {target_ms_path}" + cmd = f"flocs-run vlbi delay-calibration --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --ms-suffix dp3concat --delay-calibrator {delay_cat} --image-catalogue {image_cat} {target_ms_path}" if not os.path.isdir(outdir): os.mkdir(outdir) print(cmd) @@ -435,9 +426,10 @@ def run_vlbi_delay(field): cmd, shell=True, text=True, stdout=f_out, stderr=f_err ) success = False - pattern = re.compile(r"Workflow.* stopped. Success: False") + pattern = re.compile(r"Workflow.* stopped. Success: True") if not proc.returncode: - if not pattern.search(proc.stderr): + f_err.seek(0) + if pattern.search(f_err.read()): success = True if success: @@ -469,7 +461,7 @@ def run_vlbi_ddcal(field): outdir, field["sas_id_target"], "VLBI_delay" ) sols_path = sols_path / "results_VLBI_delay-calibration" - sols = sols_path.glob("merged_*_selfcalcycle???_linearfulljones*.h5") + sols = list(sols_path.glob("merged*selfcalcycle???_linearfulljones*.h5"))[0] print(f"Using PILOT delay calibration solutions: {sols}") source_cat = os.path.join(DATA_DIR, field["target_name"], "vlbi_target.csv") @@ -479,22 +471,27 @@ def run_vlbi_ddcal(field): set_status_processing( field["target_name"], "vlbi_dd", field["sas_id_target"] ) - cmd = f"flocs-run vlbi dd-calibration --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --delay-solset {sols} --phasediff-score 10.0 --source-catalogue {source_cat} --ms-suffix .dp3concat {target_ms_path}" + cmd = f"flocs-run vlbi dd-calibration --runner toil --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --delay-solset {sols} --phasediff-score 10.0 --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat {target_ms_path}" if not os.path.isdir(outdir): os.mkdir(outdir) print(cmd) with open( f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt", - "w", + "w+", ) as f_out, open( f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}_err.txt", - "w", + "w+", ) as f_err: proc = subprocess.run( cmd, shell=True, text=True, stdout=f_out, stderr=f_err ) + success = False + pattern = re.compile(r"Workflow.* stopped. Success: True") if not proc.returncode: - return True + f_err.seek(0) + if pattern.search(f_err.read()): + success = True + if success: set_status_finished( field["target_name"], "vlbi_dd", field["sas_id_target"] ) From f59f56fc0dcfba536c313104e7e7a8cd38150113 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Sun, 21 Jun 2026 15:14:56 +0100 Subject: [PATCH 025/120] Add some documentation --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..af403d5 --- /dev/null +++ b/README.md @@ -0,0 +1,18 @@ +# End-to-end processing of ILT HBA data with flocs + +This package aims to provide relatively simple end-to-end automatic processing of ILT HBA data. Where `flocs-runners` provides the interface to running pipelines, `flocs-processing` is the scaffolding to tie it together. Data reduction is coordinated via a dedicated SQLite database that holds information on which observations to process, which pipelines to run for them and all of the related statuses. Orchestration of all the pipelines is handled via Airflow through a DAG. + +## Database setup +A database for processing is created via `flocs-processing create-database`. This will create an empty database with the necessary columns. Datasets to process can be added via `flocs-processing add-field`. + +## Processing data +To start processing data, Airflow needs to be running. This will be delegated to `flocs-processing process-from-database` in the future, but for now requires running Airflow manually. For setup do the following: + +1. Set up a folder that wil contain all of Airflow's own stuff and assign it to the `AIRFLOW_HOME` environment variable. +2. Run `airflow config list --defaults > "${AIRFLOW_HOME}/airflow.cfg"` +3. Define `AIRFLOW__CORE__DAGS_FOLDER` as `${AIRFLOW_HOME}/dags` and create the folder. Copy the DAGs inside `flocs_processing/dags` to this folder. +4. Define `AIRFLOW__CORE__LOAD_EXAMPLES` as `False` + +Next start a screen or tmux session and run `airflow standalone` to start the Airflow instance. It will echo a user name and password on first start, but will also store the credentials in `${AIRFLOW_HOME}/simple_auth_manager_passwords.json.generated`. The Airflow instance will start on port 8080. You can access it via `localhost:8080` in your browser. If it is running on a remote cluster, you can set up a tunnel via e.g. `ssh -N -L 8080:localhost:8080 ` to forward it to your local machine. + +Once `flocs-processing` is complete the processing loop will be automatic, but for now the user must trigger the DAG manually. On the "Dags" tab you should now see the flocs DAGs available. To manually trigger one, click on the name and on the subsequent page use the "Trigger" button in the top right. From 062e8bc64078af4cd3a77ee65c36c41952c2fb2f Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Sun, 21 Jun 2026 17:03:52 +0100 Subject: [PATCH 026/120] Implement calibrator2 and homogenise success triggers --- flocs_processing/dags/vlbi_single_target.py | 106 +++++++++++++++----- 1 file changed, 81 insertions(+), 25 deletions(-) diff --git a/flocs_processing/dags/vlbi_single_target.py b/flocs_processing/dags/vlbi_single_target.py index 97b8ff4..827c06d 100644 --- a/flocs_processing/dags/vlbi_single_target.py +++ b/flocs_processing/dags/vlbi_single_target.py @@ -15,14 +15,14 @@ from stager_access import get_surls_requested, get_surls_online # Need to replace this with a config file -TABLE_NAME = "" -DATABASE = "" -SLURM_ACCOUNT = "" -SLURM_QUEUE = "" -DATA_DIR = "" -OUTPUT_DIR = "" -PROCESSING_DIR = "" -NN_MODEL_CACHE = "" +TABLE_NAME = "processing_banados" +DATABASE = "/project/lofarvlbi/Data/fsweijen/banados-high-z/banados_airflow.sqlite" +SLURM_ACCOUNT = "lofarvlbi" +SLURM_QUEUE = "normal" +DATA_DIR = "/project/lofarvlbi/Data/fsweijen/banados-high-z" +OUTPUT_DIR = "/project/lofarvlbi/Data/fsweijen/banados-high-z" +PROCESSING_DIR = "/project/lofarvlbi/Data/fsweijen/banados-high-z/processing" +NN_MODEL_CACHE = "/project/lofarvlbi/Software/fsweijen/nn_cache" @functools.total_ordering @@ -85,9 +85,7 @@ def set_status_downloaded(name, target): def set_field_finished(name, target): - # name = str(field_dict["target_name"]) - # target = str(field_dict["sas_id_target"]) - query = f"update {TABLE_NAME} set downloaded=1 where target_name=='{name}' and sas_id_target=='{target}'" + query = f"update {TABLE_NAME} set finished=1 where target_name=='{name}' and sas_id_target=='{target}'" with sqlite3.connect(DATABASE) as db: cursor = db.cursor() cursor.execute(query) @@ -204,10 +202,10 @@ def download_field(field): ) with open( f"log_download_calibrators_{field['target_name']}.txt", - "w", + "w+", ) as f_out, open( f"log_download_calibrators_{field['target_name']}.txt", - "w", + "w+", ) as f_err: proc = subprocess.run( cmd, shell=True, text=True, stdout=f_out, stderr=f_err @@ -226,10 +224,10 @@ def download_field(field): cmd = f"flocs-lta download --outdir {dl_path} {stage_id_target}" with open( f"log_download_calibrators_{field['target_name']}.txt", - "w", + "w+", ) as f_out, open( f"log_download_calibrators_{field['target_name']}.txt", - "w", + "w+", ) as f_err: proc = subprocess.run( cmd, shell=True, text=True, stdout=f_out, stderr=f_err @@ -270,15 +268,21 @@ def run_linc_calibrator1(field): print(cmd) with open( f"log_LINC_calibrator_{field['target_name']}_{field['sas_id_calibrator1']}.txt", - "w", + "w+", ) as f_out, open( f"log_LINC_calibrator_{field['target_name']}_{field['sas_id_calibrator1']}_err.txt", - "w", + "w+", ) as f_err: proc = subprocess.run( cmd, shell=True, text=True, stdout=f_out, stderr=f_err ) + success = False + pattern = re.compile(r"Workflow.* stopped. Success: True") if not proc.returncode: + f_err.seek(0) + if pattern.search(f_err.read()): + success = True + if success: set_status_finished( field["target_name"], "calibrator1", field["sas_id_target"] ) @@ -288,12 +292,59 @@ def run_linc_calibrator1(field): @task def run_linc_calibrator2(field): - raise AirflowSkipException() + if (field["status_calibrator2"] == PIPELINE_STATUS.finished) or ( + field["status_calibrator2"] == PIPELINE_STATUS.running + ): + print( + f"Flux density calibrator {field['sas_id_calibrator2']} for observation {field['target_name']} {field['sas_id_target']} already processed." + ) + return field + else: + print( + f"Processing flux density calibrator {field['sas_id_calibrator2']} for observation {field['target_name']} {field['sas_id_target']}" + ) + ms_folder = f"L{field['sas_id_calibrator2']}" + set_status_processing( + field["target_name"], "calibrator2", field["sas_id_target"] + ) + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + cmd = f"flocs-run linc calibrator --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} {os.path.join(DATA_DIR, field['target_name'], 'calibrator', ms_folder)}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with open( + f"log_LINC_calibrator_{field['target_name']}_{field['sas_id_calibrator2']}.txt", + "w+", + ) as f_out, open( + f"log_LINC_calibrator_{field['target_name']}_{field['sas_id_calibrator2']}_err.txt", + "w+", + ) as f_err: + proc = subprocess.run( + cmd, shell=True, text=True, stdout=f_out, stderr=f_err + ) + success = False + pattern = re.compile(r"Workflow.* stopped. Success: True") + if not proc.returncode: + f_err.seek(0) + if pattern.search(f_err.read()): + success = True + if success: + set_status_finished( + field["target_name"], "calibrator2", field["sas_id_target"] + ) + else: + raise RuntimeError + return field @task(trigger_rule=TriggerRule.ONE_DONE) def select_best_calibrator(result1, result2): - if result1 and result2: + if result1["sas_id_calibrator_final"]: + return result1 + elif result2["sas_id_calibrator_final"]: + return result2 + elif result1 and result2: print("Selecting between cal1 and cal2") + # Need actual selection logic here set_final_calibrator( result1["target_name"], result1["sas_id_target"], @@ -311,9 +362,9 @@ def select_best_calibrator(result1, result2): elif (not result1) and result2: print("Only cal 2 succeeded, continuing with that") set_final_calibrator( - result1["target_name"], - result1["sas_id_target"], - result1["sas_id_calibrator2"], + result2["target_name"], + result2["sas_id_target"], + result2["sas_id_calibrator2"], ) return result2 else: @@ -346,16 +397,21 @@ def run_linc_target(field): print(cmd) with open( f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}.txt", - "w", + "w+", ) as f_out, open( f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}_err.txt", - "w", + "w+", ) as f_err: proc = subprocess.run( cmd, shell=True, text=True, stdout=f_out, stderr=f_err ) + success = False + pattern = re.compile(r"Workflow.* stopped. Success: True") if not proc.returncode: - return True + f_err.seek(0) + if pattern.search(f_err.read()): + success = True + if success: set_status_finished( field["target_name"], "target", field["sas_id_target"] ) From 3e181cabea0ef129b53f3cf8acbafa4c574a9a0f Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 24 Jun 2026 14:54:03 +0100 Subject: [PATCH 027/120] Fix adding a field --- flocs_processing/flocs_processing.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flocs_processing/flocs_processing.py b/flocs_processing/flocs_processing.py index 517d3f2..a603739 100644 --- a/flocs_processing/flocs_processing.py +++ b/flocs_processing/flocs_processing.py @@ -155,10 +155,10 @@ def add_field( dbstr += ", sas_id_calibrator1, sas_id_calibrator2" dbstr += f", sas_id_target) values ('{field_name}', " if len(sas_id_calibrators) == 1: - dbstr += f"{sas_id_calibrators[0]}" + dbstr += f"'{sas_id_calibrators[0]}'" if len(sas_id_calibrators) == 2: - dbstr += f"{sas_id_calibrators[0]}, {sas_id_calibrators[1]}, " - dbstr += f"{sas_id_target})" + dbstr += f"'{sas_id_calibrators[0]}', '{sas_id_calibrators[1]}', " + dbstr += f"'{sas_id_target}')" cmd = ["sqlite3", dbname, dbstr] print(f"Adding field {field_name} to {table_name} via: {" ".join(cmd)}") From f99d1928c5f0fa628a15f8d72aab9cab1d1af7b9 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 24 Jun 2026 15:38:38 +0100 Subject: [PATCH 028/120] Draft widefield dag --- ...ingle_target.py => pilot_single_target.py} | 48 +- flocs_processing/dags/pilot_widefield.py | 604 ++++++++++++++++++ 2 files changed, 633 insertions(+), 19 deletions(-) rename flocs_processing/dags/{vlbi_single_target.py => pilot_single_target.py} (95%) create mode 100644 flocs_processing/dags/pilot_widefield.py diff --git a/flocs_processing/dags/vlbi_single_target.py b/flocs_processing/dags/pilot_single_target.py similarity index 95% rename from flocs_processing/dags/vlbi_single_target.py rename to flocs_processing/dags/pilot_single_target.py index 827c06d..a3e045f 100644 --- a/flocs_processing/dags/vlbi_single_target.py +++ b/flocs_processing/dags/pilot_single_target.py @@ -15,14 +15,14 @@ from stager_access import get_surls_requested, get_surls_online # Need to replace this with a config file -TABLE_NAME = "processing_banados" -DATABASE = "/project/lofarvlbi/Data/fsweijen/banados-high-z/banados_airflow.sqlite" -SLURM_ACCOUNT = "lofarvlbi" -SLURM_QUEUE = "normal" -DATA_DIR = "/project/lofarvlbi/Data/fsweijen/banados-high-z" -OUTPUT_DIR = "/project/lofarvlbi/Data/fsweijen/banados-high-z" -PROCESSING_DIR = "/project/lofarvlbi/Data/fsweijen/banados-high-z/processing" -NN_MODEL_CACHE = "/project/lofarvlbi/Software/fsweijen/nn_cache" +TABLE_NAME = "" +DATABASE = "" +SLURM_ACCOUNT = "" +SLURM_QUEUE = "" +DATA_DIR = "" +OUTPUT_DIR = "" +PROCESSING_DIR = "" +NN_MODEL_CACHE = "" @functools.total_ordering @@ -115,7 +115,7 @@ def get_most_recent_run(searchpath: str, sas_id: str, pipeline: str) -> pathlib. @dag(max_active_runs=1) -def single_target_vlbi(): +def pilot_single_target(): @task def get_unprocessed_target(): field = dict(get_db_columns()[0]) @@ -445,7 +445,12 @@ def run_vlbi_delay(field): delay_cat = os.path.join(outdir, "delay_calibrators.csv") image_cat = os.path.join(outdir, "image_catalogue.csv") - proc = subprocess.run("detect_bad_slurm_nodes.sh", shell=True, text=True, stdout=subprocess.PIPE) + proc = subprocess.run( + "detect_bad_slurm_nodes.sh", + shell=True, + text=True, + stdout=subprocess.PIPE, + ) bad_nodes = proc.stdout.strip() if bad_nodes: print(f"Excluding the following bad nodes from scheduling: {bad_nodes}") @@ -457,15 +462,21 @@ def run_vlbi_delay(field): else: # Extract the previous working directory flocs_workdir = "" - print(f"Scanning log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt for workdir.") - with open(f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt") as f_out: + print( + f"Scanning log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt for workdir." + ) + with open( + f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt" + ) as f_out: for line in f_out.readlines(): print(line) if "Running workflow with" in line: flocs_workdir = line.split(" ")[-1].strip() break if not flocs_workdir: - raise RuntimeError("Could not retrieve PILOT workdir. Flocs probably crashed before launching.") + raise RuntimeError( + "Could not retrieve PILOT workdir. Flocs probably crashed before launching." + ) print(f"Resuming failed PILOT run in {flocs_workdir}") cmd = f"flocs-run vlbi delay-calibration --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --ms-suffix dp3concat --delay-calibrator {delay_cat} --image-catalogue {image_cat} {target_ms_path}" if not os.path.isdir(outdir): @@ -496,6 +507,10 @@ def run_vlbi_delay(field): raise RuntimeError return field + @task + def run_ddf_subtract(field): + return field + @task def run_vlbi_ddcal(field): if (field["status_vlbi_dd"] == PIPELINE_STATUS.finished) or ( @@ -556,10 +571,6 @@ def run_vlbi_ddcal(field): raise RuntimeError return field - @task - def run_ddf_pipeline(field): - return True - proceed = check_fields() get_field = get_unprocessed_target() field = download_field(get_field) @@ -570,9 +581,8 @@ def run_ddf_pipeline(field): linc_is_valid = validate_linc_target(result_targ) result_vlbi_delay = run_vlbi_delay(linc_is_valid) result_vlbi_dd = run_vlbi_ddcal(result_vlbi_delay) - # run_ddf_pipeline(vlbi_delay_is_valid) proceed >> get_field -single_target_vlbi() +pilot_single_target() diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py new file mode 100644 index 0000000..c735f5b --- /dev/null +++ b/flocs_processing/dags/pilot_widefield.py @@ -0,0 +1,604 @@ +from enum import Enum +import datetime +import functools +import os +import pathlib +import re +import sqlite3 +import subprocess +import time + +from airflow.exceptions import AirflowFailException, AirflowSkipException +from airflow.sdk import dag, get_current_context, task +from airflow.task.trigger_rule import TriggerRule +from flocs_lta.lta_search import ObservationStager +from stager_access import get_surls_requested, get_surls_online + +# Need to replace this with a config file +TABLE_NAME = "" +DATABASE = "" +SLURM_ACCOUNT = "" +SLURM_QUEUE = "" +DATA_DIR = "" +OUTPUT_DIR = "" +PROCESSING_DIR = "" +NN_MODEL_CACHE = "" + + +@functools.total_ordering +class PIPELINE_STATUS(Enum): + nothing = 0 + downloaded = 1 + finished = 2 + running = 3 + processing = 98 + error = 99 + + def __eq__(self, other): + if other.__class__ is int: + return self.value == other + elif other.__class__ is self.__class__: + return self.value == other.value + else: + raise NotImplementedError + + def __lt__(self, other): + if self.__class__ is not other.__class__: + raise NotImplementedError + return self.value < other.value + + +def get_db_columns(): + with sqlite3.connect(DATABASE) as db: + db.row_factory = sqlite3.Row + cursor = db.cursor() + columns = "target_name,priority,finished,downloaded,sas_id_calibrator1,sas_id_calibrator2,sas_id_calibrator_final,sas_id_target,status_calibrator1,status_calibrator2,status_target,status_vlbi_delay,status_vlbi_dd" + field = cursor.execute( + f"select {columns} from {TABLE_NAME} where finished==0 order by priority desc" + ).fetchall() + print(field) + return field + + +def set_status_processing(name, identifier, target): + with sqlite3.connect(DATABASE) as db: + cursor = db.cursor() + cursor.execute( + f"update {TABLE_NAME} set status_{identifier}={PIPELINE_STATUS.processing.value} where target_name=='{name}' and sas_id_target=='{target}'" + ) + + +def set_status_finished(name, identifier, target): + with sqlite3.connect(DATABASE) as db: + cursor = db.cursor() + cursor.execute( + f"update {TABLE_NAME} set status_{identifier}={PIPELINE_STATUS.finished.value} where target_name=='{name}' and sas_id_target=='{target}'" + ) + + +def set_status_downloaded(name, target): + with sqlite3.connect(DATABASE) as db: + cursor = db.cursor() + cursor.execute( + f"update {TABLE_NAME} set downloaded=1 where target_name=='{name}' and sas_id_target=='{target}'" + ) + + +def set_field_finished(name, target): + query = f"update {TABLE_NAME} set finished=1 where target_name=='{name}' and sas_id_target=='{target}'" + with sqlite3.connect(DATABASE) as db: + cursor = db.cursor() + cursor.execute(query) + + +def set_final_calibrator(name, target, final_cal): + with sqlite3.connect(DATABASE) as db: + cursor = db.cursor() + cursor.execute( + f"update {TABLE_NAME} set sas_id_calibrator_final={final_cal} where target_name=='{name}' and sas_id_target=='{target}'" + ) + + +def get_most_recent_run(searchpath: str, sas_id: str, pipeline: str) -> pathlib.Path: + rundirs = pathlib.Path(searchpath) + rundirs_sorted = sorted(rundirs.iterdir()) + if pipeline: + rundirs_sorted_filtered = [ + d + for d in rundirs_sorted + if ((sas_id in d.parts[-1]) and (pipeline in d.parts[-1])) and d.is_dir() + ] + else: + rundirs_sorted_filtered = [d for d in rundirs_sorted if sas_id in d.parts[-1]] + rundir_final = rundirs_sorted_filtered[-1].absolute() + return rundir_final + + +@dag(max_active_runs=1) +def pilot_widefield(): + @task + def get_unprocessed_target(): + field = dict(get_db_columns()[0]) + print(field["target_name"]) + return field + + @task.short_circuit + def check_fields(): + fields = get_db_columns() + return bool(fields) + + @task + def download_field(field): + if field["downloaded"]: + return field + else: + has_cal1 = False + stage_calibrators = False + if field["sas_id_calibrator1"]: + ms_folder = f"L{field['sas_id_calibrator1']}" + cal1_full_path = os.path.join( + DATA_DIR, field["target_name"], "calibrator", ms_folder + ) + if os.path.exists(cal1_full_path): + has_cal1 = True + else: + stage_calibrators = True + + has_cal2 = False + if field["sas_id_calibrator2"]: + ms_folder = f"L{field['sas_id_calibrator2']}" + cal2_full_path = os.path.join( + DATA_DIR, field["target_name"], "calibrator", ms_folder + ) + if os.path.exists(cal2_full_path): + has_cal2 = True + else: + stage_calibrators = True + if field["sas_id_target"]: + ms_folder = f"L{field['sas_id_target']}" + target_full_path = os.path.join( + DATA_DIR, field["target_name"], "target", ms_folder + ) + if os.path.exists(target_full_path): + stage_target = False + else: + stage_target = True + else: + raise AirflowFailException( + f"No target SAS ID in database for field {field['target_name']}" + ) + + if stage_calibrators or stage_target: + print(f"Field {field['sas_id_target']} is not downloaded.") + stager = ObservationStager(get_surls=True) + stager.find_observation_by_sasid( + "ALL", + field["sas_id_target"], + None, + 120e6, + 168e6, + ) + if stage_calibrators: + stager.find_nearest_calibrators(2, 120e6, 168e6) + stage_id_calibrators = stager.stage_calibrators() + if stage_target: + stage_id_target = stager.stage_target() + else: + return field + + calibrator_staged = False + target_staged = False + calibrator_downloaded = has_cal1 or has_cal2 + target_downloaded = not stage_target + while True: + if len(get_surls_online(stage_id_calibrators)) == len( + get_surls_requested(stage_id_calibrators) + ): + calibrator_staged = True + if calibrator_staged and not calibrator_downloaded: + dl_path = os.path.join(DATA_DIR, field["target_name"], "calibrator") + cmd = ( + f"flocs-lta download --outdir {dl_path} {stage_id_calibrators}" + ) + with open( + f"log_download_calibrators_{field['target_name']}.txt", + "w+", + ) as f_out, open( + f"log_download_calibrators_{field['target_name']}.txt", + "w+", + ) as f_err: + proc = subprocess.run( + cmd, shell=True, text=True, stdout=f_out, stderr=f_err + ) + if not proc.returncode: + calibrator_downloaded = True + else: + raise RuntimeError + + if len(get_surls_online(stage_id_target)) == len( + get_surls_requested(stage_id_target) + ): + calibrator_staged = True + if target_staged and not target_downloaded: + dl_path = os.path.join(DATA_DIR, field["target_name"], "target") + cmd = f"flocs-lta download --outdir {dl_path} {stage_id_target}" + with open( + f"log_download_calibrators_{field['target_name']}.txt", + "w+", + ) as f_out, open( + f"log_download_calibrators_{field['target_name']}.txt", + "w+", + ) as f_err: + proc = subprocess.run( + cmd, shell=True, text=True, stdout=f_out, stderr=f_err + ) + if not proc.returncode: + set_status_downloaded( + field["target_name"], + field["sas_id_target"], + ) + target_downloaded = True + else: + raise RuntimeError + if calibrator_downloaded and target_downloaded: + break + time.sleep(60) + + @task + def run_linc_calibrator1(field): + if (field["status_calibrator1"] == PIPELINE_STATUS.finished) or ( + field["status_calibrator1"] == PIPELINE_STATUS.running + ): + print( + f"Flux density calibrator {field['sas_id_calibrator1']} for observation {field['target_name']} {field['sas_id_target']} already processed." + ) + return field + else: + print( + f"Processing flux density calibrator {field['sas_id_calibrator1']} for observation {field['target_name']} {field['sas_id_target']}" + ) + ms_folder = f"L{field['sas_id_calibrator1']}" + set_status_processing( + field["target_name"], "calibrator1", field["sas_id_target"] + ) + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + cmd = f"flocs-run linc calibrator --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} {os.path.join(DATA_DIR, field['target_name'], 'calibrator', ms_folder)}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with open( + f"log_LINC_calibrator_{field['target_name']}_{field['sas_id_calibrator1']}.txt", + "w+", + ) as f_out, open( + f"log_LINC_calibrator_{field['target_name']}_{field['sas_id_calibrator1']}_err.txt", + "w+", + ) as f_err: + proc = subprocess.run( + cmd, shell=True, text=True, stdout=f_out, stderr=f_err + ) + success = False + pattern = re.compile(r"Workflow.* stopped. Success: True") + if not proc.returncode: + f_err.seek(0) + if pattern.search(f_err.read()): + success = True + if success: + set_status_finished( + field["target_name"], "calibrator1", field["sas_id_target"] + ) + else: + raise RuntimeError + return field + + @task + def run_linc_calibrator2(field): + if (field["status_calibrator2"] == PIPELINE_STATUS.finished) or ( + field["status_calibrator2"] == PIPELINE_STATUS.running + ): + print( + f"Flux density calibrator {field['sas_id_calibrator2']} for observation {field['target_name']} {field['sas_id_target']} already processed." + ) + return field + else: + print( + f"Processing flux density calibrator {field['sas_id_calibrator2']} for observation {field['target_name']} {field['sas_id_target']}" + ) + ms_folder = f"L{field['sas_id_calibrator2']}" + set_status_processing( + field["target_name"], "calibrator2", field["sas_id_target"] + ) + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + cmd = f"flocs-run linc calibrator --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} {os.path.join(DATA_DIR, field['target_name'], 'calibrator', ms_folder)}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with open( + f"log_LINC_calibrator_{field['target_name']}_{field['sas_id_calibrator2']}.txt", + "w+", + ) as f_out, open( + f"log_LINC_calibrator_{field['target_name']}_{field['sas_id_calibrator2']}_err.txt", + "w+", + ) as f_err: + proc = subprocess.run( + cmd, shell=True, text=True, stdout=f_out, stderr=f_err + ) + success = False + pattern = re.compile(r"Workflow.* stopped. Success: True") + if not proc.returncode: + f_err.seek(0) + if pattern.search(f_err.read()): + success = True + if success: + set_status_finished( + field["target_name"], "calibrator2", field["sas_id_target"] + ) + else: + raise RuntimeError + return field + + @task(trigger_rule=TriggerRule.ONE_DONE) + def select_best_calibrator(result1, result2): + if result1["sas_id_calibrator_final"]: + return result1 + elif result2["sas_id_calibrator_final"]: + return result2 + elif result1 and result2: + print("Selecting between cal1 and cal2") + # Need actual selection logic here + set_final_calibrator( + result1["target_name"], + result1["sas_id_target"], + result1["sas_id_calibrator1"], + ) + return result1 + elif result1 and (not result2): + print("Only cal 1 succeeded, continuing with that") + set_final_calibrator( + result1["target_name"], + result1["sas_id_target"], + result1["sas_id_calibrator1"], + ) + return result1 + elif (not result1) and result2: + print("Only cal 2 succeeded, continuing with that") + set_final_calibrator( + result2["target_name"], + result2["sas_id_target"], + result2["sas_id_calibrator2"], + ) + return result2 + else: + raise AirflowFailException("No calibrators succeeded; stopping processing.") + + @task + def run_linc_target(field): + if (field["status_target"] == PIPELINE_STATUS.finished) or ( + field["status_target"] == PIPELINE_STATUS.running + ): + return field + else: + print( + f"Processing target observation {field['target_name']} {field['sas_id_target']} with calibrator {field['sas_id_calibrator_final']}" + ) + ms_folder = f"L{field['sas_id_target']}" + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + calibrator_path = get_most_recent_run( + outdir, field["sas_id_calibrator_final"], "LINC_calibrator" + ) + calibrator_solutions = ( + calibrator_path / "results_LINC_calibrator" / "cal_solutions.h5" + ) + set_status_processing( + field["target_name"], "target", field["sas_id_target"] + ) + cmd = f"flocs-run linc target --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --cal-solutions {calibrator_solutions} {os.path.join(DATA_DIR, field['target_name'], 'target', ms_folder)}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with open( + f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}.txt", + "w+", + ) as f_out, open( + f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w+", + ) as f_err: + proc = subprocess.run( + cmd, shell=True, text=True, stdout=f_out, stderr=f_err + ) + success = False + pattern = re.compile(r"Workflow.* stopped. Success: True") + if not proc.returncode: + f_err.seek(0) + if pattern.search(f_err.read()): + success = True + if success: + set_status_finished( + field["target_name"], "target", field["sas_id_target"] + ) + else: + raise RuntimeError + return field + + @task + def validate_linc_target(field): + return field + + @task(retries=0, retry_delay=datetime.timedelta(seconds=5)) + def run_vlbi_delay(field): + if (field["status_vlbi_delay"] == PIPELINE_STATUS.finished) or ( + field["status_vlbi_delay"] == PIPELINE_STATUS.running + ): + return field + else: + print( + f"Processing delay calibration for {field['target_name']} {field['sas_id_target']}" + ) + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + target_path = get_most_recent_run( + outdir, field["sas_id_target"], "LINC_target" + ) + target_ms_path = target_path / "results_LINC_target" / "results" + set_status_processing( + field["target_name"], "vlbi_delay", field["sas_id_target"] + ) + + delay_cat = os.path.join(outdir, "delay_calibrators.csv") + image_cat = os.path.join(outdir, "image_catalogue.csv") + + proc = subprocess.run( + "detect_bad_slurm_nodes.sh", + shell=True, + text=True, + stdout=subprocess.PIPE, + ) + bad_nodes = proc.stdout.strip() + if bad_nodes: + print(f"Excluding the following bad nodes from scheduling: {bad_nodes}") + os.environ["TOIL_SLURM_ARGS"] = f"--exclude={bad_nodes}" + + context = get_current_context() + if context["ti"].try_number == 1: + cmd = f"flocs-run vlbi delay-calibration --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --ms-suffix dp3concat --delay-calibrator {delay_cat} --image-catalogue {image_cat} {target_ms_path}" + else: + # Extract the previous working directory + flocs_workdir = "" + print( + f"Scanning log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt for workdir." + ) + with open( + f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt" + ) as f_out: + for line in f_out.readlines(): + print(line) + if "Running workflow with" in line: + flocs_workdir = line.split(" ")[-1].strip() + break + if not flocs_workdir: + raise RuntimeError( + "Could not retrieve PILOT workdir. Flocs probably crashed before launching." + ) + print(f"Resuming failed PILOT run in {flocs_workdir}") + cmd = f"flocs-run vlbi delay-calibration --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --ms-suffix dp3concat --delay-calibrator {delay_cat} --image-catalogue {image_cat} {target_ms_path}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with open( + f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt", + "w+", + ) as f_out, open( + f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w+", + ) as f_err: + proc = subprocess.run( + cmd, shell=True, text=True, stdout=f_out, stderr=f_err + ) + success = False + pattern = re.compile(r"Workflow.* stopped. Success: True") + if not proc.returncode: + f_err.seek(0) + if pattern.search(f_err.read()): + success = True + + if success: + set_status_finished( + field["target_name"], "vlbi_delay", field["sas_id_target"] + ) + else: + raise RuntimeError + return field + + @task + def run_ddf_pipeline(field): + if (field["status_ddf"] == PIPELINE_STATUS.finished) or ( + field["status_ddf"] == PIPELINE_STATUS.running + ): + return field + else: + print( + f"Starting ddf-pipeline for {field['target_name']} {field['sas_id_target']}" + ) + return field + + + @task + def run_ddf_subtract(field): + return field + + @task + def run_vlbi_ddcal(field): + if (field["status_vlbi_dd"] == PIPELINE_STATUS.finished) or ( + field["status_vlbi_dd"] == PIPELINE_STATUS.running + ): + return field + else: + print( + f"Processing ILT dd calibration for {field['target_name']} {field['sas_id_target']}" + ) + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + target_path = get_most_recent_run( + outdir, field["sas_id_target"], "LINC_target" + ) + target_ms_path = target_path / "results_LINC_target" / "results" + print(f"Using LINC target run: {target_path}") + + sols_path = get_most_recent_run( + outdir, field["sas_id_target"], "VLBI_delay" + ) + sols_path = sols_path / "results_VLBI_delay-calibration" + sols = list(sols_path.glob("merged*selfcalcycle???_linearfulljones*.h5"))[0] + print(f"Using PILOT delay calibration solutions: {sols}") + + source_cat = os.path.join(DATA_DIR, field["target_name"], "vlbi_target.csv") + if not os.path.isfile(source_cat): + raise AirflowFailException(f"{source_cat} not found.") + + set_status_processing( + field["target_name"], "vlbi_dd", field["sas_id_target"] + ) + cmd = f"flocs-run vlbi dd-calibration --runner toil --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --delay-solset {sols} --phasediff-score 10.0 --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat {target_ms_path}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with open( + f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt", + "w+", + ) as f_out, open( + f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w+", + ) as f_err: + proc = subprocess.run( + cmd, shell=True, text=True, stdout=f_out, stderr=f_err + ) + success = False + pattern = re.compile(r"Workflow.* stopped. Success: True") + if not proc.returncode: + f_err.seek(0) + if pattern.search(f_err.read()): + success = True + if success: + set_status_finished( + field["target_name"], "vlbi_dd", field["sas_id_target"] + ) + set_field_finished(field["target_name"], field["sas_id_target"]) + else: + raise RuntimeError + return field + + proceed = check_fields() + get_field = get_unprocessed_target() + field = download_field(get_field) + result_cal1 = run_linc_calibrator1(field) + result_cal2 = run_linc_calibrator2(field) + best_cal = select_best_calibrator(result_cal1, result_cal2) + result_targ = run_linc_target(best_cal) + linc_is_valid = validate_linc_target(result_targ) + result_vlbi_delay = run_vlbi_delay(linc_is_valid) + result_ddf = run_ddf_pipeline(result_vlbi_delay) + result_ddf_subtract = run_ddf_subtract(result_ddf) + result_vlbi_dd = run_vlbi_ddcal(result_vlbi_delay) + + proceed >> get_field + result_ddf >> result_ddf_subtract >> result_vlbi_dd + + +pilot_widefield() From 52a36d462e80bfe393268908c62495c267580ab0 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 24 Jun 2026 16:38:38 +0100 Subject: [PATCH 029/120] Update readme --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index af403d5..cd88973 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,11 @@ A database for processing is created via `flocs-processing create-database`. Thi ## Processing data To start processing data, Airflow needs to be running. This will be delegated to `flocs-processing process-from-database` in the future, but for now requires running Airflow manually. For setup do the following: -1. Set up a folder that wil contain all of Airflow's own stuff and assign it to the `AIRFLOW_HOME` environment variable. -2. Run `airflow config list --defaults > "${AIRFLOW_HOME}/airflow.cfg"` -3. Define `AIRFLOW__CORE__DAGS_FOLDER` as `${AIRFLOW_HOME}/dags` and create the folder. Copy the DAGs inside `flocs_processing/dags` to this folder. -4. Define `AIRFLOW__CORE__LOAD_EXAMPLES` as `False` +1. Install airflow: `uv pip install apache-airflow` +2. Set up a folder that wil contain all of Airflow's own stuff and assign it to the `AIRFLOW_HOME` environment variable. +3. Run `airflow config list --defaults > "${AIRFLOW_HOME}/airflow.cfg"` +4. Define `AIRFLOW__CORE__DAGS_FOLDER` as `${AIRFLOW_HOME}/dags` and create the folder. Copy the DAGs inside `flocs_processing/dags` to this folder. +5. Define `AIRFLOW__CORE__LOAD_EXAMPLES` as `False` Next start a screen or tmux session and run `airflow standalone` to start the Airflow instance. It will echo a user name and password on first start, but will also store the credentials in `${AIRFLOW_HOME}/simple_auth_manager_passwords.json.generated`. The Airflow instance will start on port 8080. You can access it via `localhost:8080` in your browser. If it is running on a remote cluster, you can set up a tunnel via e.g. `ssh -N -L 8080:localhost:8080 ` to forward it to your local machine. From 354b1544ef4330b2a8770007695a79e9cfc2e683 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 24 Jun 2026 16:43:18 +0100 Subject: [PATCH 030/120] Expand readme with expected folder setup --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index cd88973..7594ded 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,15 @@ This package aims to provide relatively simple end-to-end automatic processing of ILT HBA data. Where `flocs-runners` provides the interface to running pipelines, `flocs-processing` is the scaffolding to tie it together. Data reduction is coordinated via a dedicated SQLite database that holds information on which observations to process, which pipelines to run for them and all of the related statuses. Orchestration of all the pipelines is handled via Airflow through a DAG. +## Folder setup +Flocs-processing requires three folders to be setup: + +* A processing folder -- this is where data is stored while processing +* A data folder -- this is where the input data is found +* An output folder -- this is where finished pipeline outputs are copied to, and searched for in steps that depend on it. + +The expected naming directory structure for input data is `//{calibrator,target}`. Inside the calibrator and target folders, the observations should follow the usual `LXXXXXX` naming scheme. These **must** match the SAS IDs in the database for flocs to be able to find them. + ## Database setup A database for processing is created via `flocs-processing create-database`. This will create an empty database with the necessary columns. Datasets to process can be added via `flocs-processing add-field`. @@ -17,3 +26,4 @@ To start processing data, Airflow needs to be running. This will be delegated to Next start a screen or tmux session and run `airflow standalone` to start the Airflow instance. It will echo a user name and password on first start, but will also store the credentials in `${AIRFLOW_HOME}/simple_auth_manager_passwords.json.generated`. The Airflow instance will start on port 8080. You can access it via `localhost:8080` in your browser. If it is running on a remote cluster, you can set up a tunnel via e.g. `ssh -N -L 8080:localhost:8080 ` to forward it to your local machine. Once `flocs-processing` is complete the processing loop will be automatic, but for now the user must trigger the DAG manually. On the "Dags" tab you should now see the flocs DAGs available. To manually trigger one, click on the name and on the subsequent page use the "Trigger" button in the top right. + From e50a28de2c65816dd4f2c7c637baa1e7d418c0c5 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 24 Jun 2026 21:35:26 +0100 Subject: [PATCH 031/120] Allow manual approval of delay solutions for widefield --- flocs_processing/dags/pilot_widefield.py | 236 ++++++++++++++++------- 1 file changed, 171 insertions(+), 65 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index c735f5b..6675ada 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -8,8 +8,9 @@ import subprocess import time -from airflow.exceptions import AirflowFailException, AirflowSkipException +from airflow.exceptions import AirflowFailException from airflow.sdk import dag, get_current_context, task +from airflow.providers.standard.sensors.python import PythonSensor from airflow.task.trigger_rule import TriggerRule from flocs_lta.lta_search import ObservationStager from stager_access import get_surls_requested, get_surls_online @@ -23,6 +24,23 @@ OUTPUT_DIR = "" PROCESSING_DIR = "" NN_MODEL_CACHE = "" +DDF_CONFIG = "" + +NEEDS_MANUAL_APPROVAL_DELAY = True + + +def get_approval(field, identifier, needs_approval): + if not needs_approval: + return True + with sqlite3.connect(DATABASE) as db: + db.row_factory = sqlite3.Row + cursor = db.cursor() + columns = f"sas_id_target,status_{identifier}" + field = cursor.execute( + f"select {columns} from {TABLE_NAME} where sas_id_target=='{field['sas_id_target']}'" + ).fetchall() + status = field[0][f"status_{identifier}"] + return status == PIPELINE_STATUS.finished.value @functools.total_ordering @@ -30,7 +48,7 @@ class PIPELINE_STATUS(Enum): nothing = 0 downloaded = 1 finished = 2 - running = 3 + await_approval = 3 processing = 98 error = 99 @@ -68,6 +86,14 @@ def set_status_processing(name, identifier, target): ) +def set_status_await_approval(name, identifier, target): + with sqlite3.connect(DATABASE) as db: + cursor = db.cursor() + cursor.execute( + f"update {TABLE_NAME} set status_{identifier}={PIPELINE_STATUS.await_approval.value} where target_name=='{name}' and sas_id_target=='{target}'" + ) + + def set_status_finished(name, identifier, target): with sqlite3.connect(DATABASE) as db: cursor = db.cursor() @@ -200,13 +226,16 @@ def download_field(field): cmd = ( f"flocs-lta download --outdir {dl_path} {stage_id_calibrators}" ) - with open( - f"log_download_calibrators_{field['target_name']}.txt", - "w+", - ) as f_out, open( - f"log_download_calibrators_{field['target_name']}.txt", - "w+", - ) as f_err: + with ( + open( + f"log_download_calibrators_{field['target_name']}.txt", + "w+", + ) as f_out, + open( + f"log_download_calibrators_{field['target_name']}.txt", + "w+", + ) as f_err, + ): proc = subprocess.run( cmd, shell=True, text=True, stdout=f_out, stderr=f_err ) @@ -222,13 +251,16 @@ def download_field(field): if target_staged and not target_downloaded: dl_path = os.path.join(DATA_DIR, field["target_name"], "target") cmd = f"flocs-lta download --outdir {dl_path} {stage_id_target}" - with open( - f"log_download_calibrators_{field['target_name']}.txt", - "w+", - ) as f_out, open( - f"log_download_calibrators_{field['target_name']}.txt", - "w+", - ) as f_err: + with ( + open( + f"log_download_calibrators_{field['target_name']}.txt", + "w+", + ) as f_out, + open( + f"log_download_calibrators_{field['target_name']}.txt", + "w+", + ) as f_err, + ): proc = subprocess.run( cmd, shell=True, text=True, stdout=f_out, stderr=f_err ) @@ -247,7 +279,7 @@ def download_field(field): @task def run_linc_calibrator1(field): if (field["status_calibrator1"] == PIPELINE_STATUS.finished) or ( - field["status_calibrator1"] == PIPELINE_STATUS.running + field["status_calibrator1"] == PIPELINE_STATUS.processing ): print( f"Flux density calibrator {field['sas_id_calibrator1']} for observation {field['target_name']} {field['sas_id_target']} already processed." @@ -266,13 +298,16 @@ def run_linc_calibrator1(field): if not os.path.isdir(outdir): os.mkdir(outdir) print(cmd) - with open( - f"log_LINC_calibrator_{field['target_name']}_{field['sas_id_calibrator1']}.txt", - "w+", - ) as f_out, open( - f"log_LINC_calibrator_{field['target_name']}_{field['sas_id_calibrator1']}_err.txt", - "w+", - ) as f_err: + with ( + open( + f"log_LINC_calibrator_{field['target_name']}_{field['sas_id_calibrator1']}.txt", + "w+", + ) as f_out, + open( + f"log_LINC_calibrator_{field['target_name']}_{field['sas_id_calibrator1']}_err.txt", + "w+", + ) as f_err, + ): proc = subprocess.run( cmd, shell=True, text=True, stdout=f_out, stderr=f_err ) @@ -293,7 +328,7 @@ def run_linc_calibrator1(field): @task def run_linc_calibrator2(field): if (field["status_calibrator2"] == PIPELINE_STATUS.finished) or ( - field["status_calibrator2"] == PIPELINE_STATUS.running + field["status_calibrator2"] == PIPELINE_STATUS.processing ): print( f"Flux density calibrator {field['sas_id_calibrator2']} for observation {field['target_name']} {field['sas_id_target']} already processed." @@ -312,13 +347,16 @@ def run_linc_calibrator2(field): if not os.path.isdir(outdir): os.mkdir(outdir) print(cmd) - with open( - f"log_LINC_calibrator_{field['target_name']}_{field['sas_id_calibrator2']}.txt", - "w+", - ) as f_out, open( - f"log_LINC_calibrator_{field['target_name']}_{field['sas_id_calibrator2']}_err.txt", - "w+", - ) as f_err: + with ( + open( + f"log_LINC_calibrator_{field['target_name']}_{field['sas_id_calibrator2']}.txt", + "w+", + ) as f_out, + open( + f"log_LINC_calibrator_{field['target_name']}_{field['sas_id_calibrator2']}_err.txt", + "w+", + ) as f_err, + ): proc = subprocess.run( cmd, shell=True, text=True, stdout=f_out, stderr=f_err ) @@ -373,7 +411,7 @@ def select_best_calibrator(result1, result2): @task def run_linc_target(field): if (field["status_target"] == PIPELINE_STATUS.finished) or ( - field["status_target"] == PIPELINE_STATUS.running + field["status_target"] == PIPELINE_STATUS.processing ): return field else: @@ -395,13 +433,16 @@ def run_linc_target(field): if not os.path.isdir(outdir): os.mkdir(outdir) print(cmd) - with open( - f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}.txt", - "w+", - ) as f_out, open( - f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}_err.txt", - "w+", - ) as f_err: + with ( + open( + f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}.txt", + "w+", + ) as f_out, + open( + f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w+", + ) as f_err, + ): proc = subprocess.run( cmd, shell=True, text=True, stdout=f_out, stderr=f_err ) @@ -426,7 +467,7 @@ def validate_linc_target(field): @task(retries=0, retry_delay=datetime.timedelta(seconds=5)) def run_vlbi_delay(field): if (field["status_vlbi_delay"] == PIPELINE_STATUS.finished) or ( - field["status_vlbi_delay"] == PIPELINE_STATUS.running + field["status_vlbi_delay"] == PIPELINE_STATUS.processing ): return field else: @@ -482,13 +523,16 @@ def run_vlbi_delay(field): if not os.path.isdir(outdir): os.mkdir(outdir) print(cmd) - with open( - f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt", - "w+", - ) as f_out, open( - f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}_err.txt", - "w+", - ) as f_err: + with ( + open( + f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt", + "w+", + ) as f_out, + open( + f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w+", + ) as f_err, + ): proc = subprocess.run( cmd, shell=True, text=True, stdout=f_out, stderr=f_err ) @@ -500,9 +544,14 @@ def run_vlbi_delay(field): success = True if success: - set_status_finished( - field["target_name"], "vlbi_delay", field["sas_id_target"] - ) + if NEEDS_MANUAL_APPROVAL_DELAY: + set_status_await_approval( + field["target_name"], "vlbi_delay", field["sas_id_target"] + ) + else: + set_status_finished( + field["target_name"], "vlbi_delay", field["sas_id_target"] + ) else: raise RuntimeError return field @@ -510,15 +559,59 @@ def run_vlbi_delay(field): @task def run_ddf_pipeline(field): if (field["status_ddf"] == PIPELINE_STATUS.finished) or ( - field["status_ddf"] == PIPELINE_STATUS.running + field["status_ddf"] == PIPELINE_STATUS.processing ): return field else: print( f"Starting ddf-pipeline for {field['target_name']} {field['sas_id_target']}" ) - return field + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + target_path = get_most_recent_run( + outdir, field["sas_id_target"], "LINC_target" + ) + target_ms_path = target_path / "results_LINC_target" / "results" + + cmd = f"flocs-run ddf-pipeline --scheduler slurm --slurm-time 72:00:00 --slurm-cores 32 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {OUTPUT_DIR} --config-file {DDF_CONFIG} {target_ms_path}" + with ( + open( + f"log_DDF-pipeline_{field['target_name']}_{field['sas_id_target']}.txt", + "w+", + ) as f_out, + open( + f"log_DDF-pipeline_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w+", + ) as f_err, + ): + proc = subprocess.run( + cmd, shell=True, text=True, stdout=f_out, stderr=f_err + ) + jobid = None + if not proc.returncode: + f_out.seek(0) + for line in f_out.readlines(): + if "Submitted batch job" in line: + jobid = line.strip().split()[-1] + else: + raise RuntimeError("Failed to submit job.") + if not jobid: + raise RuntimeError("Failed to retrieve job id") + else: + while True: + poll_cmd = f"sacct -X -j {jobid} --format=State --noheader" + status = subprocess.run( + poll_cmd, shell=True, text=True, capture_output=True + ).stdout.strip() + if status == "RUNNING": + time.sleep(60) + elif status == "COMPLETED": + break + elif status == "FAILED": + raise RuntimeError( + f"DDF-pipeline for {field['target_name']} {field['sas_id_target']} failed." + ) + return field @task def run_ddf_subtract(field): @@ -527,7 +620,7 @@ def run_ddf_subtract(field): @task def run_vlbi_ddcal(field): if (field["status_vlbi_dd"] == PIPELINE_STATUS.finished) or ( - field["status_vlbi_dd"] == PIPELINE_STATUS.running + field["status_vlbi_dd"] == PIPELINE_STATUS.processing ): return field else: @@ -559,13 +652,16 @@ def run_vlbi_ddcal(field): if not os.path.isdir(outdir): os.mkdir(outdir) print(cmd) - with open( - f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt", - "w+", - ) as f_out, open( - f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}_err.txt", - "w+", - ) as f_err: + with ( + open( + f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt", + "w+", + ) as f_out, + open( + f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w+", + ) as f_err, + ): proc = subprocess.run( cmd, shell=True, text=True, stdout=f_out, stderr=f_err ) @@ -593,12 +689,22 @@ def run_vlbi_ddcal(field): result_targ = run_linc_target(best_cal) linc_is_valid = validate_linc_target(result_targ) result_vlbi_delay = run_vlbi_delay(linc_is_valid) - result_ddf = run_ddf_pipeline(result_vlbi_delay) - result_ddf_subtract = run_ddf_subtract(result_ddf) - result_vlbi_dd = run_vlbi_ddcal(result_vlbi_delay) proceed >> get_field - result_ddf >> result_ddf_subtract >> result_vlbi_dd + await_approval_delay = PythonSensor( + task_id="approve_delay", + python_callable=get_approval, + poke_interval=60, + timeout=86400 * 7, + mode="poke", + op_args=[result_vlbi_delay, "vlbi_delay", NEEDS_MANUAL_APPROVAL_DELAY], + ) + # result_ddf = run_ddf_pipeline(await_approval_delay) + result_ddf = run_ddf_pipeline(result_vlbi_delay) + + result_ddf_subtract = run_ddf_subtract(result_ddf) + result_vlbi_dd = run_vlbi_ddcal(result_ddf_subtract) + await_approval_delay >> result_ddf >> result_ddf_subtract >> result_vlbi_dd pilot_widefield() From 7869c9541d485039b5e38f27ea8334447bd6a0c4 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Thu, 16 Jul 2026 13:14:05 +0100 Subject: [PATCH 032/120] Update widefield DAG with subtract and ddcal --- flocs_processing/dags/pilot_widefield.py | 488 +++++++++++++++++++++-- 1 file changed, 447 insertions(+), 41 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 6675ada..d7d3cd7 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -13,6 +13,7 @@ from airflow.providers.standard.sensors.python import PythonSensor from airflow.task.trigger_rule import TriggerRule from flocs_lta.lta_search import ObservationStager +from losoto.h5parm import h5parm from stager_access import get_surls_requested, get_surls_online # Need to replace this with a config file @@ -31,7 +32,7 @@ def get_approval(field, identifier, needs_approval): if not needs_approval: - return True + return field with sqlite3.connect(DATABASE) as db: db.row_factory = sqlite3.Row cursor = db.cursor() @@ -40,7 +41,8 @@ def get_approval(field, identifier, needs_approval): f"select {columns} from {TABLE_NAME} where sas_id_target=='{field['sas_id_target']}'" ).fetchall() status = field[0][f"status_{identifier}"] - return status == PIPELINE_STATUS.finished.value + if status == PIPELINE_STATUS.finished.value: + return field @functools.total_ordering @@ -66,18 +68,31 @@ def __lt__(self, other): return self.value < other.value -def get_db_columns(): +def get_db_columns(obsid: str = None): with sqlite3.connect(DATABASE) as db: db.row_factory = sqlite3.Row cursor = db.cursor() - columns = "target_name,priority,finished,downloaded,sas_id_calibrator1,sas_id_calibrator2,sas_id_calibrator_final,sas_id_target,status_calibrator1,status_calibrator2,status_target,status_vlbi_delay,status_vlbi_dd" - field = cursor.execute( - f"select {columns} from {TABLE_NAME} where finished==0 order by priority desc" - ).fetchall() + columns = "target_name,priority,finished,downloaded,sas_id_calibrator1,sas_id_calibrator2,sas_id_calibrator_final,sas_id_target,status_calibrator1,status_calibrator2,status_target,status_vlbi_delay,status_vlbi_dd,status_ddf,status_ddf_subtract" + if obsid: + field = cursor.execute( + f"select {columns} from {TABLE_NAME} where sas_id_target=='{obsid}' and finished==0 order by priority desc" + ).fetchall() + else: + field = cursor.execute( + f"select {columns} from {TABLE_NAME} where finished==0 order by priority desc" + ).fetchall() print(field) return field +def set_status_failed(name, identifier, target): + with sqlite3.connect(DATABASE) as db: + cursor = db.cursor() + cursor.execute( + f"update {TABLE_NAME} set status_{identifier}={PIPELINE_STATUS.error.value} where target_name=='{name}' and sas_id_target=='{target}'" + ) + + def set_status_processing(name, identifier, target): with sqlite3.connect(DATABASE) as db: cursor = db.cursor() @@ -466,8 +481,10 @@ def validate_linc_target(field): @task(retries=0, retry_delay=datetime.timedelta(seconds=5)) def run_vlbi_delay(field): - if (field["status_vlbi_delay"] == PIPELINE_STATUS.finished) or ( - field["status_vlbi_delay"] == PIPELINE_STATUS.processing + if ( + (field["status_vlbi_delay"] == PIPELINE_STATUS.finished) + or (field["status_vlbi_delay"] == PIPELINE_STATUS.processing) + or (field["status_vlbi_delay"] == PIPELINE_STATUS.await_approval) ): return field else: @@ -499,7 +516,7 @@ def run_vlbi_delay(field): context = get_current_context() if context["ti"].try_number == 1: - cmd = f"flocs-run vlbi delay-calibration --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --ms-suffix dp3concat --delay-calibrator {delay_cat} --image-catalogue {image_cat} {target_ms_path}" + cmd = f"flocs-run vlbi delay-calibration --record-toil-stats --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --ms-suffix dp3concat --delay-calibrator {delay_cat} --image-catalogue {image_cat} --apply-delay-solutions {target_ms_path}" else: # Extract the previous working directory flocs_workdir = "" @@ -558,9 +575,43 @@ def run_vlbi_delay(field): @task def run_ddf_pipeline(field): - if (field["status_ddf"] == PIPELINE_STATUS.finished) or ( - field["status_ddf"] == PIPELINE_STATUS.processing - ): + field = dict(get_db_columns(field["sas_id_target"])[0]) + if field["status_ddf"] == PIPELINE_STATUS.processing: + print( + f"ddf-pipeline for {field['target_name']} {field['sas_id_target']} should be running, attempting to resume polling..." + ) + with ( + open( + f"log_DDF-pipeline_{field['target_name']}_{field['sas_id_target']}.txt", + "r", + ) as f_out, + open( + f"log_DDF-pipeline_{field['target_name']}_{field['sas_id_target']}_err.txt", + "r", + ) as f_err, + ): + jobid = None + for line in f_out.readlines(): + if "Submitted batch job" in line: + jobid = line.strip().split()[-1] + else: + raise AirflowFailException("Failed to recover job id from log.") + + while True: + print(f"Polling DDF-pipeine job {jobid}") + poll_cmd = f"sacct -X -j {jobid} --format=State --noheader" + status = subprocess.run( + poll_cmd, shell=True, text=True, capture_output=True + ).stdout.strip() + if (status == "RUNNING") or (status == "PENDING"): + time.sleep(60) + elif status == "COMPLETED": + return field + elif (status == "FAILED") or ("TIMEOUT" in status): + raise RuntimeError( + f"DDF-pipeline for {field['target_name']} {field['sas_id_target']} failed." + ) + if field["status_ddf"] == PIPELINE_STATUS.finished: return field else: print( @@ -568,11 +619,18 @@ def run_ddf_pipeline(field): ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) target_path = get_most_recent_run( - outdir, field["sas_id_target"], "LINC_target" + outdir, field["sas_id_target"], "VLBI_delay-calibration" ) - target_ms_path = target_path / "results_LINC_target" / "results" + target_ms_path = target_path / "results_VLBI_delay-calibration" + if not list(target_ms_path.glob("*pre-cal.ms")): + target_path = get_most_recent_run( + outdir, field["sas_id_target"], "LINC_target" + ) + target_ms_path = target_path / "results_LINC_target" / "results" cmd = f"flocs-run ddf-pipeline --scheduler slurm --slurm-time 72:00:00 --slurm-cores 32 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {OUTPUT_DIR} --config-file {DDF_CONFIG} {target_ms_path}" + print(cmd) + set_status_processing(field["target_name"], "ddf", field["sas_id_target"]) with ( open( f"log_DDF-pipeline_{field['target_name']}_{field['sas_id_target']}.txt", @@ -599,15 +657,20 @@ def run_ddf_pipeline(field): raise RuntimeError("Failed to retrieve job id") else: while True: + print(f"Polling DDF-pipeine job {jobid}") poll_cmd = f"sacct -X -j {jobid} --format=State --noheader" status = subprocess.run( poll_cmd, shell=True, text=True, capture_output=True ).stdout.strip() - if status == "RUNNING": + if (status == "RUNNING") or (status == "PENDING"): time.sleep(60) elif status == "COMPLETED": break - elif status == "FAILED": + elif ( + (status == "FAILED") + or ("TIMEOUT" in status) + or ("CANCELLED" in status) + ): raise RuntimeError( f"DDF-pipeline for {field['target_name']} {field['sas_id_target']} failed." ) @@ -615,10 +678,86 @@ def run_ddf_pipeline(field): @task def run_ddf_subtract(field): - return field + if (field["status_ddf_subtract"] == PIPELINE_STATUS.finished) or ( + field["status_ddf_subtract"] == PIPELINE_STATUS.processing + ): + return field + else: + print( + f"Running ddf subtract for {field['target_name']} {field['sas_id_target']}" + ) + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + target_path = get_most_recent_run( + outdir, field["sas_id_target"], "VLBI_delay-calibration" + ) + target_ms_path = target_path / "results_VLBI_delay-calibration" + print(f"Using data at: {target_path}/*.dp3concat") + + ddf_path = get_most_recent_run( + outdir, field["sas_id_target"], "DDF-pipeline" + ) + ddf_sols_path = ddf_path / "SOLSDIR" + print(f"Using DDF run at: {ddf_path}") + + context = get_current_context() + if context["ti"].try_number == 1: + set_status_processing( + field["target_name"], "ddf_subtract", field["sas_id_target"] + ) + cmd = f"flocs-run vlbi process-ddf --runner toil --record-toil-stats --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --ms-suffix .dp3concat --ddf-rundir {ddf_path} --solsdir {ddf_sols_path} --do-subtraction {target_ms_path}" + else: + # Extract the previous working directory + flocs_workdir = "" + print( + f"Scanning log_VLBI_process-ddf_{field['target_name']}_{field['sas_id_target']}.txt for workdir." + ) + with open( + f"log_VLBI_process-ddf_{field['target_name']}_{field['sas_id_target']}.txt" + ) as f_out: + for line in f_out.readlines(): + print(line) + if "Running workflow with" in line: + flocs_workdir = line.split(" ")[-1].strip() + break + if not flocs_workdir: + raise RuntimeError( + "Could not retrieve PILOT workdir. Flocs probably crashed before launching." + ) + print(f"Resuming failed PILOT run in {flocs_workdir}") + cmd = f"flocs-run vlbi process-ddf --runner toil --record-toil-stats --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --ms-suffix .dp3concat --ddf-rundir {ddf_path} --solsdir {ddf_sols_path} --do-subtraction {target_ms_path}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with ( + open( + f"log_VLBI_process-ddf_{field['target_name']}_{field['sas_id_target']}.txt", + "w+", + ) as f_out, + open( + f"log_VLBI_process-ddf_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w+", + ) as f_err, + ): + proc = subprocess.run( + cmd, shell=True, text=True, stdout=f_out, stderr=f_err + ) + success = False + pattern = re.compile(r"Workflow.* stopped. Success: True") + if not proc.returncode: + f_err.seek(0) + if pattern.search(f_err.read()): + success = True + if success: + set_status_finished( + field["target_name"], "ddf_subtract", field["sas_id_target"] + ) + else: + raise RuntimeError + return field @task def run_vlbi_ddcal(field): + field = dict(get_db_columns(field["sas_id_target"])[0]) if (field["status_vlbi_dd"] == PIPELINE_STATUS.finished) or ( field["status_vlbi_dd"] == PIPELINE_STATUS.processing ): @@ -628,27 +767,78 @@ def run_vlbi_ddcal(field): f"Processing ILT dd calibration for {field['target_name']} {field['sas_id_target']}" ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - target_path = get_most_recent_run( - outdir, field["sas_id_target"], "LINC_target" - ) - target_ms_path = target_path / "results_LINC_target" / "results" - print(f"Using LINC target run: {target_path}") + if "status_ddf" not in field: + print("Not a widefield imaging run, checking LINC + delay calibration.") + target_path = get_most_recent_run( + outdir, field["sas_id_target"], "LINC_target" + ) + target_ms_path = target_path / "results_LINC_target" / "results" + print(f"Using LINC target run: {target_path}") - sols_path = get_most_recent_run( - outdir, field["sas_id_target"], "VLBI_delay" - ) - sols_path = sols_path / "results_VLBI_delay-calibration" - sols = list(sols_path.glob("merged*selfcalcycle???_linearfulljones*.h5"))[0] - print(f"Using PILOT delay calibration solutions: {sols}") + sols_path = get_most_recent_run( + outdir, field["sas_id_target"], "VLBI_delay" + ) + sols_path = sols_path / "results_VLBI_delay-calibration" + sols = list( + sols_path.glob("merged*selfcalcycle???_linearfulljones*.h5") + )[0] + print(f"Using PILOT delay calibration solutions: {sols}") - source_cat = os.path.join(DATA_DIR, field["target_name"], "vlbi_target.csv") - if not os.path.isfile(source_cat): - raise AirflowFailException(f"{source_cat} not found.") + source_cat = os.path.join( + DATA_DIR, field["target_name"], "vlbi_target.csv" + ) + if not os.path.isfile(source_cat): + raise AirflowFailException(f"{source_cat} not found.") - set_status_processing( - field["target_name"], "vlbi_dd", field["sas_id_target"] - ) - cmd = f"flocs-run vlbi dd-calibration --runner toil --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --delay-solset {sols} --phasediff-score 10.0 --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat {target_ms_path}" + set_status_processing( + field["target_name"], "vlbi_dd", field["sas_id_target"] + ) + cmd = f"flocs-run vlbi dd-calibration --runner toil --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --delay-solset {sols} --phasediff-score 10.0 --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat {target_ms_path}" + else: + print("Widefield imaging run, checking subtraction output.") + target_path = get_most_recent_run( + outdir, field["sas_id_target"], "VLBI_process-ddf" + ) + target_ms_path = target_path / "results_VLBI_process-ddf" + print(f"Using subtracted data at: {target_path}") + + source_cat = os.path.join( + DATA_DIR, field["target_name"], "image_catalogue.csv" + ) + if not os.path.isfile(source_cat): + raise AirflowFailException(f"{source_cat} not found.") + + set_status_processing( + field["target_name"], "vlbi_dd", field["sas_id_target"] + ) + + context = get_current_context() + if context["ti"].try_number == 1: + cmd = f"flocs-run vlbi dd-calibration --record-toil-stats --runner toil --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat.sub.ms {target_ms_path}" + else: + if field["status_vlbi_dd"] == PIPELINE_STATUS.downloaded: + # This way we can force a clean restart in the database. + cmd = f"flocs-run vlbi dd-calibration --record-toil-stats --runner toil --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat.sub.ms {target_ms_path}" + else: + # Extract the previous working directory + flocs_workdir = "" + print( + f"Scanning log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt for workdir." + ) + with open( + f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt" + ) as f_out: + for line in f_out.readlines(): + print(line) + if "Running workflow with" in line: + flocs_workdir = line.split(" ")[-1].strip() + break + if not flocs_workdir: + raise RuntimeError( + "Could not retrieve PILOT workdir. Flocs probably crashed before launching." + ) + print(f"Resuming failed PILOT run in {flocs_workdir}") + cmd = f"flocs-run vlbi dd-calibration --record-toil-stats --runner toil --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat.sub.ms {target_ms_path}" if not os.path.isdir(outdir): os.mkdir(outdir) print(cmd) @@ -677,9 +867,217 @@ def run_vlbi_ddcal(field): ) set_field_finished(field["target_name"], field["sas_id_target"]) else: + set_status_failed( + field["target_name"], "vlbi_dd", field["sas_id_target"] + ) raise RuntimeError return field + @task + def prepare_ddf(field): + print( + f"Preparing DDF input for {field['target_name']} {field['sas_id_target']}" + ) + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + target_path = get_most_recent_run(outdir, field["sas_id_target"], "VLBI_delay") + target_ms_path = target_path / "results_VLBI_delay-calibration" + mses_unaveraged = list(target_ms_path.glob("*.dp3concat")) + delay_sols = "" + if not mses_unaveraged: + print( + "No MSes found in delay-calibration output, will apply delay solutions to LINC." + ) + sols_path = get_most_recent_run( + outdir, field["sas_id_target"], "VLBI_delay" + ) + sols_path = sols_path / "results_VLBI_delay-calibration" + delay_sols = list( + sols_path.glob("merged*selfcalcycle???_linearfulljones*.h5") + )[0] + print(f"Using PILOT delay calibration solutions: {delay_sols}") + + linc_path = get_most_recent_run( + outdir, field["sas_id_target"], "LINC_target" + ) + print(f"Using LINC MSes at {linc_path}") + linc_ms_path = linc_path / "results_LINC_target" / "results" + mses_unaveraged = list(linc_ms_path.glob("*.dp3concat")) + else: + print("Found unaveraged data in delay calibration output.") + mses_averaged = list(target_ms_path.glob("*_pre-cal.ms")) + if not mses_unaveraged: + raise RuntimeError( + f"No unaveraged input MSes found at {linc_ms_path}/*.dp3concat" + ) + if mses_unaveraged and (len(mses_averaged) == len(mses_unaveraged)): + print("Appropriate input exists for ddf-pipeline.") + return field + + jobids = [] + averaged_mses = [] + for ms in mses_unaveraged: + out_ms = target_ms_path / f"{ms.stem}_pre-cal.ms" + if out_ms.exists(): + print(f"Skipping {out_ms.name}, already exists.") + averaged_mses.append(str(out_ms)) + continue + if delay_sols: + h5 = h5parm(str(delay_sols)) + ss = h5.getSolset("sol000") + # We only expect there to be one direction: the delay calibrator. + dirname = list(ss.getSou())[0] + sourcedir = ss.getSou()[dirname] + delaydir = f"[{sourcedir[0]},{sourcedir[1]}]" + dp3_cmd = f"apptainer exec $CWL_SINGULARITY_CACHE/astronrd_linc_latest.sif DP3 numthreads=2 msin={ms} msout={out_ms} msout.uvwcompression=False msout.antennacompression=False msout.scalarflags=False msout.storagemanager=Dysco steps=[average,applybeamdelay,applycal,applybeamtarget,filter] average.timeresolution=8 average.freqresolution=97.64kHz applybeamdelay.type=applybeam applybeamdelay.beammode=full applybeamdelay.updateweights=True applybeamdelay.direction={delaydir} applycal.parmdb={delay_sols} applycal.correction=fulljones applycal.soltab=[amplitude000,phase000] applybeamtarget.type=applybeam applybeamtarget.beammode=full applybeamtarget.updateweights=True filter.remove=True filter.baseline='[CR]S*&&'" + print(dp3_cmd) + else: + dp3_cmd = f"apptainer exec $CWL_SINGULARITY_CACHE/astronrd_linc_latest.sif DP3 numthreads=2 msin={ms} msout={out_ms} msout.uvwcompression=False msout.antennacompression=False msout.scalarflags=False msout.storagemanager=Dysco steps=[filter,average] average.timeresolution=8 average.freqresolution=97.64kHz filter.remove=True filter.baseline='[CR]S*&&'" + submit_cmd = f'sbatch -A {SLURM_ACCOUNT} -p {SLURM_QUEUE} --time=02:00:00 -c 2 --job-name=dp3_avg_{ms.stem} --wrap="{dp3_cmd}"' + print(f"Submitting: {submit_cmd}") + proc = subprocess.run( + submit_cmd, shell=True, text=True, capture_output=True + ) + if proc.returncode: + print(proc.stdout) + print(proc.stderr) + raise RuntimeError(f"Failed to submit SLURM job for {ms}") + jobid = proc.stdout.strip().split()[-1] + print(f"Submitted job {jobid} for {ms.name}") + jobids.append((jobid, out_ms)) + averaged_mses.append(str(out_ms)) + + while jobids: + print(f"Polling {len(jobids)} SLURM jobs...") + remaining = [] + for jobid, out_ms in jobids: + poll_cmd = f"sacct -X -j {jobid} --format=State --noheader" + status = subprocess.run( + poll_cmd, shell=True, text=True, capture_output=True + ).stdout.strip() + if status == "COMPLETED": + print(f"Job {jobid} completed ({out_ms.name})") + elif status == "FAILED": + raise RuntimeError( + f"DP3 averaging job {jobid} failed for {out_ms.name}" + ) + elif status in ("PENDING", "RUNNING"): + remaining.append((jobid, out_ms)) + else: + remaining.append((jobid, out_ms)) + jobids = remaining + time.sleep(30) + + mses_averaged = list(target_ms_path.glob("*_pre-cal.ms")) + if mses_averaged: + return field + else: + raise RuntimeError("No averaged MSes for ddf-pipeline found.") + + @task + def prepare_ddf_subtract(field): + print( + f"Preparing input for DDF subtract of {field['target_name']} {field['sas_id_target']}" + ) + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + target_path = get_most_recent_run(outdir, field["sas_id_target"], "VLBI_delay") + target_ms_path = target_path / "results_VLBI_delay-calibration" + mses_unaveraged = list(target_ms_path.glob("*.dp3concat")) + delay_sols = "" + if not mses_unaveraged: + print( + "No MSes found in delay-calibration output, will apply delay solutions to LINC." + ) + sols_path = get_most_recent_run( + outdir, field["sas_id_target"], "VLBI_delay" + ) + sols_path = sols_path / "results_VLBI_delay-calibration" + delay_sols = list( + sols_path.glob("merged*selfcalcycle???_linearfulljones*.h5") + )[0] + print(f"Using PILOT delay calibration solutions: {delay_sols}") + + linc_path = get_most_recent_run( + outdir, field["sas_id_target"], "LINC_target" + ) + print(f"Using LINC MSes at {linc_path}") + linc_ms_path = linc_path / "results_LINC_target" / "results" + mses_unaveraged = list(linc_ms_path.glob("*.dp3concat")) + mses_averaged = list(target_ms_path.glob("*_pre-cal.ms")) + mses_unaveraged_pilot = list(target_ms_path.glob("*.dp3concat")) + if not mses_unaveraged: + raise RuntimeError( + f"No unaveraged input MSes found at {linc_ms_path}/*.dp3concat" + ) + if mses_unaveraged_pilot and ( + len(mses_unaveraged_pilot) == len(mses_unaveraged) + ): + print("Appropriate input exists for ddf-pipeline.") + return field + + jobids = [] + averaged_mses = [] + for ms in mses_unaveraged: + out_ms = target_ms_path / f"{ms.stem}.dp3concat" + if out_ms.exists(): + print(f"Skipping {out_ms.name}, already exists.") + averaged_mses.append(str(out_ms)) + continue + if delay_sols: + h5 = h5parm(str(delay_sols)) + ss = h5.getSolset("sol000") + # We only expect there to be one direction: the delay calibrator. + dirname = list(ss.getSou())[0] + sourcedir = ss.getSou()[dirname] + delaydir = f"[{sourcedir[0]},{sourcedir[1]}]" + dp3_cmd = f"apptainer exec $CWL_SINGULARITY_CACHE/astronrd_linc_latest.sif DP3 numthreads=2 msin={ms} msout={out_ms} msout.uvwcompression=False msout.antennacompression=False msout.scalarflags=False msout.storagemanager=Dysco steps=[applybeamdelay,applycal,applybeamtarget] applybeamdelay.type=applybeam applybeamdelay.beammode=full applybeamdelay.updateweights=True applybeamdelay.direction={delaydir} applycal.parmdb={delay_sols} applycal.correction=fulljones applycal.soltab=[amplitude000,phase000] applybeamtarget.type=applybeam applybeamtarget.beammode=full applybeamtarget.updateweights=True" + print(dp3_cmd) + else: + dp3_cmd = f"apptainer exec $CWL_SINGULARITY_CACHE/astronrd_linc_latest.sif DP3 numthreads=2 msin={ms} msout={out_ms} msout.uvwcompression=False msout.antennacompression=False msout.scalarflags=False msout.storagemanager=Dysco steps=[]" + submit_cmd = f'sbatch -A {SLURM_ACCOUNT} -p {SLURM_QUEUE} --time=08:00:00 -c 2 --job-name=dp3_avg_{ms.stem} --wrap="{dp3_cmd}"' + print(f"Submitting: {submit_cmd}") + proc = subprocess.run( + submit_cmd, shell=True, text=True, capture_output=True + ) + if proc.returncode: + print(proc.stdout) + print(proc.stderr) + raise RuntimeError(f"Failed to submit SLURM job for {ms}") + jobid = proc.stdout.strip().split()[-1] + print(f"Submitted job {jobid} for {ms.name}") + jobids.append((jobid, out_ms)) + averaged_mses.append(str(out_ms)) + + while jobids: + print(f"Polling {len(jobids)} SLURM jobs...") + remaining = [] + for jobid, out_ms in jobids: + poll_cmd = f"sacct -X -j {jobid} --format=State --noheader" + status = subprocess.run( + poll_cmd, shell=True, text=True, capture_output=True + ).stdout.strip() + if status == "COMPLETED": + print(f"Job {jobid} completed ({out_ms.name})") + elif ( + (status == "FAILED") + or ("TIMEOUT" in status) + or ("CANCELLED" in status) + ): + raise RuntimeError( + f"DP3 averaging job {jobid} failed for {out_ms.name}" + ) + elif status in ("PENDING", "RUNNING"): + remaining.append((jobid, out_ms)) + else: + remaining.append((jobid, out_ms)) + jobids = remaining + time.sleep(30) + + mses_averaged = list(target_ms_path.glob("*_pre-cal.ms")) + if mses_averaged: + return field + else: + raise RuntimeError("No averaged MSes for ddf-pipeline found.") + proceed = check_fields() get_field = get_unprocessed_target() field = download_field(get_field) @@ -699,12 +1097,20 @@ def run_vlbi_ddcal(field): mode="poke", op_args=[result_vlbi_delay, "vlbi_delay", NEEDS_MANUAL_APPROVAL_DELAY], ) - # result_ddf = run_ddf_pipeline(await_approval_delay) - result_ddf = run_ddf_pipeline(result_vlbi_delay) - - result_ddf_subtract = run_ddf_subtract(result_ddf) + result_prepare_ddf = prepare_ddf(result_vlbi_delay) + result_ddf = run_ddf_pipeline(result_prepare_ddf) + result_prepare_ddf_subtract = prepare_ddf_subtract(result_ddf) + result_ddf_subtract = run_ddf_subtract(result_prepare_ddf_subtract) result_vlbi_dd = run_vlbi_ddcal(result_ddf_subtract) - await_approval_delay >> result_ddf >> result_ddf_subtract >> result_vlbi_dd + + ( + await_approval_delay + >> result_prepare_ddf + >> result_ddf + >> result_prepare_ddf_subtract + >> result_ddf_subtract + >> result_vlbi_dd + ) pilot_widefield() From c48cb9201956583ff7ab7f0a07e8d9594762831f Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Mon, 3 Aug 2026 12:53:12 +0200 Subject: [PATCH 033/120] Update readme --- README.md | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7594ded..9a57e2c 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,30 @@ To start processing data, Airflow needs to be running. This will be delegated to 4. Define `AIRFLOW__CORE__DAGS_FOLDER` as `${AIRFLOW_HOME}/dags` and create the folder. Copy the DAGs inside `flocs_processing/dags` to this folder. 5. Define `AIRFLOW__CORE__LOAD_EXAMPLES` as `False` -Next start a screen or tmux session and run `airflow standalone` to start the Airflow instance. It will echo a user name and password on first start, but will also store the credentials in `${AIRFLOW_HOME}/simple_auth_manager_passwords.json.generated`. The Airflow instance will start on port 8080. You can access it via `localhost:8080` in your browser. If it is running on a remote cluster, you can set up a tunnel via e.g. `ssh -N -L 8080:localhost:8080 ` to forward it to your local machine. +Finally, define the following airflow variables: + +``` +export AIRFLOW_HOME=/path/to/some/folder/for/airflow +export AIRFLOW__CORE__DAGS_FOLDER=$AIRFLOW_HOME/dags +export AIRFLOW__CORE__LOAD_EXAMPLES=False +export AIRFLOW__CORE__PARALLELISM=32 +export AIRFLOW__LOGGING__DAG_PROCESSOR_CHILD_PROCESS_LOG_DIRECTORY=$AIRFLOW_HOME/logs/dag_processor +export AIRFLOW__CORE__PLUGINS_FOLDER=$AIRFLOW_HOME/plugins +export AIRFLOW__DATABASE__SQL_ALCHEMY_CONN="sqlite:///$AIRFLOW_HOME/airflow.db" +export AIRFLOW__LOGGING__BASE_LOG_FOLDER=$AIRFLOW_HOME/logs +``` + +For a small test, you can run `airflow standalone` to start the Airflow instance for a small test. +For proper deployment, it is recommended by the Airflow docs to not use `standalone`. To start the necessary Airflow services, execute them like follows: + +``` +tmux new-session -d -s airflow-api-server "bash -c 'source $HOME/source_airflow.sh && airflow api-server; exec bash'" +tmux new-session -d -s airflow-triggerer "bash -c 'source $HOME/source_airflow.sh && airflow api-server; exec bash'" +tmux new-session -d -s airflow-dag-processor "bash -c 'source $HOME/source_airflow.sh && airflow api-server; exec bash'" +tmux new-session -d -s airflow-scheduler "bash -c 'source $HOME/source_airflow.sh && airflow scheduler; exec bash'" +``` + +This should start four tmux sessions with these services running in the background. The credentials to log into e.g. the web interface will be stored in `${AIRFLOW_HOME}/simple_auth_manager_passwords.json.generated`. The Airflow instance will start on port 8080. You can access it via `localhost:8080` in your browser. If it is running on a remote cluster, you can set up a tunnel via e.g. `ssh -N -L 8080:localhost:8080 ` to forward it to your local machine. Once `flocs-processing` is complete the processing loop will be automatic, but for now the user must trigger the DAG manually. On the "Dags" tab you should now see the flocs DAGs available. To manually trigger one, click on the name and on the subsequent page use the "Trigger" button in the top right. From a69c0a406c0bbd68bfd041d783e6ffd05c008d09 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Mon, 3 Aug 2026 13:29:56 +0200 Subject: [PATCH 034/120] Add intermediate resolution imaging workflow --- flocs_processing/dags/pilot_widefield.py | 97 ++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index d7d3cd7..e9e4573 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -1078,6 +1078,102 @@ def prepare_ddf_subtract(field): else: raise RuntimeError("No averaged MSes for ddf-pipeline found.") + @task + def run_vlbi_image_intermediate(field): + field = dict(get_db_columns(field["sas_id_target"])[0]) + if (field["status_vlbi_intermediate_img"] == PIPELINE_STATUS.finished) or ( + field["status_vlbi_intermediate_img"] == PIPELINE_STATUS.processing + ): + return field + else: + print( + f"Processing ILT intermediate resolution imaging for {field['target_name']} {field['sas_id_target']}" + ) + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + target_path = get_most_recent_run( + outdir, field["sas_id_target"], "VLBI_process-ddf" + ) + target_ms_path = target_path / "results_VLBI_process-ddf" + print(f"Using subtracted data at: {target_path}") + + dd_sols_path = get_most_recent_run( + outdir, field["sas_id_target"], "VLBI_dd-calibration" + ) + dd_sols = dd_sols_path / "results_VLBI_dd-calibration" / "merged.h5" + print(f"Using dd solutions at: {target_path}") + + if not os.path.isfile(dd_sols): + raise AirflowFailException(f"{dd_sols} not found.") + + set_status_processing( + field["target_name"], "vlbi_intermediate_img", field["sas_id_target"] + ) + + context = get_current_context() + if context["ti"].try_number == 1: + cmd = f"flocs-run vlbi image-intermediate-resolution --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --dd-solutions {dd_sols} --ms-suffix .dp3concat.sub.ms {target_ms_path}" + else: + if field["status_vlbi_intermediate_img"] == PIPELINE_STATUS.downloaded: + # This way we can force a clean restart in the database. + cmd = f"flocs-run vlbi image-intermediate-resolution --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --dd-solutions {dd_sols} --ms-suffix .dp3concat.sub.ms {target_ms_path}" + else: + # Extract the previous working directory + flocs_workdir = "" + print( + f"Scanning log_VLBI_image_intermediate_resolution_{field['target_name']}_{field['sas_id_target']}.txt for workdir." + ) + with open( + f"log_VLBI_image_intermediate_resolution_{field['target_name']}_{field['sas_id_target']}.txt" + ) as f_out: + for line in f_out.readlines(): + print(line) + if "Running workflow with" in line: + flocs_workdir = line.split(" ")[-1].strip() + break + if not flocs_workdir: + raise RuntimeError( + "Could not retrieve PILOT workdir. Flocs probably crashed before launching." + ) + print(f"Resuming failed PILOT run in {flocs_workdir}") + cmd = f"flocs-run vlbi image-intermediate-resolution --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --rundir {PROCESSING_DIR} --outdir {outdir} --dd-solutions {dd_sols} --ms-suffix .dp3concat.sub.ms {target_ms_path}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with ( + open( + f"log_VLBI_image_intermediate_resolution_{field['target_name']}_{field['sas_id_target']}.txt", + "w+", + ) as f_out, + open( + f"log_VLBI_image_intermediate_resolution_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w+", + ) as f_err, + ): + proc = subprocess.run( + cmd, shell=True, text=True, stdout=f_out, stderr=f_err + ) + success = False + pattern = re.compile(r"Workflow.* stopped. Success: True") + if not proc.returncode: + f_err.seek(0) + if pattern.search(f_err.read()): + success = True + if success: + set_status_finished( + field["target_name"], + "vlbi_intermediate_img", + field["sas_id_target"], + ) + set_field_finished(field["target_name"], field["sas_id_target"]) + else: + set_status_failed( + field["target_name"], + "vlbi_intermediate_img", + field["sas_id_target"], + ) + raise RuntimeError + return field + proceed = check_fields() get_field = get_unprocessed_target() field = download_field(get_field) @@ -1102,6 +1198,7 @@ def prepare_ddf_subtract(field): result_prepare_ddf_subtract = prepare_ddf_subtract(result_ddf) result_ddf_subtract = run_ddf_subtract(result_prepare_ddf_subtract) result_vlbi_dd = run_vlbi_ddcal(result_ddf_subtract) + result_vlbi_interm_img = run_vlbi_image_intermediate(result_vlbi_dd) ( await_approval_delay From f31ff4c4f0e0d5385fe1c6f98fbf944cc5006275 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Mon, 3 Aug 2026 15:38:24 +0200 Subject: [PATCH 035/120] Update tmux commands --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9a57e2c..c2c52f5 100644 --- a/README.md +++ b/README.md @@ -41,8 +41,8 @@ For proper deployment, it is recommended by the Airflow docs to not use `standal ``` tmux new-session -d -s airflow-api-server "bash -c 'source $HOME/source_airflow.sh && airflow api-server; exec bash'" -tmux new-session -d -s airflow-triggerer "bash -c 'source $HOME/source_airflow.sh && airflow api-server; exec bash'" -tmux new-session -d -s airflow-dag-processor "bash -c 'source $HOME/source_airflow.sh && airflow api-server; exec bash'" +tmux new-session -d -s airflow-triggerer "bash -c 'source $HOME/source_airflow.sh && airflow triggerer; exec bash'" +tmux new-session -d -s airflow-dag-processor "bash -c 'source $HOME/source_airflow.sh && airflow dag-processor; exec bash'" tmux new-session -d -s airflow-scheduler "bash -c 'source $HOME/source_airflow.sh && airflow scheduler; exec bash'" ``` From b5a59f52d156acc52969603dd562a44a27d9abb3 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Mon, 3 Aug 2026 15:39:54 +0200 Subject: [PATCH 036/120] Fix database addition --- flocs_processing/flocs_processing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flocs_processing/flocs_processing.py b/flocs_processing/flocs_processing.py index a603739..d4f1b3b 100644 --- a/flocs_processing/flocs_processing.py +++ b/flocs_processing/flocs_processing.py @@ -155,7 +155,7 @@ def add_field( dbstr += ", sas_id_calibrator1, sas_id_calibrator2" dbstr += f", sas_id_target) values ('{field_name}', " if len(sas_id_calibrators) == 1: - dbstr += f"'{sas_id_calibrators[0]}'" + dbstr += f"'{sas_id_calibrators[0]}', " if len(sas_id_calibrators) == 2: dbstr += f"'{sas_id_calibrators[0]}', '{sas_id_calibrators[1]}', " dbstr += f"'{sas_id_target}')" From 95a4d7408fed754c28ffa19121e5eb2bccefdee1 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Mon, 3 Aug 2026 16:11:52 +0200 Subject: [PATCH 037/120] Update database column creation --- flocs_processing/flocs_processing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flocs_processing/flocs_processing.py b/flocs_processing/flocs_processing.py index a603739..1df6884 100644 --- a/flocs_processing/flocs_processing.py +++ b/flocs_processing/flocs_processing.py @@ -109,10 +109,11 @@ def create_database( if "vlbi-delay-widefield" in pipelines: dbstr += f", status_vlbi_delay smallint default {PIPELINE_STATUS.nothing.value}" dbstr += f", status_vlbi_dd smallint default {PIPELINE_STATUS.nothing.value}" + dbstr += f", status_vlbi_ddf_subtract smallint default {PIPELINE_STATUS.nothing.value}" dbstr += f", status_vlbi_intermediate_img smallint default {PIPELINE_STATUS.nothing.value}" dbstr += f", status_vlbi_facet_subtract smallint default {PIPELINE_STATUS.nothing.value}" dbstr += ( - f", status_vlbi_facet_img smallint default {PIPELINE_STATUS.nothing.value}" + f", status_vlbi_facet_image smallint default {PIPELINE_STATUS.nothing.value}" ) if "vlbi-delay-single-target" in pipelines: dbstr += f", status_vlbi_delay smallint default {PIPELINE_STATUS.nothing.value}" From 99a21996756398ae363f330d1d3e17d584dad45c Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Mon, 3 Aug 2026 16:31:27 +0200 Subject: [PATCH 038/120] Add facet subtract task --- flocs_processing/dags/pilot_widefield.py | 114 ++++++++++++++++++++++- 1 file changed, 113 insertions(+), 1 deletion(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index e9e4573..4d2c52a 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -1135,7 +1135,7 @@ def run_vlbi_image_intermediate(field): "Could not retrieve PILOT workdir. Flocs probably crashed before launching." ) print(f"Resuming failed PILOT run in {flocs_workdir}") - cmd = f"flocs-run vlbi image-intermediate-resolution --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --rundir {PROCESSING_DIR} --outdir {outdir} --dd-solutions {dd_sols} --ms-suffix .dp3concat.sub.ms {target_ms_path}" + cmd = f"flocs-run vlbi image-intermediate-resolution --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --dd-solutions {dd_sols} --ms-suffix .dp3concat.sub.ms {target_ms_path}" if not os.path.isdir(outdir): os.mkdir(outdir) print(cmd) @@ -1174,6 +1174,118 @@ def run_vlbi_image_intermediate(field): raise RuntimeError return field + @task + def run_vlbi_facet_subtract(field): + field = dict(get_db_columns(field["sas_id_target"])[0]) + if (field["status_vlbi_facet_subtract"] == PIPELINE_STATUS.finished) or ( + field["status_vlbi_facet_subtract"] == PIPELINE_STATUS.processing + ): + return field + else: + print( + f"Processing ILT facet subtraction for {field['target_name']} {field['sas_id_target']}" + ) + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + target_path = get_most_recent_run( + outdir, field["sas_id_target"], "VLBI_process-ddf" + ) + target_ms_path = target_path / "results_VLBI_process-ddf" + print(f"Using subtracted data at: {target_path}") + + dd_sols_path = get_most_recent_run( + outdir, field["sas_id_target"], "VLBI_dd-calibration" + ) + dd_sols = dd_sols_path / "results_VLBI_dd-calibration" / "merged.h5" + print(f"Using dd solutions at: {target_path}") + + if not os.path.isfile(dd_sols): + raise AirflowFailException(f"{dd_sols} not found.") + + model_images_path = get_most_recent_run( + outdir, field["sas_id_target"], "VLBI_intermediate_resolution_imaging" + ) + model_images = ( + model_images_path + / "results_VLBI_intermediate_resolution_imaging" + / "*-????-model-fpb.fits" + ) + model_images = list(model_images_path.glob("*-????-model-fpb.fits")) + print(f"Using model image at: {model_images_path}/*-????-model-fpb.fits") + + if not model_images: + raise AirflowFailException( + "No suitable intermediate resolution model images found." + ) + + set_status_processing( + field["target_name"], "vlbi_facet_subtract", field["sas_id_target"] + ) + + context = get_current_context() + if context["ti"].try_number == 1: + cmd = f"flocs-run vlbi facet-subtract --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --dd-solutions {dd_sols} --model-image-directory {model_images_path} --ms-suffix .dp3concat.sub.ms {target_ms_path}" + else: + if field["status_vlbi_facet_subtract"] == PIPELINE_STATUS.downloaded: + # This way we can force a clean restart in the database. + cmd = f"flocs-run vlbi facet-subtract --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --dd-solutions {dd_sols} --model-image-directory {model_images_path} --ms-suffix .dp3concat.sub.ms {target_ms_path}" + else: + # Extract the previous working directory + flocs_workdir = "" + print( + f"Scanning log_VLBI_facet_subtract_{field['target_name']}_{field['sas_id_target']}.txt for workdir." + ) + with open( + f"log_VLBI_facet_subtract_{field['target_name']}_{field['sas_id_target']}.txt" + ) as f_out: + for line in f_out.readlines(): + print(line) + if "Running workflow with" in line: + flocs_workdir = line.split(" ")[-1].strip() + break + if not flocs_workdir: + raise RuntimeError( + "Could not retrieve PILOT workdir. Flocs probably crashed before launching." + ) + print(f"Resuming failed PILOT run in {flocs_workdir}") + cmd = f"flocs-run vlbi facet-subtract --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --dd-solutions {dd_sols} --model-image-directory {model_images_path} --ms-suffix .dp3concat.sub.ms {target_ms_path}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with ( + open( + f"log_VLBI_facet_subtract_{field['target_name']}_{field['sas_id_target']}.txt", + "w+", + ) as f_out, + open( + f"log_VLBI_facet_subtract_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w+", + ) as f_err, + ): + proc = subprocess.run( + cmd, shell=True, text=True, stdout=f_out, stderr=f_err + ) + success = False + pattern = re.compile(r"Workflow.* stopped. Success: True") + if not proc.returncode: + f_err.seek(0) + if pattern.search(f_err.read()): + success = True + if success: + set_status_finished( + field["target_name"], + "vlbi_facet_subtract", + field["sas_id_target"], + ) + set_field_finished(field["target_name"], field["sas_id_target"]) + else: + set_status_failed( + field["target_name"], + "vlbi_facet_subtract", + field["sas_id_target"], + ) + raise RuntimeError + return field + proceed = check_fields() get_field = get_unprocessed_target() field = download_field(get_field) From c478d2fcd30df68f72a1c242fa8fb8e2a854c374 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Mon, 3 Aug 2026 16:46:46 +0200 Subject: [PATCH 039/120] Add facet imaging for 0.3 arcsec initially --- flocs_processing/dags/pilot_widefield.py | 96 ++++++++++++++++++++++++ flocs_processing/flocs_processing.py | 10 ++- 2 files changed, 103 insertions(+), 3 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 4d2c52a..2b99336 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -1286,6 +1286,100 @@ def run_vlbi_facet_subtract(field): raise RuntimeError return field + @task + def run_vlbi_facet_imaging(field): + field = dict(get_db_columns(field["sas_id_target"])[0]) + if (field["status_vlbi_facet_imaging"] == PIPELINE_STATUS.finished) or ( + field["status_vlbi_facet_imaging"] == PIPELINE_STATUS.processing + ): + return field + else: + print( + f"Processing ILT facet imaging for {field['target_name']} {field['sas_id_target']}" + ) + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + target_path = get_most_recent_run( + outdir, field["sas_id_target"], "VLBI_facet_subtract" + ) + target_ms_path = target_path / "results_VLBI_facet_subtract" + print(f"Using subtracted data at: {target_path}") + + facet_mses = list(target_ms_path.glob("facet*.ms")) + + if not facet_mses: + raise AirflowFailException( + "No suitable intermediate resolution model images found." + ) + + set_status_processing( + field["target_name"], "vlbi_facet_imaging", field["sas_id_target"] + ) + + context = get_current_context() + if context["ti"].try_number == 1: + cmd = f"flocs-run vlbi facet-imaging --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --resolution 0.3asec --pixel-scale 0.1 --ms-suffix .ms {target_ms_path}" + else: + if field["status_vlbi_facet_imaging"] == PIPELINE_STATUS.downloaded: + # This way we can force a clean restart in the database. + cmd = f"flocs-run vlbi facet-imaging --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --resolution 0.3asec --pixel-scale 0.1 --ms-suffix .ms {target_ms_path}" + else: + # Extract the previous working directory + flocs_workdir = "" + print( + f"Scanning log_VLBI_facet_imaging_{field['target_name']}_{field['sas_id_target']}.txt for workdir." + ) + with open( + f"log_VLBI_facet_imaging_{field['target_name']}_{field['sas_id_target']}.txt" + ) as f_out: + for line in f_out.readlines(): + print(line) + if "Running workflow with" in line: + flocs_workdir = line.split(" ")[-1].strip() + break + if not flocs_workdir: + raise RuntimeError( + "Could not retrieve PILOT workdir. Flocs probably crashed before launching." + ) + print(f"Resuming failed PILOT run in {flocs_workdir}") + cmd = f"flocs-run vlbi facet-imaging --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --resolution 0.3asec --pixel-scale 0.1 --ms-suffix .ms {target_ms_path}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with ( + open( + f"log_VLBI_facet_imaging_{field['target_name']}_{field['sas_id_target']}.txt", + "w+", + ) as f_out, + open( + f"log_VLBI_facet_imaging_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w+", + ) as f_err, + ): + proc = subprocess.run( + cmd, shell=True, text=True, stdout=f_out, stderr=f_err + ) + success = False + pattern = re.compile(r"Workflow.* stopped. Success: True") + if not proc.returncode: + f_err.seek(0) + if pattern.search(f_err.read()): + success = True + if success: + set_status_finished( + field["target_name"], + "vlbi_facet_imaging", + field["sas_id_target"], + ) + set_field_finished(field["target_name"], field["sas_id_target"]) + else: + set_status_failed( + field["target_name"], + "vlbi_facet_imaging", + field["sas_id_target"], + ) + raise RuntimeError + return field + proceed = check_fields() get_field = get_unprocessed_target() field = download_field(get_field) @@ -1311,6 +1405,8 @@ def run_vlbi_facet_subtract(field): result_ddf_subtract = run_ddf_subtract(result_prepare_ddf_subtract) result_vlbi_dd = run_vlbi_ddcal(result_ddf_subtract) result_vlbi_interm_img = run_vlbi_image_intermediate(result_vlbi_dd) + result_vlbi_facet_subtract = run_vlbi_facet_subtract(result_vlbi_interm_img) + result_vlbi_facet_img = run_vlbi_facet_imaging(result_vlbi_facet_subtract) ( await_approval_delay diff --git a/flocs_processing/flocs_processing.py b/flocs_processing/flocs_processing.py index 2480bb9..8aa86eb 100644 --- a/flocs_processing/flocs_processing.py +++ b/flocs_processing/flocs_processing.py @@ -18,6 +18,12 @@ app = cyclopts.App() +# At some point we need to spawn these instead of running standalone +# tmux new-session -d -s airflow-api-server "bash -c 'source $HOME/source_airflow.sh && airflow api-server; exec bash'" +# tmux new-session -d -s airflow-scheduler "bash -c 'source $HOME/source_airflow.sh && airflow scheduler; exec bash'" +# tmux new-session -d -s airflow-dag-processor "bash -c 'source $HOME/source_airflow.sh && airflow dag-processor; exec bash'" +# tmux new-session -d -s airflow-triggerer "bash -c 'source $HOME/source_airflow.sh && airflow triggerer; exec bash'" + @functools.total_ordering class PIPELINE(Enum): @@ -112,9 +118,7 @@ def create_database( dbstr += f", status_vlbi_ddf_subtract smallint default {PIPELINE_STATUS.nothing.value}" dbstr += f", status_vlbi_intermediate_img smallint default {PIPELINE_STATUS.nothing.value}" dbstr += f", status_vlbi_facet_subtract smallint default {PIPELINE_STATUS.nothing.value}" - dbstr += ( - f", status_vlbi_facet_image smallint default {PIPELINE_STATUS.nothing.value}" - ) + dbstr += f", status_vlbi_facet_imaging smallint default {PIPELINE_STATUS.nothing.value}" if "vlbi-delay-single-target" in pipelines: dbstr += f", status_vlbi_delay smallint default {PIPELINE_STATUS.nothing.value}" dbstr += f", status_vlbi_dd smallint default {PIPELINE_STATUS.nothing.value}" From 0e7b476d3d42f548b7d0f85b831a4fbc6f256c37 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Mon, 3 Aug 2026 16:50:47 +0200 Subject: [PATCH 040/120] Fix ddf subtract status column query --- flocs_processing/dags/pilot_widefield.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 2b99336..507cbe5 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -72,7 +72,7 @@ def get_db_columns(obsid: str = None): with sqlite3.connect(DATABASE) as db: db.row_factory = sqlite3.Row cursor = db.cursor() - columns = "target_name,priority,finished,downloaded,sas_id_calibrator1,sas_id_calibrator2,sas_id_calibrator_final,sas_id_target,status_calibrator1,status_calibrator2,status_target,status_vlbi_delay,status_vlbi_dd,status_ddf,status_ddf_subtract" + columns = "target_name,priority,finished,downloaded,sas_id_calibrator1,sas_id_calibrator2,sas_id_calibrator_final,sas_id_target,status_calibrator1,status_calibrator2,status_target,status_vlbi_delay,status_vlbi_dd,status_ddf,status_vlbi_ddf_subtract" if obsid: field = cursor.execute( f"select {columns} from {TABLE_NAME} where sas_id_target=='{obsid}' and finished==0 order by priority desc" @@ -678,8 +678,8 @@ def run_ddf_pipeline(field): @task def run_ddf_subtract(field): - if (field["status_ddf_subtract"] == PIPELINE_STATUS.finished) or ( - field["status_ddf_subtract"] == PIPELINE_STATUS.processing + if (field["status_vlbi_ddf_subtract"] == PIPELINE_STATUS.finished) or ( + field["status_vlbi_ddf_subtract"] == PIPELINE_STATUS.processing ): return field else: From 9ddbc4afd31caedae5d49a4a481a675d08790b10 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Mon, 3 Aug 2026 17:06:05 +0200 Subject: [PATCH 041/120] Fix frequency range for downloading --- flocs_processing/dags/pilot_widefield.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 507cbe5..0ba13dd 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -216,11 +216,11 @@ def download_field(field): "ALL", field["sas_id_target"], None, - 120e6, - 168e6, + 120, + 168, ) if stage_calibrators: - stager.find_nearest_calibrators(2, 120e6, 168e6) + stager.find_nearest_calibrators(2, 120, 168) stage_id_calibrators = stager.stage_calibrators() if stage_target: stage_id_target = stager.stage_target() From 0944be7ab20f3459de1040fafba3cdde65b8ad0a Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Tue, 4 Aug 2026 10:29:58 +0200 Subject: [PATCH 042/120] Update readme with secret --- README.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c2c52f5..53782ba 100644 --- a/README.md +++ b/README.md @@ -36,8 +36,17 @@ export AIRFLOW__DATABASE__SQL_ALCHEMY_CONN="sqlite:///$AIRFLOW_HOME/airflow.db" export AIRFLOW__LOGGING__BASE_LOG_FOLDER=$AIRFLOW_HOME/logs ``` -For a small test, you can run `airflow standalone` to start the Airflow instance for a small test. -For proper deployment, it is recommended by the Airflow docs to not use `standalone`. To start the necessary Airflow services, execute them like follows: +For a small test, you can run `airflow standalone` to start the Airflow instance for a small test. For proper deployment, it is recommended by the Airflow docs to not use `standalone`. First we'll set up a persistent JWT secret for authentication purpose. + +``` +mkdir -p "$HOME/.config/airflow" +chmod 700 "$HOME/.config/airflow" +openssl rand -hex 32 > "$HOME/.config/airflow/jwt_secret" +chmod 600 "$HOME/.config/airflow/jwt_secret" +export AIRFLOW__API_AUTH__JWT_SECRET="$(cat "$HOME/.config/airflow/jwt_secret")" +``` + +Finally, to start the necessary Airflow services, execute them like follows: ``` tmux new-session -d -s airflow-api-server "bash -c 'source $HOME/source_airflow.sh && airflow api-server; exec bash'" From 6a6c62d49dc176523cdd365fdf1b1451b76c8683 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Tue, 4 Aug 2026 11:05:39 +0200 Subject: [PATCH 043/120] Update readme with DB initialisation --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 53782ba..b71f806 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,12 @@ chmod 600 "$HOME/.config/airflow/jwt_secret" export AIRFLOW__API_AUTH__JWT_SECRET="$(cat "$HOME/.config/airflow/jwt_secret")" ``` +Next, initialise Airflow's own database with + +``` +airflow db migrate +``` + Finally, to start the necessary Airflow services, execute them like follows: ``` From ef6892cfcf5c3bd03ff37581d0c5af9b814cd1d7 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Tue, 4 Aug 2026 12:28:20 +0200 Subject: [PATCH 044/120] Don't succeed delay if processing --- flocs_processing/dags/pilot_widefield.py | 1 - 1 file changed, 1 deletion(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 0ba13dd..d4d4497 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -483,7 +483,6 @@ def validate_linc_target(field): def run_vlbi_delay(field): if ( (field["status_vlbi_delay"] == PIPELINE_STATUS.finished) - or (field["status_vlbi_delay"] == PIPELINE_STATUS.processing) or (field["status_vlbi_delay"] == PIPELINE_STATUS.await_approval) ): return field From 0532dcb2cff083fdf195839b2aa7d970f6b7365e Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Tue, 4 Aug 2026 13:17:00 +0200 Subject: [PATCH 045/120] Fix target staged flag --- flocs_processing/dags/pilot_widefield.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index d4d4497..6fd6ae3 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -262,7 +262,7 @@ def download_field(field): if len(get_surls_online(stage_id_target)) == len( get_surls_requested(stage_id_target) ): - calibrator_staged = True + target_staged = True if target_staged and not target_downloaded: dl_path = os.path.join(DATA_DIR, field["target_name"], "target") cmd = f"flocs-lta download --outdir {dl_path} {stage_id_target}" From f65bff52b2e64ab8100656038fc6a231bae473da Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Tue, 4 Aug 2026 13:58:45 +0200 Subject: [PATCH 046/120] Run lofar-vlbi-plot if no delay catalogue is found --- flocs_processing/dags/pilot_widefield.py | 33 +++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 6fd6ae3..10fdee2 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -481,9 +481,8 @@ def validate_linc_target(field): @task(retries=0, retry_delay=datetime.timedelta(seconds=5)) def run_vlbi_delay(field): - if ( - (field["status_vlbi_delay"] == PIPELINE_STATUS.finished) - or (field["status_vlbi_delay"] == PIPELINE_STATUS.await_approval) + if (field["status_vlbi_delay"] == PIPELINE_STATUS.finished) or ( + field["status_vlbi_delay"] == PIPELINE_STATUS.await_approval ): return field else: @@ -502,6 +501,34 @@ def run_vlbi_delay(field): delay_cat = os.path.join(outdir, "delay_calibrators.csv") image_cat = os.path.join(outdir, "image_catalogue.csv") + if not os.path.isfile(delay_cat): + ms = list(target_ms_path.glob("*.dp3concat"))[0] + cmd = f"lofar-vlbi-plot --MS {ms}" + with ( + open( + f"log_plot_field_{field['target_name']}_{field['sas_id_target']}.txt", + "w+", + ) as f_out, + open( + f"log_plot_field_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w+", + ) as f_err, + ): + proc = subprocess.run( + cmd, shell=True, text=True, stdout=f_out, stderr=f_err + ) + if not proc.returncode: + raise RuntimeError( + "Failed to download necessary products for delay calibration." + ) + delay_cat = os.path.join(outdir, "delay_calibrators.csv") + image_cat = os.path.join(outdir, "image_catalogue.csv") + + if not os.path.isfile(delay_cat): + raise RuntimeError("Delay calibrator catalogue is missing or invalid.") + if not os.path.isfile(image_cat): + raise RuntimeError("Image source catalogue is missing or invalid.") + proc = subprocess.run( "detect_bad_slurm_nodes.sh", shell=True, From d67066f6f17440c35431e1ad02f1b683c63018f4 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Tue, 4 Aug 2026 15:02:09 +0200 Subject: [PATCH 047/120] Make downloading a bit more robust with restarts --- flocs_processing/dags/pilot_widefield.py | 72 ++++++++++++++---------- 1 file changed, 42 insertions(+), 30 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 10fdee2..3e1734c 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -195,15 +195,26 @@ def download_field(field): has_cal2 = True else: stage_calibrators = True + if field["sas_id_target"]: ms_folder = f"L{field['sas_id_target']}" target_full_path = os.path.join( DATA_DIR, field["target_name"], "target", ms_folder ) if os.path.exists(target_full_path): - stage_target = False - else: - stage_target = True + out = subprocess.check_output( + "wc -l srms_{field['sas_id_target']}.txt | cut -f 1 -d ' '", + text=True, + ) + num_staged = int(out.strip()) + + num_downloaded = len( + list(pathlib.Path(target_full_path).glob("*.MS")) + ) + if num_downloaded == num_staged: + stage_target = False + else: + stage_target = True else: raise AirflowFailException( f"No target SAS ID in database for field {field['target_name']}" @@ -247,7 +258,7 @@ def download_field(field): "w+", ) as f_out, open( - f"log_download_calibrators_{field['target_name']}.txt", + f"log_download_calibrators_{field['target_name']}_err.txt", "w+", ) as f_err, ): @@ -259,34 +270,35 @@ def download_field(field): else: raise RuntimeError - if len(get_surls_online(stage_id_target)) == len( - get_surls_requested(stage_id_target) - ): - target_staged = True - if target_staged and not target_downloaded: - dl_path = os.path.join(DATA_DIR, field["target_name"], "target") - cmd = f"flocs-lta download --outdir {dl_path} {stage_id_target}" - with ( - open( - f"log_download_calibrators_{field['target_name']}.txt", - "w+", - ) as f_out, - open( - f"log_download_calibrators_{field['target_name']}.txt", - "w+", - ) as f_err, + if not target_downloaded: + if len(get_surls_online(stage_id_target)) == len( + get_surls_requested(stage_id_target) ): - proc = subprocess.run( - cmd, shell=True, text=True, stdout=f_out, stderr=f_err - ) - if not proc.returncode: - set_status_downloaded( - field["target_name"], - field["sas_id_target"], + target_staged = True + if target_staged and not target_downloaded: + dl_path = os.path.join(DATA_DIR, field["target_name"], "target") + cmd = f"flocs-lta download --outdir {dl_path} {stage_id_target}" + with ( + open( + f"log_download_target_{field['target_name']}.txt", + "w+", + ) as f_out, + open( + f"log_download_target_{field['target_name']}_err.txt", + "w+", + ) as f_err, + ): + proc = subprocess.run( + cmd, shell=True, text=True, stdout=f_out, stderr=f_err ) - target_downloaded = True - else: - raise RuntimeError + if not proc.returncode: + set_status_downloaded( + field["target_name"], + field["sas_id_target"], + ) + target_downloaded = True + else: + raise RuntimeError if calibrator_downloaded and target_downloaded: break time.sleep(60) From 598fa468c20f2e6856e8fd0f252eb7f0b127f810 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Tue, 4 Aug 2026 15:09:12 +0200 Subject: [PATCH 048/120] Add more download guards --- flocs_processing/dags/pilot_widefield.py | 43 +++++++++++++++++++----- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 3e1734c..9a0ba58 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -175,6 +175,24 @@ def download_field(field): else: has_cal1 = False stage_calibrators = False + num_downloaded_calib1 = 0 + num_downloaded_calib2 = 0 + num_staged_calib = 0 + num_staged_targ = 0 + if os.path.exists("srms_{field['sas_id_target']}_calibrators.txt"): + out = subprocess.check_output( + "wc -l srms_{field['sas_id_target']}_calibrators.txt | cut -f 1 -d ' '", + text=True, + ) + num_staged_calib = int(out.strip()) + + if os.path.exists("srms_{field['sas_id_target']}.txt"): + out = subprocess.check_output( + "wc -l srms_{field['sas_id_target']}.txt | cut -f 1 -d ' '", + text=True, + ) + num_staged_targ = int(out.strip()) + if field["sas_id_calibrator1"]: ms_folder = f"L{field['sas_id_calibrator1']}" cal1_full_path = os.path.join( @@ -182,6 +200,9 @@ def download_field(field): ) if os.path.exists(cal1_full_path): has_cal1 = True + num_downloaded_calib1 = len( + list(pathlib.Path(cal1_full_path).glob("*.MS")) + ) else: stage_calibrators = True @@ -193,25 +214,29 @@ def download_field(field): ) if os.path.exists(cal2_full_path): has_cal2 = True + num_downloaded_calib2 = len( + list(pathlib.Path(cal2_full_path).glob("*.MS")) + ) else: stage_calibrators = True + num_downloaded_calib = num_downloaded_calib1 + num_downloaded_calib2 + if num_downloaded_calib == num_staged_calib: + stage_calibrators = False + else: + stage_calibrators = True + + stage_target = False if field["sas_id_target"]: ms_folder = f"L{field['sas_id_target']}" target_full_path = os.path.join( DATA_DIR, field["target_name"], "target", ms_folder ) if os.path.exists(target_full_path): - out = subprocess.check_output( - "wc -l srms_{field['sas_id_target']}.txt | cut -f 1 -d ' '", - text=True, - ) - num_staged = int(out.strip()) - - num_downloaded = len( + num_downloaded_targ = len( list(pathlib.Path(target_full_path).glob("*.MS")) ) - if num_downloaded == num_staged: + if num_downloaded_targ == num_staged_targ: stage_target = False else: stage_target = True @@ -1444,7 +1469,7 @@ def run_vlbi_facet_imaging(field): result_vlbi_dd = run_vlbi_ddcal(result_ddf_subtract) result_vlbi_interm_img = run_vlbi_image_intermediate(result_vlbi_dd) result_vlbi_facet_subtract = run_vlbi_facet_subtract(result_vlbi_interm_img) - result_vlbi_facet_img = run_vlbi_facet_imaging(result_vlbi_facet_subtract) + _result_vlbi_facet_img = run_vlbi_facet_imaging(result_vlbi_facet_subtract) ( await_approval_delay From ba85ba730743a3f7ad1f0a9355887a078f0b78a5 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Tue, 4 Aug 2026 15:11:11 +0200 Subject: [PATCH 049/120] More dl guards --- flocs_processing/dags/pilot_widefield.py | 55 +++++++++++------------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 9a0ba58..f5b3319 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -173,7 +173,6 @@ def download_field(field): if field["downloaded"]: return field else: - has_cal1 = False stage_calibrators = False num_downloaded_calib1 = 0 num_downloaded_calib2 = 0 @@ -199,21 +198,18 @@ def download_field(field): DATA_DIR, field["target_name"], "calibrator", ms_folder ) if os.path.exists(cal1_full_path): - has_cal1 = True num_downloaded_calib1 = len( list(pathlib.Path(cal1_full_path).glob("*.MS")) ) else: stage_calibrators = True - has_cal2 = False if field["sas_id_calibrator2"]: ms_folder = f"L{field['sas_id_calibrator2']}" cal2_full_path = os.path.join( DATA_DIR, field["target_name"], "calibrator", ms_folder ) if os.path.exists(cal2_full_path): - has_cal2 = True num_downloaded_calib2 = len( list(pathlib.Path(cal2_full_path).glob("*.MS")) ) @@ -265,35 +261,36 @@ def download_field(field): calibrator_staged = False target_staged = False - calibrator_downloaded = has_cal1 or has_cal2 + calibrator_downloaded = not stage_calibrators target_downloaded = not stage_target while True: - if len(get_surls_online(stage_id_calibrators)) == len( - get_surls_requested(stage_id_calibrators) - ): - calibrator_staged = True - if calibrator_staged and not calibrator_downloaded: - dl_path = os.path.join(DATA_DIR, field["target_name"], "calibrator") - cmd = ( - f"flocs-lta download --outdir {dl_path} {stage_id_calibrators}" - ) - with ( - open( - f"log_download_calibrators_{field['target_name']}.txt", - "w+", - ) as f_out, - open( - f"log_download_calibrators_{field['target_name']}_err.txt", - "w+", - ) as f_err, + if not calibrator_downloaded: + if len(get_surls_online(stage_id_calibrators)) == len( + get_surls_requested(stage_id_calibrators) ): - proc = subprocess.run( - cmd, shell=True, text=True, stdout=f_out, stderr=f_err + calibrator_staged = True + if calibrator_staged and not calibrator_downloaded: + dl_path = os.path.join( + DATA_DIR, field["target_name"], "calibrator" ) - if not proc.returncode: - calibrator_downloaded = True - else: - raise RuntimeError + cmd = f"flocs-lta download --outdir {dl_path} {stage_id_calibrators}" + with ( + open( + f"log_download_calibrators_{field['target_name']}.txt", + "w+", + ) as f_out, + open( + f"log_download_calibrators_{field['target_name']}_err.txt", + "w+", + ) as f_err, + ): + proc = subprocess.run( + cmd, shell=True, text=True, stdout=f_out, stderr=f_err + ) + if not proc.returncode: + calibrator_downloaded = True + else: + raise RuntimeError if not target_downloaded: if len(get_surls_online(stage_id_target)) == len( From cd7d4c5f891c4aa8ae6062b7ed4b94949f3d9446 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Tue, 4 Aug 2026 15:20:31 +0200 Subject: [PATCH 050/120] Be more verbose about downloading --- flocs_processing/dags/pilot_widefield.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index f5b3319..030dfbd 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -179,6 +179,7 @@ def download_field(field): num_staged_calib = 0 num_staged_targ = 0 if os.path.exists("srms_{field['sas_id_target']}_calibrators.txt"): + print("Found srm file; ounting calibrator SRMs.") out = subprocess.check_output( "wc -l srms_{field['sas_id_target']}_calibrators.txt | cut -f 1 -d ' '", text=True, @@ -186,6 +187,7 @@ def download_field(field): num_staged_calib = int(out.strip()) if os.path.exists("srms_{field['sas_id_target']}.txt"): + print("Found srm file; counting target SRMs.") out = subprocess.check_output( "wc -l srms_{field['sas_id_target']}.txt | cut -f 1 -d ' '", text=True, @@ -218,8 +220,14 @@ def download_field(field): num_downloaded_calib = num_downloaded_calib1 + num_downloaded_calib2 if num_downloaded_calib == num_staged_calib: + print( + f"Number of staged calibrator MSes ({num_staged_calib}) equals number of downloaded MSes ({num_downloaded_calib}); not staging calibrators again." + ) stage_calibrators = False else: + print( + f"Number of staged calibrator MSes ({num_staged_calib}) does NOT equal number of downloaded MSes ({num_downloaded_calib}); restaging calibrators and resuming download." + ) stage_calibrators = True stage_target = False @@ -233,8 +241,14 @@ def download_field(field): list(pathlib.Path(target_full_path).glob("*.MS")) ) if num_downloaded_targ == num_staged_targ: + print( + f"Number of staged target MSes ({num_staged_calib}) equals number of downloaded MSes ({num_downloaded_calib}); not staging target again." + ) stage_target = False else: + print( + f"Number of staged target MSes ({num_staged_calib}) does NOT number of downloaded MSes ({num_downloaded_calib}); staging target again and resuming download." + ) stage_target = True else: raise AirflowFailException( From d1225be8a4da7de8b499c2673f1393faba875f8f Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Tue, 4 Aug 2026 16:23:26 +0200 Subject: [PATCH 051/120] Fix target dl reporting --- flocs_processing/dags/pilot_widefield.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 030dfbd..ec29ca2 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -179,7 +179,7 @@ def download_field(field): num_staged_calib = 0 num_staged_targ = 0 if os.path.exists("srms_{field['sas_id_target']}_calibrators.txt"): - print("Found srm file; ounting calibrator SRMs.") + print("Found srm file; counting calibrator SRMs.") out = subprocess.check_output( "wc -l srms_{field['sas_id_target']}_calibrators.txt | cut -f 1 -d ' '", text=True, @@ -242,12 +242,12 @@ def download_field(field): ) if num_downloaded_targ == num_staged_targ: print( - f"Number of staged target MSes ({num_staged_calib}) equals number of downloaded MSes ({num_downloaded_calib}); not staging target again." + f"Number of staged target MSes ({num_staged_targ}) equals number of downloaded MSes ({num_downloaded_targ}); not staging target again." ) stage_target = False else: print( - f"Number of staged target MSes ({num_staged_calib}) does NOT number of downloaded MSes ({num_downloaded_calib}); staging target again and resuming download." + f"Number of staged target MSes ({num_staged_targ}) does NOT equal number of downloaded MSes ({num_downloaded_targ}); staging target again and resuming download." ) stage_target = True else: From 467bf79243561818a393d4bc2c34481245ca17ea Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Tue, 4 Aug 2026 16:33:03 +0200 Subject: [PATCH 052/120] Fix string formatting --- flocs_processing/dags/pilot_widefield.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index ec29ca2..687a59a 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -178,18 +178,18 @@ def download_field(field): num_downloaded_calib2 = 0 num_staged_calib = 0 num_staged_targ = 0 - if os.path.exists("srms_{field['sas_id_target']}_calibrators.txt"): + if os.path.exists(f"srms_{field['sas_id_target']}_calibrators.txt"): print("Found srm file; counting calibrator SRMs.") out = subprocess.check_output( - "wc -l srms_{field['sas_id_target']}_calibrators.txt | cut -f 1 -d ' '", + f"wc -l srms_{field['sas_id_target']}_calibrators.txt | cut -f 1 -d ' '", text=True, ) num_staged_calib = int(out.strip()) - if os.path.exists("srms_{field['sas_id_target']}.txt"): + if os.path.exists(f"srms_{field['sas_id_target']}.txt"): print("Found srm file; counting target SRMs.") out = subprocess.check_output( - "wc -l srms_{field['sas_id_target']}.txt | cut -f 1 -d ' '", + f"wc -l srms_{field['sas_id_target']}.txt | cut -f 1 -d ' '", text=True, ) num_staged_targ = int(out.strip()) From 823d7bcee79725c98ad91253b855c76f3aab6c78 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Tue, 4 Aug 2026 16:35:01 +0200 Subject: [PATCH 053/120] Shell in subprocess --- flocs_processing/dags/pilot_widefield.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 687a59a..b4ca13d 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -183,6 +183,7 @@ def download_field(field): out = subprocess.check_output( f"wc -l srms_{field['sas_id_target']}_calibrators.txt | cut -f 1 -d ' '", text=True, + shell=True, ) num_staged_calib = int(out.strip()) @@ -191,6 +192,7 @@ def download_field(field): out = subprocess.check_output( f"wc -l srms_{field['sas_id_target']}.txt | cut -f 1 -d ' '", text=True, + shell=True, ) num_staged_targ = int(out.strip()) From cfaa858a380530c799465d28074dcd9be22294fb Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Tue, 4 Aug 2026 17:25:29 +0200 Subject: [PATCH 054/120] Only obtain field that are not currently processing --- flocs_processing/dags/pilot_widefield.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index b4ca13d..969e30e 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -159,7 +159,13 @@ def get_most_recent_run(searchpath: str, sas_id: str, pipeline: str) -> pathlib. def pilot_widefield(): @task def get_unprocessed_target(): - field = dict(get_db_columns()[0]) + for row in get_db_columns(): + status_keys = filter(lambda x: x.startswith("status_"), row.keys()) + for key in status_keys: + if row[key] == PIPELINE_STATUS.processing.value: + continue + # Only select a field if nothing is processing it. + field = dict(row) print(field["target_name"]) return field From 68e71670fb3e5d6c3008e327a6d56f33b5246964 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 5 Aug 2026 11:15:10 +0200 Subject: [PATCH 055/120] Wait for both calibrators to finish --- flocs_processing/dags/pilot_widefield.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 969e30e..07ff035 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -445,7 +445,7 @@ def run_linc_calibrator2(field): raise RuntimeError return field - @task(trigger_rule=TriggerRule.ONE_DONE) + @task(trigger_rule=TriggerRule.ALL_DONE) def select_best_calibrator(result1, result2): if result1["sas_id_calibrator_final"]: return result1 From 6b2adf0985215ffef261201aa640a79b7f519b15 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 5 Aug 2026 12:45:18 +0200 Subject: [PATCH 056/120] Do not return field for calibrator if processing --- flocs_processing/dags/pilot_widefield.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 07ff035..a0ee5b6 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -349,9 +349,7 @@ def download_field(field): @task def run_linc_calibrator1(field): - if (field["status_calibrator1"] == PIPELINE_STATUS.finished) or ( - field["status_calibrator1"] == PIPELINE_STATUS.processing - ): + if (field["status_calibrator1"] == PIPELINE_STATUS.finished): print( f"Flux density calibrator {field['sas_id_calibrator1']} for observation {field['target_name']} {field['sas_id_target']} already processed." ) @@ -398,9 +396,7 @@ def run_linc_calibrator1(field): @task def run_linc_calibrator2(field): - if (field["status_calibrator2"] == PIPELINE_STATUS.finished) or ( - field["status_calibrator2"] == PIPELINE_STATUS.processing - ): + if field["status_calibrator2"] == PIPELINE_STATUS.finished: print( f"Flux density calibrator {field['sas_id_calibrator2']} for observation {field['target_name']} {field['sas_id_target']} already processed." ) From 9abb427126d094707a2637b37ef0a2df30942673 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 5 Aug 2026 13:24:56 +0200 Subject: [PATCH 057/120] ruff --- flocs_processing/dags/pilot_widefield.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index a0ee5b6..995c867 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -349,7 +349,7 @@ def download_field(field): @task def run_linc_calibrator1(field): - if (field["status_calibrator1"] == PIPELINE_STATUS.finished): + if field["status_calibrator1"] == PIPELINE_STATUS.finished: print( f"Flux density calibrator {field['sas_id_calibrator1']} for observation {field['target_name']} {field['sas_id_target']} already processed." ) From 790892b02bd704b00e7de30c5eb58ea61259a1a2 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 5 Aug 2026 13:25:08 +0200 Subject: [PATCH 058/120] Initialise DB with enum values --- flocs_processing/flocs_processing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flocs_processing/flocs_processing.py b/flocs_processing/flocs_processing.py index 8aa86eb..8830f07 100644 --- a/flocs_processing/flocs_processing.py +++ b/flocs_processing/flocs_processing.py @@ -109,7 +109,7 @@ def create_database( dbstr = f"create table {table_name}(target_name text default NULL, pipelines text default '{pipeline_str}', priority int default 0, finished bit default 0, downloaded bit default 0" if "linc" in pipelines: - dbstr += ", sas_id_calibrator1 text default NULL, sas_id_calibrator2 text default NULL, sas_id_calibrator_final text default NULL, sas_id_target text primary key default NULL, status_calibrator1 smallint default 0, status_calibrator2 smallint default 0, status_target smallint default 0" + dbstr += f", sas_id_calibrator1 text default NULL, sas_id_calibrator2 text default NULL, sas_id_calibrator_final text default NULL, sas_id_target text primary key default NULL, status_calibrator1 smallint default {PIPELINE_STATUS.nothing.value}, status_calibrator2 smallint default {PIPELINE_STATUS.nothing.value}, status_target smallint default {PIPELINE_STATUS.nothing.value}" if "ddf-pipeline" in pipelines: dbstr += f", status_ddf smallint default {PIPELINE_STATUS.nothing.value}" if "vlbi-delay-widefield" in pipelines: From 05730ed4b1ad795b36ca02139aa46c532b3204fe Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 5 Aug 2026 13:29:38 +0200 Subject: [PATCH 059/120] Download products to output directory --- flocs_processing/dags/pilot_widefield.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 995c867..6fdafd1 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -555,7 +555,7 @@ def run_vlbi_delay(field): if not os.path.isfile(delay_cat): ms = list(target_ms_path.glob("*.dp3concat"))[0] - cmd = f"lofar-vlbi-plot --MS {ms}" + cmd = f"lofar-vlbi-plot --output_dir {outdir} --MS {ms}" with ( open( f"log_plot_field_{field['target_name']}_{field['sas_id_target']}.txt", From e19cf9cfe5be4ee7acd86a00f616d29a9064a393 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 5 Aug 2026 13:44:20 +0200 Subject: [PATCH 060/120] Move delay download check inside command --- flocs_processing/dags/pilot_widefield.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 6fdafd1..1a43deb 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -553,9 +553,9 @@ def run_vlbi_delay(field): delay_cat = os.path.join(outdir, "delay_calibrators.csv") image_cat = os.path.join(outdir, "image_catalogue.csv") - if not os.path.isfile(delay_cat): + if not os.path.isfile(delay_cat) or not os.path.isfile(image_cat): ms = list(target_ms_path.glob("*.dp3concat"))[0] - cmd = f"lofar-vlbi-plot --output_dir {outdir} --MS {ms}" + cmd = f"lofar-vlbi-plot --force --output_dir {outdir} --MS {ms}" with ( open( f"log_plot_field_{field['target_name']}_{field['sas_id_target']}.txt", @@ -569,17 +569,17 @@ def run_vlbi_delay(field): proc = subprocess.run( cmd, shell=True, text=True, stdout=f_out, stderr=f_err ) - if not proc.returncode: + delay_cat = os.path.join(outdir, "delay_calibrators.csv") + image_cat = os.path.join(outdir, "image_catalogue.csv") + + if not os.path.isfile(delay_cat): raise RuntimeError( - "Failed to download necessary products for delay calibration." + "Delay calibrator catalogue is missing or invalid." + ) + if not os.path.isfile(image_cat): + raise RuntimeError( + "Image source catalogue is missing or invalid." ) - delay_cat = os.path.join(outdir, "delay_calibrators.csv") - image_cat = os.path.join(outdir, "image_catalogue.csv") - - if not os.path.isfile(delay_cat): - raise RuntimeError("Delay calibrator catalogue is missing or invalid.") - if not os.path.isfile(image_cat): - raise RuntimeError("Image source catalogue is missing or invalid.") proc = subprocess.run( "detect_bad_slurm_nodes.sh", From c444c5431443e7ef8aa9acde7a637cad79c4127a Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 5 Aug 2026 13:48:52 +0200 Subject: [PATCH 061/120] Do not try to restart delay if log file not present --- flocs_processing/dags/pilot_widefield.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 1a43deb..3fc7ce2 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -593,7 +593,11 @@ def run_vlbi_delay(field): os.environ["TOIL_SLURM_ARGS"] = f"--exclude={bad_nodes}" context = get_current_context() - if context["ti"].try_number == 1: + if context["ti"].try_number == 1 or ( + not os.path.isfile( + f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt" + ) + ): cmd = f"flocs-run vlbi delay-calibration --record-toil-stats --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --ms-suffix dp3concat --delay-calibrator {delay_cat} --image-catalogue {image_cat} --apply-delay-solutions {target_ms_path}" else: # Extract the previous working directory From 9a3a57ad2203f762d812aaf6363ab3b0de45ab2d Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 5 Aug 2026 14:13:36 +0200 Subject: [PATCH 062/120] Skip calibrators if they don't exist --- flocs_processing/dags/pilot_widefield.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 3fc7ce2..57b15a7 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -11,6 +11,7 @@ from airflow.exceptions import AirflowFailException from airflow.sdk import dag, get_current_context, task from airflow.providers.standard.sensors.python import PythonSensor +from airflow.sdk.exceptions import AirflowSkipException from airflow.task.trigger_rule import TriggerRule from flocs_lta.lta_search import ObservationStager from losoto.h5parm import h5parm @@ -349,6 +350,9 @@ def download_field(field): @task def run_linc_calibrator1(field): + field = dict(get_db_columns(field["sas_id_target"])[0]) + if not field["sas_id_calibrator1"]: + raise AirflowSkipException("Calibrator 1 does not exist, skipping.") if field["status_calibrator1"] == PIPELINE_STATUS.finished: print( f"Flux density calibrator {field['sas_id_calibrator1']} for observation {field['target_name']} {field['sas_id_target']} already processed." @@ -396,6 +400,9 @@ def run_linc_calibrator1(field): @task def run_linc_calibrator2(field): + field = dict(get_db_columns(field["sas_id_target"])[0]) + if not field["sas_id_calibrator2"]: + raise AirflowSkipException("Calibrator 2 does not exist, skipping.") if field["status_calibrator2"] == PIPELINE_STATUS.finished: print( f"Flux density calibrator {field['sas_id_calibrator2']} for observation {field['target_name']} {field['sas_id_target']} already processed." From e26583a2fc4cca6fb10120e05a0cc2711427fcc6 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 5 Aug 2026 14:17:44 +0200 Subject: [PATCH 063/120] Make resumes more robust for the PILOT pipelines --- flocs_processing/dags/pilot_widefield.py | 38 ++++++++++++++++++------ 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 57b15a7..f36be28 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -789,7 +789,11 @@ def run_ddf_subtract(field): print(f"Using DDF run at: {ddf_path}") context = get_current_context() - if context["ti"].try_number == 1: + if context["ti"].try_number == 1 or ( + not os.path.isfile( + f"log_VLBI_process-ddf_{field['target_name']}_{field['sas_id_target']}.txt" + ) + ): set_status_processing( field["target_name"], "ddf_subtract", field["sas_id_target"] ) @@ -902,7 +906,11 @@ def run_vlbi_ddcal(field): ) context = get_current_context() - if context["ti"].try_number == 1: + if context["ti"].try_number == 1 or ( + not os.path.isfile( + f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt" + ) + ): cmd = f"flocs-run vlbi dd-calibration --record-toil-stats --runner toil --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat.sub.ms {target_ms_path}" else: if field["status_vlbi_dd"] == PIPELINE_STATUS.downloaded: @@ -1199,7 +1207,11 @@ def run_vlbi_image_intermediate(field): ) context = get_current_context() - if context["ti"].try_number == 1: + if context["ti"].try_number == 1 or ( + not os.path.isfile( + f"log_VLBI_image_intermediate_resolution_{field['target_name']}_{field['sas_id_target']}.txt" + ) + ): cmd = f"flocs-run vlbi image-intermediate-resolution --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --dd-solutions {dd_sols} --ms-suffix .dp3concat.sub.ms {target_ms_path}" else: if field["status_vlbi_intermediate_img"] == PIPELINE_STATUS.downloaded: @@ -1311,7 +1323,11 @@ def run_vlbi_facet_subtract(field): ) context = get_current_context() - if context["ti"].try_number == 1: + if context["ti"].try_number == 1 or ( + not os.path.isfile( + f"log_VLBI_facet-subtract_{field['target_name']}_{field['sas_id_target']}.txt" + ) + ): cmd = f"flocs-run vlbi facet-subtract --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --dd-solutions {dd_sols} --model-image-directory {model_images_path} --ms-suffix .dp3concat.sub.ms {target_ms_path}" else: if field["status_vlbi_facet_subtract"] == PIPELINE_STATUS.downloaded: @@ -1321,10 +1337,10 @@ def run_vlbi_facet_subtract(field): # Extract the previous working directory flocs_workdir = "" print( - f"Scanning log_VLBI_facet_subtract_{field['target_name']}_{field['sas_id_target']}.txt for workdir." + f"Scanning log_VLBI_facet-subtract_{field['target_name']}_{field['sas_id_target']}.txt for workdir." ) with open( - f"log_VLBI_facet_subtract_{field['target_name']}_{field['sas_id_target']}.txt" + f"log_VLBI_facet-subtract_{field['target_name']}_{field['sas_id_target']}.txt" ) as f_out: for line in f_out.readlines(): print(line) @@ -1342,11 +1358,11 @@ def run_vlbi_facet_subtract(field): print(cmd) with ( open( - f"log_VLBI_facet_subtract_{field['target_name']}_{field['sas_id_target']}.txt", + f"log_VLBI_facet-subtract_{field['target_name']}_{field['sas_id_target']}.txt", "w+", ) as f_out, open( - f"log_VLBI_facet_subtract_{field['target_name']}_{field['sas_id_target']}_err.txt", + f"log_VLBI_facet-subtract_{field['target_name']}_{field['sas_id_target']}_err.txt", "w+", ) as f_err, ): @@ -1405,7 +1421,11 @@ def run_vlbi_facet_imaging(field): ) context = get_current_context() - if context["ti"].try_number == 1: + if context["ti"].try_number == 1 or ( + not os.path.isfile( + f"log_VLBI_facet-imaging_{field['target_name']}_{field['sas_id_target']}.txt" + ) + ): cmd = f"flocs-run vlbi facet-imaging --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --resolution 0.3asec --pixel-scale 0.1 --ms-suffix .ms {target_ms_path}" else: if field["status_vlbi_facet_imaging"] == PIPELINE_STATUS.downloaded: From 1fc9622cd52578d01db07fa0f72bb368d54cdcf7 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 5 Aug 2026 14:33:21 +0200 Subject: [PATCH 064/120] Skip rest of DAG if no unprocessed fields --- flocs_processing/dags/pilot_widefield.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index f36be28..8025210 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -160,6 +160,7 @@ def get_most_recent_run(searchpath: str, sas_id: str, pipeline: str) -> pathlib. def pilot_widefield(): @task def get_unprocessed_target(): + field = None for row in get_db_columns(): status_keys = filter(lambda x: x.startswith("status_"), row.keys()) for key in status_keys: @@ -167,6 +168,8 @@ def get_unprocessed_target(): continue # Only select a field if nothing is processing it. field = dict(row) + if not field: + raise AirflowSkipException("No unprocessed fields found.") print(field["target_name"]) return field From c10963013f624dc0795e84e07b185a6eb9ed01ee Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 5 Aug 2026 14:54:40 +0200 Subject: [PATCH 065/120] Read DAG settings from a config file --- flocs_processing/dags/pilot_widefield.py | 30 ++++++++++++++---------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 8025210..2ae412d 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -1,4 +1,5 @@ from enum import Enum +import configparser import datetime import functools import os @@ -17,18 +18,23 @@ from losoto.h5parm import h5parm from stager_access import get_surls_requested, get_surls_online -# Need to replace this with a config file -TABLE_NAME = "" -DATABASE = "" -SLURM_ACCOUNT = "" -SLURM_QUEUE = "" -DATA_DIR = "" -OUTPUT_DIR = "" -PROCESSING_DIR = "" -NN_MODEL_CACHE = "" -DDF_CONFIG = "" - -NEEDS_MANUAL_APPROVAL_DELAY = True +CONFIG_FILE = os.getenv("FLOCS_AIRFLOW_CONFIG") + +parser = configparser.ConfigParser() +parser.optionxform = str +with open(CONFIG_FILE, "r") as config: + parser.read_string("[DEFAULT]\n" + config.read()) + +TABLE_NAME = parser["DEFAULT"]["TABLE_NAME"] +DATABASE = parser["DEFAULT"]["DATABASE"] +SLURM_ACCOUNT = parser["DEFAULT"]["SLURM_ACCOUNT"] +SLURM_QUEUE = parser["DEFAULT"]["SLURM_QUEUE"] +DATA_DIR = parser["DEFAULT"]["DATA_DIR"] +OUTPUT_DIR = parser["DEFAULT"]["OUTPUT_DIR"] +PROCESSING_DIR = parser["DEFAULT"]["PROCESSING_DIR"] +NN_MODEL_CACHE = parser["DEFAULT"]["NN_MODEL_CACHE"] +DDF_CONFIG = parser["DEFAULT"]["DDF_CONFIG"] +NEEDS_MANUAL_APPROVAL_DELAY = parser["DEFAULT"]["NEEDS_MANUAL_APPROVAL_DELAY"] def get_approval(field, identifier, needs_approval): From 53df96d8f904dc638006f400139bcdcab8558b74 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 5 Aug 2026 15:03:44 +0200 Subject: [PATCH 066/120] Convert row to dict --- flocs_processing/dags/pilot_widefield.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 2ae412d..61d64f1 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -167,7 +167,8 @@ def pilot_widefield(): @task def get_unprocessed_target(): field = None - for row in get_db_columns(): + for dbrow in get_db_columns(): + row = dict(dbrow) status_keys = filter(lambda x: x.startswith("status_"), row.keys()) for key in status_keys: if row[key] == PIPELINE_STATUS.processing.value: From 0d18311fc81f6a5ad415344194adee5cc7315b56 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 5 Aug 2026 15:27:31 +0200 Subject: [PATCH 067/120] Print summary of config on launch --- flocs_processing/dags/pilot_widefield.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 61d64f1..f1baa1b 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -19,12 +19,18 @@ from stager_access import get_surls_requested, get_surls_online CONFIG_FILE = os.getenv("FLOCS_AIRFLOW_CONFIG") +if not os.path.isfile(CONFIG_FILE): + raise RuntimeError(f"{CONFIG_FILE} is not a valid file") parser = configparser.ConfigParser() parser.optionxform = str with open(CONFIG_FILE, "r") as config: parser.read_string("[DEFAULT]\n" + config.read()) +print("Config summary:") +for k, v in parser["DEFAULT"].items(): + print(f"{k}: {v}") + TABLE_NAME = parser["DEFAULT"]["TABLE_NAME"] DATABASE = parser["DEFAULT"]["DATABASE"] SLURM_ACCOUNT = parser["DEFAULT"]["SLURM_ACCOUNT"] From e41f83bc1e1effcc7dd857f0904cf4ccf6fe8fe6 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 5 Aug 2026 15:34:15 +0200 Subject: [PATCH 068/120] evaluate bool eval for delay --- flocs_processing/dags/pilot_widefield.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index f1baa1b..8a40fcf 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -40,7 +40,7 @@ PROCESSING_DIR = parser["DEFAULT"]["PROCESSING_DIR"] NN_MODEL_CACHE = parser["DEFAULT"]["NN_MODEL_CACHE"] DDF_CONFIG = parser["DEFAULT"]["DDF_CONFIG"] -NEEDS_MANUAL_APPROVAL_DELAY = parser["DEFAULT"]["NEEDS_MANUAL_APPROVAL_DELAY"] +NEEDS_MANUAL_APPROVAL_DELAY = bool(parser["DEFAULT"]["NEEDS_MANUAL_APPROVAL_DELAY"]) def get_approval(field, identifier, needs_approval): From 3bb598868e0a9f8a4b1a0d7abdc9679e2fc0c47c Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 5 Aug 2026 15:49:41 +0200 Subject: [PATCH 069/120] Try to fix unprocessed fields --- flocs_processing/dags/pilot_widefield.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 8a40fcf..201bc5f 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -174,11 +174,14 @@ def pilot_widefield(): def get_unprocessed_target(): field = None for dbrow in get_db_columns(): + is_processing = False row = dict(dbrow) status_keys = filter(lambda x: x.startswith("status_"), row.keys()) for key in status_keys: if row[key] == PIPELINE_STATUS.processing.value: - continue + is_processing = True + break + if not is_processing: # Only select a field if nothing is processing it. field = dict(row) if not field: From b4d6ac7063d3268d9e81b08e0047c1f0bdf0188d Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 5 Aug 2026 15:50:20 +0200 Subject: [PATCH 070/120] Break if unprocessed field found --- flocs_processing/dags/pilot_widefield.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 201bc5f..f1fad0d 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -184,6 +184,7 @@ def get_unprocessed_target(): if not is_processing: # Only select a field if nothing is processing it. field = dict(row) + break if not field: raise AirflowSkipException("No unprocessed fields found.") print(field["target_name"]) From 51e14f9d6fbae6038e4598e58ee26340f19ef5c6 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 5 Aug 2026 17:53:32 +0200 Subject: [PATCH 071/120] Warn inside the dag if config not found --- flocs_processing/dags/pilot_widefield.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index f1fad0d..967e772 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -18,6 +18,11 @@ from losoto.h5parm import h5parm from stager_access import get_surls_requested, get_surls_online +if "FLOCS_AIRFLOW_CONFIG" not in os.environ: + raise RuntimeError( + "FLOCS_AIRFLOW_CONFIG environment variable not set. Please point this to a valid configuration file." + ) + CONFIG_FILE = os.getenv("FLOCS_AIRFLOW_CONFIG") if not os.path.isfile(CONFIG_FILE): raise RuntimeError(f"{CONFIG_FILE} is not a valid file") From 46ede5b6081bd7343f23e63213431202aa15835e Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 5 Aug 2026 17:53:54 +0200 Subject: [PATCH 072/120] Make a user-friendly frontend for the airflow setup --- flocs_processing/flocs_processing.py | 676 +-------------------------- flocs_processing/processors.py | 190 ++++++++ 2 files changed, 208 insertions(+), 658 deletions(-) create mode 100644 flocs_processing/processors.py diff --git a/flocs_processing/flocs_processing.py b/flocs_processing/flocs_processing.py index 8830f07..e5d6c51 100644 --- a/flocs_processing/flocs_processing.py +++ b/flocs_processing/flocs_processing.py @@ -1,57 +1,16 @@ #!/usr/bin/env python -from astropy.table import Table -from concurrent.futures import ProcessPoolExecutor +from .processors import FlocsAirflowProcessor from cyclopts import Parameter from enum import Enum -from rich.console import Console -from typing import Annotated import cyclopts import functools -import glob -import os -import pathlib -import re import subprocess -import sqlite3 -import threading -import time +import structlog +from typing import Optional, Annotated app = cyclopts.App() -# At some point we need to spawn these instead of running standalone -# tmux new-session -d -s airflow-api-server "bash -c 'source $HOME/source_airflow.sh && airflow api-server; exec bash'" -# tmux new-session -d -s airflow-scheduler "bash -c 'source $HOME/source_airflow.sh && airflow scheduler; exec bash'" -# tmux new-session -d -s airflow-dag-processor "bash -c 'source $HOME/source_airflow.sh && airflow dag-processor; exec bash'" -# tmux new-session -d -s airflow-triggerer "bash -c 'source $HOME/source_airflow.sh && airflow triggerer; exec bash'" - - -@functools.total_ordering -class PIPELINE(Enum): - download = 0 - linc_calibrator = 1 - linc_target = 2 - vlbi_delay = 3 - - def __eq__(self, other): - if self.__class__ is not other.__class__: - raise NotImplementedError - return self.value == other.value - - def __lt__(self, other): - if self.__class__ is not other.__class__: - raise NotImplementedError - return self.value < other.value - - def __hash__(self): - return hash(self.value) - - -PIPELINE_NAMES: dict[PIPELINE, str] = { - PIPELINE.download: "not downloaded", - PIPELINE.linc_calibrator: "LINC Calibrator", - PIPELINE.linc_target: "LINC Target", - PIPELINE.vlbi_delay: "PILOT delay calibration", -} +logger = structlog.getLogger() @functools.total_ordering @@ -74,24 +33,6 @@ def __lt__(self, other): return self.value < other.value -@functools.total_ordering -class STAGING_STATUS(Enum): - error = -1 - not_staged = 0 - in_progress = 1 - finished = 2 - - def __eq__(self, other): - if self.__class__ is not other.__class__: - raise NotImplementedError - return self.value == other.value - - def __lt__(self, other): - if self.__class__ is not other.__class__: - raise NotImplementedError - return self.value < other.value - - @app.command() def create_database( dbname: Annotated[ @@ -174,604 +115,23 @@ def add_field( @app.command() -def process_from_database( - dbname: Annotated[ - str, Parameter(help="Sqlite3 database from which processing will be done.") - ], - rundir: Annotated[ - str, +def start_airflow_processing( + airflow_cores: Annotated[ + int, Parameter(help="Number of cores to give to Airflow.") + ] = 6, + custom_airflow_home: Annotated[ + Optional[str], Parameter( - help="Directory where data is located and processing will take place." + help="Directory where Airflow stores its internal data. If given this overrides $AIRFLOW_HOME. If not given, $AIRFLOW_HOME is used to define other related variables." ), - ], - slurm_queues: Annotated[ - list[str], Parameter(help="Slurm queues that jobs can be submitted to.") - ], - slurm_account: Annotated[str, Parameter(help="Slurm account to submit under.")], - table_name: Annotated[ - str, Parameter(help="Database table that will be processed.") - ] = "processing_flocs", + ] = "", ): - fp = FlocsSlurmProcessor( - database=dbname, - slurm_queues=slurm_queues, - slurm_account=slurm_account, - table_name=table_name, - rundir=rundir, - ) - fp.start_processing_loop() - - -class FlocsSlurmProcessor: - def __init__( - self, - database: str, - slurm_queues: list, - slurm_account: str, - rundir: str, - table_name: Annotated[ - str, Parameter(help="Database table to start processing in.") - ] = "flocs_processing", - ): - self.DATABASE = database - self.SLURM_QUEUES = ",".join(slurm_queues) - self.SLURM_ACCOUNT = slurm_account - self.TABLE_NAME = table_name - self.RUNDIR = rundir - - def launch_calibrator(self, field_name, sas_id, restart: bool = False): - if not restart: - try: - cmd = f"flocs-run linc calibrator --record-toil-stats --scheduler slurm --rundir {self.RUNDIR}/{field_name}/rundir/ --outdir {self.RUNDIR}/{field_name} --slurm-queue {self.SLURM_QUEUES} --slurm-time 24:00:00 --slurm-account {self.SLURM_ACCOUNT} --runner toil --save-raw-solutions {self.RUNDIR}/{field_name}/calibrator/L{sas_id}" - print(cmd) - with open( - f"{field_name}/log_LINC_calibrator_{field_name}_{sas_id}.txt", "a" - ) as f_out, open( - f"{field_name}/log_LINC_calibrator_{field_name}_{sas_id}_err.txt", - "a", - ) as f_err: - proc = subprocess.run( - cmd, shell=True, text=True, stdout=f_out, stderr=f_err - ) - if not proc.returncode: - return True - else: - return False - except subprocess.CalledProcessError: - print("something went wrong") - else: - rundirs = pathlib.Path(f"{self.RUNDIR}/{field_name}/rundir") - rundirs_sorted = sorted(rundirs.iterdir(), key=os.path.getctime) - rundirs_sorted_filtered = [ - d for d in rundirs_sorted if sas_id in d.parts[-1] - ] - # Last directory touched for this source - rundir_final = rundirs_sorted_filtered[-1].parts[-1] - try: - cmd = f"flocs-run linc calibrator --record-toil-stats --scheduler slurm --rundir {self.RUNDIR}/{field_name}/rundir/{rundir_final} --outdir {self.RUNDIR}/{field_name} --restart --slurm-queue {self.SLURM_QUEUES} --slurm-time 24:00:00 --slurm-account {self.SLURM_ACCOUNT} --runner toil --save-raw-solutions {self.RUNDIR}/{field_name}/calibrator/L{sas_id}" - print(cmd) - with open( - f"{field_name}/log_LINC_calibrator_{field_name}_{sas_id}.txt", "a" - ) as f_out, open( - f"{field_name}/log_LINC_calibrator_{field_name}_{sas_id}_err.txt", - "a", - ) as f_err: - proc = subprocess.run( - cmd, shell=True, text=True, stdout=f_out, stderr=f_err - ) - if not proc.returncode: - return True - else: - return False - except subprocess.CalledProcessError: - print("something went wrong") - return False - - def launch_target(self, field_name, sas_id, sas_id_cal, restart: bool = False): - if not restart: - try: - cal_sol_path = glob.glob( - f"{self.RUNDIR}/{field_name}/LINC_calibrator_L{sas_id_cal}*/results_LINC_calibrator/cal_solutions.h5" - )[0] - cmd = f"flocs-run linc target --record-toil-stats --scheduler slurm --rundir {self.RUNDIR}/{field_name}/rundir/ --outdir {self.RUNDIR}/{field_name} --slurm-queue {self.SLURM_QUEUES} --slurm-time 48:00:00 --slurm-account {self.SLURM_ACCOUNT} --runner toil --output-fullres-data --min-unflagged-fraction 0.05 --cal-solutions {cal_sol_path} {self.RUNDIR}/{field_name}/target/L{sas_id}/" - print(cmd) - with open( - f"{field_name}/log_LINC_target_{field_name}_{sas_id}.txt", "w" - ) as f_out, open( - f"{field_name}/log_LINC_target_{field_name}_{sas_id}_err.txt", "w" - ) as f_err: - proc = subprocess.run( - cmd, shell=True, text=True, stdout=f_out, stderr=f_err - ) - if not proc.returncode: - pattern = re.compile(r"Workflow.* stopped. Success: False") - if pattern.search(proc.stderr): - return False - return True - else: - return False - except subprocess.CalledProcessError: - print("something went wrong") - else: - rundirs = pathlib.Path(f"{self.RUNDIR}/{field_name}/rundir") - rundirs_sorted = sorted(rundirs.iterdir(), key=os.path.getctime) - rundirs_sorted_filtered = [ - d for d in rundirs_sorted if sas_id in d.parts[-1] - ] - # Last directory touched for this source - rundir_final = rundirs_sorted_filtered[-1].parts[-1] - try: - cal_sol_path = glob.glob( - f"{self.RUNDIR}/{field_name}/LINC_calibrator_L{sas_id_cal}*/results_LINC_calibrator/cal_solutions.h5" - )[0] - cmd = f"flocs-run linc target --record-toil-stats --scheduler slurm --rundir {self.RUNDIR}/{field_name}/rundir/{rundir_final} --restart --outdir {self.RUNDIR}/{field_name} --slurm-queue {self.SLURM_QUEUES} --slurm-time 48:00:00 --slurm-account {self.SLURM_ACCOUNT} --runner toil --output-fullres-data --min-unflagged-fraction 0.05 --cal-solutions {cal_sol_path} {self.RUNDIR}/{field_name}/target/L{sas_id}/" - print(cmd) - with open( - f"{field_name}/log_LINC_target_{field_name}_{sas_id}.txt", "a" - ) as f_out, open( - f"{field_name}/log_LINC_target_{field_name}_{sas_id}_err.txt", "a" - ) as f_err: - proc = subprocess.run( - cmd, shell=True, text=True, stdout=f_out, stderr=f_err - ) - if not proc.returncode: - return True - else: - return False - return True - except subprocess.CalledProcessError: - print("something went wrong") - return False - - def launch_vlbi_delay(self, field_name, sas_id, restart: bool = False): - if not restart: - print(f"Generating input catalogue(s) for {field_name}") - rundirs = pathlib.Path(f"{self.RUNDIR}/{field_name}/") - rundirs_sorted = sorted(rundirs.iterdir(), key=os.path.getctime) - rundirs_sorted_filtered = [ - d - for d in rundirs_sorted - if ( - (sas_id in d.parts[-1]) - and (d.parts[-1].startswith(("LINC_target"))) - ) - ] - # Last LINC target reduction for this source - linc_target_dir = rundirs_sorted_filtered[-1] - first_ms = glob.glob( - f"{linc_target_dir}/results_LINC_target/results/*.dp3concat" - )[0] - - with open( - f"{rundirs}/log_VLBI_delay-calibration_plot_field_{field_name}_{sas_id}.txt", - "w", - ) as f_out, open( - f"{rundirs}/log_VLBI_delay-calibration_plot_field_{field_name}_{sas_id}_err.txt", - "w", - ) as f_err: - cmd = f"lofar-vlbi-plot --output_dir {rundirs} --MS {first_ms} --continue_no_lotss --vlass" - proc = subprocess.run( - cmd, shell=True, text=True, stdout=subprocess.PIPE - ) - if proc.returncode: - return False - delay_csv = rundirs / "delay_calibrators.csv" - if not os.path.isfile(delay_csv): - print(f"Failed to find delay_calibrators.csv for {field_name}") - return False - dc = Table.read(delay_csv) - model_image = rundirs / f"{dc[0]['Observation']}_vlass.fits" - try: - cmd = f"flocs-run vlbi delay-calibration --record-toil-stats --scheduler slurm --rundir {rundirs/'rundir'} --outdir {rundirs} --slurm-queue {self.SLURM_QUEUES} --slurm-time 48:00:00 --slurm-account {self.SLURM_ACCOUNT} --runner toil --delay-calibrator {delay_csv} --model-image {model_image} --ms-suffix dp3concat {linc_target_dir/'results_LINC_target'/'results'}" - print(cmd) - os.chdir(rundirs) - with open( - f"{rundirs}/log_VLBI_delay-calibration_{field_name}_{sas_id}.txt", - "w", - ) as f_out, open( - f"{rundirs}/log_VLBI_delay-calibration_{field_name}_{sas_id}_err.txt", - "w", - ) as f_err: - proc = subprocess.run( - cmd, shell=True, text=True, stdout=f_out, stderr=f_err - ) - if not proc.returncode: - pattern = re.compile(r"Workflow.* stopped. Success: False") - if pattern.search(proc.stderr): - return False - return True - else: - return False - except subprocess.CalledProcessError: - print("something went wrong") - return False - else: - rundirs = pathlib.Path(f"{self.RUNDIR}/{field_name}/") - rundirs_sorted = sorted(rundirs.iterdir(), key=os.path.getctime) - rundirs_sorted_filtered = [ - d - for d in rundirs_sorted - if ( - (sas_id in d.parts[-1]) - and (d.parts[-1].startswith(("LINC_target"))) - ) - ] - # Last LINC target reduction for this source - linc_target_dir = rundirs_sorted_filtered[-1] - print(f"{linc_target_dir=}") - - vlbi_rundirs = pathlib.Path(f"{self.RUNDIR}/{field_name}/rundir") - vlbi_rundirs_sorted = sorted(vlbi_rundirs.iterdir(), key=os.path.getctime) - # vlbi_rundirs_sorted_filtered = [d for d in vlbi_rundirs_sorted if ((sas_id in d.parts[-1]) and ("delay" in d.parts[-1]))] - vlbi_rundirs_sorted_filtered = [ - d for d in vlbi_rundirs_sorted if ("delay" in d.parts[-1]) - ] - vlbi_dir = vlbi_rundirs_sorted_filtered[-1] - print(f"{vlbi_dir=}") - - delay_csv = rundirs / "delay_calibrators.csv" - try: - cmd = f"flocs-run vlbi delay-calibration --record-toil-stats --scheduler slurm --rundir {vlbi_dir} --restart --outdir {self.RUNDIR}/{field_name} --slurm-queue {self.SLURM_QUEUES} --slurm-time 48:00:00 --slurm-account {self.SLURM_ACCOUNT} --runner toil --delay-calibrator {delay_csv} --ms-suffix dp3concat {linc_target_dir/'results_LINC_target/results'}" - print(cmd) - os.chdir(rundirs) - with open( - f"{rundirs}/log_VLBI_delay-calibration_{field_name}_{sas_id}.txt", - "w", - ) as f_out, open( - f"{rundirs}/log_VLBI_delay-calibration_{field_name}_{sas_id}_err.txt", - "w", - ) as f_err: - proc = subprocess.run( - cmd, shell=True, text=True, stdout=f_out, stderr=f_err - ) - if not proc.returncode: - pattern = re.compile(r"Workflow.* stopped. Success: False") - if pattern.search(proc.stderr): - return False - return True - else: - return False - except subprocess.CalledProcessError: - print("something went wrong") - return False - - def summarise_status(self): - console = Console(highlight=False) - console.print(f"General statistics for {self.DATABASE}", style="bold") - with sqlite3.connect(self.DATABASE) as db: - cursor = db.cursor() - not_started = cursor.execute( - f"select count(source_name) from {self.TABLE_NAME} where (status_calibrator1=={PIPELINE_STATUS.nothing.value} or status_calibrator2=={PIPELINE_STATUS.nothing.value})" - ).fetchall()[0][0] - downloaded = cursor.execute( - f"select count(source_name) from {self.TABLE_NAME} where (status_calibrator1=={PIPELINE_STATUS.downloaded.value} or status_calibrator2=={PIPELINE_STATUS.downloaded.value})" - ).fetchall()[0][0] - processing = cursor.execute( - f"select count(source_name) from {self.TABLE_NAME} where (status_calibrator1=={PIPELINE_STATUS.processing.value} or status_calibrator2=={PIPELINE_STATUS.processing.value})" - ).fetchall()[0][0] - finished = cursor.execute( - f"select count(source_name) from {self.TABLE_NAME} where (status_calibrator1=={PIPELINE_STATUS.finished.value} or status_calibrator2=={PIPELINE_STATUS.finished.value})" - ).fetchall()[0][0] - error = cursor.execute( - f"select count(source_name) from {self.TABLE_NAME} where (status_calibrator1=={PIPELINE_STATUS.error.value} or status_calibrator2=={PIPELINE_STATUS.error.value})" - ).fetchall()[0][0] - console.print("Flux density calibrators", style="bold") - console.print(f"= {not_started} calibrators not yet downloaded") - console.print(f"= {downloaded} calibrators downloaded", style="yellow") - console.print(f"= {processing} calibrators processing", style="cyan") - console.print(f"= {finished} calibrators finished", style="green") - console.print(f"= {error} calibrators failed", style="red") - - not_started = cursor.execute( - f"select count(source_name) from {self.TABLE_NAME} where status_target=={PIPELINE_STATUS.nothing.value}" - ).fetchall()[0][0] - downloaded = cursor.execute( - f"select count(source_name) from {self.TABLE_NAME} where status_target=={PIPELINE_STATUS.downloaded.value}" - ).fetchall()[0][0] - finished = cursor.execute( - f"select count(source_name) from {self.TABLE_NAME} where status_target=={PIPELINE_STATUS.finished.value}" - ).fetchall()[0][0] - processing = cursor.execute( - f"select count(source_name) from {self.TABLE_NAME} where status_target=={PIPELINE_STATUS.processing.value}" - ).fetchall()[0][0] - error = cursor.execute( - f"select count(source_name) from {self.TABLE_NAME} where status_target=={PIPELINE_STATUS.error.value}" - ).fetchall()[0][0] - console.print("\nFields / science targets", style="bold") - console.print(f"= {not_started} targets not yet downloaded") - console.print(f"= {downloaded} targets downloaded", style="yellow") - console.print(f"= {processing} targets processing", style="cyan") - console.print(f"= {finished} targets finished", style="green") - console.print(f"= {error} targets failed", style="red") - - def update_db_statuses(self, running_fields): - if not running_fields: - return - print("== UPDATING DB STATUSES ==") - console = Console(highlight=False) - futures = running_fields.keys() - to_delete = set() - for future in futures: - field = running_fields[future] - console.print(f"[bold]Field: {field['name']}[/bold]") - console.print(f"Target SAS ID: {field['sasid']}") - console.print(f"Pipeline: {PIPELINE_NAMES[field['pipeline']]}") - if future.cancelled(): - console.print("Status: [bold yellow]cancelled[/bold yellow]") - del running_fields[future] - elif future.done(): - if future.exception(timeout=10): - console.print("Status: [bold red]failed[/bold red]") - print( - f"Processing {field['identifier']} for {field['name']} failed." - ) - print("Error was: ", future.exception(timeout=10)) - with sqlite3.connect(self.DATABASE) as db: - cursor = db.cursor() - cursor.execute( - f"update {self.TABLE_NAME} set status_{field['identifier']}={PIPELINE_STATUS.error.value} where source_name=='{field['name']}'" - ) - else: - result = future.result() - print(f"Result was {result}") - with sqlite3.connect(self.DATABASE) as db: - cursor = db.cursor() - if result: - console.print("Status: [bold green]finished[/bold green]") - cursor.execute( - f"update {self.TABLE_NAME} set status_{field['identifier']}={PIPELINE_STATUS.finished.value} where source_name=='{field['name']}'" - ) - if field["identifier"] == "target": - cursor.execute( - f"update {self.TABLE_NAME} set status_delay={PIPELINE_STATUS.downloaded.value} where source_name=='{field['name']}'" - ) - else: - console.print("Status: [bold red]failed[/bold red]") - cursor.execute( - f"update {self.TABLE_NAME} set status_{field['identifier']}={PIPELINE_STATUS.error.value} where source_name=='{field['name']}'" - ) - to_delete.add(future) - else: - console.print("Status: [bold cyan]running[/bold cyan]\n") - for f in to_delete: - print(f"Deleting future for {field['name']}") - del running_fields[f] - print("== UPDATING DB STATUSES FINISHED") - - def get_not_started(self, identifier: str): - not_started = self.get_db_columns(identifier, PIPELINE_STATUS.downloaded) - return not_started - - def get_failed(self, identifier: str): - restart = self.get_db_columns(identifier, PIPELINE_STATUS.error) - return restart - - def get_db_columns(self, identifier: str, status: PIPELINE_STATUS): - with sqlite3.connect(self.DATABASE) as db: - cursor = db.cursor() - if "calibrator" in identifier: - columns = "source_name,sas_id_calibrator1,sas_id_calibrator2,sas_id_calibrator_final,sas_id_target" - elif "target" in identifier: - columns = "source_name,sas_id_calibrator_final,sas_id_target" - elif "delay" in identifier: - columns = "source_name,sas_id_target" - else: - columns = "*" - restart = cursor.execute( - f"select {columns} from {self.TABLE_NAME} where status_{identifier}=={status.value}" - ).fetchall() - return restart - - def is_processing(self, name, running_fields): - return name in [v["name"] for f, v in running_fields.items()] - - def set_status_processing(self, name, identifier, target): - with sqlite3.connect(self.DATABASE) as db: - cursor = db.cursor() - cursor.execute( - f"update {self.TABLE_NAME} set status_{identifier}={PIPELINE_STATUS.processing.value} where source_name=='{name}' and sas_id_target=='{target}'" - ) - - def check_fields_linc_calibrator(self, running_fields, tpe): - restart1 = self.get_failed("calibrator1") - restart2 = self.get_failed("calibrator2") - if restart1: - for name, cal1, cal2, cal_final, target in restart1: - if ( - not self.is_processing(name, running_fields) - and self.is_accepting_jobs - ): - print( - f"Re-starting LINC calibrator for calibrator 1 of field {name}" - ) - future = tpe.submit( - self.launch_calibrator, name, cal1, restart=True - ) - running_fields[future] = { - "name": name, - "pipeline": PIPELINE.linc_calibrator, - "identifier": "calibrator1", - "sasid": target, - } - self.set_status_processing(name, "calibrator1", target) - if restart2: - for name, cal1, cal2, cal_final, target in restart2: - if ( - not self.is_processing(name, running_fields) - and self.is_accepting_jobs - ): - print( - f"Re-starting LINC calibrator for calibrator 2 of field {name}" - ) - future = tpe.submit( - self.launch_calibrator, name, cal2, restart=True - ) - running_fields[future] = { - "name": name, - "pipeline": PIPELINE.linc_calibrator, - "identifier": "calibrator2", - "sasid": target, - } - self.set_status_processing(name, "calibrator2", target) - - not_started1 = self.get_not_started("calibrator1") - not_started2 = self.get_not_started("calibrator2") - if not_started1: - for name, cal1, cal2, cal_final, target in not_started1: - if ( - not self.is_processing(name, running_fields) - and self.is_accepting_jobs - ): - print( - f"Re-starting LINC calibrator for calibrator 1 of field {name}" - ) - future = tpe.submit(self.launch_calibrator, name, cal1) - running_fields[future] = { - "name": name, - "pipeline": PIPELINE.linc_calibrator, - "identifier": "calibrator1", - "sasid": target, - } - self.set_status_processing(name, "calibrator1", target) - if not_started2: - for name, cal1, cal2, cal_final, target in not_started2: - if ( - not self.is_processing(name, running_fields) - and self.is_accepting_jobs - ): - print( - f"Re-starting LINC calibrator for calibrator 2 of field {name}" - ) - future = tpe.submit(self.launch_calibrator, name, cal2) - running_fields[future] = { - "name": name, - "pipeline": PIPELINE.linc_calibrator, - "identifier": "calibrator2", - "sasid": target, - } - self.set_status_processing(name, "calibrator2", target) - - def check_fields_linc_target(self, running_fields, tpe): - restart = self.get_failed("target") - if restart: - for name, cal_final, target in restart: - if ( - not self.is_processing(name, running_fields) - and self.is_accepting_jobs - ): - print(f"Re-starting LINC target for field {name}") - future = tpe.submit( - self.launch_target, - name, - target, - cal_final, - restart=True, - ) - running_fields[future] = { - "name": name, - "pipeline": PIPELINE.linc_target, - "sasid": target, - "identifier": "target", - } - self.set_status_processing(name, "target", target) - self.is_accepting_jobs = len(running_fields) < self.MAX_RUNNING - print(f"Launched {name}") - - not_started = self.get_not_started("target") - if not_started: - for name, cal_final, target in not_started: - if ( - not self.is_processing(name, running_fields) - and self.is_accepting_jobs - ): - print(f"Starting LINC target for field {name}") - future = tpe.submit(self.launch_target, name, target, cal_final) - running_fields[future] = { - "name": name, - "pipeline": PIPELINE.linc_target, - "sasid": target, - "identifier": "target", - } - self.set_status_processing(name, "target", target) - self.is_accepting_jobs = len(running_fields) < self.MAX_RUNNING - print(f"Launched {name}") - - def check_fields_vlbi_delay(self, running_fields, tpe): - restart = self.get_failed("delay") - if restart: - for name, target in restart: - if ( - not self.is_processing(name, running_fields) - and self.is_accepting_jobs - ): - print(f"Re-starting VLBI delay for field {name}") - future = tpe.submit( - self.launch_vlbi_delay, - name, - target, - restart=True, - ) - running_fields[future] = { - "name": name, - "pipeline": PIPELINE.vlbi_delay, - "sasid": target, - "identifier": "delay", - } - self.set_status_processing(name, "delay", target) - print(f"Launched {name}") - - not_started = self.get_not_started("delay") - if not_started: - for name, target in not_started: - if ( - not self.is_processing(name, running_fields) - and self.is_accepting_jobs - ): - print(f"Starting VLBI delay for field {name}") - future = tpe.submit(self.launch_vlbi_delay, name, target) - running_fields[future] = { - "name": name, - "pipeline": PIPELINE.vlbi_delay, - "sasid": target, - "identifier": "delay", - } - self.set_status_processing(name, "delay", target) - print(f"Launched {name}") - - def start_processing_loop(self, allow_up_to=PIPELINE.linc_calibrator): - print("Starting processing loop") - allow_up_to = PIPELINE.vlbi_delay - self.MAX_RUNNING = 3 - max_noqueue = 5 - noqueue = 0 - lock = threading.RLock() - with ProcessPoolExecutor(max_workers=self.MAX_RUNNING + 1) as tpe: - running_fields = {} - - while True: - if len(running_fields) < 1: - noqueue += 1 - else: - noqueue = 0 - self.summarise_status() - if noqueue >= max_noqueue: - print( - f"No new jobs added in queue for {max_noqueue * 60} s, quitting processing loop." - ) - break - self.is_accepting_jobs = len(running_fields) < self.MAX_RUNNING - if allow_up_to >= PIPELINE.linc_calibrator: - with lock: - self.check_fields_linc_calibrator(running_fields, tpe) - if allow_up_to >= PIPELINE.linc_target: - with lock: - self.check_fields_linc_target(running_fields, tpe) - if allow_up_to >= PIPELINE.vlbi_delay: - with lock: - self.check_fields_vlbi_delay(running_fields, tpe) - with lock: - self.update_db_statuses(running_fields) - time.sleep(60) + fp = FlocsAirflowProcessor(custom_airflow_home, airflow_cores) + fp.generate_jwt_secret() + fp.check_environment() + fp.check_airflow_init() + fp.write_source_file() + fp.deploy_airflow_tmux() def main(): diff --git a/flocs_processing/processors.py b/flocs_processing/processors.py new file mode 100644 index 0000000..467b16c --- /dev/null +++ b/flocs_processing/processors.py @@ -0,0 +1,190 @@ +import os +import shlex +import subprocess +import sys +from pathlib import Path +from typing import Optional + +import libtmux +import structlog + +logger = structlog.getLogger() + + +class FlocsAirflowProcessor: + def __init__(self, airflow_home: Optional[str], airflow_cores: int): + os.environ["AIRFLOW__CORE__LOAD_EXAMPLES"] = "False" + os.environ["AIRFLOW__CORE__PARALLELISM"] = str(airflow_cores) + self.REQUIRED_AIRFLOW_VARS = [ + "AIRFLOW_HOME", + "AIRFLOW__CORE__DAGS_FOLDER", + "AIRFLOW__CORE__LOAD_EXAMPLES", + "AIRFLOW__CORE__PARALLELISM", + "AIRFLOW__LOGGING__DAG_PROCESSOR_CHILD_PROCESS_LOG_DIRECTORY", + "AIRFLOW__CORE__PLUGINS_FOLDER", + "AIRFLOW__DATABASE__SQL_ALCHEMY_CONN", + "AIRFLOW__LOGGING__BASE_LOG_FOLDER", + ] + if airflow_home: + os.environ["AIRFLOW_HOME"] = airflow_home + os.environ["AIRFLOW__CORE__DAGS_FOLDER"] = os.path.join( + airflow_home, "dags" + ) + os.environ[ + "AIRFLOW__LOGGING__DAG_PROCESSOR_CHILD_PROCESS_LOG_DIRECTORY" + ] = os.path.join(airflow_home, "logs/dag_processor") + os.environ["AIRFLOW__CORE__PLUGINS_FOLDER"] = os.path.join( + airflow_home, "plugins" + ) + os.environ["AIRFLOW__DATABASE__SQL_ALCHEMY_CONN"] = ( + "sqlite:///$AIRFLOW_HOME/airflow.db" + ) + os.environ["AIRFLOW__LOGGING__BASE_LOG_FOLDER"] = os.path.join( + airflow_home, "logs" + ) + + def check_airflow_init(self): + """Check if the Airflow instance is initialised already. If not, runs `airflow db migrate` to intitialise it.""" + af_home = os.environ["AIRFLOW_HOME"] + if not os.path.isdir(af_home): + logger.info(f"Airflow home {af_home} does not exist, creating it") + os.makedirs(af_home) + if not os.listdir(af_home): + logger.info( + f"Airflow home {af_home} is empty. Running `airflow db migrate` to initialise." + ) + old_dir = os.getcwd() + os.chdir(af_home) + _ = subprocess.check_output( + "airflow db migrate", + text=True, + shell=True, + ) + os.chdir(old_dir) + else: + logger.info( + f"Airflow home {af_home} is populated. Leaving it alone and assuming it is operational." + ) + + def check_environment(self): + environment_ok = True + if "FLOCS_AIRFLOW_CONFIG" not in os.environ: + logger.warning( + "FLOCS_AIRFLOW_CONFIG environment variable not set. Please point this to a valid configuration file." + ) + environment_ok = False + else: + CONFIG_FILE = os.getenv("FLOCS_AIRFLOW_CONFIG") or "" + if not os.path.isfile(CONFIG_FILE): + logger.warning(f"{CONFIG_FILE} is not a valid file") + environment_ok = False + else: + logger.info(f"Found config file {os.getenv('FLOCS_AIRFLOW_CONFIG')}.") + + if "AIRFLOW__API_AUTH__JWT_SECRET" not in os.environ: + logger.critical("AIRFLOW__API_AUTH__JWT_SECRET not defined.") + environment_ok = False + + for kw in self.REQUIRED_AIRFLOW_VARS: + if kw not in os.environ: + logger.warning(f"Required variable {kw} is not set.") + environment_ok = False + + if environment_ok: + logger.info("Flocs automated processing environment appears ok") + else: + logger.critical( + "Flocs automated processing environment is not properly set up. Please see warnings." + ) + sys.exit(1) + + def check_airflow_services(self): + status = { + "api-server": False, + "dag-processor": False, + "scheduler": False, + "triggerer": False, + } + + tmux_server = libtmux.Server() + airflow_sessions = map( + lambda x: x.name, + filter(lambda x: "airflow" in x.name, tmux_server.sessions), + ) + logger.info("Checking Airflow status:") + for service in status.keys(): + if f"airflow-{service}" not in airflow_sessions: + logger.info(f"-- {service}: not running") + status[service] = False + else: + logger.info(f"-- {service}: running") + status[service] = True + return status + + def deploy_airflow_tmux(self): + af_status = self.check_airflow_services() + + if any(not up for up in af_status.values()): + source_file = os.path.abspath("source_flocs_airflow.sh") + logger.info("Not all Airflow services are online.") + tmux_server = libtmux.Server() + for service, running in af_status.items(): + if not running: + logger.info(f"-- deploying {service}") + tmux_server.new_session( + session_name=f"airflow-{service}", + attach=False, + window_command=( + f"bash -c 'source {source_file} && airflow api-server'; bash -i" + ), + ) + + def generate_jwt_secret(self): + if os.path.isfile(os.path.expandvars("$HOME/.config/airflow/jwt_secret")): + logger.info("JWT secret found, not (re)generating.") + else: + logger.info("No JWT secret found, generating one.") + _ = subprocess.check_output( + "mkdir -p $HOME/.config/airflow", + text=True, + shell=True, + ) + + _ = subprocess.check_output( + "chmod 700 $HOME/.config/airflow", + text=True, + shell=True, + ) + + _ = subprocess.check_output( + "openssl rand -hex 32 > $HOME/.config/airflow/jwt_secret", + text=True, + shell=True, + ) + + _ = subprocess.check_output( + "chmod 600 $HOME/.config/airflow/jwt_secret", + text=True, + shell=True, + ) + logger.info("JWT secret generated") + secret = Path.home().joinpath(".config/airflow/jwt_secret").read_text().strip() + os.environ["AIRFLOW__API_AUTH__JWT_SECRET"] = secret + + def write_source_file(self): + if os.path.exists("source_flocs_airflow.sh"): + logger.info("Found source_flocs_airflow.sh, not overwriting.") + else: + logger.info("Writing environment settings to source_flocs_airflow.sh") + with open("source_flocs_airflow.sh", "w") as f: + for kw in self.REQUIRED_AIRFLOW_VARS: + f.write(f"export {kw}={os.environ[kw]}\n") + + secret = ( + Path.home() + .joinpath(".config/airflow/jwt_secret") + .read_text() + .strip() + ) + f.write(f"export AIRFLOW__API_AUTH__JWT_SECRET={shlex.quote(secret)}\n") + os.chmod("source_flocs_airflow.sh", 0o600) From 478f3720e94411d631ff9832501e5d6323268011 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 5 Aug 2026 19:03:45 +0200 Subject: [PATCH 073/120] Actually launch different services --- flocs_processing/processors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flocs_processing/processors.py b/flocs_processing/processors.py index 467b16c..f86e867 100644 --- a/flocs_processing/processors.py +++ b/flocs_processing/processors.py @@ -135,7 +135,7 @@ def deploy_airflow_tmux(self): session_name=f"airflow-{service}", attach=False, window_command=( - f"bash -c 'source {source_file} && airflow api-server'; bash -i" + f"bash -c 'source {source_file} && airflow {service}'; bash -i" ), ) From 05aefc4986e126bb4a8b394044199cb0c3b006d2 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Thu, 6 Aug 2026 14:27:34 +0200 Subject: [PATCH 074/120] Add calibrator selection based on Tim's code --- flocs_processing/dags/pilot_widefield.py | 75 +++++++++++++++++++++--- 1 file changed, 67 insertions(+), 8 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 967e772..8627109 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -4,6 +4,7 @@ import functools import os import pathlib +import random import re import sqlite3 import subprocess @@ -15,6 +16,7 @@ from airflow.sdk.exceptions import AirflowSkipException from airflow.task.trigger_rule import TriggerRule from flocs_lta.lta_search import ObservationStager +from ilotss.assess_calibrators import assess_and_compare from losoto.h5parm import h5parm from stager_access import get_surls_requested, get_surls_online @@ -45,6 +47,7 @@ PROCESSING_DIR = parser["DEFAULT"]["PROCESSING_DIR"] NN_MODEL_CACHE = parser["DEFAULT"]["NN_MODEL_CACHE"] DDF_CONFIG = parser["DEFAULT"]["DDF_CONFIG"] +FLUX_CALIBRATOR_TEMPLATE = parser["DEFAULT"]["FLUX_CALIBRATOR_TEMPLATE"] NEEDS_MANUAL_APPROVAL_DELAY = bool(parser["DEFAULT"]["NEEDS_MANUAL_APPROVAL_DELAY"]) @@ -480,14 +483,70 @@ def select_best_calibrator(result1, result2): elif result2["sas_id_calibrator_final"]: return result2 elif result1 and result2: - print("Selecting between cal1 and cal2") - # Need actual selection logic here - set_final_calibrator( - result1["target_name"], - result1["sas_id_target"], - result1["sas_id_calibrator1"], - ) - return result1 + cal_template = pathlib.Path(FLUX_CALIBRATOR_TEMPLATE) + if not cal_template.is_file(): + cal = random.randint(1, 2) + print( + f"No flux density calibrator template found. Randomly selected calibrator{cal}" + ) + if cal == 1: + set_final_calibrator( + result1["target_name"], + result1["sas_id_target"], + result1["sas_id_calibrator1"], + ) + return result1 + elif cal == 2: + set_final_calibrator( + result2["target_name"], + result2["sas_id_target"], + result2["sas_id_calibrator2"], + ) + return result2 + else: + outdir = os.path.join(OUTPUT_DIR, result1["target_name"]) + calibrator1_path = get_most_recent_run( + outdir, result1["sas_id_calibrator1"], "LINC_calibrator" + ) + calibrator1_solutions = ( + calibrator1_path / "results_LINC_calibrator" / "cal_solutions.h5" + ) + + calibrator2_path = get_most_recent_run( + outdir, result2["sas_id_calibrator2"], "LINC_calibrator" + ) + calibrator2_solutions = ( + calibrator2_path / "results_LINC_calibrator" / "cal_solutions.h5" + ) + assess_cal1 = assess_and_compare( + FLUX_CALIBRATOR_TEMPLATE, + [calibrator1_solutions], + ) + assess_cal2 = assess_and_compare( + FLUX_CALIBRATOR_TEMPLATE, + [calibrator2_solutions], + ) + score1 = assess_cal1[0]["score"] + score2 = assess_cal2[0]["score"] + print(f"Calibrator 1 score: {score1}") + print(f"Calibrator 2 score: {score2}") + match score1 < score2: + case True: + print("Best score for calibrator1") + set_final_calibrator( + result1["target_name"], + result1["sas_id_target"], + result1["sas_id_calibrator1"], + ) + return result1 + case False: + print("Best score for calibrator2") + set_final_calibrator( + result2["target_name"], + result2["sas_id_target"], + result2["sas_id_calibrator2"], + ) + return result2 elif result1 and (not result2): print("Only cal 1 succeeded, continuing with that") set_final_calibrator( From 29108eb4c097f0d7bc0e68b9ca0daa3ebfd7116a Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Thu, 6 Aug 2026 14:30:07 +0200 Subject: [PATCH 075/120] Update readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index b71f806..0001a6f 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ This package aims to provide relatively simple end-to-end automatic processing of ILT HBA data. Where `flocs-runners` provides the interface to running pipelines, `flocs-processing` is the scaffolding to tie it together. Data reduction is coordinated via a dedicated SQLite database that holds information on which observations to process, which pipelines to run for them and all of the related statuses. Orchestration of all the pipelines is handled via Airflow through a DAG. +The autoPILOT package (https://github.com/LOFAR-VLBI/autoPILOT) needs to be on PYTHONPATH to enable the automatic calibrator assessment. + ## Folder setup Flocs-processing requires three folders to be setup: From 033cb659b1fc52bc0a5cff4cc5531c0793eeabf1 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Thu, 6 Aug 2026 15:12:42 +0200 Subject: [PATCH 076/120] Mark ddf pipeline as finished in database --- flocs_processing/dags/pilot_widefield.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 8627109..cdb7b44 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -779,6 +779,9 @@ def run_ddf_pipeline(field): if (status == "RUNNING") or (status == "PENDING"): time.sleep(60) elif status == "COMPLETED": + set_status_finished( + field["target_name"], "ddf_subtract", field["sas_id_target"] + ) return field elif (status == "FAILED") or ("TIMEOUT" in status): raise RuntimeError( @@ -801,7 +804,7 @@ def run_ddf_pipeline(field): ) target_ms_path = target_path / "results_LINC_target" / "results" - cmd = f"flocs-run ddf-pipeline --scheduler slurm --slurm-time 72:00:00 --slurm-cores 32 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {OUTPUT_DIR} --config-file {DDF_CONFIG} {target_ms_path}" + cmd = f"flocs-run ddf-pipeline --scheduler slurm --slurm-time 72:00:00 --slurm-cores 32 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --config-file {DDF_CONFIG} {target_ms_path}" print(cmd) set_status_processing(field["target_name"], "ddf", field["sas_id_target"]) with ( @@ -838,6 +841,11 @@ def run_ddf_pipeline(field): if (status == "RUNNING") or (status == "PENDING"): time.sleep(60) elif status == "COMPLETED": + set_status_finished( + field["target_name"], + "ddf_subtract", + field["sas_id_target"], + ) break elif ( (status == "FAILED") @@ -851,9 +859,8 @@ def run_ddf_pipeline(field): @task def run_ddf_subtract(field): - if (field["status_vlbi_ddf_subtract"] == PIPELINE_STATUS.finished) or ( - field["status_vlbi_ddf_subtract"] == PIPELINE_STATUS.processing - ): + field = dict(get_db_columns(field["sas_id_target"])[0]) + if field["status_vlbi_ddf_subtract"] == PIPELINE_STATUS.finished: return field else: print( From 16dff1f967beedcd1a6d11a08bffb82878aefc87 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Thu, 6 Aug 2026 16:14:54 +0200 Subject: [PATCH 077/120] Implement both cwltool and toil for linc calibrator --- flocs_processing/dags/pilot_widefield.py | 186 ++++++++++++++--------- 1 file changed, 112 insertions(+), 74 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index cdb7b44..f32bf71 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -50,6 +50,106 @@ FLUX_CALIBRATOR_TEMPLATE = parser["DEFAULT"]["FLUX_CALIBRATOR_TEMPLATE"] NEEDS_MANUAL_APPROVAL_DELAY = bool(parser["DEFAULT"]["NEEDS_MANUAL_APPROVAL_DELAY"]) +CWL_RUNNER_LINC = "cwltool" + + +def run_linc_calibrator_cwltool(field, calibrator_field: int): + print( + f"Processing flux density calibrator {field[f'sas_id_calibrator{calibrator_field}']} for observation {field['target_name']} {field['sas_id_target']}" + ) + ms_folder = f"L{field[f'sas_id_calibrator{calibrator_field}']}" + set_status_processing( + field["target_name"], f"calibrator{calibrator_field}", field["sas_id_target"] + ) + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + cmd = f"flocs-run linc calibrator --runner cwltool --scheduler slurm --slurm-cores 32 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} {os.path.join(DATA_DIR, field['target_name'], 'calibrator', ms_folder)}" + with ( + open( + f"log_LINC_calibrator_{field['target_name']}_{field[f'sas_id_calibrator{calibrator_field}']}.txt", + "w+", + ) as f_out, + open( + f"log_LINC_calibrator_{field['target_name']}_{field[f'sas_id_calibrator{calibrator_field}']}_err.txt", + "w+", + ) as f_err, + ): + proc = subprocess.run(cmd, shell=True, text=True, stdout=f_out, stderr=f_err) + jobid = None + if not proc.returncode: + f_out.seek(0) + for line in f_out.readlines(): + if "Submitted batch job" in line: + jobid = line.strip().split()[-1] + else: + raise RuntimeError("Failed to submit job.") + + if not jobid: + raise RuntimeError("Failed to retrieve job id") + else: + while True: + print(f"Polling LINC calibrator job {jobid}") + poll_cmd = f"sacct -X -j {jobid} --format=State --noheader" + status = subprocess.run( + poll_cmd, shell=True, text=True, capture_output=True + ).stdout.strip() + if (status == "RUNNING") or (status == "PENDING"): + time.sleep(60) + elif status == "COMPLETED": + set_status_finished( + field["target_name"], + f"calibrator{calibrator_field}", + field["sas_id_target"], + ) + break + elif ( + (status == "FAILED") + or ("TIMEOUT" in status) + or ("CANCELLED" in status) + ): + raise RuntimeError( + f"LINC calibrator for {field['target_name']} {field['sas_id_target']} failed." + ) + + +def run_linc_calibrator_toil(field, calibrator_field: int): + print( + f"Processing flux density calibrator {field[f'sas_id_calibrator{calibrator_field}']} for observation {field['target_name']} {field['sas_id_target']}" + ) + ms_folder = f"L{field[f'sas_id_calibrator{calibrator_field}']}" + set_status_processing( + field["target_name"], f"calibrator{calibrator_field}", field["sas_id_target"] + ) + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + cmd = f"flocs-run linc calibrator --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} {os.path.join(DATA_DIR, field['target_name'], 'calibrator', ms_folder)}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with ( + open( + f"log_LINC_calibrator_{field['target_name']}_{field[f'sas_id_calibrator{calibrator_field}']}.txt", + "w+", + ) as f_out, + open( + f"log_LINC_calibrator_{field['target_name']}_{field[f'sas_id_calibrator{calibrator_field}']}_err.txt", + "w+", + ) as f_err, + ): + proc = subprocess.run(cmd, shell=True, text=True, stdout=f_out, stderr=f_err) + success = False + pattern = re.compile(r"Workflow.* stopped. Success: True") + if not proc.returncode: + f_err.seek(0) + if pattern.search(f_err.read()): + success = True + if success: + set_status_finished( + field["target_name"], + f"calibrator{calibrator_field}", + field["sas_id_target"], + ) + else: + raise RuntimeError + def get_approval(field, identifier, needs_approval): if not needs_approval: @@ -387,43 +487,12 @@ def run_linc_calibrator1(field): ) return field else: - print( - f"Processing flux density calibrator {field['sas_id_calibrator1']} for observation {field['target_name']} {field['sas_id_target']}" - ) - ms_folder = f"L{field['sas_id_calibrator1']}" - set_status_processing( - field["target_name"], "calibrator1", field["sas_id_target"] - ) - outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - cmd = f"flocs-run linc calibrator --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} {os.path.join(DATA_DIR, field['target_name'], 'calibrator', ms_folder)}" - if not os.path.isdir(outdir): - os.mkdir(outdir) - print(cmd) - with ( - open( - f"log_LINC_calibrator_{field['target_name']}_{field['sas_id_calibrator1']}.txt", - "w+", - ) as f_out, - open( - f"log_LINC_calibrator_{field['target_name']}_{field['sas_id_calibrator1']}_err.txt", - "w+", - ) as f_err, - ): - proc = subprocess.run( - cmd, shell=True, text=True, stdout=f_out, stderr=f_err - ) - success = False - pattern = re.compile(r"Workflow.* stopped. Success: True") - if not proc.returncode: - f_err.seek(0) - if pattern.search(f_err.read()): - success = True - if success: - set_status_finished( - field["target_name"], "calibrator1", field["sas_id_target"] - ) - else: - raise RuntimeError + if CWL_RUNNER_LINC == "cwltool": + run_linc_calibrator_cwltool(field, calibrator_field=1) + elif CWL_RUNNER_LINC == "toil": + run_linc_calibrator_toil(field, calibrator_field=1) + else: + raise RuntimeError("Invalid CWL runner specified.") return field @task @@ -437,43 +506,12 @@ def run_linc_calibrator2(field): ) return field else: - print( - f"Processing flux density calibrator {field['sas_id_calibrator2']} for observation {field['target_name']} {field['sas_id_target']}" - ) - ms_folder = f"L{field['sas_id_calibrator2']}" - set_status_processing( - field["target_name"], "calibrator2", field["sas_id_target"] - ) - outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - cmd = f"flocs-run linc calibrator --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} {os.path.join(DATA_DIR, field['target_name'], 'calibrator', ms_folder)}" - if not os.path.isdir(outdir): - os.mkdir(outdir) - print(cmd) - with ( - open( - f"log_LINC_calibrator_{field['target_name']}_{field['sas_id_calibrator2']}.txt", - "w+", - ) as f_out, - open( - f"log_LINC_calibrator_{field['target_name']}_{field['sas_id_calibrator2']}_err.txt", - "w+", - ) as f_err, - ): - proc = subprocess.run( - cmd, shell=True, text=True, stdout=f_out, stderr=f_err - ) - success = False - pattern = re.compile(r"Workflow.* stopped. Success: True") - if not proc.returncode: - f_err.seek(0) - if pattern.search(f_err.read()): - success = True - if success: - set_status_finished( - field["target_name"], "calibrator2", field["sas_id_target"] - ) - else: - raise RuntimeError + if CWL_RUNNER_LINC == "cwltool": + run_linc_calibrator_cwltool(field, calibrator_field=1) + elif CWL_RUNNER_LINC == "toil": + run_linc_calibrator_toil(field, calibrator_field=1) + else: + raise RuntimeError("Invalid CWL runner specified.") return field @task(trigger_rule=TriggerRule.ALL_DONE) From 175081085902bd61aca741aed3bbbf2871c4a7f2 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Thu, 6 Aug 2026 16:41:24 +0200 Subject: [PATCH 078/120] Run calibrator 2 in 2... --- flocs_processing/dags/pilot_widefield.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index f32bf71..7a7a633 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -507,9 +507,9 @@ def run_linc_calibrator2(field): return field else: if CWL_RUNNER_LINC == "cwltool": - run_linc_calibrator_cwltool(field, calibrator_field=1) + run_linc_calibrator_cwltool(field, calibrator_field=2) elif CWL_RUNNER_LINC == "toil": - run_linc_calibrator_toil(field, calibrator_field=1) + run_linc_calibrator_toil(field, calibrator_field=2) else: raise RuntimeError("Invalid CWL runner specified.") return field From 22b39d361166f7e6dc817323945346a96ce750ff Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Thu, 6 Aug 2026 17:15:35 +0200 Subject: [PATCH 079/120] Move DB logic to submodule --- flocs_processing/dags/pilot_widefield.py | 207 +++++++---------------- 1 file changed, 62 insertions(+), 145 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 7a7a633..6030d24 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -1,7 +1,7 @@ -from enum import Enum +from flocs_processing.db_utils import PIPELINE_STATUS, FlocsDB + import configparser import datetime -import functools import os import pathlib import random @@ -52,13 +52,15 @@ CWL_RUNNER_LINC = "cwltool" +CURRENT_DB = FlocsDB(DATABASE, TABLE_NAME) + def run_linc_calibrator_cwltool(field, calibrator_field: int): print( f"Processing flux density calibrator {field[f'sas_id_calibrator{calibrator_field}']} for observation {field['target_name']} {field['sas_id_target']}" ) ms_folder = f"L{field[f'sas_id_calibrator{calibrator_field}']}" - set_status_processing( + CURRENT_DB.set_status_processing( field["target_name"], f"calibrator{calibrator_field}", field["sas_id_target"] ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) @@ -95,7 +97,7 @@ def run_linc_calibrator_cwltool(field, calibrator_field: int): if (status == "RUNNING") or (status == "PENDING"): time.sleep(60) elif status == "COMPLETED": - set_status_finished( + CURRENT_DB.set_status_finished( field["target_name"], f"calibrator{calibrator_field}", field["sas_id_target"], @@ -116,7 +118,7 @@ def run_linc_calibrator_toil(field, calibrator_field: int): f"Processing flux density calibrator {field[f'sas_id_calibrator{calibrator_field}']} for observation {field['target_name']} {field['sas_id_target']}" ) ms_folder = f"L{field[f'sas_id_calibrator{calibrator_field}']}" - set_status_processing( + CURRENT_DB.set_status_processing( field["target_name"], f"calibrator{calibrator_field}", field["sas_id_target"] ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) @@ -142,7 +144,7 @@ def run_linc_calibrator_toil(field, calibrator_field: int): if pattern.search(f_err.read()): success = True if success: - set_status_finished( + CURRENT_DB.set_status_finished( field["target_name"], f"calibrator{calibrator_field}", field["sas_id_target"], @@ -166,101 +168,6 @@ def get_approval(field, identifier, needs_approval): return field -@functools.total_ordering -class PIPELINE_STATUS(Enum): - nothing = 0 - downloaded = 1 - finished = 2 - await_approval = 3 - processing = 98 - error = 99 - - def __eq__(self, other): - if other.__class__ is int: - return self.value == other - elif other.__class__ is self.__class__: - return self.value == other.value - else: - raise NotImplementedError - - def __lt__(self, other): - if self.__class__ is not other.__class__: - raise NotImplementedError - return self.value < other.value - - -def get_db_columns(obsid: str = None): - with sqlite3.connect(DATABASE) as db: - db.row_factory = sqlite3.Row - cursor = db.cursor() - columns = "target_name,priority,finished,downloaded,sas_id_calibrator1,sas_id_calibrator2,sas_id_calibrator_final,sas_id_target,status_calibrator1,status_calibrator2,status_target,status_vlbi_delay,status_vlbi_dd,status_ddf,status_vlbi_ddf_subtract" - if obsid: - field = cursor.execute( - f"select {columns} from {TABLE_NAME} where sas_id_target=='{obsid}' and finished==0 order by priority desc" - ).fetchall() - else: - field = cursor.execute( - f"select {columns} from {TABLE_NAME} where finished==0 order by priority desc" - ).fetchall() - print(field) - return field - - -def set_status_failed(name, identifier, target): - with sqlite3.connect(DATABASE) as db: - cursor = db.cursor() - cursor.execute( - f"update {TABLE_NAME} set status_{identifier}={PIPELINE_STATUS.error.value} where target_name=='{name}' and sas_id_target=='{target}'" - ) - - -def set_status_processing(name, identifier, target): - with sqlite3.connect(DATABASE) as db: - cursor = db.cursor() - cursor.execute( - f"update {TABLE_NAME} set status_{identifier}={PIPELINE_STATUS.processing.value} where target_name=='{name}' and sas_id_target=='{target}'" - ) - - -def set_status_await_approval(name, identifier, target): - with sqlite3.connect(DATABASE) as db: - cursor = db.cursor() - cursor.execute( - f"update {TABLE_NAME} set status_{identifier}={PIPELINE_STATUS.await_approval.value} where target_name=='{name}' and sas_id_target=='{target}'" - ) - - -def set_status_finished(name, identifier, target): - with sqlite3.connect(DATABASE) as db: - cursor = db.cursor() - cursor.execute( - f"update {TABLE_NAME} set status_{identifier}={PIPELINE_STATUS.finished.value} where target_name=='{name}' and sas_id_target=='{target}'" - ) - - -def set_status_downloaded(name, target): - with sqlite3.connect(DATABASE) as db: - cursor = db.cursor() - cursor.execute( - f"update {TABLE_NAME} set downloaded=1 where target_name=='{name}' and sas_id_target=='{target}'" - ) - - -def set_field_finished(name, target): - query = f"update {TABLE_NAME} set finished=1 where target_name=='{name}' and sas_id_target=='{target}'" - with sqlite3.connect(DATABASE) as db: - cursor = db.cursor() - cursor.execute(query) - - -def set_final_calibrator(name, target, final_cal): - with sqlite3.connect(DATABASE) as db: - cursor = db.cursor() - cursor.execute( - f"update {TABLE_NAME} set sas_id_calibrator_final={final_cal} where target_name=='{name}' and sas_id_target=='{target}'" - ) - - def get_most_recent_run(searchpath: str, sas_id: str, pipeline: str) -> pathlib.Path: rundirs = pathlib.Path(searchpath) rundirs_sorted = sorted(rundirs.iterdir()) @@ -281,7 +188,7 @@ def pilot_widefield(): @task def get_unprocessed_target(): field = None - for dbrow in get_db_columns(): + for dbrow in CURRENT_DB.get_db_columns(): is_processing = False row = dict(dbrow) status_keys = filter(lambda x: x.startswith("status_"), row.keys()) @@ -300,7 +207,7 @@ def get_unprocessed_target(): @task.short_circuit def check_fields(): - fields = get_db_columns() + fields = CURRENT_DB.get_db_columns() return bool(fields) @task @@ -465,7 +372,7 @@ def download_field(field): cmd, shell=True, text=True, stdout=f_out, stderr=f_err ) if not proc.returncode: - set_status_downloaded( + CURRENT_DB.set_status_downloaded( field["target_name"], field["sas_id_target"], ) @@ -478,7 +385,7 @@ def download_field(field): @task def run_linc_calibrator1(field): - field = dict(get_db_columns(field["sas_id_target"])[0]) + field = dict(CURRENT_DB.get_db_columns(field["sas_id_target"])[0]) if not field["sas_id_calibrator1"]: raise AirflowSkipException("Calibrator 1 does not exist, skipping.") if field["status_calibrator1"] == PIPELINE_STATUS.finished: @@ -497,7 +404,7 @@ def run_linc_calibrator1(field): @task def run_linc_calibrator2(field): - field = dict(get_db_columns(field["sas_id_target"])[0]) + field = dict(CURRENT_DB.get_db_columns(field["sas_id_target"])[0]) if not field["sas_id_calibrator2"]: raise AirflowSkipException("Calibrator 2 does not exist, skipping.") if field["status_calibrator2"] == PIPELINE_STATUS.finished: @@ -528,14 +435,14 @@ def select_best_calibrator(result1, result2): f"No flux density calibrator template found. Randomly selected calibrator{cal}" ) if cal == 1: - set_final_calibrator( + CURRENT_DB.set_final_calibrator( result1["target_name"], result1["sas_id_target"], result1["sas_id_calibrator1"], ) return result1 elif cal == 2: - set_final_calibrator( + CURRENT_DB.set_final_calibrator( result2["target_name"], result2["sas_id_target"], result2["sas_id_calibrator2"], @@ -571,7 +478,7 @@ def select_best_calibrator(result1, result2): match score1 < score2: case True: print("Best score for calibrator1") - set_final_calibrator( + CURRENT_DB.set_final_calibrator( result1["target_name"], result1["sas_id_target"], result1["sas_id_calibrator1"], @@ -579,7 +486,7 @@ def select_best_calibrator(result1, result2): return result1 case False: print("Best score for calibrator2") - set_final_calibrator( + CURRENT_DB.set_final_calibrator( result2["target_name"], result2["sas_id_target"], result2["sas_id_calibrator2"], @@ -587,7 +494,7 @@ def select_best_calibrator(result1, result2): return result2 elif result1 and (not result2): print("Only cal 1 succeeded, continuing with that") - set_final_calibrator( + CURRENT_DB.set_final_calibrator( result1["target_name"], result1["sas_id_target"], result1["sas_id_calibrator1"], @@ -595,7 +502,7 @@ def select_best_calibrator(result1, result2): return result1 elif (not result1) and result2: print("Only cal 2 succeeded, continuing with that") - set_final_calibrator( + CURRENT_DB.set_final_calibrator( result2["target_name"], result2["sas_id_target"], result2["sas_id_calibrator2"], @@ -622,7 +529,7 @@ def run_linc_target(field): calibrator_solutions = ( calibrator_path / "results_LINC_calibrator" / "cal_solutions.h5" ) - set_status_processing( + CURRENT_DB.set_status_processing( field["target_name"], "target", field["sas_id_target"] ) cmd = f"flocs-run linc target --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --cal-solutions {calibrator_solutions} {os.path.join(DATA_DIR, field['target_name'], 'target', ms_folder)}" @@ -649,7 +556,7 @@ def run_linc_target(field): if pattern.search(f_err.read()): success = True if success: - set_status_finished( + CURRENT_DB.set_status_finished( field["target_name"], "target", field["sas_id_target"] ) else: @@ -675,7 +582,7 @@ def run_vlbi_delay(field): outdir, field["sas_id_target"], "LINC_target" ) target_ms_path = target_path / "results_LINC_target" / "results" - set_status_processing( + CURRENT_DB.set_status_processing( field["target_name"], "vlbi_delay", field["sas_id_target"] ) @@ -773,11 +680,11 @@ def run_vlbi_delay(field): if success: if NEEDS_MANUAL_APPROVAL_DELAY: - set_status_await_approval( + CURRENT_DB.set_status_await_approval( field["target_name"], "vlbi_delay", field["sas_id_target"] ) else: - set_status_finished( + CURRENT_DB.set_status_finished( field["target_name"], "vlbi_delay", field["sas_id_target"] ) else: @@ -786,7 +693,7 @@ def run_vlbi_delay(field): @task def run_ddf_pipeline(field): - field = dict(get_db_columns(field["sas_id_target"])[0]) + field = dict(CURRENT_DB.get_db_columns(field["sas_id_target"])[0]) if field["status_ddf"] == PIPELINE_STATUS.processing: print( f"ddf-pipeline for {field['target_name']} {field['sas_id_target']} should be running, attempting to resume polling..." @@ -817,7 +724,7 @@ def run_ddf_pipeline(field): if (status == "RUNNING") or (status == "PENDING"): time.sleep(60) elif status == "COMPLETED": - set_status_finished( + CURRENT_DB.set_status_finished( field["target_name"], "ddf_subtract", field["sas_id_target"] ) return field @@ -844,7 +751,9 @@ def run_ddf_pipeline(field): cmd = f"flocs-run ddf-pipeline --scheduler slurm --slurm-time 72:00:00 --slurm-cores 32 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --config-file {DDF_CONFIG} {target_ms_path}" print(cmd) - set_status_processing(field["target_name"], "ddf", field["sas_id_target"]) + CURRENT_DB.set_status_processing( + field["target_name"], "ddf", field["sas_id_target"] + ) with ( open( f"log_DDF-pipeline_{field['target_name']}_{field['sas_id_target']}.txt", @@ -879,7 +788,7 @@ def run_ddf_pipeline(field): if (status == "RUNNING") or (status == "PENDING"): time.sleep(60) elif status == "COMPLETED": - set_status_finished( + CURRENT_DB.set_status_finished( field["target_name"], "ddf_subtract", field["sas_id_target"], @@ -897,7 +806,7 @@ def run_ddf_pipeline(field): @task def run_ddf_subtract(field): - field = dict(get_db_columns(field["sas_id_target"])[0]) + field = dict(CURRENT_DB.get_db_columns(field["sas_id_target"])[0]) if field["status_vlbi_ddf_subtract"] == PIPELINE_STATUS.finished: return field else: @@ -923,7 +832,7 @@ def run_ddf_subtract(field): f"log_VLBI_process-ddf_{field['target_name']}_{field['sas_id_target']}.txt" ) ): - set_status_processing( + CURRENT_DB.set_status_processing( field["target_name"], "ddf_subtract", field["sas_id_target"] ) cmd = f"flocs-run vlbi process-ddf --runner toil --record-toil-stats --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --ms-suffix .dp3concat --ddf-rundir {ddf_path} --solsdir {ddf_sols_path} --do-subtraction {target_ms_path}" @@ -970,7 +879,7 @@ def run_ddf_subtract(field): if pattern.search(f_err.read()): success = True if success: - set_status_finished( + CURRENT_DB.set_status_finished( field["target_name"], "ddf_subtract", field["sas_id_target"] ) else: @@ -979,7 +888,7 @@ def run_ddf_subtract(field): @task def run_vlbi_ddcal(field): - field = dict(get_db_columns(field["sas_id_target"])[0]) + field = dict(CURRENT_DB.get_db_columns(field["sas_id_target"])[0]) if (field["status_vlbi_dd"] == PIPELINE_STATUS.finished) or ( field["status_vlbi_dd"] == PIPELINE_STATUS.processing ): @@ -1012,7 +921,7 @@ def run_vlbi_ddcal(field): if not os.path.isfile(source_cat): raise AirflowFailException(f"{source_cat} not found.") - set_status_processing( + CURRENT_DB.set_status_processing( field["target_name"], "vlbi_dd", field["sas_id_target"] ) cmd = f"flocs-run vlbi dd-calibration --runner toil --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --delay-solset {sols} --phasediff-score 10.0 --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat {target_ms_path}" @@ -1030,7 +939,7 @@ def run_vlbi_ddcal(field): if not os.path.isfile(source_cat): raise AirflowFailException(f"{source_cat} not found.") - set_status_processing( + CURRENT_DB.set_status_processing( field["target_name"], "vlbi_dd", field["sas_id_target"] ) @@ -1088,12 +997,14 @@ def run_vlbi_ddcal(field): if pattern.search(f_err.read()): success = True if success: - set_status_finished( + CURRENT_DB.set_status_finished( field["target_name"], "vlbi_dd", field["sas_id_target"] ) - set_field_finished(field["target_name"], field["sas_id_target"]) + CURRENT_DB.set_field_finished( + field["target_name"], field["sas_id_target"] + ) else: - set_status_failed( + CURRENT_DB.set_status_failed( field["target_name"], "vlbi_dd", field["sas_id_target"] ) raise RuntimeError @@ -1306,7 +1217,7 @@ def prepare_ddf_subtract(field): @task def run_vlbi_image_intermediate(field): - field = dict(get_db_columns(field["sas_id_target"])[0]) + field = dict(CURRENT_DB.get_db_columns(field["sas_id_target"])[0]) if (field["status_vlbi_intermediate_img"] == PIPELINE_STATUS.finished) or ( field["status_vlbi_intermediate_img"] == PIPELINE_STATUS.processing ): @@ -1331,7 +1242,7 @@ def run_vlbi_image_intermediate(field): if not os.path.isfile(dd_sols): raise AirflowFailException(f"{dd_sols} not found.") - set_status_processing( + CURRENT_DB.set_status_processing( field["target_name"], "vlbi_intermediate_img", field["sas_id_target"] ) @@ -1389,14 +1300,16 @@ def run_vlbi_image_intermediate(field): if pattern.search(f_err.read()): success = True if success: - set_status_finished( + CURRENT_DB.set_status_finished( field["target_name"], "vlbi_intermediate_img", field["sas_id_target"], ) - set_field_finished(field["target_name"], field["sas_id_target"]) + CURRENT_DB.set_field_finished( + field["target_name"], field["sas_id_target"] + ) else: - set_status_failed( + CURRENT_DB.set_status_failed( field["target_name"], "vlbi_intermediate_img", field["sas_id_target"], @@ -1406,7 +1319,7 @@ def run_vlbi_image_intermediate(field): @task def run_vlbi_facet_subtract(field): - field = dict(get_db_columns(field["sas_id_target"])[0]) + field = dict(CURRENT_DB.CURRENT_DB.get_db_columns(field["sas_id_target"])[0]) if (field["status_vlbi_facet_subtract"] == PIPELINE_STATUS.finished) or ( field["status_vlbi_facet_subtract"] == PIPELINE_STATUS.processing ): @@ -1447,7 +1360,7 @@ def run_vlbi_facet_subtract(field): "No suitable intermediate resolution model images found." ) - set_status_processing( + CURRENT_DB.set_status_processing( field["target_name"], "vlbi_facet_subtract", field["sas_id_target"] ) @@ -1505,14 +1418,16 @@ def run_vlbi_facet_subtract(field): if pattern.search(f_err.read()): success = True if success: - set_status_finished( + CURRENT_DB.set_status_finished( field["target_name"], "vlbi_facet_subtract", field["sas_id_target"], ) - set_field_finished(field["target_name"], field["sas_id_target"]) + CURRENT_DB.set_field_finished( + field["target_name"], field["sas_id_target"] + ) else: - set_status_failed( + CURRENT_DB.set_status_failed( field["target_name"], "vlbi_facet_subtract", field["sas_id_target"], @@ -1522,7 +1437,7 @@ def run_vlbi_facet_subtract(field): @task def run_vlbi_facet_imaging(field): - field = dict(get_db_columns(field["sas_id_target"])[0]) + field = dict(CURRENT_DB.get_db_columns(field["sas_id_target"])[0]) if (field["status_vlbi_facet_imaging"] == PIPELINE_STATUS.finished) or ( field["status_vlbi_facet_imaging"] == PIPELINE_STATUS.processing ): @@ -1545,7 +1460,7 @@ def run_vlbi_facet_imaging(field): "No suitable intermediate resolution model images found." ) - set_status_processing( + CURRENT_DB.set_status_processing( field["target_name"], "vlbi_facet_imaging", field["sas_id_target"] ) @@ -1603,14 +1518,16 @@ def run_vlbi_facet_imaging(field): if pattern.search(f_err.read()): success = True if success: - set_status_finished( + CURRENT_DB.set_status_finished( field["target_name"], "vlbi_facet_imaging", field["sas_id_target"], ) - set_field_finished(field["target_name"], field["sas_id_target"]) + CURRENT_DB.set_field_finished( + field["target_name"], field["sas_id_target"] + ) else: - set_status_failed( + CURRENT_DB.set_status_failed( field["target_name"], "vlbi_facet_imaging", field["sas_id_target"], From 7c9f0cb43b32bfc9a8161dc0fe425f341439bd93 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Thu, 6 Aug 2026 17:19:57 +0200 Subject: [PATCH 080/120] Add DB utils interface --- flocs_processing/db_utils.py | 96 ++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 flocs_processing/db_utils.py diff --git a/flocs_processing/db_utils.py b/flocs_processing/db_utils.py new file mode 100644 index 0000000..ea3a7fc --- /dev/null +++ b/flocs_processing/db_utils.py @@ -0,0 +1,96 @@ +from enum import Enum +import functools +import sqlite3 + + +@functools.total_ordering +class PIPELINE_STATUS(Enum): + nothing = 0 + downloaded = 1 + finished = 2 + await_approval = 3 + processing = 98 + error = 99 + + def __eq__(self, other): + if other.__class__ is int: + return self.value == other + elif other.__class__ is self.__class__: + return self.value == other.value + else: + raise NotImplementedError + + def __lt__(self, other): + if self.__class__ is not other.__class__: + raise NotImplementedError + return self.value < other.value + + +class FlocsDB: + def __init__(self, dbname: str, db_table: str): + self.DATABASE = dbname + self.TABLE_NAME = db_table + + def get_db_columns(self, obsid: str = None): + with sqlite3.connect(self.DATABASE) as db: + db.row_factory = sqlite3.Row + cursor = db.cursor() + columns = "target_name,priority,finished,downloaded,sas_id_calibrator1,sas_id_calibrator2,sas_id_calibrator_final,sas_id_target,status_calibrator1,status_calibrator2,status_target,status_vlbi_delay,status_vlbi_dd,status_ddf,status_vlbi_ddf_subtract" + if obsid: + field = cursor.execute( + f"select {columns} from {self.TABLE_NAME} where sas_id_target=='{obsid}' and finished==0 order by priority desc" + ).fetchall() + else: + field = cursor.execute( + f"select {columns} from {self.TABLE_NAME} where finished==0 order by priority desc" + ).fetchall() + print(field) + return field + + def set_status_failed(self, name, identifier, target): + with sqlite3.connect(self.DATABASE) as db: + cursor = db.cursor() + cursor.execute( + f"update {self.TABLE_NAME} set status_{identifier}={PIPELINE_STATUS.error.value} where target_name=='{name}' and sas_id_target=='{target}'" + ) + + def set_status_processing(self, name, identifier, target): + with sqlite3.connect(self.DATABASE) as db: + cursor = db.cursor() + cursor.execute( + f"update {self.TABLE_NAME} set status_{identifier}={PIPELINE_STATUS.processing.value} where target_name=='{name}' and sas_id_target=='{target}'" + ) + + def set_status_await_approval(self, name, identifier, target): + with sqlite3.connect(self.DATABASE) as db: + cursor = db.cursor() + cursor.execute( + f"update {self.TABLE_NAME} set status_{identifier}={PIPELINE_STATUS.await_approval.value} where target_name=='{name}' and sas_id_target=='{target}'" + ) + + def set_status_finished(self, name, identifier, target): + with sqlite3.connect(self.DATABASE) as db: + cursor = db.cursor() + cursor.execute( + f"update {self.TABLE_NAME} set status_{identifier}={PIPELINE_STATUS.finished.value} where target_name=='{name}' and sas_id_target=='{target}'" + ) + + def set_status_downloaded(self, name, target): + with sqlite3.connect(self.DATABASE) as db: + cursor = db.cursor() + cursor.execute( + f"update {self.TABLE_NAME} set downloaded=1 where target_name=='{name}' and sas_id_target=='{target}'" + ) + + def set_field_finished(self, name, target): + query = f"update {self.TABLE_NAME} set finished=1 where target_name=='{name}' and sas_id_target=='{target}'" + with sqlite3.connect(self.DATABASE) as db: + cursor = db.cursor() + cursor.execute(query) + + def set_final_calibrator(self, name, target, final_cal): + with sqlite3.connect(self.DATABASE) as db: + cursor = db.cursor() + cursor.execute( + f"update {self.TABLE_NAME} set sas_id_calibrator_final={final_cal} where target_name=='{name}' and sas_id_target=='{target}'" + ) From 4a0bbe02fc8469173fdf40e588412c73eda06460 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Thu, 6 Aug 2026 17:35:28 +0200 Subject: [PATCH 081/120] Implement cwltool for linc target --- flocs_processing/dags/pilot_widefield.py | 171 ++++++++++++++++------- 1 file changed, 123 insertions(+), 48 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 6030d24..6a3c6ef 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -50,7 +50,8 @@ FLUX_CALIBRATOR_TEMPLATE = parser["DEFAULT"]["FLUX_CALIBRATOR_TEMPLATE"] NEEDS_MANUAL_APPROVAL_DELAY = bool(parser["DEFAULT"]["NEEDS_MANUAL_APPROVAL_DELAY"]) -CWL_RUNNER_LINC = "cwltool" +CWL_RUNNER_LINC_CALIBRATOR = "cwltool" +CWL_RUNNER_LINC_TARGET = "cwltool" CURRENT_DB = FlocsDB(DATABASE, TABLE_NAME) @@ -153,6 +154,117 @@ def run_linc_calibrator_toil(field, calibrator_field: int): raise RuntimeError +def run_linc_target_cwltool(field): + print( + f"Processing target observation {field['target_name']} {field['sas_id_target']} with calibrator {field['sas_id_calibrator_final']}" + ) + ms_folder = f"L{field['sas_id_target']}" + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + calibrator_path = get_most_recent_run( + outdir, field["sas_id_calibrator_final"], "LINC_calibrator" + ) + calibrator_solutions = ( + calibrator_path / "results_LINC_calibrator" / "cal_solutions.h5" + ) + CURRENT_DB.set_status_processing( + field["target_name"], "target", field["sas_id_target"] + ) + cmd = f"flocs-run linc target --runner cwltool --scheduler slurm --slurm-cores 64 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --cal-solutions {calibrator_solutions} {os.path.join(DATA_DIR, field['target_name'], 'target', ms_folder)}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with ( + open( + f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}.txt", + "w+", + ) as f_out, + open( + f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w+", + ) as f_err, + ): + proc = subprocess.run(cmd, shell=True, text=True, stdout=f_out, stderr=f_err) + jobid = None + if not proc.returncode: + f_out.seek(0) + for line in f_out.readlines(): + if "Submitted batch job" in line: + jobid = line.strip().split()[-1] + else: + raise RuntimeError("Failed to submit job.") + + if not jobid: + raise RuntimeError("Failed to retrieve job id") + else: + while True: + print(f"Polling LINC target job {jobid}") + poll_cmd = f"sacct -X -j {jobid} --format=State --noheader" + status = subprocess.run( + poll_cmd, shell=True, text=True, capture_output=True + ).stdout.strip() + if (status == "RUNNING") or (status == "PENDING"): + time.sleep(60) + elif status == "COMPLETED": + CURRENT_DB.set_status_finished( + field["target_name"], + "target", + field["sas_id_target"], + ) + break + elif ( + (status == "FAILED") + or ("TIMEOUT" in status) + or ("CANCELLED" in status) + ): + raise RuntimeError( + f"LINC target for {field['target_name']} {field['sas_id_target']} failed." + ) + + +def run_linc_target_toil(field): + print( + f"Processing target observation {field['target_name']} {field['sas_id_target']} with calibrator {field['sas_id_calibrator_final']}" + ) + ms_folder = f"L{field['sas_id_target']}" + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + calibrator_path = get_most_recent_run( + outdir, field["sas_id_calibrator_final"], "LINC_calibrator" + ) + calibrator_solutions = ( + calibrator_path / "results_LINC_calibrator" / "cal_solutions.h5" + ) + CURRENT_DB.set_status_processing( + field["target_name"], "target", field["sas_id_target"] + ) + cmd = f"flocs-run linc target --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --cal-solutions {calibrator_solutions} {os.path.join(DATA_DIR, field['target_name'], 'target', ms_folder)}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with ( + open( + f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}.txt", + "w+", + ) as f_out, + open( + f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w+", + ) as f_err, + ): + proc = subprocess.run(cmd, shell=True, text=True, stdout=f_out, stderr=f_err) + success = False + pattern = re.compile(r"Workflow.* stopped. Success: True") + if not proc.returncode: + f_err.seek(0) + if pattern.search(f_err.read()): + success = True + if success: + CURRENT_DB.set_status_finished( + field["target_name"], "target", field["sas_id_target"] + ) + else: + raise RuntimeError + + def get_approval(field, identifier, needs_approval): if not needs_approval: return field @@ -394,9 +506,9 @@ def run_linc_calibrator1(field): ) return field else: - if CWL_RUNNER_LINC == "cwltool": + if CWL_RUNNER_LINC_CALIBRATOR == "cwltool": run_linc_calibrator_cwltool(field, calibrator_field=1) - elif CWL_RUNNER_LINC == "toil": + elif CWL_RUNNER_LINC_CALIBRATOR == "toil": run_linc_calibrator_toil(field, calibrator_field=1) else: raise RuntimeError("Invalid CWL runner specified.") @@ -413,9 +525,9 @@ def run_linc_calibrator2(field): ) return field else: - if CWL_RUNNER_LINC == "cwltool": + if CWL_RUNNER_LINC_CALIBRATOR == "cwltool": run_linc_calibrator_cwltool(field, calibrator_field=2) - elif CWL_RUNNER_LINC == "toil": + elif CWL_RUNNER_LINC_CALIBRATOR == "toil": run_linc_calibrator_toil(field, calibrator_field=2) else: raise RuntimeError("Invalid CWL runner specified.") @@ -518,49 +630,12 @@ def run_linc_target(field): ): return field else: - print( - f"Processing target observation {field['target_name']} {field['sas_id_target']} with calibrator {field['sas_id_calibrator_final']}" - ) - ms_folder = f"L{field['sas_id_target']}" - outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - calibrator_path = get_most_recent_run( - outdir, field["sas_id_calibrator_final"], "LINC_calibrator" - ) - calibrator_solutions = ( - calibrator_path / "results_LINC_calibrator" / "cal_solutions.h5" - ) - CURRENT_DB.set_status_processing( - field["target_name"], "target", field["sas_id_target"] - ) - cmd = f"flocs-run linc target --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --cal-solutions {calibrator_solutions} {os.path.join(DATA_DIR, field['target_name'], 'target', ms_folder)}" - if not os.path.isdir(outdir): - os.mkdir(outdir) - print(cmd) - with ( - open( - f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}.txt", - "w+", - ) as f_out, - open( - f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}_err.txt", - "w+", - ) as f_err, - ): - proc = subprocess.run( - cmd, shell=True, text=True, stdout=f_out, stderr=f_err - ) - success = False - pattern = re.compile(r"Workflow.* stopped. Success: True") - if not proc.returncode: - f_err.seek(0) - if pattern.search(f_err.read()): - success = True - if success: - CURRENT_DB.set_status_finished( - field["target_name"], "target", field["sas_id_target"] - ) - else: - raise RuntimeError + if CWL_RUNNER_LINC_TARGET == "cwltool": + run_linc_target_cwltool(field) + elif CWL_RUNNER_LINC_TARGET == "toil": + run_linc_target_toil(field) + else: + raise RuntimeError("Invalid CWL runner specified.") return field @task From 7efc522f6e0dec6ef0a49d3a92cd914c3d3568cc Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Thu, 6 Aug 2026 17:37:37 +0200 Subject: [PATCH 082/120] default toil for target for now --- flocs_processing/dags/pilot_widefield.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 6a3c6ef..b1d7270 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -51,7 +51,7 @@ NEEDS_MANUAL_APPROVAL_DELAY = bool(parser["DEFAULT"]["NEEDS_MANUAL_APPROVAL_DELAY"]) CWL_RUNNER_LINC_CALIBRATOR = "cwltool" -CWL_RUNNER_LINC_TARGET = "cwltool" +CWL_RUNNER_LINC_TARGET = "toil" CURRENT_DB = FlocsDB(DATABASE, TABLE_NAME) From ee6604036f4338e26e99b3bdb29a90df5e50834e Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 7 Aug 2026 10:16:56 +0200 Subject: [PATCH 083/120] Update score choice to handle identical scores --- flocs_processing/dags/pilot_widefield.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 6030d24..1290418 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -475,7 +475,7 @@ def select_best_calibrator(result1, result2): score2 = assess_cal2[0]["score"] print(f"Calibrator 1 score: {score1}") print(f"Calibrator 2 score: {score2}") - match score1 < score2: + match score1 <= score2: case True: print("Best score for calibrator1") CURRENT_DB.set_final_calibrator( From 802f0508c56fc37ffa4ef41d35b8deddfdef8e13 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 7 Aug 2026 10:42:22 +0200 Subject: [PATCH 084/120] Add initial db manipulation commands --- flocs_processing/flocs_processing.py | 52 +++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/flocs_processing/flocs_processing.py b/flocs_processing/flocs_processing.py index e5d6c51..0c4cd38 100644 --- a/flocs_processing/flocs_processing.py +++ b/flocs_processing/flocs_processing.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +from .db_utils import FlocsDB from .processors import FlocsAirflowProcessor from cyclopts import Parameter from enum import Enum @@ -6,7 +7,7 @@ import functools import subprocess import structlog -from typing import Optional, Annotated +from typing import Annotated, Literal, Optional app = cyclopts.App() @@ -114,6 +115,55 @@ def add_field( raise RuntimeError(f"Failed to update table {table_name} in database {dbname}.") +@app.command() +def update_field( + field_name: Annotated[str, Parameter(help="Name of the source/field to add.")], + sas_id_target: Annotated[ + str, + Parameter(help="SAS ID of the target to add.", consume_multiple=True), + ], + dbname: Annotated[ + str, Parameter(help="Sqlite3 database from which processing will be done.") + ], + table_name: Annotated[ + str, Parameter(help="Database table that will be processed.") + ] = "processing_flocs", + pipeline: Annotated[ + Optional[ + Literal[ + "calibrator1", + "calibrator2", + "target", + "vlbi_delay", + "ddf", + "vlbi_ddf_subtract", + "vlbi_dd", + "vlbi_intermediate_img", + "vlbi_facet_subtract", + "vlbi_facet_img", + ] + ], + Parameter(help="Pipeline of which to update the status"), + ] = None, + set_status: Annotated[ + Optional[Literal["nothing", "downloaded", "processing", "failed", "success"]], + Parameter(help="Set the status of the given pipeline."), + ] = None, +): + db = FlocsDB(dbname=dbname, db_table=table_name) + match set_status: + case "nothing": + pass + case "downloaded": + db.set_status_downloaded(field_name, sas_id_target) + case "processing": + db.set_status_processing(field_name, pipeline, sas_id_target) + case "failed": + db.set_status_failed(field_name, pipeline, sas_id_target) + case "success": + db.set_status_finished(field_name, pipeline, sas_id_target) + + @app.command() def start_airflow_processing( airflow_cores: Annotated[ From fa5e48f6bcf2d98764237311b0ec2aff5ed34ade Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 7 Aug 2026 10:44:27 +0200 Subject: [PATCH 085/120] Add nothing status --- flocs_processing/db_utils.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/flocs_processing/db_utils.py b/flocs_processing/db_utils.py index ea3a7fc..9d43e70 100644 --- a/flocs_processing/db_utils.py +++ b/flocs_processing/db_utils.py @@ -47,6 +47,13 @@ def get_db_columns(self, obsid: str = None): print(field) return field + def set_status_nothing(self, name, identifier, target): + with sqlite3.connect(self.DATABASE) as db: + cursor = db.cursor() + cursor.execute( + f"update {self.TABLE_NAME} set status_{identifier}={PIPELINE_STATUS.nothing.value} where target_name=='{name}' and sas_id_target=='{target}'" + ) + def set_status_failed(self, name, identifier, target): with sqlite3.connect(self.DATABASE) as db: cursor = db.cursor() From 6eaea014c53deac45eab9d4080ef1235843e5976 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 7 Aug 2026 11:00:12 +0200 Subject: [PATCH 086/120] More db manipulation --- flocs_processing/flocs_processing.py | 84 +++++++++++++++++----------- 1 file changed, 52 insertions(+), 32 deletions(-) diff --git a/flocs_processing/flocs_processing.py b/flocs_processing/flocs_processing.py index 0c4cd38..580d3dd 100644 --- a/flocs_processing/flocs_processing.py +++ b/flocs_processing/flocs_processing.py @@ -7,13 +7,28 @@ import functools import subprocess import structlog -from typing import Annotated, Literal, Optional +from typing import Annotated, Literal, Optional, get_args app = cyclopts.App() logger = structlog.getLogger() +PIPELINES = Literal[ + "all", + "calibrator1", + "calibrator2", + "target", + "vlbi_delay", + "ddf", + "vlbi_ddf_subtract", + "vlbi_dd", + "vlbi_intermediate_img", + "vlbi_facet_subtract", + "vlbi_facet_img", +] + + @functools.total_ordering class PIPELINE_STATUS(Enum): nothing = 0 @@ -125,43 +140,48 @@ def update_field( dbname: Annotated[ str, Parameter(help="Sqlite3 database from which processing will be done.") ], - table_name: Annotated[ - str, Parameter(help="Database table that will be processed.") - ] = "processing_flocs", pipeline: Annotated[ - Optional[ - Literal[ - "calibrator1", - "calibrator2", - "target", - "vlbi_delay", - "ddf", - "vlbi_ddf_subtract", - "vlbi_dd", - "vlbi_intermediate_img", - "vlbi_facet_subtract", - "vlbi_facet_img", - ] - ], + PIPELINES, Parameter(help="Pipeline of which to update the status"), - ] = None, + ], set_status: Annotated[ - Optional[Literal["nothing", "downloaded", "processing", "failed", "success"]], + Literal["nothing", "downloaded", "processing", "failed", "success"], Parameter(help="Set the status of the given pipeline."), - ] = None, + ], + table_name: Annotated[ + str, Parameter(help="Database table that will be processed.") + ] = "processing_flocs", ): db = FlocsDB(dbname=dbname, db_table=table_name) - match set_status: - case "nothing": - pass - case "downloaded": - db.set_status_downloaded(field_name, sas_id_target) - case "processing": - db.set_status_processing(field_name, pipeline, sas_id_target) - case "failed": - db.set_status_failed(field_name, pipeline, sas_id_target) - case "success": - db.set_status_finished(field_name, pipeline, sas_id_target) + if pipeline != "all": + p_list = list(get_args(PIPELINES)) + p_list.remove("all") + for p in p_list: + logger.info(f"Setting pipeline {p} to status {set_status}") + match set_status: + case "nothing": + db.set_status_nothing(field_name, p, sas_id_target) + case "downloaded": + db.set_status_downloaded(field_name, sas_id_target) + case "processing": + db.set_status_processing(field_name, p, sas_id_target) + case "failed": + db.set_status_failed(field_name, p, sas_id_target) + case "success": + db.set_status_finished(field_name, p, sas_id_target) + else: + logger.info(f"Setting pipeline {pipeline} to status {set_status}") + match set_status: + case "nothing": + db.set_status_nothing(field_name, pipeline, sas_id_target) + case "downloaded": + db.set_status_downloaded(field_name, sas_id_target) + case "processing": + db.set_status_processing(field_name, pipeline, sas_id_target) + case "failed": + db.set_status_failed(field_name, pipeline, sas_id_target) + case "success": + db.set_status_finished(field_name, pipeline, sas_id_target) @app.command() From 2ff0a6730a0904d350461ad3b0727ae1951103fa Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 7 Aug 2026 12:10:08 +0200 Subject: [PATCH 087/120] Implement cwltool for delay cal --- flocs_processing/dags/pilot_widefield.py | 344 +++++++++++++++-------- 1 file changed, 224 insertions(+), 120 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 5604a96..7c87136 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -25,12 +25,12 @@ "FLOCS_AIRFLOW_CONFIG environment variable not set. Please point this to a valid configuration file." ) -CONFIG_FILE = os.getenv("FLOCS_AIRFLOW_CONFIG") +CONFIG_FILE: str = os.getenv("FLOCS_AIRFLOW_CONFIG") or "" if not os.path.isfile(CONFIG_FILE): raise RuntimeError(f"{CONFIG_FILE} is not a valid file") parser = configparser.ConfigParser() -parser.optionxform = str +parser.optionxform = str # ty: ignore[invalid-assignment] with open(CONFIG_FILE, "r") as config: parser.read_string("[DEFAULT]\n" + config.read()) @@ -52,6 +52,7 @@ CWL_RUNNER_LINC_CALIBRATOR = "cwltool" CWL_RUNNER_LINC_TARGET = "toil" +CWL_RUNNER_PILOT_DELAY = "toil" CURRENT_DB = FlocsDB(DATABASE, TABLE_NAME) @@ -265,6 +266,220 @@ def run_linc_target_toil(field): raise RuntimeError +def run_pilot_delay_cwltool(field): + print( + f"Processing delay calibration for {field['target_name']} {field['sas_id_target']}" + ) + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + target_path = get_most_recent_run(outdir, field["sas_id_target"], "LINC_target") + target_ms_path = target_path / "results_LINC_target" / "results" + CURRENT_DB.set_status_processing( + field["target_name"], "vlbi_delay", field["sas_id_target"] + ) + + delay_cat = os.path.join(outdir, "delay_calibrators.csv") + image_cat = os.path.join(outdir, "image_catalogue.csv") + + if not os.path.isfile(delay_cat) or not os.path.isfile(image_cat): + ms = list(target_ms_path.glob("*.dp3concat"))[0] + cmd = f"lofar-vlbi-plot --force --output_dir {outdir} --MS {ms}" + with ( + open( + f"log_plot_field_{field['target_name']}_{field['sas_id_target']}.txt", + "w+", + ) as f_out, + open( + f"log_plot_field_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w+", + ) as f_err, + ): + proc = subprocess.run( + cmd, shell=True, text=True, stdout=f_out, stderr=f_err + ) + delay_cat = os.path.join(outdir, "delay_calibrators.csv") + image_cat = os.path.join(outdir, "image_catalogue.csv") + + if not os.path.isfile(delay_cat): + raise RuntimeError("Delay calibrator catalogue is missing or invalid.") + if not os.path.isfile(image_cat): + raise RuntimeError("Image source catalogue is missing or invalid.") + + proc = subprocess.run( + "detect_bad_slurm_nodes.sh", + shell=True, + text=True, + stdout=subprocess.PIPE, + ) + bad_nodes = proc.stdout.strip() + if bad_nodes: + print(f"Excluding the following bad nodes from scheduling: {bad_nodes}") + os.environ["TOIL_SLURM_ARGS"] = f"--exclude={bad_nodes}" + + cmd = f"flocs-run vlbi delay-calibration --record-toil-stats --runner cwltool --scheduler slurm --slurm-cores 64 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --ms-suffix dp3concat --delay-calibrator {delay_cat} --image-catalogue {image_cat} --apply-delay-solutions {target_ms_path}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with ( + open( + f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt", + "w+", + ) as f_out, + open( + f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w+", + ) as f_err, + ): + proc = subprocess.run(cmd, shell=True, text=True, stdout=f_out, stderr=f_err) + jobid = None + if not proc.returncode: + f_out.seek(0) + for line in f_out.readlines(): + if "Submitted batch job" in line: + jobid = line.strip().split()[-1] + else: + raise RuntimeError("Failed to submit job.") + + if not jobid: + raise RuntimeError("Failed to retrieve job id") + else: + while True: + print(f"Polling LINC target job {jobid}") + poll_cmd = f"sacct -X -j {jobid} --format=State --noheader" + status = subprocess.run( + poll_cmd, shell=True, text=True, capture_output=True + ).stdout.strip() + if (status == "RUNNING") or (status == "PENDING"): + time.sleep(60) + elif status == "COMPLETED": + if NEEDS_MANUAL_APPROVAL_DELAY: + CURRENT_DB.set_status_await_approval( + field["target_name"], "vlbi_delay", field["sas_id_target"] + ) + else: + CURRENT_DB.set_status_finished( + field["target_name"], "vlbi_delay", field["sas_id_target"] + ) + break + elif ( + (status == "FAILED") + or ("TIMEOUT" in status) + or ("CANCELLED" in status) + ): + raise RuntimeError( + f"LINC target for {field['target_name']} {field['sas_id_target']} failed." + ) + + +def run_pilot_delay_toil(field): + print( + f"Processing delay calibration for {field['target_name']} {field['sas_id_target']}" + ) + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + target_path = get_most_recent_run(outdir, field["sas_id_target"], "LINC_target") + target_ms_path = target_path / "results_LINC_target" / "results" + CURRENT_DB.set_status_processing( + field["target_name"], "vlbi_delay", field["sas_id_target"] + ) + + delay_cat = os.path.join(outdir, "delay_calibrators.csv") + image_cat = os.path.join(outdir, "image_catalogue.csv") + + if not os.path.isfile(delay_cat) or not os.path.isfile(image_cat): + ms = list(target_ms_path.glob("*.dp3concat"))[0] + cmd = f"lofar-vlbi-plot --force --output_dir {outdir} --MS {ms}" + with ( + open( + f"log_plot_field_{field['target_name']}_{field['sas_id_target']}.txt", + "w+", + ) as f_out, + open( + f"log_plot_field_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w+", + ) as f_err, + ): + proc = subprocess.run( + cmd, shell=True, text=True, stdout=f_out, stderr=f_err + ) + delay_cat = os.path.join(outdir, "delay_calibrators.csv") + image_cat = os.path.join(outdir, "image_catalogue.csv") + + if not os.path.isfile(delay_cat): + raise RuntimeError("Delay calibrator catalogue is missing or invalid.") + if not os.path.isfile(image_cat): + raise RuntimeError("Image source catalogue is missing or invalid.") + + proc = subprocess.run( + "detect_bad_slurm_nodes.sh", + shell=True, + text=True, + stdout=subprocess.PIPE, + ) + bad_nodes = proc.stdout.strip() + if bad_nodes: + print(f"Excluding the following bad nodes from scheduling: {bad_nodes}") + os.environ["TOIL_SLURM_ARGS"] = f"--exclude={bad_nodes}" + + context = get_current_context() + if context["ti"].try_number == 1 or ( + not os.path.isfile( + f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt" + ) + ): + cmd = f"flocs-run vlbi delay-calibration --record-toil-stats --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --ms-suffix dp3concat --delay-calibrator {delay_cat} --image-catalogue {image_cat} --apply-delay-solutions {target_ms_path}" + else: + # Extract the previous working directory + flocs_workdir = "" + print( + f"Scanning log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt for workdir." + ) + with open( + f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt" + ) as f_out: + for line in f_out.readlines(): + print(line) + if "Running workflow with" in line: + flocs_workdir = line.split(" ")[-1].strip() + break + if not flocs_workdir: + raise RuntimeError( + "Could not retrieve PILOT workdir. Flocs probably crashed before launching." + ) + print(f"Resuming failed PILOT run in {flocs_workdir}") + cmd = f"flocs-run vlbi delay-calibration --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --ms-suffix dp3concat --delay-calibrator {delay_cat} --image-catalogue {image_cat} {target_ms_path}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with ( + open( + f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt", + "w+", + ) as f_out, + open( + f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w+", + ) as f_err, + ): + proc = subprocess.run(cmd, shell=True, text=True, stdout=f_out, stderr=f_err) + success = False + pattern = re.compile(r"Workflow.* stopped. Success: True") + if not proc.returncode: + f_err.seek(0) + if pattern.search(f_err.read()): + success = True + + if success: + if NEEDS_MANUAL_APPROVAL_DELAY: + CURRENT_DB.set_status_await_approval( + field["target_name"], "vlbi_delay", field["sas_id_target"] + ) + else: + CURRENT_DB.set_status_finished( + field["target_name"], "vlbi_delay", field["sas_id_target"] + ) + else: + raise RuntimeError + + def get_approval(field, identifier, needs_approval): if not needs_approval: return field @@ -644,126 +859,15 @@ def validate_linc_target(field): @task(retries=0, retry_delay=datetime.timedelta(seconds=5)) def run_vlbi_delay(field): - if (field["status_vlbi_delay"] == PIPELINE_STATUS.finished) or ( - field["status_vlbi_delay"] == PIPELINE_STATUS.await_approval - ): + if field["status_vlbi_delay"] == PIPELINE_STATUS.finished: return field else: - print( - f"Processing delay calibration for {field['target_name']} {field['sas_id_target']}" - ) - outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - target_path = get_most_recent_run( - outdir, field["sas_id_target"], "LINC_target" - ) - target_ms_path = target_path / "results_LINC_target" / "results" - CURRENT_DB.set_status_processing( - field["target_name"], "vlbi_delay", field["sas_id_target"] - ) - - delay_cat = os.path.join(outdir, "delay_calibrators.csv") - image_cat = os.path.join(outdir, "image_catalogue.csv") - - if not os.path.isfile(delay_cat) or not os.path.isfile(image_cat): - ms = list(target_ms_path.glob("*.dp3concat"))[0] - cmd = f"lofar-vlbi-plot --force --output_dir {outdir} --MS {ms}" - with ( - open( - f"log_plot_field_{field['target_name']}_{field['sas_id_target']}.txt", - "w+", - ) as f_out, - open( - f"log_plot_field_{field['target_name']}_{field['sas_id_target']}_err.txt", - "w+", - ) as f_err, - ): - proc = subprocess.run( - cmd, shell=True, text=True, stdout=f_out, stderr=f_err - ) - delay_cat = os.path.join(outdir, "delay_calibrators.csv") - image_cat = os.path.join(outdir, "image_catalogue.csv") - - if not os.path.isfile(delay_cat): - raise RuntimeError( - "Delay calibrator catalogue is missing or invalid." - ) - if not os.path.isfile(image_cat): - raise RuntimeError( - "Image source catalogue is missing or invalid." - ) - - proc = subprocess.run( - "detect_bad_slurm_nodes.sh", - shell=True, - text=True, - stdout=subprocess.PIPE, - ) - bad_nodes = proc.stdout.strip() - if bad_nodes: - print(f"Excluding the following bad nodes from scheduling: {bad_nodes}") - os.environ["TOIL_SLURM_ARGS"] = f"--exclude={bad_nodes}" - - context = get_current_context() - if context["ti"].try_number == 1 or ( - not os.path.isfile( - f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt" - ) - ): - cmd = f"flocs-run vlbi delay-calibration --record-toil-stats --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --ms-suffix dp3concat --delay-calibrator {delay_cat} --image-catalogue {image_cat} --apply-delay-solutions {target_ms_path}" + if CWL_RUNNER_PILOT_DELAY == "cwltool": + run_pilot_delay_cwltool(field) + elif CWL_RUNNER_PILOT_DELAY == "toil": + run_pilot_delay_toil(field) else: - # Extract the previous working directory - flocs_workdir = "" - print( - f"Scanning log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt for workdir." - ) - with open( - f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt" - ) as f_out: - for line in f_out.readlines(): - print(line) - if "Running workflow with" in line: - flocs_workdir = line.split(" ")[-1].strip() - break - if not flocs_workdir: - raise RuntimeError( - "Could not retrieve PILOT workdir. Flocs probably crashed before launching." - ) - print(f"Resuming failed PILOT run in {flocs_workdir}") - cmd = f"flocs-run vlbi delay-calibration --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --ms-suffix dp3concat --delay-calibrator {delay_cat} --image-catalogue {image_cat} {target_ms_path}" - if not os.path.isdir(outdir): - os.mkdir(outdir) - print(cmd) - with ( - open( - f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt", - "w+", - ) as f_out, - open( - f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}_err.txt", - "w+", - ) as f_err, - ): - proc = subprocess.run( - cmd, shell=True, text=True, stdout=f_out, stderr=f_err - ) - success = False - pattern = re.compile(r"Workflow.* stopped. Success: True") - if not proc.returncode: - f_err.seek(0) - if pattern.search(f_err.read()): - success = True - - if success: - if NEEDS_MANUAL_APPROVAL_DELAY: - CURRENT_DB.set_status_await_approval( - field["target_name"], "vlbi_delay", field["sas_id_target"] - ) - else: - CURRENT_DB.set_status_finished( - field["target_name"], "vlbi_delay", field["sas_id_target"] - ) - else: - raise RuntimeError + raise RuntimeError("Invalid CWL runner specified.") return field @task @@ -1394,7 +1498,7 @@ def run_vlbi_image_intermediate(field): @task def run_vlbi_facet_subtract(field): - field = dict(CURRENT_DB.CURRENT_DB.get_db_columns(field["sas_id_target"])[0]) + field = dict(CURRENT_DB.get_db_columns(field["sas_id_target"])[0]) if (field["status_vlbi_facet_subtract"] == PIPELINE_STATUS.finished) or ( field["status_vlbi_facet_subtract"] == PIPELINE_STATUS.processing ): From 00aa1ed280938ab7b4001f9f868adffa3821bc52 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 7 Aug 2026 12:20:20 +0200 Subject: [PATCH 088/120] Fix update if statement --- flocs_processing/flocs_processing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flocs_processing/flocs_processing.py b/flocs_processing/flocs_processing.py index 580d3dd..00be8cf 100644 --- a/flocs_processing/flocs_processing.py +++ b/flocs_processing/flocs_processing.py @@ -153,7 +153,7 @@ def update_field( ] = "processing_flocs", ): db = FlocsDB(dbname=dbname, db_table=table_name) - if pipeline != "all": + if pipeline == "all": p_list = list(get_args(PIPELINES)) p_list.remove("all") for p in p_list: From 9effc08ebc093a326fcd0d73dd55bd3ef25c087b Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 7 Aug 2026 14:35:12 +0200 Subject: [PATCH 089/120] Rename airflow deployment --- flocs_processing/flocs_processing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flocs_processing/flocs_processing.py b/flocs_processing/flocs_processing.py index 00be8cf..db9becb 100644 --- a/flocs_processing/flocs_processing.py +++ b/flocs_processing/flocs_processing.py @@ -185,7 +185,7 @@ def update_field( @app.command() -def start_airflow_processing( +def deploy_airflow( airflow_cores: Annotated[ int, Parameter(help="Number of cores to give to Airflow.") ] = 6, From df047431ae7639c91569468002e9c174a6116844 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 7 Aug 2026 14:47:49 +0200 Subject: [PATCH 090/120] Change python version bound to make ty happy --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index cc255e7..b100231 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" name = "flocs-processing" version = "0.0.0" description = "" -requires-python = ">3.9" +requires-python = ">=3.10" dependencies = ["cyclopts", "structlog", "packaging"] [project.scripts] From 4ab880c0e562af6f125313a68444e260688a2d2d Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 7 Aug 2026 14:55:00 +0200 Subject: [PATCH 091/120] Implement cwltool for ddcal --- flocs_processing/dags/pilot_widefield.py | 314 +++++++++++++++-------- 1 file changed, 200 insertions(+), 114 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 7c87136..50bc07a 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -53,6 +53,7 @@ CWL_RUNNER_LINC_CALIBRATOR = "cwltool" CWL_RUNNER_LINC_TARGET = "toil" CWL_RUNNER_PILOT_DELAY = "toil" +CWL_RUNNER_PILOT_DDCAL = "toil" CURRENT_DB = FlocsDB(DATABASE, TABLE_NAME) @@ -366,7 +367,7 @@ def run_pilot_delay_cwltool(field): or ("CANCELLED" in status) ): raise RuntimeError( - f"LINC target for {field['target_name']} {field['sas_id_target']} failed." + f"PILOT delay calibration for {field['target_name']} {field['sas_id_target']} failed." ) @@ -480,6 +481,199 @@ def run_pilot_delay_toil(field): raise RuntimeError +def run_pilot_ddcal_cwltool(field): + print( + f"Processing ILT dd calibration for {field['target_name']} {field['sas_id_target']}" + ) + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + if "status_ddf" not in field: + print("Not a widefield imaging run, checking LINC + delay calibration.") + target_path = get_most_recent_run(outdir, field["sas_id_target"], "LINC_target") + target_ms_path = target_path / "results_LINC_target" / "results" + print(f"Using LINC target run: {target_path}") + + sols_path = get_most_recent_run(outdir, field["sas_id_target"], "VLBI_delay") + sols_path = sols_path / "results_VLBI_delay-calibration" + sols = list(sols_path.glob("merged*selfcalcycle???_linearfulljones*.h5"))[0] + print(f"Using PILOT delay calibration solutions: {sols}") + + source_cat = os.path.join(DATA_DIR, field["target_name"], "vlbi_target.csv") + if not os.path.isfile(source_cat): + raise AirflowFailException(f"{source_cat} not found.") + + CURRENT_DB.set_status_processing( + field["target_name"], "vlbi_dd", field["sas_id_target"] + ) + cmd = f"flocs-run vlbi dd-calibration --runner cwltool --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --delay-solset {sols} --phasediff-score 10.0 --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat {target_ms_path}" + else: + print("Widefield imaging run, checking subtraction output.") + target_path = get_most_recent_run( + outdir, field["sas_id_target"], "VLBI_process-ddf" + ) + target_ms_path = target_path / "results_VLBI_process-ddf" + print(f"Using subtracted data at: {target_path}") + + source_cat = os.path.join(DATA_DIR, field["target_name"], "image_catalogue.csv") + if not os.path.isfile(source_cat): + raise AirflowFailException(f"{source_cat} not found.") + + CURRENT_DB.set_status_processing( + field["target_name"], "vlbi_dd", field["sas_id_target"] + ) + + cmd = f"flocs-run vlbi dd-calibration --runner cwltool --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat.sub.ms {target_ms_path}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with ( + open( + f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt", + "w+", + ) as f_out, + open( + f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w+", + ) as f_err, + ): + proc = subprocess.run(cmd, shell=True, text=True, stdout=f_out, stderr=f_err) + jobid = None + if not proc.returncode: + f_out.seek(0) + for line in f_out.readlines(): + if "Submitted batch job" in line: + jobid = line.strip().split()[-1] + else: + raise RuntimeError("Failed to submit job.") + + if not jobid: + raise RuntimeError("Failed to retrieve job id") + else: + while True: + print(f"Polling LINC target job {jobid}") + poll_cmd = f"sacct -X -j {jobid} --format=State --noheader" + status = subprocess.run( + poll_cmd, shell=True, text=True, capture_output=True + ).stdout.strip() + if (status == "RUNNING") or (status == "PENDING"): + time.sleep(60) + elif status == "COMPLETED": + CURRENT_DB.set_status_finished( + field["target_name"], "vlbi_dd", field["sas_id_target"] + ) + break + elif ( + (status == "FAILED") + or ("TIMEOUT" in status) + or ("CANCELLED" in status) + ): + raise RuntimeError( + f"PILOT direction-dependent calibration for {field['target_name']} {field['sas_id_target']} failed." + ) + + +def run_pilot_ddcal_toil(field): + print( + f"Processing ILT dd calibration for {field['target_name']} {field['sas_id_target']}" + ) + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + if "status_ddf" not in field: + print("Not a widefield imaging run, checking LINC + delay calibration.") + target_path = get_most_recent_run(outdir, field["sas_id_target"], "LINC_target") + target_ms_path = target_path / "results_LINC_target" / "results" + print(f"Using LINC target run: {target_path}") + + sols_path = get_most_recent_run(outdir, field["sas_id_target"], "VLBI_delay") + sols_path = sols_path / "results_VLBI_delay-calibration" + sols = list(sols_path.glob("merged*selfcalcycle???_linearfulljones*.h5"))[0] + print(f"Using PILOT delay calibration solutions: {sols}") + + source_cat = os.path.join(DATA_DIR, field["target_name"], "vlbi_target.csv") + if not os.path.isfile(source_cat): + raise AirflowFailException(f"{source_cat} not found.") + + CURRENT_DB.set_status_processing( + field["target_name"], "vlbi_dd", field["sas_id_target"] + ) + cmd = f"flocs-run vlbi dd-calibration --runner toil --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --delay-solset {sols} --phasediff-score 10.0 --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat {target_ms_path}" + else: + print("Widefield imaging run, checking subtraction output.") + target_path = get_most_recent_run( + outdir, field["sas_id_target"], "VLBI_process-ddf" + ) + target_ms_path = target_path / "results_VLBI_process-ddf" + print(f"Using subtracted data at: {target_path}") + + source_cat = os.path.join(DATA_DIR, field["target_name"], "image_catalogue.csv") + if not os.path.isfile(source_cat): + raise AirflowFailException(f"{source_cat} not found.") + + CURRENT_DB.set_status_processing( + field["target_name"], "vlbi_dd", field["sas_id_target"] + ) + + context = get_current_context() + if context["ti"].try_number == 1 or ( + not os.path.isfile( + f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt" + ) + ): + cmd = f"flocs-run vlbi dd-calibration --record-toil-stats --runner toil --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat.sub.ms {target_ms_path}" + else: + if field["status_vlbi_dd"] == PIPELINE_STATUS.downloaded: + # This way we can force a clean restart in the database. + cmd = f"flocs-run vlbi dd-calibration --record-toil-stats --runner toil --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat.sub.ms {target_ms_path}" + else: + # Extract the previous working directory + flocs_workdir = "" + print( + f"Scanning log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt for workdir." + ) + with open( + f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt" + ) as f_out: + for line in f_out.readlines(): + print(line) + if "Running workflow with" in line: + flocs_workdir = line.split(" ")[-1].strip() + break + if not flocs_workdir: + raise RuntimeError( + "Could not retrieve PILOT workdir. Flocs probably crashed before launching." + ) + print(f"Resuming failed PILOT run in {flocs_workdir}") + cmd = f"flocs-run vlbi dd-calibration --record-toil-stats --runner toil --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat.sub.ms {target_ms_path}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with ( + open( + f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt", + "w+", + ) as f_out, + open( + f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w+", + ) as f_err, + ): + proc = subprocess.run(cmd, shell=True, text=True, stdout=f_out, stderr=f_err) + success = False + pattern = re.compile(r"Workflow.* stopped. Success: True") + if not proc.returncode: + f_err.seek(0) + if pattern.search(f_err.read()): + success = True + if success: + CURRENT_DB.set_status_finished( + field["target_name"], "vlbi_dd", field["sas_id_target"] + ) + CURRENT_DB.set_field_finished(field["target_name"], field["sas_id_target"]) + else: + CURRENT_DB.set_status_failed( + field["target_name"], "vlbi_dd", field["sas_id_target"] + ) + raise RuntimeError + + def get_approval(field, identifier, needs_approval): if not needs_approval: return field @@ -1073,120 +1267,12 @@ def run_vlbi_ddcal(field): ): return field else: - print( - f"Processing ILT dd calibration for {field['target_name']} {field['sas_id_target']}" - ) - outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - if "status_ddf" not in field: - print("Not a widefield imaging run, checking LINC + delay calibration.") - target_path = get_most_recent_run( - outdir, field["sas_id_target"], "LINC_target" - ) - target_ms_path = target_path / "results_LINC_target" / "results" - print(f"Using LINC target run: {target_path}") - - sols_path = get_most_recent_run( - outdir, field["sas_id_target"], "VLBI_delay" - ) - sols_path = sols_path / "results_VLBI_delay-calibration" - sols = list( - sols_path.glob("merged*selfcalcycle???_linearfulljones*.h5") - )[0] - print(f"Using PILOT delay calibration solutions: {sols}") - - source_cat = os.path.join( - DATA_DIR, field["target_name"], "vlbi_target.csv" - ) - if not os.path.isfile(source_cat): - raise AirflowFailException(f"{source_cat} not found.") - - CURRENT_DB.set_status_processing( - field["target_name"], "vlbi_dd", field["sas_id_target"] - ) - cmd = f"flocs-run vlbi dd-calibration --runner toil --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --delay-solset {sols} --phasediff-score 10.0 --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat {target_ms_path}" + if CWL_RUNNER_PILOT_DDCAL == "cwltool": + run_pilot_ddcal_cwltool(field) + elif CWL_RUNNER_PILOT_DDCAL == "toil": + run_pilot_ddcal_toil(field) else: - print("Widefield imaging run, checking subtraction output.") - target_path = get_most_recent_run( - outdir, field["sas_id_target"], "VLBI_process-ddf" - ) - target_ms_path = target_path / "results_VLBI_process-ddf" - print(f"Using subtracted data at: {target_path}") - - source_cat = os.path.join( - DATA_DIR, field["target_name"], "image_catalogue.csv" - ) - if not os.path.isfile(source_cat): - raise AirflowFailException(f"{source_cat} not found.") - - CURRENT_DB.set_status_processing( - field["target_name"], "vlbi_dd", field["sas_id_target"] - ) - - context = get_current_context() - if context["ti"].try_number == 1 or ( - not os.path.isfile( - f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt" - ) - ): - cmd = f"flocs-run vlbi dd-calibration --record-toil-stats --runner toil --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat.sub.ms {target_ms_path}" - else: - if field["status_vlbi_dd"] == PIPELINE_STATUS.downloaded: - # This way we can force a clean restart in the database. - cmd = f"flocs-run vlbi dd-calibration --record-toil-stats --runner toil --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat.sub.ms {target_ms_path}" - else: - # Extract the previous working directory - flocs_workdir = "" - print( - f"Scanning log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt for workdir." - ) - with open( - f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt" - ) as f_out: - for line in f_out.readlines(): - print(line) - if "Running workflow with" in line: - flocs_workdir = line.split(" ")[-1].strip() - break - if not flocs_workdir: - raise RuntimeError( - "Could not retrieve PILOT workdir. Flocs probably crashed before launching." - ) - print(f"Resuming failed PILOT run in {flocs_workdir}") - cmd = f"flocs-run vlbi dd-calibration --record-toil-stats --runner toil --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat.sub.ms {target_ms_path}" - if not os.path.isdir(outdir): - os.mkdir(outdir) - print(cmd) - with ( - open( - f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt", - "w+", - ) as f_out, - open( - f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}_err.txt", - "w+", - ) as f_err, - ): - proc = subprocess.run( - cmd, shell=True, text=True, stdout=f_out, stderr=f_err - ) - success = False - pattern = re.compile(r"Workflow.* stopped. Success: True") - if not proc.returncode: - f_err.seek(0) - if pattern.search(f_err.read()): - success = True - if success: - CURRENT_DB.set_status_finished( - field["target_name"], "vlbi_dd", field["sas_id_target"] - ) - CURRENT_DB.set_field_finished( - field["target_name"], field["sas_id_target"] - ) - else: - CURRENT_DB.set_status_failed( - field["target_name"], "vlbi_dd", field["sas_id_target"] - ) - raise RuntimeError + raise RuntimeError("Invalid CWL runner specified.") return field @task From f53703116a8ad14a8b025edde164ccb27e22059e Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 7 Aug 2026 14:56:03 +0200 Subject: [PATCH 092/120] Add cores to cwltool calls --- flocs_processing/dags/pilot_widefield.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 50bc07a..3b4a224 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -504,7 +504,7 @@ def run_pilot_ddcal_cwltool(field): CURRENT_DB.set_status_processing( field["target_name"], "vlbi_dd", field["sas_id_target"] ) - cmd = f"flocs-run vlbi dd-calibration --runner cwltool --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --delay-solset {sols} --phasediff-score 10.0 --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat {target_ms_path}" + cmd = f"flocs-run vlbi dd-calibration --runner cwltool --scheduler slurm --slurm-cores 32 --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --delay-solset {sols} --phasediff-score 10.0 --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat {target_ms_path}" else: print("Widefield imaging run, checking subtraction output.") target_path = get_most_recent_run( @@ -521,7 +521,7 @@ def run_pilot_ddcal_cwltool(field): field["target_name"], "vlbi_dd", field["sas_id_target"] ) - cmd = f"flocs-run vlbi dd-calibration --runner cwltool --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat.sub.ms {target_ms_path}" + cmd = f"flocs-run vlbi dd-calibration --runner cwltool --scheduler slurm --slurm-cores 64 --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat.sub.ms {target_ms_path}" if not os.path.isdir(outdir): os.mkdir(outdir) print(cmd) From 2e7696c175ee8679abdf2f5d02031ae0a24af6b6 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 7 Aug 2026 15:24:45 +0200 Subject: [PATCH 093/120] Extract run commands into submodule --- flocs_processing/dags/pilot_widefield.py | 646 +--------------------- flocs_processing/pipeline_runners.py | 665 +++++++++++++++++++++++ 2 files changed, 678 insertions(+), 633 deletions(-) create mode 100644 flocs_processing/pipeline_runners.py diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 3b4a224..b97e956 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -1,4 +1,15 @@ from flocs_processing.db_utils import PIPELINE_STATUS, FlocsDB +from flocs_processing.pipeline_runners import ( + get_most_recent_run, + run_linc_calibrator_cwltool, + run_linc_calibrator_toil, + run_linc_target_cwltool, + run_linc_target_toil, + run_pilot_delay_cwltool, + run_pilot_delay_toil, + run_pilot_ddcal_cwltool, + run_pilot_ddcal_toil, +) import configparser import datetime @@ -58,622 +69,6 @@ CURRENT_DB = FlocsDB(DATABASE, TABLE_NAME) -def run_linc_calibrator_cwltool(field, calibrator_field: int): - print( - f"Processing flux density calibrator {field[f'sas_id_calibrator{calibrator_field}']} for observation {field['target_name']} {field['sas_id_target']}" - ) - ms_folder = f"L{field[f'sas_id_calibrator{calibrator_field}']}" - CURRENT_DB.set_status_processing( - field["target_name"], f"calibrator{calibrator_field}", field["sas_id_target"] - ) - outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - cmd = f"flocs-run linc calibrator --runner cwltool --scheduler slurm --slurm-cores 32 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} {os.path.join(DATA_DIR, field['target_name'], 'calibrator', ms_folder)}" - with ( - open( - f"log_LINC_calibrator_{field['target_name']}_{field[f'sas_id_calibrator{calibrator_field}']}.txt", - "w+", - ) as f_out, - open( - f"log_LINC_calibrator_{field['target_name']}_{field[f'sas_id_calibrator{calibrator_field}']}_err.txt", - "w+", - ) as f_err, - ): - proc = subprocess.run(cmd, shell=True, text=True, stdout=f_out, stderr=f_err) - jobid = None - if not proc.returncode: - f_out.seek(0) - for line in f_out.readlines(): - if "Submitted batch job" in line: - jobid = line.strip().split()[-1] - else: - raise RuntimeError("Failed to submit job.") - - if not jobid: - raise RuntimeError("Failed to retrieve job id") - else: - while True: - print(f"Polling LINC calibrator job {jobid}") - poll_cmd = f"sacct -X -j {jobid} --format=State --noheader" - status = subprocess.run( - poll_cmd, shell=True, text=True, capture_output=True - ).stdout.strip() - if (status == "RUNNING") or (status == "PENDING"): - time.sleep(60) - elif status == "COMPLETED": - CURRENT_DB.set_status_finished( - field["target_name"], - f"calibrator{calibrator_field}", - field["sas_id_target"], - ) - break - elif ( - (status == "FAILED") - or ("TIMEOUT" in status) - or ("CANCELLED" in status) - ): - raise RuntimeError( - f"LINC calibrator for {field['target_name']} {field['sas_id_target']} failed." - ) - - -def run_linc_calibrator_toil(field, calibrator_field: int): - print( - f"Processing flux density calibrator {field[f'sas_id_calibrator{calibrator_field}']} for observation {field['target_name']} {field['sas_id_target']}" - ) - ms_folder = f"L{field[f'sas_id_calibrator{calibrator_field}']}" - CURRENT_DB.set_status_processing( - field["target_name"], f"calibrator{calibrator_field}", field["sas_id_target"] - ) - outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - cmd = f"flocs-run linc calibrator --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} {os.path.join(DATA_DIR, field['target_name'], 'calibrator', ms_folder)}" - if not os.path.isdir(outdir): - os.mkdir(outdir) - print(cmd) - with ( - open( - f"log_LINC_calibrator_{field['target_name']}_{field[f'sas_id_calibrator{calibrator_field}']}.txt", - "w+", - ) as f_out, - open( - f"log_LINC_calibrator_{field['target_name']}_{field[f'sas_id_calibrator{calibrator_field}']}_err.txt", - "w+", - ) as f_err, - ): - proc = subprocess.run(cmd, shell=True, text=True, stdout=f_out, stderr=f_err) - success = False - pattern = re.compile(r"Workflow.* stopped. Success: True") - if not proc.returncode: - f_err.seek(0) - if pattern.search(f_err.read()): - success = True - if success: - CURRENT_DB.set_status_finished( - field["target_name"], - f"calibrator{calibrator_field}", - field["sas_id_target"], - ) - else: - raise RuntimeError - - -def run_linc_target_cwltool(field): - print( - f"Processing target observation {field['target_name']} {field['sas_id_target']} with calibrator {field['sas_id_calibrator_final']}" - ) - ms_folder = f"L{field['sas_id_target']}" - outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - calibrator_path = get_most_recent_run( - outdir, field["sas_id_calibrator_final"], "LINC_calibrator" - ) - calibrator_solutions = ( - calibrator_path / "results_LINC_calibrator" / "cal_solutions.h5" - ) - CURRENT_DB.set_status_processing( - field["target_name"], "target", field["sas_id_target"] - ) - cmd = f"flocs-run linc target --runner cwltool --scheduler slurm --slurm-cores 64 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --cal-solutions {calibrator_solutions} {os.path.join(DATA_DIR, field['target_name'], 'target', ms_folder)}" - if not os.path.isdir(outdir): - os.mkdir(outdir) - print(cmd) - with ( - open( - f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}.txt", - "w+", - ) as f_out, - open( - f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}_err.txt", - "w+", - ) as f_err, - ): - proc = subprocess.run(cmd, shell=True, text=True, stdout=f_out, stderr=f_err) - jobid = None - if not proc.returncode: - f_out.seek(0) - for line in f_out.readlines(): - if "Submitted batch job" in line: - jobid = line.strip().split()[-1] - else: - raise RuntimeError("Failed to submit job.") - - if not jobid: - raise RuntimeError("Failed to retrieve job id") - else: - while True: - print(f"Polling LINC target job {jobid}") - poll_cmd = f"sacct -X -j {jobid} --format=State --noheader" - status = subprocess.run( - poll_cmd, shell=True, text=True, capture_output=True - ).stdout.strip() - if (status == "RUNNING") or (status == "PENDING"): - time.sleep(60) - elif status == "COMPLETED": - CURRENT_DB.set_status_finished( - field["target_name"], - "target", - field["sas_id_target"], - ) - break - elif ( - (status == "FAILED") - or ("TIMEOUT" in status) - or ("CANCELLED" in status) - ): - raise RuntimeError( - f"LINC target for {field['target_name']} {field['sas_id_target']} failed." - ) - - -def run_linc_target_toil(field): - print( - f"Processing target observation {field['target_name']} {field['sas_id_target']} with calibrator {field['sas_id_calibrator_final']}" - ) - ms_folder = f"L{field['sas_id_target']}" - outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - calibrator_path = get_most_recent_run( - outdir, field["sas_id_calibrator_final"], "LINC_calibrator" - ) - calibrator_solutions = ( - calibrator_path / "results_LINC_calibrator" / "cal_solutions.h5" - ) - CURRENT_DB.set_status_processing( - field["target_name"], "target", field["sas_id_target"] - ) - cmd = f"flocs-run linc target --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --cal-solutions {calibrator_solutions} {os.path.join(DATA_DIR, field['target_name'], 'target', ms_folder)}" - if not os.path.isdir(outdir): - os.mkdir(outdir) - print(cmd) - with ( - open( - f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}.txt", - "w+", - ) as f_out, - open( - f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}_err.txt", - "w+", - ) as f_err, - ): - proc = subprocess.run(cmd, shell=True, text=True, stdout=f_out, stderr=f_err) - success = False - pattern = re.compile(r"Workflow.* stopped. Success: True") - if not proc.returncode: - f_err.seek(0) - if pattern.search(f_err.read()): - success = True - if success: - CURRENT_DB.set_status_finished( - field["target_name"], "target", field["sas_id_target"] - ) - else: - raise RuntimeError - - -def run_pilot_delay_cwltool(field): - print( - f"Processing delay calibration for {field['target_name']} {field['sas_id_target']}" - ) - outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - target_path = get_most_recent_run(outdir, field["sas_id_target"], "LINC_target") - target_ms_path = target_path / "results_LINC_target" / "results" - CURRENT_DB.set_status_processing( - field["target_name"], "vlbi_delay", field["sas_id_target"] - ) - - delay_cat = os.path.join(outdir, "delay_calibrators.csv") - image_cat = os.path.join(outdir, "image_catalogue.csv") - - if not os.path.isfile(delay_cat) or not os.path.isfile(image_cat): - ms = list(target_ms_path.glob("*.dp3concat"))[0] - cmd = f"lofar-vlbi-plot --force --output_dir {outdir} --MS {ms}" - with ( - open( - f"log_plot_field_{field['target_name']}_{field['sas_id_target']}.txt", - "w+", - ) as f_out, - open( - f"log_plot_field_{field['target_name']}_{field['sas_id_target']}_err.txt", - "w+", - ) as f_err, - ): - proc = subprocess.run( - cmd, shell=True, text=True, stdout=f_out, stderr=f_err - ) - delay_cat = os.path.join(outdir, "delay_calibrators.csv") - image_cat = os.path.join(outdir, "image_catalogue.csv") - - if not os.path.isfile(delay_cat): - raise RuntimeError("Delay calibrator catalogue is missing or invalid.") - if not os.path.isfile(image_cat): - raise RuntimeError("Image source catalogue is missing or invalid.") - - proc = subprocess.run( - "detect_bad_slurm_nodes.sh", - shell=True, - text=True, - stdout=subprocess.PIPE, - ) - bad_nodes = proc.stdout.strip() - if bad_nodes: - print(f"Excluding the following bad nodes from scheduling: {bad_nodes}") - os.environ["TOIL_SLURM_ARGS"] = f"--exclude={bad_nodes}" - - cmd = f"flocs-run vlbi delay-calibration --record-toil-stats --runner cwltool --scheduler slurm --slurm-cores 64 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --ms-suffix dp3concat --delay-calibrator {delay_cat} --image-catalogue {image_cat} --apply-delay-solutions {target_ms_path}" - if not os.path.isdir(outdir): - os.mkdir(outdir) - print(cmd) - with ( - open( - f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt", - "w+", - ) as f_out, - open( - f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}_err.txt", - "w+", - ) as f_err, - ): - proc = subprocess.run(cmd, shell=True, text=True, stdout=f_out, stderr=f_err) - jobid = None - if not proc.returncode: - f_out.seek(0) - for line in f_out.readlines(): - if "Submitted batch job" in line: - jobid = line.strip().split()[-1] - else: - raise RuntimeError("Failed to submit job.") - - if not jobid: - raise RuntimeError("Failed to retrieve job id") - else: - while True: - print(f"Polling LINC target job {jobid}") - poll_cmd = f"sacct -X -j {jobid} --format=State --noheader" - status = subprocess.run( - poll_cmd, shell=True, text=True, capture_output=True - ).stdout.strip() - if (status == "RUNNING") or (status == "PENDING"): - time.sleep(60) - elif status == "COMPLETED": - if NEEDS_MANUAL_APPROVAL_DELAY: - CURRENT_DB.set_status_await_approval( - field["target_name"], "vlbi_delay", field["sas_id_target"] - ) - else: - CURRENT_DB.set_status_finished( - field["target_name"], "vlbi_delay", field["sas_id_target"] - ) - break - elif ( - (status == "FAILED") - or ("TIMEOUT" in status) - or ("CANCELLED" in status) - ): - raise RuntimeError( - f"PILOT delay calibration for {field['target_name']} {field['sas_id_target']} failed." - ) - - -def run_pilot_delay_toil(field): - print( - f"Processing delay calibration for {field['target_name']} {field['sas_id_target']}" - ) - outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - target_path = get_most_recent_run(outdir, field["sas_id_target"], "LINC_target") - target_ms_path = target_path / "results_LINC_target" / "results" - CURRENT_DB.set_status_processing( - field["target_name"], "vlbi_delay", field["sas_id_target"] - ) - - delay_cat = os.path.join(outdir, "delay_calibrators.csv") - image_cat = os.path.join(outdir, "image_catalogue.csv") - - if not os.path.isfile(delay_cat) or not os.path.isfile(image_cat): - ms = list(target_ms_path.glob("*.dp3concat"))[0] - cmd = f"lofar-vlbi-plot --force --output_dir {outdir} --MS {ms}" - with ( - open( - f"log_plot_field_{field['target_name']}_{field['sas_id_target']}.txt", - "w+", - ) as f_out, - open( - f"log_plot_field_{field['target_name']}_{field['sas_id_target']}_err.txt", - "w+", - ) as f_err, - ): - proc = subprocess.run( - cmd, shell=True, text=True, stdout=f_out, stderr=f_err - ) - delay_cat = os.path.join(outdir, "delay_calibrators.csv") - image_cat = os.path.join(outdir, "image_catalogue.csv") - - if not os.path.isfile(delay_cat): - raise RuntimeError("Delay calibrator catalogue is missing or invalid.") - if not os.path.isfile(image_cat): - raise RuntimeError("Image source catalogue is missing or invalid.") - - proc = subprocess.run( - "detect_bad_slurm_nodes.sh", - shell=True, - text=True, - stdout=subprocess.PIPE, - ) - bad_nodes = proc.stdout.strip() - if bad_nodes: - print(f"Excluding the following bad nodes from scheduling: {bad_nodes}") - os.environ["TOIL_SLURM_ARGS"] = f"--exclude={bad_nodes}" - - context = get_current_context() - if context["ti"].try_number == 1 or ( - not os.path.isfile( - f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt" - ) - ): - cmd = f"flocs-run vlbi delay-calibration --record-toil-stats --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --ms-suffix dp3concat --delay-calibrator {delay_cat} --image-catalogue {image_cat} --apply-delay-solutions {target_ms_path}" - else: - # Extract the previous working directory - flocs_workdir = "" - print( - f"Scanning log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt for workdir." - ) - with open( - f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt" - ) as f_out: - for line in f_out.readlines(): - print(line) - if "Running workflow with" in line: - flocs_workdir = line.split(" ")[-1].strip() - break - if not flocs_workdir: - raise RuntimeError( - "Could not retrieve PILOT workdir. Flocs probably crashed before launching." - ) - print(f"Resuming failed PILOT run in {flocs_workdir}") - cmd = f"flocs-run vlbi delay-calibration --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --ms-suffix dp3concat --delay-calibrator {delay_cat} --image-catalogue {image_cat} {target_ms_path}" - if not os.path.isdir(outdir): - os.mkdir(outdir) - print(cmd) - with ( - open( - f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt", - "w+", - ) as f_out, - open( - f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}_err.txt", - "w+", - ) as f_err, - ): - proc = subprocess.run(cmd, shell=True, text=True, stdout=f_out, stderr=f_err) - success = False - pattern = re.compile(r"Workflow.* stopped. Success: True") - if not proc.returncode: - f_err.seek(0) - if pattern.search(f_err.read()): - success = True - - if success: - if NEEDS_MANUAL_APPROVAL_DELAY: - CURRENT_DB.set_status_await_approval( - field["target_name"], "vlbi_delay", field["sas_id_target"] - ) - else: - CURRENT_DB.set_status_finished( - field["target_name"], "vlbi_delay", field["sas_id_target"] - ) - else: - raise RuntimeError - - -def run_pilot_ddcal_cwltool(field): - print( - f"Processing ILT dd calibration for {field['target_name']} {field['sas_id_target']}" - ) - outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - if "status_ddf" not in field: - print("Not a widefield imaging run, checking LINC + delay calibration.") - target_path = get_most_recent_run(outdir, field["sas_id_target"], "LINC_target") - target_ms_path = target_path / "results_LINC_target" / "results" - print(f"Using LINC target run: {target_path}") - - sols_path = get_most_recent_run(outdir, field["sas_id_target"], "VLBI_delay") - sols_path = sols_path / "results_VLBI_delay-calibration" - sols = list(sols_path.glob("merged*selfcalcycle???_linearfulljones*.h5"))[0] - print(f"Using PILOT delay calibration solutions: {sols}") - - source_cat = os.path.join(DATA_DIR, field["target_name"], "vlbi_target.csv") - if not os.path.isfile(source_cat): - raise AirflowFailException(f"{source_cat} not found.") - - CURRENT_DB.set_status_processing( - field["target_name"], "vlbi_dd", field["sas_id_target"] - ) - cmd = f"flocs-run vlbi dd-calibration --runner cwltool --scheduler slurm --slurm-cores 32 --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --delay-solset {sols} --phasediff-score 10.0 --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat {target_ms_path}" - else: - print("Widefield imaging run, checking subtraction output.") - target_path = get_most_recent_run( - outdir, field["sas_id_target"], "VLBI_process-ddf" - ) - target_ms_path = target_path / "results_VLBI_process-ddf" - print(f"Using subtracted data at: {target_path}") - - source_cat = os.path.join(DATA_DIR, field["target_name"], "image_catalogue.csv") - if not os.path.isfile(source_cat): - raise AirflowFailException(f"{source_cat} not found.") - - CURRENT_DB.set_status_processing( - field["target_name"], "vlbi_dd", field["sas_id_target"] - ) - - cmd = f"flocs-run vlbi dd-calibration --runner cwltool --scheduler slurm --slurm-cores 64 --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat.sub.ms {target_ms_path}" - if not os.path.isdir(outdir): - os.mkdir(outdir) - print(cmd) - with ( - open( - f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt", - "w+", - ) as f_out, - open( - f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}_err.txt", - "w+", - ) as f_err, - ): - proc = subprocess.run(cmd, shell=True, text=True, stdout=f_out, stderr=f_err) - jobid = None - if not proc.returncode: - f_out.seek(0) - for line in f_out.readlines(): - if "Submitted batch job" in line: - jobid = line.strip().split()[-1] - else: - raise RuntimeError("Failed to submit job.") - - if not jobid: - raise RuntimeError("Failed to retrieve job id") - else: - while True: - print(f"Polling LINC target job {jobid}") - poll_cmd = f"sacct -X -j {jobid} --format=State --noheader" - status = subprocess.run( - poll_cmd, shell=True, text=True, capture_output=True - ).stdout.strip() - if (status == "RUNNING") or (status == "PENDING"): - time.sleep(60) - elif status == "COMPLETED": - CURRENT_DB.set_status_finished( - field["target_name"], "vlbi_dd", field["sas_id_target"] - ) - break - elif ( - (status == "FAILED") - or ("TIMEOUT" in status) - or ("CANCELLED" in status) - ): - raise RuntimeError( - f"PILOT direction-dependent calibration for {field['target_name']} {field['sas_id_target']} failed." - ) - - -def run_pilot_ddcal_toil(field): - print( - f"Processing ILT dd calibration for {field['target_name']} {field['sas_id_target']}" - ) - outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - if "status_ddf" not in field: - print("Not a widefield imaging run, checking LINC + delay calibration.") - target_path = get_most_recent_run(outdir, field["sas_id_target"], "LINC_target") - target_ms_path = target_path / "results_LINC_target" / "results" - print(f"Using LINC target run: {target_path}") - - sols_path = get_most_recent_run(outdir, field["sas_id_target"], "VLBI_delay") - sols_path = sols_path / "results_VLBI_delay-calibration" - sols = list(sols_path.glob("merged*selfcalcycle???_linearfulljones*.h5"))[0] - print(f"Using PILOT delay calibration solutions: {sols}") - - source_cat = os.path.join(DATA_DIR, field["target_name"], "vlbi_target.csv") - if not os.path.isfile(source_cat): - raise AirflowFailException(f"{source_cat} not found.") - - CURRENT_DB.set_status_processing( - field["target_name"], "vlbi_dd", field["sas_id_target"] - ) - cmd = f"flocs-run vlbi dd-calibration --runner toil --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --delay-solset {sols} --phasediff-score 10.0 --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat {target_ms_path}" - else: - print("Widefield imaging run, checking subtraction output.") - target_path = get_most_recent_run( - outdir, field["sas_id_target"], "VLBI_process-ddf" - ) - target_ms_path = target_path / "results_VLBI_process-ddf" - print(f"Using subtracted data at: {target_path}") - - source_cat = os.path.join(DATA_DIR, field["target_name"], "image_catalogue.csv") - if not os.path.isfile(source_cat): - raise AirflowFailException(f"{source_cat} not found.") - - CURRENT_DB.set_status_processing( - field["target_name"], "vlbi_dd", field["sas_id_target"] - ) - - context = get_current_context() - if context["ti"].try_number == 1 or ( - not os.path.isfile( - f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt" - ) - ): - cmd = f"flocs-run vlbi dd-calibration --record-toil-stats --runner toil --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat.sub.ms {target_ms_path}" - else: - if field["status_vlbi_dd"] == PIPELINE_STATUS.downloaded: - # This way we can force a clean restart in the database. - cmd = f"flocs-run vlbi dd-calibration --record-toil-stats --runner toil --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat.sub.ms {target_ms_path}" - else: - # Extract the previous working directory - flocs_workdir = "" - print( - f"Scanning log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt for workdir." - ) - with open( - f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt" - ) as f_out: - for line in f_out.readlines(): - print(line) - if "Running workflow with" in line: - flocs_workdir = line.split(" ")[-1].strip() - break - if not flocs_workdir: - raise RuntimeError( - "Could not retrieve PILOT workdir. Flocs probably crashed before launching." - ) - print(f"Resuming failed PILOT run in {flocs_workdir}") - cmd = f"flocs-run vlbi dd-calibration --record-toil-stats --runner toil --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat.sub.ms {target_ms_path}" - if not os.path.isdir(outdir): - os.mkdir(outdir) - print(cmd) - with ( - open( - f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt", - "w+", - ) as f_out, - open( - f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}_err.txt", - "w+", - ) as f_err, - ): - proc = subprocess.run(cmd, shell=True, text=True, stdout=f_out, stderr=f_err) - success = False - pattern = re.compile(r"Workflow.* stopped. Success: True") - if not proc.returncode: - f_err.seek(0) - if pattern.search(f_err.read()): - success = True - if success: - CURRENT_DB.set_status_finished( - field["target_name"], "vlbi_dd", field["sas_id_target"] - ) - CURRENT_DB.set_field_finished(field["target_name"], field["sas_id_target"]) - else: - CURRENT_DB.set_status_failed( - field["target_name"], "vlbi_dd", field["sas_id_target"] - ) - raise RuntimeError - - def get_approval(field, identifier, needs_approval): if not needs_approval: return field @@ -689,21 +84,6 @@ def get_approval(field, identifier, needs_approval): return field -def get_most_recent_run(searchpath: str, sas_id: str, pipeline: str) -> pathlib.Path: - rundirs = pathlib.Path(searchpath) - rundirs_sorted = sorted(rundirs.iterdir()) - if pipeline: - rundirs_sorted_filtered = [ - d - for d in rundirs_sorted - if ((sas_id in d.parts[-1]) and (pipeline in d.parts[-1])) and d.is_dir() - ] - else: - rundirs_sorted_filtered = [d for d in rundirs_sorted if sas_id in d.parts[-1]] - rundir_final = rundirs_sorted_filtered[-1].absolute() - return rundir_final - - @dag(max_active_runs=1) def pilot_widefield(): @task @@ -916,9 +296,9 @@ def run_linc_calibrator1(field): return field else: if CWL_RUNNER_LINC_CALIBRATOR == "cwltool": - run_linc_calibrator_cwltool(field, calibrator_field=1) + run_linc_calibrator_cwltool(field, calibrator_field=1, db=CURRENT_DB) elif CWL_RUNNER_LINC_CALIBRATOR == "toil": - run_linc_calibrator_toil(field, calibrator_field=1) + run_linc_calibrator_toil(field, calibrator_field=1, db=CURRENT_DB) else: raise RuntimeError("Invalid CWL runner specified.") return field diff --git a/flocs_processing/pipeline_runners.py b/flocs_processing/pipeline_runners.py new file mode 100644 index 0000000..7f0990b --- /dev/null +++ b/flocs_processing/pipeline_runners.py @@ -0,0 +1,665 @@ +import configparser +import os +import pathlib +import re +import subprocess +import time + +from airflow.exceptions import AirflowFailException +from airflow.sdk import get_current_context + +from flocs_processing.db_utils import PIPELINE_STATUS, FlocsDB + +if "FLOCS_AIRFLOW_CONFIG" not in os.environ: + raise RuntimeError( + "FLOCS_AIRFLOW_CONFIG environment variable not set. Please point this to a valid configuration file." + ) + +# Need to think of a way to centralise this and not read multiple times here and in the DAG +CONFIG_FILE: str = os.getenv("FLOCS_AIRFLOW_CONFIG") or "" +if not os.path.isfile(CONFIG_FILE): + raise RuntimeError(f"{CONFIG_FILE} is not a valid file") + +parser = configparser.ConfigParser() +parser.optionxform = str # ty: ignore[invalid-assignment] +with open(CONFIG_FILE, "r") as config: + parser.read_string("[DEFAULT]\n" + config.read()) + +print("Config summary:") +for k, v in parser["DEFAULT"].items(): + print(f"{k}: {v}") + +TABLE_NAME = parser["DEFAULT"]["TABLE_NAME"] +DATABASE = parser["DEFAULT"]["DATABASE"] +SLURM_ACCOUNT = parser["DEFAULT"]["SLURM_ACCOUNT"] +SLURM_QUEUE = parser["DEFAULT"]["SLURM_QUEUE"] +DATA_DIR = parser["DEFAULT"]["DATA_DIR"] +OUTPUT_DIR = parser["DEFAULT"]["OUTPUT_DIR"] +PROCESSING_DIR = parser["DEFAULT"]["PROCESSING_DIR"] +NN_MODEL_CACHE = parser["DEFAULT"]["NN_MODEL_CACHE"] +DDF_CONFIG = parser["DEFAULT"]["DDF_CONFIG"] +FLUX_CALIBRATOR_TEMPLATE = parser["DEFAULT"]["FLUX_CALIBRATOR_TEMPLATE"] +NEEDS_MANUAL_APPROVAL_DELAY = bool(parser["DEFAULT"]["NEEDS_MANUAL_APPROVAL_DELAY"]) + + +def get_most_recent_run(searchpath: str, sas_id: str, pipeline: str) -> pathlib.Path: + rundirs = pathlib.Path(searchpath) + rundirs_sorted = sorted(rundirs.iterdir()) + if pipeline: + rundirs_sorted_filtered = [ + d + for d in rundirs_sorted + if ((sas_id in d.parts[-1]) and (pipeline in d.parts[-1])) and d.is_dir() + ] + else: + rundirs_sorted_filtered = [d for d in rundirs_sorted if sas_id in d.parts[-1]] + rundir_final = rundirs_sorted_filtered[-1].absolute() + return rundir_final + + +def run_linc_calibrator_cwltool(field, calibrator_field: int, db: FlocsDB): + print( + f"Processing flux density calibrator {field[f'sas_id_calibrator{calibrator_field}']} for observation {field['target_name']} {field['sas_id_target']}" + ) + ms_folder = f"L{field[f'sas_id_calibrator{calibrator_field}']}" + db.set_status_processing( + field["target_name"], f"calibrator{calibrator_field}", field["sas_id_target"] + ) + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + cmd = f"flocs-run linc calibrator --runner cwltool --scheduler slurm --slurm-cores 32 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} {os.path.join(DATA_DIR, field['target_name'], 'calibrator', ms_folder)}" + with ( + open( + f"log_LINC_calibrator_{field['target_name']}_{field[f'sas_id_calibrator{calibrator_field}']}.txt", + "w+", + ) as f_out, + open( + f"log_LINC_calibrator_{field['target_name']}_{field[f'sas_id_calibrator{calibrator_field}']}_err.txt", + "w+", + ) as f_err, + ): + proc = subprocess.run(cmd, shell=True, text=True, stdout=f_out, stderr=f_err) + jobid = None + if not proc.returncode: + f_out.seek(0) + for line in f_out.readlines(): + if "Submitted batch job" in line: + jobid = line.strip().split()[-1] + else: + raise RuntimeError("Failed to submit job.") + + if not jobid: + raise RuntimeError("Failed to retrieve job id") + else: + while True: + print(f"Polling LINC calibrator job {jobid}") + poll_cmd = f"sacct -X -j {jobid} --format=State --noheader" + status = subprocess.run( + poll_cmd, shell=True, text=True, capture_output=True + ).stdout.strip() + if (status == "RUNNING") or (status == "PENDING"): + time.sleep(60) + elif status == "COMPLETED": + db.set_status_finished( + field["target_name"], + f"calibrator{calibrator_field}", + field["sas_id_target"], + ) + break + elif ( + (status == "FAILED") + or ("TIMEOUT" in status) + or ("CANCELLED" in status) + ): + raise RuntimeError( + f"LINC calibrator for {field['target_name']} {field['sas_id_target']} failed." + ) + + +def run_linc_calibrator_toil(field, calibrator_field: int, db: FlocsDB): + print( + f"Processing flux density calibrator {field[f'sas_id_calibrator{calibrator_field}']} for observation {field['target_name']} {field['sas_id_target']}" + ) + ms_folder = f"L{field[f'sas_id_calibrator{calibrator_field}']}" + db.set_status_processing( + field["target_name"], f"calibrator{calibrator_field}", field["sas_id_target"] + ) + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + cmd = f"flocs-run linc calibrator --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} {os.path.join(DATA_DIR, field['target_name'], 'calibrator', ms_folder)}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with ( + open( + f"log_LINC_calibrator_{field['target_name']}_{field[f'sas_id_calibrator{calibrator_field}']}.txt", + "w+", + ) as f_out, + open( + f"log_LINC_calibrator_{field['target_name']}_{field[f'sas_id_calibrator{calibrator_field}']}_err.txt", + "w+", + ) as f_err, + ): + proc = subprocess.run(cmd, shell=True, text=True, stdout=f_out, stderr=f_err) + success = False + pattern = re.compile(r"Workflow.* stopped. Success: True") + if not proc.returncode: + f_err.seek(0) + if pattern.search(f_err.read()): + success = True + if success: + db.set_status_finished( + field["target_name"], + f"calibrator{calibrator_field}", + field["sas_id_target"], + ) + else: + raise RuntimeError + + +def run_linc_target_cwltool(field, db: FlocsDB): + print( + f"Processing target observation {field['target_name']} {field['sas_id_target']} with calibrator {field['sas_id_calibrator_final']}" + ) + ms_folder = f"L{field['sas_id_target']}" + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + calibrator_path = get_most_recent_run( + outdir, field["sas_id_calibrator_final"], "LINC_calibrator" + ) + calibrator_solutions = ( + calibrator_path / "results_LINC_calibrator" / "cal_solutions.h5" + ) + db.set_status_processing(field["target_name"], "target", field["sas_id_target"]) + cmd = f"flocs-run linc target --runner cwltool --scheduler slurm --slurm-cores 64 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --cal-solutions {calibrator_solutions} {os.path.join(DATA_DIR, field['target_name'], 'target', ms_folder)}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with ( + open( + f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}.txt", + "w+", + ) as f_out, + open( + f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w+", + ) as f_err, + ): + proc = subprocess.run(cmd, shell=True, text=True, stdout=f_out, stderr=f_err) + jobid = None + if not proc.returncode: + f_out.seek(0) + for line in f_out.readlines(): + if "Submitted batch job" in line: + jobid = line.strip().split()[-1] + else: + raise RuntimeError("Failed to submit job.") + + if not jobid: + raise RuntimeError("Failed to retrieve job id") + else: + while True: + print(f"Polling LINC target job {jobid}") + poll_cmd = f"sacct -X -j {jobid} --format=State --noheader" + status = subprocess.run( + poll_cmd, shell=True, text=True, capture_output=True + ).stdout.strip() + if (status == "RUNNING") or (status == "PENDING"): + time.sleep(60) + elif status == "COMPLETED": + db.set_status_finished( + field["target_name"], + "target", + field["sas_id_target"], + ) + break + elif ( + (status == "FAILED") + or ("TIMEOUT" in status) + or ("CANCELLED" in status) + ): + raise RuntimeError( + f"LINC target for {field['target_name']} {field['sas_id_target']} failed." + ) + + +def run_linc_target_toil(field, db: FlocsDB): + print( + f"Processing target observation {field['target_name']} {field['sas_id_target']} with calibrator {field['sas_id_calibrator_final']}" + ) + ms_folder = f"L{field['sas_id_target']}" + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + calibrator_path = get_most_recent_run( + outdir, field["sas_id_calibrator_final"], "LINC_calibrator" + ) + calibrator_solutions = ( + calibrator_path / "results_LINC_calibrator" / "cal_solutions.h5" + ) + db.set_status_processing(field["target_name"], "target", field["sas_id_target"]) + cmd = f"flocs-run linc target --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --cal-solutions {calibrator_solutions} {os.path.join(DATA_DIR, field['target_name'], 'target', ms_folder)}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with ( + open( + f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}.txt", + "w+", + ) as f_out, + open( + f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w+", + ) as f_err, + ): + proc = subprocess.run(cmd, shell=True, text=True, stdout=f_out, stderr=f_err) + success = False + pattern = re.compile(r"Workflow.* stopped. Success: True") + if not proc.returncode: + f_err.seek(0) + if pattern.search(f_err.read()): + success = True + if success: + db.set_status_finished( + field["target_name"], "target", field["sas_id_target"] + ) + else: + raise RuntimeError + + +def run_pilot_delay_cwltool(field, db: FlocsDB): + print( + f"Processing delay calibration for {field['target_name']} {field['sas_id_target']}" + ) + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + target_path = get_most_recent_run(outdir, field["sas_id_target"], "LINC_target") + target_ms_path = target_path / "results_LINC_target" / "results" + db.set_status_processing(field["target_name"], "vlbi_delay", field["sas_id_target"]) + + delay_cat = os.path.join(outdir, "delay_calibrators.csv") + image_cat = os.path.join(outdir, "image_catalogue.csv") + + if not os.path.isfile(delay_cat) or not os.path.isfile(image_cat): + ms = list(target_ms_path.glob("*.dp3concat"))[0] + cmd = f"lofar-vlbi-plot --force --output_dir {outdir} --MS {ms}" + with ( + open( + f"log_plot_field_{field['target_name']}_{field['sas_id_target']}.txt", + "w+", + ) as f_out, + open( + f"log_plot_field_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w+", + ) as f_err, + ): + proc = subprocess.run( + cmd, shell=True, text=True, stdout=f_out, stderr=f_err + ) + delay_cat = os.path.join(outdir, "delay_calibrators.csv") + image_cat = os.path.join(outdir, "image_catalogue.csv") + + if not os.path.isfile(delay_cat): + raise RuntimeError("Delay calibrator catalogue is missing or invalid.") + if not os.path.isfile(image_cat): + raise RuntimeError("Image source catalogue is missing or invalid.") + + proc = subprocess.run( + "detect_bad_slurm_nodes.sh", + shell=True, + text=True, + stdout=subprocess.PIPE, + ) + bad_nodes = proc.stdout.strip() + if bad_nodes: + print(f"Excluding the following bad nodes from scheduling: {bad_nodes}") + os.environ["TOIL_SLURM_ARGS"] = f"--exclude={bad_nodes}" + + cmd = f"flocs-run vlbi delay-calibration --record-toil-stats --runner cwltool --scheduler slurm --slurm-cores 64 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --ms-suffix dp3concat --delay-calibrator {delay_cat} --image-catalogue {image_cat} --apply-delay-solutions {target_ms_path}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with ( + open( + f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt", + "w+", + ) as f_out, + open( + f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w+", + ) as f_err, + ): + proc = subprocess.run(cmd, shell=True, text=True, stdout=f_out, stderr=f_err) + jobid = None + if not proc.returncode: + f_out.seek(0) + for line in f_out.readlines(): + if "Submitted batch job" in line: + jobid = line.strip().split()[-1] + else: + raise RuntimeError("Failed to submit job.") + + if not jobid: + raise RuntimeError("Failed to retrieve job id") + else: + while True: + print(f"Polling LINC target job {jobid}") + poll_cmd = f"sacct -X -j {jobid} --format=State --noheader" + status = subprocess.run( + poll_cmd, shell=True, text=True, capture_output=True + ).stdout.strip() + if (status == "RUNNING") or (status == "PENDING"): + time.sleep(60) + elif status == "COMPLETED": + if NEEDS_MANUAL_APPROVAL_DELAY: + db.set_status_await_approval( + field["target_name"], "vlbi_delay", field["sas_id_target"] + ) + else: + db.set_status_finished( + field["target_name"], "vlbi_delay", field["sas_id_target"] + ) + break + elif ( + (status == "FAILED") + or ("TIMEOUT" in status) + or ("CANCELLED" in status) + ): + raise RuntimeError( + f"PILOT delay calibration for {field['target_name']} {field['sas_id_target']} failed." + ) + + +def run_pilot_delay_toil(field, db: FlocsDB): + print( + f"Processing delay calibration for {field['target_name']} {field['sas_id_target']}" + ) + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + target_path = get_most_recent_run(outdir, field["sas_id_target"], "LINC_target") + target_ms_path = target_path / "results_LINC_target" / "results" + db.set_status_processing(field["target_name"], "vlbi_delay", field["sas_id_target"]) + + delay_cat = os.path.join(outdir, "delay_calibrators.csv") + image_cat = os.path.join(outdir, "image_catalogue.csv") + + if not os.path.isfile(delay_cat) or not os.path.isfile(image_cat): + ms = list(target_ms_path.glob("*.dp3concat"))[0] + cmd = f"lofar-vlbi-plot --force --output_dir {outdir} --MS {ms}" + with ( + open( + f"log_plot_field_{field['target_name']}_{field['sas_id_target']}.txt", + "w+", + ) as f_out, + open( + f"log_plot_field_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w+", + ) as f_err, + ): + proc = subprocess.run( + cmd, shell=True, text=True, stdout=f_out, stderr=f_err + ) + delay_cat = os.path.join(outdir, "delay_calibrators.csv") + image_cat = os.path.join(outdir, "image_catalogue.csv") + + if not os.path.isfile(delay_cat): + raise RuntimeError("Delay calibrator catalogue is missing or invalid.") + if not os.path.isfile(image_cat): + raise RuntimeError("Image source catalogue is missing or invalid.") + + proc = subprocess.run( + "detect_bad_slurm_nodes.sh", + shell=True, + text=True, + stdout=subprocess.PIPE, + ) + bad_nodes = proc.stdout.strip() + if bad_nodes: + print(f"Excluding the following bad nodes from scheduling: {bad_nodes}") + os.environ["TOIL_SLURM_ARGS"] = f"--exclude={bad_nodes}" + + context = get_current_context() + if context["ti"].try_number == 1 or ( + not os.path.isfile( + f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt" + ) + ): + cmd = f"flocs-run vlbi delay-calibration --record-toil-stats --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --ms-suffix dp3concat --delay-calibrator {delay_cat} --image-catalogue {image_cat} --apply-delay-solutions {target_ms_path}" + else: + # Extract the previous working directory + flocs_workdir = "" + print( + f"Scanning log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt for workdir." + ) + with open( + f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt" + ) as f_out: + for line in f_out.readlines(): + print(line) + if "Running workflow with" in line: + flocs_workdir = line.split(" ")[-1].strip() + break + if not flocs_workdir: + raise RuntimeError( + "Could not retrieve PILOT workdir. Flocs probably crashed before launching." + ) + print(f"Resuming failed PILOT run in {flocs_workdir}") + cmd = f"flocs-run vlbi delay-calibration --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --ms-suffix dp3concat --delay-calibrator {delay_cat} --image-catalogue {image_cat} {target_ms_path}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with ( + open( + f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt", + "w+", + ) as f_out, + open( + f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w+", + ) as f_err, + ): + proc = subprocess.run(cmd, shell=True, text=True, stdout=f_out, stderr=f_err) + success = False + pattern = re.compile(r"Workflow.* stopped. Success: True") + if not proc.returncode: + f_err.seek(0) + if pattern.search(f_err.read()): + success = True + + if success: + if NEEDS_MANUAL_APPROVAL_DELAY: + db.set_status_await_approval( + field["target_name"], "vlbi_delay", field["sas_id_target"] + ) + else: + db.set_status_finished( + field["target_name"], "vlbi_delay", field["sas_id_target"] + ) + else: + raise RuntimeError + + +def run_pilot_ddcal_cwltool(field, db: FlocsDB): + print( + f"Processing ILT dd calibration for {field['target_name']} {field['sas_id_target']}" + ) + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + if "status_ddf" not in field: + print("Not a widefield imaging run, checking LINC + delay calibration.") + target_path = get_most_recent_run(outdir, field["sas_id_target"], "LINC_target") + target_ms_path = target_path / "results_LINC_target" / "results" + print(f"Using LINC target run: {target_path}") + + sols_path = get_most_recent_run(outdir, field["sas_id_target"], "VLBI_delay") + sols_path = sols_path / "results_VLBI_delay-calibration" + sols = list(sols_path.glob("merged*selfcalcycle???_linearfulljones*.h5"))[0] + print(f"Using PILOT delay calibration solutions: {sols}") + + source_cat = os.path.join(DATA_DIR, field["target_name"], "vlbi_target.csv") + if not os.path.isfile(source_cat): + raise AirflowFailException(f"{source_cat} not found.") + + db.set_status_processing( + field["target_name"], "vlbi_dd", field["sas_id_target"] + ) + cmd = f"flocs-run vlbi dd-calibration --runner cwltool --scheduler slurm --slurm-cores 32 --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --delay-solset {sols} --phasediff-score 10.0 --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat {target_ms_path}" + else: + print("Widefield imaging run, checking subtraction output.") + target_path = get_most_recent_run( + outdir, field["sas_id_target"], "VLBI_process-ddf" + ) + target_ms_path = target_path / "results_VLBI_process-ddf" + print(f"Using subtracted data at: {target_path}") + + source_cat = os.path.join(DATA_DIR, field["target_name"], "image_catalogue.csv") + if not os.path.isfile(source_cat): + raise AirflowFailException(f"{source_cat} not found.") + + db.set_status_processing( + field["target_name"], "vlbi_dd", field["sas_id_target"] + ) + + cmd = f"flocs-run vlbi dd-calibration --runner cwltool --scheduler slurm --slurm-cores 64 --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat.sub.ms {target_ms_path}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with ( + open( + f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt", + "w+", + ) as f_out, + open( + f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w+", + ) as f_err, + ): + proc = subprocess.run(cmd, shell=True, text=True, stdout=f_out, stderr=f_err) + jobid = None + if not proc.returncode: + f_out.seek(0) + for line in f_out.readlines(): + if "Submitted batch job" in line: + jobid = line.strip().split()[-1] + else: + raise RuntimeError("Failed to submit job.") + + if not jobid: + raise RuntimeError("Failed to retrieve job id") + else: + while True: + print(f"Polling LINC target job {jobid}") + poll_cmd = f"sacct -X -j {jobid} --format=State --noheader" + status = subprocess.run( + poll_cmd, shell=True, text=True, capture_output=True + ).stdout.strip() + if (status == "RUNNING") or (status == "PENDING"): + time.sleep(60) + elif status == "COMPLETED": + db.set_status_finished( + field["target_name"], "vlbi_dd", field["sas_id_target"] + ) + break + elif ( + (status == "FAILED") + or ("TIMEOUT" in status) + or ("CANCELLED" in status) + ): + raise RuntimeError( + f"PILOT direction-dependent calibration for {field['target_name']} {field['sas_id_target']} failed." + ) + + +def run_pilot_ddcal_toil(field, db: FlocsDB): + print( + f"Processing ILT dd calibration for {field['target_name']} {field['sas_id_target']}" + ) + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + if "status_ddf" not in field: + print("Not a widefield imaging run, checking LINC + delay calibration.") + target_path = get_most_recent_run(outdir, field["sas_id_target"], "LINC_target") + target_ms_path = target_path / "results_LINC_target" / "results" + print(f"Using LINC target run: {target_path}") + + sols_path = get_most_recent_run(outdir, field["sas_id_target"], "VLBI_delay") + sols_path = sols_path / "results_VLBI_delay-calibration" + sols = list(sols_path.glob("merged*selfcalcycle???_linearfulljones*.h5"))[0] + print(f"Using PILOT delay calibration solutions: {sols}") + + source_cat = os.path.join(DATA_DIR, field["target_name"], "vlbi_target.csv") + if not os.path.isfile(source_cat): + raise AirflowFailException(f"{source_cat} not found.") + + db.set_status_processing( + field["target_name"], "vlbi_dd", field["sas_id_target"] + ) + cmd = f"flocs-run vlbi dd-calibration --runner toil --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --delay-solset {sols} --phasediff-score 10.0 --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat {target_ms_path}" + else: + print("Widefield imaging run, checking subtraction output.") + target_path = get_most_recent_run( + outdir, field["sas_id_target"], "VLBI_process-ddf" + ) + target_ms_path = target_path / "results_VLBI_process-ddf" + print(f"Using subtracted data at: {target_path}") + + source_cat = os.path.join(DATA_DIR, field["target_name"], "image_catalogue.csv") + if not os.path.isfile(source_cat): + raise AirflowFailException(f"{source_cat} not found.") + + db.set_status_processing( + field["target_name"], "vlbi_dd", field["sas_id_target"] + ) + + context = get_current_context() + if context["ti"].try_number == 1 or ( + not os.path.isfile( + f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt" + ) + ): + cmd = f"flocs-run vlbi dd-calibration --record-toil-stats --runner toil --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat.sub.ms {target_ms_path}" + else: + if field["status_vlbi_dd"] == PIPELINE_STATUS.downloaded: + # This way we can force a clean restart in the database. + cmd = f"flocs-run vlbi dd-calibration --record-toil-stats --runner toil --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat.sub.ms {target_ms_path}" + else: + # Extract the previous working directory + flocs_workdir = "" + print( + f"Scanning log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt for workdir." + ) + with open( + f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt" + ) as f_out: + for line in f_out.readlines(): + print(line) + if "Running workflow with" in line: + flocs_workdir = line.split(" ")[-1].strip() + break + if not flocs_workdir: + raise RuntimeError( + "Could not retrieve PILOT workdir. Flocs probably crashed before launching." + ) + print(f"Resuming failed PILOT run in {flocs_workdir}") + cmd = f"flocs-run vlbi dd-calibration --record-toil-stats --runner toil --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat.sub.ms {target_ms_path}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with ( + open( + f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt", + "w+", + ) as f_out, + open( + f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w+", + ) as f_err, + ): + proc = subprocess.run(cmd, shell=True, text=True, stdout=f_out, stderr=f_err) + success = False + pattern = re.compile(r"Workflow.* stopped. Success: True") + if not proc.returncode: + f_err.seek(0) + if pattern.search(f_err.read()): + success = True + if success: + db.set_status_finished( + field["target_name"], "vlbi_dd", field["sas_id_target"] + ) + db.set_field_finished(field["target_name"], field["sas_id_target"]) + else: + db.set_status_failed( + field["target_name"], "vlbi_dd", field["sas_id_target"] + ) + raise RuntimeError From 2f0ac7a316f35f07fb8fcad21a8cca1965d922a7 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 7 Aug 2026 15:30:32 +0200 Subject: [PATCH 094/120] Extract intermediate resolution image --- flocs_processing/dags/pilot_widefield.py | 98 +----------------------- 1 file changed, 3 insertions(+), 95 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index b97e956..bf43e71 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -9,6 +9,7 @@ run_pilot_delay_toil, run_pilot_ddcal_cwltool, run_pilot_ddcal_toil, + run_pilot_intermediate_image, ) import configparser @@ -863,103 +864,10 @@ def prepare_ddf_subtract(field): @task def run_vlbi_image_intermediate(field): field = dict(CURRENT_DB.get_db_columns(field["sas_id_target"])[0]) - if (field["status_vlbi_intermediate_img"] == PIPELINE_STATUS.finished) or ( - field["status_vlbi_intermediate_img"] == PIPELINE_STATUS.processing - ): + if field["status_vlbi_intermediate_img"] == PIPELINE_STATUS.finished: return field else: - print( - f"Processing ILT intermediate resolution imaging for {field['target_name']} {field['sas_id_target']}" - ) - outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - target_path = get_most_recent_run( - outdir, field["sas_id_target"], "VLBI_process-ddf" - ) - target_ms_path = target_path / "results_VLBI_process-ddf" - print(f"Using subtracted data at: {target_path}") - - dd_sols_path = get_most_recent_run( - outdir, field["sas_id_target"], "VLBI_dd-calibration" - ) - dd_sols = dd_sols_path / "results_VLBI_dd-calibration" / "merged.h5" - print(f"Using dd solutions at: {target_path}") - - if not os.path.isfile(dd_sols): - raise AirflowFailException(f"{dd_sols} not found.") - - CURRENT_DB.set_status_processing( - field["target_name"], "vlbi_intermediate_img", field["sas_id_target"] - ) - - context = get_current_context() - if context["ti"].try_number == 1 or ( - not os.path.isfile( - f"log_VLBI_image_intermediate_resolution_{field['target_name']}_{field['sas_id_target']}.txt" - ) - ): - cmd = f"flocs-run vlbi image-intermediate-resolution --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --dd-solutions {dd_sols} --ms-suffix .dp3concat.sub.ms {target_ms_path}" - else: - if field["status_vlbi_intermediate_img"] == PIPELINE_STATUS.downloaded: - # This way we can force a clean restart in the database. - cmd = f"flocs-run vlbi image-intermediate-resolution --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --dd-solutions {dd_sols} --ms-suffix .dp3concat.sub.ms {target_ms_path}" - else: - # Extract the previous working directory - flocs_workdir = "" - print( - f"Scanning log_VLBI_image_intermediate_resolution_{field['target_name']}_{field['sas_id_target']}.txt for workdir." - ) - with open( - f"log_VLBI_image_intermediate_resolution_{field['target_name']}_{field['sas_id_target']}.txt" - ) as f_out: - for line in f_out.readlines(): - print(line) - if "Running workflow with" in line: - flocs_workdir = line.split(" ")[-1].strip() - break - if not flocs_workdir: - raise RuntimeError( - "Could not retrieve PILOT workdir. Flocs probably crashed before launching." - ) - print(f"Resuming failed PILOT run in {flocs_workdir}") - cmd = f"flocs-run vlbi image-intermediate-resolution --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --dd-solutions {dd_sols} --ms-suffix .dp3concat.sub.ms {target_ms_path}" - if not os.path.isdir(outdir): - os.mkdir(outdir) - print(cmd) - with ( - open( - f"log_VLBI_image_intermediate_resolution_{field['target_name']}_{field['sas_id_target']}.txt", - "w+", - ) as f_out, - open( - f"log_VLBI_image_intermediate_resolution_{field['target_name']}_{field['sas_id_target']}_err.txt", - "w+", - ) as f_err, - ): - proc = subprocess.run( - cmd, shell=True, text=True, stdout=f_out, stderr=f_err - ) - success = False - pattern = re.compile(r"Workflow.* stopped. Success: True") - if not proc.returncode: - f_err.seek(0) - if pattern.search(f_err.read()): - success = True - if success: - CURRENT_DB.set_status_finished( - field["target_name"], - "vlbi_intermediate_img", - field["sas_id_target"], - ) - CURRENT_DB.set_field_finished( - field["target_name"], field["sas_id_target"] - ) - else: - CURRENT_DB.set_status_failed( - field["target_name"], - "vlbi_intermediate_img", - field["sas_id_target"], - ) - raise RuntimeError + run_pilot_intermediate_image(field, CURRENT_DB) return field @task From 890500806e93c0bfead15b2f977e6651ca9bac39 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 7 Aug 2026 15:31:47 +0200 Subject: [PATCH 095/120] Extract facet subtract --- flocs_processing/dags/pilot_widefield.py | 110 +---------------------- 1 file changed, 2 insertions(+), 108 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index bf43e71..db05dc7 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -9,6 +9,7 @@ run_pilot_delay_toil, run_pilot_ddcal_cwltool, run_pilot_ddcal_toil, + run_pilot_facet_subtract, run_pilot_intermediate_image, ) @@ -878,114 +879,7 @@ def run_vlbi_facet_subtract(field): ): return field else: - print( - f"Processing ILT facet subtraction for {field['target_name']} {field['sas_id_target']}" - ) - outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - target_path = get_most_recent_run( - outdir, field["sas_id_target"], "VLBI_process-ddf" - ) - target_ms_path = target_path / "results_VLBI_process-ddf" - print(f"Using subtracted data at: {target_path}") - - dd_sols_path = get_most_recent_run( - outdir, field["sas_id_target"], "VLBI_dd-calibration" - ) - dd_sols = dd_sols_path / "results_VLBI_dd-calibration" / "merged.h5" - print(f"Using dd solutions at: {target_path}") - - if not os.path.isfile(dd_sols): - raise AirflowFailException(f"{dd_sols} not found.") - - model_images_path = get_most_recent_run( - outdir, field["sas_id_target"], "VLBI_intermediate_resolution_imaging" - ) - model_images = ( - model_images_path - / "results_VLBI_intermediate_resolution_imaging" - / "*-????-model-fpb.fits" - ) - model_images = list(model_images_path.glob("*-????-model-fpb.fits")) - print(f"Using model image at: {model_images_path}/*-????-model-fpb.fits") - - if not model_images: - raise AirflowFailException( - "No suitable intermediate resolution model images found." - ) - - CURRENT_DB.set_status_processing( - field["target_name"], "vlbi_facet_subtract", field["sas_id_target"] - ) - - context = get_current_context() - if context["ti"].try_number == 1 or ( - not os.path.isfile( - f"log_VLBI_facet-subtract_{field['target_name']}_{field['sas_id_target']}.txt" - ) - ): - cmd = f"flocs-run vlbi facet-subtract --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --dd-solutions {dd_sols} --model-image-directory {model_images_path} --ms-suffix .dp3concat.sub.ms {target_ms_path}" - else: - if field["status_vlbi_facet_subtract"] == PIPELINE_STATUS.downloaded: - # This way we can force a clean restart in the database. - cmd = f"flocs-run vlbi facet-subtract --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --dd-solutions {dd_sols} --model-image-directory {model_images_path} --ms-suffix .dp3concat.sub.ms {target_ms_path}" - else: - # Extract the previous working directory - flocs_workdir = "" - print( - f"Scanning log_VLBI_facet-subtract_{field['target_name']}_{field['sas_id_target']}.txt for workdir." - ) - with open( - f"log_VLBI_facet-subtract_{field['target_name']}_{field['sas_id_target']}.txt" - ) as f_out: - for line in f_out.readlines(): - print(line) - if "Running workflow with" in line: - flocs_workdir = line.split(" ")[-1].strip() - break - if not flocs_workdir: - raise RuntimeError( - "Could not retrieve PILOT workdir. Flocs probably crashed before launching." - ) - print(f"Resuming failed PILOT run in {flocs_workdir}") - cmd = f"flocs-run vlbi facet-subtract --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --dd-solutions {dd_sols} --model-image-directory {model_images_path} --ms-suffix .dp3concat.sub.ms {target_ms_path}" - if not os.path.isdir(outdir): - os.mkdir(outdir) - print(cmd) - with ( - open( - f"log_VLBI_facet-subtract_{field['target_name']}_{field['sas_id_target']}.txt", - "w+", - ) as f_out, - open( - f"log_VLBI_facet-subtract_{field['target_name']}_{field['sas_id_target']}_err.txt", - "w+", - ) as f_err, - ): - proc = subprocess.run( - cmd, shell=True, text=True, stdout=f_out, stderr=f_err - ) - success = False - pattern = re.compile(r"Workflow.* stopped. Success: True") - if not proc.returncode: - f_err.seek(0) - if pattern.search(f_err.read()): - success = True - if success: - CURRENT_DB.set_status_finished( - field["target_name"], - "vlbi_facet_subtract", - field["sas_id_target"], - ) - CURRENT_DB.set_field_finished( - field["target_name"], field["sas_id_target"] - ) - else: - CURRENT_DB.set_status_failed( - field["target_name"], - "vlbi_facet_subtract", - field["sas_id_target"], - ) - raise RuntimeError + run_pilot_facet_subtract(field, CURRENT_DB) return field @task From 7bae7771c0fe0d7338819b187d1131584d519a7c Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 7 Aug 2026 15:33:28 +0200 Subject: [PATCH 096/120] Extract facet imaging --- flocs_processing/dags/pilot_widefield.py | 108 ++--------------------- 1 file changed, 8 insertions(+), 100 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index db05dc7..28330c0 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -9,8 +9,9 @@ run_pilot_delay_toil, run_pilot_ddcal_cwltool, run_pilot_ddcal_toil, - run_pilot_facet_subtract, - run_pilot_intermediate_image, + run_pilot_facet_imaging_toil, + run_pilot_facet_subtract_toil, + run_pilot_intermediate_image_toil, ) import configparser @@ -868,118 +869,25 @@ def run_vlbi_image_intermediate(field): if field["status_vlbi_intermediate_img"] == PIPELINE_STATUS.finished: return field else: - run_pilot_intermediate_image(field, CURRENT_DB) + run_pilot_intermediate_image_toil(field, CURRENT_DB) return field @task def run_vlbi_facet_subtract(field): field = dict(CURRENT_DB.get_db_columns(field["sas_id_target"])[0]) - if (field["status_vlbi_facet_subtract"] == PIPELINE_STATUS.finished) or ( - field["status_vlbi_facet_subtract"] == PIPELINE_STATUS.processing - ): + if field["status_vlbi_facet_subtract"] == PIPELINE_STATUS.finished: return field else: - run_pilot_facet_subtract(field, CURRENT_DB) + run_pilot_facet_subtract_toil(field, CURRENT_DB) return field @task def run_vlbi_facet_imaging(field): field = dict(CURRENT_DB.get_db_columns(field["sas_id_target"])[0]) - if (field["status_vlbi_facet_imaging"] == PIPELINE_STATUS.finished) or ( - field["status_vlbi_facet_imaging"] == PIPELINE_STATUS.processing - ): + if field["status_vlbi_facet_imaging"] == PIPELINE_STATUS.finished: return field else: - print( - f"Processing ILT facet imaging for {field['target_name']} {field['sas_id_target']}" - ) - outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - target_path = get_most_recent_run( - outdir, field["sas_id_target"], "VLBI_facet_subtract" - ) - target_ms_path = target_path / "results_VLBI_facet_subtract" - print(f"Using subtracted data at: {target_path}") - - facet_mses = list(target_ms_path.glob("facet*.ms")) - - if not facet_mses: - raise AirflowFailException( - "No suitable intermediate resolution model images found." - ) - - CURRENT_DB.set_status_processing( - field["target_name"], "vlbi_facet_imaging", field["sas_id_target"] - ) - - context = get_current_context() - if context["ti"].try_number == 1 or ( - not os.path.isfile( - f"log_VLBI_facet-imaging_{field['target_name']}_{field['sas_id_target']}.txt" - ) - ): - cmd = f"flocs-run vlbi facet-imaging --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --resolution 0.3asec --pixel-scale 0.1 --ms-suffix .ms {target_ms_path}" - else: - if field["status_vlbi_facet_imaging"] == PIPELINE_STATUS.downloaded: - # This way we can force a clean restart in the database. - cmd = f"flocs-run vlbi facet-imaging --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --resolution 0.3asec --pixel-scale 0.1 --ms-suffix .ms {target_ms_path}" - else: - # Extract the previous working directory - flocs_workdir = "" - print( - f"Scanning log_VLBI_facet_imaging_{field['target_name']}_{field['sas_id_target']}.txt for workdir." - ) - with open( - f"log_VLBI_facet_imaging_{field['target_name']}_{field['sas_id_target']}.txt" - ) as f_out: - for line in f_out.readlines(): - print(line) - if "Running workflow with" in line: - flocs_workdir = line.split(" ")[-1].strip() - break - if not flocs_workdir: - raise RuntimeError( - "Could not retrieve PILOT workdir. Flocs probably crashed before launching." - ) - print(f"Resuming failed PILOT run in {flocs_workdir}") - cmd = f"flocs-run vlbi facet-imaging --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --resolution 0.3asec --pixel-scale 0.1 --ms-suffix .ms {target_ms_path}" - if not os.path.isdir(outdir): - os.mkdir(outdir) - print(cmd) - with ( - open( - f"log_VLBI_facet_imaging_{field['target_name']}_{field['sas_id_target']}.txt", - "w+", - ) as f_out, - open( - f"log_VLBI_facet_imaging_{field['target_name']}_{field['sas_id_target']}_err.txt", - "w+", - ) as f_err, - ): - proc = subprocess.run( - cmd, shell=True, text=True, stdout=f_out, stderr=f_err - ) - success = False - pattern = re.compile(r"Workflow.* stopped. Success: True") - if not proc.returncode: - f_err.seek(0) - if pattern.search(f_err.read()): - success = True - if success: - CURRENT_DB.set_status_finished( - field["target_name"], - "vlbi_facet_imaging", - field["sas_id_target"], - ) - CURRENT_DB.set_field_finished( - field["target_name"], field["sas_id_target"] - ) - else: - CURRENT_DB.set_status_failed( - field["target_name"], - "vlbi_facet_imaging", - field["sas_id_target"], - ) - raise RuntimeError + run_pilot_facet_imaging_toil(field, CURRENT_DB) return field proceed = check_fields() From 21dd0d393f4bc5ed7e712453b2f45a01b6abded9 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 7 Aug 2026 15:35:18 +0200 Subject: [PATCH 097/120] Add missing db arguments --- flocs_processing/dags/pilot_widefield.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 28330c0..b0a69cb 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -1,3 +1,4 @@ +from threading import currentThread from flocs_processing.db_utils import PIPELINE_STATUS, FlocsDB from flocs_processing.pipeline_runners import ( get_most_recent_run, @@ -318,9 +319,9 @@ def run_linc_calibrator2(field): return field else: if CWL_RUNNER_LINC_CALIBRATOR == "cwltool": - run_linc_calibrator_cwltool(field, calibrator_field=2) + run_linc_calibrator_cwltool(field, calibrator_field=2, db=CURRENT_DB) elif CWL_RUNNER_LINC_CALIBRATOR == "toil": - run_linc_calibrator_toil(field, calibrator_field=2) + run_linc_calibrator_toil(field, calibrator_field=2, db=CURRENT_DB) else: raise RuntimeError("Invalid CWL runner specified.") return field @@ -423,9 +424,9 @@ def run_linc_target(field): return field else: if CWL_RUNNER_LINC_TARGET == "cwltool": - run_linc_target_cwltool(field) + run_linc_target_cwltool(field, CURRENT_DB) elif CWL_RUNNER_LINC_TARGET == "toil": - run_linc_target_toil(field) + run_linc_target_toil(field, CURRENT_DB) else: raise RuntimeError("Invalid CWL runner specified.") return field @@ -440,9 +441,9 @@ def run_vlbi_delay(field): return field else: if CWL_RUNNER_PILOT_DELAY == "cwltool": - run_pilot_delay_cwltool(field) + run_pilot_delay_cwltool(field, CURRENT_DB) elif CWL_RUNNER_PILOT_DELAY == "toil": - run_pilot_delay_toil(field) + run_pilot_delay_toil(field, CURRENT_DB) else: raise RuntimeError("Invalid CWL runner specified.") return field @@ -651,9 +652,9 @@ def run_vlbi_ddcal(field): return field else: if CWL_RUNNER_PILOT_DDCAL == "cwltool": - run_pilot_ddcal_cwltool(field) + run_pilot_ddcal_cwltool(field, CURRENT_DB) elif CWL_RUNNER_PILOT_DDCAL == "toil": - run_pilot_ddcal_toil(field) + run_pilot_ddcal_toil(field, CURRENT_DB) else: raise RuntimeError("Invalid CWL runner specified.") return field From 20e3c09235c7c992745fac9e299b2098947d4158 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 7 Aug 2026 15:40:57 +0200 Subject: [PATCH 098/120] Extract ddf subtract prep step --- flocs_processing/dags/pilot_widefield.py | 101 +---------------------- 1 file changed, 2 insertions(+), 99 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index b0a69cb..3e04774 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -1,4 +1,3 @@ -from threading import currentThread from flocs_processing.db_utils import PIPELINE_STATUS, FlocsDB from flocs_processing.pipeline_runners import ( get_most_recent_run, @@ -13,6 +12,7 @@ run_pilot_facet_imaging_toil, run_pilot_facet_subtract_toil, run_pilot_intermediate_image_toil, + run_prepare_ddf_subtract, ) import configparser @@ -761,104 +761,7 @@ def prepare_ddf(field): @task def prepare_ddf_subtract(field): - print( - f"Preparing input for DDF subtract of {field['target_name']} {field['sas_id_target']}" - ) - outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - target_path = get_most_recent_run(outdir, field["sas_id_target"], "VLBI_delay") - target_ms_path = target_path / "results_VLBI_delay-calibration" - mses_unaveraged = list(target_ms_path.glob("*.dp3concat")) - delay_sols = "" - if not mses_unaveraged: - print( - "No MSes found in delay-calibration output, will apply delay solutions to LINC." - ) - sols_path = get_most_recent_run( - outdir, field["sas_id_target"], "VLBI_delay" - ) - sols_path = sols_path / "results_VLBI_delay-calibration" - delay_sols = list( - sols_path.glob("merged*selfcalcycle???_linearfulljones*.h5") - )[0] - print(f"Using PILOT delay calibration solutions: {delay_sols}") - - linc_path = get_most_recent_run( - outdir, field["sas_id_target"], "LINC_target" - ) - print(f"Using LINC MSes at {linc_path}") - linc_ms_path = linc_path / "results_LINC_target" / "results" - mses_unaveraged = list(linc_ms_path.glob("*.dp3concat")) - mses_averaged = list(target_ms_path.glob("*_pre-cal.ms")) - mses_unaveraged_pilot = list(target_ms_path.glob("*.dp3concat")) - if not mses_unaveraged: - raise RuntimeError( - f"No unaveraged input MSes found at {linc_ms_path}/*.dp3concat" - ) - if mses_unaveraged_pilot and ( - len(mses_unaveraged_pilot) == len(mses_unaveraged) - ): - print("Appropriate input exists for ddf-pipeline.") - return field - - jobids = [] - averaged_mses = [] - for ms in mses_unaveraged: - out_ms = target_ms_path / f"{ms.stem}.dp3concat" - if out_ms.exists(): - print(f"Skipping {out_ms.name}, already exists.") - averaged_mses.append(str(out_ms)) - continue - if delay_sols: - h5 = h5parm(str(delay_sols)) - ss = h5.getSolset("sol000") - # We only expect there to be one direction: the delay calibrator. - dirname = list(ss.getSou())[0] - sourcedir = ss.getSou()[dirname] - delaydir = f"[{sourcedir[0]},{sourcedir[1]}]" - dp3_cmd = f"apptainer exec $CWL_SINGULARITY_CACHE/astronrd_linc_latest.sif DP3 numthreads=2 msin={ms} msout={out_ms} msout.uvwcompression=False msout.antennacompression=False msout.scalarflags=False msout.storagemanager=Dysco steps=[applybeamdelay,applycal,applybeamtarget] applybeamdelay.type=applybeam applybeamdelay.beammode=full applybeamdelay.updateweights=True applybeamdelay.direction={delaydir} applycal.parmdb={delay_sols} applycal.correction=fulljones applycal.soltab=[amplitude000,phase000] applybeamtarget.type=applybeam applybeamtarget.beammode=full applybeamtarget.updateweights=True" - print(dp3_cmd) - else: - dp3_cmd = f"apptainer exec $CWL_SINGULARITY_CACHE/astronrd_linc_latest.sif DP3 numthreads=2 msin={ms} msout={out_ms} msout.uvwcompression=False msout.antennacompression=False msout.scalarflags=False msout.storagemanager=Dysco steps=[]" - submit_cmd = f'sbatch -A {SLURM_ACCOUNT} -p {SLURM_QUEUE} --time=08:00:00 -c 2 --job-name=dp3_avg_{ms.stem} --wrap="{dp3_cmd}"' - print(f"Submitting: {submit_cmd}") - proc = subprocess.run( - submit_cmd, shell=True, text=True, capture_output=True - ) - if proc.returncode: - print(proc.stdout) - print(proc.stderr) - raise RuntimeError(f"Failed to submit SLURM job for {ms}") - jobid = proc.stdout.strip().split()[-1] - print(f"Submitted job {jobid} for {ms.name}") - jobids.append((jobid, out_ms)) - averaged_mses.append(str(out_ms)) - - while jobids: - print(f"Polling {len(jobids)} SLURM jobs...") - remaining = [] - for jobid, out_ms in jobids: - poll_cmd = f"sacct -X -j {jobid} --format=State --noheader" - status = subprocess.run( - poll_cmd, shell=True, text=True, capture_output=True - ).stdout.strip() - if status == "COMPLETED": - print(f"Job {jobid} completed ({out_ms.name})") - elif ( - (status == "FAILED") - or ("TIMEOUT" in status) - or ("CANCELLED" in status) - ): - raise RuntimeError( - f"DP3 averaging job {jobid} failed for {out_ms.name}" - ) - elif status in ("PENDING", "RUNNING"): - remaining.append((jobid, out_ms)) - else: - remaining.append((jobid, out_ms)) - jobids = remaining - time.sleep(30) - - mses_averaged = list(target_ms_path.glob("*_pre-cal.ms")) + mses_averaged = run_prepare_ddf_subtract(field, CURRENT_DB) if mses_averaged: return field else: From 17a654d5b7a9235e63a924d70d036c4df0f03c96 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 7 Aug 2026 15:48:07 +0200 Subject: [PATCH 099/120] Extract ddf subtract prep step --- flocs_processing/dags/pilot_widefield.py | 102 +---- flocs_processing/pipeline_runners.py | 470 +++++++++++++++++++++++ 2 files changed, 475 insertions(+), 97 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 3e04774..e997cc0 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -12,6 +12,7 @@ run_pilot_facet_imaging_toil, run_pilot_facet_subtract_toil, run_pilot_intermediate_image_toil, + run_prepare_ddf, run_prepare_ddf_subtract, ) @@ -32,7 +33,6 @@ from airflow.task.trigger_rule import TriggerRule from flocs_lta.lta_search import ObservationStager from ilotss.assess_calibrators import assess_and_compare -from losoto.h5parm import h5parm from stager_access import get_surls_requested, get_surls_online if "FLOCS_AIRFLOW_CONFIG" not in os.environ: @@ -661,99 +661,7 @@ def run_vlbi_ddcal(field): @task def prepare_ddf(field): - print( - f"Preparing DDF input for {field['target_name']} {field['sas_id_target']}" - ) - outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - target_path = get_most_recent_run(outdir, field["sas_id_target"], "VLBI_delay") - target_ms_path = target_path / "results_VLBI_delay-calibration" - mses_unaveraged = list(target_ms_path.glob("*.dp3concat")) - delay_sols = "" - if not mses_unaveraged: - print( - "No MSes found in delay-calibration output, will apply delay solutions to LINC." - ) - sols_path = get_most_recent_run( - outdir, field["sas_id_target"], "VLBI_delay" - ) - sols_path = sols_path / "results_VLBI_delay-calibration" - delay_sols = list( - sols_path.glob("merged*selfcalcycle???_linearfulljones*.h5") - )[0] - print(f"Using PILOT delay calibration solutions: {delay_sols}") - - linc_path = get_most_recent_run( - outdir, field["sas_id_target"], "LINC_target" - ) - print(f"Using LINC MSes at {linc_path}") - linc_ms_path = linc_path / "results_LINC_target" / "results" - mses_unaveraged = list(linc_ms_path.glob("*.dp3concat")) - else: - print("Found unaveraged data in delay calibration output.") - mses_averaged = list(target_ms_path.glob("*_pre-cal.ms")) - if not mses_unaveraged: - raise RuntimeError( - f"No unaveraged input MSes found at {linc_ms_path}/*.dp3concat" - ) - if mses_unaveraged and (len(mses_averaged) == len(mses_unaveraged)): - print("Appropriate input exists for ddf-pipeline.") - return field - - jobids = [] - averaged_mses = [] - for ms in mses_unaveraged: - out_ms = target_ms_path / f"{ms.stem}_pre-cal.ms" - if out_ms.exists(): - print(f"Skipping {out_ms.name}, already exists.") - averaged_mses.append(str(out_ms)) - continue - if delay_sols: - h5 = h5parm(str(delay_sols)) - ss = h5.getSolset("sol000") - # We only expect there to be one direction: the delay calibrator. - dirname = list(ss.getSou())[0] - sourcedir = ss.getSou()[dirname] - delaydir = f"[{sourcedir[0]},{sourcedir[1]}]" - dp3_cmd = f"apptainer exec $CWL_SINGULARITY_CACHE/astronrd_linc_latest.sif DP3 numthreads=2 msin={ms} msout={out_ms} msout.uvwcompression=False msout.antennacompression=False msout.scalarflags=False msout.storagemanager=Dysco steps=[average,applybeamdelay,applycal,applybeamtarget,filter] average.timeresolution=8 average.freqresolution=97.64kHz applybeamdelay.type=applybeam applybeamdelay.beammode=full applybeamdelay.updateweights=True applybeamdelay.direction={delaydir} applycal.parmdb={delay_sols} applycal.correction=fulljones applycal.soltab=[amplitude000,phase000] applybeamtarget.type=applybeam applybeamtarget.beammode=full applybeamtarget.updateweights=True filter.remove=True filter.baseline='[CR]S*&&'" - print(dp3_cmd) - else: - dp3_cmd = f"apptainer exec $CWL_SINGULARITY_CACHE/astronrd_linc_latest.sif DP3 numthreads=2 msin={ms} msout={out_ms} msout.uvwcompression=False msout.antennacompression=False msout.scalarflags=False msout.storagemanager=Dysco steps=[filter,average] average.timeresolution=8 average.freqresolution=97.64kHz filter.remove=True filter.baseline='[CR]S*&&'" - submit_cmd = f'sbatch -A {SLURM_ACCOUNT} -p {SLURM_QUEUE} --time=02:00:00 -c 2 --job-name=dp3_avg_{ms.stem} --wrap="{dp3_cmd}"' - print(f"Submitting: {submit_cmd}") - proc = subprocess.run( - submit_cmd, shell=True, text=True, capture_output=True - ) - if proc.returncode: - print(proc.stdout) - print(proc.stderr) - raise RuntimeError(f"Failed to submit SLURM job for {ms}") - jobid = proc.stdout.strip().split()[-1] - print(f"Submitted job {jobid} for {ms.name}") - jobids.append((jobid, out_ms)) - averaged_mses.append(str(out_ms)) - - while jobids: - print(f"Polling {len(jobids)} SLURM jobs...") - remaining = [] - for jobid, out_ms in jobids: - poll_cmd = f"sacct -X -j {jobid} --format=State --noheader" - status = subprocess.run( - poll_cmd, shell=True, text=True, capture_output=True - ).stdout.strip() - if status == "COMPLETED": - print(f"Job {jobid} completed ({out_ms.name})") - elif status == "FAILED": - raise RuntimeError( - f"DP3 averaging job {jobid} failed for {out_ms.name}" - ) - elif status in ("PENDING", "RUNNING"): - remaining.append((jobid, out_ms)) - else: - remaining.append((jobid, out_ms)) - jobids = remaining - time.sleep(30) - - mses_averaged = list(target_ms_path.glob("*_pre-cal.ms")) + mses_averaged = run_prepare_ddf(field) if mses_averaged: return field else: @@ -761,11 +669,11 @@ def prepare_ddf(field): @task def prepare_ddf_subtract(field): - mses_averaged = run_prepare_ddf_subtract(field, CURRENT_DB) - if mses_averaged: + mses_delay_corrected = run_prepare_ddf_subtract(field) + if mses_delay_corrected: return field else: - raise RuntimeError("No averaged MSes for ddf-pipeline found.") + raise RuntimeError("No delay-corrected MSes for ddf subtract found.") @task def run_vlbi_image_intermediate(field): diff --git a/flocs_processing/pipeline_runners.py b/flocs_processing/pipeline_runners.py index 7f0990b..6922eb7 100644 --- a/flocs_processing/pipeline_runners.py +++ b/flocs_processing/pipeline_runners.py @@ -7,6 +7,8 @@ from airflow.exceptions import AirflowFailException from airflow.sdk import get_current_context +from losoto.h5parm import h5parm +from pydantic_core.core_schema import model_ser_schema from flocs_processing.db_utils import PIPELINE_STATUS, FlocsDB @@ -663,3 +665,471 @@ def run_pilot_ddcal_toil(field, db: FlocsDB): field["target_name"], "vlbi_dd", field["sas_id_target"] ) raise RuntimeError + + +def run_pilot_intermediate_image_toil(field, db: FlocsDB): + print( + f"Processing ILT intermediate resolution imaging for {field['target_name']} {field['sas_id_target']}" + ) + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + target_path = get_most_recent_run( + outdir, field["sas_id_target"], "VLBI_process-ddf" + ) + target_ms_path = target_path / "results_VLBI_process-ddf" + print(f"Using subtracted data at: {target_path}") + + dd_sols_path = get_most_recent_run( + outdir, field["sas_id_target"], "VLBI_dd-calibration" + ) + dd_sols = dd_sols_path / "results_VLBI_dd-calibration" / "merged.h5" + print(f"Using dd solutions at: {target_path}") + + if not os.path.isfile(dd_sols): + raise AirflowFailException(f"{dd_sols} not found.") + + db.set_status_processing( + field["target_name"], "vlbi_intermediate_img", field["sas_id_target"] + ) + + context = get_current_context() + if context["ti"].try_number == 1 or ( + not os.path.isfile( + f"log_VLBI_image_intermediate_resolution_{field['target_name']}_{field['sas_id_target']}.txt" + ) + ): + cmd = f"flocs-run vlbi image-intermediate-resolution --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --dd-solutions {dd_sols} --ms-suffix .dp3concat.sub.ms {target_ms_path}" + else: + if field["status_vlbi_intermediate_img"] == PIPELINE_STATUS.downloaded: + # This way we can force a clean restart in the database. + cmd = f"flocs-run vlbi image-intermediate-resolution --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --dd-solutions {dd_sols} --ms-suffix .dp3concat.sub.ms {target_ms_path}" + else: + # Extract the previous working directory + flocs_workdir = "" + print( + f"Scanning log_VLBI_image_intermediate_resolution_{field['target_name']}_{field['sas_id_target']}.txt for workdir." + ) + with open( + f"log_VLBI_image_intermediate_resolution_{field['target_name']}_{field['sas_id_target']}.txt" + ) as f_out: + for line in f_out.readlines(): + print(line) + if "Running workflow with" in line: + flocs_workdir = line.split(" ")[-1].strip() + break + if not flocs_workdir: + raise RuntimeError( + "Could not retrieve PILOT workdir. Flocs probably crashed before launching." + ) + print(f"Resuming failed PILOT run in {flocs_workdir}") + cmd = f"flocs-run vlbi image-intermediate-resolution --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --dd-solutions {dd_sols} --ms-suffix .dp3concat.sub.ms {target_ms_path}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with ( + open( + f"log_VLBI_image_intermediate_resolution_{field['target_name']}_{field['sas_id_target']}.txt", + "w+", + ) as f_out, + open( + f"log_VLBI_image_intermediate_resolution_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w+", + ) as f_err, + ): + proc = subprocess.run(cmd, shell=True, text=True, stdout=f_out, stderr=f_err) + success = False + pattern = re.compile(r"Workflow.* stopped. Success: True") + if not proc.returncode: + f_err.seek(0) + if pattern.search(f_err.read()): + success = True + if success: + db.set_status_finished( + field["target_name"], + "vlbi_intermediate_img", + field["sas_id_target"], + ) + db.set_field_finished(field["target_name"], field["sas_id_target"]) + else: + db.set_status_failed( + field["target_name"], + "vlbi_intermediate_img", + field["sas_id_target"], + ) + raise RuntimeError + + +def run_pilot_facet_subtract_toil(field, db: FlocsDB): + print( + f"Processing ILT facet subtraction for {field['target_name']} {field['sas_id_target']}" + ) + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + target_path = get_most_recent_run( + outdir, field["sas_id_target"], "VLBI_process-ddf" + ) + target_ms_path = target_path / "results_VLBI_process-ddf" + print(f"Using subtracted data at: {target_path}") + + dd_sols_path = get_most_recent_run( + outdir, field["sas_id_target"], "VLBI_dd-calibration" + ) + dd_sols = dd_sols_path / "results_VLBI_dd-calibration" / "merged.h5" + print(f"Using dd solutions at: {target_path}") + + if not os.path.isfile(dd_sols): + raise AirflowFailException(f"{dd_sols} not found.") + + model_images_path = get_most_recent_run( + outdir, field["sas_id_target"], "VLBI_intermediate_resolution_imaging" + ) + model_images = ( + model_images_path + / "results_VLBI_intermediate_resolution_imaging" + / "*-????-model-fpb.fits" + ) + model_images = list(model_images_path.glob("*-????-model-fpb.fits")) + print(f"Using model image at: {model_images_path}/*-????-model-fpb.fits") + + if not model_images: + raise AirflowFailException( + "No suitable intermediate resolution model images found." + ) + + db.set_status_processing( + field["target_name"], "vlbi_facet_subtract", field["sas_id_target"] + ) + + context = get_current_context() + if context["ti"].try_number == 1 or ( + not os.path.isfile( + f"log_VLBI_facet-subtract_{field['target_name']}_{field['sas_id_target']}.txt" + ) + ): + cmd = f"flocs-run vlbi facet-subtract --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --dd-solutions {dd_sols} --model-image-directory {model_images_path} --ms-suffix .dp3concat.sub.ms {target_ms_path}" + else: + if field["status_vlbi_facet_subtract"] == PIPELINE_STATUS.downloaded: + # This way we can force a clean restart in the database. + cmd = f"flocs-run vlbi facet-subtract --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --dd-solutions {dd_sols} --model-image-directory {model_images_path} --ms-suffix .dp3concat.sub.ms {target_ms_path}" + else: + # Extract the previous working directory + flocs_workdir = "" + print( + f"Scanning log_VLBI_facet-subtract_{field['target_name']}_{field['sas_id_target']}.txt for workdir." + ) + with open( + f"log_VLBI_facet-subtract_{field['target_name']}_{field['sas_id_target']}.txt" + ) as f_out: + for line in f_out.readlines(): + print(line) + if "Running workflow with" in line: + flocs_workdir = line.split(" ")[-1].strip() + break + if not flocs_workdir: + raise RuntimeError( + "Could not retrieve PILOT workdir. Flocs probably crashed before launching." + ) + print(f"Resuming failed PILOT run in {flocs_workdir}") + cmd = f"flocs-run vlbi facet-subtract --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --dd-solutions {dd_sols} --model-image-directory {model_images_path} --ms-suffix .dp3concat.sub.ms {target_ms_path}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with ( + open( + f"log_VLBI_facet-subtract_{field['target_name']}_{field['sas_id_target']}.txt", + "w+", + ) as f_out, + open( + f"log_VLBI_facet-subtract_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w+", + ) as f_err, + ): + proc = subprocess.run(cmd, shell=True, text=True, stdout=f_out, stderr=f_err) + success = False + pattern = re.compile(r"Workflow.* stopped. Success: True") + if not proc.returncode: + f_err.seek(0) + if pattern.search(f_err.read()): + success = True + if success: + db.set_status_finished( + field["target_name"], + "vlbi_facet_subtract", + field["sas_id_target"], + ) + db.set_field_finished(field["target_name"], field["sas_id_target"]) + else: + db.set_status_failed( + field["target_name"], + "vlbi_facet_subtract", + field["sas_id_target"], + ) + raise RuntimeError + + +def run_pilot_facet_imaging_toil(field, db: FlocsDB): + print( + f"Processing ILT facet imaging for {field['target_name']} {field['sas_id_target']}" + ) + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + target_path = get_most_recent_run( + outdir, field["sas_id_target"], "VLBI_facet_subtract" + ) + target_ms_path = target_path / "results_VLBI_facet_subtract" + print(f"Using subtracted data at: {target_path}") + + facet_mses = list(target_ms_path.glob("facet*.ms")) + + if not facet_mses: + raise AirflowFailException( + "No suitable intermediate resolution model images found." + ) + + db.set_status_processing( + field["target_name"], "vlbi_facet_imaging", field["sas_id_target"] + ) + + context = get_current_context() + if context["ti"].try_number == 1 or ( + not os.path.isfile( + f"log_VLBI_facet-imaging_{field['target_name']}_{field['sas_id_target']}.txt" + ) + ): + cmd = f"flocs-run vlbi facet-imaging --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --resolution 0.3asec --pixel-scale 0.1 --ms-suffix .ms {target_ms_path}" + else: + if field["status_vlbi_facet_imaging"] == PIPELINE_STATUS.downloaded: + # This way we can force a clean restart in the database. + cmd = f"flocs-run vlbi facet-imaging --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --resolution 0.3asec --pixel-scale 0.1 --ms-suffix .ms {target_ms_path}" + else: + # Extract the previous working directory + flocs_workdir = "" + print( + f"Scanning log_VLBI_facet_imaging_{field['target_name']}_{field['sas_id_target']}.txt for workdir." + ) + with open( + f"log_VLBI_facet_imaging_{field['target_name']}_{field['sas_id_target']}.txt" + ) as f_out: + for line in f_out.readlines(): + print(line) + if "Running workflow with" in line: + flocs_workdir = line.split(" ")[-1].strip() + break + if not flocs_workdir: + raise RuntimeError( + "Could not retrieve PILOT workdir. Flocs probably crashed before launching." + ) + print(f"Resuming failed PILOT run in {flocs_workdir}") + cmd = f"flocs-run vlbi facet-imaging --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --resolution 0.3asec --pixel-scale 0.1 --ms-suffix .ms {target_ms_path}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with ( + open( + f"log_VLBI_facet_imaging_{field['target_name']}_{field['sas_id_target']}.txt", + "w+", + ) as f_out, + open( + f"log_VLBI_facet_imaging_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w+", + ) as f_err, + ): + proc = subprocess.run(cmd, shell=True, text=True, stdout=f_out, stderr=f_err) + success = False + pattern = re.compile(r"Workflow.* stopped. Success: True") + if not proc.returncode: + f_err.seek(0) + if pattern.search(f_err.read()): + success = True + if success: + db.set_status_finished( + field["target_name"], + "vlbi_facet_imaging", + field["sas_id_target"], + ) + db.set_field_finished(field["target_name"], field["sas_id_target"]) + else: + db.set_status_failed( + field["target_name"], + "vlbi_facet_imaging", + field["sas_id_target"], + ) + raise RuntimeError + + +def run_prepare_ddf_subtract(field): + print( + f"Preparing input for DDF subtract of {field['target_name']} {field['sas_id_target']}" + ) + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + target_path = get_most_recent_run(outdir, field["sas_id_target"], "VLBI_delay") + target_ms_path = target_path / "results_VLBI_delay-calibration" + mses_unaveraged = list(target_ms_path.glob("*.dp3concat")) + delay_sols = "" + if not mses_unaveraged: + print( + "No MSes found in delay-calibration output, will apply delay solutions to LINC." + ) + sols_path = get_most_recent_run(outdir, field["sas_id_target"], "VLBI_delay") + sols_path = sols_path / "results_VLBI_delay-calibration" + delay_sols = list(sols_path.glob("merged*selfcalcycle???_linearfulljones*.h5"))[ + 0 + ] + print(f"Using PILOT delay calibration solutions: {delay_sols}") + + linc_path = get_most_recent_run(outdir, field["sas_id_target"], "LINC_target") + print(f"Using LINC MSes at {linc_path}") + linc_ms_path = linc_path / "results_LINC_target" / "results" + mses_unaveraged = list(linc_ms_path.glob("*.dp3concat")) + mses_averaged = list(target_ms_path.glob("*_pre-cal.ms")) + mses_unaveraged_pilot = list(target_ms_path.glob("*.dp3concat")) + if not mses_unaveraged: + raise RuntimeError( + f"No unaveraged input MSes found at {linc_ms_path}/*.dp3concat" + ) + if mses_unaveraged_pilot and (len(mses_unaveraged_pilot) == len(mses_unaveraged)): + print("Appropriate input exists for ddf-pipeline.") + return field + + jobids = [] + delay_corrected_mses = [] + for ms in mses_unaveraged: + out_ms = target_ms_path / f"{ms.stem}.dp3concat" + if out_ms.exists(): + print(f"Skipping {out_ms.name}, already exists.") + delay_corrected_mses.append(str(out_ms)) + continue + if delay_sols: + h5 = h5parm(str(delay_sols)) + ss = h5.getSolset("sol000") + # We only expect there to be one direction: the delay calibrator. + dirname = list(ss.getSou())[0] + sourcedir = ss.getSou()[dirname] + delaydir = f"[{sourcedir[0]},{sourcedir[1]}]" + dp3_cmd = f"apptainer exec $CWL_SINGULARITY_CACHE/astronrd_linc_latest.sif DP3 numthreads=2 msin={ms} msout={out_ms} msout.uvwcompression=False msout.antennacompression=False msout.scalarflags=False msout.storagemanager=Dysco steps=[applybeamdelay,applycal,applybeamtarget] applybeamdelay.type=applybeam applybeamdelay.beammode=full applybeamdelay.updateweights=True applybeamdelay.direction={delaydir} applycal.parmdb={delay_sols} applycal.correction=fulljones applycal.soltab=[amplitude000,phase000] applybeamtarget.type=applybeam applybeamtarget.beammode=full applybeamtarget.updateweights=True" + print(dp3_cmd) + else: + dp3_cmd = f"apptainer exec $CWL_SINGULARITY_CACHE/astronrd_linc_latest.sif DP3 numthreads=2 msin={ms} msout={out_ms} msout.uvwcompression=False msout.antennacompression=False msout.scalarflags=False msout.storagemanager=Dysco steps=[]" + submit_cmd = f'sbatch -A {SLURM_ACCOUNT} -p {SLURM_QUEUE} --time=08:00:00 -c 2 --job-name=dp3_avg_{ms.stem} --wrap="{dp3_cmd}"' + print(f"Submitting: {submit_cmd}") + proc = subprocess.run(submit_cmd, shell=True, text=True, capture_output=True) + if proc.returncode: + print(proc.stdout) + print(proc.stderr) + raise RuntimeError(f"Failed to submit SLURM job for {ms}") + jobid = proc.stdout.strip().split()[-1] + print(f"Submitted job {jobid} for {ms.name}") + jobids.append((jobid, out_ms)) + delay_corrected_mses.append(str(out_ms)) + + while jobids: + print(f"Polling {len(jobids)} SLURM jobs...") + remaining = [] + for jobid, out_ms in jobids: + poll_cmd = f"sacct -X -j {jobid} --format=State --noheader" + status = subprocess.run( + poll_cmd, shell=True, text=True, capture_output=True + ).stdout.strip() + if status == "COMPLETED": + print(f"Job {jobid} completed ({out_ms.name})") + elif ( + (status == "FAILED") or ("TIMEOUT" in status) or ("CANCELLED" in status) + ): + raise RuntimeError( + f"DP3 averaging job {jobid} failed for {out_ms.name}" + ) + elif status in ("PENDING", "RUNNING"): + remaining.append((jobid, out_ms)) + else: + remaining.append((jobid, out_ms)) + jobids = remaining + time.sleep(30) + + mses_delay_corrected = list(target_ms_path.glob("*.dp3concat")) + return mses_delay_corrected + + +def run_prepare_ddf(field): + print(f"Preparing DDF input for {field['target_name']} {field['sas_id_target']}") + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + target_path = get_most_recent_run(outdir, field["sas_id_target"], "VLBI_delay") + target_ms_path = target_path / "results_VLBI_delay-calibration" + mses_unaveraged = list(target_ms_path.glob("*.dp3concat")) + delay_sols = "" + if not mses_unaveraged: + print( + "No MSes found in delay-calibration output, will apply delay solutions to LINC." + ) + sols_path = get_most_recent_run(outdir, field["sas_id_target"], "VLBI_delay") + sols_path = sols_path / "results_VLBI_delay-calibration" + delay_sols = list(sols_path.glob("merged*selfcalcycle???_linearfulljones*.h5"))[ + 0 + ] + print(f"Using PILOT delay calibration solutions: {delay_sols}") + + linc_path = get_most_recent_run(outdir, field["sas_id_target"], "LINC_target") + print(f"Using LINC MSes at {linc_path}") + linc_ms_path = linc_path / "results_LINC_target" / "results" + mses_unaveraged = list(linc_ms_path.glob("*.dp3concat")) + else: + print("Found unaveraged data in delay calibration output.") + mses_averaged = list(target_ms_path.glob("*_pre-cal.ms")) + if not mses_unaveraged: + raise RuntimeError( + f"No unaveraged input MSes found at {linc_ms_path}/*.dp3concat" + ) + if mses_unaveraged and (len(mses_averaged) == len(mses_unaveraged)): + print("Appropriate input exists for ddf-pipeline.") + return field + + jobids = [] + averaged_mses = [] + for ms in mses_unaveraged: + out_ms = target_ms_path / f"{ms.stem}_pre-cal.ms" + if out_ms.exists(): + print(f"Skipping {out_ms.name}, already exists.") + averaged_mses.append(str(out_ms)) + continue + if delay_sols: + h5 = h5parm(str(delay_sols)) + ss = h5.getSolset("sol000") + # We only expect there to be one direction: the delay calibrator. + dirname = list(ss.getSou())[0] + sourcedir = ss.getSou()[dirname] + delaydir = f"[{sourcedir[0]},{sourcedir[1]}]" + dp3_cmd = f"apptainer exec $CWL_SINGULARITY_CACHE/astronrd_linc_latest.sif DP3 numthreads=2 msin={ms} msout={out_ms} msout.uvwcompression=False msout.antennacompression=False msout.scalarflags=False msout.storagemanager=Dysco steps=[average,applybeamdelay,applycal,applybeamtarget,filter] average.timeresolution=8 average.freqresolution=97.64kHz applybeamdelay.type=applybeam applybeamdelay.beammode=full applybeamdelay.updateweights=True applybeamdelay.direction={delaydir} applycal.parmdb={delay_sols} applycal.correction=fulljones applycal.soltab=[amplitude000,phase000] applybeamtarget.type=applybeam applybeamtarget.beammode=full applybeamtarget.updateweights=True filter.remove=True filter.baseline='[CR]S*&&'" + print(dp3_cmd) + else: + dp3_cmd = f"apptainer exec $CWL_SINGULARITY_CACHE/astronrd_linc_latest.sif DP3 numthreads=2 msin={ms} msout={out_ms} msout.uvwcompression=False msout.antennacompression=False msout.scalarflags=False msout.storagemanager=Dysco steps=[filter,average] average.timeresolution=8 average.freqresolution=97.64kHz filter.remove=True filter.baseline='[CR]S*&&'" + submit_cmd = f'sbatch -A {SLURM_ACCOUNT} -p {SLURM_QUEUE} --time=02:00:00 -c 2 --job-name=dp3_avg_{ms.stem} --wrap="{dp3_cmd}"' + print(f"Submitting: {submit_cmd}") + proc = subprocess.run(submit_cmd, shell=True, text=True, capture_output=True) + if proc.returncode: + print(proc.stdout) + print(proc.stderr) + raise RuntimeError(f"Failed to submit SLURM job for {ms}") + jobid = proc.stdout.strip().split()[-1] + print(f"Submitted job {jobid} for {ms.name}") + jobids.append((jobid, out_ms)) + averaged_mses.append(str(out_ms)) + + while jobids: + print(f"Polling {len(jobids)} SLURM jobs...") + remaining = [] + for jobid, out_ms in jobids: + poll_cmd = f"sacct -X -j {jobid} --format=State --noheader" + status = subprocess.run( + poll_cmd, shell=True, text=True, capture_output=True + ).stdout.strip() + if status == "COMPLETED": + print(f"Job {jobid} completed ({out_ms.name})") + elif status == "FAILED": + raise RuntimeError( + f"DP3 averaging job {jobid} failed for {out_ms.name}" + ) + elif status in ("PENDING", "RUNNING"): + remaining.append((jobid, out_ms)) + else: + remaining.append((jobid, out_ms)) + jobids = remaining + time.sleep(30) + + mses_averaged = list(target_ms_path.glob("*_pre-cal.ms")) + return mses_averaged From 16b24054fa569f0ebd3e57f8320a5981dd56ffb9 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 7 Aug 2026 15:51:32 +0200 Subject: [PATCH 100/120] Extract ddf subtract --- flocs_processing/dags/pilot_widefield.py | 76 +----------------------- flocs_processing/pipeline_runners.py | 72 +++++++++++++++++++++- 2 files changed, 73 insertions(+), 75 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index e997cc0..50a8832 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -12,6 +12,7 @@ run_pilot_facet_imaging_toil, run_pilot_facet_subtract_toil, run_pilot_intermediate_image_toil, + run_pilot_process_ddf_toil, run_prepare_ddf, run_prepare_ddf_subtract, ) @@ -567,80 +568,7 @@ def run_ddf_subtract(field): if field["status_vlbi_ddf_subtract"] == PIPELINE_STATUS.finished: return field else: - print( - f"Running ddf subtract for {field['target_name']} {field['sas_id_target']}" - ) - outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - target_path = get_most_recent_run( - outdir, field["sas_id_target"], "VLBI_delay-calibration" - ) - target_ms_path = target_path / "results_VLBI_delay-calibration" - print(f"Using data at: {target_path}/*.dp3concat") - - ddf_path = get_most_recent_run( - outdir, field["sas_id_target"], "DDF-pipeline" - ) - ddf_sols_path = ddf_path / "SOLSDIR" - print(f"Using DDF run at: {ddf_path}") - - context = get_current_context() - if context["ti"].try_number == 1 or ( - not os.path.isfile( - f"log_VLBI_process-ddf_{field['target_name']}_{field['sas_id_target']}.txt" - ) - ): - CURRENT_DB.set_status_processing( - field["target_name"], "ddf_subtract", field["sas_id_target"] - ) - cmd = f"flocs-run vlbi process-ddf --runner toil --record-toil-stats --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --ms-suffix .dp3concat --ddf-rundir {ddf_path} --solsdir {ddf_sols_path} --do-subtraction {target_ms_path}" - else: - # Extract the previous working directory - flocs_workdir = "" - print( - f"Scanning log_VLBI_process-ddf_{field['target_name']}_{field['sas_id_target']}.txt for workdir." - ) - with open( - f"log_VLBI_process-ddf_{field['target_name']}_{field['sas_id_target']}.txt" - ) as f_out: - for line in f_out.readlines(): - print(line) - if "Running workflow with" in line: - flocs_workdir = line.split(" ")[-1].strip() - break - if not flocs_workdir: - raise RuntimeError( - "Could not retrieve PILOT workdir. Flocs probably crashed before launching." - ) - print(f"Resuming failed PILOT run in {flocs_workdir}") - cmd = f"flocs-run vlbi process-ddf --runner toil --record-toil-stats --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --ms-suffix .dp3concat --ddf-rundir {ddf_path} --solsdir {ddf_sols_path} --do-subtraction {target_ms_path}" - if not os.path.isdir(outdir): - os.mkdir(outdir) - print(cmd) - with ( - open( - f"log_VLBI_process-ddf_{field['target_name']}_{field['sas_id_target']}.txt", - "w+", - ) as f_out, - open( - f"log_VLBI_process-ddf_{field['target_name']}_{field['sas_id_target']}_err.txt", - "w+", - ) as f_err, - ): - proc = subprocess.run( - cmd, shell=True, text=True, stdout=f_out, stderr=f_err - ) - success = False - pattern = re.compile(r"Workflow.* stopped. Success: True") - if not proc.returncode: - f_err.seek(0) - if pattern.search(f_err.read()): - success = True - if success: - CURRENT_DB.set_status_finished( - field["target_name"], "ddf_subtract", field["sas_id_target"] - ) - else: - raise RuntimeError + run_pilot_process_ddf_toil(field, CURRENT_DB) return field @task diff --git a/flocs_processing/pipeline_runners.py b/flocs_processing/pipeline_runners.py index 6922eb7..aa8ff2f 100644 --- a/flocs_processing/pipeline_runners.py +++ b/flocs_processing/pipeline_runners.py @@ -8,7 +8,6 @@ from airflow.exceptions import AirflowFailException from airflow.sdk import get_current_context from losoto.h5parm import h5parm -from pydantic_core.core_schema import model_ser_schema from flocs_processing.db_utils import PIPELINE_STATUS, FlocsDB @@ -1133,3 +1132,74 @@ def run_prepare_ddf(field): mses_averaged = list(target_ms_path.glob("*_pre-cal.ms")) return mses_averaged + + +def run_pilot_process_ddf_toil(field, db: FlocsDB): + print(f"Running ddf subtract for {field['target_name']} {field['sas_id_target']}") + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + target_path = get_most_recent_run( + outdir, field["sas_id_target"], "VLBI_delay-calibration" + ) + target_ms_path = target_path / "results_VLBI_delay-calibration" + print(f"Using data at: {target_path}/*.dp3concat") + + ddf_path = get_most_recent_run(outdir, field["sas_id_target"], "DDF-pipeline") + ddf_sols_path = ddf_path / "SOLSDIR" + print(f"Using DDF run at: {ddf_path}") + + context = get_current_context() + if context["ti"].try_number == 1 or ( + not os.path.isfile( + f"log_VLBI_process-ddf_{field['target_name']}_{field['sas_id_target']}.txt" + ) + ): + db.set_status_processing( + field["target_name"], "ddf_subtract", field["sas_id_target"] + ) + cmd = f"flocs-run vlbi process-ddf --runner toil --record-toil-stats --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --ms-suffix .dp3concat --ddf-rundir {ddf_path} --solsdir {ddf_sols_path} --do-subtraction {target_ms_path}" + else: + # Extract the previous working directory + flocs_workdir = "" + print( + f"Scanning log_VLBI_process-ddf_{field['target_name']}_{field['sas_id_target']}.txt for workdir." + ) + with open( + f"log_VLBI_process-ddf_{field['target_name']}_{field['sas_id_target']}.txt" + ) as f_out: + for line in f_out.readlines(): + print(line) + if "Running workflow with" in line: + flocs_workdir = line.split(" ")[-1].strip() + break + if not flocs_workdir: + raise RuntimeError( + "Could not retrieve PILOT workdir. Flocs probably crashed before launching." + ) + print(f"Resuming failed PILOT run in {flocs_workdir}") + cmd = f"flocs-run vlbi process-ddf --runner toil --record-toil-stats --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --ms-suffix .dp3concat --ddf-rundir {ddf_path} --solsdir {ddf_sols_path} --do-subtraction {target_ms_path}" + if not os.path.isdir(outdir): + os.mkdir(outdir) + print(cmd) + with ( + open( + f"log_VLBI_process-ddf_{field['target_name']}_{field['sas_id_target']}.txt", + "w+", + ) as f_out, + open( + f"log_VLBI_process-ddf_{field['target_name']}_{field['sas_id_target']}_err.txt", + "w+", + ) as f_err, + ): + proc = subprocess.run(cmd, shell=True, text=True, stdout=f_out, stderr=f_err) + success = False + pattern = re.compile(r"Workflow.* stopped. Success: True") + if not proc.returncode: + f_err.seek(0) + if pattern.search(f_err.read()): + success = True + if success: + db.set_status_finished( + field["target_name"], "ddf_subtract", field["sas_id_target"] + ) + else: + raise RuntimeError From 8cb3991b2f4aefb04b4c2a244c5da28998d32bc9 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 7 Aug 2026 17:31:05 +0200 Subject: [PATCH 101/120] Implement a logs directory for most steps --- flocs_processing/pipeline_runners.py | 138 +++++++++++++++++++++++---- 1 file changed, 118 insertions(+), 20 deletions(-) diff --git a/flocs_processing/pipeline_runners.py b/flocs_processing/pipeline_runners.py index aa8ff2f..08dcce4 100644 --- a/flocs_processing/pipeline_runners.py +++ b/flocs_processing/pipeline_runners.py @@ -67,14 +67,23 @@ def run_linc_calibrator_cwltool(field, calibrator_field: int, db: FlocsDB): field["target_name"], f"calibrator{calibrator_field}", field["sas_id_target"] ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + logsdir = os.path.join(OUTPUT_DIR, field["target_name"]) + if not os.path.isdir(logsdir): + os.mkdir(logsdir) cmd = f"flocs-run linc calibrator --runner cwltool --scheduler slurm --slurm-cores 32 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} {os.path.join(DATA_DIR, field['target_name'], 'calibrator', ms_folder)}" with ( open( - f"log_LINC_calibrator_{field['target_name']}_{field[f'sas_id_calibrator{calibrator_field}']}.txt", + os.path.join( + logsdir, + f"log_LINC_calibrator_{field['target_name']}_{field[f'sas_id_calibrator{calibrator_field}']}.txt", + ), "w+", ) as f_out, open( - f"log_LINC_calibrator_{field['target_name']}_{field[f'sas_id_calibrator{calibrator_field}']}_err.txt", + os.path.join( + logsdir, + f"log_LINC_calibrator_{field['target_name']}_{field[f'sas_id_calibrator{calibrator_field}']}_err.txt", + ), "w+", ) as f_err, ): @@ -125,17 +134,26 @@ def run_linc_calibrator_toil(field, calibrator_field: int, db: FlocsDB): field["target_name"], f"calibrator{calibrator_field}", field["sas_id_target"] ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + logsdir = os.path.join(OUTPUT_DIR, field["target_name"]) + if not os.path.isdir(logsdir): + os.mkdir(logsdir) cmd = f"flocs-run linc calibrator --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} {os.path.join(DATA_DIR, field['target_name'], 'calibrator', ms_folder)}" if not os.path.isdir(outdir): os.mkdir(outdir) print(cmd) with ( open( - f"log_LINC_calibrator_{field['target_name']}_{field[f'sas_id_calibrator{calibrator_field}']}.txt", + os.path.join( + logsdir, + f"log_LINC_calibrator_{field['target_name']}_{field[f'sas_id_calibrator{calibrator_field}']}.txt", + ), "w+", ) as f_out, open( - f"log_LINC_calibrator_{field['target_name']}_{field[f'sas_id_calibrator{calibrator_field}']}_err.txt", + os.path.join( + logsdir, + f"log_LINC_calibrator_{field['target_name']}_{field[f'sas_id_calibrator{calibrator_field}']}_err.txt", + ), "w+", ) as f_err, ): @@ -162,6 +180,9 @@ def run_linc_target_cwltool(field, db: FlocsDB): ) ms_folder = f"L{field['sas_id_target']}" outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + logsdir = os.path.join(OUTPUT_DIR, field["target_name"]) + if not os.path.isdir(logsdir): + os.mkdir(logsdir) calibrator_path = get_most_recent_run( outdir, field["sas_id_calibrator_final"], "LINC_calibrator" ) @@ -175,11 +196,17 @@ def run_linc_target_cwltool(field, db: FlocsDB): print(cmd) with ( open( - f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}.txt", + os.path.join( + logsdir, + f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}.txt", + ), "w+", ) as f_out, open( - f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}_err.txt", + os.path.join( + logsdir, + f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}_err.txt", + ), "w+", ) as f_err, ): @@ -227,6 +254,9 @@ def run_linc_target_toil(field, db: FlocsDB): ) ms_folder = f"L{field['sas_id_target']}" outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + logsdir = os.path.join(OUTPUT_DIR, field["target_name"]) + if not os.path.isdir(logsdir): + os.mkdir(logsdir) calibrator_path = get_most_recent_run( outdir, field["sas_id_calibrator_final"], "LINC_calibrator" ) @@ -240,11 +270,17 @@ def run_linc_target_toil(field, db: FlocsDB): print(cmd) with ( open( - f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}.txt", + os.path.join( + logsdir, + f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}.txt", + ), "w+", ) as f_out, open( - f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}_err.txt", + os.path.join( + logsdir, + f"log_LINC_target_{field['target_name']}_{field['sas_id_target']}_err.txt", + ), "w+", ) as f_err, ): @@ -268,6 +304,9 @@ def run_pilot_delay_cwltool(field, db: FlocsDB): f"Processing delay calibration for {field['target_name']} {field['sas_id_target']}" ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + logsdir = os.path.join(OUTPUT_DIR, field["target_name"]) + if not os.path.isdir(logsdir): + os.mkdir(logsdir) target_path = get_most_recent_run(outdir, field["sas_id_target"], "LINC_target") target_ms_path = target_path / "results_LINC_target" / "results" db.set_status_processing(field["target_name"], "vlbi_delay", field["sas_id_target"]) @@ -280,11 +319,17 @@ def run_pilot_delay_cwltool(field, db: FlocsDB): cmd = f"lofar-vlbi-plot --force --output_dir {outdir} --MS {ms}" with ( open( - f"log_plot_field_{field['target_name']}_{field['sas_id_target']}.txt", + os.path.join( + logsdir, + f"log_plot_field_{field['target_name']}_{field['sas_id_target']}.txt", + ), "w+", ) as f_out, open( - f"log_plot_field_{field['target_name']}_{field['sas_id_target']}_err.txt", + os.path.join( + logsdir, + f"log_plot_field_{field['target_name']}_{field['sas_id_target']}_err.txt", + ), "w+", ) as f_err, ): @@ -370,6 +415,9 @@ def run_pilot_delay_toil(field, db: FlocsDB): f"Processing delay calibration for {field['target_name']} {field['sas_id_target']}" ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + logsdir = os.path.join(OUTPUT_DIR, field["target_name"]) + if not os.path.isdir(logsdir): + os.mkdir(logsdir) target_path = get_most_recent_run(outdir, field["sas_id_target"], "LINC_target") target_ms_path = target_path / "results_LINC_target" / "results" db.set_status_processing(field["target_name"], "vlbi_delay", field["sas_id_target"]) @@ -382,11 +430,17 @@ def run_pilot_delay_toil(field, db: FlocsDB): cmd = f"lofar-vlbi-plot --force --output_dir {outdir} --MS {ms}" with ( open( - f"log_plot_field_{field['target_name']}_{field['sas_id_target']}.txt", + os.path.join( + logsdir, + f"log_plot_field_{field['target_name']}_{field['sas_id_target']}.txt", + ), "w+", ) as f_out, open( - f"log_plot_field_{field['target_name']}_{field['sas_id_target']}_err.txt", + os.path.join( + logsdir, + f"log_plot_field_{field['target_name']}_{field['sas_id_target']}_err.txt", + ), "w+", ) as f_err, ): @@ -478,6 +532,9 @@ def run_pilot_ddcal_cwltool(field, db: FlocsDB): f"Processing ILT dd calibration for {field['target_name']} {field['sas_id_target']}" ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + logsdir = os.path.join(OUTPUT_DIR, field["target_name"]) + if not os.path.isdir(logsdir): + os.mkdir(logsdir) if "status_ddf" not in field: print("Not a widefield imaging run, checking LINC + delay calibration.") target_path = get_most_recent_run(outdir, field["sas_id_target"], "LINC_target") @@ -519,11 +576,17 @@ def run_pilot_ddcal_cwltool(field, db: FlocsDB): print(cmd) with ( open( - f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt", + os.path.join( + logsdir, + f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt", + ), "w+", ) as f_out, open( - f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}_err.txt", + os.path.join( + logsdir, + f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}_err.txt", + ), "w+", ) as f_err, ): @@ -568,6 +631,9 @@ def run_pilot_ddcal_toil(field, db: FlocsDB): f"Processing ILT dd calibration for {field['target_name']} {field['sas_id_target']}" ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + logsdir = os.path.join(OUTPUT_DIR, field["target_name"]) + if not os.path.isdir(logsdir): + os.mkdir(logsdir) if "status_ddf" not in field: print("Not a widefield imaging run, checking LINC + delay calibration.") target_path = get_most_recent_run(outdir, field["sas_id_target"], "LINC_target") @@ -621,7 +687,10 @@ def run_pilot_ddcal_toil(field, db: FlocsDB): f"Scanning log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt for workdir." ) with open( - f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt" + os.path.join( + logsdir, + f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt", + ) ) as f_out: for line in f_out.readlines(): print(line) @@ -671,6 +740,9 @@ def run_pilot_intermediate_image_toil(field, db: FlocsDB): f"Processing ILT intermediate resolution imaging for {field['target_name']} {field['sas_id_target']}" ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + logsdir = os.path.join(OUTPUT_DIR, field["target_name"]) + if not os.path.isdir(logsdir): + os.mkdir(logsdir) target_path = get_most_recent_run( outdir, field["sas_id_target"], "VLBI_process-ddf" ) @@ -708,7 +780,10 @@ def run_pilot_intermediate_image_toil(field, db: FlocsDB): f"Scanning log_VLBI_image_intermediate_resolution_{field['target_name']}_{field['sas_id_target']}.txt for workdir." ) with open( - f"log_VLBI_image_intermediate_resolution_{field['target_name']}_{field['sas_id_target']}.txt" + os.path.join( + logsdir, + f"log_VLBI_image_intermediate_resolution_{field['target_name']}_{field['sas_id_target']}.txt", + ) ) as f_out: for line in f_out.readlines(): print(line) @@ -762,6 +837,9 @@ def run_pilot_facet_subtract_toil(field, db: FlocsDB): f"Processing ILT facet subtraction for {field['target_name']} {field['sas_id_target']}" ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + logsdir = os.path.join(OUTPUT_DIR, field["target_name"]) + if not os.path.isdir(logsdir): + os.mkdir(logsdir) target_path = get_most_recent_run( outdir, field["sas_id_target"], "VLBI_process-ddf" ) @@ -815,7 +893,10 @@ def run_pilot_facet_subtract_toil(field, db: FlocsDB): f"Scanning log_VLBI_facet-subtract_{field['target_name']}_{field['sas_id_target']}.txt for workdir." ) with open( - f"log_VLBI_facet-subtract_{field['target_name']}_{field['sas_id_target']}.txt" + os.path.join( + logsdir, + f"log_VLBI_facet-subtract_{field['target_name']}_{field['sas_id_target']}.txt", + ) ) as f_out: for line in f_out.readlines(): print(line) @@ -869,6 +950,9 @@ def run_pilot_facet_imaging_toil(field, db: FlocsDB): f"Processing ILT facet imaging for {field['target_name']} {field['sas_id_target']}" ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + logsdir = os.path.join(OUTPUT_DIR, field["target_name"]) + if not os.path.isdir(logsdir): + os.mkdir(logsdir) target_path = get_most_recent_run( outdir, field["sas_id_target"], "VLBI_facet_subtract" ) @@ -904,7 +988,10 @@ def run_pilot_facet_imaging_toil(field, db: FlocsDB): f"Scanning log_VLBI_facet_imaging_{field['target_name']}_{field['sas_id_target']}.txt for workdir." ) with open( - f"log_VLBI_facet_imaging_{field['target_name']}_{field['sas_id_target']}.txt" + os.path.join( + logsdir, + f"log_VLBI_facet_imaging_{field['target_name']}_{field['sas_id_target']}.txt", + ) ) as f_out: for line in f_out.readlines(): print(line) @@ -958,6 +1045,9 @@ def run_prepare_ddf_subtract(field): f"Preparing input for DDF subtract of {field['target_name']} {field['sas_id_target']}" ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + logsdir = os.path.join(OUTPUT_DIR, field["target_name"]) + if not os.path.isdir(logsdir): + os.mkdir(logsdir) target_path = get_most_recent_run(outdir, field["sas_id_target"], "VLBI_delay") target_ms_path = target_path / "results_VLBI_delay-calibration" mses_unaveraged = list(target_ms_path.glob("*.dp3concat")) @@ -977,7 +1067,6 @@ def run_prepare_ddf_subtract(field): print(f"Using LINC MSes at {linc_path}") linc_ms_path = linc_path / "results_LINC_target" / "results" mses_unaveraged = list(linc_ms_path.glob("*.dp3concat")) - mses_averaged = list(target_ms_path.glob("*_pre-cal.ms")) mses_unaveraged_pilot = list(target_ms_path.glob("*.dp3concat")) if not mses_unaveraged: raise RuntimeError( @@ -1048,6 +1137,9 @@ def run_prepare_ddf_subtract(field): def run_prepare_ddf(field): print(f"Preparing DDF input for {field['target_name']} {field['sas_id_target']}") outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + logsdir = os.path.join(OUTPUT_DIR, field["target_name"]) + if not os.path.isdir(logsdir): + os.mkdir(logsdir) target_path = get_most_recent_run(outdir, field["sas_id_target"], "VLBI_delay") target_ms_path = target_path / "results_VLBI_delay-calibration" mses_unaveraged = list(target_ms_path.glob("*.dp3concat")) @@ -1137,6 +1229,9 @@ def run_prepare_ddf(field): def run_pilot_process_ddf_toil(field, db: FlocsDB): print(f"Running ddf subtract for {field['target_name']} {field['sas_id_target']}") outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + logsdir = os.path.join(OUTPUT_DIR, field["target_name"]) + if not os.path.isdir(logsdir): + os.mkdir(logsdir) target_path = get_most_recent_run( outdir, field["sas_id_target"], "VLBI_delay-calibration" ) @@ -1164,7 +1259,10 @@ def run_pilot_process_ddf_toil(field, db: FlocsDB): f"Scanning log_VLBI_process-ddf_{field['target_name']}_{field['sas_id_target']}.txt for workdir." ) with open( - f"log_VLBI_process-ddf_{field['target_name']}_{field['sas_id_target']}.txt" + os.path.join( + logsdir, + f"log_VLBI_process-ddf_{field['target_name']}_{field['sas_id_target']}.txt", + ) ) as f_out: for line in f_out.readlines(): print(line) From 396627474e8b53877e079e38598c4af85b8a7480 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 7 Aug 2026 18:05:33 +0200 Subject: [PATCH 102/120] Extract initial ddf pipeline and actually make a logs sub directory --- flocs_processing/dags/pilot_widefield.py | 68 +--------------- flocs_processing/pipeline_runners.py | 99 ++++++++++++++++++++---- 2 files changed, 86 insertions(+), 81 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 50a8832..13c5c78 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -493,73 +493,7 @@ def run_ddf_pipeline(field): if field["status_ddf"] == PIPELINE_STATUS.finished: return field else: - print( - f"Starting ddf-pipeline for {field['target_name']} {field['sas_id_target']}" - ) - outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - target_path = get_most_recent_run( - outdir, field["sas_id_target"], "VLBI_delay-calibration" - ) - target_ms_path = target_path / "results_VLBI_delay-calibration" - if not list(target_ms_path.glob("*pre-cal.ms")): - target_path = get_most_recent_run( - outdir, field["sas_id_target"], "LINC_target" - ) - target_ms_path = target_path / "results_LINC_target" / "results" - - cmd = f"flocs-run ddf-pipeline --scheduler slurm --slurm-time 72:00:00 --slurm-cores 32 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --config-file {DDF_CONFIG} {target_ms_path}" - print(cmd) - CURRENT_DB.set_status_processing( - field["target_name"], "ddf", field["sas_id_target"] - ) - with ( - open( - f"log_DDF-pipeline_{field['target_name']}_{field['sas_id_target']}.txt", - "w+", - ) as f_out, - open( - f"log_DDF-pipeline_{field['target_name']}_{field['sas_id_target']}_err.txt", - "w+", - ) as f_err, - ): - proc = subprocess.run( - cmd, shell=True, text=True, stdout=f_out, stderr=f_err - ) - jobid = None - if not proc.returncode: - f_out.seek(0) - for line in f_out.readlines(): - if "Submitted batch job" in line: - jobid = line.strip().split()[-1] - else: - raise RuntimeError("Failed to submit job.") - - if not jobid: - raise RuntimeError("Failed to retrieve job id") - else: - while True: - print(f"Polling DDF-pipeine job {jobid}") - poll_cmd = f"sacct -X -j {jobid} --format=State --noheader" - status = subprocess.run( - poll_cmd, shell=True, text=True, capture_output=True - ).stdout.strip() - if (status == "RUNNING") or (status == "PENDING"): - time.sleep(60) - elif status == "COMPLETED": - CURRENT_DB.set_status_finished( - field["target_name"], - "ddf_subtract", - field["sas_id_target"], - ) - break - elif ( - (status == "FAILED") - or ("TIMEOUT" in status) - or ("CANCELLED" in status) - ): - raise RuntimeError( - f"DDF-pipeline for {field['target_name']} {field['sas_id_target']} failed." - ) + launch_ddf_pipeline(field, CURRENT_DB) return field @task diff --git a/flocs_processing/pipeline_runners.py b/flocs_processing/pipeline_runners.py index 08dcce4..718164e 100644 --- a/flocs_processing/pipeline_runners.py +++ b/flocs_processing/pipeline_runners.py @@ -67,7 +67,7 @@ def run_linc_calibrator_cwltool(field, calibrator_field: int, db: FlocsDB): field["target_name"], f"calibrator{calibrator_field}", field["sas_id_target"] ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - logsdir = os.path.join(OUTPUT_DIR, field["target_name"]) + logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") if not os.path.isdir(logsdir): os.mkdir(logsdir) cmd = f"flocs-run linc calibrator --runner cwltool --scheduler slurm --slurm-cores 32 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} {os.path.join(DATA_DIR, field['target_name'], 'calibrator', ms_folder)}" @@ -134,7 +134,7 @@ def run_linc_calibrator_toil(field, calibrator_field: int, db: FlocsDB): field["target_name"], f"calibrator{calibrator_field}", field["sas_id_target"] ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - logsdir = os.path.join(OUTPUT_DIR, field["target_name"]) + logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") if not os.path.isdir(logsdir): os.mkdir(logsdir) cmd = f"flocs-run linc calibrator --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} {os.path.join(DATA_DIR, field['target_name'], 'calibrator', ms_folder)}" @@ -180,7 +180,7 @@ def run_linc_target_cwltool(field, db: FlocsDB): ) ms_folder = f"L{field['sas_id_target']}" outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - logsdir = os.path.join(OUTPUT_DIR, field["target_name"]) + logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") if not os.path.isdir(logsdir): os.mkdir(logsdir) calibrator_path = get_most_recent_run( @@ -254,7 +254,7 @@ def run_linc_target_toil(field, db: FlocsDB): ) ms_folder = f"L{field['sas_id_target']}" outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - logsdir = os.path.join(OUTPUT_DIR, field["target_name"]) + logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") if not os.path.isdir(logsdir): os.mkdir(logsdir) calibrator_path = get_most_recent_run( @@ -304,7 +304,7 @@ def run_pilot_delay_cwltool(field, db: FlocsDB): f"Processing delay calibration for {field['target_name']} {field['sas_id_target']}" ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - logsdir = os.path.join(OUTPUT_DIR, field["target_name"]) + logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") if not os.path.isdir(logsdir): os.mkdir(logsdir) target_path = get_most_recent_run(outdir, field["sas_id_target"], "LINC_target") @@ -415,7 +415,7 @@ def run_pilot_delay_toil(field, db: FlocsDB): f"Processing delay calibration for {field['target_name']} {field['sas_id_target']}" ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - logsdir = os.path.join(OUTPUT_DIR, field["target_name"]) + logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") if not os.path.isdir(logsdir): os.mkdir(logsdir) target_path = get_most_recent_run(outdir, field["sas_id_target"], "LINC_target") @@ -532,7 +532,7 @@ def run_pilot_ddcal_cwltool(field, db: FlocsDB): f"Processing ILT dd calibration for {field['target_name']} {field['sas_id_target']}" ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - logsdir = os.path.join(OUTPUT_DIR, field["target_name"]) + logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") if not os.path.isdir(logsdir): os.mkdir(logsdir) if "status_ddf" not in field: @@ -631,7 +631,7 @@ def run_pilot_ddcal_toil(field, db: FlocsDB): f"Processing ILT dd calibration for {field['target_name']} {field['sas_id_target']}" ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - logsdir = os.path.join(OUTPUT_DIR, field["target_name"]) + logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") if not os.path.isdir(logsdir): os.mkdir(logsdir) if "status_ddf" not in field: @@ -740,7 +740,7 @@ def run_pilot_intermediate_image_toil(field, db: FlocsDB): f"Processing ILT intermediate resolution imaging for {field['target_name']} {field['sas_id_target']}" ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - logsdir = os.path.join(OUTPUT_DIR, field["target_name"]) + logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") if not os.path.isdir(logsdir): os.mkdir(logsdir) target_path = get_most_recent_run( @@ -837,7 +837,7 @@ def run_pilot_facet_subtract_toil(field, db: FlocsDB): f"Processing ILT facet subtraction for {field['target_name']} {field['sas_id_target']}" ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - logsdir = os.path.join(OUTPUT_DIR, field["target_name"]) + logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") if not os.path.isdir(logsdir): os.mkdir(logsdir) target_path = get_most_recent_run( @@ -950,7 +950,7 @@ def run_pilot_facet_imaging_toil(field, db: FlocsDB): f"Processing ILT facet imaging for {field['target_name']} {field['sas_id_target']}" ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - logsdir = os.path.join(OUTPUT_DIR, field["target_name"]) + logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") if not os.path.isdir(logsdir): os.mkdir(logsdir) target_path = get_most_recent_run( @@ -1045,7 +1045,7 @@ def run_prepare_ddf_subtract(field): f"Preparing input for DDF subtract of {field['target_name']} {field['sas_id_target']}" ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - logsdir = os.path.join(OUTPUT_DIR, field["target_name"]) + logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") if not os.path.isdir(logsdir): os.mkdir(logsdir) target_path = get_most_recent_run(outdir, field["sas_id_target"], "VLBI_delay") @@ -1137,7 +1137,7 @@ def run_prepare_ddf_subtract(field): def run_prepare_ddf(field): print(f"Preparing DDF input for {field['target_name']} {field['sas_id_target']}") outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - logsdir = os.path.join(OUTPUT_DIR, field["target_name"]) + logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") if not os.path.isdir(logsdir): os.mkdir(logsdir) target_path = get_most_recent_run(outdir, field["sas_id_target"], "VLBI_delay") @@ -1229,7 +1229,7 @@ def run_prepare_ddf(field): def run_pilot_process_ddf_toil(field, db: FlocsDB): print(f"Running ddf subtract for {field['target_name']} {field['sas_id_target']}") outdir = os.path.join(OUTPUT_DIR, field["target_name"]) - logsdir = os.path.join(OUTPUT_DIR, field["target_name"]) + logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") if not os.path.isdir(logsdir): os.mkdir(logsdir) target_path = get_most_recent_run( @@ -1301,3 +1301,74 @@ def run_pilot_process_ddf_toil(field, db: FlocsDB): ) else: raise RuntimeError + + +def launch_ddf_pipeline(field, db: FlocsDB): + print(f"Starting ddf-pipeline for {field['target_name']} {field['sas_id_target']}") + outdir = os.path.join(OUTPUT_DIR, field["target_name"]) + logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") + if not os.path.isdir(logsdir): + os.mkdir(logsdir) + target_path = get_most_recent_run( + outdir, field["sas_id_target"], "VLBI_delay-calibration" + ) + target_ms_path = target_path / "results_VLBI_delay-calibration" + if not list(target_ms_path.glob("*pre-cal.ms")): + target_path = get_most_recent_run(outdir, field["sas_id_target"], "LINC_target") + target_ms_path = target_path / "results_LINC_target" / "results" + + cmd = f"flocs-run ddf-pipeline --scheduler slurm --slurm-time 72:00:00 --slurm-cores 32 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --config-file {DDF_CONFIG} {target_ms_path}" + print(cmd) + db.set_status_processing(field["target_name"], "ddf", field["sas_id_target"]) + with ( + open( + os.path.join( + logsdir, + f"log_DDF-pipeline_{field['target_name']}_{field['sas_id_target']}.txt", + ), + "w+", + ) as f_out, + open( + os.path.join( + logsdir, + f"log_DDF-pipeline_{field['target_name']}_{field['sas_id_target']}_err.txt", + ), + "w+", + ) as f_err, + ): + proc = subprocess.run(cmd, shell=True, text=True, stdout=f_out, stderr=f_err) + jobid = None + if not proc.returncode: + f_out.seek(0) + for line in f_out.readlines(): + if "Submitted batch job" in line: + jobid = line.strip().split()[-1] + else: + raise RuntimeError("Failed to submit job.") + + if not jobid: + raise RuntimeError("Failed to retrieve job id") + else: + while True: + print(f"Polling DDF-pipeine job {jobid}") + poll_cmd = f"sacct -X -j {jobid} --format=State --noheader" + status = subprocess.run( + poll_cmd, shell=True, text=True, capture_output=True + ).stdout.strip() + if (status == "RUNNING") or (status == "PENDING"): + time.sleep(60) + elif status == "COMPLETED": + db.set_status_finished( + field["target_name"], + "ddf_subtract", + field["sas_id_target"], + ) + break + elif ( + (status == "FAILED") + or ("TIMEOUT" in status) + or ("CANCELLED" in status) + ): + raise RuntimeError( + f"DDF-pipeline for {field['target_name']} {field['sas_id_target']} failed." + ) From 2eb8e5735ef89478bfe73d149e528ee55f1f8367 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 7 Aug 2026 20:56:24 +0200 Subject: [PATCH 103/120] Fix unprocessed target assignment --- flocs_processing/dags/pilot_widefield.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 13c5c78..91933d0 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -104,7 +104,7 @@ def get_unprocessed_target(): break if not is_processing: # Only select a field if nothing is processing it. - field = dict(row) + field = row break if not field: raise AirflowSkipException("No unprocessed fields found.") From 9b2508b2b9c55f7b924468c8636dc4c87672632b Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 7 Aug 2026 20:56:54 +0200 Subject: [PATCH 104/120] Fix unprocessed target assignment --- flocs_processing/dags/pilot_widefield.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 3b4a224..769d5ca 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -719,7 +719,7 @@ def get_unprocessed_target(): break if not is_processing: # Only select a field if nothing is processing it. - field = dict(row) + field = row break if not field: raise AirflowSkipException("No unprocessed fields found.") From 4945af2fe176ae5587f420f5eb69d277b01f7185 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 14 Aug 2026 16:02:02 +0100 Subject: [PATCH 105/120] Check user home for a config to be a bit more flexible for screens/tmuxes --- flocs_processing/pipeline_runners.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/flocs_processing/pipeline_runners.py b/flocs_processing/pipeline_runners.py index 718164e..b8467c4 100644 --- a/flocs_processing/pipeline_runners.py +++ b/flocs_processing/pipeline_runners.py @@ -11,15 +11,16 @@ from flocs_processing.db_utils import PIPELINE_STATUS, FlocsDB -if "FLOCS_AIRFLOW_CONFIG" not in os.environ: - raise RuntimeError( - "FLOCS_AIRFLOW_CONFIG environment variable not set. Please point this to a valid configuration file." - ) - # Need to think of a way to centralise this and not read multiple times here and in the DAG -CONFIG_FILE: str = os.getenv("FLOCS_AIRFLOW_CONFIG") or "" -if not os.path.isfile(CONFIG_FILE): - raise RuntimeError(f"{CONFIG_FILE} is not a valid file") +if "FLOCS_AIRFLOW_CONFIG" not in os.environ: + if not pathlib.Path(os.path.expandvars("$HOME/.flocs_airflow.cfg")).is_file(): + raise RuntimeError( + "FLOCS_AIRFLOW_CONFIG environment variable not set and no $HOME/.flocs_airflow.cfg exists. Please create a valid configuration file." + ) + else: + CONFIG_FILE = os.path.expandvars("$HOME/.flocs_airflow.cfg") +else: + CONFIG_FILE = os.getenv("FLOCS_AIRFLOW_CONFIG") or "" parser = configparser.ConfigParser() parser.optionxform = str # ty: ignore[invalid-assignment] @@ -30,8 +31,6 @@ for k, v in parser["DEFAULT"].items(): print(f"{k}: {v}") -TABLE_NAME = parser["DEFAULT"]["TABLE_NAME"] -DATABASE = parser["DEFAULT"]["DATABASE"] SLURM_ACCOUNT = parser["DEFAULT"]["SLURM_ACCOUNT"] SLURM_QUEUE = parser["DEFAULT"]["SLURM_QUEUE"] DATA_DIR = parser["DEFAULT"]["DATA_DIR"] From b6b7e6eda2deaf5cb1c1caa1756ad30ba041e1b2 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 14 Aug 2026 16:07:46 +0100 Subject: [PATCH 106/120] Resolve ddf launching --- flocs_processing/dags/pilot_widefield.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 91933d0..9cbe4da 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -15,6 +15,7 @@ run_pilot_process_ddf_toil, run_prepare_ddf, run_prepare_ddf_subtract, + launch_ddf_pipeline, ) import configparser From fce2eaa0398f832d548ea1138ce1aac7a6502869 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 14 Aug 2026 16:17:17 +0100 Subject: [PATCH 107/120] Output logs to the proper directory everywhere --- flocs_processing/pipeline_runners.py | 120 +++++++++++++++++++++------ 1 file changed, 94 insertions(+), 26 deletions(-) diff --git a/flocs_processing/pipeline_runners.py b/flocs_processing/pipeline_runners.py index b8467c4..758a956 100644 --- a/flocs_processing/pipeline_runners.py +++ b/flocs_processing/pipeline_runners.py @@ -360,11 +360,17 @@ def run_pilot_delay_cwltool(field, db: FlocsDB): print(cmd) with ( open( - f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt", + os.path.join( + logsdir, + f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt", + ), "w+", ) as f_out, open( - f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}_err.txt", + os.path.join( + logsdir, + f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}_err.txt", + ), "w+", ) as f_err, ): @@ -468,7 +474,10 @@ def run_pilot_delay_toil(field, db: FlocsDB): context = get_current_context() if context["ti"].try_number == 1 or ( not os.path.isfile( - f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt" + os.path.join( + logsdir, + f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt", + ) ) ): cmd = f"flocs-run vlbi delay-calibration --record-toil-stats --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --ms-suffix dp3concat --delay-calibrator {delay_cat} --image-catalogue {image_cat} --apply-delay-solutions {target_ms_path}" @@ -497,11 +506,17 @@ def run_pilot_delay_toil(field, db: FlocsDB): print(cmd) with ( open( - f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt", + os.path.join( + logsdir, + f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt", + ), "w+", ) as f_out, open( - f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}_err.txt", + os.path.join( + logsdir, + f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}_err.txt", + ), "w+", ) as f_err, ): @@ -671,7 +686,10 @@ def run_pilot_ddcal_toil(field, db: FlocsDB): context = get_current_context() if context["ti"].try_number == 1 or ( not os.path.isfile( - f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt" + os.path.join( + logsdir, + f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt", + ) ) ): cmd = f"flocs-run vlbi dd-calibration --record-toil-stats --runner toil --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat.sub.ms {target_ms_path}" @@ -707,11 +725,17 @@ def run_pilot_ddcal_toil(field, db: FlocsDB): print(cmd) with ( open( - f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt", + os.path.join( + logsdir, + f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}.txt", + ), "w+", ) as f_out, open( - f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}_err.txt", + os.path.join( + logsdir, + f"log_VLBI_dd-calibration_{field['target_name']}_{field['sas_id_target']}_err.txt", + ), "w+", ) as f_err, ): @@ -764,7 +788,10 @@ def run_pilot_intermediate_image_toil(field, db: FlocsDB): context = get_current_context() if context["ti"].try_number == 1 or ( not os.path.isfile( - f"log_VLBI_image_intermediate_resolution_{field['target_name']}_{field['sas_id_target']}.txt" + os.path.join( + logsdir, + f"log_VLBI_image_intermediate_resolution_{field['target_name']}_{field['sas_id_target']}.txt", + ) ) ): cmd = f"flocs-run vlbi image-intermediate-resolution --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --dd-solutions {dd_sols} --ms-suffix .dp3concat.sub.ms {target_ms_path}" @@ -800,11 +827,17 @@ def run_pilot_intermediate_image_toil(field, db: FlocsDB): print(cmd) with ( open( - f"log_VLBI_image_intermediate_resolution_{field['target_name']}_{field['sas_id_target']}.txt", + os.path.join( + logsdir, + f"log_VLBI_image_intermediate_resolution_{field['target_name']}_{field['sas_id_target']}.txt", + ), "w+", ) as f_out, open( - f"log_VLBI_image_intermediate_resolution_{field['target_name']}_{field['sas_id_target']}_err.txt", + os.path.join( + logsdir, + f"log_VLBI_image_intermediate_resolution_{field['target_name']}_{field['sas_id_target']}_err.txt", + ), "w+", ) as f_err, ): @@ -877,7 +910,10 @@ def run_pilot_facet_subtract_toil(field, db: FlocsDB): context = get_current_context() if context["ti"].try_number == 1 or ( not os.path.isfile( - f"log_VLBI_facet-subtract_{field['target_name']}_{field['sas_id_target']}.txt" + os.path.join( + logsdir, + f"log_VLBI_facet-subtract_{field['target_name']}_{field['sas_id_target']}.txt", + ) ) ): cmd = f"flocs-run vlbi facet-subtract --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --dd-solutions {dd_sols} --model-image-directory {model_images_path} --ms-suffix .dp3concat.sub.ms {target_ms_path}" @@ -913,11 +949,17 @@ def run_pilot_facet_subtract_toil(field, db: FlocsDB): print(cmd) with ( open( - f"log_VLBI_facet-subtract_{field['target_name']}_{field['sas_id_target']}.txt", + os.path.join( + logsdir, + f"log_VLBI_facet-subtract_{field['target_name']}_{field['sas_id_target']}.txt", + ), "w+", ) as f_out, open( - f"log_VLBI_facet-subtract_{field['target_name']}_{field['sas_id_target']}_err.txt", + os.path.join( + logsdir, + f"log_VLBI_facet-subtract_{field['target_name']}_{field['sas_id_target']}_err.txt", + ), "w+", ) as f_err, ): @@ -972,7 +1014,10 @@ def run_pilot_facet_imaging_toil(field, db: FlocsDB): context = get_current_context() if context["ti"].try_number == 1 or ( not os.path.isfile( - f"log_VLBI_facet-imaging_{field['target_name']}_{field['sas_id_target']}.txt" + os.path.join( + logsdir, + f"log_VLBI_facet-imaging_{field['target_name']}_{field['sas_id_target']}.txt", + ) ) ): cmd = f"flocs-run vlbi facet-imaging --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --resolution 0.3asec --pixel-scale 0.1 --ms-suffix .ms {target_ms_path}" @@ -1008,11 +1053,17 @@ def run_pilot_facet_imaging_toil(field, db: FlocsDB): print(cmd) with ( open( - f"log_VLBI_facet_imaging_{field['target_name']}_{field['sas_id_target']}.txt", + os.path.join( + logsdir, + f"log_VLBI_facet_imaging_{field['target_name']}_{field['sas_id_target']}.txt", + ), "w+", ) as f_out, open( - f"log_VLBI_facet_imaging_{field['target_name']}_{field['sas_id_target']}_err.txt", + os.path.join( + logsdir, + f"log_VLBI_facet_imaging_{field['target_name']}_{field['sas_id_target']}_err.txt", + ), "w+", ) as f_err, ): @@ -1244,7 +1295,10 @@ def run_pilot_process_ddf_toil(field, db: FlocsDB): context = get_current_context() if context["ti"].try_number == 1 or ( not os.path.isfile( - f"log_VLBI_process-ddf_{field['target_name']}_{field['sas_id_target']}.txt" + os.path.join( + logsdir, + f"log_VLBI_process-ddf_{field['target_name']}_{field['sas_id_target']}.txt", + ) ) ): db.set_status_processing( @@ -1257,12 +1311,20 @@ def run_pilot_process_ddf_toil(field, db: FlocsDB): print( f"Scanning log_VLBI_process-ddf_{field['target_name']}_{field['sas_id_target']}.txt for workdir." ) - with open( - os.path.join( - logsdir, - f"log_VLBI_process-ddf_{field['target_name']}_{field['sas_id_target']}.txt", - ) - ) as f_out: + with ( + open( + os.path.join( + logsdir, + f"log_VLBI_process-ddf_{field['target_name']}_{field['sas_id_target']}.txt", + ) + ) as f_out, + open( + os.path.join( + logsdir, + f"log_VLBI_process-ddf_{field['target_name']}_{field['sas_id_target']}_err.txt", + ) + ) as f_err, + ): for line in f_out.readlines(): print(line) if "Running workflow with" in line: @@ -1279,11 +1341,17 @@ def run_pilot_process_ddf_toil(field, db: FlocsDB): print(cmd) with ( open( - f"log_VLBI_process-ddf_{field['target_name']}_{field['sas_id_target']}.txt", + os.path.join( + logsdir, + f"log_VLBI_process-ddf_{field['target_name']}_{field['sas_id_target']}.txt", + ), "w+", ) as f_out, open( - f"log_VLBI_process-ddf_{field['target_name']}_{field['sas_id_target']}_err.txt", + os.path.join( + logsdir, + f"log_VLBI_process-ddf_{field['target_name']}_{field['sas_id_target']}_err.txt", + ), "w+", ) as f_err, ): From 8f2bc2aaeb7a2c1848615fe10367a15175f90c20 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 14 Aug 2026 16:19:18 +0100 Subject: [PATCH 108/120] Cleanup remaining ddf logs --- flocs_processing/dags/pilot_widefield.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 9cbe4da..6f4cf72 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -453,20 +453,18 @@ def run_vlbi_delay(field): @task def run_ddf_pipeline(field): field = dict(CURRENT_DB.get_db_columns(field["sas_id_target"])[0]) + logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") if field["status_ddf"] == PIPELINE_STATUS.processing: print( f"ddf-pipeline for {field['target_name']} {field['sas_id_target']} should be running, attempting to resume polling..." ) - with ( - open( + with open( + os.path.join( + logsdir, f"log_DDF-pipeline_{field['target_name']}_{field['sas_id_target']}.txt", - "r", - ) as f_out, - open( - f"log_DDF-pipeline_{field['target_name']}_{field['sas_id_target']}_err.txt", - "r", - ) as f_err, - ): + ), + "r", + ) as f_out: jobid = None for line in f_out.readlines(): if "Submitted batch job" in line: From 2b57ab5b9f246538d38b79b99a96cb5c57b65580 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 14 Aug 2026 16:20:36 +0100 Subject: [PATCH 109/120] Better boolean parsing --- flocs_processing/dags/pilot_widefield.py | 2 +- flocs_processing/pipeline_runners.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 6f4cf72..2e55c5c 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -65,7 +65,7 @@ NN_MODEL_CACHE = parser["DEFAULT"]["NN_MODEL_CACHE"] DDF_CONFIG = parser["DEFAULT"]["DDF_CONFIG"] FLUX_CALIBRATOR_TEMPLATE = parser["DEFAULT"]["FLUX_CALIBRATOR_TEMPLATE"] -NEEDS_MANUAL_APPROVAL_DELAY = bool(parser["DEFAULT"]["NEEDS_MANUAL_APPROVAL_DELAY"]) +NEEDS_MANUAL_APPROVAL_DELAY = parser.getboolean("DEFAULT", "NEEDS_MANUAL_APPROVAL_DELAY") CWL_RUNNER_LINC_CALIBRATOR = "cwltool" CWL_RUNNER_LINC_TARGET = "toil" diff --git a/flocs_processing/pipeline_runners.py b/flocs_processing/pipeline_runners.py index 758a956..d025d9d 100644 --- a/flocs_processing/pipeline_runners.py +++ b/flocs_processing/pipeline_runners.py @@ -39,7 +39,7 @@ NN_MODEL_CACHE = parser["DEFAULT"]["NN_MODEL_CACHE"] DDF_CONFIG = parser["DEFAULT"]["DDF_CONFIG"] FLUX_CALIBRATOR_TEMPLATE = parser["DEFAULT"]["FLUX_CALIBRATOR_TEMPLATE"] -NEEDS_MANUAL_APPROVAL_DELAY = bool(parser["DEFAULT"]["NEEDS_MANUAL_APPROVAL_DELAY"]) +NEEDS_MANUAL_APPROVAL_DELAY = parser.getboolean("DEFAULT", "NEEDS_MANUAL_APPROVAL_DELAY") def get_most_recent_run(searchpath: str, sas_id: str, pipeline: str) -> pathlib.Path: From 04690d6fecde432996636652e997add89891db53 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 14 Aug 2026 16:25:25 +0100 Subject: [PATCH 110/120] Safeguard directory creations --- flocs_processing/pipeline_runners.py | 36 +++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/flocs_processing/pipeline_runners.py b/flocs_processing/pipeline_runners.py index d025d9d..bde78de 100644 --- a/flocs_processing/pipeline_runners.py +++ b/flocs_processing/pipeline_runners.py @@ -67,6 +67,8 @@ def run_linc_calibrator_cwltool(field, calibrator_field: int, db: FlocsDB): ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") + if not os.path.isdir(outdir): + os.mkdir(outdir) if not os.path.isdir(logsdir): os.mkdir(logsdir) cmd = f"flocs-run linc calibrator --runner cwltool --scheduler slurm --slurm-cores 32 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} {os.path.join(DATA_DIR, field['target_name'], 'calibrator', ms_folder)}" @@ -134,11 +136,11 @@ def run_linc_calibrator_toil(field, calibrator_field: int, db: FlocsDB): ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") + if not os.path.isdir(outdir): + os.mkdir(outdir) if not os.path.isdir(logsdir): os.mkdir(logsdir) cmd = f"flocs-run linc calibrator --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} {os.path.join(DATA_DIR, field['target_name'], 'calibrator', ms_folder)}" - if not os.path.isdir(outdir): - os.mkdir(outdir) print(cmd) with ( open( @@ -180,6 +182,8 @@ def run_linc_target_cwltool(field, db: FlocsDB): ms_folder = f"L{field['sas_id_target']}" outdir = os.path.join(OUTPUT_DIR, field["target_name"]) logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") + if not os.path.isdir(outdir): + os.mkdir(outdir) if not os.path.isdir(logsdir): os.mkdir(logsdir) calibrator_path = get_most_recent_run( @@ -190,8 +194,6 @@ def run_linc_target_cwltool(field, db: FlocsDB): ) db.set_status_processing(field["target_name"], "target", field["sas_id_target"]) cmd = f"flocs-run linc target --runner cwltool --scheduler slurm --slurm-cores 64 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --cal-solutions {calibrator_solutions} {os.path.join(DATA_DIR, field['target_name'], 'target', ms_folder)}" - if not os.path.isdir(outdir): - os.mkdir(outdir) print(cmd) with ( open( @@ -254,6 +256,8 @@ def run_linc_target_toil(field, db: FlocsDB): ms_folder = f"L{field['sas_id_target']}" outdir = os.path.join(OUTPUT_DIR, field["target_name"]) logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") + if not os.path.isdir(outdir): + os.mkdir(outdir) if not os.path.isdir(logsdir): os.mkdir(logsdir) calibrator_path = get_most_recent_run( @@ -264,8 +268,6 @@ def run_linc_target_toil(field, db: FlocsDB): ) db.set_status_processing(field["target_name"], "target", field["sas_id_target"]) cmd = f"flocs-run linc target --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --cal-solutions {calibrator_solutions} {os.path.join(DATA_DIR, field['target_name'], 'target', ms_folder)}" - if not os.path.isdir(outdir): - os.mkdir(outdir) print(cmd) with ( open( @@ -304,6 +306,8 @@ def run_pilot_delay_cwltool(field, db: FlocsDB): ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") + if not os.path.isdir(outdir): + os.mkdir(outdir) if not os.path.isdir(logsdir): os.mkdir(logsdir) target_path = get_most_recent_run(outdir, field["sas_id_target"], "LINC_target") @@ -421,6 +425,8 @@ def run_pilot_delay_toil(field, db: FlocsDB): ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") + if not os.path.isdir(outdir): + os.mkdir(outdir) if not os.path.isdir(logsdir): os.mkdir(logsdir) target_path = get_most_recent_run(outdir, field["sas_id_target"], "LINC_target") @@ -547,6 +553,8 @@ def run_pilot_ddcal_cwltool(field, db: FlocsDB): ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") + if not os.path.isdir(outdir): + os.mkdir(outdir) if not os.path.isdir(logsdir): os.mkdir(logsdir) if "status_ddf" not in field: @@ -646,6 +654,8 @@ def run_pilot_ddcal_toil(field, db: FlocsDB): ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") + if not os.path.isdir(outdir): + os.mkdir(outdir) if not os.path.isdir(logsdir): os.mkdir(logsdir) if "status_ddf" not in field: @@ -764,6 +774,8 @@ def run_pilot_intermediate_image_toil(field, db: FlocsDB): ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") + if not os.path.isdir(outdir): + os.mkdir(outdir) if not os.path.isdir(logsdir): os.mkdir(logsdir) target_path = get_most_recent_run( @@ -870,6 +882,8 @@ def run_pilot_facet_subtract_toil(field, db: FlocsDB): ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") + if not os.path.isdir(outdir): + os.mkdir(outdir) if not os.path.isdir(logsdir): os.mkdir(logsdir) target_path = get_most_recent_run( @@ -992,6 +1006,8 @@ def run_pilot_facet_imaging_toil(field, db: FlocsDB): ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") + if not os.path.isdir(outdir): + os.mkdir(outdir) if not os.path.isdir(logsdir): os.mkdir(logsdir) target_path = get_most_recent_run( @@ -1096,6 +1112,8 @@ def run_prepare_ddf_subtract(field): ) outdir = os.path.join(OUTPUT_DIR, field["target_name"]) logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") + if not os.path.isdir(outdir): + os.mkdir(outdir) if not os.path.isdir(logsdir): os.mkdir(logsdir) target_path = get_most_recent_run(outdir, field["sas_id_target"], "VLBI_delay") @@ -1188,6 +1206,8 @@ def run_prepare_ddf(field): print(f"Preparing DDF input for {field['target_name']} {field['sas_id_target']}") outdir = os.path.join(OUTPUT_DIR, field["target_name"]) logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") + if not os.path.isdir(outdir): + os.mkdir(outdir) if not os.path.isdir(logsdir): os.mkdir(logsdir) target_path = get_most_recent_run(outdir, field["sas_id_target"], "VLBI_delay") @@ -1280,6 +1300,8 @@ def run_pilot_process_ddf_toil(field, db: FlocsDB): print(f"Running ddf subtract for {field['target_name']} {field['sas_id_target']}") outdir = os.path.join(OUTPUT_DIR, field["target_name"]) logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") + if not os.path.isdir(outdir): + os.mkdir(outdir) if not os.path.isdir(logsdir): os.mkdir(logsdir) target_path = get_most_recent_run( @@ -1374,6 +1396,8 @@ def launch_ddf_pipeline(field, db: FlocsDB): print(f"Starting ddf-pipeline for {field['target_name']} {field['sas_id_target']}") outdir = os.path.join(OUTPUT_DIR, field["target_name"]) logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") + if not os.path.isdir(outdir): + os.mkdir(outdir) if not os.path.isdir(logsdir): os.mkdir(logsdir) target_path = get_most_recent_run( From afc7cef347500a03f61c904fe3c242f0824ecc44 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 14 Aug 2026 16:27:15 +0100 Subject: [PATCH 111/120] No crash if dir exists --- flocs_processing/pipeline_runners.py | 76 ++++++++++++++-------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/flocs_processing/pipeline_runners.py b/flocs_processing/pipeline_runners.py index bde78de..2df6b94 100644 --- a/flocs_processing/pipeline_runners.py +++ b/flocs_processing/pipeline_runners.py @@ -68,9 +68,9 @@ def run_linc_calibrator_cwltool(field, calibrator_field: int, db: FlocsDB): outdir = os.path.join(OUTPUT_DIR, field["target_name"]) logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") if not os.path.isdir(outdir): - os.mkdir(outdir) + os.makedirs(outdir, exist_ok=True) if not os.path.isdir(logsdir): - os.mkdir(logsdir) + os.makedirs(logsdir, exist_ok=True) cmd = f"flocs-run linc calibrator --runner cwltool --scheduler slurm --slurm-cores 32 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} {os.path.join(DATA_DIR, field['target_name'], 'calibrator', ms_folder)}" with ( open( @@ -137,9 +137,9 @@ def run_linc_calibrator_toil(field, calibrator_field: int, db: FlocsDB): outdir = os.path.join(OUTPUT_DIR, field["target_name"]) logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") if not os.path.isdir(outdir): - os.mkdir(outdir) + os.makedirs(outdir, exist_ok=True) if not os.path.isdir(logsdir): - os.mkdir(logsdir) + os.makedirs(logsdir, exist_ok=True) cmd = f"flocs-run linc calibrator --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} {os.path.join(DATA_DIR, field['target_name'], 'calibrator', ms_folder)}" print(cmd) with ( @@ -183,9 +183,9 @@ def run_linc_target_cwltool(field, db: FlocsDB): outdir = os.path.join(OUTPUT_DIR, field["target_name"]) logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") if not os.path.isdir(outdir): - os.mkdir(outdir) + os.makedirs(outdir, exist_ok=True) if not os.path.isdir(logsdir): - os.mkdir(logsdir) + os.makedirs(logsdir, exist_ok=True) calibrator_path = get_most_recent_run( outdir, field["sas_id_calibrator_final"], "LINC_calibrator" ) @@ -257,9 +257,9 @@ def run_linc_target_toil(field, db: FlocsDB): outdir = os.path.join(OUTPUT_DIR, field["target_name"]) logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") if not os.path.isdir(outdir): - os.mkdir(outdir) + os.makedirs(outdir, exist_ok=True) if not os.path.isdir(logsdir): - os.mkdir(logsdir) + os.makedirs(logsdir, exist_ok=True) calibrator_path = get_most_recent_run( outdir, field["sas_id_calibrator_final"], "LINC_calibrator" ) @@ -307,9 +307,9 @@ def run_pilot_delay_cwltool(field, db: FlocsDB): outdir = os.path.join(OUTPUT_DIR, field["target_name"]) logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") if not os.path.isdir(outdir): - os.mkdir(outdir) + os.makedirs(outdir, exist_ok=True) if not os.path.isdir(logsdir): - os.mkdir(logsdir) + os.makedirs(logsdir, exist_ok=True) target_path = get_most_recent_run(outdir, field["sas_id_target"], "LINC_target") target_ms_path = target_path / "results_LINC_target" / "results" db.set_status_processing(field["target_name"], "vlbi_delay", field["sas_id_target"]) @@ -360,7 +360,7 @@ def run_pilot_delay_cwltool(field, db: FlocsDB): cmd = f"flocs-run vlbi delay-calibration --record-toil-stats --runner cwltool --scheduler slurm --slurm-cores 64 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --ms-suffix dp3concat --delay-calibrator {delay_cat} --image-catalogue {image_cat} --apply-delay-solutions {target_ms_path}" if not os.path.isdir(outdir): - os.mkdir(outdir) + os.makedirs(outdir, exist_ok=True) print(cmd) with ( open( @@ -426,9 +426,9 @@ def run_pilot_delay_toil(field, db: FlocsDB): outdir = os.path.join(OUTPUT_DIR, field["target_name"]) logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") if not os.path.isdir(outdir): - os.mkdir(outdir) + os.makedirs(outdir, exist_ok=True) if not os.path.isdir(logsdir): - os.mkdir(logsdir) + os.makedirs(logsdir, exist_ok=True) target_path = get_most_recent_run(outdir, field["sas_id_target"], "LINC_target") target_ms_path = target_path / "results_LINC_target" / "results" db.set_status_processing(field["target_name"], "vlbi_delay", field["sas_id_target"]) @@ -508,7 +508,7 @@ def run_pilot_delay_toil(field, db: FlocsDB): print(f"Resuming failed PILOT run in {flocs_workdir}") cmd = f"flocs-run vlbi delay-calibration --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --ms-suffix dp3concat --delay-calibrator {delay_cat} --image-catalogue {image_cat} {target_ms_path}" if not os.path.isdir(outdir): - os.mkdir(outdir) + os.makedirs(outdir, exist_ok=True) print(cmd) with ( open( @@ -554,9 +554,9 @@ def run_pilot_ddcal_cwltool(field, db: FlocsDB): outdir = os.path.join(OUTPUT_DIR, field["target_name"]) logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") if not os.path.isdir(outdir): - os.mkdir(outdir) + os.makedirs(outdir, exist_ok=True) if not os.path.isdir(logsdir): - os.mkdir(logsdir) + os.makedirs(logsdir, exist_ok=True) if "status_ddf" not in field: print("Not a widefield imaging run, checking LINC + delay calibration.") target_path = get_most_recent_run(outdir, field["sas_id_target"], "LINC_target") @@ -594,7 +594,7 @@ def run_pilot_ddcal_cwltool(field, db: FlocsDB): cmd = f"flocs-run vlbi dd-calibration --runner cwltool --scheduler slurm --slurm-cores 64 --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat.sub.ms {target_ms_path}" if not os.path.isdir(outdir): - os.mkdir(outdir) + os.makedirs(outdir, exist_ok=True) print(cmd) with ( open( @@ -655,9 +655,9 @@ def run_pilot_ddcal_toil(field, db: FlocsDB): outdir = os.path.join(OUTPUT_DIR, field["target_name"]) logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") if not os.path.isdir(outdir): - os.mkdir(outdir) + os.makedirs(outdir, exist_ok=True) if not os.path.isdir(logsdir): - os.mkdir(logsdir) + os.makedirs(logsdir, exist_ok=True) if "status_ddf" not in field: print("Not a widefield imaging run, checking LINC + delay calibration.") target_path = get_most_recent_run(outdir, field["sas_id_target"], "LINC_target") @@ -731,7 +731,7 @@ def run_pilot_ddcal_toil(field, db: FlocsDB): print(f"Resuming failed PILOT run in {flocs_workdir}") cmd = f"flocs-run vlbi dd-calibration --record-toil-stats --runner toil --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --source-catalogue {source_cat} --model-cache {NN_MODEL_CACHE} --ms-suffix .dp3concat.sub.ms {target_ms_path}" if not os.path.isdir(outdir): - os.mkdir(outdir) + os.makedirs(outdir, exist_ok=True) print(cmd) with ( open( @@ -775,9 +775,9 @@ def run_pilot_intermediate_image_toil(field, db: FlocsDB): outdir = os.path.join(OUTPUT_DIR, field["target_name"]) logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") if not os.path.isdir(outdir): - os.mkdir(outdir) + os.makedirs(outdir, exist_ok=True) if not os.path.isdir(logsdir): - os.mkdir(logsdir) + os.makedirs(logsdir, exist_ok=True) target_path = get_most_recent_run( outdir, field["sas_id_target"], "VLBI_process-ddf" ) @@ -835,7 +835,7 @@ def run_pilot_intermediate_image_toil(field, db: FlocsDB): print(f"Resuming failed PILOT run in {flocs_workdir}") cmd = f"flocs-run vlbi image-intermediate-resolution --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --dd-solutions {dd_sols} --ms-suffix .dp3concat.sub.ms {target_ms_path}" if not os.path.isdir(outdir): - os.mkdir(outdir) + os.makedirs(outdir, exist_ok=True) print(cmd) with ( open( @@ -883,9 +883,9 @@ def run_pilot_facet_subtract_toil(field, db: FlocsDB): outdir = os.path.join(OUTPUT_DIR, field["target_name"]) logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") if not os.path.isdir(outdir): - os.mkdir(outdir) + os.makedirs(outdir, exist_ok=True) if not os.path.isdir(logsdir): - os.mkdir(logsdir) + os.makedirs(logsdir, exist_ok=True) target_path = get_most_recent_run( outdir, field["sas_id_target"], "VLBI_process-ddf" ) @@ -959,7 +959,7 @@ def run_pilot_facet_subtract_toil(field, db: FlocsDB): print(f"Resuming failed PILOT run in {flocs_workdir}") cmd = f"flocs-run vlbi facet-subtract --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --dd-solutions {dd_sols} --model-image-directory {model_images_path} --ms-suffix .dp3concat.sub.ms {target_ms_path}" if not os.path.isdir(outdir): - os.mkdir(outdir) + os.makedirs(outdir, exist_ok=True) print(cmd) with ( open( @@ -1007,9 +1007,9 @@ def run_pilot_facet_imaging_toil(field, db: FlocsDB): outdir = os.path.join(OUTPUT_DIR, field["target_name"]) logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") if not os.path.isdir(outdir): - os.mkdir(outdir) + os.makedirs(outdir, exist_ok=True) if not os.path.isdir(logsdir): - os.mkdir(logsdir) + os.makedirs(logsdir, exist_ok=True) target_path = get_most_recent_run( outdir, field["sas_id_target"], "VLBI_facet_subtract" ) @@ -1065,7 +1065,7 @@ def run_pilot_facet_imaging_toil(field, db: FlocsDB): print(f"Resuming failed PILOT run in {flocs_workdir}") cmd = f"flocs-run vlbi facet-imaging --record-toil-stats --runner toil --scheduler slurm --slurm-time 72:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --resolution 0.3asec --pixel-scale 0.1 --ms-suffix .ms {target_ms_path}" if not os.path.isdir(outdir): - os.mkdir(outdir) + os.makedirs(outdir, exist_ok=True) print(cmd) with ( open( @@ -1113,9 +1113,9 @@ def run_prepare_ddf_subtract(field): outdir = os.path.join(OUTPUT_DIR, field["target_name"]) logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") if not os.path.isdir(outdir): - os.mkdir(outdir) + os.makedirs(outdir, exist_ok=True) if not os.path.isdir(logsdir): - os.mkdir(logsdir) + os.makedirs(logsdir, exist_ok=True) target_path = get_most_recent_run(outdir, field["sas_id_target"], "VLBI_delay") target_ms_path = target_path / "results_VLBI_delay-calibration" mses_unaveraged = list(target_ms_path.glob("*.dp3concat")) @@ -1207,9 +1207,9 @@ def run_prepare_ddf(field): outdir = os.path.join(OUTPUT_DIR, field["target_name"]) logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") if not os.path.isdir(outdir): - os.mkdir(outdir) + os.makedirs(outdir, exist_ok=True) if not os.path.isdir(logsdir): - os.mkdir(logsdir) + os.makedirs(logsdir, exist_ok=True) target_path = get_most_recent_run(outdir, field["sas_id_target"], "VLBI_delay") target_ms_path = target_path / "results_VLBI_delay-calibration" mses_unaveraged = list(target_ms_path.glob("*.dp3concat")) @@ -1301,9 +1301,9 @@ def run_pilot_process_ddf_toil(field, db: FlocsDB): outdir = os.path.join(OUTPUT_DIR, field["target_name"]) logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") if not os.path.isdir(outdir): - os.mkdir(outdir) + os.makedirs(outdir, exist_ok=True) if not os.path.isdir(logsdir): - os.mkdir(logsdir) + os.makedirs(logsdir, exist_ok=True) target_path = get_most_recent_run( outdir, field["sas_id_target"], "VLBI_delay-calibration" ) @@ -1359,7 +1359,7 @@ def run_pilot_process_ddf_toil(field, db: FlocsDB): print(f"Resuming failed PILOT run in {flocs_workdir}") cmd = f"flocs-run vlbi process-ddf --runner toil --record-toil-stats --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --ms-suffix .dp3concat --ddf-rundir {ddf_path} --solsdir {ddf_sols_path} --do-subtraction {target_ms_path}" if not os.path.isdir(outdir): - os.mkdir(outdir) + os.makedirs(outdir, exist_ok=True) print(cmd) with ( open( @@ -1397,9 +1397,9 @@ def launch_ddf_pipeline(field, db: FlocsDB): outdir = os.path.join(OUTPUT_DIR, field["target_name"]) logsdir = os.path.join(OUTPUT_DIR, field["target_name"], "logs") if not os.path.isdir(outdir): - os.mkdir(outdir) + os.makedirs(outdir, exist_ok=True) if not os.path.isdir(logsdir): - os.mkdir(logsdir) + os.makedirs(logsdir, exist_ok=True) target_path = get_most_recent_run( outdir, field["sas_id_target"], "VLBI_delay-calibration" ) From d74264e14cbe641fdf089c724960742ae9089a84 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 14 Aug 2026 16:32:22 +0100 Subject: [PATCH 112/120] Bunch of fixes --- flocs_processing/dags/pilot_widefield.py | 8 ++++++-- flocs_processing/pipeline_runners.py | 19 +++++++++++++++---- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 2e55c5c..20165a6 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -65,7 +65,9 @@ NN_MODEL_CACHE = parser["DEFAULT"]["NN_MODEL_CACHE"] DDF_CONFIG = parser["DEFAULT"]["DDF_CONFIG"] FLUX_CALIBRATOR_TEMPLATE = parser["DEFAULT"]["FLUX_CALIBRATOR_TEMPLATE"] -NEEDS_MANUAL_APPROVAL_DELAY = parser.getboolean("DEFAULT", "NEEDS_MANUAL_APPROVAL_DELAY") +NEEDS_MANUAL_APPROVAL_DELAY = parser.getboolean( + "DEFAULT", "NEEDS_MANUAL_APPROVAL_DELAY" +) CWL_RUNNER_LINC_CALIBRATOR = "cwltool" CWL_RUNNER_LINC_TARGET = "toil" @@ -482,7 +484,9 @@ def run_ddf_pipeline(field): time.sleep(60) elif status == "COMPLETED": CURRENT_DB.set_status_finished( - field["target_name"], "ddf_subtract", field["sas_id_target"] + field["target_name"], + "vlbi_ddf_subtract", + field["sas_id_target"], ) return field elif (status == "FAILED") or ("TIMEOUT" in status): diff --git a/flocs_processing/pipeline_runners.py b/flocs_processing/pipeline_runners.py index 2df6b94..c21df8a 100644 --- a/flocs_processing/pipeline_runners.py +++ b/flocs_processing/pipeline_runners.py @@ -39,7 +39,9 @@ NN_MODEL_CACHE = parser["DEFAULT"]["NN_MODEL_CACHE"] DDF_CONFIG = parser["DEFAULT"]["DDF_CONFIG"] FLUX_CALIBRATOR_TEMPLATE = parser["DEFAULT"]["FLUX_CALIBRATOR_TEMPLATE"] -NEEDS_MANUAL_APPROVAL_DELAY = parser.getboolean("DEFAULT", "NEEDS_MANUAL_APPROVAL_DELAY") +NEEDS_MANUAL_APPROVAL_DELAY = parser.getboolean( + "DEFAULT", "NEEDS_MANUAL_APPROVAL_DELAY" +) def get_most_recent_run(searchpath: str, sas_id: str, pipeline: str) -> pathlib.Path: @@ -53,8 +55,12 @@ def get_most_recent_run(searchpath: str, sas_id: str, pipeline: str) -> pathlib. ] else: rundirs_sorted_filtered = [d for d in rundirs_sorted if sas_id in d.parts[-1]] - rundir_final = rundirs_sorted_filtered[-1].absolute() - return rundir_final + try: + rundir_final = rundirs_sorted_filtered[-1].absolute() + return rundir_final + except IndexError: + print("No {pipeline} run for {sas_id} found") + raise RuntimeError("No {pipeline} run for {sas_id} found") def run_linc_calibrator_cwltool(field, calibrator_field: int, db: FlocsDB): @@ -1281,7 +1287,9 @@ def run_prepare_ddf(field): ).stdout.strip() if status == "COMPLETED": print(f"Job {jobid} completed ({out_ms.name})") - elif status == "FAILED": + elif ( + (status == "FAILED") or ("TIMEOUT" in status) or ("CANCELLED" in status) + ): raise RuntimeError( f"DP3 averaging job {jobid} failed for {out_ms.name}" ) @@ -1389,6 +1397,9 @@ def run_pilot_process_ddf_toil(field, db: FlocsDB): field["target_name"], "ddf_subtract", field["sas_id_target"] ) else: + db.set_status_failed( + field["target_name"], "ddf_subtract", field["sas_id_target"] + ) raise RuntimeError From 7c2cba3e8b6e2e23d312c7bcb53268c31fa8d5bb Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 14 Aug 2026 16:37:39 +0100 Subject: [PATCH 113/120] More fixes --- flocs_processing/pipeline_runners.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flocs_processing/pipeline_runners.py b/flocs_processing/pipeline_runners.py index c21df8a..11ec97b 100644 --- a/flocs_processing/pipeline_runners.py +++ b/flocs_processing/pipeline_runners.py @@ -512,7 +512,7 @@ def run_pilot_delay_toil(field, db: FlocsDB): "Could not retrieve PILOT workdir. Flocs probably crashed before launching." ) print(f"Resuming failed PILOT run in {flocs_workdir}") - cmd = f"flocs-run vlbi delay-calibration --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --ms-suffix dp3concat --delay-calibrator {delay_cat} --image-catalogue {image_cat} {target_ms_path}" + cmd = f"flocs-run vlbi delay-calibration --record-toil-stats --runner toil --scheduler slurm --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {flocs_workdir} --restart --outdir {outdir} --ms-suffix dp3concat --delay-calibrator {delay_cat} --image-catalogue {image_cat} {target_ms_path}" if not os.path.isdir(outdir): os.makedirs(outdir, exist_ok=True) print(cmd) @@ -794,7 +794,7 @@ def run_pilot_intermediate_image_toil(field, db: FlocsDB): outdir, field["sas_id_target"], "VLBI_dd-calibration" ) dd_sols = dd_sols_path / "results_VLBI_dd-calibration" / "merged.h5" - print(f"Using dd solutions at: {target_path}") + print(f"Using dd solutions: {dd_sols}") if not os.path.isfile(dd_sols): raise AirflowFailException(f"{dd_sols} not found.") @@ -910,13 +910,13 @@ def run_pilot_facet_subtract_toil(field, db: FlocsDB): model_images_path = get_most_recent_run( outdir, field["sas_id_target"], "VLBI_intermediate_resolution_imaging" ) - model_images = ( + model_images_path = ( model_images_path / "results_VLBI_intermediate_resolution_imaging" / "*-????-model-fpb.fits" ) model_images = list(model_images_path.glob("*-????-model-fpb.fits")) - print(f"Using model image at: {model_images_path}/*-????-model-fpb.fits") + print(f"Using model images at: {model_images_path}") if not model_images: raise AirflowFailException( @@ -1205,7 +1205,7 @@ def run_prepare_ddf_subtract(field): time.sleep(30) mses_delay_corrected = list(target_ms_path.glob("*.dp3concat")) - return mses_delay_corrected + return field def run_prepare_ddf(field): @@ -1301,7 +1301,7 @@ def run_prepare_ddf(field): time.sleep(30) mses_averaged = list(target_ms_path.glob("*_pre-cal.ms")) - return mses_averaged + return field def run_pilot_process_ddf_toil(field, db: FlocsDB): From 7ded59e1096a14985efb46805d6af0981251feb3 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 14 Aug 2026 16:46:08 +0100 Subject: [PATCH 114/120] More fixes --- flocs_processing/pipeline_runners.py | 29 +++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/flocs_processing/pipeline_runners.py b/flocs_processing/pipeline_runners.py index 11ec97b..25a1fca 100644 --- a/flocs_processing/pipeline_runners.py +++ b/flocs_processing/pipeline_runners.py @@ -59,7 +59,7 @@ def get_most_recent_run(searchpath: str, sas_id: str, pipeline: str) -> pathlib. rundir_final = rundirs_sorted_filtered[-1].absolute() return rundir_final except IndexError: - print("No {pipeline} run for {sas_id} found") + print(f"No {pipeline} run for {sas_id} found") raise RuntimeError("No {pipeline} run for {sas_id} found") @@ -500,7 +500,10 @@ def run_pilot_delay_toil(field, db: FlocsDB): f"Scanning log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt for workdir." ) with open( - f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt" + os.path.join( + logsdir, + f"log_VLBI_delay-calibration_{field['target_name']}_{field['sas_id_target']}.txt", + ) ) as f_out: for line in f_out.readlines(): print(line) @@ -902,7 +905,7 @@ def run_pilot_facet_subtract_toil(field, db: FlocsDB): outdir, field["sas_id_target"], "VLBI_dd-calibration" ) dd_sols = dd_sols_path / "results_VLBI_dd-calibration" / "merged.h5" - print(f"Using dd solutions at: {target_path}") + print(f"Using dd solutions at: {dd_sols}") if not os.path.isfile(dd_sols): raise AirflowFailException(f"{dd_sols} not found.") @@ -910,12 +913,12 @@ def run_pilot_facet_subtract_toil(field, db: FlocsDB): model_images_path = get_most_recent_run( outdir, field["sas_id_target"], "VLBI_intermediate_resolution_imaging" ) - model_images_path = ( + model_images = ( model_images_path / "results_VLBI_intermediate_resolution_imaging" / "*-????-model-fpb.fits" ) - model_images = list(model_images_path.glob("*-????-model-fpb.fits")) + model_images = list(model_images.glob("*-????-model-fpb.fits")) print(f"Using model images at: {model_images_path}") if not model_images: @@ -1148,7 +1151,7 @@ def run_prepare_ddf_subtract(field): ) if mses_unaveraged_pilot and (len(mses_unaveraged_pilot) == len(mses_unaveraged)): print("Appropriate input exists for ddf-pipeline.") - return field + return mses_unaveraged_pilot jobids = [] delay_corrected_mses = [] @@ -1205,7 +1208,7 @@ def run_prepare_ddf_subtract(field): time.sleep(30) mses_delay_corrected = list(target_ms_path.glob("*.dp3concat")) - return field + return mses_delay_corrected def run_prepare_ddf(field): @@ -1244,7 +1247,7 @@ def run_prepare_ddf(field): ) if mses_unaveraged and (len(mses_averaged) == len(mses_unaveraged)): print("Appropriate input exists for ddf-pipeline.") - return field + return mses_averaged jobids = [] averaged_mses = [] @@ -1301,7 +1304,7 @@ def run_prepare_ddf(field): time.sleep(30) mses_averaged = list(target_ms_path.glob("*_pre-cal.ms")) - return field + return mses_averaged def run_pilot_process_ddf_toil(field, db: FlocsDB): @@ -1332,7 +1335,7 @@ def run_pilot_process_ddf_toil(field, db: FlocsDB): ) ): db.set_status_processing( - field["target_name"], "ddf_subtract", field["sas_id_target"] + field["target_name"], "vlbi_ddf_subtract", field["sas_id_target"] ) cmd = f"flocs-run vlbi process-ddf --runner toil --record-toil-stats --scheduler slurm --slurm-time 24:00:00 --slurm-account {SLURM_ACCOUNT} --slurm-queue {SLURM_QUEUE} --rundir {PROCESSING_DIR} --outdir {outdir} --ms-suffix .dp3concat --ddf-rundir {ddf_path} --solsdir {ddf_sols_path} --do-subtraction {target_ms_path}" else: @@ -1394,11 +1397,11 @@ def run_pilot_process_ddf_toil(field, db: FlocsDB): success = True if success: db.set_status_finished( - field["target_name"], "ddf_subtract", field["sas_id_target"] + field["target_name"], "vlbi_ddf_subtract", field["sas_id_target"] ) else: db.set_status_failed( - field["target_name"], "ddf_subtract", field["sas_id_target"] + field["target_name"], "vlbi_ddf_subtract", field["sas_id_target"] ) raise RuntimeError @@ -1462,7 +1465,7 @@ def launch_ddf_pipeline(field, db: FlocsDB): elif status == "COMPLETED": db.set_status_finished( field["target_name"], - "ddf_subtract", + "ddf", field["sas_id_target"], ) break From c1fa0566f044eb8a504680450543287d1b302eb0 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 14 Aug 2026 16:50:21 +0100 Subject: [PATCH 115/120] Fix model image glob --- flocs_processing/pipeline_runners.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/flocs_processing/pipeline_runners.py b/flocs_processing/pipeline_runners.py index 25a1fca..5f19d8e 100644 --- a/flocs_processing/pipeline_runners.py +++ b/flocs_processing/pipeline_runners.py @@ -913,12 +913,10 @@ def run_pilot_facet_subtract_toil(field, db: FlocsDB): model_images_path = get_most_recent_run( outdir, field["sas_id_target"], "VLBI_intermediate_resolution_imaging" ) - model_images = ( - model_images_path - / "results_VLBI_intermediate_resolution_imaging" - / "*-????-model-fpb.fits" + model_images_path = ( + model_images_path / "results_VLBI_intermediate_resolution_imaging" ) - model_images = list(model_images.glob("*-????-model-fpb.fits")) + model_images = list(model_images_path.glob("*-????-model-fpb.fits")) print(f"Using model images at: {model_images_path}") if not model_images: From 30d537049f09312fbb26f447962fac86e2da91e9 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 14 Aug 2026 16:51:38 +0100 Subject: [PATCH 116/120] More fixes --- flocs_processing/pipeline_runners.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flocs_processing/pipeline_runners.py b/flocs_processing/pipeline_runners.py index 5f19d8e..27e3944 100644 --- a/flocs_processing/pipeline_runners.py +++ b/flocs_processing/pipeline_runners.py @@ -60,7 +60,7 @@ def get_most_recent_run(searchpath: str, sas_id: str, pipeline: str) -> pathlib. return rundir_final except IndexError: print(f"No {pipeline} run for {sas_id} found") - raise RuntimeError("No {pipeline} run for {sas_id} found") + raise RuntimeError(f"No {pipeline} run for {sas_id} found") def run_linc_calibrator_cwltool(field, calibrator_field: int, db: FlocsDB): From bd2f269f1a79d8df9970b85515a08047f7bb281d Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 14 Aug 2026 17:00:00 +0100 Subject: [PATCH 117/120] Really fix loop this time --- flocs_processing/dags/pilot_widefield.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index 20165a6..f88c007 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -471,8 +471,9 @@ def run_ddf_pipeline(field): for line in f_out.readlines(): if "Submitted batch job" in line: jobid = line.strip().split()[-1] - else: - raise AirflowFailException("Failed to recover job id from log.") + break + else: + raise AirflowFailException("Failed to recover job id from log.") while True: print(f"Polling DDF-pipeine job {jobid}") @@ -485,7 +486,7 @@ def run_ddf_pipeline(field): elif status == "COMPLETED": CURRENT_DB.set_status_finished( field["target_name"], - "vlbi_ddf_subtract", + "ddf", field["sas_id_target"], ) return field From 90af5bac47f4e8198c65fc8afeddea2b8271edd1 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 14 Aug 2026 17:07:10 +0100 Subject: [PATCH 118/120] Add oom slurm failure --- flocs_processing/dags/pilot_widefield.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/flocs_processing/dags/pilot_widefield.py b/flocs_processing/dags/pilot_widefield.py index f88c007..75a4961 100644 --- a/flocs_processing/dags/pilot_widefield.py +++ b/flocs_processing/dags/pilot_widefield.py @@ -490,7 +490,12 @@ def run_ddf_pipeline(field): field["sas_id_target"], ) return field - elif (status == "FAILED") or ("TIMEOUT" in status): + elif ( + (status == "FAILED") + or ("TIMEOUT" in status) + or ("CANCELELD" in status) + or ("OUT_OF_MEM" in status) + ): raise RuntimeError( f"DDF-pipeline for {field['target_name']} {field['sas_id_target']} failed." ) From aa179aefa5b6777b56cf78fcad44539aa7625678 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 14 Aug 2026 17:28:13 +0100 Subject: [PATCH 119/120] Add missing status columns --- flocs_processing/db_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flocs_processing/db_utils.py b/flocs_processing/db_utils.py index 9d43e70..af85aa7 100644 --- a/flocs_processing/db_utils.py +++ b/flocs_processing/db_utils.py @@ -35,7 +35,7 @@ def get_db_columns(self, obsid: str = None): with sqlite3.connect(self.DATABASE) as db: db.row_factory = sqlite3.Row cursor = db.cursor() - columns = "target_name,priority,finished,downloaded,sas_id_calibrator1,sas_id_calibrator2,sas_id_calibrator_final,sas_id_target,status_calibrator1,status_calibrator2,status_target,status_vlbi_delay,status_vlbi_dd,status_ddf,status_vlbi_ddf_subtract" + columns = "target_name,priority,finished,downloaded,sas_id_calibrator1,sas_id_calibrator2,sas_id_calibrator_final,sas_id_target,status_calibrator1,status_calibrator2,status_target,status_vlbi_delay,status_vlbi_dd,status_ddf,status_vlbi_ddf_subtract,status_vlbi_intermediate_img,status_vlbi_facet_subtract,status_vlbi_facet_img" if obsid: field = cursor.execute( f"select {columns} from {self.TABLE_NAME} where sas_id_target=='{obsid}' and finished==0 order by priority desc" From f5c7649d3b0390bedaa07837733c5361a73b497f Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Sat, 15 Aug 2026 10:24:20 +0100 Subject: [PATCH 120/120] Output err log for intermediate img --- flocs_processing/pipeline_runners.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/flocs_processing/pipeline_runners.py b/flocs_processing/pipeline_runners.py index 27e3944..de1e2ce 100644 --- a/flocs_processing/pipeline_runners.py +++ b/flocs_processing/pipeline_runners.py @@ -826,12 +826,20 @@ def run_pilot_intermediate_image_toil(field, db: FlocsDB): print( f"Scanning log_VLBI_image_intermediate_resolution_{field['target_name']}_{field['sas_id_target']}.txt for workdir." ) - with open( - os.path.join( - logsdir, - f"log_VLBI_image_intermediate_resolution_{field['target_name']}_{field['sas_id_target']}.txt", - ) - ) as f_out: + with ( + open( + os.path.join( + logsdir, + f"log_VLBI_image_intermediate_resolution_{field['target_name']}_{field['sas_id_target']}.txt", + ) + ) as f_out, + open( + os.path.join( + logsdir, + f"log_VLBI_image_intermediate_resolution_{field['target_name']}_{field['sas_id_target']}_err.txt", + ) + ) as f_err, + ): for line in f_out.readlines(): print(line) if "Running workflow with" in line: