diff --git a/.github/workflows/temporary-format.yml b/.github/workflows/temporary-format.yml new file mode 100644 index 00000000..808e5ddc --- /dev/null +++ b/.github/workflows/temporary-format.yml @@ -0,0 +1,44 @@ +name: One-time Python formatting + +on: + push: + branches: + - chore/black-formatting + +permissions: + contents: write + +jobs: + format: + if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + submodules: false + fetch-depth: 0 + + - uses: actions/setup-python@v6 + with: + python-version: '3.12' + + - name: Install formatters + run: python -m pip install 'black>=23' 'isort>=5' + + - name: Format ROBIN-owned Python + run: | + # Deliberately leave vendored/upstream code under src/robin/submodules untouched. + isort src/robin tests --skip-glob 'src/robin/submodules/*' + black src/robin tests --exclude 'src/robin/submodules/' + + - name: Commit formatting + run: | + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git add src/robin tests + if git diff --cached --quiet; then + echo 'No formatting changes required.' + exit 0 + fi + git commit -m 'style: format ROBIN Python with Black and isort' + git push diff --git a/pyproject.toml b/pyproject.toml index 324444f5..c14cd729 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -142,7 +142,7 @@ allow-direct-references = true [tool.black] line-length = 88 -target-version = ['py38'] +target-version = ['py312'] include = '\.pyi?$' extend-exclude = ''' /( @@ -155,6 +155,7 @@ extend-exclude = ''' | \.venv | build | dist + | src/robin/submodules )/ ''' @@ -163,6 +164,7 @@ profile = "black" multi_line_output = 3 line_length = 88 known_first_party = ["robin"] +skip_glob = ["src/robin/submodules/*"] [tool.mypy] python_version = "3.8" diff --git a/src/robin/__init__.py b/src/robin/__init__.py index 6db3d9f9..a52d711b 100644 --- a/src/robin/__init__.py +++ b/src/robin/__init__.py @@ -2,6 +2,7 @@ # Suppress pkg_resources deprecation warnings from sorted_nearest import warnings + warnings.filterwarnings( "ignore", message="pkg_resources is deprecated", category=UserWarning ) diff --git a/src/robin/analysis/__init__.py b/src/robin/analysis/__init__.py index a570d7f2..083b8d4a 100644 --- a/src/robin/analysis/__init__.py +++ b/src/robin/analysis/__init__.py @@ -26,8 +26,8 @@ pass try: - from .mgmt_analysis import mgmt_handler # noqa: F401 from .mgmt_analysis import extract_mgmt_site_rows_from_bed # noqa: F401 + from .mgmt_analysis import mgmt_handler # noqa: F401 __all__.extend(["mgmt_handler", "extract_mgmt_site_rows_from_bed"]) except Exception: @@ -99,8 +99,8 @@ try: from .methylation_wrapper import ( # noqa: F401 figure_is_renderable, - locus_figure, load_figure_pickle, + locus_figure, save_figure_pickle, try_load_figure_pickle, ) diff --git a/src/robin/analysis/bam_preprocessor.py b/src/robin/analysis/bam_preprocessor.py index 8894890e..d4f60637 100644 --- a/src/robin/analysis/bam_preprocessor.py +++ b/src/robin/analysis/bam_preprocessor.py @@ -2,26 +2,27 @@ Requires Python 3.12+ (slots=True, str.removeprefix, PEP 709 comprehensions). """ + from __future__ import annotations +import hashlib import os +import re import sys import time -import re -import hashlib from dataclasses import dataclass, field -from typing import Dict, Any, Optional, List, Tuple from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple if sys.version_info < (3, 12): raise RuntimeError("robin bam_preprocessor requires Python 3.12 or newer") import pysam from dateutil import parser + from robin.analysis.master_csv_manager import MasterCSVManager from robin.logging_config import get_job_logger - # ============================================================================ # CONSTANTS AND CONFIGURATION # ============================================================================ @@ -53,7 +54,9 @@ # When True, BAMs with >50k reads are processed (downstream batching uses batch size 1 per file). # Set ROBIN_PROCESS_LARGE_BAMS=1 to enable. -_PROCESS_LARGE_BAMS_INDIVIDUALLY = os.getenv("ROBIN_PROCESS_LARGE_BAMS", "0").strip().lower() in ("1", "true", "yes", "on") +_PROCESS_LARGE_BAMS_INDIVIDUALLY = os.getenv( + "ROBIN_PROCESS_LARGE_BAMS", "0" +).strip().lower() in ("1", "true", "yes", "on") # Constants for MGMT locus (chr10:129,466,536-129,467,536) _MGMT_CHR = "chr10" @@ -99,7 +102,9 @@ def _persist_supplementary_read_ids( sample_id = metadata.extracted_data.get("sample_id", "unknown") supp_dir = os.path.join(work_dir, sample_id, "_supplementary_read_ids") os.makedirs(supp_dir, exist_ok=True) - path_hash = hashlib.sha256(os.path.abspath(bam_path).encode("utf-8")).hexdigest()[:16] + path_hash = hashlib.sha256(os.path.abspath(bam_path).encode("utf-8")).hexdigest()[ + :16 + ] supp_path = os.path.join( supp_dir, f"{os.path.basename(bam_path)}.{path_hash}.txt", @@ -411,19 +416,19 @@ def process_bam_reads(bam_file: str) -> Optional[Dict[str, Any]]: # A read has supplementary alignments if: # 1. This alignment itself is supplementary (is_supplementary=True), OR # 2. ANY alignment (primary or secondary) has an SA tag (indicating supplementary alignments exist) - # + # # We check ALL reads (not just primaries) to ensure we catch every read with supplementary mappings, # even if the SA tag is only present on certain alignment records has_supplementary_alignments = False if is_supplementary: supplementary_reads += 1 has_supplementary_alignments = True - + # Also check SA tag on ALL reads (primary, secondary, supplementary) to catch any we might miss # Some BAM files may have SA tag on different records than expected if read.has_tag("SA"): has_supplementary_alignments = True - + if has_supplementary_alignments: reads_with_supplementary.add(query_name) @@ -718,7 +723,7 @@ def _send_alignment_warning_notification( ) -> None: """ Send an alignment warning notification to the GUI. - + Args: warning_msg: The warning message to display sample_id: Sample ID for context @@ -727,7 +732,7 @@ def _send_alignment_warning_notification( try: from robin.gui.app import send_gui_update from robin.gui_launcher import UpdateType - + send_gui_update( UpdateType.WARNING_NOTIFICATION, { @@ -848,7 +853,7 @@ def bam_preprocessing_handler(job, center: str = None): metadata.extracted_data["modbase_warning_level"] = warning_level job.context.add_metadata("modbase_warning", modbase_warning) job.context.add_metadata("modbase_warning_level", warning_level) - + if total_reads > 0: # Check if BAM file has no mapped reads (no alignment data) if mapped_reads == 0: @@ -885,8 +890,7 @@ def bam_preprocessing_handler(job, center: str = None): elif total_reads == 0: # Edge case: BAM file has no reads at all warning_msg = ( - f"BAM file contains no reads. " - f"This file may be empty or corrupted." + f"BAM file contains no reads. " f"This file may be empty or corrupted." ) logger.warning(f"WARNING: {warning_msg}") metadata.extracted_data["alignment_warning"] = warning_msg @@ -901,19 +905,23 @@ def bam_preprocessing_handler(job, center: str = None): job.context.add_metadata("file_size", metadata.file_size) job.context.add_metadata("creation_time", metadata.creation_time) job.context.add_metadata("processing_steps", metadata.processing_steps) - + # Store center information if center: job.context.add_metadata("center", center) logger.info(f"Center set to: {center}") - + # Preserve target_panel metadata from job context (set by file classifier) existing_target_panel = job.context.metadata.get("target_panel") if existing_target_panel: - logger.info(f"Preserving target panel from job context: {existing_target_panel}") + logger.info( + f"Preserving target panel from job context: {existing_target_panel}" + ) else: - logger.warning("No target_panel found in job context - this may cause panel assignment issues") - + logger.warning( + "No target_panel found in job context - this may cause panel assignment issues" + ) + # Preserve reference metadata from job context (set by file classifier) existing_reference = job.context.metadata.get("reference") if existing_reference: @@ -928,7 +936,7 @@ def bam_preprocessing_handler(job, center: str = None): ) except Exception: total_reads = 0 - + if total_reads > 50000: if _PROCESS_LARGE_BAMS_INDIVIDUALLY: # Process this BAM; downstream batching will pass it to workers as a single-file batch @@ -962,9 +970,7 @@ def bam_preprocessing_handler(job, center: str = None): # Persist the complete ID set per BAM to avoid retaining large lists in memory. try: - work_dir = job.context.metadata.get( - "work_dir", os.path.dirname(bam_path) - ) + work_dir = job.context.metadata.get("work_dir", os.path.dirname(bam_path)) _persist_supplementary_read_ids(metadata, bam_path, work_dir) except Exception: # Keep the complete in-memory list as a safe fallback. @@ -1003,27 +1009,33 @@ def bam_preprocessing_handler(job, center: str = None): "has_mgmt_reads", "mgmt_read_count", ) - bam_stats = {key: metadata.extracted_data.get(key, 0) for key in _bam_stat_keys} + bam_stats = { + key: metadata.extracted_data.get(key, 0) for key in _bam_stat_keys + } # Update master.csv csv_manager.update_master_csv( sample_id, bam_stats, metadata.extracted_data ) - + # Update analysis panel in master.csv (preserve from job context) existing_target_panel = job.context.metadata.get("target_panel") if existing_target_panel: csv_manager.update_analysis_panel(sample_id, existing_target_panel) - logger.info(f"Updated master.csv with analysis panel '{existing_target_panel}' for sample {sample_id}") + logger.info( + f"Updated master.csv with analysis panel '{existing_target_panel}' for sample {sample_id}" + ) else: - logger.warning(f"No target_panel found in job context for sample {sample_id} - analysis_panel not set in master.csv") + logger.warning( + f"No target_panel found in job context for sample {sample_id} - analysis_panel not set in master.csv" + ) except Exception as e: logger.warning(f"Could not update master.csv for {sample_id}: {e}") # Step 5: Add result to context sample_id = metadata.extracted_data.get("sample_id", "unknown") - + job.context.add_result( "preprocessing", { diff --git a/src/robin/analysis/bed_conversion.py b/src/robin/analysis/bed_conversion.py index b837a7c3..70eaeb40 100644 --- a/src/robin/analysis/bed_conversion.py +++ b/src/robin/analysis/bed_conversion.py @@ -5,24 +5,26 @@ Requires Python 3.12+. Automated conversion of BAM files to parquet using matkit; integrates with robin's preprocessing pipeline and CPG master file. """ + from __future__ import annotations import sys + if sys.version_info < (3, 12): raise RuntimeError("robin bed_conversion requires Python 3.12 or newer") +import logging import os -import time import tempfile -import logging +import time from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import dataclass, field -from typing import Dict, Any, List, Optional +from typing import Any, Dict, List, Optional # Import robin utilities and resources try: - from robin.analysis.utilities.matkit import run_matkit from robin import resources + from robin.analysis.utilities.matkit import run_matkit except ImportError: run_matkit = None resources = None @@ -288,6 +290,7 @@ def process_single_bam(bam: str) -> str: # Fallback if robin is not available (write empty parquet not used in real runs) import pyarrow as pa import pyarrow.parquet as pq + empty = pa.table( { "chrom": pa.array([], type=pa.binary()), @@ -311,7 +314,9 @@ def process_single_bam(bam: str) -> str: try: os.remove(temp_file.name) except Exception as e: - logger.error(f"Failed to delete temporary file {temp_file.name}: {e}") + logger.error( + f"Failed to delete temporary file {temp_file.name}: {e}" + ) pass raise @@ -321,13 +326,17 @@ def process_single_bam(bam: str) -> str: # Parallel path: tolerate per-file failures so one bad BAM does not fail the batch try: with ThreadPoolExecutor(max_workers=threads) as executor: - futures = {executor.submit(process_single_bam, bam): bam for bam in bams} + futures = { + executor.submit(process_single_bam, bam): bam for bam in bams + } for future in as_completed(futures): try: processed_files.append(future.result()) except Exception as e: bam_path = futures[future] - logger.warning(f"Failed to process BAM {os.path.basename(bam_path)}: {e}") + logger.warning( + f"Failed to process BAM {os.path.basename(bam_path)}: {e}" + ) return processed_files except Exception: cleanup_temp_files(processed_files) @@ -409,6 +418,7 @@ def _update_state( return state + def process_multiple_files( bam_paths, metadata_list, @@ -422,7 +432,7 @@ def process_multiple_files( ): """ Process multiple BAM files for bed conversion analysis. - + This function processes multiple BAM files for the same sample. Each file is processed individually and results are accumulated in the same parquet file. @@ -439,17 +449,17 @@ def process_multiple_files( """ if not bam_paths or not metadata_list: raise ValueError("bam_paths and metadata_list must not be empty") - + if len(bam_paths) != len(metadata_list): raise ValueError("bam_paths and metadata_list must have the same length") - + # Get sample ID from first metadata (assuming all BAM files are from same sample) sample_id = metadata_list[0].get("sample_id", "unknown") - + logger.info(f"🔄 Starting multi-file bed conversion for sample: {sample_id}") logger.info(f"Processing {len(bam_paths)} BAM files for sample {sample_id}") logger.info(f"Using {threads} threads for processing") - + analysis_result = { "sample_id": sample_id, "bam_paths": bam_paths, @@ -465,10 +475,10 @@ def process_multiple_files( # Create sample-specific output directory sample_dir = os.path.join(work_dir, sample_id) os.makedirs(sample_dir, exist_ok=True) - + parquet_path = os.path.join(sample_dir, f"{sample_id}.parquet") analysis_result["parquet_path"] = parquet_path - + logger.info(f"Created output directory: {sample_dir}") logger.info(f"Parquet file: {parquet_path}") analysis_result["processing_steps"].append("directory_created") @@ -480,7 +490,7 @@ def process_multiple_files( reference_fasta=reference_fasta, cpg_mode=cpg_mode, ) - + logger.info("Initialized bed conversion analyzer") analysis_result["processing_steps"].append("analyzer_initialized") @@ -496,12 +506,16 @@ def process_multiple_files( analysis_result["processing_steps"].append("no_files_processed") return analysis_result - logger.info(f"Processing {len(valid_bam_paths)} BAM files (up to {bed_analyzer.threads} in parallel)") + logger.info( + f"Processing {len(valid_bam_paths)} BAM files (up to {bed_analyzer.threads} in parallel)" + ) all_processed_data = bed_analyzer._process_bams(valid_bam_paths, sample_dir) processed_files = len(all_processed_data) if processed_files == 0: - analysis_result["error_message"] = "No files could be processed successfully" + analysis_result["error_message"] = ( + "No files could be processed successfully" + ) analysis_result["processing_steps"].append("no_files_processed") return analysis_result @@ -510,19 +524,21 @@ def process_multiple_files( # Create parquet file from all processed data if all_processed_data: - logger.info(f"Creating parquet file from {len(all_processed_data)} processed files") + logger.info( + f"Creating parquet file from {len(all_processed_data)} processed files" + ) try: # Use merge_modkit_files to create the final parquet file bed_analyzer._update_state( - parquet_path, - all_processed_data, - sample_id, - processed_files # Use number of processed files as file_number + parquet_path, + all_processed_data, + sample_id, + processed_files, # Use number of processed files as file_number ) - + analysis_result["processing_steps"].append("parquet_created") logger.info(f"Parquet file created successfully: {parquet_path}") - + # Verify parquet file was created if os.path.exists(parquet_path): parquet_size = os.path.getsize(parquet_path) @@ -535,21 +551,25 @@ def process_multiple_files( logger.error(msg) analysis_result["error_message"] = "Parquet file creation failed" return analysis_result - + except Exception as e: logger.error(f"Error creating parquet file: {e}") analysis_result["error_message"] = f"Parquet creation failed: {str(e)}" return analysis_result else: - analysis_result["error_message"] = "No processed data available for parquet creation" + analysis_result["error_message"] = ( + "No processed data available for parquet creation" + ) analysis_result["processing_steps"].append("no_data_for_parquet") return analysis_result analysis_result["processing_steps"].append("analysis_complete") logger.info(f"Multi-file bed conversion completed for {sample_id}") - logger.info(f"Files successfully processed: {analysis_result['files_processed']}/{analysis_result['total_files']}") + logger.info( + f"Files successfully processed: {analysis_result['files_processed']}/{analysis_result['total_files']}" + ) logger.info(f"Output parquet file: {analysis_result['parquet_path']}") - + return analysis_result except Exception as e: @@ -597,27 +617,35 @@ def bed_conversion_handler(job, work_dir=None, reference=None): # Get job-specific logger logger = get_job_logger(str(job.job_id), job.job_type, job.context.filepath) suppress_expected = _is_fail_only_expected(job) - + # Check if this is a batched job batched_job = job.context.metadata.get("_batched_job") if batched_job: batch_size = batched_job.get_file_count() sample_id = batched_job.get_sample_id() batch_id = batched_job.batch_id - logger.info(f"Processing bed conversion batch: {batch_size} files for sample '{sample_id}' (batch_id: {batch_id})") - + logger.info( + f"Processing bed conversion batch: {batch_size} files for sample '{sample_id}' (batch_id: {batch_id})" + ) + # Get all filepaths in the batch filepaths = batched_job.get_filepaths() - + # Log individual files in the batch for i, filepath in enumerate(filepaths): - logger.info(f" Batch file {i+1}/{batch_size}: {os.path.basename(filepath)}") - + logger.info( + f" Batch file {i+1}/{batch_size}: {os.path.basename(filepath)}" + ) + # Prepare metadata list for all BAM files in the batch (list comp inlined in 3.12) def _batch_metadata(i: int) -> dict: ctx = batched_job.contexts[i] sid = ctx.get_sample_id() - return {**ctx.metadata.get("bam_metadata", {}), "sample_id": sid if sid != "unknown" else sample_id} + return { + **ctx.metadata.get("bam_metadata", {}), + "sample_id": sid if sid != "unknown" else sample_id, + } + metadata_list = [_batch_metadata(i) for i in range(len(filepaths))] # Determine work directory for the batch @@ -629,7 +657,7 @@ def _batch_metadata(i: int) -> dict: os.makedirs(work_dir, exist_ok=True) batch_work_dir = work_dir logger.debug(f"Using specified work directory: {batch_work_dir}") - + ref_fasta, cpg_mode = _bed_conversion_matkit_options( job, batch_work_dir, reference, sample_id ) @@ -639,7 +667,9 @@ def _batch_metadata(i: int) -> dict: ) # Process all BAM files in the batch using the new aggregated function - logger.info(f"Processing {batch_size} BAM files as aggregated batch for sample '{sample_id}'") + logger.info( + f"Processing {batch_size} BAM files as aggregated batch for sample '{sample_id}'" + ) batch_result = process_multiple_files( bam_paths=filepaths, metadata_list=metadata_list, @@ -650,20 +680,27 @@ def _batch_metadata(i: int) -> dict: reference_fasta=ref_fasta, cpg_mode=cpg_mode, ) - + # Store batch results in job context (maintain compatibility with existing structure) - job.context.add_metadata("bed_conversion", { - "batch_result": batch_result, # Single aggregated result - "batch_size": batch_size, - "sample_id": sample_id, - "batch_id": batch_id, - "files_processed": batch_result.get("files_processed", batch_size), - "total_files": batch_result.get("total_files", batch_size) - }) - - logger.info(f"Completed bed conversion batch processing: {batch_size} files for sample '{sample_id}'") - logger.info(f"Files successfully processed: {batch_result.get('files_processed', batch_size)}/{batch_result.get('total_files', batch_size)}") - + job.context.add_metadata( + "bed_conversion", + { + "batch_result": batch_result, # Single aggregated result + "batch_size": batch_size, + "sample_id": sample_id, + "batch_id": batch_id, + "files_processed": batch_result.get("files_processed", batch_size), + "total_files": batch_result.get("total_files", batch_size), + }, + ) + + logger.info( + f"Completed bed conversion batch processing: {batch_size} files for sample '{sample_id}'" + ) + logger.info( + f"Files successfully processed: {batch_result.get('files_processed', batch_size)}/{batch_result.get('total_files', batch_size)}" + ) + if batch_result.get("error_message"): if suppress_expected: logger.warning( @@ -682,9 +719,13 @@ def _batch_metadata(i: int) -> dict: logger.error( f"Batch processing completed with errors: {batch_result['error_message']}" ) - job.context.add_error("bed_conversion", batch_result["error_message"]) + job.context.add_error( + "bed_conversion", batch_result["error_message"] + ) else: - logger.info("Batch processing completed successfully with aggregated bed conversion") + logger.info( + "Batch processing completed successfully with aggregated bed conversion" + ) job.context.add_result( "bed_conversion", { @@ -693,15 +734,17 @@ def _batch_metadata(i: int) -> dict: "analysis_time": batch_result.get("analysis_timestamp", 0), "parquet_path": batch_result.get("parquet_path", ""), "processing_steps": batch_result.get("processing_steps", []), - "files_processed": batch_result.get("files_processed", batch_size), + "files_processed": batch_result.get( + "files_processed", batch_size + ), "total_files": batch_result.get("total_files", batch_size), "matkit_cpg_mode": cpg_mode, "reference_fasta": ref_fasta, }, ) - + return - + else: # Single file processing (backward compatibility) bam_path = job.context.filepath @@ -743,7 +786,9 @@ def _batch_metadata(i: int) -> dict: # Store results in job context job.context.add_metadata("bed_conversion", bed_result.results) - job.context.add_metadata("bed_processing_steps", bed_result.processing_steps) + job.context.add_metadata( + "bed_processing_steps", bed_result.processing_steps + ) if bed_result.error_message: if suppress_expected: @@ -782,7 +827,9 @@ def _batch_metadata(i: int) -> dict: ) logger.info(f"Sample ID: {bed_result.sample_id}") logger.info(f"Parquet file: {bed_result.parquet_path}") - logger.debug(f"Processing steps: {', '.join(bed_result.processing_steps)}") + logger.debug( + f"Processing steps: {', '.join(bed_result.processing_steps)}" + ) except Exception as e: suppress_expected = _is_fail_only_expected(job) diff --git a/src/robin/analysis/cnv_analysis.py b/src/robin/analysis/cnv_analysis.py index 3d982110..7bff55e5 100644 --- a/src/robin/analysis/cnv_analysis.py +++ b/src/robin/analysis/cnv_analysis.py @@ -74,24 +74,24 @@ Matt Loose """ -import os +import gc import logging -import time +import os import pickle -import gc import subprocess import sys -from functools import lru_cache -from typing import Any, Dict, Optional, List, Tuple +import time from dataclasses import dataclass +from functools import lru_cache +from typing import Any, Dict, List, Optional, Tuple import numpy as np import pysam +import ruptures as rpt from scipy.ndimage import uniform_filter1d -import ruptures as rpt -from robin.logging_config import get_job_logger import robin.resources as resources +from robin.logging_config import get_job_logger os.environ["CI"] = "1" @@ -123,64 +123,71 @@ def get_cached_ref_cnv_dict(ref_cnv_dict_path: str, logger) -> dict: """ Get reference CNV dictionary with caching to avoid reloading for every sample. - + Args: ref_cnv_dict_path: Path to reference CNV pickle file logger: Logger instance - + Returns: Reference CNV dictionary """ global _ref_cnv_dict_cache, _ref_cnv_dict_path_cache - + # Check if we need to load the reference dict if _ref_cnv_dict_cache is None or _ref_cnv_dict_path_cache != ref_cnv_dict_path: - logger.info(f"Loading reference CNV dict from {ref_cnv_dict_path} (first time or path changed)") + logger.info( + f"Loading reference CNV dict from {ref_cnv_dict_path} (first time or path changed)" + ) load_start = time.time() - + with open(ref_cnv_dict_path, "rb") as f: _ref_cnv_dict_cache = pickle.load(f) _ref_cnv_dict_path_cache = ref_cnv_dict_path - + load_time = time.time() - load_start - logger.info(f"Reference CNV dict loaded in {load_time:.3f}s (cached for subsequent samples)") + logger.info( + f"Reference CNV dict loaded in {load_time:.3f}s (cached for subsequent samples)" + ) else: logger.debug("Using cached reference CNV dict") - + return _ref_cnv_dict_cache + def clear_ref_cnv_dict_cache(): """Clear the reference CNV dict cache (useful when switching reference files)""" global _ref_cnv_dict_cache, _ref_cnv_dict_path_cache _ref_cnv_dict_cache = None _ref_cnv_dict_path_cache = None + def get_sample_cache(sample_id: str) -> dict: """ Get the cache for a specific sample, creating it if it doesn't exist. - + Args: sample_id: Sample identifier - + Returns: Dictionary containing cached data for the sample """ global _sample_cache if sample_id not in _sample_cache: _sample_cache[sample_id] = { - 'copy_numbers': None, - 'copy_numbers_path': None, - 'analysis_counter': None, - 'last_accessed': time.time(), - 'bam_count': 0 + "copy_numbers": None, + "copy_numbers_path": None, + "analysis_counter": None, + "last_accessed": time.time(), + "bam_count": 0, } - _sample_cache[sample_id]['last_accessed'] = time.time() + _sample_cache[sample_id]["last_accessed"] = time.time() return _sample_cache[sample_id] + def update_sample_cache(sample_id: str, **kwargs): """ Update the cache for a specific sample. - + Args: sample_id: Sample identifier **kwargs: Key-value pairs to update in the cache @@ -188,10 +195,11 @@ def update_sample_cache(sample_id: str, **kwargs): cache = get_sample_cache(sample_id) cache.update(kwargs) + def clear_sample_cache(sample_id: str = None): """ Clear the cache for a specific sample or all samples. - + Args: sample_id: Sample identifier to clear, or None to clear all """ @@ -204,74 +212,84 @@ def clear_sample_cache(sample_id: str = None): if _current_sample_id == sample_id: _current_sample_id = None + def set_current_sample(sample_id: str, logger) -> bool: """ Set the current sample being processed and manage cache transitions. - + Args: sample_id: Sample identifier logger: Logger instance - + Returns: True if this is a new sample (cache was cleared), False if same sample """ global _current_sample_id - + if _current_sample_id != sample_id: - logger.info(f"Switching from sample '{_current_sample_id}' to '{sample_id}' - managing cache") + logger.info( + f"Switching from sample '{_current_sample_id}' to '{sample_id}' - managing cache" + ) _current_sample_id = sample_id - + # Update BAM count for the new sample cache = get_sample_cache(sample_id) - cache['bam_count'] += 1 - + cache["bam_count"] += 1 + return True # New sample else: # Same sample, just increment BAM count cache = get_sample_cache(sample_id) - cache['bam_count'] += 1 - logger.debug(f"Processing BAM file {cache['bam_count']} for sample '{sample_id}'") + cache["bam_count"] += 1 + logger.debug( + f"Processing BAM file {cache['bam_count']} for sample '{sample_id}'" + ) return False # Same sample + def cleanup_sample_cache_on_completion(sample_id: str, logger) -> None: """ Clean up sample cache when all BAM files for a sample are complete. This should be called by the workflow system when a sample is fully processed. - + Args: sample_id: Sample identifier logger: Logger instance """ cache = get_sample_cache(sample_id) - bam_count = cache.get('bam_count', 0) - logger.info(f"Sample '{sample_id}' completed processing {bam_count} BAM files - cache can be cleaned up") - + bam_count = cache.get("bam_count", 0) + logger.info( + f"Sample '{sample_id}' completed processing {bam_count} BAM files - cache can be cleaned up" + ) + # Note: We don't actually clear the cache here as it might be needed for other operations # The workflow system should call clear_sample_cache() when appropriate + def get_sample_cache_stats() -> dict: """ Get statistics about the current sample cache for monitoring purposes. - + Returns: Dictionary with cache statistics """ global _sample_cache, _current_sample_id return { - 'current_sample': _current_sample_id, - 'cached_samples': list(_sample_cache.keys()), - 'cache_size': len(_sample_cache), - 'sample_details': { + "current_sample": _current_sample_id, + "cached_samples": list(_sample_cache.keys()), + "cache_size": len(_sample_cache), + "sample_details": { sid: { - 'bam_count': cache.get('bam_count', 0), - 'last_accessed': cache.get('last_accessed', 0), - 'has_copy_numbers': cache.get('copy_numbers') is not None, - 'analysis_counter': cache.get('analysis_counter') + "bam_count": cache.get("bam_count", 0), + "last_accessed": cache.get("last_accessed", 0), + "has_copy_numbers": cache.get("copy_numbers") is not None, + "analysis_counter": cache.get("analysis_counter"), } for sid, cache in _sample_cache.items() - } + }, } + def run_cnv_analysis_direct( bam_path, copy_numbers, @@ -298,7 +316,7 @@ def run_cnv_analysis_direct( Dictionary with analysis results or None if failed """ import cnv_from_bam - + try: # Use provided copy_numbers dict directly (no file I/O) if copy_numbers is None: @@ -322,7 +340,9 @@ def run_cnv_analysis_direct( log_level=int(logging.ERROR), ) pass1_time = time.time() - pass1_start - logger.info(f"Pass 1 completed in {pass1_time:.2f}s (bin_width: {result.bin_width}, variance: {result.variance:.6f})") + logger.info( + f"Pass 1 completed in {pass1_time:.2f}s (bin_width: {result.bin_width}, variance: {result.variance:.6f})" + ) # Reference track from packaged control counts only (no sample BAM). # Re-reading the sample BAM onto the control previously contaminated CNV2. @@ -351,7 +371,7 @@ def run_cnv_analysis_direct( "pass1_time": pass1_time, "pass2_time": pass2_time, "total_time": pass1_time + pass2_time, - } + }, } logger.debug("CNV analysis completed successfully (direct mode)") @@ -360,6 +380,7 @@ def run_cnv_analysis_direct( except Exception as e: logger.error(f"Error in direct CNV analysis: {e}") import traceback + logger.error(traceback.format_exc()) return { "success": False, @@ -367,6 +388,7 @@ def run_cnv_analysis_direct( "error_type": type(e).__name__, } + def run_cnv_analysis_subprocess( bam_path, copy_numbers, @@ -432,7 +454,7 @@ def run_cnv_analysis_subprocess( "--mapq-filter", str(mapq_filter), ] - + # Use per-sample copy_numbers file if provided (OPTIMIZED APPROACH) if copy_numbers_path: cmd.extend(["--copy-numbers-path", copy_numbers_path]) @@ -446,12 +468,12 @@ def run_cnv_analysis_subprocess( logger.debug("Using temporary copy_numbers file (fallback)") logger.debug(f"Running CNV analysis in subprocess: {' '.join(cmd)}") - + # Log file size for performance monitoring try: bam_size_mb = os.path.getsize(bam_path) / (1024 * 1024) logger.info(f"Processing BAM file: {bam_size_mb:.1f} MB") - + # Warn about very large files that might take a long time if bam_size_mb > 500: logger.warning( @@ -464,7 +486,7 @@ def run_cnv_analysis_subprocess( # Run subprocess with stdout/stderr redirected to files to avoid capture overhead stdout_log_path = os.path.join(temp_dir, "cnv_subprocess.stdout.log") stderr_log_path = os.path.join(temp_dir, "cnv_subprocess.stderr.log") - + try: with open(stdout_log_path, "w") as _out, open(stderr_log_path, "w") as _err: result = subprocess.run( @@ -501,7 +523,7 @@ def run_cnv_analysis_subprocess( else: logger.error("Results file not found") return None - + except subprocess.TimeoutExpired: logger.error( f"CNV analysis subprocess timed out after {timeout}s for {os.path.basename(bam_path)}. " @@ -513,13 +535,16 @@ def run_cnv_analysis_subprocess( with open(stderr_log_path, "r") as f: stderr_content = f.read() if stderr_content.strip(): - logger.error(f"subprocess stderr before timeout:\n{stderr_content[-1000:]}") # Last 1000 chars + logger.error( + f"subprocess stderr before timeout:\n{stderr_content[-1000:]}" + ) # Last 1000 chars except Exception: pass return None except Exception as e: logger.error(f"Error running CNV analysis subprocess: {e}") import traceback + logger.error(traceback.format_exc()) return None @@ -958,7 +983,9 @@ def prepare_cnv_calling_track( ref_cnv_map, ref_mappability_eps=ref_mappability_eps, ) - calling_bw = resolve_cnv_calling_bin_width(analysis_bin_width, min_calling_bin_width) + calling_bw = resolve_cnv_calling_bin_width( + analysis_bin_width, min_calling_bin_width + ) return ( coarsen_cnv_track(log2_track, analysis_bin_width, calling_bw), calling_bw, @@ -1016,7 +1043,9 @@ def downsample_cnv_for_plot( where=finite_counts > 0, ) # Centre each display bin in analysis-bin coordinates (not nominal plot_bin_width). - x_bp = (np.arange(len(values_out)) * group_size + (group_size / 2.0)) * analysis_bin_width + x_bp = ( + np.arange(len(values_out)) * group_size + (group_size / 2.0) + ) * analysis_bin_width return x_bp, values_out @@ -1032,7 +1061,9 @@ def downsample_cnv_chromosome_track( """ values = np.asarray(values_1d, dtype=float) resolved_plot_bw = resolve_cnv_plot_bin_width(analysis_bin_width, plot_bin_width) - x_bp, plot_values = downsample_cnv_for_plot(values, analysis_bin_width, resolved_plot_bw) + x_bp, plot_values = downsample_cnv_for_plot( + values, analysis_bin_width, resolved_plot_bw + ) x_max_mb = len(values) * analysis_bin_width / 1_000_000.0 return x_bp / 1_000_000.0, plot_values, x_max_mb @@ -1320,15 +1351,17 @@ def load_analysis_counter(sample_id: str, work_dir: str, logger) -> int: """Load the analysis counter for a sample from disk with caching""" # Check cache first cache = get_sample_cache(sample_id) - if cache['analysis_counter'] is not None: - logger.debug(f"Using cached analysis counter for {sample_id}: {cache['analysis_counter']}") - return cache['analysis_counter'] - + if cache["analysis_counter"] is not None: + logger.debug( + f"Using cached analysis counter for {sample_id}: {cache['analysis_counter']}" + ) + return cache["analysis_counter"] + # Load from disk result = _read_analysis_counter_from_disk(sample_id, work_dir) - + # Cache the result - cache['analysis_counter'] = result + cache["analysis_counter"] = result logger.debug(f"Loaded and cached analysis counter for {sample_id}: {result}") return result @@ -1341,7 +1374,7 @@ def save_analysis_counter(sample_id: str, counter: int, work_dir: str, logger) - os.makedirs(os.path.dirname(counter_file), exist_ok=True) with open(counter_file, "w") as f: f.write(str(counter)) - + # Update cache update_sample_cache(sample_id, analysis_counter=counter) logger.debug(f"Saved and cached analysis counter for {sample_id}: {counter}") @@ -1349,9 +1382,7 @@ def save_analysis_counter(sample_id: str, counter: int, work_dir: str, logger) - logger.error(f"Error saving counter for {sample_id}: {e}") -def allocate_next_analysis_counter( - sample_id: str, work_dir: str, logger=None -) -> int: +def allocate_next_analysis_counter(sample_id: str, work_dir: str, logger=None) -> int: """ Atomically increment the shared BED/CNV analysis counter and return the new value. @@ -1630,15 +1661,15 @@ def _atomic_pickle_dump(obj, target_path: str) -> None: if generate_master_bed: try: from robin.analysis.master_bed_generator import ( - generate_master_bed_async, _try_get_target_panel_from_fusion_metadata, + generate_master_bed_async, ) - + # Extract sample_id from sample_dir sample_id = os.path.basename(sample_dir) # work_dir is the parent of sample_dir work_dir = os.path.dirname(sample_dir) - + # Prefer the workflow panel passed from cnv_handler; fall back to # fusion metadata only when the caller did not supply one. resolved_panel = target_panel @@ -1652,7 +1683,7 @@ def _atomic_pickle_dump(obj, target_path: str) -> None: "(neither workflow target_panel nor fusion metadata available)", sample_id, ) - + # Generate asynchronously (non-blocking) generate_master_bed_async( sample_id=sample_id, @@ -1671,8 +1702,6 @@ def _atomic_pickle_dump(obj, target_path: str) -> None: logger.error(f"Error saving CNV files: {e}") - - def process_single_bam( bam_path, metadata, @@ -1698,10 +1727,10 @@ def process_single_bam( Dictionary with CNV analysis results """ sample_id = metadata.extracted_data.get("sample_id", "unknown") - + # Set current sample and check if this is a new sample is_new_sample = set_current_sample(sample_id, logger) - + if is_new_sample: logger.info(f"🧬 Starting CNV analysis for NEW sample: {sample_id}") else: @@ -1749,7 +1778,7 @@ def process_single_bam( # Check file size and adjust timeout accordingly bam_size_mb = os.path.getsize(bam_path) / (1024 * 1024) logger.debug(f"BAM file size: {bam_size_mb:.1f} MB") - + # Adaptive timeout based on file size # Base: 1 hour for files up to 100 MB # Add 1 minute per additional 10 MB for large files @@ -1765,35 +1794,50 @@ def process_single_bam( # Use per-sample copy_numbers file (OPTIMIZED APPROACH) # This eliminates the multi-sample dictionary bottleneck - copy_numbers_path = os.path.join(sample_output_dir, f"{sample_id}_copy_numbers.pkl") - + copy_numbers_path = os.path.join( + sample_output_dir, f"{sample_id}_copy_numbers.pkl" + ) + # Check cache first for copy_numbers cache = get_sample_cache(sample_id) - if cache['copy_numbers'] is not None and cache['copy_numbers_path'] == copy_numbers_path: + if ( + cache["copy_numbers"] is not None + and cache["copy_numbers_path"] == copy_numbers_path + ): logger.debug(f"Using cached copy_numbers for {sample_id}") - copy_numbers = cache['copy_numbers'] + copy_numbers = cache["copy_numbers"] else: # Load from disk or start fresh copy_numbers = None - + # Backward compatibility: Migrate from old multi-sample dict to per-sample file legacy_dict_path = os.path.join(sample_output_dir, "update_cnv_dict.pkl") - if not os.path.exists(copy_numbers_path) and os.path.exists(legacy_dict_path): - logger.info("Migrating from legacy multi-sample dict to per-sample file...") + if not os.path.exists(copy_numbers_path) and os.path.exists( + legacy_dict_path + ): + logger.info( + "Migrating from legacy multi-sample dict to per-sample file..." + ) try: with open(legacy_dict_path, "rb") as f: update_cnv_dict = pickle.load(f) if sample_id in update_cnv_dict: # Extract this sample's data and save to per-sample file with open(copy_numbers_path, "wb") as f: - pickle.dump(update_cnv_dict[sample_id], f, protocol=pickle.HIGHEST_PROTOCOL) - logger.info(f"Migrated copy_numbers for {sample_id} to per-sample file") - + pickle.dump( + update_cnv_dict[sample_id], + f, + protocol=pickle.HIGHEST_PROTOCOL, + ) + logger.info( + f"Migrated copy_numbers for {sample_id} to per-sample file" + ) + # Optionally remove legacy file after migration (commented out for safety) # os.remove(legacy_dict_path) except Exception as e: logger.warning(f"Could not migrate legacy copy_numbers: {e}") - + if os.path.exists(copy_numbers_path): logger.debug(f"Loading copy_numbers from disk: {copy_numbers_path}") try: @@ -1806,9 +1850,13 @@ def process_single_bam( else: logger.debug("No previous copy_numbers found, starting fresh") copy_numbers = {} - + # Cache the loaded copy_numbers - update_sample_cache(sample_id, copy_numbers=copy_numbers, copy_numbers_path=copy_numbers_path) + update_sample_cache( + sample_id, + copy_numbers=copy_numbers, + copy_numbers_path=copy_numbers_path, + ) # Load reference CNV dict once (cached for subsequent samples) ref_cnv_path = os.path.join( @@ -1819,7 +1867,9 @@ def process_single_bam( # Process BAM file with cnv_from_bam using configurable execution mode execution_mode = "subprocess" if USE_CNV_SUBPROCESS else "direct" - logger.debug(f"Processing BAM file with cnv_from_bam ({execution_mode}, {threads} threads)") + logger.debug( + f"Processing BAM file with cnv_from_bam ({execution_mode}, {threads} threads)" + ) analysis_start = time.time() try: # Run CNV analysis using configurable execution mode @@ -1864,25 +1914,31 @@ def process_single_bam( genome_length = subprocess_result["genome_length"] r2_cnv = subprocess_result["r2_cnv"] updated_copy_numbers = subprocess_result["updated_copy_numbers"] - + # Log timing information from analysis if "timing" in subprocess_result: timing = subprocess_result["timing"] - logger.info(f"CNV analysis timing: Pass1={timing['pass1_time']:.2f}s, Pass2={timing['pass2_time']:.2f}s, Total={timing['total_time']:.2f}s") - + logger.info( + f"CNV analysis timing: Pass1={timing['pass1_time']:.2f}s, Pass2={timing['pass2_time']:.2f}s, Total={timing['total_time']:.2f}s" + ) + analysis_elapsed = time.time() - analysis_start - logger.info(f"CNV analysis ({execution_mode}) completed in {analysis_elapsed:.2f}s (total with overhead)") + logger.info( + f"CNV analysis ({execution_mode}) completed in {analysis_elapsed:.2f}s (total with overhead)" + ) # Save updated copy_numbers back to per-sample file (OPTIMIZED) save_start = time.time() with open(copy_numbers_path, "wb") as f: pickle.dump(updated_copy_numbers, f, protocol=pickle.HIGHEST_PROTOCOL) save_time = time.time() - save_start - logger.debug(f"Saved updated copy_numbers to {copy_numbers_path} in {save_time:.3f}s") - + logger.debug( + f"Saved updated copy_numbers to {copy_numbers_path} in {save_time:.3f}s" + ) + # Update cache with the saved copy_numbers update_sample_cache(sample_id, copy_numbers=updated_copy_numbers) - + # Update the local copy_numbers variable for subsequent processing copy_numbers = updated_copy_numbers @@ -1898,10 +1954,9 @@ def process_single_bam( analysis_result["processing_steps"].append("cnv_extraction_failed") return analysis_result - # Calculate normalized CNV data (difference between sample and reference) logger.debug("Calculating normalized CNV data") - + result3_cnv = {} for key in r_cnv.keys(): if key != "chrM" and key in r2_cnv: @@ -1913,7 +1968,7 @@ def process_single_bam( result3_cnv[key] = moving_avg_data1 - moving_avg_data2 analysis_result["processing_steps"].append("normalized_cnv_calculated") - + # Estimate sex from CNV data sex_estimate = estimate_sex_from_cnv(result3_cnv, logger) analysis_result["sex_estimate"] = sex_estimate @@ -1984,7 +2039,7 @@ def process_single_bam( logger.info(f"CNV analysis complete for {sample_id}") logger.info(f"Sex Estimate: {analysis_result['sex_estimate']}") logger.info(f"Breakpoints: {len(analysis_result['breakpoints'])} detected") - + return analysis_result except Exception as e: @@ -2006,7 +2061,7 @@ def process_multiple_bams( ): """ Process multiple BAM files for CNV analysis using aggregated CNV data. - + This function processes multiple BAM files for the same sample, accumulating CNV data across all files before performing downstream analysis. This is more efficient than processing each BAM file individually and then trying @@ -2027,27 +2082,29 @@ def process_multiple_bams( """ if not bam_paths or not metadata_list: raise ValueError("bam_paths and metadata_list must not be empty") - + if len(bam_paths) != len(metadata_list): raise ValueError("bam_paths and metadata_list must have the same length") - + # Get sample ID from first metadata (assuming all BAMs are from same sample) sample_id = metadata_list[0].extracted_data.get("sample_id", "unknown") - + # Set current sample and check if this is a new sample is_new_sample = set_current_sample(sample_id, logger) - + if is_new_sample: logger.info(f"🧬 Starting multi-BAM CNV analysis for NEW sample: {sample_id}") else: logger.info(f"🧬 Continuing multi-BAM CNV analysis for sample: {sample_id}") logger.info(f"Processing {len(bam_paths)} BAM files for sample {sample_id}") - + # Log essential metadata only for i, (bam_path, metadata) in enumerate(zip(bam_paths, metadata_list)): - logger.debug(f"BAM file {i+1}: {metadata.file_path} ({metadata.file_size:,} bytes)") - + logger.debug( + f"BAM file {i+1}: {metadata.file_path} ({metadata.file_size:,} bytes)" + ) + logger.debug(f"Sample ID: {sample_id}") logger.debug(f"Threads: {threads}") @@ -2081,51 +2138,68 @@ def process_multiple_bams( # Check if any BAM has reads valid_bam_paths = [] valid_metadata_list = [] - + for bam_path, metadata in zip(bam_paths, metadata_list): if has_reads(bam_path): valid_bam_paths.append(bam_path) valid_metadata_list.append(metadata) else: logger.warning(f"No reads found in BAM file: {bam_path}") - + if not valid_bam_paths: logger.warning(f"No valid BAM files found for {sample_id}") analysis_result["error_message"] = "No reads found in any BAM files" analysis_result["processing_steps"].append("no_reads_found") return analysis_result - logger.info(f"Processing {len(valid_bam_paths)} valid BAM files out of {len(bam_paths)} total") + logger.info( + f"Processing {len(valid_bam_paths)} valid BAM files out of {len(bam_paths)} total" + ) analysis_result["files_processed"] = len(valid_bam_paths) analysis_result["processing_steps"].append("reads_found") # Use per-sample copy_numbers file (OPTIMIZED APPROACH) - copy_numbers_path = os.path.join(sample_output_dir, f"{sample_id}_copy_numbers.pkl") - + copy_numbers_path = os.path.join( + sample_output_dir, f"{sample_id}_copy_numbers.pkl" + ) + # Check cache first for copy_numbers cache = get_sample_cache(sample_id) - if cache['copy_numbers'] is not None and cache['copy_numbers_path'] == copy_numbers_path: + if ( + cache["copy_numbers"] is not None + and cache["copy_numbers_path"] == copy_numbers_path + ): logger.debug(f"Using cached copy_numbers for {sample_id}") - copy_numbers = cache['copy_numbers'] + copy_numbers = cache["copy_numbers"] else: # Load from disk or start fresh copy_numbers = None - + # Backward compatibility: Migrate from old multi-sample dict to per-sample file legacy_dict_path = os.path.join(sample_output_dir, "update_cnv_dict.pkl") - if not os.path.exists(copy_numbers_path) and os.path.exists(legacy_dict_path): - logger.info("Migrating from legacy multi-sample dict to per-sample file...") + if not os.path.exists(copy_numbers_path) and os.path.exists( + legacy_dict_path + ): + logger.info( + "Migrating from legacy multi-sample dict to per-sample file..." + ) try: with open(legacy_dict_path, "rb") as f: update_cnv_dict = pickle.load(f) if sample_id in update_cnv_dict: # Extract this sample's data and save to per-sample file with open(copy_numbers_path, "wb") as f: - pickle.dump(update_cnv_dict[sample_id], f, protocol=pickle.HIGHEST_PROTOCOL) - logger.info(f"Migrated copy_numbers for {sample_id} to per-sample file") + pickle.dump( + update_cnv_dict[sample_id], + f, + protocol=pickle.HIGHEST_PROTOCOL, + ) + logger.info( + f"Migrated copy_numbers for {sample_id} to per-sample file" + ) except Exception as e: logger.warning(f"Could not migrate legacy copy_numbers: {e}") - + if os.path.exists(copy_numbers_path): logger.debug(f"Loading copy_numbers from disk: {copy_numbers_path}") try: @@ -2138,9 +2212,13 @@ def process_multiple_bams( else: logger.debug("No previous copy_numbers found, starting fresh") copy_numbers = {} - + # Cache the loaded copy_numbers - update_sample_cache(sample_id, copy_numbers=copy_numbers, copy_numbers_path=copy_numbers_path) + update_sample_cache( + sample_id, + copy_numbers=copy_numbers, + copy_numbers_path=copy_numbers_path, + ) # Load reference CNV dict once (cached for subsequent samples) ref_cnv_path = os.path.join( @@ -2151,12 +2229,16 @@ def process_multiple_bams( # Process all BAM files with cnv_from_bam using configurable execution mode execution_mode = "subprocess" if USE_CNV_SUBPROCESS else "direct" - logger.debug(f"Processing {len(valid_bam_paths)} BAM files with cnv_from_bam ({execution_mode}, {threads} threads)") - + logger.debug( + f"Processing {len(valid_bam_paths)} BAM files with cnv_from_bam ({execution_mode}, {threads} threads)" + ) + # Calculate adaptive timeout based on total file sizes - total_bam_size_mb = sum(os.path.getsize(bam_path) for bam_path in valid_bam_paths) / (1024 * 1024) + total_bam_size_mb = sum( + os.path.getsize(bam_path) for bam_path in valid_bam_paths + ) / (1024 * 1024) logger.debug(f"Total BAM file size: {total_bam_size_mb:.1f} MB") - + # Adaptive timeout based on total file size base_timeout = 3600 # 1 hour if total_bam_size_mb > 100: @@ -2171,9 +2253,13 @@ def process_multiple_bams( analysis_start = time.time() try: # Process each BAM file, accumulating CNV data in copy_numbers - for i, (bam_path, metadata) in enumerate(zip(valid_bam_paths, valid_metadata_list)): - logger.info(f"Processing BAM file {i+1}/{len(valid_bam_paths)}: {os.path.basename(bam_path)}") - + for i, (bam_path, metadata) in enumerate( + zip(valid_bam_paths, valid_metadata_list) + ): + logger.info( + f"Processing BAM file {i+1}/{len(valid_bam_paths)}: {os.path.basename(bam_path)}" + ) + # Run CNV analysis using configurable execution mode if USE_CNV_SUBPROCESS: # Use subprocess execution (original approach) @@ -2201,29 +2287,39 @@ def process_multiple_bams( sample_id=sample_id, ) - if subprocess_result is None or not subprocess_result.get("success", False): + if subprocess_result is None or not subprocess_result.get( + "success", False + ): error_msg = ( subprocess_result.get("error", "Unknown error") if subprocess_result else "Subprocess failed" ) - raise RuntimeError(f"CNV analysis subprocess failed for {os.path.basename(bam_path)}: {error_msg}") + raise RuntimeError( + f"CNV analysis subprocess failed for {os.path.basename(bam_path)}: {error_msg}" + ) # Extract results from subprocess (only need the final accumulated copy_numbers) updated_copy_numbers = subprocess_result["updated_copy_numbers"] - + # Log timing information from analysis if "timing" in subprocess_result: timing = subprocess_result["timing"] - logger.debug(f"BAM {i+1} CNV analysis timing: Pass1={timing['pass1_time']:.2f}s, Pass2={timing['pass2_time']:.2f}s, Total={timing['total_time']:.2f}s") - + logger.debug( + f"BAM {i+1} CNV analysis timing: Pass1={timing['pass1_time']:.2f}s, Pass2={timing['pass2_time']:.2f}s, Total={timing['total_time']:.2f}s" + ) + # Update copy_numbers for next iteration copy_numbers = updated_copy_numbers - - logger.info(f"Completed BAM file {i+1}/{len(valid_bam_paths)}: {os.path.basename(bam_path)}") + + logger.info( + f"Completed BAM file {i+1}/{len(valid_bam_paths)}: {os.path.basename(bam_path)}" + ) if job_id is not None: try: - from robin.workflow_ray import notify_coordinator_files_completed + from robin.workflow_ray import ( + notify_coordinator_files_completed, + ) notify_coordinator_files_completed("cnv", 1, job_id=job_id) except Exception: @@ -2250,9 +2346,11 @@ def process_multiple_bams( genome_length = final_result["genome_length"] r2_cnv = final_result["r2_cnv"] final_copy_numbers = final_result["updated_copy_numbers"] - + analysis_elapsed = time.time() - analysis_start - logger.info(f"Multi-BAM CNV analysis ({execution_mode}) completed in {analysis_elapsed:.2f}s (total with overhead)") + logger.info( + f"Multi-BAM CNV analysis ({execution_mode}) completed in {analysis_elapsed:.2f}s (total with overhead)" + ) # Save updated copy_numbers back to per-sample file (OPTIMIZED) save_start = time.time() @@ -2288,7 +2386,9 @@ def process_multiple_bams( moving_avg_data1, moving_avg_data2 ) result3_cnv[key] = moving_avg_data1 - moving_avg_data2 - logger.info(f"[cnv] Normalized CNV (moving avg + diff) completed in {time.time() - t0:.2f}s") + logger.info( + f"[cnv] Normalized CNV (moving avg + diff) completed in {time.time() - t0:.2f}s" + ) analysis_result["processing_steps"].append("normalized_cnv_calculated") @@ -2351,7 +2451,9 @@ def process_multiple_bams( min_contiguous_bins=min_contiguous_bins, target_panel=target_panel, ) - logger.info(f"[cnv] save_cnv_files (incl. master BED) completed in {time.time() - t0:.2f}s") + logger.info( + f"[cnv] save_cnv_files (incl. master BED) completed in {time.time() - t0:.2f}s" + ) analysis_result["cnv_data_path"] = os.path.join( sample_output_dir, f"{analysis_counter}_cnv_data.json" @@ -2368,10 +2470,12 @@ def process_multiple_bams( # Replace print with logging logger.debug(f"Analysis result: {analysis_result}") logger.info(f"Multi-BAM CNV analysis complete for {sample_id}") - logger.info(f"Files processed: {analysis_result['files_processed']}/{analysis_result['total_files']}") + logger.info( + f"Files processed: {analysis_result['files_processed']}/{analysis_result['total_files']}" + ) logger.info(f"Sex Estimate: {analysis_result['sex_estimate']}") logger.info(f"Breakpoints: {len(analysis_result['breakpoints'])} detected") - + return analysis_result except Exception as e: @@ -2395,53 +2499,59 @@ def cnv_handler(job, work_dir=None, target_panel=None, threads=2): # Validate required parameters if not target_panel: raise ValueError("target_panel is required for CNV analysis") - + # Get job-specific logger logger = get_job_logger(str(job.job_id), job.job_type, job.context.filepath) - + # Check if this is a batched job batched_job = job.context.metadata.get("_batched_job") if batched_job: batch_size = batched_job.get_file_count() sample_id = batched_job.get_sample_id() batch_id = batched_job.batch_id - logger.info(f"Processing CNV batch: {batch_size} files for sample '{sample_id}' (batch_id: {batch_id})") - + logger.info( + f"Processing CNV batch: {batch_size} files for sample '{sample_id}' (batch_id: {batch_id})" + ) + # Get all filepaths in the batch filepaths = batched_job.get_filepaths() - + # Log individual files in the batch for i, filepath in enumerate(filepaths): - logger.info(f" Batch file {i+1}/{batch_size}: {os.path.basename(filepath)}") - + logger.info( + f" Batch file {i+1}/{batch_size}: {os.path.basename(filepath)}" + ) + # Prepare metadata list for all BAM files in the batch metadata_list = [] for i, bam_path in enumerate(filepaths): # Get metadata from preprocessing for this specific file # Note: Each file in the batch should have its own metadata file_metadata = batched_job.contexts[i].metadata.get("bam_metadata", {}) - + # Get sample ID from preprocessing results for this specific file file_context = batched_job.contexts[i] file_sample_id = file_context.get_sample_id() - + # Use the sample ID from the file's context (which should have preprocessing results) if file_sample_id != "unknown": file_metadata["sample_id"] = file_sample_id else: file_metadata["sample_id"] = sample_id - + # Create BamMetadata object for compatibility from robin.analysis.bam_preprocessor import BamMetadata - + metadata = BamMetadata( file_path=bam_path, file_size=batched_job.contexts[i].metadata.get("file_size", 0), - creation_time=batched_job.contexts[i].metadata.get("creation_time", time.time()), + creation_time=batched_job.contexts[i].metadata.get( + "creation_time", time.time() + ), extracted_data=file_metadata, ) metadata_list.append(metadata) - + # Determine work directory for the batch if work_dir is None: # Default to first BAM file directory @@ -2451,17 +2561,19 @@ def cnv_handler(job, work_dir=None, target_panel=None, threads=2): os.makedirs(work_dir, exist_ok=True) batch_work_dir = work_dir logger.debug(f"Using specified work directory: {batch_work_dir}") - + # Get reference from job metadata reference = job.context.metadata.get("reference") if reference: # Expand user home directory if present reference = os.path.expanduser(reference) logger.debug(f"Using reference genome from job metadata: {reference}") - + # Process all BAM files in the batch using the new aggregated function - logger.info(f"Processing {batch_size} BAM files as aggregated batch for sample '{sample_id}'") - + logger.info( + f"Processing {batch_size} BAM files as aggregated batch for sample '{sample_id}'" + ) + batch_result = process_multiple_bams( bam_paths=filepaths, metadata_list=metadata_list, @@ -2472,25 +2584,36 @@ def cnv_handler(job, work_dir=None, target_panel=None, threads=2): job_id=job.job_id, target_panel=target_panel, ) - + # Store batch results in job context (maintain compatibility with existing structure) - job.context.add_metadata("cnv_analysis", { - "batch_result": batch_result, # Single aggregated result - "batch_size": batch_size, - "sample_id": sample_id, - "batch_id": batch_id, - "files_processed": batch_result.get("files_processed", batch_size), - "total_files": batch_result.get("total_files", batch_size) - }) - - logger.info(f"Completed CNV batch processing: {batch_size} files for sample '{sample_id}'") - logger.info(f"Files successfully processed: {batch_result.get('files_processed', batch_size)}/{batch_result.get('total_files', batch_size)}") - + job.context.add_metadata( + "cnv_analysis", + { + "batch_result": batch_result, # Single aggregated result + "batch_size": batch_size, + "sample_id": sample_id, + "batch_id": batch_id, + "files_processed": batch_result.get("files_processed", batch_size), + "total_files": batch_result.get("total_files", batch_size), + }, + ) + + logger.info( + f"Completed CNV batch processing: {batch_size} files for sample '{sample_id}'" + ) + logger.info( + f"Files successfully processed: {batch_result.get('files_processed', batch_size)}/{batch_result.get('total_files', batch_size)}" + ) + if batch_result.get("error_message"): - logger.error(f"Batch processing completed with errors: {batch_result['error_message']}") + logger.error( + f"Batch processing completed with errors: {batch_result['error_message']}" + ) job.context.add_error("cnv_analysis", batch_result["error_message"]) else: - logger.info("Batch processing completed successfully with aggregated CNV analysis") + logger.info( + "Batch processing completed successfully with aggregated CNV analysis" + ) job.context.add_result( "cnv_analysis", { @@ -2503,9 +2626,9 @@ def cnv_handler(job, work_dir=None, target_panel=None, threads=2): "cnv_data_path": batch_result.get("cnv_data_path", ""), }, ) - + return - + else: # CNV jobs should always be batched - raise an error if not error_msg = f"CNV job received without batching metadata. Expected batched job but got single file: {os.path.basename(job.context.filepath)}" diff --git a/src/robin/analysis/cnv_classification.py b/src/robin/analysis/cnv_classification.py index 3d7660e7..aa3bbe85 100644 --- a/src/robin/analysis/cnv_classification.py +++ b/src/robin/analysis/cnv_classification.py @@ -8,27 +8,27 @@ import logging import pickle from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple +import natsort import numpy as np import pandas as pd -import natsort -from typing import Dict, List, Tuple, Optional, Any try: from importlib import resources as importlib_resources except ImportError: # pragma: no cover import importlib_resources # type: ignore -from robin.classification_config import ( - get_cnv_thresholds, - is_whole_chromosome_event, - is_arm_event, - is_resolution_sufficient -) from robin.analysis.cnv_analysis import ( CNV_MAPPABILITY_MIN_FRACTION, region_has_mappable_support, ) +from robin.classification_config import ( + get_cnv_thresholds, + is_arm_event, + is_resolution_sufficient, + is_whole_chromosome_event, +) logger = logging.getLogger(__name__) @@ -89,7 +89,7 @@ def _arm_has_mappable_support( class CNVEvent: """Represents a CNV event with metadata.""" - + def __init__( self, chromosome: str, @@ -101,10 +101,12 @@ def __init__( genes: List[str] = None, confidence: str = "Unknown", arm: Optional[str] = None, - proportion_affected: float = 0.0 + proportion_affected: float = 0.0, ): self.chromosome = chromosome - self.event_type = event_type # 'GAIN', 'LOSS', 'WHOLE_CHR_GAIN', 'WHOLE_CHR_LOSS' + self.event_type = ( + event_type # 'GAIN', 'LOSS', 'WHOLE_CHR_GAIN', 'WHOLE_CHR_LOSS' + ) self.mean_cnv = mean_cnv self.start_pos = start_pos self.end_pos = end_pos @@ -113,7 +115,7 @@ def __init__( self.confidence = confidence self.arm = arm self.proportion_affected = proportion_affected - + def to_dict(self) -> Dict[str, Any]: """Convert to dictionary for GUI/reporting.""" return { @@ -155,7 +157,7 @@ def analyze_chromosome_arms( chromosome: str, bin_width: int, sex_estimate: str, - cytobands_df: pd.DataFrame + cytobands_df: pd.DataFrame, ) -> Tuple[ Optional[float], Optional[float], @@ -188,9 +190,7 @@ def analyze_chromosome_arms( logger.debug(f"{chromosome} cytoband names: {chr_cytobands['name'].tolist()}") - p_arm_cytobands = chr_cytobands[ - chr_cytobands["name"].str.startswith("p", na=False) - ] + p_arm_cytobands = chr_cytobands[chr_cytobands["name"].str.startswith("p", na=False)] p_arm_mean = None p_arm_proportion_gain = 0.0 @@ -200,9 +200,11 @@ def analyze_chromosome_arms( p_arm_values: List[float] = [] for _, band in p_arm_cytobands.iterrows(): start_pos_bin = max(0, int(band["start_pos"] // bin_width)) - end_pos_bin = min(len(cnv_data[chromosome]) - 1, int(band["end_pos"] // bin_width)) + end_pos_bin = min( + len(cnv_data[chromosome]) - 1, int(band["end_pos"] // bin_width) + ) if end_pos_bin >= start_pos_bin: - region_values = cnv_data[chromosome][start_pos_bin:end_pos_bin + 1] + region_values = cnv_data[chromosome][start_pos_bin : end_pos_bin + 1] p_arm_values.extend(region_values) p_arm_values = _finite_arm_values(p_arm_values) if p_arm_values: @@ -216,9 +218,7 @@ def analyze_chromosome_arms( f"gain_prop={p_arm_proportion_gain:.3f}, loss_prop={p_arm_proportion_loss:.3f}" ) - q_arm_cytobands = chr_cytobands[ - chr_cytobands["name"].str.startswith("q", na=False) - ] + q_arm_cytobands = chr_cytobands[chr_cytobands["name"].str.startswith("q", na=False)] q_arm_mean = None q_arm_proportion_gain = 0.0 @@ -228,9 +228,11 @@ def analyze_chromosome_arms( q_arm_values: List[float] = [] for _, band in q_arm_cytobands.iterrows(): start_pos_bin = max(0, int(band["start_pos"] // bin_width)) - end_pos_bin = min(len(cnv_data[chromosome]) - 1, int(band["end_pos"] // bin_width)) + end_pos_bin = min( + len(cnv_data[chromosome]) - 1, int(band["end_pos"] // bin_width) + ) if end_pos_bin >= start_pos_bin: - region_values = cnv_data[chromosome][start_pos_bin:end_pos_bin + 1] + region_values = cnv_data[chromosome][start_pos_bin : end_pos_bin + 1] q_arm_values.extend(region_values) q_arm_values = _finite_arm_values(q_arm_values) if q_arm_values: @@ -287,30 +289,34 @@ def detect_cnv_events( List of CNVEvent objects """ events = [] - - logger.debug(f"Detecting CNV events with sex_estimate='{sex_estimate}', bin_width={bin_width}") - + + logger.debug( + f"Detecting CNV events with sex_estimate='{sex_estimate}', bin_width={bin_width}" + ) + # Check if resolution is sufficient if not is_resolution_sufficient(bin_width): - logger.warning(f"Resolution insufficient for CNV calling: bin_width={bin_width}") + logger.warning( + f"Resolution insufficient for CNV calling: bin_width={bin_width}" + ) return events - + # Analyze each chromosome logger.debug(f"Available chromosomes: {list(cnv_data.keys())}") for chromosome in natsort.natsorted(cnv_data.keys()): if chromosome == "chrM" or not chromosome.startswith("chr"): continue - + # Skip Y chromosome for male samples (expected absence) if chromosome == "chrY" and sex_estimate.upper() in ("XY", "MALE"): logger.debug(f"Skipping {chromosome} for male sample") continue - + logger.debug(f"Analyzing chromosome {chromosome} for CNV events") - + # Get thresholds gain_threshold, loss_threshold = get_cnv_thresholds(chromosome, sex_estimate) - + ( p_arm_mean, q_arm_mean, @@ -350,7 +356,7 @@ def detect_cnv_events( 100 * CNV_MAPPABILITY_MIN_FRACTION, ) q_arm_mean = None - + # Check for whole chromosome events if p_arm_mean is not None and q_arm_mean is not None: logger.debug( @@ -368,7 +374,7 @@ def detect_cnv_events( gain_threshold, loss_threshold, ) - + if is_whole_chr: # Create whole chromosome event chr_cytobands = cytobands_df[cytobands_df["chrom"] == chromosome] @@ -376,19 +382,23 @@ def detect_cnv_events( start_pos = int(chr_cytobands["start_pos"].min()) end_pos = int(chr_cytobands["end_pos"].max()) length = end_pos - start_pos - + # Get genes in chromosome genes = [] if gene_df is not None: - genes = gene_df[gene_df["chrom"] == chromosome]["gene"].astype(str).tolist() - + genes = ( + gene_df[gene_df["chrom"] == chromosome]["gene"] + .astype(str) + .tolist() + ) + chr_vals = cnv_data[chromosome] chr_mean = float(np.nanmean(np.asarray(chr_vals, dtype=float))) if event_type == "GAIN": arm_prop = max(p_arm_proportion_gain, q_arm_proportion_gain) else: arm_prop = max(p_arm_proportion_loss, q_arm_proportion_loss) - + event = CNVEvent( chromosome=chromosome, event_type=f"WHOLE_CHR_{event_type}", @@ -401,7 +411,9 @@ def detect_cnv_events( proportion_affected=arm_prop, ) events.append(event) - logger.info(f"Detected whole chromosome {event_type} for {chromosome}") + logger.info( + f"Detected whole chromosome {event_type} for {chromosome}" + ) elif chromosome != "chrY": # Arm-level events only when no whole-chromosome call on this chromosome @@ -414,22 +426,30 @@ def detect_cnv_events( loss_threshold, ) if is_p_event: - chr_cytobands = cytobands_df[cytobands_df["chrom"] == chromosome] - p_bands = chr_cytobands[chr_cytobands["name"].str.startswith("p", na=False)] + chr_cytobands = cytobands_df[ + cytobands_df["chrom"] == chromosome + ] + p_bands = chr_cytobands[ + chr_cytobands["name"].str.startswith("p", na=False) + ] if not p_bands.empty: start_pos = int(p_bands["start_pos"].min()) end_pos = int(p_bands["end_pos"].max()) length = end_pos - start_pos - + # Get genes in p arm genes = [] if gene_df is not None: - genes = gene_df[ - (gene_df["chrom"] == chromosome) & - (gene_df["start_pos"] <= end_pos) & - (gene_df["end_pos"] >= start_pos) - ]["gene"].astype(str).tolist() - + genes = ( + gene_df[ + (gene_df["chrom"] == chromosome) + & (gene_df["start_pos"] <= end_pos) + & (gene_df["end_pos"] >= start_pos) + ]["gene"] + .astype(str) + .tolist() + ) + p_prop = ( p_arm_proportion_gain if p_event_type == "GAIN" @@ -448,8 +468,10 @@ def detect_cnv_events( proportion_affected=p_prop, ) events.append(event) - logger.info(f"Detected p-arm {p_event_type} for {chromosome}") - + logger.info( + f"Detected p-arm {p_event_type} for {chromosome}" + ) + # Check q arm if q_arm_mean is not None: is_q_event, q_event_type = is_arm_event( @@ -460,22 +482,30 @@ def detect_cnv_events( loss_threshold, ) if is_q_event: - chr_cytobands = cytobands_df[cytobands_df["chrom"] == chromosome] - q_bands = chr_cytobands[chr_cytobands["name"].str.startswith("q", na=False)] + chr_cytobands = cytobands_df[ + cytobands_df["chrom"] == chromosome + ] + q_bands = chr_cytobands[ + chr_cytobands["name"].str.startswith("q", na=False) + ] if not q_bands.empty: start_pos = int(q_bands["start_pos"].min()) end_pos = int(q_bands["end_pos"].max()) length = end_pos - start_pos - + # Get genes in q arm genes = [] if gene_df is not None: - genes = gene_df[ - (gene_df["chrom"] == chromosome) & - (gene_df["start_pos"] <= end_pos) & - (gene_df["end_pos"] >= start_pos) - ]["gene"].astype(str).tolist() - + genes = ( + gene_df[ + (gene_df["chrom"] == chromosome) + & (gene_df["start_pos"] <= end_pos) + & (gene_df["end_pos"] >= start_pos) + ]["gene"] + .astype(str) + .tolist() + ) + q_prop = ( q_arm_proportion_gain if q_event_type == "GAIN" @@ -494,7 +524,9 @@ def detect_cnv_events( proportion_affected=q_prop, ) events.append(event) - logger.info(f"Detected q-arm {q_event_type} for {chromosome}") + logger.info( + f"Detected q-arm {q_event_type} for {chromosome}" + ) else: # Single arm chromosome - use stricter threshold # Only use this logic if we truly have only one arm (like chrY in some cases) @@ -520,11 +552,15 @@ def detect_cnv_events( length = end_pos - start_pos genes = [] if gene_df is not None: - genes = gene_df[ - (gene_df["chrom"] == chromosome) - & (gene_df["start_pos"] <= end_pos) - & (gene_df["end_pos"] >= start_pos) - ]["gene"].astype(str).tolist() + genes = ( + gene_df[ + (gene_df["chrom"] == chromosome) + & (gene_df["start_pos"] <= end_pos) + & (gene_df["end_pos"] >= start_pos) + ]["gene"] + .astype(str) + .tolist() + ) prop = prop_g if evt_type == "GAIN" else prop_l events.append( CNVEvent( @@ -548,8 +584,10 @@ def detect_cnv_events( ) continue - logger.warning(f"{chromosome}: Only one arm detected - this may indicate a cytoband parsing issue") - + logger.warning( + f"{chromosome}: Only one arm detected - this may indicate a cytoband parsing issue" + ) + arr = np.asarray(cnv_data[chromosome], dtype=float) if not np.any(np.isfinite(arr)): continue @@ -557,19 +595,23 @@ def detect_cnv_events( if not np.isfinite(whole_chr_mean): continue single_arm_multiplier = 1.5 # From CNV_EVENT_RULES - + if abs(whole_chr_mean) > abs(gain_threshold) * single_arm_multiplier: chr_cytobands = cytobands_df[cytobands_df["chrom"] == chromosome] if not chr_cytobands.empty: start_pos = int(chr_cytobands["start_pos"].min()) end_pos = int(chr_cytobands["end_pos"].max()) length = end_pos - start_pos - + # Get genes in chromosome genes = [] if gene_df is not None: - genes = gene_df[gene_df["chrom"] == chromosome]["gene"].astype(str).tolist() - + genes = ( + gene_df[gene_df["chrom"] == chromosome]["gene"] + .astype(str) + .tolist() + ) + event_type = "GAIN" if whole_chr_mean > gain_threshold else "LOSS" event = CNVEvent( chromosome=chromosome, @@ -580,21 +622,23 @@ def detect_cnv_events( length=length, genes=genes, confidence="Medium", # Single arm events are less certain - proportion_affected=1.0 # Entire chromosome + proportion_affected=1.0, # Entire chromosome ) events.append(event) - logger.info(f"Detected single-arm whole chromosome {event_type} for {chromosome}") - + logger.info( + f"Detected single-arm whole chromosome {event_type} for {chromosome}" + ) + return events def get_cnv_summary(events: List[CNVEvent]) -> Dict[str, Any]: """ Generate a summary of CNV events. - + Args: events: List of CNVEvent objects - + Returns: Dictionary with summary statistics """ @@ -607,24 +651,24 @@ def get_cnv_summary(events: List[CNVEvent]) -> Dict[str, Any]: "high_confidence_events": 0, "medium_confidence_events": 0, } - + for event in events: if event.event_type.startswith("WHOLE_CHR_"): summary["whole_chromosome_events"].append(event) else: summary["arm_events"].append(event) - + if event.genes: summary["gene_containing_events"].append(event) summary["total_genes_affected"].update(event.genes) - + if event.confidence == "High": summary["high_confidence_events"] += 1 elif event.confidence == "Medium": summary["medium_confidence_events"] += 1 - + summary["total_genes_affected"] = len(summary["total_genes_affected"]) - + return summary diff --git a/src/robin/analysis/cnv_regional.py b/src/robin/analysis/cnv_regional.py index 601090a8..1ec5f9b3 100644 --- a/src/robin/analysis/cnv_regional.py +++ b/src/robin/analysis/cnv_regional.py @@ -36,7 +36,10 @@ def load_panel_gene_bed(output_dir: str) -> tuple[str | None, pd.DataFrame]: master_df = pd.read_csv(master_csv) if not master_df.empty and "analysis_panel" in master_df.columns: panel_val = master_df.iloc[0]["analysis_panel"] - if panel_val is not None and str(panel_val).strip().lower() not in ("", "nan"): + if panel_val is not None and str(panel_val).strip().lower() not in ( + "", + "nan", + ): panel = str(panel_val).strip() except Exception as exc: logger.debug("Could not read analysis panel from master.csv: %s", exc) @@ -50,7 +53,9 @@ def load_panel_gene_bed(output_dir: str) -> tuple[str | None, pd.DataFrame]: panel_bed_filename(panel), ) if not os.path.exists(bed_path): - logger.warning("Target panel BED not found for panel '%s' at %s", panel, bed_path) + logger.warning( + "Target panel BED not found for panel '%s' at %s", panel, bed_path + ) return panel, empty return panel, pd.read_csv( @@ -190,7 +195,10 @@ def build_regional_cnv_events( "mean_cnv": float(row["mean_cnv"]), "state": str(row["cnv_state"]), "panel_genes": panel_genes_in_region( - panel_genes_df, chrom, start_pos, end_pos, + panel_genes_df, + chrom, + start_pos, + end_pos, ), } ) @@ -346,9 +354,7 @@ def analyze_cytoband_cnv( current_group["name"] = ( f"{current_group['chrom']} {current_group['bands'][0]}-{current_group['bands'][-1]}" ) - finite_means = [ - v for v in current_group["mean_cnv"] if np.isfinite(v) - ] + finite_means = [v for v in current_group["mean_cnv"] if np.isfinite(v)] current_group["mean_cnv"] = ( float(np.mean(finite_means)) if finite_means else float("nan") ) @@ -388,9 +394,7 @@ def analyze_cytoband_cnv( current_group["mean_cnv"] = ( float(np.mean(finite_means)) if finite_means else float("nan") ) - current_group["length"] = ( - current_group["end_pos"] - current_group["start_pos"] - ) + current_group["length"] = current_group["end_pos"] - current_group["start_pos"] if current_group["cnv_state"] not in ("NORMAL", "NO_DATA"): merged_cytobands[merged_idx] = current_group diff --git a/src/robin/analysis/cnv_subprocess.py b/src/robin/analysis/cnv_subprocess.py index a7559229..aa4b3c66 100644 --- a/src/robin/analysis/cnv_subprocess.py +++ b/src/robin/analysis/cnv_subprocess.py @@ -4,12 +4,12 @@ This isolates the cnv_from_bam module from the main process to prevent signal handling issues. """ -import sys -import os +import argparse import json -import pickle import logging -import argparse +import os +import pickle +import sys # Import cnv_from_bam only in this subprocess import cnv_from_bam @@ -42,7 +42,7 @@ def run_cnv_analysis( Dictionary with analysis results """ import time - + if os.environ.get("LJ_CNV_SUBPROCESS_DEBUG") == "1": print("CNV Subprocess started") print(f"BAM path: {bam_path}") @@ -72,17 +72,25 @@ def run_cnv_analysis( # Per-sample file approach (optimized) with open(copy_numbers_path, "rb") as f: copy_numbers = pickle.load(f) - print(f"Loaded per-sample copy_numbers from {copy_numbers_path} in {time.time() - load_start:.3f}s", file=sys.stderr) + print( + f"Loaded per-sample copy_numbers from {copy_numbers_path} in {time.time() - load_start:.3f}s", + file=sys.stderr, + ) elif update_cnv_dict_path is not None and sample_id is not None: # Legacy multi-sample dict approach (deprecated but supported for backward compat) if not os.path.exists(update_cnv_dict_path): copy_numbers = {} - print(f"No existing copy_numbers found, starting fresh", file=sys.stderr) + print( + f"No existing copy_numbers found, starting fresh", file=sys.stderr + ) else: with open(update_cnv_dict_path, "rb") as f: multi_sample_dict = pickle.load(f) copy_numbers = multi_sample_dict.get(sample_id, {}) - print(f"Loaded copy_numbers from legacy multi-sample dict in {time.time() - load_start:.3f}s", file=sys.stderr) + print( + f"Loaded copy_numbers from legacy multi-sample dict in {time.time() - load_start:.3f}s", + file=sys.stderr, + ) else: # No existing data copy_numbers = {} @@ -92,10 +100,16 @@ def run_cnv_analysis( ref_load_start = time.time() with open(ref_cnv_dict_path, "rb") as f: ref_cnv_dict = pickle.load(f) - print(f"Loaded reference CNV dict in {time.time() - ref_load_start:.3f}s", file=sys.stderr) + print( + f"Loaded reference CNV dict in {time.time() - ref_load_start:.3f}s", + file=sys.stderr, + ) # First pass: process sample with accumulated copy numbers - print(f"Starting Pass 1: Sample CNV extraction with {threads} threads", file=sys.stderr) + print( + f"Starting Pass 1: Sample CNV extraction with {threads} threads", + file=sys.stderr, + ) pass1_start = time.time() result = cnv_from_bam.iterate_bam_file( bam_path, @@ -105,7 +119,10 @@ def run_cnv_analysis( log_level=int(logging.ERROR), ) pass1_time = time.time() - pass1_start - print(f"Pass 1 completed in {pass1_time:.2f}s (bin_width: {result.bin_width}, variance: {result.variance:.6f})", file=sys.stderr) + print( + f"Pass 1 completed in {pass1_time:.2f}s (bin_width: {result.bin_width}, variance: {result.variance:.6f})", + file=sys.stderr, + ) # Uncontaminated reference: rebin control counts only (no sample BAM). print( @@ -118,7 +135,10 @@ def run_cnv_analysis( r2_cnv = build_reference_cnv_from_counts(ref_cnv_dict, int(result.bin_width)) pass2_time = time.time() - pass2_start print(f"Pass 2 completed in {pass2_time:.2f}s", file=sys.stderr) - print(f"Total CNV extraction time: {pass1_time + pass2_time:.2f}s", file=sys.stderr) + print( + f"Total CNV extraction time: {pass1_time + pass2_time:.2f}s", + file=sys.stderr, + ) # Prepare results analysis_results = { @@ -133,7 +153,7 @@ def run_cnv_analysis( "pass1_time": pass1_time, "pass2_time": pass2_time, "total_time": pass1_time + pass2_time, - } + }, } # Save results to output directory diff --git a/src/robin/analysis/fusion_analysis.py b/src/robin/analysis/fusion_analysis.py index b0517735..6b11675b 100644 --- a/src/robin/analysis/fusion_analysis.py +++ b/src/robin/analysis/fusion_analysis.py @@ -11,6 +11,7 @@ - Standalone file processing (process_single_file) - Metadata management (FusionMetadata) """ + from __future__ import annotations import os @@ -18,40 +19,41 @@ if sys.version_info < (3, 12): raise RuntimeError("robin fusion analysis requires Python 3.12 or newer") -import tempfile +import json import logging +import tempfile import time -import json -from typing import Dict, Any, Optional, List, Tuple, Set from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Set, Tuple + import numpy as np import pandas as pd import pysam -from robin.logging_config import get_job_logger # Import core fusion detection logic from fusion_work.py from robin.analysis.fusion_work import ( - process_bam_file, - _generate_output_files, FusionMetadata, GeneRegion, - _setup_file_paths, - _load_bed_regions, _ensure_gene_regions_loaded, - has_supplementary_alignments, - find_reads_with_supplementary, - _find_gene_intersections, - _process_reads_for_fusions, - _optimize_fusion_dataframe, _filter_fusion_candidates, - process_bam_for_fusions_work, - _merge_fusion_metadata_objects, + _find_gene_intersections, + _generate_output_files, + _load_bed_regions, _load_fusion_metadata, - preprocess_fusion_data_standalone, - process_bam_with_staging, + _merge_fusion_metadata_objects, + _optimize_fusion_dataframe, + _process_reads_for_fusions, + _setup_file_paths, accumulate_fusion_candidates, finalize_fusion_accumulation_for_sample, + find_reads_with_supplementary, + has_supplementary_alignments, + preprocess_fusion_data_standalone, + process_bam_file, + process_bam_for_fusions_work, + process_bam_with_staging, ) +from robin.logging_config import get_job_logger logger = logging.getLogger(__name__) @@ -109,7 +111,9 @@ def _load_supplementary_read_ids( return [] expected_count = metadata.get("supplementary_read_ids_count") - if expected_count is not None and len(supplementary_read_ids) != int(expected_count): + if expected_count is not None and len(supplementary_read_ids) != int( + expected_count + ): log.warning( "Supplementary-read ID count mismatch for %s: expected %s, found %s", supp_ids_path, @@ -296,11 +300,16 @@ def process_single_file( reference = os.path.expanduser(reference) except Exception: pass - + # Generate output files using fusion_work.py # For single file processing, don't generate master BED (it should be done at batch end) output_files = _generate_output_files( - sample_id, analysis_results, fusion_metadata, work_dir, reference=reference, generate_master_bed=False + sample_id, + analysis_results, + fusion_metadata, + work_dir, + reference=reference, + generate_master_bed=False, ) # Update metadata with results (now includes merged data) @@ -368,10 +377,12 @@ def process_single_file( } -def process_multiple_files(bam_paths, metadata_list, work_dir, logger, target_panel=None, reference=None): +def process_multiple_files( + bam_paths, metadata_list, work_dir, logger, target_panel=None, reference=None +): """ Process multiple BAM files for fusion analysis using staged processing. - + This function processes multiple BAM files for the same sample using the existing staging infrastructure. Each file is processed individually and staged, then all staged files are accumulated in a single batch operation. @@ -390,19 +401,19 @@ def process_multiple_files(bam_paths, metadata_list, work_dir, logger, target_pa """ if not bam_paths or not metadata_list: raise ValueError("bam_paths and metadata_list must not be empty") - + if len(bam_paths) != len(metadata_list): raise ValueError("bam_paths and metadata_list must have the same length") - + # Get sample ID from first metadata (assuming all BAMs are from same sample) sample_id = metadata_list[0].get("sample_id", "unknown") - + logger.info(f"🔗 Starting multi-file fusion analysis for sample: {sample_id}") logger.info(f"Processing {len(bam_paths)} BAM files for sample {sample_id}") - + # Log essential metadata only (if debug logging is enabled) # JobLogger wraps a standard logger, access it via .logger attribute - if hasattr(logger, 'logger') and logger.logger.isEnabledFor(logging.DEBUG): + if hasattr(logger, "logger") and logger.logger.isEnabledFor(logging.DEBUG): for i, (bam_path, metadata) in enumerate(zip(bam_paths, metadata_list)): logger.debug(f"BAM file {i+1}: {os.path.basename(bam_path)}") @@ -424,52 +435,64 @@ def process_multiple_files(bam_paths, metadata_list, work_dir, logger, target_pa # Filter files that have supplementary reads valid_bam_paths = [] valid_metadata_list = [] - + for bam_path, metadata in zip(bam_paths, metadata_list): has_supplementary = metadata.get("has_supplementary_reads", False) if has_supplementary: valid_bam_paths.append(bam_path) valid_metadata_list.append(metadata) analysis_result["files_with_supplementary"] += 1 - logger.debug(f"BAM {os.path.basename(bam_path)}: has supplementary reads") + logger.debug( + f"BAM {os.path.basename(bam_path)}: has supplementary reads" + ) else: - logger.debug(f"BAM {os.path.basename(bam_path)}: no supplementary reads - skipping") - + logger.debug( + f"BAM {os.path.basename(bam_path)}: no supplementary reads - skipping" + ) + if not valid_bam_paths: logger.info(f"No BAM files with supplementary reads found for {sample_id}") - analysis_result["error_message"] = "No supplementary reads found in any BAM files" + analysis_result["error_message"] = ( + "No supplementary reads found in any BAM files" + ) analysis_result["processing_steps"].append("no_supplementary_reads") return analysis_result - logger.info(f"Processing {len(valid_bam_paths)} valid BAM files out of {len(bam_paths)} total") + logger.info( + f"Processing {len(valid_bam_paths)} valid BAM files out of {len(bam_paths)} total" + ) analysis_result["files_processed"] = len(valid_bam_paths) analysis_result["processing_steps"].append("supplementary_reads_found") # Pre-initialize batch-level resources (done once per batch, not per file) # 1. Ensure gene regions are loaded (cached, but ensure it's done once) from robin.analysis.fusion_work import _ensure_gene_regions_loaded + _ensure_gene_regions_loaded(target_panel) - + # 2. Pre-create staging directory (avoids repeated os.makedirs calls) from robin.analysis.fusion_work import _get_staging_dir + staging_dir = _get_staging_dir(work_dir, sample_id) logger.debug(f"Pre-created staging directory: {staging_dir}") - + # Process each BAM file individually using staging logger.info("Processing files with fusion staging (fast path)") processed_files = 0 staging_batch_size = _get_fusion_batch_size() - - for i, (bam_path, metadata) in enumerate(zip(valid_bam_paths, valid_metadata_list)): - logger.info(f"Processing BAM file {i+1}/{len(valid_bam_paths)}: {os.path.basename(bam_path)}") - + + for i, (bam_path, metadata) in enumerate( + zip(valid_bam_paths, valid_metadata_list) + ): + logger.info( + f"Processing BAM file {i+1}/{len(valid_bam_paths)}: {os.path.basename(bam_path)}" + ) + try: # Get supplementary read information has_supplementary = metadata.get("has_supplementary_reads", False) - supplementary_read_ids = _load_supplementary_read_ids( - metadata, logger - ) - + supplementary_read_ids = _load_supplementary_read_ids(metadata, logger) + # Create fusion metadata fusion_metadata = FusionMetadata( sample_id=sample_id, @@ -477,7 +500,7 @@ def process_multiple_files(bam_paths, metadata_list, work_dir, logger, target_pa analysis_timestamp=time.time(), target_panel=target_panel, ) - + # Create temporary directory for processing with tempfile.TemporaryDirectory() as temp_dir: # Process with staging @@ -493,20 +516,26 @@ def process_multiple_files(bam_paths, metadata_list, work_dir, logger, target_pa work_dir=work_dir, batch_size=staging_batch_size, ) - + if analysis_results.get("error_message"): - logger.warning(f"Error processing {os.path.basename(bam_path)}: {analysis_results['error_message']}") + logger.warning( + f"Error processing {os.path.basename(bam_path)}: {analysis_results['error_message']}" + ) continue - + processed_files += 1 - logger.debug(f"Successfully staged file {i+1}: {os.path.basename(bam_path)}") - + logger.debug( + f"Successfully staged file {i+1}: {os.path.basename(bam_path)}" + ) + except Exception as e: logger.warning(f"Error processing {os.path.basename(bam_path)}: {e}") continue if processed_files == 0: - analysis_result["error_message"] = "No files could be processed successfully" + analysis_result["error_message"] = ( + "No files could be processed successfully" + ) analysis_result["processing_steps"].append("no_files_processed") return analysis_result @@ -517,23 +546,32 @@ def process_multiple_files(bam_paths, metadata_list, work_dir, logger, target_pa # Expand reference path if provided if reference: reference = os.path.expanduser(reference) - - logger.info(f"Accumulating {processed_files} staged fusion files for sample {sample_id}") + + logger.info( + f"Accumulating {processed_files} staged fusion files for sample {sample_id}" + ) accumulation_result = accumulate_fusion_candidates( - work_dir, sample_id, target_panel, force=True, batch_size=1, reference=reference + work_dir, + sample_id, + target_panel, + force=True, + batch_size=1, + reference=reference, ) - + if accumulation_result.get("status") != "success": - analysis_result["error_message"] = f"Accumulation failed: {accumulation_result.get('error', 'Unknown error')}" + analysis_result["error_message"] = ( + f"Accumulation failed: {accumulation_result.get('error', 'Unknown error')}" + ) analysis_result["processing_steps"].append("accumulation_failed") return analysis_result analysis_result["processing_steps"].append("accumulation_complete") logger.info(f"Fusion accumulation completed: {accumulation_result}") - + # Load final accumulated data for result metadata sample_output_dir = os.path.join(work_dir, sample_id) - + # Set output file paths analysis_result["target_fusion_path"] = os.path.join( sample_output_dir, "fusion_candidates_master.csv" @@ -541,7 +579,7 @@ def process_multiple_files(bam_paths, metadata_list, work_dir, logger, target_pa analysis_result["genome_wide_fusion_path"] = os.path.join( sample_output_dir, "fusion_candidates_all.csv" ) - + # Store final results analysis_result["fusion_data"] = { "target_candidates_count": accumulation_result.get( @@ -555,14 +593,22 @@ def process_multiple_files(bam_paths, metadata_list, work_dir, logger, target_pa "files_with_supplementary": analysis_result["files_with_supplementary"], "files_processed": processed_files, } - + analysis_result["processing_steps"].append("analysis_complete") logger.info(f"Multi-file fusion analysis completed for {sample_id}") - logger.info(f"Files successfully processed: {analysis_result['files_processed']}/{analysis_result['total_files']}") - logger.info(f"Files with supplementary reads: {analysis_result['files_with_supplementary']}") - logger.info(f"Target fusion candidates: {analysis_result['fusion_data']['target_candidates_count']}") - logger.info(f"Genome-wide fusion candidates: {analysis_result['fusion_data']['genome_wide_candidates_count']}") - + logger.info( + f"Files successfully processed: {analysis_result['files_processed']}/{analysis_result['total_files']}" + ) + logger.info( + f"Files with supplementary reads: {analysis_result['files_with_supplementary']}" + ) + logger.info( + f"Target fusion candidates: {analysis_result['fusion_data']['target_candidates_count']}" + ) + logger.info( + f"Genome-wide fusion candidates: {analysis_result['fusion_data']['genome_wide_candidates_count']}" + ) + return analysis_result except Exception as e: @@ -584,44 +630,48 @@ def fusion_handler(job, work_dir=None, target_panel=None): # Validate required parameters if not target_panel: raise ValueError("target_panel is required for fusion analysis") - + try: # Get logger with proper parameters logger = get_job_logger(str(job.job_id), "fusion", job.context.filepath) - + # Check if this is a batched job batched_job = job.context.metadata.get("_batched_job") if batched_job: batch_size = batched_job.get_file_count() sample_id = batched_job.get_sample_id() batch_id = batched_job.batch_id - logger.info(f"Processing fusion analysis batch: {batch_size} files for sample '{sample_id}' (batch_id: {batch_id})") - + logger.info( + f"Processing fusion analysis batch: {batch_size} files for sample '{sample_id}' (batch_id: {batch_id})" + ) + # Get all filepaths in the batch filepaths = batched_job.get_filepaths() - + # Log individual files in the batch for i, filepath in enumerate(filepaths): - logger.info(f" Batch file {i+1}/{batch_size}: {os.path.basename(filepath)}") - + logger.info( + f" Batch file {i+1}/{batch_size}: {os.path.basename(filepath)}" + ) + # Prepare metadata list for all BAM files in the batch metadata_list = [] for i, bam_path in enumerate(filepaths): # Get metadata from preprocessing for this specific file file_metadata = batched_job.contexts[i].metadata.get("bam_metadata", {}) - + # Get sample ID from preprocessing results for this specific file file_context = batched_job.contexts[i] file_sample_id = file_context.get_sample_id() - + # Use the sample ID from the file's context (which should have preprocessing results) if file_sample_id != "unknown": file_metadata["sample_id"] = file_sample_id else: file_metadata["sample_id"] = sample_id - + metadata_list.append(file_metadata) - + # Determine work directory for the batch if work_dir is None: # Default to first BAM file directory @@ -631,69 +681,100 @@ def fusion_handler(job, work_dir=None, target_panel=None): os.makedirs(work_dir, exist_ok=True) batch_work_dir = work_dir logger.debug(f"Using specified work directory: {batch_work_dir}") - + # Log and validate target panel - job_panel = batched_job.contexts[0].metadata.get("target_panel", target_panel) + job_panel = batched_job.contexts[0].metadata.get( + "target_panel", target_panel + ) if job_panel != target_panel: - logger.warning(f"Panel mismatch: job metadata has '{job_panel}' but handler received '{target_panel}'. Using '{job_panel}' from metadata.") + logger.warning( + f"Panel mismatch: job metadata has '{job_panel}' but handler received '{target_panel}'. Using '{job_panel}' from metadata." + ) target_panel = job_panel logger.info(f"Using target panel: {target_panel}") - + # Get reference from job metadata reference = job.context.metadata.get("reference") if reference: reference = os.path.expanduser(reference) logger.debug(f"Using reference genome from job metadata: {reference}") - + # Process all BAM files in the batch using the new aggregated function - logger.info(f"Processing {batch_size} BAM files as aggregated batch for sample '{sample_id}'") + logger.info( + f"Processing {batch_size} BAM files as aggregated batch for sample '{sample_id}'" + ) batch_result = process_multiple_files( bam_paths=filepaths, metadata_list=metadata_list, work_dir=batch_work_dir, logger=logger, target_panel=target_panel, - reference=reference + reference=reference, ) - + # Store batch results in job context (maintain compatibility with existing structure) - job.context.add_metadata("fusion_analysis", { - "batch_result": batch_result, # Single aggregated result - "batch_size": batch_size, - "sample_id": sample_id, - "batch_id": batch_id, - "files_processed": batch_result.get("files_processed", batch_size), - "total_files": batch_result.get("total_files", batch_size) - }) - - logger.info(f"Completed fusion analysis batch processing: {batch_size} files for sample '{sample_id}'") - logger.info(f"Files successfully processed: {batch_result.get('files_processed', batch_size)}/{batch_result.get('total_files', batch_size)}") - logger.info(f"Files with supplementary reads: {batch_result.get('files_with_supplementary', 0)}") - + job.context.add_metadata( + "fusion_analysis", + { + "batch_result": batch_result, # Single aggregated result + "batch_size": batch_size, + "sample_id": sample_id, + "batch_id": batch_id, + "files_processed": batch_result.get("files_processed", batch_size), + "total_files": batch_result.get("total_files", batch_size), + }, + ) + + logger.info( + f"Completed fusion analysis batch processing: {batch_size} files for sample '{sample_id}'" + ) + logger.info( + f"Files successfully processed: {batch_result.get('files_processed', batch_size)}/{batch_result.get('total_files', batch_size)}" + ) + logger.info( + f"Files with supplementary reads: {batch_result.get('files_with_supplementary', 0)}" + ) + if batch_result.get("error_message"): - logger.error(f"Batch processing completed with errors: {batch_result['error_message']}") + logger.error( + f"Batch processing completed with errors: {batch_result['error_message']}" + ) job.context.add_error("fusion_analysis", batch_result["error_message"]) else: - logger.info("Batch processing completed successfully with aggregated fusion analysis") + logger.info( + "Batch processing completed successfully with aggregated fusion analysis" + ) job.context.add_result( "fusion_analysis", { "success": True, "sample_id": sample_id, "analysis_time": batch_result.get("analysis_timestamp", 0), - "target_candidates_count": batch_result.get("fusion_data", {}).get("target_candidates_count", 0), - "genome_wide_candidates_count": batch_result.get("fusion_data", {}).get("genome_wide_candidates_count", 0), + "target_candidates_count": batch_result.get( + "fusion_data", {} + ).get("target_candidates_count", 0), + "genome_wide_candidates_count": batch_result.get( + "fusion_data", {} + ).get("genome_wide_candidates_count", 0), "processing_steps": batch_result.get("processing_steps", []), - "target_fusion_path": batch_result.get("target_fusion_path", ""), - "genome_wide_fusion_path": batch_result.get("genome_wide_fusion_path", ""), - "files_processed": batch_result.get("files_processed", batch_size), + "target_fusion_path": batch_result.get( + "target_fusion_path", "" + ), + "genome_wide_fusion_path": batch_result.get( + "genome_wide_fusion_path", "" + ), + "files_processed": batch_result.get( + "files_processed", batch_size + ), "total_files": batch_result.get("total_files", batch_size), - "files_with_supplementary": batch_result.get("files_with_supplementary", 0), + "files_with_supplementary": batch_result.get( + "files_with_supplementary", 0 + ), }, ) - + return - + else: # Single file processing (backward compatibility) # Extract file path and metadata from job @@ -703,7 +784,9 @@ def fusion_handler(job, work_dir=None, target_panel=None): # Log and validate target panel job_panel = job.context.metadata.get("target_panel", target_panel) if job_panel != target_panel: - logger.warning(f"Panel mismatch: job metadata has '{job_panel}' but handler received '{target_panel}'. Using '{job_panel}' from metadata.") + logger.warning( + f"Panel mismatch: job metadata has '{job_panel}' but handler received '{target_panel}'. Using '{job_panel}' from metadata." + ) target_panel = job_panel logger.info(f"Using target panel: {target_panel}") logger.info(f"DEBUG: Job metadata: {job.context.metadata}") @@ -714,9 +797,7 @@ def fusion_handler(job, work_dir=None, target_panel=None): logger.info(f"Starting fusion analysis for {file_path}") logger.info(f"Metadata: {metadata}") - supplementary_read_ids = _load_supplementary_read_ids( - metadata, logger - ) + supplementary_read_ids = _load_supplementary_read_ids(metadata, logger) # Set default work directory if not provided if work_dir is None: @@ -728,7 +809,7 @@ def fusion_handler(job, work_dir=None, target_panel=None): # Use staging-based processing for performance logger.info("Using fusion staging-based processing (fast path)") staging_batch_size = _get_fusion_batch_size() - + # Create fusion metadata sample_id = metadata.get("sample_id", "unknown") fusion_metadata = FusionMetadata( @@ -737,9 +818,10 @@ def fusion_handler(job, work_dir=None, target_panel=None): analysis_timestamp=time.time(), target_panel=target_panel, ) - + # Create temporary directory for processing import tempfile + with tempfile.TemporaryDirectory() as temp_dir: # Process with staging analysis_results, should_accumulate = process_bam_with_staging( @@ -753,7 +835,7 @@ def fusion_handler(job, work_dir=None, target_panel=None): work_dir=work_dir, batch_size=staging_batch_size, ) - + # Convert to result format result = { "success": True, @@ -764,34 +846,49 @@ def fusion_handler(job, work_dir=None, target_panel=None): "error_message": None, "analysis_results": analysis_results, } - + # Add result to job context job.context.add_result("fusion_analysis", result) - + # Trigger accumulation if threshold reached # The accumulation function re-checks the staging threshold before proceeding. if should_accumulate: - logger.info("Fusion accumulation threshold reached - attempting batch accumulation") + logger.info( + "Fusion accumulation threshold reached - attempting batch accumulation" + ) # Get reference from job metadata if available reference = job.context.metadata.get("reference") if reference: reference = os.path.expanduser(reference) - logger.debug(f"Using reference genome from job metadata: {reference}") - + logger.debug( + f"Using reference genome from job metadata: {reference}" + ) + accumulation_result = accumulate_fusion_candidates( - work_dir, sample_id, target_panel, force=False, batch_size=staging_batch_size, reference=reference + work_dir, + sample_id, + target_panel, + force=False, + batch_size=staging_batch_size, + reference=reference, ) if accumulation_result.get("status") == "below_threshold": logger.debug("Accumulation skipped - below threshold") else: - logger.info(f"Fusion accumulation result: {accumulation_result}") - job.context.add_metadata("fusion_accumulation_result", accumulation_result) - + logger.info( + f"Fusion accumulation result: {accumulation_result}" + ) + job.context.add_metadata( + "fusion_accumulation_result", accumulation_result + ) + # Store flag for potential end-of-queue accumulation job.context.add_metadata("needs_final_fusion_accumulation", True) if result["success"]: - logger.info(f"Fusion analysis completed successfully for {file_path}") + logger.info( + f"Fusion analysis completed successfully for {file_path}" + ) logger.info(f"Results: {result}") else: error_msg = result.get("error_message", "Unknown error") @@ -838,47 +935,48 @@ def fusion_handler(job, work_dir=None, target_panel=None): def _get_available_panels() -> List[str]: """Get list of available panels from resources directory.""" panels = ["rCNS2", "AML"] # Built-in panels - + try: from pathlib import Path + # Look for the resources directory relative to this file current_file = Path(__file__) resources_dir = current_file.parent.parent.parent / "robin" / "resources" - + if resources_dir.exists(): # Look for custom panels (files ending with _panel_name_uniq.bed) for bed_file in resources_dir.glob("*_panel_name_uniq.bed"): panel_name = bed_file.stem.replace("_panel_name_uniq", "") if panel_name not in panels: panels.append(panel_name) - + panels.sort() - + except Exception: # Fallback to built-in panels only pass - + return panels if __name__ == "__main__": """ Standalone CLI for fusion analysis testing. - + Usage: # Analyze a single BAM file python -m robin.analysis.fusion_analysis file.bam --target-panel rCNS2 --work-dir output/ - + # Analyze all BAM files in a folder python -m robin.analysis.fusion_analysis /path/to/bam/folder --target-panel rCNS2 --work-dir output/ - + # Analyze specific BAM files python -m robin.analysis.fusion_analysis file1.bam file2.bam --target-panel rCNS2 --work-dir output/ """ import argparse import glob from pathlib import Path - + parser = argparse.ArgumentParser( description="Standalone fusion analysis for BAM files", formatter_class=argparse.RawDescriptionHelpFormatter, @@ -895,60 +993,55 @@ def _get_available_panels() -> List[str]: # With reference genome python -m robin.analysis.fusion_analysis folder/ --target-panel rCNS2 --work-dir output/ --reference /path/to/reference.fa - """ + """, ) - + parser.add_argument( - "input", - nargs="+", - help="BAM file(s) or folder containing BAM files to analyze" + "input", nargs="+", help="BAM file(s) or folder containing BAM files to analyze" ) - + parser.add_argument( "--target-panel", type=str, choices=_get_available_panels(), required=True, - help=f"Target gene panel for fusion analysis. Available: {', '.join(_get_available_panels())}" + help=f"Target gene panel for fusion analysis. Available: {', '.join(_get_available_panels())}", ) - + parser.add_argument( "--work-dir", type=str, default="fusion_output", - help="Working directory for output files (default: fusion_output)" + help="Working directory for output files (default: fusion_output)", ) - + parser.add_argument( "--reference", type=str, - help="Path to reference genome FASTA file (optional, for master BED generation)" + help="Path to reference genome FASTA file (optional, for master BED generation)", ) - + parser.add_argument( "--sample-id", type=str, - help="Sample ID to use (default: auto-detect from first BAM file name)" + help="Sample ID to use (default: auto-detect from first BAM file name)", ) - + parser.add_argument( - "--verbose", - "-v", - action="store_true", - help="Enable verbose logging" + "--verbose", "-v", action="store_true", help="Enable verbose logging" ) - + args = parser.parse_args() - + # Set up logging log_level = logging.DEBUG if args.verbose else logging.INFO logging.basicConfig( level=log_level, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', - datefmt='%Y-%m-%d %H:%M:%S' + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", ) logger = logging.getLogger("fusion_analysis_standalone") - + # Collect BAM files bam_files = [] for input_path in args.input: @@ -963,13 +1056,13 @@ def _get_available_panels() -> List[str]: bam_files.extend(glob.glob(str(path / "*.bam"))) else: logger.warning(f"Path does not exist: {input_path}") - + if not bam_files: logger.error("No BAM files found to process!") sys.exit(1) - + logger.info(f"Found {len(bam_files)} BAM file(s) to process") - + # Expand reference path if provided reference = None if args.reference: @@ -977,34 +1070,38 @@ def _get_available_panels() -> List[str]: if not os.path.exists(reference): logger.error(f"Reference genome file not found: {reference}") sys.exit(1) - + # Create work directory work_dir = os.path.expanduser(args.work_dir) os.makedirs(work_dir, exist_ok=True) logger.info(f"Using work directory: {work_dir}") - + # Prepare metadata for each BAM file metadata_list = [] sample_id = args.sample_id - + for bam_path in bam_files: # Check for supplementary reads logger.info(f"Checking {os.path.basename(bam_path)} for supplementary reads...") has_supplementary = has_supplementary_alignments(bam_path) - + if not has_supplementary: - logger.warning(f"No supplementary reads found in {os.path.basename(bam_path)} - will be skipped") - + logger.warning( + f"No supplementary reads found in {os.path.basename(bam_path)} - will be skipped" + ) + # Get supplementary read IDs supplementary_read_ids = [] if has_supplementary: try: supplementary_read_ids = list(find_reads_with_supplementary(bam_path)) - logger.info(f"Found {len(supplementary_read_ids)} reads with supplementary alignments") + logger.info( + f"Found {len(supplementary_read_ids)} reads with supplementary alignments" + ) except Exception as e: logger.warning(f"Error finding supplementary reads: {e}") has_supplementary = False - + # Auto-detect sample ID from first file if not provided if sample_id is None: # Extract sample ID from filename (remove .bam extension and common prefixes) @@ -1016,7 +1113,7 @@ def _get_available_panels() -> List[str]: sample_id = sample_id.removeprefix(prefix) break logger.info(f"Auto-detected sample ID: {sample_id}") - + metadata = { "sample_id": sample_id, "has_supplementary_reads": has_supplementary, @@ -1026,11 +1123,11 @@ def _get_available_panels() -> List[str]: "file_path": bam_path, } metadata_list.append(metadata) - + # Process all files logger.info(f"Starting fusion analysis for sample: {sample_id}") logger.info(f"Target panel: {args.target_panel}") - + try: result = process_multiple_files( bam_paths=bam_files, @@ -1038,29 +1135,37 @@ def _get_available_panels() -> List[str]: work_dir=work_dir, logger=logger, target_panel=args.target_panel, - reference=reference + reference=reference, ) - + if result.get("error_message"): logger.error(f"Analysis failed: {result['error_message']}") sys.exit(1) - + # Print summary - print("\n" + "="*60) + print("\n" + "=" * 60) print("FUSION ANALYSIS COMPLETE") - print("="*60) + print("=" * 60) print(f"Sample ID: {result.get('sample_id', 'unknown')}") - print(f"Files processed: {result.get('files_processed', 0)}/{result.get('total_files', 0)}") - print(f"Files with supplementary reads: {result.get('files_with_supplementary', 0)}") - print(f"Target fusion candidates: {result.get('fusion_data', {}).get('target_candidates_count', 0)}") - print(f"Genome-wide fusion candidates: {result.get('fusion_data', {}).get('genome_wide_candidates_count', 0)}") + print( + f"Files processed: {result.get('files_processed', 0)}/{result.get('total_files', 0)}" + ) + print( + f"Files with supplementary reads: {result.get('files_with_supplementary', 0)}" + ) + print( + f"Target fusion candidates: {result.get('fusion_data', {}).get('target_candidates_count', 0)}" + ) + print( + f"Genome-wide fusion candidates: {result.get('fusion_data', {}).get('genome_wide_candidates_count', 0)}" + ) print(f"\nOutput files:") - if result.get('target_fusion_path'): + if result.get("target_fusion_path"): print(f" - Target fusions: {result['target_fusion_path']}") - if result.get('genome_wide_fusion_path'): + if result.get("genome_wide_fusion_path"): print(f" - Genome-wide fusions: {result['genome_wide_fusion_path']}") - print("="*60) - + print("=" * 60) + except KeyboardInterrupt: logger.info("Analysis interrupted by user") sys.exit(130) diff --git a/src/robin/analysis/fusion_work.py b/src/robin/analysis/fusion_work.py index f267f2b1..5dfae621 100644 --- a/src/robin/analysis/fusion_work.py +++ b/src/robin/analysis/fusion_work.py @@ -11,47 +11,52 @@ 4. True fusions require reads to map to multiple genomic locations 5. Fusions must be supported by a configurable minimum number of reads (default: 3) """ + from __future__ import annotations import sys + if sys.version_info < (3, 12): raise RuntimeError("robin fusion_work requires Python 3.12 or newer") +import bisect +import glob +import json +import logging + # Standard library imports import os -import random -import logging -import json import pickle -import glob -import time -import bisect +import random import re -from datetime import datetime -from pathlib import Path +import shutil +import time from collections import defaultdict +from dataclasses import asdict, dataclass +from datetime import datetime from itertools import combinations -from typing import Dict, Any, Optional, List, Tuple, Set, Union -from dataclasses import dataclass, asdict +from pathlib import Path +from typing import Any, Dict, List, Optional, Set, Tuple, Union + +import networkx as nx # Third-party imports import numpy as np import pandas as pd -import pysam -import networkx as nx -from sklearn.cluster import DBSCAN import pyarrow.parquet as pq -import shutil +import pysam from ncls import NCLS +from sklearn.cluster import DBSCAN + _HAS_NCLS = True # Local imports from robin.analysis.master_bed_generator import FileLock from robin.classification_config import ( - get_fusion_threshold, + are_coordinates_similar, get_fusion_rule, + get_fusion_threshold, validate_fusion_candidate, - are_coordinates_similar ) # FusionMetadata is now defined in this file @@ -62,7 +67,9 @@ # TEMP: Disable master BED interactions for performance testing ENABLE_MASTER_BED = True # Debug flag for incremental master BED breakpoint logging -DEBUG_MASTER_BED_INCREMENTAL = os.getenv("ROBIN_DEBUG_MASTER_BED_INCREMENTAL", "0") == "1" +DEBUG_MASTER_BED_INCREMENTAL = ( + os.getenv("ROBIN_DEBUG_MASTER_BED_INCREMENTAL", "0") == "1" +) # Minimum primary alignment QS (BAM tag "qs") to include a read in fusion analysis MIN_PRIMARY_QS = 12 @@ -77,7 +84,10 @@ def _replace_file_if_changed(temporary_path: str, destination_path: str) -> bool try: if os.path.exists(destination_path): if os.path.getsize(temporary_path) == os.path.getsize(destination_path): - with open(temporary_path, "rb") as new_file, open(destination_path, "rb") as existing_file: + with ( + open(temporary_path, "rb") as new_file, + open(destination_path, "rb") as existing_file, + ): while True: new_chunk = new_file.read(1024 * 1024) existing_chunk = existing_file.read(1024 * 1024) @@ -256,7 +266,9 @@ def overlaps_with(self, other_start: int, other_end: int) -> bool: # Cache for NCLS indexes _gene_region_ncls_cache: Dict[str, Dict[str, Tuple["NCLS", List[GeneRegion]]]] = {} _all_gene_region_ncls_cache: Dict[str, Dict[str, Tuple["NCLS", List[GeneRegion]]]] = {} -_combined_gene_region_ncls_cache: Dict[str, Dict[str, Tuple["NCLS", List[TaggedGeneRegion]]]] = {} +_combined_gene_region_ncls_cache: Dict[ + str, Dict[str, Tuple["NCLS", List[TaggedGeneRegion]]] +] = {} # ============================================================================= # UTILITY FUNCTIONS @@ -489,7 +501,10 @@ def _append_gene_intersections( gene_region = indexed_regions[region_id] overlap_start = max(gene_region.start, ref_start) overlap_end = min(gene_region.end, ref_end) - if overlap_end > overlap_start and (overlap_end - overlap_start) > min_overlap: + if ( + overlap_end > overlap_start + and (overlap_end - overlap_start) > min_overlap + ): _append_fusion_row( data, ref_name, @@ -578,9 +593,7 @@ def _check_read_alignments_overlap_columnar(data: Dict[str, List[Any]]) -> bool: genomic_alignments: Dict[str, List[str]] = {} for i in range(len(reference_ids)): - genomic_key = ( - f"{reference_ids[i]}:{reference_starts[i]}-{reference_ends[i]}" - ) + genomic_key = f"{reference_ids[i]}:{reference_starts[i]}-{reference_ends[i]}" genomic_alignments.setdefault(genomic_key, []).append(gene_names[i]) for genes in genomic_alignments.values(): @@ -645,7 +658,9 @@ def _setup_file_paths(target_panel: str) -> Tuple[str, str]: gene_bed = "AML_panel_name_uniq.bed" else: # Check for custom panel - custom_panel_path = os.path.join(resources_dir, f"{target_panel}_panel_name_uniq.bed") + custom_panel_path = os.path.join( + resources_dir, f"{target_panel}_panel_name_uniq.bed" + ) if os.path.exists(custom_panel_path): gene_bed = custom_panel_path else: @@ -684,7 +699,7 @@ def _load_bed_regions(bed_file: str) -> Dict[str, List[GeneRegion]]: if not os.path.exists(bed_file): logger.warning(f"BED file does not exist: {bed_file}") return dict(regions) - + logger.debug(f"Loading BED file: {bed_file}") try: @@ -722,7 +737,7 @@ def _load_bed_regions(bed_file: str) -> Dict[str, List[GeneRegion]]: # Sort by start position, then by end position for consistency sorted_regions = sorted(region_list, key=lambda r: (r.start, r.end)) result[chrom] = sorted_regions - + total_regions = sum(len(regions) for regions in result.values()) logger.debug(f"Loaded and sorted {total_regions} total regions from {bed_file}") return result @@ -730,19 +745,27 @@ def _load_bed_regions(bed_file: str) -> Dict[str, List[GeneRegion]]: def _ensure_gene_regions_loaded(target_panel: str) -> None: """Ensure gene regions are loaded into cache for the given target panel.""" - logger.debug("_ensure_gene_regions_loaded called with target_panel='%s'", target_panel) + logger.debug( + "_ensure_gene_regions_loaded called with target_panel='%s'", target_panel + ) logger.debug("Current cache keys: %s", list(_gene_regions_cache.keys())) - + if target_panel not in _gene_regions_cache: logger.debug("Loading gene regions for target_panel='%s'", target_panel) gene_bed, all_gene_bed = _setup_file_paths(target_panel) - logger.debug("Resolved gene_bed='%s', all_gene_bed='%s'", gene_bed, all_gene_bed) + logger.debug( + "Resolved gene_bed='%s', all_gene_bed='%s'", gene_bed, all_gene_bed + ) # Load target panel gene regions _gene_regions_cache[target_panel] = _load_bed_regions(gene_bed) - total_regions = sum(len(regions) for regions in _gene_regions_cache[target_panel].values()) - logger.info(f"Loaded {len(_gene_regions_cache[target_panel])} chromosomes, {total_regions} total regions for target panel {target_panel}") - + total_regions = sum( + len(regions) for regions in _gene_regions_cache[target_panel].values() + ) + logger.info( + f"Loaded {len(_gene_regions_cache[target_panel])} chromosomes, {total_regions} total regions for target panel {target_panel}" + ) + # Pre-compute start position lists for binary search optimization _gene_region_starts_cache[target_panel] = { chrom: [region.start for region in regions] @@ -758,9 +781,13 @@ def _ensure_gene_regions_loaded(target_panel: str) -> None: # Load genome-wide gene regions (shared across all panels) if not _all_gene_regions_cache: _all_gene_regions_cache["shared"] = _load_bed_regions(all_gene_bed) - total_genome_regions = sum(len(regions) for regions in _all_gene_regions_cache["shared"].values()) - logger.info(f"Loaded {len(_all_gene_regions_cache['shared'])} chromosomes, {total_genome_regions} total regions for genome-wide genes") - + total_genome_regions = sum( + len(regions) for regions in _all_gene_regions_cache["shared"].values() + ) + logger.info( + f"Loaded {len(_all_gene_regions_cache['shared'])} chromosomes, {total_genome_regions} total regions for genome-wide genes" + ) + # Pre-compute start position lists for binary search optimization _all_gene_region_starts_cache["shared"] = { chrom: [region.start for region in regions] @@ -781,11 +808,15 @@ def _ensure_gene_regions_loaded(target_panel: str) -> None: combined_regions: List[TaggedGeneRegion] = [] for region in target_regions.get(chrom, []): combined_regions.append( - TaggedGeneRegion(region.start, region.end, region.name, "target") + TaggedGeneRegion( + region.start, region.end, region.name, "target" + ) ) for region in genome_regions.get(chrom, []): combined_regions.append( - TaggedGeneRegion(region.start, region.end, region.name, "genome") + TaggedGeneRegion( + region.start, region.end, region.name, "genome" + ) ) if combined_regions: index = _build_tagged_ncls_index(combined_regions) @@ -798,47 +829,47 @@ def _load_master_bed_regions(bed_file: str) -> Dict[str, List[MasterBedRegion]]: """ Load master BED file regions into memory for efficient lookup. Master BED regions don't require gene overlap - breaks can occur anywhere. - + Args: bed_file: Path to master BED file - + Returns: Dictionary mapping chromosome names to lists of MasterBedRegion objects """ regions = defaultdict(list) - + if not os.path.exists(bed_file): logger.debug(f"Master BED file does not exist: {bed_file}") return dict(regions) - + logger.info(f"Loading master BED file: {bed_file}") - + try: with open(bed_file, "r") as f: for line_num, line in enumerate(f, 1): line = line.strip() if not line or line.startswith("#"): continue - + parts = line.split("\t") if len(parts) < 3: continue - + try: chrom = parts[0] start = int(parts[1]) end = int(parts[2]) name = parts[3] if len(parts) > 3 else f"region_{line_num}" - + regions[chrom].append(MasterBedRegion(start, end, name)) except (ValueError, IndexError) as e: logger.debug(f"Skipping invalid line {line_num} in {bed_file}: {e}") continue - + except Exception as e: logger.error(f"Error loading master BED file {bed_file}: {e}") return dict(regions) - + result = dict(regions) total_regions = sum(len(regions) for regions in result.values()) logger.info(f"Loaded {total_regions} total regions from master BED file") @@ -848,11 +879,11 @@ def _load_master_bed_regions(bed_file: str) -> Dict[str, List[MasterBedRegion]]: def _get_master_bed_path(work_dir: str, sample_id: str) -> Optional[str]: """ Get the path to the master BED file for a sample. - + Args: work_dir: Working directory sample_id: Sample ID - + Returns: Path to master BED file, or None if not found """ @@ -866,7 +897,7 @@ def _get_master_bed_path(work_dir: str, sample_id: str) -> Optional[str]: return latest except Exception as e: logger.debug(f"Error finding master BED file: {e}") - + return None @@ -895,25 +926,25 @@ def find_reads_mapping_to_master_bed( ) -> Set[str]: """ Find all read names that map to regions in the master BED file. - + Args: bamfile: Path to BAM file master_bed_regions: Dictionary of master BED regions by chromosome - + Returns: Set of read names that map to master BED regions """ reads_mapping_to_master = set() - + if not master_bed_regions: return reads_mapping_to_master - + try: with pysam.AlignmentFile(bamfile, "rb") as bam: for read in bam: if read.is_unmapped: continue - + # Get reference information ref_name = ( bam.get_reference_name(read.reference_id) @@ -922,7 +953,7 @@ def find_reads_mapping_to_master_bed( ) if not ref_name or ref_name == "chrM": continue - + # Only consider primary alignments with QS >= MIN_PRIMARY_QS if not _primary_meets_min_qs(read): continue @@ -931,17 +962,19 @@ def find_reads_mapping_to_master_bed( if ref_name in master_bed_regions: ref_start = read.reference_start ref_end = read.reference_end - + for region in master_bed_regions[ref_name]: if region.overlaps_with(ref_start, ref_end): reads_mapping_to_master.add(read.query_name) break # No need to check other regions for this read - + except Exception as e: logger.error(f"Error finding reads mapping to master BED in {bamfile}: {e}") raise - - logger.debug(f"Found {len(reads_mapping_to_master)} reads mapping to master BED regions") + + logger.debug( + f"Found {len(reads_mapping_to_master)} reads mapping to master BED regions" + ) return reads_mapping_to_master @@ -981,65 +1014,71 @@ def _check_read_alignments_overlap(read_rows: List[Dict]) -> bool: Check for false positives in read alignments: - Same genomic alignment annotated with multiple genes (overlapping gene regions) - Very similar (but not identical) alignments that are likely mapping artifacts - + Args: read_rows: List of alignment dictionaries for the same read - + Returns: True if any false positive pattern is detected, False otherwise """ if len(read_rows) < 2: return False - + # Check: Same genomic alignment annotated with multiple genes # Group by genomic coordinates (chr:start-end) genomic_alignments = {} for row in read_rows: - genomic_key = f"{row['reference_id']}:{row['reference_start']}-{row['reference_end']}" + genomic_key = ( + f"{row['reference_id']}:{row['reference_start']}-{row['reference_end']}" + ) if genomic_key not in genomic_alignments: genomic_alignments[genomic_key] = [] genomic_alignments[genomic_key].append(row) - + # Only filter out if we have the EXACT same genomic coordinates with different genes # This is more restrictive - only remove true mapping artifacts for genomic_key, alignments in genomic_alignments.items(): if len(alignments) > 1: # Check if all alignments have the same gene annotation - genes = [align['col4'] for align in alignments] + genes = [align["col4"] for align in alignments] if len(set(genes)) > 1: # Same genomic coordinates with different gene annotations = false positive return True - + # Check: Very similar alignments (coordinate similarity filter) if get_fusion_rule("coordinate_similarity_filter"): max_diff = get_fusion_threshold("coordinate_similarity") - + # Optimize: Group by chromosome first to reduce comparisons # Only compare alignments on the same chromosome alignments_by_chrom = defaultdict(list) for i, row in enumerate(read_rows): - alignments_by_chrom[row['reference_id']].append((i, row)) - + alignments_by_chrom[row["reference_id"]].append((i, row)) + # Compare pairs within each chromosome (reduces comparisons significantly) for chrom, chrom_alignments in alignments_by_chrom.items(): if len(chrom_alignments) < 2: continue - + # Only compare pairs on the same chromosome for i in range(len(chrom_alignments)): for j in range(i + 1, len(chrom_alignments)): _, row1 = chrom_alignments[i] _, row2 = chrom_alignments[j] - + if are_coordinates_similar( - row1['reference_start'], row1['reference_end'], - row2['reference_start'], row2['reference_end'], - max_diff + row1["reference_start"], + row1["reference_end"], + row2["reference_start"], + row2["reference_end"], + max_diff, ): # Very similar alignments detected - likely mapping artifact - logger.debug(f"Filtering similar alignments: {row1['reference_id']}:{row1['reference_start']}-{row1['reference_end']} vs {row2['reference_id']}:{row2['reference_start']}-{row2['reference_end']}") + logger.debug( + f"Filtering similar alignments: {row1['reference_id']}:{row1['reference_start']}-{row1['reference_end']} vs {row2['reference_id']}:{row2['reference_start']}-{row2['reference_end']}" + ) return True - + return False @@ -1059,8 +1098,7 @@ def _canonicalize_gene_alignment_rows( ) existing = unique.get(key) if existing is None or ( - existing.get("_from_sa_tag", False) - and not row.get("_from_sa_tag", False) + existing.get("_from_sa_tag", False) and not row.get("_from_sa_tag", False) ): unique[key] = row @@ -1084,7 +1122,7 @@ def _find_gene_intersections( """ Find intersections between a read and gene regions using optimized binary search. Only processes reads that have supplementary alignments (true fusion candidates). - + Uses binary search on sorted gene regions for O(log n + k) complexity where: - n = number of gene regions - k = number of overlapping regions (typically small) @@ -1118,7 +1156,10 @@ def _find_gene_intersections( gene_region = indexed_regions[region_id] overlap_start = max(gene_region.start, ref_start) overlap_end = min(gene_region.end, ref_end) - if overlap_end > overlap_start and (overlap_end - overlap_start) > min_overlap: + if ( + overlap_end > overlap_start + and (overlap_end - overlap_start) > min_overlap + ): read_rows.append( _build_fusion_row_dict( ref_name, @@ -1135,26 +1176,26 @@ def _find_gene_intersections( # Binary search optimization: # 1. Find the first region that could overlap (regions with start <= ref_end) # 2. Iterate backwards checking overlaps until we pass ref_start - + # Use cached start positions if provided, otherwise create on-the-fly if region_starts is None: region_starts = [region.start for region in gene_regions] - + # Find the rightmost region with start <= ref_end # This is the last region that could potentially overlap rightmost_idx = bisect.bisect_right(region_starts, ref_end) - + # Now iterate backwards from rightmost_idx to find all overlapping regions # We iterate backwards because regions are sorted by start, and we want to find # all regions that overlap with [ref_start, ref_end] for i in range(rightmost_idx - 1, -1, -1): gene_region = gene_regions[i] - + # If this region ends before ref_start, we've gone too far (no more overlaps) # Since regions are sorted by start, all previous regions will also end before ref_start if gene_region.end < ref_start: break - + # Check if this region overlaps with the read if gene_region.overlaps_with(ref_start, ref_end, min_overlap=min_overlap): read_rows.append( @@ -1345,9 +1386,9 @@ def _process_reads_for_fusions( # Apply filtering thresholds using centralized configuration min_mq = get_fusion_threshold("mapping_quality") min_span = get_fusion_threshold("mapping_span") - df = df[(df["mapping_quality"] > min_mq) & (df["mapping_span"] > min_span)].reset_index( - drop=True - ) + df = df[ + (df["mapping_quality"] > min_mq) & (df["mapping_span"] > min_span) + ].reset_index(drop=True) return df if not df.empty else None @@ -1399,38 +1440,38 @@ def _process_reads_for_master_bed_fusions( """ Process reads mapping to master BED regions and track ALL their supplementary alignments. Unlike regular fusion detection, we don't require gene overlap - breaks can occur anywhere. - + Args: bamfile: Path to BAM file reads_mapping_to_master: Set of read names that map to master BED regions - + Returns: DataFrame with fusion candidates or None if no candidates found """ read_rows = [] - + if not reads_mapping_to_master: return None - + try: with pysam.AlignmentFile(bamfile, "rb") as bam: for read in bam: # Only process reads that map to master BED regions if read.query_name not in reads_mapping_to_master: continue - + # Skip secondary alignments and unmapped reads if read.is_secondary or read.is_unmapped: continue - + # Only process reads with supplementary alignments (SA tag) if not read.has_tag("SA"): continue - + # Only include reads where the primary mapping has QS >= MIN_PRIMARY_QS if not _primary_meets_min_qs(read): continue - + # Get reference information ref_name = ( bam.get_reference_name(read.reference_id) @@ -1439,10 +1480,10 @@ def _process_reads_for_master_bed_fusions( ) if not ref_name or ref_name == "chrM": continue - + ref_start = read.reference_start ref_end = read.reference_end - + # For master BED fusions, we track ALL supplementary alignments # without requiring gene overlap. Create a row for each alignment. read_rows.append( @@ -1464,7 +1505,7 @@ def _process_reads_for_master_bed_fusions( "mapping_span": ref_end - ref_start, # Mapping span } ) - + # Also parse SA tag to get all supplementary alignments try: sa_tag = read.get_tag("SA") @@ -1479,7 +1520,7 @@ def _process_reads_for_master_bed_fusions( sa_pos = int(sa_parts[1]) sa_strand = sa_parts[2] sa_mapq = int(sa_parts[4]) if len(sa_parts) > 4 else 0 - + # Estimate end position from CIGAR if available if len(sa_parts) > 3: cigar_str = sa_parts[3] @@ -1487,11 +1528,11 @@ def _process_reads_for_master_bed_fusions( sa_end = sa_pos + read.query_length else: sa_end = sa_pos + 100 # Default small span - + # Skip mitochondrial chromosomes if sa_chrom == "chrM" or sa_chrom == "M": continue - + # Add supplementary alignment as a row read_rows.append( { @@ -1513,33 +1554,35 @@ def _process_reads_for_master_bed_fusions( } ) except (ValueError, KeyError) as e: - logger.debug(f"Could not parse SA tag for read {read.query_name}: {e}") + logger.debug( + f"Could not parse SA tag for read {read.query_name}: {e}" + ) continue - + except Exception as e: logger.error(f"Error processing reads for master BED fusions: {str(e)}") raise - + if not read_rows: return None - + # Create DataFrame df = pd.DataFrame(read_rows) - + # Apply memory optimizations df = _optimize_fusion_dataframe(df) - + # Apply basic filtering thresholds (mapping quality, mapping span) # but NOT gene overlap requirement min_mq = get_fusion_threshold("mapping_quality") min_span = get_fusion_threshold("mapping_span") - df = df[(df["mapping_quality"] > min_mq) & (df["mapping_span"] > min_span)].reset_index( - drop=True - ) - + df = df[ + (df["mapping_quality"] > min_mq) & (df["mapping_span"] > min_span) + ].reset_index(drop=True) + if df.empty: return None - + return df @@ -1559,10 +1602,10 @@ def _filter_fusion_candidates(df: pd.DataFrame) -> Optional[pd.DataFrame]: try: # Count unique genes per read_id to find fusion candidates gene_counts = df.groupby("read_id", observed=True)["col4"].nunique() - + # Filter for reads that map to more than 1 gene (fusion candidates) fusion_read_ids = gene_counts[gene_counts > 1].index - + if len(fusion_read_ids) == 0: return None @@ -1596,35 +1639,37 @@ def process_bam_for_master_bed_fusions( No longer requires primary alignment to overlap master BED regions. Tracks supplementary mappings from all reads with supplementary alignments, without requiring gene overlap (breaks can occur anywhere). - + Args: bamfile: Path to BAM file work_dir: Working directory (kept for API compatibility, not used for filtering) sample_id: Sample ID (kept for API compatibility, not used for filtering) supplementary_read_ids: Optional list of read IDs with supplementary alignments (if available from metadata, avoids checking SA tag for all reads) - + Returns: DataFrame with master BED fusion candidates or None if no candidates found """ try: # Convert supplementary_read_ids to set for fast lookup if provided - supplementary_reads_set = set(supplementary_read_ids) if supplementary_read_ids else None - + supplementary_reads_set = ( + set(supplementary_read_ids) if supplementary_read_ids else None + ) + # Process ALL reads with supplementary alignments (no master BED overlap requirement) read_rows = [] reads_with_supplementary_count = 0 - + try: with pysam.AlignmentFile(bamfile, "rb") as bam: for read in bam: if read.is_unmapped: continue - + # Skip secondary alignments (we only process primary alignments) if read.is_secondary: continue - + # Get reference information ref_name = ( bam.get_reference_name(read.reference_id) @@ -1633,9 +1678,12 @@ def process_bam_for_master_bed_fusions( ) if not ref_name or ref_name == "chrM": continue - + # Check for supplementary alignments - if supplementary_reads_set is not None and supplementary_read_ids_complete: + if ( + supplementary_reads_set is not None + and supplementary_read_ids_complete + ): if read.query_name not in supplementary_reads_set: continue has_supplementary = True @@ -1646,17 +1694,17 @@ def process_bam_for_master_bed_fusions( has_supplementary = read.has_tag("SA") if not has_supplementary: continue - + # Only include reads where the primary mapping has QS >= MIN_PRIMARY_QS if not _primary_meets_min_qs(read): continue - + # Process this read: it has supplementary alignments reads_with_supplementary_count += 1 - + ref_start = read.reference_start ref_end = read.reference_end - + # Process this read: add primary alignment read_rows.append( { @@ -1677,9 +1725,12 @@ def process_bam_for_master_bed_fusions( "mapping_span": ref_end - ref_start, # Mapping span } ) - + # Parse SA tag to get all supplementary alignments - if supplementary_reads_set is not None and supplementary_read_ids_complete: + if ( + supplementary_reads_set is not None + and supplementary_read_ids_complete + ): try: sa_tag = read.get_tag("SA") except KeyError: @@ -1697,8 +1748,10 @@ def process_bam_for_master_bed_fusions( sa_chrom = sa_parts[0] sa_pos = int(sa_parts[1]) sa_strand = sa_parts[2] - sa_mapq = int(sa_parts[4]) if len(sa_parts) > 4 else 0 - + sa_mapq = ( + int(sa_parts[4]) if len(sa_parts) > 4 else 0 + ) + # Estimate end position from CIGAR if available if len(sa_parts) > 3: cigar_str = sa_parts[3] @@ -1706,11 +1759,11 @@ def process_bam_for_master_bed_fusions( sa_end = sa_pos + read.query_length else: sa_end = sa_pos + 100 # Default small span - + # Skip mitochondrial chromosomes if sa_chrom == "chrM" or sa_chrom == "M": continue - + # Add supplementary alignment as a row read_rows.append( { @@ -1728,49 +1781,56 @@ def process_bam_for_master_bed_fusions( "read_end": read.query_length, # Not available from SA tag "is_secondary": False, # SA tag entries are supplementary "is_supplementary": True, # This is a supplementary alignment - "mapping_span": sa_end - sa_pos, # Mapping span + "mapping_span": sa_end + - sa_pos, # Mapping span } ) except (ValueError, KeyError) as e: - logger.debug(f"Could not parse SA tag for read {read.query_name}: {e}") + logger.debug( + f"Could not parse SA tag for read {read.query_name}: {e}" + ) continue - + except Exception as e: logger.error(f"Error processing BAM file for master BED fusions: {e}") raise - + if reads_with_supplementary_count == 0: logger.debug("No reads with supplementary alignments found") return None - - logger.info(f"Found {reads_with_supplementary_count} reads with supplementary alignments") - + + logger.info( + f"Found {reads_with_supplementary_count} reads with supplementary alignments" + ) + if not read_rows: - logger.debug("No master BED fusion candidates found (no supplementary alignments)") + logger.debug( + "No master BED fusion candidates found (no supplementary alignments)" + ) return None - + # Create DataFrame df = pd.DataFrame(read_rows) - + # Apply memory optimizations df = _optimize_fusion_dataframe(df) - + # Apply basic filtering thresholds (mapping quality, mapping span) # but NOT gene overlap requirement min_mq = get_fusion_threshold("mapping_quality") min_span = get_fusion_threshold("mapping_span") - + before_filter = len(df) - df = df[(df["mapping_quality"] > min_mq) & (df["mapping_span"] > min_span)].reset_index( - drop=True - ) - + df = df[ + (df["mapping_quality"] > min_mq) & (df["mapping_span"] > min_span) + ].reset_index(drop=True) + if df.empty: return None - + logger.info(f"Found {len(df)} master BED fusion candidates") return df - + except Exception as e: logger.error(f"Error processing master BED fusions: {str(e)}") logger.error("Exception details:", exc_info=True) @@ -1787,12 +1847,12 @@ def process_bam_single_pass( ) -> Tuple[Optional[pd.DataFrame], Optional[pd.DataFrame], Optional[pd.DataFrame]]: """ Process BAM file in a single pass to find all fusion candidates. - + This optimized function combines three separate BAM file passes into one: 1. Finding reads with supplementary alignments 2. Processing target panel and genome-wide fusions 3. Processing master BED fusions - + This provides a 2-3x speedup by eliminating redundant BAM file I/O. Args: @@ -1808,10 +1868,10 @@ def process_bam_single_pass( """ try: logger.debug(f"Processing BAM file in single pass: {bamfile}") - + # Ensure gene regions are loaded (cached, so this is fast after first call) _ensure_gene_regions_loaded(target_panel) - + # Get gene region dictionaries target_regions = _gene_regions_cache.get(target_panel, {}) if "shared" not in _all_gene_regions_cache: @@ -1819,7 +1879,7 @@ def process_bam_single_pass( genome_regions = {} else: genome_regions = _all_gene_regions_cache["shared"] - + # Get cached start position lists for binary search optimization target_region_starts = _gene_region_starts_cache.get(target_panel, {}) genome_region_starts = _all_gene_region_starts_cache.get("shared", {}) @@ -1834,10 +1894,12 @@ def process_bam_single_pass( combined_region_indexes = ( _combined_gene_region_ncls_cache.get(target_panel, {}) if _HAS_NCLS else {} ) - + # Convert supplementary_read_ids to set for fast lookup if provided - supplementary_reads_set = set(supplementary_read_ids) if supplementary_read_ids else None - + supplementary_reads_set = ( + set(supplementary_read_ids) if supplementary_read_ids else None + ) + # Collectors for all three fusion types target_read_alignments = {} # read_id -> list of alignment rows genome_read_alignments = {} # read_id -> list of alignment rows @@ -1853,7 +1915,7 @@ def _flush_master_bed_rows(rows: List[Dict[str, Any]]) -> None: if not df.empty: master_bed_chunks.append(df) rows.clear() - + reads_with_supplementary_count = 0 # Reads whose primary alignment failed QS (exclude all alignments for these reads) primary_qs_excluded: Set[str] = set() @@ -1868,11 +1930,11 @@ def _flush_master_bed_rows(rows: List[Dict[str, Any]]) -> None: # Skip unmapped reads if read.is_unmapped: continue - + # Skip secondary alignments (we only process primary alignments) if read.is_secondary: continue - + # Get reference information ref_name = ( bam.get_reference_name(read.reference_id) @@ -1881,11 +1943,14 @@ def _flush_master_bed_rows(rows: List[Dict[str, Any]]) -> None: ) if not ref_name or ref_name == "chrM": continue - + # Single SA fetch for incomplete-list path (reused below for master BED parsing). # Complete-list path trusts preprocessing and fetches SA once at master BED stage. sa_tag_cached: Optional[str] = None - if supplementary_reads_set is not None and supplementary_read_ids_complete: + if ( + supplementary_reads_set is not None + and supplementary_read_ids_complete + ): if read.query_name not in supplementary_reads_set: continue else: @@ -1898,22 +1963,25 @@ def _flush_master_bed_rows(rows: List[Dict[str, Any]]) -> None: ) # Optional: if we have a supplementary_read_ids list and the read is not in it, # keep the read only when it still has an SA tag (same as has_tag("SA") before). - if supplementary_reads_set is not None and read.query_name not in supplementary_reads_set: + if ( + supplementary_reads_set is not None + and read.query_name not in supplementary_reads_set + ): if not has_supplementary: continue if not has_supplementary: continue - + # Skip all alignments for reads whose primary failed QS (decided when we saw the primary) if read.query_name in primary_qs_excluded: continue - + # Only check primary QS when we're on the primary alignment; then include/exclude the whole read if not read.is_supplementary: if not _primary_meets_min_qs(read): primary_qs_excluded.add(read.query_name) continue - + # This read has supplementary alignments - process it for all fusion types (primary + supp) reads_with_supplementary_count += 1 read_id = read.query_name @@ -1924,7 +1992,7 @@ def _flush_master_bed_rows(rows: List[Dict[str, Any]]) -> None: # Early quality gates to avoid expensive region intersection work if read.mapping_quality <= min_mq or mapping_span <= min_span: continue - + # 1-2. Unified target + genome overlap (NCLS only) if ref_name in combined_region_indexes: _append_combined_gene_intersections( @@ -1957,7 +2025,7 @@ def _flush_master_bed_rows(rows: List[Dict[str, Any]]) -> None: if read_id not in target_read_alignments: target_read_alignments[read_id] = [] target_read_alignments[read_id].extend(target_rows) - + # 2. Process for genome-wide fusions if ref_name in genome_regions: genome_starts = genome_region_starts.get(ref_name) @@ -1976,7 +2044,7 @@ def _flush_master_bed_rows(rows: List[Dict[str, Any]]) -> None: if read_id not in genome_read_alignments: genome_read_alignments[read_id] = [] genome_read_alignments[read_id].extend(genome_rows) - + # 3. Process for master BED fusions # Add primary alignment master_bed_rows.append( @@ -2000,11 +2068,14 @@ def _flush_master_bed_rows(rows: List[Dict[str, Any]]) -> None: ) if len(master_bed_rows) >= master_bed_chunk_size: _flush_master_bed_rows(master_bed_rows) - + # Parse SA tag for supplementary master BED rows (one fetch: reuse sa_tag_cached or get once) if read.is_supplementary: sa_tag_to_parse = None - elif supplementary_reads_set is not None and supplementary_read_ids_complete: + elif ( + supplementary_reads_set is not None + and supplementary_read_ids_complete + ): try: sa_tag_to_parse = read.get_tag("SA") except KeyError: @@ -2026,19 +2097,23 @@ def _flush_master_bed_rows(rows: List[Dict[str, Any]]) -> None: sa_strand = sa_parts[2] sa_cigar = sa_parts[3] sa_mapq = int(sa_parts[4]) - sa_span, sa_query_span = _sa_alignment_spans(sa_cigar) + sa_span, sa_query_span = _sa_alignment_spans( + sa_cigar + ) sa_end = sa_pos + sa_span sa_span = sa_end - sa_pos - + # Apply same quality thresholds to supplementary mappings if sa_mapq <= min_mq or sa_span <= min_span: continue - + # Skip mitochondrial chromosomes if sa_chrom == "chrM" or sa_chrom == "M": continue - sa_tokens = re.findall(r"(\d+)([MIDNSHP=X])", sa_cigar) + sa_tokens = re.findall( + r"(\d+)([MIDNSHP=X])", sa_cigar + ) sa_read_start = ( int(sa_tokens[0][0]) if sa_tokens and sa_tokens[0][1] == "S" @@ -2047,19 +2122,25 @@ def _flush_master_bed_rows(rows: List[Dict[str, Any]]) -> None: sa_read_end = sa_read_start + sa_query_span if sa_chrom in target_regions: - target_rows = _find_gene_intersections_for_values( - ref_name=sa_chrom, - ref_start=sa_pos, - ref_end=sa_end, - read_id=read_id, - mapping_quality=sa_mapq, - strand=sa_strand, - read_start=sa_read_start, - read_end=sa_read_end, - gene_regions=target_regions[sa_chrom], - region_starts=target_region_starts.get(sa_chrom), - region_index=target_region_indexes.get(sa_chrom), - min_overlap=min_overlap, + target_rows = ( + _find_gene_intersections_for_values( + ref_name=sa_chrom, + ref_start=sa_pos, + ref_end=sa_end, + read_id=read_id, + mapping_quality=sa_mapq, + strand=sa_strand, + read_start=sa_read_start, + read_end=sa_read_end, + gene_regions=target_regions[sa_chrom], + region_starts=target_region_starts.get( + sa_chrom + ), + region_index=target_region_indexes.get( + sa_chrom + ), + min_overlap=min_overlap, + ) ) if target_rows: target_read_alignments.setdefault( @@ -2067,25 +2148,31 @@ def _flush_master_bed_rows(rows: List[Dict[str, Any]]) -> None: ).extend(target_rows) if sa_chrom in genome_regions: - genome_rows = _find_gene_intersections_for_values( - ref_name=sa_chrom, - ref_start=sa_pos, - ref_end=sa_end, - read_id=read_id, - mapping_quality=sa_mapq, - strand=sa_strand, - read_start=sa_read_start, - read_end=sa_read_end, - gene_regions=genome_regions[sa_chrom], - region_starts=genome_region_starts.get(sa_chrom), - region_index=genome_region_indexes.get(sa_chrom), - min_overlap=min_overlap, + genome_rows = ( + _find_gene_intersections_for_values( + ref_name=sa_chrom, + ref_start=sa_pos, + ref_end=sa_end, + read_id=read_id, + mapping_quality=sa_mapq, + strand=sa_strand, + read_start=sa_read_start, + read_end=sa_read_end, + gene_regions=genome_regions[sa_chrom], + region_starts=genome_region_starts.get( + sa_chrom + ), + region_index=genome_region_indexes.get( + sa_chrom + ), + min_overlap=min_overlap, + ) ) if genome_rows: genome_read_alignments.setdefault( read_id, [] ).extend(genome_rows) - + # Add supplementary alignment as a row master_bed_rows.append( { @@ -2109,15 +2196,19 @@ def _flush_master_bed_rows(rows: List[Dict[str, Any]]) -> None: if len(master_bed_rows) >= master_bed_chunk_size: _flush_master_bed_rows(master_bed_rows) except (ValueError, KeyError) as e: - logger.debug(f"Could not parse SA tag for read {read.query_name}: {e}") + logger.debug( + f"Could not parse SA tag for read {read.query_name}: {e}" + ) continue - + except Exception as e: logger.error(f"Error reading BAM file: {e}") raise - - logger.info(f"Found {reads_with_supplementary_count} reads with supplementary alignments") - + + logger.info( + f"Found {reads_with_supplementary_count} reads with supplementary alignments" + ) + # Process target panel candidates (comprehension inlined in 3.12 for speed) target_candidates = None if target_read_alignments: @@ -2127,13 +2218,14 @@ def _flush_master_bed_rows(rows: List[Dict[str, Any]]) -> None: for deduplicated in [_canonicalize_gene_alignment_rows(alignments)] if not _check_read_alignments_overlap(deduplicated) for align in deduplicated - if align.get("mapping_quality", 0) > min_mq and align.get("mapping_span", 0) > min_span + if align.get("mapping_quality", 0) > min_mq + and align.get("mapping_span", 0) > min_span ] if filtered_target_rows: target_df = pd.DataFrame(filtered_target_rows) target_df = _optimize_fusion_dataframe(target_df) target_candidates = target_df if not target_df.empty else None - + # Process genome-wide candidates (comprehension inlined in 3.12 for speed) genome_wide_candidates = None if genome_read_alignments: @@ -2143,13 +2235,14 @@ def _flush_master_bed_rows(rows: List[Dict[str, Any]]) -> None: for deduplicated in [_canonicalize_gene_alignment_rows(alignments)] if not _check_read_alignments_overlap(deduplicated) for align in deduplicated - if align.get("mapping_quality", 0) > min_mq and align.get("mapping_span", 0) > min_span + if align.get("mapping_quality", 0) > min_mq + and align.get("mapping_span", 0) > min_span ] if filtered_genome_rows: genome_df = pd.DataFrame(filtered_genome_rows) genome_df = _optimize_fusion_dataframe(genome_df) genome_wide_candidates = genome_df if not genome_df.empty else None - + # Process master BED candidates master_bed_candidates = None if master_bed_rows: @@ -2161,13 +2254,13 @@ def _flush_master_bed_rows(rows: List[Dict[str, Any]]) -> None: master_bed_candidates = pd.concat(master_bed_chunks, ignore_index=True) else: master_bed_candidates = None - + logger.info( f"Single-pass results: {len(target_candidates) if target_candidates is not None else 0} target, " f"{len(genome_wide_candidates) if genome_wide_candidates is not None else 0} genome-wide, " f"{len(master_bed_candidates) if master_bed_candidates is not None else 0} master BED candidates" ) - + return target_candidates, genome_wide_candidates, master_bed_candidates except Exception as e: @@ -2182,7 +2275,7 @@ def process_bam_for_fusions_work( """ Process BAM file to find fusion candidates with memory optimization. This method now uses the optimized single-pass approach. - + NOTE: This function is kept for backward compatibility but now uses the single-pass implementation internally. @@ -2194,15 +2287,21 @@ def process_bam_for_fusions_work( Tuple of (target_panel_candidates, genome_wide_candidates) DataFrames """ try: - logger.info(f"DEBUG: process_bam_for_fusions_work called with target_panel='{target_panel}'") - + logger.info( + f"DEBUG: process_bam_for_fusions_work called with target_panel='{target_panel}'" + ) + # Use single-pass processing (master BED not needed here) target_candidates, genome_wide_candidates, _ = process_bam_single_pass( bamfile, target_panel ) - - logger.info(f"Target panel candidates found: {len(target_candidates) if target_candidates is not None else 0}") - logger.info(f"Genome-wide candidates found: {len(genome_wide_candidates) if genome_wide_candidates is not None else 0}") + + logger.info( + f"Target panel candidates found: {len(target_candidates) if target_candidates is not None else 0}" + ) + logger.info( + f"Genome-wide candidates found: {len(genome_wide_candidates) if genome_wide_candidates is not None else 0}" + ) return target_candidates, genome_wide_candidates @@ -2240,7 +2339,9 @@ def _get_pending_count(work_dir: str, sample_id: str) -> int: for pattern in ("target_*.parquet", "genome_*.parquet", "master_bed_*.parquet"): for path in glob.glob(os.path.join(staging_dir, pattern)): try: - counters.add(int(os.path.basename(path).rsplit("_", 1)[1].split(".", 1)[0])) + counters.add( + int(os.path.basename(path).rsplit("_", 1)[1].split(".", 1)[0]) + ) except (IndexError, ValueError): continue return len(counters) @@ -2315,7 +2416,7 @@ def process_bam_with_staging( """ Fast per-file processing that saves results to staging area. Does NOT merge with accumulated data - much faster for large datasets. - + Args: file_path: Path to BAM file temp_dir: Temporary directory @@ -2326,12 +2427,12 @@ def process_bam_with_staging( supplementary_read_ids: List of supplementary read IDs work_dir: Working directory for staging batch_size: Number of files before accumulation triggers - + Returns: Tuple of (results_dict, should_accumulate) """ sample_id = fusion_metadata.sample_id - + if not has_supplementary: return { "has_supplementary": False, @@ -2340,15 +2441,24 @@ def process_bam_with_staging( "target_candidates": None, "genome_wide_candidates": None, }, False - + if not work_dir: # Fallback to non-staging mode logger.warning("No work_dir provided - falling back to non-staging mode") - return process_bam_file( - file_path, temp_dir, metadata, fusion_metadata, - target_panel, has_supplementary, supplementary_read_ids, work_dir - ), False - + return ( + process_bam_file( + file_path, + temp_dir, + metadata, + fusion_metadata, + target_panel, + has_supplementary, + supplementary_read_ids, + work_dir, + ), + False, + ) + try: supplementary_read_ids_complete = bool( metadata.get("supplementary_read_ids_complete", False) @@ -2356,17 +2466,19 @@ def process_bam_with_staging( # Get atomic counter (thread-safe) counter = _atomic_counter_increment(work_dir, sample_id) logger.debug(f"Assigned fusion file counter: {counter}") - + # Process BAM file in single pass (combines all three operations) - target_candidates, genome_wide_candidates, master_bed_candidates = process_bam_single_pass( - file_path, - target_panel, - work_dir, - sample_id, - supplementary_read_ids, - supplementary_read_ids_complete, + target_candidates, genome_wide_candidates, master_bed_candidates = ( + process_bam_single_pass( + file_path, + target_panel, + work_dir, + sample_id, + supplementary_read_ids, + supplementary_read_ids_complete, + ) ) - + # Save only non-empty candidate types. The pending counter records processed # BAMs independently, so empty placeholder Parquets are unnecessary. staging_dir = _get_staging_dir(work_dir, sample_id) @@ -2374,33 +2486,39 @@ def process_bam_with_staging( target_staging = os.path.join(staging_dir, f"target_{counter:06d}.parquet") genome_staging = os.path.join(staging_dir, f"genome_{counter:06d}.parquet") - master_bed_staging = os.path.join(staging_dir, f"master_bed_{counter:06d}.parquet") + master_bed_staging = os.path.join( + staging_dir, f"master_bed_{counter:06d}.parquet" + ) if target_candidates is not None and not target_candidates.empty: target_candidates.to_parquet(target_staging, **_staging_opts) logger.info(f"Saved {len(target_candidates)} target candidates to staging") if genome_wide_candidates is not None and not genome_wide_candidates.empty: genome_wide_candidates.to_parquet(genome_staging, **_staging_opts) - logger.debug(f"Saved {len(genome_wide_candidates)} genome-wide candidates to staging") + logger.debug( + f"Saved {len(genome_wide_candidates)} genome-wide candidates to staging" + ) if master_bed_candidates is not None and not master_bed_candidates.empty: master_bed_candidates.to_parquet(master_bed_staging, **_staging_opts) - logger.info(f"Saved {len(master_bed_candidates)} master BED candidates to staging") - + logger.info( + f"Saved {len(master_bed_candidates)} master BED candidates to staging" + ) + # Check if accumulation should run # Note: This check is not atomic - multiple workers might see threshold reached # The actual accumulation function will re-check inside the lock to prevent duplicate work pending_count = _increment_pending_count(work_dir, sample_id, delta=1) should_accumulate = pending_count >= batch_size - + logger.info( f"Fusion staging complete. Pending files: {pending_count}/{batch_size}" ) - + if should_accumulate: logger.info( f"Accumulation threshold reached ({pending_count} >= {batch_size}) - will attempt accumulation" ) - + results = { "has_supplementary": True, "target_candidates_count": ( @@ -2416,18 +2534,28 @@ def process_bam_with_staging( "genome_wide_candidates": genome_wide_candidates, "master_bed_candidates": master_bed_candidates, } - + return results, should_accumulate - + except Exception as e: logger.error(f"Error in fusion staging for {sample_id}: {e}") import traceback + logger.error(traceback.format_exc()) # Fall back to non-staging mode on error - return process_bam_file( - file_path, temp_dir, metadata, fusion_metadata, - target_panel, has_supplementary, supplementary_read_ids, work_dir - ), False + return ( + process_bam_file( + file_path, + temp_dir, + metadata, + fusion_metadata, + target_panel, + has_supplementary, + supplementary_read_ids, + work_dir, + ), + False, + ) def accumulate_fusion_candidates( @@ -2440,35 +2568,39 @@ def accumulate_fusion_candidates( ) -> Dict[str, Any]: """ Batch accumulation of staged fusion candidates. - + This method: 1. Loads all staged files 2. Efficiently concatenates them (faster than iterative merge) 3. Merges batch with existing accumulated data 4. Saves updated accumulated data 5. Cleans up staging files - + Args: work_dir: Working directory sample_id: Sample identifier target_panel: Target panel type force: If True, accumulate even if below threshold (for end-of-run) batch_size: Minimum number of files to accumulate - + Returns: Dictionary with accumulation results """ try: - logger.info(f"Starting fusion batch accumulation for {sample_id} (force={force})") + logger.info( + f"Starting fusion batch accumulation for {sample_id} (force={force})" + ) start_time = time.time() - + staging_dir = _get_staging_dir(work_dir, sample_id) logger.debug(f"Scanning staging directory: {staging_dir}") - + # Find all staging files target_files = sorted(glob.glob(os.path.join(staging_dir, "target_*.parquet"))) genome_files = sorted(glob.glob(os.path.join(staging_dir, "genome_*.parquet"))) - master_bed_files = sorted(glob.glob(os.path.join(staging_dir, "master_bed_*.parquet"))) + master_bed_files = sorted( + glob.glob(os.path.join(staging_dir, "master_bed_*.parquet")) + ) pending_count = _get_pending_count(work_dir, sample_id) logger.debug( "Staging file counts - target: %d, genome: %d, master_bed: %d, pending BAMs: %d", @@ -2477,7 +2609,7 @@ def accumulate_fusion_candidates( len(master_bed_files), pending_count, ) - + all_staging_files = target_files + genome_files + master_bed_files if pending_count == 0 and not all_staging_files: logger.info( @@ -2493,7 +2625,7 @@ def accumulate_fusion_candidates( "master_bed_candidates": 0, "message": "No new data to accumulate", } - + # Re-check if we should accumulate based on count if not force and pending_count < batch_size: logger.info( @@ -2505,13 +2637,13 @@ def accumulate_fusion_candidates( "files_pending": pending_count, "error": f"Below batch threshold ({pending_count} < {batch_size})", } - + logger.info( f"Accumulating candidates from {pending_count} staged BAMs for {sample_id}" ) - + # Stream staged files and append to datasets to cap memory usage - + def _extract_batch_id(file_path: str) -> int: name = os.path.basename(file_path) try: @@ -2574,9 +2706,7 @@ def _append_staged_files( ) append_start = time.time() - batch_target_rows, _ = _append_staged_files( - target_files, "target_candidates" - ) + batch_target_rows, _ = _append_staged_files(target_files, "target_candidates") batch_genome_rows, _ = _append_staged_files( genome_files, "genome_wide_candidates" ) @@ -2594,12 +2724,18 @@ def _append_staged_files( f"Batch appended: {batch_target_rows} target, {batch_genome_rows} genome-wide, " f"{batch_master_bed_rows} master BED candidates" ) - - counts["target_candidates"] = counts.get("target_candidates", 0) + batch_target_rows - counts["genome_wide_candidates"] = counts.get("genome_wide_candidates", 0) + batch_genome_rows - counts["master_bed_candidates"] = counts.get("master_bed_candidates", 0) + batch_master_bed_rows - save_counts_start = time.time() - _save_fusion_counts(work_dir, sample_id, counts) + + counts["target_candidates"] = ( + counts.get("target_candidates", 0) + batch_target_rows + ) + counts["genome_wide_candidates"] = ( + counts.get("genome_wide_candidates", 0) + batch_genome_rows + ) + counts["master_bed_candidates"] = ( + counts.get("master_bed_candidates", 0) + batch_master_bed_rows + ) + save_counts_start = time.time() + _save_fusion_counts(work_dir, sample_id, counts) logger.debug( "Saved fusion counts in %.3fs (updated: target=%d, genome=%d, master_bed=%d)", time.time() - save_counts_start, @@ -2613,7 +2749,7 @@ def _append_staged_files( f"{counts.get('genome_wide_candidates', 0)} genome-wide, " f"{counts.get('master_bed_candidates', 0)} master BED candidates" ) - + # Create updated metadata (with empty lists - data is in Parquet files) # This keeps the metadata structure but avoids storing large lists in JSON fusion_metadata = FusionMetadata( @@ -2633,7 +2769,7 @@ def _append_staged_files( }, processing_steps=["accumulated"], ) - + # Save metadata (without large candidate lists - they're in Parquet) metadata_start = time.time() _save_fusion_metadata(fusion_metadata, work_dir, sample_id) @@ -2642,19 +2778,21 @@ def _append_staged_files( time.time() - metadata_start, sample_id, ) - + # Generate output files only on final accumulation (force=True) # This avoids expensive groupby operations and CSV generation during intermediate accumulations # Output files are only needed at the end, not after every batch # However, master BED breakpoint extraction should still run incrementally to build up the BED file if force and all_staging_files: - logger.info("Final accumulation detected - generating output files (CSV, BED, etc.)") + logger.info( + "Final accumulation detected - generating output files (CSV, BED, etc.)" + ) output_start = time.time() _generate_output_files( - sample_id, - fusion_metadata.analysis_results, - fusion_metadata, - work_dir, + sample_id, + fusion_metadata.analysis_results, + fusion_metadata, + work_dir, reference=reference, generate_master_bed=True, # Generate master BED on final accumulation new_master_bed_files=set(new_master_bed_dataset_files), @@ -2665,8 +2803,10 @@ def _append_staged_files( sample_id, ) else: - logger.debug("Intermediate accumulation - skipping output file generation (will be generated on final accumulation)") - + logger.debug( + "Intermediate accumulation - skipping output file generation (will be generated on final accumulation)" + ) + # Clean up staging files logger.info("Cleaning up fusion staging files...") cleanup_start = time.time() @@ -2688,14 +2828,14 @@ def _append_staged_files( removed, failed, ) - + elapsed = time.time() - start_time logger.info( f"Fusion batch accumulation complete for {sample_id}: " f"{pending_count} BAMs in {elapsed:.2f}s " f"({elapsed/pending_count:.3f}s per BAM)" ) - + return { "status": "success", "files_processed": pending_count, @@ -2703,18 +2843,15 @@ def _append_staged_files( "genome_wide_candidates": counts.get("genome_wide_candidates", 0), "master_bed_candidates": counts.get("master_bed_candidates", 0), "target_candidates_count": counts.get("target_candidates", 0), - "genome_wide_candidates_count": counts.get( - "genome_wide_candidates", 0 - ), - "master_bed_candidates_count": counts.get( - "master_bed_candidates", 0 - ), + "genome_wide_candidates_count": counts.get("genome_wide_candidates", 0), + "master_bed_candidates_count": counts.get("master_bed_candidates", 0), "elapsed_time": elapsed, } - + except Exception as e: logger.error(f"Error during fusion batch accumulation for {sample_id}: {e}") import traceback + logger.error(traceback.format_exc()) return {"status": "error", "error": str(e)} @@ -2762,22 +2899,24 @@ def process_bam_file( if has_sup: # Process BAM file in single pass (combines all three operations) - target_candidates, genome_wide_candidates, master_bed_candidates = process_bam_single_pass( - file_path, - target_panel, - work_dir, - sample_id, - supplementary_read_ids, - supplementary_read_ids_complete, + target_candidates, genome_wide_candidates, master_bed_candidates = ( + process_bam_single_pass( + file_path, + target_panel, + work_dir, + sample_id, + supplementary_read_ids, + supplementary_read_ids_complete, + ) ) # Apply fusion candidate filtering to get the final results if target_candidates is not None and not target_candidates.empty: target_candidates = _filter_fusion_candidates(target_candidates) - + if genome_wide_candidates is not None and not genome_wide_candidates.empty: genome_wide_candidates = _filter_fusion_candidates(genome_wide_candidates) - + fusion_metadata.processing_steps.append("supplementary_found") results = { "has_supplementary": True, @@ -2803,14 +2942,30 @@ def process_bam_file( target_candidates, "target_candidates", work_dir, sample_id, batch_id ) _append_fusion_candidates_parquet( - genome_wide_candidates, "genome_wide_candidates", work_dir, sample_id, batch_id + genome_wide_candidates, + "genome_wide_candidates", + work_dir, + sample_id, + batch_id, ) _append_fusion_candidates_parquet( - master_bed_candidates, "master_bed_candidates", work_dir, sample_id, batch_id + master_bed_candidates, + "master_bed_candidates", + work_dir, + sample_id, + batch_id, + ) + counts["target_candidates"] = ( + counts.get("target_candidates", 0) + results["target_candidates_count"] + ) + counts["genome_wide_candidates"] = ( + counts.get("genome_wide_candidates", 0) + + results["genome_wide_candidates_count"] + ) + counts["master_bed_candidates"] = ( + counts.get("master_bed_candidates", 0) + + results["master_bed_candidates_count"] ) - counts["target_candidates"] = counts.get("target_candidates", 0) + results["target_candidates_count"] - counts["genome_wide_candidates"] = counts.get("genome_wide_candidates", 0) + results["genome_wide_candidates_count"] - counts["master_bed_candidates"] = counts.get("master_bed_candidates", 0) + results["master_bed_candidates_count"] _save_fusion_counts(work_dir, sample_id, counts) # Keep metadata structure without embedding large candidate lists @@ -2976,6 +3131,7 @@ def _generate_output_files( Dictionary mapping file type to file path """ output_start = time.time() + def _build_filtered_candidates( candidate_type: str, output_csv_name: str, @@ -2990,7 +3146,9 @@ def _build_filtered_candidates( ) return - state_path = os.path.join(work_dir, sample_id, f"{candidate_type}_filter_state.pkl") + state_path = os.path.join( + work_dir, sample_id, f"{candidate_type}_filter_state.pkl" + ) processed_read_ids: Set[str] = set() read_to_genes: Dict[str, Set[str]] = {} tag_counts: Dict[str, int] = defaultdict(int) @@ -3033,7 +3191,9 @@ def _build_filtered_candidates( dataset_dir = _get_parquet_dataset_dir(work_dir, sample_id, candidate_type) dataset_files = glob.glob(os.path.join(dataset_dir, "*.parquet")) legacy_path = _get_parquet_paths(work_dir, sample_id).get(candidate_type) - has_parquet = bool(dataset_files) or (legacy_path and os.path.exists(legacy_path)) + has_parquet = bool(dataset_files) or ( + legacy_path and os.path.exists(legacy_path) + ) # Fallback to in-memory fusion_metadata when Parquet isn't available if not has_parquet and getattr(fusion_metadata, "fusion_data", None): @@ -3047,20 +3207,28 @@ def _build_filtered_candidates( candidate_df = None if candidate_df is not None and not candidate_df.empty: - gene_counts = candidate_df.groupby("read_id", observed=True)["col4"].nunique() + gene_counts = candidate_df.groupby("read_id", observed=True)[ + "col4" + ].nunique() fusion_read_ids = gene_counts[gene_counts > 1].index if len(fusion_read_ids) == 0: return - result = candidate_df[candidate_df["read_id"].isin(fusion_read_ids)].copy() + result = candidate_df[ + candidate_df["read_id"].isin(fusion_read_ids) + ].copy() lookup = result.groupby("read_id", observed=True)["col4"].agg( lambda x: ",".join(sorted(set(x))) ) result["tag"] = result["read_id"].map(lookup) min_support = get_fusion_threshold("read_support") - gene_pair_read_counts = result.groupby("tag", observed=True)["read_id"].nunique() - valid_gene_pairs = gene_pair_read_counts[gene_pair_read_counts >= min_support].index + gene_pair_read_counts = result.groupby("tag", observed=True)[ + "read_id" + ].nunique() + valid_gene_pairs = gene_pair_read_counts[ + gene_pair_read_counts >= min_support + ].index result = result[result["tag"].isin(valid_gene_pairs)] if result.empty: @@ -3114,9 +3282,13 @@ def _build_filtered_candidates( state = { "processed_read_ids": list(processed_read_ids), "written_read_ids": list(written_read_ids), - "read_to_genes": {rid: list(genes) for rid, genes in read_to_genes.items()}, + "read_to_genes": { + rid: list(genes) for rid, genes in read_to_genes.items() + }, "tag_counts": dict(tag_counts), - "tag_to_read_ids": {tag: list(rids) for tag, rids in tag_to_read_ids.items()}, + "tag_to_read_ids": { + tag: list(rids) for tag, rids in tag_to_read_ids.items() + }, "valid_tags": list(valid_tags), "updated_at": time.time(), } @@ -3136,9 +3308,13 @@ def _build_filtered_candidates( state = { "processed_read_ids": list(processed_read_ids), "written_read_ids": list(written_read_ids), - "read_to_genes": {rid: list(genes) for rid, genes in read_to_genes.items()}, + "read_to_genes": { + rid: list(genes) for rid, genes in read_to_genes.items() + }, "tag_counts": dict(tag_counts), - "tag_to_read_ids": {tag: list(rids) for tag, rids in tag_to_read_ids.items()}, + "tag_to_read_ids": { + tag: list(rids) for tag, rids in tag_to_read_ids.items() + }, "valid_tags": list(valid_tags), "updated_at": time.time(), } @@ -3150,10 +3326,14 @@ def _build_filtered_candidates( logger.warning("Could not save %s filter state: %s", candidate_type, e) return - read_ids_to_write = {rid for rid in valid_read_ids if rid not in written_read_ids} + read_ids_to_write = { + rid for rid in valid_read_ids if rid not in written_read_ids + } for tag in newly_valid_tags: read_ids_to_write.update( - rid for rid in tag_to_read_ids.get(tag, set()) if rid not in written_read_ids + rid + for rid in tag_to_read_ids.get(tag, set()) + if rid not in written_read_ids ) if not read_ids_to_write: @@ -3161,9 +3341,13 @@ def _build_filtered_candidates( state = { "processed_read_ids": list(processed_read_ids), "written_read_ids": list(written_read_ids), - "read_to_genes": {rid: list(genes) for rid, genes in read_to_genes.items()}, + "read_to_genes": { + rid: list(genes) for rid, genes in read_to_genes.items() + }, "tag_counts": dict(tag_counts), - "tag_to_read_ids": {tag: list(rids) for tag, rids in tag_to_read_ids.items()}, + "tag_to_read_ids": { + tag: list(rids) for tag, rids in tag_to_read_ids.items() + }, "valid_tags": list(valid_tags), "updated_at": time.time(), } @@ -3193,9 +3377,11 @@ def _build_filtered_candidates( continue subset = subset.copy() subset["tag"] = subset["read_id"].map( - lambda rid: ",".join(sorted(read_to_genes.get(rid, set()))) - if len(read_to_genes.get(rid, set())) > 1 - else "" + lambda rid: ( + ",".join(sorted(read_to_genes.get(rid, set()))) + if len(read_to_genes.get(rid, set())) > 1 + else "" + ) ) subset = subset[subset["tag"] != ""] if subset.empty: @@ -3212,9 +3398,13 @@ def _build_filtered_candidates( state = { "processed_read_ids": list(processed_read_ids), "written_read_ids": list(written_read_ids), - "read_to_genes": {rid: list(genes) for rid, genes in read_to_genes.items()}, + "read_to_genes": { + rid: list(genes) for rid, genes in read_to_genes.items() + }, "tag_counts": dict(tag_counts), - "tag_to_read_ids": {tag: list(rids) for tag, rids in tag_to_read_ids.items()}, + "tag_to_read_ids": { + tag: list(rids) for tag, rids in tag_to_read_ids.items() + }, "valid_tags": list(valid_tags), "updated_at": time.time(), } @@ -3233,7 +3423,12 @@ def _build_filtered_candidates( os.path.join(work_dir, sample_id, output_pickle_name), ) except Exception as e: - logger.warning("Could not preprocess %s from %s: %s", candidate_type, output_csv_path, e) + logger.warning( + "Could not preprocess %s from %s: %s", + candidate_type, + output_csv_path, + e, + ) _build_filtered_candidates( "target_candidates", @@ -3248,7 +3443,7 @@ def _build_filtered_candidates( # Master BED candidate handling temporarily disabled for performance testing master_bed_candidates = None - + # Create sv_count.txt file with content "0" if it doesn't exist # Create sv_count.txt file with content "0" if it doesn't exist @@ -3257,45 +3452,50 @@ def _build_filtered_candidates( with open(sv_count_file, "w") as f: f.write("0") - # Generate fusion breakpoint BED file only when content has changed. fusion_breakpoint_changed = _generate_fusion_breakpoint_bed( sample_id, fusion_metadata, work_dir ) - + # Generate master BED breakpoint BED file (new target regions from supplementary alignments) # This is called incrementally as data accumulates. For large datasets, we use an incremental # approach: only process NEW staging files and merge with existing breakpoints. if generate_master_bed: - master_bed_candidates = _load_fusion_candidates_parquet("master_bed_candidates", work_dir, sample_id) - - #ToDo: This is the slow code from here. - + master_bed_candidates = _load_fusion_candidates_parquet( + "master_bed_candidates", work_dir, sample_id + ) + + # ToDo: This is the slow code from here. + master_bed_breakpoint_changed = False if master_bed_candidates is not None and not master_bed_candidates.empty: master_bed_breakpoint_changed = _generate_master_bed_breakpoint_bed( - sample_id, - fusion_metadata, + sample_id, + fusion_metadata, work_dir, new_master_bed_files=new_master_bed_files, # Pass new files for incremental processing ) - - + if ENABLE_MASTER_BED: - - + # Ask the master BED generator to refresh. It is content-signature gated, # so unchanged source BEDs return without doing the expensive merge. if generate_master_bed: try: - from robin.analysis.master_bed_generator import generate_master_bed_async - + from robin.analysis.master_bed_generator import ( + generate_master_bed_async, + ) + # Get analysis counter analysis_counter = _load_analysis_counter(sample_id, work_dir) - + # Get target_panel from fusion_metadata - target_panel = fusion_metadata.target_panel if hasattr(fusion_metadata, 'target_panel') else None - + target_panel = ( + fusion_metadata.target_panel + if hasattr(fusion_metadata, "target_panel") + else None + ) + # Generate asynchronously (non-blocking) generate_master_bed_async( sample_id=sample_id, @@ -3323,27 +3523,27 @@ def _build_filtered_candidates( work_dir, sample_id, "fusion_candidates_all.csv" ), } - + # Master BED CSV generation has been deprecated - Parquet file is used instead - + return output_paths def _get_cnv_bin_width(work_dir: str, sample_id: str) -> int: """ Try to load bin_width from CNV analysis results if available. - + Args: work_dir: Working directory sample_id: Sample ID - + Returns: bin_width in base pairs, or default 10000 (10kb) if not available """ try: sample_dir = os.path.join(work_dir, sample_id) cnv_dict_path = os.path.join(sample_dir, "CNV_dict.npy") - + if os.path.exists(cnv_dict_path): cnv_dict = np.load(cnv_dict_path, allow_pickle=True).item() bin_width = cnv_dict.get("bin_width") @@ -3352,29 +3552,31 @@ def _get_cnv_bin_width(work_dir: str, sample_id: str) -> int: return int(bin_width) except Exception as e: logger.debug(f"Could not load bin_width from CNV analysis: {e}") - + # Default to 10kb if CNV data not available default_bin_width = 10000 return default_bin_width -def _extract_fusion_breakpoints(fusion_metadata: FusionMetadata) -> List[Dict[str, Any]]: +def _extract_fusion_breakpoints( + fusion_metadata: FusionMetadata, +) -> List[Dict[str, Any]]: """ Extract fusion breakpoint coordinates from fusion metadata. Only includes fusions that meet the minimum read support threshold. - + Args: fusion_metadata: FusionMetadata object with fusion candidates - + Returns: List of dictionaries with fusion breakpoint information """ breakpoints = [] breakpoint_set = set() # For deduplication - + fusion_data = fusion_metadata.fusion_data or {} min_support = get_fusion_threshold("read_support") - + # Process target candidates - load from in-memory data (Parquet loading handled by _load_fusion_metadata) target_candidates = fusion_data.get("target_candidates", []) target_df = None @@ -3383,17 +3585,17 @@ def _extract_fusion_breakpoints(fusion_metadata: FusionMetadata) -> List[Dict[st target_df = pd.DataFrame(target_candidates) elif isinstance(target_candidates, pd.DataFrame): target_df = target_candidates - + if target_df is not None and not target_df.empty: - + if not target_df.empty and "reference_start" in target_df.columns: # Filter for fusion candidates (reads mapping to multiple genes) gene_counts = target_df.groupby("read_id", observed=True)["col4"].nunique() fusion_read_ids = gene_counts[gene_counts > 1].index - + if len(fusion_read_ids) > 0: fusion_df = target_df[target_df["read_id"].isin(fusion_read_ids)] - + # Apply minimum read support threshold if not fusion_df.empty: # Create tag column by grouping genes per read_id @@ -3401,32 +3603,38 @@ def _extract_fusion_breakpoints(fusion_metadata: FusionMetadata) -> List[Dict[st lambda x: ",".join(sorted(set(x))) ) fusion_df["tag"] = fusion_df["read_id"].map(lookup) - + # Group by gene pair (tag) and count supporting reads - gene_pair_read_counts = fusion_df.groupby("tag", observed=True)["read_id"].nunique() - valid_gene_pairs = gene_pair_read_counts[gene_pair_read_counts >= min_support].index + gene_pair_read_counts = fusion_df.groupby("tag", observed=True)[ + "read_id" + ].nunique() + valid_gene_pairs = gene_pair_read_counts[ + gene_pair_read_counts >= min_support + ].index filtered_df = fusion_df[fusion_df["tag"].isin(valid_gene_pairs)] - + # Extract breakpoint coordinates for valid fusions for row in filtered_df.itertuples(index=False, name="Row"): chrom = getattr(row, "reference_id", "Unknown") start = int(getattr(row, "reference_start", 0)) end = int(getattr(row, "reference_end", 0)) gene = getattr(row, "col4", "Unknown") - + if start > 0 and end > start: bp_key = (chrom, start, end) if bp_key not in breakpoint_set: breakpoint_set.add(bp_key) - breakpoints.append({ - "chromosome": chrom, - "start": start, - "end": end, - "gene": gene, - "read_id": getattr(row, "read_id", "Unknown"), - "source": "target" - }) - + breakpoints.append( + { + "chromosome": chrom, + "start": start, + "end": end, + "gene": gene, + "read_id": getattr(row, "read_id", "Unknown"), + "source": "target", + } + ) + # Process genome-wide candidates - load from in-memory data (Parquet loading handled by _load_fusion_metadata) genome_wide_candidates = fusion_data.get("genome_wide_candidates", []) genome_df = None @@ -3435,61 +3643,73 @@ def _extract_fusion_breakpoints(fusion_metadata: FusionMetadata) -> List[Dict[st genome_df = pd.DataFrame(genome_wide_candidates) elif isinstance(genome_wide_candidates, pd.DataFrame): genome_df = genome_wide_candidates - + if genome_df is not None and not genome_df.empty: - + if not genome_df.empty and "reference_start" in genome_df.columns: # Filter for fusion candidates (reads mapping to multiple genes) - gene_counts_all = genome_df.groupby("read_id", observed=True)["col4"].nunique() + gene_counts_all = genome_df.groupby("read_id", observed=True)[ + "col4" + ].nunique() fusion_read_ids_all = gene_counts_all[gene_counts_all > 1].index - + if len(fusion_read_ids_all) > 0: - fusion_df_all = genome_df[genome_df["read_id"].isin(fusion_read_ids_all)] - + fusion_df_all = genome_df[ + genome_df["read_id"].isin(fusion_read_ids_all) + ] + # Apply minimum read support threshold if not fusion_df_all.empty: # Create tag column by grouping genes per read_id - lookup_all = fusion_df_all.groupby("read_id", observed=True)["col4"].agg( - lambda x: ",".join(sorted(set(x))) - ) + lookup_all = fusion_df_all.groupby("read_id", observed=True)[ + "col4" + ].agg(lambda x: ",".join(sorted(set(x)))) fusion_df_all["tag"] = fusion_df_all["read_id"].map(lookup_all) - + # Group by gene pair (tag) and count supporting reads - gene_pair_read_counts_all = fusion_df_all.groupby("tag", observed=True)["read_id"].nunique() - valid_gene_pairs_all = gene_pair_read_counts_all[gene_pair_read_counts_all >= min_support].index - filtered_df_all = fusion_df_all[fusion_df_all["tag"].isin(valid_gene_pairs_all)] - + gene_pair_read_counts_all = fusion_df_all.groupby( + "tag", observed=True + )["read_id"].nunique() + valid_gene_pairs_all = gene_pair_read_counts_all[ + gene_pair_read_counts_all >= min_support + ].index + filtered_df_all = fusion_df_all[ + fusion_df_all["tag"].isin(valid_gene_pairs_all) + ] + # Extract breakpoint coordinates for valid fusions for row in filtered_df_all.itertuples(index=False, name="Row"): chrom = getattr(row, "reference_id", "Unknown") start = int(getattr(row, "reference_start", 0)) end = int(getattr(row, "reference_end", 0)) gene = getattr(row, "col4", "Unknown") - + if start > 0 and end > start: bp_key = (chrom, start, end) if bp_key not in breakpoint_set: breakpoint_set.add(bp_key) - breakpoints.append({ - "chromosome": chrom, - "start": start, - "end": end, - "gene": gene, - "read_id": getattr(row, "read_id", "Unknown"), - "source": "genome_wide" - }) - + breakpoints.append( + { + "chromosome": chrom, + "start": start, + "end": end, + "gene": gene, + "read_id": getattr(row, "read_id", "Unknown"), + "source": "genome_wide", + } + ) + return breakpoints def _load_analysis_counter(sample_id: str, work_dir: str) -> int: """ Load the analysis counter for a sample from disk. - + Args: sample_id: Sample ID work_dir: Working directory - + Returns: Analysis counter value, or 0 if not found """ @@ -3510,21 +3730,21 @@ def _create_breakpoint_pairs_vectorized( ) -> List[Dict[str, Any]]: """ Create breakpoint pairs using vectorized Pandas operations (much faster than nested loops). - + This function uses Pandas merge to create the cartesian product efficiently, which is significantly faster than nested Python loops. - + Args: primary_df: DataFrame with primary alignments for a single read supplementary_df: DataFrame with supplementary alignments for a single read read_id: Read ID - + Returns: List of breakpoint pair dictionaries """ if primary_df.empty or supplementary_df.empty: return [] - + # Prepare primary alignments primary_cols = ["reference_id", "reference_start", "reference_end"] if "mapping_quality" in primary_df.columns: @@ -3533,7 +3753,7 @@ def _create_breakpoint_pairs_vectorized( primary_cols.append("mapping_span") if "strand" in primary_df.columns: primary_cols.append("strand") - + # Prepare supplementary alignments supp_cols = ["reference_id", "reference_start", "reference_end"] if "mapping_quality" in supplementary_df.columns: @@ -3542,22 +3762,20 @@ def _create_breakpoint_pairs_vectorized( supp_cols.append("mapping_span") if "strand" in supplementary_df.columns: supp_cols.append("strand") - + # Select only the columns we need primaries = primary_df[primary_cols].copy() supplementaries = supplementary_df[supp_cols].copy() - + # Add a key column for cross join primaries["_key"] = 1 supplementaries["_key"] = 1 - + # Perform cross join using merge (much faster than nested loops) pairs_df = primaries.merge( - supplementaries, - on="_key", - suffixes=("_primary", "_supplementary") + supplementaries, on="_key", suffixes=("_primary", "_supplementary") ).drop("_key", axis=1) - + # Convert to list of dictionaries def _opt_val(r, name, default): v = getattr(r, name, None) @@ -3583,7 +3801,7 @@ def _opt_val(r, name, default): pair["primary_strand"] = _opt_val(row, "strand_primary", ".") pair["supp_strand"] = _opt_val(row, "strand_supplementary", ".") breakpoint_pairs.append(pair) - + return breakpoint_pairs @@ -3593,32 +3811,34 @@ def _merge_nearby_events( ) -> pd.DataFrame: """ Merge nearby events on the same chromosome to remove duplicates. - + Events are merged if they: - Are on the same chromosome - Have the same event_type - Overlap or are within cluster_distance of each other - + Args: events_df: DataFrame with events (chromosome, start, end, event_type, read_count, etc.) cluster_distance: Maximum distance for merging events (in base pairs) - + Returns: DataFrame with merged events """ if events_df.empty: return events_df - + # Group by chromosome and event_type merged_events = [] - - for (chrom, event_type), group in events_df.groupby(["chromosome", "event_type"], observed=True): + + for (chrom, event_type), group in events_df.groupby( + ["chromosome", "event_type"], observed=True + ): if group.empty: continue - + # Sort by start position group_sorted = group.sort_values("start").copy() - + # Merge overlapping or nearby events current_events = [] for row in group_sorted.itertuples(index=False, name="Row"): @@ -3662,21 +3882,23 @@ def _merge_nearby_events( break if not merged: - current_events.append({ - "chromosome": chrom, - "start": row_start, - "end": row_end, - "event_type": event_type, - "read_count": row_read_count, - "avg_mapping_quality": row_avg_mq, - "avg_mapping_span": row_avg_span, - }) - + current_events.append( + { + "chromosome": chrom, + "start": row_start, + "end": row_end, + "event_type": event_type, + "read_count": row_read_count, + "avg_mapping_quality": row_avg_mq, + "avg_mapping_span": row_avg_span, + } + ) + merged_events.extend(current_events) - + if not merged_events: return pd.DataFrame() - + return pd.DataFrame(merged_events) @@ -3687,23 +3909,23 @@ def _cluster_breakpoint_pairs_dbscan( ) -> List[Dict[str, Any]]: """ Cluster breakpoint pairs using DBSCAN algorithm for efficient O(n log n) clustering. - + This replaces the O(n²) nested loop approach with DBSCAN, which is much faster for large datasets. Two breakpoint pairs are clustered if: - Primary locations are close (within cluster_distance) - Supplementary locations are close (within cluster_distance) - + Args: breakpoint_pairs: List of breakpoint pair dictionaries cluster_distance: Maximum distance for clustering (in base pairs) min_read_support: Minimum number of reads required per cluster - + Returns: List of clustered breakpoint pairs with aggregated information """ if not breakpoint_pairs: return [] - + logger.debug( "DBSCAN clustering start: pairs=%d, cluster_distance=%d, min_read_support=%d", len(breakpoint_pairs), @@ -3712,7 +3934,7 @@ def _cluster_breakpoint_pairs_dbscan( ) cluster_start = time.time() max_dbscan_pairs = int(os.environ.get("ROBIN_DBSCAN_MAX_PAIRS", "100000")) - + # Group pairs by chromosome combination (required for clustering) pairs_by_chrom = {} for i, pair in enumerate(breakpoint_pairs): @@ -3720,14 +3942,14 @@ def _cluster_breakpoint_pairs_dbscan( if chrom_key not in pairs_by_chrom: pairs_by_chrom[chrom_key] = [] pairs_by_chrom[chrom_key].append((i, pair)) - + clustered_pairs = [] - + # Cluster within each chromosome combination for chrom_key, chrom_pairs in pairs_by_chrom.items(): if len(chrom_pairs) == 0: continue - + chrom_start = time.time() logger.debug( "DBSCAN chrom group: primary=%s, supp=%s, pairs=%d", @@ -3735,11 +3957,11 @@ def _cluster_breakpoint_pairs_dbscan( chrom_key[1], len(chrom_pairs), ) - + # Extract indices and pairs indices = [idx for idx, _ in chrom_pairs] pairs = [pair for _, pair in chrom_pairs] - + if max_dbscan_pairs > 0 and len(pairs) > max_dbscan_pairs: fallback_start = time.time() logger.warning( @@ -3759,26 +3981,30 @@ def _cluster_breakpoint_pairs_dbscan( time.time() - fallback_start, ) continue - + # Create feature matrix for DBSCAN: [primary_midpoint, supp_midpoint] # We use midpoints for clustering to reduce dimensionality - features = np.array([ + features = np.array( [ - (pair["primary_start"] + pair["primary_end"]) / 2, # Primary midpoint - (pair["supp_start"] + pair["supp_end"]) / 2, # Supplementary midpoint + [ + (pair["primary_start"] + pair["primary_end"]) + / 2, # Primary midpoint + (pair["supp_start"] + pair["supp_end"]) + / 2, # Supplementary midpoint + ] + for pair in pairs ] - for pair in pairs - ]) - + ) + # Use DBSCAN with Manhattan distance (L1 norm) for genomic coordinates # eps is the maximum distance between samples in the same cluster # We use cluster_distance * 1.5 to account for coordinate ranges eps = cluster_distance * 1.5 min_samples = min_read_support # Minimum samples in a cluster - + # Cluster using DBSCAN dbscan_start = time.time() - clustering = DBSCAN(eps=eps, min_samples=min_samples, metric='manhattan') + clustering = DBSCAN(eps=eps, min_samples=min_samples, metric="manhattan") labels = clustering.fit_predict(features) logger.debug( "DBSCAN fitted: eps=%.1f, min_samples=%d, labels=%d, time=%.3fs", @@ -3787,13 +4013,13 @@ def _cluster_breakpoint_pairs_dbscan( len(labels), time.time() - dbscan_start, ) - + # Group pairs by cluster label clusters = {} for idx, label in enumerate(labels): if label == -1: # Noise points (not in any cluster) continue - + if label not in clusters: clusters[label] = { "indices": [], @@ -3808,7 +4034,7 @@ def _cluster_breakpoint_pairs_dbscan( "supp_mapqs": [], "supp_spans": [], } - + clusters[label]["indices"].append(indices[idx]) clusters[label]["pairs"].append(pairs[idx]) clusters[label]["read_ids"].add(pairs[idx]["read_id"]) @@ -3816,7 +4042,7 @@ def _cluster_breakpoint_pairs_dbscan( clusters[label]["primary_ends"].append(pairs[idx]["primary_end"]) clusters[label]["supp_starts"].append(pairs[idx].get("supp_start", 0)) clusters[label]["supp_ends"].append(pairs[idx].get("supp_end", 0)) - + # Optional fields if "primary_mapq" in pairs[idx]: clusters[label]["primary_mapqs"].append(pairs[idx]["primary_mapq"]) @@ -3826,14 +4052,14 @@ def _cluster_breakpoint_pairs_dbscan( clusters[label]["supp_mapqs"].append(pairs[idx]["supp_mapq"]) if "supp_span" in pairs[idx]: clusters[label]["supp_spans"].append(pairs[idx]["supp_span"]) - + # Create clustered breakpoint pairs clustered_before_support = 0 for label, cluster_data in clusters.items(): clustered_before_support += 1 if len(cluster_data["read_ids"]) < min_read_support: continue - + clustered_pair = { "primary_chrom": pairs[0]["primary_chrom"], "primary_start": min(cluster_data["primary_starts"]), @@ -3844,32 +4070,47 @@ def _cluster_breakpoint_pairs_dbscan( "read_count": len(cluster_data["read_ids"]), "read_ids": cluster_data["read_ids"], } - + # Add optional fields if available if cluster_data["primary_mapqs"]: - clustered_pair["primary_avg_mapq"] = sum(cluster_data["primary_mapqs"]) / len(cluster_data["primary_mapqs"]) + clustered_pair["primary_avg_mapq"] = sum( + cluster_data["primary_mapqs"] + ) / len(cluster_data["primary_mapqs"]) if cluster_data["primary_spans"]: - clustered_pair["primary_avg_span"] = sum(cluster_data["primary_spans"]) / len(cluster_data["primary_spans"]) + clustered_pair["primary_avg_span"] = sum( + cluster_data["primary_spans"] + ) / len(cluster_data["primary_spans"]) if cluster_data["supp_mapqs"]: - clustered_pair["supp_avg_mapq"] = sum(cluster_data["supp_mapqs"]) / len(cluster_data["supp_mapqs"]) + clustered_pair["supp_avg_mapq"] = sum(cluster_data["supp_mapqs"]) / len( + cluster_data["supp_mapqs"] + ) if cluster_data["supp_spans"]: - clustered_pair["supp_avg_span"] = sum(cluster_data["supp_spans"]) / len(cluster_data["supp_spans"]) - + clustered_pair["supp_avg_span"] = sum(cluster_data["supp_spans"]) / len( + cluster_data["supp_spans"] + ) + clustered_pairs.append(clustered_pair) - + logger.debug( "DBSCAN chrom group done: clusters=%d, retained=%d, time=%.3fs", clustered_before_support, - len([p for p in clustered_pairs if p["primary_chrom"] == chrom_key[0] and p["supp_chrom"] == chrom_key[1]]), + len( + [ + p + for p in clustered_pairs + if p["primary_chrom"] == chrom_key[0] + and p["supp_chrom"] == chrom_key[1] + ] + ), time.time() - chrom_start, ) - + logger.debug( "DBSCAN clustering complete: clustered_pairs=%d, time=%.3fs", len(clustered_pairs), time.time() - cluster_start, ) - + return clustered_pairs @@ -3949,40 +4190,44 @@ def _extract_master_bed_breakpoints( Extract breakpoint coordinates from master BED fusion candidates. Identifies genomic rearrangements (deletions, inversions, translocations) by finding breakpoint pairs (primary + supplementary alignments) supported by multiple reads. - + A rearrangement event is defined by a breakpoint pair: - Primary alignment location (where the read starts) - Supplementary alignment location (where the read continues) - + Only includes breakpoint pairs supported by at least min_read_support reads. The breakpoint pairs can be on different chromosomes (translocations). - + Args: fusion_metadata: FusionMetadata object with master BED candidates work_dir: Working directory (required to load from Parquet files) min_read_support: Minimum number of reads required to support a breakpoint pair (default: 3) - + Returns: List of dictionaries with master BED breakpoint information (both primary and supplementary regions) """ breakpoints = [] overall_start = time.time() - + # Try loading from Parquet files first (fast path, new storage format) master_bed_df = None sample_id = fusion_metadata.sample_id - + if work_dir and sample_id: load_start = time.time() - master_bed_df = _load_fusion_candidates_parquet("master_bed_candidates", work_dir, sample_id) + master_bed_df = _load_fusion_candidates_parquet( + "master_bed_candidates", work_dir, sample_id + ) if master_bed_df is not None and not master_bed_df.empty: - logger.debug(f"Loaded {len(master_bed_df)} master BED candidates from Parquet for breakpoint extraction") + logger.debug( + f"Loaded {len(master_bed_df)} master BED candidates from Parquet for breakpoint extraction" + ) logger.debug( "Loaded master BED candidates from Parquet in %.3fs (rows=%d)", time.time() - load_start, len(master_bed_df) if master_bed_df is not None else 0, ) - + # Fallback to in-memory data (for backward compatibility) if master_bed_df is None or master_bed_df.empty: fusion_data = fusion_metadata.fusion_data or {} @@ -3991,10 +4236,14 @@ def _extract_master_bed_breakpoints( fallback_start = time.time() if isinstance(master_bed_candidates, list): master_bed_df = pd.DataFrame(master_bed_candidates) - logger.debug(f"Loaded {len(master_bed_candidates)} master BED candidates from in-memory data (fallback)") + logger.debug( + f"Loaded {len(master_bed_candidates)} master BED candidates from in-memory data (fallback)" + ) elif isinstance(master_bed_candidates, pd.DataFrame): master_bed_df = master_bed_candidates - logger.debug(f"Loaded {len(master_bed_candidates)} master BED candidates from in-memory DataFrame (fallback)") + logger.debug( + f"Loaded {len(master_bed_candidates)} master BED candidates from in-memory DataFrame (fallback)" + ) logger.debug( "Loaded master BED candidates from fallback in %.3fs (rows=%d)", time.time() - fallback_start, @@ -4002,16 +4251,20 @@ def _extract_master_bed_breakpoints( ) else: logger.debug("No master BED candidates found in Parquet or in-memory data") - + # Process master BED candidates if we have data # TEMPORARY: Skip breakpoint extraction for performance testing # Set SKIP_MASTER_BED_BREAKPOINTS=True to disable this section - SKIP_MASTER_BED_BREAKPOINTS = os.environ.get("SKIP_MASTER_BED_BREAKPOINTS", "False").lower() == "true" - + SKIP_MASTER_BED_BREAKPOINTS = ( + os.environ.get("SKIP_MASTER_BED_BREAKPOINTS", "False").lower() == "true" + ) + if SKIP_MASTER_BED_BREAKPOINTS: - logger.info("SKIPPING master BED breakpoint extraction (SKIP_MASTER_BED_BREAKPOINTS=True)") + logger.info( + "SKIPPING master BED breakpoint extraction (SKIP_MASTER_BED_BREAKPOINTS=True)" + ) return breakpoints - + if master_bed_df is not None and not master_bed_df.empty: extract_start = time.time() # Group reads by their breakpoint pairs (primary + supplementary alignments) @@ -4019,42 +4272,60 @@ def _extract_master_bed_breakpoints( if "read_id" not in master_bed_df.columns: logger.debug("Missing required column (read_id) in master BED candidates") return breakpoints - + # Separate primary and supplementary alignments using col4 # col4 is more reliable: "master_bed_region" = primary alignment, "master_bed_supplementary" = supplementary # The is_supplementary flag can be inconsistent (some primary alignments may have is_supplementary=True # if they're part of a chimeric read set in the BAM file) filter_start = time.time() if "col4" in master_bed_df.columns: - primary_df = master_bed_df[master_bed_df["col4"] == "master_bed_region"].copy() - supplementary_df = master_bed_df[master_bed_df["col4"] == "master_bed_supplementary"].copy() + primary_df = master_bed_df[ + master_bed_df["col4"] == "master_bed_region" + ].copy() + supplementary_df = master_bed_df[ + master_bed_df["col4"] == "master_bed_supplementary" + ].copy() else: # Fallback: use is_supplementary if col4 is not available logger.debug("col4 column not found, using is_supplementary as fallback") if "is_supplementary" not in master_bed_df.columns: - logger.debug("Missing required columns (col4 or is_supplementary) in master BED candidates") + logger.debug( + "Missing required columns (col4 or is_supplementary) in master BED candidates" + ) return breakpoints - primary_df = master_bed_df[master_bed_df["is_supplementary"] == False].copy() - supplementary_df = master_bed_df[master_bed_df["is_supplementary"] == True].copy() + primary_df = master_bed_df[ + master_bed_df["is_supplementary"] == False + ].copy() + supplementary_df = master_bed_df[ + master_bed_df["is_supplementary"] == True + ].copy() logger.debug( "Separated primary/supplementary in %.3fs (primary_rows=%d, supplementary_rows=%d)", time.time() - filter_start, len(primary_df), len(supplementary_df), ) - + # Debug: check for any misclassified alignments - if "is_supplementary" in master_bed_df.columns and "col4" in master_bed_df.columns: + if ( + "is_supplementary" in master_bed_df.columns + and "col4" in master_bed_df.columns + ): misclassified = master_bed_df[ - (master_bed_df["col4"] == "master_bed_region") & (master_bed_df["is_supplementary"] == True) + (master_bed_df["col4"] == "master_bed_region") + & (master_bed_df["is_supplementary"] == True) ] if not misclassified.empty: - logger.debug(f"Found {len(misclassified)} primary alignments with is_supplementary=True (using col4 for classification)") - + logger.debug( + f"Found {len(misclassified)} primary alignments with is_supplementary=True (using col4 for classification)" + ) + if primary_df.empty or supplementary_df.empty: - logger.debug("Need both primary and supplementary alignments to identify breakpoint pairs") + logger.debug( + "Need both primary and supplementary alignments to identify breakpoint pairs" + ) return breakpoints - + # Find reads that have both primary and supplementary alignments read_id_start = time.time() primary_read_ids = set(primary_df["read_id"].unique()) @@ -4074,12 +4345,12 @@ def _extract_master_bed_breakpoints( time.time() - read_id_start, len(reads_with_both), ) - + logger.debug( f"Found {len(reads_with_both)} reads with both primary and supplementary alignments " f"(out of {len(primary_read_ids)} reads with primary, {len(supplementary_read_ids)} with supplementary)" ) - + # Debug: log some example read IDs if available if reads_with_both: example_reads = list(reads_with_both)[:3] @@ -4090,15 +4361,19 @@ def _extract_master_bed_breakpoints( elif supplementary_read_ids: example_supp = list(supplementary_read_ids)[:3] logger.debug(f"Example reads with only supplementary: {example_supp}") - + if not reads_with_both: logger.debug("No reads have both primary and supplementary alignments") return breakpoints - + # Filter to only reads with both primary and supplementary alignments filter_both_start = time.time() - primary_filtered = primary_df[primary_df["read_id"].isin(reads_with_both)].copy() - supplementary_filtered = supplementary_df[supplementary_df["read_id"].isin(reads_with_both)].copy() + primary_filtered = primary_df[ + primary_df["read_id"].isin(reads_with_both) + ].copy() + supplementary_filtered = supplementary_df[ + supplementary_df["read_id"].isin(reads_with_both) + ].copy() logger.debug( "Filtered reads_with_both in %.3fs (reads=%d, primary_rows=%d, supplementary_rows=%d)", time.time() - filter_both_start, @@ -4106,26 +4381,30 @@ def _extract_master_bed_breakpoints( len(primary_filtered), len(supplementary_filtered), ) - + # For each read, create breakpoint pairs (primary + supplementary) # A breakpoint pair represents a potential rearrangement event cluster_distance = 500 # Maximum distance for clustering breakpoint pairs (5kb) - + # Build breakpoint pairs: for each read, pair its primary alignment with each supplementary alignment # Optimized: use vectorized operations instead of iterrows() breakpoint_pairs = [] - + # Group by read_id for efficient lookup groupby_start = time.time() - primary_indices_by_read = primary_filtered.groupby("read_id", observed=True).indices - supplementary_indices_by_read = supplementary_filtered.groupby("read_id", observed=True).indices + primary_indices_by_read = primary_filtered.groupby( + "read_id", observed=True + ).indices + supplementary_indices_by_read = supplementary_filtered.groupby( + "read_id", observed=True + ).indices logger.debug( "Grouped by read_id in %.3fs (primary_groups=%d, supplementary_groups=%d)", time.time() - groupby_start, len(primary_indices_by_read), len(supplementary_indices_by_read), ) - + pair_creation_start = time.time() # Sort reads_with_both to ensure deterministic processing order # This prevents different results when DataFrames have different row orders @@ -4149,20 +4428,22 @@ def _extract_master_bed_breakpoints( if supplementary_idx is not None else empty_supplementary ) - + if read_primaries.empty or read_supplementaries.empty: if primary_idx is None: missing_primary += 1 if supplementary_idx is None: missing_supplementary += 1 continue - + # Create pairs using vectorized Pandas operations (much faster than nested loops) reads_processed += 1 - pairs = _create_breakpoint_pairs_vectorized(read_primaries, read_supplementaries, read_id) + pairs = _create_breakpoint_pairs_vectorized( + read_primaries, read_supplementaries, read_id + ) breakpoint_pairs.extend(pairs) total_pairs += len(pairs) - + logger.debug( "Created breakpoint pairs in %.3fs (reads_processed=%d, total_pairs=%d, " "missing_primary=%d, missing_supplementary=%d)", @@ -4174,11 +4455,11 @@ def _extract_master_bed_breakpoints( ) if processed_read_ids is not None and reads_with_both: processed_read_ids.update(reads_with_both) - + if not breakpoint_pairs: logger.debug("No breakpoint pairs created") return breakpoints - + # Cluster breakpoint pairs using canonicalized endpoint keys clustering_start = time.time() clustered_pairs = _cluster_breakpoint_pairs_canonical( @@ -4190,12 +4471,11 @@ def _extract_master_bed_breakpoints( len(clustered_pairs), len(breakpoint_pairs), ) - + # Filter for breakpoint pairs with sufficient read support (already filtered in DBSCAN, but keep for safety) filter_support_start = time.time() supported_pairs = [ - p for p in clustered_pairs - if p["read_count"] >= min_read_support + p for p in clustered_pairs if p["read_count"] >= min_read_support ] logger.debug( "Supported pairs filter: input=%d, output=%d, min_read_support=%d, time=%.3fs", @@ -4204,36 +4484,43 @@ def _extract_master_bed_breakpoints( min_read_support, time.time() - filter_support_start, ) - + if len(supported_pairs) > 0: logger.info( f"Found {len(supported_pairs)} master BED breakpoint pairs " f"supported by >= {min_read_support} reads (out of {len(clustered_pairs)} clustered pairs, " f"{len(breakpoint_pairs)} original pairs)" ) - + # Extract breakpoint coordinates for both primary and supplementary regions coord_extract_start = time.time() for pair in supported_pairs: # Add primary region breakpoint - if pair["primary_start"] > 0 and pair["primary_end"] > pair["primary_start"]: - breakpoints.append({ - "chromosome": pair["primary_chrom"], - "start": pair["primary_start"], - "end": pair["primary_end"], - "read_count": pair["read_count"], - "source": "master_bed" - }) - + if ( + pair["primary_start"] > 0 + and pair["primary_end"] > pair["primary_start"] + ): + breakpoints.append( + { + "chromosome": pair["primary_chrom"], + "start": pair["primary_start"], + "end": pair["primary_end"], + "read_count": pair["read_count"], + "source": "master_bed", + } + ) + # Add supplementary region breakpoint (new target region) if pair["supp_start"] > 0 and pair["supp_end"] > pair["supp_start"]: - breakpoints.append({ - "chromosome": pair["supp_chrom"], - "start": pair["supp_start"], - "end": pair["supp_end"], - "read_count": pair["read_count"], - "source": "master_bed" - }) + breakpoints.append( + { + "chromosome": pair["supp_chrom"], + "start": pair["supp_start"], + "end": pair["supp_end"], + "read_count": pair["read_count"], + "source": "master_bed", + } + ) logger.debug( "Extracted breakpoint coordinates in %.3fs (breakpoints=%d)", time.time() - coord_extract_start, @@ -4252,7 +4539,7 @@ def _extract_master_bed_breakpoints( logger.debug( f"No clustered breakpoint pairs created from {len(breakpoint_pairs)} original pairs" ) - + logger.debug( "Master BED breakpoint extraction completed in %.3fs (breakpoints=%d)", time.time() - overall_start, @@ -4271,22 +4558,22 @@ def _extract_master_bed_breakpoints_incremental( """ Incrementally extract master BED breakpoints by processing only NEW staging files and merging with existing breakpoints from the previous BED file. - + This avoids reprocessing the entire accumulated dataset (which can be 1M+ rows) by only processing the new data and merging results. - + Args: fusion_metadata: FusionMetadata object work_dir: Working directory sample_id: Sample ID new_master_bed_files: Set of new staging file paths to process min_read_support: Minimum read support threshold - + Returns: List of all breakpoint dictionaries (existing + new) """ incremental_start = time.time() - + # Load existing breakpoints from the previous BED file (if it exists) existing_breakpoints = [] sample_dir = os.path.join(work_dir, sample_id) @@ -4297,7 +4584,7 @@ def _extract_master_bed_breakpoints_incremental( processed_reads_path = os.path.join(sample_dir, "master_bed_processed_read_ids.pkl") cluster_state_path = os.path.join(sample_dir, "master_bed_breakpoint_clusters.pkl") bin_size = 500 - + if previous_bed_file and os.path.exists(previous_bed_file): # Parse existing BED file to get breakpoint coordinates # BED format: chrom, start, end, name, score, strand @@ -4318,21 +4605,25 @@ def _extract_master_bed_breakpoints_incremental( bin_width = _get_cnv_bin_width(work_dir, sample_id) original_start = max(0, midpoint - bin_width) original_end = midpoint + bin_width - existing_breakpoints.append({ - "chromosome": chrom, - "start": original_start, - "end": original_end, - "read_count": 1, # Unknown from BED file, use minimum - "source": "master_bed" - }) + existing_breakpoints.append( + { + "chromosome": chrom, + "start": original_start, + "end": original_end, + "read_count": 1, # Unknown from BED file, use minimum + "source": "master_bed", + } + ) except Exception as e: - logger.warning(f"Could not load existing breakpoints from {previous_bed_file}: {e}") + logger.warning( + f"Could not load existing breakpoints from {previous_bed_file}: {e}" + ) logger.debug( "Loaded existing breakpoints in %.3fs (count=%d)", time.time() - existing_start, len(existing_breakpoints), ) - + # Load previously processed read_ids (optional optimization) processed_read_ids: Set[str] = set() if os.path.exists(processed_reads_path): @@ -4348,9 +4639,11 @@ def _extract_master_bed_breakpoints_incremental( len(processed_read_ids), ) except Exception as e: - logger.warning(f"Could not load processed read IDs from {processed_reads_path}: {e}") + logger.warning( + f"Could not load processed read IDs from {processed_reads_path}: {e}" + ) processed_read_ids = set() - + # Load or initialize cluster state for incremental updates cluster_state = {"bin_size": bin_size, "pair_support": {}, "supported_keys": set()} if os.path.exists(cluster_state_path): @@ -4365,15 +4658,25 @@ def _extract_master_bed_breakpoints_incremental( len(cluster_state.get("supported_keys", set())), ) except Exception as e: - logger.warning(f"Could not load cluster state from {cluster_state_path}: {e}") - cluster_state = {"bin_size": bin_size, "pair_support": {}, "supported_keys": set()} + logger.warning( + f"Could not load cluster state from {cluster_state_path}: {e}" + ) + cluster_state = { + "bin_size": bin_size, + "pair_support": {}, + "supported_keys": set(), + } if cluster_state.get("bin_size") != bin_size: logger.warning( "Cluster state bin_size mismatch (found=%s, expected=%s). Resetting state.", cluster_state.get("bin_size"), bin_size, ) - cluster_state = {"bin_size": bin_size, "pair_support": {}, "supported_keys": set()} + cluster_state = { + "bin_size": bin_size, + "pair_support": {}, + "supported_keys": set(), + } if not isinstance(cluster_state.get("pair_support"), dict): cluster_state["pair_support"] = {} if not isinstance(cluster_state.get("supported_keys"), set): @@ -4388,7 +4691,9 @@ def _extract_master_bed_breakpoints_incremental( if not df.empty: new_master_bed_dfs.append(df) except Exception as e: - logger.warning(f"Error loading staging file {os.path.basename(staging_file)}: {e}") + logger.warning( + f"Error loading staging file {os.path.basename(staging_file)}: {e}" + ) if not new_master_bed_dfs: logger.debug( @@ -4396,10 +4701,14 @@ def _extract_master_bed_breakpoints_incremental( time.time() - new_data_start, ) return existing_breakpoints - + # Combine new staging files concat_start = time.time() - new_master_bed_df = pd.concat(new_master_bed_dfs, ignore_index=True) if len(new_master_bed_dfs) > 1 else new_master_bed_dfs[0] + new_master_bed_df = ( + pd.concat(new_master_bed_dfs, ignore_index=True) + if len(new_master_bed_dfs) > 1 + else new_master_bed_dfs[0] + ) logger.debug( "Loaded new master BED staging files in %.3fs (files=%d, rows=%d)", time.time() - new_data_start, @@ -4410,7 +4719,7 @@ def _extract_master_bed_breakpoints_incremental( "Concatenated new master BED staging data in %.3fs", time.time() - concat_start, ) - + # Extract breakpoint pairs from new data only (incremental clustering) new_breakpoints: List[Dict[str, Any]] = [] if "read_id" not in new_master_bed_df.columns: @@ -4418,13 +4727,21 @@ def _extract_master_bed_breakpoints_incremental( # Separate primary and supplementary alignments using col4 if "col4" in new_master_bed_df.columns: - primary_df = new_master_bed_df[new_master_bed_df["col4"] == "master_bed_region"].copy() - supplementary_df = new_master_bed_df[new_master_bed_df["col4"] == "master_bed_supplementary"].copy() + primary_df = new_master_bed_df[ + new_master_bed_df["col4"] == "master_bed_region" + ].copy() + supplementary_df = new_master_bed_df[ + new_master_bed_df["col4"] == "master_bed_supplementary" + ].copy() else: if "is_supplementary" not in new_master_bed_df.columns: return existing_breakpoints - primary_df = new_master_bed_df[new_master_bed_df["is_supplementary"] == False].copy() - supplementary_df = new_master_bed_df[new_master_bed_df["is_supplementary"] == True].copy() + primary_df = new_master_bed_df[ + new_master_bed_df["is_supplementary"] == False + ].copy() + supplementary_df = new_master_bed_df[ + new_master_bed_df["is_supplementary"] == True + ].copy() if primary_df.empty or supplementary_df.empty: return existing_breakpoints @@ -4444,9 +4761,13 @@ def _extract_master_bed_breakpoints_incremental( return existing_breakpoints primary_filtered = primary_df[primary_df["read_id"].isin(reads_with_both)].copy() - supplementary_filtered = supplementary_df[supplementary_df["read_id"].isin(reads_with_both)].copy() + supplementary_filtered = supplementary_df[ + supplementary_df["read_id"].isin(reads_with_both) + ].copy() primary_indices_by_read = primary_filtered.groupby("read_id", observed=True).indices - supplementary_indices_by_read = supplementary_filtered.groupby("read_id", observed=True).indices + supplementary_indices_by_read = supplementary_filtered.groupby( + "read_id", observed=True + ).indices logger.debug( "Filtered reads_with_both in %.3fs (reads=%d, primary_rows=%d, supplementary_rows=%d)", time.time() - new_data_start, @@ -4474,7 +4795,9 @@ def _extract_master_bed_breakpoints_incremental( ) if read_primaries.empty or read_supplementaries.empty: continue - pairs = _create_breakpoint_pairs_vectorized(read_primaries, read_supplementaries, read_id) + pairs = _create_breakpoint_pairs_vectorized( + read_primaries, read_supplementaries, read_id + ) breakpoint_pairs.extend(pairs) logger.debug( "Created incremental breakpoint pairs in %.3fs (pairs=%d, reads=%d)", @@ -4578,7 +4901,7 @@ def _endpoint_key(chrom: str, pos: int, strand: str) -> Tuple[str, int, str]: touched_keys, len(new_supported_pairs), ) - + # Persist cluster state and processed read_ids try: save_state_start = time.time() @@ -4592,7 +4915,7 @@ def _endpoint_key(chrom: str, pos: int, strand: str) -> Tuple[str, int, str]: ) except Exception as e: logger.warning(f"Could not save cluster state to {cluster_state_path}: {e}") - + # Persist processed read_ids to avoid reprocessing the same reads in future batches try: save_reads_start = time.time() @@ -4606,12 +4929,14 @@ def _endpoint_key(chrom: str, pos: int, strand: str) -> Tuple[str, int, str]: len(processed_read_ids), ) except Exception as e: - logger.warning(f"Could not save processed read IDs to {processed_reads_path}: {e}") - + logger.warning( + f"Could not save processed read IDs to {processed_reads_path}: {e}" + ) + # Merge new breakpoints with existing ones # For now, simple merge (could be optimized to cluster nearby breakpoints) all_breakpoints = existing_breakpoints + new_breakpoints - + logger.debug( "Incremental master BED breakpoint extraction completed in %.3fs (existing=%d, new=%d, total=%d)", time.time() - incremental_start, @@ -4632,7 +4957,7 @@ def _generate_master_bed_breakpoint_bed( Generate BED file for master BED breakpoints (new target regions from supplementary alignments). Creates regions with +/- 1 bin_width around breakpoints. Uses counter-based naming consistent with other BED files. - + Args: sample_id: Sample ID fusion_metadata: FusionMetadata object with master BED candidates @@ -4644,15 +4969,20 @@ def _generate_master_bed_breakpoint_bed( # For large datasets, use incremental approach: only process new staging files # and merge with existing breakpoints from previous BED file load_start = time.time() - master_bed_candidates = _load_fusion_candidates_parquet("master_bed_candidates", work_dir, sample_id) - master_bed_size = len(master_bed_candidates) if master_bed_candidates is not None and not master_bed_candidates.empty else 0 + master_bed_candidates = _load_fusion_candidates_parquet( + "master_bed_candidates", work_dir, sample_id + ) + master_bed_size = ( + len(master_bed_candidates) + if master_bed_candidates is not None and not master_bed_candidates.empty + else 0 + ) logger.debug( "Loaded master BED candidates in %.3fs (rows=%d)", time.time() - load_start, master_bed_size, ) - - + # Always prefer incremental extraction when new staging files are available # Only process NEW staging files and merge with existing breakpoints extract_start = time.time() @@ -4662,30 +4992,34 @@ def _generate_master_bed_breakpoint_bed( ) else: # For smaller datasets, process all data (faster for small datasets) - master_bed_breakpoints = _extract_master_bed_breakpoints(fusion_metadata, work_dir=work_dir) + master_bed_breakpoints = _extract_master_bed_breakpoints( + fusion_metadata, work_dir=work_dir + ) logger.debug( "Extracted master BED breakpoints in %.3fs (count=%d)", time.time() - extract_start, len(master_bed_breakpoints) if master_bed_breakpoints else 0, ) - + if not master_bed_breakpoints: - logger.debug("No master BED breakpoints found - skipping BED file generation") + logger.debug( + "No master BED breakpoints found - skipping BED file generation" + ) return False # Get bin_width from CNV analysis if available, otherwise use default bin_width = _get_cnv_bin_width(work_dir, sample_id) - + # Create bed_files directory if it doesn't exist sample_dir = os.path.join(work_dir, sample_id) bed_dir = os.path.join(sample_dir, "bed_files") os.makedirs(bed_dir, exist_ok=True) - + # Sort breakpoints by chromosome, then start position, then end position def sort_key(bp): chrom = bp["chromosome"] start = bp["start"] end = bp["end"] - + # Extract chromosome number for numeric sorting chrom_num = None if chrom.startswith("chr"): @@ -4703,9 +5037,9 @@ def sort_key(bp): chrom_num = 200 # Put non-standard chromosomes at the end else: chrom_num = 200 # Put non-standard chromosomes at the end - + return (chrom_num, start, end) - + sort_start = time.time() sorted_breakpoints = sorted(master_bed_breakpoints, key=sort_key) logger.debug( @@ -4721,14 +5055,14 @@ def sort_key(bp): chrom = bp["chromosome"] start = bp["start"] end = bp["end"] - + # Calculate midpoint for breakpoint midpoint = (start + end) // 2 - + # Create region +/- 1 bin_width around breakpoint region_start = max(0, midpoint - bin_width) region_end = midpoint + bin_width - + # Write BED entry: chrom, start, end, name (master_bed-breakpoint) name = "master_bed-breakpoint" f.write(f"{chrom}\t{region_start}\t{region_end}\t{name}\t0\t.\n") @@ -4741,7 +5075,7 @@ def sort_key(bp): master_bed_bp_file, changed, ) - + # Log summary of read support if master_bed_breakpoints: read_counts = [bp.get("read_count", 0) for bp in master_bed_breakpoints] @@ -4754,8 +5088,10 @@ def sort_key(bp): f"(read support: min={min_reads}, max={max_reads}, avg={avg_reads:.1f})" ) else: - logger.info(f"Generated master BED breakpoint BED file: {master_bed_bp_file} with 0 breakpoints") - + logger.info( + f"Generated master BED breakpoint BED file: {master_bed_bp_file} with 0 breakpoints" + ) + # Generate master BED events summary CSV for GUI (pre-computed to avoid blocking UI) summary_start = time.time() _generate_master_bed_events_summary(sample_id, fusion_metadata, work_dir) @@ -4768,10 +5104,11 @@ def sort_key(bp): time.time() - step_start, ) return changed - + except Exception as e: logger.warning(f"Error generating master BED breakpoint BED file: {e}") import traceback + logger.debug(f"Traceback: {traceback.format_exc()}") return False @@ -4786,16 +5123,16 @@ def _generate_master_bed_events_summary( ) -> None: """ Generate master BED events summary CSV file for GUI display. - + This pre-computes the expensive breakpoint pair clustering so the GUI can just read the results instead of computing them on the UI thread. - + Uses the same breakpoint pair extraction logic as _extract_master_bed_breakpoints: - Identifies reads with both primary and supplementary alignments - Creates breakpoint pairs (primary + supplementary for each read) - Clusters similar breakpoint pairs (both primary and supplementary locations must be close) - Writes both primary and supplementary regions as separate events to CSV - + Args: sample_id: Sample ID fusion_metadata: FusionMetadata object with master BED candidates @@ -4830,37 +5167,47 @@ def _generate_master_bed_events_summary( ) # Load master BED candidates from Parquet load_start = time.time() - master_bed_df = _load_fusion_candidates_parquet("master_bed_candidates", work_dir, sample_id) + master_bed_df = _load_fusion_candidates_parquet( + "master_bed_candidates", work_dir, sample_id + ) logger.debug( "Loaded master BED candidates in %.3fs (rows=%d)", time.time() - load_start, len(master_bed_df) if master_bed_df is not None else 0, ) - + if master_bed_df is None or master_bed_df.empty: - logger.debug("No master BED candidates found - skipping events summary generation") + logger.debug( + "No master BED candidates found - skipping events summary generation" + ) # Write empty file so GUI knows there's no data - summary_file = os.path.join(work_dir, sample_id, "master_bed_events_summary.csv") + summary_file = os.path.join( + work_dir, sample_id, "master_bed_events_summary.csv" + ) pd.DataFrame().to_csv(summary_file, index=False) return - + # Check required columns if "read_id" not in master_bed_df.columns: logger.debug("Missing required column (read_id) in master BED candidates") return - + # Filter to only high-quality supplementary mappings (MapQ >= min_mapq) filter_start = time.time() - if "mapping_quality" in master_bed_df.columns and "col4" in master_bed_df.columns: - high_quality_mask = ( - (master_bed_df["col4"] == "master_bed_region") | - (master_bed_df["mapping_quality"] >= min_mapq) + if ( + "mapping_quality" in master_bed_df.columns + and "col4" in master_bed_df.columns + ): + high_quality_mask = (master_bed_df["col4"] == "master_bed_region") | ( + master_bed_df["mapping_quality"] >= min_mapq ) master_bed_df = master_bed_df[high_quality_mask].copy() - + if master_bed_df.empty: logger.debug(f"No high-quality mappings found (MapQ >= {min_mapq})") - summary_file = os.path.join(work_dir, sample_id, "master_bed_events_summary.csv") + summary_file = os.path.join( + work_dir, sample_id, "master_bed_events_summary.csv" + ) pd.DataFrame().to_csv(summary_file, index=False) return logger.debug( @@ -4868,24 +5215,34 @@ def _generate_master_bed_events_summary( time.time() - filter_start, len(master_bed_df), ) - + # Separate primary and supplementary alignments using col4 if "col4" in master_bed_df.columns: - primary_df = master_bed_df[master_bed_df["col4"] == "master_bed_region"].copy() - supplementary_df = master_bed_df[master_bed_df["col4"] == "master_bed_supplementary"].copy() + primary_df = master_bed_df[ + master_bed_df["col4"] == "master_bed_region" + ].copy() + supplementary_df = master_bed_df[ + master_bed_df["col4"] == "master_bed_supplementary" + ].copy() else: if "is_supplementary" not in master_bed_df.columns: logger.debug("Missing required columns (col4 or is_supplementary)") return - primary_df = master_bed_df[master_bed_df["is_supplementary"] == False].copy() - supplementary_df = master_bed_df[master_bed_df["is_supplementary"] == True].copy() - + primary_df = master_bed_df[ + master_bed_df["is_supplementary"] == False + ].copy() + supplementary_df = master_bed_df[ + master_bed_df["is_supplementary"] == True + ].copy() + if primary_df.empty or supplementary_df.empty: logger.debug("Need both primary and supplementary alignments") - summary_file = os.path.join(work_dir, sample_id, "master_bed_events_summary.csv") + summary_file = os.path.join( + work_dir, sample_id, "master_bed_events_summary.csv" + ) pd.DataFrame().to_csv(summary_file, index=False) return - + # Find reads with both primary and supplementary alignments primary_read_ids = set(primary_df["read_id"].unique()) supplementary_read_ids = set(supplementary_df["read_id"].unique()) @@ -4904,17 +5261,23 @@ def _generate_master_bed_events_summary( ) return reads_with_both = new_reads_with_both - + if not reads_with_both: logger.debug("No reads have both primary and supplementary alignments") - summary_file = os.path.join(work_dir, sample_id, "master_bed_events_summary.csv") + summary_file = os.path.join( + work_dir, sample_id, "master_bed_events_summary.csv" + ) pd.DataFrame().to_csv(summary_file, index=False) return - + # Filter to only reads with both - primary_filtered = primary_df[primary_df["read_id"].isin(reads_with_both)].copy() - supplementary_filtered = supplementary_df[supplementary_df["read_id"].isin(reads_with_both)].copy() - + primary_filtered = primary_df[ + primary_df["read_id"].isin(reads_with_both) + ].copy() + supplementary_filtered = supplementary_df[ + supplementary_df["read_id"].isin(reads_with_both) + ].copy() + # Build breakpoint pairs pair_start = time.time() breakpoint_pairs = [] @@ -4939,17 +5302,31 @@ def _generate_master_bed_events_summary( "Breakpoint pair progress: %d/%d (%.1f%%)", idx, total_reads_with_both, - (idx / total_reads_with_both) * 100 if total_reads_with_both else 100.0, + ( + (idx / total_reads_with_both) * 100 + if total_reads_with_both + else 100.0 + ), ) - read_primaries = primary_by_read.get_group(read_id) if read_id in primary_by_read.groups else pd.DataFrame() - read_supplementaries = supplementary_by_read.get_group(read_id) if read_id in supplementary_by_read.groups else pd.DataFrame() - + read_primaries = ( + primary_by_read.get_group(read_id) + if read_id in primary_by_read.groups + else pd.DataFrame() + ) + read_supplementaries = ( + supplementary_by_read.get_group(read_id) + if read_id in supplementary_by_read.groups + else pd.DataFrame() + ) + if read_primaries.empty or read_supplementaries.empty: empty_pair_skips += 1 continue - + # Use vectorized breakpoint pair creation (much faster than nested loops) - pairs = _create_breakpoint_pairs_vectorized(read_primaries, read_supplementaries, read_id) + pairs = _create_breakpoint_pairs_vectorized( + read_primaries, read_supplementaries, read_id + ) breakpoint_pairs.extend(pairs) logger.debug( "Breakpoint pair build complete: reads_processed=%d, empty_pair_skips=%d, pairs_total=%d", @@ -4958,10 +5335,12 @@ def _generate_master_bed_events_summary( len(breakpoint_pairs), ) processed_read_ids.update(reads_with_both) - + if not breakpoint_pairs: logger.debug("No breakpoint pairs created") - summary_file = os.path.join(work_dir, sample_id, "master_bed_events_summary.csv") + summary_file = os.path.join( + work_dir, sample_id, "master_bed_events_summary.csv" + ) pd.DataFrame().to_csv(summary_file, index=False) return logger.debug(f"breakpoint pairs: {len(breakpoint_pairs)}") @@ -4972,7 +5351,9 @@ def _generate_master_bed_events_summary( # Cluster similar breakpoint pairs using DBSCAN (much faster than O(n²) nested loops) cluster_start = time.time() clustered_pairs = _cluster_breakpoint_pairs_dbscan( - breakpoint_pairs, cluster_distance=cluster_distance, min_read_support=min_read_support + breakpoint_pairs, + cluster_distance=cluster_distance, + min_read_support=min_read_support, ) logger.debug(f"clustered pairs: {len(clustered_pairs)}") logger.debug( @@ -4981,13 +5362,16 @@ def _generate_master_bed_events_summary( ) # Filter for breakpoint pairs with sufficient read support (already filtered in DBSCAN, but keep for safety) supported_pairs = [ - p for p in clustered_pairs - if p["read_count"] >= min_read_support + p for p in clustered_pairs if p["read_count"] >= min_read_support ] - + if not supported_pairs: - logger.debug(f"No breakpoint pairs found with >= {min_read_support} read support") - summary_file = os.path.join(work_dir, sample_id, "master_bed_events_summary.csv") + logger.debug( + f"No breakpoint pairs found with >= {min_read_support} read support" + ) + summary_file = os.path.join( + work_dir, sample_id, "master_bed_events_summary.csv" + ) pd.DataFrame().to_csv(summary_file, index=False) return logger.debug(f"supported pairs: {len(supported_pairs)}") @@ -4995,33 +5379,44 @@ def _generate_master_bed_events_summary( events_start = time.time() events = [] for pair in supported_pairs: - if pair["primary_start"] > 0 and pair["primary_end"] > pair["primary_start"]: - events.append({ - "chromosome": pair["primary_chrom"], - "start": pair["primary_start"], - "end": pair["primary_end"], - "event_type": "breakpoint_pair_primary", - "read_count": pair["read_count"], - "avg_mapping_quality": round(pair.get("primary_avg_mapq", 0.0), 1), - "avg_mapping_span": round(pair.get("primary_avg_span", 0.0), 0), - }) - + if ( + pair["primary_start"] > 0 + and pair["primary_end"] > pair["primary_start"] + ): + events.append( + { + "chromosome": pair["primary_chrom"], + "start": pair["primary_start"], + "end": pair["primary_end"], + "event_type": "breakpoint_pair_primary", + "read_count": pair["read_count"], + "avg_mapping_quality": round( + pair.get("primary_avg_mapq", 0.0), 1 + ), + "avg_mapping_span": round(pair.get("primary_avg_span", 0.0), 0), + } + ) + if pair["supp_start"] > 0 and pair["supp_end"] > pair["supp_start"]: - events.append({ - "chromosome": pair["supp_chrom"], - "start": pair["supp_start"], - "end": pair["supp_end"], - "event_type": "breakpoint_pair_supplementary", - "read_count": pair["read_count"], - "avg_mapping_quality": round(pair.get("supp_avg_mapq", 0.0), 1), - "avg_mapping_span": round(pair.get("supp_avg_span", 0.0), 0), - }) - + events.append( + { + "chromosome": pair["supp_chrom"], + "start": pair["supp_start"], + "end": pair["supp_end"], + "event_type": "breakpoint_pair_supplementary", + "read_count": pair["read_count"], + "avg_mapping_quality": round(pair.get("supp_avg_mapq", 0.0), 1), + "avg_mapping_span": round(pair.get("supp_avg_span", 0.0), 0), + } + ) + if not events: - summary_file = os.path.join(work_dir, sample_id, "master_bed_events_summary.csv") + summary_file = os.path.join( + work_dir, sample_id, "master_bed_events_summary.csv" + ) pd.DataFrame().to_csv(summary_file, index=False) return - + result_df = pd.DataFrame(events) logger.debug(f"merging nearby events: {len(result_df)} events") # Merge nearby duplicate events on the same chromosome @@ -5033,17 +5428,20 @@ def _generate_master_bed_events_summary( time.time() - merge_start, len(result_df), ) - + result_df = result_df.sort_values( - ["read_count", "chromosome", "start"], - ascending=[False, True, True] + ["read_count", "chromosome", "start"], ascending=[False, True, True] ) - + # Write summary CSV file - summary_file = os.path.join(work_dir, sample_id, "master_bed_events_summary.csv") + summary_file = os.path.join( + work_dir, sample_id, "master_bed_events_summary.csv" + ) write_start = time.time() result_df.to_csv(summary_file, index=False) - logger.info(f"Generated master BED events summary: {summary_file} with {len(result_df)} events") + logger.info( + f"Generated master BED events summary: {summary_file} with {len(result_df)} events" + ) logger.debug( "Wrote master BED events summary in %.3fs", time.time() - write_start, @@ -5071,10 +5469,11 @@ def _generate_master_bed_events_summary( "Master BED events summary pipeline completed in %.3fs", time.time() - step_start, ) - + except Exception as e: logger.warning(f"Error generating master BED events summary: {e}") import traceback + logger.debug(f"Traceback: {traceback.format_exc()}") @@ -5087,7 +5486,7 @@ def _generate_fusion_breakpoint_bed( Generate BED file for fusion breakpoints with +/- 1 bin_width regions. Only includes fusions that meet the minimum read support threshold. Uses counter-based naming consistent with other BED files. - + Args: sample_id: Sample ID fusion_metadata: FusionMetadata object with fusion candidates @@ -5109,7 +5508,13 @@ def _generate_fusion_breakpoint_bed( fusion_breakpoints = list(state.get("breakpoints", [])) updated_at = float(state.get("updated_at", 0.0)) breakpoint_set = { - (bp.get("chromosome"), bp.get("start"), bp.get("end"), bp.get("gene", "Unknown"), bp.get("source", "unknown")) + ( + bp.get("chromosome"), + bp.get("start"), + bp.get("end"), + bp.get("gene", "Unknown"), + bp.get("source", "unknown"), + ) for bp in fusion_breakpoints } state_loaded = True @@ -5204,25 +5609,25 @@ def _generate_fusion_breakpoint_bed( ) else: fusion_breakpoints = _extract_fusion_breakpoints(fusion_metadata) - + if not fusion_breakpoints: logger.debug("No fusion breakpoints found - skipping BED file generation") return False - + # Get bin_width from CNV analysis if available, otherwise use default bin_width = _get_cnv_bin_width(work_dir, sample_id) - + # Create bed_files directory if it doesn't exist bed_dir = os.path.join(sample_dir, "bed_files") os.makedirs(bed_dir, exist_ok=True) - + # Sort breakpoints by chromosome, then start position, then end position # Handle chromosome sorting (chr1, chr2, ..., chr10, chr11, ..., chrX, chrY, chrM) def sort_key(bp): chrom = bp["chromosome"] start = bp["start"] end = bp["end"] - + # Extract chromosome number for numeric sorting chrom_num = None if chrom.startswith("chr"): @@ -5240,11 +5645,11 @@ def sort_key(bp): chrom_num = 200 # Put non-standard chromosomes at the end else: chrom_num = 200 # Put non-standard chromosomes at the end - + return (chrom_num, start, end) - + sorted_breakpoints = sorted(fusion_breakpoints, key=sort_key) - + temporary_path = os.path.join(bed_dir, "fusion_breakpoints.tmp") with open(temporary_path, "w") as f: for bp in sorted_breakpoints: @@ -5253,23 +5658,25 @@ def sort_key(bp): end = bp["end"] gene = bp.get("gene", "Unknown") source = bp.get("source", "unknown") - + # Calculate midpoint for breakpoint midpoint = (start + end) // 2 - + # Create region +/- 1 bin_width around breakpoint region_start = max(0, midpoint - bin_width) region_end = midpoint + bin_width - + # Write BED entry: chrom, start, end, name (gene-source) name = f"{gene}-{source}" f.write(f"{chrom}\t{region_start}\t{region_end}\t{name}\n") changed, fusion_bed_file = _commit_versioned_bed_if_changed( temporary_path, bed_dir, "fusion_breakpoints", sample_id, work_dir ) - + if changed: - logger.info(f"Generated fusion breakpoint BED file: {fusion_bed_file} with {len(fusion_breakpoints)} breakpoints") + logger.info( + f"Generated fusion breakpoint BED file: {fusion_bed_file} with {len(fusion_breakpoints)} breakpoints" + ) try: state = { "processed_read_ids": list(processed_read_ids), @@ -5283,10 +5690,11 @@ def sort_key(bp): except Exception as e: logger.warning(f"Could not save fusion breakpoint state: {e}") return changed - + except Exception as e: logger.warning(f"Error generating fusion breakpoint BED file: {e}") import traceback + logger.debug(f"Traceback: {traceback.format_exc()}") return False @@ -5310,21 +5718,27 @@ def find_and_process_bam_files(root_dir): def _get_parquet_paths(work_dir: str, sample_id: str) -> Dict[str, str]: """ Get file paths for Parquet storage of fusion candidates. - + Args: work_dir: Working directory sample_id: Sample identifier - + Returns: Dictionary mapping candidate type to file path """ sample_dir = os.path.join(work_dir, sample_id) os.makedirs(sample_dir, exist_ok=True) - + return { - "target_candidates": os.path.join(sample_dir, f"{sample_id}_target_candidates.parquet"), - "genome_wide_candidates": os.path.join(sample_dir, f"{sample_id}_genome_wide_candidates.parquet"), - "master_bed_candidates": os.path.join(sample_dir, f"{sample_id}_master_bed_candidates.parquet"), + "target_candidates": os.path.join( + sample_dir, f"{sample_id}_target_candidates.parquet" + ), + "genome_wide_candidates": os.path.join( + sample_dir, f"{sample_id}_genome_wide_candidates.parquet" + ), + "master_bed_candidates": os.path.join( + sample_dir, f"{sample_id}_master_bed_candidates.parquet" + ), } @@ -5337,7 +5751,11 @@ def _get_parquet_dataset_dir(work_dir: str, sample_id: str, candidate_type: str) def _migrate_legacy_parquet_to_dataset(work_dir: str, sample_id: str) -> None: """Migrate legacy single-file Parquet into append-only dataset layout.""" - for candidate_type in ["target_candidates", "genome_wide_candidates", "master_bed_candidates"]: + for candidate_type in [ + "target_candidates", + "genome_wide_candidates", + "master_bed_candidates", + ]: dataset_dir = _get_parquet_dataset_dir(work_dir, sample_id, candidate_type) dataset_files = glob.glob(os.path.join(dataset_dir, "*.parquet")) if dataset_files: @@ -5431,7 +5849,9 @@ def _append_fusion_candidates_parquet( os.makedirs(dataset_dir, exist_ok=True) part_path = os.path.join(dataset_dir, f"part_{batch_id:06d}.parquet") if os.path.exists(part_path): - part_path = os.path.join(dataset_dir, f"part_{batch_id:06d}_{int(time.time())}.parquet") + part_path = os.path.join( + dataset_dir, f"part_{batch_id:06d}_{int(time.time())}.parquet" + ) candidates_df.to_parquet( part_path, index=False, engine="pyarrow", compression="snappy" ) @@ -5463,7 +5883,11 @@ def _load_fusion_counts(work_dir: str, sample_id: str) -> Dict[str, int]: except Exception as e: logger.debug(f"Could not read fusion_counts for {sample_id}: {e}") counts = {} - for candidate_type in ["target_candidates", "genome_wide_candidates", "master_bed_candidates"]: + for candidate_type in [ + "target_candidates", + "genome_wide_candidates", + "master_bed_candidates", + ]: dataset_dir = _get_parquet_dataset_dir(work_dir, sample_id, candidate_type) dataset_files = sorted(glob.glob(os.path.join(dataset_dir, "*.parquet"))) if dataset_files: @@ -5494,7 +5918,7 @@ def _save_fusion_candidates_parquet( ) -> None: """ Save fusion candidates DataFrame to Parquet file. - + Args: candidates_df: DataFrame to save candidate_type: Type of candidates ('target_candidates', 'genome_wide_candidates', 'master_bed_candidates') @@ -5503,14 +5927,16 @@ def _save_fusion_candidates_parquet( """ if candidates_df is None or candidates_df.empty: return - + try: parquet_paths = _get_parquet_paths(work_dir, sample_id) parquet_path = parquet_paths.get(candidate_type) - + if parquet_path: - candidates_df.to_parquet(parquet_path, index=False, engine='pyarrow') - logger.debug(f"Saved {len(candidates_df)} {candidate_type} to {parquet_path}") + candidates_df.to_parquet(parquet_path, index=False, engine="pyarrow") + logger.debug( + f"Saved {len(candidates_df)} {candidate_type} to {parquet_path}" + ) except Exception as e: logger.error(f"Error saving {candidate_type} to Parquet: {e}") @@ -5522,12 +5948,12 @@ def _load_fusion_candidates_parquet( ) -> Optional[pd.DataFrame]: """ Load fusion candidates from Parquet file. - + Args: candidate_type: Type of candidates ('target_candidates', 'genome_wide_candidates', 'master_bed_candidates') work_dir: Working directory sample_id: Sample identifier - + Returns: DataFrame if file exists and is readable, None otherwise """ @@ -5540,7 +5966,9 @@ def _load_fusion_candidates_parquet( if df.empty: logger.debug(f"Parquet dataset {dataset_dir} exists but is empty") return None - logger.debug(f"Loaded {len(df)} {candidate_type} from dataset {dataset_dir}") + logger.debug( + f"Loaded {len(df)} {candidate_type} from dataset {dataset_dir}" + ) return df parquet_paths = _get_parquet_paths(work_dir, sample_id) @@ -5554,7 +5982,7 @@ def _load_fusion_candidates_parquet( return df except Exception as e: logger.warning(f"Error loading {candidate_type} from Parquet: {e}") - + return None @@ -5585,7 +6013,9 @@ def _iter_batches(): for path in paths: try: parquet_file = pq.ParquetFile(path) - for batch in parquet_file.iter_batches(columns=columns, batch_size=batch_size): + for batch in parquet_file.iter_batches( + columns=columns, batch_size=batch_size + ): df = batch.to_pandas() if not df.empty: yield df @@ -5604,7 +6034,7 @@ def search_fusion_candidates_by_read_id( ) -> Dict[str, pd.DataFrame]: """ Search for one or more read IDs across all fusion candidate Parquet files. - + Args: work_dir: Working directory sample_id: Sample identifier @@ -5612,7 +6042,7 @@ def search_fusion_candidates_by_read_id( candidate_types: Optional list of candidate types to search. If None, searches all types. Valid types: 'target_candidates', 'genome_wide_candidates', 'master_bed_candidates' report_totals: If True, print summary of total unique reads in each candidate type (default: True) - + Returns: Dictionary mapping candidate_type to DataFrame containing matching rows (empty DataFrame if no matches) """ @@ -5620,38 +6050,50 @@ def search_fusion_candidates_by_read_id( if isinstance(read_ids, str): read_ids = [read_ids] elif not isinstance(read_ids, list): - raise ValueError(f"read_ids must be a string or list of strings, got {type(read_ids)}") - + raise ValueError( + f"read_ids must be a string or list of strings, got {type(read_ids)}" + ) + # Convert to set for efficient lookup read_ids_set = set(read_ids) - + if candidate_types is None: - candidate_types = ["target_candidates", "genome_wide_candidates", "master_bed_candidates"] - + candidate_types = [ + "target_candidates", + "genome_wide_candidates", + "master_bed_candidates", + ] + results = {} totals_summary = {} - + for candidate_type in candidate_types: try: # Load the Parquet file df = _load_fusion_candidates_parquet(candidate_type, work_dir, sample_id) - + if df is not None and not df.empty: if "read_id" not in df.columns: - logger.warning(f"read_id column not found in {candidate_type} DataFrame. Columns: {list(df.columns)}") + logger.warning( + f"read_id column not found in {candidate_type} DataFrame. Columns: {list(df.columns)}" + ) results[candidate_type] = pd.DataFrame() - totals_summary[candidate_type] = {"total_entries": len(df), "unique_reads": 0, "file_exists": True} + totals_summary[candidate_type] = { + "total_entries": len(df), + "unique_reads": 0, + "file_exists": True, + } continue - + # Count unique reads unique_read_count = df["read_id"].nunique() total_entries = len(df) totals_summary[candidate_type] = { "total_entries": total_entries, "unique_reads": unique_read_count, - "file_exists": True + "file_exists": True, } - + # Filter for any of the read IDs in the list matches = df[df["read_id"].isin(read_ids_set)] if not matches.empty: @@ -5671,17 +6113,33 @@ def search_fusion_candidates_by_read_id( f"Found similar read IDs in {candidate_type} (case/whitespace differences). " f"Searching for: {read_ids_set}, found similar: {read_ids_lower & df_read_ids_lower}" ) - results[candidate_type] = pd.DataFrame() # Empty DataFrame to indicate searched but no matches + results[candidate_type] = ( + pd.DataFrame() + ) # Empty DataFrame to indicate searched but no matches else: - results[candidate_type] = pd.DataFrame() # Empty DataFrame if file doesn't exist or is empty - totals_summary[candidate_type] = {"total_entries": 0, "unique_reads": 0, "file_exists": False} + results[candidate_type] = ( + pd.DataFrame() + ) # Empty DataFrame if file doesn't exist or is empty + totals_summary[candidate_type] = { + "total_entries": 0, + "unique_reads": 0, + "file_exists": False, + } except Exception as e: - logger.warning(f"Error searching {candidate_type} for read_ids {read_ids}: {e}") + logger.warning( + f"Error searching {candidate_type} for read_ids {read_ids}: {e}" + ) import traceback + logger.debug(f"Traceback: {traceback.format_exc()}") results[candidate_type] = pd.DataFrame() - totals_summary[candidate_type] = {"total_entries": 0, "unique_reads": 0, "file_exists": False, "error": str(e)} - + totals_summary[candidate_type] = { + "total_entries": 0, + "unique_reads": 0, + "file_exists": False, + "error": str(e), + } + # Log summary of totals if requested if report_totals: logger.info("Total unique reads in each candidate type:") @@ -5695,7 +6153,7 @@ def search_fusion_candidates_by_read_id( logger.warning(f" {candidate_type}: Error - {stats['error']}") else: logger.debug(f" {candidate_type}: File empty or doesn't exist") - + return results @@ -5708,10 +6166,10 @@ def diagnose_master_bed_breakpoint_extraction( ) -> None: """ Diagnostic function to debug why master BED breakpoints aren't being detected. - + This function loads the master_bed_candidates data and traces through the entire breakpoint extraction logic to identify where the process fails. - + Args: work_dir: Working directory sample_id: Sample identifier @@ -5719,38 +6177,50 @@ def diagnose_master_bed_breakpoint_extraction( min_read_support: Minimum read support threshold (default: 3) cluster_distance: Clustering distance threshold in bp (default: 5000) """ - logger.debug("\n" + "="*80) + logger.debug("\n" + "=" * 80) logger.debug(f"DIAGNOSTIC: Master BED Breakpoint Extraction") logger.debug(f"Sample: {sample_id}") logger.debug(f"Work dir: {work_dir}") - logger.debug("="*80 + "\n") - + logger.debug("=" * 80 + "\n") + # Load master_bed_candidates logger.debug("[STEP 1] Loading master_bed_candidates from Parquet...") - master_bed_df = _load_fusion_candidates_parquet("master_bed_candidates", work_dir, sample_id) - + master_bed_df = _load_fusion_candidates_parquet( + "master_bed_candidates", work_dir, sample_id + ) + if master_bed_df is None or master_bed_df.empty: logger.debug(f" ERROR: No master_bed_candidates found or file is empty") - logger.debug(f" Parquet path: {_get_parquet_paths(work_dir, sample_id).get('master_bed_candidates', 'N/A')}") + logger.debug( + f" Parquet path: {_get_parquet_paths(work_dir, sample_id).get('master_bed_candidates', 'N/A')}" + ) return - + logger.debug(f" Loaded {len(master_bed_df)} total entries") logger.debug(f" Columns: {list(master_bed_df.columns)}") - + # Filter to specific read IDs if provided if read_ids: read_ids_set = set(read_ids) - master_bed_df = master_bed_df[master_bed_df["read_id"].isin(read_ids_set)].copy() - logger.debug(f"\n[FILTER] Filtered to {len(master_bed_df)} entries for {len(read_ids)} read IDs") + master_bed_df = master_bed_df[ + master_bed_df["read_id"].isin(read_ids_set) + ].copy() + logger.debug( + f"\n[FILTER] Filtered to {len(master_bed_df)} entries for {len(read_ids)} read IDs" + ) if master_bed_df.empty: - logger.debug(f" ERROR: None of the specified read IDs found in master_bed_candidates") + logger.debug( + f" ERROR: None of the specified read IDs found in master_bed_candidates" + ) logger.debug(f" Looking for: {read_ids}") - all_read_ids = _load_fusion_candidates_parquet("master_bed_candidates", work_dir, sample_id) + all_read_ids = _load_fusion_candidates_parquet( + "master_bed_candidates", work_dir, sample_id + ) if all_read_ids is not None and not all_read_ids.empty: unique_reads = all_read_ids["read_id"].unique()[:10] logger.debug(f" Example read IDs in file: {list(unique_reads)}") return - + # Check required columns logger.debug("\n[STEP 2] Checking required columns...") required_cols = ["read_id", "reference_id", "reference_start", "reference_end"] @@ -5759,72 +6229,88 @@ def diagnose_master_bed_breakpoint_extraction( logger.debug(f" ERROR: Missing required columns: {missing_cols}") return logger.debug(f" All required columns present") - + # Check classification columns logger.debug("\n[STEP 3] Checking alignment classification...") has_col4 = "col4" in master_bed_df.columns has_is_supp = "is_supplementary" in master_bed_df.columns - + if has_col4: logger.debug(f" col4 column found") col4_values = master_bed_df["col4"].value_counts() logger.debug(f" col4 value counts:\n{col4_values}") else: logger.debug(f" WARNING: col4 column not found") - + if has_is_supp: logger.debug(f" is_supplementary column found") is_supp_counts = master_bed_df["is_supplementary"].value_counts() logger.debug(f" is_supplementary value counts:\n{is_supp_counts}") else: logger.debug(f" WARNING: is_supplementary column not found") - + if not has_col4 and not has_is_supp: - logger.debug(f" ERROR: Neither col4 nor is_supplementary column found - cannot classify alignments") + logger.debug( + f" ERROR: Neither col4 nor is_supplementary column found - cannot classify alignments" + ) return - + # Separate primary and supplementary logger.debug("\n[STEP 4] Separating primary and supplementary alignments...") if has_col4: primary_df = master_bed_df[master_bed_df["col4"] == "master_bed_region"].copy() - supplementary_df = master_bed_df[master_bed_df["col4"] == "master_bed_supplementary"].copy() + supplementary_df = master_bed_df[ + master_bed_df["col4"] == "master_bed_supplementary" + ].copy() logger.debug(f" Using col4 for classification") else: primary_df = master_bed_df[master_bed_df["is_supplementary"] == False].copy() - supplementary_df = master_bed_df[master_bed_df["is_supplementary"] == True].copy() + supplementary_df = master_bed_df[ + master_bed_df["is_supplementary"] == True + ].copy() logger.debug(f" Using is_supplementary for classification (fallback)") - + logger.debug(f" Primary alignments: {len(primary_df)}") logger.debug(f" Supplementary alignments: {len(supplementary_df)}") - + if primary_df.empty: logger.debug(f" ERROR: No primary alignments found") return - + if supplementary_df.empty: logger.debug(f" ERROR: No supplementary alignments found") return - + # Check for reads with both - logger.debug("\n[STEP 5] Finding reads with both primary and supplementary alignments...") + logger.debug( + "\n[STEP 5] Finding reads with both primary and supplementary alignments..." + ) primary_read_ids = set(primary_df["read_id"].unique()) supplementary_read_ids = set(supplementary_df["read_id"].unique()) reads_with_both = primary_read_ids & supplementary_read_ids - - logger.debug(f" Reads with primary only: {len(primary_read_ids - supplementary_read_ids)}") - logger.debug(f" Reads with supplementary only: {len(supplementary_read_ids - primary_read_ids)}") + + logger.debug( + f" Reads with primary only: {len(primary_read_ids - supplementary_read_ids)}" + ) + logger.debug( + f" Reads with supplementary only: {len(supplementary_read_ids - primary_read_ids)}" + ) logger.debug(f" Reads with both: {len(reads_with_both)}") - + if not reads_with_both: - logger.debug(f" ERROR: No reads have both primary and supplementary alignments") + logger.debug( + f" ERROR: No reads have both primary and supplementary alignments" + ) if read_ids: logger.debug(f"\n Checking specific read IDs:") for rid in read_ids: has_primary = rid in primary_read_ids has_supp = rid in supplementary_read_ids - logger.debug(f" {rid}: primary={has_primary}, supplementary={has_supp}") + logger.debug( + f" {rid}: primary={has_primary}, supplementary={has_supp}" + ) return - + # Show example reads if read_ids: logger.debug(f"\n Checking specific read IDs:") @@ -5843,7 +6329,9 @@ def diagnose_master_bed_breakpoint_extraction( getattr(row, "col4", "N/A"), getattr(row, "is_supplementary", "N/A"), ) - logger.debug(f" Supplementary: {len(supplementaries)} alignment(s)") + logger.debug( + f" Supplementary: {len(supplementaries)} alignment(s)" + ) for row in supplementaries.itertuples(index=False, name="Row"): logger.debug( " %s:%s-%s (col4=%s, is_supp=%s)", @@ -5853,44 +6341,62 @@ def diagnose_master_bed_breakpoint_extraction( getattr(row, "col4", "N/A"), getattr(row, "is_supplementary", "N/A"), ) - + # Create breakpoint pairs logger.debug("\n[STEP 6] Creating breakpoint pairs...") primary_filtered = primary_df[primary_df["read_id"].isin(reads_with_both)].copy() - supplementary_filtered = supplementary_df[supplementary_df["read_id"].isin(reads_with_both)].copy() - + supplementary_filtered = supplementary_df[ + supplementary_df["read_id"].isin(reads_with_both) + ].copy() + breakpoint_pairs = [] primary_by_read = primary_filtered.groupby("read_id", observed=True) supplementary_by_read = supplementary_filtered.groupby("read_id", observed=True) - + for read_id in reads_with_both: - read_primaries = primary_by_read.get_group(read_id) if read_id in primary_by_read.groups else pd.DataFrame() - read_supplementaries = supplementary_by_read.get_group(read_id) if read_id in supplementary_by_read.groups else pd.DataFrame() - + read_primaries = ( + primary_by_read.get_group(read_id) + if read_id in primary_by_read.groups + else pd.DataFrame() + ) + read_supplementaries = ( + supplementary_by_read.get_group(read_id) + if read_id in supplementary_by_read.groups + else pd.DataFrame() + ) + if read_primaries.empty or read_supplementaries.empty: continue - + # Create pairs using vectorized Pandas operations (much faster than nested loops) - pairs = _create_breakpoint_pairs_vectorized(read_primaries, read_supplementaries, read_id) + pairs = _create_breakpoint_pairs_vectorized( + read_primaries, read_supplementaries, read_id + ) breakpoint_pairs.extend(pairs) - - logger.debug(f" Created {len(breakpoint_pairs)} breakpoint pairs from {len(reads_with_both)} reads") - + + logger.debug( + f" Created {len(breakpoint_pairs)} breakpoint pairs from {len(reads_with_both)} reads" + ) + if not breakpoint_pairs: logger.debug(f" ERROR: No breakpoint pairs created") return - + # Show example pairs if read_ids: logger.debug(f"\n Breakpoint pairs for specified read IDs:") for pair in breakpoint_pairs[:20]: # Show first 20 if pair["read_id"] in read_ids: - logger.debug(f" {pair['read_id']}: primary={pair['primary_chrom']}:{pair['primary_start']}-{pair['primary_end']}, " - f"supp={pair['supp_chrom']}:{pair['supp_start']}-{pair['supp_end']}") - + logger.debug( + f" {pair['read_id']}: primary={pair['primary_chrom']}:{pair['primary_start']}-{pair['primary_end']}, " + f"supp={pair['supp_chrom']}:{pair['supp_start']}-{pair['supp_end']}" + ) + # Cluster pairs - logger.debug(f"\n[STEP 7] Clustering breakpoint pairs (distance={cluster_distance}bp)...") - + logger.debug( + f"\n[STEP 7] Clustering breakpoint pairs (distance={cluster_distance}bp)..." + ) + # Group by chromosome combination pairs_by_chrom = {} for i, pair in enumerate(breakpoint_pairs): @@ -5898,95 +6404,130 @@ def diagnose_master_bed_breakpoint_extraction( if chrom_key not in pairs_by_chrom: pairs_by_chrom[chrom_key] = [] pairs_by_chrom[chrom_key].append((i, pair)) - + logger.debug(f" Grouped into {len(pairs_by_chrom)} chromosome combinations") for chrom_key, pairs in pairs_by_chrom.items(): logger.debug(f" {chrom_key[0]} -> {chrom_key[1]}: {len(pairs)} pairs") - + # Cluster within each chromosome combination clustered_pairs = [] used_indices = set() - + for chrom_key, chrom_pairs in pairs_by_chrom.items(): if len(chrom_pairs) == 0: continue - + # Use DBSCAN clustering for this chromosome combination # Convert to list format expected by DBSCAN function chrom_pair_list = [pair for _, pair in chrom_pairs] chrom_clustered = _cluster_breakpoint_pairs_dbscan( - chrom_pair_list, cluster_distance=cluster_distance, min_read_support=min_read_support + chrom_pair_list, + cluster_distance=cluster_distance, + min_read_support=min_read_support, ) clustered_pairs.extend(chrom_clustered) - + logger.debug(f" Clustered into {len(clustered_pairs)} clusters") - + # Show cluster details if clustered_pairs: logger.debug(f"\n Cluster details:") - sorted_clusters = sorted(clustered_pairs, key=lambda p: p["read_count"], reverse=True) + sorted_clusters = sorted( + clustered_pairs, key=lambda p: p["read_count"], reverse=True + ) for i, cluster in enumerate(sorted_clusters[:10]): # Show top 10 logger.debug(f" Cluster {i+1}: {cluster['read_count']} reads") - logger.debug(f" Primary: {cluster['primary_chrom']}:{cluster['primary_start']}-{cluster['primary_end']}") - logger.debug(f" Supplementary: {cluster['supp_chrom']}:{cluster['supp_start']}-{cluster['supp_end']}") + logger.debug( + f" Primary: {cluster['primary_chrom']}:{cluster['primary_start']}-{cluster['primary_end']}" + ) + logger.debug( + f" Supplementary: {cluster['supp_chrom']}:{cluster['supp_start']}-{cluster['supp_end']}" + ) if read_ids: - cluster_read_ids_list = list(cluster['read_ids']) - matching_reads = [rid for rid in read_ids if rid in cluster_read_ids_list] + cluster_read_ids_list = list(cluster["read_ids"]) + matching_reads = [ + rid for rid in read_ids if rid in cluster_read_ids_list + ] if matching_reads: logger.debug(f" Matching specified reads: {matching_reads}") - + # Filter by min_read_support logger.debug(f"\n[STEP 8] Filtering by min_read_support (>= {min_read_support})...") supported_pairs = [ - p for p in clustered_pairs - if p["read_count"] >= min_read_support + p for p in clustered_pairs if p["read_count"] >= min_read_support ] - - logger.debug(f" Clusters with >= {min_read_support} reads: {len(supported_pairs)} (out of {len(clustered_pairs)} total)") - + + logger.debug( + f" Clusters with >= {min_read_support} reads: {len(supported_pairs)} (out of {len(clustered_pairs)} total)" + ) + if supported_pairs: - logger.debug(f"\n SUCCESS: Found {len(supported_pairs)} supported breakpoint pairs") + logger.debug( + f"\n SUCCESS: Found {len(supported_pairs)} supported breakpoint pairs" + ) for i, pair in enumerate(supported_pairs): - logger.debug(f" {i+1}. {pair['read_count']} reads: " - f"primary={pair['primary_chrom']}:{pair['primary_start']}-{pair['primary_end']}, " - f"supp={pair['supp_chrom']}:{pair['supp_start']}-{pair['supp_end']}") + logger.debug( + f" {i+1}. {pair['read_count']} reads: " + f"primary={pair['primary_chrom']}:{pair['primary_start']}-{pair['primary_end']}, " + f"supp={pair['supp_chrom']}:{pair['supp_start']}-{pair['supp_end']}" + ) else: - logger.debug(f" ERROR: No clusters meet the min_read_support threshold of {min_read_support}") + logger.debug( + f" ERROR: No clusters meet the min_read_support threshold of {min_read_support}" + ) if clustered_pairs: max_read_count = max(p["read_count"] for p in clustered_pairs) logger.debug(f" Maximum read count in any cluster: {max_read_count}") logger.debug(f" Top clusters by read count:") - sorted_clusters = sorted(clustered_pairs, key=lambda p: p["read_count"], reverse=True) + sorted_clusters = sorted( + clustered_pairs, key=lambda p: p["read_count"], reverse=True + ) for i, cluster in enumerate(sorted_clusters[:5]): - logger.debug(f" {i+1}. {cluster['read_count']} reads: " - f"primary={cluster['primary_chrom']}:{cluster['primary_start']}-{cluster['primary_end']}, " - f"supp={cluster['supp_chrom']}:{cluster['supp_start']}-{cluster['supp_end']}") - - logger.debug("\n" + "="*80) + logger.debug( + f" {i+1}. {cluster['read_count']} reads: " + f"primary={cluster['primary_chrom']}:{cluster['primary_start']}-{cluster['primary_end']}, " + f"supp={cluster['supp_chrom']}:{cluster['supp_start']}-{cluster['supp_end']}" + ) + + logger.debug("\n" + "=" * 80) logger.debug("DIAGNOSTIC COMPLETE") - logger.debug("="*80 + "\n") + logger.debug("=" * 80 + "\n") -def _migrate_json_to_parquet(work_dir: str, sample_id: str, fusion_data: Dict[str, Any]) -> None: +def _migrate_json_to_parquet( + work_dir: str, sample_id: str, fusion_data: Dict[str, Any] +) -> None: """ Migrate fusion candidates from JSON lists to Parquet files. This is called when loading old JSON-based metadata. - + Args: work_dir: Working directory sample_id: Sample identifier fusion_data: Dictionary containing lists of candidate dicts """ - for candidate_type in ["target_candidates", "genome_wide_candidates", "master_bed_candidates"]: + for candidate_type in [ + "target_candidates", + "genome_wide_candidates", + "master_bed_candidates", + ]: candidates_list = fusion_data.get(candidate_type, []) - if candidates_list and isinstance(candidates_list, list) and len(candidates_list) > 0: + if ( + candidates_list + and isinstance(candidates_list, list) + and len(candidates_list) > 0 + ): try: # Convert list of dicts to DataFrame df = pd.DataFrame(candidates_list) if not df.empty: # Save to Parquet - _save_fusion_candidates_parquet(df, candidate_type, work_dir, sample_id) - logger.info(f"Migrated {len(candidates_list)} {candidate_type} from JSON to Parquet") + _save_fusion_candidates_parquet( + df, candidate_type, work_dir, sample_id + ) + logger.info( + f"Migrated {len(candidates_list)} {candidate_type} from JSON to Parquet" + ) except Exception as e: logger.warning(f"Error migrating {candidate_type} to Parquet: {e}") @@ -6010,18 +6551,26 @@ def _save_fusion_metadata( # Save fusion candidates to Parquet files (fast storage) if fusion_metadata.fusion_data: - for candidate_type in ["target_candidates", "genome_wide_candidates", "master_bed_candidates"]: + for candidate_type in [ + "target_candidates", + "genome_wide_candidates", + "master_bed_candidates", + ]: candidates = fusion_metadata.fusion_data.get(candidate_type) - + # Handle both DataFrame and list of dicts (for backward compatibility) if candidates is not None: if isinstance(candidates, pd.DataFrame): # Already a DataFrame - save directly - _save_fusion_candidates_parquet(candidates, candidate_type, work_dir, sample_id) + _save_fusion_candidates_parquet( + candidates, candidate_type, work_dir, sample_id + ) elif isinstance(candidates, list) and len(candidates) > 0: # List of dicts - convert to DataFrame first df = pd.DataFrame(candidates) - _save_fusion_candidates_parquet(df, candidate_type, work_dir, sample_id) + _save_fusion_candidates_parquet( + df, candidate_type, work_dir, sample_id + ) # Empty lists are skipped (no file created) # Create metadata file path (for non-candidate metadata only) @@ -6029,12 +6578,16 @@ def _save_fusion_metadata( # Convert dataclass to dictionary for JSON serialization metadata_dict = asdict(fusion_metadata) - + # Remove large candidate lists from JSON (they're now in Parquet files) # Keep empty lists or None to indicate structure if "fusion_data" in metadata_dict and metadata_dict["fusion_data"]: # Replace large lists with empty lists or file indicators - for key in ["target_candidates", "genome_wide_candidates", "master_bed_candidates"]: + for key in [ + "target_candidates", + "genome_wide_candidates", + "master_bed_candidates", + ]: if key in metadata_dict["fusion_data"]: candidates = metadata_dict["fusion_data"][key] # Only store in JSON if it's a small list (for backward compatibility) @@ -6048,7 +6601,9 @@ def _save_fusion_metadata( # Save to JSON file (metadata only, candidates are in Parquet) with open(metadata_path, "w") as f: - json.dump(metadata_dict, f, indent=2, default=str) # default=str handles any non-serializable types + json.dump( + metadata_dict, f, indent=2, default=str + ) # default=str handles any non-serializable types logger.debug(f"Saved fusion metadata to {metadata_path}") @@ -6060,18 +6615,18 @@ def _save_fusion_metadata( def _migrate_old_fusion_metadata(metadata_dict: Dict[str, Any]) -> Dict[str, Any]: """ Migrate old format fusion metadata to new format for backward compatibility. - + This function handles data generated from the main branch and converts it to be compatible with the fusion_bug branch format. - + Args: metadata_dict: Dictionary loaded from JSON file (may be old format) - + Returns: Updated dictionary compatible with current FusionMetadata structure """ migrated = metadata_dict.copy() - + # Ensure all required fields exist with defaults required_fields = { "sample_id": "unknown", @@ -6085,32 +6640,32 @@ def _migrate_old_fusion_metadata(metadata_dict: Dict[str, Any]) -> Dict[str, Any "fusion_data": {}, "target_panel": None, } - + # Fill in missing fields for field, default_value in required_fields.items(): if field not in migrated: migrated[field] = default_value - + # Ensure processing_steps is a list if not isinstance(migrated.get("processing_steps"), list): migrated["processing_steps"] = [] - + # Ensure analysis_results is a dict if not isinstance(migrated.get("analysis_results"), dict): migrated["analysis_results"] = {} - + # Ensure fusion_data is a dict with required structure if not isinstance(migrated.get("fusion_data"), dict): migrated["fusion_data"] = {} - + fusion_data = migrated["fusion_data"] - + # Ensure required fusion_data keys exist if "target_candidates" not in fusion_data: fusion_data["target_candidates"] = [] if "genome_wide_candidates" not in fusion_data: fusion_data["genome_wide_candidates"] = [] - + # Migrate old CSV-based data to fusion_data structure if needed # Check if we have CSV files but no fusion_data entries # Try to determine sample directory from various paths @@ -6127,60 +6682,80 @@ def _migrate_old_fusion_metadata(metadata_dict: Dict[str, Any]) -> Dict[str, Any potential_dir = os.path.dirname(file_path) if os.path.basename(potential_dir): # If there's a parent directory sample_dir = potential_dir - + if sample_dir and os.path.exists(sample_dir): target_csv = os.path.join(sample_dir, "target_fusion.csv") genome_csv = os.path.join(sample_dir, "genome_wide_fusion.csv") - + # If fusion_data is empty but CSV files exist, try to load from CSVs - if (not fusion_data.get("target_candidates") and - not fusion_data.get("genome_wide_candidates") and - (os.path.exists(target_csv) or os.path.exists(genome_csv))): - - logger.info(f"Migrating fusion data from CSV files for {migrated.get('sample_id', 'unknown')}") - + if ( + not fusion_data.get("target_candidates") + and not fusion_data.get("genome_wide_candidates") + and (os.path.exists(target_csv) or os.path.exists(genome_csv)) + ): + + logger.info( + f"Migrating fusion data from CSV files for {migrated.get('sample_id', 'unknown')}" + ) + try: # Load target fusion CSV if it exists if os.path.exists(target_csv): try: target_df = pd.read_csv(target_csv) if not target_df.empty: - fusion_data["target_candidates"] = target_df.to_dict("records") - logger.info(f"Migrated {len(fusion_data['target_candidates'])} target candidates from CSV") + fusion_data["target_candidates"] = target_df.to_dict( + "records" + ) + logger.info( + f"Migrated {len(fusion_data['target_candidates'])} target candidates from CSV" + ) except Exception as e: logger.warning(f"Could not migrate target fusion CSV: {e}") - + # Load genome-wide fusion CSV if it exists if os.path.exists(genome_csv): try: genome_df = pd.read_csv(genome_csv) if not genome_df.empty: - fusion_data["genome_wide_candidates"] = genome_df.to_dict("records") - logger.info(f"Migrated {len(fusion_data['genome_wide_candidates'])} genome-wide candidates from CSV") + fusion_data["genome_wide_candidates"] = genome_df.to_dict( + "records" + ) + logger.info( + f"Migrated {len(fusion_data['genome_wide_candidates'])} genome-wide candidates from CSV" + ) except Exception as e: logger.warning(f"Could not migrate genome-wide fusion CSV: {e}") - + # Update analysis results counts - if fusion_data.get("target_candidates") or fusion_data.get("genome_wide_candidates"): - migrated["analysis_results"]["target_candidates_count"] = len(fusion_data.get("target_candidates", [])) - migrated["analysis_results"]["genome_wide_candidates_count"] = len(fusion_data.get("genome_wide_candidates", [])) + if fusion_data.get("target_candidates") or fusion_data.get( + "genome_wide_candidates" + ): + migrated["analysis_results"]["target_candidates_count"] = len( + fusion_data.get("target_candidates", []) + ) + migrated["analysis_results"]["genome_wide_candidates_count"] = len( + fusion_data.get("genome_wide_candidates", []) + ) migrated["processing_steps"].append("migrated_from_csv") - + except Exception as e: logger.warning(f"Error during CSV migration: {e}") - + return migrated -def _create_metadata_from_csv_files(work_dir: str, sample_id: str) -> Optional[FusionMetadata]: +def _create_metadata_from_csv_files( + work_dir: str, sample_id: str +) -> Optional[FusionMetadata]: """ Create FusionMetadata from CSV files when metadata JSON doesn't exist. This provides backward compatibility for data generated from main branch. - + Args: work_dir: Working directory sample_id: Sample ID - + Returns: FusionMetadata object if CSV files found, None otherwise """ @@ -6188,19 +6763,19 @@ def _create_metadata_from_csv_files(work_dir: str, sample_id: str) -> Optional[F sample_dir = os.path.join(work_dir, sample_id) if not os.path.exists(sample_dir): return None - + target_csv = os.path.join(sample_dir, "target_fusion.csv") genome_csv = os.path.join(sample_dir, "genome_wide_fusion.csv") - + # Check if CSV files exist has_target = os.path.exists(target_csv) has_genome = os.path.exists(genome_csv) - + if not (has_target or has_genome): return None - + logger.info(f"Creating fusion metadata from CSV files for {sample_id}") - + # Create new metadata object fusion_metadata = FusionMetadata( sample_id=sample_id, @@ -6210,50 +6785,56 @@ def _create_metadata_from_csv_files(work_dir: str, sample_id: str) -> Optional[F genome_wide_fusion_path=genome_csv if has_genome else None, target_panel=None, # Unknown for old data ) - + # Load CSV data into fusion_data fusion_data = {} - + if has_target: try: target_df = pd.read_csv(target_csv) if not target_df.empty: fusion_data["target_candidates"] = target_df.to_dict("records") - logger.info(f"Loaded {len(fusion_data['target_candidates'])} target candidates from CSV") + logger.info( + f"Loaded {len(fusion_data['target_candidates'])} target candidates from CSV" + ) except Exception as e: logger.warning(f"Could not load target fusion CSV: {e}") fusion_data["target_candidates"] = [] else: fusion_data["target_candidates"] = [] - + if has_genome: try: genome_df = pd.read_csv(genome_csv) if not genome_df.empty: fusion_data["genome_wide_candidates"] = genome_df.to_dict("records") - logger.info(f"Loaded {len(fusion_data['genome_wide_candidates'])} genome-wide candidates from CSV") + logger.info( + f"Loaded {len(fusion_data['genome_wide_candidates'])} genome-wide candidates from CSV" + ) except Exception as e: logger.warning(f"Could not load genome-wide fusion CSV: {e}") fusion_data["genome_wide_candidates"] = [] else: fusion_data["genome_wide_candidates"] = [] - + fusion_metadata.fusion_data = fusion_data fusion_metadata.analysis_results = { "target_candidates_count": len(fusion_data.get("target_candidates", [])), - "genome_wide_candidates_count": len(fusion_data.get("genome_wide_candidates", [])), + "genome_wide_candidates_count": len( + fusion_data.get("genome_wide_candidates", []) + ), } fusion_metadata.processing_steps = ["created_from_csv_files"] - + # Save the created metadata for future use try: _save_fusion_metadata(fusion_metadata, work_dir, sample_id) logger.info(f"Saved created fusion metadata for {sample_id}") except Exception as e: logger.warning(f"Could not save created metadata: {e}") - + return fusion_metadata - + except Exception as e: logger.warning(f"Error creating metadata from CSV files: {e}") return None @@ -6290,19 +6871,25 @@ def _load_fusion_metadata(work_dir: str, sample_id: str) -> Optional[FusionMetad # Check if we have Parquet files (new format) or JSON lists (old format) fusion_data = metadata_dict.get("fusion_data", {}) parquet_loaded = False - - for candidate_type in ["target_candidates", "genome_wide_candidates", "master_bed_candidates"]: + + for candidate_type in [ + "target_candidates", + "genome_wide_candidates", + "master_bed_candidates", + ]: dataset_dir = _get_parquet_dataset_dir(work_dir, sample_id, candidate_type) dataset_files = glob.glob(os.path.join(dataset_dir, "*.parquet")) legacy_path = _get_parquet_paths(work_dir, sample_id).get(candidate_type) - has_parquet = bool(dataset_files) or (legacy_path and os.path.exists(legacy_path)) - + has_parquet = bool(dataset_files) or ( + legacy_path and os.path.exists(legacy_path) + ) + if has_parquet: # Parquet exists - avoid loading into memory, keep empty list for metadata fusion_data[candidate_type] = [] parquet_loaded = True continue - + if candidate_type in fusion_data: # Check if JSON has data (old format) candidates_list = fusion_data.get(candidate_type, []) @@ -6311,7 +6898,7 @@ def _load_fusion_metadata(work_dir: str, sample_id: str) -> Optional[FusionMetad _migrate_json_to_parquet(work_dir, sample_id, fusion_data) parquet_loaded = True # Keep the list for now (will be replaced by Parquet on next save) - + # Update metadata_dict with loaded/migrated fusion_data metadata_dict["fusion_data"] = fusion_data @@ -6323,8 +6910,11 @@ def _load_fusion_metadata(work_dir: str, sample_id: str) -> Optional[FusionMetad logger.warning(f"Extra fields in metadata, filtering: {e}") # Get only the fields that FusionMetadata expects from dataclasses import fields + valid_fields = {f.name for f in fields(FusionMetadata)} - filtered_dict = {k: v for k, v in metadata_dict.items() if k in valid_fields} + filtered_dict = { + k: v for k, v in metadata_dict.items() if k in valid_fields + } fusion_metadata = FusionMetadata(**filtered_dict) # Ensure the loaded metadata has the correct structure @@ -6350,11 +6940,11 @@ def _load_fusion_metadata(work_dir: str, sample_id: str) -> Optional[FusionMetad fusion_metadata.analysis_results["target_candidates_count"] = counts.get( "target_candidates", 0 ) - fusion_metadata.analysis_results["genome_wide_candidates_count"] = counts.get( - "genome_wide_candidates", 0 + fusion_metadata.analysis_results["genome_wide_candidates_count"] = ( + counts.get("genome_wide_candidates", 0) ) - fusion_metadata.analysis_results["master_bed_candidates_count"] = counts.get( - "master_bed_candidates", 0 + fusion_metadata.analysis_results["master_bed_candidates_count"] = ( + counts.get("master_bed_candidates", 0) ) # Save migrated metadata back to disk if migration occurred @@ -6375,6 +6965,7 @@ def _load_fusion_metadata(work_dir: str, sample_id: str) -> Optional[FusionMetad except Exception as e: logger.warning(f"Error loading fusion metadata: {str(e)}") import traceback + logger.debug(f"Traceback: {traceback.format_exc()}") return None @@ -6448,7 +7039,7 @@ def _merge_fusion_metadata_objects( def _annotate_results(result: pd.DataFrame) -> Tuple[pd.DataFrame, pd.Series]: """Annotates the result DataFrame with tags and colors with memory optimization. - + Filters fusion candidates to require at least 3 supporting reads per gene pair to ensure reliable fusion detection and reduce false positives. """ @@ -6474,7 +7065,10 @@ def _annotate_results(result: pd.DataFrame) -> Tuple[pd.DataFrame, pd.Series]: # Find good pairs (gene pairs supported by minimum threshold for reliable fusion detection) min_support = get_fusion_threshold("read_support") - goodpairs = result.groupby("tag", observed=True)["read_id"].transform("nunique") >= min_support + goodpairs = ( + result.groupby("tag", observed=True)["read_id"].transform("nunique") + >= min_support + ) return result, goodpairs @@ -6569,8 +7163,12 @@ def _get_reads(reads: pd.DataFrame) -> pd.DataFrame: df["end"] = df["end"].astype(int) except ValueError as e: logger.error(f"Error converting start/end to int: {str(e)}") - problematic_start = df[df['start'].apply(lambda x: not str(x).isdigit())]['start'].tolist() - problematic_end = df[df['end'].apply(lambda x: not str(x).isdigit())]['end'].tolist() + problematic_start = df[df["start"].apply(lambda x: not str(x).isdigit())][ + "start" + ].tolist() + problematic_end = df[df["end"].apply(lambda x: not str(x).isdigit())][ + "end" + ].tolist() logger.debug(f"Problematic values in start: {problematic_start}") logger.debug(f"Problematic values in end: {problematic_end}") raise @@ -6602,23 +7200,23 @@ def _get_reads(reads: pd.DataFrame) -> pd.DataFrame: def _generate_fusion_summary_files(output_file: str, processed_data: dict) -> None: """Generate summary files for the summary component. - + Creates sv_count.txt and fusion_results.csv files that can be easily read by the summary component without needing to parse pickle files. """ try: import os from pathlib import Path - + # Get the directory where the output file is located output_dir = Path(output_file).parent - + # Extract fusion counts candidate_count = processed_data.get("candidate_count", 0) - + # Determine if this is target panel or genome-wide based on filename is_target_panel = "master" in Path(output_file).name - + # Generate sv_count.txt file (simple count) sv_count_file = output_dir / "sv_count.txt" if is_target_panel: @@ -6629,29 +7227,34 @@ def _generate_fusion_summary_files(output_file: str, processed_data: dict) -> No # For genome-wide, write the genome-wide count with open(sv_count_file, "w") as f: f.write(str(candidate_count)) - + # Generate fusion_results.csv file with detailed information fusion_results_file = output_dir / "fusion_results.csv" with open(fusion_results_file, "w", newline="") as f: import csv + writer = csv.writer(f) - + # Write header if is_target_panel: writer.writerow(["target_fusions", "genome_fusions"]) - writer.writerow([candidate_count, 0]) # Target panel only has target fusions + writer.writerow( + [candidate_count, 0] + ) # Target panel only has target fusions else: writer.writerow(["target_fusions", "genome_fusions"]) - writer.writerow([0, candidate_count]) # Genome-wide only has genome fusions - + writer.writerow( + [0, candidate_count] + ) # Genome-wide only has genome fusions + # Generate a combined summary file that has both counts # This will be overwritten each time, with the final one containing the correct totals summary_file = output_dir / "fusion_summary.csv" - + # Try to read existing summary to get both counts target_count = 0 genome_count = 0 - + if summary_file.exists(): try: with open(summary_file, "r", newline="") as f: @@ -6662,21 +7265,23 @@ def _generate_fusion_summary_files(output_file: str, processed_data: dict) -> No break except Exception: pass # If we can't read it, we'll start fresh - + # Update the appropriate count - preserve existing counts from other processing if is_target_panel: target_count = candidate_count else: genome_count = candidate_count - + # Write the updated summary with both counts preserved with open(summary_file, "w", newline="") as f: writer = csv.writer(f) writer.writerow(["target_fusions", "genome_fusions"]) writer.writerow([target_count, genome_count]) - - logger.info(f"[Fusion] Updated summary file: target={target_count}, genome={genome_count}") - + + logger.info( + f"[Fusion] Updated summary file: target={target_count}, genome={genome_count}" + ) + except Exception as e: logger.warning(f"Failed to generate fusion summary files: {e}") @@ -6685,7 +7290,7 @@ def preprocess_fusion_data_standalone( fusion_data: pd.DataFrame, output_file: str ) -> None: """Standalone version of fusion data preprocessing for CPU-bound execution with memory optimization. - + Applies filtering to require at least 3 supporting reads per gene pair to ensure reliable fusion detection while maintaining quality control. """ @@ -6723,7 +7328,7 @@ def preprocess_fusion_data_standalone( .unique() .tolist() ) - + # Filter out empty strings and create valid gene pairs valid_gene_pairs = [] for pair in gene_pairs: @@ -6746,7 +7351,9 @@ def preprocess_fusion_data_standalone( ] unique_read_count = gene_group_reads["read_id"].nunique() min_support = get_fusion_threshold("read_support") - if unique_read_count >= min_support: # Require minimum supporting reads for reliable fusion detection + if ( + unique_read_count >= min_support + ): # Require minimum supporting reads for reliable fusion detection gene_groups.append(gene_group) processed_data.update( @@ -6760,7 +7367,7 @@ def preprocess_fusion_data_standalone( # Save processed data as pickle for efficient loading # Use atomic writes to prevent truncation from read/write clashes import pickle - + # Write to temporary file first, then atomically rename to prevent truncation # This ensures readers never see a partially written file temp_file = output_file + ".tmp" @@ -6777,16 +7384,17 @@ def preprocess_fusion_data_standalone( except: pass raise - + # Generate summary files for the summary component _generate_fusion_summary_files(output_file, processed_data) - + # Free the processed data immediately del processed_data except Exception as e: logger.error(f"Error pre-processing fusion data: {str(e)}") import traceback + logger.debug(f"Exception details: {traceback.format_exc()}") @@ -6881,51 +7489,62 @@ def finalize_fusion_accumulation_for_sample( """ Force final accumulation of any remaining staged fusion files for a sample. This will also generate the final master BED file once all files are processed. - + This should be called when: - All files for a sample have been processed - The workflow is completing - There are staged files that haven't been accumulated yet - + Args: sample_id: Sample identifier work_dir: Working directory containing sample data target_panel: Target panel type reference: Optional reference genome path - + Returns: Dictionary with accumulation results """ try: logger.info(f"Finalizing fusion accumulation for sample {sample_id}") - + # Check if there are pending files pending_count = _get_pending_count(work_dir, sample_id) - + if pending_count == 0: - logger.info(f"No pending fusion files for {sample_id} - checking if final master BED generation is needed") + logger.info( + f"No pending fusion files for {sample_id} - checking if final master BED generation is needed" + ) # Even if no pending files, we should generate master BED if it hasn't been generated yet # This handles the case where all files were accumulated but master BED wasn't generated else: - logger.info(f"Found {pending_count} pending fusion files for {sample_id} - forcing accumulation") - + logger.info( + f"Found {pending_count} pending fusion files for {sample_id} - forcing accumulation" + ) + # Force accumulation of remaining files (batch_size=1 ensures accumulation runs) # force=True will trigger master BED generation in _generate_output_files result = accumulate_fusion_candidates( - work_dir, sample_id, target_panel, force=True, batch_size=1, reference=reference + work_dir, + sample_id, + target_panel, + force=True, + batch_size=1, + reference=reference, ) - + # If accumulation succeeded but master BED wasn't generated (e.g., no pending files), # generate it now as a final step if result.get("status") == "success" or pending_count == 0: try: from robin.analysis.master_bed_generator import generate_master_bed - + # Get analysis counter analysis_counter = _load_analysis_counter(sample_id, work_dir) - + # Generate master BED file (final generation after all files processed) - logger.info(f"Generating final master BED file for sample {sample_id} (counter: {analysis_counter})") + logger.info( + f"Generating final master BED file for sample {sample_id} (counter: {analysis_counter})" + ) master_bed_path = generate_master_bed( sample_id=sample_id, work_dir=work_dir, @@ -6940,12 +7559,13 @@ def finalize_fusion_accumulation_for_sample( result["master_bed_path"] = master_bed_path except Exception as e: logger.warning(f"Could not generate final master BED file: {e}") - + logger.info(f"Final fusion accumulation complete for {sample_id}: {result}") return result - + except Exception as e: logger.error(f"Error during final fusion accumulation for {sample_id}: {e}") import traceback + logger.error(traceback.format_exc()) return {"status": "error", "error": str(e), "sample_id": sample_id} diff --git a/src/robin/analysis/itd_work.py b/src/robin/analysis/itd_work.py index 52ad6a68..3d23896c 100644 --- a/src/robin/analysis/itd_work.py +++ b/src/robin/analysis/itd_work.py @@ -192,7 +192,9 @@ def load_itd_hotspots(path: Optional[str | Path] = None) -> Dict[str, ItdHotspot min_supporting_reads=int(entry.get("min_supporting_reads", 2)), label=str(entry.get("label", "ITD")), transcript=( - str(entry["transcript"]) if entry.get("transcript") is not None else None + str(entry["transcript"]) + if entry.get("transcript") is not None + else None ), ) return hotspots @@ -658,9 +660,7 @@ def filter_hotspots_by_panel( """Keep hotspots whose gene symbol appears in the active target panel.""" symbols = panel_gene_symbols(panel) filtered = { - gene: hotspot - for gene, hotspot in hotspots.items() - if gene.upper() in symbols + gene: hotspot for gene, hotspot in hotspots.items() if gene.upper() in symbols } logger.info( "ITD hotspots after panel filter (%s): %s (from %d configured)", @@ -869,7 +869,9 @@ def call_events_from_counts( changed = True # Representative = highest-support member; support = sum of unique bins. - representative = max(members, key=lambda row: (row["support"], -row["position"])) + representative = max( + members, key=lambda row: (row["support"], -row["position"]) + ) support = sum(member["support"] for member in members) # Spanning depth at the representative anchor (includes ref + alt reads). depth = int(representative["coverage"]) @@ -913,8 +915,8 @@ def call_events_from_counts( exon = hotspot.exon_at(int(event["position"])) event["exon_number"] = exon.number if exon else None event["transcript_id"] = ( - (exon.transcript_id if exon else None) or hotspot.transcript - ) + exon.transcript_id if exon else None + ) or hotspot.transcript event["exon_id"] = exon.exon_id if exon else None filtered.append(event) @@ -1057,7 +1059,9 @@ def process_bam_itd_with_staging( staging_path = os.path.join(staging_dir, f"itd_{counter:06d}.parquet") if not counts.empty: - counts.to_parquet(staging_path, index=False, engine="pyarrow", compression="snappy") + counts.to_parquet( + staging_path, index=False, engine="pyarrow", compression="snappy" + ) logger.info( "ITD staging: wrote %d indel count rows from %s", len(counts), @@ -1065,7 +1069,9 @@ def process_bam_itd_with_staging( ) else: # Touch an empty marker so pending accounting stays aligned with fusion. - counts.to_parquet(staging_path, index=False, engine="pyarrow", compression="snappy") + counts.to_parquet( + staging_path, index=False, engine="pyarrow", compression="snappy" + ) logger.debug("ITD staging: no insertions in hotspots for %s", bam_path) pending = _increment_pending_count(work_dir, sample_id, delta=1) @@ -1088,10 +1094,9 @@ def _aggregate_count_frames(frames: Iterable[pd.DataFrame]) -> pd.DataFrame: return pd.DataFrame(columns=ITD_COUNT_COLUMNS) combined = pd.concat(pieces, ignore_index=True) - grouped = ( - combined.groupby(["gene", "chrom", "position", "length", "label"], as_index=False) - .agg(support=("support", "sum"), coverage=("coverage", "sum")) - ) + grouped = combined.groupby( + ["gene", "chrom", "position", "length", "label"], as_index=False + ).agg(support=("support", "sum"), coverage=("coverage", "sum")) grouped["bam_path"] = "" return grouped[ITD_COUNT_COLUMNS] @@ -1184,7 +1189,9 @@ def accumulate_itd_candidates( if not batch.empty: part_id = len(list(Path(dataset_dir).glob("part_*.parquet"))) part_path = os.path.join(dataset_dir, f"part_{part_id:06d}.parquet") - batch.to_parquet(part_path, index=False, engine="pyarrow", compression="snappy") + batch.to_parquet( + part_path, index=False, engine="pyarrow", compression="snappy" + ) for path in staging_files: try: @@ -1224,8 +1231,12 @@ def accumulate_itd_candidates( "start": hotspot.start, "end": hotspot.end, "n_events": int(len(gene_events)), - "max_support": int(gene_events["support"].max()) if len(gene_events) else 0, - "max_vaf": float(gene_events["vaf"].max()) if len(gene_events) else 0.0, + "max_support": ( + int(gene_events["support"].max()) if len(gene_events) else 0 + ), + "max_vaf": ( + float(gene_events["vaf"].max()) if len(gene_events) else 0.0 + ), } ) pd.DataFrame(summary_rows).to_csv(summary_path, index=False) @@ -1401,7 +1412,9 @@ def collect_supporting_read_qc_for_event( "softclip_left": soft_l, "softclip_right": soft_r, "softclip_frac": round(soft_frac, 4), - "mean_ins_baseq": None if mean_ins_q != mean_ins_q else round(mean_ins_q, 2), + "mean_ins_baseq": ( + None if mean_ins_q != mean_ins_q else round(mean_ins_q, 2) + ), "min_ins_baseq": None if min_ins_q != min_ins_q else int(min_ins_q), "nm": int(nm) if nm is not None else None, } @@ -1526,9 +1539,7 @@ def write_itd_read_qc( sample_dir = Path(sample_dir) if bam_paths is None: bam_paths = sorted( - str(path) - for path in sample_dir.glob("batch_*.bam") - if path.is_file() + str(path) for path in sample_dir.glob("batch_*.bam") if path.is_file() ) read_qc, event_qc = collect_itd_read_qc(events, list(bam_paths or [])) read_path = sample_dir / "itd_read_qc.csv" diff --git a/src/robin/analysis/lamprey_analysis.py b/src/robin/analysis/lamprey_analysis.py index af865c68..9ffdbff8 100644 --- a/src/robin/analysis/lamprey_analysis.py +++ b/src/robin/analysis/lamprey_analysis.py @@ -111,7 +111,9 @@ def load_probe_names(bed_path: str | os.PathLike) -> List[str]: return names -def load_probe_position_map(probes_bed_path: str | os.PathLike) -> Dict[tuple[str, int], str]: +def load_probe_position_map( + probes_bed_path: str | os.PathLike, +) -> Dict[tuple[str, int], str]: """ Build (chrom, position) → probe_id from Lamprey ``probe_hg38.bed``. @@ -163,21 +165,21 @@ def bedmethyl_to_probe_calls( start_col = ( "chromStart" if "chromStart" in df.columns - else "start_pos" - if "start_pos" in df.columns - else "start" + else "start_pos" if "start_pos" in df.columns else "start" ) value_col = ( "percent_modified" if "percent_modified" in df.columns - else "fraction" - if "fraction" in df.columns - else "score" - if "score" in df.columns - else None + else ( + "fraction" + if "fraction" in df.columns + else "score" if "score" in df.columns else None + ) ) if value_col is None: - raise ValueError("Methylation dataframe missing percent_modified/fraction/score") + raise ValueError( + "Methylation dataframe missing percent_modified/fraction/score" + ) sums: Dict[str, float] = {} counts: Dict[str, int] = {} @@ -334,19 +336,13 @@ def predict_from_probe_calls( self, probe_calls: Mapping[str, int] ) -> LampreyPrediction: vector, n_used = build_feature_vector(self.probe_names, probe_calls) - temperature = float( - self.temps[np.argmin(np.abs(self.bin_centers - n_used))] - ) + temperature = float(self.temps[np.argmin(np.abs(self.bin_centers - n_used))]) batch = vector.reshape(1, -1) - outputs = self.session.run( - [self.output_name], {self.input_name: batch} - )[0] + outputs = self.session.run([self.output_name], {self.input_name: batch})[0] outputs = outputs / np.exp(temperature) outputs = _softmax(outputs, axis=1) probs = outputs[0] - scores = { - name: float(score) for name, score in zip(self.class_names, probs) - } + scores = {name: float(score) for name, score in zip(self.class_names, probs)} top_idx = int(np.argmax(probs)) top_score = float(probs[top_idx]) return LampreyPrediction( @@ -544,7 +540,9 @@ def process_multiple_files( except Exception: pass if analysis_result["files_processed"] == 0: - analysis_result["error_message"] = "No files could be processed successfully" + analysis_result["error_message"] = ( + "No files could be processed successfully" + ) analysis_result["processing_steps"].append("no_files_processed") return analysis_result analysis_result["processing_steps"].append("analysis_complete") @@ -596,7 +594,9 @@ def lamprey_handler(job, work_dir=None): os.path.basename(bam_path), ) if not parquet_paths: - error_msg = "No parquet paths found from bed conversion results in batch" + error_msg = ( + "No parquet paths found from bed conversion results in batch" + ) if suppress_expected: job.context.add_result( "lamprey_analysis", @@ -673,9 +673,7 @@ def lamprey_handler(job, work_dir=None): else: os.makedirs(work_dir, exist_ok=True) - analyzer = LampreyAnalysis( - work_dir=work_dir, genome_build=DEFAULT_GENOME_BUILD - ) + analyzer = LampreyAnalysis(work_dir=work_dir, genome_build=DEFAULT_GENOME_BUILD) result = analyzer.process_parquet_file(parquet_path, sample_id) job.context.add_metadata("lamprey_analysis", result.results) job.context.add_metadata("lamprey_processing_steps", result.processing_steps) @@ -720,4 +718,4 @@ def lamprey_handler(job, work_dir=None): ) return job.context.add_error("lamprey_analysis", str(exc)) - logger.error("Error in Lamprey analysis for %s: %s", job.context.filepath, exc) \ No newline at end of file + logger.error("Error in Lamprey analysis for %s: %s", job.context.filepath, exc) diff --git a/src/robin/analysis/lightweight_gene_analysis.py b/src/robin/analysis/lightweight_gene_analysis.py index 70805cb5..b694ce53 100644 --- a/src/robin/analysis/lightweight_gene_analysis.py +++ b/src/robin/analysis/lightweight_gene_analysis.py @@ -1,4 +1,4 @@ -q#!/usr/bin/env python3 +q #!/usr/bin/env python3 """ Lightweight Gene Analysis Module for robin @@ -56,16 +56,18 @@ while still providing valuable insights into genes of interest. """ -import os import logging -from typing import Dict, Any, Optional, List, Tuple +import os from dataclasses import dataclass from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + import numpy as np import pandas as pd import pysam -from robin.logging_config import get_job_logger + from robin.analysis.variant_classification import is_clinvar_significant_from_mapping +from robin.logging_config import get_job_logger @dataclass @@ -775,9 +777,7 @@ def _perform_targeted_pileup( start_pos = gene_coords["start"] end_pos = gene_coords["end"] - print( - f" Analyzing region: {chrom_name}:{start_pos:,}-{end_pos:,}" - ) + print(f" Analyzing region: {chrom_name}:{start_pos:,}-{end_pos:,}") self.logger.debug( f"Analyzing pileup for {chrom_name}:{start_pos}-{end_pos}" ) @@ -927,7 +927,7 @@ def _analyze_variant_evidence( # For indels, we need to check if the reference sequence matches what we expect if is_indel: - + # NOTE: Indel analysis is complex and requires: # 1. Reference genome sequence validation # 2. Multi-position pileup analysis @@ -966,7 +966,7 @@ def _analyze_variant_evidence( alt_support = 0 vaf = 0.0 else: - + # For insertions, we need to look at the actual sequence context # This requires more sophisticated pileup analysis # For now, mark as requiring manual review diff --git a/src/robin/analysis/marlin_analysis.py b/src/robin/analysis/marlin_analysis.py index 8c78eacd..0b5d996a 100644 --- a/src/robin/analysis/marlin_analysis.py +++ b/src/robin/analysis/marlin_analysis.py @@ -73,7 +73,9 @@ def _normalize_chrom(value: object) -> str: return f"chr{text}" -def load_probe_position_map(probes_bed_path: str | os.PathLike) -> Dict[tuple[str, int], str]: +def load_probe_position_map( + probes_bed_path: str | os.PathLike, +) -> Dict[tuple[str, int], str]: """ Build (chrom, ref_position) → probe_id from a MARLIN probe BED. @@ -121,18 +123,16 @@ def bedmethyl_to_probe_values( start_col = ( "chromStart" if "chromStart" in df.columns - else "start_pos" - if "start_pos" in df.columns - else "start" + else "start_pos" if "start_pos" in df.columns else "start" ) value_col = ( "percent_modified" if "percent_modified" in df.columns - else "fraction" - if "fraction" in df.columns - else "score" - if "score" in df.columns - else None + else ( + "fraction" + if "fraction" in df.columns + else "score" if "score" in df.columns else None + ) ) if value_col is None: raise ValueError( @@ -263,9 +263,7 @@ def _ensure_predictor(self): ) return self._predictor - def process_parquet_file( - self, parquet_path: str, sample_id: str - ) -> MarlinMetadata: + def process_parquet_file(self, parquet_path: str, sample_id: str) -> MarlinMetadata: logger = logging.getLogger("robin.marlin") start_time = time.time() @@ -345,7 +343,9 @@ def process_parquet_file( return result except Exception as exc: - logger.error("MARLIN analysis failed for %s: %s", sample_id, exc, exc_info=True) + logger.error( + "MARLIN analysis failed for %s: %s", sample_id, exc, exc_info=True + ) result.error_message = str(exc) result.processing_steps.append("analysis_failed") return result @@ -421,7 +421,9 @@ def process_multiple_files( pass if analysis_result["files_processed"] == 0: - analysis_result["error_message"] = "No files could be processed successfully" + analysis_result["error_message"] = ( + "No files could be processed successfully" + ) analysis_result["processing_steps"].append("no_files_processed") return analysis_result @@ -493,9 +495,13 @@ def marlin_handler(job, work_dir=None): ) if not parquet_paths: - error_msg = "No parquet paths found from bed conversion results in batch" + error_msg = ( + "No parquet paths found from bed conversion results in batch" + ) if suppress_expected: - logger.warning("%s (expected for fail-only BAM submission)", error_msg) + logger.warning( + "%s (expected for fail-only BAM submission)", error_msg + ) job.context.add_result( "marlin_analysis", {"status": "expected_failure", "reason": error_msg}, @@ -582,7 +588,10 @@ def marlin_handler(job, work_dir=None): if suppress_expected: job.context.add_result( "marlin_analysis", - {"status": "expected_failure", "error_message": result.error_message}, + { + "status": "expected_failure", + "error_message": result.error_message, + }, ) else: job.context.add_error("marlin_analysis", result.error_message) @@ -619,7 +628,9 @@ def marlin_handler(job, work_dir=None): except Exception as exc: if suppress_expected: - logger.warning("Expected MARLIN failure for fail-only BAM submission: %s", exc) + logger.warning( + "Expected MARLIN failure for fail-only BAM submission: %s", exc + ) job.context.add_result( "marlin_analysis", {"status": "expected_failure", "error_message": str(exc)}, diff --git a/src/robin/analysis/marlin_python/predictor.py b/src/robin/analysis/marlin_python/predictor.py index e1a6307b..7cced762 100644 --- a/src/robin/analysis/marlin_python/predictor.py +++ b/src/robin/analysis/marlin_python/predictor.py @@ -11,7 +11,6 @@ from typing import Iterable, Mapping, MutableMapping, Optional, Sequence from zipfile import ZipFile - _CG_PATTERN = re.compile(rb"cg\d{8}") _XML_NS = {"main": "http://schemas.openxmlformats.org/spreadsheetml/2006/main"} @@ -139,7 +138,9 @@ def from_paths( model = _load_tensorflow_model(model_path, custom_objects=custom_objects) probe_names = extract_probe_names_from_rdata(feature_path) class_names = ( - read_class_names_from_xlsx(annotation_path) if annotation_path is not None else None + read_class_names_from_xlsx(annotation_path) + if annotation_path is not None + else None ) return cls(model=model, probe_names=probe_names, class_names=class_names) @@ -171,7 +172,9 @@ def build_feature_vector( vector[index] = 1 if beta >= 0.5 else -1 return vector - def _predict_from_feature_vector(self, feature_vector: Sequence[int]) -> MARLINPrediction: + def _predict_from_feature_vector( + self, feature_vector: Sequence[int] + ) -> MARLINPrediction: # Keras 3 / TF 2.16+ rejects bare Python lists; use a float32 batch array. # Prefer model(batch, training=False) over model.predict(...): on macOS, # predict() can hang indefinitely in the TF data-adapter path. @@ -188,8 +191,12 @@ def _predict_from_feature_vector(self, feature_vector: Sequence[int]) -> MARLINP except TypeError: raw_predictions = self.model.predict(batch, verbose=0) row = _coerce_prediction_row(raw_predictions) - class_names = self.class_names or [f"class_{idx + 1}" for idx in range(len(row))] - scores = OrderedDict((name, float(score)) for name, score in zip(class_names, row)) + class_names = self.class_names or [ + f"class_{idx + 1}" for idx in range(len(row)) + ] + scores = OrderedDict( + (name, float(score)) for name, score in zip(class_names, row) + ) covered_cpgs = sum(1 for value in feature_vector if value != 0) return MARLINPrediction( scores=scores, diff --git a/src/robin/analysis/master_bed_generator.py b/src/robin/analysis/master_bed_generator.py index 166d3ab0..7e6bb43c 100644 --- a/src/robin/analysis/master_bed_generator.py +++ b/src/robin/analysis/master_bed_generator.py @@ -12,21 +12,21 @@ merged (per strand) to remove overlaps. """ -import os -import sys -import logging -import glob -import fcntl -import time import argparse +import fcntl +import glob import hashlib import json +import logging +import os +import sys import threading -from typing import List, Tuple, Dict, Optional, Any +import time from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple -import pandas as pd import numpy as np +import pandas as pd logger = logging.getLogger("robin.analysis.master_bed") @@ -36,23 +36,23 @@ class FileLock: Simple file-based lock for coordinating access across processes/threads. Uses fcntl for POSIX systems. """ - + def __init__(self, lock_file: str, timeout: float = 30.0): self.lock_file = lock_file self.timeout = timeout self.fd = None - + def __enter__(self): self.acquire() return self - + def __exit__(self, exc_type, exc_val, exc_tb): self.release() - + def acquire(self): """Acquire the lock with timeout""" os.makedirs(os.path.dirname(self.lock_file), exist_ok=True) - self.fd = open(self.lock_file, 'w') + self.fd = open(self.lock_file, "w") start_time = time.time() try: while True: @@ -75,7 +75,7 @@ def acquire(self): pass self.fd = None raise - + def release(self): """Release the lock""" if self.fd: @@ -87,22 +87,22 @@ def release(self): def _get_latest_bed_file(bed_dir: str, pattern: str) -> Optional[str]: """ Find the latest BED file matching a pattern based on counter. - + Args: bed_dir: Directory containing BED files pattern: Glob pattern to match (e.g., "new_file_*.bed") - + Returns: Path to the latest BED file, or None if not found """ bed_files = glob.glob(os.path.join(bed_dir, pattern)) if not bed_files: return None - + # Extract counter from filename and find the latest latest_file = None latest_counter = -1 - + for bed_file in bed_files: try: # Extract counter from filename (e.g., "new_file_001.bed" -> 1) @@ -117,26 +117,28 @@ def _get_latest_bed_file(bed_dir: str, pattern: str) -> Optional[str]: latest_file = bed_file except (ValueError, IndexError): # If we can't parse the counter, use modification time as fallback - if latest_file is None or os.path.getmtime(bed_file) > os.path.getmtime(latest_file): + if latest_file is None or os.path.getmtime(bed_file) > os.path.getmtime( + latest_file + ): latest_file = bed_file - + return latest_file def _load_bed_file(bed_path: str, require_bed6: bool = False) -> pd.DataFrame: """ Load a BED file into a DataFrame, preserving strand information. - + Args: bed_path: Path to BED file require_bed6: If True, expects BED6 format (chrom, start, end, name, score, strand) - + Returns: DataFrame with columns: chrom, start, end, name, score, strand """ if not os.path.exists(bed_path): return pd.DataFrame() - + try: # Read BED file - handle variable number of columns (BED format can have 3-12 columns) # First, read without column names to detect actual number of columns @@ -147,10 +149,10 @@ def _load_bed_file(bed_path: str, require_bed6: bool = False) -> pd.DataFrame: nrows=1, # Just read first row to check column count ) num_cols = len(df_temp.columns) - + # Standard BED6 column names bed6_cols = ["chrom", "start", "end", "name", "score", "strand"] - + # Read full file with appropriate column names based on actual column count if num_cols >= 6: # File has at least 6 columns (BED6 format) @@ -160,14 +162,23 @@ def _load_bed_file(bed_path: str, require_bed6: bool = False) -> pd.DataFrame: sep="\t", header=None, names=col_names[:num_cols], - dtype={"chrom": str, "start": int, "end": int, "name": str, "score": str, "strand": str}, + dtype={ + "chrom": str, + "start": int, + "end": int, + "name": str, + "score": str, + "strand": str, + }, na_values=["."], ) # Keep BED6 columns df = df[bed6_cols].copy() elif num_cols >= 4: # File has at least 4 columns (chrom, start, end, name) - col_names = ["chrom", "start", "end", "name"] + [f"col{i}" for i in range(4, num_cols)] + col_names = ["chrom", "start", "end", "name"] + [ + f"col{i}" for i in range(4, num_cols) + ] df = pd.read_csv( bed_path, sep="\t", @@ -197,15 +208,15 @@ def _load_bed_file(bed_path: str, require_bed6: bool = False) -> pd.DataFrame: else: logger.warning(f"BED file has too few columns ({num_cols}): {bed_path}") return pd.DataFrame() - + # Fill missing values df["name"] = df["name"].fillna(".") df["score"] = df["score"].fillna("0") df["strand"] = df["strand"].fillna(".") - + # Normalize strand values df["strand"] = df["strand"].replace({"": ".", "?": "."}) - + return df[bed6_cols] except Exception as e: logger.warning(f"Error loading BED file {bed_path}: {e}") @@ -244,76 +255,84 @@ def _expand_cnv_regions(df: pd.DataFrame, expand_size: int = 10000) -> pd.DataFr """ Convert CNV regions into breakpoint regions at start and end positions. Each breakpoint is padded by +/- expand_size and duplicated for both strands. - + Processing: 1. Split each CNV region into two breakpoints: one at start, one at end 2. Pad each breakpoint by +/- expand_size (creates 2*expand_size wide regions) 3. Duplicate each breakpoint for both + and - strands - + Args: df: DataFrame with chrom, start, end, name, score, strand columns expand_size: Size to pad around each breakpoint in base pairs (normally the CNV analysis bin_width) - + Returns: DataFrame with breakpoint regions, duplicated for both strands """ if df.empty: return df - + breakpoint_regions = [] - + for _, row in df.iterrows(): chrom = row["chrom"] region_start = row["start"] region_end = row["end"] name = row.get("name", ".") score = row.get("score", "0") - + # Create start breakpoint: pad around region_start start_bp_start = max(0, region_start - expand_size) start_bp_end = region_start + expand_size - + # Create end breakpoint: pad around region_end end_bp_start = max(0, region_end - expand_size) end_bp_end = region_end + expand_size - + # Add start breakpoint for both strands - breakpoint_regions.append({ - "chrom": chrom, - "start": start_bp_start, - "end": start_bp_end, - "name": name, - "score": score, - "strand": "+" - }) - breakpoint_regions.append({ - "chrom": chrom, - "start": start_bp_start, - "end": start_bp_end, - "name": name, - "score": score, - "strand": "-" - }) - + breakpoint_regions.append( + { + "chrom": chrom, + "start": start_bp_start, + "end": start_bp_end, + "name": name, + "score": score, + "strand": "+", + } + ) + breakpoint_regions.append( + { + "chrom": chrom, + "start": start_bp_start, + "end": start_bp_end, + "name": name, + "score": score, + "strand": "-", + } + ) + # Add end breakpoint for both strands - breakpoint_regions.append({ - "chrom": chrom, - "start": end_bp_start, - "end": end_bp_end, - "name": name, - "score": score, - "strand": "+" - }) - breakpoint_regions.append({ - "chrom": chrom, - "start": end_bp_start, - "end": end_bp_end, - "name": name, - "score": score, - "strand": "-" - }) - + breakpoint_regions.append( + { + "chrom": chrom, + "start": end_bp_start, + "end": end_bp_end, + "name": name, + "score": score, + "strand": "+", + } + ) + breakpoint_regions.append( + { + "chrom": chrom, + "start": end_bp_start, + "end": end_bp_end, + "name": name, + "score": score, + "strand": "-", + } + ) + result = pd.DataFrame(breakpoint_regions) return result @@ -322,36 +341,36 @@ def _merge_overlapping_regions(df: pd.DataFrame) -> pd.DataFrame: """ Merge overlapping regions in a BED DataFrame, preserving strand information. Regions are merged separately by chromosome and strand. - + Args: df: DataFrame with chrom, start, end, name, score, strand columns - + Returns: DataFrame with merged regions """ if df.empty: return df - + merged_regions = [] - + # Sort by chromosome, strand, then start position df_sorted = df.sort_values(by=["chrom", "strand", "start", "end"]).copy() - + # Group by chromosome and strand for (chrom, strand), group in df_sorted.groupby(["chrom", "strand"], observed=True): if group.empty: continue - + # Merge overlapping regions within chromosome and strand current_start = None current_end = None current_name = None - + for _, row in group.iterrows(): start = row["start"] end = row["end"] name = row.get("name", "merged") - + if current_start is None: # First region current_start = start @@ -368,133 +387,140 @@ def _merge_overlapping_regions(df: pd.DataFrame) -> pd.DataFrame: current_name = f"{current_name},{name}" else: # No overlap - save current region and start new one - merged_regions.append({ - "chrom": chrom, - "start": current_start, - "end": current_end, - "name": current_name if current_name else "merged", - "score": row.get("score", "0"), - "strand": strand - }) + merged_regions.append( + { + "chrom": chrom, + "start": current_start, + "end": current_end, + "name": current_name if current_name else "merged", + "score": row.get("score", "0"), + "strand": strand, + } + ) current_start = start current_end = end current_name = name - + # Don't forget the last region if current_start is not None: - merged_regions.append({ - "chrom": chrom, - "start": current_start, - "end": current_end, - "name": current_name if current_name else "merged", - "score": "0", - "strand": strand - }) - + merged_regions.append( + { + "chrom": chrom, + "start": current_start, + "end": current_end, + "name": current_name if current_name else "merged", + "score": "0", + "strand": strand, + } + ) + if not merged_regions: return pd.DataFrame() - + return pd.DataFrame(merged_regions) def _duplicate_unstranded_regions(df: pd.DataFrame) -> pd.DataFrame: """ Duplicate regions with unstranded ('.') strand to both + and - strands. - + Args: df: DataFrame with chrom, start, end, name, score, strand columns - + Returns: DataFrame with unstranded regions duplicated for both strands """ if df.empty: return df - + # Separate stranded and unstranded regions stranded = df[df["strand"].isin(["+", "-"])].copy() unstranded = df[df["strand"] == "."].copy() - + if unstranded.empty: return stranded - + # Duplicate unstranded regions for both strands plus_strand = unstranded.copy() plus_strand["strand"] = "+" - + minus_strand = unstranded.copy() minus_strand["strand"] = "-" - + # Combine stranded regions with duplicated unstranded regions result = pd.concat([stranded, plus_strand, minus_strand], ignore_index=True) - + return result def _intersect_with_target_genes( - regions_df: pd.DataFrame, - target_bed_path: str + regions_df: pd.DataFrame, target_bed_path: str ) -> pd.DataFrame: """ Filter regions to only include those that overlap with target gene BED file. Keeps the original regions that overlap, preserving strand information. - + Args: regions_df: DataFrame with chrom, start, end, name, score, strand columns target_bed_path: Path to target gene BED file - + Returns: DataFrame with regions that overlap target genes """ if regions_df.empty: return pd.DataFrame() - + if not os.path.exists(target_bed_path): - logger.warning(f"Target BED file not found: {target_bed_path}, skipping intersection") + logger.warning( + f"Target BED file not found: {target_bed_path}, skipping intersection" + ) return regions_df - + try: # Load target genes (BED6 format) target_df = _load_bed_file(target_bed_path, require_bed6=True) if target_df.empty: logger.warning(f"Target BED file is empty: {target_bed_path}") return pd.DataFrame() - + # Duplicate unstranded target regions for both strands target_df = _duplicate_unstranded_regions(target_df) - + # Filter regions to only those that overlap with target genes overlapping_regions = [] - + for _, region_row in regions_df.iterrows(): region_chrom = region_row["chrom"] region_start = region_row["start"] region_end = region_row["end"] region_strand = region_row.get("strand", ".") - + # Check if this region overlaps with any target gene (same chromosome) overlapping_targets = target_df[ - (target_df["chrom"] == region_chrom) & - (target_df["start"] < region_end) & - (target_df["end"] > region_start) + (target_df["chrom"] == region_chrom) + & (target_df["start"] < region_end) + & (target_df["end"] > region_start) ] - + if not overlapping_targets.empty: # Keep the original region (it overlaps with target genes) # Preserve the region's strand information - overlapping_regions.append({ - "chrom": region_chrom, - "start": region_start, - "end": region_end, - "name": region_row.get("name", "overlap"), - "score": region_row.get("score", "0"), - "strand": region_strand - }) - + overlapping_regions.append( + { + "chrom": region_chrom, + "start": region_start, + "end": region_end, + "name": region_row.get("name", "overlap"), + "score": region_row.get("score", "0"), + "strand": region_strand, + } + ) + if not overlapping_regions: return pd.DataFrame() - + return pd.DataFrame(overlapping_regions) - + except Exception as e: logger.error(f"Error intersecting with target genes: {e}") return regions_df @@ -503,87 +529,94 @@ def _intersect_with_target_genes( def _load_fai_file(fai_path: str) -> Dict[str, int]: """ Load a FASTA index (.fai) file and return chromosome lengths. - + Args: fai_path: Path to .fai file - + Returns: Dictionary mapping chromosome names to lengths """ chrom_lengths = {} - + if not os.path.exists(fai_path): logger.warning(f"FAI file not found: {fai_path}") return chrom_lengths - + try: - with open(fai_path, 'r') as f: + with open(fai_path, "r") as f: for line in f: line = line.strip() if not line: continue # FAI format: chrom_name length offset linebases linewidth - parts = line.split('\t') + parts = line.split("\t") if len(parts) >= 2: chrom_name = parts[0] chrom_length = int(parts[1]) chrom_lengths[chrom_name] = chrom_length except Exception as e: logger.error(f"Error loading FAI file {fai_path}: {e}") - + return chrom_lengths def _calculate_genome_coverage( - bed_df: pd.DataFrame, - fai_path: Optional[str] = None + bed_df: pd.DataFrame, fai_path: Optional[str] = None ) -> Optional[Dict[str, float]]: """ Calculate the proportion of the genome covered by BED regions, accounting for strand. - + Args: bed_df: DataFrame with chrom, start, end, strand columns fai_path: Optional path to .fai file for genome sizes - + Returns: Dictionary with coverage statistics, or None if calculation fails """ if bed_df.empty: return None - + if not fai_path or not os.path.exists(fai_path): - logger.warning("FAI file not provided or not found - cannot calculate genome coverage") + logger.warning( + "FAI file not provided or not found - cannot calculate genome coverage" + ) return None - + # Load chromosome lengths chrom_lengths = _load_fai_file(fai_path) if not chrom_lengths: logger.warning("No chromosome lengths loaded from FAI file") return None - + # Calculate total genome size (sum of all chromosome lengths) total_genome_size = sum(chrom_lengths.values()) if total_genome_size == 0: logger.warning("Total genome size is 0") return None - + # Check for chromosomes in BED file that aren't in FAI file bed_chroms = set(bed_df["chrom"].unique()) fai_chroms = set(chrom_lengths.keys()) missing_chroms = bed_chroms - fai_chroms if missing_chroms: - logger.warning(f"Chromosomes in BED file not found in FAI file: {sorted(missing_chroms)}") - + logger.warning( + f"Chromosomes in BED file not found in FAI file: {sorted(missing_chroms)}" + ) + # Calculate coverage for each strand separately strand_coverage = {"+": 0, "-": 0, ".": 0} - strand_total_genome = {"+": total_genome_size, "-": total_genome_size, ".": total_genome_size} - + strand_total_genome = { + "+": total_genome_size, + "-": total_genome_size, + ".": total_genome_size, + } + # Group by strand for strand in ["+", "-", "."]: strand_df = bed_df[bed_df["strand"] == strand].copy() if strand_df.empty: continue - + # Calculate total covered bases for this strand # IMPORTANT: Regions must be merged before calling this function to avoid double-counting overlaps covered_bases = 0 @@ -592,7 +625,7 @@ def _calculate_genome_coverage( chrom = row["chrom"] start = row["start"] end = row["end"] - + # Only count if chromosome exists in FAI file if chrom in chrom_lengths: # Clip region to chromosome boundaries @@ -600,23 +633,37 @@ def _calculate_genome_coverage( covered_bases += region_length else: skipped_regions += 1 - + if skipped_regions > 0: - logger.debug(f"Skipped {skipped_regions} {strand} strand regions due to missing chromosomes in FAI file") - + logger.debug( + f"Skipped {skipped_regions} {strand} strand regions due to missing chromosomes in FAI file" + ) + strand_coverage[strand] = covered_bases - + # Calculate proportions results = { "total_genome_size": total_genome_size, "plus_strand_coverage": strand_coverage["+"], "minus_strand_coverage": strand_coverage["-"], "unstranded_coverage": strand_coverage["."], - "plus_strand_proportion": strand_coverage["+"] / strand_total_genome["+"] if strand_total_genome["+"] > 0 else 0.0, - "minus_strand_proportion": strand_coverage["-"] / strand_total_genome["-"] if strand_total_genome["-"] > 0 else 0.0, - "unstranded_proportion": strand_coverage["."] / strand_total_genome["."] if strand_total_genome["."] > 0 else 0.0, + "plus_strand_proportion": ( + strand_coverage["+"] / strand_total_genome["+"] + if strand_total_genome["+"] > 0 + else 0.0 + ), + "minus_strand_proportion": ( + strand_coverage["-"] / strand_total_genome["-"] + if strand_total_genome["-"] > 0 + else 0.0 + ), + "unstranded_proportion": ( + strand_coverage["."] / strand_total_genome["."] + if strand_total_genome["."] > 0 + else 0.0 + ), } - + # Calculate combined coverage # + and - strands are treated separately (each can have different coverage) # Unstranded regions (if any) are counted once @@ -625,29 +672,31 @@ def _calculate_genome_coverage( # plus unstranded regions if they exist total_possible_coverage = 2 * total_genome_size results["total_covered_bases"] = total_covered - results["total_proportion"] = total_covered / total_possible_coverage if total_possible_coverage > 0 else 0.0 - + results["total_proportion"] = ( + total_covered / total_possible_coverage if total_possible_coverage > 0 else 0.0 + ) + return results def _sort_bed_regions(df: pd.DataFrame) -> pd.DataFrame: """ Sort BED regions by chromosome (numeric order), strand, then start, then end. - + Args: df: DataFrame with chrom, start, end, strand columns - + Returns: Sorted DataFrame """ if df.empty: return df - + def chrom_sort_key(chrom): """Sort chromosomes: chr1-22, then chrX, chrY, chrM, then others""" if not chrom.startswith("chr"): return (300, chrom) # Non-standard chromosomes at end - + chrom_suffix = chrom[3:] if chrom_suffix == "X": return (100, 0) @@ -660,7 +709,7 @@ def chrom_sort_key(chrom): return (int(chrom_suffix), 0) except ValueError: return (300, chrom_suffix) - + def strand_sort_key(strand): """Sort strands: +, then -, then .""" if strand == "+": @@ -669,13 +718,13 @@ def strand_sort_key(strand): return 1 else: return 2 - + df = df.copy() df["_chrom_sort_key"] = df["chrom"].apply(chrom_sort_key) df["_strand_sort_key"] = df.get("strand", ".").apply(strand_sort_key) df = df.sort_values(by=["_chrom_sort_key", "_strand_sort_key", "start", "end"]) df = df.drop(columns=["_chrom_sort_key", "_strand_sort_key"]) - + return df.reset_index(drop=True) @@ -710,6 +759,7 @@ def _get_reference_path() -> Optional[str]: # Try to load from config file try: import yaml # type: ignore + config_paths = [ os.path.join(os.getcwd(), "config.yaml"), os.path.expanduser("~/.robin/config.yaml"), @@ -717,7 +767,7 @@ def _get_reference_path() -> Optional[str]: ] for config_path in config_paths: if os.path.exists(config_path): - with open(config_path, 'r') as f: + with open(config_path, "r") as f: config = yaml.safe_load(f) if config and isinstance(config, dict): reference = config.get("reference") @@ -728,7 +778,7 @@ def _get_reference_path() -> Optional[str]: return reference except (ImportError, Exception): pass - + # Environment variables (prefer ROBIN_REFERENCE; keep robin_REFERENCE for compat) for env_key in ("ROBIN_REFERENCE", "robin_REFERENCE"): reference_env = os.environ.get(env_key) @@ -736,7 +786,7 @@ def _get_reference_path() -> Optional[str]: reference_env = os.path.expanduser(reference_env) if os.path.exists(reference_env): return reference_env - + return None @@ -744,17 +794,17 @@ def _find_fai_file(reference: Optional[str] = None) -> Optional[str]: """ Find the FAI (FASTA index) file path from reference genome or environment variable. If reference exists but FAI doesn't, attempts to create it using samtools faidx. - + Args: reference: Optional path to reference FASTA file - + Returns: Path to FAI file, or None if not found """ # Get reference if not provided if not reference: reference = _get_reference_path() - + # If reference is provided, expand user home directory and check for corresponding .fai file if reference: # Expand user home directory if present (handles ~/path) @@ -762,28 +812,33 @@ def _find_fai_file(reference: Optional[str] = None) -> Optional[str]: fai_path = f"{reference}.fai" if os.path.exists(fai_path): return fai_path - + # If reference exists but FAI doesn't, try to create it if os.path.exists(reference): try: import subprocess - logger.debug(f"FAI file not found for {reference}, attempting to create it with samtools faidx") + + logger.debug( + f"FAI file not found for {reference}, attempting to create it with samtools faidx" + ) result = subprocess.run( ["samtools", "faidx", reference], capture_output=True, text=True, - timeout=300 + timeout=300, ) if result.returncode == 0 and os.path.exists(fai_path): logger.info(f"Successfully created FAI index: {fai_path}") return fai_path else: - logger.warning(f"Failed to create FAI index for {reference}: {result.stderr}") + logger.warning( + f"Failed to create FAI index for {reference}: {result.stderr}" + ) except (subprocess.TimeoutExpired, FileNotFoundError, Exception) as e: logger.debug(f"Could not create FAI index for {reference}: {e}") else: logger.debug(f"Reference file does not exist: {reference}") - + # Check environment variables directly (fallback) for env_key in ("ROBIN_REFERENCE", "robin_REFERENCE"): reference_env = os.environ.get(env_key) @@ -793,18 +848,18 @@ def _find_fai_file(reference: Optional[str] = None) -> Optional[str]: fai_path = f"{reference_env}.fai" if os.path.exists(fai_path): return fai_path - + # Common locations to check common_paths = [ "/data/reference/genome.fa.fai", "/usr/local/share/reference/genome.fa.fai", os.path.expanduser("~/reference/genome.fa.fai"), ] - + for path in common_paths: if os.path.exists(path): return path - + return None @@ -813,12 +868,12 @@ def _log_bed_coverage_data( work_dir: str, analysis_counter: int, coverage_data: Dict[str, Dict[str, float]], - log_file: str = "bed_coverage_log.json" + log_file: str = "bed_coverage_log.json", ) -> None: """ Log genome coverage data for each BED file type to a JSON file. Appends to existing log if it exists. - + Args: sample_id: Sample ID work_dir: Working directory @@ -872,7 +927,9 @@ def _round_sigfigs(value: float, digits: int = 3) -> float: return float(f"{value:.{digits}g}") -def build_bed_coverage_series(log_entries: List[Dict[str, Any]]) -> List[Dict[str, Any]]: +def build_bed_coverage_series( + log_entries: List[Dict[str, Any]], +) -> List[Dict[str, Any]]: """Build ECharts series from ``bed_coverage_log.json`` entries. When many masters are written in a short wall-clock burst (folder @@ -980,7 +1037,9 @@ def _log_entries_are_bursty(log_entries: List[Dict[str, Any]]) -> bool: return len(timestamps) <= 1 and len(counters) > 1 -def _generate_visualization_data(sample_dir: str, log_entries: List[Dict[str, Any]]) -> None: +def _generate_visualization_data( + sample_dir: str, log_entries: List[Dict[str, Any]] +) -> None: """ Generate pre-processed visualization data for the GUI. This avoids doing heavy DataFrame operations in the GUI for each viewer. @@ -1070,19 +1129,22 @@ def _get_target_bed_path( try: from robin import resources + resources_dir = os.path.dirname(resources.__file__) - + if target_panel == "rCNS2": bed_path = os.path.join(resources_dir, "rCNS2_panel_name_uniq.bed") elif target_panel == "AML": bed_path = os.path.join(resources_dir, "AML_panel_name_uniq.bed") else: # Try custom panel - bed_path = os.path.join(resources_dir, f"{target_panel}_panel_name_uniq.bed") - + bed_path = os.path.join( + resources_dir, f"{target_panel}_panel_name_uniq.bed" + ) + if os.path.exists(bed_path): return bed_path - + # Fallback paths if target_panel == "rCNS2": fallback = "rCNS2_panel_name_uniq.bed" @@ -1090,13 +1152,13 @@ def _get_target_bed_path( fallback = "AML_panel_name_uniq.bed" else: fallback = f"{target_panel}_panel_name_uniq.bed" - + if os.path.exists(fallback): return fallback - + except ImportError: pass - + return None @@ -1106,23 +1168,23 @@ def _try_get_target_panel_from_fusion_metadata( ) -> Optional[str]: """ Try to get target_panel from fusion metadata if available. - + Args: sample_id: Sample ID work_dir: Working directory - + Returns: Target panel name if found, None otherwise """ try: from robin.analysis.fusion_work import _load_fusion_metadata - + fusion_metadata = _load_fusion_metadata(work_dir, sample_id) - if fusion_metadata and hasattr(fusion_metadata, 'target_panel'): + if fusion_metadata and hasattr(fusion_metadata, "target_panel"): return fusion_metadata.target_panel except Exception: pass - + return None @@ -1161,7 +1223,9 @@ def _master_bed_source_signature( target_bed_path = _get_target_bed_path(target_panel, _resolve_reference(reference)) if target_bed_path and os.path.exists(target_bed_path): - sources.append({"type": "target_panel", "sha256": _file_sha256(target_bed_path)}) + sources.append( + {"type": "target_panel", "sha256": _file_sha256(target_bed_path)} + ) return {"version": 1, "target_panel": target_panel, "sources": sources} @@ -1234,17 +1298,27 @@ def _build_master_bed_data( ) cnv_expanded_df = _expand_cnv_regions(cnv_df, expand_size=expand_size) all_regions.append(cnv_expanded_df) - log.debug(f"Loaded {len(cnv_df)} CNV regions (converted to {len(cnv_expanded_df)} breakpoint regions)") + log.debug( + f"Loaded {len(cnv_df)} CNV regions (converted to {len(cnv_expanded_df)} breakpoint regions)" + ) cnv_df_merged = _merge_overlapping_regions(cnv_df) cnv_entry = {"region_count": len(cnv_df)} if fai_path: cnv_coverage = _calculate_genome_coverage(cnv_df_merged, fai_path) if cnv_coverage: - cnv_entry.update({ - "total_proportion": cnv_coverage.get("total_proportion", 0.0), - "plus_strand_proportion": cnv_coverage.get("plus_strand_proportion", 0.0), - "minus_strand_proportion": cnv_coverage.get("minus_strand_proportion", 0.0), - }) + cnv_entry.update( + { + "total_proportion": cnv_coverage.get( + "total_proportion", 0.0 + ), + "plus_strand_proportion": cnv_coverage.get( + "plus_strand_proportion", 0.0 + ), + "minus_strand_proportion": cnv_coverage.get( + "minus_strand_proportion", 0.0 + ), + } + ) coverage_data["cnv_regions"] = cnv_entry # 2. CNV breakpoints @@ -1255,17 +1329,27 @@ def _build_master_bed_data( if not bp_df.empty: bp_df_processed = _duplicate_unstranded_regions(bp_df) all_regions.append(bp_df_processed) - log.debug(f"Loaded {len(bp_df_processed)} CNV breakpoints (with both strands)") + log.debug( + f"Loaded {len(bp_df_processed)} CNV breakpoints (with both strands)" + ) bp_df_merged = _merge_overlapping_regions(bp_df_processed) bp_entry = {"region_count": len(bp_df_processed)} if fai_path: bp_coverage = _calculate_genome_coverage(bp_df_merged, fai_path) if bp_coverage: - bp_entry.update({ - "total_proportion": bp_coverage.get("total_proportion", 0.0), - "plus_strand_proportion": bp_coverage.get("plus_strand_proportion", 0.0), - "minus_strand_proportion": bp_coverage.get("minus_strand_proportion", 0.0), - }) + bp_entry.update( + { + "total_proportion": bp_coverage.get( + "total_proportion", 0.0 + ), + "plus_strand_proportion": bp_coverage.get( + "plus_strand_proportion", 0.0 + ), + "minus_strand_proportion": bp_coverage.get( + "minus_strand_proportion", 0.0 + ), + } + ) coverage_data["cnv_breakpoints"] = bp_entry # 3. Fusion breakpoints @@ -1276,17 +1360,27 @@ def _build_master_bed_data( if not fusion_df.empty: fusion_df_processed = _duplicate_unstranded_regions(fusion_df) all_regions.append(fusion_df_processed) - log.debug(f"Loaded {len(fusion_df_processed)} fusion breakpoints (preserving strand)") + log.debug( + f"Loaded {len(fusion_df_processed)} fusion breakpoints (preserving strand)" + ) fusion_df_merged = _merge_overlapping_regions(fusion_df_processed) fusion_entry = {"region_count": len(fusion_df_processed)} if fai_path: fusion_coverage = _calculate_genome_coverage(fusion_df_merged, fai_path) if fusion_coverage: - fusion_entry.update({ - "total_proportion": fusion_coverage.get("total_proportion", 0.0), - "plus_strand_proportion": fusion_coverage.get("plus_strand_proportion", 0.0), - "minus_strand_proportion": fusion_coverage.get("minus_strand_proportion", 0.0), - }) + fusion_entry.update( + { + "total_proportion": fusion_coverage.get( + "total_proportion", 0.0 + ), + "plus_strand_proportion": fusion_coverage.get( + "plus_strand_proportion", 0.0 + ), + "minus_strand_proportion": fusion_coverage.get( + "minus_strand_proportion", 0.0 + ), + } + ) coverage_data["fusion_breakpoints"] = fusion_entry # 4. Master BED breakpoints @@ -1297,17 +1391,31 @@ def _build_master_bed_data( if not master_bed_bp_df.empty: master_bed_bp_df_processed = _duplicate_unstranded_regions(master_bed_bp_df) all_regions.append(master_bed_bp_df_processed) - log.debug(f"Loaded {len(master_bed_bp_df_processed)} master BED breakpoints (with both strands)") - master_bed_bp_df_merged = _merge_overlapping_regions(master_bed_bp_df_processed) + log.debug( + f"Loaded {len(master_bed_bp_df_processed)} master BED breakpoints (with both strands)" + ) + master_bed_bp_df_merged = _merge_overlapping_regions( + master_bed_bp_df_processed + ) master_bed_bp_entry = {"region_count": len(master_bed_bp_df_processed)} if fai_path: - master_bed_bp_coverage = _calculate_genome_coverage(master_bed_bp_df_merged, fai_path) + master_bed_bp_coverage = _calculate_genome_coverage( + master_bed_bp_df_merged, fai_path + ) if master_bed_bp_coverage: - master_bed_bp_entry.update({ - "total_proportion": master_bed_bp_coverage.get("total_proportion", 0.0), - "plus_strand_proportion": master_bed_bp_coverage.get("plus_strand_proportion", 0.0), - "minus_strand_proportion": master_bed_bp_coverage.get("minus_strand_proportion", 0.0), - }) + master_bed_bp_entry.update( + { + "total_proportion": master_bed_bp_coverage.get( + "total_proportion", 0.0 + ), + "plus_strand_proportion": master_bed_bp_coverage.get( + "plus_strand_proportion", 0.0 + ), + "minus_strand_proportion": master_bed_bp_coverage.get( + "minus_strand_proportion", 0.0 + ), + } + ) coverage_data["master_bed_breakpoints"] = master_bed_bp_entry if not all_regions: @@ -1322,31 +1430,51 @@ def _build_master_bed_data( if target_panel: target_bed_path = _get_target_bed_path(target_panel, ref) if target_bed_path: - log.debug(f"Loading target panel regions: {target_panel} ({target_bed_path})") + log.debug( + f"Loading target panel regions: {target_panel} ({target_bed_path})" + ) target_df = _load_bed_file(target_bed_path, require_bed6=True) if not target_df.empty: target_df_processed = _duplicate_unstranded_regions(target_df) target_df_merged = _merge_overlapping_regions(target_df_processed) target_entry = {"region_count": len(target_df_processed)} if fai_path: - target_coverage = _calculate_genome_coverage(target_df_merged, fai_path) + target_coverage = _calculate_genome_coverage( + target_df_merged, fai_path + ) if target_coverage: - target_entry.update({ - "total_proportion": target_coverage.get("total_proportion", 0.0), - "plus_strand_proportion": target_coverage.get("plus_strand_proportion", 0.0), - "minus_strand_proportion": target_coverage.get("minus_strand_proportion", 0.0), - }) + target_entry.update( + { + "total_proportion": target_coverage.get( + "total_proportion", 0.0 + ), + "plus_strand_proportion": target_coverage.get( + "plus_strand_proportion", 0.0 + ), + "minus_strand_proportion": target_coverage.get( + "minus_strand_proportion", 0.0 + ), + } + ) coverage_data["target_panel"] = target_entry - log.debug(f"Keeping all {len(merged_df)} breakpoint regions and adding {len(target_df_processed)} target gene regions") - all_regions_with_target = pd.concat([merged_df, target_df_processed], ignore_index=True) + log.debug( + f"Keeping all {len(merged_df)} breakpoint regions and adding {len(target_df_processed)} target gene regions" + ) + all_regions_with_target = pd.concat( + [merged_df, target_df_processed], ignore_index=True + ) merged_df = _merge_overlapping_regions(all_regions_with_target) - log.debug(f"Final merged regions including target genes: {len(merged_df)}") + log.debug( + f"Final merged regions including target genes: {len(merged_df)}" + ) else: log.warning(f"Target panel BED file is empty: {target_bed_path}") else: log.warning(f"Target panel BED file not found for {target_panel}") else: - log.debug("No target panel specified - master BED will include all breakpoint regions") + log.debug( + "No target panel specified - master BED will include all breakpoint regions" + ) if merged_df.empty: log.debug("No regions remaining after processing") @@ -1366,11 +1494,17 @@ def _build_master_bed_data( if fai_path: master_coverage = _calculate_genome_coverage(sorted_df, fai_path) if master_coverage: - master_entry.update({ - "total_proportion": master_coverage.get("total_proportion", 0.0), - "plus_strand_proportion": master_coverage.get("plus_strand_proportion", 0.0), - "minus_strand_proportion": master_coverage.get("minus_strand_proportion", 0.0), - }) + master_entry.update( + { + "total_proportion": master_coverage.get("total_proportion", 0.0), + "plus_strand_proportion": master_coverage.get( + "plus_strand_proportion", 0.0 + ), + "minus_strand_proportion": master_coverage.get( + "minus_strand_proportion", 0.0 + ), + } + ) coverage_data["master_bed"] = master_entry return (sorted_df, coverage_data) @@ -1412,7 +1546,9 @@ def generate_master_bed( ) with FileLock(lock_file, timeout=lock_timeout): latest_master = _get_latest_bed_file(bed_dir, "master_*.bed") - if latest_master and _master_bed_is_current(latest_master, source_signature): + if latest_master and _master_bed_is_current( + latest_master, source_signature + ): log.debug(f"Master BED sources unchanged: {latest_master}") return latest_master @@ -1435,17 +1571,25 @@ def generate_master_bed( sample_id, work_dir, target_panel, reference ) if current_signature != source_signature: - log.debug(f"Master BED sources changed during generation for {sample_id}") + log.debug( + f"Master BED sources changed during generation for {sample_id}" + ) continue latest_master = _get_latest_bed_file(bed_dir, "master_*.bed") - if latest_master and _master_bed_is_current(latest_master, current_signature): - log.debug(f"Master BED was generated by another process: {latest_master}") + if latest_master and _master_bed_is_current( + latest_master, current_signature + ): + log.debug( + f"Master BED was generated by another process: {latest_master}" + ) return latest_master from robin.analysis.cnv_analysis import allocate_next_analysis_counter write_counter = allocate_next_analysis_counter(sample_id, work_dir, log) - master_bed_path = os.path.join(bed_dir, f"master_{write_counter:03d}.bed") + master_bed_path = os.path.join( + bed_dir, f"master_{write_counter:03d}.bed" + ) temporary_path = f"{master_bed_path}.tmp" sorted_df[bed6_cols].to_csv( @@ -1518,6 +1662,7 @@ def generate_master_bed( except Exception as e: log.error(f"Error generating master BED file: {e}") import traceback + log.debug(f"Traceback: {traceback.format_exc()}") return None @@ -1566,6 +1711,7 @@ def _generate_in_background(): log = logger_instance if logger_instance else logger log.error(f"Error in background master BED generation for {sample_id}: {e}") import traceback + log.debug(f"Traceback: {traceback.format_exc()}") thread = threading.Thread(target=_generate_in_background, daemon=True) @@ -1591,7 +1737,7 @@ def generate_master_bed_from_files( ) -> Optional[str]: """ Generate master BED file from explicit file paths (for CLI/testing). - + Args: cnv_regions_file: Path to CNV regions BED file (will be expanded by +/- expand_cnv_size) cnv_breakpoints_file: Path to CNV breakpoints BED file @@ -1602,7 +1748,7 @@ def generate_master_bed_from_files( ``bin_width`` from ``sample_dir`` / inferred sample directory. sample_dir: Sample directory containing ``CNV_dict.npy`` (optional) logger_instance: Optional logger instance - + Returns: Path to generated master BED file, or None if generation failed """ @@ -1610,10 +1756,10 @@ def generate_master_bed_from_files( log = logger_instance else: log = logger - + try: all_regions = [] - + # 1. Load and convert CNV regions to breakpoints if cnv_regions_file and os.path.exists(cnv_regions_file): log.debug(f"Loading CNV regions from: {cnv_regions_file}") @@ -1638,8 +1784,10 @@ def generate_master_bed_from_files( ) expanded_df = _expand_cnv_regions(cnv_df, expand_size=expand_size) all_regions.append(expanded_df) - log.debug(f"Loaded {len(cnv_df)} CNV regions (converted to {len(expanded_df)} breakpoint regions)") - + log.debug( + f"Loaded {len(cnv_df)} CNV regions (converted to {len(expanded_df)} breakpoint regions)" + ) + # 2. Load CNV breakpoints if cnv_breakpoints_file and os.path.exists(cnv_breakpoints_file): log.debug(f"Loading CNV breakpoints from: {cnv_breakpoints_file}") @@ -1649,7 +1797,7 @@ def generate_master_bed_from_files( bp_df = _duplicate_unstranded_regions(bp_df) all_regions.append(bp_df) log.debug(f"Loaded {len(bp_df)} CNV breakpoints (with both strands)") - + # 3. Load fusion breakpoints if fusion_breakpoints_file and os.path.exists(fusion_breakpoints_file): log.debug(f"Loading fusion breakpoints from: {fusion_breakpoints_file}") @@ -1658,32 +1806,38 @@ def generate_master_bed_from_files( # Fusion breakpoints preserve their strand fusion_df = _duplicate_unstranded_regions(fusion_df) all_regions.append(fusion_df) - log.debug(f"Loaded {len(fusion_df)} fusion breakpoints (preserving strand)") - + log.debug( + f"Loaded {len(fusion_df)} fusion breakpoints (preserving strand)" + ) + # 4. Load master BED breakpoints (if provided as argument) # Note: In the main generate_master_bed function, these are loaded automatically # This parameter is for the CLI/testing function if master_bed_breakpoints_file and os.path.exists(master_bed_breakpoints_file): - log.debug(f"Loading master BED breakpoints from: {master_bed_breakpoints_file}") + log.debug( + f"Loading master BED breakpoints from: {master_bed_breakpoints_file}" + ) master_bed_bp_df = _load_bed_file(master_bed_breakpoints_file) if not master_bed_bp_df.empty: # Master BED breakpoints should have both strands master_bed_bp_df = _duplicate_unstranded_regions(master_bed_bp_df) all_regions.append(master_bed_bp_df) - log.debug(f"Loaded {len(master_bed_bp_df)} master BED breakpoints (with both strands)") - + log.debug( + f"Loaded {len(master_bed_bp_df)} master BED breakpoints (with both strands)" + ) + if not all_regions: log.error("No BED regions found to merge") return None - + # Combine all regions combined_df = pd.concat(all_regions, ignore_index=True) log.debug(f"Combined {len(combined_df)} total regions") - + # Merge overlapping regions merged_df = _merge_overlapping_regions(combined_df) log.debug(f"Merged to {len(merged_df)} regions") - + # Add target gene panel regions if available if target_bed_file and os.path.exists(target_bed_file): log.debug(f"Loading target panel regions from: {target_bed_file}") @@ -1691,26 +1845,34 @@ def generate_master_bed_from_files( if not target_df.empty: # Duplicate unstranded target regions for both strands target_df = _duplicate_unstranded_regions(target_df) - + # Keep all breakpoint regions and add target gene regions - log.debug(f"Keeping all {len(merged_df)} breakpoint regions and adding {len(target_df)} target gene regions") - + log.debug( + f"Keeping all {len(merged_df)} breakpoint regions and adding {len(target_df)} target gene regions" + ) + # Combine all breakpoints with target gene regions - all_regions_with_target = pd.concat([merged_df, target_df], ignore_index=True) - - log.debug(f"Combined {len(merged_df)} breakpoints with {len(target_df)} target gene regions") - + all_regions_with_target = pd.concat( + [merged_df, target_df], ignore_index=True + ) + + log.debug( + f"Combined {len(merged_df)} breakpoints with {len(target_df)} target gene regions" + ) + # Merge overlapping regions merged_df = _merge_overlapping_regions(all_regions_with_target) - log.debug(f"Final merged regions including target genes: {len(merged_df)}") - + log.debug( + f"Final merged regions including target genes: {len(merged_df)}" + ) + if merged_df.empty: log.error("No regions remaining after processing") return None - + # Sort regions sorted_df = _sort_bed_regions(merged_df) - + # Ensure we have all BED6 columns bed6_cols = ["chrom", "start", "end", "name", "score", "strand"] for col in bed6_cols: @@ -1721,7 +1883,7 @@ def generate_master_bed_from_files( sorted_df[col] = "." else: sorted_df[col] = "." - + # Write master BED file in BED6 format sorted_df[bed6_cols].to_csv( output_file, @@ -1729,13 +1891,16 @@ def generate_master_bed_from_files( header=False, index=False, ) - - log.info(f"Generated master BED file: {output_file} with {len(sorted_df)} regions") + + log.info( + f"Generated master BED file: {output_file} with {len(sorted_df)} regions" + ) return output_file - + except Exception as e: log.error(f"Error generating master BED file: {e}") import traceback + log.debug(f"Traceback: {traceback.format_exc()}") return None @@ -1770,9 +1935,9 @@ def main(): python master_bed_generator.py --cnv-regions new_file_001.bed \\ --expand-size 5000 \\ --output master.bed - """ + """, ) - + parser.add_argument( "--cnv-regions", type=str, @@ -1782,7 +1947,7 @@ def main(): "Each breakpoint is padded by +/- expand-size (default 10kb) around the breakpoint position. " "Each breakpoint region is duplicated for both + and - strands. " "Regions are then merged if they overlap (separately by strand)." - ) + ), ) parser.add_argument( "--cnv-breakpoints", @@ -1791,7 +1956,7 @@ def main(): "Path to CNV breakpoints BED file (breakpoints_*.bed). " "Processing: Regions are loaded as-is. If regions have unstranded ('.') strand, " "they are duplicated for both + and - strands. Regions are merged if they overlap (separately by strand)." - ) + ), ) parser.add_argument( "--fusion-breakpoints", @@ -1801,7 +1966,7 @@ def main(): "Processing: Regions preserve their original strand information. " "If regions have unstranded ('.') strand, they are duplicated for both + and - strands. " "Regions are merged if they overlap (separately by strand)." - ) + ), ) parser.add_argument( "--master-bed-breakpoints", @@ -1812,7 +1977,7 @@ def main(): "Processing: Regions are loaded as-is. If regions have unstranded ('.') strand, " "they are duplicated for both + and - strands (breaks can occur on either strand). " "Regions are merged if they overlap (separately by strand)." - ) + ), ) parser.add_argument( "--target-panel", @@ -1823,13 +1988,10 @@ def main(): "All breakpoint regions are kept (not filtered). " "All target gene regions are then added to the final output. " "Final regions are merged if they overlap (separately by strand)." - ) + ), ) parser.add_argument( - "--output", - type=str, - required=True, - help="Path to output master BED file" + "--output", type=str, required=True, help="Path to output master BED file" ) parser.add_argument( "--expand-size", @@ -1847,31 +2009,34 @@ def main(): "Path to FASTA index (.fai) file (optional). " "If provided, calculates the total proportion of the genome covered by the BED file, " "accounting for strand information (+ and - strands are calculated separately)." - ) + ), ) parser.add_argument( - "--verbose", - "-v", - action="store_true", - help="Enable verbose logging" + "--verbose", "-v", action="store_true", help="Enable verbose logging" ) - + args = parser.parse_args() - + # Setup logging log_level = logging.DEBUG if args.verbose else logging.INFO logging.basicConfig( - level=log_level, - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s" + level=log_level, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s" ) - + # Check that at least one input file is provided - if not any([args.cnv_regions, args.cnv_breakpoints, args.fusion_breakpoints, args.master_bed_breakpoints]): + if not any( + [ + args.cnv_regions, + args.cnv_breakpoints, + args.fusion_breakpoints, + args.master_bed_breakpoints, + ] + ): parser.error( "At least one input BED file must be provided " "(--cnv-regions, --cnv-breakpoints, --fusion-breakpoints, or --master-bed-breakpoints)" ) - + # Generate master BED file result = generate_master_bed_from_files( cnv_regions_file=args.cnv_regions, @@ -1882,10 +2047,10 @@ def main(): output_file=args.output, expand_cnv_size=args.expand_size, ) - + if result: print(f"Successfully generated master BED file: {result}") - + # Calculate genome coverage if FAI file is provided if args.fai_file: try: @@ -1894,29 +2059,49 @@ def main(): if not bed_df.empty: coverage_stats = _calculate_genome_coverage(bed_df, args.fai_file) if coverage_stats: - print("\n" + "="*60) + print("\n" + "=" * 60) print("Genome Coverage Statistics") - print("="*60) - print(f"Total genome size: {coverage_stats['total_genome_size']:,} bp") + print("=" * 60) + print( + f"Total genome size: {coverage_stats['total_genome_size']:,} bp" + ) print(f"\nCoverage by strand:") - print(f" + strand: {coverage_stats['plus_strand_coverage']:,} bp " - f"({coverage_stats['plus_strand_proportion']*100:.4f}%)") - print(f" - strand: {coverage_stats['minus_strand_coverage']:,} bp " - f"({coverage_stats['minus_strand_proportion']*100:.4f}%)") - if coverage_stats['unstranded_coverage'] > 0: - print(f" . strand: {coverage_stats['unstranded_coverage']:,} bp " - f"({coverage_stats['unstranded_proportion']*100:.4f}%)") - print(f"\nTotal covered bases: {coverage_stats['total_covered_bases']:,} bp") - print(f"Total proportion covered: {coverage_stats['total_proportion']*100:.4f}%") - print("="*60) + print( + f" + strand: {coverage_stats['plus_strand_coverage']:,} bp " + f"({coverage_stats['plus_strand_proportion']*100:.4f}%)" + ) + print( + f" - strand: {coverage_stats['minus_strand_coverage']:,} bp " + f"({coverage_stats['minus_strand_proportion']*100:.4f}%)" + ) + if coverage_stats["unstranded_coverage"] > 0: + print( + f" . strand: {coverage_stats['unstranded_coverage']:,} bp " + f"({coverage_stats['unstranded_proportion']*100:.4f}%)" + ) + print( + f"\nTotal covered bases: {coverage_stats['total_covered_bases']:,} bp" + ) + print( + f"Total proportion covered: {coverage_stats['total_proportion']*100:.4f}%" + ) + print("=" * 60) else: - print("Warning: Could not calculate genome coverage statistics", file=sys.stderr) + print( + "Warning: Could not calculate genome coverage statistics", + file=sys.stderr, + ) else: - print("Warning: Generated BED file is empty, cannot calculate coverage", file=sys.stderr) + print( + "Warning: Generated BED file is empty, cannot calculate coverage", + file=sys.stderr, + ) except Exception as e: logger.error(f"Error calculating genome coverage: {e}") - print(f"Warning: Error calculating genome coverage: {e}", file=sys.stderr) - + print( + f"Warning: Error calculating genome coverage: {e}", file=sys.stderr + ) + sys.exit(0) else: print("Failed to generate master BED file", file=sys.stderr) diff --git a/src/robin/analysis/master_csv_manager.py b/src/robin/analysis/master_csv_manager.py index 8a10a8ac..d6e14244 100644 --- a/src/robin/analysis/master_csv_manager.py +++ b/src/robin/analysis/master_csv_manager.py @@ -6,13 +6,14 @@ for each sample, tracking comprehensive metadata across multiple BAM files. """ +import logging import os import tempfile -import pandas as pd -from typing import Dict, Any -from dataclasses import dataclass -import logging import time +from dataclasses import dataclass +from typing import Any, Dict + +import pandas as pd # Per-sample file lock for master.csv read-modify-write from robin.analysis.master_bed_generator import FileLock @@ -38,9 +39,11 @@ def _write_csv_internal(self, data: Dict[str, Any], csv_path: str) -> None: """ df = pd.DataFrame([data]) csv_dir = os.path.dirname(csv_path) - temp_fd, temp_path = tempfile.mkstemp(dir=csv_dir, prefix='.master_', suffix='.csv.tmp') + temp_fd, temp_path = tempfile.mkstemp( + dir=csv_dir, prefix=".master_", suffix=".csv.tmp" + ) try: - with os.fdopen(temp_fd, 'w') as f: + with os.fdopen(temp_fd, "w") as f: df.to_csv(f, index=False) f.flush() os.fsync(f.fileno()) @@ -84,7 +87,7 @@ def update_master_csv( def _load_existing_data(self, csv_path: str) -> Dict[str, Any]: """ Load existing data from CSV or create default structure. - + No read locking is used because: 1. Writers use atomic write-and-rename, so readers never see partial files 2. Worst case is reading slightly stale data (acceptable for monitoring) @@ -92,25 +95,35 @@ def _load_existing_data(self, csv_path: str) -> Dict[str, Any]: """ # Start with default structure to ensure all fields are present data = self._get_default_structure() - + if os.path.exists(csv_path): try: # Read without locking - atomic writes ensure we never see partial data df = pd.read_csv(csv_path) if not df.empty: csv_data = df.iloc[0].to_dict() - + # Merge CSV data with default structure (CSV data takes precedence) for key, value in csv_data.items(): - if value is not None and not (isinstance(value, float) and pd.isna(value)): + if value is not None and not ( + isinstance(value, float) and pd.isna(value) + ): data[key] = value - + # Ensure string fields are properly converted to strings # to prevent float objects from being passed to split() methods string_fields = [ - "devices", "basecall_models", "modbase_models", "run_time", "flowcell_ids", - "run_info_run_time", "run_info_device", "run_info_model", - "run_info_flow_cell", "samples_overview_job_types", "analysis_panel" + "devices", + "basecall_models", + "modbase_models", + "run_time", + "flowcell_ids", + "run_info_run_time", + "run_info_device", + "run_info_model", + "run_info_flow_cell", + "samples_overview_job_types", + "analysis_panel", ] for field in string_fields: if field in data and data[field] is not None: @@ -119,7 +132,7 @@ def _load_existing_data(self, csv_path: str) -> Dict[str, Any]: print(f"Warning: Error reading existing CSV: {e}") return data - + def _get_default_structure(self) -> Dict[str, Any]: return { "counter_bam_passed": 0, @@ -324,7 +337,8 @@ def update_sample_overview(self, sample_id: str, overview: Dict[str, Any]) -> No existing_data = self._load_existing_data(master_csv_path) existing_data["samples_overview_active_jobs"] = int( overview.get( - "active_jobs", existing_data.get("samples_overview_active_jobs", 0) + "active_jobs", + existing_data.get("samples_overview_active_jobs", 0), ) ) existing_data["samples_overview_pending_jobs"] = int( @@ -335,7 +349,8 @@ def update_sample_overview(self, sample_id: str, overview: Dict[str, Any]) -> No ) existing_data["samples_overview_total_jobs"] = int( overview.get( - "total_jobs", existing_data.get("samples_overview_total_jobs", 0) + "total_jobs", + existing_data.get("samples_overview_total_jobs", 0), ) ) existing_data["samples_overview_completed_jobs"] = int( @@ -346,7 +361,8 @@ def update_sample_overview(self, sample_id: str, overview: Dict[str, Any]) -> No ) existing_data["samples_overview_failed_jobs"] = int( overview.get( - "failed_jobs", existing_data.get("samples_overview_failed_jobs", 0) + "failed_jobs", + existing_data.get("samples_overview_failed_jobs", 0), ) ) jt_value = overview.get("job_types") @@ -374,7 +390,9 @@ def update_sample_overview(self, sample_id: str, overview: Dict[str, Any]) -> No try: last_seen = float(overview.get("last_seen")) except Exception: - last_seen = float(existing_data.get("samples_overview_last_seen", 0.0)) + last_seen = float( + existing_data.get("samples_overview_last_seen", 0.0) + ) existing_data["samples_overview_last_seen"] = last_seen self._write_csv_internal(existing_data, master_csv_path) diff --git a/src/robin/analysis/methylation_wrapper.py b/src/robin/analysis/methylation_wrapper.py index db3d9c0e..dd9ed268 100644 --- a/src/robin/analysis/methylation_wrapper.py +++ b/src/robin/analysis/methylation_wrapper.py @@ -1,12 +1,12 @@ # methylartist_locus_capture.py -import sys -import shutil +import os import runpy +import shutil +import sys from contextlib import contextmanager, redirect_stderr -import os +from typing import Any, Dict, List, Optional from matplotlib.figure import Figure -from typing import List, Optional, Dict, Any @contextmanager @@ -63,15 +63,17 @@ def _prepend_conda_env_lib_to_ld_library_path(): /lib/x86_64-linux-gnu/ (fixes CXXABI_1.3.15 on older hosts when methylartist loads via runpy). """ - prefix = os.environ.get("CONDA_PREFIX") or getattr(sys, "base_prefix", None) or sys.prefix + prefix = ( + os.environ.get("CONDA_PREFIX") + or getattr(sys, "base_prefix", None) + or sys.prefix + ) lib_dir = os.path.join(prefix, "lib") if not os.path.isdir(lib_dir): yield return previous = os.environ.get("LD_LIBRARY_PATH") - os.environ["LD_LIBRARY_PATH"] = ( - f"{lib_dir}:{previous}" if previous else lib_dir - ) + os.environ["LD_LIBRARY_PATH"] = f"{lib_dir}:{previous}" if previous else lib_dir try: yield finally: @@ -184,9 +186,10 @@ def locus_figure( argv = [str(a) for a in argv] # Check if BAM file is indexed and the index is valid before calling methylartist + import logging import os + import pysam - import logging if not has_bam_index(bam_path): raise RuntimeError( @@ -194,7 +197,7 @@ def locus_figure( f"Index file (.bai or .csi) not found or cannot be used for fetch." ) bai_path = f"{bam_path}.bai" - + # Check if index file has content (not empty) try: bai_size = os.path.getsize(bai_path) @@ -208,20 +211,20 @@ def locus_figure( except OSError as e: logging.warning(f"[MGMT] Could not check index file size: {bai_path} - {e}") # Continue anyway - might be a permission issue - + # Verify the index is actually readable and valid try: with pysam.AlignmentFile(bam_path, "rb") as test_bam: # Try to access the header first _ = test_bam.header - + # Verify file has references if not test_bam.references: raise RuntimeError( f"BAM file has no references: {bam_path}. " f"The file may be empty or corrupted." ) - + # Verify the index is actually readable try: test_bam.check_index() @@ -230,9 +233,17 @@ def locus_figure( # Try to actually use the index by fetching a small region first_chrom = test_bam.references[0] # Try fetching a small region - this will fail if index is invalid - list(test_bam.fetch(first_chrom, 0, min(1000, test_bam.get_reference_length(first_chrom) or 1000))) + list( + test_bam.fetch( + first_chrom, + 0, + min(1000, test_bam.get_reference_length(first_chrom) or 1000), + ) + ) except Exception as idx_error: - logging.warning(f"[MGMT] BAM index exists but is not readable: {bai_path} - {idx_error}") + logging.warning( + f"[MGMT] BAM index exists but is not readable: {bai_path} - {idx_error}" + ) raise RuntimeError( f"BAM file index is invalid or corrupted: {bam_path}. " f"The index file exists but cannot be read. " @@ -254,9 +265,12 @@ def locus_figure( captured = {"fig": None} - with _prepend_conda_env_lib_to_ld_library_path(), _patch_fig_savefig( - captured - ), _patch_argv(argv), _suppress_methylartist_warnings(): + with ( + _prepend_conda_env_lib_to_ld_library_path(), + _patch_fig_savefig(captured), + _patch_argv(argv), + _suppress_methylartist_warnings(), + ): # Execute the installed CLI script as __main__ # Catch SystemExit exceptions from methylartist (e.g., when BAM is not indexed) try: @@ -264,19 +278,22 @@ def locus_figure( except SystemExit as e: # SystemExit(0) is normal for successful completion # Only raise error if exit code is non-zero - exit_code = e.code if hasattr(e, 'code') else 0 + exit_code = e.code if hasattr(e, "code") else 0 if exit_code != 0: - error_msg = str(e) if str(e) else f"methylartist exited with code {exit_code}" + error_msg = ( + str(e) if str(e) else f"methylartist exited with code {exit_code}" + ) raise RuntimeError(f"methylartist failed: {error_msg}") # Try to get figure from capture first fig = captured["fig"] - + # Fallback: if savefig wasn't called, try to get the figure from matplotlib's figure manager if fig is None: - import matplotlib.pyplot as plt import logging - + + import matplotlib.pyplot as plt + # Try to get the current figure try: # Get all figure numbers @@ -285,7 +302,9 @@ def locus_figure( # Get the most recent figure (highest number) latest_fig_num = max(figure_numbers) fig = plt.figure(latest_fig_num) - logging.debug(f"[MGMT] Captured figure from matplotlib figure manager (figure {latest_fig_num})") + logging.debug( + f"[MGMT] Captured figure from matplotlib figure manager (figure {latest_fig_num})" + ) else: # Try to get current figure (might be None) fig = plt.gcf() @@ -294,33 +313,41 @@ def locus_figure( else: fig = None except Exception as e: - logging.debug(f"[MGMT] Failed to get figure from matplotlib figure manager: {e}") + logging.debug( + f"[MGMT] Failed to get figure from matplotlib figure manager: {e}" + ) fig = None - + if fig is None: # Provide more helpful error message import logging - logging.error(f"[MGMT] methylartist locus did not produce a figure. " - f"BAM: {bam_path}, Interval: {interval}") + + logging.error( + f"[MGMT] methylartist locus did not produce a figure. " + f"BAM: {bam_path}, Interval: {interval}" + ) raise RuntimeError( f"methylartist locus did not produce a figure. " f"This may indicate that the BAM file has no reads in the specified interval " f"({interval}) or methylartist encountered an error. " f"Check that the BAM file contains methylation data (MM/ML tags) for this region." ) - + # Verify the figure has content (axes with data) try: axes = fig.get_axes() if not axes: import logging - logging.warning(f"[MGMT] Figure has no axes - methylartist may have produced an empty figure") + + logging.warning( + f"[MGMT] Figure has no axes - methylartist may have produced an empty figure" + ) # Still return the figure - let the caller handle empty figures else: # Check if at least one axis has data has_data = False for ax in axes: - if hasattr(ax, 'has_data') and ax.has_data(): + if hasattr(ax, "has_data") and ax.has_data(): has_data = True break # Check if axis has any artists (lines, patches, etc.) @@ -329,9 +356,13 @@ def locus_figure( break if not has_data: import logging - logging.debug(f"[MGMT] Figure axes exist but contain no data - this may indicate no reads in interval") + + logging.debug( + f"[MGMT] Figure axes exist but contain no data - this may indicate no reads in interval" + ) except Exception as e: import logging + logging.debug(f"[MGMT] Could not verify figure content: {e}") # Continue anyway - figure might still be valid @@ -339,7 +370,7 @@ def locus_figure( # Check if interval matches MGMT region (chr10:129466536-129467536) try: import logging - + # Determine which sites to annotate mgmt_sites = [] if "chr10" in interval and "129466" in interval: @@ -349,14 +380,19 @@ def locus_figure( pos_str = str(row.get("pos", "")) site_label = str(row.get("site", "")) # Extract just the number (e.g., "1" from "Site 1" or "1" from "Site 1 (CpG ...)") - site_num = site_label.split("(")[0].strip().replace("Site ", "").strip() + site_num = ( + site_label.split("(")[0].strip().replace("Site ", "").strip() + ) if not site_num: # Fallback: try to extract number from the label import re - match = re.search(r'\d+', site_label) + + match = re.search(r"\d+", site_label) site_num = match.group(0) if match else site_label.strip() - - parts = [p.strip() for p in pos_str.split("/") if p.strip().isdigit()] + + parts = [ + p.strip() for p in pos_str.split("/") if p.strip().isdigit() + ] if len(parts) >= 2: try: p1 = float(parts[0]) @@ -364,7 +400,7 @@ def locus_figure( mgmt_sites.append((p1, p2, site_num)) except (ValueError, IndexError): continue - + # If no sites from site_rows, use default MGMT positions if not mgmt_sites: mgmt_sites = [ @@ -373,13 +409,13 @@ def locus_figure( (129467262, 129467263, "3"), (129467272, 129467273, "4"), ] - + # Get all axes in the figure - axes = [ax for ax in fig.get_axes() if hasattr(ax, 'get_xlim')] - + axes = [ax for ax in fig.get_axes() if hasattr(ax, "get_xlim")] + if not axes: axes = [fig.gca()] - + # Parse the interval to get the start position (methylartist uses relative coordinates) interval_start = None try: @@ -392,7 +428,7 @@ def locus_figure( interval_end = int(end_str) except Exception: pass - + # Convert absolute genomic positions to relative positions (relative to interval start) if interval_start: relative_mgmt_sites = [] @@ -401,10 +437,10 @@ def locus_figure( rel_p2 = p2 - interval_start relative_mgmt_sites.append((rel_p1, rel_p2, label)) mgmt_sites = relative_mgmt_sites - + # Find the main axis for text annotations (only calculate once) main_ax = _choose_main_axis(fig) - + # Pre-calculate all label positions with proper staggering # First, collect all sites that need annotations label_positions_dict = {} @@ -413,45 +449,53 @@ def locus_figure( y_range = ylim[1] - ylim[0] xlim = main_ax.get_xlim() x_range = xlim[1] - xlim[0] - + # Estimate label width in data coordinates (approximate) # Font size 9 with padding means roughly 0.02-0.03 of x_range per character - max_label_len = max([len(l) for _, _, l in mgmt_sites]) if mgmt_sites else 1 + max_label_len = ( + max([len(l) for _, _, l in mgmt_sites]) if mgmt_sites else 1 + ) label_width_data = (max_label_len + 2) * 0.015 * x_range - + # Calculate center positions for all sites site_centers = [] for p1, p2, label in mgmt_sites: center_x = (p1 + p2) / 2.0 if xlim[0] <= center_x <= xlim[1]: site_centers.append((center_x, label)) - + # Sort by x position to process left to right site_centers.sort(key=lambda x: x[0]) - + # Calculate staggered y positions to avoid overlap base_y_offset = 0.02 vertical_spacing = 0.025 # Fraction of y_range for spacing - + for idx, (center_x, label) in enumerate(site_centers): # Start with base position annotation_y = ylim[1] - (base_y_offset * y_range) - + # Check for horizontal overlap with previously placed labels overlap_threshold = label_width_data * 0.8 # 80% of label width stagger_level = 0 - + # Find all overlapping labels and determine best stagger position overlapping_labels = [] - for existing_label, (existing_x, existing_y, existing_stagger) in label_positions_dict.items(): + for existing_label, ( + existing_x, + existing_y, + existing_stagger, + ) in label_positions_dict.items(): if abs(center_x - existing_x) < overlap_threshold: - overlapping_labels.append((existing_x, existing_y, existing_stagger)) - + overlapping_labels.append( + (existing_x, existing_y, existing_stagger) + ) + if overlapping_labels: # Find the maximum stagger level among overlapping labels max_stagger = max([s for _, _, s in overlapping_labels]) stagger_level = max_stagger + 1 - + # Alternate direction: even stagger levels go up, odd levels go down # This creates a zigzag pattern if stagger_level % 2 == 0: @@ -462,44 +506,51 @@ def locus_figure( # Go down from the lowest overlapping label min_y = min([y for _, y, _ in overlapping_labels]) annotation_y = min_y - (vertical_spacing * y_range) - + # Ensure we don't go outside the plot area - annotation_y = max(ylim[0] + 0.05 * y_range, min(ylim[1] - 0.02 * y_range, annotation_y)) - + annotation_y = max( + ylim[0] + 0.05 * y_range, + min(ylim[1] - 0.02 * y_range, annotation_y), + ) + # Store the position with stagger level - label_positions_dict[label] = (center_x, annotation_y, stagger_level) - + label_positions_dict[label] = ( + center_x, + annotation_y, + stagger_level, + ) + # Add vertical lines to ALL axes (all subplots) for ax_idx, ax in enumerate(axes): try: xlim = ax.get_xlim() - + # For each MGMT site, draw vertical lines at both CpG positions for p1, p2, label in mgmt_sites: # Only draw if positions are within the visible x-axis range if xlim[0] <= p1 <= xlim[1] or xlim[0] <= p2 <= xlim[1]: # Draw dashed vertical lines - more subtle, lower zorder so data points appear on top ax.axvline( - p1, - color="crimson", + p1, + color="crimson", linestyle="--", linewidth=1.0, alpha=0.5, - zorder=5 + zorder=5, ) ax.axvline( - p2, - color="crimson", + p2, + color="crimson", linestyle="--", linewidth=1.0, alpha=0.5, - zorder=5 + zorder=5, ) - + # Add text annotation on the main axis only if ax == main_ax and label in label_positions_dict: center_x, annotation_y, _ = label_positions_dict[label] - + ax.text( center_x, annotation_y, @@ -534,11 +585,16 @@ def locus_figure( except Exception: pass - logging.debug(f"[MGMT] Successfully added annotations for {len(mgmt_sites)} sites") + logging.debug( + f"[MGMT] Successfully added annotations for {len(mgmt_sites)} sites" + ) else: - logging.debug(f"[MGMT] Interval {interval} is not MGMT region, skipping annotations") + logging.debug( + f"[MGMT] Interval {interval} is not MGMT region, skipping annotations" + ) except Exception as e: import logging + logging.exception(f"[MGMT] Failed to add annotations: {e}") return fig @@ -604,13 +660,16 @@ def load_figure_pickle(path: str) -> Figure: return pickle.load(f) -def try_load_figure_pickle(path: str, *, remove_if_invalid: bool = True) -> Optional[Figure]: +def try_load_figure_pickle( + path: str, *, remove_if_invalid: bool = True +) -> Optional[Figure]: """ Load a pickled figure when it is renderable in the current matplotlib. Returns None if unpickling fails, axes are incompatible, or savefig would fail. """ import logging + import matplotlib.pyplot as plt try: diff --git a/src/robin/analysis/mgmt_analysis.py b/src/robin/analysis/mgmt_analysis.py index 8108b324..7fb0b95c 100644 --- a/src/robin/analysis/mgmt_analysis.py +++ b/src/robin/analysis/mgmt_analysis.py @@ -16,15 +16,17 @@ - Comprehensive metadata extraction and logging """ +import logging import os -import time +import shutil import subprocess import tempfile -import shutil -import pysam -import logging +import time from dataclasses import dataclass, field -from typing import Dict, Any, List, Optional +from typing import Any, Dict, List, Optional + +import pysam + from robin.logging_config import get_job_logger # Try to find HV path from robin package, fallback to common locations @@ -63,24 +65,24 @@ def safely_sort_and_index_bam( output_bam: str, logger: logging.Logger, threads: int = 4, - verify_readable: bool = True + verify_readable: bool = True, ) -> bool: """ Safely sort and index a BAM file with proper verification to prevent truncated file issues. - + This function ensures that: 1. The input BAM file exists and is readable 2. The sorted BAM file is fully written before indexing 3. The indexed BAM file is verified to be readable 4. Proper error handling and logging throughout - + Args: input_bam: Path to input BAM file output_bam: Path to output sorted BAM file logger: Logger instance for logging threads: Number of threads to use for sorting verify_readable: If True, verify the output BAM is readable after creation - + Returns: True if successful, False otherwise """ @@ -89,14 +91,16 @@ def safely_sort_and_index_bam( if not os.path.exists(input_bam): logger.error(f"Input BAM file does not exist: {input_bam}") return False - + input_size = os.path.getsize(input_bam) if input_size == 0: logger.warning(f"Input BAM file is empty: {input_bam}") return False - - logger.debug(f"Sorting BAM file: {input_bam} -> {output_bam} (input size: {input_size} bytes)") - + + logger.debug( + f"Sorting BAM file: {input_bam} -> {output_bam} (input size: {input_size} bytes)" + ) + # Remove output file if it exists (to avoid issues with partial writes) if os.path.exists(output_bam): try: @@ -104,8 +108,10 @@ def safely_sort_and_index_bam( if os.path.exists(f"{output_bam}.bai"): os.remove(f"{output_bam}.bai") except OSError as e: - logger.warning(f"Could not remove existing output file {output_bam}: {e}") - + logger.warning( + f"Could not remove existing output file {output_bam}: {e}" + ) + # Sort the BAM file try: if threads > 1: @@ -115,19 +121,21 @@ def safely_sort_and_index_bam( except Exception as e: logger.error(f"Failed to sort BAM file {input_bam}: {e}") return False - + # Verify sorted file was created and has content if not os.path.exists(output_bam): logger.error(f"Sorted BAM file was not created: {output_bam}") return False - + output_size = os.path.getsize(output_bam) if output_size == 0: logger.error(f"Sorted BAM file is empty: {output_bam}") return False - - logger.debug(f"Sorted BAM file created: {output_bam} (size: {output_size} bytes)") - + + logger.debug( + f"Sorted BAM file created: {output_bam} (size: {output_size} bytes)" + ) + # Verify the sorted BAM is readable before indexing if verify_readable: try: @@ -140,7 +148,9 @@ def safely_sort_and_index_bam( read_count += 1 if read_count >= 10: # Just verify first 10 reads break - logger.debug(f"Verified sorted BAM is readable (checked {read_count} reads)") + logger.debug( + f"Verified sorted BAM is readable (checked {read_count} reads)" + ) except Exception as e: logger.error(f"Sorted BAM file appears corrupted or truncated: {e}") # Clean up corrupted file @@ -149,17 +159,17 @@ def safely_sort_and_index_bam( except OSError: pass return False - + # Index the sorted BAM file try: index_file = f"{output_bam}.bai" pysam.index(output_bam, index_file) - + # Verify index was created if not os.path.exists(index_file): logger.error(f"BAM index file was not created: {index_file}") return False - + logger.debug(f"BAM index created: {index_file}") except Exception as e: logger.error(f"Failed to index BAM file {output_bam}: {e}") @@ -170,7 +180,7 @@ def safely_sort_and_index_bam( except OSError: pass return False - + # Final verification: ensure both BAM and index are readable if verify_readable: try: @@ -182,20 +192,25 @@ def safely_sort_and_index_bam( # Get first chromosome from header if final_bam.references: first_chrom = final_bam.references[0] - list(final_bam.fetch(first_chrom, 0, 1000)) # Try fetching a small region + list( + final_bam.fetch(first_chrom, 0, 1000) + ) # Try fetching a small region except Exception: # If fetch fails, that's okay - file might be empty or have no reads pass logger.info(f"Successfully sorted and indexed BAM: {output_bam}") except Exception as e: - logger.error(f"Final verification failed for sorted BAM {output_bam}: {e}") + logger.error( + f"Final verification failed for sorted BAM {output_bam}: {e}" + ) return False - + return True - + except Exception as e: logger.error(f"Unexpected error in safely_sort_and_index_bam: {e}") import traceback + logger.error(traceback.format_exc()) return False @@ -203,11 +218,11 @@ def safely_sort_and_index_bam( def validate_methylation_data(bam_file: str, logger: logging.Logger) -> Dict[str, Any]: """ Validate that BAM file has sufficient methylation data for methylartist. - + Args: bam_file (str): Path to BAM file logger (logging.Logger): Logger instance - + Returns: Dict containing validation results and metadata """ @@ -217,18 +232,18 @@ def validate_methylation_data(bam_file: str, logger: logging.Logger) -> Dict[str "reads_with_mm_tags": 0, "reads_with_ml_tags": 0, "modification_types": set(), - "error_message": None + "error_message": None, } - + try: with pysam.AlignmentFile(bam_file, "rb") as bam: read_count = 0 mm_count = 0 ml_count = 0 - + for read in bam.fetch(until_eof=True): read_count += 1 - + # Check for MM tags (modification calls) if read.has_tag("MM"): mm_count += 1 @@ -240,75 +255,89 @@ def validate_methylation_data(bam_file: str, logger: logging.Logger) -> Dict[str if ":" in mod: mod_type = mod.split(":")[0] validation_result["modification_types"].add(mod_type) - + # Check for ML tags (modification likelihoods) if read.has_tag("ML"): ml_count += 1 - + # Stop counting after reasonable sample size to avoid long processing if read_count >= 10000: break - + validation_result["total_reads"] = read_count validation_result["reads_with_mm_tags"] = mm_count validation_result["reads_with_ml_tags"] = ml_count - + # Determine if we have sufficient data # Need at least some reads with methylation data min_reads_with_mods = 10 has_modifications = len(validation_result["modification_types"]) > 0 - + if mm_count >= min_reads_with_mods and has_modifications: validation_result["has_sufficient_data"] = True - logger.info(f"Methylation data validation passed: {mm_count} reads with MM tags, mods: {validation_result['modification_types']}") + logger.info( + f"Methylation data validation passed: {mm_count} reads with MM tags, mods: {validation_result['modification_types']}" + ) else: - validation_result["error_message"] = f"Insufficient methylation data: {mm_count} reads with MM tags (min: {min_reads_with_mods}), modifications: {validation_result['modification_types']}" + validation_result["error_message"] = ( + f"Insufficient methylation data: {mm_count} reads with MM tags (min: {min_reads_with_mods}), modifications: {validation_result['modification_types']}" + ) logger.warning(validation_result["error_message"]) - + except Exception as e: - validation_result["error_message"] = f"Error validating methylation data: {str(e)}" + validation_result["error_message"] = ( + f"Error validating methylation data: {str(e)}" + ) logger.error(validation_result["error_message"]) - - return validation_result + return validation_result def extract_mgmt_site_rows_from_bed(bed_path: str) -> List[Dict[str, Any]]: """ Extract MGMT-specific CpG site information from a bedmethyl file. - + Args: bed_path (str): Path to bedmethyl file - + Returns: List of dictionaries containing site information for annotation """ try: import pandas as pd - + if not os.path.exists(bed_path): return [] - + df = pd.read_csv(bed_path, sep="\t", header=None) - + # Check if column 10 contains space-separated values (old format) has_space_separated_col10 = False if df.shape[1] > 10 and len(df) > 0: sample_val = str(df.iloc[0, 9]) - has_space_separated_col10 = ' ' in sample_val or '\t' in sample_val - + has_space_separated_col10 = " " in sample_val or "\t" in sample_val + # Check if this is the new bedmethyl format (separate columns) or old format if df.shape[1] >= 12 and not has_space_separated_col10: # New bedmethyl format with separate columns cols = [ - "Chromosome", "Start", "End", "Modified_Base_Code", "Score", - "Strand", "Start2", "End2", "RGB", - "Nvalid_cov", "Fraction_Modified", "Nmod", + "Chromosome", + "Start", + "End", + "Modified_Base_Code", + "Score", + "Strand", + "Start2", + "End2", + "RGB", + "Nvalid_cov", + "Fraction_Modified", + "Nmod", ] num_cols_to_read = min(len(cols), df.shape[1]) df = df.iloc[:, :num_cols_to_read] df.columns = cols[:num_cols_to_read] - + df["Nvalid_cov"] = df["Nvalid_cov"].astype(float) df["Fraction_Modified"] = df["Fraction_Modified"].astype(float) # Some bedmethyl outputs store fraction as percent (0-100). @@ -319,37 +348,47 @@ def extract_mgmt_site_rows_from_bed(bed_path: str) -> List[Dict[str, Any]]: df["Nmod"] = df["Nmod"].astype(float) else: df["Nmod"] = df["Nvalid_cov"] * df["Fraction_Modified"] - + df["Start"] = df["Start"].astype(int) df["Coverage"] = df["Nvalid_cov"] df["Modified_Fraction"] = df["Fraction_Modified"] * 100.0 elif df.shape[1] >= 10: # Old format with space-separated Coverage_Info in column 10 cols = [ - "Chromosome", "Start", "End", "Name", "Score", - "Strand", "Start2", "End2", "RGB", "Coverage_Info", + "Chromosome", + "Start", + "End", + "Name", + "Score", + "Strand", + "Start2", + "End2", + "RGB", + "Coverage_Info", ] - df = df.iloc[:, :len(cols)] + df = df.iloc[:, : len(cols)] df.columns = cols - + cov_split = df["Coverage_Info"].astype(str).str.split() df["Coverage"] = cov_split.str[0].astype(float) fraction_val = cov_split.str[1].astype(float).fillna(0.0) - is_percentage = (fraction_val > 1.0).any() if len(fraction_val) > 0 else False - + is_percentage = ( + (fraction_val > 1.0).any() if len(fraction_val) > 0 else False + ) + if is_percentage: df["Modified_Fraction"] = fraction_val df["Fraction_Modified"] = df["Modified_Fraction"] / 100.0 else: df["Fraction_Modified"] = fraction_val df["Modified_Fraction"] = df["Fraction_Modified"] * 100.0 - + df["Nvalid_cov"] = df["Coverage"] df["Nmod"] = df["Coverage"] * df["Fraction_Modified"] df["Start"] = df["Start"].astype(int) else: return [] - + cpg_pairs = [ (129467255, 129467256), (129467258, 129467259), @@ -363,25 +402,25 @@ def extract_mgmt_site_rows_from_bed(bed_path: str) -> List[Dict[str, Any]]: "129467262/129467263": "3", "129467272/129467273": "4", } - + for p1, p2 in cpg_pairs: pos_key = f"{p1}/{p2}" site_label = label_map.get(pos_key, "Unknown") - + # Check forward strand reads at position p1 fwd_p1 = df[ (df["Chromosome"] == "chr10") & (df["Start"] == p1 - 1) & (df["Strand"] == "+") ] - + # Check reverse strand reads at position p2 rev_p2 = df[ (df["Chromosome"] == "chr10") & (df["Start"] == p2 - 1) & (df["Strand"] == "-") ] - + # Get forward strand data from p1 if not fwd_p1.empty: cov_f = float(fwd_p1["Nvalid_cov"].iloc[0]) @@ -392,7 +431,7 @@ def extract_mgmt_site_rows_from_bed(bed_path: str) -> List[Dict[str, Any]]: cov_f = 0.0 mf = 0.0 meth_fwd_count = 0 - + # Get reverse strand data from p2 if not rev_p2.empty: cov_r = float(rev_p2["Nvalid_cov"].iloc[0]) @@ -403,29 +442,33 @@ def extract_mgmt_site_rows_from_bed(bed_path: str) -> List[Dict[str, Any]]: cov_r = 0.0 mr = 0.0 meth_rev_count = 0 - + # Only add row if we have data if cov_f > 0 or cov_r > 0: tot = cov_f + cov_r weighted = ((cov_f * mf) + (cov_r * mr)) / tot if tot > 0 else 0.0 weighted_pct = weighted * 100.0 - - rows.append({ - "site": f"{site_label} (CpG {pos_key})", - "chr": "chr10", - "pos": pos_key, - "cov_fwd": int(cov_f), - "cov_rev": int(cov_r), - "cov_total": int(tot), - "meth": round(weighted_pct, 2), - "meth_fwd": int(meth_fwd_count), - "meth_rev": int(meth_rev_count), - "notes": "Combined methylation from both strands of CpG pair", - }) - + + rows.append( + { + "site": f"{site_label} (CpG {pos_key})", + "chr": "chr10", + "pos": pos_key, + "cov_fwd": int(cov_f), + "cov_rev": int(cov_r), + "cov_total": int(tot), + "meth": round(weighted_pct, 2), + "meth_fwd": int(meth_fwd_count), + "meth_rev": int(meth_rev_count), + "notes": "Combined methylation from both strands of CpG pair", + } + ) + return rows except Exception as e: - logging.getLogger("robin.mgmt").debug(f"Failed to extract site rows from {bed_path}: {e}") + logging.getLogger("robin.mgmt").debug( + f"Failed to extract site rows from {bed_path}: {e}" + ) return [] @@ -437,15 +480,15 @@ def generate_mgmt_visualization( bed_file: Optional[str] = None, logger: Optional[logging.Logger] = None, extra_cli: Optional[List[str]] = None, - use_fallback: bool = False + use_fallback: bool = False, ) -> Dict[str, Any]: """ Generate MGMT methylation visualization with both PNG and pickle outputs. - + This unified function uses the locus_figure approach to capture the matplotlib figure object, then saves both a PNG file (for reports) and a pickle file (for fast GUI loading). The figure includes MGMT CpG site annotations. - + Args: bam_file (str): Path to BAM file output_png (str): Output PNG file path @@ -455,13 +498,13 @@ def generate_mgmt_visualization( logger (Optional[logging.Logger]): Logger instance extra_cli (Optional[List[str]]): Extra CLI parameters for methylartist use_fallback (bool): Whether to use fallback parameters (more lenient) - + Returns: Dict containing execution results, metadata, and paths to generated files """ if logger is None: logger = logging.getLogger("robin.mgmt") - + result = { "success": False, "error_message": None, @@ -469,40 +512,44 @@ def generate_mgmt_visualization( "fallback_used": use_fallback, "png_path": output_png, "pickle_path": None, - "figure": None + "figure": None, } - + # Determine pickle path (same directory as PNG, with .pkl extension) pickle_path = os.path.splitext(output_png)[0] + ".pkl" result["pickle_path"] = pickle_path - + try: # Step 1: Validate methylation data logger.info(f"Validating methylation data in {os.path.basename(bam_file)}") validation = validate_methylation_data(bam_file, logger) - + if not validation["has_sufficient_data"]: - result["error_message"] = f"Methylation data validation failed: {validation['error_message']}" + result["error_message"] = ( + f"Methylation data validation failed: {validation['error_message']}" + ) logger.warning(result["error_message"]) return result - + result["validation_passed"] = True logger.info("Methylation data validation passed, proceeding with methylartist") - + # Step 2: Extract site_rows from bed file if provided site_rows = None if bed_file and os.path.exists(bed_file): try: site_rows = extract_mgmt_site_rows_from_bed(bed_file) if site_rows: - logger.debug(f"Extracted {len(site_rows)} site rows from {bed_file}") + logger.debug( + f"Extracted {len(site_rows)} site rows from {bed_file}" + ) except Exception as e: logger.debug(f"Failed to extract site rows from {bed_file}: {e}") - + # Step 3: Build extra_cli parameters if extra_cli is None: extra_cli = [] - + # Add figure size parameters if not already specified has_width = any("--width" in str(arg) for arg in extra_cli) has_height = any("--height" in str(arg) for arg in extra_cli) @@ -510,7 +557,7 @@ def generate_mgmt_visualization( extra_cli.extend(["--width", "24"]) if not has_height: extra_cli.extend(["--height", "12"]) - + # Add quality/read parameters based on fallback mode if use_fallback: # More lenient parameters for fallback @@ -530,11 +577,14 @@ def generate_mgmt_visualization( extra_cli.extend(["--minqual", "10"]) if not any("--smoothwindowsize" in str(arg) for arg in extra_cli): extra_cli.extend(["--smoothwindowsize", "5"]) - + # Step 4: Generate figure using locus_figure try: - from robin.analysis.methylation_wrapper import locus_figure, save_figure_pickle - + from robin.analysis.methylation_wrapper import ( + locus_figure, + save_figure_pickle, + ) + logger.info(f"Generating methylation visualization using locus_figure") fig = locus_figure( interval=interval, @@ -542,41 +592,45 @@ def generate_mgmt_visualization( motif="CG", mods="m", extra_cli=extra_cli, - site_rows=site_rows + site_rows=site_rows, ) - + result["figure"] = fig - + # Step 5: Save PNG file try: - fig.savefig(output_png, dpi=150, bbox_inches='tight') + fig.savefig(output_png, dpi=150, bbox_inches="tight") logger.info(f"Saved PNG visualization: {os.path.basename(output_png)}") except Exception as e: logger.warning(f"Failed to save PNG file: {e}") # Continue anyway - pickle is more important for GUI - + # Step 6: Save pickle file for fast GUI loading try: save_figure_pickle(fig, pickle_path) - logger.info(f"Saved pickle file for fast GUI loading: {os.path.basename(pickle_path)}") + logger.info( + f"Saved pickle file for fast GUI loading: {os.path.basename(pickle_path)}" + ) except Exception as e: logger.warning(f"Failed to save pickle file: {e}") # Not fatal, but less optimal - + result["success"] = True logger.info("Methylation visualization generated successfully") - + except RuntimeError as e: # Check if this is a validation error we can handle error_msg = str(e) if "not indexed" in error_msg or "index" in error_msg.lower(): result["error_message"] = f"BAM file indexing issue: {error_msg}" elif "did not produce a figure" in error_msg: - result["error_message"] = f"Methylartist failed to produce figure: {error_msg}" + result["error_message"] = ( + f"Methylartist failed to produce figure: {error_msg}" + ) else: result["error_message"] = f"Methylartist error: {error_msg}" logger.error(result["error_message"]) - + # Try fallback if not already using it if not use_fallback: logger.info("Attempting fallback with more lenient parameters") @@ -588,12 +642,14 @@ def generate_mgmt_visualization( bed_file=bed_file, logger=logger, extra_cli=extra_cli, # Keep any custom extra_cli - use_fallback=True + use_fallback=True, ) except Exception as e: - result["error_message"] = f"Unexpected error generating visualization: {str(e)}" + result["error_message"] = ( + f"Unexpected error generating visualization: {str(e)}" + ) logger.error(result["error_message"], exc_info=True) - + # Try fallback if not already using it if not use_fallback: logger.info("Attempting fallback with more lenient parameters") @@ -605,30 +661,32 @@ def generate_mgmt_visualization( bed_file=bed_file, logger=logger, extra_cli=extra_cli, - use_fallback=True + use_fallback=True, ) - + except Exception as e: - result["error_message"] = f"Unexpected error in visualization generation: {str(e)}" + result["error_message"] = ( + f"Unexpected error in visualization generation: {str(e)}" + ) logger.error(result["error_message"], exc_info=True) - + return result def run_methylartist_safely( - bam_file: str, - output_file: str, + bam_file: str, + output_file: str, interval: str = "chr10:129466536-129467536", reference: Optional[str] = None, logger: Optional[logging.Logger] = None, - bed_file: Optional[str] = None + bed_file: Optional[str] = None, ) -> Dict[str, Any]: """ Run methylartist with robust error handling and fallback options. - + This function now uses the unified generate_mgmt_visualization function which saves both PNG and pickle files for optimal performance. - + Args: bam_file (str): Path to BAM file output_file (str): Output PNG file path @@ -636,7 +694,7 @@ def run_methylartist_safely( reference (Optional[str]): Reference genome path logger (Optional[logging.Logger]): Logger instance bed_file (Optional[str]): Path to bedmethyl file for extracting site annotations - + Returns: Dict containing execution results and metadata (backward compatible format) """ @@ -648,18 +706,18 @@ def run_methylartist_safely( reference=reference, bed_file=bed_file, logger=logger, - use_fallback=False + use_fallback=False, ) - + # Convert to backward-compatible format result = { "success": viz_result["success"], "error_message": viz_result["error_message"], "command_used": None, # Not applicable with locus_figure approach "validation_passed": viz_result["validation_passed"], - "fallback_used": viz_result["fallback_used"] + "fallback_used": viz_result["fallback_used"], } - + return result @@ -809,7 +867,7 @@ def run_matkit( output_bam=sorted_bam, logger=logger, threads=4, - verify_readable=True + verify_readable=True, ): logger.error(f"Failed to sort/index BAM file for matkit: {mgmt_bamfile}") return False @@ -872,7 +930,11 @@ def run_matkit( def process_bam_file( - bam_path: str, metadata: Dict[str, Any], work_dir: str, threads: int = 4, reference: Optional[str] = None + bam_path: str, + metadata: Dict[str, Any], + work_dir: str, + threads: int = 4, + reference: Optional[str] = None, ) -> MGMTMetadata: """Process a single BAM file for MGMT analysis""" # print(f"Processing MGMT for BAM file: {bam_path}\n\n") @@ -1025,25 +1087,27 @@ def process_bam_file( output_bam=sorted_mgmt_bam, logger=logger, threads=4, - verify_readable=True + verify_readable=True, ): logger.info( f"Sorted and indexed accumulated MGMT BAM: {sorted_mgmt_bam}" ) mgmt_bam_for_plot = sorted_mgmt_bam else: - logger.warning(f"Failed to sort/index accumulated MGMT BAM, using unsorted BAM") + logger.warning( + f"Failed to sort/index accumulated MGMT BAM, using unsorted BAM" + ) mgmt_bam_for_plot = mgmt_bam_output plot_out = os.path.join(sample_dir, f"{file_number}_mgmt.png") - + # Look for corresponding bed file for site annotations bed_file = os.path.join(sample_dir, f"{file_number}_mgmt.bed") if not os.path.exists(bed_file): # Try alternative naming alt_bed = os.path.join(sample_dir, f"{file_number}_mgmt_mgmt.bed") bed_file = alt_bed if os.path.exists(alt_bed) else None - + # Use the new safe methylartist wrapper methylartist_result = run_methylartist_safely( bam_file=mgmt_bam_for_plot, @@ -1051,7 +1115,7 @@ def process_bam_file( interval="chr10:129466536-129467536", reference=reference, logger=logger, - bed_file=bed_file if bed_file and os.path.exists(bed_file) else None + bed_file=bed_file if bed_file and os.path.exists(bed_file) else None, ) if methylartist_result["success"]: @@ -1060,7 +1124,9 @@ def process_bam_file( if methylartist_result["fallback_used"]: logger.info("Visualization completed using fallback parameters") else: - logger.warning(f"Methylartist visualization failed: {methylartist_result['error_message']}") + logger.warning( + f"Methylartist visualization failed: {methylartist_result['error_message']}" + ) # Don't treat this as a fatal error - analysis can continue without visualization except Exception as e: @@ -1159,7 +1225,7 @@ def run_final_combined_analysis(sample_id: str, work_dir: str) -> bool: def process_multiple_bams(bam_paths, metadata_list, work_dir, logger, reference=None): """ Process multiple BAM files for MGMT analysis using aggregated MGMT data. - + This function processes multiple BAM files for the same sample, accumulating MGMT reads across all files before performing downstream analysis. This is more efficient than processing each BAM file individually and then trying @@ -1177,16 +1243,16 @@ def process_multiple_bams(bam_paths, metadata_list, work_dir, logger, reference= """ if not bam_paths or not metadata_list: raise ValueError("bam_paths and metadata_list must not be empty") - + if len(bam_paths) != len(metadata_list): raise ValueError("bam_paths and metadata_list must have the same length") - + # Get sample ID from first metadata (assuming all BAMs are from same sample) sample_id = metadata_list[0].get("sample_id", "unknown") - + logger.info(f"🧬 Starting multi-BAM MGMT analysis for sample: {sample_id}") logger.info(f"Processing {len(bam_paths)} BAM files for sample {sample_id}") - + # Log essential metadata only for i, (bam_path, metadata) in enumerate(zip(bam_paths, metadata_list)): logger.debug(f"BAM file {i+1}: {os.path.basename(bam_path)}") @@ -1222,44 +1288,63 @@ def process_multiple_bams(bam_paths, metadata_list, work_dir, logger, reference= valid_bam_paths = [] valid_metadata_list = [] total_mgmt_read_count = 0 - + for bam_path, metadata in zip(bam_paths, metadata_list): # Check MGMT reads from preprocessing metadata has_mgmt_reads = metadata.get("has_mgmt_reads", False) mgmt_read_count = metadata.get("mgmt_read_count", 0) - + if has_mgmt_reads: valid_bam_paths.append(bam_path) valid_metadata_list.append(metadata) total_mgmt_read_count += mgmt_read_count - logger.debug(f"BAM {os.path.basename(bam_path)}: {mgmt_read_count} MGMT reads") + logger.debug( + f"BAM {os.path.basename(bam_path)}: {mgmt_read_count} MGMT reads" + ) else: - logger.warning(f"No MGMT reads found in BAM file: {os.path.basename(bam_path)}") - + logger.warning( + f"No MGMT reads found in BAM file: {os.path.basename(bam_path)}" + ) + if not valid_bam_paths: - logger.info(f"No MGMT reads found in any BAM files for sample {sample_id} - this is normal") + logger.info( + f"No MGMT reads found in any BAM files for sample {sample_id} - this is normal" + ) analysis_result["processing_steps"].append("no_mgmt_reads_found") analysis_result["status"] = "no_mgmt_reads" analysis_result["message"] = "No MGMT reads found in any BAM files" return analysis_result - logger.info(f"Processing {len(valid_bam_paths)} valid BAM files out of {len(bam_paths)} total") + logger.info( + f"Processing {len(valid_bam_paths)} valid BAM files out of {len(bam_paths)} total" + ) analysis_result["files_processed"] = len(valid_bam_paths) analysis_result["mgmt_read_count_from_preprocessing"] = total_mgmt_read_count analysis_result["processing_steps"].append("mgmt_reads_found") # Process each BAM file, accumulating MGMT reads mgmt_bam_output = os.path.join(sample_dir, "mgmt.bam") - - for i, (bam_path, metadata) in enumerate(zip(valid_bam_paths, valid_metadata_list)): - logger.info(f"Processing BAM file {i+1}/{len(valid_bam_paths)}: {os.path.basename(bam_path)}") - + + for i, (bam_path, metadata) in enumerate( + zip(valid_bam_paths, valid_metadata_list) + ): + logger.info( + f"Processing BAM file {i+1}/{len(valid_bam_paths)}: {os.path.basename(bam_path)}" + ) + # Extract MGMT region using bedtools temp_bamfile = tempfile.NamedTemporaryFile(suffix=".bam", delete=False) temp_bamfile.close() # Extract MGMT region to temporary file - bedtools_cmd = ["bedtools", "intersect", "-a", bam_path, "-b", mgmt_bed_path] + bedtools_cmd = [ + "bedtools", + "intersect", + "-a", + bam_path, + "-b", + mgmt_bed_path, + ] try: with open(temp_bamfile.name, "wb") as f: @@ -1276,7 +1361,9 @@ def process_multiple_bams(bam_paths, metadata_list, work_dir, logger, reference= not os.path.exists(temp_bamfile.name) or os.path.getsize(temp_bamfile.name) == 0 ): - logger.warning(f"Bedtools failed to create valid BAM file for {os.path.basename(bam_path)}") + logger.warning( + f"Bedtools failed to create valid BAM file for {os.path.basename(bam_path)}" + ) continue # Create index for the BAM file @@ -1286,10 +1373,15 @@ def process_multiple_bams(bam_paths, metadata_list, work_dir, logger, reference= logger.warning(f"Failed to create BAM index: {e}") # Check if the extracted BAM has reads - if pysam.AlignmentFile(temp_bamfile.name, "rb").count(until_eof=True) > 0: + if ( + pysam.AlignmentFile(temp_bamfile.name, "rb").count(until_eof=True) + > 0 + ): if os.path.exists(mgmt_bam_output): # Concatenate with existing mgmt.bam - temp_holder = tempfile.NamedTemporaryFile(suffix=".bam", delete=False) + temp_holder = tempfile.NamedTemporaryFile( + suffix=".bam", delete=False + ) temp_holder.close() pysam.cat( @@ -1309,12 +1401,18 @@ def process_multiple_bams(bam_paths, metadata_list, work_dir, logger, reference= except FileNotFoundError: pass - logger.debug(f"Accumulated MGMT reads from {os.path.basename(bam_path)}") + logger.debug( + f"Accumulated MGMT reads from {os.path.basename(bam_path)}" + ) else: - logger.warning(f"No reads found in extracted MGMT region for {os.path.basename(bam_path)}") + logger.warning( + f"No reads found in extracted MGMT region for {os.path.basename(bam_path)}" + ) except subprocess.TimeoutExpired: - logger.warning(f"Bedtools extraction timed out for {os.path.basename(bam_path)}") + logger.warning( + f"Bedtools extraction timed out for {os.path.basename(bam_path)}" + ) except FileNotFoundError: logger.error("bedtools not found in PATH") raise RuntimeError("bedtools not found in PATH") @@ -1347,23 +1445,27 @@ def process_multiple_bams(bam_paths, metadata_list, work_dir, logger, reference= output_bam=sorted_mgmt_bam, logger=logger, threads=4, - verify_readable=True + verify_readable=True, ): - logger.info(f"Sorted and indexed accumulated MGMT BAM: {sorted_mgmt_bam}") + logger.info( + f"Sorted and indexed accumulated MGMT BAM: {sorted_mgmt_bam}" + ) mgmt_bam_for_plot = sorted_mgmt_bam else: - logger.warning(f"Failed to sort/index accumulated MGMT BAM, using unsorted BAM") + logger.warning( + f"Failed to sort/index accumulated MGMT BAM, using unsorted BAM" + ) mgmt_bam_for_plot = mgmt_bam_output plot_out = os.path.join(sample_dir, "final_mgmt.png") - + # Look for corresponding bed file for site annotations bed_file = os.path.join(sample_dir, "final_mgmt.bed") if not os.path.exists(bed_file): # Try alternative naming alt_bed = os.path.join(sample_dir, "final_mgmt_mgmt.bed") bed_file = alt_bed if os.path.exists(alt_bed) else None - + # Use the new safe methylartist wrapper methylartist_result = run_methylartist_safely( bam_file=mgmt_bam_for_plot, @@ -1371,7 +1473,7 @@ def process_multiple_bams(bam_paths, metadata_list, work_dir, logger, reference= interval="chr10:129466536-129467536", reference=reference, logger=logger, - bed_file=bed_file if bed_file and os.path.exists(bed_file) else None + bed_file=bed_file if bed_file and os.path.exists(bed_file) else None, ) if methylartist_result["success"]: @@ -1380,7 +1482,9 @@ def process_multiple_bams(bam_paths, metadata_list, work_dir, logger, reference= if methylartist_result["fallback_used"]: logger.info("Visualization completed using fallback parameters") else: - logger.warning(f"Methylartist visualization failed: {methylartist_result['error_message']}") + logger.warning( + f"Methylartist visualization failed: {methylartist_result['error_message']}" + ) # Don't treat this as a fatal error - analysis can continue without visualization except Exception as e: @@ -1414,9 +1518,13 @@ def process_multiple_bams(bam_paths, metadata_list, work_dir, logger, reference= analysis_result["processing_steps"].append("analysis_complete") logger.info(f"Multi-BAM MGMT analysis completed for {sample_id}") - logger.info(f"Files successfully processed: {analysis_result['files_processed']}/{analysis_result['total_files']}") - logger.info(f"Total MGMT reads: {analysis_result['mgmt_read_count_from_preprocessing']}") - + logger.info( + f"Files successfully processed: {analysis_result['files_processed']}/{analysis_result['total_files']}" + ) + logger.info( + f"Total MGMT reads: {analysis_result['mgmt_read_count_from_preprocessing']}" + ) + return analysis_result except Exception as e: @@ -1438,40 +1546,44 @@ def mgmt_handler(job, work_dir=None, reference=None): """ # Get job-specific logger logger = get_job_logger(str(job.job_id), job.job_type, job.context.filepath) - + # Check if this is a batched job batched_job = job.context.metadata.get("_batched_job") if batched_job: batch_size = batched_job.get_file_count() sample_id = batched_job.get_sample_id() batch_id = batched_job.batch_id - logger.info(f"Processing MGMT batch: {batch_size} files for sample '{sample_id}' (batch_id: {batch_id})") - + logger.info( + f"Processing MGMT batch: {batch_size} files for sample '{sample_id}' (batch_id: {batch_id})" + ) + # Get all filepaths in the batch filepaths = batched_job.get_filepaths() - + # Log individual files in the batch for i, filepath in enumerate(filepaths): - logger.info(f" Batch file {i+1}/{batch_size}: {os.path.basename(filepath)}") - + logger.info( + f" Batch file {i+1}/{batch_size}: {os.path.basename(filepath)}" + ) + # Prepare metadata list for all BAM files in the batch metadata_list = [] for i, bam_path in enumerate(filepaths): # Get metadata from preprocessing for this specific file file_metadata = batched_job.contexts[i].metadata.get("bam_metadata", {}) - + # Get sample ID from preprocessing results for this specific file file_context = batched_job.contexts[i] file_sample_id = file_context.get_sample_id() - + # Use the sample ID from the file's context (which should have preprocessing results) if file_sample_id != "unknown": file_metadata["sample_id"] = file_sample_id else: file_metadata["sample_id"] = sample_id - + metadata_list.append(file_metadata) - + # Determine work directory for the batch if work_dir is None: # Default to first BAM file directory @@ -1481,30 +1593,39 @@ def mgmt_handler(job, work_dir=None, reference=None): os.makedirs(work_dir, exist_ok=True) batch_work_dir = work_dir logger.debug(f"Using specified work directory: {batch_work_dir}") - + # Process all BAM files in the batch using the new aggregated function - logger.info(f"Processing {batch_size} BAM files as aggregated batch for sample '{sample_id}'") + logger.info( + f"Processing {batch_size} BAM files as aggregated batch for sample '{sample_id}'" + ) batch_result = process_multiple_bams( bam_paths=filepaths, metadata_list=metadata_list, work_dir=batch_work_dir, logger=logger, - reference=reference + reference=reference, ) - + # Store batch results in job context (maintain compatibility with existing structure) - job.context.add_metadata("mgmt_analysis", { - "batch_result": batch_result, # Single aggregated result - "batch_size": batch_size, - "sample_id": sample_id, - "batch_id": batch_id, - "files_processed": batch_result.get("files_processed", batch_size), - "total_files": batch_result.get("total_files", batch_size) - }) - - logger.info(f"Completed MGMT batch processing: {batch_size} files for sample '{sample_id}'") - logger.info(f"Files successfully processed: {batch_result.get('files_processed', batch_size)}/{batch_result.get('total_files', batch_size)}") - + job.context.add_metadata( + "mgmt_analysis", + { + "batch_result": batch_result, # Single aggregated result + "batch_size": batch_size, + "sample_id": sample_id, + "batch_id": batch_id, + "files_processed": batch_result.get("files_processed", batch_size), + "total_files": batch_result.get("total_files", batch_size), + }, + ) + + logger.info( + f"Completed MGMT batch processing: {batch_size} files for sample '{sample_id}'" + ) + logger.info( + f"Files successfully processed: {batch_result.get('files_processed', batch_size)}/{batch_result.get('total_files', batch_size)}" + ) + # Check if this is an expected condition (no MGMT reads) vs actual error if batch_result.get("status") == "no_mgmt_reads": # This is an expected condition, not an error @@ -1514,18 +1635,28 @@ def mgmt_handler(job, work_dir=None, reference=None): "status": "no_mgmt_reads", "sample_id": sample_id, "processing_steps": batch_result.get("processing_steps", []), - "message": batch_result.get("message", "No MGMT reads found in any BAM files"), - "mgmt_read_count_from_preprocessing": batch_result.get("mgmt_read_count_from_preprocessing", 0), + "message": batch_result.get( + "message", "No MGMT reads found in any BAM files" + ), + "mgmt_read_count_from_preprocessing": batch_result.get( + "mgmt_read_count_from_preprocessing", 0 + ), "files_processed": batch_result.get("files_processed", batch_size), "total_files": batch_result.get("total_files", batch_size), }, ) - logger.info(f"MGMT batch analysis completed - {batch_result.get('message', 'No MGMT reads found')}") + logger.info( + f"MGMT batch analysis completed - {batch_result.get('message', 'No MGMT reads found')}" + ) elif batch_result.get("error_message"): - logger.error(f"Batch processing completed with errors: {batch_result['error_message']}") + logger.error( + f"Batch processing completed with errors: {batch_result['error_message']}" + ) job.context.add_error("mgmt_analysis", batch_result["error_message"]) else: - logger.info("Batch processing completed successfully with aggregated MGMT analysis") + logger.info( + "Batch processing completed successfully with aggregated MGMT analysis" + ) job.context.add_result( "mgmt_analysis", { @@ -1534,14 +1665,16 @@ def mgmt_handler(job, work_dir=None, reference=None): "mgmt_bam_file": batch_result.get("mgmt_bam_file", ""), "processing_steps": batch_result.get("processing_steps", []), "tools_available": batch_result.get("tools_available", {}), - "mgmt_read_count_from_preprocessing": batch_result.get("mgmt_read_count_from_preprocessing", 0), + "mgmt_read_count_from_preprocessing": batch_result.get( + "mgmt_read_count_from_preprocessing", 0 + ), "files_processed": batch_result.get("files_processed", batch_size), "total_files": batch_result.get("total_files", batch_size), }, ) - + return - + else: # Single file processing (backward compatibility) try: @@ -1561,11 +1694,15 @@ def mgmt_handler(job, work_dir=None, reference=None): logger.debug(f"Using specified work directory: {work_dir}") # Process the BAM file - mgmt_result = process_bam_file(bam_path, bam_metadata, work_dir, reference=reference) + mgmt_result = process_bam_file( + bam_path, bam_metadata, work_dir, reference=reference + ) # Store results in job context job.context.add_metadata("mgmt_analysis", mgmt_result.results) - job.context.add_metadata("mgmt_processing_steps", mgmt_result.processing_steps) + job.context.add_metadata( + "mgmt_processing_steps", mgmt_result.processing_steps + ) if mgmt_result.error_message: # Check if this is an expected condition (no MGMT reads) vs actual error @@ -1583,7 +1720,9 @@ def mgmt_handler(job, work_dir=None, reference=None): ), }, ) - logger.info(f"MGMT analysis completed - {mgmt_result.error_message}") + logger.info( + f"MGMT analysis completed - {mgmt_result.error_message}" + ) else: # This is an actual error job.context.add_error("mgmt_analysis", mgmt_result.error_message) @@ -1597,7 +1736,9 @@ def mgmt_handler(job, work_dir=None, reference=None): "analysis_time": mgmt_result.results.get("analysis_time", 0), "mgmt_bam_file": mgmt_result.results.get("mgmt_bam_file", ""), "processing_steps": mgmt_result.processing_steps, - "tools_available": mgmt_result.results.get("tools_available", {}), + "tools_available": mgmt_result.results.get( + "tools_available", {} + ), "mgmt_read_count_from_preprocessing": mgmt_result.results.get( "mgmt_read_count_from_preprocessing", 0 ), @@ -1608,7 +1749,9 @@ def mgmt_handler(job, work_dir=None, reference=None): logger.info( f"Analysis time: {mgmt_result.results.get('analysis_time', 0):.2f}s" ) - logger.debug(f"Processing steps: {', '.join(mgmt_result.processing_steps)}") + logger.debug( + f"Processing steps: {', '.join(mgmt_result.processing_steps)}" + ) logger.debug( f"Output directory: {os.path.dirname(mgmt_result.results.get('mgmt_bam_file', ''))}" ) diff --git a/src/robin/analysis/mnpflex_docker.py b/src/robin/analysis/mnpflex_docker.py index b1148615..e414f32a 100644 --- a/src/robin/analysis/mnpflex_docker.py +++ b/src/robin/analysis/mnpflex_docker.py @@ -214,7 +214,9 @@ def _resolve_hierarchy_predictions( except ValueError: logger.warning("[MNPFlex] Falling back from LIMS layout in %s", lims_path) - matches = sorted(docker_dir.glob(f"{prefix}_*_missing_sites.mnp-flex_all_preds.csv")) + matches = sorted( + docker_dir.glob(f"{prefix}_*_missing_sites.mnp-flex_all_preds.csv") + ) if not matches: matches = sorted(docker_dir.glob(f"{prefix}_*.mnp-flex_all_preds.csv")) if matches: @@ -358,7 +360,9 @@ def build_bundle_summary_from_docker_dir( description = (annotation_row.get("Description") or "").strip() classifier = _parse_classifier_label(lims_row.get("Classifier", "")) - hierarchy_predictions = _resolve_hierarchy_predictions(docker_dir, prefix, lims_path) + hierarchy_predictions = _resolve_hierarchy_predictions( + docker_dir, prefix, lims_path + ) hierarchy = _build_hierarchy_tree(hierarchy_predictions, description) scores = _build_scores_from_cal(scores_path) if scores_path.exists() else [] scores = _enrich_scores_with_hierarchy(scores, hierarchy_predictions) diff --git a/src/robin/analysis/mnpflex_eligibility.py b/src/robin/analysis/mnpflex_eligibility.py index 3086571c..7a562b02 100644 --- a/src/robin/analysis/mnpflex_eligibility.py +++ b/src/robin/analysis/mnpflex_eligibility.py @@ -42,22 +42,13 @@ def _float_field(data: Mapping[str, Any], *keys: str, default: float = 0.0) -> f def sample_workflow_jobs_complete(overview: Mapping[str, Any]) -> bool: - active = _int_field( - overview, "active_jobs", "samples_overview_active_jobs" - ) - pending = _int_field( - overview, "pending_jobs", "samples_overview_pending_jobs" - ) + active = _int_field(overview, "active_jobs", "samples_overview_active_jobs") + pending = _int_field(overview, "pending_jobs", "samples_overview_pending_jobs") total = _int_field(overview, "total_jobs", "samples_overview_total_jobs") completed = _int_field( overview, "completed_jobs", "samples_overview_completed_jobs" ) - return ( - total > 0 - and completed >= total - and active == 0 - and pending == 0 - ) + return total > 0 and completed >= total and active == 0 and pending == 0 def sample_data_last_seen(overview: Mapping[str, Any]) -> float: @@ -119,6 +110,4 @@ def sample_ready_for_mnpflex_auto_run_from_dir( row = read_master_csv_overview_row(sample_dir) if not row: return False - return sample_ready_for_mnpflex_auto_run( - row, idle_seconds=idle_seconds, now=now - ) + return sample_ready_for_mnpflex_auto_run(row, idle_seconds=idle_seconds, now=now) diff --git a/src/robin/analysis/mnpflex_runner.py b/src/robin/analysis/mnpflex_runner.py index 163c7ce5..5d259c8a 100644 --- a/src/robin/analysis/mnpflex_runner.py +++ b/src/robin/analysis/mnpflex_runner.py @@ -8,7 +8,11 @@ from robin.analysis.mnpflex_bed import select_input_bed_for_config from robin.analysis.mnpflex_config import MNPFlexConfig, load_mnpflex_config -from robin.analysis.mnpflex_docker import format_mnpflex_runtime_error, run_docker_mnpflex, validate_docker_runtime +from robin.analysis.mnpflex_docker import ( + format_mnpflex_runtime_error, + run_docker_mnpflex, + validate_docker_runtime, +) from robin.utils.mnpflex_client_standalone import MNPFlexClient logger = logging.getLogger(__name__) @@ -23,9 +27,7 @@ def preflight_mnpflex_runtime( if err: return err if cfg.backend == "disabled": - return ( - "MNP-Flex is disabled. Set MNPFLEX_BACKEND to docker or api." - ) + return "MNP-Flex is disabled. Set MNPFLEX_BACKEND to docker or api." if cfg.backend == "docker": try: validate_docker_runtime(cfg) diff --git a/src/robin/analysis/nanodx_analysis.py b/src/robin/analysis/nanodx_analysis.py index d862c717..eadd087a 100644 --- a/src/robin/analysis/nanodx_analysis.py +++ b/src/robin/analysis/nanodx_analysis.py @@ -6,29 +6,29 @@ using NanoDX and PanNanoDX models from the robin package. """ +import gc +import json +import logging import os +import subprocess import sys -import time import tempfile -import subprocess -import logging -import gc -import json -from typing import Tuple, Dict, Any, Optional +import time from dataclasses import dataclass, field +from typing import Any, Dict, Optional, Tuple -import pandas as pd import numpy as np +import pandas as pd # Import robin utilities try: - from robin.analysis.utilities.merge_bedmethyl import collapse_minimal_bedmethyl - from robin.submodules.nanoDX.workflow.scripts.NN_model import NN_classifier - from robin import resources - # ToDo: Resolve models into robin - from robin import models - from robin.analysis.utilities.merge_bedmethyl import load_modkit_data + from robin import models, resources + from robin.analysis.utilities.merge_bedmethyl import ( + collapse_minimal_bedmethyl, + load_modkit_data, + ) + from robin.submodules.nanoDX.workflow.scripts.NN_model import NN_classifier except ImportError as e: logging.warning(f"Some robin dependencies not available: {e}") collapse_minimal_bedmethyl = None @@ -441,10 +441,12 @@ def __init__(self, work_dir=None, model: str = "pancan_devel_v5i_NN_v2.pkl"): logger.debug(f"Store file: {self.storefile}") -def process_multiple_files(parquet_paths, metadata_list, work_dir, logger, model="Capper_et_al_NN_v2.pkl"): +def process_multiple_files( + parquet_paths, metadata_list, work_dir, logger, model="Capper_et_al_NN_v2.pkl" +): """ Process multiple parquet files for NanoDX analysis. - + This function processes multiple parquet files for the same sample. Each file is processed individually and results are accumulated in the same NanoDX_scores.csv or PanNanoDX_scores.csv file. @@ -461,21 +463,23 @@ def process_multiple_files(parquet_paths, metadata_list, work_dir, logger, model """ if not parquet_paths or not metadata_list: raise ValueError("parquet_paths and metadata_list must not be empty") - + if len(parquet_paths) != len(metadata_list): raise ValueError("parquet_paths and metadata_list must have the same length") - + # Get sample ID from first metadata (assuming all parquet files are from same sample) sample_id = metadata_list[0].get("sample_id", "unknown") - + # Determine analysis type based on model is_pannanodx = model != "Capper_et_al_NN_v2.pkl" analysis_type = "PanNanoDX" if is_pannanodx else "NanoDX" - - logger.info(f"🧠 Starting multi-file {analysis_type} analysis for sample: {sample_id}") + + logger.info( + f"🧠 Starting multi-file {analysis_type} analysis for sample: {sample_id}" + ) logger.info(f"Processing {len(parquet_paths)} parquet files for sample {sample_id}") logger.info(f"Using model: {model}") - + # Log essential metadata only for i, (parquet_path, metadata) in enumerate(zip(parquet_paths, metadata_list)): logger.debug(f"Parquet file {i+1}: {os.path.basename(parquet_path)}") @@ -498,16 +502,16 @@ def process_multiple_files(parquet_paths, metadata_list, work_dir, logger, model # Create sample-specific output directory sample_dir = os.path.join(work_dir, sample_id) os.makedirs(sample_dir, exist_ok=True) - + # Determine store file name if is_pannanodx: storefile = "PanNanoDX_scores.csv" else: storefile = "NanoDX_scores.csv" - + store_path = os.path.join(sample_dir, storefile) analysis_result["store_path"] = store_path - + logger.info(f"Created output directory: {sample_dir}") logger.info(f"Store file: {storefile}") analysis_result["processing_steps"].append("directory_created") @@ -517,7 +521,7 @@ def process_multiple_files(parquet_paths, metadata_list, work_dir, logger, model nanodx_analyzer = PanNanodxAnalysis(work_dir=work_dir, model=model) else: nanodx_analyzer = NanodxAnalysis(work_dir=work_dir, model=model) - + logger.info(f"Initialized {analysis_type} analyzer") analysis_result["processing_steps"].append("analyzer_initialized") @@ -525,34 +529,46 @@ def process_multiple_files(parquet_paths, metadata_list, work_dir, logger, model logger.info("Processing parquet files individually") processed_files = 0 total_features = 0 - + for i, (parquet_path, metadata) in enumerate(zip(parquet_paths, metadata_list)): - logger.info(f"Processing parquet file {i+1}/{len(parquet_paths)}: {os.path.basename(parquet_path)}") - + logger.info( + f"Processing parquet file {i+1}/{len(parquet_paths)}: {os.path.basename(parquet_path)}" + ) + try: # Check if parquet file exists if not os.path.exists(parquet_path): logger.warning(f"Parquet file not found: {parquet_path}") continue - + # Process the parquet file - nanodx_result = nanodx_analyzer.process_parquet_file(parquet_path, sample_id) - + nanodx_result = nanodx_analyzer.process_parquet_file( + parquet_path, sample_id + ) + if nanodx_result.error_message: - logger.warning(f"Error processing {os.path.basename(parquet_path)}: {nanodx_result.error_message}") + logger.warning( + f"Error processing {os.path.basename(parquet_path)}: {nanodx_result.error_message}" + ) continue - + processed_files += 1 total_features += nanodx_result.n_features - logger.debug(f"Successfully processed file {i+1}: {os.path.basename(parquet_path)}") + logger.debug( + f"Successfully processed file {i+1}: {os.path.basename(parquet_path)}" + ) logger.debug(f"Features from this file: {nanodx_result.n_features}") - + except Exception as e: - logger.warning(f"Error processing {os.path.basename(parquet_path)}: {e}") + logger.warning( + f"Error processing {os.path.basename(parquet_path)}: {e}" + ) continue if processed_files == 0: - analysis_result["error_message"] = "No files could be processed successfully" + analysis_result["error_message"] = ( + "No files could be processed successfully" + ) analysis_result["processing_steps"].append("no_files_processed") return analysis_result @@ -564,7 +580,7 @@ def process_multiple_files(parquet_paths, metadata_list, work_dir, logger, model if os.path.exists(store_path): analysis_result["processing_steps"].append("scores_file_created") logger.info(f"{analysis_type} scores accumulated in: {store_path}") - + # Load and log summary of results try: scores_df = pd.read_csv(store_path, index_col=0) @@ -577,14 +593,18 @@ def process_multiple_files(parquet_paths, metadata_list, work_dir, logger, model analysis_result["processing_steps"].append("analysis_complete") logger.info(f"Multi-file {analysis_type} analysis completed for {sample_id}") - logger.info(f"Files successfully processed: {analysis_result['files_processed']}/{analysis_result['total_files']}") + logger.info( + f"Files successfully processed: {analysis_result['files_processed']}/{analysis_result['total_files']}" + ) logger.info(f"Total features processed: {analysis_result['total_features']}") logger.info(f"Output directory: {sample_dir}") - + return analysis_result except Exception as e: - logger.error(f"Error in multi-file {analysis_type} analysis for {sample_id}: {e}") + logger.error( + f"Error in multi-file {analysis_type} analysis for {sample_id}: {e}" + ) analysis_result["error_message"] = str(e) analysis_result["processing_steps"].append("analysis_failed") return analysis_result @@ -610,48 +630,62 @@ def nanodx_handler(job, work_dir=None): batch_size = batched_job.get_file_count() sample_id = batched_job.get_sample_id() batch_id = batched_job.batch_id - logger.info(f"Processing NanoDX analysis batch: {batch_size} files for sample '{sample_id}' (batch_id: {batch_id})") - + logger.info( + f"Processing NanoDX analysis batch: {batch_size} files for sample '{sample_id}' (batch_id: {batch_id})" + ) + # Get all filepaths in the batch filepaths = batched_job.get_filepaths() - + # Log individual files in the batch for i, filepath in enumerate(filepaths): - logger.info(f" Batch file {i+1}/{batch_size}: {os.path.basename(filepath)}") - + logger.info( + f" Batch file {i+1}/{batch_size}: {os.path.basename(filepath)}" + ) + # Prepare metadata list and extract parquet paths from bed conversion results metadata_list = [] parquet_paths = [] - + for i, bam_path in enumerate(filepaths): # Get metadata from preprocessing for this specific file file_metadata = batched_job.contexts[i].metadata.get("bam_metadata", {}) - + # Get sample ID from preprocessing results for this specific file file_context = batched_job.contexts[i] file_sample_id = file_context.get_sample_id() - + # Use the sample ID from the file's context (which should have preprocessing results) if file_sample_id != "unknown": file_metadata["sample_id"] = file_sample_id else: file_metadata["sample_id"] = sample_id - + # Get parquet path from bed conversion results for this file - bed_conversion_result = batched_job.contexts[i].results.get("bed_conversion", {}) + bed_conversion_result = batched_job.contexts[i].results.get( + "bed_conversion", {} + ) parquet_path = bed_conversion_result.get("parquet_path") - + if parquet_path: parquet_paths.append(parquet_path) metadata_list.append(file_metadata) - logger.debug(f"Found parquet path for file {i+1}: {os.path.basename(parquet_path)}") + logger.debug( + f"Found parquet path for file {i+1}: {os.path.basename(parquet_path)}" + ) else: - logger.warning(f"No parquet path found for file {i+1}: {os.path.basename(bam_path)}") - + logger.warning( + f"No parquet path found for file {i+1}: {os.path.basename(bam_path)}" + ) + if not parquet_paths: - error_msg = "No parquet paths found from bed conversion results in batch" + error_msg = ( + "No parquet paths found from bed conversion results in batch" + ) if suppress_expected: - logger.warning(f"{error_msg} (expected for fail-only BAM submission)") + logger.warning( + f"{error_msg} (expected for fail-only BAM submission)" + ) job.context.add_result( "nanodx_analysis", {"status": "expected_failure", "reason": error_msg}, @@ -660,7 +694,7 @@ def nanodx_handler(job, work_dir=None): logger.error(error_msg) job.context.add_error("nanodx_analysis", error_msg) return - + # Determine work directory for the batch if work_dir is None: # Default to first parquet file directory @@ -670,31 +704,44 @@ def nanodx_handler(job, work_dir=None): os.makedirs(work_dir, exist_ok=True) batch_work_dir = work_dir logger.debug(f"Using specified work directory: {batch_work_dir}") - + # Process all parquet files in the batch using the new aggregated function - logger.info(f"Processing {len(parquet_paths)} parquet files as aggregated batch for sample '{sample_id}'") + logger.info( + f"Processing {len(parquet_paths)} parquet files as aggregated batch for sample '{sample_id}'" + ) batch_result = process_multiple_files( parquet_paths=parquet_paths, metadata_list=metadata_list, work_dir=batch_work_dir, logger=logger, - model="Capper_et_al_NN_v2.pkl" # NanoDX model + model="Capper_et_al_NN_v2.pkl", # NanoDX model ) - + # Store batch results in job context (maintain compatibility with existing structure) - job.context.add_metadata("nanodx_analysis", { - "batch_result": batch_result, # Single aggregated result - "batch_size": batch_size, - "sample_id": sample_id, - "batch_id": batch_id, - "files_processed": batch_result.get("files_processed", len(parquet_paths)), - "total_files": batch_result.get("total_files", len(parquet_paths)) - }) - - logger.info(f"Completed NanoDX analysis batch processing: {batch_size} files for sample '{sample_id}'") - logger.info(f"Files successfully processed: {batch_result.get('files_processed', len(parquet_paths))}/{batch_result.get('total_files', len(parquet_paths))}") - logger.info(f"Total features processed: {batch_result.get('total_features', 0)}") - + job.context.add_metadata( + "nanodx_analysis", + { + "batch_result": batch_result, # Single aggregated result + "batch_size": batch_size, + "sample_id": sample_id, + "batch_id": batch_id, + "files_processed": batch_result.get( + "files_processed", len(parquet_paths) + ), + "total_files": batch_result.get("total_files", len(parquet_paths)), + }, + ) + + logger.info( + f"Completed NanoDX analysis batch processing: {batch_size} files for sample '{sample_id}'" + ) + logger.info( + f"Files successfully processed: {batch_result.get('files_processed', len(parquet_paths))}/{batch_result.get('total_files', len(parquet_paths))}" + ) + logger.info( + f"Total features processed: {batch_result.get('total_features', 0)}" + ) + if batch_result.get("error_message"): if suppress_expected: logger.warning( @@ -712,26 +759,36 @@ def nanodx_handler(job, work_dir=None): logger.error( f"Batch processing completed with errors: {batch_result['error_message']}" ) - job.context.add_error("nanodx_analysis", batch_result["error_message"]) + job.context.add_error( + "nanodx_analysis", batch_result["error_message"] + ) else: - logger.info("Batch processing completed successfully with aggregated NanoDX analysis") + logger.info( + "Batch processing completed successfully with aggregated NanoDX analysis" + ) job.context.add_result( "nanodx_analysis", { "status": "success", "sample_id": sample_id, "analysis_time": batch_result.get("analysis_timestamp", 0), - "model_used": batch_result.get("model_used", "Capper_et_al_NN_v2.pkl"), + "model_used": batch_result.get( + "model_used", "Capper_et_al_NN_v2.pkl" + ), "n_features": batch_result.get("total_features", 0), "store_path": batch_result.get("store_path", ""), "processing_steps": batch_result.get("processing_steps", []), - "files_processed": batch_result.get("files_processed", len(parquet_paths)), - "total_files": batch_result.get("total_files", len(parquet_paths)), + "files_processed": batch_result.get( + "files_processed", len(parquet_paths) + ), + "total_files": batch_result.get( + "total_files", len(parquet_paths) + ), }, ) - + return - + else: # Single file processing (backward compatibility) # Get metadata from previous steps @@ -743,7 +800,9 @@ def nanodx_handler(job, work_dir=None): if not parquet_path: raise ValueError("No parquet path found from bed conversion step") - logger.info(f"Starting NanoDX analysis for: {os.path.basename(parquet_path)}") + logger.info( + f"Starting NanoDX analysis for: {os.path.basename(parquet_path)}" + ) # Get sample ID sample_id = bam_metadata.get("sample_id", "unknown") @@ -759,7 +818,9 @@ def nanodx_handler(job, work_dir=None): nanodx_analyzer = NanodxAnalysis(work_dir=work_dir) # Process the parquet file - nanodx_result = nanodx_analyzer.process_parquet_file(parquet_path, sample_id) + nanodx_result = nanodx_analyzer.process_parquet_file( + parquet_path, sample_id + ) # Store results in job context job.context.add_metadata("nanodx_analysis", nanodx_result.results) @@ -795,9 +856,7 @@ def nanodx_handler(job, work_dir=None): except Exception as e: if suppress_expected: - logger.warning( - f"Expected NanoDX failure for fail-only BAM submission: {e}" - ) + logger.warning(f"Expected NanoDX failure for fail-only BAM submission: {e}") job.context.add_result( "nanodx_analysis", {"status": "expected_failure", "error_message": str(e)}, @@ -827,48 +886,62 @@ def pannanodx_handler(job, work_dir=None): batch_size = batched_job.get_file_count() sample_id = batched_job.get_sample_id() batch_id = batched_job.batch_id - logger.info(f"Processing PanNanoDX analysis batch: {batch_size} files for sample '{sample_id}' (batch_id: {batch_id})") - + logger.info( + f"Processing PanNanoDX analysis batch: {batch_size} files for sample '{sample_id}' (batch_id: {batch_id})" + ) + # Get all filepaths in the batch filepaths = batched_job.get_filepaths() - + # Log individual files in the batch for i, filepath in enumerate(filepaths): - logger.info(f" Batch file {i+1}/{batch_size}: {os.path.basename(filepath)}") - + logger.info( + f" Batch file {i+1}/{batch_size}: {os.path.basename(filepath)}" + ) + # Prepare metadata list and extract parquet paths from bed conversion results metadata_list = [] parquet_paths = [] - + for i, bam_path in enumerate(filepaths): # Get metadata from preprocessing for this specific file file_metadata = batched_job.contexts[i].metadata.get("bam_metadata", {}) - + # Get sample ID from preprocessing results for this specific file file_context = batched_job.contexts[i] file_sample_id = file_context.get_sample_id() - + # Use the sample ID from the file's context (which should have preprocessing results) if file_sample_id != "unknown": file_metadata["sample_id"] = file_sample_id else: file_metadata["sample_id"] = sample_id - + # Get parquet path from bed conversion results for this file - bed_conversion_result = batched_job.contexts[i].results.get("bed_conversion", {}) + bed_conversion_result = batched_job.contexts[i].results.get( + "bed_conversion", {} + ) parquet_path = bed_conversion_result.get("parquet_path") - + if parquet_path: parquet_paths.append(parquet_path) metadata_list.append(file_metadata) - logger.debug(f"Found parquet path for file {i+1}: {os.path.basename(parquet_path)}") + logger.debug( + f"Found parquet path for file {i+1}: {os.path.basename(parquet_path)}" + ) else: - logger.warning(f"No parquet path found for file {i+1}: {os.path.basename(bam_path)}") - + logger.warning( + f"No parquet path found for file {i+1}: {os.path.basename(bam_path)}" + ) + if not parquet_paths: - error_msg = "No parquet paths found from bed conversion results in batch" + error_msg = ( + "No parquet paths found from bed conversion results in batch" + ) if suppress_expected: - logger.warning(f"{error_msg} (expected for fail-only BAM submission)") + logger.warning( + f"{error_msg} (expected for fail-only BAM submission)" + ) job.context.add_result( "pannanodx_analysis", {"status": "expected_failure", "reason": error_msg}, @@ -877,7 +950,7 @@ def pannanodx_handler(job, work_dir=None): logger.error(error_msg) job.context.add_error("pannanodx_analysis", error_msg) return - + # Determine work directory for the batch if work_dir is None: # Default to first parquet file directory @@ -887,31 +960,44 @@ def pannanodx_handler(job, work_dir=None): os.makedirs(work_dir, exist_ok=True) batch_work_dir = work_dir logger.debug(f"Using specified work directory: {batch_work_dir}") - + # Process all parquet files in the batch using the new aggregated function - logger.info(f"Processing {len(parquet_paths)} parquet files as aggregated batch for sample '{sample_id}'") + logger.info( + f"Processing {len(parquet_paths)} parquet files as aggregated batch for sample '{sample_id}'" + ) batch_result = process_multiple_files( parquet_paths=parquet_paths, metadata_list=metadata_list, work_dir=batch_work_dir, logger=logger, - model="pancan_devel_v5i_NN_v2.pkl" # PanNanoDX model + model="pancan_devel_v5i_NN_v2.pkl", # PanNanoDX model ) - + # Store batch results in job context (maintain compatibility with existing structure) - job.context.add_metadata("pannanodx_analysis", { - "batch_result": batch_result, # Single aggregated result - "batch_size": batch_size, - "sample_id": sample_id, - "batch_id": batch_id, - "files_processed": batch_result.get("files_processed", len(parquet_paths)), - "total_files": batch_result.get("total_files", len(parquet_paths)) - }) - - logger.info(f"Completed PanNanoDX analysis batch processing: {batch_size} files for sample '{sample_id}'") - logger.info(f"Files successfully processed: {batch_result.get('files_processed', len(parquet_paths))}/{batch_result.get('total_files', len(parquet_paths))}") - logger.info(f"Total features processed: {batch_result.get('total_features', 0)}") - + job.context.add_metadata( + "pannanodx_analysis", + { + "batch_result": batch_result, # Single aggregated result + "batch_size": batch_size, + "sample_id": sample_id, + "batch_id": batch_id, + "files_processed": batch_result.get( + "files_processed", len(parquet_paths) + ), + "total_files": batch_result.get("total_files", len(parquet_paths)), + }, + ) + + logger.info( + f"Completed PanNanoDX analysis batch processing: {batch_size} files for sample '{sample_id}'" + ) + logger.info( + f"Files successfully processed: {batch_result.get('files_processed', len(parquet_paths))}/{batch_result.get('total_files', len(parquet_paths))}" + ) + logger.info( + f"Total features processed: {batch_result.get('total_features', 0)}" + ) + if batch_result.get("error_message"): if suppress_expected: logger.warning( @@ -933,24 +1019,32 @@ def pannanodx_handler(job, work_dir=None): "pannanodx_analysis", batch_result["error_message"] ) else: - logger.info("Batch processing completed successfully with aggregated PanNanoDX analysis") + logger.info( + "Batch processing completed successfully with aggregated PanNanoDX analysis" + ) job.context.add_result( "pannanodx_analysis", { "status": "success", "sample_id": sample_id, "analysis_time": batch_result.get("analysis_timestamp", 0), - "model_used": batch_result.get("model_used", "pancan_devel_v5i_NN_v2.pkl"), + "model_used": batch_result.get( + "model_used", "pancan_devel_v5i_NN_v2.pkl" + ), "n_features": batch_result.get("total_features", 0), "store_path": batch_result.get("store_path", ""), "processing_steps": batch_result.get("processing_steps", []), - "files_processed": batch_result.get("files_processed", len(parquet_paths)), - "total_files": batch_result.get("total_files", len(parquet_paths)), + "files_processed": batch_result.get( + "files_processed", len(parquet_paths) + ), + "total_files": batch_result.get( + "total_files", len(parquet_paths) + ), }, ) - + return - + else: # Single file processing (backward compatibility) # Get metadata from previous steps @@ -980,7 +1074,9 @@ def pannanodx_handler(job, work_dir=None): pannanodx_analyzer = PanNanodxAnalysis(work_dir=work_dir) # Process the parquet file - nanodx_result = pannanodx_analyzer.process_parquet_file(parquet_path, sample_id) + nanodx_result = pannanodx_analyzer.process_parquet_file( + parquet_path, sample_id + ) # Store results in job context job.context.add_metadata("pannanodx_analysis", nanodx_result.results) @@ -990,7 +1086,9 @@ def pannanodx_handler(job, work_dir=None): if nanodx_result.error_message: job.context.add_error("pannanodx_analysis", nanodx_result.error_message) - logger.error(f"PanNanoDX analysis failed: {nanodx_result.error_message}") + logger.error( + f"PanNanoDX analysis failed: {nanodx_result.error_message}" + ) else: job.context.add_result( "pannanodx_analysis", diff --git a/src/robin/analysis/random_forest_analysis.py b/src/robin/analysis/random_forest_analysis.py index 2bf4a99d..c3dccbd2 100644 --- a/src/robin/analysis/random_forest_analysis.py +++ b/src/robin/analysis/random_forest_analysis.py @@ -26,14 +26,14 @@ assumes the presence of necessary R scripts and model files. """ +import logging import os +import shutil +import subprocess import tempfile import time -import subprocess -import logging -import shutil from dataclasses import dataclass, field -from typing import Dict, Any, Optional, List +from typing import Any, Dict, List, Optional import pandas as pd @@ -81,8 +81,6 @@ def _compute_hvpath() -> Optional[str]: HVPATH = _compute_hvpath() - - @dataclass class RandomForestMetadata: """Container for Random Forest analysis metadata and results""" @@ -524,7 +522,7 @@ def process_multiple_files( ): """ Process multiple parquet files for Random Forest analysis. - + This function processes multiple parquet files for the same sample. Each file is processed individually and results are accumulated in the same random_forest_scores.csv file. @@ -542,17 +540,19 @@ def process_multiple_files( """ if not parquet_paths or not metadata_list: raise ValueError("parquet_paths and metadata_list must not be empty") - + if len(parquet_paths) != len(metadata_list): raise ValueError("parquet_paths and metadata_list must have the same length") - + # Get sample ID from first metadata (assuming all parquet files are from same sample) sample_id = metadata_list[0].get("sample_id", "unknown") - - logger.info(f"🌲 Starting multi-file Random Forest analysis for sample: {sample_id}") + + logger.info( + f"🌲 Starting multi-file Random Forest analysis for sample: {sample_id}" + ) logger.info(f"Processing {len(parquet_paths)} parquet files for sample {sample_id}") logger.info(f"Using {threads} threads for R script execution") - + # Log essential metadata only for i, (parquet_path, metadata) in enumerate(zip(parquet_paths, metadata_list)): logger.debug(f"Parquet file {i+1}: {os.path.basename(parquet_path)}") @@ -574,21 +574,19 @@ def process_multiple_files( # Create sample-specific output directory sample_dir = os.path.join(work_dir, sample_id) os.makedirs(sample_dir, exist_ok=True) - + scores_file_path = os.path.join(sample_dir, "random_forest_scores.csv") analysis_result["scores_file_path"] = scores_file_path - + logger.info(f"Created output directory: {sample_dir}") logger.info(f"Scores file: {scores_file_path}") analysis_result["processing_steps"].append("directory_created") # Initialize Random Forest analysis random_forest_analyzer = RandomForestAnalysis( - work_dir=work_dir, - threads=threads, - showerrors=showerrors + work_dir=work_dir, threads=threads, showerrors=showerrors ) - + logger.info("Initialized Random Forest analyzer") analysis_result["processing_steps"].append("analyzer_initialized") @@ -596,43 +594,57 @@ def process_multiple_files( logger.info("Processing parquet files individually") processed_files = 0 total_batches = 0 - + for i, (parquet_path, metadata) in enumerate(zip(parquet_paths, metadata_list)): - logger.info(f"Processing parquet file {i+1}/{len(parquet_paths)}: {os.path.basename(parquet_path)}") - + logger.info( + f"Processing parquet file {i+1}/{len(parquet_paths)}: {os.path.basename(parquet_path)}" + ) + try: # Check if parquet file exists if not os.path.exists(parquet_path): logger.warning(f"Parquet file not found: {parquet_path}") continue - + # Process the parquet file - rf_result = random_forest_analyzer.process_parquet_file(parquet_path, sample_id) - + rf_result = random_forest_analyzer.process_parquet_file( + parquet_path, sample_id + ) + if rf_result.error_message: - logger.warning(f"Error processing {os.path.basename(parquet_path)}: {rf_result.error_message}") + logger.warning( + f"Error processing {os.path.basename(parquet_path)}: {rf_result.error_message}" + ) continue - + processed_files += 1 total_batches += 1 - logger.debug(f"Successfully processed file {i+1}: {os.path.basename(parquet_path)}") + logger.debug( + f"Successfully processed file {i+1}: {os.path.basename(parquet_path)}" + ) logger.debug(f"Batch number: {rf_result.batch_number}") if job_id is not None: try: - from robin.workflow_ray import notify_coordinator_files_completed + from robin.workflow_ray import ( + notify_coordinator_files_completed, + ) notify_coordinator_files_completed( "random_forest", 1, job_id=job_id ) except Exception: pass - + except Exception as e: - logger.warning(f"Error processing {os.path.basename(parquet_path)}: {e}") + logger.warning( + f"Error processing {os.path.basename(parquet_path)}: {e}" + ) continue if processed_files == 0: - analysis_result["error_message"] = "No files could be processed successfully" + analysis_result["error_message"] = ( + "No files could be processed successfully" + ) analysis_result["processing_steps"].append("no_files_processed") return analysis_result @@ -644,7 +656,7 @@ def process_multiple_files( if os.path.exists(scores_file_path): analysis_result["processing_steps"].append("scores_file_created") logger.info(f"Random Forest scores accumulated in: {scores_file_path}") - + # Load and log summary of results try: scores_df = pd.read_csv(scores_file_path) @@ -664,10 +676,12 @@ def process_multiple_files( analysis_result["processing_steps"].append("analysis_complete") logger.info(f"Multi-file Random Forest analysis completed for {sample_id}") - logger.info(f"Files successfully processed: {analysis_result['files_processed']}/{analysis_result['total_files']}") + logger.info( + f"Files successfully processed: {analysis_result['files_processed']}/{analysis_result['total_files']}" + ) logger.info(f"Total batches processed: {analysis_result['total_batches']}") logger.info(f"Output directory: {sample_dir}") - + return analysis_result except Exception as e: @@ -697,48 +711,62 @@ def random_forest_handler(job, work_dir=None): batch_size = batched_job.get_file_count() sample_id = batched_job.get_sample_id() batch_id = batched_job.batch_id - logger.info(f"Processing Random Forest analysis batch: {batch_size} files for sample '{sample_id}' (batch_id: {batch_id})") - + logger.info( + f"Processing Random Forest analysis batch: {batch_size} files for sample '{sample_id}' (batch_id: {batch_id})" + ) + # Get all filepaths in the batch filepaths = batched_job.get_filepaths() - + # Log individual files in the batch for i, filepath in enumerate(filepaths): - logger.info(f" Batch file {i+1}/{batch_size}: {os.path.basename(filepath)}") - + logger.info( + f" Batch file {i+1}/{batch_size}: {os.path.basename(filepath)}" + ) + # Prepare metadata list and extract parquet paths from bed conversion results metadata_list = [] parquet_paths = [] - + for i, bam_path in enumerate(filepaths): # Get metadata from preprocessing for this specific file file_metadata = batched_job.contexts[i].metadata.get("bam_metadata", {}) - + # Get sample ID from preprocessing results for this specific file file_context = batched_job.contexts[i] file_sample_id = file_context.get_sample_id() - + # Use the sample ID from the file's context (which should have preprocessing results) if file_sample_id != "unknown": file_metadata["sample_id"] = file_sample_id else: file_metadata["sample_id"] = sample_id - + # Get parquet path from bed conversion results for this file - bed_conversion_result = batched_job.contexts[i].results.get("bed_conversion", {}) + bed_conversion_result = batched_job.contexts[i].results.get( + "bed_conversion", {} + ) parquet_path = bed_conversion_result.get("parquet_path") - + if parquet_path: parquet_paths.append(parquet_path) metadata_list.append(file_metadata) - logger.debug(f"Found parquet path for file {i+1}: {os.path.basename(parquet_path)}") + logger.debug( + f"Found parquet path for file {i+1}: {os.path.basename(parquet_path)}" + ) else: - logger.warning(f"No parquet path found for file {i+1}: {os.path.basename(bam_path)}") - + logger.warning( + f"No parquet path found for file {i+1}: {os.path.basename(bam_path)}" + ) + if not parquet_paths: - error_msg = "No parquet paths found from bed conversion results in batch" + error_msg = ( + "No parquet paths found from bed conversion results in batch" + ) if suppress_expected: - logger.warning(f"{error_msg} (expected for fail-only BAM submission)") + logger.warning( + f"{error_msg} (expected for fail-only BAM submission)" + ) job.context.add_result( "random_forest_analysis", {"status": "expected_failure", "reason": error_msg}, @@ -747,7 +775,7 @@ def random_forest_handler(job, work_dir=None): logger.error(error_msg) job.context.add_error("random_forest_analysis", error_msg) return - + # Determine work directory for the batch if work_dir is None: # Default to first parquet file directory @@ -757,9 +785,11 @@ def random_forest_handler(job, work_dir=None): os.makedirs(work_dir, exist_ok=True) batch_work_dir = work_dir logger.debug(f"Using specified work directory: {batch_work_dir}") - + # Process all parquet files in the batch using the new aggregated function - logger.info(f"Processing {len(parquet_paths)} parquet files as aggregated batch for sample '{sample_id}'") + logger.info( + f"Processing {len(parquet_paths)} parquet files as aggregated batch for sample '{sample_id}'" + ) batch_result = process_multiple_files( parquet_paths=parquet_paths, metadata_list=metadata_list, @@ -769,21 +799,32 @@ def random_forest_handler(job, work_dir=None): showerrors=False, # Default error display setting job_id=job.job_id, ) - + # Store batch results in job context (maintain compatibility with existing structure) - job.context.add_metadata("random_forest_analysis", { - "batch_result": batch_result, # Single aggregated result - "batch_size": batch_size, - "sample_id": sample_id, - "batch_id": batch_id, - "files_processed": batch_result.get("files_processed", len(parquet_paths)), - "total_files": batch_result.get("total_files", len(parquet_paths)) - }) - - logger.info(f"Completed Random Forest analysis batch processing: {batch_size} files for sample '{sample_id}'") - logger.info(f"Files successfully processed: {batch_result.get('files_processed', len(parquet_paths))}/{batch_result.get('total_files', len(parquet_paths))}") - logger.info(f"Total batches processed: {batch_result.get('total_batches', 0)}") - + job.context.add_metadata( + "random_forest_analysis", + { + "batch_result": batch_result, # Single aggregated result + "batch_size": batch_size, + "sample_id": sample_id, + "batch_id": batch_id, + "files_processed": batch_result.get( + "files_processed", len(parquet_paths) + ), + "total_files": batch_result.get("total_files", len(parquet_paths)), + }, + ) + + logger.info( + f"Completed Random Forest analysis batch processing: {batch_size} files for sample '{sample_id}'" + ) + logger.info( + f"Files successfully processed: {batch_result.get('files_processed', len(parquet_paths))}/{batch_result.get('total_files', len(parquet_paths))}" + ) + logger.info( + f"Total batches processed: {batch_result.get('total_batches', 0)}" + ) + if batch_result.get("error_message"): if suppress_expected: logger.warning( @@ -805,7 +846,9 @@ def random_forest_handler(job, work_dir=None): "random_forest_analysis", batch_result["error_message"] ) else: - logger.info("Batch processing completed successfully with aggregated Random Forest analysis") + logger.info( + "Batch processing completed successfully with aggregated Random Forest analysis" + ) job.context.add_result( "random_forest_analysis", { @@ -816,13 +859,17 @@ def random_forest_handler(job, work_dir=None): "processing_steps": batch_result.get("processing_steps", []), "scores_file": batch_result.get("scores_file_path", ""), "bed_file": batch_result.get("bed_file_path", ""), - "files_processed": batch_result.get("files_processed", len(parquet_paths)), - "total_files": batch_result.get("total_files", len(parquet_paths)), + "files_processed": batch_result.get( + "files_processed", len(parquet_paths) + ), + "total_files": batch_result.get( + "total_files", len(parquet_paths) + ), }, ) - + return - + else: # Single file processing (backward compatibility) # Get the parquet file path from bed_conversion results @@ -853,7 +900,9 @@ def random_forest_handler(job, work_dir=None): random_forest_analyzer = RandomForestAnalysis(work_dir=work_dir) # Process the parquet file - result = random_forest_analyzer.process_parquet_file(parquet_path, sample_id) + result = random_forest_analyzer.process_parquet_file( + parquet_path, sample_id + ) # Store results in job context job.context.add_metadata("random_forest_analysis", result.results) diff --git a/src/robin/analysis/snp_processing.py b/src/robin/analysis/snp_processing.py index ee7dd679..208de279 100644 --- a/src/robin/analysis/snp_processing.py +++ b/src/robin/analysis/snp_processing.py @@ -1,19 +1,20 @@ from __future__ import annotations +import logging from pathlib import Path from typing import Any, Dict, List, Optional, Tuple -import logging import numpy as np import pandas as pd from robin.analysis.variant_classification import classify_clinvar_significance - logger = logging.getLogger(__name__) -def _process_annotations(record: Dict[str, Any]) -> Tuple[Dict[int, Dict[str, Any]], Dict[str, Any]]: +def _process_annotations( + record: Dict[str, Any], +) -> Tuple[Dict[int, Dict[str, Any]], Dict[str, Any]]: """ Expand annotation information from a VCF record. @@ -123,12 +124,19 @@ def parse_vcf(vcf_path: Path) -> Optional[pd.DataFrame]: if "Allele" in vcf_df.columns: shared_columns.append("Allele") - non_shared_columns = [col for col in vcf_df.columns if col not in shared_columns] + non_shared_columns = [ + col for col in vcf_df.columns if col not in shared_columns + ] vcf_df = vcf_df.replace({np.nan: None}) aggregated = ( vcf_df.groupby(shared_columns)[non_shared_columns] - .agg(lambda series: ", ".join(sorted({str(item) for item in series.dropna()})) or None) + .agg( + lambda series: ", ".join( + sorted({str(item) for item in series.dropna()}) + ) + or None + ) .reset_index() ) return aggregated @@ -217,9 +225,7 @@ def add_column(field_name: str) -> None: add_column(col) remaining_columns = [ - col - for col in vcf_df.columns - if col not in added_fields and col != "INFO" + col for col in vcf_df.columns if col not in added_fields and col != "INFO" ] for col in remaining_columns: add_column(col) @@ -300,4 +306,3 @@ def _as_bool_flag(value: Any) -> bool: "summary": summary, "snp_regions_map": snp_regions_map, } - diff --git a/src/robin/analysis/sturgeon_analysis.py b/src/robin/analysis/sturgeon_analysis.py index 850c658e..f50f970e 100644 --- a/src/robin/analysis/sturgeon_analysis.py +++ b/src/robin/analysis/sturgeon_analysis.py @@ -14,37 +14,36 @@ - Error handling and result tracking """ +import gc +import json +import logging import os +import sys +import tempfile import time -import logging +import zipfile from dataclasses import dataclass, field -from typing import Dict, Any, Optional, List -from robin.logging_config import get_job_logger -from robin.analysis.utilities.merge_bedmethyl import ( - load_modkit_data, - modkit_pileup_file_to_bed, -) -import tempfile -import gc -import pandas as pd -import numpy as np - -# from robin.subpages.Sturgeon_object import predict_sample_from_dataframe +from typing import Any, Dict, List, Optional +import numpy as np +import onnxruntime +import pandas as pd import sturgeon +from sturgeon.constants import NOMEASURE_VALUE # Sturgeon-related imports (must be installed) from sturgeon.prediction import bed_to_numpy - -import sys - -import zipfile -import json -import onnxruntime from sturgeon.utils import load_bed_file, softmax -from sturgeon.constants import NOMEASURE_VALUE from robin import models +from robin.analysis.utilities.merge_bedmethyl import ( + load_modkit_data, + modkit_pileup_file_to_bed, +) +from robin.logging_config import get_job_logger + +# from robin.subpages.Sturgeon_object import predict_sample_from_dataframe + logger = logging.getLogger(__name__) @@ -191,7 +190,7 @@ def _run_sturgeon_analysis( def process_multiple_files(parquet_paths, metadata_list, work_dir, logger): """ Process multiple parquet files for sturgeon analysis. - + This function processes multiple parquet files for the same sample. Each file is processed individually and results are accumulated in the same sturgeon_scores.csv file. @@ -207,16 +206,16 @@ def process_multiple_files(parquet_paths, metadata_list, work_dir, logger): """ if not parquet_paths or not metadata_list: raise ValueError("parquet_paths and metadata_list must not be empty") - + if len(parquet_paths) != len(metadata_list): raise ValueError("parquet_paths and metadata_list must have the same length") - + # Get sample ID from first metadata (assuming all parquet files are from same sample) sample_id = metadata_list[0].get("sample_id", "unknown") - + logger.info(f"🐟 Starting multi-file sturgeon analysis for sample: {sample_id}") logger.info(f"Processing {len(parquet_paths)} parquet files for sample {sample_id}") - + # Log essential metadata only for i, (parquet_path, metadata) in enumerate(zip(parquet_paths, metadata_list)): logger.debug(f"Parquet file {i+1}: {os.path.basename(parquet_path)}") @@ -238,14 +237,16 @@ def process_multiple_files(parquet_paths, metadata_list, work_dir, logger): output_dir = os.path.join(work_dir, sample_id) os.makedirs(output_dir, exist_ok=True) analysis_result["output_dir"] = output_dir - analysis_result["sturgeon_scores_path"] = os.path.join(output_dir, "sturgeon_scores.csv") - + analysis_result["sturgeon_scores_path"] = os.path.join( + output_dir, "sturgeon_scores.csv" + ) + logger.info(f"Created output directory: {output_dir}") analysis_result["processing_steps"].append("directory_created") # Initialize sturgeon analysis sturgeon_analyzer = SturgeonAnalysis(work_dir=work_dir) - + # Get reference genome and probes file (shared across all files) reference_genome = "hg38" probes_file = sturgeon_analyzer._get_probes_file(reference_genome) @@ -255,33 +256,41 @@ def process_multiple_files(parquet_paths, metadata_list, work_dir, logger): # Process each parquet file individually logger.info("Processing parquet files individually") processed_files = 0 - + for i, (parquet_path, metadata) in enumerate(zip(parquet_paths, metadata_list)): - logger.info(f"Processing parquet file {i+1}/{len(parquet_paths)}: {os.path.basename(parquet_path)}") - + logger.info( + f"Processing parquet file {i+1}/{len(parquet_paths)}: {os.path.basename(parquet_path)}" + ) + try: # Check if parquet file exists if not os.path.exists(parquet_path): logger.warning(f"Parquet file not found: {parquet_path}") continue - + # Get current timestamp in milliseconds current_time = time.time() * 1000 - + # Run sturgeon analysis for this file sturgeon_analyzer._run_sturgeon_analysis( parquet_path, output_dir, probes_file, current_time ) - + processed_files += 1 - logger.debug(f"Successfully processed file {i+1}: {os.path.basename(parquet_path)}") - + logger.debug( + f"Successfully processed file {i+1}: {os.path.basename(parquet_path)}" + ) + except Exception as e: - logger.warning(f"Error processing {os.path.basename(parquet_path)}: {e}") + logger.warning( + f"Error processing {os.path.basename(parquet_path)}: {e}" + ) continue if processed_files == 0: - analysis_result["error_message"] = "No files could be processed successfully" + analysis_result["error_message"] = ( + "No files could be processed successfully" + ) analysis_result["processing_steps"].append("no_files_processed") return analysis_result @@ -291,15 +300,19 @@ def process_multiple_files(parquet_paths, metadata_list, work_dir, logger): # Check if sturgeon_scores.csv was created if os.path.exists(analysis_result["sturgeon_scores_path"]): analysis_result["processing_steps"].append("sturgeon_scores_created") - logger.info(f"Sturgeon scores accumulated in: {analysis_result['sturgeon_scores_path']}") + logger.info( + f"Sturgeon scores accumulated in: {analysis_result['sturgeon_scores_path']}" + ) else: logger.warning("No sturgeon_scores.csv file was created") analysis_result["processing_steps"].append("analysis_complete") logger.info(f"Multi-file sturgeon analysis completed for {sample_id}") - logger.info(f"Files successfully processed: {analysis_result['files_processed']}/{analysis_result['total_files']}") + logger.info( + f"Files successfully processed: {analysis_result['files_processed']}/{analysis_result['total_files']}" + ) logger.info(f"Output directory: {analysis_result['output_dir']}") - + return analysis_result except Exception as e: @@ -322,55 +335,69 @@ def sturgeon_handler(job, work_dir=None): # Get job-specific logger logger = get_job_logger(str(job.job_id), job.job_type, job.context.filepath) suppress_expected = _is_fail_only_expected(job) - + # Check if this is a batched job batched_job = job.context.metadata.get("_batched_job") if batched_job: batch_size = batched_job.get_file_count() sample_id = batched_job.get_sample_id() batch_id = batched_job.batch_id - logger.info(f"Processing sturgeon analysis batch: {batch_size} files for sample '{sample_id}' (batch_id: {batch_id})") - + logger.info( + f"Processing sturgeon analysis batch: {batch_size} files for sample '{sample_id}' (batch_id: {batch_id})" + ) + # Get all filepaths in the batch filepaths = batched_job.get_filepaths() - + # Log individual files in the batch for i, filepath in enumerate(filepaths): - logger.info(f" Batch file {i+1}/{batch_size}: {os.path.basename(filepath)}") - + logger.info( + f" Batch file {i+1}/{batch_size}: {os.path.basename(filepath)}" + ) + # Prepare metadata list and extract parquet paths from bed conversion results metadata_list = [] parquet_paths = [] - + for i, bam_path in enumerate(filepaths): # Get metadata from preprocessing for this specific file file_metadata = batched_job.contexts[i].metadata.get("bam_metadata", {}) - + # Get sample ID from preprocessing results for this specific file file_context = batched_job.contexts[i] file_sample_id = file_context.get_sample_id() - + # Use the sample ID from the file's context (which should have preprocessing results) if file_sample_id != "unknown": file_metadata["sample_id"] = file_sample_id else: file_metadata["sample_id"] = sample_id - + # Get parquet path from bed conversion results for this file - bed_conversion_result = batched_job.contexts[i].results.get("bed_conversion", {}) + bed_conversion_result = batched_job.contexts[i].results.get( + "bed_conversion", {} + ) parquet_path = bed_conversion_result.get("parquet_path") - + if parquet_path: parquet_paths.append(parquet_path) metadata_list.append(file_metadata) - logger.debug(f"Found parquet path for file {i+1}: {os.path.basename(parquet_path)}") + logger.debug( + f"Found parquet path for file {i+1}: {os.path.basename(parquet_path)}" + ) else: - logger.warning(f"No parquet path found for file {i+1}: {os.path.basename(bam_path)}") - + logger.warning( + f"No parquet path found for file {i+1}: {os.path.basename(bam_path)}" + ) + if not parquet_paths: - error_msg = "No parquet paths found from bed conversion results in batch" + error_msg = ( + "No parquet paths found from bed conversion results in batch" + ) if suppress_expected: - logger.warning(f"{error_msg} (expected for fail-only BAM submission)") + logger.warning( + f"{error_msg} (expected for fail-only BAM submission)" + ) job.context.add_result( "sturgeon_analysis", {"status": "expected_failure", "reason": error_msg}, @@ -379,7 +406,7 @@ def sturgeon_handler(job, work_dir=None): logger.error(error_msg) job.context.add_error("sturgeon_analysis", error_msg) return - + # Determine work directory for the batch if work_dir is None: # Default to first parquet file directory @@ -389,29 +416,40 @@ def sturgeon_handler(job, work_dir=None): os.makedirs(work_dir, exist_ok=True) batch_work_dir = work_dir logger.debug(f"Using specified work directory: {batch_work_dir}") - + # Process all parquet files in the batch using the new aggregated function - logger.info(f"Processing {len(parquet_paths)} parquet files as aggregated batch for sample '{sample_id}'") + logger.info( + f"Processing {len(parquet_paths)} parquet files as aggregated batch for sample '{sample_id}'" + ) batch_result = process_multiple_files( parquet_paths=parquet_paths, metadata_list=metadata_list, work_dir=batch_work_dir, - logger=logger + logger=logger, ) - + # Store batch results in job context (maintain compatibility with existing structure) - job.context.add_metadata("sturgeon_analysis", { - "batch_result": batch_result, # Single aggregated result - "batch_size": batch_size, - "sample_id": sample_id, - "batch_id": batch_id, - "files_processed": batch_result.get("files_processed", len(parquet_paths)), - "total_files": batch_result.get("total_files", len(parquet_paths)) - }) - - logger.info(f"Completed sturgeon analysis batch processing: {batch_size} files for sample '{sample_id}'") - logger.info(f"Files successfully processed: {batch_result.get('files_processed', len(parquet_paths))}/{batch_result.get('total_files', len(parquet_paths))}") - + job.context.add_metadata( + "sturgeon_analysis", + { + "batch_result": batch_result, # Single aggregated result + "batch_size": batch_size, + "sample_id": sample_id, + "batch_id": batch_id, + "files_processed": batch_result.get( + "files_processed", len(parquet_paths) + ), + "total_files": batch_result.get("total_files", len(parquet_paths)), + }, + ) + + logger.info( + f"Completed sturgeon analysis batch processing: {batch_size} files for sample '{sample_id}'" + ) + logger.info( + f"Files successfully processed: {batch_result.get('files_processed', len(parquet_paths))}/{batch_result.get('total_files', len(parquet_paths))}" + ) + if batch_result.get("error_message"): if suppress_expected: logger.warning( @@ -429,9 +467,13 @@ def sturgeon_handler(job, work_dir=None): logger.error( f"Batch processing completed with errors: {batch_result['error_message']}" ) - job.context.add_error("sturgeon_analysis", batch_result["error_message"]) + job.context.add_error( + "sturgeon_analysis", batch_result["error_message"] + ) else: - logger.info("Batch processing completed successfully with aggregated sturgeon analysis") + logger.info( + "Batch processing completed successfully with aggregated sturgeon analysis" + ) job.context.add_result( "sturgeon_analysis", { @@ -440,14 +482,20 @@ def sturgeon_handler(job, work_dir=None): "analysis_time": batch_result.get("analysis_timestamp", 0), "output_dir": batch_result.get("output_dir", ""), "processing_steps": batch_result.get("processing_steps", []), - "sturgeon_scores_path": batch_result.get("sturgeon_scores_path", ""), - "files_processed": batch_result.get("files_processed", len(parquet_paths)), - "total_files": batch_result.get("total_files", len(parquet_paths)), + "sturgeon_scores_path": batch_result.get( + "sturgeon_scores_path", "" + ), + "files_processed": batch_result.get( + "files_processed", len(parquet_paths) + ), + "total_files": batch_result.get( + "total_files", len(parquet_paths) + ), }, ) - + return - + else: # Single file processing (backward compatibility) logger.info( @@ -490,8 +538,12 @@ def sturgeon_handler(job, work_dir=None): ) if sturgeon_result.error_message: - job.context.add_error("sturgeon_analysis", sturgeon_result.error_message) - logger.error(f"Sturgeon analysis failed: {sturgeon_result.error_message}") + job.context.add_error( + "sturgeon_analysis", sturgeon_result.error_message + ) + logger.error( + f"Sturgeon analysis failed: {sturgeon_result.error_message}" + ) else: job.context.add_result( "sturgeon_analysis", @@ -519,7 +571,9 @@ def sturgeon_handler(job, work_dir=None): except Exception: suppress_expected = False if suppress_expected: - logger.warning(f"Expected Sturgeon failure for fail-only BAM submission: {e}") + logger.warning( + f"Expected Sturgeon failure for fail-only BAM submission: {e}" + ) job.context.add_result( "sturgeon_analysis", {"status": "expected_failure", "error_message": str(e)}, @@ -622,8 +676,8 @@ def predict_sample_from_dataframe( Returns: pd.DataFrame: The prediction results. """ - import subprocess import json + import subprocess import tempfile # Alternative multiprocessing approach (uncomment to use): diff --git a/src/robin/analysis/target_analysis.py b/src/robin/analysis/target_analysis.py index d26902cc..5adbc71b 100644 --- a/src/robin/analysis/target_analysis.py +++ b/src/robin/analysis/target_analysis.py @@ -11,29 +11,32 @@ from __future__ import annotations import sys + if sys.version_info < (3, 12): raise RuntimeError("robin target_analysis requires Python 3.12 or newer") +import fcntl +import glob +import json +import logging import os +import re +import shutil +import subprocess import tempfile -import logging import time -import json -import subprocess -import shutil -import glob -import fcntl import uuid -import re -from pathlib import Path -from typing import Dict, Any, Optional, List, Tuple from dataclasses import dataclass from io import StringIO +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + import numpy as np import pandas as pd import pysam -from robin.logging_config import get_job_logger + from robin.analysis.snp_processing import build_snp_display_data +from robin.logging_config import get_job_logger from robin.utils.docker_fs import chown_tree_to_host_user, docker_host_user_spec # Optional import for Docker functionality @@ -56,7 +59,10 @@ def is_docker_available_for_snp_analysis() -> tuple[bool, str]: Returns (True, "") if Docker is ready, (False, "error message") otherwise. """ if docker is None: - return False, "Docker Python package is not installed. Install it with: pip install docker" + return ( + False, + "Docker Python package is not installed. Install it with: pip install docker", + ) try: client = docker.from_env() client.ping() @@ -103,9 +109,7 @@ def _resolve_clinvar_db_for_snpsift(logger: logging.Logger) -> Optional[str]: clinvar_tbi = resources_dir / "clinvar.vcf.gz.tbi" if not _is_non_empty_file(clinvar_gz): - logger.warning( - "ClinVar bgzipped VCF not found or empty at %s", clinvar_gz - ) + logger.warning("ClinVar bgzipped VCF not found or empty at %s", clinvar_gz) return None from robin.utils.clinvar_manager import _ensure_tabix_index @@ -251,24 +255,24 @@ class FileLock: Simple file-based lock for coordinating access across processes/threads. Uses fcntl for POSIX systems. """ - + def __init__(self, lock_file: str, timeout: float = 30.0): self.lock_file = lock_file self.timeout = timeout self.fd = None - + def __enter__(self): self.acquire() return self - + def __exit__(self, exc_type, exc_val, exc_tb): self.release() - + def acquire(self): """Acquire the lock with timeout""" os.makedirs(os.path.dirname(self.lock_file), exist_ok=True) - self.fd = open(self.lock_file, 'w') - + self.fd = open(self.lock_file, "w") + start_time = time.time() while True: try: @@ -276,9 +280,11 @@ def acquire(self): return except IOError: if time.time() - start_time > self.timeout: - raise TimeoutError(f"Could not acquire lock {self.lock_file} within {self.timeout}s") + raise TimeoutError( + f"Could not acquire lock {self.lock_file} within {self.timeout}s" + ) time.sleep(0.1) - + def release(self): """Release the lock""" if self.fd: @@ -313,12 +319,12 @@ def _bam_has_any_alignment(bam_path: str) -> bool: def _load_bed_regions(bedfile: str) -> List[Tuple[str, int, int]]: """Load BED regions into a list of (chrom, start, end) tuples.""" regions: List[Tuple[str, int, int]] = [] - with open(bedfile, 'r') as f: + with open(bedfile, "r") as f: for line in f: line = line.strip() - if not line or line.startswith('#'): + if not line or line.startswith("#"): continue - parts = line.split('\t') + parts = line.split("\t") if len(parts) >= 3: chrom = parts[0] start = int(parts[1]) @@ -327,15 +333,17 @@ def _load_bed_regions(bedfile: str) -> List[Tuple[str, int, int]]: return regions -def run_bedtools(bamfile, bedfile, tempbamfile, regions: Optional[List[Tuple[str, int, int]]] = None): +def run_bedtools( + bamfile, bedfile, tempbamfile, regions: Optional[List[Tuple[str, int, int]]] = None +): """ Extract target regions from BAM file, keeping all mappings (primary, secondary, supplementary) for reads that overlap the target regions. - + This function uses a two-step process: 1. Extract read names from primary alignments that overlap target regions 2. Extract ALL alignments (primary, secondary, supplementary) for those read names - + Parameters ---------- bamfile : str @@ -354,12 +362,14 @@ def run_bedtools(bamfile, bedfile, tempbamfile, regions: Optional[List[Tuple[str # Use bedtools to find primary alignments (filtering out supplementary and secondary) # Flag 2304 = 0x800 (supplementary) | 0x100 (secondary) # We use -F 2304 to exclude supplementary and secondary, keeping only primary alignments - logger.debug(f"Step 1: Extracting read names from primary alignments overlapping {bedfile}") - + logger.debug( + f"Step 1: Extracting read names from primary alignments overlapping {bedfile}" + ) + # Read BED file to get regions (unless preloaded) if regions is None: regions = _load_bed_regions(bedfile) - + if not regions: logger.warning(f"No valid regions found in BED file: {bedfile}") # Create empty BAM file @@ -369,7 +379,7 @@ def run_bedtools(bamfile, bedfile, tempbamfile, regions: Optional[List[Tuple[str pass pysam.index(tempbamfile) return - + # Open input BAM and collect read names from primary alignments overlapping regions read_names = set() with pysam.AlignmentFile(bamfile, "rb") as in_bam: @@ -377,7 +387,7 @@ def run_bedtools(bamfile, bedfile, tempbamfile, regions: Optional[List[Tuple[str index_file = f"{bamfile}.bai" if not os.path.exists(index_file): raise FileNotFoundError(f"BAM index (.bai) not found for {bamfile}") - + # Use indexed access (faster) for chrom, start, end in regions: try: @@ -390,9 +400,11 @@ def run_bedtools(bamfile, bedfile, tempbamfile, regions: Optional[List[Tuple[str # Chromosome not found in BAM, skip logger.debug(f"Chromosome {chrom} not found in BAM file, skipping") continue - - logger.debug(f"Found {len(read_names)} unique read names overlapping target regions") - + + logger.debug( + f"Found {len(read_names)} unique read names overlapping target regions" + ) + if not read_names: logger.warning("No reads found overlapping target regions") # Create empty BAM file with same header @@ -402,10 +414,12 @@ def run_bedtools(bamfile, bedfile, tempbamfile, regions: Optional[List[Tuple[str pass pysam.index(tempbamfile) return - + # Step 2: Extract ALL alignments (primary, secondary, supplementary) for those read names - logger.debug(f"Step 2: Extracting all alignments for {len(read_names)} read names") - + logger.debug( + f"Step 2: Extracting all alignments for {len(read_names)} read names" + ) + reads_written = 0 names = read_names with pysam.AlignmentFile(bamfile, "rb") as in_bam: @@ -416,42 +430,49 @@ def run_bedtools(bamfile, bedfile, tempbamfile, regions: Optional[List[Tuple[str if read.query_name in names: out_bam.write(read) reads_written += 1 - + # Ensure all data is written to disk before closing out_bam.flush() - logger.debug(f"Wrote {reads_written} alignments (including secondary/supplementary) to output BAM") - + logger.debug( + f"Wrote {reads_written} alignments (including secondary/supplementary) to output BAM" + ) + # Verify the file was written successfully before indexing if not os.path.exists(tempbamfile): raise RuntimeError(f"Output BAM file was not created: {tempbamfile}") - + file_size = os.path.getsize(tempbamfile) if file_size == 0: logger.warning(f"Output BAM file is empty: {tempbamfile}") else: logger.debug(f"Output BAM file size: {file_size} bytes") - + # Index the output BAM (only if file has content) if file_size > 0: try: pysam.index(tempbamfile) - logger.info(f"Successfully extracted target regions to {tempbamfile} ({reads_written} alignments)") + logger.info( + f"Successfully extracted target regions to {tempbamfile} ({reads_written} alignments)" + ) except Exception as e: logger.error(f"Failed to index BAM file {tempbamfile}: {e}") # Try to verify if the BAM file is valid try: with pysam.AlignmentFile(tempbamfile, "rb") as test_bam: test_count = test_bam.count(until_eof=True) - logger.info(f"BAM file is readable, contains {test_count} reads") + logger.info( + f"BAM file is readable, contains {test_count} reads" + ) except Exception as verify_error: logger.error(f"BAM file appears corrupted: {verify_error}") raise else: logger.warning(f"Skipping indexing for empty BAM file: {tempbamfile}") - + except Exception as e: logger.error(f"Error in run_bedtools: {e}") import traceback + logger.error(traceback.format_exc()) @@ -562,14 +583,14 @@ def get_covdfs(bamfile, bedfile=None): def get_read_counts_per_target(bamfile, bedfile): """ Count reads overlapping each target region in a BED file. - + Parameters ---------- bamfile : str Path to the input BAM file bedfile : str Path to the BED file defining target regions - + Returns ------- pd.DataFrame @@ -577,27 +598,29 @@ def get_read_counts_per_target(bamfile, bedfile): Returns empty DataFrame if extraction fails """ logger = logging.getLogger("robin.target") - + try: # Read BED file to get regions bed_regions = [] - with open(bedfile, 'r') as f: + with open(bedfile, "r") as f: for line in f: line = line.strip() - if not line or line.startswith('#'): + if not line or line.startswith("#"): continue - parts = line.split('\t') + parts = line.split("\t") if len(parts) >= 4: chrom = parts[0] start = int(parts[1]) end = int(parts[2]) name = parts[3] if parts[3].strip() else f"{chrom}:{start}-{end}" bed_regions.append((chrom, start, end, name)) - + if not bed_regions: logger.warning(f"No valid regions found in BED file: {bedfile}") - return pd.DataFrame(columns=["chrom", "startpos", "endpos", "name", "reads"]) - + return pd.DataFrame( + columns=["chrom", "startpos", "endpos", "name", "reads"] + ) + # Count reads per region read_counts = [] with pysam.AlignmentFile(bamfile, "rb") as bam: @@ -613,32 +636,39 @@ def get_read_counts_per_target(bamfile, bedfile): for read in bam.fetch(chrom, start, end): if (read.flag & _BAM_NON_PRIMARY_MASK) == 0: read_count += 1 - - read_counts.append({ - 'chrom': chrom, - 'startpos': start, - 'endpos': end, - 'name': name, - 'reads': read_count - }) + + read_counts.append( + { + "chrom": chrom, + "startpos": start, + "endpos": end, + "name": name, + "reads": read_count, + } + ) except ValueError: logger.debug(f"Chromosome {chrom} not found in BAM file, skipping") - read_counts.append({ - 'chrom': chrom, - 'startpos': start, - 'endpos': end, - 'name': name, - 'reads': 0 - }) + read_counts.append( + { + "chrom": chrom, + "startpos": start, + "endpos": end, + "name": name, + "reads": 0, + } + ) continue - + df = pd.DataFrame(read_counts) - logger.debug(f"Extracted read counts for {len(df)} target regions from {bamfile}") + logger.debug( + f"Extracted read counts for {len(df)} target regions from {bamfile}" + ) return df - + except Exception as e: logger.error(f"Error extracting read counts per target: {e}") import traceback + logger.error(traceback.format_exc()) return pd.DataFrame(columns=["chrom", "startpos", "endpos", "name", "reads"]) @@ -769,8 +799,15 @@ def __post_init__(self): class TargetAnalysis: """Target analysis worker""" - def __init__(self, work_dir=None, config_path=None, threads=4, target_panel=None, - batch_size=10, use_staging=True): + def __init__( + self, + work_dir=None, + config_path=None, + threads=4, + target_panel=None, + batch_size=10, + use_staging=True, + ): logger = logging.getLogger("robin.target") self.work_dir = work_dir or os.getcwd() @@ -809,41 +846,43 @@ def __init__(self, work_dir=None, config_path=None, threads=4, target_panel=None "No reference genome configured - SNP calling will not be available" ) - logger.info(f"Target Analysis initialized (staging={'enabled' if use_staging else 'disabled'}, batch_size={batch_size})") + logger.info( + f"Target Analysis initialized (staging={'enabled' if use_staging else 'disabled'}, batch_size={batch_size})" + ) def _get_master_bed_path(self, sample_id: str) -> Optional[str]: """ Get the path to the master BED file for a sample if it exists. Master BED includes target panel + CNV breakpoints + fusion breakpoints + master BED breakpoints. - + Args: sample_id: Sample ID - + Returns: Path to master BED file, or None if not found """ logger = logging.getLogger("robin.target") try: from robin.analysis.master_bed_generator import _get_latest_bed_file - + sample_dir = os.path.join(self.work_dir, sample_id) bed_dir = os.path.join(sample_dir, "bed_files") - + if not os.path.exists(bed_dir): return None - + latest = _get_latest_bed_file(bed_dir, "master_*.bed") if latest: return latest except Exception as e: logger.debug(f"Error finding master BED file: {e}") - + return None def _find_target_bed(self, target_panel: str) -> str: """Find the target BED file from robin resources based on panel type""" logger = logging.getLogger("robin.target") - + # Determine the correct BED file name based on panel type bed_filename = None if target_panel == "rCNS2": @@ -854,7 +893,7 @@ def _find_target_bed(self, target_panel: str) -> str: # Check for custom panel bed_filename = f"{target_panel}_panel_name_uniq.bed" logger.info(f"Using custom panel: {target_panel}") - + if resources is not None: try: bed_path = os.path.join( @@ -880,7 +919,9 @@ def _find_target_bed(self, target_panel: str) -> str: return path # If not found, create a placeholder (this will cause an error later) - logger.warning(f"Target BED file '{bed_filename}' not found for panel '{target_panel}', will use placeholder") + logger.warning( + f"Target BED file '{bed_filename}' not found for panel '{target_panel}', will use placeholder" + ) return bed_filename def _load_config(self) -> Dict[str, Any]: @@ -912,19 +953,19 @@ def _check_and_create_folder(self, base_dir: str, sample_id: str) -> str: sample_dir = os.path.join(base_dir, sample_id) os.makedirs(sample_dir, exist_ok=True) return sample_dir - + def _get_staging_dir(self, sample_id: str) -> str: """Get staging directory for temporary per-file results""" staging_dir = os.path.join(self.work_dir, sample_id, "_staging") os.makedirs(staging_dir, exist_ok=True) return staging_dir - + def _get_lock_file(self, sample_id: str, lock_type: str = "counter") -> str: """Get lock file path for coordinating concurrent access""" lock_dir = os.path.join(self.work_dir, sample_id, "_locks") os.makedirs(lock_dir, exist_ok=True) return os.path.join(lock_dir, f"{lock_type}.lock") - + def _get_pending_count(self, sample_id: str) -> int: """Get count of complete staging sets pending accumulation (thread-safe).""" staging_dir = self._get_staging_dir(sample_id) @@ -979,18 +1020,20 @@ def _collect_complete_staging_sets( ] return complete_sets, counts, incomplete_ids - + def _atomic_counter_increment(self, sample_id: str) -> int: """ Atomically increment and return the file counter for a sample. Uses file locking to prevent race conditions. - + Returns: The counter value to use for this file """ lock_file = self._get_lock_file(sample_id, "counter") - counter_file = os.path.join(self.work_dir, sample_id, "target_analysis_counter.txt") - + counter_file = os.path.join( + self.work_dir, sample_id, "target_analysis_counter.txt" + ) + with FileLock(lock_file, timeout=30.0): # Read current counter if os.path.exists(counter_file): @@ -1001,14 +1044,14 @@ def _atomic_counter_increment(self, sample_id: str) -> int: counter = 0 else: counter = 0 - + # Write incremented counter os.makedirs(os.path.dirname(counter_file), exist_ok=True) with open(counter_file, "w") as f: f.write(str(counter + 1)) - + return counter - + def process_file_with_staging( self, file_path: str, @@ -1018,62 +1061,62 @@ def process_file_with_staging( """ Fast per-file processing that saves results to staging area. Does NOT merge with accumulated data - much faster for large datasets. - + Args: file_path: Path to the input file metadata: File metadata from preprocessing timestamp: Optional timestamp for coverage tracking - + Returns: Tuple of (TargetMetadata, should_accumulate) - TargetMetadata: Results from this file - should_accumulate: True if batch accumulation should run now """ logger = logging.getLogger("robin.target") - + logger.info(f"Processing file with staging: {file_path}") start_time = time.time() - + # Extract sample ID from metadata sample_id = metadata.get("sample_id", "unknown") logger.debug(f"Extracted sample_id: {sample_id}") - + target_result = TargetMetadata( sample_id=sample_id, file_path=file_path, analysis_timestamp=start_time ) - + try: # Step 1: Validate input file if not os.path.exists(file_path): raise FileNotFoundError(f"Input file not found: {file_path}") - + target_result.processing_steps.append("file_validation") - + # Step 2: Create sample-specific output directory sample_output_dir = self._check_and_create_folder(self.work_dir, sample_id) logger.info(f"Sample output directory: {sample_output_dir}") - + # Step 3: Get atomic counter (thread-safe) analysis_counter = self._atomic_counter_increment(sample_id) logger.info(f"Assigned file counter: {analysis_counter}") - + target_result.processing_steps.append("counter_assigned") - + # Step 4: Extract coverage data (no loading of accumulated data) logger.info("Extracting coverage data...") newcovdf, bedcovdf = get_covdfs(file_path, self.bedfile) - + if newcovdf is None or bedcovdf is None: raise RuntimeError("Failed to extract coverage data from BAM file") - + target_result.processing_steps.append("coverage_extracted") logger.info( f"Coverage data extracted: genome={newcovdf.shape}, targets={bedcovdf.shape}" ) - + # Step 5: Save to staging (Parquet is ~5-10x faster than CSV) staging_dir = self._get_staging_dir(sample_id) - + coverage_staging = os.path.join( staging_dir, f"coverage_{analysis_counter:06d}.parquet" ) @@ -1083,54 +1126,56 @@ def process_file_with_staging( timestamp_staging = os.path.join( staging_dir, f"timestamp_{analysis_counter:06d}.txt" ) - + # Save coverage data to staging newcovdf.to_parquet(coverage_staging, **_PARQUET_WRITE_KWARGS) bedcovdf.to_parquet(bedcov_staging, **_PARQUET_WRITE_KWARGS) - + # Save timestamp for coverage tracking - current_timestamp = timestamp * 1000 if (self.simtime and timestamp) else time.time() * 1000 + current_timestamp = ( + timestamp * 1000 if (self.simtime and timestamp) else time.time() * 1000 + ) with open(timestamp_staging, "w") as f: f.write(str(current_timestamp)) - + # Save source BAM file path for target.bam creation during accumulation source_bam_staging = os.path.join( staging_dir, f"source_bam_{analysis_counter:06d}.txt" ) with open(source_bam_staging, "w") as f: f.write(file_path) - + target_result.processing_steps.append("saved_to_staging") logger.info(f"Saved to staging: {coverage_staging}") - + # Step 6: Store minimal coverage data in metadata (for logging) target_result.coverage_data = { "genome_coverage_shape": newcovdf.shape, "target_coverage_shape": bedcovdf.shape, "staging_file": coverage_staging, } - + # Step 7: Check if accumulation should run pending_count = self._get_pending_count(sample_id) should_accumulate = pending_count >= self.batch_size - + logger.info( f"File staged successfully. Pending files: {pending_count}/{self.batch_size}" ) - + if should_accumulate: logger.info( f"Accumulation threshold reached ({pending_count} >= {self.batch_size})" ) - + target_result.processing_steps.append("staging_complete") elapsed = time.time() - start_time logger.info( f"Staging complete for {sample_id} in {elapsed:.2f}s (vs ~{elapsed*10:.1f}s without staging)" ) - + return target_result, should_accumulate - + except Exception as e: error_details = f"Error in staging for {sample_id}: {str(e)}" logger.error(error_details) @@ -1236,7 +1281,7 @@ def process_file( # Master BED includes target panel + CNV + fusion + master BED breakpoints sample_id = metadata.get("sample_id", "unknown") targets_bed = self._get_master_bed_path(sample_id) or self.bedfile - + # Run bedtools intersection run_bedtools(file_path, targets_bed, tempbamfile.name) @@ -1404,12 +1449,16 @@ def process_file( # Step 13b: Create target coverage with timestamp and read counts (optimized) logger.info("Calculating read counts per target...") new_read_counts_df = get_read_counts_per_target(file_path, self.bedfile) - + if not new_read_counts_df.empty: # Use optimized approach: store latest cumulative reads in separate Parquet file for fast access - time_coverage_file = os.path.join(sample_output_dir, "target_coverage_time.csv") - latest_reads_cache = os.path.join(sample_output_dir, "_target_coverage_latest_reads.parquet") - + time_coverage_file = os.path.join( + sample_output_dir, "target_coverage_time.csv" + ) + latest_reads_cache = os.path.join( + sample_output_dir, "_target_coverage_latest_reads.parquet" + ) + # Load previous cumulative reads from cache (much faster than reading entire CSV) previous_cumulative_reads = None if os.path.exists(latest_reads_cache): @@ -1417,100 +1466,150 @@ def process_file( previous_cumulative_reads = pd.read_parquet( latest_reads_cache, **_PARQUET_READ_KWARGS ) - previous_cumulative_reads.rename(columns={'reads': 'previous_reads'}, inplace=True) + previous_cumulative_reads.rename( + columns={"reads": "previous_reads"}, inplace=True + ) except Exception as e: - logger.debug(f"Could not load cached latest reads, will try CSV: {e}") + logger.debug( + f"Could not load cached latest reads, will try CSV: {e}" + ) # Fallback to CSV if cache doesn't exist if os.path.exists(time_coverage_file): try: # Only read last chunk for efficiency existing_time_df = pd.read_csv(time_coverage_file) if not existing_time_df.empty: - latest_timestamp = existing_time_df['timestamp'].max() + latest_timestamp = existing_time_df[ + "timestamp" + ].max() previous_cumulative_reads = existing_time_df[ - existing_time_df['timestamp'] == latest_timestamp - ][['chrom', 'startpos', 'endpos', 'name', 'reads']].copy() - previous_cumulative_reads.rename(columns={'reads': 'previous_reads'}, inplace=True) + existing_time_df["timestamp"] + == latest_timestamp + ][ + [ + "chrom", + "startpos", + "endpos", + "name", + "reads", + ] + ].copy() + previous_cumulative_reads.rename( + columns={"reads": "previous_reads"}, + inplace=True, + ) except Exception as e2: - logger.warning(f"Error loading existing target_coverage_time.csv: {e2}") - + logger.warning( + f"Error loading existing target_coverage_time.csv: {e2}" + ) + # Merge new read counts with target coverage data target_coverage_with_reads = target_coverage_df.merge( - new_read_counts_df[['chrom', 'startpos', 'endpos', 'name', 'reads']], - on=['chrom', 'startpos', 'endpos', 'name'], - how='left' + new_read_counts_df[ + ["chrom", "startpos", "endpos", "name", "reads"] + ], + on=["chrom", "startpos", "endpos", "name"], + how="left", ) # Fill missing reads with 0 - target_coverage_with_reads['reads'] = target_coverage_with_reads['reads'].fillna(0).astype(int) - + target_coverage_with_reads["reads"] = ( + target_coverage_with_reads["reads"].fillna(0).astype(int) + ) + # Accumulate with previous cumulative reads if available if previous_cumulative_reads is not None: target_coverage_with_reads = target_coverage_with_reads.merge( previous_cumulative_reads, - on=['chrom', 'startpos', 'endpos', 'name'], - how='left' + on=["chrom", "startpos", "endpos", "name"], + how="left", + ) + target_coverage_with_reads["previous_reads"] = ( + target_coverage_with_reads["previous_reads"] + .fillna(0) + .astype(int) ) - target_coverage_with_reads['previous_reads'] = target_coverage_with_reads['previous_reads'].fillna(0).astype(int) # Add new reads to previous cumulative reads - target_coverage_with_reads['reads'] = ( - target_coverage_with_reads['reads'] + target_coverage_with_reads['previous_reads'] + target_coverage_with_reads["reads"] = ( + target_coverage_with_reads["reads"] + + target_coverage_with_reads["previous_reads"] + ) + target_coverage_with_reads.drop( + columns=["previous_reads"], inplace=True ) - target_coverage_with_reads.drop(columns=['previous_reads'], inplace=True) - + # Calculate normalized reads (reads per length) - target_coverage_with_reads['reads_per_length'] = ( - target_coverage_with_reads['reads'] / target_coverage_with_reads['length'] + target_coverage_with_reads["reads_per_length"] = ( + target_coverage_with_reads["reads"] + / target_coverage_with_reads["length"] ) - + # Add timestamp if self.simtime and timestamp: current_timestamp = timestamp * 1000 else: current_timestamp = time.time() * 1000 - target_coverage_with_reads['timestamp'] = current_timestamp - + target_coverage_with_reads["timestamp"] = current_timestamp + # Reorder columns: chrom, startpos, endpos, name, length, coverage, bases, timestamp, reads, reads_per_length target_coverage_with_reads = target_coverage_with_reads[ - ['chrom', 'startpos', 'endpos', 'name', 'length', 'coverage', 'bases', - 'timestamp', 'reads', 'reads_per_length'] + [ + "chrom", + "startpos", + "endpos", + "name", + "length", + "coverage", + "bases", + "timestamp", + "reads", + "reads_per_length", + ] ] - + # Save latest cumulative reads to cache for next time (fast access) try: target_coverage_with_reads[ - ['chrom', 'startpos', 'endpos', 'name', 'reads'] + ["chrom", "startpos", "endpos", "name", "reads"] ].to_parquet(latest_reads_cache, **_PARQUET_WRITE_KWARGS) except Exception as e: logger.debug(f"Could not save latest reads cache: {e}") - + # Append to CSV using append mode (much faster than reading entire file) try: # Check if file exists to determine if we need header file_exists = os.path.exists(time_coverage_file) target_coverage_with_reads.to_csv( - time_coverage_file, - mode='a', + time_coverage_file, + mode="a", header=not file_exists, - index=False + index=False, ) except Exception as e: - logger.warning(f"Error appending to target_coverage_time.csv: {e}") + logger.warning( + f"Error appending to target_coverage_time.csv: {e}" + ) # Fallback: read and concat (slower but works) if os.path.exists(time_coverage_file): try: existing_time_df = pd.read_csv(time_coverage_file) target_coverage_with_reads = pd.concat( [existing_time_df, target_coverage_with_reads], - ignore_index=True + ignore_index=True, + ) + target_coverage_with_reads.to_csv( + time_coverage_file, index=False ) - target_coverage_with_reads.to_csv(time_coverage_file, index=False) except Exception as e2: logger.error(f"Error in fallback CSV write: {e2}") - - logger.info(f"Saved target coverage with timestamp and cumulative read counts: {time_coverage_file}") + + logger.info( + f"Saved target coverage with timestamp and cumulative read counts: {time_coverage_file}" + ) target_result.processing_steps.append("target_coverage_time_saved") else: - logger.warning("No read counts extracted, skipping target_coverage_time.csv") + logger.warning( + "No read counts extracted, skipping target_coverage_time.csv" + ) # Step 13: Identify targets exceeding threshold run_list = target_coverage_df[ @@ -1706,7 +1805,9 @@ def _perform_target_analysis( "position": f"{row['chrom']}:{row['startpos']}-{row['endpos']}", "coverage": row["bases"], "significance": ( - "high" if row["bases"] > significant_threshold * max_bases else "medium" + "high" + if row["bases"] > significant_threshold * max_bases + else "medium" ), } for idx, row in bedcovdf.iterrows() @@ -1835,8 +1936,10 @@ def _load_existing_coverage_over_time( logger.warning(f"Error loading existing coverage over time data: {e}") return None return None - - def accumulate_staged_files(self, sample_id: str, force: bool = False) -> Dict[str, Any]: + + def accumulate_staged_files( + self, sample_id: str, force: bool = False + ) -> Dict[str, Any]: """ Batch accumulation of staged files with minimal lock hold time. Lock is held only to: (1) claim staging files, (2) load existing + merge, (3) write outputs + cleanup. @@ -1853,8 +1956,8 @@ def accumulate_staged_files(self, sample_id: str, force: bool = False) -> Dict[s batch_dir = None num_claimed = 0 with FileLock(lock_file, timeout=60.0): - complete_sets, staging_counts, incomplete_ids = self._collect_complete_staging_sets( - staging_dir + complete_sets, staging_counts, incomplete_ids = ( + self._collect_complete_staging_sets(staging_dir) ) if not complete_sets: if any(staging_counts.values()): @@ -1890,20 +1993,34 @@ def accumulate_staged_files(self, sample_id: str, force: bool = False) -> Dict[s "files_pending": complete_count, "incomplete_staging_sets": len(incomplete_ids), } - n_claim = complete_count if force else min(self.batch_size, complete_count) + n_claim = ( + complete_count if force else min(self.batch_size, complete_count) + ) batch_id = str(uuid.uuid4()) batch_dir = os.path.join(staging_dir, f"_batch_{batch_id}") os.makedirs(batch_dir, exist_ok=True) - for coverage_src, bedcov_src, timestamp_src, source_bam_src in complete_sets[:n_claim]: - for src in (coverage_src, bedcov_src, timestamp_src, source_bam_src): + for ( + coverage_src, + bedcov_src, + timestamp_src, + source_bam_src, + ) in complete_sets[:n_claim]: + for src in ( + coverage_src, + bedcov_src, + timestamp_src, + source_bam_src, + ): if os.path.exists(src): - shutil.move(src, os.path.join(batch_dir, os.path.basename(src))) + shutil.move( + src, os.path.join(batch_dir, os.path.basename(src)) + ) num_claimed = n_claim # Lock released # --- No lock: load from batch dir --- - batch_sets, batch_counts, batch_incomplete = self._collect_complete_staging_sets( - batch_dir + batch_sets, batch_counts, batch_incomplete = ( + self._collect_complete_staging_sets(batch_dir) ) if batch_incomplete: logger.warning( @@ -1919,12 +2036,8 @@ def accumulate_staged_files(self, sample_id: str, force: bool = False) -> Dict[s loaded_files = 0 for cov_file, bed_file, ts_file, source_bam_file in batch_sets: try: - cov_frames.append( - pd.read_parquet(cov_file, **_PARQUET_READ_KWARGS) - ) - bed_frames.append( - pd.read_parquet(bed_file, **_PARQUET_READ_KWARGS) - ) + cov_frames.append(pd.read_parquet(cov_file, **_PARQUET_READ_KWARGS)) + bed_frames.append(pd.read_parquet(bed_file, **_PARQUET_READ_KWARGS)) with open(ts_file, "r") as f: timestamps.append(float(f.read().strip())) with open(source_bam_file, "r") as f: @@ -1973,7 +2086,9 @@ def accumulate_staged_files(self, sample_id: str, force: bool = False) -> Dict[s except OSError: pass return {"status": "load_failed", "files_attempted": num_claimed} - logger.info(f"Loaded {loaded_files} staging files; merging with accumulated data...") + logger.info( + f"Loaded {loaded_files} staging files; merging with accumulated data..." + ) # --- Critical section 2: load existing + merge (short) --- existing_covdf = None @@ -1999,7 +2114,9 @@ def accumulate_staged_files(self, sample_id: str, force: bool = False) -> Dict[s updated_covdf, updated_bedcovdf = batch_covdf, batch_bedcovdf # Lock released - logger.info(f"Final accumulated: genome={updated_covdf.shape}, targets={updated_bedcovdf.shape}") + logger.info( + f"Final accumulated: genome={updated_covdf.shape}, targets={updated_bedcovdf.shape}" + ) bases = updated_covdf["covbases"].sum() genome = updated_covdf["endpos"].sum() coverage = bases / genome if genome > 0 else 0.0 @@ -2022,7 +2139,9 @@ def accumulate_staged_files(self, sample_id: str, force: bool = False) -> Dict[s target_coverage_df = target_coverage_df[ ["chrom", "startpos", "endpos", "name", "length", "coverage", "bases"] ] - run_list = target_coverage_df[target_coverage_df["coverage"].ge(self.callthreshold)] + run_list = target_coverage_df[ + target_coverage_df["coverage"].ge(self.callthreshold) + ] # --- No lock: BAM read counts and filtered BAMs --- batch_read_counts = None @@ -2033,39 +2152,64 @@ def accumulate_staged_files(self, sample_id: str, force: bool = False) -> Dict[s if not read_counts_df.empty: batch_read_counts_list.append(read_counts_df) if batch_read_counts_list: - batch_read_counts = pd.concat(batch_read_counts_list, ignore_index=True) + batch_read_counts = pd.concat( + batch_read_counts_list, ignore_index=True + ) batch_read_counts = batch_read_counts.groupby( - ['chrom', 'startpos', 'endpos', 'name'], as_index=False - ).agg({'reads': 'sum'}) + ["chrom", "startpos", "endpos", "name"], as_index=False + ).agg({"reads": "sum"}) if len(run_list) > 0: logger.info("Processing filtered BAM files...") try: original_bam_files = list(set(source_bam_paths)) if original_bam_files: - targets_bed = self._get_master_bed_path(sample_id) or self.bedfile + targets_bed = ( + self._get_master_bed_path(sample_id) or self.bedfile + ) if targets_bed != self.bedfile: - logger.info(f"Using master BED file for target.bam: {targets_bed}") + logger.info( + f"Using master BED file for target.bam: {targets_bed}" + ) else: - logger.info(f"Using original target panel BED file for target.bam: {targets_bed}") - filtered_bams_dir = os.path.join(sample_output_dir, "_filtered_bams") + logger.info( + f"Using original target panel BED file for target.bam: {targets_bed}" + ) + filtered_bams_dir = os.path.join( + sample_output_dir, "_filtered_bams" + ) os.makedirs(filtered_bams_dir, exist_ok=True) new_filtered_bams = [] bed_regions_cache = None try: bed_regions_cache = _load_bed_regions(targets_bed) except Exception as e: - logger.warning(f"Could not preload BED regions from {targets_bed}: {e}") + logger.warning( + f"Could not preload BED regions from {targets_bed}: {e}" + ) for i, source_bam in enumerate(original_bam_files): - filtered_bam_name = f"filtered_{i:06d}_{os.path.basename(source_bam)}" - filtered_bam_path = os.path.join(filtered_bams_dir, filtered_bam_name) - logger.info(f"Filtering BAM {i+1}/{len(original_bam_files)}: {os.path.basename(source_bam)}") - run_bedtools(source_bam, targets_bed, filtered_bam_path, regions=bed_regions_cache) + filtered_bam_name = ( + f"filtered_{i:06d}_{os.path.basename(source_bam)}" + ) + filtered_bam_path = os.path.join( + filtered_bams_dir, filtered_bam_name + ) + logger.info( + f"Filtering BAM {i+1}/{len(original_bam_files)}: {os.path.basename(source_bam)}" + ) + run_bedtools( + source_bam, + targets_bed, + filtered_bam_path, + regions=bed_regions_cache, + ) if os.path.exists(filtered_bam_path): new_filtered_bams.append(filtered_bam_path) if new_filtered_bams: batch_timestamp = int(time.time() * 1000) - batch_merged_bam = os.path.join(sample_output_dir, f"batch_{batch_timestamp}.bam") + batch_merged_bam = os.path.join( + sample_output_dir, f"batch_{batch_timestamp}.bam" + ) if len(new_filtered_bams) > 1: pysam.merge("-o", batch_merged_bam, *new_filtered_bams) else: @@ -2078,7 +2222,9 @@ def accumulate_staged_files(self, sample_id: str, force: bool = False) -> Dict[s if os.path.exists(f"{filtered_bam}.bai"): os.remove(f"{filtered_bam}.bai") except OSError as e: - logger.warning(f"Could not remove filtered BAM {filtered_bam}: {e}") + logger.warning( + f"Could not remove filtered BAM {filtered_bam}: {e}" + ) try: if not os.listdir(filtered_bams_dir): os.rmdir(filtered_bams_dir) @@ -2087,11 +2233,16 @@ def accumulate_staged_files(self, sample_id: str, force: bool = False) -> Dict[s except Exception as e: logger.error(f"Error creating target.bam: {e}") import traceback + logger.error(traceback.format_exc()) # --- Critical section 3: write all outputs + cleanup (short) --- - time_coverage_file = os.path.join(sample_output_dir, "target_coverage_time.csv") - latest_reads_cache = os.path.join(sample_output_dir, "_target_coverage_latest_reads.parquet") + time_coverage_file = os.path.join( + sample_output_dir, "target_coverage_time.csv" + ) + latest_reads_cache = os.path.join( + sample_output_dir, "_target_coverage_latest_reads.parquet" + ) with FileLock(lock_file, timeout=60.0): np.save( os.path.join(sample_output_dir, "coverage_time_chart.npy"), @@ -2102,16 +2253,27 @@ def accumulate_staged_files(self, sample_id: str, force: bool = False) -> Dict[s ) bed_coverage_main_df = updated_bedcovdf.copy() bed_coverage_main_df["length"] = ( - bed_coverage_main_df["endpos"] - bed_coverage_main_df["startpos"] + 1 + bed_coverage_main_df["endpos"] + - bed_coverage_main_df["startpos"] + + 1 ) bed_coverage_main_df["coverage"] = ( bed_coverage_main_df["bases"] / bed_coverage_main_df["length"] ) bed_coverage_main_df = bed_coverage_main_df[ - ["chrom", "startpos", "endpos", "name", "length", "coverage", "bases"] + [ + "chrom", + "startpos", + "endpos", + "name", + "length", + "coverage", + "bases", + ] ] bed_coverage_main_df.to_csv( - os.path.join(sample_output_dir, "bed_coverage_main.csv"), index=False + os.path.join(sample_output_dir, "bed_coverage_main.csv"), + index=False, ) target_coverage_df.to_csv( os.path.join(sample_output_dir, "target_coverage.csv"), index=False @@ -2123,52 +2285,87 @@ def accumulate_staged_files(self, sample_id: str, force: bool = False) -> Dict[s previous_cumulative_reads = pd.read_parquet( latest_reads_cache, **_PARQUET_READ_KWARGS ) - previous_cumulative_reads.rename(columns={'reads': 'previous_reads'}, inplace=True) + previous_cumulative_reads.rename( + columns={"reads": "previous_reads"}, inplace=True + ) except Exception: pass - if previous_cumulative_reads is None and os.path.exists(time_coverage_file): + if previous_cumulative_reads is None and os.path.exists( + time_coverage_file + ): try: existing_time_df = pd.read_csv(time_coverage_file) if not existing_time_df.empty: - latest_ts = existing_time_df['timestamp'].max() + latest_ts = existing_time_df["timestamp"].max() previous_cumulative_reads = existing_time_df[ - existing_time_df['timestamp'] == latest_ts - ][['chrom', 'startpos', 'endpos', 'name', 'reads']].copy() - previous_cumulative_reads.rename(columns={'reads': 'previous_reads'}, inplace=True) + existing_time_df["timestamp"] == latest_ts + ][ + ["chrom", "startpos", "endpos", "name", "reads"] + ].copy() + previous_cumulative_reads.rename( + columns={"reads": "previous_reads"}, inplace=True + ) except Exception: pass target_coverage_with_reads = target_coverage_df.merge( - batch_read_counts[['chrom', 'startpos', 'endpos', 'name', 'reads']], - on=['chrom', 'startpos', 'endpos', 'name'], how='left' + batch_read_counts[ + ["chrom", "startpos", "endpos", "name", "reads"] + ], + on=["chrom", "startpos", "endpos", "name"], + how="left", + ) + target_coverage_with_reads["reads"] = ( + target_coverage_with_reads["reads"].fillna(0).astype(int) ) - target_coverage_with_reads['reads'] = target_coverage_with_reads['reads'].fillna(0).astype(int) if previous_cumulative_reads is not None: target_coverage_with_reads = target_coverage_with_reads.merge( previous_cumulative_reads, - on=['chrom', 'startpos', 'endpos', 'name'], how='left' + on=["chrom", "startpos", "endpos", "name"], + how="left", ) - target_coverage_with_reads['previous_reads'] = target_coverage_with_reads['previous_reads'].fillna(0).astype(int) - target_coverage_with_reads['reads'] = ( - target_coverage_with_reads['reads'] + target_coverage_with_reads['previous_reads'] + target_coverage_with_reads["previous_reads"] = ( + target_coverage_with_reads["previous_reads"] + .fillna(0) + .astype(int) ) - target_coverage_with_reads.drop(columns=['previous_reads'], inplace=True) - target_coverage_with_reads['reads_per_length'] = ( - target_coverage_with_reads['reads'] / target_coverage_with_reads['length'] + target_coverage_with_reads["reads"] = ( + target_coverage_with_reads["reads"] + + target_coverage_with_reads["previous_reads"] + ) + target_coverage_with_reads.drop( + columns=["previous_reads"], inplace=True + ) + target_coverage_with_reads["reads_per_length"] = ( + target_coverage_with_reads["reads"] + / target_coverage_with_reads["length"] ) - target_coverage_with_reads['timestamp'] = current_timestamp + target_coverage_with_reads["timestamp"] = current_timestamp target_coverage_with_reads = target_coverage_with_reads[ - ['chrom', 'startpos', 'endpos', 'name', 'length', 'coverage', 'bases', - 'timestamp', 'reads', 'reads_per_length'] + [ + "chrom", + "startpos", + "endpos", + "name", + "length", + "coverage", + "bases", + "timestamp", + "reads", + "reads_per_length", + ] ] try: target_coverage_with_reads[ - ['chrom', 'startpos', 'endpos', 'name', 'reads'] + ["chrom", "startpos", "endpos", "name", "reads"] ].to_parquet(latest_reads_cache, **_PARQUET_WRITE_KWARGS) except Exception: pass file_exists = os.path.exists(time_coverage_file) target_coverage_with_reads.to_csv( - time_coverage_file, mode='a', header=not file_exists, index=False + time_coverage_file, + mode="a", + header=not file_exists, + index=False, ) targets_exceeding_file = os.path.join( sample_output_dir, "targets_exceeding_threshold_count.txt" @@ -2177,12 +2374,19 @@ def accumulate_staged_files(self, sample_id: str, force: bool = False) -> Dict[s with open(targets_exceeding_file, "w") as f: f.write(str(len(run_list))) run_list[["chrom", "startpos", "endpos"]].to_csv( - os.path.join(sample_output_dir, "targets_exceeding_threshold.bed"), - sep="\t", header=None, index=None, + os.path.join( + sample_output_dir, "targets_exceeding_threshold.bed" + ), + sep="\t", + header=None, + index=None, ) else: with open( - os.path.join(sample_output_dir, "targets_exceeding_threshold.bed"), "w" + os.path.join( + sample_output_dir, "targets_exceeding_threshold.bed" + ), + "w", ) as f: pass if batch_dir and os.path.exists(batch_dir): @@ -2205,7 +2409,9 @@ def accumulate_staged_files(self, sample_id: str, force: bool = False) -> Dict[s "status": "success", "files_processed": num_claimed, "coverage": coverage, - "targets_exceeding_threshold": len(run_list) if len(run_list) > 0 else 0, + "targets_exceeding_threshold": ( + len(run_list) if len(run_list) > 0 else 0 + ), "elapsed_time": elapsed, } @@ -2215,6 +2421,7 @@ def accumulate_staged_files(self, sample_id: str, force: bool = False) -> Dict[s except Exception as e: logger.error(f"Error during batch accumulation for {sample_id}: {e}") import traceback + logger.error(traceback.format_exc()) return {"status": "error", "error": str(e)} @@ -2313,10 +2520,12 @@ def process_single_file( return analysis_result -def process_multiple_files(bam_paths, metadata_list, work_dir, logger, reference=None, target_panel=None): +def process_multiple_files( + bam_paths, metadata_list, work_dir, logger, reference=None, target_panel=None +): """ Process multiple BAM files for target analysis using staged processing. - + This function processes multiple BAM files for the same sample using the existing staging infrastructure. Each file is processed individually and staged, then all staged files are accumulated in a single batch operation. @@ -2334,16 +2543,16 @@ def process_multiple_files(bam_paths, metadata_list, work_dir, logger, reference """ if not bam_paths or not metadata_list: raise ValueError("bam_paths and metadata_list must not be empty") - + if len(bam_paths) != len(metadata_list): raise ValueError("bam_paths and metadata_list must have the same length") - + # Get sample ID from first metadata (assuming all BAMs are from same sample) sample_id = metadata_list[0].get("sample_id", "unknown") - + logger.info(f"🎯 Starting multi-file target analysis for sample: {sample_id}") logger.info(f"Processing {len(bam_paths)} BAM files for sample {sample_id}") - + # Log essential metadata only for i, (bam_path, metadata) in enumerate(zip(bam_paths, metadata_list)): logger.debug(f"BAM file {i+1}: {os.path.basename(bam_path)}") @@ -2364,12 +2573,12 @@ def process_multiple_files(bam_paths, metadata_list, work_dir, logger, reference try: # Initialize target analysis with staging enabled target_analysis = TargetAnalysis( - work_dir=work_dir, + work_dir=work_dir, target_panel=target_panel, batch_size=1, # Force accumulation after each batch - use_staging=True + use_staging=True, ) - + # Set reference genome if provided if reference: target_analysis.reference = reference @@ -2378,29 +2587,37 @@ def process_multiple_files(bam_paths, metadata_list, work_dir, logger, reference # Process each BAM file individually using staging logger.info("Processing files with staging (fast path)") processed_files = 0 - + for i, (bam_path, metadata) in enumerate(zip(bam_paths, metadata_list)): - logger.info(f"Processing BAM file {i+1}/{len(bam_paths)}: {os.path.basename(bam_path)}") - + logger.info( + f"Processing BAM file {i+1}/{len(bam_paths)}: {os.path.basename(bam_path)}" + ) + try: # Process file with staging - target_metadata, should_accumulate = target_analysis.process_file_with_staging( - bam_path, metadata + target_metadata, should_accumulate = ( + target_analysis.process_file_with_staging(bam_path, metadata) ) - + if target_metadata.error_message: - logger.warning(f"Error processing {os.path.basename(bam_path)}: {target_metadata.error_message}") + logger.warning( + f"Error processing {os.path.basename(bam_path)}: {target_metadata.error_message}" + ) continue - + processed_files += 1 - logger.debug(f"Successfully staged file {i+1}: {os.path.basename(bam_path)}") - + logger.debug( + f"Successfully staged file {i+1}: {os.path.basename(bam_path)}" + ) + except Exception as e: logger.warning(f"Error processing {os.path.basename(bam_path)}: {e}") continue if processed_files == 0: - analysis_result["error_message"] = "No files could be processed successfully" + analysis_result["error_message"] = ( + "No files could be processed successfully" + ) analysis_result["processing_steps"].append("no_files_processed") return analysis_result @@ -2408,13 +2625,17 @@ def process_multiple_files(bam_paths, metadata_list, work_dir, logger, reference analysis_result["processing_steps"].append("files_staged") # Force accumulation of all staged files - logger.info(f"Accumulating {processed_files} staged files for sample {sample_id}") + logger.info( + f"Accumulating {processed_files} staged files for sample {sample_id}" + ) accumulation_result = target_analysis.accumulate_staged_files( sample_id, force=True ) - + if accumulation_result.get("status") != "success": - analysis_result["error_message"] = f"Accumulation failed: {accumulation_result.get('error', 'Unknown error')}" + analysis_result["error_message"] = ( + f"Accumulation failed: {accumulation_result.get('error', 'Unknown error')}" + ) analysis_result["processing_steps"].append("accumulation_failed") return analysis_result @@ -2423,7 +2644,7 @@ def process_multiple_files(bam_paths, metadata_list, work_dir, logger, reference # Load final accumulated data for result metadata sample_output_dir = os.path.join(work_dir, sample_id) - + # Load final coverage data final_covdf = target_analysis._load_existing_coverage_data( sample_output_dir, "coverage_main.csv", logger @@ -2434,24 +2655,38 @@ def process_multiple_files(bam_paths, metadata_list, work_dir, logger, reference final_coverage_over_time = target_analysis._load_existing_coverage_over_time( sample_output_dir, logger ) - + # Store final results analysis_result["coverage_data"] = { - "genome_coverage_shape": final_covdf.shape if final_covdf is not None else (0, 0), - "target_coverage_shape": final_bedcovdf.shape if final_bedcovdf is not None else (0, 0), + "genome_coverage_shape": ( + final_covdf.shape if final_covdf is not None else (0, 0) + ), + "target_coverage_shape": ( + final_bedcovdf.shape if final_bedcovdf is not None else (0, 0) + ), "coverage": accumulation_result.get("coverage", 0.0), - "targets_exceeding_threshold": accumulation_result.get("targets_exceeding_threshold", 0), + "targets_exceeding_threshold": accumulation_result.get( + "targets_exceeding_threshold", 0 + ), } - - analysis_result["target_bam_path"] = os.path.join(sample_output_dir, "target.bam") + + analysis_result["target_bam_path"] = os.path.join( + sample_output_dir, "target.bam" + ) analysis_result["coverage_over_time"] = final_coverage_over_time - + analysis_result["processing_steps"].append("analysis_complete") logger.info(f"Multi-file target analysis completed for {sample_id}") - logger.info(f"Files successfully processed: {analysis_result['files_processed']}/{analysis_result['total_files']}") - logger.info(f"Final coverage: {analysis_result['coverage_data']['coverage']:.4f}") - logger.info(f"Targets exceeding threshold: {analysis_result['coverage_data']['targets_exceeding_threshold']}") - + logger.info( + f"Files successfully processed: {analysis_result['files_processed']}/{analysis_result['total_files']}" + ) + logger.info( + f"Final coverage: {analysis_result['coverage_data']['coverage']:.4f}" + ) + logger.info( + f"Targets exceeding threshold: {analysis_result['coverage_data']['targets_exceeding_threshold']}" + ) + return analysis_result except Exception as e: @@ -2475,30 +2710,38 @@ def target_handler(job, work_dir=None, reference=None, target_panel=None): # Validate required parameters if not target_panel: raise ValueError("target_panel is required for target analysis") - + # Get job-specific logger logger = get_job_logger(str(job.job_id), job.job_type, job.context.filepath) - + # Check if this is a batched job batched_job = job.context.metadata.get("_batched_job") if batched_job: batch_size = batched_job.get_file_count() sample_id = batched_job.get_sample_id() batch_id = batched_job.batch_id - logger.info(f"Processing target analysis batch: {batch_size} files for sample '{sample_id}' (batch_id: {batch_id})") - + logger.info( + f"Processing target analysis batch: {batch_size} files for sample '{sample_id}' (batch_id: {batch_id})" + ) + # Get all filepaths in the batch filepaths = batched_job.get_filepaths() - + # Log individual files in the batch for i, filepath in enumerate(filepaths): - logger.info(f" Batch file {i+1}/{batch_size}: {os.path.basename(filepath)}") - + logger.info( + f" Batch file {i+1}/{batch_size}: {os.path.basename(filepath)}" + ) + # Prepare metadata list for all BAM files in the batch (list comp inlined in 3.12) def _batch_metadata(i: int) -> dict: ctx = batched_job.contexts[i] sid = ctx.get_sample_id() - return {**ctx.metadata.get("bam_metadata", {}), "sample_id": sid if sid != "unknown" else sample_id} + return { + **ctx.metadata.get("bam_metadata", {}), + "sample_id": sid if sid != "unknown" else sample_id, + } + metadata_list = [_batch_metadata(i) for i in range(len(filepaths))] # Determine work directory for the batch @@ -2510,14 +2753,16 @@ def _batch_metadata(i: int) -> dict: os.makedirs(work_dir, exist_ok=True) batch_work_dir = work_dir logger.debug(f"Using specified work directory: {batch_work_dir}") - + # Log and validate target panel job_panel = batched_job.contexts[0].metadata.get("target_panel", target_panel) if job_panel != target_panel: - logger.warning(f"Panel mismatch: job metadata has '{job_panel}' but handler received '{target_panel}'. Using '{job_panel}' from metadata.") + logger.warning( + f"Panel mismatch: job metadata has '{job_panel}' but handler received '{target_panel}'. Using '{job_panel}' from metadata." + ) target_panel = job_panel logger.info(f"Using target panel: {target_panel}") - + # Debug: Log reference genome status if reference: logger.info(f"Reference genome provided to target_handler: {reference}") @@ -2534,43 +2779,58 @@ def _batch_metadata(i: int) -> dict: logger.info(f"Using reference from job metadata: {reference}") else: logger.info("No reference genome found in job metadata") - + # Process all BAM files in the batch using the new aggregated function - logger.info(f"Processing {batch_size} BAM files as aggregated batch for sample '{sample_id}'") + logger.info( + f"Processing {batch_size} BAM files as aggregated batch for sample '{sample_id}'" + ) batch_result = process_multiple_files( bam_paths=filepaths, metadata_list=metadata_list, work_dir=batch_work_dir, logger=logger, reference=reference, - target_panel=target_panel + target_panel=target_panel, ) - + # Store batch results in job context (maintain compatibility with existing structure) - job.context.add_metadata("target_analysis", { - "batch_result": batch_result, # Single aggregated result - "batch_size": batch_size, - "sample_id": sample_id, - "batch_id": batch_id, - "files_processed": batch_result.get("files_processed", batch_size), - "total_files": batch_result.get("total_files", batch_size) - }) - - logger.info(f"Completed target analysis batch processing: {batch_size} files for sample '{sample_id}'") - logger.info(f"Files successfully processed: {batch_result.get('files_processed', batch_size)}/{batch_result.get('total_files', batch_size)}") - + job.context.add_metadata( + "target_analysis", + { + "batch_result": batch_result, # Single aggregated result + "batch_size": batch_size, + "sample_id": sample_id, + "batch_id": batch_id, + "files_processed": batch_result.get("files_processed", batch_size), + "total_files": batch_result.get("total_files", batch_size), + }, + ) + + logger.info( + f"Completed target analysis batch processing: {batch_size} files for sample '{sample_id}'" + ) + logger.info( + f"Files successfully processed: {batch_result.get('files_processed', batch_size)}/{batch_result.get('total_files', batch_size)}" + ) + if batch_result.get("error_message"): - logger.error(f"Batch processing completed with errors: {batch_result['error_message']}") + logger.error( + f"Batch processing completed with errors: {batch_result['error_message']}" + ) job.context.add_error("target_analysis", batch_result["error_message"]) else: - logger.info("Batch processing completed successfully with aggregated target analysis") + logger.info( + "Batch processing completed successfully with aggregated target analysis" + ) job.context.add_result( "target_analysis", { "status": "success", "sample_id": sample_id, "analysis_time": batch_result.get("analysis_timestamp", 0), - "targets_found": batch_result.get("coverage_data", {}).get("targets_exceeding_threshold", 0), + "targets_found": batch_result.get("coverage_data", {}).get( + "targets_exceeding_threshold", 0 + ), "processing_steps": batch_result.get("processing_steps", []), "target_data_path": batch_result.get("target_data_path", ""), "target_plot_path": batch_result.get("target_plot_path", ""), @@ -2580,33 +2840,39 @@ def _batch_metadata(i: int) -> dict: "total_files": batch_result.get("total_files", batch_size), }, ) - + # Update master.csv with panel information if sample_id and not batch_result.get("error_message"): try: from robin.analysis.master_csv_manager import MasterCSVManager - + # Update master.csv with panel information csv_manager = MasterCSVManager(batch_work_dir) csv_manager.update_analysis_panel(sample_id, target_panel) - logger.info(f"Updated master.csv with panel '{target_panel}' for sample {sample_id}") - + logger.info( + f"Updated master.csv with panel '{target_panel}' for sample {sample_id}" + ) + except Exception as e: - logger.warning(f"Could not update master.csv with panel info for {sample_id}: {e}") - + logger.warning( + f"Could not update master.csv with panel info for {sample_id}: {e}" + ) + return - + else: # Single file processing (backward compatibility) try: file_path = job.context.filepath logger.info(f"Starting target analysis for: {os.path.basename(file_path)}") - + # Log and validate target panel job_panel = job.context.metadata.get("target_panel", target_panel) if job_panel != target_panel: - logger.warning(f"Panel mismatch: job metadata has '{job_panel}' but handler received '{target_panel}'. Using '{job_panel}' from metadata.") + logger.warning( + f"Panel mismatch: job metadata has '{job_panel}' but handler received '{target_panel}'. Using '{job_panel}' from metadata." + ) target_panel = job_panel logger.info(f"Using target panel: {target_panel}") @@ -2640,22 +2906,22 @@ def _batch_metadata(i: int) -> dict: # Initialize target analysis with staging enabled target_analysis = TargetAnalysis( - work_dir=work_dir, + work_dir=work_dir, target_panel=target_panel, batch_size=10, # Accumulate every 10 files - use_staging=True + use_staging=True, ) - + # Set reference genome if provided if reference: target_analysis.reference = reference - + # Use fast staging-based processing logger.info("Using staging-based processing (fast path)") - target_metadata, should_accumulate = target_analysis.process_file_with_staging( - file_path, file_metadata + target_metadata, should_accumulate = ( + target_analysis.process_file_with_staging(file_path, file_metadata) ) - + # Convert TargetMetadata to dict for storage result = { "sample_id": target_metadata.sample_id, @@ -2665,19 +2931,21 @@ def _batch_metadata(i: int) -> dict: "error_message": target_metadata.error_message, "coverage_data": target_metadata.coverage_data, } - + # Store results in job context job.context.add_metadata("target_analysis", result) - + # Trigger accumulation if threshold reached if should_accumulate: - logger.info("Accumulation threshold reached - running batch accumulation") + logger.info( + "Accumulation threshold reached - running batch accumulation" + ) accumulation_result = target_analysis.accumulate_staged_files( target_metadata.sample_id, force=False ) logger.info(f"Accumulation result: {accumulation_result}") job.context.add_metadata("accumulation_result", accumulation_result) - + # Store flag for potential end-of-queue accumulation job.context.add_metadata("needs_final_accumulation", True) @@ -2685,18 +2953,22 @@ def _batch_metadata(i: int) -> dict: if result.get("sample_id") and not result.get("error_message"): try: from robin.analysis.master_csv_manager import MasterCSVManager - + # Determine work directory if work_dir is None: work_dir = os.path.dirname(file_path) - + # Update master.csv with panel information csv_manager = MasterCSVManager(work_dir) csv_manager.update_analysis_panel(result["sample_id"], target_panel) - logger.info(f"Updated master.csv with panel '{target_panel}' for sample {result['sample_id']}") - + logger.info( + f"Updated master.csv with panel '{target_panel}' for sample {result['sample_id']}" + ) + except Exception as e: - logger.warning(f"Could not update master.csv with panel info for {result.get('sample_id', 'unknown')}: {e}") + logger.warning( + f"Could not update master.csv with panel info for {result.get('sample_id', 'unknown')}: {e}" + ) if result.get("error_message"): job.context.add_error("target_analysis", result["error_message"]) @@ -2720,14 +2992,18 @@ def _batch_metadata(i: int) -> dict: ), }, ) - logger.info(f"Target analysis complete for {os.path.basename(file_path)}") + logger.info( + f"Target analysis complete for {os.path.basename(file_path)}" + ) logger.info(f"Sample ID: {result.get('sample_id', 'unknown')}") logger.info( f"Targets found: {result.get('analysis_results', {}).get('targets_found', 0)}" ) except Exception as e: - error_details = f"Error in target analysis for {job.context.filepath}: {str(e)}" + error_details = ( + f"Error in target analysis for {job.context.filepath}: {str(e)}" + ) job.context.add_error("target_analysis", error_details) logger.error(error_details) @@ -2738,60 +3014,66 @@ def finalize_accumulation_for_sample( """ Force final accumulation of any remaining staged files for a sample and merge all batch BAMs into final target.bam. - + This should be called when: - All files for a sample have been processed - The workflow is completing - There are staged files that haven't been accumulated yet - + Args: sample_id: Sample identifier work_dir: Working directory containing sample data target_panel: Target panel type - + Returns: Dictionary with accumulation results """ logger = logging.getLogger("robin.target") - + try: logger.info(f"Finalizing accumulation for sample {sample_id}") - + sample_output_dir = os.path.join(work_dir, sample_id) - + # Initialize target analysis target_analysis = TargetAnalysis( work_dir=work_dir, target_panel=target_panel, batch_size=1, # Force accumulation regardless of count - use_staging=True + use_staging=True, ) if reference: target_analysis.reference = reference logger.info(f"Finalize path using reference genome: {reference}") - + # Check if there are pending files pending_count = target_analysis._get_pending_count(sample_id) - + if pending_count > 0: - logger.info(f"Found {pending_count} pending files for {sample_id} - forcing accumulation") + logger.info( + f"Found {pending_count} pending files for {sample_id} - forcing accumulation" + ) # Force accumulation of remaining files result = target_analysis.accumulate_staged_files(sample_id, force=True) logger.info(f"Final accumulation complete for {sample_id}: {result}") else: logger.info(f"No pending files for {sample_id} - skipping accumulation") result = {"status": "no_pending_files", "sample_id": sample_id} - + # Now merge all batch BAMs into target.bam logger.info(f"Merging all batch BAMs into final target.bam for {sample_id}") - + target_bam_path = os.path.join(sample_output_dir, "target.bam") - + # Check if target.bam already exists but batch files weren't cleaned up from a previous run if os.path.exists(target_bam_path) and os.path.exists(f"{target_bam_path}.bai"): - existing_batch_bams = sorted(glob.glob(os.path.join(sample_output_dir, "batch_*.bam"))) + existing_batch_bams = sorted( + glob.glob(os.path.join(sample_output_dir, "batch_*.bam")) + ) if existing_batch_bams: - logger.info(f"Found {len(existing_batch_bams)} leftover batch BAM files - cleaning up since target.bam already exists") + logger.info( + f"Found {len(existing_batch_bams)} leftover batch BAM files - cleaning up since target.bam already exists" + ) cleaned_count = 0 for batch_bam in existing_batch_bams: try: @@ -2801,54 +3083,66 @@ def finalize_accumulation_for_sample( if os.path.exists(f"{batch_bam}.bai"): os.remove(f"{batch_bam}.bai") except OSError as e: - logger.warning(f"Could not remove leftover batch BAM {os.path.basename(batch_bam)}: {e}") + logger.warning( + f"Could not remove leftover batch BAM {os.path.basename(batch_bam)}: {e}" + ) logger.info(f"Cleaned up {cleaned_count} leftover batch files") - + # Find all batch BAM files batch_bams = sorted(glob.glob(os.path.join(sample_output_dir, "batch_*.bam"))) - + if batch_bams: logger.info(f"Found {len(batch_bams)} batch BAM files to merge") - + # Create temp merged output temp_merged_bam = os.path.join(sample_output_dir, ".final_merged.bam.tmp") - + # Clean stale temp outputs from a prior interrupted finalize, then force overwrite. if os.path.exists(temp_merged_bam): try: os.remove(temp_merged_bam) except OSError as e: - logger.warning(f"Could not remove stale temp merged BAM {temp_merged_bam}: {e}") + logger.warning( + f"Could not remove stale temp merged BAM {temp_merged_bam}: {e}" + ) if os.path.exists(f"{temp_merged_bam}.bai"): try: os.remove(f"{temp_merged_bam}.bai") except OSError as e: - logger.warning(f"Could not remove stale temp merged BAI {temp_merged_bam}.bai: {e}") + logger.warning( + f"Could not remove stale temp merged BAI {temp_merged_bam}.bai: {e}" + ) # Merge all batch BAMs (force overwrite in case file appears between checks) pysam.merge("-f", "-o", temp_merged_bam, *batch_bams) logger.info("Merged all batch BAMs into temporary file") - + # Index the merged BAM pysam.index(temp_merged_bam) logger.info("Indexed merged target.bam") existing_target_read_count = 0 existing_target_has_reads = False - if os.path.exists(target_bam_path) and os.path.exists(f"{target_bam_path}.bai"): + if os.path.exists(target_bam_path) and os.path.exists( + f"{target_bam_path}.bai" + ): try: with pysam.AlignmentFile(target_bam_path, "rb") as bam_file: existing_target_read_count = bam_file.count(until_eof=True) existing_target_has_reads = existing_target_read_count > 0 except Exception as e: - logger.warning(f"Could not read existing target.bam before replacement: {e}") + logger.warning( + f"Could not read existing target.bam before replacement: {e}" + ) merged_read_count = 0 try: with pysam.AlignmentFile(temp_merged_bam, "rb") as bam_file: merged_read_count = bam_file.count(until_eof=True) except Exception as e: - logger.warning(f"Could not read merged temp BAM before replacement: {e}") + logger.warning( + f"Could not read merged temp BAM before replacement: {e}" + ) replaced_target_bam = False @@ -2865,9 +3159,13 @@ def finalize_accumulation_for_sample( if os.path.exists(f"{temp_merged_bam}.bai"): os.remove(f"{temp_merged_bam}.bai") except OSError as e: - logger.warning(f"Could not clean temp merged BAM after preserve decision: {e}") + logger.warning( + f"Could not clean temp merged BAM after preserve decision: {e}" + ) - target_bam_exists = os.path.exists(target_bam_path) and os.path.exists(f"{target_bam_path}.bai") + target_bam_exists = os.path.exists(target_bam_path) and os.path.exists( + f"{target_bam_path}.bai" + ) result["final_merge"] = "preserved_existing" result["warning"] = ( "Merged BAM was empty; preserved existing target.bam and continued." @@ -2885,7 +3183,9 @@ def finalize_accumulation_for_sample( replaced_target_bam = True # Verify target.bam was created successfully - target_bam_exists = os.path.exists(target_bam_path) and os.path.exists(f"{target_bam_path}.bai") + target_bam_exists = os.path.exists(target_bam_path) and os.path.exists( + f"{target_bam_path}.bai" + ) if target_bam_exists: try: @@ -2896,57 +3196,76 @@ def finalize_accumulation_for_sample( f"Successfully created target.bam with {read_count} reads from {len(batch_bams)} batch files" ) else: - logger.warning("target.bam created but contains no reads") + logger.warning( + "target.bam created but contains no reads" + ) except Exception as e: logger.warning(f"Could not verify target.bam: {e}") else: logger.error("Failed to create target.bam file") result["final_merge"] = "success" if target_bam_exists else "failed" - + # Clean up batch BAM files and their associated BAI files # Only clean up if target.bam was successfully created if target_bam_exists and replaced_target_bam: - logger.info(f"Cleaning up {len(batch_bams)} batch BAM files and their index files after final merge") + logger.info( + f"Cleaning up {len(batch_bams)} batch BAM files and their index files after final merge" + ) cleaned_count = 0 failed_count = 0 - + for batch_bam in batch_bams: try: # Remove the batch BAM file if os.path.exists(batch_bam): os.remove(batch_bam) cleaned_count += 1 - logger.debug(f"Removed batch BAM: {os.path.basename(batch_bam)}") - + logger.debug( + f"Removed batch BAM: {os.path.basename(batch_bam)}" + ) + # Remove the associated BAI file batch_bai = f"{batch_bam}.bai" if os.path.exists(batch_bai): os.remove(batch_bai) - logger.debug(f"Removed batch BAI: {os.path.basename(batch_bai)}") + logger.debug( + f"Removed batch BAI: {os.path.basename(batch_bai)}" + ) except OSError as e: failed_count += 1 - logger.warning(f"Could not remove batch BAM {os.path.basename(batch_bam)}: {e}") - - logger.info(f"Batch cleanup complete: {cleaned_count} batch files removed, {failed_count} failures") - + logger.warning( + f"Could not remove batch BAM {os.path.basename(batch_bam)}: {e}" + ) + + logger.info( + f"Batch cleanup complete: {cleaned_count} batch files removed, {failed_count} failures" + ) + if failed_count > 0: - logger.warning(f"Failed to remove {failed_count} batch file(s) - they may need manual cleanup") + logger.warning( + f"Failed to remove {failed_count} batch file(s) - they may need manual cleanup" + ) elif target_bam_exists: - logger.info("Preserved existing target.bam; keeping batch files for investigation/retry.") + logger.info( + "Preserved existing target.bam; keeping batch files for investigation/retry." + ) else: - logger.warning("Skipping batch cleanup - target.bam was not successfully created") - + logger.warning( + "Skipping batch cleanup - target.bam was not successfully created" + ) + logger.info(f"Final merge complete for {sample_id}") result["batch_files_merged"] = len(batch_bams) else: logger.info(f"No batch BAM files found for {sample_id}") result["final_merge"] = "no_batch_files" - + return result - + except Exception as e: logger.error(f"Error during final accumulation for {sample_id}: {e}") import traceback + logger.error(traceback.format_exc()) return {"status": "error", "error": str(e), "sample_id": sample_id} @@ -2995,6 +3314,7 @@ def target_bam_finalize_handler(job, work_dir: Optional[str] = None) -> None: if not target_panel and sample_dir and os.path.isdir(sample_dir): try: import csv + master_csv = os.path.join(sample_dir, "master.csv") if os.path.exists(master_csv): with open(master_csv, "r", newline="") as fh: @@ -3015,11 +3335,16 @@ def target_bam_finalize_handler(job, work_dir: Optional[str] = None) -> None: ) result = finalize_accumulation_for_sample( - sample_id=sample_id, work_dir=base, target_panel=target_panel, reference=reference + sample_id=sample_id, + work_dir=base, + target_panel=target_panel, + reference=reference, ) if result.get("status") == "error": - job.context.add_error("target_bam_finalize", result.get("error", "Unknown error")) + job.context.add_error( + "target_bam_finalize", result.get("error", "Unknown error") + ) else: job.context.add_result("target_bam_finalize", result) @@ -3231,7 +3556,9 @@ def run_snp_analysis( ] if annotation_only: - logger.info("Annotation-only mode enabled; skipping existing output shortcut.") + logger.info( + "Annotation-only mode enabled; skipping existing output shortcut." + ) elif not force_regenerate and all(os.path.exists(f) for f in snp_output_files): logger.info(f"SNP analysis already present in {clair_dir}") logger.info("Rebuilding SNP display JSON from existing snpsift output.") @@ -3249,7 +3576,9 @@ def run_snp_analysis( provenance = load_sample_clinvar_provenance(sample_dir) summary = snp_display.setdefault("summary", {}) - summary["clinvar_release"] = provenance.get("file_date") or "" + summary["clinvar_release"] = ( + provenance.get("file_date") or "" + ) summary["clinvar_label"] = format_clinvar_version_label( provenance ) @@ -3257,9 +3586,7 @@ def run_snp_analysis( pass with snp_display_path.open("w", encoding="utf-8") as f_out: json.dump(snp_display, f_out) - logger.info( - f"SNP display data refreshed at {snp_display_path}" - ) + logger.info(f"SNP display data refreshed at {snp_display_path}") else: logger.warning( "Could not regenerate SNP display data from existing VCF." @@ -3274,7 +3601,9 @@ def run_snp_analysis( logger.info("STEP 3: Checking for required input files") target_bam = os.path.join(sample_dir, "target.bam") - threshold_targets_bed = os.path.join(sample_dir, "targets_exceeding_threshold.bed") + threshold_targets_bed = os.path.join( + sample_dir, "targets_exceeding_threshold.bed" + ) targets_bed = threshold_targets_bed output_snv_vcf = os.path.join(clair_dir, "output_done.vcf.gz") output_indel_vcf = os.path.join(clair_dir, "output_indel_done.vcf.gz") @@ -3301,7 +3630,11 @@ def resolve_full_targets_bed() -> str: except Exception: pass panel_paths.extend( - [bed_filename, f"data/{bed_filename}", f"/usr/local/share/{bed_filename}"] + [ + bed_filename, + f"data/{bed_filename}", + f"/usr/local/share/{bed_filename}", + ] ) for path in panel_paths: if os.path.exists(path): @@ -3332,7 +3665,9 @@ def resolve_full_targets_bed() -> str: if annotation_only: missing_annotation_inputs = [ - path for path in [output_snv_vcf, output_indel_vcf] if not os.path.exists(path) + path + for path in [output_snv_vcf, output_indel_vcf] + if not os.path.exists(path) ] if missing_annotation_inputs: for missing in missing_annotation_inputs: @@ -3385,7 +3720,9 @@ def resolve_full_targets_bed() -> str: logger.info(f"Targets BED: {targets_bed}") if annotation_only: - logger.info("Annotation-only mode enabled; skipping Clair3 variant calling.") + logger.info( + "Annotation-only mode enabled; skipping Clair3 variant calling." + ) sorted_bam = os.path.join(clair_dir, "sorted_targets_exceeding.bam") else: logger.info("STEP 4: Sorting target BAM for Clair3") @@ -3521,14 +3858,12 @@ def resolve_full_targets_bed() -> str: # Verify the specific files exist in their directories logger.info("Verifying input files in volume directories...") - + if not os.path.exists(sorted_bam): logger.error(f"Sorted BAM file not found in directory: {sorted_bam}") return "" if not os.path.exists(targets_bed): - logger.error( - f"Targets BED file not found in directory: {targets_bed}" - ) + logger.error(f"Targets BED file not found in directory: {targets_bed}") return "" if not os.path.exists(reference): logger.error(f"Reference file not found in directory: {reference}") @@ -3586,7 +3921,9 @@ def split_bed_into_chunks(bed_file, max_chunk_size=150000000): def covered_span(chrom_bounds): # Sum per-chromosome spans so sparse targets cannot inflate a chunk indefinitely. - return sum((end - start) for start, end in chrom_bounds.values()) + return sum( + (end - start) for start, end in chrom_bounds.values() + ) current_chunk = [] current_bounds = {} @@ -3595,18 +3932,27 @@ def covered_span(chrom_bounds): projected_bounds = dict(current_bounds) if chrom in projected_bounds: prev_start, prev_end = projected_bounds[chrom] - projected_bounds[chrom] = (min(prev_start, start), max(prev_end, end)) + projected_bounds[chrom] = ( + min(prev_start, start), + max(prev_end, end), + ) else: projected_bounds[chrom] = (start, end) - if current_chunk and covered_span(projected_bounds) > max_chunk_size: + if ( + current_chunk + and covered_span(projected_bounds) > max_chunk_size + ): chunks.append(current_chunk) current_chunk = [] current_bounds = {} if chrom in current_bounds: prev_start, prev_end = current_bounds[chrom] - current_bounds[chrom] = (min(prev_start, start), max(prev_end, end)) + current_bounds[chrom] = ( + min(prev_start, start), + max(prev_end, end), + ) else: current_bounds[chrom] = (start, end) current_chunk.append((chrom, start, end, raw_line, entry_len)) @@ -3628,13 +3974,20 @@ def covered_span(chrom_bounds): if split_regions_env is None: use_split_regions = True else: - use_split_regions = split_regions_env.lower() in ("1", "true", "yes", "on") + use_split_regions = split_regions_env.lower() in ( + "1", + "true", + "yes", + "on", + ) if use_split_regions: logger.info( "Using region-split ClairS mode (recommended for INDEL memory stability)." ) - chunked_entries = split_bed_into_chunks(targets_bed, max_chunk_size=250000000) + chunked_entries = split_bed_into_chunks( + targets_bed, max_chunk_size=250000000 + ) if not chunked_entries: logger.error("Failed to split BED file into chunks") return "" @@ -3654,13 +4007,20 @@ def covered_span(chrom_bounds): for chrom, start, end, _, _ in chunk: if chrom in chrom_bounds: prev_start, prev_end = chrom_bounds[chrom] - chrom_bounds[chrom] = (min(prev_start, start), max(prev_end, end)) + chrom_bounds[chrom] = ( + min(prev_start, start), + max(prev_end, end), + ) else: chrom_bounds[chrom] = (start, end) - span_bases = sum((end - start) for start, end in chrom_bounds.values()) + span_bases = sum( + (end - start) for start, end in chrom_bounds.values() + ) label = f"{first_chrom}:{first_start+1}-{last_end}" if first_chrom != last_chrom: - label = f"{first_chrom}:{first_start+1}..{last_chrom}:{last_end}" + label = ( + f"{first_chrom}:{first_start+1}..{last_chrom}:{last_end}" + ) regions.append( { @@ -3674,9 +4034,7 @@ def covered_span(chrom_bounds): f"Chunk {i}: {label} (entries: {len(chunk)}, span: {span_bases:,} bases)" ) else: - logger.info( - "Running ClairS in single-pass mode over full BED." - ) + logger.info("Running ClairS in single-pass mode over full BED.") logger.info( "Set ROBIN_CLAIRS_SPLIT_REGIONS=1 to re-enable region-splitting mode." ) @@ -3821,7 +4179,9 @@ def merge_vcf_files(vcf_files, output_file, variant_type): oom_killed = False kill_reason = None try: - inspect_data = client.api.inspect_container(container=container_id) + inspect_data = client.api.inspect_container( + container=container_id + ) state = inspect_data.get("State", {}) if inspect_data else {} oom_killed = bool(state.get("OOMKilled", False)) except Exception as inspect_exc: @@ -3927,7 +4287,9 @@ def merge_vcf_files(vcf_files, output_file, variant_type): logger.error(f"Single-pass SNV output not found: {single_snv}") return "" shutil.copy2(single_snv, f"{clair_dir}/output_done.vcf.gz") - logger.info(f"Single-pass SNV output copied to: {clair_dir}/output_done.vcf.gz") + logger.info( + f"Single-pass SNV output copied to: {clair_dir}/output_done.vcf.gz" + ) if os.path.exists(single_indel): shutil.copy2(single_indel, f"{clair_dir}/output_indel_done.vcf.gz") @@ -3935,12 +4297,18 @@ def merge_vcf_files(vcf_files, output_file, variant_type): f"Single-pass INDEL output copied to: {clair_dir}/output_indel_done.vcf.gz" ) else: - logger.warning(f"Single-pass INDEL output not found: {single_indel}") - logger.info("Clair3 pipeline completed successfully in single-pass mode") + logger.warning( + f"Single-pass INDEL output not found: {single_indel}" + ) + logger.info( + "Clair3 pipeline completed successfully in single-pass mode" + ) try: if chown_tree_to_host_user(Path(clair_dir)): - logger.info("Clair3 output ownership normalized under %s", clair_dir) + logger.info( + "Clair3 output ownership normalized under %s", clair_dir + ) else: logger.warning( "Clair3 outputs under %s may include root-owned files " @@ -3955,7 +4323,6 @@ def merge_vcf_files(vcf_files, output_file, variant_type): chown_exc, ) - if annotation_only: logger.info( "Proceeding directly to annotation pipeline using existing Clair3 outputs." @@ -3993,9 +4360,7 @@ def merge_vcf_files(vcf_files, output_file, variant_type): except Exception as e: logger.warning(f"Could not read input VCF file: {e}") else: - logger.error( - f"Input VCF file does not exist: {output_snv_vcf}" - ) + logger.error(f"Input VCF file does not exist: {output_snv_vcf}") snpeff_cmd = ["snpEff"] snpeff_cmd.append("-v" if annotation_verbose else "-q") @@ -4082,9 +4447,7 @@ def merge_vcf_files(vcf_files, output_file, variant_type): try: if clinvar_db_path: clinvar_size = os.path.getsize(clinvar_db_path) - logger.info( - f"ClinVar DB found, size: {clinvar_size} bytes" - ) + logger.info(f"ClinVar DB found, size: {clinvar_size} bytes") try: from robin.utils.clinvar_manager import ( format_clinvar_version_label, @@ -4194,9 +4557,7 @@ def merge_vcf_files(vcf_files, output_file, variant_type): except Exception as e: logger.warning(f"Could not read INDEL input VCF file: {e}") else: - logger.error( - f"INDEL input VCF file does not exist: {output_indel_vcf}" - ) + logger.error(f"INDEL input VCF file does not exist: {output_indel_vcf}") snpeff_indel_cmd = ["snpEff"] snpeff_indel_cmd.append("-v" if annotation_verbose else "-q") @@ -4388,7 +4749,9 @@ def basic_vcf_to_csv(vcf_file, csv_file): # Build pre-formatted SNP display data for the GUI try: snp_display_path = Path(clair_dir) / "snpsift_output_display.json" - snp_display = build_snp_display_data(Path(clair_dir) / "snpsift_output.vcf") + snp_display = build_snp_display_data( + Path(clair_dir) / "snpsift_output.vcf" + ) if snp_display is not None: try: from robin.utils.clinvar_manager import ( @@ -4399,7 +4762,9 @@ def basic_vcf_to_csv(vcf_file, csv_file): provenance = load_sample_clinvar_provenance(sample_dir) summary = snp_display.setdefault("summary", {}) summary["clinvar_release"] = provenance.get("file_date") or "" - summary["clinvar_label"] = format_clinvar_version_label(provenance) + summary["clinvar_label"] = format_clinvar_version_label( + provenance + ) except Exception: pass with snp_display_path.open("w", encoding="utf-8") as f_out: diff --git a/src/robin/analysis/temp_utilities.py b/src/robin/analysis/temp_utilities.py index 3dbcf201..5cc29ca6 100644 --- a/src/robin/analysis/temp_utilities.py +++ b/src/robin/analysis/temp_utilities.py @@ -5,18 +5,19 @@ This module provides utilities for merging modkit files and creating parquet files. """ -import warnings -from typing import List, Optional import gc -import os +import json import logging +import os import pickle +import tempfile +import warnings +from contextlib import contextmanager from datetime import datetime +from typing import List, Optional + import pandas as pd import polars as pl -from contextlib import contextmanager -import tempfile -import json # Suppress pkg_resources deprecation warnings from sorted_nearest warnings.filterwarnings( @@ -29,12 +30,14 @@ try: import pyranges as pr - from robin.analysis.utilities.mnp_flex import APIClient as MnpFlexClient #ToDo: Maintain to future integration. + + from robin.analysis.utilities.mnp_flex import ( + APIClient as MnpFlexClient, # ToDo: Maintain to future integration. + ) except ImportError as e: logging.warning(f"Some dependencies not available: {e}") - # Simple cross-process file lock using POSIX flock when available (no-op on unsupported platforms) try: import fcntl # type: ignore @@ -143,7 +146,6 @@ def merge_modkit_files( f"Total cumulative BAM files contributing to parquet: {cumulative_bam_file_count} (added {num_bam_files_seen} new files)" ) - # Cache or build PyRanges filter with improved caching # Use distinct cache for .txt (1-based converted) vs .gz (0-based) to avoid stale data cache_suffix = "_1based" if filter_bed_file.endswith(".txt") else "" @@ -176,7 +178,9 @@ def merge_modkit_files( rename[c] = "End" bed_df = bed_df.rename(columns=rename) bed_df["Start"] = bed_df["Start"].astype(int) - 1 # 1-based -> 0-based - bed_df["End"] = bed_df["End"].astype(int) # 1-based end inclusive -> 0-based exclusive + bed_df["End"] = bed_df["End"].astype( + int + ) # 1-based end inclusive -> 0-based exclusive else: bed_df = pd.read_csv( filter_bed_file, @@ -245,7 +249,10 @@ def merge_modkit_files( infer_schema_length=0, ).select(essential_cols) pl_df = pl_df.with_columns( - [pl.col(c).cast(pl.UInt32, strict=False) for c in unsigned_int_cols] + [ + pl.col(c).cast(pl.UInt32, strict=False) + for c in unsigned_int_cols + ] + [pl.col(c).cast(pl.Float32, strict=False) for c in float_cols] ) @@ -254,9 +261,9 @@ def merge_modkit_files( continue # Build PyRanges for intersection (only need chrom/start/end) - pr_df = pl_df.rename({"chrom": "Chromosome", "chromStart": "Start"}).with_columns( - (pl.col("Start") + 1).alias("End") - ) + pr_df = pl_df.rename( + {"chrom": "Chromosome", "chromStart": "Start"} + ).with_columns((pl.col("Start") + 1).alias("End")) gr = pr.PyRanges(pr_df.to_pandas()[["Chromosome", "Start", "End"]]) inter = gr.intersect(filter_ranges).df diff --git a/src/robin/analysis/tucan_analysis.py b/src/robin/analysis/tucan_analysis.py index 407075ed..33703d93 100644 --- a/src/robin/analysis/tucan_analysis.py +++ b/src/robin/analysis/tucan_analysis.py @@ -117,9 +117,7 @@ def append_tucan_scores( # Prefer timestamp / coverage meta first, then class columns. cols = list(rows.columns) ordered = [ - c - for c in ("timestamp", "covered_cpgs", "number_probes", "probes") - if c in cols + c for c in ("timestamp", "covered_cpgs", "number_probes", "probes") if c in cols ] ordered.extend(c for c in cols if c not in ordered) new_df = rows[ordered] @@ -137,7 +135,9 @@ def append_tucan_scores( combined.to_csv(scores_path, index=False) -def _top_prediction(prediction_df: pd.DataFrame) -> tuple[Optional[str], Optional[float], int]: +def _top_prediction( + prediction_df: pd.DataFrame, +) -> tuple[Optional[str], Optional[float], int]: """Return (top_class, top_score, covered_cpgs) from a Tucan output frame.""" if prediction_df is None or prediction_df.empty: return None, None, 0 @@ -215,9 +215,7 @@ def __init__( self.bambatch: Dict[str, int] = {} self._assets: Optional[Dict[str, Any]] = None - logger.info( - "Tucan Analysis initialized (probe_margin=%s)", self.probe_margin - ) + logger.info("Tucan Analysis initialized (probe_margin=%s)", self.probe_margin) def _ensure_assets(self) -> Dict[str, Any]: if self._assets is not None: @@ -229,9 +227,7 @@ def _ensure_assets(self) -> Dict[str, Any]: ) return self._assets - def process_parquet_file( - self, parquet_path: str, sample_id: str - ) -> TucanMetadata: + def process_parquet_file(self, parquet_path: str, sample_id: str) -> TucanMetadata: start_time = time.time() if sample_id not in self.bambatch: @@ -320,9 +316,9 @@ def process_parquet_file( "top_score": top_score, "scores_file": scores_path, "bed_file": bed_path, - "num_cpgs": self.num_cpgs - if self.num_cpgs is not None - else DEFAULT_NUM_CPGS, + "num_cpgs": ( + self.num_cpgs if self.num_cpgs is not None else DEFAULT_NUM_CPGS + ), "probe_margin": self.probe_margin, "processing_steps": result.processing_steps.copy(), } @@ -419,7 +415,9 @@ def process_multiple_files( pass if analysis_result["files_processed"] == 0: - analysis_result["error_message"] = "No files could be processed successfully" + analysis_result["error_message"] = ( + "No files could be processed successfully" + ) analysis_result["processing_steps"].append("no_files_processed") return analysis_result @@ -473,9 +471,13 @@ def tucan_handler(job, work_dir=None): ) if not parquet_paths: - error_msg = "No parquet paths found from bed conversion results in batch" + error_msg = ( + "No parquet paths found from bed conversion results in batch" + ) if suppress_expected: - logger.warning("%s (expected for fail-only BAM submission)", error_msg) + logger.warning( + "%s (expected for fail-only BAM submission)", error_msg + ) job.context.add_result( "tucan_analysis", {"status": "expected_failure", "reason": error_msg}, @@ -560,7 +562,10 @@ def tucan_handler(job, work_dir=None): if suppress_expected: job.context.add_result( "tucan_analysis", - {"status": "expected_failure", "error_message": result.error_message}, + { + "status": "expected_failure", + "error_message": result.error_message, + }, ) else: job.context.add_error("tucan_analysis", result.error_message) @@ -596,7 +601,9 @@ def tucan_handler(job, work_dir=None): except Exception as exc: if suppress_expected: - logger.warning("Expected Tucan failure for fail-only BAM submission: %s", exc) + logger.warning( + "Expected Tucan failure for fail-only BAM submission: %s", exc + ) job.context.add_result( "tucan_analysis", {"status": "expected_failure", "error_message": str(exc)}, diff --git a/src/robin/analysis/utilities/ReadBam.py b/src/robin/analysis/utilities/ReadBam.py index f8c5acbf..157eb52e 100644 --- a/src/robin/analysis/utilities/ReadBam.py +++ b/src/robin/analysis/utilities/ReadBam.py @@ -1,17 +1,20 @@ """BAM read and RG tag extraction. Requires Python 3.12+.""" + from __future__ import annotations import sys + if sys.version_info < (3, 12): raise RuntimeError("robin ReadBam utilities require Python 3.12 or newer") -import pysam -import os -from typing import Optional, Tuple, Dict, Any, Generator, Set -from dataclasses import dataclass, field, asdict import logging -from dateutil import parser +import os import re +from dataclasses import asdict, dataclass, field +from typing import Any, Dict, Generator, Optional, Set, Tuple + +import pysam +from dateutil import parser # Create a logger for this module logger = logging.getLogger(__name__) diff --git a/src/robin/analysis/utilities/matkit.py b/src/robin/analysis/utilities/matkit.py index bd401be9..b0a4f1f7 100644 --- a/src/robin/analysis/utilities/matkit.py +++ b/src/robin/analysis/utilities/matkit.py @@ -1,35 +1,38 @@ """ Modkit/matkit utilities for BAM methylation. Requires Python 3.12+. """ + from __future__ import annotations -import warnings import sys +import warnings + if sys.version_info < (3, 12): raise RuntimeError("robin matkit utilities require Python 3.12 or newer") -from typing import List, Optional import bisect import gc -import os +import json import logging +import os +import pickle import subprocess import time -import pickle from datetime import datetime from pathlib import Path +from typing import List, Optional + import pandas as pd import polars as pl import pyarrow as pa import pyarrow.parquet as pq - import pyranges as pr import pysam from alive_progress import alive_bar -from robin.analysis.utilities.ReadBam import ReadBam -from robin.analysis.utilities.mnp_flex import APIClient as MnpFlexClient + from robin import resources -import json +from robin.analysis.utilities.mnp_flex import APIClient as MnpFlexClient +from robin.analysis.utilities.ReadBam import ReadBam # Suppress pkg_resources deprecation warnings from sorted_nearest warnings.filterwarnings( @@ -54,16 +57,18 @@ # Per-BAM parquet schema: string columns stored as binary to avoid UTF-8 validation on read _PARQUET_STR_COLS = ("chrom", "mod_code", "strand") -PARQUET_SCHEMA_BINARY = pa.schema([ - ("chrom", pa.binary()), - ("chromStart", pa.int64()), - ("mod_code", pa.binary()), - ("strand", pa.binary()), - ("valid_cov", pa.uint32()), - ("percent_modified", pa.float32()), - ("n_mod", pa.uint32()), - ("n_canonical", pa.uint32()), -]) +PARQUET_SCHEMA_BINARY = pa.schema( + [ + ("chrom", pa.binary()), + ("chromStart", pa.int64()), + ("mod_code", pa.binary()), + ("strand", pa.binary()), + ("valid_cov", pa.uint32()), + ("percent_modified", pa.float32()), + ("n_mod", pa.uint32()), + ("n_canonical", pa.uint32()), + ] +) # Minimum primary alignment QS (BAM tag "qs") to include a read in methylation analysis. # Same rule as fusion_work: only process alignments for reads whose primary has qs >= this. @@ -150,39 +155,39 @@ def _read_parquet_robust(path: str, columns: list[str]): def _ensure_fasta_index(ref_fasta: str) -> None: """ Ensure the reference FASTA file has an index (.fai file). - + This function checks if the FASTA file has a corresponding .fai index file. If the index is missing or older than the FASTA file, it creates/updates it using pysam.faidx. - + Args: ref_fasta: Path to the reference FASTA file - + Raises: FileNotFoundError: If the reference FASTA file doesn't exist RuntimeError: If the index creation fails """ if not ref_fasta or not os.path.exists(ref_fasta): raise FileNotFoundError(f"Reference FASTA file not found: {ref_fasta}") - + fai_file = f"{ref_fasta}.fai" - + # Check if index exists and is up-to-date if os.path.exists(fai_file): # Check if index is newer than the FASTA file fai_mtime = os.path.getmtime(fai_file) fa_mtime = os.path.getmtime(ref_fasta) - + if fai_mtime >= fa_mtime: # Index exists and is up-to-date, no action needed return - + # Create or update the index using pysam print(f"Creating FASTA index for {ref_fasta}") try: # pysam.faidx creates the .fai index file pysam.faidx(ref_fasta) - + # Verify the index was created if not os.path.exists(fai_file): error_msg = ( @@ -191,9 +196,9 @@ def _ensure_fasta_index(ref_fasta: str) -> None: ) print(f"ERROR: {error_msg}") raise RuntimeError(error_msg) - + print(f"Successfully created FASTA index: {fai_file}") - + except Exception as e: error_msg = ( f"Failed to create FASTA index for {ref_fasta}. " @@ -257,7 +262,9 @@ def merge_modkit_files( if os.path.exists(existing_file): try: # Read existing metadata - metadata_file = existing_file.removesuffix(".parquet") + "_metadata.json" + metadata_file = ( + existing_file.removesuffix(".parquet") + "_metadata.json" + ) if os.path.exists(metadata_file): with open(metadata_file, "r") as f: metadata = json.load(f) @@ -300,10 +307,16 @@ def merge_modkit_files( ) # Map to expected column names (handle chr/start/end) – dict comp inlined in 3.12 col_map = {"chr": "Chromosome", "start": "Start", "end": "End"} - rename = {c: col_map[c.lower()] for c in bed_df.columns if c.lower() in col_map} + rename = { + c: col_map[c.lower()] + for c in bed_df.columns + if c.lower() in col_map + } bed_df = bed_df.rename(columns=rename) bed_df["Start"] = bed_df["Start"].astype(int) - 1 # 1-based -> 0-based - bed_df["End"] = bed_df["End"].astype(int) # 1-based end inclusive -> 0-based exclusive + bed_df["End"] = bed_df["End"].astype( + int + ) # 1-based end inclusive -> 0-based exclusive else: bed_df = pd.read_csv( filter_bed_file, @@ -351,7 +364,9 @@ def merge_modkit_files( sep=r"\s+", header=None, names=full_cols, - dtype={c: str for c in ["chrom", "mod_code", "strand", "color"]}, + dtype={ + c: str for c in ["chrom", "mod_code", "strand", "color"] + }, ) missing_cols = set(full_cols) - set(df.columns) if missing_cols: @@ -405,7 +420,9 @@ def merge_modkit_files( if c in pl_df.columns: pl_df = pl_df.with_columns(pl.col(c).cast(pl.UInt32, strict=False)) if "percent_modified" in pl_df.columns: - pl_df = pl_df.with_columns(pl.col("percent_modified").cast(pl.Float32, strict=False)) + pl_df = pl_df.with_columns( + pl.col("percent_modified").cast(pl.Float32, strict=False) + ) pl_df.write_parquet(output_file) # Save metadata with cumulative BAM file count @@ -437,7 +454,9 @@ def merge_modkit_files( if c in pl_df.columns: pl_df = pl_df.with_columns(pl.col(c).cast(pl.UInt32, strict=False)) if "percent_modified" in pl_df.columns: - pl_df = pl_df.with_columns(pl.col("percent_modified").cast(pl.Float32, strict=False)) + pl_df = pl_df.with_columns( + pl.col("percent_modified").cast(pl.Float32, strict=False) + ) pl_df.write_parquet(output_file) metadata = { "bam_file_count": cumulative_bam_file_count, @@ -488,14 +507,22 @@ def merge_modkit_files( # Cast count/float columns to canonical types so concat never sees Int64 vs UInt32. for c in ["valid_cov", "n_mod", "n_canonical"]: if c in existing_df.columns: - existing_df = existing_df.with_columns(pl.col(c).cast(pl.UInt32, strict=False)) + existing_df = existing_df.with_columns( + pl.col(c).cast(pl.UInt32, strict=False) + ) if c in pl_new_df.columns: - pl_new_df = pl_new_df.with_columns(pl.col(c).cast(pl.UInt32, strict=False)) + pl_new_df = pl_new_df.with_columns( + pl.col(c).cast(pl.UInt32, strict=False) + ) for c in ["percent_modified"]: if c in existing_df.columns: - existing_df = existing_df.with_columns(pl.col(c).cast(pl.Float32, strict=False)) + existing_df = existing_df.with_columns( + pl.col(c).cast(pl.Float32, strict=False) + ) if c in pl_new_df.columns: - pl_new_df = pl_new_df.with_columns(pl.col(c).cast(pl.Float32, strict=False)) + pl_new_df = pl_new_df.with_columns( + pl.col(c).cast(pl.Float32, strict=False) + ) # Combine existing and new data combined = pl.concat([existing_df, pl_new_df]) @@ -690,9 +717,7 @@ def merge_modkit_files( logging.info("Continuing with processing despite MNP-FLEX error") # Don't re-raise the exception - allow processing to continue - logging.debug( - f"Merged with optimized Polars and cache saved to: {output_file}" - ) + logging.debug(f"Merged with optimized Polars and cache saved to: {output_file}") except Exception as e: logging.error(f"Error in merge_modkit_files: {str(e)}") @@ -814,7 +839,10 @@ def cpg_cytosine_site( try: if ref_fasta_obj.fetch(chrom, refpos, refpos + 2).upper() == "CG": return refpos, PLUS_STRAND - if refpos >= 1 and ref_fasta_obj.fetch(chrom, refpos - 1, refpos + 1).upper() == "CG": + if ( + refpos >= 1 + and ref_fasta_obj.fetch(chrom, refpos - 1, refpos + 1).upper() == "CG" + ): return refpos, MINUS_STRAND except ValueError: return None @@ -1527,7 +1555,6 @@ def process_bam_counts_improved( # This ensures we only output sites that modkit would output mod_sites = set() - # Load reference genome if provided for validation. Reuse cached handle when the same # path is used across calls (e.g. one ref for many BAMs) to avoid re-indexing/re-opening. ref_fasta_obj = None @@ -1601,7 +1628,9 @@ def process_bam_counts_improved( # Collect per-site max probability per mod_code for this read. # Avoids allocating lists of probs when we only ever use max(probs). - read_sites: dict[tuple[int, str], dict[str, int]] = {} # (refpos, strand) -> {mod_code: max_prob_255} + read_sites: dict[tuple[int, str], dict[str, int]] = ( + {} + ) # (refpos, strand) -> {mod_code: max_prob_255} seen_sites: set[tuple[int, str]] = set() ref_map_get = ref_map.get read_sites_get = read_sites.get @@ -1694,7 +1723,9 @@ def process_bam_counts_improved( if local_max_prob_h > c[COUNT_IDX_MAX_PROB_H]: c[COUNT_IDX_MAX_PROB_H] = local_max_prob_h - max_prob = max(canonical_prob_255, local_max_prob_m, local_max_prob_h) + max_prob = max( + canonical_prob_255, local_max_prob_m, local_max_prob_h + ) if max_prob >= thresh: if canonical_prob_255 == max_prob: @@ -1722,7 +1753,9 @@ def process_bam_counts_improved( if mod_probs is not None: for mod_code, mod_prob in mod_probs.items(): if mod_code in ["C", "m", "h"]: - debug_data[debug_key]["probs"][mod_code].append(mod_prob) + debug_data[debug_key]["probs"][mod_code].append( + mod_prob + ) debug_data[debug_key]["classification"] = classification else: @@ -1812,7 +1845,6 @@ def process_bam_counts_improved( else: c[COUNT_IDX_FAIL] += 1 # Failed call (0 < prob < threshold) - bam.close() # Only close the ref handle when we opened it in this call and it is not in the cache. if ref_fasta_obj and not ref_fasta_cached: diff --git a/src/robin/analysis/utilities/merge_bedmethyl.py b/src/robin/analysis/utilities/merge_bedmethyl.py index daddff0c..71cffbf7 100644 --- a/src/robin/analysis/utilities/merge_bedmethyl.py +++ b/src/robin/analysis/utilities/merge_bedmethyl.py @@ -1,21 +1,23 @@ """ Helper functions to sort and merge bedmethyl files. Requires Python 3.12+. """ + from __future__ import annotations import sys + if sys.version_info < (3, 12): raise RuntimeError("robin merge_bedmethyl utilities require Python 3.12 or newer") -import pandas as pd import csv +import gc import logging -from typing import List, Dict, Optional import os -import gc from copy import deepcopy -import numpy as np +from typing import Dict, List, Optional +import numpy as np +import pandas as pd # Sturgeon-related imports (must be installed) from sturgeon.utils import read_probes_file @@ -233,8 +235,10 @@ def modkit_pileup_file_to_bed( "strand", ] # Check if all expected columns exist in the DataFrame - has_expected_cols = all(col in modkit_df.columns for col in expected_columns) - + has_expected_cols = all( + col in modkit_df.columns for col in expected_columns + ) + if has_expected_cols: # Data has the essential columns, just filter and select them modkit_df = modkit_df[expected_columns].copy() @@ -319,19 +323,19 @@ def modkit_pileup_file_to_bed( # Load probes file # The probes file has a header and uses whitespace (spaces) as delimiter - probes_df = pd.read_csv(probes_file, sep=r'\s+', header=0) - + probes_df = pd.read_csv(probes_file, sep=r"\s+", header=0) + # Rename columns to expected names if needed - if 'ID_REF' in probes_df.columns: - probes_df = probes_df.rename(columns={'ID_REF': 'probe_name'}) - + if "ID_REF" in probes_df.columns: + probes_df = probes_df.rename(columns={"ID_REF": "probe_name"}) + # Keep only the columns we need - probes_df = probes_df[['chr', 'start', 'end', 'probe_name']].copy() + probes_df = probes_df[["chr", "start", "end", "probe_name"]].copy() # Ensure chromosome names match probes_df["chr"] = probes_df["chr"].astype(str).str.removeprefix("chr") modkit_df["chr"] = modkit_df["chr"].astype(str).str.removeprefix("chr") - + # Get unique chromosomes chromosomes = np.unique(probes_df["chr"].astype(str)) @@ -364,8 +368,10 @@ def modkit_pileup_file_to_bed( ) # Rename 'probe_name' to 'probe_id' for consistency - if 'probe_name' in calls_per_probe_chr.columns: - calls_per_probe_chr = calls_per_probe_chr.rename(columns={'probe_name': 'probe_id'}) + if "probe_name" in calls_per_probe_chr.columns: + calls_per_probe_chr = calls_per_probe_chr.rename( + columns={"probe_name": "probe_id"} + ) calls_per_probe.append(calls_per_probe_chr) @@ -411,10 +417,10 @@ def modkit_pileup_file_to_bed( # Store result for return result_df = calls_per_probe.copy() - + # Rename 'probe_name' to 'probe_id' for Sturgeon compatibility - if 'probe_name' in result_df.columns: - result_df = result_df.rename(columns={'probe_name': 'probe_id'}) + if "probe_name" in result_df.columns: + result_df = result_df.rename(columns={"probe_name": "probe_id"}) finally: # Clean up large DataFrames that are no longer needed diff --git a/src/robin/analysis/utilities/mnp_flex.py b/src/robin/analysis/utilities/mnp_flex.py index 9ea6be22..db1cb8eb 100644 --- a/src/robin/analysis/utilities/mnp_flex.py +++ b/src/robin/analysis/utilities/mnp_flex.py @@ -1,7 +1,8 @@ -import requests -from typing import Any, Dict, Tuple, List import logging import os +from typing import Any, Dict, List, Tuple + +import requests class APIClient: diff --git a/src/robin/analysis/utilities/vcf_chromosomes.py b/src/robin/analysis/utilities/vcf_chromosomes.py index 4062c680..b95cc8f0 100644 --- a/src/robin/analysis/utilities/vcf_chromosomes.py +++ b/src/robin/analysis/utilities/vcf_chromosomes.py @@ -70,9 +70,10 @@ def rewrite_vcf_chromosomes( dst_path.parent.mkdir(parents=True, exist_ok=True) variant_count = 0 - with src_path.open("r", encoding="utf-8", errors="replace") as fin, dst_path.open( - "w", encoding="utf-8" - ) as fout: + with ( + src_path.open("r", encoding="utf-8", errors="replace") as fin, + dst_path.open("w", encoding="utf-8") as fout, + ): variant_count = _stream_rewrite_vcf(fin, fout, transform) return variant_count diff --git a/src/robin/analysis/variant_classification.py b/src/robin/analysis/variant_classification.py index 863e3586..16eb95b0 100644 --- a/src/robin/analysis/variant_classification.py +++ b/src/robin/analysis/variant_classification.py @@ -27,7 +27,6 @@ from dataclasses import dataclass from typing import Any, Iterable, Mapping, Optional - PATHOGENIC_TERMS: tuple[str, ...] = ( "pathogenic/likely_pathogenic", "pathogenic/likely pathogenic", diff --git a/src/robin/classification_config.py b/src/robin/classification_config.py index 4558cfcd..1ff151d3 100644 --- a/src/robin/classification_config.py +++ b/src/robin/classification_config.py @@ -5,7 +5,7 @@ and CNV classification rules used across the robin application (GUI, reporting, etc.). """ -from typing import Dict, Tuple, Any +from typing import Any, Dict, Tuple # Confidence thresholds for different classifiers CLASSIFIER_CONFIDENCE_THRESHOLDS: Dict[str, Dict[str, float]] = { @@ -55,21 +55,22 @@ "low": 20.0, } + def get_confidence_level(classifier: str, confidence: float) -> str: """ Get confidence level based on classifier-specific thresholds. - + Args: classifier: Name of the classifier (e.g., 'sturgeon', 'nanodx') confidence: Confidence value as percentage (0-100) - + Returns: Confidence level string ('High confidence', 'Medium confidence', etc.) """ thresholds = CLASSIFIER_CONFIDENCE_THRESHOLDS.get( classifier, DEFAULT_CONFIDENCE_THRESHOLDS ) - + if confidence >= thresholds["high"]: return "High confidence" elif confidence >= thresholds["medium"]: @@ -101,21 +102,21 @@ def get_confidence_ui_tier(classifier: str, confidence: float) -> str: def get_confidence_status(classifier: str, confidence: float) -> tuple[str, str]: """ Get confidence status and color for reporting. - + Args: classifier: Name of the classifier (e.g., 'sturgeon', 'nanodx') confidence: Confidence value as decimal (0-1) - + Returns: Tuple of (status, color_hex) where status is 'High', 'Medium', or 'Low' """ # Convert to percentage for threshold comparison confidence_percent = confidence * 100 - + thresholds = CLASSIFIER_CONFIDENCE_THRESHOLDS.get( classifier, DEFAULT_CONFIDENCE_THRESHOLDS ) - + if confidence_percent >= thresholds["high"]: return "High", "#059669" # Green elif confidence_percent >= thresholds["medium"]: @@ -123,6 +124,7 @@ def get_confidence_status(classifier: str, confidence: float) -> tuple[str, str] else: return "Low", "#DC2626" # Red + # CNV Analysis Configuration # Thresholds apply to log2(ploidy / expected copy number); 0 = normal. # Call thresholds are set for shifts visible on the genome-wide log2 plot @@ -141,7 +143,7 @@ def get_confidence_status(classifier: str, confidence: float) -> tuple[str, str] "female": { "gain": 0.3, "loss": -0.3, - } + }, }, "chrY": { "male": { @@ -151,8 +153,8 @@ def get_confidence_status(classifier: str, confidence: float) -> tuple[str, str] "female": { "gain": 0.3, "loss": -0.3, - } - } + }, + }, } # CNV Event Detection Rules @@ -167,23 +169,24 @@ def get_confidence_status(classifier: str, confidence: float) -> tuple[str, str] }, "resolution": { "max_bin_width": 10_000_000, # 10Mb - resolution too low for accurate CNV calling - } + }, } + def get_cnv_thresholds(chromosome: str, sex_estimate: str) -> Tuple[float, float]: """ Get CNV gain/loss thresholds for a specific chromosome and sex. - + Args: chromosome: Chromosome name (e.g., 'chr1', 'chrX', 'chrY') sex_estimate: Sex estimate ('XY', 'XX', 'Male', 'Female') - + Returns: Tuple of (gain_threshold, loss_threshold) """ # Normalize sex estimate sex_key = "male" if sex_estimate.upper() in ("XY", "MALE") else "female" - + if chromosome == "chrX": thresholds = CNV_THRESHOLDS["chrX"][sex_key] elif chromosome == "chrY": @@ -191,9 +194,10 @@ def get_cnv_thresholds(chromosome: str, sex_estimate: str) -> Tuple[float, float else: # Autosomes thresholds = CNV_THRESHOLDS["autosomes"] - + return thresholds["gain"], thresholds["loss"] + def is_whole_chromosome_event( p_arm_mean: float, q_arm_mean: float, @@ -245,12 +249,13 @@ def is_whole_chromosome_event( return True, "LOSS" return False, "NORMAL" + def is_arm_event( arm_mean: float, arm_proportion_gain: float, arm_proportion_loss: float, gain_threshold: float, - loss_threshold: float + loss_threshold: float, ) -> Tuple[bool, str]: """ Determine if a chromosome arm shows a significant event. @@ -259,90 +264,99 @@ def is_arm_event( """ rules = CNV_EVENT_RULES["arm_specific"] - if arm_mean > gain_threshold and arm_proportion_gain > rules["min_proportion_affected"]: + if ( + arm_mean > gain_threshold + and arm_proportion_gain > rules["min_proportion_affected"] + ): return True, "GAIN" - if arm_mean < loss_threshold and arm_proportion_loss > rules["min_proportion_affected"]: + if ( + arm_mean < loss_threshold + and arm_proportion_loss > rules["min_proportion_affected"] + ): return True, "LOSS" return False, "NORMAL" + def is_resolution_sufficient(bin_width: int) -> bool: """ Check if the resolution is sufficient for CNV calling. - + Args: bin_width: Bin width in base pairs - + Returns: True if resolution is sufficient for CNV calling """ return bin_width <= CNV_EVENT_RULES["resolution"]["max_bin_width"] + # Fusion Detection Configuration FUSION_DETECTION_THRESHOLDS = { "mapping_quality": { "min_threshold": 49, # Minimum mapping quality score - "description": "Minimum mapping quality to consider read alignment reliable" + "description": "Minimum mapping quality to consider read alignment reliable", }, "mapping_span": { "min_threshold": 249, # Minimum mapping span in bases - "description": "Minimum read mapping length for reliable fusion detection" + "description": "Minimum read mapping length for reliable fusion detection", }, "gene_overlap": { "min_threshold": 99, # Minimum overlap with gene region - "description": "Minimum overlap between read and gene region (0 = any overlap)" + "description": "Minimum overlap between read and gene region (0 = any overlap)", }, "read_support": { "min_threshold": 3, # Minimum supporting reads per gene pair - "description": "Minimum number of supporting reads required for reliable fusion detection" + "description": "Minimum number of supporting reads required for reliable fusion detection", }, "read_coordinate_overlap": { "max_threshold": 100, # Maximum allowed overlap between read alignments - "description": "Maximum allowed overlap between read alignments to prevent false positives" + "description": "Maximum allowed overlap between read alignments to prevent false positives", }, "coordinate_similarity": { "max_threshold": 50, # Maximum allowed coordinate difference for similar alignments - "description": "Maximum coordinate difference (start or end) to consider alignments as similar and filter duplicates" - } + "description": "Maximum coordinate difference (start or end) to consider alignments as similar and filter duplicates", + }, } # Fusion Detection Rules FUSION_DETECTION_RULES = { "supplementary_alignment": { "required": True, - "description": "Only process reads with supplementary alignments (SA tag)" + "description": "Only process reads with supplementary alignments (SA tag)", }, "multi_gene_mapping": { "required": True, - "description": "Reads must map to more than 1 gene to be considered fusion candidates" + "description": "Reads must map to more than 1 gene to be considered fusion candidates", }, "overlapping_gene_filter": { "enabled": True, - "description": "Filter out reads where same genomic alignment is annotated with multiple overlapping genes" + "description": "Filter out reads where same genomic alignment is annotated with multiple overlapping genes", }, "exact_coordinate_matching": { "enabled": True, - "description": "Use exact genomic coordinate matching to identify identical alignments" + "description": "Use exact genomic coordinate matching to identify identical alignments", }, "coordinate_similarity_filter": { "enabled": True, - "description": "Filter out reads with very similar (but not identical) alignments to reduce mapping artifacts" - } + "description": "Filter out reads with very similar (but not identical) alignments to reduce mapping artifacts", + }, } + def get_fusion_threshold(threshold_name: str) -> int: """ Get fusion detection threshold by name. - + Args: threshold_name: Name of the threshold ('mapping_quality', 'mapping_span', etc.) - + Returns: Threshold value as integer """ if threshold_name not in FUSION_DETECTION_THRESHOLDS: raise ValueError(f"Unknown fusion threshold: {threshold_name}") - + # Handle both min_threshold and max_threshold keys threshold_config = FUSION_DETECTION_THRESHOLDS[threshold_name] if "min_threshold" in threshold_config: @@ -350,22 +364,30 @@ def get_fusion_threshold(threshold_name: str) -> int: elif "max_threshold" in threshold_config: return threshold_config["max_threshold"] else: - raise ValueError(f"Threshold {threshold_name} has no min_threshold or max_threshold defined") + raise ValueError( + f"Threshold {threshold_name} has no min_threshold or max_threshold defined" + ) + def get_fusion_rule(rule_name: str) -> bool: """ Get fusion detection rule setting by name. - + Args: rule_name: Name of the rule ('supplementary_alignment', 'multi_gene_mapping', etc.) - + Returns: Rule setting as boolean """ if rule_name not in FUSION_DETECTION_RULES: raise ValueError(f"Unknown fusion rule: {rule_name}") - - return FUSION_DETECTION_RULES[rule_name]["required"] if "required" in FUSION_DETECTION_RULES[rule_name] else FUSION_DETECTION_RULES[rule_name]["enabled"] + + return ( + FUSION_DETECTION_RULES[rule_name]["required"] + if "required" in FUSION_DETECTION_RULES[rule_name] + else FUSION_DETECTION_RULES[rule_name]["enabled"] + ) + def validate_fusion_candidate( mapping_quality: int, @@ -373,11 +395,11 @@ def validate_fusion_candidate( gene_overlap: int, supporting_reads: int, has_supplementary: bool = True, - maps_multiple_genes: bool = True + maps_multiple_genes: bool = True, ) -> Tuple[bool, str]: """ Validate a fusion candidate against all detection rules and thresholds. - + Args: mapping_quality: Read mapping quality score mapping_span: Read mapping span in bases @@ -385,76 +407,85 @@ def validate_fusion_candidate( supporting_reads: Number of supporting reads for the gene pair has_supplementary: Whether read has supplementary alignments maps_multiple_genes: Whether read maps to multiple genes - + Returns: Tuple of (is_valid, reason) where reason explains why validation failed """ # Check supplementary alignment requirement if get_fusion_rule("supplementary_alignment") and not has_supplementary: return False, "Missing supplementary alignment (SA tag)" - + # Check multi-gene mapping requirement if get_fusion_rule("multi_gene_mapping") and not maps_multiple_genes: return False, "Read does not map to multiple genes" - + # Check mapping quality threshold min_mq = get_fusion_threshold("mapping_quality") if mapping_quality <= min_mq: return False, f"Mapping quality {mapping_quality} below threshold {min_mq}" - + # Check mapping span threshold min_span = get_fusion_threshold("mapping_span") if mapping_span <= min_span: return False, f"Mapping span {mapping_span} below threshold {min_span}" - + # Check gene overlap threshold min_overlap = get_fusion_threshold("gene_overlap") if gene_overlap < min_overlap: return False, f"Gene overlap {gene_overlap} below threshold {min_overlap}" - + # Check read support threshold min_support = get_fusion_threshold("read_support") if supporting_reads < min_support: - return False, f"Supporting reads {supporting_reads} below threshold {min_support}" - + return ( + False, + f"Supporting reads {supporting_reads} below threshold {min_support}", + ) + return True, "All validation criteria passed" + def are_coordinates_similar( - coord1_start: int, coord1_end: int, - coord2_start: int, coord2_end: int, - max_difference: int = None + coord1_start: int, + coord1_end: int, + coord2_start: int, + coord2_end: int, + max_difference: int = None, ) -> bool: """ Check if two coordinate ranges are similar (within threshold). - + Args: coord1_start: Start of first coordinate range coord1_end: End of first coordinate range coord2_start: Start of second coordinate range coord2_end: End of second coordinate range max_difference: Maximum allowed difference in coordinates (uses config if None) - + Returns: True if coordinates are similar (within threshold) """ if max_difference is None: - max_difference = FUSION_DETECTION_THRESHOLDS["coordinate_similarity"]["max_threshold"] - + max_difference = FUSION_DETECTION_THRESHOLDS["coordinate_similarity"][ + "max_threshold" + ] + # Check if start and end coordinates are within the threshold start_diff = abs(coord1_start - coord2_start) end_diff = abs(coord1_end - coord2_end) - + return start_diff <= max_difference and end_diff <= max_difference + def get_fusion_config_summary() -> Dict[str, Any]: """ Get a summary of all fusion detection configuration. - + Returns: Dictionary containing all thresholds and rules """ return { "thresholds": FUSION_DETECTION_THRESHOLDS, "rules": FUSION_DETECTION_RULES, - "description": "Centralized fusion detection configuration for consistent analysis across the application" + "description": "Centralized fusion detection configuration for consistent analysis across the application", } diff --git a/src/robin/cli.py b/src/robin/cli.py index f657c7be..0568d7c7 100644 --- a/src/robin/cli.py +++ b/src/robin/cli.py @@ -26,6 +26,7 @@ # Suppress pkg_resources deprecation warnings from sorted_nearest import warnings + warnings.filterwarnings( "ignore", message="pkg_resources is deprecated", category=UserWarning ) @@ -34,41 +35,48 @@ "ignore", message="The figure layout has changed to tight", category=UserWarning ) -import os -import sys import csv +import logging +import os import shutil +import sys import tempfile from pathlib import Path -from typing import Optional, List, Dict, Tuple, Any, Iterable +from typing import Any, Dict, Iterable, List, Optional, Tuple import click -import logging # Check if we're in development mode -is_development_mode = os.environ.get("ROBIN_DEV_MODE", "").lower() in ("1", "true", "yes", "on") +is_development_mode = os.environ.get("ROBIN_DEV_MODE", "").lower() in ( + "1", + "true", + "yes", + "on", +) -from robin.workflow_simple import default_file_classifier, Job +from robin.workflow_simple import Job, default_file_classifier # Many analysis handlers have optional third-party dependencies. Import them lazily # so lightweight commands (e.g. `robin utils update-models`) still work. _analysis_import_error: Optional[BaseException] = None try: from robin.analysis.bam_preprocessor import bam_preprocessing_handler - from robin.analysis.mgmt_analysis import mgmt_handler - from robin.analysis.cnv_analysis import cnv_handler from robin.analysis.bed_conversion import bed_conversion_handler - from robin.analysis.sturgeon_analysis import sturgeon_handler + from robin.analysis.cnv_analysis import cnv_handler + from robin.analysis.fusion_analysis import fusion_handler + from robin.analysis.itd_analysis import itd_handler + from robin.analysis.lamprey_analysis import lamprey_handler + from robin.analysis.marlin_analysis import marlin_handler + from robin.analysis.mgmt_analysis import ( + extract_mgmt_site_rows_from_bed, + mgmt_handler, + ) from robin.analysis.nanodx_analysis import nanodx_handler, pannanodx_handler from robin.analysis.random_forest_analysis import random_forest_handler - from robin.analysis.marlin_analysis import marlin_handler - from robin.analysis.lamprey_analysis import lamprey_handler - from robin.analysis.tucan_analysis import tucan_handler + from robin.analysis.sturgeon_analysis import sturgeon_handler from robin.analysis.target_analysis import target_handler - from robin.analysis.fusion_analysis import fusion_handler - from robin.analysis.itd_analysis import itd_handler + from robin.analysis.tucan_analysis import tucan_handler from robin.analysis.utilities.matkit import run_matkit - from robin.analysis.mgmt_analysis import extract_mgmt_site_rows_from_bed except Exception as e: _analysis_import_error = e bam_preprocessing_handler = None # type: ignore[assignment] @@ -105,73 +113,75 @@ def _download_missing_models(missing_files, models_dir): """Download missing model files (same asset manifest logic as ``robin utils update-models``).""" - import json import hashlib - import urllib.request - import urllib.error + import json import os - + import urllib.error + import urllib.request + print("\n🔄 Attempting to download missing models...") - + # Find project root from models_dir location # models_dir is src/robin/models, so project root is 3 levels up project_root = models_dir.parent.parent.parent - + # Load assets manifest try: assets_file = project_root / "assets.json" if not assets_file.exists(): - print(f"❌ assets.json not found at {assets_file}. Cannot download models automatically.") + print( + f"❌ assets.json not found at {assets_file}. Cannot download models automatically." + ) return False - - with open(assets_file, 'r') as f: + + with open(assets_file, "r") as f: manifest = json.load(f) except Exception as e: print(f"❌ Failed to load assets manifest: {e}") return False - + # Asset name mapping asset_mapping = { "general.zip": "general_model", - "Capper_et_al_NN_v2.pkl": "capper_model", - "pancan_devel_v5i_NN_v2.pkl": "pancan_model" + "Capper_et_al_NN_v2.pkl": "capper_model", + "pancan_devel_v5i_NN_v2.pkl": "pancan_model", } - - github_token = os.getenv('GITHUB_TOKEN') + + github_token = os.getenv("GITHUB_TOKEN") if not github_token: print("ℹ️ No GITHUB_TOKEN found. Trying public download...") - + success_count = 0 for filename in missing_files: if filename not in asset_mapping: print(f"⚠️ Unknown model file: {filename}") continue - + asset_name = asset_mapping[filename] if asset_name not in manifest["assets"]: print(f"❌ Asset '{asset_name}' not found in manifest") continue - + asset_info = manifest["assets"][asset_name] asset_url = asset_info["url"] expected_sha256 = asset_info["sha256"] - + target_path = models_dir / filename - + try: print(f"\n📥 Downloading {filename}...") - + # Download the file headers = {} if github_token: headers["Authorization"] = f"Bearer {github_token}" - + request = urllib.request.Request(asset_url, headers=headers) - + with urllib.request.urlopen(request) as response: - with open(target_path, 'wb') as f: + with open(target_path, "wb") as f: f.write(response.read()) - + # Verify checksum print("🔍 Verifying checksum...") sha256_hash = hashlib.sha256() @@ -179,17 +189,17 @@ def _download_missing_models(missing_files, models_dir): for chunk in iter(lambda: f.read(4096), b""): sha256_hash.update(chunk) calculated_sha256 = sha256_hash.hexdigest() - + if calculated_sha256 != expected_sha256: print(f"❌ Checksum mismatch for {filename}") print(f"Expected: {expected_sha256}") print(f"Got: {calculated_sha256}") target_path.unlink() continue - + print(f"✅ Successfully downloaded {filename}") success_count += 1 - + except urllib.error.HTTPError as e: if e.code == 401: print(f"❌ Authentication failed for {filename}. Need GitHub token.") @@ -199,16 +209,16 @@ def _download_missing_models(missing_files, models_dir): print(f"❌ HTTP error {e.code} downloading {filename}: {e.reason}") except Exception as e: print(f"❌ Failed to download {filename}: {e}") - + return success_count == len(missing_files) def _check_models_or_exit(): """Ensure required runtime assets exist; auto-download missing ones.""" try: - from robin.utils.model_checker import get_models_directory, check_model_files - from robin.utils.model_updater import update_models as _update_models from robin.utils.clinvar_manager import ensure_clinvar_files + from robin.utils.model_checker import check_model_files, get_models_directory + from robin.utils.model_updater import update_models as _update_models except Exception as e: click.echo(f"❌ Could not load asset bootstrap helpers: {e}", err=True) sys.exit(1) @@ -335,6 +345,7 @@ def _echo_styled(message: str, level: str = "info") -> None: style = styles.get(level, "white") _RICH_CONSOLE.print(message, style=style) + # Disclaimer text for user acknowledgment DISCLAIMER_TEXT = EXTENDED_DISCLAIMER_TEXT @@ -394,7 +405,9 @@ def _get_user_acknowledgment() -> bool: click.echo("=" * 70) click.echo(DISCLAIMER_TEXT) click.echo("=" * 70) - _echo_styled("\nTo proceed, please type 'I agree' (exactly as shown):", level="warn") + _echo_styled( + "\nTo proceed, please type 'I agree' (exactly as shown):", level="warn" + ) try: response = input().strip() except (KeyboardInterrupt, EOFError): @@ -413,7 +426,12 @@ def _get_user_acknowledgment() -> bool: def _warn_if_process_large_bams() -> None: """If ROBIN_PROCESS_LARGE_BAMS is set, print a warning not to use with live runs.""" - if os.environ.get("ROBIN_PROCESS_LARGE_BAMS", "0").strip().lower() in ("1", "true", "yes", "on"): + if os.environ.get("ROBIN_PROCESS_LARGE_BAMS", "0").strip().lower() in ( + "1", + "true", + "yes", + "on", + ): _echo_styled( "Warning: ROBIN_PROCESS_LARGE_BAMS is enabled. Do not use this option alongside live runs.", level="warn", @@ -505,7 +523,9 @@ def _get_security_services(): @users.command("bootstrap-admin") -@click.option("--username", default="admin", show_default=True, help="Initial admin username.") +@click.option( + "--username", default="admin", show_default=True, help="Initial admin username." +) @click.option( "--from-legacy-hash", is_flag=True, @@ -554,7 +574,9 @@ def users_bootstrap_admin(username: str, from_legacy_hash: bool) -> None: else: password = click.prompt("Password", hide_input=True, confirmation_prompt=True) try: - user_id = auth.create_user(username, password, role="admin", must_change_password=False) + user_id = auth.create_user( + username, password, role="admin", must_change_password=False + ) except Exception as e: click.echo(f"Failed to create admin user '{username}': {e}", err=True) sys.exit(1) @@ -575,7 +597,9 @@ def users_bootstrap_admin(username: str, from_legacy_hash: bool) -> None: @users.command("create") @click.argument("username", type=str) -@click.option("--role", type=click.Choice(["admin", "user"]), default="user", show_default=True) +@click.option( + "--role", type=click.Choice(["admin", "user"]), default="user", show_default=True +) @click.option("--email", type=str, default="", help="Contact email for this account.") @click.option( "--clinical-role", @@ -601,7 +625,6 @@ def users_create( ) -> None: """Create a GUI user account.""" try: - from robin.security.user_metadata import CLINICAL_ROLE_KEY, EMAIL_KEY from robin.security.user_approvals import ( ADMIN_USER_APPROVALS_UPDATED_EVENT, MINKNOW_REMOTE_CONTROL_KEY, @@ -610,6 +633,7 @@ def users_create( approval_audit_details, default_approvals, ) + from robin.security.user_metadata import CLINICAL_ROLE_KEY, EMAIL_KEY store, auth, audit = _get_security_services() except ImportError as e: @@ -763,7 +787,9 @@ def users_list() -> None: default=None, help="Clinical role label (e.g. Consultant, Scientist).", ) -@click.option("--notes", type=str, default=None, help="Internal notes about this account.") +@click.option( + "--notes", type=str, default=None, help="Internal notes about this account." +) def users_set_profile( username: str, email: Optional[str], @@ -792,7 +818,9 @@ def users_set_profile( if notes is not None: updates[NOTES_KEY] = notes if not updates: - click.echo("Provide at least one of --email, --clinical-role, or --notes.", err=True) + click.echo( + "Provide at least one of --email, --clinical-role, or --notes.", err=True + ) sys.exit(1) try: @@ -862,7 +890,9 @@ def users_set_approvals( click.echo(f"User '{username}' not found.", err=True) sys.exit(1) if store.user_has_role(user.id, "admin"): - click.echo("Administrators always have all approvals; nothing to update.", err=True) + click.echo( + "Administrators always have all approvals; nothing to update.", err=True + ) sys.exit(1) updates = {} @@ -934,9 +964,7 @@ def users_consent_status(consent_version: str) -> None: status = "accepted" if row["has_consent"] else "pending" agreed = row["agreed_at"] or "never" active = "active" if row["is_active"] else "inactive" - click.echo( - f" - {row['username']} ({active}): {status} (agreed_at={agreed})" - ) + click.echo(f" - {row['username']} ({active}): {status} (agreed_at={agreed})") @users.command("deactivate") @@ -1030,7 +1058,11 @@ def users_revoke_role(username: str, role: str) -> None: if user is None: click.echo(f"User '{username}' not found.", err=True) sys.exit(1) - if role == "admin" and store.user_has_role(user.id, "admin") and store.count_active_admins() <= 1: + if ( + role == "admin" + and store.user_has_role(user.id, "admin") + and store.count_active_admins() <= 1 + ): click.echo("Cannot revoke admin role from the last active admin.", err=True) sys.exit(1) if not store.revoke_role(user.id, role): @@ -1053,11 +1085,15 @@ def audit() -> None: @audit.command("list") @click.option("--user", "username", type=str, default="", help="Filter by username.") -@click.option("--event", "event_type", type=str, default="", help="Filter by event type.") +@click.option( + "--event", "event_type", type=str, default="", help="Filter by event type." +) @click.option("--from-ts", type=str, default="", help="Start timestamp (UTC ISO8601).") @click.option("--to-ts", type=str, default="", help="End timestamp (UTC ISO8601).") @click.option("--limit", type=int, default=50, show_default=True) -def audit_list(username: str, event_type: str, from_ts: str, to_ts: str, limit: int) -> None: +def audit_list( + username: str, event_type: str, from_ts: str, to_ts: str, limit: int +) -> None: """List recent audit events.""" try: store, _, _ = _get_security_services() @@ -1084,12 +1120,16 @@ def audit_list(username: str, event_type: str, from_ts: str, to_ts: str, limit: @audit.command("export") @click.option("--user", "username", type=str, default="", help="Filter by username.") -@click.option("--event", "event_type", type=str, default="", help="Filter by event type.") +@click.option( + "--event", "event_type", type=str, default="", help="Filter by event type." +) @click.option("--from-ts", type=str, default="", help="Start timestamp (UTC ISO8601).") @click.option("--to-ts", type=str, default="", help="End timestamp (UTC ISO8601).") @click.option("--limit", type=int, default=5000, show_default=True) @click.option("--out", "out_path", type=click.Path(path_type=Path), required=True) -def audit_export(username: str, event_type: str, from_ts: str, to_ts: str, limit: int, out_path: Path) -> None: +def audit_export( + username: str, event_type: str, from_ts: str, to_ts: str, limit: int, out_path: Path +) -> None: """Export audit events to CSV.""" try: store, _, _ = _get_security_services() @@ -1172,7 +1212,9 @@ def mgmt(output_dir: Path, recursive: bool, out_path: Path) -> None: ) sys.exit(1) - output_stream = sys.stdout if str(out_path) == "-" else open(out_path, "w", newline="") + output_stream = ( + sys.stdout if str(out_path) == "-" else open(out_path, "w", newline="") + ) try: writer = csv.writer(output_stream, delimiter="\t") writer.writerow( @@ -1197,9 +1239,7 @@ def mgmt(output_dir: Path, recursive: bool, out_path: Path) -> None: sample_id = run_dir.name rel_run_path = os.path.relpath(run_dir, output_dir) - with tempfile.NamedTemporaryFile( - suffix=".bed", delete=False - ) as temp_bed: + with tempfile.NamedTemporaryFile(suffix=".bed", delete=False) as temp_bed: temp_bed_path = temp_bed.name try: @@ -1229,8 +1269,12 @@ def mgmt(output_dir: Path, recursive: bool, out_path: Path) -> None: meth_rev = int(row.get("meth_rev", 0)) meth_total = meth_fwd + meth_rev cov_total = int(row.get("cov_total", 0)) - meth_pct = round((meth_total / cov_total) * 100.0, 2) if cov_total else 0.0 - site_label = str(row.get("site", "")).split(" ")[0] if row.get("site") else "" + meth_pct = ( + round((meth_total / cov_total) * 100.0, 2) if cov_total else 0.0 + ) + site_label = ( + str(row.get("site", "")).split(" ")[0] if row.get("site") else "" + ) writer.writerow( [ sample_id, @@ -1298,7 +1342,9 @@ def update_clinvar() -> None: is_flag=True, help="Overwrite existing model files (default: skip existing).", ) -def update_models(models_dir: Optional[Path], manifest_path: Optional[Path], overwrite: bool) -> None: +def update_models( + models_dir: Optional[Path], manifest_path: Optional[Path], overwrite: bool +) -> None: """Download/update ROBIN model files using the assets manifest.""" try: from robin.utils.model_checker import get_models_directory @@ -1329,24 +1375,25 @@ def _remove_panel_from_system(panel_name: str) -> bool: click.echo(f"Error: Cannot remove built-in panel '{panel_name}'", err=True) click.echo("Built-in panels (rCNS2, AML) cannot be removed.", err=True) return False - + # Get resources directory try: from robin import resources + resources_dir = Path(resources.__file__).parent except ImportError: click.echo("Error: Could not locate ROBIN resources directory", err=True) return False - + # Check if panel exists panel_filename = f"{panel_name}_panel_name_uniq.bed" panel_path = resources_dir / panel_filename - + if not panel_path.exists(): click.echo(f"Error: Panel '{panel_name}' not found", err=True) click.echo(f"Expected file: {panel_path}", err=True) return False - + # Confirm removal source_preview = resources_dir / panel_source_filename(panel_name) click.echo(f"Panel '{panel_name}' will be removed:") @@ -1355,17 +1402,19 @@ def _remove_panel_from_system(panel_name: str) -> bool: if source_preview.exists(): click.echo(f" Original upload: {source_preview}") click.echo(f" Size: {source_preview.stat().st_size} bytes") - + # Ask for confirmation try: - confirm = input(f"\nAre you sure you want to remove panel '{panel_name}'? Type 'yes' to confirm: ").strip() - if confirm.lower() != 'yes': + confirm = input( + f"\nAre you sure you want to remove panel '{panel_name}'? Type 'yes' to confirm: " + ).strip() + if confirm.lower() != "yes": click.echo("Panel removal cancelled.") return False except (KeyboardInterrupt, EOFError): click.echo("\nPanel removal cancelled.") return False - + # Remove processed BED and optional stored original upload panel_path.unlink() source_path = resources_dir / panel_source_filename(panel_name) @@ -1377,7 +1426,7 @@ def _remove_panel_from_system(panel_name: str) -> bool: click.echo(f"Removed file: {panel_path}") return True - + except Exception as e: click.echo(f"Error removing panel: {e}", err=True) return False @@ -1386,43 +1435,41 @@ def _remove_panel_from_system(panel_name: str) -> bool: @main.command() @click.argument("panel_name", type=str) @click.option( - "--force", - "-f", - is_flag=True, - help="Skip confirmation prompt (use with caution)" + "--force", "-f", is_flag=True, help="Skip confirmation prompt (use with caution)" ) def remove_panel(panel_name: str, force: bool) -> None: """Remove a custom panel from ROBIN. - + PANEL_NAME: Name of the panel to remove - + Built-in panels (rCNS2, AML) cannot be removed. """ if not _get_user_acknowledgment(): sys.exit(1) - + # Validate panel name if not panel_name or not panel_name.strip(): click.echo("Error: Panel name cannot be empty", err=True) sys.exit(1) - + panel_name = panel_name.strip() - + # Check if it's a built-in panel built_in_panels = {"rCNS2", "AML"} if panel_name in built_in_panels: click.echo(f"Error: Cannot remove built-in panel '{panel_name}'", err=True) click.echo("Built-in panels (rCNS2, AML) cannot be removed.", err=True) sys.exit(1) - + # Get resources directory try: from robin import resources + resources_dir = Path(resources.__file__).parent except ImportError: click.echo("Error: Could not locate ROBIN resources directory", err=True) sys.exit(1) - + # Check if panel exists panel_filename = f"{panel_name}_panel_name_uniq.bed" panel_path = resources_dir / panel_filename @@ -1441,18 +1488,20 @@ def remove_panel(panel_name: str, force: bool) -> None: if source_path.exists(): click.echo(f" Original upload: {source_path}") click.echo(f" Size: {source_path.stat().st_size} bytes") - + # Confirm removal unless --force is used if not force: try: - confirm = input(f"\nAre you sure you want to remove panel '{panel_name}'? Type 'yes' to confirm: ").strip() - if confirm.lower() != 'yes': + confirm = input( + f"\nAre you sure you want to remove panel '{panel_name}'? Type 'yes' to confirm: " + ).strip() + if confirm.lower() != "yes": click.echo("Panel removal cancelled.") return except (KeyboardInterrupt, EOFError): click.echo("\nPanel removal cancelled.") return - + # Remove processed BED and optional stored original try: panel_path.unlink() @@ -1472,19 +1521,19 @@ def list_panels() -> None: """List all available panels in ROBIN.""" if not _get_user_acknowledgment(): sys.exit(1) - + panels = _get_available_panels() - + click.echo("Available panels in ROBIN:\n") - + # Built-in panels built_in_panels = ["rCNS2", "AML"] custom_panels = [p for p in panels if p not in built_in_panels] - + click.echo("BUILT-IN PANELS:") for panel in built_in_panels: click.echo(f" • {panel}") - + if custom_panels: click.echo("\nCUSTOM PANELS:") for panel in custom_panels: @@ -1492,10 +1541,12 @@ def list_panels() -> None: else: click.echo("\nCUSTOM PANELS:") click.echo(" (none)") - + click.echo(f"\nTotal panels: {len(panels)}") click.echo("\nUsage: Use --target-panel in workflow commands") - click.echo("Example: robin workflow /path/to/bams --workflow mgmt,target --target-panel rCNS2") + click.echo( + "Example: robin workflow /path/to/bams --workflow mgmt,target --target-panel rCNS2" + ) click.echo("\nPanel management:") click.echo(" • Add panel: robin add-panel ") click.echo(" • Remove panel: robin remove-panel ") @@ -1569,49 +1620,59 @@ def list_job_types() -> None: def _validate_bed_file(bed_path: Path) -> Tuple[bool, List[str]]: """Validate BED file format and return (is_valid, error_messages).""" errors = [] - + if not bed_path.exists(): errors.append(f"BED file does not exist: {bed_path}") return False, errors - + if not bed_path.is_file(): errors.append(f"Path is not a file: {bed_path}") return False, errors - + try: - with open(bed_path, 'r') as f: + with open(bed_path, "r") as f: line_count = 0 for line_num, line in enumerate(f, 1): line = line.strip() - if not line or line.startswith('#'): + if not line or line.startswith("#"): continue - + line_count += 1 - parts = line.split('\t') - + parts = line.split("\t") + if len(parts) < 3: - errors.append(f"Line {line_num}: Invalid BED format - must have at least 3 columns (chromosome, start, end)") + errors.append( + f"Line {line_num}: Invalid BED format - must have at least 3 columns (chromosome, start, end)" + ) continue - + # Validate chromosome chrom = parts[0] - if not chrom.startswith('chr'): - errors.append(f"Line {line_num}: Chromosome must start with 'chr': {chrom}") - + if not chrom.startswith("chr"): + errors.append( + f"Line {line_num}: Chromosome must start with 'chr': {chrom}" + ) + # Validate start and end positions try: start = int(parts[1]) end = int(parts[2]) if start < 0 or end < 0: - errors.append(f"Line {line_num}: Start and end positions must be non-negative") + errors.append( + f"Line {line_num}: Start and end positions must be non-negative" + ) if start >= end: - errors.append(f"Line {line_num}: Start position must be less than end position") + errors.append( + f"Line {line_num}: Start position must be less than end position" + ) except ValueError: - errors.append(f"Line {line_num}: Start and end positions must be integers") - + errors.append( + f"Line {line_num}: Start and end positions must be integers" + ) + # Column 4 may be a gene name, a placeholder ('.'), or absent (3-col BED). # Placeholders are annotated from all_genes2.bed during add-panel. - + # Optional: validate 6-column BED format if present if len(parts) >= 6: # Validate score (5th column) - should be numeric or "." @@ -1620,26 +1681,32 @@ def _validate_bed_file(bed_path: Path) -> Tuple[bool, List[str]]: try: score_int = int(score) if score_int < 0: - errors.append(f"Line {line_num}: Score must be non-negative") + errors.append( + f"Line {line_num}: Score must be non-negative" + ) except ValueError: - errors.append(f"Line {line_num}: Score must be an integer or '.'") - + errors.append( + f"Line {line_num}: Score must be an integer or '.'" + ) + # Validate strand (6th column) - should be + or - strand = parts[5].strip() - if strand not in ['+', '-']: - errors.append(f"Line {line_num}: Strand must be '+' or '-', got: {strand}") - + if strand not in ["+", "-"]: + errors.append( + f"Line {line_num}: Strand must be '+' or '-', got: {strand}" + ) + # Limit error reporting to first 10 errors if len(errors) >= 10: errors.append("... (additional errors truncated)") break - + if line_count == 0: errors.append("BED file contains no valid data lines") - + except Exception as e: errors.append(f"Error reading BED file: {e}") - + return len(errors) == 0, errors @@ -1689,14 +1756,18 @@ def _load_all_genes_dataframe(genes_bed: Path): return genes.reset_index(drop=True) -def _annotate_placeholder_intervals_with_genes(panel_df, genes_df) -> List[Dict[str, object]]: +def _annotate_placeholder_intervals_with_genes( + panel_df, genes_df +) -> List[Dict[str, object]]: """Intersect placeholder panel intervals with the gene reference. Returns gene-body rows (chrom/start/end/gene) for each overlapping gene. """ annotated: List[Dict[str, object]] = [] seen_genes: set[str] = set() - genes_by_chrom = {chrom: group for chrom, group in genes_df.groupby("chrom", sort=False)} + genes_by_chrom = { + chrom: group for chrom, group in genes_df.groupby("chrom", sort=False) + } for chrom, intervals in panel_df.groupby("chrom", sort=False): gene_chrom = genes_by_chrom.get(chrom) @@ -1742,18 +1813,18 @@ def _generate_unique_gene_bed(input_bed_path: Path, output_bed_path: Path) -> bo # Try 6-column format first (chrom, start, end, gene, score, strand) df = pd.read_csv( input_bed_path, - sep='\t', + sep="\t", header=None, - names=['chrom', 'start', 'end', 'gene', 'score', 'strand'], - comment='#' + names=["chrom", "start", "end", "gene", "score", "strand"], + comment="#", ) except ValueError: # Fallback to 4-column / 3-column format raw = pd.read_csv( input_bed_path, - sep='\t', + sep="\t", header=None, - comment='#', + comment="#", ) if raw.shape[1] < 3: click.echo("Error: BED file must have at least 3 columns", err=True) @@ -1761,7 +1832,7 @@ def _generate_unique_gene_bed(input_bed_path: Path, output_bed_path: Path) -> bo raw = raw.iloc[:, :4].copy() while raw.shape[1] < 4: raw[raw.shape[1]] = "." - raw.columns = ['chrom', 'start', 'end', 'gene'] + raw.columns = ["chrom", "start", "end", "gene"] df = raw df["chrom"] = df["chrom"].astype(str) @@ -1832,17 +1903,14 @@ def _generate_unique_gene_bed(input_bed_path: Path, output_bed_path: Path) -> bo processed_df = pd.DataFrame(processed_regions) # Remove duplicates based on gene name (keep first occurrence) - processed_df = processed_df.drop_duplicates(subset=['gene'], keep='first') + processed_df = processed_df.drop_duplicates(subset=["gene"], keep="first") # Sort by chromosome and position - processed_df = processed_df.sort_values(['chrom', 'start', 'end']) + processed_df = processed_df.sort_values(["chrom", "start", "end"]) # Write to output file in standard 4-column BED format - processed_df[['chrom', 'start', 'end', 'gene']].to_csv( - output_bed_path, - sep='\t', - header=False, - index=False + processed_df[["chrom", "start", "end", "gene"]].to_csv( + output_bed_path, sep="\t", header=False, index=False ) click.echo( @@ -1854,7 +1922,7 @@ def _generate_unique_gene_bed(input_bed_path: Path, output_bed_path: Path) -> bo ) ) return True - + except Exception as e: click.echo(f"Error generating unique gene BED file: {e}", err=True) return False @@ -1863,26 +1931,26 @@ def _generate_unique_gene_bed(input_bed_path: Path, output_bed_path: Path) -> bo def _get_available_panels() -> List[str]: """Get list of available panels from resources directory.""" panels = ["rCNS2", "AML"] # Built-in panels - + try: # Try to find the resources directory without importing robin module # Look for the resources directory relative to this file current_file = Path(__file__) resources_dir = current_file.parent.parent / "robin" / "resources" - + if resources_dir.exists(): # Look for custom panels (files ending with _panel_name_uniq.bed) for bed_file in resources_dir.glob("*_panel_name_uniq.bed"): panel_name = bed_file.stem.replace("_panel_name_uniq", "") if panel_name not in panels: panels.append(panel_name) - + panels.sort() - + except Exception: # Fallback to built-in panels only - don't fail on any import or other errors pass - + return panels @@ -2001,17 +2069,17 @@ def _register_panel_in_system(panel_name: str, bed_path: Path) -> bool: try: # Create a simple registration by copying the BED file to resources # and updating any necessary configuration files - + # For now, we'll just ensure the file is in the right location # The actual registration happens when the panel is referenced by name # in analysis modules like target_analysis.py and fusion_work.py - + click.echo(f"Panel '{panel_name}' registered successfully") click.echo(f"BED file location: {bed_path}") click.echo(f"Panel can now be used with --target-panel {panel_name}") - + return True - + except Exception as e: click.echo(f"Error registering panel: {e}", err=True) return False @@ -2023,22 +2091,22 @@ def _register_panel_in_system(panel_name: str, bed_path: Path) -> bool: @click.option( "--validate-only", is_flag=True, - help="Only validate the BED file format without adding the panel" + help="Only validate the BED file format without adding the panel", ) def add_panel(bed_file: Path, panel_name: str, validate_only: bool) -> None: """Add a custom panel to ROBIN. - + BED_FILE: Path to the BED file containing panel regions PANEL_NAME: Name for the panel (e.g., 'CustomPanel', 'MyPanel') - + The BED file should be in standard format with at least 3 columns: chromosome, start, end [, gene_name(s) [, score, strand]] - + Supported formats: - 3-column: chr1, 1000000, 2000000 - 4-column: chr1, 1000000, 2000000, GENE1 - 6-column: chr1, 1000000, 2000000, GENE1, 0, + - + Gene names can be comma-separated for regions covering multiple genes. When gene names are missing or '.', intervals are annotated by intersecting with the packaged all_genes2.bed reference. Named intervals keep their @@ -2050,53 +2118,62 @@ def add_panel(bed_file: Path, panel_name: str, validate_only: bool) -> None: """ if not _get_user_acknowledgment(): sys.exit(1) - + # Validate panel name if not panel_name or not panel_name.strip(): click.echo("Error: Panel name cannot be empty", err=True) sys.exit(1) - + panel_name = panel_name.strip() - + # Check for reserved panel names reserved_names = {"rCNS2", "AML"} if panel_name in reserved_names: - click.echo(f"Error: Panel name '{panel_name}' is reserved. Please choose a different name.", err=True) + click.echo( + f"Error: Panel name '{panel_name}' is reserved. Please choose a different name.", + err=True, + ) sys.exit(1) - + click.echo(f"Validating BED file: {bed_file}") - + # Validate BED file format is_valid, errors = _validate_bed_file(bed_file) - + if not is_valid: click.echo("BED file validation failed:", err=True) for error in errors: click.echo(f" • {error}", err=True) sys.exit(1) - + click.echo("BED file format validation passed") - + if validate_only: click.echo("Validation complete. Use without --validate-only to add the panel.") return - + # Generate output paths try: from robin import resources + resources_dir = Path(resources.__file__).parent except ImportError: click.echo("Error: Could not locate ROBIN resources directory", err=True) sys.exit(1) - + output_filename = f"{panel_name}_panel_name_uniq.bed" output_path = resources_dir / output_filename source_path = resources_dir / panel_source_filename(panel_name) # Check if panel already exists if output_path.exists(): - click.echo(f"Error: Panel '{panel_name}' already exists at {output_path}", err=True) - click.echo("Please choose a different panel name or remove the existing panel first.", err=True) + click.echo( + f"Error: Panel '{panel_name}' already exists at {output_path}", err=True + ) + click.echo( + "Please choose a different panel name or remove the existing panel first.", + err=True, + ) sys.exit(1) if source_path.exists(): click.echo(f"Error: File already exists: {source_path}", err=True) @@ -2137,7 +2214,9 @@ def add_panel(bed_file: Path, panel_name: str, validate_only: bool) -> None: click.echo(f" Processed BED (unique genes): {output_path}") click.echo(f" Original upload: {source_path}") click.echo(f"Usage: Use --target-panel {panel_name} in workflow commands") - click.echo(f"Example: robin workflow /path/to/bams --workflow mgmt,target --target-panel {panel_name}") + click.echo( + f"Example: robin workflow /path/to/bams --workflow mgmt,target --target-panel {panel_name}" + ) click.echo(f"Remove: robin remove-panel {panel_name}") @@ -2265,10 +2344,13 @@ def _create_ray_workflow_runner( try: # Prefer Ray Core implementation import asyncio + from robin import workflow_ray as wrn class _RayCoreWrapper: - def __init__(self, reference: Optional[Path] = None, target_panel: str = None): + def __init__( + self, reference: Optional[Path] = None, target_panel: str = None + ): self.manager = type( "_DummyManager", (), {"get_priority_info": lambda _self: {}} )() @@ -2300,6 +2382,7 @@ def submit_sample_job( try: # Try to get coordinator with retries import time + from robin import workflow_ray as wrn max_retries = 5 @@ -2360,6 +2443,7 @@ def submit_snp_analysis_job( ) # Try to get coordinator with retries import time + from robin import workflow_ray as wrn max_retries = 5 @@ -2442,6 +2526,7 @@ def submit_target_bam_finalize_job( f"[Finalize] Submitting target BAM finalization for sample '{sid_for_log}'..." ) import time + from robin import workflow_ray as wrn max_retries = 5 @@ -2592,8 +2677,8 @@ def _initialize_ray(num_cpus: Optional[int], include_dashboard: bool = True) -> if not ray.is_initialized(): # Suppress Ray logging to prevent interference with progress bars - import logging import json + import logging ray_logger = logging.getLogger("ray") ray_logger.setLevel(logging.ERROR) @@ -2760,7 +2845,7 @@ def _register_handlers( center: str = None, ) -> None: """Register all workflow handlers with the runner.""" - + # Validate target panel available_panels = _get_available_panels() if target_panel not in available_panels: @@ -2772,11 +2857,11 @@ def _register_handlers( # Don't reset to rCNS2 - allow custom panels to be used else: click.echo(f"Using target panel: {target_panel}") - + # Track handlers that should accept target_panel handlers_requiring_panel = {"target", "fusion", "cnv", "itd"} registered_panel_handlers = set() - + for ( queue_type, job_type, @@ -2799,7 +2884,12 @@ def create_handler_with_work_dir_and_ref( handler, work_dir_path, ref_path, center_param, panel_param ): return lambda job: handler( - job, work_dir=str(work_dir_path), reference=str(ref_path), target_panel=job.context.metadata.get("target_panel", panel_param) + job, + work_dir=str(work_dir_path), + reference=str(ref_path), + target_panel=job.context.metadata.get( + "target_panel", panel_param + ), ) final_handler = create_handler_with_work_dir_and_ref( @@ -2818,6 +2908,7 @@ def create_mgmt_handler_with_work_dir_and_ref( handler_func, work_dir, reference ) elif reference and job_type == "bed_conversion": + def create_bed_conversion_handler_with_work_dir_and_ref( handler, work_dir_path, ref_path ): @@ -2881,22 +2972,37 @@ def _analysis_handler(job): ) else: # Standard work directory handling for other job types - def create_handler_with_work_dir(handler, work_dir_path, center_param): + def create_handler_with_work_dir( + handler, work_dir_path, center_param + ): return lambda job: handler(job, work_dir=str(work_dir_path)) - final_handler = create_handler_with_work_dir(handler_func, work_dir, center) + final_handler = create_handler_with_work_dir( + handler_func, work_dir, center + ) elif reference and job_type == "target": # Reference genome only (no work_dir needed) with target panel def create_handler_with_ref(handler, ref_path, center_param, panel_param): - return lambda job: handler(job, reference=str(ref_path), target_panel=job.context.metadata.get("target_panel", panel_param)) + return lambda job: handler( + job, + reference=str(ref_path), + target_panel=job.context.metadata.get("target_panel", panel_param), + ) - final_handler = create_handler_with_ref(handler_func, reference, center, target_panel) + final_handler = create_handler_with_ref( + handler_func, reference, center, target_panel + ) elif job_type in ["fusion", "cnv", "itd"]: # Analysis with target panel only (no work_dir needed) def create_analysis_handler_with_panel(handler, panel_param): - return lambda job: handler(job, target_panel=job.context.metadata.get("target_panel", panel_param)) + return lambda job: handler( + job, + target_panel=job.context.metadata.get("target_panel", panel_param), + ) - final_handler = create_analysis_handler_with_panel(handler_func, target_panel) + final_handler = create_analysis_handler_with_panel( + handler_func, target_panel + ) elif job_type == "preprocessing": # Special handling for preprocessing to pass center def create_preprocessing_handler(handler, center_param): @@ -2909,6 +3015,7 @@ def create_preprocessing_handler(handler, center_param): # Track handlers that require target_panel if job_type in handlers_requiring_panel: import inspect + sig = inspect.signature(handler_func) if "target_panel" in sig.parameters: registered_panel_handlers.add(job_type) @@ -2926,7 +3033,7 @@ def create_preprocessing_handler(handler, center_param): f"Warning: Failed to register handler for {queue_type}:{job_type}: {e}", err=True, ) - + # Report on panel handler registration missing_panel_handlers = handlers_requiring_panel - registered_panel_handlers if missing_panel_handlers: @@ -2935,7 +3042,9 @@ def create_preprocessing_handler(handler, center_param): err=True, ) else: - click.echo(f"Successfully registered panel-aware handlers: {registered_panel_handlers}") + click.echo( + f"Successfully registered panel-aware handlers: {registered_panel_handlers}" + ) def _register_command_handlers( @@ -3038,15 +3147,15 @@ def _display_workflow_config( ) -> None: """Display workflow configuration information.""" _echo_styled(f"Center: {center}", level="info") - + if no_process_existing: - _echo_styled( + _echo_styled( f"Starting workflow on {path} for BAM files (skipping existing files)..." - ) + ) else: - _echo_styled( + _echo_styled( f"Starting workflow on {path} for BAM files (will process existing files first)..." - ) + ) if work_dir: _echo_styled(f"Output directory: {work_dir}", level="info") @@ -3099,9 +3208,7 @@ def _display_workflow_config( if queue_priority: click.echo(f" - Queue priorities: {list(queue_priority)}") else: - _echo_styled( - "Distributed computing: Disabled (using threading)", level="warn" - ) + _echo_styled("Distributed computing: Disabled (using threading)", level="warn") click.echo("Worker configuration:") if legacy_analysis_queue: click.echo( @@ -3421,22 +3528,22 @@ def workflow( # Check for required model files first _check_models_or_exit() - + # Validate reference genome if provided if reference: try: # Import the validation function from matkit from robin.analysis.utilities.matkit import _ensure_fasta_index - + # Convert Path to string for the validation function ref_path = str(reference) if isinstance(reference, Path) else reference - + # Use click.echo for visibility even when log level is ERROR _echo_styled(f"Validating reference genome: {reference}", level="info") - + # Validate and ensure index exists _ensure_fasta_index(ref_path) - + _echo_styled( f"Reference genome validated and indexed: {reference}", level="success", @@ -3449,7 +3556,7 @@ def workflow( ) click.echo(f"❌ {error_msg}", err=True) sys.exit(1) - + # Require user acknowledgment before proceeding if not _get_user_acknowledgment(): sys.exit(1) @@ -3520,7 +3627,9 @@ def workflow( ) if preset in {"p2i", "standard"}: - init_kwargs["num_cpus"] = 2 if preset == "p2i" else 6 # Increased from 4 to 6 for standard + init_kwargs["num_cpus"] = ( + 2 if preset == "p2i" else 6 + ) # Increased from 4 to 6 for standard try: ray.init(**init_kwargs) except TypeError: @@ -3570,6 +3679,7 @@ def workflow( # Run Ray Core implementation try: import asyncio + from robin import workflow_ray as wrn asyncio.run( @@ -3606,6 +3716,7 @@ def workflow( # Attempt to shutdown Ray gracefully try: import ray + if ray.is_initialized(): print("[SHUTDOWN] Shutting down Ray coordinator...") # Get the coordinator and shutdown gracefully @@ -3690,7 +3801,9 @@ def workflow( click.echo(f"Job deduplication enabled for: {valid_dedup_jobs}") # Register handlers and command handlers - _register_handlers(runner, legacy_analysis_queue, work_dir, target_panel, reference, center) + _register_handlers( + runner, legacy_analysis_queue, work_dir, target_panel, reference, center + ) _register_command_handlers(runner, command_map, legacy_analysis_queue) # For Ray workflow, reinitialize processors after handlers are registered @@ -3753,7 +3866,9 @@ def workflow( workflow_steps=workflow_steps, monitored_directory=str(work_dir) if work_dir else str(path), center=center, - workflow_toml=str(toml_config.resolve()) if toml_config else None, + workflow_toml=( + str(toml_config.resolve()) if toml_config else None + ), ) # Now install workflow hooks for real-time monitoring diff --git a/src/robin/gui/admin.py b/src/robin/gui/admin.py index 75fd17b6..351b3a14 100644 --- a/src/robin/gui/admin.py +++ b/src/robin/gui/admin.py @@ -21,7 +21,6 @@ effective_section_map, ) from robin.security import get_consent_version -from robin.security.user_metadata import CLINICAL_ROLE_KEY, EMAIL_KEY, NOTES_KEY from robin.security.user_approvals import ( ADMIN_USER_APPROVALS_UPDATED_EVENT, MINKNOW_REMOTE_CONTROL_KEY, @@ -32,6 +31,7 @@ default_approvals, effective_approvals, ) +from robin.security.user_metadata import CLINICAL_ROLE_KEY, EMAIL_KEY, NOTES_KEY if TYPE_CHECKING: from robin.gui_launcher import GUILauncher @@ -69,7 +69,9 @@ def _user_table_rows(launcher: "GUILauncher") -> List[Dict[str, Any]]: return rows -def _audit_table_rows(launcher: "GUILauncher", filters: Dict[str, Any]) -> List[Dict[str, Any]]: +def _audit_table_rows( + launcher: "GUILauncher", filters: Dict[str, Any] +) -> List[Dict[str, Any]]: events = launcher.security_store.query_audit_events( username=str(filters.get("username") or ""), event_type=str(filters.get("event_type") or ""), @@ -107,7 +109,9 @@ def create_admin_page(launcher: "GUILauncher") -> None: ): with ui.element("div").classes("w-full min-w-0").props("id=admin-page"): with ui.column().classes("w-full max-w-6xl mx-auto gap-3 p-2 md:p-3"): - with ui.element("div").classes("classification-insight-shell w-full min-w-0"): + with ui.element("div").classes( + "classification-insight-shell w-full min-w-0" + ): ui.label("Administration").classes( "classification-insight-heading text-headline-small" ) @@ -143,7 +147,9 @@ def create_admin_page(launcher: "GUILauncher") -> None: def _build_users_panel(launcher: "GUILauncher", consent_version: str) -> None: with ui.element("div").classes("classification-insight-card w-full min-w-0"): with ui.column().classes("w-full min-w-0 gap-3 p-2 md:p-3"): - with ui.row().classes("w-full items-center justify-between gap-2 flex-wrap"): + with ui.row().classes( + "w-full items-center justify-between gap-2 flex-wrap" + ): ui.label("User accounts").classes("classification-insight-model") with ui.row().classes("gap-2"): refresh_btn = ui.button("Refresh", icon="refresh").props( @@ -152,11 +158,18 @@ def _build_users_panel(launcher: "GUILauncher", consent_version: str) -> None: ui.button( "Create user", icon="person_add", - on_click=lambda: _open_create_user_dialog(launcher, refresh_users), + on_click=lambda: _open_create_user_dialog( + launcher, refresh_users + ), ).props("color=primary no-caps") user_columns = [ - {"name": "username", "label": "Username", "field": "username", "align": "left"}, + { + "name": "username", + "label": "Username", + "field": "username", + "align": "left", + }, {"name": "email", "label": "Email", "field": "email", "align": "left"}, { "name": "clinical_role", @@ -183,12 +196,42 @@ def _build_users_panel(launcher: "GUILauncher", consent_version: str) -> None: "align": "left", }, {"name": "roles", "label": "Roles", "field": "roles", "align": "left"}, - {"name": "active", "label": "Active", "field": "active", "align": "left"}, - {"name": "password", "label": "Password", "field": "password", "align": "left"}, - {"name": "last_login", "label": "Last login", "field": "last_login", "align": "left"}, - {"name": "consent", "label": "Consent", "field": "consent", "align": "left"}, - {"name": "consent_at", "label": "Consent at", "field": "consent_at", "align": "left"}, - {"name": "actions", "label": "Actions", "field": "actions", "align": "left"}, + { + "name": "active", + "label": "Active", + "field": "active", + "align": "left", + }, + { + "name": "password", + "label": "Password", + "field": "password", + "align": "left", + }, + { + "name": "last_login", + "label": "Last login", + "field": "last_login", + "align": "left", + }, + { + "name": "consent", + "label": "Consent", + "field": "consent", + "align": "left", + }, + { + "name": "consent_at", + "label": "Consent at", + "field": "consent_at", + "align": "left", + }, + { + "name": "actions", + "label": "Actions", + "field": "actions", + "align": "left", + }, ] _, user_table = theme.styled_table( columns=user_columns, @@ -231,24 +274,60 @@ def _build_audit_panel(launcher: "GUILauncher", audit_filters: Dict[str, Any]) - ui.label("Audit events").classes("classification-insight-model") with ui.row().classes("w-full gap-2 flex-wrap items-end"): - username_filter = ui.input("Username").classes("min-w-[10rem]").props( - "dense outlined clearable" + username_filter = ( + ui.input("Username") + .classes("min-w-[10rem]") + .props("dense outlined clearable") + ) + event_filter = ( + ui.input("Event type") + .classes("min-w-[12rem]") + .props("dense outlined clearable") ) - event_filter = ui.input("Event type").classes("min-w-[12rem]").props( - "dense outlined clearable" + limit_filter = ( + ui.number("Limit", value=100, min=1, max=5000, step=1) + .classes("w-28") + .props("dense outlined") ) - limit_filter = ui.number( - "Limit", value=100, min=1, max=5000, step=1 - ).classes("w-28").props("dense outlined") audit_columns = [ - {"name": "occurred_at", "label": "Time (UTC)", "field": "occurred_at", "align": "left"}, - {"name": "username", "label": "User", "field": "username", "align": "left"}, - {"name": "event_type", "label": "Event", "field": "event_type", "align": "left"}, - {"name": "target", "label": "Target", "field": "target", "align": "left"}, - {"name": "result", "label": "Result", "field": "result", "align": "left"}, + { + "name": "occurred_at", + "label": "Time (UTC)", + "field": "occurred_at", + "align": "left", + }, + { + "name": "username", + "label": "User", + "field": "username", + "align": "left", + }, + { + "name": "event_type", + "label": "Event", + "field": "event_type", + "align": "left", + }, + { + "name": "target", + "label": "Target", + "field": "target", + "align": "left", + }, + { + "name": "result", + "label": "Result", + "field": "result", + "align": "left", + }, {"name": "ip", "label": "IP", "field": "ip", "align": "left"}, - {"name": "details", "label": "Details", "field": "details", "align": "left"}, + { + "name": "details", + "label": "Details", + "field": "details", + "align": "left", + }, ] _, audit_table = theme.styled_table( columns=audit_columns, @@ -295,7 +374,9 @@ def _export_csv() -> None: writer.writeheader() for event in events: row = dict(event) - row["details"] = json.dumps(event.get("details") or {}, ensure_ascii=True) + row["details"] = json.dumps( + event.get("details") or {}, ensure_ascii=True + ) writer.writerow({k: row.get(k, "") for k in fieldnames}) launcher._audit_log( event_type="admin.audit.exported", @@ -307,9 +388,9 @@ def _export_csv() -> None: ui.download(buf.getvalue().encode("utf-8"), "robin_audit_export.csv") with ui.row().classes("w-full gap-2 flex-wrap"): - ui.button("Apply filters", icon="filter_alt", on_click=_apply_filters).props( - "color=primary no-caps" - ) + ui.button( + "Apply filters", icon="filter_alt", on_click=_apply_filters + ).props("color=primary no-caps") ui.button("Export CSV", icon="download", on_click=_export_csv).props( "flat no-caps outline" ) @@ -324,9 +405,7 @@ def _build_sample_display_panel(launcher: "GUILauncher") -> None: if getattr(launcher, "display_config", None) is not None else SampleDisplayConfig() ) - checkbox_state: Dict[str, Dict[str, Any]] = { - role: {} for role in DISPLAY_ROLES - } + checkbox_state: Dict[str, Dict[str, Any]] = {role: {} for role in DISPLAY_ROLES} def _initial_visible(section_id: str, role: str) -> bool: return effective_section_map( @@ -350,7 +429,9 @@ def _initial_visible(section_id: str, role: str) -> bool: for role in DISPLAY_ROLES } - with ui.tab_panels(role_tabs, value=role_tab_items["user"]).classes("w-full"): + with ui.tab_panels(role_tabs, value=role_tab_items["user"]).classes( + "w-full" + ): for role in DISPLAY_ROLES: with ui.tab_panel(role_tab_items[role]): ui.label( @@ -437,7 +518,9 @@ def _reset_to_workflow() -> None: ) with ui.row().classes("w-full gap-2 flex-wrap mt-2"): - ui.button("Save", icon="save", on_click=_save).props("color=primary no-caps") + ui.button("Save", icon="save", on_click=_save).props( + "color=primary no-caps" + ) ui.button( "Reset active role to workflow defaults", icon="restart_alt", @@ -505,9 +588,7 @@ def _labeled_switch( right="Log2", value=current.cnv_report_scale == CNV_REPORT_SCALE_NORMALIZED_DIFFERENCE, - tooltip=( - "Left: estimated ploidy · Right: log2(ploidy / expected)" - ), + tooltip=("Left: estimated ploidy · Right: log2(ploidy / expected)"), ) ui.label("CNV coverage genes (GUI)").classes( @@ -528,9 +609,7 @@ def _labeled_switch( left="Chromosome", right="Up/Down", value=current.cnv_gui_color_mode == CNV_GUI_COLOR_MODE_VALUE, - tooltip=( - "Left: colour by chromosome · Right: gain/loss (up/down)" - ), + tooltip=("Left: colour by chromosome · Right: gain/loss (up/down)"), ) ui.label("CNV breakpoints (GUI)").classes( @@ -572,14 +651,18 @@ def _labeled_switch( "Controls which chromosomes and contigs appear in coverage and CNV " "figures in the GUI and PDF reports." ).classes("classification-insight-foot mb-2") - contig_scope_select = ui.select( - { - scope: REFERENCE_CONTIG_SCOPE_LABELS[scope] - for scope in REFERENCE_CONTIG_SCOPES - }, - value=current.reference_contig_scope, - label="Contigs shown in plots", - ).classes("w-full").props("dense outlined") + contig_scope_select = ( + ui.select( + { + scope: REFERENCE_CONTIG_SCOPE_LABELS[scope] + for scope in REFERENCE_CONTIG_SCOPES + }, + value=current.reference_contig_scope, + label="Contigs shown in plots", + ) + .classes("w-full") + .props("dense outlined") + ) status_label = ui.label("").classes("classification-insight-meta") @@ -636,27 +719,36 @@ def _open_create_user_dialog( launcher: "GUILauncher", on_created: Callable[[], None] | None = None, ) -> None: - with ui.dialog() as dialog, ui.card().classes( - "robin-dialog-surface p-4 md:p-5 min-w-[18rem] max-w-md w-full" + with ( + ui.dialog() as dialog, + ui.card().classes( + "robin-dialog-surface p-4 md:p-5 min-w-[18rem] max-w-md w-full" + ), ): ui.label("Create user").classes( "classification-insight-heading text-headline-small q-mb-sm" ) username_input = ui.input("Username").classes("w-full").props("outlined dense") - password_input = ui.input("Password").classes("w-full").props( - "outlined dense type=password" + password_input = ( + ui.input("Password").classes("w-full").props("outlined dense type=password") + ) + confirm_input = ( + ui.input("Confirm password") + .classes("w-full") + .props("outlined dense type=password") ) - confirm_input = ui.input("Confirm password").classes("w-full").props( - "outlined dense type=password" + role_select = ( + ui.select(["user", "admin"], value="user", label="Role") + .classes("w-full") + .props("outlined dense") ) - role_select = ui.select(["user", "admin"], value="user", label="Role").classes( - "w-full" - ).props("outlined dense") - email_input = ui.input("Email (optional)").classes("w-full").props( - "outlined dense" + email_input = ( + ui.input("Email (optional)").classes("w-full").props("outlined dense") ) - clinical_role_input = ui.input("Clinical role (optional)").classes("w-full").props( - "outlined dense" + clinical_role_input = ( + ui.input("Clinical role (optional)") + .classes("w-full") + .props("outlined dense") ) approval_boxes: Dict[str, Any] = {} is_admin_role = {"value": str(role_select.value or "user") == "admin"} @@ -755,7 +847,9 @@ def _create() -> None: with ui.row().classes("w-full justify-end gap-2 mt-3"): ui.button("Cancel", on_click=dialog.close).props("flat no-caps outline") - ui.button("Create", on_click=_create, icon="check").props("color=primary no-caps") + ui.button("Create", on_click=_create, icon="check").props( + "color=primary no-caps" + ) dialog.open() @@ -772,31 +866,38 @@ def _open_manage_user_dialog( roles = store.get_user_roles(user.id) is_admin = store.user_has_role(user.id, "admin") - with ui.dialog() as dialog, ui.card().classes( - "robin-dialog-surface p-4 md:p-5 min-w-[18rem] max-w-md w-full" + with ( + ui.dialog() as dialog, + ui.card().classes( + "robin-dialog-surface p-4 md:p-5 min-w-[18rem] max-w-md w-full" + ), ): ui.label(f"Manage {username}").classes( "classification-insight-heading text-headline-small q-mb-sm" ) - ui.label(f"Roles: {', '.join(roles) or 'none'}").classes("classification-insight-foot") - ui.label( - f"Status: {'active' if user.is_active else 'inactive'}" - ).classes("classification-insight-foot q-mb-md") + ui.label(f"Roles: {', '.join(roles) or 'none'}").classes( + "classification-insight-foot" + ) + ui.label(f"Status: {'active' if user.is_active else 'inactive'}").classes( + "classification-insight-foot q-mb-md" + ) email_input = ui.input("Email").classes("w-full").props("outlined dense") email_input.value = user.metadata.get(EMAIL_KEY, "") - clinical_role_input = ui.input("Clinical role").classes("w-full").props( - "outlined dense" + clinical_role_input = ( + ui.input("Clinical role").classes("w-full").props("outlined dense") ) clinical_role_input.value = user.metadata.get(CLINICAL_ROLE_KEY, "") - notes_input = ui.textarea("Notes").classes("w-full").props("outlined dense autogrow") + notes_input = ( + ui.textarea("Notes").classes("w-full").props("outlined dense autogrow") + ) notes_input.value = user.metadata.get(NOTES_KEY, "") ui.label("Approvals").classes("classification-insight-meta font-medium mt-2") if is_admin: - ui.label( - "Administrators always have all approvals granted." - ).classes("classification-insight-foot q-mb-sm") + ui.label("Administrators always have all approvals granted.").classes( + "classification-insight-foot q-mb-sm" + ) approval_boxes: Dict[str, Any] = {} for field in USER_APPROVAL_FIELDS: approval_boxes[field.key] = ui.checkbox( @@ -807,11 +908,15 @@ def _open_manage_user_dialog( approval_boxes[field.key].set_value(True) approval_boxes[field.key].disable() - new_password = ui.input("New password (optional)").classes("w-full").props( - "outlined dense type=password" + new_password = ( + ui.input("New password (optional)") + .classes("w-full") + .props("outlined dense type=password") ) - confirm_password = ui.input("Confirm new password").classes("w-full").props( - "outlined dense type=password" + confirm_password = ( + ui.input("Confirm new password") + .classes("w-full") + .props("outlined dense type=password") ) def _reset_password() -> None: @@ -824,7 +929,9 @@ def _reset_password() -> None: ui.notify("Passwords do not match", type="negative") return new_hash = launcher.auth_service.hash_password(pwd) - if not store.set_user_password_hash(username, new_hash, must_change_password=True): + if not store.set_user_password_hash( + username, new_hash, must_change_password=True + ): ui.notify("Password update failed", type="negative") return launcher._audit_log( @@ -854,7 +961,9 @@ def _save_profile() -> None: if not store.update_user_metadata(username, metadata): ui.notify("Profile update failed", type="negative") return - if not is_admin and not store.update_user_approvals(username, approvals): + if not is_admin and not store.update_user_approvals( + username, approvals + ): ui.notify("Approvals update failed", type="negative") return except ValueError as exc: @@ -887,8 +996,13 @@ def _save_profile() -> None: def _toggle_active() -> None: if user.is_active: - if store.user_has_role(user.id, "admin") and store.count_active_admins() <= 1: - ui.notify("Cannot deactivate the last active admin", type="negative") + if ( + store.user_has_role(user.id, "admin") + and store.count_active_admins() <= 1 + ): + ui.notify( + "Cannot deactivate the last active admin", type="negative" + ) return if not store.set_user_active(username, False): ui.notify("Deactivate failed", type="negative") @@ -928,8 +1042,13 @@ def _grant_admin() -> None: on_changed() def _revoke_admin() -> None: - if store.user_has_role(user.id, "admin") and store.count_active_admins() <= 1: - ui.notify("Cannot revoke admin from the last active admin", type="negative") + if ( + store.user_has_role(user.id, "admin") + and store.count_active_admins() <= 1 + ): + ui.notify( + "Cannot revoke admin from the last active admin", type="negative" + ) return if not store.revoke_role(user.id, "admin"): ui.notify("User does not have admin role", type="warning") @@ -953,21 +1072,21 @@ def _revoke_admin() -> None: "flat no-caps outline" ) if user.is_active: - ui.button("Deactivate user", on_click=_toggle_active, icon="person_off").props( - "flat no-caps outline color=negative" - ) + ui.button( + "Deactivate user", on_click=_toggle_active, icon="person_off" + ).props("flat no-caps outline color=negative") else: - ui.button("Activate user", on_click=_toggle_active, icon="person").props( - "flat no-caps outline" - ) + ui.button( + "Activate user", on_click=_toggle_active, icon="person" + ).props("flat no-caps outline") if "admin" in roles: - ui.button("Revoke admin role", on_click=_revoke_admin, icon="shield").props( - "flat no-caps outline" - ) + ui.button( + "Revoke admin role", on_click=_revoke_admin, icon="shield" + ).props("flat no-caps outline") else: - ui.button("Grant admin role", on_click=_grant_admin, icon="shield").props( - "flat no-caps outline" - ) + ui.button( + "Grant admin role", on_click=_grant_admin, icon="shield" + ).props("flat no-caps outline") with ui.row().classes("w-full justify-end gap-2 mt-3"): ui.button("Close", on_click=dialog.close).props("flat no-caps outline") diff --git a/src/robin/gui/admin_sample_lifecycle.py b/src/robin/gui/admin_sample_lifecycle.py index d4e4f2c5..10e1fe36 100644 --- a/src/robin/gui/admin_sample_lifecycle.py +++ b/src/robin/gui/admin_sample_lifecycle.py @@ -137,9 +137,7 @@ def build_sample_lifecycle_panel(launcher: "GUILauncher") -> None: async def pick_archive_folder() -> None: start = ( - state.get("archive_destination") - or work_dir - or str(Path.home()) + state.get("archive_destination") or work_dir or str(Path.home()) ) picker = local_folder_picker(start, upper_limit=None) result = await picker @@ -202,7 +200,9 @@ def refresh_table() -> None: sample_table.rows = _lifecycle_rows(launcher) sample_table.update() - with ui.row().classes("w-full items-center justify-between gap-2 flex-wrap"): + with ui.row().classes( + "w-full items-center justify-between gap-2 flex-wrap" + ): ui.button("Refresh", icon="refresh", on_click=refresh_table).props( "flat no-caps outline" ) @@ -218,8 +218,11 @@ def refresh_table() -> None: on_click=lambda: None, ).props("color=negative no-caps") - with ui.dialog() as confirm_dialog, ui.card().classes( - "robin-dialog-surface p-4 md:p-5 min-w-[18rem] max-w-md" + with ( + ui.dialog() as confirm_dialog, + ui.card().classes( + "robin-dialog-surface p-4 md:p-5 min-w-[18rem] max-w-md" + ), ): confirm_title = ui.label("").classes( "text-headline-small text-slate-900 dark:text-slate-50" @@ -388,9 +391,7 @@ def _open_confirm(kind: str) -> None: if work_path is None: return try: - dest = validate_archive_destination( - Path(dest_raw), work_path - ) + dest = validate_archive_destination(Path(dest_raw), work_path) except ValueError as exc: ui.notify(str(exc), type="warning") return diff --git a/src/robin/gui/app.py b/src/robin/gui/app.py index 72c7257c..ea4fba75 100644 --- a/src/robin/gui/app.py +++ b/src/robin/gui/app.py @@ -1,7 +1,7 @@ from __future__ import annotations -from typing import Any, Optional, Dict from pathlib import Path +from typing import Any, Dict, Optional try: from nicegui import ui diff --git a/src/robin/gui/change_password.py b/src/robin/gui/change_password.py index 18076a48..8f1b9ae5 100644 --- a/src/robin/gui/change_password.py +++ b/src/robin/gui/change_password.py @@ -22,12 +22,17 @@ def create_change_password_page( user_id = launcher._get_current_user_id() username = launcher._get_current_username() or "" forced = bool( - user_id is not None and launcher.security_store.user_must_change_password(user_id) + user_id is not None + and launcher.security_store.user_must_change_password(user_id) ) if voluntary and forced: voluntary = False - safe_target = redirect_to if redirect_to and redirect_to not in ("/login", "/change-password") else "/" + safe_target = ( + redirect_to + if redirect_to and redirect_to not in ("/login", "/change-password") + else "/" + ) with theme.frame( "R.O.B.I.N - Change password", @@ -36,7 +41,9 @@ def create_change_password_page( center=launcher.center, setup_notifications=launcher._setup_notification_system, ): - with ui.element("div").classes("w-full min-w-0").props("id=change-password-page"): + with ( + ui.element("div").classes("w-full min-w-0").props("id=change-password-page") + ): with ui.column().classes( "w-full max-w-md mx-auto items-center justify-center min-h-[60vh] p-4 gap-3" ): @@ -65,28 +72,39 @@ def create_change_password_page( current_input = ( ui.input("Current password") .classes("w-full") - .props("outlined dense type=password autocomplete=current-password") + .props( + "outlined dense type=password autocomplete=current-password" + ) ) new_input = ( ui.input("New password") .classes("w-full") - .props("outlined dense type=password autocomplete=new-password") + .props( + "outlined dense type=password autocomplete=new-password" + ) ) confirm_input = ( ui.input("Confirm new password") .classes("w-full") - .props("outlined dense type=password autocomplete=new-password") + .props( + "outlined dense type=password autocomplete=new-password" + ) ) def _submit() -> None: if user_id is None: - ui.notify("Session expired. Please sign in again.", type="negative") + ui.notify( + "Session expired. Please sign in again.", + type="negative", + ) ui.navigate.to("/login") return new_password = str(new_input.value or "") confirm = str(confirm_input.value or "") if new_password != confirm: - ui.notify("New passwords do not match", type="negative") + ui.notify( + "New passwords do not match", type="negative" + ) return current_password: Optional[str] = None if current_input is not None: diff --git a/src/robin/gui/components/__init__.py b/src/robin/gui/components/__init__.py index 8229d4fa..37ba8fc4 100644 --- a/src/robin/gui/components/__init__.py +++ b/src/robin/gui/components/__init__.py @@ -1,6 +1,6 @@ -#from .coverage import add_coverage_section -#from .cnv import add_cnv_section -#from .mgmt import add_mgmt_section -#from .classification import add_classification_section -#from .fusion import add_fusion_section -#from .summary import add_summary_section \ No newline at end of file +# from .coverage import add_coverage_section +# from .cnv import add_cnv_section +# from .mgmt import add_mgmt_section +# from .classification import add_classification_section +# from .fusion import add_fusion_section +# from .summary import add_summary_section diff --git a/src/robin/gui/components/bed_coverage.py b/src/robin/gui/components/bed_coverage.py index beb495e6..6d662349 100644 --- a/src/robin/gui/components/bed_coverage.py +++ b/src/robin/gui/components/bed_coverage.py @@ -1,15 +1,15 @@ from __future__ import annotations -from typing import Any, Dict, Optional -from pathlib import Path -import logging import json +import logging import os -from datetime import datetime -import threading import queue +import threading +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, Optional -from robin.gui.theme import get_user_dark_mode, client_timer, stop_timer +from robin.gui.theme import client_timer, get_user_dark_mode, stop_timer try: from nicegui import ui @@ -124,7 +124,9 @@ def _load_visualization_data(sample_dir: Path) -> Optional[Dict[str, Any]]: return { "series": series, "entry_count": len(entries), - "point_count": sum(len(s.get("data") or []) for s in series), + "point_count": sum( + len(s.get("data") or []) for s in series + ), } except Exception as e: logger.warning(f"Could not rebuild BED coverage viz from log: {e}") @@ -217,13 +219,13 @@ def _update_ui(result: Optional[Dict[str, Any]]) -> None: n_series = len(chart_data.get("series") or []) n_points = chart_data.get("point_count") if n_points is None: - n_points = sum(len(s.get("data") or []) for s in (chart_data.get("series") or [])) + n_points = sum( + len(s.get("data") or []) for s in (chart_data.get("series") or []) + ) n_entries = chart_data.get("entry_count") x_mode = chart_data.get("x_axis_mode") or "wall_clock" if state["bed_summary"]: - bits = [ - f"{n_series} BED type{'s' if n_series != 1 else ''} tracked" - ] + bits = [f"{n_series} BED type{'s' if n_series != 1 else ''} tracked"] if n_entries: bits.append(f"{n_entries} master generations logged") if n_points: @@ -266,7 +268,9 @@ def _update_ui(result: Optional[Dict[str, Any]]) -> None: state["chart"].options["xAxis"]["type"] = "time" state["chart"].options["xAxis"]["name"] = "Time" state["chart"].options["xAxis"].pop("min", None) - state["chart"].options["xAxis"].pop("axisLabel", None) # Update yAxis to format values as percentages + state["chart"].options["xAxis"].pop( + "axisLabel", None + ) # Update yAxis to format values as percentages if "yAxis" in state["chart"].options: state["chart"].options["yAxis"]["axisLabel"] = { "formatter": "{value}%" @@ -336,7 +340,10 @@ def _do_heavy_work() -> Optional[Dict[str, Any]]: return {"mtime": None, "chart_data": None} # Skip if mtime hasn't changed and we already have data - if current_mtime == state.get("mtime") and state.get("chart_data") is not None: + if ( + current_mtime == state.get("mtime") + and state.get("chart_data") is not None + ): logger.debug("[BED Coverage] Data unchanged, skipping update") return None # Signal to skip UI update @@ -385,12 +392,18 @@ def _run_update(): state["update_in_progress"] = False # Build UI — design.md §9: insight shell, cards, explicit .body--dark CSS - with ui.element("div").classes("w-full min-w-0").props("id=analysis-detail-bed-coverage"): + with ( + ui.element("div") + .classes("w-full min-w-0") + .props("id=analysis-detail-bed-coverage") + ): with ui.element("div").classes("classification-insight-shell w-full min-w-0"): ui.label("BED coverage").classes( "classification-insight-heading text-headline-small" ) - with ui.element("div").classes("classification-insight-card w-full min-w-0"): + with ui.element("div").classes( + "classification-insight-card w-full min-w-0" + ): with ui.column().classes("w-full min-w-0 gap-2 p-2 md:p-3"): with ui.row().classes("items-center gap-2 min-w-0"): ui.icon("layers").classes("classification-insight-icon") diff --git a/src/robin/gui/components/classification.py b/src/robin/gui/components/classification.py index 12182472..a6249ec4 100644 --- a/src/robin/gui/components/classification.py +++ b/src/robin/gui/components/classification.py @@ -1,11 +1,11 @@ from __future__ import annotations import asyncio -from pathlib import Path -from typing import Any, Dict, List, Optional import csv import logging import zlib +from pathlib import Path +from typing import Any, Dict, List, Optional # Classification charts — design.md §8.8 (ECharts; brand / diagnostic palette) _BRAND_GREEN = "#10b981" @@ -146,7 +146,9 @@ def _apply_palette_to_ts_chart(ts: Any, palette: Dict[str, str]) -> None: def _active_tumour_palette() -> List[str]: """Categorical bar/line colours: deep on light UI, brighter on dark UI.""" - return _CLASS_TUMOUR_PALETTE_DARK if _is_dark_mode() else _CLASS_TUMOUR_PALETTE_LIGHT + return ( + _CLASS_TUMOUR_PALETTE_DARK if _is_dark_mode() else _CLASS_TUMOUR_PALETTE_LIGHT + ) def _tumour_category_color(index: int) -> str: @@ -161,6 +163,7 @@ def _color_for_class_name(name: str) -> str: raw = zlib.adler32(name.encode("utf-8", errors="replace")) & 0xFFFFFFFF return pal[raw % len(pal)] + try: from robin.classification_config import ( CLASSIFIER_CONFIDENCE_THRESHOLDS, @@ -303,9 +306,9 @@ def _bar_chart_mark_line( # Import section visibility helpers try: from robin.gui.config import ( + CLASSIFICATION_STEPS, get_visible_classification_steps, launcher_visibility_context, - CLASSIFICATION_STEPS, ) except ImportError: get_visible_classification_steps = lambda *a, **k: {"sturgeon", "nanodx", "random_forest", "pannanodx", "marlin", "lamprey", "tucan"} # type: ignore[assignment] @@ -330,7 +333,7 @@ def add_classification_section(sample_dir: Path, launcher: Any = None) -> None: ) if not enabled_classification_steps: return - + # Map workflow step names to tool display names tool_to_step_map = { "Sturgeon": "sturgeon", @@ -341,9 +344,11 @@ def add_classification_section(sample_dir: Path, launcher: Any = None) -> None: "Lamprey (research)": "lamprey", "Tucan": "tucan", } - - with ui.element("div").classes("classification-insight-shell w-full min-w-0").props( - "id=classification-section" + + with ( + ui.element("div") + .classes("classification-insight-shell w-full min-w-0") + .props("id=classification-section") ): ui.label("Classification").classes( "classification-insight-heading text-headline-small" @@ -366,7 +371,7 @@ def add_classification_section(sample_dir: Path, launcher: Any = None) -> None: exp = ( ui.expansion(tool_name, icon="analytics") .classes("w-full") - .props(f'id=classification-detail-{tool_step}') + .props(f"id=classification-detail-{tool_step}") ) with exp: summary_labels = None @@ -425,7 +430,12 @@ def add_classification_section(sample_dir: Path, launcher: Any = None) -> None: "conf": ndx_conf, "probes": ndx_feats, } - elif tool_name in ("Random Forest", "MARLIN", "Lamprey (research)", "Tucan"): + elif tool_name in ( + "Random Forest", + "MARLIN", + "Lamprey (research)", + "Tucan", + ): with ui.element("div").classes( "classification-insight-card w-full min-w-0 mb-2" ): @@ -457,7 +467,9 @@ def add_classification_section(sample_dir: Path, launcher: Any = None) -> None: ui.label(f"{tool_name} current classification").classes( "classification-insight-meta" ) - _thr = _confidence_thresholds_for_classifier(tool_to_step_map[tool_name]) + _thr = _confidence_thresholds_for_classifier( + tool_to_step_map[tool_name] + ) pal = _echart_surface_palette() bar = ui.echart( { @@ -527,9 +539,7 @@ def add_classification_section(sample_dir: Path, launcher: Any = None) -> None: ], "media": _echart_media_responsive_bar(), } - ).classes( - "w-full min-h-[300px] h-[340px] sm:min-h-[320px] sm:h-80" - ) + ).classes("w-full min-h-[300px] h-[340px] sm:min-h-[320px] sm:h-80") ui.label(f"{tool_name} confidence over time").classes( "classification-insight-meta mt-2" ) @@ -606,9 +616,7 @@ def add_classification_section(sample_dir: Path, launcher: Any = None) -> None: "series": [], "media": _echart_media_responsive_ts(pal), } - ).classes( - "w-full min-h-[320px] h-[380px] sm:min-h-[280px] sm:h-72" - ) + ).classes("w-full min-h-[320px] h-[380px] sm:min-h-[280px] sm:h-72") charts[tool_name] = { "bar": bar, "ts": ts, @@ -652,8 +660,8 @@ def _sync_classification_theme(force: bool = False) -> None: pass from robin.gui.theme import ( - register_theme_sync_callback, client_timer, + register_theme_sync_callback, stop_timer, ) @@ -734,7 +742,12 @@ def _sort_key(p): number_probes: Optional[int] = None try: lr = rows[-1] - for nk in ("number_probes", "Number_probes", "covered_cpgs", "probes"): + for nk in ( + "number_probes", + "Number_probes", + "covered_cpgs", + "probes", + ): raw = lr.get(nk) if raw is not None and str(raw).strip() != "": number_probes = int(float(raw)) @@ -765,7 +778,9 @@ def _bar_values_to_rich_data( bar_color = _BRAND_GREEN else: lab = labels[i] if i < len(labels) else "" - bar_color = _color_for_class_name(lab) if lab else _tumour_category_color(i) + bar_color = ( + _color_for_class_name(lab) if lab else _tumour_category_color(i) + ) out.append( { "value": v, @@ -797,7 +812,11 @@ def _update_charts_from_file( if not file_path or not file_path.exists(): return mode = charts[tool_name].get("mode", "percent") - data = preloaded if preloaded is not None else _read_scores_csv(file_path, mode) + data = ( + preloaded + if preloaded is not None + else _read_scores_csv(file_path, mode) + ) if not data: return bar = charts[tool_name]["bar"] @@ -1007,9 +1026,7 @@ def _line_pts(k: str) -> Any: npv = data.get("number_probes") if npv is not None: label = ( - "Features" - if tool_name == "Random Forest" - else "Probes" + "Features" if tool_name == "Random Forest" else "Probes" ) labels_map["probes"].set_text(f"{label}: {int(npv)}") except Exception: @@ -1030,7 +1047,9 @@ def _refresh_classification_sync_impl() -> None: """Refresh classification data (synchronous; for contexts without asyncio loop).""" try: if not sample_dir or not sample_dir.exists(): - logging.warning(f"[Classification] Sample directory not found: {sample_dir}") + logging.warning( + f"[Classification] Sample directory not found: {sample_dir}" + ) return import time @@ -1091,7 +1110,9 @@ async def _refresh_classification_async() -> None: """Parse score CSVs off the event loop, then update ECharts on the main thread.""" try: if not sample_dir or not sample_dir.exists(): - logging.warning(f"[Classification] Sample directory not found: {sample_dir}") + logging.warning( + f"[Classification] Sample directory not found: {sample_dir}" + ) return import time @@ -1146,9 +1167,7 @@ async def _refresh_classification_async() -> None: mode = charts[tool_name].get("mode", "percent") data = await asyncio.to_thread(_read_scores_csv, file_path, mode) if data: - _update_charts_from_file( - tool_name, cfg["file"], preloaded=data - ) + _update_charts_from_file(tool_name, cfg["file"], preloaded=data) if tool_name in file_mtimes: charts[tool_name]["last_mtime"] = file_mtimes[tool_name] except Exception as e: @@ -1163,9 +1182,11 @@ def _refresh_classification() -> None: return asyncio.create_task(_refresh_classification_async()) - def _check_and_update_file(tool_name: str, filename: str, file_path: Path, charts: Dict[str, Any]) -> None: + def _check_and_update_file( + tool_name: str, filename: str, file_path: Path, charts: Dict[str, Any] + ) -> None: """Check file and update charts if needed. - + Note: This function is now called only when the file has changed, so we can skip the mtime check here (it's done upstream). """ @@ -1176,7 +1197,9 @@ def _check_and_update_file(tool_name: str, filename: str, file_path: Path, chart pass # Start the refresh timer (every 30 seconds) - refresh_timer = client_timer(30.0, _refresh_classification, active=True, immediate=False) + refresh_timer = client_timer( + 30.0, _refresh_classification, active=True, immediate=False + ) # Initial refresh after the page is rendered client_timer(0.5, _refresh_classification, once=True) try: diff --git a/src/robin/gui/components/cnv.py b/src/robin/gui/components/cnv.py index 30ce58f2..8c535dab 100644 --- a/src/robin/gui/components/cnv.py +++ b/src/robin/gui/components/cnv.py @@ -1,17 +1,17 @@ from __future__ import annotations -from typing import Any, Dict, List, Optional, Sequence, Tuple -from pathlib import Path - import asyncio +import importlib.resources as importlib_resources import json -import natsort -import numpy as np -from functools import lru_cache import logging import pickle import time -import importlib.resources as importlib_resources +from functools import lru_cache +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence, Tuple + +import natsort +import numpy as np import pandas as pd try: @@ -19,19 +19,6 @@ except ImportError: # pragma: no cover ui = None -from robin.gui.theme import ( - styled_table, - register_theme_sync_callback, - get_user_dark_mode, - client_timer, - stop_timer, -) -from robin.analysis.cnv_classification import ( - CNVEvent, - detect_cnv_events, - format_cnv_events_card_lines, - format_cnv_events_section_summary, -) from robin.analysis.cnv_analysis import ( compute_cnv_log2_from_ploidy, downsample_cnv_for_plot, @@ -40,6 +27,12 @@ resolve_cnv_calling_track, resolve_cnv_plot_bin_width, ) +from robin.analysis.cnv_classification import ( + CNVEvent, + detect_cnv_events, + format_cnv_events_card_lines, + format_cnv_events_section_summary, +) from robin.analysis.cnv_regional import ( SIGNIFICANT_CNV_STATES, analyze_cytoband_cnv, @@ -50,11 +43,16 @@ ) from robin.analysis.itd_work import load_gene_target_coverage from robin.classification_config import get_cnv_thresholds +from robin.gui.theme import ( + client_timer, + get_user_dark_mode, + register_theme_sync_callback, + stop_timer, + styled_table, +) # Same chromosome set as reporting (plotting.py): chr0–chr22, chrX, chrY only -CNV_PLOT_CONTIGS = frozenset( - ["chr" + str(i) for i in range(0, 23)] + ["chrX", "chrY"] -) +CNV_PLOT_CONTIGS = frozenset(["chr" + str(i) for i in range(0, 23)] + ["chrX", "chrY"]) _CNV_PLOT_BIN_KEY_DEFAULT = "Data default" _CNV_PLOT_BIN_OPTIONS = { @@ -156,7 +154,10 @@ def _load_cnv_gene_locations(gene_names: tuple[str, ...]) -> tuple[Dict[str, Any return tuple( sorted( intervals.values(), - key=lambda row: (order.get(str(row["gene"]).casefold(), 10**9), row["chrom"]), + key=lambda row: ( + order.get(str(row["gene"]).casefold(), 10**9), + row["chrom"], + ), ) ) @@ -432,7 +433,11 @@ def _build_configured_gene_coverage_points( for row in _configured_genes_on_chrom(gene_locations, selected): gene = str(row["gene"]) chrom = str(row["chrom"]) - if selected == "All" and chrom not in chrom_offsets and chrom not in abs_plot_map: + if ( + selected == "All" + and chrom not in chrom_offsets + and chrom not in abs_plot_map + ): continue coverage = cov_lookup.get(gene.casefold()) if coverage is None or not np.isfinite(coverage): @@ -838,7 +843,11 @@ def _apply_cnv_abs_y_window(chart: Any, y_lo: float, y_hi: float) -> None: """Pin the abs-chart Y axis and slider to a marker-aware window.""" try: dz_list = chart.options.get("dataZoom") - if isinstance(dz_list, list) and len(dz_list) > 1 and isinstance(dz_list[1], dict): + if ( + isinstance(dz_list, list) + and len(dz_list) > 1 + and isinstance(dz_list[1], dict) + ): dz_list[1]["startValue"] = float(y_lo) dz_list[1]["endValue"] = float(y_hi) dz_list[1]["filterMode"] = "none" @@ -1442,9 +1451,7 @@ def _apply_cnv_echart_chrome(echart: Any, dark: bool) -> None: dz[":labelFormatter"] = _CNV_Y_VALUE_FORMATTER_JS dz["borderColor"] = p["axis_line"] dz["fillerColor"] = ( - "rgba(51, 65, 85, 0.35)" - if dark - else "rgba(148, 163, 184, 0.2)" + "rgba(51, 65, 85, 0.35)" if dark else "rgba(148, 163, 184, 0.2)" ) dz["handleStyle"] = { "color": p["text"], @@ -1498,7 +1505,9 @@ def add_cnv_section(launcher: Any, sample_dir: Path) -> None: configured_gene_locations = _load_cnv_gene_locations(configured_gene_names) if configured_gene_names: found = {str(row["gene"]).casefold() for row in configured_gene_locations} - missing = [name for name in configured_gene_names if name.casefold() not in found] + missing = [ + name for name in configured_gene_names if name.casefold() not in found + ] if missing: logging.warning( "No packaged GRCh38 location found for configured CNV genes: %s", @@ -1540,9 +1549,7 @@ def add_cnv_section(launcher: Any, sample_dir: Path) -> None: cnv_var = ui.label("Variance: --").classes( "classification-insight-meta" ) - with ui.row().classes( - "w-full gap-3 items-center mb-2 flex-wrap mt-2" - ): + with ui.row().classes("w-full gap-3 items-center mb-2 flex-wrap mt-2"): ui.label("Chromosome").classes("classification-insight-meta") cnv_chrom_select = ui.select(options={"All": "All"}, value="All").style( "width: 160px" @@ -1590,9 +1597,7 @@ def add_cnv_section(launcher: Any, sample_dir: Path) -> None: == _CNV_GENE_COVERAGE_FILTER_OUTLIERS, ) .props("dense") - .tooltip( - "Left: all configured genes · Right: outliers only" - ) + .tooltip("Left: all configured genes · Right: outliers only") ) cnv_gene_cov_filter.value = ( _gene_cov_filter == _CNV_GENE_COVERAGE_FILTER_OUTLIERS @@ -1625,9 +1630,7 @@ def add_cnv_section(launcher: Any, sample_dir: Path) -> None: cnv_scale = ( ui.switch(value=_y_scale == "log") .props("dense") - .tooltip( - "Left: linear ploidy · Right: log2(ploidy / expected)" - ) + .tooltip("Left: linear ploidy · Right: log2(ploidy / expected)") ) cnv_scale.value = _y_scale == "log" ui.label("Log2").classes("classification-insight-meta") @@ -1636,12 +1639,16 @@ def add_cnv_section(launcher: Any, sample_dir: Path) -> None: options=_CNV_PLOT_BIN_OPTIONS, value=_CNV_PLOT_BIN_KEY_DEFAULT, ).style("width: 120px") - cnv_bp_label = ui.label("Breakpoints").classes( - "classification-insight-meta ml-2" - ).style("display: none") - with ui.row().classes("items-center gap-1").style( - "display: none" - ) as cnv_bp_row: + cnv_bp_label = ( + ui.label("Breakpoints") + .classes("classification-insight-meta ml-2") + .style("display: none") + ) + with ( + ui.row() + .classes("items-center gap-1") + .style("display: none") as cnv_bp_row + ): ui.label("Hide").classes("classification-insight-meta") cnv_bp = ( ui.switch(value=bool(_cnv_ui_state.get("show_bp", True))) @@ -1660,9 +1667,7 @@ def add_cnv_section(launcher: Any, sample_dir: Path) -> None: cnv_height = ( ui.switch(value=_double_height) .props("dense") - .tooltip( - "Left: standard scatter height · Right: double height" - ) + .tooltip("Left: standard scatter height · Right: double height") ) cnv_height.value = _double_height ui.label("Tall").classes("classification-insight-meta") @@ -1670,7 +1675,11 @@ def add_cnv_section(launcher: Any, sample_dir: Path) -> None: cnv_abs = ui.echart( { "backgroundColor": "transparent", - "title": {"text": "CNV scatter plot", "left": "center", "top": 10}, + "title": { + "text": "CNV scatter plot", + "left": "center", + "top": 10, + }, "grid": { "left": "5%", "right": "5%", @@ -1701,7 +1710,12 @@ def add_cnv_section(launcher: Any, sample_dir: Path) -> None: }, ], "series": [ - {"type": "scatter", "name": "CNV", "symbolSize": 3, "data": []}, + { + "type": "scatter", + "name": "CNV", + "symbolSize": 3, + "data": [], + }, { "type": "scatter", "name": "centromeres_highlight", @@ -1728,11 +1742,17 @@ def add_cnv_section(launcher: Any, sample_dir: Path) -> None: ).classes( f"w-full {_CNV_ABS_HEIGHT_CLASS_TALL if _double_height else _CNV_ABS_HEIGHT_CLASS} cnv-genome-abs-chart" ) - with ui.element("div").classes("w-full target-coverage-panel__plot-wrap mt-2"): + with ui.element("div").classes( + "w-full target-coverage-panel__plot-wrap mt-2" + ): cnv_diff = ui.echart( { "backgroundColor": "transparent", - "title": {"text": "Difference plot", "left": "center", "top": 10}, + "title": { + "text": "Difference plot", + "left": "center", + "top": 10, + }, "grid": { "left": "5%", "right": "5%", @@ -1804,7 +1824,12 @@ def add_cnv_section(launcher: Any, sample_dir: Path) -> None: ) regional_cnv_columns = [ {"name": "chrom", "label": "Chr", "field": "chrom", "sortable": True}, - {"name": "region", "label": "Region", "field": "region", "sortable": True}, + { + "name": "region", + "label": "Region", + "field": "region", + "sortable": True, + }, { "name": "start_mb", "label": "Start (Mb)", @@ -1843,10 +1868,15 @@ def add_cnv_section(launcher: Any, sample_dir: Path) -> None: {"name": "panel_genes", "label": "Panel genes", "field": "panel_genes"}, ] _, regional_cnv_table = styled_table( - columns=regional_cnv_columns, rows=[], pagination=20, class_size="table-xs" + columns=regional_cnv_columns, + rows=[], + pagination=20, + class_size="table-xs", ) try: - regional_cnv_table.props('multi-sort rows-per-page-options="[10,20,50,0]"') + regional_cnv_table.props( + 'multi-sort rows-per-page-options="[10,20,50,0]"' + ) except Exception: pass @@ -1857,18 +1887,64 @@ def add_cnv_section(launcher: Any, sample_dir: Path) -> None: cnv_events_summary = ui.label("No CNV events detected").classes( "classification-insight-meta mb-2" ) - + # CNV Events Table cnv_events_columns = [ - {"name": "chromosome", "label": "Chr", "field": "chromosome", "sortable": True}, - {"name": "event_type", "label": "Event Type", "field": "event_type", "sortable": True}, + { + "name": "chromosome", + "label": "Chr", + "field": "chromosome", + "sortable": True, + }, + { + "name": "event_type", + "label": "Event Type", + "field": "event_type", + "sortable": True, + }, {"name": "arm", "label": "Arm", "field": "arm", "sortable": True}, - {"name": "start_mb", "label": "Start (Mb)", "field": "start_mb", "sortable": True, "align": "right"}, - {"name": "end_mb", "label": "End (Mb)", "field": "end_mb", "sortable": True, "align": "right"}, - {"name": "length_mb", "label": "Length (Mb)", "field": "length_mb", "sortable": True, "align": "right"}, - {"name": "mean_cnv_str", "label": "Mean CNV", "field": "mean_cnv_str", "sortable": True, "align": "right"}, - {"name": "confidence", "label": "Confidence", "field": "confidence", "sortable": True, "align": "center"}, - {"name": "proportion_affected", "label": "% Affected", "field": "proportion_affected", "sortable": True, "align": "right"}, + { + "name": "start_mb", + "label": "Start (Mb)", + "field": "start_mb", + "sortable": True, + "align": "right", + }, + { + "name": "end_mb", + "label": "End (Mb)", + "field": "end_mb", + "sortable": True, + "align": "right", + }, + { + "name": "length_mb", + "label": "Length (Mb)", + "field": "length_mb", + "sortable": True, + "align": "right", + }, + { + "name": "mean_cnv_str", + "label": "Mean CNV", + "field": "mean_cnv_str", + "sortable": True, + "align": "right", + }, + { + "name": "confidence", + "label": "Confidence", + "field": "confidence", + "sortable": True, + "align": "center", + }, + { + "name": "proportion_affected", + "label": "% Affected", + "field": "proportion_affected", + "sortable": True, + "align": "right", + }, {"name": "genes_str", "label": "Genes", "field": "genes_str"}, ] _, cnv_events_table = styled_table( @@ -1995,7 +2071,7 @@ def _thin_chart_series(chart, max_points: int = MAX_POINTS_PER_CHART) -> None: return # Allocate budgets proportional to visible counts with a small floor budgets = [] - + for sub in vis_data: share = int(max(1, round((len(sub) / total) * max_points))) budgets.append(share) @@ -2070,12 +2146,16 @@ def _load_cytobands_df() -> pd.DataFrame: ) return df except Exception: - return pd.DataFrame(columns=["chrom", "start_pos", "end_pos", "name", "stain"]) + return pd.DataFrame( + columns=["chrom", "start_pos", "end_pos", "name", "stain"] + ) @lru_cache(maxsize=1) def _load_centromere_bed_df() -> pd.DataFrame: try: - res_path = importlib_resources.files("robin.resources") / "cenSatRegions.bed" + res_path = ( + importlib_resources.files("robin.resources") / "cenSatRegions.bed" + ) return pd.read_csv( res_path, sep="\t", @@ -2118,6 +2198,7 @@ def _load_gene_bed(sample_dir: Path = None) -> pd.DataFrame: master_csv_path = sample_dir / "master.csv" if master_csv_path.exists(): import pandas as pd + df = pd.read_csv(master_csv_path) if not df.empty and "analysis_panel" in df.columns: panel_val = df.iloc[0]["analysis_panel"] @@ -2125,7 +2206,7 @@ def _load_gene_bed(sample_dir: Path = None) -> pd.DataFrame: panel = str(panel_val).strip() except Exception: pass - + # Map panel to BED filename bed_filename = None if not panel: @@ -2138,7 +2219,7 @@ def _load_gene_bed(sample_dir: Path = None) -> pd.DataFrame: else: # Check for custom panel bed_filename = f"{panel}_panel_name_uniq.bed" - + # Try to load the panel-specific BED file try: res_path = importlib_resources.files("robin.resources") / bed_filename @@ -2151,10 +2232,12 @@ def _load_gene_bed(sample_dir: Path = None) -> pd.DataFrame: ) except Exception: pass - + # Fallback to unique_genes.bed if panel-specific file not found try: - res_path = importlib_resources.files("robin.resources") / "unique_genes.bed" + res_path = ( + importlib_resources.files("robin.resources") / "unique_genes.bed" + ) if res_path.exists(): return pd.read_csv( res_path, @@ -2164,10 +2247,10 @@ def _load_gene_bed(sample_dir: Path = None) -> pd.DataFrame: ) except Exception: pass - + except Exception: pass - + return pd.DataFrame(columns=["chrom", "start_pos", "end_pos", "gene"]) def _sex_label(xy_val: Any) -> str: @@ -2229,6 +2312,7 @@ def _compute_all_cytoband_df( if frames: out = pd.concat(frames, ignore_index=True) if not out.empty: + def _rank(label: Any) -> int: try: s = str(label) @@ -2284,7 +2368,7 @@ def _update_cnv_events_analysis(state: Dict[str, Any]) -> None: # Load cytobands and genes cyto_df = _load_cytobands_df() gene_df = _load_gene_bed(sample_dir) - + # Detect CNV events using centralized rules events = detect_cnv_events( cnv_data=data, @@ -2295,21 +2379,23 @@ def _update_cnv_events_analysis(state: Dict[str, Any]) -> None: support_cnv_data=analysis_log2, support_bin_width=int(binw), ) - + # Update events table events_rows = [] for event in events: event_dict = event.to_dict() # Format proportion as percentage - event_dict["proportion_affected"] = f"{event.proportion_affected:.1%}" + event_dict["proportion_affected"] = ( + f"{event.proportion_affected:.1%}" + ) events_rows.append(event_dict) - + cnv_events_table.rows = events_rows try: cnv_events_table.update() except Exception: pass - + # Update summaries (insight card + events section) whole_text, arm_text = format_cnv_events_card_lines(events) cnv_whole_chr_summary.set_text(whole_text) @@ -2376,11 +2462,7 @@ def _render_cnv_from_state(state: Dict[str, Any]) -> None: pass selected = state.get("selected_chrom", "All") use_log = state.get("y_scale", "linear") == "log" - abs_plot_map = ( - cnv_log2_map - if use_log and cnv_log2_map - else cnv_map - ) + abs_plot_map = cnv_log2_map if use_log and cnv_log2_map else cnv_map raw_color_mode = state.get("color_mode", "chromosome") # normalize color mode to expected keys lval = str(raw_color_mode).strip().lower() @@ -2411,9 +2493,9 @@ def _render_cnv_from_state(state: Dict[str, Any]) -> None: cnv_abs.options["yAxis"][0]["name"] = "Log2 ratio (ploidy / expected)" cnv_abs.options["title"]["text"] = "CNV scatter plot" cnv_abs.options["title"]["top"] = 4 - cnv_abs.options["title"]["subtext"] = ( - "log2(ploidy / expected copy number); 0 = normal" - ) + cnv_abs.options["title"][ + "subtext" + ] = "log2(ploidy / expected copy number); 0 = normal" cnv_abs.options["grid"]["top"] = "26%" else: cnv_abs.options["yAxis"][0]["name"] = "Ploidy" @@ -2471,9 +2553,7 @@ def _apply_abs_gene_coverage_overlay() -> None: ) if filter_mode not in _CNV_GENE_COVERAGE_FILTERS: filter_mode = _CNV_GENE_COVERAGE_FILTER_OUTLIERS - plot_map = ( - abs_plot_map if isinstance(abs_plot_map, dict) else cnv_map - ) + plot_map = abs_plot_map if isinstance(abs_plot_map, dict) else cnv_map points, _mean_cov = _build_configured_gene_coverage_points( configured_gene_locations, selected=selected, @@ -2546,18 +2626,14 @@ def _apply_abs_gene_coverage_overlay() -> None: chrom_bounds.append((contig, start_bp, end_bp)) offset_bp = end_bp if color_mode == "chromosome": - ci = len( - [s for s in series_abs if s.get("type") == "scatter"] - ) + ci = len([s for s in series_abs if s.get("type") == "scatter"]) series_abs.append( { "type": "scatter", "name": contig, "symbolSize": 3, "itemStyle": { - "color": chrom_palette[ - ci % len(chrom_palette) - ] + "color": chrom_palette[ci % len(chrom_palette)] }, "data": pts, } @@ -2831,7 +2907,7 @@ def _clear_overlays(chart): else np.array([]) ) band_areas = [] - + # Get CNV events for this chromosome to highlight significant events events = [] try: @@ -2860,28 +2936,32 @@ def _clear_overlays(chart): ) except Exception: pass - + # Create event lookup for highlighting event_regions = {} for event in events: key = f"{event.start_pos}-{event.end_pos}" event_regions[key] = event - + for _, row in bands.iterrows(): - s_bp, e_bp = int(row["start_pos"]), int(row["end_pos"]) + s_bp, e_bp = int(row["start_pos"]), int( + row["end_pos"] + ) s_bin = max(0, s_bp // binw_analysis) - e_bin = min(len(vals) - 1, max(0, e_bp // binw_analysis)) + e_bin = min( + len(vals) - 1, max(0, e_bp // binw_analysis) + ) if len(vals) > 0 and e_bin >= s_bin: mean_val = float( np.mean(vals[s_bin : e_bin + 1]) ) else: mean_val = 0.0 - + # Check if this region has a significant CNV event region_key = f"{s_bp}-{e_bp}" event = event_regions.get(region_key) - + fill_neutral = ( "rgba(255, 255, 255, 0.07)" if dark_ui @@ -2889,9 +2969,15 @@ def _clear_overlays(chart): ) if event: # Highlight significant events with stronger colors - if event.event_type in ("GAIN", "WHOLE_CHR_GAIN"): + if event.event_type in ( + "GAIN", + "WHOLE_CHR_GAIN", + ): color = "rgba(52, 199, 89, 0.3)" # gains - elif event.event_type in ("LOSS", "WHOLE_CHR_LOSS"): + elif event.event_type in ( + "LOSS", + "WHOLE_CHR_LOSS", + ): color = "rgba(255, 45, 85, 0.3)" # losses else: color = fill_neutral @@ -2914,9 +3000,7 @@ def _clear_overlays(chart): "show": True, "position": "insideTop", "color": ( - "#cbd5e1" - if dark_ui - else "#555" + "#cbd5e1" if dark_ui else "#555" ), "fontSize": 11, }, @@ -2996,7 +3080,7 @@ def _clear_overlays(chart): chrom_offsets=chrom_offsets, dark=dark_ui, ) - + # Breakpoint candidates as dashed vertical lines try: idx_cyto_abs = next( @@ -3013,7 +3097,9 @@ def _clear_overlays(chart): ): arr = state["bp_array"] pos = [ - int(r["end_pos"]) for r in arr if r["name"] == selected + int(r["end_pos"]) + for r in arr + if r["name"] == selected ] lines = [ { @@ -3043,9 +3129,9 @@ def _clear_overlays(chart): # Apply gene zoom before updating chart try: - sel_gene = launcher._cnv_state.setdefault( - str(sample_dir), {} - ).get("selected_gene", "All") + sel_gene = launcher._cnv_state.setdefault(str(sample_dir), {}).get( + "selected_gene", "All" + ) gene_interval = None if sel_gene and sel_gene != "All" and selected != "All": @@ -3084,7 +3170,10 @@ def _clear_overlays(chart): else: # Reset zoom when "All" is selected try: - if isinstance(cnv_abs.options.get("dataZoom"), list) and cnv_abs.options["dataZoom"]: + if ( + isinstance(cnv_abs.options.get("dataZoom"), list) + and cnv_abs.options["dataZoom"] + ): dz = cnv_abs.options["dataZoom"][0] dz.pop("startValue", None) dz.pop("endValue", None) @@ -3142,9 +3231,9 @@ def _clear_overlays(chart): # Gene zoom on difference chart (single-chromosome view) try: - sel_gene = launcher._cnv_state.setdefault( - str(sample_dir), {} - ).get("selected_gene", "All") + sel_gene = launcher._cnv_state.setdefault(str(sample_dir), {}).get( + "selected_gene", "All" + ) for rel_chart in (cnv_diff,): gene_interval = None if sel_gene and sel_gene != "All" and selected != "All": @@ -3216,7 +3305,9 @@ def _clear_overlays(chart): f"{int(binw)}:{sex_lbl}" ) if state.get("cyto_cache_key") != cache_key: - panel_name, panel_genes_df = load_panel_gene_bed(str(sample_dir)) + panel_name, panel_genes_df = load_panel_gene_bed( + str(sample_dir) + ) df_all = _compute_all_cytoband_df(data, int(binw), sex_lbl) state["cyto_df_all"] = df_all state["panel_name"] = panel_name @@ -3269,9 +3360,7 @@ def _clear_overlays(chart): regional_cnv_table.update() except Exception: pass - regional_cnv_summary.set_text( - "No regional CNV events detected" - ) + regional_cnv_summary.set_text("No regional CNV events detected") else: regional_cnv_summary.set_text("CNV data not available") except Exception: @@ -3331,7 +3420,9 @@ def _prepare_cnv_refresh() -> Optional[Dict[str, Any]]: want_scale = "linear" else: want_scale = ( - "log" if "log" in scale_key else state.get("y_scale", "linear") + "log" + if "log" in scale_key + else state.get("y_scale", "linear") ) if want_scale != state.get("y_scale"): state["y_scale"] = want_scale @@ -3412,8 +3503,12 @@ def _prepare_cnv_refresh() -> Optional[Dict[str, Any]]: cnv_npy_mtime = cnv_npy.stat().st_mtime if cnv_npy.exists() else 0 cnv2_npy_mtime = cnv2_npy.stat().st_mtime if cnv2_npy.exists() else 0 cnv3_npy_mtime = cnv3_npy.stat().st_mtime if cnv3_npy.exists() else 0 - cnv_dict_npy_mtime = cnv_dict_npy.stat().st_mtime if cnv_dict_npy.exists() else 0 - data_array_npy_mtime = data_array_npy.stat().st_mtime if data_array_npy.exists() else 0 + cnv_dict_npy_mtime = ( + cnv_dict_npy.stat().st_mtime if cnv_dict_npy.exists() else 0 + ) + data_array_npy_mtime = ( + data_array_npy.stat().st_mtime if data_array_npy.exists() else 0 + ) xy_pkl_mtime = xy_pkl.stat().st_mtime if xy_pkl.exists() else 0 prev_cnv_npy_mtime = state.get("cnv_m", 0) @@ -3735,9 +3830,9 @@ def _apply_cnv_refresh_after_load( arr = np.load(data_array_npy, allow_pickle=True) if hasattr(arr, "dtype") and "name" in arr.dtype.names: state["bp_array"] = arr - selected = launcher._cnv_state.setdefault( - str(sample_dir), {} - ).get("selected_chrom", "All") + selected = launcher._cnv_state.setdefault(str(sample_dir), {}).get( + "selected_chrom", "All" + ) breakpoint_lines = [] for r in arr: if selected == "All" or r["name"] == selected: @@ -3753,9 +3848,9 @@ def _apply_cnv_refresh_after_load( elif data_array_npy.exists() and not data_array_npy_changed: if state.get("bp_array") is not None: try: - selected = launcher._cnv_state.setdefault( - str(sample_dir), {} - ).get("selected_chrom", "All") + selected = launcher._cnv_state.setdefault(str(sample_dir), {}).get( + "selected_chrom", "All" + ) arr = state["bp_array"] breakpoint_lines = [] for r in arr: @@ -3845,10 +3940,10 @@ def _update_breakpoints_visibility() -> None: key = str(sample_dir) state = launcher._cnv_state.get(key, {}) selected = state.get("selected_chrom", "All") - + # Show breakpoints controls only when viewing individual chromosomes should_show = selected != "All" - + try: display_value = "block" if should_show else "none" cnv_bp_label.style(f"display: {display_value}") @@ -3956,7 +4051,9 @@ def _apply_cnv_abs_height(tall: bool) -> None: try: cnv_abs.run_chart_method("resize") except Exception: - logging.debug("CNV scatter resize after height change failed", exc_info=True) + logging.debug( + "CNV scatter resize after height change failed", exc_info=True + ) def _on_height(ev): st = launcher._cnv_state.setdefault(str(sample_dir), {}) @@ -3971,7 +4068,9 @@ def _on_height(ev): def _on_color(ev): st = launcher._cnv_state.setdefault(str(sample_dir), {}) - st["color_mode"] = "value" if _switch_bool(ev, default=False) else "chromosome" + st["color_mode"] = ( + "value" if _switch_bool(ev, default=False) else "chromosome" + ) try: if isinstance(getattr(cnv_color, "value", None), bool): st["color_mode"] = "value" if cnv_color.value else "chromosome" @@ -3983,7 +4082,11 @@ def _on_color(ev): def _on_plot_bin(ev): st = launcher._cnv_state.setdefault(str(sample_dir), {}) - v = getattr(ev, "args", None) if hasattr(ev, "args") else getattr(ev, "value", None) + v = ( + getattr(ev, "args", None) + if hasattr(ev, "args") + else getattr(ev, "value", None) + ) if v is None and hasattr(ev, "value"): v = ev.value st["plot_bin_width"] = _cnv_plot_bin_bp_from_ui(v) diff --git a/src/robin/gui/components/coverage.py b/src/robin/gui/components/coverage.py index f1c8021e..e3c4d58f 100644 --- a/src/robin/gui/components/coverage.py +++ b/src/robin/gui/components/coverage.py @@ -1,35 +1,34 @@ from __future__ import annotations -import asyncio -from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple import asyncio import json -import time +import logging import os +import time +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple import natsort import numpy as np import pandas as pd -import logging try: - from nicegui import ui, app + from nicegui import app, ui except ImportError: # pragma: no cover ui = None app = None from robin.gui.theme import ( - styled_table, - register_theme_sync_callback, - get_user_dark_mode, client_timer, + get_user_dark_mode, + register_theme_sync_callback, stop_timer, + styled_table, ui_element_exists, ) - from robin.reference_contigs import is_visible_contig + # Shared paged table renderer to avoid materializing full row lists for large DataFrames. def _render_paged_df_table( df: pd.DataFrame, @@ -138,13 +137,16 @@ def _apply_search(term: str) -> None: search_input = ui.input(placeholder=search_placeholder).props( "type=search dense clearable" ) - search_input.on("update:model-value", lambda e: _apply_search(getattr(e, "value", ""))) + search_input.on( + "update:model-value", lambda e: _apply_search(getattr(e, "value", "")) + ) for col in table.columns: col["sortable"] = False _fill_from_pagination(init_pagination) try: + def _cleanup_table_state() -> None: page_state["filtered_positions"] = [] table.rows = [] @@ -154,6 +156,7 @@ def _cleanup_table_state() -> None: pass return table + # --- Coverage charts (design.md §9.5): on/off colours, mean line, outlier palette --- _COV_OUTLIER_LINE_PALETTE_LIGHT = [ "#059669", # emerald-600 @@ -205,7 +208,12 @@ def _cov_per_timestamp_envelope( """ low_data: list[list[float]] = [] span_data: list[list[float]] = [] - if df is None or df.empty or "timestamp" not in df.columns or "coverage" not in df.columns: + if ( + df is None + or df.empty + or "timestamp" not in df.columns + or "coverage" not in df.columns + ): return low_data, span_data for ts, timepoint in df.groupby("timestamp", sort=True): @@ -649,7 +657,8 @@ def _apply_target_coverage_time_analysis_chrome(ec: Any) -> None: ): # Outlier target lines: solid brand colours + luminous hint (dark) lc = (s.get("lineStyle") or {}).get("color") or ( - (s.get("itemStyle") or {}).get("color")) + (s.get("itemStyle") or {}).get("color") + ) ols = { "width": 2, "type": "solid", @@ -751,13 +760,17 @@ def add_igv_viewer(launcher: Any, sample_dir: Path) -> None: # IGV viewer section - only show if target.bam exists target_bam = sample_dir / "target.bam" if not (target_bam.exists() and target_bam.is_file()): - with ui.element("div").classes( - "classification-insight-shell w-full min-w-0" - ).props("id=sample-details-igv"): + with ( + ui.element("div") + .classes("classification-insight-shell w-full min-w-0") + .props("id=sample-details-igv") + ): ui.label("IGV browser").classes( "classification-insight-heading text-headline-small" ) - with ui.element("div").classes("classification-insight-card w-full min-w-0"): + with ui.element("div").classes( + "classification-insight-card w-full min-w-0" + ): with ui.column().classes("w-full min-w-0 gap-2 p-2 md:p-3"): with ui.row().classes("items-center gap-2 min-w-0"): ui.icon("biotech").classes("classification-insight-icon") @@ -769,9 +782,11 @@ def add_igv_viewer(launcher: Any, sample_dir: Path) -> None: ).classes("classification-insight-foot") return - with ui.element("div").classes( - "classification-insight-shell w-full min-w-0" - ).props("id=sample-details-igv"): + with ( + ui.element("div") + .classes("classification-insight-shell w-full min-w-0") + .props("id=sample-details-igv") + ): ui.label("IGV browser").classes( "classification-insight-heading text-headline-small" ) @@ -1303,12 +1318,16 @@ def _poll_new_browser_ready(attempt: int = 0) -> None: try: existing = ui.run_javascript(js_check_existing, timeout=5.0) if existing: - if state.get("igv_initialized") and state.get("igv_browser_ready"): + if state.get("igv_initialized") and state.get( + "igv_browser_ready" + ): # Browser is ready, just add the track pass # Will fall through to track loading else: # Wait a bit more and check again - ui.timer(1.0, lambda: _load_igv_bam(bam_path), once=True) + ui.timer( + 1.0, lambda: _load_igv_bam(bam_path), once=True + ) return except Exception: pass @@ -1581,16 +1600,26 @@ async def _reload_bam_track(): if bam_path and bam_path.exists(): igv_status.set_text("Checking if BAM file is ready...") if not await asyncio.to_thread(_wait_for_bam_ready, bam_path): - igv_status.set_text("BAM file is still being updated. Please wait and try again.") - ui.notify("BAM file is still being updated. Please wait and try again.", type="warning") + igv_status.set_text( + "BAM file is still being updated. Please wait and try again." + ) + ui.notify( + "BAM file is still being updated. Please wait and try again.", + type="warning", + ) return # Also check if BAI file is ready bai_path = bam_path.with_suffix(bam_path.suffix + ".bai") if bai_path.exists(): if not await asyncio.to_thread(_wait_for_bam_ready, bai_path): - igv_status.set_text("BAM index is still being updated. Please wait and try again.") - ui.notify("BAM index is still being updated. Please wait and try again.", type="warning") + igv_status.set_text( + "BAM index is still being updated. Please wait and try again." + ) + ui.notify( + "BAM index is still being updated. Please wait and try again.", + type="warning", + ) return # JavaScript to reload the BAM track @@ -1718,25 +1747,31 @@ def _load_target_bed(): master_csv_path = sample_dir / "master.csv" if master_csv_path.exists(): import pandas as pd + df = pd.read_csv(master_csv_path) if not df.empty and "analysis_panel" in df.columns: target_panel = str(df.iloc[0]["analysis_panel"]).strip() # Map panel to BED filename bed_file_mapping = { - "rCNS2": "rCNS2_panel_name_uniq.bed", - "AML": "AML_panel_name_uniq.bed", - "Sarcoma": "Sarcoma_panel_name_uniq.bed" + "rCNS2": "rCNS2_panel_name_uniq.bed", + "AML": "AML_panel_name_uniq.bed", + "Sarcoma": "Sarcoma_panel_name_uniq.bed", } - bed_filename = bed_file_mapping.get(target_panel, f"{target_panel}_panel_name_uniq.bed") + bed_filename = bed_file_mapping.get( + target_panel, f"{target_panel}_panel_name_uniq.bed" + ) # Try to find the BED file in robin resources try: from robin import resources + bed_file_path = os.path.join( - os.path.dirname(os.path.abspath(resources.__file__)), - bed_filename + os.path.dirname( + os.path.abspath(resources.__file__) + ), + bed_filename, ) if not os.path.exists(bed_file_path): bed_file_path = None @@ -1746,10 +1781,10 @@ def _load_target_bed(): # Fallback paths if not bed_file_path: possible_paths = [ - bed_filename, - f"data/{bed_filename}", - f"/usr/local/share/{bed_filename}", - ] + bed_filename, + f"data/{bed_filename}", + f"/usr/local/share/{bed_filename}", + ] for path in possible_paths: if os.path.exists(path): bed_file_path = path @@ -1758,8 +1793,13 @@ def _load_target_bed(): print(f"Error reading panel information: {e}") if not bed_file_path or not os.path.exists(bed_file_path): - igv_status.set_text(f"Could not find BED file for panel: {target_panel}") - ui.notify(f"Target BED file not found for panel: {target_panel}", type="warning") + igv_status.set_text( + f"Could not find BED file for panel: {target_panel}" + ) + ui.notify( + f"Target BED file not found for panel: {target_panel}", + type="warning", + ) return # Mount the BED file's directory @@ -2126,9 +2166,7 @@ def chr_key(label: str) -> int: bed_df = bed_df.copy() bed_df["length"] = (bed_df["endpos"] - bed_df["startpos"] + 1).astype(float) grouped = ( - bed_df.groupby("chrom") - .agg({"bases": "sum", "length": "sum"}) - .reset_index() + bed_df.groupby("chrom").agg({"bases": "sum", "length": "sum"}).reset_index() ) grouped["meandepth"] = grouped["bases"] / grouped["length"] name_col = "#rname" if "#rname" in cov_df.columns else "rname" @@ -2150,9 +2188,11 @@ def chr_key(label: str) -> int: temp_df_grouped = temp_df.groupby(name_col)["meandepth"].mean() off_target_data = [ - float(temp_df_grouped.get(chrom, 0.0)) - if pd.notna(temp_df_grouped.get(chrom, 0.0)) - else 0.0 + ( + float(temp_df_grouped.get(chrom, 0.0)) + if pd.notna(temp_df_grouped.get(chrom, 0.0)) + else 0.0 + ) for chrom in names ] @@ -2183,7 +2223,9 @@ def _compute_boxplot_chart_data( df["length"] = (df["endpos"] - df["startpos"] + 1).astype(float) df["coverage"] = df["bases"] / df["length"] chrom_str = df["chrom"].astype(str).str.strip() - df["chrom"] = chrom_str.where(chrom_str.str.startswith("chr"), "chr" + chrom_str) + df["chrom"] = chrom_str.where( + chrom_str.str.startswith("chr"), "chr" + chrom_str + ) df = df[ df["chrom"] .astype(str) @@ -2211,10 +2253,9 @@ def _compute_boxplot_chart_data( ) agg["chrom"] = pd.Categorical(agg["chrom"], categories=chroms, ordered=True) agg = agg.sort_values("chrom").reset_index(drop=True) - result = ( - [["chrom", "min", "Q1", "median", "Q3", "max", "chrom_index"]] - + agg.values.tolist() - ) + result = [ + ["chrom", "min", "Q1", "median", "Q3", "max", "chrom_index"] + ] + agg.values.tolist() def iqr_bounds(sub: pd.DataFrame) -> Tuple[float, float]: q1 = np.percentile(sub["coverage"], 25) @@ -2405,8 +2446,7 @@ def _coverage_load_refresh_data( cov_df = cov_df.copy() with np.errstate(divide="ignore", invalid="ignore"): cov_df["meandepth"] = ( - cov_df["covbases"] - / cov_df["endpos"].replace(0, np.nan) + cov_df["covbases"] / cov_df["endpos"].replace(0, np.nan) ).fillna(0) state_updates["cov_df"] = cov_df loaded_cov_df = True @@ -2517,11 +2557,7 @@ def _coverage_load_refresh_data( bdf["length"] = (bdf["endpos"] - bdf["startpos"] + 1).astype(float) if bdf["length"].sum() > 0: target_cov_v = float(bdf["bases"].sum()) / float(bdf["length"].sum()) - if ( - global_cov is not None - and target_cov_v is not None - and global_cov > 0 - ): + if global_cov is not None and target_cov_v is not None and global_cov > 0: enrich_v = target_cov_v / global_cov quality: Dict[str, Any] | None = None @@ -2583,7 +2619,12 @@ def add_coverage_section(launcher: Any, sample_dir: Path) -> None: This mirrors the existing inline implementation but lives in a reusable module. """ # Check for development environment variable to show/hide testing features - is_development_mode = os.environ.get("ROBIN_DEV_MODE", "").lower() in ("1", "true", "yes", "on") + is_development_mode = os.environ.get("ROBIN_DEV_MODE", "").lower() in ( + "1", + "true", + "yes", + "on", + ) from robin.gui.plotting_preferences import resolve_plotting_reference_contig_scope @@ -2618,58 +2659,58 @@ def add_coverage_section(launcher: Any, sample_dir: Path) -> None: # Per Chromosome Target Coverage (grouped bar — design.md §9.5) _cp0 = _cov_chrome_palette() echart_target_cov = ui.echart( - { - "backgroundColor": "transparent", - "textStyle": {"color": _cp0["axis"]}, - "title": { - "text": "Per Chromosome Target Coverage", - "left": "center", - "top": 10, - "textStyle": {"fontSize": 15, "color": _cp0["title"]}, - }, - "legend": { - "data": ["On Target", "Off Target"], - "left": 10, - "top": "center", - "orient": "vertical", - "itemGap": 10, - "textStyle": {"color": _cp0["legend"]}, - }, - "tooltip": { - "trigger": "axis", - "axisPointer": {"type": "shadow"}, - **_cov_tooltip_option(), - }, - "grid": { - "left": "12%", - "right": "8%", - "bottom": "12%", - "top": "20%", - "containLabel": True, - }, - "xAxis": { - "type": "category", - "data": [], - "axisLabel": { - "rotate": 0, - "interval": 0, - "fontSize": 11, - "color": _cp0["axis"], - }, - "axisLine": {"lineStyle": {"color": _cp0["axis"]}}, + { + "backgroundColor": "transparent", + "textStyle": {"color": _cp0["axis"]}, + "title": { + "text": "Per Chromosome Target Coverage", + "left": "center", + "top": 10, + "textStyle": {"fontSize": 15, "color": _cp0["title"]}, + }, + "legend": { + "data": ["On Target", "Off Target"], + "left": 10, + "top": "center", + "orient": "vertical", + "itemGap": 10, + "textStyle": {"color": _cp0["legend"]}, + }, + "tooltip": { + "trigger": "axis", + "axisPointer": {"type": "shadow"}, + **_cov_tooltip_option(), + }, + "grid": { + "left": "12%", + "right": "8%", + "bottom": "12%", + "top": "20%", + "containLabel": True, + }, + "xAxis": { + "type": "category", + "data": [], + "axisLabel": { + "rotate": 0, + "interval": 0, + "fontSize": 11, + "color": _cp0["axis"], }, - "yAxis": { - "type": "value", - "name": "Coverage (×)", - "nameTextStyle": {"color": _cp0["axis"]}, - "axisLabel": {"color": _cp0["axis"]}, - "splitLine": { - "lineStyle": {"color": _cp0["split"], "type": "dashed"}, - }, + "axisLine": {"lineStyle": {"color": _cp0["axis"]}}, + }, + "yAxis": { + "type": "value", + "name": "Coverage (×)", + "nameTextStyle": {"color": _cp0["axis"]}, + "axisLabel": {"color": _cp0["axis"]}, + "splitLine": { + "lineStyle": {"color": _cp0["split"], "type": "dashed"}, }, - "series": [], - } - ).classes("w-full h-64") + }, + "series": [], + } + ).classes("w-full h-64") with ui.card().classes("w-full"): ui.label("Coverage Over Time").classes("text-lg font-semibold mb-2") @@ -2736,7 +2777,9 @@ def add_coverage_section(launcher: Any, sample_dir: Path) -> None: # Target Coverage Over Time Analysis with ui.card().classes("w-full mt-4"): - ui.label("Target Coverage Over Time Analysis").classes("text-lg font-semibold mb-2") + ui.label("Target Coverage Over Time Analysis").classes( + "text-lg font-semibold mb-2" + ) ui.label( "Signal-first view of target coverage over time: a population mean and ±2σ envelope, " "ghosted in-range targets, and highlighted outliers. Outliers are points where a target’s " @@ -2778,7 +2821,9 @@ def _plot_target_coverage_over_time(): if not time_coverage_file.exists(): with container: container.clear() - ui.label("No target_coverage_time.csv file found.").classes("text-gray-600") + ui.label("No target_coverage_time.csv file found.").classes( + "text-gray-600" + ) return # Load data @@ -2786,16 +2831,20 @@ def _plot_target_coverage_over_time(): if df.empty: with container: container.clear() - ui.label("No data available in target_coverage_time.csv").classes("text-gray-600") + ui.label( + "No data available in target_coverage_time.csv" + ).classes("text-gray-600") return - if not {'chrom', 'startpos', 'endpos', 'name'}.issubset(df.columns): + if not {"chrom", "startpos", "endpos", "name"}.issubset(df.columns): with container: container.clear() - ui.label("target_coverage_time.csv missing chrom/startpos/endpos/name columns").classes("text-gray-600") + ui.label( + "target_coverage_time.csv missing chrom/startpos/endpos/name columns" + ).classes("text-gray-600") return # Convert timestamp to datetime (milliseconds to datetime) - df['datetime'] = pd.to_datetime(df['timestamp'], unit='ms') + df["datetime"] = pd.to_datetime(df["timestamp"], unit="ms") # Create unique target identifier (chrom, startpos, endpos) — vectorized df["target_key"] = list( @@ -2817,8 +2866,12 @@ def _plot_target_coverage_over_time(): ) # Calculate mean coverage per timepoint - mean_coverage = df.groupby('timestamp')['coverage'].mean().reset_index() - mean_coverage['datetime'] = pd.to_datetime(mean_coverage['timestamp'], unit='ms') + mean_coverage = ( + df.groupby("timestamp")["coverage"].mean().reset_index() + ) + mean_coverage["datetime"] = pd.to_datetime( + mean_coverage["timestamp"], unit="ms" + ) # Detect outliers using standard deviation method (Z-score) def detect_outliers_sd(series, num_sd=2.0): @@ -2848,14 +2901,16 @@ def detect_outliers_sd(series, num_sd=2.0): # Calculate global statistics across all genes at each timepoint # This allows us to detect genes that are outliers relative to the population - for timestamp in df['timestamp'].unique(): - timepoint_data = df[df['timestamp'] == timestamp].copy() - if len(timepoint_data) < 3: # Need at least 3 genes to calculate SD + for timestamp in df["timestamp"].unique(): + timepoint_data = df[df["timestamp"] == timestamp].copy() + if ( + len(timepoint_data) < 3 + ): # Need at least 3 genes to calculate SD continue # Calculate mean and SD across all genes at this timepoint - global_mean = timepoint_data['coverage'].mean() - global_std = timepoint_data['coverage'].std() + global_mean = timepoint_data["coverage"].mean() + global_std = timepoint_data["coverage"].std() if global_std == 0: # All genes have same coverage continue @@ -2864,25 +2919,41 @@ def detect_outliers_sd(series, num_sd=2.0): lower_bound = global_mean - 2.0 * global_std upper_bound = global_mean + 2.0 * global_std - outlier_mask = (timepoint_data['coverage'] < lower_bound) | (timepoint_data['coverage'] > upper_bound) + outlier_mask = (timepoint_data["coverage"] < lower_bound) | ( + timepoint_data["coverage"] > upper_bound + ) outlier_points = timepoint_data[outlier_mask] for _, row in outlier_points.iterrows(): - tk = row['target_key'] - tk = (str(tk[0]), int(tk[1]), int(tk[2])) if isinstance(tk, (list, tuple)) else (str(row['chrom']), int(row['startpos']), int(row['endpos'])) - outliers.append({ - 'target_key': tk, - 'target_label': row['target_label'], - 'gene': row['name'], - 'timestamp': row['timestamp'], - 'datetime': row['datetime'], - 'coverage': row['coverage'], - 'reads': row['reads'], - 'reads_per_length': row['reads_per_length'], - 'type': 'high' if row['coverage'] > global_mean else 'low', - 'global_mean': global_mean, - 'global_std': global_std - }) + tk = row["target_key"] + tk = ( + (str(tk[0]), int(tk[1]), int(tk[2])) + if isinstance(tk, (list, tuple)) + else ( + str(row["chrom"]), + int(row["startpos"]), + int(row["endpos"]), + ) + ) + outliers.append( + { + "target_key": tk, + "target_label": row["target_label"], + "gene": row["name"], + "timestamp": row["timestamp"], + "datetime": row["datetime"], + "coverage": row["coverage"], + "reads": row["reads"], + "reads_per_length": row["reads_per_length"], + "type": ( + "high" + if row["coverage"] > global_mean + else "low" + ), + "global_mean": global_mean, + "global_std": global_std, + } + ) outliers_df = pd.DataFrame(outliers) if outliers else pd.DataFrame() @@ -2892,12 +2963,12 @@ def detect_outliers_sd(series, num_sd=2.0): # Preserve order: (target_key, target_label) for consistent display seen = set() for _, row in outliers_df.iterrows(): - key = row['target_key'] + key = row["target_key"] if isinstance(key, (list, tuple)): key = tuple(key) if key not in seen: seen.add(key) - outlier_targets.append((key, row['target_label'])) + outlier_targets.append((key, row["target_label"])) try: outlier_limit = int(outlier_limit_state["value"]) @@ -2959,7 +3030,9 @@ def _sort_key(item: Any) -> float: ) targets_to_plot = outlier_targets_sorted[:outlier_limit] - for idx, (target_key, target_label) in enumerate(targets_to_plot): + for idx, (target_key, target_label) in enumerate( + targets_to_plot + ): key = ( tuple(target_key) if isinstance(target_key, (list, tuple)) @@ -3077,9 +3150,7 @@ def _sort_key(item: Any) -> float: ] dim_gene_series.append( { - "name": _cov_legend_label( - f"{target_label} · in-range" - ), + "name": _cov_legend_label(f"{target_label} · in-range"), "type": "line", "smooth": 0.42, "data": series_data, @@ -3111,9 +3182,15 @@ def _sort_key(item: Any) -> float: # Summary statistics with ui.row().classes("w-full mb-4 gap-3"): - ui.label(f"Total timepoints: {len(mean_coverage)}").classes("text-sm") - ui.label(f"Total targets: {len(df['target_key'].unique())}").classes("text-sm") - ui.label(f"Outliers detected: {len(outliers_df)}").classes("text-sm") + ui.label(f"Total timepoints: {len(mean_coverage)}").classes( + "text-sm" + ) + ui.label( + f"Total targets: {len(df['target_key'].unique())}" + ).classes("text-sm") + ui.label(f"Outliers detected: {len(outliers_df)}").classes( + "text-sm" + ) # Chart (design.md §9.5.C — signal-first trend, vertical scrubber, right legend) _cp_t = _cov_chrome_palette() @@ -3200,13 +3277,18 @@ def _sort_key(item: Any) -> float: # Show summary of outlier targets if outlier_targets: outlier_count = len(outlier_targets) - ui.label(f"Showing profiles for {min(outlier_count, outlier_limit)} outlier targets (out of {outlier_count} total)").classes("text-sm text-gray-600 mt-2") + ui.label( + f"Showing profiles for {min(outlier_count, outlier_limit)} outlier targets (out of {outlier_count} total)" + ).classes("text-sm text-gray-600 mt-2") else: - ui.label("No significant outliers detected.").classes("text-gray-600 mt-2") + ui.label("No significant outliers detected.").classes( + "text-gray-600 mt-2" + ) except Exception as e: logging.error(f"Error plotting target coverage over time: {e}") import traceback + logging.error(traceback.format_exc()) if ui_element_exists(container): with container: @@ -3222,7 +3304,9 @@ def _set_outlier_limit(e) -> None: except (TypeError, ValueError): outlier_limit_state["value"] = 10 outlier_limit_state["value"] = max(1, outlier_limit_state["value"]) - coverage_state["target_cov_outlier_limit"] = outlier_limit_state["value"] + coverage_state["target_cov_outlier_limit"] = outlier_limit_state[ + "value" + ] _plot_target_coverage_over_time() # Outlier limit (plot updates automatically when target_coverage_time.csv changes) @@ -3237,8 +3321,10 @@ def _set_outlier_limit(e) -> None: on_change=_set_outlier_limit, ).props("dense").classes("w-24") - with ui.column().classes("w-full target-coverage-panel").props( - "id=analysis-detail-target-coverage" + with ( + ui.column() + .classes("w-full target-coverage-panel") + .props("id=analysis-detail-target-coverage") ): # Add target panel legend def _get_target_panel_info(): @@ -3248,10 +3334,15 @@ def _get_target_panel_info(): master_csv_path = sample_dir / "master.csv" if master_csv_path.exists(): import pandas as pd + df = pd.read_csv(master_csv_path) if not df.empty and "analysis_panel" in df.columns: panel = df.iloc[0]["analysis_panel"] - if panel and str(panel).strip() != "" and str(panel).strip().lower() != "nan": + if ( + panel + and str(panel).strip() != "" + and str(panel).strip().lower() != "nan" + ): return str(panel).strip() # Fallback 1: Try to detect panel from BED files in the sample directory @@ -3268,7 +3359,9 @@ def _get_target_panel_info(): return "Sarcoma" # Fallback 2: Try to detect from target analysis output files - target_files = list(sample_dir.glob("*target*.csv")) + list(sample_dir.glob("*coverage*.csv")) + target_files = list(sample_dir.glob("*target*.csv")) + list( + sample_dir.glob("*coverage*.csv") + ) if target_files: # This is a heuristic - if we have target analysis files, # we can assume it's likely a known panel @@ -3276,7 +3369,11 @@ def _get_target_panel_info(): return "" # No panel found except Exception as e: - _log_notify(f"Exception in _get_target_panel_info: {e}", level="error", notify=False) + _log_notify( + f"Exception in _get_target_panel_info: {e}", + level="error", + notify=False, + ) return "" # No default fallback target_panel = _get_target_panel_info() @@ -3286,15 +3383,22 @@ def _get_target_panel_info(): "rCNS2": ("bg-blue-100", "text-blue-800", "rCNS2 Panel"), "AML": ("bg-green-100", "text-green-800", "AML Panel"), "Sarcoma": ("bg-orange-100", "text-orange-800", "Sarcoma Panel"), - "Unknown Panel": ("bg-yellow-100", "text-yellow-800", "Unknown Panel") + "Unknown Panel": ("bg-yellow-100", "text-yellow-800", "Unknown Panel"), } if not target_panel: # No panel found - show warning - panel_color_classes, panel_text_classes, panel_display_name = ("bg-red-100", "text-red-800", "Panel Not Found") + panel_color_classes, panel_text_classes, panel_display_name = ( + "bg-red-100", + "text-red-800", + "Panel Not Found", + ) else: - panel_color_classes, panel_text_classes, panel_display_name = panel_colors.get( - target_panel, ("bg-gray-100", "text-gray-800", f"{target_panel} Panel") + panel_color_classes, panel_text_classes, panel_display_name = ( + panel_colors.get( + target_panel, + ("bg-gray-100", "text-gray-800", f"{target_panel} Panel"), + ) ) # Store panel information in state for use by other functions @@ -3307,16 +3411,24 @@ def _get_target_panel_info(): "w-full items-center justify-between mb-2 gap-2 flex-wrap" ): ui.label("Target Coverage").classes("text-lg font-semibold") - target_coverage_back_button = ui.button( - "← Back to overview", - on_click=lambda: _show_target_coverage_overview(), - ).props("flat dense no-caps outline").classes("hidden") + target_coverage_back_button = ( + ui.button( + "← Back to overview", + on_click=lambda: _show_target_coverage_overview(), + ) + .props("flat dense no-caps outline") + .classes("hidden") + ) with ui.row().classes("w-full items-center gap-2 mb-2"): ui.label("Panel:").classes("text-sm font-medium text-gray-600") - ui.label(panel_display_name).classes(f"px-2 py-1 rounded text-sm font-medium {panel_color_classes} {panel_text_classes}") + ui.label(panel_display_name).classes( + f"px-2 py-1 rounded text-sm font-medium {panel_color_classes} {panel_text_classes}" + ) ui.label("•").classes("text-gray-400") - ui.label("Target regions defined by gene panel").classes("text-xs text-gray-500") + ui.label("Target regions defined by gene panel").classes( + "text-xs text-gray-500" + ) # Add detailed panel information in an expansion with ui.expansion().classes("w-full mb-2").props("icon=info dense"): @@ -3339,7 +3451,10 @@ def _get_target_panel_info(): if bed_filename and bed_filename != "Unknown": try: from robin import resources - resources_dir = os.path.dirname(os.path.abspath(resources.__file__)) + + resources_dir = os.path.dirname( + os.path.abspath(resources.__file__) + ) bed_path = os.path.join(resources_dir, bed_filename) if not os.path.exists(bed_path): # File doesn't exist, but we'll still show the expected filename @@ -3364,9 +3479,11 @@ def _get_target_panel_info(): "rCNS2": "Central Nervous System genes (244 regions)", "AML": "Acute Myeloid Leukemia genes (1,181 regions)", "Sarcoma": "Sarcoma-specific gene panel", - "Unknown Panel": "Panel type could not be determined" + "Unknown Panel": "Panel type could not be determined", } - ui.label(panel_descriptions.get(target_panel, "Custom gene panel")).classes("text-xs") + ui.label( + panel_descriptions.get(target_panel, "Custom gene panel") + ).classes("text-xs") # Define helper functions before chart creation def _show_chromosome_scatter(chromosome: str) -> None: @@ -3388,7 +3505,9 @@ def _show_chromosome_scatter(chromosome: str) -> None: chrom_data = df[df["chrom"].astype(str) == chromosome].copy() if chrom_data.empty: - ui.notify(f"No data found for chromosome {chromosome}", type="warning") + ui.notify( + f"No data found for chromosome {chromosome}", type="warning" + ) return # Sort by position for better visualization @@ -3409,11 +3528,17 @@ def _show_chromosome_scatter(chromosome: str) -> None: label = f"{name} ({start_mb:.2f}-{end_mb:.2f} Mb)" else: label = name - scatter_data.append([label, row["coverage"], row["startpos"], row["endpos"]]) + scatter_data.append( + [label, row["coverage"], row["startpos"], row["endpos"]] + ) # Update chart to show scatter plot - target_boxplot.options["title"]["text"] = f"Gene Coverage - {chromosome}" - target_boxplot.options["title"]["subtext"] = f"{len(scatter_data)} targets" + target_boxplot.options["title"][ + "text" + ] = f"Gene Coverage - {chromosome}" + target_boxplot.options["title"][ + "subtext" + ] = f"{len(scatter_data)} targets" # Update x-axis to show gene names (with position when duplicated) gene_names = [d[0] for d in scatter_data] @@ -3435,16 +3560,13 @@ def _show_chromosome_scatter(chromosome: str) -> None: "symbolSize": 8, "universalTransition": True, # Enable universal transition "animationDurationUpdate": 1000, # Set transition duration - "itemStyle": { - "color": "#3b82f6", - "opacity": 0.7 - }, + "itemStyle": {"color": "#3b82f6", "opacity": 0.7}, "emphasis": { "itemStyle": { "color": "#1d4ed8", "opacity": 1, "borderColor": _cp_sc["title"], - "borderWidth": 2 + "borderWidth": 2, } }, "label": { @@ -3456,7 +3578,7 @@ def _show_chromosome_scatter(chromosome: str) -> None: }, "tooltip": { ":formatter": "function(params) { const data = params.data; return 'Gene: ' + data[0] + '
Coverage: ' + Number(data[1]).toFixed(2) + 'x
Position: ' + data[2].toLocaleString() + '-' + data[3].toLocaleString(); }" - } + }, } ] @@ -3471,10 +3593,18 @@ def _show_chromosome_scatter(chromosome: str) -> None: except Exception: target_coverage_back_button.set_visibility(True) - _log_notify(f"Showing gene coverage for {chromosome}", level="info", notify=False) + _log_notify( + f"Showing gene coverage for {chromosome}", + level="info", + notify=False, + ) except Exception as e: - _log_notify(f"Failed to show chromosome scatter: {e}", level="error", notify=True) + _log_notify( + f"Failed to show chromosome scatter: {e}", + level="error", + notify=True, + ) async def _show_target_coverage_overview_async() -> None: """Return to the original box plot overview (CSV + chart off event loop).""" @@ -3535,13 +3665,24 @@ def _show_target_coverage_overview() -> None: def handle_boxplot_click(params): """Handle clicks on the ECharts box plot and show chromosome scatter""" try: - if params.series_name == 'box plot' and params.data: + if params.series_name == "box plot" and params.data: chromosome = params.name - ui.notify(f"Showing coverage for chromosome: {chromosome}", type="info") - _log_notify(f"User clicked on chromosome: {chromosome}", level="info", notify=False) + ui.notify( + f"Showing coverage for chromosome: {chromosome}", + type="info", + ) + _log_notify( + f"User clicked on chromosome: {chromosome}", + level="info", + notify=False, + ) _show_chromosome_scatter(chromosome) except Exception as e: - _log_notify(f"Error handling chart click: {e}", level="warning", notify=False) + _log_notify( + f"Error handling chart click: {e}", + level="warning", + notify=False, + ) _cp_bp = _cov_chrome_palette() target_boxplot = ui.echart( @@ -3665,7 +3806,7 @@ def handle_boxplot_click(params): }, ], }, - on_point_click=handle_boxplot_click + on_point_click=handle_boxplot_click, ).classes("w-full h-80 target-coverage-boxplot") _apply_coverage_boxplot_chrome(target_boxplot) with ui.card().classes("w-full"): @@ -3750,9 +3891,9 @@ def handle_boxplot_click(params): ui.label("IGV").classes("text-lg font-semibold mb-2") igv_div = ui.element("div").classes("w-full h-[600px] border") igv_div._props["id"] = "igv-container" - igv_status = ui.label("Checking for IGV-ready BAM files...").classes( - "text-sm text-gray-600" - ) + igv_status = ui.label( + "Checking for IGV-ready BAM files..." + ).classes("text-sm text-gray-600") # Add IGV library status indicator igv_lib_status = ui.label("IGV library: Checking...").classes( @@ -3868,7 +4009,9 @@ def _check_igv_library(): result = ui.run_javascript(js_check, timeout=10.0) # Update the status indicator if result: - igv_lib_status.set_text("IGV library: ✓ Loaded and ready") + igv_lib_status.set_text( + "IGV library: ✓ Loaded and ready" + ) igv_lib_status.classes("text-xs text-green-600") else: igv_lib_status.set_text("IGV library: ✗ Not ready") @@ -3932,7 +4075,9 @@ def _load_igv_bam(bam_path: Path): # Prevent multiple simultaneous IGV loading attempts if state.get("igv_loading", False): - igv_status.set_text("IGV is already being loaded, please wait...") + igv_status.set_text( + "IGV is already being loaded, please wait..." + ) return # First check if IGV library is available @@ -3945,7 +4090,9 @@ def _load_igv_bam(bam_path: Path): # Check if we already have this BAM loaded bam_url = f"/samples/{sample_dir.name}/{bam_path.name}" if state.get("igv_loaded_bam") == bam_url and _is_igv_ready(): - igv_status.set_text(f"BAM {bam_path.name} already loaded in IGV.") + igv_status.set_text( + f"BAM {bam_path.name} already loaded in IGV." + ) return # Mark that we're loading IGV @@ -3976,8 +4123,12 @@ def _load_igv_bam(bam_path: Path): ): # Wait for element to be ready before creating IGV browser if not _wait_for_element_ready(): - igv_status.set_text("Waiting for IGV element to be ready...") - ui.timer(0.5, lambda: _load_igv_bam(bam_path), once=True) + igv_status.set_text( + "Waiting for IGV element to be ready..." + ) + ui.timer( + 0.5, lambda: _load_igv_bam(bam_path), once=True + ) return # Create new IGV browser @@ -4008,17 +4159,25 @@ def _load_igv_bam(bam_path: Path): try: ui.run_javascript(js_create, timeout=30.0) _set_igv_ready(bam_url) - igv_status.set_text(f"IGV browser created with {bam_path.name}") + igv_status.set_text( + f"IGV browser created with {bam_path.name}" + ) # Clear loading flag on success state["igv_loading"] = False except Exception as e: - igv_status.set_text(f"Failed to create IGV browser: {e}") + igv_status.set_text( + f"Failed to create IGV browser: {e}" + ) print(f"IGV browser creation error: {e}") # Clear loading flag on failure state["igv_loading"] = False # Try to retry after a delay - ui.timer(2.0, lambda: _retry_igv_creation(bam_path), once=True) + ui.timer( + 2.0, + lambda: _retry_igv_creation(bam_path), + once=True, + ) _clear_igv_state() else: # Browser exists, just add/update the track @@ -4041,7 +4200,9 @@ def _load_igv_bam(bam_path: Path): try: ui.run_javascript(js_add_track, timeout=30.0) _set_igv_ready(bam_url) - igv_status.set_text(f"Track updated in IGV: {bam_path.name}") + igv_status.set_text( + f"Track updated in IGV: {bam_path.name}" + ) # Clear loading flag on success state["igv_loading"] = False except Exception as e: @@ -4078,9 +4239,12 @@ def _refresh_igv_check(): ] if any( - p.exists() and Path(f"{p}.bai").exists() for p in candidates + p.exists() and Path(f"{p}.bai").exists() + for p in candidates ): - igv_status.set_text("IGV is ready and BAM file is current.") + igv_status.set_text( + "IGV is ready and BAM file is current." + ) return # If we get here, we need to refresh @@ -4096,7 +4260,9 @@ def _clear_igv_tracks(): try: # First check if IGV is actually ready if not _is_igv_ready(): - igv_status.set_text("IGV is not ready - cannot clear tracks.") + igv_status.set_text( + "IGV is not ready - cannot clear tracks." + ) return # Simple JavaScript to clear tracks @@ -4146,7 +4312,9 @@ def _clear_igv_tracks(): print(f"Error in _clear_igv_tracks: {e}") # Function to wait for BAM file to stabilize (not being written to) - def _wait_for_bam_ready(bam_path: Path, max_wait_time: int = 30) -> bool: + def _wait_for_bam_ready( + bam_path: Path, max_wait_time: int = 30 + ) -> bool: """ Wait for BAM file to be ready (not actively being written to). Returns True if file is ready, False if timeout. @@ -4156,7 +4324,9 @@ def _wait_for_bam_ready(bam_path: Path, max_wait_time: int = 30) -> bool: start_time = time.time() last_size = -1 stable_count = 0 - required_stable_checks = 2 # Need 2 consecutive checks with same size + required_stable_checks = ( + 2 # Need 2 consecutive checks with same size + ) while time.time() - start_time < max_wait_time: try: @@ -4201,7 +4371,9 @@ def _reload_bam_track(): current_bam_url = state.get("igv_loaded_bam") if not current_bam_url: - igv_status.set_text("No BAM file currently loaded - cannot reload.") + igv_status.set_text( + "No BAM file currently loaded - cannot reload." + ) return # Extract BAM name for display @@ -4221,16 +4393,26 @@ def _reload_bam_track(): if bam_path and bam_path.exists(): igv_status.set_text("Checking if BAM file is ready...") if not _wait_for_bam_ready(bam_path): - igv_status.set_text("BAM file is still being updated. Please wait and try again.") - ui.notify("BAM file is still being updated. Please wait and try again.", type="warning") + igv_status.set_text( + "BAM file is still being updated. Please wait and try again." + ) + ui.notify( + "BAM file is still being updated. Please wait and try again.", + type="warning", + ) return # Also check if BAI file is ready bai_path = bam_path.with_suffix(bam_path.suffix + ".bai") if bai_path.exists(): if not _wait_for_bam_ready(bai_path): - igv_status.set_text("BAM index is still being updated. Please wait and try again.") - ui.notify("BAM index is still being updated. Please wait and try again.", type="warning") + igv_status.set_text( + "BAM index is still being updated. Please wait and try again." + ) + ui.notify( + "BAM index is still being updated. Please wait and try again.", + type="warning", + ) return # JavaScript to reload the BAM track @@ -4339,7 +4521,9 @@ def _load_target_bed(): try: # First check if IGV is actually ready if not _is_igv_ready(): - igv_status.set_text("IGV is not ready - cannot load BED file.") + igv_status.set_text( + "IGV is not ready - cannot load BED file." + ) return # Get the target panel information @@ -4356,25 +4540,34 @@ def _load_target_bed(): master_csv_path = sample_dir / "master.csv" if master_csv_path.exists(): import pandas as pd + df = pd.read_csv(master_csv_path) if not df.empty and "analysis_panel" in df.columns: - target_panel = str(df.iloc[0]["analysis_panel"]).strip() + target_panel = str( + df.iloc[0]["analysis_panel"] + ).strip() # Map panel to BED filename bed_file_mapping = { - "rCNS2": "rCNS2_panel_name_uniq.bed", - "AML": "AML_panel_name_uniq.bed", - "Sarcoma": "Sarcoma_panel_name_uniq.bed" + "rCNS2": "rCNS2_panel_name_uniq.bed", + "AML": "AML_panel_name_uniq.bed", + "Sarcoma": "Sarcoma_panel_name_uniq.bed", } - bed_filename = bed_file_mapping.get(target_panel, f"{target_panel}_panel_name_uniq.bed") + bed_filename = bed_file_mapping.get( + target_panel, + f"{target_panel}_panel_name_uniq.bed", + ) # Try to find the BED file in robin resources try: from robin import resources + bed_file_path = os.path.join( - os.path.dirname(os.path.abspath(resources.__file__)), - bed_filename + os.path.dirname( + os.path.abspath(resources.__file__) + ), + bed_filename, ) if not os.path.exists(bed_file_path): bed_file_path = None @@ -4384,10 +4577,10 @@ def _load_target_bed(): # Fallback paths if not bed_file_path: possible_paths = [ - bed_filename, - f"data/{bed_filename}", - f"/usr/local/share/{bed_filename}", - ] + bed_filename, + f"data/{bed_filename}", + f"/usr/local/share/{bed_filename}", + ] for path in possible_paths: if os.path.exists(path): bed_file_path = path @@ -4396,8 +4589,13 @@ def _load_target_bed(): print(f"Error reading panel information: {e}") if not bed_file_path or not os.path.exists(bed_file_path): - igv_status.set_text(f"Could not find BED file for panel: {target_panel}") - ui.notify(f"Target BED file not found for panel: {target_panel}", type="warning") + igv_status.set_text( + f"Could not find BED file for panel: {target_panel}" + ) + ui.notify( + f"Target BED file not found for panel: {target_panel}", + type="warning", + ) return # Mount the BED file's directory @@ -4449,7 +4647,9 @@ def _load_target_bed(): # Update status igv_status.set_text(f"Loaded BED file: {bed_name}") - ui.notify(f"Target BED file loaded: {bed_name}", type="positive") + ui.notify( + f"Target BED file loaded: {bed_name}", type="positive" + ) except Exception as e: igv_status.set_text(f"Error loading BED: {e}") @@ -4620,7 +4820,9 @@ def _debug_igv_state(): ui.timer(3.0, _check_igv_library, once=True) # BAM generation buttons - async def _trigger_build_sorted_bam(force_regenerate: bool = False) -> None: + async def _trigger_build_sorted_bam( + force_regenerate: bool = False, + ) -> None: try: # Debug: Check what's in the launcher debug_info = f"launcher type: {type(launcher)}, workflow_runner: {getattr(launcher, 'workflow_runner', 'None')}" @@ -4647,7 +4849,9 @@ async def _trigger_build_sorted_bam(force_regenerate: bool = False) -> None: and igv_bam.exists() and (sample_dir / "igv" / "igv_ready.bam.bai").exists() ): - ui.notify("IGV BAM already exists and is ready.", type="positive") + ui.notify( + "IGV BAM already exists and is ready.", type="positive" + ) return if force_regenerate: @@ -4676,7 +4880,10 @@ async def _trigger_build_sorted_bam(force_regenerate: bool = False) -> None: if hasattr(runner, "submit_sample_job"): # Simple workflow success = runner.submit_sample_job( - str(sample_dir), "igv_bam", sample_id, force_regenerate + str(sample_dir), + "igv_bam", + sample_id, + force_regenerate, ) elif hasattr(runner, "manager") and hasattr( runner.manager, "submit_sample_job" @@ -4698,7 +4905,9 @@ async def _trigger_build_sorted_bam(force_regenerate: bool = False) -> None: return if success: - action = "regenerated" if force_regenerate else "generated" + action = ( + "regenerated" if force_regenerate else "generated" + ) ui.notify( f"IGV BAM {action} job submitted to workflow queue!", type="positive", @@ -4712,17 +4921,19 @@ async def _trigger_build_sorted_bam(force_regenerate: bool = False) -> None: ) except Exception as e: - ui.notify(f"Error submitting IGV BAM job: {e}", type="negative") + ui.notify( + f"Error submitting IGV BAM job: {e}", type="negative" + ) except Exception as e: try: - ui.notify(f"Error checking IGV BAM status: {e}", type="negative") + ui.notify( + f"Error checking IGV BAM status: {e}", type="negative" + ) except Exception: # Client may have disconnected, ignore UI errors pass - - # SNP Analysis section (only shown in development mode) if is_development_mode: with ui.card().classes("w-full"): @@ -4813,7 +5024,9 @@ def _check_snp_results(): indel_csv = clair_dir / "snpsift_indel_output.vcf.csv" if snp_vcf.exists() and indel_vcf.exists(): - snp_results_status.set_text("SNP analysis completed successfully!") + snp_results_status.set_text( + "SNP analysis completed successfully!" + ) # Clear previous results snp_results_container.clear() @@ -4826,9 +5039,9 @@ def _check_snp_results(): if snp_csv.exists(): try: snp_df = pd.read_csv(snp_csv) - ui.label(f"Total SNPs: {len(snp_df)}").classes( - "text-xs text-gray-600" - ) + ui.label( + f"Total SNPs: {len(snp_df)}" + ).classes("text-xs text-gray-600") except Exception: ui.label("SNP data available").classes( "text-xs text-green-600" @@ -4861,15 +5074,19 @@ def _check_snp_results(): # Add detailed results viewer with ui.expansion().classes("w-full").props("icon=table_chart"): - ui.label("Detailed Results").classes("text-sm font-medium mb-2") + ui.label("Detailed Results").classes( + "text-sm font-medium mb-2" + ) # Tabs for SNPs and INDELs - with ui.tabs().classes("w-full"):# as tabs: + with ui.tabs().classes("w-full"): # as tabs: with ui.tab("SNPs", icon="dna"): _display_variant_table(snp_csv, "SNP", clair_dir) with ui.tab("INDELs", icon="straighten"): - _display_variant_table(indel_csv, "INDEL", clair_dir) + _display_variant_table( + indel_csv, "INDEL", clair_dir + ) else: snp_results_status.set_text("No SNP analysis results found") @@ -4924,12 +5141,16 @@ def _display_variant_table(csv_file, variant_type, clair_dir): with gzip.open(vcf_file, "rt") as f: lines = [ - line.strip() for line in f if not line.startswith("#") + line.strip() + for line in f + if not line.startswith("#") ] else: with open(vcf_file, "r") as f: lines = [ - line.strip() for line in f if not line.startswith("#") + line.strip() + for line in f + if not line.startswith("#") ] if not lines: @@ -5002,7 +5223,9 @@ def _display_variant_table(csv_file, variant_type, clair_dir): "text-2xl font-bold text-gray-400" ) else: - ui.label("N/A").classes("text-2xl font-bold text-gray-400") + ui.label("N/A").classes( + "text-2xl font-bold text-gray-400" + ) with ui.card().classes("flex-1"): ui.label("Avg Quality").classes("text-sm font-medium") @@ -5010,7 +5233,9 @@ def _display_variant_table(csv_file, variant_type, clair_dir): if "QUAL" in df.columns: try: # Calculate average quality from QUAL column - qual_values = pd.to_numeric(df["QUAL"], errors="coerce") + qual_values = pd.to_numeric( + df["QUAL"], errors="coerce" + ) avg_qual = qual_values.mean() if pd.notna(avg_qual): ui.label(f"{avg_qual:.1f}").classes( @@ -5026,7 +5251,9 @@ def _display_variant_table(csv_file, variant_type, clair_dir): "text-2xl font-bold text-gray-400" ) else: - ui.label("N/A").classes("text-2xl font-bold text-gray-400") + ui.label("N/A").classes( + "text-2xl font-bold text-gray-400" + ) # Add filtering controls with ui.row().classes("w-full gap-3 mb-4"): @@ -5075,7 +5302,10 @@ def apply_filters(): pass # Invalid quality value, skip filtering # Apply filter status - if filter_status.value != "All" and "FILTER" in filtered_df.columns: + if ( + filter_status.value != "All" + and "FILTER" in filtered_df.columns + ): # Filter by status in FILTER column filter_mask = filtered_df["FILTER"] == filter_status.value filtered_df = filtered_df[filter_mask] @@ -5165,12 +5395,14 @@ def truncate_info(info_val): # Create columns definition from DataFrame columns = [] for col in display_df.columns: - columns.append({ - "name": col, - "label": col, - "field": col, - "sortable": True - }) + columns.append( + { + "name": col, + "label": col, + "field": col, + "sortable": True, + } + ) # Create paged table (slice rows per page, avoid full row materialization). variant_table = _render_paged_df_table( @@ -5231,7 +5463,9 @@ def toggle_column(column: dict, visible: bool) -> None: ui.button( f"Export {variant_type}s to CSV", icon="download", - on_click=lambda: _export_variants(filtered_df, variant_type), + on_click=lambda: _export_variants( + filtered_df, variant_type + ), ).classes("w-full") # Add row count display @@ -5248,8 +5482,11 @@ def toggle_column(column: dict, visible: bool) -> None: def _show_variant_details(variant_row, variant_type, clair_dir): """Show detailed information for a specific variant""" try: - with ui.dialog() as dialog, ui.card().classes( - "robin-dialog-surface p-4 md:p-5 w-full max-w-2xl max-h-[85vh] overflow-auto" + with ( + ui.dialog() as dialog, + ui.card().classes( + "robin-dialog-surface p-4 md:p-5 w-full max-w-2xl max-h-[85vh] overflow-auto" + ), ): ui.label(f"{variant_type} details").classes( "classification-insight-heading text-headline-small mb-2" @@ -5290,7 +5527,9 @@ def _export_variants(data_df, variant_type): temp_path = f.name # Download the file - ui.download(temp_path, filename=f"{variant_type.lower()}_variants.csv") + ui.download( + temp_path, filename=f"{variant_type.lower()}_variants.csv" + ) # Clean up os.unlink(temp_path) @@ -5426,7 +5665,9 @@ def _trigger_snp_analysis(): ) if hasattr(launcher, "workflow_runner"): - print(f"launcher.workflow_runner: {launcher.workflow_runner}") + print( + f"launcher.workflow_runner: {launcher.workflow_runner}" + ) if launcher.workflow_runner: print( f"workflow_runner type: {type(launcher.workflow_runner)}" @@ -5439,8 +5680,12 @@ def _trigger_snp_analysis(): ) if hasattr(launcher.workflow_runner, "reference"): - reference_genome = launcher.workflow_runner.reference - print(f"workflow_runner.reference: {reference_genome}") + reference_genome = ( + launcher.workflow_runner.reference + ) + print( + f"workflow_runner.reference: {reference_genome}" + ) if reference_genome: print( f"SUCCESS: Using reference genome from workflow runner: {reference_genome}" @@ -5607,8 +5852,7 @@ def run_snp_analysis_fallback(): except Exception as e: # Log error instead of trying to update UI from background thread - ui.run_javascript( - f""" + ui.run_javascript(f""" // Update status var statusElement = document.querySelector('{snp_status_label.id}'); if (statusElement) {{ @@ -5621,8 +5865,7 @@ def run_snp_analysis_fallback(): if (buttonElement) {{ buttonElement.disabled = false; }} - """ - ) + """) # Run in background thread as fallback import threading @@ -5766,8 +6009,12 @@ def _display_lga_results(json_report): ui.label("Available").classes("text-xs text-green-600") with ui.card().classes("flex-1"): - ui.label("Analysis Complete").classes("text-sm font-medium") - ui.label("Ready to view").classes("text-xs text-green-600") + ui.label("Analysis Complete").classes( + "text-sm font-medium" + ) + ui.label("Ready to view").classes( + "text-xs text-green-600" + ) # Add view results button with ui.row().classes("w-full mt-4"): @@ -5844,9 +6091,9 @@ def _view_lga_json_results_inline(json_path): with ui.row().classes( "w-full items-center justify-between p-4 bg-purple-50 rounded" ): - ui.label("Total Pathogenic Variants Found").classes( - "text-lg font-medium text-gray-700" - ) + ui.label( + "Total Pathogenic Variants Found" + ).classes("text-lg font-medium text-gray-700") ui.label( f"{metadata.get('total_variants_found', 0):,}" ).classes("text-3xl font-bold text-purple-600") @@ -5854,9 +6101,9 @@ def _view_lga_json_results_inline(json_path): with ui.row().classes( "w-full items-center justify-between p-4 bg-green-50 rounded" ): - ui.label("Genes with Good Coverage (≥10x)").classes( - "text-lg font-medium text-gray-700" - ) + ui.label( + "Genes with Good Coverage (≥10x)" + ).classes("text-lg font-medium text-gray-700") ui.label( f"{metadata.get('genes_with_good_coverage_variants', 0)}" ).classes("text-3xl font-bold text-green-600") @@ -5864,9 +6111,9 @@ def _view_lga_json_results_inline(json_path): with ui.row().classes( "w-full items-center justify-between p-4 bg-orange-50 rounded" ): - ui.label("Genes with Low Coverage (<10x)").classes( - "text-lg font-medium text-gray-700" - ) + ui.label( + "Genes with Low Coverage (<10x)" + ).classes("text-lg font-medium text-gray-700") low_coverage = metadata.get( "total_genes_analyzed", 0 ) - metadata.get( @@ -5917,8 +6164,12 @@ def _hide_lga_results(): with lga_results_container: with ui.row().classes("w-full gap-3"): with ui.card().classes("flex-1"): - ui.label("JSON Results").classes("text-sm font-medium") - ui.label("Available").classes("text-xs text-green-600") + ui.label("JSON Results").classes( + "text-sm font-medium" + ) + ui.label("Available").classes( + "text-xs text-green-600" + ) with ui.card().classes("flex-1"): ui.label("Analysis Complete").classes( @@ -5976,7 +6227,9 @@ def _display_gene_overview(data): ) # Sort by mean coverage - coverage_data.sort(key=lambda x: x["mean_coverage"], reverse=True) + coverage_data.sort( + key=lambda x: x["mean_coverage"], reverse=True + ) # Create coverage distribution chart with ui.card().classes("w-full mb-4"): @@ -6035,7 +6288,9 @@ def _display_gene_overview(data): "nameLocation": "middle", "nameGap": 50, "nameTextStyle": {"color": _lga_p["axis"]}, - "axisLine": {"lineStyle": {"color": _lga_p["axis"]}}, + "axisLine": { + "lineStyle": {"color": _lga_p["axis"]} + }, }, "yAxis": { "type": "value", @@ -6111,7 +6366,9 @@ def _display_gene_overview(data): with ui.row().classes( "items-center gap-3 p-2 bg-green-50 rounded" ): - ui.element("div").classes("w-4 h-4 bg-green-500 rounded") + ui.element("div").classes( + "w-4 h-4 bg-green-500 rounded" + ) ui.label( "≥10x (Good Coverage) - Reliable variant detection" ).classes("text-sm text-gray-700") @@ -6119,7 +6376,9 @@ def _display_gene_overview(data): with ui.row().classes( "items-center gap-3 p-2 bg-orange-50 rounded" ): - ui.element("div").classes("w-4 h-4 bg-orange-500 rounded") + ui.element("div").classes( + "w-4 h-4 bg-orange-500 rounded" + ) ui.label( "5-10x (Moderate Coverage) - Limited reliability" ).classes("text-sm text-gray-700") @@ -6127,12 +6386,12 @@ def _display_gene_overview(data): with ui.row().classes( "items-center gap-3 p-2 bg-red-50 rounded" ): - ui.element("div").classes("w-4 h-4 bg-red-500 rounded") + ui.element("div").classes( + "w-4 h-4 bg-red-500 rounded" + ) ui.label( "<5x (Low Coverage) - Poor reliability" - ).classes( - "text-sm text-gray-700" - ) + ).classes("text-sm text-gray-700") # Summary statistics with ui.card().classes("w-full mb-4"): @@ -6141,9 +6400,13 @@ def _display_gene_overview(data): ) # Calculate additional statistics - total_coverage = sum(g["mean_coverage"] for g in coverage_data) + total_coverage = sum( + g["mean_coverage"] for g in coverage_data + ) avg_coverage = ( - total_coverage / len(coverage_data) if coverage_data else 0 + total_coverage / len(coverage_data) + if coverage_data + else 0 ) high_coverage_count = sum( 1 for g in coverage_data if g["mean_coverage"] >= 10 @@ -6157,9 +6420,9 @@ def _display_gene_overview(data): with ui.row().classes( "w-full items-center justify-between p-3 bg-blue-50 rounded" ): - ui.label("Average Coverage Across All Genes").classes( - "text-base font-medium text-gray-700" - ) + ui.label( + "Average Coverage Across All Genes" + ).classes("text-base font-medium text-gray-700") ui.label(f"{avg_coverage:.1f}x").classes( "text-2xl font-bold text-blue-600" ) @@ -6204,13 +6467,17 @@ def _display_gene_overview(data): gene_table_data = [] for gene_name, gene_data in genes.items(): summary = gene_data.get("summary", {}) - coverage_stats = gene_data.get("coverage_statistics", {}) + coverage_stats = gene_data.get( + "coverage_statistics", {} + ) gene_table_data.append( { "gene": gene_name, "mean_coverage": f"{coverage_stats.get('mean_coverage', 0):.1f}x", - "total_variants": summary.get("total_variants", 0), + "total_variants": summary.get( + "total_variants", 0 + ), "high_coverage_variants": summary.get( "high_coverage_variants", 0 ), @@ -6227,7 +6494,9 @@ def _display_gene_overview(data): # Sort by mean coverage gene_table_data.sort( - key=lambda x: float(x["mean_coverage"].replace("x", "")), + key=lambda x: float( + x["mean_coverage"].replace("x", "") + ), reverse=True, ) @@ -6237,26 +6506,31 @@ def _display_gene_overview(data): # Create columns definition columns = [] for col in gene_table_data[0].keys(): - columns.append({ - "name": col, - "label": col.replace("_", " ").title(), - "field": col, - "sortable": True - }) + columns.append( + { + "name": col, + "label": col.replace("_", " ").title(), + "field": col, + "sortable": True, + } + ) # Create styled table table_container, gene_table = styled_table( columns=columns, rows=gene_table_data, pagination=25, - class_size="table-xs" + class_size="table-xs", ) # Add search functionality with gene_table.add_slot("top-right"): - with ui.input(placeholder="Search genes...").props( - "type=search" - ).bind_value(gene_table, "filter").add_slot("append"): + with ( + ui.input(placeholder="Search genes...") + .props("type=search") + .bind_value(gene_table, "filter") + .add_slot("append") + ): ui.icon("search") # Make columns sortable @@ -6284,9 +6558,9 @@ def _display_high_coverage_genes(data): high_coverage_genes[gene_name] = gene_data if not high_coverage_genes: - ui.label("No genes with high coverage variants found.").classes( - "text-sm text-gray-500" - ) + ui.label( + "No genes with high coverage variants found." + ).classes("text-sm text-gray-500") return ui.label( @@ -6295,11 +6569,15 @@ def _display_high_coverage_genes(data): # Display each gene with its high coverage variants for gene_name, gene_data in high_coverage_genes.items(): - with ui.expansion().classes("w-full mb-2").props( - f"icon=dna label={gene_name}" + with ( + ui.expansion() + .classes("w-full mb-2") + .props(f"icon=dna label={gene_name}") ): summary = gene_data.get("summary", {}) - coverage_stats = gene_data.get("coverage_statistics", {}) + coverage_stats = gene_data.get( + "coverage_statistics", {} + ) variants = gene_data.get("variants", []) # Gene summary @@ -6363,8 +6641,12 @@ def _display_high_coverage_genes(data): { "genomic_locus": genomic_locus, "position": f"{position:,}", - "reference": variant.get("reference", "N"), - "alternate": variant.get("alternate", "N"), + "reference": variant.get( + "reference", "N" + ), + "alternate": variant.get( + "alternate", "N" + ), "variant_type": evidence.get( "variant_type", "unknown" ).upper(), @@ -6378,7 +6660,9 @@ def _display_high_coverage_genes(data): )[:50] + ( "..." - if len(variant.get("disease_name", "")) + if len( + variant.get("disease_name", "") + ) > 50 else "" ), @@ -6411,12 +6695,16 @@ def _display_high_coverage_genes(data): # Create columns definition columns = [] for col in df.columns: - columns.append({ - "name": col, - "label": col.replace("_", " ").title(), - "field": col, - "sortable": True - }) + columns.append( + { + "name": col, + "label": col.replace( + "_", " " + ).title(), + "field": col, + "sortable": True, + } + ) variant_table = _render_paged_df_table( df, @@ -6482,7 +6770,9 @@ def _display_low_coverage_genes(data): "low_coverage_variants": summary.get( "low_coverage_variants", 0 ), - "status": summary.get("pathogenic_status", "unknown") + "status": summary.get( + "pathogenic_status", "unknown" + ) .replace("_", " ") .title(), } @@ -6499,26 +6789,31 @@ def _display_low_coverage_genes(data): # Create columns definition columns = [] for col in low_cov_data[0].keys(): - columns.append({ - "name": col, - "label": col.replace("_", " ").title(), - "field": col, - "sortable": True - }) + columns.append( + { + "name": col, + "label": col.replace("_", " ").title(), + "field": col, + "sortable": True, + } + ) # Create styled table table_container, low_cov_table = styled_table( columns=columns, rows=low_cov_data, pagination=25, - class_size="table-xs" + class_size="table-xs", ) # Add search functionality with low_cov_table.add_slot("top-right"): - with ui.input(placeholder="Search genes...").props( - "type=search" - ).bind_value(low_cov_table, "filter").add_slot("append"): + with ( + ui.input(placeholder="Search genes...") + .props("type=search") + .bind_value(low_cov_table, "filter") + .add_slot("append") + ): ui.icon("search") # Make columns sortable @@ -6579,7 +6874,9 @@ def update_gene_details(): ) ui.label( f"{coverage_stats.get('mean_coverage', 0):.1f}x" - ).classes("text-2xl font-bold text-blue-600") + ).classes( + "text-2xl font-bold text-blue-600" + ) with ui.row().classes( "w-full items-center justify-between p-3 bg-purple-50 rounded" @@ -6589,27 +6886,37 @@ def update_gene_details(): ) ui.label( f"{summary.get('total_variants', 0)}" - ).classes("text-2xl font-bold text-purple-600") + ).classes( + "text-2xl font-bold text-purple-600" + ) with ui.row().classes( "w-full items-center justify-between p-3 bg-green-50 rounded" ): ui.label( "High Coverage Variants (≥10x)" - ).classes("text-base font-medium text-gray-700") + ).classes( + "text-base font-medium text-gray-700" + ) ui.label( f"{summary.get('high_coverage_variants', 0)}" - ).classes("text-2xl font-bold text-green-600") + ).classes( + "text-2xl font-bold text-green-600" + ) with ui.row().classes( "w-full items-center justify-between p-3 bg-orange-50 rounded" ): ui.label( "Low Coverage Variants (<10x)" - ).classes("text-base font-medium text-gray-700") + ).classes( + "text-base font-medium text-gray-700" + ) ui.label( f"{summary.get('low_coverage_variants', 0)}" - ).classes("text-2xl font-bold text-orange-600") + ).classes( + "text-2xl font-bold text-orange-600" + ) # Variants table if variants: @@ -6647,7 +6954,8 @@ def update_gene_details(): "variant_type", "unknown" ).upper(), "clinical_significance": variant.get( - "clinical_significance", "unknown" + "clinical_significance", + "unknown", ) .replace("_", " ") .title(), @@ -6657,7 +6965,9 @@ def update_gene_details(): + ( "..." if len( - variant.get("disease_name", "") + variant.get( + "disease_name", "" + ) ) > 40 else "" @@ -6703,12 +7013,14 @@ def update_gene_details(): # Create columns definition columns = [] for col in df.columns: - columns.append({ - "name": col, - "label": col.replace("_", " ").title(), - "field": col, - "sortable": True - }) + columns.append( + { + "name": col, + "label": col.replace("_", " ").title(), + "field": col, + "sortable": True, + } + ) detailed_variant_table = _render_paged_df_table( df, @@ -6733,9 +7045,9 @@ def update_gene_details(): ), ).classes("w-full") else: - ui.label("No variants found for this gene.").classes( - "text-sm text-gray-500" - ) + ui.label( + "No variants found for this gene." + ).classes("text-sm text-gray-500") # Initial display update_gene_details() @@ -7124,9 +7436,7 @@ def _update_target_cov(cov_df: pd.DataFrame, bed_df: pd.DataFrame) -> None: asyncio.get_running_loop() except RuntimeError: _apply_target_cov_payload( - _compute_target_cov_series_data( - cov_df, bed_df, reference_contig_scope - ) + _compute_target_cov_series_data(cov_df, bed_df, reference_contig_scope) ) return asyncio.create_task(_apply_target_cov_async(cov_df, bed_df)) @@ -7202,9 +7512,9 @@ def _apply_boxplot_payload(result: Dict[str, Any]) -> None: }, }, ] - target_boxplot.options["title"]["text"] = ( - f"Target Coverage ({panel_display_name})" - ) + target_boxplot.options["title"][ + "text" + ] = f"Target Coverage ({panel_display_name})" target_boxplot.options["title"]["subtext"] = "" target_boxplot.options["legend"]["show"] = True _apply_coverage_boxplot_chrome(target_boxplot) @@ -7213,8 +7523,7 @@ def _apply_boxplot_payload(result: Dict[str, Any]) -> None: options_clean = _echarts_option_to_json(target_boxplot.options) options_json = json.dumps(options_clean) options_escaped = json.dumps(options_json) - ui.run_javascript( - f""" + ui.run_javascript(f""" (function() {{ var el = document.querySelector('.target-coverage-boxplot'); if (!el) return; @@ -7229,8 +7538,7 @@ def _apply_boxplot_payload(result: Dict[str, Any]) -> None: chart.setOption(options, {{ replaceMerge: ['series', 'dataset'] }}); }} catch (e) {{ console.warn('Target coverage replaceMerge:', e); }} }})(); - """ - ) + """) except Exception as js_err: logging.debug("Target coverage replaceMerge JS skip: %s", js_err) @@ -7385,18 +7693,18 @@ async def _refresh_coverage_apply(load_result: Dict[str, Any]) -> None: if loaded_bed_df and state.get("bed_df") is not None: await _apply_boxplot_async(state["bed_df"], panel_display_name) if state.get("cov_df") is not None: - await _apply_target_cov_async( - state["cov_df"], state["bed_df"] - ) + await _apply_target_cov_async(state["cov_df"], state["bed_df"]) - if load_result.get("loaded_target_df") and load_result.get( - "target_df" - ) is not None: + if ( + load_result.get("loaded_target_df") + and load_result.get("target_df") is not None + ): await _apply_target_table_async(load_result["target_df"]) - if load_result.get("cov_time_refresh") and load_result.get( - "cov_time_path" - ) is not None: + if ( + load_result.get("cov_time_refresh") + and load_result.get("cov_time_path") is not None + ): await _apply_time_series_async(load_result["cov_time_path"]) if load_result.get("target_cov_time_analysis_refresh"): @@ -7432,7 +7740,9 @@ async def _refresh_coverage_apply(load_result: Dict[str, Any]) -> None: f"Targets Estimated Coverage: {target_cov_v:.2f}x" ) if enrich_v is not None: - cov_enrich_lbl.set_text(f"Estimated enrichment: {enrich_v:.2f}x") + cov_enrich_lbl.set_text( + f"Estimated enrichment: {enrich_v:.2f}x" + ) except Exception as e: _log_notify( f"Coverage summary update failed: {e}", @@ -7496,7 +7806,6 @@ async def _refresh_coverage_apply(load_result: Dict[str, Any]) -> None: target_bam = sample_dir / "target.bam" targets_bed = sample_dir / "targets_exceeding_threshold.bed" - except Exception as e: logging.debug(f" LGA: : {e}") pass diff --git a/src/robin/gui/components/folder_picker.py b/src/robin/gui/components/folder_picker.py index 1ea62752..7f47a479 100644 --- a/src/robin/gui/components/folder_picker.py +++ b/src/robin/gui/components/folder_picker.py @@ -38,9 +38,12 @@ def __init__( self.multiple = multiple self.selected_paths: List[str] = [] - with self, ui.card().classes( - "robin-dialog-surface workflow-folder-picker p-4 " - "w-full max-w-2xl min-w-[18rem]" + with ( + self, + ui.card().classes( + "robin-dialog-surface workflow-folder-picker p-4 " + "w-full max-w-2xl min-w-[18rem]" + ), ): with ui.row().classes("w-full items-start gap-3 min-w-0"): with ui.column().classes("gap-0 flex-1 min-w-0"): diff --git a/src/robin/gui/components/fusion.py b/src/robin/gui/components/fusion.py index 7cb095a4..1693fb2e 100644 --- a/src/robin/gui/components/fusion.py +++ b/src/robin/gui/components/fusion.py @@ -1,16 +1,16 @@ from __future__ import annotations import asyncio -from typing import Any, Dict, List, Optional -from pathlib import Path +import hashlib import logging -import pickle import os -import hashlib +import pickle +from pathlib import Path +from typing import Any, Dict, List, Optional -import pandas as pd -import numpy as np import matplotlib +import numpy as np +import pandas as pd # Use Agg backend for compatibility with DNA Features Viewer and NiceGUI matplotlib.use("Agg") @@ -18,7 +18,7 @@ # Configure matplotlib to allow more open figures and suppress warnings # During preprocessing, many figures may be created -plt.rcParams['figure.max_open_warning'] = 100 # Increase threshold +plt.rcParams["figure.max_open_warning"] = 100 # Increase threshold # Module-level variable to track last fusion plot figure for cleanup _last_fusion_figure = None @@ -36,7 +36,7 @@ # chrov ideograms removed - not working properly try: - from nicegui import ui, background_tasks + from nicegui import background_tasks, ui except ImportError: # pragma: no cover ui = None background_tasks = None @@ -151,13 +151,16 @@ def _apply_search(term: str) -> None: search_input = ui.input(placeholder=search_placeholder).props( "type=search dense clearable" ) - search_input.on("update:model-value", lambda e: _apply_search(getattr(e, "value", ""))) + search_input.on( + "update:model-value", lambda e: _apply_search(getattr(e, "value", "")) + ) for col in table.columns: col["sortable"] = False _fill_from_pagination(init_pagination) try: + def _cleanup_table_state() -> None: page_state["filtered_positions"] = [] table.rows = [] @@ -300,7 +303,9 @@ def _count_unique_fusion_pairs(data: Dict[str, Any]) -> int: # Filter to good pairs if available if not goodpairs.empty and goodpairs.sum() > 0: - aligned_goodpairs = goodpairs.reindex(annotated_data.index, fill_value=False) + aligned_goodpairs = goodpairs.reindex( + annotated_data.index, fill_value=False + ) filtered_data = annotated_data[aligned_goodpairs] else: filtered_data = annotated_data @@ -309,7 +314,9 @@ def _count_unique_fusion_pairs(data: Dict[str, Any]) -> int: return 0 # Get validated fusion pairs using breakpoint validation - clustered_data = _cluster_fusion_reads(filtered_data, max_distance=10000, use_breakpoint_validation=True) + clustered_data = _cluster_fusion_reads( + filtered_data, max_distance=10000, use_breakpoint_validation=True + ) if clustered_data.empty: return 0 @@ -348,7 +355,9 @@ def _get_validated_fusion_groups(data: Dict[str, Any]) -> List[List[str]]: # Filter to good pairs if available if not goodpairs.empty and goodpairs.sum() > 0: - aligned_goodpairs = goodpairs.reindex(annotated_data.index, fill_value=False) + aligned_goodpairs = goodpairs.reindex( + annotated_data.index, fill_value=False + ) filtered_data = annotated_data[aligned_goodpairs] else: filtered_data = annotated_data @@ -357,7 +366,9 @@ def _get_validated_fusion_groups(data: Dict[str, Any]) -> List[List[str]]: return [] # Get validated fusion pairs (these meet the minimum read support threshold) - clustered_data = _cluster_fusion_reads(filtered_data, max_distance=10000, use_breakpoint_validation=True) + clustered_data = _cluster_fusion_reads( + filtered_data, max_distance=10000, use_breakpoint_validation=True + ) if clustered_data.empty: return [] @@ -398,7 +409,9 @@ def _get_validated_fusion_groups(data: Dict[str, Any]) -> List[List[str]]: if group_has_validated_pair: validated_groups.append(normalized_group_genes) - logging.info(f"[Fusion] Validated {len(validated_groups)} fusion groups from {len(gene_groups)} total groups") + logging.info( + f"[Fusion] Validated {len(validated_groups)} fusion groups from {len(gene_groups)} total groups" + ) return validated_groups except Exception as e: @@ -427,7 +440,9 @@ def _count_unique_fusion_groups(data: Dict[str, Any]) -> int: return 0 -def _generate_summary_files_from_pickle(sample_dir: Path, force_regenerate: bool = False) -> bool: +def _generate_summary_files_from_pickle( + sample_dir: Path, force_regenerate: bool = False +) -> bool: """Generate summary files from existing pickle files if they don't exist. This provides backward compatibility for existing analyses that were run @@ -457,26 +472,38 @@ def _generate_summary_files_from_pickle(sample_dir: Path, force_regenerate: bool # Debug: List all fusion-related files in the directory fusion_files = list(sample_dir.glob("*fusion*")) - logging.info(f"[Fusion] All fusion files in directory: {[f.name for f in fusion_files]}") + logging.info( + f"[Fusion] All fusion files in directory: {[f.name for f in fusion_files]}" + ) # Debug: Check file sizes if target_file.exists(): - logging.info(f"[Fusion] Target file size: {target_file.stat().st_size} bytes") + logging.info( + f"[Fusion] Target file size: {target_file.stat().st_size} bytes" + ) if genome_file.exists(): - logging.info(f"[Fusion] Genome-wide file size: {genome_file.stat().st_size} bytes") + logging.info( + f"[Fusion] Genome-wide file size: {genome_file.stat().st_size} bytes" + ) target_data = _load_processed_pickle(target_file) genome_data = _load_processed_pickle(genome_file) if target_data is not None: - logging.info(f"[Fusion] Target data loaded: candidate_count={target_data.get('candidate_count', 0)}") + logging.info( + f"[Fusion] Target data loaded: candidate_count={target_data.get('candidate_count', 0)}" + ) else: logging.warning(f"[Fusion] Failed to load target data from: {target_file}") if genome_data is not None: - logging.info(f"[Fusion] Genome-wide data loaded: candidate_count={genome_data.get('candidate_count', 0)}") + logging.info( + f"[Fusion] Genome-wide data loaded: candidate_count={genome_data.get('candidate_count', 0)}" + ) else: - logging.warning(f"[Fusion] Failed to load genome-wide data from: {genome_file}") + logging.warning( + f"[Fusion] Failed to load genome-wide data from: {genome_file}" + ) # Extract counts using the same filtering logic as the display code # This ensures consistency between summary panel and fusion section @@ -489,7 +516,9 @@ def _generate_summary_files_from_pickle(sample_dir: Path, force_regenerate: bool if genome_data is not None and isinstance(genome_data, dict): genome_count = _count_unique_fusion_pairs(genome_data) - logging.info(f"[Fusion] Summary: Genome-wide count (filtered): {genome_count}") + logging.info( + f"[Fusion] Summary: Genome-wide count (filtered): {genome_count}" + ) # Generate fusion_summary.csv with open(summary_file, "w", newline="") as f: @@ -497,7 +526,9 @@ def _generate_summary_files_from_pickle(sample_dir: Path, force_regenerate: bool writer.writerow(["target_fusions", "genome_fusions"]) writer.writerow([target_count, genome_count]) - logging.info(f"[Fusion] Generated summary file from pickle: target={target_count}, genome={genome_count}") + logging.info( + f"[Fusion] Generated summary file from pickle: target={target_count}, genome={genome_count}" + ) # Generate fusion_results.csv (use the one with more data) results_file = sample_dir / "fusion_results.csv" @@ -519,7 +550,9 @@ def _generate_summary_files_from_pickle(sample_dir: Path, force_regenerate: bool with open(sv_count_file, "w") as f: f.write(str(genome_count)) - logging.info(f"[Fusion] Generated summary files from pickle data - target: {target_count}, genome: {genome_count}") + logging.info( + f"[Fusion] Generated summary files from pickle data - target: {target_count}, genome: {genome_count}" + ) return True except Exception as e: @@ -547,21 +580,29 @@ def _load_processed_pickle(file_path: Path) -> Optional[Dict[str, Any]]: data = pickle.load(f) except (pickle.UnpicklingError, EOFError) as e: # If pickle is truncated, try to load what we can - logging.warning(f"[Fusion] Pickle file appears truncated, attempting recovery: {file_path}") + logging.warning( + f"[Fusion] Pickle file appears truncated, attempting recovery: {file_path}" + ) f.seek(0) try: # Try loading with protocol 0 which is more forgiving data = pickle.load(f) except: # If all else fails, return None and let the system regenerate - logging.error(f"[Fusion] Could not recover truncated pickle: {file_path}") + logging.error( + f"[Fusion] Could not recover truncated pickle: {file_path}" + ) return None # Expected keys: annotated_data (DataFrame), goodpairs (Series), gene_groups (list), candidate_count (int) if isinstance(data, dict): logging.info(f"[Fusion] Loaded pickle data with keys: {list(data.keys())}") - logging.info(f"[Fusion] Raw candidate_count: {data.get('candidate_count', 'not found')}") - logging.info(f"[Fusion] Raw gene_groups count: {len(data.get('gene_groups', []))}") + logging.info( + f"[Fusion] Raw candidate_count: {data.get('candidate_count', 'not found')}" + ) + logging.info( + f"[Fusion] Raw gene_groups count: {len(data.get('gene_groups', []))}" + ) # Apply the same filtering logic as the reporting code annotated_data = data.get("annotated_data", pd.DataFrame()) @@ -573,12 +614,20 @@ def _load_processed_pickle(file_path: Path) -> Optional[Dict[str, Any]]: if not annotated_data.empty and not goodpairs.empty: # Only keep the good pairs (same as reporting code does) data["annotated_data"] = annotated_data[goodpairs] - logging.info(f"[Fusion] Filtered data: {len(annotated_data)} -> {len(data['annotated_data'])} good pairs") + logging.info( + f"[Fusion] Filtered data: {len(annotated_data)} -> {len(data['annotated_data'])} good pairs" + ) else: - logging.info(f"[Fusion] No filtering applied - annotated_data empty: {annotated_data.empty}, goodpairs empty: {goodpairs.empty}") + logging.info( + f"[Fusion] No filtering applied - annotated_data empty: {annotated_data.empty}, goodpairs empty: {goodpairs.empty}" + ) - logging.info(f"[Fusion] Final candidate_count: {data.get('candidate_count', 'not found')}") - logging.info(f"[Fusion] Final gene_groups count: {len(data.get('gene_groups', []))}") + logging.info( + f"[Fusion] Final candidate_count: {data.get('candidate_count', 'not found')}" + ) + logging.info( + f"[Fusion] Final gene_groups count: {len(data.get('gene_groups', []))}" + ) return data else: @@ -608,12 +657,14 @@ def _make_fusion_table(container: Any, df: pd.DataFrame) -> Any: # Create columns definition from DataFrame columns = [] for col in processed_df.columns: - columns.append({ - "name": col, - "label": col.replace("_", " ").title(), - "field": col, - "sortable": True - }) + columns.append( + { + "name": col, + "label": col.replace("_", " ").title(), + "field": col, + "sortable": True, + } + ) table = _render_paged_df_table( processed_df, @@ -625,7 +676,9 @@ def _make_fusion_table(container: Any, df: pd.DataFrame) -> Any: return table -def _cluster_nearby_regions(regions_df: pd.DataFrame, max_distance: int = 200) -> pd.DataFrame: +def _cluster_nearby_regions( + regions_df: pd.DataFrame, max_distance: int = 200 +) -> pd.DataFrame: """ Cluster nearby regions together to merge similar breakpoint events. @@ -668,15 +721,21 @@ def _cluster_nearby_regions(regions_df: pd.DataFrame, max_distance: int = 200) - if isinstance(read_ids, set): cluster_reads.update(read_ids) elif isinstance(read_ids, list): - cluster_reads.update(read_ids) # set.update() handles duplicates automatically + cluster_reads.update( + read_ids + ) # set.update() handles duplicates automatically elif "read_count" in row: # Fallback: if we don't have actual read IDs, we can't properly track uniqueness - logging.warning(f"[Fusion] Missing read_ids for region, using read_count estimate") + logging.warning( + f"[Fusion] Missing read_ids for region, using read_count estimate" + ) for k in range(int(row["read_count"])): cluster_reads.add(f"read_{i}_{k}") elif "read_count" in row: # Fallback: if we don't have actual read IDs, we can't properly track uniqueness - logging.warning(f"[Fusion] Missing read_ids for region, using read_count estimate") + logging.warning( + f"[Fusion] Missing read_ids for region, using read_count estimate" + ) for k in range(int(row["read_count"])): cluster_reads.add(f"read_{i}_{k}") @@ -720,15 +779,21 @@ def _cluster_nearby_regions(regions_df: pd.DataFrame, max_distance: int = 200) - if isinstance(other_read_ids, set): cluster_reads.update(other_read_ids) elif isinstance(other_read_ids, list): - cluster_reads.update(other_read_ids) # set.update() handles duplicates automatically + cluster_reads.update( + other_read_ids + ) # set.update() handles duplicates automatically elif "read_count" in other_row: # Fallback: if we don't have actual read IDs, we can't properly track uniqueness - logging.warning(f"[Fusion] Missing read_ids for merged region, using read_count estimate") + logging.warning( + f"[Fusion] Missing read_ids for merged region, using read_count estimate" + ) for k in range(int(other_row["read_count"])): cluster_reads.add(f"read_{j}_{k}") elif "read_count" in other_row: # Fallback: if we don't have actual read IDs, we can't properly track uniqueness - logging.warning(f"[Fusion] Missing read_ids for merged region, using read_count estimate") + logging.warning( + f"[Fusion] Missing read_ids for merged region, using read_count estimate" + ) for k in range(int(other_row["read_count"])): cluster_reads.add(f"read_{j}_{k}") @@ -742,15 +807,31 @@ def _cluster_nearby_regions(regions_df: pd.DataFrame, max_distance: int = 200) - # Create merged region # read_count is the number of UNIQUE reads supporting this clustered event # (using len() on the set ensures we count each read only once, even if it appears in multiple merged regions) - clustered_results.append({ - "chromosome": chromosome, - "start": cluster_start, - "end": cluster_end, - "event_type": "master_bed_region" if "master_bed_region" in cluster_types else "supplementary", - "read_count": len(cluster_reads), # Count of unique reads (set automatically handles deduplication) - "avg_mapping_quality": round(sum(cluster_mapqs) / len(cluster_mapqs), 1) if cluster_mapqs else 0, - "avg_mapping_span": round(sum(cluster_spans) / len(cluster_spans), 0) if cluster_spans else 0, - }) + clustered_results.append( + { + "chromosome": chromosome, + "start": cluster_start, + "end": cluster_end, + "event_type": ( + "master_bed_region" + if "master_bed_region" in cluster_types + else "supplementary" + ), + "read_count": len( + cluster_reads + ), # Count of unique reads (set automatically handles deduplication) + "avg_mapping_quality": ( + round(sum(cluster_mapqs) / len(cluster_mapqs), 1) + if cluster_mapqs + else 0 + ), + "avg_mapping_span": ( + round(sum(cluster_spans) / len(cluster_spans), 0) + if cluster_spans + else 0 + ), + } + ) if not clustered_results: return pd.DataFrame() @@ -758,8 +839,12 @@ def _cluster_nearby_regions(regions_df: pd.DataFrame, max_distance: int = 200) - return pd.DataFrame(clustered_results) -def _summarize_master_bed_events(df: pd.DataFrame, min_read_support: int = 3, min_mapq: int = 50, - cluster_distance: int = 5000) -> pd.DataFrame: +def _summarize_master_bed_events( + df: pd.DataFrame, + min_read_support: int = 3, + min_mapq: int = 50, + cluster_distance: int = 5000, +) -> pd.DataFrame: """ Summarize master BED data into detected breakpoint pair events. @@ -788,13 +873,21 @@ def _summarize_master_bed_events(df: pd.DataFrame, min_read_support: int = 3, mi # This prevents the GUI from freezing when processing very large datasets MAX_ROWS_TO_PROCESS = 20000 # Limit to prevent UI blocking (reduced from 50000 for better responsiveness) if len(df) > MAX_ROWS_TO_PROCESS: - logging.warning(f"[Fusion] Master BED dataset too large ({len(df)} rows), limiting to {MAX_ROWS_TO_PROCESS} rows for UI performance") + logging.warning( + f"[Fusion] Master BED dataset too large ({len(df)} rows), limiting to {MAX_ROWS_TO_PROCESS} rows for UI performance" + ) # Sample rows rather than just taking head() to get better coverage - df = df.sample(n=MAX_ROWS_TO_PROCESS, random_state=42).copy() if len(df) > MAX_ROWS_TO_PROCESS else df.copy() + df = ( + df.sample(n=MAX_ROWS_TO_PROCESS, random_state=42).copy() + if len(df) > MAX_ROWS_TO_PROCESS + else df.copy() + ) # Check required columns if "read_id" not in df.columns: - logging.debug("[Fusion] Missing required column (read_id) in master BED candidates") + logging.debug( + "[Fusion] Missing required column (read_id) in master BED candidates" + ) return pd.DataFrame() df = df.copy() @@ -805,14 +898,15 @@ def _summarize_master_bed_events(df: pd.DataFrame, min_read_support: int = 3, mi if "mapping_quality" in df.columns and "col4" in df.columns: # Keep master_bed_region entries regardless of MapQ (they're primary alignments) # Filter supplementary entries to only high-quality ones - high_quality_mask = ( - (df["col4"] == "master_bed_region") | - (df["mapping_quality"] >= min_mapq) + high_quality_mask = (df["col4"] == "master_bed_region") | ( + df["mapping_quality"] >= min_mapq ) df = df[high_quality_mask].copy() if df.empty: - logging.debug(f"[Fusion] No high-quality mappings found (MapQ >= {min_mapq})") + logging.debug( + f"[Fusion] No high-quality mappings found (MapQ >= {min_mapq})" + ) return pd.DataFrame() # Separate primary and supplementary alignments using col4 @@ -823,13 +917,17 @@ def _summarize_master_bed_events(df: pd.DataFrame, min_read_support: int = 3, mi else: # Fallback: use is_supplementary if col4 is not available if "is_supplementary" not in df.columns: - logging.debug("[Fusion] Missing required columns (col4 or is_supplementary) in master BED candidates") + logging.debug( + "[Fusion] Missing required columns (col4 or is_supplementary) in master BED candidates" + ) return pd.DataFrame() primary_df = df[df["is_supplementary"] == False].copy() supplementary_df = df[df["is_supplementary"] == True].copy() if primary_df.empty or supplementary_df.empty: - logging.debug("[Fusion] Need both primary and supplementary alignments to identify breakpoint pairs") + logging.debug( + "[Fusion] Need both primary and supplementary alignments to identify breakpoint pairs" + ) return pd.DataFrame() # Find reads that have both primary and supplementary alignments @@ -838,12 +936,18 @@ def _summarize_master_bed_events(df: pd.DataFrame, min_read_support: int = 3, mi reads_with_both = primary_read_ids & supplementary_read_ids if not reads_with_both: - logging.debug("[Fusion] No reads have both primary and supplementary alignments") + logging.debug( + "[Fusion] No reads have both primary and supplementary alignments" + ) return pd.DataFrame() # Filter to only reads with both primary and supplementary alignments - primary_filtered = primary_df[primary_df["read_id"].isin(reads_with_both)].copy() - supplementary_filtered = supplementary_df[supplementary_df["read_id"].isin(reads_with_both)].copy() + primary_filtered = primary_df[ + primary_df["read_id"].isin(reads_with_both) + ].copy() + supplementary_filtered = supplementary_df[ + supplementary_df["read_id"].isin(reads_with_both) + ].copy() # Build breakpoint pairs: for each read, pair its primary alignment with each supplementary alignment breakpoint_pairs = [] @@ -854,8 +958,16 @@ def _summarize_master_bed_events(df: pd.DataFrame, min_read_support: int = 3, mi for read_id in reads_with_both: # Get all primary and supplementary alignments for this read - read_primaries = primary_by_read.get_group(read_id) if read_id in primary_by_read.groups else pd.DataFrame() - read_supplementaries = supplementary_by_read.get_group(read_id) if read_id in supplementary_by_read.groups else pd.DataFrame() + read_primaries = ( + primary_by_read.get_group(read_id) + if read_id in primary_by_read.groups + else pd.DataFrame() + ) + read_supplementaries = ( + supplementary_by_read.get_group(read_id) + if read_id in supplementary_by_read.groups + else pd.DataFrame() + ) if read_primaries.empty or read_supplementaries.empty: continue @@ -873,43 +985,49 @@ def _summarize_master_bed_events(df: pd.DataFrame, min_read_support: int = 3, mi if "mapping_span" in read_supplementaries.columns: supp_cols.append("mapping_span") - primaries_list = list(read_primaries[primary_cols].itertuples(index=False, name=None)) - supplementaries_list = list(read_supplementaries[supp_cols].itertuples(index=False, name=None)) + primaries_list = list( + read_primaries[primary_cols].itertuples(index=False, name=None) + ) + supplementaries_list = list( + read_supplementaries[supp_cols].itertuples(index=False, name=None) + ) # Create all combinations p_idx = {name: i for i, name in enumerate(primary_cols)} s_idx = {name: i for i, name in enumerate(supp_cols)} for primary_row in primaries_list: for supp_row in supplementaries_list: - breakpoint_pairs.append({ - "read_id": read_id, - "primary_chrom": primary_row[p_idx["reference_id"]], - "primary_start": int(primary_row[p_idx["reference_start"]]), - "primary_end": int(primary_row[p_idx["reference_end"]]), - "primary_mapq": ( - primary_row[p_idx["mapping_quality"]] - if "mapping_quality" in p_idx - else 0 - ), - "primary_span": ( - primary_row[p_idx["mapping_span"]] - if "mapping_span" in p_idx - else 0 - ), - "supp_chrom": supp_row[s_idx["reference_id"]], - "supp_start": int(supp_row[s_idx["reference_start"]]), - "supp_end": int(supp_row[s_idx["reference_end"]]), - "supp_mapq": ( - supp_row[s_idx["mapping_quality"]] - if "mapping_quality" in s_idx - else 0 - ), - "supp_span": ( - supp_row[s_idx["mapping_span"]] - if "mapping_span" in s_idx - else 0 - ), - }) + breakpoint_pairs.append( + { + "read_id": read_id, + "primary_chrom": primary_row[p_idx["reference_id"]], + "primary_start": int(primary_row[p_idx["reference_start"]]), + "primary_end": int(primary_row[p_idx["reference_end"]]), + "primary_mapq": ( + primary_row[p_idx["mapping_quality"]] + if "mapping_quality" in p_idx + else 0 + ), + "primary_span": ( + primary_row[p_idx["mapping_span"]] + if "mapping_span" in p_idx + else 0 + ), + "supp_chrom": supp_row[s_idx["reference_id"]], + "supp_start": int(supp_row[s_idx["reference_start"]]), + "supp_end": int(supp_row[s_idx["reference_end"]]), + "supp_mapq": ( + supp_row[s_idx["mapping_quality"]] + if "mapping_quality" in s_idx + else 0 + ), + "supp_span": ( + supp_row[s_idx["mapping_span"]] + if "mapping_span" in s_idx + else 0 + ), + } + ) if not breakpoint_pairs: logging.debug("[Fusion] No breakpoint pairs created") @@ -918,9 +1036,12 @@ def _summarize_master_bed_events(df: pd.DataFrame, min_read_support: int = 3, mi # Early exit: if too many breakpoint pairs, limit to prevent UI blocking MAX_PAIRS_TO_PROCESS = 10000 # Limit clustering to prevent UI blocking if len(breakpoint_pairs) > MAX_PAIRS_TO_PROCESS: - logging.warning(f"[Fusion] Too many breakpoint pairs ({len(breakpoint_pairs)}), limiting to {MAX_PAIRS_TO_PROCESS} for UI performance") + logging.warning( + f"[Fusion] Too many breakpoint pairs ({len(breakpoint_pairs)}), limiting to {MAX_PAIRS_TO_PROCESS} for UI performance" + ) # Sample pairs to get better coverage import random + breakpoint_pairs = random.sample(breakpoint_pairs, MAX_PAIRS_TO_PROCESS) # Cluster similar breakpoint pairs @@ -944,9 +1065,9 @@ def _summarize_master_bed_events(df: pd.DataFrame, min_read_support: int = 3, mi continue # Sort pairs within this chromosome combination - sorted_chrom_pairs = sorted(chrom_pairs, key=lambda x: ( - x[1]["primary_start"], x[1]["supp_start"] - )) + sorted_chrom_pairs = sorted( + chrom_pairs, key=lambda x: (x[1]["primary_start"], x[1]["supp_start"]) + ) # Cluster pairs in this chromosome group for idx, (orig_i, pair) in enumerate(sorted_chrom_pairs): @@ -983,14 +1104,18 @@ def _summarize_master_bed_events(df: pd.DataFrame, min_read_support: int = 3, mi # Check if primary locations are similar (same chromosome, close coordinates) primary_similar = ( - other_pair["primary_start"] <= cluster_primary_max + cluster_distance and - other_pair["primary_end"] >= cluster_primary_min - cluster_distance + other_pair["primary_start"] + <= cluster_primary_max + cluster_distance + and other_pair["primary_end"] + >= cluster_primary_min - cluster_distance ) # Check if supplementary locations are similar (same chromosome, close coordinates) supp_similar = ( - other_pair["supp_start"] <= cluster_supp_max + cluster_distance and - other_pair["supp_end"] >= cluster_supp_min - cluster_distance + other_pair["supp_start"] + <= cluster_supp_max + cluster_distance + and other_pair["supp_end"] + >= cluster_supp_min - cluster_distance ) # Both primary and supplementary must be similar to cluster @@ -1013,56 +1138,82 @@ def _summarize_master_bed_events(df: pd.DataFrame, min_read_support: int = 3, mi cluster_supp_max = max(cluster_supp_ends) # Create clustered breakpoint pair - clustered_pairs.append({ - "primary_chrom": cluster_primary_chrom, - "primary_start": min(cluster_primary_starts), - "primary_end": max(cluster_primary_ends), - "primary_avg_mapq": sum(cluster_primary_mapqs) / len(cluster_primary_mapqs) if cluster_primary_mapqs else 0, - "primary_avg_span": sum(cluster_primary_spans) / len(cluster_primary_spans) if cluster_primary_spans else 0, - "supp_chrom": cluster_supp_chrom, - "supp_start": min(cluster_supp_starts), - "supp_end": max(cluster_supp_ends), - "supp_avg_mapq": sum(cluster_supp_mapqs) / len(cluster_supp_mapqs) if cluster_supp_mapqs else 0, - "supp_avg_span": sum(cluster_supp_spans) / len(cluster_supp_spans) if cluster_supp_spans else 0, - "read_count": len(cluster_read_ids), - }) + clustered_pairs.append( + { + "primary_chrom": cluster_primary_chrom, + "primary_start": min(cluster_primary_starts), + "primary_end": max(cluster_primary_ends), + "primary_avg_mapq": ( + sum(cluster_primary_mapqs) / len(cluster_primary_mapqs) + if cluster_primary_mapqs + else 0 + ), + "primary_avg_span": ( + sum(cluster_primary_spans) / len(cluster_primary_spans) + if cluster_primary_spans + else 0 + ), + "supp_chrom": cluster_supp_chrom, + "supp_start": min(cluster_supp_starts), + "supp_end": max(cluster_supp_ends), + "supp_avg_mapq": ( + sum(cluster_supp_mapqs) / len(cluster_supp_mapqs) + if cluster_supp_mapqs + else 0 + ), + "supp_avg_span": ( + sum(cluster_supp_spans) / len(cluster_supp_spans) + if cluster_supp_spans + else 0 + ), + "read_count": len(cluster_read_ids), + } + ) # Filter for breakpoint pairs with sufficient read support supported_pairs = [ - p for p in clustered_pairs - if p["read_count"] >= min_read_support + p for p in clustered_pairs if p["read_count"] >= min_read_support ] if not supported_pairs: - logging.debug(f"[Fusion] No breakpoint pairs found with >= {min_read_support} read support") + logging.debug( + f"[Fusion] No breakpoint pairs found with >= {min_read_support} read support" + ) return pd.DataFrame() # Convert to DataFrame format - include both primary and supplementary regions events = [] for pair in supported_pairs: # Add primary region event - if pair["primary_start"] > 0 and pair["primary_end"] > pair["primary_start"]: - events.append({ - "chromosome": pair["primary_chrom"], - "start": pair["primary_start"], - "end": pair["primary_end"], - "event_type": "breakpoint_pair_primary", - "read_count": pair["read_count"], - "avg_mapping_quality": round(pair["primary_avg_mapq"], 1), - "avg_mapping_span": round(pair["primary_avg_span"], 0), - }) + if ( + pair["primary_start"] > 0 + and pair["primary_end"] > pair["primary_start"] + ): + events.append( + { + "chromosome": pair["primary_chrom"], + "start": pair["primary_start"], + "end": pair["primary_end"], + "event_type": "breakpoint_pair_primary", + "read_count": pair["read_count"], + "avg_mapping_quality": round(pair["primary_avg_mapq"], 1), + "avg_mapping_span": round(pair["primary_avg_span"], 0), + } + ) # Add supplementary region event if pair["supp_start"] > 0 and pair["supp_end"] > pair["supp_start"]: - events.append({ - "chromosome": pair["supp_chrom"], - "start": pair["supp_start"], - "end": pair["supp_end"], - "event_type": "breakpoint_pair_supplementary", - "read_count": pair["read_count"], - "avg_mapping_quality": round(pair["supp_avg_mapq"], 1), - "avg_mapping_span": round(pair["supp_avg_span"], 0), - }) + events.append( + { + "chromosome": pair["supp_chrom"], + "start": pair["supp_start"], + "end": pair["supp_end"], + "event_type": "breakpoint_pair_supplementary", + "read_count": pair["read_count"], + "avg_mapping_quality": round(pair["supp_avg_mapq"], 1), + "avg_mapping_span": round(pair["supp_avg_span"], 0), + } + ) if not events: return pd.DataFrame() @@ -1071,8 +1222,7 @@ def _summarize_master_bed_events(df: pd.DataFrame, min_read_support: int = 3, mi # Sort by read count (descending), then by chromosome and start result_df = result_df.sort_values( - ["read_count", "chromosome", "start"], - ascending=[False, True, True] + ["read_count", "chromosome", "start"], ascending=[False, True, True] ) return result_df @@ -1080,11 +1230,14 @@ def _summarize_master_bed_events(df: pd.DataFrame, min_read_support: int = 3, mi except Exception as e: logging.warning(f"[Fusion] Failed to summarize master BED events: {e}") import traceback + logging.debug(f"[Fusion] Traceback: {traceback.format_exc()}") return pd.DataFrame() -def _make_master_bed_summary_table(container: Any, df: pd.DataFrame, sample_dir: Optional[Path] = None) -> Any: +def _make_master_bed_summary_table( + container: Any, df: pd.DataFrame, sample_dir: Optional[Path] = None +) -> Any: """Create a summary table for master BED events (regions) instead of individual reads. Args: @@ -1099,14 +1252,20 @@ def _make_master_bed_summary_table(container: Any, df: pd.DataFrame, sample_dir: if summary_file.exists(): try: summary_df = pd.read_csv(summary_file) - logging.debug(f"[Fusion] Loaded pre-computed master BED events summary from {summary_file}") + logging.debug( + f"[Fusion] Loaded pre-computed master BED events summary from {summary_file}" + ) except Exception as e: logging.warning(f"[Fusion] Failed to load pre-computed summary: {e}") # Fallback to computing on the fly if summary file doesn't exist (backward compatibility) if summary_df.empty and df is not None and not df.empty: - logging.debug("[Fusion] Pre-computed summary not found, computing on the fly (this may be slow)") - summary_df = _summarize_master_bed_events(df, min_read_support=3, min_mapq=50, cluster_distance=5000) + logging.debug( + "[Fusion] Pre-computed summary not found, computing on the fly (this may be slow)" + ) + summary_df = _summarize_master_bed_events( + df, min_read_support=3, min_mapq=50, cluster_distance=5000 + ) if summary_df.empty: with container: @@ -1181,8 +1340,11 @@ def _make_master_bed_summary_table(container: Any, df: pd.DataFrame, sample_dir: return table -def _cluster_fusion_reads(filtered_data: pd.DataFrame, max_distance: int = 10000, - use_breakpoint_validation: bool = True) -> pd.DataFrame: +def _cluster_fusion_reads( + filtered_data: pd.DataFrame, + max_distance: int = 10000, + use_breakpoint_validation: bool = True, +) -> pd.DataFrame: """ Cluster fusion reads by similar mapping coordinates with optional breakpoint validation. @@ -1202,7 +1364,7 @@ def _cluster_fusion_reads(filtered_data: pd.DataFrame, max_distance: int = 10000 validated_breakpoints = _validate_fusion_breakpoints( filtered_data, min_read_support=4, - max_breakpoint_distance=100 # Much tighter clustering for breakpoints + max_breakpoint_distance=100, # Much tighter clustering for breakpoints ) if validated_breakpoints.empty: @@ -1212,21 +1374,25 @@ def _cluster_fusion_reads(filtered_data: pd.DataFrame, max_distance: int = 10000 # Convert to the expected format for the summary table clustered_results = [] for _, row in validated_breakpoints.iterrows(): - clustered_results.append({ - "fusion_pair": row["gene_pair"], - "chr1": row["gene1_chr"], - "chr2": row["gene2_chr"], - "gene1": row["gene1"], - "gene1_position": row["gene1_breakpoint"], - "gene2": row["gene2"], - "gene2_position": row["gene2_breakpoint"], - "reads": row["supporting_reads"], - "avg_mapping_quality": row["avg_mapping_quality"], - "avg_mapping_span": row["avg_mapping_span"], - "cluster_id": row["cluster_id"] - }) - - logging.info(f"[Fusion] Breakpoint validation found {len(clustered_results)} validated fusion clusters") + clustered_results.append( + { + "fusion_pair": row["gene_pair"], + "chr1": row["gene1_chr"], + "chr2": row["gene2_chr"], + "gene1": row["gene1"], + "gene1_position": row["gene1_breakpoint"], + "gene2": row["gene2"], + "gene2_position": row["gene2_breakpoint"], + "reads": row["supporting_reads"], + "avg_mapping_quality": row["avg_mapping_quality"], + "avg_mapping_span": row["avg_mapping_span"], + "cluster_id": row["cluster_id"], + } + ) + + logging.info( + f"[Fusion] Breakpoint validation found {len(clustered_results)} validated fusion clusters" + ) return pd.DataFrame(clustered_results) else: @@ -1251,7 +1417,7 @@ def _cluster_fusion_reads(filtered_data: pd.DataFrame, max_distance: int = 10000 gene_info[gene] = { "chromosome": first_row.get("reference_id", "Unknown"), "start": first_row.get("reference_start", 0), - "end": first_row.get("reference_end", 0) + "end": first_row.get("reference_end", 0), } # Only add if we have info for at least 2 genes @@ -1260,18 +1426,20 @@ def _cluster_fusion_reads(filtered_data: pd.DataFrame, max_distance: int = 10000 if len(genes_with_info) >= 2: gene1, gene2 = genes_with_info[0], genes_with_info[1] - fusion_summary.append({ - "fusion_pair": gene_pair, - "chr1": gene_info[gene1]["chromosome"], - "chr2": gene_info[gene2]["chromosome"], - "gene1": gene1, - "gene1_start": gene_info[gene1]["start"], - "gene1_end": gene_info[gene1]["end"], - "gene2": gene2, - "gene2_start": gene_info[gene2]["start"], - "gene2_end": gene_info[gene2]["end"], - "read_id": read_id - }) + fusion_summary.append( + { + "fusion_pair": gene_pair, + "chr1": gene_info[gene1]["chromosome"], + "chr2": gene_info[gene2]["chromosome"], + "gene1": gene1, + "gene1_start": gene_info[gene1]["start"], + "gene1_end": gene_info[gene1]["end"], + "gene2": gene2, + "gene2_start": gene_info[gene2]["start"], + "gene2_end": gene_info[gene2]["end"], + "read_id": read_id, + } + ) if not fusion_summary: return pd.DataFrame() @@ -1300,12 +1468,21 @@ def _cluster_fusion_reads(filtered_data: pd.DataFrame, max_distance: int = 10000 # Find reads that belong to both clusters cluster_reads = [] for idx, row in chr_group.iterrows(): - gene1_start, gene1_end = row["gene1_start"], row["gene1_end"] - gene2_start, gene2_end = row["gene2_start"], row["gene2_end"] + gene1_start, gene1_end = ( + row["gene1_start"], + row["gene1_end"], + ) + gene2_start, gene2_end = ( + row["gene2_start"], + row["gene2_end"], + ) # Check if this read belongs to both clusters - if (_position_in_cluster(gene1_start, gene1_end, gene1_cluster, max_distance) and - _position_in_cluster(gene2_start, gene2_end, gene2_cluster, max_distance)): + if _position_in_cluster( + gene1_start, gene1_end, gene1_cluster, max_distance + ) and _position_in_cluster( + gene2_start, gene2_end, gene2_cluster, max_distance + ): cluster_reads.append(row["read_id"]) # Only include clusters with minimum read support (4 or more reads) @@ -1316,16 +1493,18 @@ def _cluster_fusion_reads(filtered_data: pd.DataFrame, max_distance: int = 10000 gene2_min_start = min(gene2_cluster[:, 0]) gene2_max_end = max(gene2_cluster[:, 1]) - clustered_results.append({ - "fusion_pair": fusion_pair, - "chr1": chr1, - "chr2": chr2, - "gene1": chr_group.iloc[0]["gene1"], - "gene1_position": f"{gene1_min_start}-{gene1_max_end}", - "gene2": chr_group.iloc[0]["gene2"], - "gene2_position": f"{gene2_min_start}-{gene2_max_end}", - "reads": len(cluster_reads) - }) + clustered_results.append( + { + "fusion_pair": fusion_pair, + "chr1": chr1, + "chr2": chr2, + "gene1": chr_group.iloc[0]["gene1"], + "gene1_position": f"{gene1_min_start}-{gene1_max_end}", + "gene2": chr_group.iloc[0]["gene2"], + "gene2_position": f"{gene2_min_start}-{gene2_max_end}", + "reads": len(cluster_reads), + } + ) return pd.DataFrame(clustered_results) @@ -1351,8 +1530,9 @@ def _cluster_positions(positions: np.ndarray, max_distance: int) -> List[np.ndar continue # Check if positions overlap or are close - if (_positions_overlap(start, end, other_start, other_end) or - _positions_close(start, end, other_start, other_end, max_distance)): + if _positions_overlap( + start, end, other_start, other_end + ) or _positions_close(start, end, other_start, other_end, max_distance): cluster.append(positions[j]) used.add(j) @@ -1366,18 +1546,24 @@ def _positions_overlap(start1: int, end1: int, start2: int, end2: int) -> bool: return not (end1 < start2 or end2 < start1) -def _positions_close(start1: int, end1: int, start2: int, end2: int, max_distance: int) -> bool: +def _positions_close( + start1: int, end1: int, start2: int, end2: int, max_distance: int +) -> bool: """Check if two genomic positions are within max_distance.""" - distance = min(abs(start1 - start2), abs(end1 - end2), - abs(start1 - end2), abs(end1 - start2)) + distance = min( + abs(start1 - start2), abs(end1 - end2), abs(start1 - end2), abs(end1 - start2) + ) return distance <= max_distance -def _position_in_cluster(start: int, end: int, cluster: np.ndarray, max_distance: int) -> bool: +def _position_in_cluster( + start: int, end: int, cluster: np.ndarray, max_distance: int +) -> bool: """Check if a position belongs to a cluster.""" for cluster_start, cluster_end in cluster: - if (_positions_overlap(start, end, cluster_start, cluster_end) or - _positions_close(start, end, cluster_start, cluster_end, max_distance)): + if _positions_overlap( + start, end, cluster_start, cluster_end + ) or _positions_close(start, end, cluster_start, cluster_end, max_distance): return True return False @@ -1386,6 +1572,7 @@ def _position_in_cluster(start: int, end: int, cluster: np.ndarray, max_distance # BREAKPOINT VALIDATION FUNCTIONS # ============================================================================= + def _extract_fusion_breakpoints(annotated_data: pd.DataFrame) -> pd.DataFrame: """ Extract fusion breakpoints from annotated fusion data. @@ -1425,7 +1612,7 @@ def _extract_fusion_breakpoints(annotated_data: pd.DataFrame) -> pd.DataFrame: "end": first_row.get("reference_end", 0), "strand": first_row.get("strand", "+"), "mapping_quality": first_row.get("mapping_quality", 0), - "mapping_span": first_row.get("mapping_span", 0) + "mapping_span": first_row.get("mapping_span", 0), } # Only proceed if we have breakpoint info for at least 2 genes @@ -1436,33 +1623,37 @@ def _extract_fusion_breakpoints(annotated_data: pd.DataFrame) -> pd.DataFrame: for j in range(i + 1, len(gene_list)): gene1, gene2 = gene_list[i], gene_list[j] - breakpoint_data.append({ - "read_id": read_id, - "gene_pair": f"{gene1}-{gene2}", - "gene1": gene1, - "gene1_chr": gene_breakpoints[gene1]["chromosome"], - "gene1_start": gene_breakpoints[gene1]["start"], - "gene1_end": gene_breakpoints[gene1]["end"], - "gene1_strand": gene_breakpoints[gene1]["strand"], - "gene2": gene2, - "gene2_chr": gene_breakpoints[gene2]["chromosome"], - "gene2_start": gene_breakpoints[gene2]["start"], - "gene2_end": gene_breakpoints[gene2]["end"], - "gene2_strand": gene_breakpoints[gene2]["strand"], - "min_mapping_quality": min( - gene_breakpoints[gene1]["mapping_quality"], - gene_breakpoints[gene2]["mapping_quality"] - ), - "min_mapping_span": min( - gene_breakpoints[gene1]["mapping_span"], - gene_breakpoints[gene2]["mapping_span"] - ) - }) + breakpoint_data.append( + { + "read_id": read_id, + "gene_pair": f"{gene1}-{gene2}", + "gene1": gene1, + "gene1_chr": gene_breakpoints[gene1]["chromosome"], + "gene1_start": gene_breakpoints[gene1]["start"], + "gene1_end": gene_breakpoints[gene1]["end"], + "gene1_strand": gene_breakpoints[gene1]["strand"], + "gene2": gene2, + "gene2_chr": gene_breakpoints[gene2]["chromosome"], + "gene2_start": gene_breakpoints[gene2]["start"], + "gene2_end": gene_breakpoints[gene2]["end"], + "gene2_strand": gene_breakpoints[gene2]["strand"], + "min_mapping_quality": min( + gene_breakpoints[gene1]["mapping_quality"], + gene_breakpoints[gene2]["mapping_quality"], + ), + "min_mapping_span": min( + gene_breakpoints[gene1]["mapping_span"], + gene_breakpoints[gene2]["mapping_span"], + ), + } + ) return pd.DataFrame(breakpoint_data) -def _cluster_breakpoints(breakpoint_data: pd.DataFrame, max_distance: int = 100) -> pd.DataFrame: +def _cluster_breakpoints( + breakpoint_data: pd.DataFrame, max_distance: int = 100 +) -> pd.DataFrame: """ Cluster breakpoints by similar coordinates within each gene pair. @@ -1482,7 +1673,9 @@ def _cluster_breakpoints(breakpoint_data: pd.DataFrame, max_distance: int = 100) clustered_results = [] # Group by gene pair and chromosome combination - for (gene_pair, chr1, chr2), group in breakpoint_data.groupby(["gene_pair", "gene1_chr", "gene2_chr"]): + for (gene_pair, chr1, chr2), group in breakpoint_data.groupby( + ["gene_pair", "gene1_chr", "gene2_chr"] + ): if group.empty: continue @@ -1507,8 +1700,11 @@ def _cluster_breakpoints(breakpoint_data: pd.DataFrame, max_distance: int = 100) gene2_start, gene2_end = row["gene2_start"], row["gene2_end"] # Check if this read belongs to both clusters - if (_position_in_cluster(gene1_start, gene1_end, gene1_cluster, max_distance) and - _position_in_cluster(gene2_start, gene2_end, gene2_cluster, max_distance)): + if _position_in_cluster( + gene1_start, gene1_end, gene1_cluster, max_distance + ) and _position_in_cluster( + gene2_start, gene2_end, gene2_cluster, max_distance + ): cluster_reads.append(row["read_id"]) cluster_mapping_qualities.append(row["min_mapping_quality"]) cluster_mapping_spans.append(row["min_mapping_span"]) @@ -1521,33 +1717,44 @@ def _cluster_breakpoints(breakpoint_data: pd.DataFrame, max_distance: int = 100) gene2_max_end = max(gene2_cluster[:, 1]) # Calculate average quality metrics - avg_mapping_quality = np.mean(cluster_mapping_qualities) if cluster_mapping_qualities else 0 - avg_mapping_span = np.mean(cluster_mapping_spans) if cluster_mapping_spans else 0 - - clustered_results.append({ - "gene_pair": gene_pair, - "gene1": group.iloc[0]["gene1"], - "gene1_chr": chr1, - "gene1_breakpoint": f"{gene1_min_start}-{gene1_max_end}", - "gene1_start": gene1_min_start, - "gene1_end": gene1_max_end, - "gene2": group.iloc[0]["gene2"], - "gene2_chr": chr2, - "gene2_breakpoint": f"{gene2_min_start}-{gene2_max_end}", - "gene2_start": gene2_min_start, - "gene2_end": gene2_max_end, - "supporting_reads": len(cluster_reads), - "read_ids": cluster_reads, - "avg_mapping_quality": avg_mapping_quality, - "avg_mapping_span": avg_mapping_span, - "cluster_id": f"{gene_pair}_{i}_{j}" - }) + avg_mapping_quality = ( + np.mean(cluster_mapping_qualities) + if cluster_mapping_qualities + else 0 + ) + avg_mapping_span = ( + np.mean(cluster_mapping_spans) if cluster_mapping_spans else 0 + ) + + clustered_results.append( + { + "gene_pair": gene_pair, + "gene1": group.iloc[0]["gene1"], + "gene1_chr": chr1, + "gene1_breakpoint": f"{gene1_min_start}-{gene1_max_end}", + "gene1_start": gene1_min_start, + "gene1_end": gene1_max_end, + "gene2": group.iloc[0]["gene2"], + "gene2_chr": chr2, + "gene2_breakpoint": f"{gene2_min_start}-{gene2_max_end}", + "gene2_start": gene2_min_start, + "gene2_end": gene2_max_end, + "supporting_reads": len(cluster_reads), + "read_ids": cluster_reads, + "avg_mapping_quality": avg_mapping_quality, + "avg_mapping_span": avg_mapping_span, + "cluster_id": f"{gene_pair}_{i}_{j}", + } + ) return pd.DataFrame(clustered_results) -def _validate_fusion_breakpoints(annotated_data: pd.DataFrame, min_read_support: int = 4, - max_breakpoint_distance: int = 100) -> pd.DataFrame: +def _validate_fusion_breakpoints( + annotated_data: pd.DataFrame, + min_read_support: int = 4, + max_breakpoint_distance: int = 100, +) -> pd.DataFrame: """ Validate fusion candidates by requiring consistent breakpoint support. @@ -1567,7 +1774,9 @@ def _validate_fusion_breakpoints(annotated_data: pd.DataFrame, min_read_support: if annotated_data.empty: return pd.DataFrame() - logging.info(f"[Fusion] Validating breakpoints for {len(annotated_data)} fusion candidates") + logging.info( + f"[Fusion] Validating breakpoints for {len(annotated_data)} fusion candidates" + ) # Extract breakpoints from fusion data breakpoint_data = _extract_fusion_breakpoints(annotated_data) @@ -1579,28 +1788,38 @@ def _validate_fusion_breakpoints(annotated_data: pd.DataFrame, min_read_support: logging.info(f"[Fusion] Extracted {len(breakpoint_data)} breakpoint records") # Cluster breakpoints by similar coordinates - clustered_breakpoints = _cluster_breakpoints(breakpoint_data, max_breakpoint_distance) + clustered_breakpoints = _cluster_breakpoints( + breakpoint_data, max_breakpoint_distance + ) if clustered_breakpoints.empty: logging.info("[Fusion] No clustered breakpoints found") return pd.DataFrame() - logging.info(f"[Fusion] Found {len(clustered_breakpoints)} clustered breakpoint groups") + logging.info( + f"[Fusion] Found {len(clustered_breakpoints)} clustered breakpoint groups" + ) # Filter by minimum read support validated_breakpoints = clustered_breakpoints[ clustered_breakpoints["supporting_reads"] >= min_read_support ] - logging.info(f"[Fusion] {len(validated_breakpoints)} breakpoint groups meet minimum support threshold ({min_read_support})") + logging.info( + f"[Fusion] {len(validated_breakpoints)} breakpoint groups meet minimum support threshold ({min_read_support})" + ) # Sort by supporting reads (descending) - validated_breakpoints = validated_breakpoints.sort_values("supporting_reads", ascending=False) + validated_breakpoints = validated_breakpoints.sort_values( + "supporting_reads", ascending=False + ) return validated_breakpoints -def _get_validated_fusion_pairs(annotated_data: pd.DataFrame, goodpairs: pd.Series) -> List[List[str]]: +def _get_validated_fusion_pairs( + annotated_data: pd.DataFrame, goodpairs: pd.Series +) -> List[List[str]]: """Extract validated fusion pairs from annotated_data for dropdown options. Returns a list of gene pairs as lists (e.g., [["GENE1", "GENE2"], ...]) @@ -1610,7 +1829,9 @@ def _get_validated_fusion_pairs(annotated_data: pd.DataFrame, goodpairs: pd.Seri # Filter to good pairs if available if not goodpairs.empty and goodpairs.sum() > 0: # Align indices to avoid reindexing warning - aligned_goodpairs = goodpairs.reindex(annotated_data.index, fill_value=False) + aligned_goodpairs = goodpairs.reindex( + annotated_data.index, fill_value=False + ) filtered_data = annotated_data[aligned_goodpairs] else: filtered_data = annotated_data @@ -1619,7 +1840,9 @@ def _get_validated_fusion_pairs(annotated_data: pd.DataFrame, goodpairs: pd.Seri return [] # Get validated fusion pairs using breakpoint validation - clustered_data = _cluster_fusion_reads(filtered_data, max_distance=10000, use_breakpoint_validation=True) + clustered_data = _cluster_fusion_reads( + filtered_data, max_distance=10000, use_breakpoint_validation=True + ) if clustered_data.empty: return [] @@ -1641,7 +1864,9 @@ def _get_validated_fusion_pairs(annotated_data: pd.DataFrame, goodpairs: pd.Seri validated_pairs.append(genes_sorted) seen_pairs.add(pair_key) - logging.info(f"[Fusion] Extracted {len(validated_pairs)} validated fusion pairs for dropdown") + logging.info( + f"[Fusion] Extracted {len(validated_pairs)} validated fusion pairs for dropdown" + ) return validated_pairs except Exception as e: @@ -1649,7 +1874,9 @@ def _get_validated_fusion_pairs(annotated_data: pd.DataFrame, goodpairs: pd.Seri return [] -def _make_fusion_summary_table(container: Any, annotated_data: pd.DataFrame, goodpairs: pd.Series) -> Any: +def _make_fusion_summary_table( + container: Any, annotated_data: pd.DataFrame, goodpairs: pd.Series +) -> Any: """Create a summary table showing fusion pairs with chromosomes, positions, and read counts.""" if annotated_data is None or annotated_data.empty: with container: @@ -1664,7 +1891,9 @@ def _make_fusion_summary_table(container: Any, annotated_data: pd.DataFrame, goo # Filter to good pairs if available if not goodpairs.empty and goodpairs.sum() > 0: # Align indices to avoid reindexing warning - aligned_goodpairs = goodpairs.reindex(annotated_data.index, fill_value=False) + aligned_goodpairs = goodpairs.reindex( + annotated_data.index, fill_value=False + ) filtered_data = annotated_data[aligned_goodpairs] else: filtered_data = annotated_data @@ -1674,7 +1903,9 @@ def _make_fusion_summary_table(container: Any, annotated_data: pd.DataFrame, goo return None # Cluster fusion reads by similar coordinates - clustered_data = _cluster_fusion_reads(filtered_data, max_distance=10000, use_breakpoint_validation=True) + clustered_data = _cluster_fusion_reads( + filtered_data, max_distance=10000, use_breakpoint_validation=True + ) if clustered_data.empty: ui.label("No fusion pairs found").classes("text-gray-600") @@ -1685,21 +1916,65 @@ def _make_fusion_summary_table(container: Any, annotated_data: pd.DataFrame, goo # Create columns definition with breakpoint validation info columns = [ - {"name": "fusion_pair", "label": "Fusion Pair", "field": "fusion_pair", "sortable": True}, + { + "name": "fusion_pair", + "label": "Fusion Pair", + "field": "fusion_pair", + "sortable": True, + }, {"name": "chr1", "label": "Chr 1", "field": "chr1", "sortable": True}, {"name": "chr2", "label": "Chr 2", "field": "chr2", "sortable": True}, - {"name": "gene1", "label": "Gene 1", "field": "gene1", "sortable": True}, - {"name": "gene1_position", "label": "Gene 1 Breakpoint", "field": "gene1_position", "sortable": True}, - {"name": "gene2", "label": "Gene 2", "field": "gene2", "sortable": True}, - {"name": "gene2_position", "label": "Gene 2 Breakpoint", "field": "gene2_position", "sortable": True}, - {"name": "reads", "label": "Supporting Reads", "field": "reads", "sortable": True} + { + "name": "gene1", + "label": "Gene 1", + "field": "gene1", + "sortable": True, + }, + { + "name": "gene1_position", + "label": "Gene 1 Breakpoint", + "field": "gene1_position", + "sortable": True, + }, + { + "name": "gene2", + "label": "Gene 2", + "field": "gene2", + "sortable": True, + }, + { + "name": "gene2_position", + "label": "Gene 2 Breakpoint", + "field": "gene2_position", + "sortable": True, + }, + { + "name": "reads", + "label": "Supporting Reads", + "field": "reads", + "sortable": True, + }, ] # Add quality metrics if available (from breakpoint validation) if "avg_mapping_quality" in aggregated.columns: - columns.append({"name": "avg_mapping_quality", "label": "Avg MapQ", "field": "avg_mapping_quality", "sortable": True}) + columns.append( + { + "name": "avg_mapping_quality", + "label": "Avg MapQ", + "field": "avg_mapping_quality", + "sortable": True, + } + ) if "avg_mapping_span" in aggregated.columns: - columns.append({"name": "avg_mapping_span", "label": "Avg Span", "field": "avg_mapping_span", "sortable": True}) + columns.append( + { + "name": "avg_mapping_span", + "label": "Avg Span", + "field": "avg_mapping_span", + "sortable": True, + } + ) # Format the data for display rows = [] @@ -1717,7 +1992,9 @@ def _make_fusion_summary_table(container: Any, annotated_data: pd.DataFrame, goo # Add quality metrics if available if "avg_mapping_quality" in row: - formatted_row["avg_mapping_quality"] = f"{row['avg_mapping_quality']:.1f}" + formatted_row["avg_mapping_quality"] = ( + f"{row['avg_mapping_quality']:.1f}" + ) if "avg_mapping_span" in row: formatted_row["avg_mapping_span"] = f"{row['avg_mapping_span']:.0f}" @@ -1728,18 +2005,18 @@ def _make_fusion_summary_table(container: Any, annotated_data: pd.DataFrame, goo # Create styled table table_container, table = styled_table( - columns=columns, - rows=rows, - pagination=20, - class_size="table-xs" + columns=columns, rows=rows, pagination=20, class_size="table-xs" ) # Add search functionality try: with table.add_slot("top-right"): - with ui.input(placeholder="Search fusions...").props("type=search").bind_value( - table, "filter" - ).add_slot("append"): + with ( + ui.input(placeholder="Search fusions...") + .props("type=search") + .bind_value(table, "filter") + .add_slot("append") + ): ui.icon("search") except Exception: pass @@ -1747,7 +2024,9 @@ def _make_fusion_summary_table(container: Any, annotated_data: pd.DataFrame, goo # Add summary information total_fusions = len(aggregated) total_reads = aggregated["reads"].sum() - ui.label(f"Total fusions: {total_fusions} | Total supporting reads: {total_reads}").classes("text-xs text-gray-500 mt-1") + ui.label( + f"Total fusions: {total_fusions} | Total supporting reads: {total_reads}" + ).classes("text-xs text-gray-500 mt-1") return table @@ -1781,9 +2060,19 @@ def _make_fusion_groups_table(container: Any, data: Dict[str, Any]) -> Any: try: # Create columns definition columns = [ - {"name": "group_id", "label": "Group ID", "field": "group_id", "sortable": True}, + { + "name": "group_id", + "label": "Group ID", + "field": "group_id", + "sortable": True, + }, {"name": "genes", "label": "Genes", "field": "genes", "sortable": True}, - {"name": "gene_count", "label": "Gene Count", "field": "gene_count", "sortable": True}, + { + "name": "gene_count", + "label": "Gene Count", + "field": "gene_count", + "sortable": True, + }, ] # Format the data for display @@ -1797,11 +2086,13 @@ def _make_fusion_groups_table(container: Any, data: Dict[str, Any]) -> Any: if len(sorted_genes) == 0: continue - rows.append({ - "group_id": idx + 1, - "genes": " - ".join(sorted_genes), - "gene_count": len(sorted_genes), - }) + rows.append( + { + "group_id": idx + 1, + "genes": " - ".join(sorted_genes), + "gene_count": len(sorted_genes), + } + ) if not rows: ui.label("No valid fusion groups found").classes("text-gray-600") @@ -1815,18 +2106,18 @@ def _make_fusion_groups_table(container: Any, data: Dict[str, Any]) -> Any: # Create styled table table_container, table = styled_table( - columns=columns, - rows=rows, - pagination=20, - class_size="table-xs" + columns=columns, rows=rows, pagination=20, class_size="table-xs" ) # Add search functionality try: with table.add_slot("top-right"): - with ui.input(placeholder="Search groups...").props("type=search").bind_value( - table, "filter" - ).add_slot("append"): + with ( + ui.input(placeholder="Search groups...") + .props("type=search") + .bind_value(table, "filter") + .add_slot("append") + ): ui.icon("search") except Exception: pass @@ -1834,17 +2125,23 @@ def _make_fusion_groups_table(container: Any, data: Dict[str, Any]) -> Any: # Add summary information total_groups = len(rows) multi_gene_groups = sum(1 for r in rows if r["gene_count"] > 2) - ui.label(f"Total groups: {total_groups} | Multi-gene groups (>2): {multi_gene_groups}").classes("text-xs text-gray-500 mt-1") + ui.label( + f"Total groups: {total_groups} | Multi-gene groups (>2): {multi_gene_groups}" + ).classes("text-xs text-gray-500 mt-1") return table except Exception as e: logging.exception(f"[Fusion] Failed to create fusion groups table: {e}") - ui.label(f"Error creating fusion groups table: {str(e)}").classes("text-red-600") + ui.label(f"Error creating fusion groups table: {str(e)}").classes( + "text-red-600" + ) return None -def _make_fusion_reads_table(container: Any, reads_df: pd.DataFrame, gene_pair: List[str]) -> Any: +def _make_fusion_reads_table( + container: Any, reads_df: pd.DataFrame, gene_pair: List[str] +) -> Any: """Create a table showing reads and specific locations for a selected gene pair.""" if reads_df is None or reads_df.empty: with container: @@ -1860,11 +2157,19 @@ def _make_fusion_reads_table(container: Any, reads_df: pd.DataFrame, gene_pair: # Debug logging logging.info(f"[Fusion] Reads table - gene_pair: {gene_pair}") - logging.info(f"[Fusion] Reads table - filtered_reads shape: {filtered_reads.shape}") - logging.info(f"[Fusion] Reads table - filtered_reads columns: {list(filtered_reads.columns)}") + logging.info( + f"[Fusion] Reads table - filtered_reads shape: {filtered_reads.shape}" + ) + logging.info( + f"[Fusion] Reads table - filtered_reads columns: {list(filtered_reads.columns)}" + ) if not filtered_reads.empty: - logging.info(f"[Fusion] Reads table - unique genes in data: {filtered_reads['col4'].unique()}") - logging.info(f"[Fusion] Reads table - sample data: {filtered_reads.head(2).to_dict('records')}") + logging.info( + f"[Fusion] Reads table - unique genes in data: {filtered_reads['col4'].unique()}" + ) + logging.info( + f"[Fusion] Reads table - sample data: {filtered_reads.head(2).to_dict('records')}" + ) if filtered_reads.empty: ui.label("No reads found for selected gene pair").classes("text-gray-600") @@ -1872,18 +2177,31 @@ def _make_fusion_reads_table(container: Any, reads_df: pd.DataFrame, gene_pair: # Select and rename relevant columns for the reads table columns_to_show = [ - "read_id", "col4", "reference_id", "reference_start", "reference_end", - "mapping_quality", "strand", "read_start", "read_end", "is_secondary", - "is_supplementary", "mapping_span" + "read_id", + "col4", + "reference_id", + "reference_start", + "reference_end", + "mapping_quality", + "strand", + "read_start", + "read_end", + "is_secondary", + "is_supplementary", + "mapping_span", ] # Only include columns that exist in the DataFrame - available_columns = [col for col in columns_to_show if col in filtered_reads.columns] + available_columns = [ + col for col in columns_to_show if col in filtered_reads.columns + ] logging.info(f"[Fusion] Reads table - available columns: {available_columns}") if not available_columns: # Fallback: show all available columns if none of the expected ones exist - logging.warning(f"[Fusion] No expected columns found, using all available columns: {list(filtered_reads.columns)}") + logging.warning( + f"[Fusion] No expected columns found, using all available columns: {list(filtered_reads.columns)}" + ) available_columns = list(filtered_reads.columns) if not available_columns: ui.label("No columns found in fusion data").classes("text-gray-600") @@ -1904,11 +2222,13 @@ def _make_fusion_reads_table(container: Any, reads_df: pd.DataFrame, gene_pair: "read_end": "Query End", "is_secondary": "Secondary", "is_supplementary": "Supplementary", - "mapping_span": "Span" + "mapping_span": "Span", } # Only rename columns that exist - final_rename_map = {k: v for k, v in column_rename_map.items() if k in reads_subset.columns} + final_rename_map = { + k: v for k, v in column_rename_map.items() if k in reads_subset.columns + } reads_subset = reads_subset.rename(columns=final_rename_map) # Sort by gene and then by read start position (if available) @@ -1926,15 +2246,12 @@ def _make_fusion_reads_table(container: Any, reads_df: pd.DataFrame, gene_pair: # Create columns definition columns = [] for col in reads_subset.columns: - columns.append({ - "name": col, - "label": col, - "field": col, - "sortable": True - }) + columns.append({"name": col, "label": col, "field": col, "sortable": True}) # Add title - ui.label(f"Reads supporting fusion: {'-'.join(gene_pair)}").classes("text-sm font-medium mb-2") + ui.label(f"Reads supporting fusion: {'-'.join(gene_pair)}").classes( + "text-sm font-medium mb-2" + ) table = _render_paged_df_table( reads_subset, columns, @@ -1947,9 +2264,13 @@ def _make_fusion_reads_table(container: Any, reads_df: pd.DataFrame, gene_pair: total_reads = len(reads_subset) if "Read ID" in reads_subset.columns: unique_reads = len(reads_subset["Read ID"].unique()) - ui.label(f"Total reads: {total_reads} | Unique reads: {unique_reads}").classes("text-xs text-gray-500 mt-1") + ui.label( + f"Total reads: {total_reads} | Unique reads: {unique_reads}" + ).classes("text-xs text-gray-500 mt-1") else: - ui.label(f"Total reads: {total_reads}").classes("text-xs text-gray-500 mt-1") + ui.label(f"Total reads: {total_reads}").classes( + "text-xs text-gray-500 mt-1" + ) return table @@ -1975,25 +2296,35 @@ def _plot_gene_group( # No good pairs - use raw data (like reporting code does) # Strip whitespace from gene names to handle leading/trailing spaces subset = annotated_data[annotated_data["col4"].str.strip().isin(gene_group)] - logging.info(f"[Fusion] Using raw data for genome-wide gene group {gene_group}: {len(subset)} rows") + logging.info( + f"[Fusion] Using raw data for genome-wide gene group {gene_group}: {len(subset)} rows" + ) else: # Target panel fusions - use filtered data # Fix pandas reindexing warning by ensuring indices match try: # Align indices to avoid reindexing warning - aligned_goodpairs = goodpairs.reindex(annotated_data.index, fill_value=False) + aligned_goodpairs = goodpairs.reindex( + annotated_data.index, fill_value=False + ) subset = annotated_data[aligned_goodpairs] # Strip whitespace from gene names to handle leading/trailing spaces subset = subset[subset["col4"].str.strip().isin(gene_group)] - logging.info(f"[Fusion] Using filtered data for target gene group {gene_group}: {len(subset)} rows") + logging.info( + f"[Fusion] Using filtered data for target gene group {gene_group}: {len(subset)} rows" + ) except Exception as e: # Fallback to raw data if indexing fails logging.warning(f"[Fusion] Indexing failed, using raw data: {e}") # Strip whitespace from gene names to handle leading/trailing spaces - subset = annotated_data[annotated_data["col4"].str.strip().isin(gene_group)] - logging.info(f"[Fusion] Using raw data (fallback) for gene group {gene_group}: {len(subset)} rows") + subset = annotated_data[ + annotated_data["col4"].str.strip().isin(gene_group) + ] + logging.info( + f"[Fusion] Using raw data (fallback) for gene group {gene_group}: {len(subset)} rows" + ) if subset.empty: with container: @@ -2022,7 +2353,10 @@ def _plot_gene_group( _apply_fusion_figure_theme(fig, _dark) # Update the matplotlib element # Close previous figure if it exists before assigning new one - if hasattr(mpl_element, 'figure') and mpl_element.figure is not None: + if ( + hasattr(mpl_element, "figure") + and mpl_element.figure is not None + ): plt.close(mpl_element.figure) mpl_element.figure = fig mpl_element.update() @@ -2071,18 +2405,24 @@ def _create_advanced_fusion_plot( # Get unique genes and their data unique_genes = list(sorted(subset["col4"].unique())) if len(unique_genes) == 0: - logging.warning(f"[Fusion] No genes found for group {gene_group}, using fallback plot") + logging.warning( + f"[Fusion] No genes found for group {gene_group}, using fallback plot" + ) return _create_simple_fallback_plot(gene_group, subset) # Ensure we have valid data to prevent zero-size axes if subset.empty: - logging.warning(f"[Fusion] Empty subset for group {gene_group}, using fallback plot") + logging.warning( + f"[Fusion] Empty subset for group {gene_group}, using fallback plot" + ) return _create_simple_fallback_plot(gene_group, subset) # Create the unified side-by-side layout without ideograms # For 2 genes, we'll create 2 columns with 2 rows each (gene structure + read mapping) num_genes = len(unique_genes) - logging.info(f"[Fusion] Creating {num_genes} gene layout with 2 rows (gene structure + read mapping)") + logging.info( + f"[Fusion] Creating {num_genes} gene layout with 2 rows (gene structure + read mapping)" + ) # Increase figure size and add more padding to prevent tight layout warnings # Ensure minimum figure size to prevent zero-size axes warnings # Close previous figure if it exists to prevent accumulation @@ -2142,20 +2482,18 @@ def _create_advanced_fusion_plot( return fig - - def _format_ticks_to_megabases(ax: plt.Axes) -> None: """Safely format x-axis ticks to megabases, handling custom formatters from DNA Features Viewer.""" try: ticks = ax.get_xticks() # Set the tick positions first, then the labels to avoid warnings ax.set_xticks(ticks) - ax.set_xticklabels([f'{t/1e6:.1f}' for t in ticks]) + ax.set_xticklabels([f"{t/1e6:.1f}" for t in ticks]) except Exception as e: logging.warning(f"Could not format ticks to megabases: {e}") # Fallback: try to use ticklabel_format if possible try: - ax.ticklabel_format(style='scientific', axis='x', scilimits=(0,0)) + ax.ticklabel_format(style="scientific", axis="x", scilimits=(0, 0)) except Exception: pass # If both methods fail, just leave the default formatting @@ -2169,6 +2507,7 @@ def _load_gene_annotations() -> Optional[pd.DataFrame]: # First try to use robin resources to find the correct path try: from robin import resources + resources_dir = os.path.dirname(resources.__file__) gene_data_path = os.path.join(resources_dir, "rCNS2_data.csv.gz") if not os.path.exists(gene_data_path): @@ -2182,7 +2521,13 @@ def _load_gene_annotations() -> Optional[pd.DataFrame]: possible_paths = [ "src/robin/resources/rCNS2_data.csv.gz", "robin/resources/rCNS2_data.csv.gz", - os.path.join(os.path.dirname(__file__), "..", "..", "resources", "rCNS2_data.csv.gz"), + os.path.join( + os.path.dirname(__file__), + "..", + "..", + "resources", + "rCNS2_data.csv.gz", + ), ] for path in possible_paths: @@ -2216,15 +2561,20 @@ def _plot_gene_structure_with_dna_features( # Filter gene table for this specific gene and chromosome # Also strip whitespace from gene_table gene_name column for matching gene_table_clean = gene_table.copy() - gene_table_clean["gene_name"] = gene_table_clean["gene_name"].astype(str).str.strip() + gene_table_clean["gene_name"] = ( + gene_table_clean["gene_name"].astype(str).str.strip() + ) gene_info = gene_table_clean[ - (gene_table_clean["gene_name"] == gene_name_clean) & (gene_table_clean["Seqid"] == chrom) + (gene_table_clean["gene_name"] == gene_name_clean) + & (gene_table_clean["Seqid"] == chrom) ] if gene_info.empty: # Fallback if no gene info found - ax.set_title(f"Gene Structure: {gene_name} ({chrom})", fontsize=10, fontweight="bold") + ax.set_title( + f"Gene Structure: {gene_name} ({chrom})", fontsize=10, fontweight="bold" + ) # Extend plot range to show full context plot_start = start - (end - start) * 0.1 plot_end = end + (end - start) * 0.1 @@ -2300,7 +2650,9 @@ def _plot_gene_structure_with_dna_features( record.plot( ax=ax, with_ruler=False, draw_line=True, strand_in_label_threshold=4 ) - ax.set_title(f"Gene Structure: {gene_name} ({chrom})", fontsize=10, fontweight="bold") + ax.set_title( + f"Gene Structure: {gene_name} ({chrom})", fontsize=10, fontweight="bold" + ) ax.set_xlabel(f"Position (Mb) - {chrom}", fontsize=10) # Extend plot range to show full context ax.margins(x=0.15, y=0.1) @@ -2308,7 +2660,9 @@ def _plot_gene_structure_with_dna_features( _format_ticks_to_megabases(ax) else: # Fallback if no features found - ax.set_title(f"Gene Structure: {gene_name} ({chrom})", fontsize=10, fontweight="bold") + ax.set_title( + f"Gene Structure: {gene_name} ({chrom})", fontsize=10, fontweight="bold" + ) # Extend plot range to show full context plot_start = start - (end - start) * 0.1 plot_end = end + (end - start) * 0.1 @@ -2331,7 +2685,9 @@ def _plot_gene_structure_with_dna_features( except Exception as e: logging.error(f"Error plotting gene structure with DNA Features Viewer: {e}") # Fallback to simple text - ax.set_title(f"Gene Structure: {gene_name} ({chrom})", fontsize=10, fontweight="bold") + ax.set_title( + f"Gene Structure: {gene_name} ({chrom})", fontsize=10, fontweight="bold" + ) # Extend plot range to show full context plot_start = start - (end - start) * 0.1 plot_end = end + (end - start) * 0.1 @@ -2425,7 +2781,9 @@ def _plot_read_mapping_sophisticated( ) # Customize plot - ax.set_title(f"Read Mapping: {gene_name} ({chrom})", fontsize=10, fontweight="bold") + ax.set_title( + f"Read Mapping: {gene_name} ({chrom})", fontsize=10, fontweight="bold" + ) # Extend plot range to clearly show read ends plot_start = start - (end - start) * 0.15 plot_end = end + (end - start) * 0.15 @@ -2440,10 +2798,10 @@ def _plot_read_mapping_sophisticated( ax.grid(True, alpha=0.3) # Remove box edges/borders - ax.spines['top'].set_visible(False) - ax.spines['right'].set_visible(False) - ax.spines['bottom'].set_visible(False) - ax.spines['left'].set_visible(False) + ax.spines["top"].set_visible(False) + ax.spines["right"].set_visible(False) + ax.spines["bottom"].set_visible(False) + ax.spines["left"].set_visible(False) # Add legend if not too many reads and we have labeled artists if len(unique_reads) <= 10 and len(ax.get_legend_handles_labels()[0]) > 0: @@ -2452,7 +2810,9 @@ def _plot_read_mapping_sophisticated( except Exception as e: logging.error(f"Error plotting sophisticated read mapping: {e}") # Fallback to simple text - ax.set_title(f"Read Mapping: {gene_name} ({chrom})", fontsize=10, fontweight="bold") + ax.set_title( + f"Read Mapping: {gene_name} ({chrom})", fontsize=10, fontweight="bold" + ) ax.text( 0.5, 0.5, @@ -2465,7 +2825,7 @@ def _plot_read_mapping_sophisticated( # Convert x-axis to megabases even in error case ax.set_xlabel(f"Position (Mb) - {chrom}") # Format x-axis ticks to show megabases - ax.ticklabel_format(style='scientific', axis='x', scilimits=(0,0)) + ax.ticklabel_format(style="scientific", axis="x", scilimits=(0, 0)) # Convert tick labels to megabases _format_ticks_to_megabases(ax) @@ -2519,7 +2879,9 @@ def _plot_gene_structure_original( try: # This would need access to the gene_table data from the original code # For now, create a placeholder that matches the original structure - ax.set_title(f"Gene Structure: {data['gene']} ({chrom})", fontsize=10, fontweight="bold") + ax.set_title( + f"Gene Structure: {data['gene']} ({chrom})", fontsize=10, fontweight="bold" + ) ax.set_xlim(start, end) ax.set_ylim(0, 1) @@ -2575,7 +2937,9 @@ def _plot_read_mapping_original( # This would need the original plotting logic with overlapping ranges, ranks, etc. # For now, create a simplified version that shows the structure - ax.set_title(f"Read Mapping: {data['gene']} ({chrom})", fontsize=10, fontweight="bold") + ax.set_title( + f"Read Mapping: {data['gene']} ({chrom})", fontsize=10, fontweight="bold" + ) # Extend plot range to clearly show read ends plot_start = start - (end - start) * 0.15 plot_end = end + (end - start) * 0.15 @@ -2594,10 +2958,10 @@ def _plot_read_mapping_original( ax.grid(True, alpha=0.3) # Remove box edges/borders - ax.spines['top'].set_visible(False) - ax.spines['right'].set_visible(False) - ax.spines['bottom'].set_visible(False) - ax.spines['left'].set_visible(False) + ax.spines["top"].set_visible(False) + ax.spines["right"].set_visible(False) + ax.spines["bottom"].set_visible(False) + ax.spines["left"].set_visible(False) except Exception as e: logging.error(f"Error plotting read mapping: {e}") @@ -2619,7 +2983,9 @@ def _plot_gene_structure( try: # This would integrate with your gene annotation data # For now, create a placeholder gene structure - ax.set_title(f"Gene Structure: {gene_name} ({chrom})", fontsize=10, fontweight="bold") + ax.set_title( + f"Gene Structure: {gene_name} ({chrom})", fontsize=10, fontweight="bold" + ) ax.set_xlim(start, end) ax.set_ylim(0, 1) @@ -2690,7 +3056,9 @@ def _plot_read_mapping( ) # Limit legend to first 10 reads # Customize plot - ax.set_title(f"Read Mapping: {gene_name} ({chrom})", fontsize=10, fontweight="bold") + ax.set_title( + f"Read Mapping: {gene_name} ({chrom})", fontsize=10, fontweight="bold" + ) # Extend plot range to clearly show read ends plot_start = start - (end - start) * 0.15 plot_end = end + (end - start) * 0.15 @@ -2705,10 +3073,10 @@ def _plot_read_mapping( ax.grid(True, alpha=0.3) # Remove box edges/borders - ax.spines['top'].set_visible(False) - ax.spines['right'].set_visible(False) - ax.spines['bottom'].set_visible(False) - ax.spines['left'].set_visible(False) + ax.spines["top"].set_visible(False) + ax.spines["right"].set_visible(False) + ax.spines["bottom"].set_visible(False) + ax.spines["left"].set_visible(False) # Add legend if not too many reads and we have labeled artists if len(unique_reads) <= 10 and len(ax.get_legend_handles_labels()[0]) > 0: @@ -2833,7 +3201,9 @@ def add_fusion_section(launcher: Any, sample_dir: Path) -> None: }, } - def _handle_gene_pair_selection(section: str, gene_pair: List[str], data: Dict[str, Any]) -> None: + def _handle_gene_pair_selection( + section: str, gene_pair: List[str], data: Dict[str, Any] + ) -> None: """Handle gene pair selection and update visualization.""" try: state[section]["selected_gene_pair"] = gene_pair @@ -2855,7 +3225,9 @@ async def refresh_fusion_async() -> None: t_start = _time.perf_counter() try: - logging.info(f"[Fusion] refresh_fusion() called for sample_dir: {sample_dir}") + logging.info( + f"[Fusion] refresh_fusion() called for sample_dir: {sample_dir}" + ) if not sample_dir or not sample_dir.exists(): logging.warning(f"[Fusion] Sample directory not found: {sample_dir}") return @@ -2892,7 +3264,9 @@ def refresh_fusion() -> None: except RuntimeError: try: if not sample_dir or not sample_dir.exists(): - logging.warning(f"[Fusion] Sample directory not found: {sample_dir}") + logging.warning( + f"[Fusion] Sample directory not found: {sample_dir}" + ) return fusion_data = _load_fusion_data(sample_dir) _update_fusion_ui(fusion_data, state, sample_dir) @@ -2904,12 +3278,17 @@ def refresh_fusion() -> None: def _load_fusion_data(sample_dir: Path) -> Dict[str, Any]: """Load fusion data from files with optional breakpoint validation.""" import time as _time + t_load_start = _time.perf_counter() try: - logging.info(f"[Fusion] _load_fusion_data() called with sample_dir: {sample_dir}") + logging.info( + f"[Fusion] _load_fusion_data() called with sample_dir: {sample_dir}" + ) # Configuration for breakpoint validation - USE_BREAKPOINT_VALIDATION = True # Set to False to disable breakpoint validation + USE_BREAKPOINT_VALIDATION = ( + True # Set to False to disable breakpoint validation + ) MIN_BREAKPOINT_SUPPORT = 4 # Minimum reads supporting same breakpoint MAX_BREAKPOINT_DISTANCE = 100 # Maximum distance for breakpoint clustering @@ -2931,40 +3310,52 @@ def _load_fusion_data(sample_dir: Path) -> Dict[str, Any]: # Apply breakpoint validation if enabled if USE_BREAKPOINT_VALIDATION: if t: - logging.info("[Fusion] Applying breakpoint validation to target data") + logging.info( + "[Fusion] Applying breakpoint validation to target data" + ) annotated_data = t.get("annotated_data", pd.DataFrame()) if not annotated_data.empty: validated_breakpoints = _validate_fusion_breakpoints( annotated_data, min_read_support=MIN_BREAKPOINT_SUPPORT, - max_breakpoint_distance=MAX_BREAKPOINT_DISTANCE + max_breakpoint_distance=MAX_BREAKPOINT_DISTANCE, ) if not validated_breakpoints.empty: t["validated_breakpoints"] = validated_breakpoints t["original_candidate_count"] = t.get("candidate_count", 0) t["candidate_count"] = len(validated_breakpoints) - logging.info(f"[Fusion] Target breakpoint validation: {t['original_candidate_count']} -> {t['candidate_count']} candidates") + logging.info( + f"[Fusion] Target breakpoint validation: {t['original_candidate_count']} -> {t['candidate_count']} candidates" + ) else: t["candidate_count"] = 0 - logging.info("[Fusion] No target fusions passed breakpoint validation") + logging.info( + "[Fusion] No target fusions passed breakpoint validation" + ) if g: - logging.info("[Fusion] Applying breakpoint validation to genome-wide data") + logging.info( + "[Fusion] Applying breakpoint validation to genome-wide data" + ) annotated_data = g.get("annotated_data", pd.DataFrame()) if not annotated_data.empty: validated_breakpoints = _validate_fusion_breakpoints( annotated_data, min_read_support=MIN_BREAKPOINT_SUPPORT, - max_breakpoint_distance=MAX_BREAKPOINT_DISTANCE + max_breakpoint_distance=MAX_BREAKPOINT_DISTANCE, ) if not validated_breakpoints.empty: g["validated_breakpoints"] = validated_breakpoints g["original_candidate_count"] = g.get("candidate_count", 0) g["candidate_count"] = len(validated_breakpoints) - logging.info(f"[Fusion] Genome-wide breakpoint validation: {g['original_candidate_count']} -> {g['candidate_count']} candidates") + logging.info( + f"[Fusion] Genome-wide breakpoint validation: {g['original_candidate_count']} -> {g['candidate_count']} candidates" + ) else: g["candidate_count"] = 0 - logging.info("[Fusion] No genome-wide fusions passed breakpoint validation") + logging.info( + "[Fusion] No genome-wide fusions passed breakpoint validation" + ) # Debug logging logging.info(f"[Fusion] Loaded target data: {t is not None}") @@ -2972,58 +3363,104 @@ def _load_fusion_data(sample_dir: Path) -> Dict[str, Any]: # Apply same logic to target panel as genome-wide if t is not None: - logging.info(f"[Fusion] Target candidate count: {t.get('candidate_count', 0)}") - logging.info(f"[Fusion] Target gene groups: {len(t.get('gene_groups', []))}") - logging.info(f"[Fusion] Target gene groups content: {t.get('gene_groups', [])}") - logging.info(f"[Fusion] Target annotated_data shape: {t.get('annotated_data', pd.DataFrame()).shape}") - logging.info(f"[Fusion] Target goodpairs shape: {t.get('goodpairs', pd.Series()).shape}") - logging.info(f"[Fusion] Target gene_pairs count: {len(t.get('gene_pairs', []))}") - logging.info(f"[Fusion] Target gene_pairs: {t.get('gene_pairs', [])[:10]}...") # Show first 10 + logging.info( + f"[Fusion] Target candidate count: {t.get('candidate_count', 0)}" + ) + logging.info( + f"[Fusion] Target gene groups: {len(t.get('gene_groups', []))}" + ) + logging.info( + f"[Fusion] Target gene groups content: {t.get('gene_groups', [])}" + ) + logging.info( + f"[Fusion] Target annotated_data shape: {t.get('annotated_data', pd.DataFrame()).shape}" + ) + logging.info( + f"[Fusion] Target goodpairs shape: {t.get('goodpairs', pd.Series()).shape}" + ) + logging.info( + f"[Fusion] Target gene_pairs count: {len(t.get('gene_pairs', []))}" + ) + logging.info( + f"[Fusion] Target gene_pairs: {t.get('gene_pairs', [])[:10]}..." + ) # Show first 10 # Use the same logic as reporting code - count gene_pairs instead of relying on candidate_count - if t.get('gene_pairs') and len(t.get('gene_pairs', [])) > 0: + if t.get("gene_pairs") and len(t.get("gene_pairs", [])) > 0: # Override candidate_count with the actual number of gene pairs (like reporting code does) - t['candidate_count'] = len(t.get('gene_pairs', [])) - logging.info(f"[Fusion] Override target candidate_count to: {t['candidate_count']}") + t["candidate_count"] = len(t.get("gene_pairs", [])) + logging.info( + f"[Fusion] Override target candidate_count to: {t['candidate_count']}" + ) # Generate gene_groups from gene_pairs if missing (like reporting code does) - if not t.get('gene_groups') or len(t.get('gene_groups', [])) == 0: + if not t.get("gene_groups") or len(t.get("gene_groups", [])) == 0: # Convert gene_pairs to gene_groups format gene_groups = [] - for gene_pair in t.get('gene_pairs', []): - if isinstance(gene_pair, (tuple, list)) and len(gene_pair) >= 2: + for gene_pair in t.get("gene_pairs", []): + if ( + isinstance(gene_pair, (tuple, list)) + and len(gene_pair) >= 2 + ): gene_groups.append(list(gene_pair)) - t['gene_groups'] = gene_groups - logging.info(f"[Fusion] Generated {len(gene_groups)} gene_groups from gene_pairs") + t["gene_groups"] = gene_groups + logging.info( + f"[Fusion] Generated {len(gene_groups)} gene_groups from gene_pairs" + ) if g is not None: - logging.info(f"[Fusion] Genome-wide candidate count: {g.get('candidate_count', 0)}") - logging.info(f"[Fusion] Genome-wide gene groups: {len(g.get('gene_groups', []))}") - logging.info(f"[Fusion] Genome-wide gene groups content: {g.get('gene_groups', [])}") - logging.info(f"[Fusion] Genome-wide annotated_data shape: {g.get('annotated_data', pd.DataFrame()).shape}") - logging.info(f"[Fusion] Genome-wide goodpairs shape: {g.get('goodpairs', pd.Series()).shape}") - logging.info(f"[Fusion] Genome-wide gene_pairs count: {len(g.get('gene_pairs', []))}") - logging.info(f"[Fusion] Genome-wide gene_pairs: {g.get('gene_pairs', [])[:10]}...") # Show first 10 + logging.info( + f"[Fusion] Genome-wide candidate count: {g.get('candidate_count', 0)}" + ) + logging.info( + f"[Fusion] Genome-wide gene groups: {len(g.get('gene_groups', []))}" + ) + logging.info( + f"[Fusion] Genome-wide gene groups content: {g.get('gene_groups', [])}" + ) + logging.info( + f"[Fusion] Genome-wide annotated_data shape: {g.get('annotated_data', pd.DataFrame()).shape}" + ) + logging.info( + f"[Fusion] Genome-wide goodpairs shape: {g.get('goodpairs', pd.Series()).shape}" + ) + logging.info( + f"[Fusion] Genome-wide gene_pairs count: {len(g.get('gene_pairs', []))}" + ) + logging.info( + f"[Fusion] Genome-wide gene_pairs: {g.get('gene_pairs', [])[:10]}..." + ) # Show first 10 # Use the same logic as reporting code - count gene_pairs instead of relying on candidate_count - if g.get('gene_pairs') and len(g.get('gene_pairs', [])) > 0: + if g.get("gene_pairs") and len(g.get("gene_pairs", [])) > 0: # Override candidate_count with the actual number of gene pairs (like reporting code does) - g['candidate_count'] = len(g.get('gene_pairs', [])) - logging.info(f"[Fusion] Override genome-wide candidate_count to: {g['candidate_count']}") + g["candidate_count"] = len(g.get("gene_pairs", [])) + logging.info( + f"[Fusion] Override genome-wide candidate_count to: {g['candidate_count']}" + ) # Generate gene_groups from gene_pairs if missing (like reporting code does) - if not g.get('gene_groups') or len(g.get('gene_groups', [])) == 0: + if not g.get("gene_groups") or len(g.get("gene_groups", [])) == 0: # Convert gene_pairs to gene_groups format gene_groups = [] - for gene_pair in g.get('gene_pairs', []): - if isinstance(gene_pair, (tuple, list)) and len(gene_pair) >= 2: + for gene_pair in g.get("gene_pairs", []): + if ( + isinstance(gene_pair, (tuple, list)) + and len(gene_pair) >= 2 + ): gene_groups.append(list(gene_pair)) - g['gene_groups'] = gene_groups - logging.info(f"[Fusion] Generated {len(gene_groups)} gene_groups from gene_pairs") + g["gene_groups"] = gene_groups + logging.info( + f"[Fusion] Generated {len(gene_groups)} gene_groups from gene_pairs" + ) else: - logging.info(f"[Fusion] Genome-wide file exists: {genome_file.exists()}") + logging.info( + f"[Fusion] Genome-wide file exists: {genome_file.exists()}" + ) if genome_file.exists(): - logging.info(f"[Fusion] Genome-wide file size: {genome_file.stat().st_size} bytes") + logging.info( + f"[Fusion] Genome-wide file size: {genome_file.stat().st_size} bytes" + ) # Get file modification times target_mtime = target_file.stat().st_mtime if target_file.exists() else None @@ -3041,12 +3478,12 @@ def _load_fusion_data(sample_dir: Path) -> Dict[str, Any]: "target": { "data": t, "mtime": target_mtime, - "data_hash": target_data_hash + "data_hash": target_data_hash, }, "genome": { "data": g, "mtime": genome_mtime, - "data_hash": genome_data_hash + "data_hash": genome_data_hash, }, "master_bed": { "data": master_bed_df, @@ -3056,14 +3493,18 @@ def _load_fusion_data(sample_dir: Path) -> Dict[str, Any]: } except Exception as e: elapsed = _time.perf_counter() - t_load_start - logging.exception(f"[Fusion] Failed to load fusion data after {elapsed:.2f}s: {e}") + logging.exception( + f"[Fusion] Failed to load fusion data after {elapsed:.2f}s: {e}" + ) return { "target": {"data": None, "mtime": None, "data_hash": None}, "genome": {"data": None, "mtime": None, "data_hash": None}, - "master_bed": {"data": None, "mtime": None} + "master_bed": {"data": None, "mtime": None}, } - def _update_fusion_ui(fusion_data: Dict[str, Any], state: Dict[str, Any], sample_dir: Path) -> None: + def _update_fusion_ui( + fusion_data: Dict[str, Any], state: Dict[str, Any], sample_dir: Path + ) -> None: """Update fusion UI elements - runs on main UI thread.""" try: target_data = fusion_data.get("target", {}) @@ -3083,9 +3524,9 @@ def _update_fusion_ui(fusion_data: Dict[str, Any], state: Dict[str, Any], sample # Update if mtime changed or if this is initial load (both mtimes are None) mtime_changed = target_mtime != state["target"].get("mtime") is_initial_load = ( - target_mtime is None and - state["target"].get("mtime") is None and - state["target"].get("data") is None + target_mtime is None + and state["target"].get("mtime") is None + and state["target"].get("data") is None ) if mtime_changed or is_initial_load: @@ -3096,7 +3537,9 @@ def _update_fusion_ui(fusion_data: Dict[str, Any], state: Dict[str, Any], sample # Update summary label try: if "summary" in state and state["summary"].get("target_lbl"): - state["summary"]["target_lbl"].text = "Target: -- pairs, -- groups" + state["summary"][ + "target_lbl" + ].text = "Target: -- pairs, -- groups" except Exception: pass @@ -3114,12 +3557,20 @@ def _update_fusion_ui(fusion_data: Dict[str, Any], state: Dict[str, Any], sample with state["target"]["summary_table_container"].classes("w-full"): ui.label("No fusion data available").classes("text-gray-600") with state["target"]["groups_table_container"].classes("w-full"): - ui.label("No validated fusion groups found").classes("text-gray-600") + ui.label("No validated fusion groups found").classes( + "text-gray-600" + ) with state["target"]["table_container"].classes("w-full"): - ui.label("No fusion candidates found yet").classes("text-gray-600") + ui.label("No fusion candidates found yet").classes( + "text-gray-600" + ) with state["target"]["status_container"].classes("w-full"): - ui.label("Target panel fusion analysis not available").classes("text-gray-600 text-sm") - ui.label("(Fusion data file not found or could not be loaded)").classes("text-gray-500 text-xs") + ui.label("Target panel fusion analysis not available").classes( + "text-gray-600 text-sm" + ) + ui.label( + "(Fusion data file not found or could not be loaded)" + ).classes("text-gray-500 text-xs") # Always update summary and table when file changes elif t is not None and target_mtime != state["target"].get("mtime"): @@ -3135,9 +3586,7 @@ def _update_fusion_ui(fusion_data: Dict[str, Any], state: Dict[str, Any], sample unique_groups = _count_unique_fusion_groups(t) state["summary"][ "target_lbl" - ].text = ( - f"Target: {unique_pairs} pairs, {unique_groups} groups" - ) + ].text = f"Target: {unique_pairs} pairs, {unique_groups} groups" except Exception: pass # summary table @@ -3180,17 +3629,21 @@ def _update_fusion_ui(fusion_data: Dict[str, Any], state: Dict[str, Any], sample # Get validated fusion pairs from summary table data validated_pairs = _get_validated_fusion_pairs( t.get("annotated_data", pd.DataFrame()), - t.get("goodpairs", pd.Series()) + t.get("goodpairs", pd.Series()), + ) + logging.info( + f"[Fusion] Target panel plotting check: candidate_count={t.get('candidate_count', 0)}, validated_pairs={len(validated_pairs)}" ) - logging.info(f"[Fusion] Target panel plotting check: candidate_count={t.get('candidate_count', 0)}, validated_pairs={len(validated_pairs)}") if validated_pairs: with state["target"]["plot_container"].classes("w-full"): with ui.row().classes("w-full"): state["target"]["dropdown"] = ui.select( options=validated_pairs, with_input=False, - on_change=lambda e, t=t: _handle_gene_pair_selection("target", e.value, t), - value=state["target"]["selected_gene_pair"] + on_change=lambda e, t=t: _handle_gene_pair_selection( + "target", e.value, t + ), + value=state["target"]["selected_gene_pair"], ).classes("w-40") with ui.row().classes("w-full"): state["target"]["card"] = ui.card() @@ -3201,16 +3654,29 @@ def _update_fusion_ui(fusion_data: Dict[str, Any], state: Dict[str, Any], sample else: # Show status message when target panel fusion data is not available with state["target"]["status_container"].classes("w-full"): - ui.label("Target panel fusion analysis not available").classes("text-gray-600 text-sm") - ui.label("(No validated fusion pairs found in target panel)").classes("text-gray-500 text-xs") + ui.label("Target panel fusion analysis not available").classes( + "text-gray-600 text-sm" + ) + ui.label( + "(No validated fusion pairs found in target panel)" + ).classes("text-gray-500 text-xs") # Restore selected gene pair if it exists (for both cases above) - if state["target"]["selected_gene_pair"] and state["target"]["dropdown"]: + if ( + state["target"]["selected_gene_pair"] + and state["target"]["dropdown"] + ): try: - state["target"]["dropdown"].value = state["target"]["selected_gene_pair"] - _handle_gene_pair_selection("target", state["target"]["selected_gene_pair"], t) + state["target"]["dropdown"].value = state["target"][ + "selected_gene_pair" + ] + _handle_gene_pair_selection( + "target", state["target"]["selected_gene_pair"], t + ) except Exception as e: - logging.warning(f"[Fusion] Failed to restore target selection: {e}") + logging.warning( + f"[Fusion] Failed to restore target selection: {e}" + ) # Only update visualization when data content actually changes (for background refreshes) elif t is not None and target_data_hash != state["target"].get("data_hash"): @@ -3224,17 +3690,21 @@ def _update_fusion_ui(fusion_data: Dict[str, Any], state: Dict[str, Any], sample # Get validated fusion pairs from summary table data validated_pairs = _get_validated_fusion_pairs( t.get("annotated_data", pd.DataFrame()), - t.get("goodpairs", pd.Series()) + t.get("goodpairs", pd.Series()), + ) + logging.info( + f"[Fusion] Target panel plotting check: candidate_count={t.get('candidate_count', 0)}, validated_pairs={len(validated_pairs)}" ) - logging.info(f"[Fusion] Target panel plotting check: candidate_count={t.get('candidate_count', 0)}, validated_pairs={len(validated_pairs)}") if validated_pairs: with state["target"]["plot_container"].classes("w-full"): with ui.row().classes("w-full"): state["target"]["dropdown"] = ui.select( options=validated_pairs, with_input=False, - on_change=lambda e, t=t: _handle_gene_pair_selection("target", e.value, t), - value=state["target"]["selected_gene_pair"] + on_change=lambda e, t=t: _handle_gene_pair_selection( + "target", e.value, t + ), + value=state["target"]["selected_gene_pair"], ).classes("w-40") with ui.row().classes("w-full"): state["target"]["card"] = ui.card() @@ -3245,21 +3715,36 @@ def _update_fusion_ui(fusion_data: Dict[str, Any], state: Dict[str, Any], sample else: # Show status message when target panel fusion data is not available with state["target"]["status_container"].classes("w-full"): - ui.label("Target panel fusion analysis not available").classes("text-gray-600 text-sm") - ui.label("(No validated fusion pairs found in target panel)").classes("text-gray-500 text-xs") + ui.label("Target panel fusion analysis not available").classes( + "text-gray-600 text-sm" + ) + ui.label( + "(No validated fusion pairs found in target panel)" + ).classes("text-gray-500 text-xs") # Restore selected gene pair if it exists (for both cases above) - if state["target"]["selected_gene_pair"] and state["target"]["dropdown"]: + if ( + state["target"]["selected_gene_pair"] + and state["target"]["dropdown"] + ): try: - state["target"]["dropdown"].value = state["target"]["selected_gene_pair"] - _handle_gene_pair_selection("target", state["target"]["selected_gene_pair"], t) + state["target"]["dropdown"].value = state["target"][ + "selected_gene_pair" + ] + _handle_gene_pair_selection( + "target", state["target"]["selected_gene_pair"], t + ) except Exception as e: - logging.warning(f"[Fusion] Failed to restore target selection: {e}") + logging.warning( + f"[Fusion] Failed to restore target selection: {e}" + ) # Update genome-wide UI genome_mtime = genome_data.get("mtime") genome_data_hash = genome_data.get("data_hash") - logging.info(f"[Fusion] Genome-wide update check: g={g is not None}, mtime_changed={genome_mtime != state['genome'].get('mtime')}") + logging.info( + f"[Fusion] Genome-wide update check: g={g is not None}, mtime_changed={genome_mtime != state['genome'].get('mtime')}" + ) # Always update summary and table when file changes if g is not None and genome_mtime != state["genome"].get("mtime"): @@ -3275,7 +3760,9 @@ def _update_fusion_ui(fusion_data: Dict[str, Any], state: Dict[str, Any], sample unique_groups = _count_unique_fusion_groups(g) state["summary"][ "genome_lbl" - ].text = f"Genome-wide: {unique_pairs} pairs, {unique_groups} groups" + ].text = ( + f"Genome-wide: {unique_pairs} pairs, {unique_groups} groups" + ) except Exception: pass # summary table @@ -3318,17 +3805,21 @@ def _update_fusion_ui(fusion_data: Dict[str, Any], state: Dict[str, Any], sample # Get validated fusion pairs from summary table data validated_pairs = _get_validated_fusion_pairs( g.get("annotated_data", pd.DataFrame()), - g.get("goodpairs", pd.Series()) + g.get("goodpairs", pd.Series()), + ) + logging.info( + f"[Fusion] Genome-wide plotting check: candidate_count={g.get('candidate_count', 0)}, validated_pairs={len(validated_pairs)}" ) - logging.info(f"[Fusion] Genome-wide plotting check: candidate_count={g.get('candidate_count', 0)}, validated_pairs={len(validated_pairs)}") if validated_pairs: with state["genome"]["plot_container"].classes("w-full"): with ui.row().classes("w-full"): state["genome"]["dropdown"] = ui.select( options=validated_pairs, with_input=False, - on_change=lambda e, g=g: _handle_gene_pair_selection("genome", e.value, g), - value=state["genome"]["selected_gene_pair"] + on_change=lambda e, g=g: _handle_gene_pair_selection( + "genome", e.value, g + ), + value=state["genome"]["selected_gene_pair"], ).classes("w-40") with ui.row().classes("w-full"): state["genome"]["card"] = ui.card() @@ -3339,16 +3830,29 @@ def _update_fusion_ui(fusion_data: Dict[str, Any], state: Dict[str, Any], sample else: # Show status message when genome-wide data is not available with state["genome"]["status_container"].classes("w-full"): - ui.label("Genome-wide fusion analysis not available").classes("text-gray-600 text-sm") - ui.label("(Requires supplementary reads in BAM file)").classes("text-gray-500 text-xs") + ui.label("Genome-wide fusion analysis not available").classes( + "text-gray-600 text-sm" + ) + ui.label("(Requires supplementary reads in BAM file)").classes( + "text-gray-500 text-xs" + ) # Restore selected gene pair if it exists (for both cases above) - if state["genome"]["selected_gene_pair"] and state["genome"]["dropdown"]: + if ( + state["genome"]["selected_gene_pair"] + and state["genome"]["dropdown"] + ): try: - state["genome"]["dropdown"].value = state["genome"]["selected_gene_pair"] - _handle_gene_pair_selection("genome", state["genome"]["selected_gene_pair"], g) + state["genome"]["dropdown"].value = state["genome"][ + "selected_gene_pair" + ] + _handle_gene_pair_selection( + "genome", state["genome"]["selected_gene_pair"], g + ) except Exception as e: - logging.warning(f"[Fusion] Failed to restore genome selection: {e}") + logging.warning( + f"[Fusion] Failed to restore genome selection: {e}" + ) # Only update visualization when data content actually changes (for background refreshes) elif g is not None and genome_data_hash != state["genome"].get("data_hash"): @@ -3362,17 +3866,21 @@ def _update_fusion_ui(fusion_data: Dict[str, Any], state: Dict[str, Any], sample # Get validated fusion pairs from summary table data validated_pairs = _get_validated_fusion_pairs( g.get("annotated_data", pd.DataFrame()), - g.get("goodpairs", pd.Series()) + g.get("goodpairs", pd.Series()), + ) + logging.info( + f"[Fusion] Genome-wide plotting check: candidate_count={g.get('candidate_count', 0)}, validated_pairs={len(validated_pairs)}" ) - logging.info(f"[Fusion] Genome-wide plotting check: candidate_count={g.get('candidate_count', 0)}, validated_pairs={len(validated_pairs)}") if validated_pairs: with state["genome"]["plot_container"].classes("w-full"): with ui.row().classes("w-full"): state["genome"]["dropdown"] = ui.select( options=validated_pairs, with_input=False, - on_change=lambda e, g=g: _handle_gene_pair_selection("genome", e.value, g), - value=state["genome"]["selected_gene_pair"] + on_change=lambda e, g=g: _handle_gene_pair_selection( + "genome", e.value, g + ), + value=state["genome"]["selected_gene_pair"], ).classes("w-40") with ui.row().classes("w-full"): state["genome"]["card"] = ui.card() @@ -3383,16 +3891,29 @@ def _update_fusion_ui(fusion_data: Dict[str, Any], state: Dict[str, Any], sample else: # Show status message when genome-wide data is not available with state["genome"]["status_container"].classes("w-full"): - ui.label("Genome-wide fusion analysis not available").classes("text-gray-600 text-sm") - ui.label("(Requires supplementary reads in BAM file)").classes("text-gray-500 text-xs") + ui.label("Genome-wide fusion analysis not available").classes( + "text-gray-600 text-sm" + ) + ui.label("(Requires supplementary reads in BAM file)").classes( + "text-gray-500 text-xs" + ) # Restore selected gene pair if it exists (for both cases above) - if state["genome"]["selected_gene_pair"] and state["genome"]["dropdown"]: + if ( + state["genome"]["selected_gene_pair"] + and state["genome"]["dropdown"] + ): try: - state["genome"]["dropdown"].value = state["genome"]["selected_gene_pair"] - _handle_gene_pair_selection("genome", state["genome"]["selected_gene_pair"], g) + state["genome"]["dropdown"].value = state["genome"][ + "selected_gene_pair" + ] + _handle_gene_pair_selection( + "genome", state["genome"]["selected_gene_pair"], g + ) except Exception as e: - logging.warning(f"[Fusion] Failed to restore genome selection: {e}") + logging.warning( + f"[Fusion] Failed to restore genome selection: {e}" + ) # Update master BED UI master_bed_mtime = master_bed_data.get("mtime") @@ -3405,7 +3926,9 @@ def _update_fusion_ui(fusion_data: Dict[str, Any], state: Dict[str, Any], sample # Master BED summary label has been deprecated try: if "summary" in state and state["summary"].get("master_bed_lbl"): - state["summary"]["master_bed_lbl"].text = "Master BED: (deprecated)" + state["summary"][ + "master_bed_lbl" + ].text = "Master BED: (deprecated)" except Exception: pass @@ -3414,8 +3937,12 @@ def _update_fusion_ui(fusion_data: Dict[str, Any], state: Dict[str, Any], sample state["master_bed"]["table_container"].clear() state["master_bed"]["status_container"].clear() with state["master_bed"]["status_container"].classes("w-full"): - ui.label("Master BED table display has been deprecated").classes("text-gray-500 text-sm") - ui.label("(Data is still processed and used for BED generation)").classes("text-gray-400 text-xs") + ui.label( + "Master BED table display has been deprecated" + ).classes("text-gray-500 text-sm") + ui.label( + "(Data is still processed and used for BED generation)" + ).classes("text-gray-400 text-xs") except Exception: pass except Exception as e: @@ -3458,7 +3985,9 @@ def _update_fusion_ui(fusion_data: Dict[str, Any], state: Dict[str, Any], sample ui.label("Target panel").classes( "target-coverage-panel__meta-label mt-4 mb-1" ) - state["target"]["summary_table_container"] = ui.column().classes("w-full") + state["target"]["summary_table_container"] = ui.column().classes( + "w-full" + ) state["target"]["groups_table_container"] = ui.column().classes( "w-full mt-2" ) @@ -3472,7 +4001,9 @@ def _update_fusion_ui(fusion_data: Dict[str, Any], state: Dict[str, Any], sample ui.label("Genome-wide").classes( "target-coverage-panel__meta-label mt-2 mb-1" ) - state["genome"]["summary_table_container"] = ui.column().classes("w-full") + state["genome"]["summary_table_container"] = ui.column().classes( + "w-full" + ) state["genome"]["groups_table_container"] = ui.column().classes( "w-full mt-2" ) @@ -3505,9 +4036,13 @@ def _update_fusion_ui(fusion_data: Dict[str, Any], state: Dict[str, Any], sample ) if "summary" in state: t_pairs0 = _count_unique_fusion_pairs(t0) if isinstance(t0, dict) else 0 - t_groups0 = _count_unique_fusion_groups(t0) if isinstance(t0, dict) else 0 + t_groups0 = ( + _count_unique_fusion_groups(t0) if isinstance(t0, dict) else 0 + ) g_pairs0 = _count_unique_fusion_pairs(g0) if isinstance(g0, dict) else 0 - g_groups0 = _count_unique_fusion_groups(g0) if isinstance(g0, dict) else 0 + g_groups0 = ( + _count_unique_fusion_groups(g0) if isinstance(g0, dict) else 0 + ) try: state["summary"][ "target_lbl" @@ -3518,7 +4053,9 @@ def _update_fusion_ui(fusion_data: Dict[str, Any], state: Dict[str, Any], sample # Master BED summary has been deprecated try: - state["summary"]["master_bed_lbl"].text = "Master BED: (deprecated)" + state["summary"][ + "master_bed_lbl" + ].text = "Master BED: (deprecated)" except Exception: pass except Exception: @@ -3527,10 +4064,12 @@ def _update_fusion_ui(fusion_data: Dict[str, Any], state: Dict[str, Any], sample pass # Start the refresh timer (every 30 seconds) - logging.info("[Fusion] Setting up refresh timer (30s interval + immediate async load)") + logging.info( + "[Fusion] Setting up refresh timer (30s interval + immediate async load)" + ) from robin.gui.theme import ( - register_theme_sync_callback, client_timer, + register_theme_sync_callback, stop_timer, ) diff --git a/src/robin/gui/components/itd.py b/src/robin/gui/components/itd.py index 33625ce1..43944b93 100644 --- a/src/robin/gui/components/itd.py +++ b/src/robin/gui/components/itd.py @@ -15,7 +15,7 @@ normalize_itd_events_df, ) from robin.gui.components.snp import navigate_igv_to_snp -from robin.gui.theme import styled_table, client_timer +from robin.gui.theme import client_timer, styled_table logger = logging.getLogger(__name__) @@ -118,9 +118,12 @@ def _format_event_rows(df: pd.DataFrame) -> List[Dict[str, Any]]: def _add_table_search(table: Any, placeholder: str) -> None: try: with table.add_slot("top-right"): - with ui.input(placeholder=placeholder).props( - "type=search dense clearable" - ).bind_value(table, "filter").add_slot("append"): + with ( + ui.input(placeholder=placeholder) + .props("type=search dense clearable") + .bind_value(table, "filter") + .add_slot("append") + ): ui.icon("search") except Exception: pass @@ -234,9 +237,7 @@ def add_itd_section( "(else local hotspot depth) and is the VAF denominator." ) if include_igv: - blurb += ( - " Use View in IGV (or click a called-event row) to inspect the locus." - ) + blurb += " Use View in IGV (or click a called-event row) to inspect the locus." with ui.element("div").classes("classification-insight-shell w-full min-w-0"): ui.label("ITDs / insertions").classes( @@ -281,10 +282,7 @@ def refresh() -> None: if summary is not None and not summary.empty: ui.label("Gene summary").classes("text-subtitle2 q-mt-sm") view = summary - if ( - not show_empty.value - and "n_events" in summary.columns - ): + if not show_empty.value and "n_events" in summary.columns: view = summary[summary["n_events"] > 0] if view.empty: ui.label( @@ -313,7 +311,9 @@ def refresh() -> None: c for c in ("support", "vaf", "gene") if c in events.columns ] ordered = ( - events.sort_values(sort_cols, ascending=[False] * len(sort_cols)) + events.sort_values( + sort_cols, ascending=[False] * len(sort_cols) + ) if sort_cols else events ) diff --git a/src/robin/gui/components/mgmt.py b/src/robin/gui/components/mgmt.py index d83acd08..7a4e0231 100644 --- a/src/robin/gui/components/mgmt.py +++ b/src/robin/gui/components/mgmt.py @@ -1,25 +1,24 @@ from __future__ import annotations import asyncio -from pathlib import Path -from typing import Any, Dict, List import logging import time +from pathlib import Path +from typing import Any, Dict, List import pandas as pd - try: from nicegui import ui except ImportError: # pragma: no cover ui = None from robin.gui.theme import ( - styled_table, - register_theme_sync_callback, - get_user_dark_mode, client_timer, + get_user_dark_mode, + register_theme_sync_callback, stop_timer, + styled_table, ) @@ -142,8 +141,16 @@ def add_mgmt_section(launcher: Any, sample_dir: Path) -> None: {"name": "cov_rev", "label": "Reverse Cov", "field": "cov_rev"}, {"name": "cov_total", "label": "Total Cov", "field": "cov_total"}, {"name": "meth", "label": "% Methylation", "field": "meth"}, - {"name": "meth_fwd", "label": "Forward Methylated", "field": "meth_fwd"}, - {"name": "meth_rev", "label": "Reverse Methylated", "field": "meth_rev"}, + { + "name": "meth_fwd", + "label": "Forward Methylated", + "field": "meth_fwd", + }, + { + "name": "meth_rev", + "label": "Reverse Methylated", + "field": "meth_rev", + }, {"name": "notes", "label": "Notes", "field": "notes"}, ], rows=[], @@ -156,15 +163,15 @@ def _extract_mgmt_specific_sites(bed_path: Path) -> List[Dict[str, Any]]: import pandas as _pd df = _pd.read_csv(bed_path, sep="\t", header=None) - + # Check if column 10 contains space-separated values (old format) # Even if file has 18 columns, column 10 might still be space-separated has_space_separated_col10 = False if df.shape[1] > 10 and len(df) > 0: # Check if column 10 (index 9) contains space-separated values sample_val = str(df.iloc[0, 9]) - has_space_separated_col10 = ' ' in sample_val or '\t' in sample_val - + has_space_separated_col10 = " " in sample_val or "\t" in sample_val + # Check if this is the new bedmethyl format (separate columns) or old format (space-separated column 10) if df.shape[1] >= 12 and not has_space_separated_col10: # New bedmethyl format with separate columns @@ -178,15 +185,15 @@ def _extract_mgmt_specific_sites(bed_path: Path) -> List[Dict[str, Any]]: "Start2", "End2", "RGB", - "Nvalid_cov", # Column 10: Valid coverage (absolute count) + "Nvalid_cov", # Column 10: Valid coverage (absolute count) "Fraction_Modified", # Column 11: Nmod / Nvalid_cov (fraction 0-1, not percentage) - "Nmod", # Column 12: Absolute count of modified reads + "Nmod", # Column 12: Absolute count of modified reads ] # Read at least the first 12 columns num_cols_to_read = min(len(cols), df.shape[1]) df = df.iloc[:, :num_cols_to_read] df.columns = cols[:num_cols_to_read] - + # Convert to proper types df["Nvalid_cov"] = df["Nvalid_cov"].astype(float) df["Fraction_Modified"] = df["Fraction_Modified"].astype(float) @@ -195,10 +202,10 @@ def _extract_mgmt_specific_sites(bed_path: Path) -> List[Dict[str, Any]]: else: # If Nmod column doesn't exist, calculate it from fraction * coverage df["Nmod"] = df["Nvalid_cov"] * df["Fraction_Modified"] - + # Ensure Start is integer for proper comparison df["Start"] = df["Start"].astype(int) - + # For backward compatibility, keep Coverage and Modified_Fraction columns df["Coverage"] = df["Nvalid_cov"] df["Modified_Fraction"] = df["Fraction_Modified"] * 100.0 @@ -219,31 +226,37 @@ def _extract_mgmt_specific_sites(bed_path: Path) -> List[Dict[str, Any]]: # Only use first 10 columns for old format, even if file has more columns df = df.iloc[:, : len(cols)] df.columns = cols - + # Parse Coverage_Info (space-separated: coverage fraction/percentage) # Format: "coverage fraction" or "coverage percentage" cov_split = df["Coverage_Info"].astype(str).str.split() df["Coverage"] = cov_split.str[0].astype(float) - + # Get second value (fraction or percentage) # pandas str accessor will return NaN for missing values fraction_val = cov_split.str[1].astype(float).fillna(0.0) - + # Determine if second value is fraction (0-1) or percentage (0-100) # If any value is > 1, assume it's percentage, otherwise assume fraction - is_percentage = (fraction_val > 1.0).any() if len(fraction_val) > 0 else False - + is_percentage = ( + (fraction_val > 1.0).any() if len(fraction_val) > 0 else False + ) + if is_percentage: df["Modified_Fraction"] = fraction_val - df["Fraction_Modified"] = df["Modified_Fraction"] / 100.0 # Convert percentage to fraction + df["Fraction_Modified"] = ( + df["Modified_Fraction"] / 100.0 + ) # Convert percentage to fraction else: df["Fraction_Modified"] = fraction_val - df["Modified_Fraction"] = df["Fraction_Modified"] * 100.0 # Convert fraction to percentage - + df["Modified_Fraction"] = ( + df["Fraction_Modified"] * 100.0 + ) # Convert fraction to percentage + # Convert to new format columns for consistency df["Nvalid_cov"] = df["Coverage"] df["Nmod"] = df["Coverage"] * df["Fraction_Modified"] - + # Ensure Start is integer for proper comparison df["Start"] = df["Start"].astype(int) else: @@ -261,11 +274,11 @@ def _extract_mgmt_specific_sites(bed_path: Path) -> List[Dict[str, Any]]: "129467262/129467263": "3", "129467272/129467273": "4", } - + for p1, p2 in cpg_pairs: pos_key = f"{p1}/{p2}" site_label = label_map.get(pos_key, "Unknown") - + # For a CpG pair (p1, p2) where p1 and p2 are consecutive: # The CpG site consists of two cytosines: # - Forward strand: C at p1, G at p1+1 (p2) @@ -278,14 +291,14 @@ def _extract_mgmt_specific_sites(bed_path: Path) -> List[Dict[str, Any]]: # # IMPORTANT: We must ensure we're checking the correct positions for this specific CpG site # and not accidentally assigning methylation from adjacent sites. - + # Check forward strand reads at position p1 (the C on forward strand for this CpG) fwd_p1 = df[ (df["Chromosome"] == "chr10") & (df["Start"] == p1 - 1) & (df["Strand"] == "+") ] - + # Check reverse strand reads at position p2 (the C on reverse strand for this CpG) # This is correct because reverse strand reads see the C at p2 for this CpG site rev_p2 = df[ @@ -293,34 +306,46 @@ def _extract_mgmt_specific_sites(bed_path: Path) -> List[Dict[str, Any]]: & (df["Start"] == p2 - 1) & (df["Strand"] == "-") ] - + # IMPORTANT: We should NOT check reverse strand at p1 or forward strand at p2, # as those would represent methylation from adjacent CpG sites or the wrong cytosine. # For example, reverse strand at p1 would be the G position (not a C), and # forward strand at p2 would be the G position (not a C) for this CpG site. - + # Get forward strand data from p1 if not fwd_p1.empty: cov_f = float(fwd_p1["Nvalid_cov"].iloc[0]) - mf = float(fwd_p1["Fraction_Modified"].iloc[0]) # Fraction (0-1), not percentage - nmod_f = float(fwd_p1["Nmod"].iloc[0]) # Direct count from bedmethyl file - meth_fwd_count = int(round(nmod_f)) # Use Nmod directly from bedmethyl + mf = float( + fwd_p1["Fraction_Modified"].iloc[0] + ) # Fraction (0-1), not percentage + nmod_f = float( + fwd_p1["Nmod"].iloc[0] + ) # Direct count from bedmethyl file + meth_fwd_count = int( + round(nmod_f) + ) # Use Nmod directly from bedmethyl else: cov_f = 0.0 mf = 0.0 meth_fwd_count = 0 - + # Get reverse strand data from p2 (the C position on reverse strand for this CpG) if not rev_p2.empty: cov_r = float(rev_p2["Nvalid_cov"].iloc[0]) - mr = float(rev_p2["Fraction_Modified"].iloc[0]) # Fraction (0-1), not percentage - nmod_r = float(rev_p2["Nmod"].iloc[0]) # Direct count from bedmethyl file - meth_rev_count = int(round(nmod_r)) # Use Nmod directly from bedmethyl + mr = float( + rev_p2["Fraction_Modified"].iloc[0] + ) # Fraction (0-1), not percentage + nmod_r = float( + rev_p2["Nmod"].iloc[0] + ) # Direct count from bedmethyl file + meth_rev_count = int( + round(nmod_r) + ) # Use Nmod directly from bedmethyl else: cov_r = 0.0 mr = 0.0 meth_rev_count = 0 - + # Only add row if we have data if cov_f > 0 or cov_r > 0: tot = cov_f + cov_r @@ -329,7 +354,7 @@ def _extract_mgmt_specific_sites(bed_path: Path) -> List[Dict[str, Any]]: # Or equivalently: weighted = (nmod_f + nmod_r) / tot weighted = ((cov_f * mf) + (cov_r * mr)) / tot if tot > 0 else 0.0 weighted_pct = weighted * 100.0 # Convert to percentage for display - + rows.append( { "site": f"{site_label} (CpG {pos_key})", @@ -338,13 +363,15 @@ def _extract_mgmt_specific_sites(bed_path: Path) -> List[Dict[str, Any]]: "cov_fwd": int(cov_f), "cov_rev": int(cov_r), "cov_total": int(tot), - "meth": round(weighted_pct, 2), # Store as percentage for display + "meth": round( + weighted_pct, 2 + ), # Store as percentage for display "meth_fwd": int(meth_fwd_count), # Direct from Nmod column "meth_rev": int(meth_rev_count), # Direct from Nmod column "notes": "Combined methylation from both strands of CpG pair", } ) - + return rows except Exception: return [] @@ -501,8 +528,10 @@ def _count_from_name(p: Path) -> int: if bam_path.exists() and figure_needs_update: def _build_mgmt_figure(): - import matplotlib.pyplot as plt import warnings + + import matplotlib.pyplot as plt + from robin.analysis.methylation_wrapper import ( has_bam_index, locus_figure, @@ -540,9 +569,10 @@ def _build_mgmt_figure(): try: fig = await asyncio.to_thread(_build_mgmt_figure) - import matplotlib.pyplot as plt import warnings + import matplotlib.pyplot as plt + with warnings.catch_warnings(): warnings.simplefilter("ignore", UserWarning) if hasattr(mgmt_mpl, "figure") and mgmt_mpl.figure is not None: @@ -653,12 +683,12 @@ def _count_from_name(p: Path) -> int: latest_csv = max(csv_files, key=_count_from_name) key = str(sample_dir) state = launcher._mgmt_state.get(key, {}) - + # Check if this is a fresh page visit - force updates on fresh visits is_fresh_visit = "last_visit_time" not in state if is_fresh_visit: state["last_visit_time"] = time.time() - + current_count = _count_from_name(latest_csv) csv_mtime = latest_csv.stat().st_mtime # Force update on fresh page visit or if file changed @@ -667,14 +697,14 @@ def _count_from_name(p: Path) -> int: or state.get("csv_path") != str(latest_csv) or state.get("csv_mtime") != csv_mtime ) - + # Always read CSV data (needed for plot even if CSV hasn't changed) try: df = pd.read_csv(latest_csv) except Exception as e: logging.error(f"[MGMT] Failed to read CSV file {latest_csv}: {e}") return # Cannot proceed without CSV data - + # Only update CSV-related UI elements if CSV has changed if csv_needs_update: try: @@ -708,7 +738,7 @@ def _count_from_name(p: Path) -> int: except Exception as e: logging.error(f"[MGMT] Failed to update CSV UI elements: {e}") pass - + # Gather site_rows for table and plot annotations site_rows: List[Dict[str, Any]] = [] # Handle bed file lookup based on CSV file type @@ -724,7 +754,7 @@ def _count_from_name(p: Path) -> int: if not bed_path.exists(): alt_bed = sample_dir / f"{current_count}_mgmt_mgmt.bed" bed_path = alt_bed if alt_bed.exists() else bed_path - + if bed_path.exists(): bed_mtime = bed_path.stat().st_mtime bed_needs_update = ( @@ -732,10 +762,10 @@ def _count_from_name(p: Path) -> int: or state.get("bed_path") != str(bed_path) or state.get("bed_mtime") != bed_mtime ) - + # Always extract site_rows for plotting (needed even if bed hasn't changed) site_rows = _extract_mgmt_specific_sites(bed_path) - + # Only update table if bed file has changed if bed_needs_update: try: @@ -752,38 +782,40 @@ def _count_from_name(p: Path) -> int: pass bam_path = sample_dir / "mgmt_sorted.bam" - + # Determine pickle file path based on CSV file type if is_final_file: pickle_path = sample_dir / "final_mgmt.pkl" else: pickle_path = sample_dir / f"{current_count}_mgmt.pkl" - + # Check if figure needs to be updated (only if pickle or BAM changed) pickle_mtime = pickle_path.stat().st_mtime if pickle_path.exists() else 0 bam_mtime = bam_path.stat().st_mtime if bam_path.exists() else 0 - + # Check if pickle or BAM file has changed since last update current_bam_path_str = str(bam_path) if bam_path.exists() else "" current_pickle_path_str = str(pickle_path) if pickle_path.exists() else "" - + # Get previous state values (use sentinel values if not present) prev_pickle_path = state.get("pickle_path", "") prev_pickle_mtime = state.get("pickle_mtime", 0) prev_bam_path = state.get("bam_path", "") prev_bam_mtime = state.get("bam_mtime", 0) - + # Debug: log state comparison pickle_changed = prev_pickle_path != current_pickle_path_str pickle_mtime_changed = prev_pickle_mtime != pickle_mtime bam_changed = prev_bam_path != current_bam_path_str bam_mtime_changed = prev_bam_mtime != bam_mtime - - logging.debug(f"[MGMT] State comparison - pickle_path: {pickle_changed} (prev='{prev_pickle_path}' vs curr='{current_pickle_path_str}'), " - f"pickle_mtime: {pickle_mtime_changed} (prev={prev_pickle_mtime} vs curr={pickle_mtime}), " - f"bam_path: {bam_changed} (prev='{prev_bam_path}' vs curr='{current_bam_path_str}'), " - f"bam_mtime: {bam_mtime_changed} (prev={prev_bam_mtime} vs curr={bam_mtime}), fresh={is_fresh_visit}") - + + logging.debug( + f"[MGMT] State comparison - pickle_path: {pickle_changed} (prev='{prev_pickle_path}' vs curr='{current_pickle_path_str}'), " + f"pickle_mtime: {pickle_mtime_changed} (prev={prev_pickle_mtime} vs curr={pickle_mtime}), " + f"bam_path: {bam_changed} (prev='{prev_bam_path}' vs curr='{current_bam_path_str}'), " + f"bam_mtime: {bam_mtime_changed} (prev={prev_bam_mtime} vs curr={bam_mtime}), fresh={is_fresh_visit}" + ) + figure_needs_update = ( is_fresh_visit or pickle_changed @@ -791,7 +823,7 @@ def _count_from_name(p: Path) -> int: or bam_changed or bam_mtime_changed ) - + if figure_needs_update: reasons = [] if is_fresh_visit: @@ -804,34 +836,46 @@ def _count_from_name(p: Path) -> int: reasons.append("bam_path_changed") if bam_mtime_changed: reasons.append("bam_mtime_changed") - logging.debug(f"[MGMT] Figure update needed. Reasons: {', '.join(reasons)}") - + logging.debug( + f"[MGMT] Figure update needed. Reasons: {', '.join(reasons)}" + ) + if not figure_needs_update and bam_path.exists(): - logging.debug(f"[MGMT] Skipping figure update - no changes detected (pickle: {pickle_path.name if pickle_path.exists() else 'N/A'}, BAM: {bam_path.name})") - + logging.debug( + f"[MGMT] Skipping figure update - no changes detected (pickle: {pickle_path.name if pickle_path.exists() else 'N/A'}, BAM: {bam_path.name})" + ) + if bam_path.exists() and figure_needs_update: try: - import matplotlib.pyplot as plt import warnings + + import matplotlib.pyplot as plt + from robin.analysis.methylation_wrapper import ( has_bam_index, locus_figure, try_load_figure_pickle, ) - + # Check if pickle file exists and is newer than BAM file fig = None use_pickle = False if pickle_path.exists(): if pickle_mtime >= bam_mtime: - logging.debug(f"[MGMT] Loading figure from pickle: {pickle_path}") + logging.debug( + f"[MGMT] Loading figure from pickle: {pickle_path}" + ) fig = try_load_figure_pickle(str(pickle_path)) use_pickle = fig is not None if use_pickle: - logging.debug(f"[MGMT] Successfully loaded figure from pickle") + logging.debug( + f"[MGMT] Successfully loaded figure from pickle" + ) else: - logging.debug(f"[MGMT] Pickle file is older than BAM, regenerating figure") - + logging.debug( + f"[MGMT] Pickle file is older than BAM, regenerating figure" + ) + # If pickle doesn't exist or failed to load, generate new figure # Skip locus_figure if BAM has no index (avoids "fetch on bamfile without index") if fig is None: @@ -847,31 +891,42 @@ def _count_from_name(p: Path) -> int: mods="m", extra_cli=[ # Set figure size to match our UI element (width 2x height) - "--width", "18", - "--height", "8", + "--width", + "18", + "--height", + "8", # "--minqual","20", "--reads","2000" ], site_rows=site_rows, ) - logging.debug(f"[MGMT] Locus figure created successfully, figure number: {fig.number}") - + logging.debug( + f"[MGMT] Locus figure created successfully, figure number: {fig.number}" + ) + # Save pickle for future use (if not already saved by analysis) if not use_pickle: try: - from robin.analysis.methylation_wrapper import save_figure_pickle + from robin.analysis.methylation_wrapper import ( + save_figure_pickle, + ) + save_figure_pickle(fig, str(pickle_path)) - logging.debug(f"[MGMT] Saved figure to pickle for future use: {pickle_path}") + logging.debug( + f"[MGMT] Saved figure to pickle for future use: {pickle_path}" + ) # Update pickle_mtime after saving pickle_mtime = pickle_path.stat().st_mtime except Exception as e: - logging.debug(f"[MGMT] Failed to save pickle file (non-fatal): {e}") - + logging.debug( + f"[MGMT] Failed to save pickle file (non-fatal): {e}" + ) + # Update matplotlib element with the figure # Suppress GridSpec warnings when updating figure with warnings.catch_warnings(): warnings.simplefilter("ignore", UserWarning) # Close previous figure if it exists before assigning new one - if hasattr(mgmt_mpl, 'figure') and mgmt_mpl.figure is not None: + if hasattr(mgmt_mpl, "figure") and mgmt_mpl.figure is not None: try: plt.close(mgmt_mpl.figure) except Exception: @@ -884,23 +939,30 @@ def _count_from_name(p: Path) -> int: except Exception as e: # If methylartist fails, create a simple placeholder plot logging.exception(f"[MGMT] Failed to create methylation plot: {e}") - import matplotlib.pyplot as plt import matplotlib.patches as patches - + import matplotlib.pyplot as plt + # Close previous figure if it exists - if hasattr(mgmt_mpl, 'figure') and mgmt_mpl.figure is not None: + if hasattr(mgmt_mpl, "figure") and mgmt_mpl.figure is not None: try: plt.close(mgmt_mpl.figure) except Exception: pass - + fig, ax = plt.subplots(figsize=(24, 12)) - ax.text(0.5, 0.5, f"Methylation plot unavailable\n({str(e)})", - ha='center', va='center', transform=ax.transAxes, - fontsize=10, color='red') + ax.text( + 0.5, + 0.5, + f"Methylation plot unavailable\n({str(e)})", + ha="center", + va="center", + transform=ax.transAxes, + fontsize=10, + color="red", + ) ax.set_xlim(0, 1) ax.set_ylim(0, 1) - ax.axis('off') + ax.axis("off") ax.set_title("MGMT Methylation Plot") _apply_mgmt_figure_theme(fig, _is_dark_mode()) mgmt_mpl.figure = fig @@ -911,18 +973,26 @@ def _count_from_name(p: Path) -> int: logging.warning(f"[MGMT] BAM file not found: {bam_path}") # Create a placeholder plot indicating BAM file is missing import matplotlib.pyplot as plt - if hasattr(mgmt_mpl, 'figure') and mgmt_mpl.figure is not None: + + if hasattr(mgmt_mpl, "figure") and mgmt_mpl.figure is not None: try: plt.close(mgmt_mpl.figure) except Exception: pass fig, ax = plt.subplots(figsize=(24, 12)) - ax.text(0.5, 0.5, "MGMT BAM file not found\n(mgmt_sorted.bam)", - ha='center', va='center', transform=ax.transAxes, - fontsize=10, color='orange') + ax.text( + 0.5, + 0.5, + "MGMT BAM file not found\n(mgmt_sorted.bam)", + ha="center", + va="center", + transform=ax.transAxes, + fontsize=10, + color="orange", + ) ax.set_xlim(0, 1) ax.set_ylim(0, 1) - ax.axis('off') + ax.axis("off") ax.set_title("MGMT Methylation Plot") _apply_mgmt_figure_theme(fig, _is_dark_mode()) mgmt_mpl.figure = fig @@ -939,13 +1009,17 @@ def _count_from_name(p: Path) -> int: "pickle_mtime": pickle_mtime, "bam_path": current_bam_path_str, "bam_mtime": bam_mtime, - "last_visit_time": state.get("last_visit_time", time.time()), # Preserve visit time + "last_visit_time": state.get( + "last_visit_time", time.time() + ), # Preserve visit time "mgmt_plot_theme_dark": _is_dark_mode(), } - + # Debug: log state after update - logging.debug(f"[MGMT] State saved - pickle_path='{current_pickle_path_str}', pickle_mtime={pickle_mtime}, " - f"bam_path='{current_bam_path_str}', bam_mtime={bam_mtime}") + logging.debug( + f"[MGMT] State saved - pickle_path='{current_pickle_path_str}', pickle_mtime={pickle_mtime}, " + f"bam_path='{current_bam_path_str}', bam_mtime={bam_mtime}" + ) except Exception as e: raise Exception(f"Failed to refresh MGMT section: {e}") diff --git a/src/robin/gui/components/minknow.py b/src/robin/gui/components/minknow.py index c1bfbdab..11726777 100644 --- a/src/robin/gui/components/minknow.py +++ b/src/robin/gui/components/minknow.py @@ -12,7 +12,11 @@ from nicegui import run, ui from robin.gui.theme import client_timer, stop_timer -from robin.minknow.config import MinKnowSettings, preset_path_from_environ, workflow_toml_from_environ +from robin.minknow.config import ( + MinKnowSettings, + preset_path_from_environ, + workflow_toml_from_environ, +) from robin.minknow.monitor import ( MinKnowPollResult, fetch_sequencer_status, @@ -34,25 +38,79 @@ ) from robin.minknow.stream_monitor import acquire_stream_monitor from robin.minknow.toml_config import MinKnowWorkflowConfig, load_minknow_toml +from robin.minknow.watch import process_auto_watch, watch_position_run from robin.minknow.workflow_refs import ( load_workflow_config_for_refs, workflow_context_from_runner, ) from robin.workflow_config import load_minknow_from_workflow_toml -from robin.minknow.watch import process_auto_watch, watch_position_run LOGGER = logging.getLogger(__name__) _TABLE_COLUMNS = [ - {"name": "position", "label": "Position", "field": "position", "sortable": True, "align": "left"}, - {"name": "state", "label": "State", "field": "state", "sortable": True, "align": "left"}, - {"name": "protocol_state", "label": "Protocol", "field": "protocol_state", "sortable": True, "align": "left"}, - {"name": "sample_id", "label": "Sample ID", "field": "sample_id", "sortable": True, "align": "left"}, - {"name": "protocol_run_id", "label": "Run ID", "field": "protocol_run_id", "sortable": True, "align": "left"}, - {"name": "flow_cell_id", "label": "Flow cell", "field": "flow_cell_id", "sortable": True, "align": "left"}, - {"name": "passed_reads", "label": "Passed reads", "field": "passed_reads", "sortable": True, "align": "left"}, - {"name": "watch_path", "label": "Watch path", "field": "watch_path", "sortable": True, "align": "left"}, - {"name": "actions", "label": "", "field": "actions", "sortable": False, "align": "right"}, + { + "name": "position", + "label": "Position", + "field": "position", + "sortable": True, + "align": "left", + }, + { + "name": "state", + "label": "State", + "field": "state", + "sortable": True, + "align": "left", + }, + { + "name": "protocol_state", + "label": "Protocol", + "field": "protocol_state", + "sortable": True, + "align": "left", + }, + { + "name": "sample_id", + "label": "Sample ID", + "field": "sample_id", + "sortable": True, + "align": "left", + }, + { + "name": "protocol_run_id", + "label": "Run ID", + "field": "protocol_run_id", + "sortable": True, + "align": "left", + }, + { + "name": "flow_cell_id", + "label": "Flow cell", + "field": "flow_cell_id", + "sortable": True, + "align": "left", + }, + { + "name": "passed_reads", + "label": "Passed reads", + "field": "passed_reads", + "sortable": True, + "align": "left", + }, + { + "name": "watch_path", + "label": "Watch path", + "field": "watch_path", + "sortable": True, + "align": "left", + }, + { + "name": "actions", + "label": "", + "field": "actions", + "sortable": False, + "align": "right", + }, ] _ACTIONS_SLOT = """ @@ -104,7 +162,9 @@ def _workflow_toml_path( return None -def _load_workflow_minknow_config(path: Optional[Path]) -> Optional[MinKnowWorkflowConfig]: +def _load_workflow_minknow_config( + path: Optional[Path], +) -> Optional[MinKnowWorkflowConfig]: if path is None or not path.is_file(): return None try: @@ -283,10 +343,14 @@ def _resolve_start_position(preset: RobinRunPreset) -> Optional[str]: manual_controls = ui.row().classes("w-full gap-2 flex-wrap items-end") with manual_controls: - host_input = ui.input( - "MinKNOW host", - value=state["host"], - ).props("outlined dense").classes("min-w-[12rem] flex-1") + host_input = ( + ui.input( + "MinKNOW host", + value=state["host"], + ) + .props("outlined dense") + .classes("min-w-[12rem] flex-1") + ) enabled_switch = ui.switch( "Monitor", value=state["enabled"], @@ -350,10 +414,14 @@ def _apply_selected_position(name: str) -> None: position_picker_row = ui.row().classes( "w-full gap-2 flex-wrap items-center" ) - position_fallback_input = ui.input( - "Position", - placeholder="e.g. P2S_000000-A", - ).props("outlined dense").classes("w-full") + position_fallback_input = ( + ui.input( + "Position", + placeholder="e.g. P2S_000000-A", + ) + .props("outlined dense") + .classes("w-full") + ) with ui.expansion( "Run settings", @@ -367,10 +435,14 @@ def _apply_selected_position(name: str) -> None: "([minknow] section) or set host and preset " "path below." ).classes("text-xs text-slate-500") - preset_input = ui.input( - "Preset TOML", - value=state.get("workflow_toml") or "", - ).props("outlined dense").classes("w-full") + preset_input = ( + ui.input( + "Preset TOML", + value=state.get("workflow_toml") or "", + ) + .props("outlined dense") + .classes("w-full") + ) preset_input.on( "blur", lambda: state.update( @@ -383,45 +455,65 @@ def _apply_selected_position(name: str) -> None: ) with ui.row().classes("w-full gap-2 flex-wrap"): - experiment_group_input = ui.input( - "Experiment group", - value="ROBIN_RUN", - ).props("outlined dense readonly").classes( - "flex-1 min-w-[12rem]" + experiment_group_input = ( + ui.input( + "Experiment group", + value="ROBIN_RUN", + ) + .props("outlined dense readonly") + .classes("flex-1 min-w-[12rem]") ) - duration_input = ui.number( - "Duration (hours)", - value=24, - min=0.1, - step=0.5, - ).props("outlined dense").classes( - "flex-1 min-w-[10rem]" + duration_input = ( + ui.number( + "Duration (hours)", + value=24, + min=0.1, + step=0.5, + ) + .props("outlined dense") + .classes("flex-1 min-w-[10rem]") ) - kit_input = ui.input( - "Sequencing kit", - value="SQK-LSK114", - ).props("outlined dense readonly").classes("w-full") - simplex_input = ui.input( - "Basecall simplex model", - ).props("outlined dense readonly").classes("w-full") - modified_input = ui.input( - "Modified models (comma-separated)", - ).props("outlined dense readonly").classes("w-full") + kit_input = ( + ui.input( + "Sequencing kit", + value="SQK-LSK114", + ) + .props("outlined dense readonly") + .classes("w-full") + ) + simplex_input = ( + ui.input( + "Basecall simplex model", + ) + .props("outlined dense readonly") + .classes("w-full") + ) + modified_input = ( + ui.input( + "Modified models (comma-separated)", + ) + .props("outlined dense readonly") + .classes("w-full") + ) with ui.row().classes("w-full gap-2 flex-wrap"): - bam_reads_input = ui.number( - "BAM reads per file", - value=50_000, - min=1, - step=1000, - ).props("outlined dense readonly").classes( - "flex-1 min-w-[12rem]" + bam_reads_input = ( + ui.number( + "BAM reads per file", + value=50_000, + min=1, + step=1000, + ) + .props("outlined dense readonly") + .classes("flex-1 min-w-[12rem]") ) if show_simulation_field: - simulation_input = ui.input( - "Simulation bulk FAST5", - ).props("outlined dense").classes( - "flex-1 min-w-[12rem]" + simulation_input = ( + ui.input( + "Simulation bulk FAST5", + ) + .props("outlined dense") + .classes("flex-1 min-w-[12rem]") ) reference_label = ui.label("").classes( @@ -434,10 +526,14 @@ def _apply_selected_position(name: str) -> None: "text-xs text-slate-500 w-full" ) - sample_id_input = ui.input( - "Sample ID / MinKNOW RUN ID", - placeholder="Registered MD5 or MinKNOW RUN ID", - ).props("outlined dense").classes("w-full") + sample_id_input = ( + ui.input( + "Sample ID / MinKNOW RUN ID", + placeholder="Registered MD5 or MinKNOW RUN ID", + ) + .props("outlined dense") + .classes("w-full") + ) with ui.expansion( "Register sample identifiers", @@ -454,48 +550,70 @@ def _apply_selected_position(name: str) -> None: "text-xs text-slate-600 dark:text-slate-400 w-full mb-2" ) - id_mode = ui.toggle( - { - "custom": "Use my sample ID", - "md5": "Generate MD5 ID", - }, - value="custom", - ).props("no-caps dense").classes("w-full") + id_mode = ( + ui.toggle( + { + "custom": "Use my sample ID", + "md5": "Generate MD5 ID", + }, + value="custom", + ) + .props("no-caps dense") + .classes("w-full") + ) md5_fields = ui.column().classes("w-full min-w-0 gap-2") with md5_fields: - gen_test_id = ui.input( - "Test ID (required for MD5)" - ).props("outlined dense").classes("w-full") + gen_test_id = ( + ui.input("Test ID (required for MD5)") + .props("outlined dense") + .classes("w-full") + ) md5_fields.set_visibility(False) custom_fields = ui.column().classes("w-full min-w-0 gap-2") with custom_fields: - custom_run_id = ui.input( - "MinKNOW RUN ID (required)", - placeholder="e.g. HOSP-2024-8841", - ).props("outlined dense").classes("w-full font-mono") - custom_test_id = ui.input( - "Test ID (optional)" - ).props("outlined dense").classes("w-full") - - gen_first = ui.input("First name (optional)").props( - "outlined dense" - ).classes("w-full") - gen_last = ui.input("Last name (optional)").props( - "outlined dense" - ).classes("w-full") + custom_run_id = ( + ui.input( + "MinKNOW RUN ID (required)", + placeholder="e.g. HOSP-2024-8841", + ) + .props("outlined dense") + .classes("w-full font-mono") + ) + custom_test_id = ( + ui.input("Test ID (optional)") + .props("outlined dense") + .classes("w-full") + ) + + gen_first = ( + ui.input("First name (optional)") + .props("outlined dense") + .classes("w-full") + ) + gen_last = ( + ui.input("Last name (optional)") + .props("outlined dense") + .classes("w-full") + ) gen_dob = ui.date_input( "Date of birth (required when encrypting)", value=None, ).classes("w-full") - gen_nhs = ui.input( - "Hospital number (optional)" - ).props("outlined dense").classes("w-full") - gen_notes = ui.textarea( - "Notes (optional)", - placeholder="Free-text notes stored encrypted with identifiers", - ).props("outlined dense autogrow").classes("w-full") + gen_nhs = ( + ui.input("Hospital number (optional)") + .props("outlined dense") + .classes("w-full") + ) + gen_notes = ( + ui.textarea( + "Notes (optional)", + placeholder="Free-text notes stored encrypted with identifiers", + ) + .props("outlined dense autogrow") + .classes("w-full") + ) def _sync_id_mode() -> None: is_md5 = id_mode.value == "md5" @@ -602,9 +720,13 @@ def _position_has_active_run(name: str) -> bool: if not name: return False result = state.get("last_result") - status = getattr(result, "status", None) if result is not None else None + status = ( + getattr(result, "status", None) if result is not None else None + ) if status is not None: - from robin.minknow.watch import position_has_active_run as _row_active + from robin.minknow.watch import ( + position_has_active_run as _row_active, + ) for position in status.positions: if position.name == name: @@ -659,13 +781,17 @@ def _sync_position_options() -> None: radio_value = ( preferred if preferred in names else names[0] ) - state["position_radio"] = ui.radio( - names, - value=radio_value, - on_change=lambda e: _apply_selected_position( - e.value - ), - ).props("inline").classes("w-full") + state["position_radio"] = ( + ui.radio( + names, + value=radio_value, + on_change=lambda e: _apply_selected_position( + e.value + ), + ) + .props("inline") + .classes("w-full") + ) position_fallback_input.set_visibility(False) _apply_selected_position(radio_value) else: @@ -726,9 +852,7 @@ def _populate_form_from_preset(preset: RobinRunPreset) -> None: def _build_preset_from_form(base: RobinRunPreset) -> RobinRunPreset: if simulation_input is not None: - simulation_path = ( - (simulation_input.value or "").strip() or None - ) + simulation_path = (simulation_input.value or "").strip() or None else: simulation_path = base.simulation_bulk_file position = (state.get("selected_position") or "").strip() @@ -749,9 +873,7 @@ def _build_preset_from_form(base: RobinRunPreset) -> RobinRunPreset: summary_label = ui.label("Waiting for stream connection…").classes( "classification-insight-foot w-full" ) - meta_label = ui.label("").classes( - "text-xs workflow-monitor-meta w-full" - ) + meta_label = ui.label("").classes("text-xs workflow-monitor-meta w-full") warning_label = ui.label("").classes( "text-xs text-amber-700 dark:text-amber-300 w-full" ) @@ -781,9 +903,13 @@ def _build_preset_from_form(base: RobinRunPreset) -> RobinRunPreset: LOGGER.debug("MinKNOW actions slot failed", exc_info=True) if not compact: + def _on_position_row_click(event) -> None: row = event.args - if isinstance(event.args, (list, tuple)) and len(event.args) > 1: + if ( + isinstance(event.args, (list, tuple)) + and len(event.args) > 1 + ): row = event.args[1] if not isinstance(row, dict): return @@ -839,9 +965,7 @@ def _apply_result(result: MinKnowPollResult) -> None: if result.error: error_label.set_text(result.error) - summary_label.set_text( - format_poll_summary(result, host=state["host"]) - ) + summary_label.set_text(format_poll_summary(result, host=state["host"])) meta_label.set_text("") warning_label.set_text("") stream_indicator.set_visibility(False) @@ -1044,7 +1168,11 @@ def _open_start_dialog() -> None: ) return - mode = (start_controls.get("id_mode").value if start_controls.get("id_mode") else None) or "custom" + mode = ( + start_controls.get("id_mode").value + if start_controls.get("id_mode") + else None + ) or "custom" sample_id_field = (start_controls["sample_id_input"].value or "").strip() custom_run = ( (start_controls["custom_run_id"].value or "").strip() diff --git a/src/robin/gui/components/mnpflex.py b/src/robin/gui/components/mnpflex.py index b2213e56..13955957 100644 --- a/src/robin/gui/components/mnpflex.py +++ b/src/robin/gui/components/mnpflex.py @@ -1,14 +1,14 @@ from __future__ import annotations import asyncio -from pathlib import Path -from typing import Any, Dict, List, Optional import base64 import json import logging import queue import threading import time +from pathlib import Path +from typing import Any, Dict, List, Optional try: from nicegui import ui @@ -23,15 +23,18 @@ format_mnpflex_runtime_error, hierarchy_aggregate_display, ) +from robin.analysis.mnpflex_eligibility import ( + sample_ready_for_mnpflex_auto_run_from_dir, +) from robin.analysis.mnpflex_hierarchy import ( format_mnpflex_hierarchy_score, mnpflex_hierarchy_has_content, ) -from robin.analysis.mnpflex_eligibility import ( - sample_ready_for_mnpflex_auto_run_from_dir, +from robin.analysis.mnpflex_runner import ( + preflight_mnpflex_runtime, + run_mnpflex_analysis, ) -from robin.analysis.mnpflex_runner import preflight_mnpflex_runtime, run_mnpflex_analysis -from robin.gui.theme import styled_table, client_timer, stop_timer +from robin.gui.theme import client_timer, stop_timer, styled_table def add_mnpflex_section(launcher: Any, sample_dir: Path, sample_id: str) -> None: @@ -88,25 +91,23 @@ def _execute_mnpflex_analysis(output_dir: Path) -> None: config=mnpflex_config, ) - with ui.element("div").classes("classification-insight-shell w-full min-w-0").props( - "id=mnpflex-results" + with ( + ui.element("div") + .classes("classification-insight-shell w-full min-w-0") + .props("id=mnpflex-results") ): ui.label("MNP-Flex results").classes( "classification-insight-heading text-headline-small" ) with ui.column().classes("w-full min-w-0 gap-3"): with ui.row().classes("mnpflex-notice"): - ui.icon("schedule", size="sm").classes( - "mnpflex-notice-icon mt-0.5" - ) + ui.icon("schedule", size="sm").classes("mnpflex-notice-icon mt-0.5") ui.label( "MNP-Flex is recommended only after at least 12 hours of " "sequencing data have been generated for this sample." ).classes("mnpflex-notice-text") - with ui.row().classes( - "w-full justify-between items-start gap-3 flex-wrap" - ): + with ui.row().classes("w-full justify-between items-start gap-3 flex-wrap"): with ui.column().classes("gap-1 min-w-0"): last_updated_label = ui.label("Last updated: --").classes( "classification-insight-meta" @@ -145,7 +146,9 @@ def _execute_mnpflex_analysis(output_dir: Path) -> None: results_container = ui.column().classes("w-full min-w-0") with results_container: - with ui.element("div").classes("classification-insight-card w-full min-w-0"): + with ui.element("div").classes( + "classification-insight-card w-full min-w-0" + ): with ui.column().classes("w-full gap-2 p-2 md:p-3"): with ui.row().classes("items-center gap-2 min-w-0"): ui.icon("psychology").classes("classification-insight-icon") @@ -161,7 +164,9 @@ def _execute_mnpflex_analysis(output_dir: Path) -> None: classifier_type = ui.label("Type: --").classes( "classification-insight-meta" ) - with ui.element("div").classes("classification-insight-card w-full min-w-0"): + with ui.element("div").classes( + "classification-insight-card w-full min-w-0" + ): with ui.column().classes("w-full gap-2 p-2 md:p-3"): with ui.row().classes("items-center gap-2 min-w-0"): ui.icon("account_tree").classes("classification-insight-icon") @@ -171,9 +176,7 @@ def _execute_mnpflex_analysis(output_dir: Path) -> None: hierarchy_tree_container = ui.element("div").classes( "mnpflex-hierarchy-tree w-full min-w-0" ) - with ui.row().classes( - "w-full items-center gap-2 mt-4 flex-wrap" - ): + with ui.row().classes("w-full items-center gap-2 mt-4 flex-wrap"): ui.label("Top path").classes("classification-insight-meta") top_path_badge = ui.badge("--").classes( "mnpflex-score-badge mnpflex-score-badge--neutral" @@ -201,7 +204,9 @@ def _execute_mnpflex_analysis(output_dir: Path) -> None: "classification-insight-card w-full min-w-0" ): with ui.column().classes("gap-2 p-3 md:p-4"): - ui.label("Subclass").classes("classification-insight-model") + ui.label("Subclass").classes( + "classification-insight-model" + ) agg_subclass_name = ui.label("--").classes( "classification-insight-result text-sm" ) @@ -212,7 +217,9 @@ def _execute_mnpflex_analysis(output_dir: Path) -> None: "classification-insight-card w-full min-w-0" ): with ui.column().classes("gap-2 p-3 md:p-4"): - ui.label("Class").classes("classification-insight-model") + ui.label("Class").classes( + "classification-insight-model" + ) agg_class_name = ui.label("--").classes( "classification-insight-result text-sm" ) @@ -223,7 +230,9 @@ def _execute_mnpflex_analysis(output_dir: Path) -> None: "classification-insight-card w-full min-w-0" ): with ui.column().classes("gap-2 p-3 md:p-4"): - ui.label("Family").classes("classification-insight-model") + ui.label("Family").classes( + "classification-insight-model" + ) agg_family_name = ui.label("--").classes( "classification-insight-result text-sm" ) @@ -259,7 +268,9 @@ def _execute_mnpflex_analysis(output_dir: Path) -> None: "classification-insight-card w-full min-w-0" ): with ui.column().classes("gap-2 p-3 md:p-4"): - ui.label("Subclass").classes("classification-insight-model") + ui.label("Subclass").classes( + "classification-insight-model" + ) agg_subclass_name_exp = ui.label("--").classes( "classification-insight-result text-sm" ) @@ -270,7 +281,9 @@ def _execute_mnpflex_analysis(output_dir: Path) -> None: "classification-insight-card w-full min-w-0" ): with ui.column().classes("gap-2 p-3 md:p-4"): - ui.label("Class").classes("classification-insight-model") + ui.label("Class").classes( + "classification-insight-model" + ) agg_class_name_exp = ui.label("--").classes( "classification-insight-result text-sm" ) @@ -281,7 +294,9 @@ def _execute_mnpflex_analysis(output_dir: Path) -> None: "classification-insight-card w-full min-w-0" ): with ui.column().classes("gap-2 p-3 md:p-4"): - ui.label("Family").classes("classification-insight-model") + ui.label("Family").classes( + "classification-insight-model" + ) agg_family_name_exp = ui.label("--").classes( "classification-insight-result text-sm" ) @@ -307,7 +322,11 @@ def _execute_mnpflex_analysis(output_dir: Path) -> None: _, classifier_scores_table = styled_table( columns=[ {"name": "score", "label": "Score", "field": "score"}, - {"name": "subclass", "label": "Subclass", "field": "subclass"}, + { + "name": "subclass", + "label": "Subclass", + "field": "subclass", + }, {"name": "class", "label": "Class", "field": "class"}, {"name": "family", "label": "Family", "field": "family"}, { @@ -322,9 +341,7 @@ def _execute_mnpflex_analysis(output_dir: Path) -> None: ) ui.separator().classes("my-4") - with ui.element("div").classes( - "classification-insight-grid--2 w-full" - ): + with ui.element("div").classes("classification-insight-grid--2 w-full"): with ui.element("div").classes( "classification-insight-card w-full min-w-0" ): @@ -348,7 +365,9 @@ def _execute_mnpflex_analysis(output_dir: Path) -> None: qc_missing = ui.label("Missing sites: --").classes( "classification-insight-meta" ) - with ui.expansion("QC plots", value=False).classes("w-full mt-3"): + with ui.expansion("QC plots", value=False).classes( + "w-full mt-3" + ): qc_plots_container = ui.row().classes("w-full gap-3 mt-2") with ui.element("div").classes( "classification-insight-card w-full min-w-0" @@ -370,14 +389,11 @@ def _execute_mnpflex_analysis(output_dir: Path) -> None: mgmt_sites = ui.label("MGMT sites: --").classes( "classification-insight-meta" ) - with ui.expansion("MGMT plot", value=False).classes("w-full mt-3"): + with ui.expansion("MGMT plot", value=False).classes( + "w-full mt-3" + ): mgmt_plot_container = ui.row().classes("w-full gap-3 mt-2") - - - - - def _format_score(value: Optional[float]) -> str: try: return f"{float(value):.4f}" @@ -423,7 +439,9 @@ def _set_badge_value(badge, value: Optional[float]) -> None: except Exception: badge.classes(classes) - def _extract_classifier_rows(classifier_summary: Dict[str, Any], limit: int = 10): + def _extract_classifier_rows( + classifier_summary: Dict[str, Any], limit: int = 10 + ): scores = classifier_summary.get("scores") or [] rows = [] for item in scores: @@ -481,9 +499,11 @@ def _render_hierarchy_nodes( description = (node.get("description") or "").strip() members = node.get("members") or [] depth_class = f"mnpflex-hierarchy-node--depth-{min(depth, 3)}" - with ui.element("div").classes( - f"mnpflex-hierarchy-node {depth_class} w-full min-w-0" - ).style(f"padding-left: {depth * 1.25}rem"): + with ( + ui.element("div") + .classes(f"mnpflex-hierarchy-node {depth_class} w-full min-w-0") + .style(f"padding-left: {depth * 1.25}rem") + ): with ui.row().classes( "mnpflex-hierarchy-node__row items-start w-full min-w-0" ): @@ -509,7 +529,9 @@ def _populate_hierarchy_tree(nodes: List[Dict[str, Any]]) -> None: with hierarchy_tree_container: _render_hierarchy_nodes(nodes) - def _update_labels(summary: Optional[Dict[str, Any]], summary_path: Optional[Path]) -> None: + def _update_labels( + summary: Optional[Dict[str, Any]], summary_path: Optional[Path] + ) -> None: has_results = summary is not None try: results_container.visible = has_results @@ -576,13 +598,9 @@ def _update_labels(summary: Optional[Dict[str, Any]], summary_path: Optional[Pat mgmt_status_value = mgmt.get("status", "Unknown") mgmt_status_badge.set_text(mgmt_status_value) mgmt_status_badge.classes(replace=_status_badge_classes(mgmt_status_value)) - mgmt_average.set_text( - f"MGMT average: {mgmt.get('average', 'Unknown')}" - ) + mgmt_average.set_text(f"MGMT average: {mgmt.get('average', 'Unknown')}") mgmt_sites.set_text(f"MGMT sites: {mgmt.get('site_count', 'Unknown')}") - classifier_name.set_text( - f"Classifier: {classifier.get('name', 'Unknown')}" - ) + classifier_name.set_text(f"Classifier: {classifier.get('name', 'Unknown')}") classifier_version.set_text( f"Version: {classifier.get('version', 'Unknown')}" ) @@ -600,7 +618,9 @@ def _update_labels(summary: Optional[Dict[str, Any]], summary_path: Optional[Pat disp_subclass = ( hierarchy_preds.get("molecular_subclass", {}).get("label") or "N/A" ) - subclass_sum = hierarchy_preds.get("molecular_subclass", {}).get("score") + subclass_sum = hierarchy_preds.get("molecular_subclass", {}).get( + "score" + ) disp_class = ( hierarchy_preds.get("molecular_class", {}).get("label") or "N/A" ) @@ -613,9 +633,9 @@ def _update_labels(summary: Optional[Dict[str, Any]], summary_path: Optional[Pat hierarchy_preds.get("molecular_superfamily", {}).get("label") or "N/A" ) - superfamily_sum = hierarchy_preds.get( - "molecular_superfamily", {} - ).get("score") + superfamily_sum = hierarchy_preds.get("molecular_superfamily", {}).get( + "score" + ) agg_subclass_name.set_text(disp_subclass) agg_class_name.set_text(disp_class) agg_family_name.set_text(disp_family) @@ -640,7 +660,9 @@ def _update_labels(summary: Optional[Dict[str, Any]], summary_path: Optional[Pat scores, "molecular_subclass", top_subclass ) class_sum = _sum_scores_by_field(scores, "molecular_class", top_class) - family_sum = _sum_scores_by_field(scores, "molecular_family", top_family) + family_sum = _sum_scores_by_field( + scores, "molecular_family", top_family + ) superfamily_sum = _sum_scores_by_field( scores, "molecular_superfamily", top_superfamily ) @@ -692,7 +714,9 @@ def _update_labels(summary: Optional[Dict[str, Any]], summary_path: Optional[Pat agg_class_name_exp.set_text(disp_class) agg_family_name_exp.set_text(disp_family) agg_superfamily_name_exp.set_text(disp_superfamily) - _set_badge_value(agg_subclass_badge_exp, subclass_sum if scores else None) + _set_badge_value( + agg_subclass_badge_exp, subclass_sum if scores else None + ) _set_badge_value(agg_class_badge_exp, class_sum if scores else None) _set_badge_value(agg_family_badge_exp, family_sum if scores else None) _set_badge_value( @@ -935,17 +959,13 @@ def _worker() -> None: f"Full BED file was not created at {full_bed_path}" ) if full_bed_path.stat().st_size == 0: - raise RuntimeError( - f"Full BED file is empty at {full_bed_path}" - ) + raise RuntimeError(f"Full BED file is empty at {full_bed_path}") if not subset_path.exists(): raise RuntimeError( f"Subset BED file was not created at {subset_path}" ) if subset_path.stat().st_size == 0: - raise RuntimeError( - f"Subset BED file is empty at {subset_path}" - ) + raise RuntimeError(f"Subset BED file is empty at {subset_path}") state["last_updated"] = time.time() outcome = { "ok": True, diff --git a/src/robin/gui/components/news_feed.py b/src/robin/gui/components/news_feed.py index a4ae5e5a..5ac68a97 100644 --- a/src/robin/gui/components/news_feed.py +++ b/src/robin/gui/components/news_feed.py @@ -6,11 +6,12 @@ import json import logging -import requests -from datetime import datetime, timedelta -from typing import Dict, Any, List, Optional from dataclasses import dataclass -from nicegui import ui, run +from datetime import datetime, timedelta +from typing import Any, Dict, List, Optional + +import requests +from nicegui import run, ui from robin.gui.theme import client_timer, stop_timer, ui_element_exists @@ -162,9 +163,7 @@ def create_news_element(self) -> None: ui.icon("update", color="gray").classes("text-sm") ui.label( f'Updated {self.last_update.strftime("%H:%M %d/%m/%Y")}' - ).classes( - "text-body-small text-slate-600 dark:text-slate-400" - ) + ).classes("text-body-small text-slate-600 dark:text-slate-400") # Scrollable news container with elegant styling with ui.scroll_area().classes("w-full h-96"): @@ -202,7 +201,9 @@ def _update_news_display(self) -> None: ui.icon("wifi_off", color="negative") ui.label( "News feed unavailable - please check your network connection" - ).classes("text-body-medium text-[color:var(--md-on-error-container)]") + ).classes( + "text-body-medium text-[color:var(--md-on-error-container)]" + ) return if not self.news_items: diff --git a/src/robin/gui/components/sample_audit.py b/src/robin/gui/components/sample_audit.py index e16aaa02..3d3622fb 100644 --- a/src/robin/gui/components/sample_audit.py +++ b/src/robin/gui/components/sample_audit.py @@ -15,7 +15,12 @@ from robin.gui_launcher import GUILauncher _AUDIT_COLUMNS = [ - {"name": "occurred_at", "label": "Time (UTC)", "field": "occurred_at", "align": "left"}, + { + "name": "occurred_at", + "label": "Time (UTC)", + "field": "occurred_at", + "align": "left", + }, {"name": "username", "label": "User", "field": "username", "align": "left"}, {"name": "event_type", "label": "Event", "field": "event_type", "align": "left"}, {"name": "target", "label": "Target", "field": "target", "align": "left"}, @@ -42,7 +47,9 @@ ] -def _audit_rows_for_sample(launcher: "GUILauncher", sample_id: str, *, limit: int = 500) -> List[Dict[str, Any]]: +def _audit_rows_for_sample( + launcher: "GUILauncher", sample_id: str, *, limit: int = 500 +) -> List[Dict[str, Any]]: events = launcher.security_store.query_audit_events( sample_id=sample_id, limit=limit, @@ -65,7 +72,9 @@ def _audit_rows_for_sample(launcher: "GUILauncher", sample_id: str, *, limit: in return rows -def _export_sample_audit_csv(launcher: "GUILauncher", sample_id: str, *, limit: int = 5000) -> bytes: +def _export_sample_audit_csv( + launcher: "GUILauncher", sample_id: str, *, limit: int = 5000 +) -> bytes: events = launcher.security_store.query_audit_events( sample_id=sample_id, limit=limit, @@ -91,8 +100,11 @@ def open_sample_audit_dialog(launcher: "GUILauncher", sample_id: str) -> None: """Show audit history for a single sample with CSV export.""" limit = 500 - with ui.dialog() as dialog, ui.card().classes( - "robin-dialog-surface p-4 md:p-5 w-full max-w-5xl min-w-[18rem]" + with ( + ui.dialog() as dialog, + ui.card().classes( + "robin-dialog-surface p-4 md:p-5 w-full max-w-5xl min-w-[18rem]" + ), ): ui.label(f"Audit history — {sample_id}").classes( "classification-insight-heading text-headline-small q-mb-sm" diff --git a/src/robin/gui/components/snp.py b/src/robin/gui/components/snp.py index 8c6c4deb..6af15bf2 100644 --- a/src/robin/gui/components/snp.py +++ b/src/robin/gui/components/snp.py @@ -1,11 +1,12 @@ from __future__ import annotations +import json +import logging import os import threading -from typing import Any, Callable, Dict, List, Optional from pathlib import Path -import logging -import json +from typing import Any, Callable, Dict, List, Optional + from robin.analysis.snp_processing import parse_vcf from robin.utils.clinvar_manager import compare_sample_clinvar_to_installed @@ -95,7 +96,7 @@ def _apply_variant_column_labels(columns: List[Dict[str, Any]]) -> None: def navigate_igv_to_snp(chrom: str, pos: int, flank: int = 100) -> None: """ Navigate IGV browser to a specific SNP location. - + Args: chrom: Chromosome name (e.g., "chr1") pos: Position on the chromosome @@ -105,16 +106,16 @@ def navigate_igv_to_snp(chrom: str, pos: int, flank: int = 100) -> None: # Ensure chromosome name has 'chr' prefix if needed if not chrom.startswith("chr"): chrom = f"chr{chrom}" - + # Calculate window around the SNP start = max(1, pos - flank) end = pos + flank - + region = f"{chrom}:{start}-{end}" - + # Escape region string for JavaScript escaped_region = region.replace('"', '\\"').replace("'", "\\'") - + js_navigate = f""" (function() {{ try {{ @@ -134,9 +135,9 @@ def navigate_igv_to_snp(chrom: str, pos: int, flank: int = 100) -> None: }} }})(); """ - + ui.run_javascript(js_navigate, timeout=5.0) - + except Exception as e: logger.error(f"Error navigating IGV to SNP {chrom}:{pos}: {e}") @@ -205,7 +206,9 @@ def _submit_snp_workflow_job( ) workflow_runner = getattr(launcher, "workflow_runner", None) - if workflow_runner is not None and hasattr(workflow_runner, "submit_snp_analysis_job"): + if workflow_runner is not None and hasattr( + workflow_runner, "submit_snp_analysis_job" + ): try: success = workflow_runner.submit_snp_analysis_job( sample_dir=str(sample_dir), @@ -256,7 +259,9 @@ def _set_status(text: str, tone: str = "meta") -> None: if not compact else "w-full gap-2 mb-2 flex-wrap items-center" ): - ui.label(f"Annotated with: {sample_label}").classes("classification-insight-meta") + ui.label(f"Annotated with: {sample_label}").classes( + "classification-insight-meta" + ) ui.label(f"Installed: {installed_label}").classes( "classification-insight-level classification-insight-level--low w-auto" if is_stale @@ -311,12 +316,12 @@ def add_snp_section(launcher: Any, sample_dir: Path) -> None: """ if not sample_dir or not sample_dir.exists(): return - + # Look for VCF files in clair3 directory clair3_dir = sample_dir / "clair3" if not clair3_dir.exists(): return - + # Look for snpsift_output.vcf (preferred) or other VCF files display_file = clair3_dir / "snpsift_output_display.json" @@ -325,7 +330,9 @@ def add_snp_section(launcher: Any, sample_dir: Path) -> None: ui.label("SNP analysis").classes( "classification-insight-heading text-headline-small" ) - with ui.element("div").classes("classification-insight-card w-full min-w-0"): + with ui.element("div").classes( + "classification-insight-card w-full min-w-0" + ): with ui.column().classes("w-full min-w-0 gap-2 p-2 md:p-3"): ui.label( "Precomputed SNP display data was not found. " @@ -342,11 +349,15 @@ def add_snp_section(launcher: Any, sample_dir: Path) -> None: ui.label("SNP analysis").classes( "classification-insight-heading text-headline-small" ) - with ui.element("div").classes("classification-insight-card w-full min-w-0"): + with ui.element("div").classes( + "classification-insight-card w-full min-w-0" + ): with ui.column().classes("w-full min-w-0 gap-2 p-2 md:p-3"): ui.label( "Could not load SNP variant data. Check logs for details." - ).classes("classification-insight-level classification-insight-level--low w-full") + ).classes( + "classification-insight-level classification-insight-level--low w-full" + ) return columns: List[Dict[str, Any]] = snp_display.get("columns", []) @@ -420,7 +431,9 @@ def _is_truthy(value: Any) -> bool: max_field_length = 80 column_lookup = { - col.get("field"): col for col in columns if isinstance(col, dict) and col.get("field") + col.get("field"): col + for col in columns + if isinstance(col, dict) and col.get("field") } visible_fields = [ @@ -428,7 +441,9 @@ def _is_truthy(value: Any) -> bool: ] if not visible_fields: visible_fields = [ - col.get("field") for col in columns[:12] if isinstance(col, dict) and col.get("field") + col.get("field") + for col in columns[:12] + if isinstance(col, dict) and col.get("field") ] display_columns = [column_lookup[field].copy() for field in visible_fields] @@ -514,7 +529,9 @@ def _update_snp_clinvar_status(text: str, tone: str = "meta") -> None: "(only the current page is sent to the browser)." ).classes("classification-insight-meta w-full") if significant_count > 0: - ui.label(f"ClinVar significant variants: {significant_count:,}").classes( + ui.label( + f"ClinVar significant variants: {significant_count:,}" + ).classes( "classification-insight-level classification-insight-level--low w-auto" ) elif pathogenic_count > 0: @@ -532,19 +549,27 @@ def _update_snp_clinvar_status(text: str, tone: str = "meta") -> None: with ui.row().classes("w-full gap-2 mb-2 flex-wrap items-end"): snp_pass_only = ui.checkbox("PASS only").props("dense") - snp_significant_only = ui.checkbox("ClinVar significant only").props("dense") - snp_min_qual = ui.number("Min QUAL", value=None).props( - "dense outlined clearable" - ).classes("w-32") + snp_significant_only = ui.checkbox("ClinVar significant only").props( + "dense" + ) + snp_min_qual = ( + ui.number("Min QUAL", value=None) + .props("dense outlined clearable") + .classes("w-32") + ) if snp_has_dp: - snp_min_dp = ui.number("Min DP", value=None).props( - "dense outlined clearable" - ).classes("w-32") + snp_min_dp = ( + ui.number("Min DP", value=None) + .props("dense outlined clearable") + .classes("w-32") + ) else: snp_min_dp = None - snp_search = ui.input("Search (gene/variant)").props( - "dense outlined clearable debounce=400" - ).classes("w-64") + snp_search = ( + ui.input("Search (gene/variant)") + .props("dense outlined clearable debounce=400") + .classes("w-64") + ) snp_reset_button = ui.button("Reset").props("dense no-caps") _snp_total_filtered = len(page_state["filtered_indices"]) @@ -590,9 +615,7 @@ def _fill_snp_from_pagination(pag: Dict[str, Any]) -> None: rows_out[-1]["__row_idx"] = idx snp_table.rows = rows_out snp_table.pagination = pag - snp_filtered_count_label.text = ( - f"{total_filtered:,} variants match filters (of {snp_total_rows:,} total)" - ) + snp_filtered_count_label.text = f"{total_filtered:,} variants match filters (of {snp_total_rows:,} total)" snp_table.update() wire_qtable_server_pagination_handlers(snp_table, _fill_snp_from_pagination) @@ -601,15 +624,22 @@ def _apply_snp_filters() -> None: pass_only = bool(getattr(snp_pass_only, "value", False)) significant_only = bool(getattr(snp_significant_only, "value", False)) min_qual = _to_float(getattr(snp_min_qual, "value", None)) - min_dp = _to_float(getattr(snp_min_dp, "value", None)) if snp_has_dp else None + min_dp = ( + _to_float(getattr(snp_min_dp, "value", None)) if snp_has_dp else None + ) search_text = str(getattr(snp_search, "value", "") or "").strip().lower() filtered_indices: List[int] = [] for idx, full_row in enumerate(snp_rows_source): - if pass_only and str(full_row.get("FILTER", "")).strip().upper() != "PASS": + if ( + pass_only + and str(full_row.get("FILTER", "")).strip().upper() != "PASS" + ): continue if significant_only and not _is_truthy( - full_row.get("is_clinvar_significant", full_row.get("is_pathogenic", "")) + full_row.get( + "is_clinvar_significant", full_row.get("is_pathogenic", "") + ) ): continue @@ -662,9 +692,12 @@ def _apply_snp_filters() -> None: if any(col.get("field") in {"action", "details"} for col in display_columns): try: - with ui.dialog() as details_dialog, ui.card().classes( - "robin-dialog-surface w-[95vw] max-w-6xl max-h-[85vh] overflow-auto " - "p-4 md:p-5" + with ( + ui.dialog() as details_dialog, + ui.card().classes( + "robin-dialog-surface w-[95vw] max-w-6xl max-h-[85vh] overflow-auto " + "p-4 md:p-5" + ), ): ui.label("Variant details").classes( "classification-insight-heading text-headline-small" @@ -691,7 +724,8 @@ def show_variant_details(row_idx: int) -> None: [ k for k in row_data.keys() - if k not in VARIANT_DETAIL_FIELDS and k not in ui_only_fields + if k not in VARIANT_DETAIL_FIELDS + and k not in ui_only_fields ] ) with details_container: @@ -699,7 +733,9 @@ def show_variant_details(row_idx: int) -> None: value = row_data.get(field, "") if value is None or str(value) == "": continue - label = VARIANT_COLUMN_LABELS.get(field, field.replace("_", " ")) + label = VARIANT_COLUMN_LABELS.get( + field, field.replace("_", " ") + ) with ui.row().classes("w-full items-start gap-2"): ui.label(f"{label}:").classes( "text-xs font-semibold min-w-[180px]" @@ -779,9 +815,11 @@ def on_snp_show_details(e): col["sortable"] = False _fill_snp_from_pagination(snp_init_pagination) try: + def _cleanup_snp_page() -> None: page_state["filtered_indices"] = [] snp_table.rows = [] + ui.context.client.on_disconnect(_cleanup_snp_page) except Exception: pass @@ -804,9 +842,9 @@ def _cleanup_snp_page() -> None: indel_df = parse_vcf(indel_vcf) if indel_df is None: - ui.label( - "Could not parse INDEL VCF data. Check logs for details." - ).classes("classification-insight-level classification-insight-level--low w-full") + ui.label("Could not parse INDEL VCF data. Check logs for details.").classes( + "classification-insight-level classification-insight-level--low w-full" + ) return if indel_df.empty: @@ -868,13 +906,15 @@ def _indel_row_text_map(idx: int) -> Dict[str, str]: # Normalize boolean display consistency for filtering and details. row_data["is_clinvar_significant"] = ( "Yes" - if _is_truthy(row_data.get("is_clinvar_significant", row_data.get("is_pathogenic", ""))) + if _is_truthy( + row_data.get( + "is_clinvar_significant", row_data.get("is_pathogenic", "") + ) + ) else "No" ) row_data["is_pathogenic"] = ( - "Yes" - if _is_truthy(row_data.get("is_pathogenic", "")) - else "No" + "Yes" if _is_truthy(row_data.get("is_pathogenic", "")) else "No" ) return row_data @@ -939,9 +979,7 @@ def _compact_indel_row(idx: int) -> Dict[str, Any]: "classification-insight-level classification-insight-level--low w-auto" ) elif pathogenic_indel_count: - ui.label( - f"Pathogenic variants: {pathogenic_indel_count:,}" - ).classes( + ui.label(f"Pathogenic variants: {pathogenic_indel_count:,}").classes( "classification-insight-level classification-insight-level--low w-auto" ) @@ -955,19 +993,27 @@ def _compact_indel_row(idx: int) -> Dict[str, Any]: with ui.row().classes("w-full gap-2 mb-2 flex-wrap items-end"): indel_pass_only = ui.checkbox("PASS only").props("dense") - indel_significant_only = ui.checkbox("ClinVar significant only").props("dense") - indel_min_qual = ui.number("Min QUAL", value=None).props( - "dense outlined clearable" - ).classes("w-32") + indel_significant_only = ui.checkbox("ClinVar significant only").props( + "dense" + ) + indel_min_qual = ( + ui.number("Min QUAL", value=None) + .props("dense outlined clearable") + .classes("w-32") + ) if indel_has_dp: - indel_min_dp = ui.number("Min DP", value=None).props( - "dense outlined clearable" - ).classes("w-32") + indel_min_dp = ( + ui.number("Min DP", value=None) + .props("dense outlined clearable") + .classes("w-32") + ) else: indel_min_dp = None - indel_search = ui.input("Search (gene/variant)").props( - "dense outlined clearable debounce=400" - ).classes("w-64") + indel_search = ( + ui.input("Search (gene/variant)") + .props("dense outlined clearable debounce=400") + .classes("w-64") + ) indel_reset_button = ui.button("Reset").props("dense no-caps") indel_page_state: Dict[str, Any] = { @@ -1012,9 +1058,7 @@ def _fill_indel_from_pagination(pag: Dict[str, Any]) -> None: _compact_indel_row(idx) for idx in filtered_indices[start:end] ] indel_table.pagination = pag - indel_filtered_count_label.text = ( - f"{total_filtered:,} variants match filters (of {total_indel_rows:,} total)" - ) + indel_filtered_count_label.text = f"{total_filtered:,} variants match filters (of {total_indel_rows:,} total)" indel_table.update() wire_qtable_server_pagination_handlers(indel_table, _fill_indel_from_pagination) @@ -1024,7 +1068,9 @@ def _apply_indel_filters() -> None: significant_only = bool(getattr(indel_significant_only, "value", False)) min_qual = _to_float(getattr(indel_min_qual, "value", None)) min_dp = ( - _to_float(getattr(indel_min_dp, "value", None)) if indel_has_dp else None + _to_float(getattr(indel_min_dp, "value", None)) + if indel_has_dp + else None ) search_text = str(getattr(indel_search, "value", "") or "").strip().lower() @@ -1032,10 +1078,15 @@ def _apply_indel_filters() -> None: for idx in range(total_indel_rows): full_row = _indel_row_text_map(idx) - if pass_only and str(full_row.get("FILTER", "")).strip().upper() != "PASS": + if ( + pass_only + and str(full_row.get("FILTER", "")).strip().upper() != "PASS" + ): continue if significant_only and not _is_truthy( - full_row.get("is_clinvar_significant", full_row.get("is_pathogenic", "")) + full_row.get( + "is_clinvar_significant", full_row.get("is_pathogenic", "") + ) ): continue @@ -1070,7 +1121,9 @@ def _apply_indel_filters() -> None: _fill_indel_from_pagination(pag) indel_pass_only.on("update:model-value", lambda _e: _apply_indel_filters()) - indel_significant_only.on("update:model-value", lambda _e: _apply_indel_filters()) + indel_significant_only.on( + "update:model-value", lambda _e: _apply_indel_filters() + ) indel_min_qual.on("update:model-value", lambda _e: _apply_indel_filters()) if indel_has_dp and indel_min_dp is not None: indel_min_dp.on("update:model-value", lambda _e: _apply_indel_filters()) @@ -1086,9 +1139,12 @@ def _apply_indel_filters() -> None: ) ) - with ui.dialog() as indel_details_dialog, ui.card().classes( - "robin-dialog-surface w-[95vw] max-w-6xl max-h-[85vh] overflow-auto " - "p-4 md:p-5" + with ( + ui.dialog() as indel_details_dialog, + ui.card().classes( + "robin-dialog-surface w-[95vw] max-w-6xl max-h-[85vh] overflow-auto " + "p-4 md:p-5" + ), ): ui.label("INDEL details").classes( "classification-insight-heading text-headline-small" @@ -1121,8 +1177,12 @@ def show_indel_details(row_idx: int) -> None: continue label = VARIANT_COLUMN_LABELS.get(field, field.replace("_", " ")) with ui.row().classes("w-full items-start gap-2"): - ui.label(f"{label}:").classes("text-xs font-semibold min-w-[180px]") - ui.label(str(value)).classes("text-xs whitespace-pre-wrap break-all flex-1") + ui.label(f"{label}:").classes( + "text-xs font-semibold min-w-[180px]" + ) + ui.label(str(value)).classes( + "text-xs whitespace-pre-wrap break-all flex-1" + ) indel_details_dialog.open() indel_table.add_slot( @@ -1200,9 +1260,11 @@ def on_indel_show_details(e): col["sortable"] = False _fill_indel_from_pagination(indel_init_pagination) try: + def _cleanup_indel_page() -> None: indel_page_state["filtered_indices"] = [] indel_table.rows = [] + ui.context.client.on_disconnect(_cleanup_indel_page) except Exception: pass diff --git a/src/robin/gui/components/summary.py b/src/robin/gui/components/summary.py index a411d17c..9b69f01a 100644 --- a/src/robin/gui/components/summary.py +++ b/src/robin/gui/components/summary.py @@ -1,43 +1,44 @@ from __future__ import annotations -from pathlib import Path -from typing import Any, Dict, List, Optional import asyncio -import threading -import json import csv -import time -from datetime import datetime import hashlib +import json import logging +import threading +import time from collections import OrderedDict +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, List, Optional try: - from nicegui import ui, background_tasks + from nicegui import background_tasks, ui except ImportError: # pragma: no cover ui = None background_tasks = None -from robin.classification_config import get_confidence_ui_tier -from robin.analysis.cnv_classification import ( - detect_cnv_events_for_sample, - format_cnv_events_card_lines, -) from robin.analysis.bam_preprocessor import ( _get_modbase_model_warning, _get_modbase_model_warning_level, _is_unresolved_modbase_model, ) +from robin.analysis.cnv_classification import ( + detect_cnv_events_for_sample, + format_cnv_events_card_lines, +) +from robin.classification_config import get_confidence_ui_tier from robin.gui.config import ( - get_confidence_level as get_classifier_confidence_level, - is_section_visible, - get_visible_classification_steps, + CLASSIFICATION_STEPS, any_classification_visible, +) +from robin.gui.config import get_confidence_level as get_classifier_confidence_level +from robin.gui.config import ( + get_visible_classification_steps, + is_section_visible, launcher_visibility_context, - CLASSIFICATION_STEPS, ) - _SUMMARY_CACHE: "OrderedDict[str, Dict[str, Any]]" = OrderedDict() _SUMMARY_CACHE_LOCK = threading.Lock() _SUMMARY_CACHE_MAX_SAMPLES = 64 @@ -49,7 +50,8 @@ def _evict_summary_cache_locked(now_ts: Optional[float] = None) -> None: stale_keys = [ key for key, payload in _SUMMARY_CACHE.items() - if (now_ts - float(payload.get("_cached_at", 0.0))) > _SUMMARY_CACHE_MAX_AGE_SECONDS + if (now_ts - float(payload.get("_cached_at", 0.0))) + > _SUMMARY_CACHE_MAX_AGE_SECONDS ] for key in stale_keys: _SUMMARY_CACHE.pop(key, None) @@ -64,7 +66,9 @@ def _get_summary_cache(sample_dir: Path) -> Dict[str, Any]: if not payload: return {} now_ts = time.time() - if (now_ts - float(payload.get("_cached_at", 0.0))) > _SUMMARY_CACHE_MAX_AGE_SECONDS: + if ( + now_ts - float(payload.get("_cached_at", 0.0)) + ) > _SUMMARY_CACHE_MAX_AGE_SECONDS: _SUMMARY_CACHE.pop(key, None) return {} _SUMMARY_CACHE.move_to_end(key) @@ -134,9 +138,7 @@ def _run_info_section(sample_dir: Path, sample_id: str): model = run_info.get("model", "Missing") modbase_model = run_info.get("modbase_model", "Missing") modbase_display = ( - "Unknown" - if _is_unresolved_modbase_model(modbase_model) - else modbase_model + "Unknown" if _is_unresolved_modbase_model(modbase_model) else modbase_model ) device = run_info.get("device", "Not available") flow = run_info.get("flow_cell", "Not available") @@ -185,9 +187,7 @@ def _run_info_section(sample_dir: Path, sample_id: str): with ui.element("div").classes("run-summary-grid"): for icn, lab, val, span in cells: ht = hints.get(lab, "") - _run_summary_cell( - icn, lab, val, col_class=span, hint=ht - ) + _run_summary_cell(icn, lab, val, col_class=span, hint=ht) modbase_warning = _get_modbase_model_warning( None if modbase_model == "Missing" else modbase_model ) @@ -269,16 +269,18 @@ def _classification_section(sample_dir: Path, launcher: Any = None): }, }, ) - + # Get workflow steps from launcher if available workflow_steps, display_config, viewer_role = launcher_visibility_context(launcher) enabled_classification_steps = get_visible_classification_steps( workflow_steps, display_config, viewer_role=viewer_role ) - - if not any_classification_visible(workflow_steps, display_config, viewer_role=viewer_role): + + if not any_classification_visible( + workflow_steps, display_config, viewer_role=viewer_role + ): return - + with ui.element("div").classes("classification-insight-shell w-full min-w-0"): ui.label("Classification details").classes( "classification-insight-heading text-headline-small" @@ -390,18 +392,25 @@ def _analysis_section(sample_dir: Path, launcher: Any = None): workflow_steps, display_config, viewer_role = launcher_visibility_context(launcher) cache = _get_summary_cache(sample_dir) analysis_data = cache.get("analysis_data", {}) - _vis = lambda sid: is_section_visible(sid, workflow_steps=workflow_steps, display_config=display_config, viewer_role=viewer_role) + _vis = lambda sid: is_section_visible( + sid, + workflow_steps=workflow_steps, + display_config=display_config, + viewer_role=viewer_role, + ) coverage_data = analysis_data.get("coverage", {}) if _vis("target") else {} cnv_data = analysis_data.get("cnv", {}) if _vis("cnv") else {} mgmt_data = analysis_data.get("mgmt", {}) if _vis("mgmt") else {} fusion_data = analysis_data.get("fusion", {}) if _vis("fusion") else {} - + should_show_target = _vis("target") should_show_cnv = _vis("cnv") should_show_mgmt = _vis("mgmt") should_show_fusion = _vis("fusion") - - if not any([should_show_target, should_show_cnv, should_show_mgmt, should_show_fusion]): + + if not any( + [should_show_target, should_show_cnv, should_show_mgmt, should_show_fusion] + ): return with ui.element("div").classes("classification-insight-shell w-full min-w-0"): @@ -505,6 +514,7 @@ def _schedule_refresh() -> None: 30.0, _refresh_summary_cache_async, active=True, immediate=False ) try: + def _on_disconnect_cleanup() -> None: stop_timer(refresh_timer) _clear_summary_cache(sample_dir) @@ -527,13 +537,33 @@ def _refresh_summary_cache_sync( # Analysis data (only compute enabled sections) analysis_data: Dict[str, Any] = {} - if is_section_visible("target", workflow_steps=workflow_steps, display_config=display_config, viewer_role=viewer_role): + if is_section_visible( + "target", + workflow_steps=workflow_steps, + display_config=display_config, + viewer_role=viewer_role, + ): analysis_data["coverage"] = _extract_coverage_data(sample_dir) - if is_section_visible("cnv", workflow_steps=workflow_steps, display_config=display_config, viewer_role=viewer_role): + if is_section_visible( + "cnv", + workflow_steps=workflow_steps, + display_config=display_config, + viewer_role=viewer_role, + ): analysis_data["cnv"] = _extract_cnv_data(sample_dir) - if is_section_visible("mgmt", workflow_steps=workflow_steps, display_config=display_config, viewer_role=viewer_role): + if is_section_visible( + "mgmt", + workflow_steps=workflow_steps, + display_config=display_config, + viewer_role=viewer_role, + ): analysis_data["mgmt"] = _extract_mgmt_data(sample_dir) - if is_section_visible("fusion", workflow_steps=workflow_steps, display_config=display_config, viewer_role=viewer_role): + if is_section_visible( + "fusion", + workflow_steps=workflow_steps, + display_config=display_config, + viewer_role=viewer_role, + ): analysis_data["fusion"] = _extract_fusion_data(sample_dir) data["analysis_data"] = analysis_data @@ -620,36 +650,50 @@ def _on_card_click() -> None: ui.label(description).classes("classification-insight-foot") -def _create_classification_dashboard_card(title: str, classification: str, icon: str, description: str) -> Dict[str, Any]: +def _create_classification_dashboard_card( + title: str, classification: str, icon: str, description: str +) -> Dict[str, Any]: """Create a compact classification dashboard card with detailed information. Returns labels for updating.""" - with ui.card().classes("flex-1 bg-white rounded-lg shadow-sm border border-gray-200 p-4"): + with ui.card().classes( + "flex-1 bg-white rounded-lg shadow-sm border border-gray-200 p-4" + ): with ui.row().classes("flex items-center justify-between mb-2"): # Title ui.label(title).classes("text-sm font-medium text-gray-600") - + # Icon in circular background - with ui.row().classes("w-7 h-7 bg-blue-100 rounded-full flex items-center justify-center"): + with ui.row().classes( + "w-7 h-7 bg-blue-100 rounded-full flex items-center justify-center" + ): ui.icon(icon).classes("w-3.5 h-3.5 text-blue-600") - + # Main classification result with confidence badge with ui.row().classes("flex items-center justify-between mb-1"): - classification_label = ui.label(classification).classes("text-xl font-bold text-gray-900") + classification_label = ui.label(classification).classes( + "text-xl font-bold text-gray-900" + ) # Confidence level badge - confidence_badge = ui.label("Loading...").classes("px-2 py-1 text-xs font-medium rounded-full bg-gray-100 text-gray-600") - + confidence_badge = ui.label("Loading...").classes( + "px-2 py-1 text-xs font-medium rounded-full bg-gray-100 text-gray-600" + ) + # Compact details in a single row with ui.row().classes("flex items-center justify-between mb-1"): - confidence_label = ui.label("Confidence: Loading...").classes("text-xs font-medium text-gray-700") - features_label = ui.label("Features: Loading...").classes("text-xs text-gray-500") - + confidence_label = ui.label("Confidence: Loading...").classes( + "text-xs font-medium text-gray-700" + ) + features_label = ui.label("Features: Loading...").classes( + "text-xs text-gray-500" + ) + # Description ui.label(description).classes("text-xs text-gray-500") - + return { "classification": classification_label, "confidence": confidence_label, "confidence_badge": confidence_badge, - "features": features_label + "features": features_label, } @@ -664,17 +708,27 @@ def _create_classification_card( ) -> Dict[str, Any]: """Create a classification summary card. Returns labels for updating.""" labels = {} - with ui.card().classes("flex-1 elevation-4 rounded-xl bg-gradient-to-br from-blue-50 to-indigo-50 border-l-4 border-blue-500"): + with ui.card().classes( + "flex-1 elevation-4 rounded-xl bg-gradient-to-br from-blue-50 to-indigo-50 border-l-4 border-blue-500" + ): ui.label(f"{title}").classes("font-bold text-blue-800 mb-2") ui.separator().classes().style("border: 1px solid var(--md-primary)") - labels["classification"] = ui.label(f"Class: {classification}").classes("font-bold text-medium text-blue-600") + labels["classification"] = ui.label(f"Class: {classification}").classes( + "font-bold text-medium text-blue-600" + ) confidence_color = _get_confidence_color(confidence_level) - labels["confidence"] = ui.label(f"Confidence: {confidence}%").classes(f"text-sm text-{confidence_color}-600") - labels["confidence_level"] = ui.label(confidence_level).classes(f"text-sm text-{confidence_color}-600") + labels["confidence"] = ui.label(f"Confidence: {confidence}%").classes( + f"text-sm text-{confidence_color}-600" + ) + labels["confidence_level"] = ui.label(confidence_level).classes( + f"text-sm text-{confidence_color}-600" + ) if model: ui.label(f"Model: {model}").classes("text-sm text-blue-600") if features and features_label: - labels["features"] = ui.label(f"Features: {features_label}").classes("text-sm text-blue-600") + labels["features"] = ui.label(f"Features: {features_label}").classes( + "text-sm text-blue-600" + ) return labels @@ -749,11 +803,15 @@ def _on_click() -> None: ui.label("≥30x").classes( "analysis-insight-pill analysis-insight-pill--emerald" ) - ui.label("≥20x").classes("analysis-insight-pill analysis-insight-pill--sky") + ui.label("≥20x").classes( + "analysis-insight-pill analysis-insight-pill--sky" + ) ui.label("≥10x").classes( "analysis-insight-pill analysis-insight-pill--amber" ) - ui.label("<10x").classes("analysis-insight-pill analysis-insight-pill--rose") + ui.label("<10x").classes( + "analysis-insight-pill analysis-insight-pill--rose" + ) ui.label("Target panel coverage quality and depth").classes( "classification-insight-foot" ) @@ -761,40 +819,62 @@ def _on_click() -> None: def _create_coverage_dashboard_card() -> Dict[str, Any]: """Create a compact coverage analysis dashboard card. Returns labels for updating.""" - with ui.card().classes("flex-1 bg-white rounded-lg shadow-sm border border-gray-200 p-4"): + with ui.card().classes( + "flex-1 bg-white rounded-lg shadow-sm border border-gray-200 p-4" + ): with ui.row().classes("flex items-center justify-between mb-2"): # Title ui.label("Coverage Analysis").classes("text-sm font-medium text-gray-600") - + # Icon in circular background - with ui.row().classes("w-7 h-7 bg-blue-100 rounded-full flex items-center justify-center"): + with ui.row().classes( + "w-7 h-7 bg-blue-100 rounded-full flex items-center justify-center" + ): ui.icon("analytics").classes("w-3.5 h-3.5 text-blue-600") - + # Main quality result with badge with ui.row().classes("flex items-center justify-between mb-1"): - quality_label = ui.label("Loading...").classes("text-xl font-bold text-gray-900") + quality_label = ui.label("Loading...").classes( + "text-xl font-bold text-gray-900" + ) # Coverage badge - coverage_badge = ui.label("--x").classes("px-2 py-1 text-xs font-medium rounded-full bg-gray-100 text-gray-600") - + coverage_badge = ui.label("--x").classes( + "px-2 py-1 text-xs font-medium rounded-full bg-gray-100 text-gray-600" + ) + # Coverage details in compact layout with ui.column().classes("mb-2"): - global_coverage_label = ui.label("Global: Loading...").classes("text-xs text-gray-600") - target_coverage_label = ui.label("Targets: Loading...").classes("text-xs text-gray-600") - enrichment_label = ui.label("Enrichment: Loading...").classes("text-xs text-gray-600") - + global_coverage_label = ui.label("Global: Loading...").classes( + "text-xs text-gray-600" + ) + target_coverage_label = ui.label("Targets: Loading...").classes( + "text-xs text-gray-600" + ) + enrichment_label = ui.label("Enrichment: Loading...").classes( + "text-xs text-gray-600" + ) + # Coverage thresholds as small badges with ui.row().classes("gap-1 flex-wrap"): - ui.label("≥30x").classes("px-1 py-0.5 text-xs bg-green-100 text-green-800 rounded") - ui.label("≥20x").classes("px-1 py-0.5 text-xs bg-blue-100 text-blue-800 rounded") - ui.label("≥10x").classes("px-1 py-0.5 text-xs bg-yellow-100 text-yellow-800 rounded") - ui.label("<10x").classes("px-1 py-0.5 text-xs bg-red-100 text-red-800 rounded") - + ui.label("≥30x").classes( + "px-1 py-0.5 text-xs bg-green-100 text-green-800 rounded" + ) + ui.label("≥20x").classes( + "px-1 py-0.5 text-xs bg-blue-100 text-blue-800 rounded" + ) + ui.label("≥10x").classes( + "px-1 py-0.5 text-xs bg-yellow-100 text-yellow-800 rounded" + ) + ui.label("<10x").classes( + "px-1 py-0.5 text-xs bg-red-100 text-red-800 rounded" + ) + return { "quality": quality_label, "coverage_badge": coverage_badge, "global_coverage": global_coverage_label, "target_coverage": target_coverage_label, - "enrichment": enrichment_label + "enrichment": enrichment_label, } @@ -809,6 +889,7 @@ def _create_cnv_dashboard_card_with_data( anchor_key: str = "cnv", ) -> None: """CNV insight card — design.md §9.""" + def _on_click() -> None: _scroll_to_analysis_detail(anchor_key) @@ -839,7 +920,9 @@ def _on_click() -> None: "classification-insight-meta w-full truncate" ).props(f'title="{_arm}"') with ui.column().classes("w-full gap-1"): - ui.label(f"Bin width: {bin_width}").classes("classification-insight-meta") + ui.label(f"Bin width: {bin_width}").classes( + "classification-insight-meta" + ) ui.label(f"Variance: {variance}").classes("classification-insight-meta") with ui.row().classes("gap-1 flex-wrap"): ui.label(f"Whole chr: {whole_chromosome_count}").classes( @@ -848,44 +931,62 @@ def _on_click() -> None: ui.label(f"Arm: {arm_count}").classes( "analysis-insight-pill analysis-insight-pill--sky" ) - ui.label( - "Copy number across the genome with breakpoint detection" - ).classes("classification-insight-foot") + ui.label("Copy number across the genome with breakpoint detection").classes( + "classification-insight-foot" + ) def _create_cnv_dashboard_card() -> Dict[str, Any]: """Create a compact CNV analysis dashboard card. Returns labels for updating.""" - with ui.card().classes("flex-1 bg-white rounded-lg shadow-sm border border-gray-200 p-4"): + with ui.card().classes( + "flex-1 bg-white rounded-lg shadow-sm border border-gray-200 p-4" + ): with ui.row().classes("flex items-center justify-between mb-2"): # Title - ui.label("Copy Number Analysis").classes("text-sm font-medium text-gray-600") - + ui.label("Copy Number Analysis").classes( + "text-sm font-medium text-gray-600" + ) + # Icon in circular background - with ui.row().classes("w-7 h-7 bg-purple-100 rounded-full flex items-center justify-center"): + with ui.row().classes( + "w-7 h-7 bg-purple-100 rounded-full flex items-center justify-center" + ): ui.icon("person").classes("w-3.5 h-3.5 text-purple-600") - + # Main genetic sex result - genetic_sex_label = ui.label("Loading...").classes("text-xl font-bold text-gray-900 mb-1") - + genetic_sex_label = ui.label("Loading...").classes( + "text-xl font-bold text-gray-900 mb-1" + ) + # Analysis details in compact layout with ui.column().classes("mb-2"): - bin_width_label = ui.label("Bin Width: Loading...").classes("text-xs text-gray-600") - variance_label = ui.label("Variance: Loading...").classes("text-xs text-gray-600") - + bin_width_label = ui.label("Bin Width: Loading...").classes( + "text-xs text-gray-600" + ) + variance_label = ui.label("Variance: Loading...").classes( + "text-xs text-gray-600" + ) + # CNV counts as badges with ui.row().classes("gap-2 mb-1"): - gained_badge = ui.label("Gained: --").classes("px-2 py-1 text-xs font-medium rounded-full bg-green-100 text-green-800") - lost_badge = ui.label("Lost: --").classes("px-2 py-1 text-xs font-medium rounded-full bg-red-100 text-red-800") - + gained_badge = ui.label("Gained: --").classes( + "px-2 py-1 text-xs font-medium rounded-full bg-green-100 text-green-800" + ) + lost_badge = ui.label("Lost: --").classes( + "px-2 py-1 text-xs font-medium rounded-full bg-red-100 text-red-800" + ) + # Description - ui.label("Copy number analysis across genome with breakpoint detection").classes("text-xs text-gray-500") - + ui.label( + "Copy number analysis across genome with breakpoint detection" + ).classes("text-xs text-gray-500") + return { "genetic_sex": genetic_sex_label, "bin_width": bin_width_label, "variance": variance_label, "gained": gained_badge, - "lost": lost_badge + "lost": lost_badge, } @@ -948,42 +1049,56 @@ def _on_click() -> None: ui.label(f"Score: {prediction_score}").classes( "classification-insight-meta" ) - ui.label( - f"Status from methylation at {cpg_sites} CpG sites" - ).classes("classification-insight-foot") + ui.label(f"Status from methylation at {cpg_sites} CpG sites").classes( + "classification-insight-foot" + ) def _create_mgmt_dashboard_card() -> Dict[str, Any]: """Create a compact MGMT analysis dashboard card. Returns labels for updating.""" - with ui.card().classes("flex-1 bg-white rounded-lg shadow-sm border border-gray-200 p-4"): + with ui.card().classes( + "flex-1 bg-white rounded-lg shadow-sm border border-gray-200 p-4" + ): with ui.row().classes("flex items-center justify-between mb-2"): # Title ui.label("MGMT Analysis").classes("text-sm font-medium text-gray-600") - + # Icon in circular background - with ui.row().classes("w-7 h-7 bg-orange-100 rounded-full flex items-center justify-center"): + with ui.row().classes( + "w-7 h-7 bg-orange-100 rounded-full flex items-center justify-center" + ): ui.icon("science").classes("w-3.5 h-3.5 text-orange-600") - + # Main status result with badge with ui.row().classes("flex items-center justify-between mb-1"): - status_label = ui.label("Loading...").classes("text-xl font-bold text-gray-900") + status_label = ui.label("Loading...").classes( + "text-xl font-bold text-gray-900" + ) # Methylation badge - methylation_badge = ui.label("--%").classes("px-2 py-1 text-xs font-medium rounded-full bg-gray-100 text-gray-600") - + methylation_badge = ui.label("--%").classes( + "px-2 py-1 text-xs font-medium rounded-full bg-gray-100 text-gray-600" + ) + # Analysis details in compact layout with ui.column().classes("mb-2"): - average_methylation_label = ui.label("Average: Loading...").classes("text-xs text-gray-600") - prediction_score_label = ui.label("Score: Loading...").classes("text-xs text-gray-600") - + average_methylation_label = ui.label("Average: Loading...").classes( + "text-xs text-gray-600" + ) + prediction_score_label = ui.label("Score: Loading...").classes( + "text-xs text-gray-600" + ) + # Description - cpg_sites_label = ui.label("MGMT status determined from methylation analysis of -- CpG sites").classes("text-xs text-gray-500") - + cpg_sites_label = ui.label( + "MGMT status determined from methylation analysis of -- CpG sites" + ).classes("text-xs text-gray-500") + return { "status": status_label, "methylation_badge": methylation_badge, "average_methylation": average_methylation_label, "prediction_score": prediction_score_label, - "cpg_sites": cpg_sites_label + "cpg_sites": cpg_sites_label, } @@ -1031,53 +1146,59 @@ def _on_click() -> None: ui.label(main_line).classes("classification-insight-result w-full").props( f'title="{_panel}"' ) - with ui.row().classes( - "w-full justify-between items-start gap-2 flex-wrap" - ): + with ui.row().classes("w-full justify-between items-start gap-2 flex-wrap"): ui.label(target_badge).classes( "analysis-insight-pill analysis-insight-pill--sky" ) with ui.column().classes("w-full gap-1"): ui.label(genome_line).classes("classification-insight-meta") - ui.label( - "Candidates from reads with supplementary alignments" - ).classes("classification-insight-foot") + ui.label("Candidates from reads with supplementary alignments").classes( + "classification-insight-foot" + ) def _create_fusion_dashboard_card() -> Dict[str, Any]: """Create a compact fusion analysis dashboard card. Returns labels for updating.""" - with ui.card().classes("flex-1 bg-white rounded-lg shadow-sm border border-gray-200 p-4"): + with ui.card().classes( + "flex-1 bg-white rounded-lg shadow-sm border border-gray-200 p-4" + ): with ui.row().classes("flex items-center justify-between mb-2"): # Title ui.label("Fusion Analysis").classes("text-sm font-medium text-gray-600") - + # Icon in circular background - with ui.row().classes("w-7 h-7 bg-green-100 rounded-full flex items-center justify-center"): + with ui.row().classes( + "w-7 h-7 bg-green-100 rounded-full flex items-center justify-center" + ): ui.icon("merge").classes("w-3.5 h-3.5 text-green-600") - + # Panel info and main result with ui.row().classes("flex items-center justify-between mb-1"): - panel_label = ui.label("Panel: --").classes("text-sm font-medium text-gray-700") - target_fusions_badge = ui.label("-- target fusions").classes("px-2 py-1 text-xs font-medium rounded-full bg-blue-100 text-blue-800") - + panel_label = ui.label("Panel: --").classes( + "text-sm font-medium text-gray-700" + ) + target_fusions_badge = ui.label("-- target fusions").classes( + "px-2 py-1 text-xs font-medium rounded-full bg-blue-100 text-blue-800" + ) + # Analysis details in compact layout with ui.column().classes("mb-2"): - genome_fusions_label = ui.label("-- genome wide fusions").classes("text-xs text-gray-600") - + genome_fusions_label = ui.label("-- genome wide fusions").classes( + "text-xs text-gray-600" + ) + # Description - ui.label("Fusion candidates identified from reads with supplementary alignments").classes("text-xs text-gray-500") - + ui.label( + "Fusion candidates identified from reads with supplementary alignments" + ).classes("text-xs text-gray-500") + return { "panel": panel_label, "target_fusions": target_fusions_badge, - "genome_fusions": genome_fusions_label + "genome_fusions": genome_fusions_label, } - - - - def _fmt_master_csv_integer(v: Optional[str]) -> Optional[str]: """Format integer counters from master.csv without float precision loss on large values.""" if v is None or not str(v).strip(): @@ -1159,9 +1280,7 @@ def get_ci(r: Dict[str, str], key: str) -> Optional[str]: if device_val and str(device_val).strip(): run_info["device"] = str(device_val).strip() - flowcell_val = get_ci(row, "run_info_flow_cell") or get_ci( - row, "flowcell_ids" - ) + flowcell_val = get_ci(row, "run_info_flow_cell") or get_ci(row, "flowcell_ids") if flowcell_val and str(flowcell_val).strip(): run_info["flow_cell"] = str(flowcell_val).strip() @@ -1633,7 +1752,9 @@ def _extract_classification_data(sample_dir: Path) -> Dict[str, Any]: classification_data["sturgeon"] = { "classification": best_class, "confidence": max_score * 100, - "confidence_level": _get_confidence_level(max_score * 100, "sturgeon"), + "confidence_level": _get_confidence_level( + max_score * 100, "sturgeon" + ), "features": features, } except Exception as e: @@ -1666,7 +1787,9 @@ def _extract_classification_data(sample_dir: Path) -> Dict[str, Any]: classification_data["nanodx"] = { "classification": best_class, "confidence": max_score * 100, - "confidence_level": _get_confidence_level(max_score * 100, "nanodx"), + "confidence_level": _get_confidence_level( + max_score * 100, "nanodx" + ), "features": features, } except Exception as e: @@ -1699,7 +1822,9 @@ def _extract_classification_data(sample_dir: Path) -> Dict[str, Any]: classification_data["pannanodx"] = { "classification": best_class, "confidence": max_score * 100, - "confidence_level": _get_confidence_level(max_score * 100, "pannanodx"), + "confidence_level": _get_confidence_level( + max_score * 100, "pannanodx" + ), "features": features, } except Exception as e: @@ -1726,7 +1851,9 @@ def _extract_classification_data(sample_dir: Path) -> Dict[str, Any]: max_score = score best_class = col except Exception as e: - logging.debug(f" Random Forest: : {e}") + logging.debug( + f" Random Forest: : {e}" + ) pass # Some random forest outputs are already in percent (0-100), @@ -1755,9 +1882,7 @@ def _extract_classification_data(sample_dir: Path) -> Dict[str, Any]: for row in reader: features = int( float( - row.get("covered_cpgs") - or row.get("number_probes") - or 0 + row.get("covered_cpgs") or row.get("number_probes") or 0 ) ) max_score = 0.0 @@ -1801,9 +1926,7 @@ def _extract_classification_data(sample_dir: Path) -> Dict[str, Any]: for row in reader: features = int( float( - row.get("covered_cpgs") - or row.get("number_probes") - or 0 + row.get("covered_cpgs") or row.get("number_probes") or 0 ) ) max_score = 0.0 @@ -1899,108 +2022,143 @@ def _extract_classification_data(sample_dir: Path) -> Dict[str, Any]: def _extract_fusion_data(sample_dir: Path) -> Dict[str, Any]: """Extract fusion analysis data from generated summary files.""" fusion_data = { - "target_fusions": 0, + "target_fusions": 0, "genome_fusions": 0, "target_pairs": 0, "target_groups": 0, "genome_pairs": 0, - "genome_groups": 0 + "genome_groups": 0, } - - logging.info(f"[Summary] _extract_fusion_data() called with sample_dir: {sample_dir}") + + logging.info( + f"[Summary] _extract_fusion_data() called with sample_dir: {sample_dir}" + ) try: # Debug: List all fusion-related files fusion_files = list(sample_dir.glob("*fusion*")) logging.info(f"[Summary] Found fusion files: {[f.name for f in fusion_files]}") - + # Debug: Check if genome-wide processed file exists genome_file = sample_dir / "fusion_candidates_all_processed.pkl" - logging.info(f"[Summary] Genome-wide processed file exists: {genome_file.exists()}") + logging.info( + f"[Summary] Genome-wide processed file exists: {genome_file.exists()}" + ) if genome_file.exists(): - logging.info(f"[Summary] Genome-wide processed file size: {genome_file.stat().st_size} bytes") + logging.info( + f"[Summary] Genome-wide processed file size: {genome_file.stat().st_size} bytes" + ) # First try to read from the new fusion_summary.csv file summary_file = sample_dir / "fusion_summary.csv" if summary_file.exists(): try: with open(summary_file, "r") as f: content = f.read() - + with open(summary_file, "r") as f: reader = csv.DictReader(f) for row in reader: - fusion_data["target_fusions"] = int(row.get("target_fusions", 0)) - fusion_data["genome_fusions"] = int(row.get("genome_fusions", 0)) + fusion_data["target_fusions"] = int( + row.get("target_fusions", 0) + ) + fusion_data["genome_fusions"] = int( + row.get("genome_fusions", 0) + ) break - + # Check if the summary file has incorrect genome-wide count (0) if fusion_data["genome_fusions"] == 0: # Try to regenerate the summary file with correct data try: - from robin.gui.components.fusion import _generate_summary_files_from_pickle - if _generate_summary_files_from_pickle(sample_dir, force_regenerate=True): + from robin.gui.components.fusion import ( + _generate_summary_files_from_pickle, + ) + + if _generate_summary_files_from_pickle( + sample_dir, force_regenerate=True + ): # Re-read the regenerated summary file with open(summary_file, "r") as f: reader = csv.DictReader(f) for row in reader: - fusion_data["target_fusions"] = int(row.get("target_fusions", 0)) - fusion_data["genome_fusions"] = int(row.get("genome_fusions", 0)) + fusion_data["target_fusions"] = int( + row.get("target_fusions", 0) + ) + fusion_data["genome_fusions"] = int( + row.get("genome_fusions", 0) + ) break return fusion_data except Exception as e: pass # Don't return early, continue to pickle file loading else: - logging.info(f"[Summary] Fusion data extracted from summary file - target: {fusion_data['target_fusions']}, genome: {fusion_data['genome_fusions']}") + logging.info( + f"[Summary] Fusion data extracted from summary file - target: {fusion_data['target_fusions']}, genome: {fusion_data['genome_fusions']}" + ) logging.info(f"[Summary] Summary file path: {summary_file}") return fusion_data except Exception as e: logging.debug(f" Fusion: Failed to read summary file: {e}") - + # If summary files don't exist, try to generate them from pickle files try: from robin.gui.components.fusion import _generate_summary_files_from_pickle + if _generate_summary_files_from_pickle(sample_dir, force_regenerate=False): # Try reading the newly generated summary file if summary_file.exists(): with open(summary_file, "r") as f: reader = csv.DictReader(f) for row in reader: - fusion_data["target_fusions"] = int(row.get("target_fusions", 0)) - fusion_data["genome_fusions"] = int(row.get("genome_fusions", 0)) + fusion_data["target_fusions"] = int( + row.get("target_fusions", 0) + ) + fusion_data["genome_fusions"] = int( + row.get("genome_fusions", 0) + ) break - logging.info(f"[Summary] Fusion data extracted from generated summary file - target: {fusion_data['target_fusions']}, genome: {fusion_data['genome_fusions']}") + logging.info( + f"[Summary] Fusion data extracted from generated summary file - target: {fusion_data['target_fusions']}, genome: {fusion_data['genome_fusions']}" + ) return fusion_data except Exception as e: - logging.debug(f" Fusion: Failed to generate summary files from pickle: {e}") - + logging.debug( + f" Fusion: Failed to generate summary files from pickle: {e}" + ) + # If still no data, try to load directly from pickle files and count gene_pairs try: - from robin.gui.components.fusion import _load_processed_pickle, _count_unique_fusion_pairs, _count_unique_fusion_groups + from robin.gui.components.fusion import ( + _count_unique_fusion_groups, + _count_unique_fusion_pairs, + _load_processed_pickle, + ) + target_file = sample_dir / "fusion_candidates_master_processed.pkl" genome_file = sample_dir / "fusion_candidates_all_processed.pkl" - - + target_data = _load_processed_pickle(target_file) genome_data = _load_processed_pickle(genome_file) - - + if target_data and isinstance(target_data, dict): # Use filtered counts to match what's displayed in fusion section fusion_data["target_fusions"] = _count_unique_fusion_pairs(target_data) fusion_data["target_pairs"] = _count_unique_fusion_pairs(target_data) fusion_data["target_groups"] = _count_unique_fusion_groups(target_data) - + if genome_data and isinstance(genome_data, dict): # Use filtered counts to match what's displayed in fusion section fusion_data["genome_fusions"] = _count_unique_fusion_pairs(genome_data) fusion_data["genome_pairs"] = _count_unique_fusion_pairs(genome_data) fusion_data["genome_groups"] = _count_unique_fusion_groups(genome_data) - - logging.info(f"[Summary] Fusion data loaded directly from pickle files - target: {fusion_data['target_fusions']} fusions, {fusion_data['target_pairs']} pairs, {fusion_data['target_groups']} groups; genome: {fusion_data['genome_fusions']} fusions, {fusion_data['genome_pairs']} pairs, {fusion_data['genome_groups']} groups") + + logging.info( + f"[Summary] Fusion data loaded directly from pickle files - target: {fusion_data['target_fusions']} fusions, {fusion_data['target_pairs']} pairs, {fusion_data['target_groups']} groups; genome: {fusion_data['genome_fusions']} fusions, {fusion_data['genome_pairs']} pairs, {fusion_data['genome_groups']} groups" + ) return fusion_data except Exception as e: logging.debug(f" Fusion: Failed to load directly from pickle files: {e}") - + # Fallback to individual fusion_results.csv file fusion_results_file = sample_dir / "fusion_results.csv" if fusion_results_file.exists(): @@ -2008,14 +2166,20 @@ def _extract_fusion_data(sample_dir: Path) -> Dict[str, Any]: with open(fusion_results_file, "r") as f: reader = csv.DictReader(f) for row in reader: - fusion_data["target_fusions"] = int(row.get("target_fusions", 0)) - fusion_data["genome_fusions"] = int(row.get("genome_fusions", 0)) + fusion_data["target_fusions"] = int( + row.get("target_fusions", 0) + ) + fusion_data["genome_fusions"] = int( + row.get("genome_fusions", 0) + ) break - logging.debug(f"[Summary] Fusion data extracted from results file - target: {fusion_data['target_fusions']}, genome: {fusion_data['genome_fusions']}") + logging.debug( + f"[Summary] Fusion data extracted from results file - target: {fusion_data['target_fusions']}, genome: {fusion_data['genome_fusions']}" + ) return fusion_data except Exception as e: logging.debug(f" Fusion: Failed to read results file: {e}") - + # Final fallback to legacy sv_count.txt file sv_count_file = sample_dir / "sv_count.txt" if sv_count_file.exists(): @@ -2026,7 +2190,9 @@ def _extract_fusion_data(sample_dir: Path) -> Dict[str, Any]: # Legacy behavior - assume this is genome-wide count fusion_data["genome_fusions"] = int(content) fusion_data["target_fusions"] = 0 - logging.debug(f"[Summary] Fusion data extracted from legacy sv_count file - target: {fusion_data['target_fusions']}, genome: {fusion_data['genome_fusions']}") + logging.debug( + f"[Summary] Fusion data extracted from legacy sv_count file - target: {fusion_data['target_fusions']}, genome: {fusion_data['genome_fusions']}" + ) except Exception as e: logging.debug(f" Fusion: Failed to read legacy sv_count file: {e}") @@ -2099,7 +2265,9 @@ def _get_confidence_badge_classes(confidence: float) -> str: if confidence >= 80: return "px-2 py-1 text-xs font-medium rounded-full bg-green-100 text-green-800" elif confidence >= 50: - return "px-2 py-1 text-xs font-medium rounded-full bg-yellow-100 text-yellow-800" + return ( + "px-2 py-1 text-xs font-medium rounded-full bg-yellow-100 text-yellow-800" + ) else: return "px-2 py-1 text-xs font-medium rounded-full bg-red-100 text-red-800" diff --git a/src/robin/gui/config.py b/src/robin/gui/config.py index 992c27f5..752fa8d0 100644 --- a/src/robin/gui/config.py +++ b/src/robin/gui/config.py @@ -55,54 +55,54 @@ def get_confidence_level(classifier: str, confidence: float) -> str: def get_enabled_sections(workflow_steps: Optional[List[str]]) -> Set[str]: """ Determine which sections should be enabled based on workflow steps. - + Args: workflow_steps: List of workflow step names (e.g., ['target', 'mgmt', 'sturgeon']) - + Returns: Set of enabled section names """ if not workflow_steps: # If no workflow steps specified, show all sections (backward compatibility) return set(WORKFLOW_STEP_TO_SECTION.values()) - + enabled = set() for step in workflow_steps: # Handle workflow steps that might have queue prefixes (e.g., "classification:sturgeon") step_name = step.split(":")[-1] if ":" in step else step if step_name in WORKFLOW_STEP_TO_SECTION: enabled.add(WORKFLOW_STEP_TO_SECTION[step_name]) - + return enabled def is_section_enabled(section_name: str, workflow_steps: Optional[List[str]]) -> bool: """ Check if a specific section should be enabled. - + Args: section_name: Name of the section to check workflow_steps: List of workflow step names - + Returns: True if section should be enabled, False otherwise """ enabled_sections = get_enabled_sections(workflow_steps) - + # If no workflow steps specified, show all sections (backward compatibility) if not workflow_steps: return True - + return section_name in enabled_sections def get_enabled_classification_steps(workflow_steps: Optional[List[str]]) -> Set[str]: """ Get the set of enabled classification steps. - + Args: workflow_steps: List of workflow step names - + Returns: Set of enabled classification step names (e.g., {'sturgeon', 'nanodx'}) """ @@ -121,8 +121,8 @@ def is_section_visible( """Check workflow and admin display config for section visibility.""" from robin.gui.display_config import ( DEFAULT_VIEWER_ROLE, - is_section_visible as _resolve, ) + from robin.gui.display_config import is_section_visible as _resolve return _resolve( section_id, @@ -143,8 +143,8 @@ def get_visible_classification_steps( """Classification steps visible on the sample page or in reports.""" from robin.gui.display_config import ( DEFAULT_VIEWER_ROLE, - get_visible_classification_steps as _resolve, ) + from robin.gui.display_config import get_visible_classification_steps as _resolve return _resolve( workflow_steps, @@ -163,8 +163,8 @@ def any_classification_visible( ) -> bool: from robin.gui.display_config import ( DEFAULT_VIEWER_ROLE, - any_classification_visible as _resolve, ) + from robin.gui.display_config import any_classification_visible as _resolve return _resolve( workflow_steps, diff --git a/src/robin/gui/display_config.py b/src/robin/gui/display_config.py index 92649f8d..ff4e2922 100644 --- a/src/robin/gui/display_config.py +++ b/src/robin/gui/display_config.py @@ -121,7 +121,13 @@ class DisplaySection: ), } -DISPLAY_GROUP_ORDER = ("classification", "v12_classifier", "analysis", "sample_details", "other") +DISPLAY_GROUP_ORDER = ( + "classification", + "v12_classifier", + "analysis", + "sample_details", + "other", +) DISPLAY_GROUP_LABELS = { "classification": "Classification", "v12_classifier": "V12 Classifier", @@ -145,8 +151,7 @@ class SampleDisplayConfig: def to_dict(self) -> Dict[str, Any]: role_sections = { - role: dict(self.role_sections.get(role) or {}) - for role in DISPLAY_ROLES + role: dict(self.role_sections.get(role) or {}) for role in DISPLAY_ROLES } out: Dict[str, Any] = { "schema_version": self.schema_version, @@ -197,7 +202,9 @@ def from_dict(cls, data: Optional[Dict[str, Any]]) -> "SampleDisplayConfig": role_sections[str(role)] = { str(k): bool(v) for k, v in mapping.items() } - legacy_sections = {str(k): bool(v) for k, v in (data.get("sections") or {}).items()} + legacy_sections = { + str(k): bool(v) for k, v in (data.get("sections") or {}).items() + } if legacy_sections and not role_sections.get("user"): role_sections["user"] = dict(legacy_sections) if "admin" not in role_sections: @@ -265,16 +272,12 @@ def with_role_updates( updated_by: Optional[str] = None, ) -> "SampleDisplayConfig": role_key = role if role in DISPLAY_ROLES else DEFAULT_VIEWER_ROLE - merged_roles = { - r: dict(self.role_sections.get(r) or {}) for r in DISPLAY_ROLES - } + merged_roles = {r: dict(self.role_sections.get(r) or {}) for r in DISPLAY_ROLES} role_map = dict(merged_roles.get(role_key) or {}) role_map.update(sections) merged_roles[role_key] = role_map legacy_sections = ( - dict(merged_roles["user"]) - if role_key == "user" - else dict(self.sections) + dict(merged_roles["user"]) if role_key == "user" else dict(self.sections) ) return SampleDisplayConfig( schema_version=self.schema_version, @@ -495,12 +498,9 @@ def effective_section_map( def sections_for_group(group: str) -> List[DisplaySection]: return [ - s - for s in DISPLAY_SECTIONS.values() - if s.group == group and s.parent_id is None + s for s in DISPLAY_SECTIONS.values() if s.group == group and s.parent_id is None ] + [ s for s in DISPLAY_SECTIONS.values() if s.group == group and s.parent_id is not None ] - diff --git a/src/robin/gui/plotting_preferences.py b/src/robin/gui/plotting_preferences.py index 33510d48..9e4fc443 100644 --- a/src/robin/gui/plotting_preferences.py +++ b/src/robin/gui/plotting_preferences.py @@ -242,9 +242,7 @@ def resolve_cnv_report_scale(scale: Optional[str]) -> str: def cnv_summary_normalized_from_scale(scale: Optional[str]) -> bool: """Return True when the configured CNV report scale is log2 ratio mode.""" - return ( - resolve_cnv_report_scale(scale) == CNV_REPORT_SCALE_NORMALIZED_DIFFERENCE - ) + return resolve_cnv_report_scale(scale) == CNV_REPORT_SCALE_NORMALIZED_DIFFERENCE def load_plotting_preferences(store=None) -> PlottingPreferencesConfig: diff --git a/src/robin/gui/progress_notifications.py b/src/robin/gui/progress_notifications.py index 158a8255..524f8046 100644 --- a/src/robin/gui/progress_notifications.py +++ b/src/robin/gui/progress_notifications.py @@ -5,23 +5,23 @@ """ import logging -from typing import Dict, Any, Optional, Callable from datetime import datetime +from typing import Any, Callable, Dict, Optional logger = logging.getLogger(__name__) class ReportProgressNotifier: """Handles progress notifications for report generation.""" - + def __init__(self): """Initialize the progress notifier.""" self.active_reports: Dict[str, Dict[str, Any]] = {} self.notification_ids: Dict[str, str] = {} - + def handle_progress_update(self, progress_data: Dict[str, Any]): """Handle a progress update event. - + Args: progress_data: Dictionary containing progress information """ @@ -29,11 +29,11 @@ def handle_progress_update(self, progress_data: Dict[str, Any]): stage = progress_data.get("stage") message = progress_data.get("message") progress = progress_data.get("progress", 0.0) - + if not sample_id: logger.warning("Progress update missing sample_id") return - + # Update active reports tracking self.active_reports[sample_id] = { "stage": stage, @@ -44,42 +44,42 @@ def handle_progress_update(self, progress_data: Dict[str, Any]): "total_sections": progress_data.get("total_sections", 0), "current_section": progress_data.get("current_section", ""), } - + # Send update to GUI using existing system self._send_gui_update(sample_id, progress_data) - + # Show notification self._update_notification(sample_id, progress_data) - + # Clean up completed/error reports if stage in ["completed", "error"]: if sample_id in self.active_reports: del self.active_reports[sample_id] - + def _send_gui_update(self, sample_id: str, progress_data: Dict[str, Any]): """Send progress update to GUI using existing update system.""" try: - from robin.gui_launcher import send_gui_update, UpdateType - + from robin.gui_launcher import UpdateType, send_gui_update + # Send progress update to GUI send_gui_update( UpdateType.PROGRESS_UPDATE, { "sample_id": sample_id, "report_progress": progress_data, - "active_reports": self.active_reports.copy() + "active_reports": self.active_reports.copy(), }, - priority=1 # High priority for progress updates + priority=1, # High priority for progress updates ) except Exception as e: logger.error(f"Error sending GUI update: {e}") - + def _update_notification(self, sample_id: str, progress_data: Dict[str, Any]): """Update or create a progress notification.""" stage = progress_data.get("stage") message = progress_data.get("message") progress = progress_data.get("progress", 0.0) - + # Determine notification type and content if stage == "completed": self._show_completion_notification(sample_id, progress_data) @@ -87,8 +87,10 @@ def _update_notification(self, sample_id: str, progress_data: Dict[str, Any]): self._show_error_notification(sample_id, progress_data) else: self._show_progress_notification(sample_id, progress_data) - - def _show_progress_notification(self, sample_id: str, progress_data: Dict[str, Any]): + + def _show_progress_notification( + self, sample_id: str, progress_data: Dict[str, Any] + ): """Show a progress notification.""" stage = progress_data.get("stage") message = progress_data.get("message") @@ -96,7 +98,7 @@ def _show_progress_notification(self, sample_id: str, progress_data: Dict[str, A completed_sections = progress_data.get("completed_sections", 0) total_sections = progress_data.get("total_sections", 0) current_section = progress_data.get("current_section", "") - + # Create detailed message if total_sections > 0: section_info = f" ({completed_sections}/{total_sections} sections)" @@ -104,12 +106,12 @@ def _show_progress_notification(self, sample_id: str, progress_data: Dict[str, A section_info += f" - {current_section}" else: section_info = "" - + detailed_message = f"{message}{section_info}" - + # Calculate progress percentage progress_percent = int(progress * 100) - + # Create notification with progress notification_type = "info" if stage in ["initializing", "loading_data"]: @@ -118,58 +120,62 @@ def _show_progress_notification(self, sample_id: str, progress_data: Dict[str, A notification_type = "ongoing" elif stage in ["building_pdf", "exporting_csv", "creating_zip"]: notification_type = "info" - + # Show notification with progress information try: from nicegui import ui + ui.notify( f"[{sample_id}] {detailed_message} ({progress_percent}%)", type=notification_type, - timeout=0 if stage != "completed" else 5000, # Persistent until completion - position="top-right" + timeout=( + 0 if stage != "completed" else 5000 + ), # Persistent until completion + position="top-right", ) except Exception as e: logger.error(f"Error showing notification: {e}") - - def _show_completion_notification(self, sample_id: str, progress_data: Dict[str, Any]): + + def _show_completion_notification( + self, sample_id: str, progress_data: Dict[str, Any] + ): """Show a completion notification.""" filename = progress_data.get("filename", "report.pdf") - + try: from nicegui import ui + ui.notify( f"[{sample_id}] Report generation completed: {filename}", type="positive", timeout=5000, - position="top-right" + position="top-right", ) except Exception as e: logger.error(f"Error showing completion notification: {e}") - + def _show_error_notification(self, sample_id: str, progress_data: Dict[str, Any]): """Show an error notification.""" error_message = progress_data.get("error_message", "Unknown error") error_details = progress_data.get("error_details", "") - + full_message = f"[{sample_id}] Report generation failed: {error_message}" if error_details: full_message += f" ({error_details})" - + try: from nicegui import ui + ui.notify( - full_message, - type="negative", - timeout=10000, - position="top-right" + full_message, type="negative", timeout=10000, position="top-right" ) except Exception as e: logger.error(f"Error showing error notification: {e}") - + def get_active_reports(self) -> Dict[str, Dict[str, Any]]: """Get information about currently active report generations.""" return self.active_reports.copy() - + def is_report_active(self, sample_id: str) -> bool: """Check if a report is currently being generated.""" return sample_id in self.active_reports @@ -181,17 +187,18 @@ def is_report_active(self, sample_id: str) -> bool: def create_progress_callback(sample_id: str): """Create a progress callback function for a specific sample. - + Args: sample_id: ID of the sample being processed - + Returns: Callback function that can be passed to report generation """ + def progress_callback(progress_data: Dict[str, Any]): """Progress callback function.""" # Ensure sample_id is set progress_data["sample_id"] = sample_id progress_notifier.handle_progress_update(progress_data) - - return progress_callback \ No newline at end of file + + return progress_callback diff --git a/src/robin/gui/report_progress.py b/src/robin/gui/report_progress.py index bc4427f9..2212a531 100644 --- a/src/robin/gui/report_progress.py +++ b/src/robin/gui/report_progress.py @@ -7,8 +7,8 @@ import logging import queue import threading -from typing import Dict, Any, Optional from datetime import datetime +from typing import Any, Dict, Optional logger = logging.getLogger(__name__) @@ -23,48 +23,47 @@ def normalize_report_progress(progress: Optional[float]) -> Optional[float]: class ReportProgressManager: """Manages report generation progress using a queue-based system.""" - + def __init__(self): """Initialize the progress manager.""" self.progress_queue = queue.Queue() self.active_reports: Dict[str, Dict[str, Any]] = {} self._notification_ids: Dict[str, str] = {} - + def start_report(self, sample_id: str) -> str: """Start tracking a new report generation. - + Args: sample_id: ID of the sample being processed - + Returns: Notification ID for tracking """ try: # Store tracking info self.active_reports[sample_id] = { - 'notification_id': None, # Will be set when notification is shown - 'stage': 'initializing', - 'start_time': datetime.now(), - 'sample_id': sample_id, - 'progress': 0.0 + "notification_id": None, # Will be set when notification is shown + "stage": "initializing", + "start_time": datetime.now(), + "sample_id": sample_id, + "progress": 0.0, } - + # Queue the initial notification - self.progress_queue.put({ - 'type': 'start', - 'sample_id': sample_id - }) - + self.progress_queue.put({"type": "start", "sample_id": sample_id}) + logger.info(f"Started tracking report generation for {sample_id}") return sample_id # Return sample_id as identifier - + except Exception as e: logger.error(f"Error starting report tracking: {e}") return None - - def update_progress(self, sample_id: str, stage: str, message: str, progress: float = None): + + def update_progress( + self, sample_id: str, stage: str, message: str, progress: float = None + ): """Update progress for a specific report. - + Args: sample_id: ID of the sample stage: Current stage (initializing, processing_sections, building_pdf, etc.) @@ -74,22 +73,24 @@ def update_progress(self, sample_id: str, stage: str, message: str, progress: fl if sample_id not in self.active_reports: logger.warning(f"No active report found for {sample_id}") return - + # Put update in queue for UI thread processing try: - self.progress_queue.put({ - 'type': 'update', - 'sample_id': sample_id, - 'stage': stage, - 'message': message, - 'progress': normalize_report_progress(progress), - }) + self.progress_queue.put( + { + "type": "update", + "sample_id": sample_id, + "stage": stage, + "message": message, + "progress": normalize_report_progress(progress), + } + ) except Exception as e: logger.error(f"Error queuing progress update: {e}") - + def complete_report(self, sample_id: str, filename: str = None): """Mark a report as completed. - + Args: sample_id: ID of the sample filename: Name of the generated file @@ -97,20 +98,18 @@ def complete_report(self, sample_id: str, filename: str = None): if sample_id not in self.active_reports: logger.warning(f"No active report found for {sample_id}") return - + # Put completion in queue for UI thread processing try: - self.progress_queue.put({ - 'type': 'complete', - 'sample_id': sample_id, - 'filename': filename - }) + self.progress_queue.put( + {"type": "complete", "sample_id": sample_id, "filename": filename} + ) except Exception as e: logger.error(f"Error queuing completion: {e}") - + def error_report(self, sample_id: str, error_message: str): """Mark a report as failed. - + Args: sample_id: ID of the sample error_message: Error description @@ -118,23 +117,25 @@ def error_report(self, sample_id: str, error_message: str): if sample_id not in self.active_reports: logger.warning(f"No active report found for {sample_id}") return - + # Put error in queue for UI thread processing try: - self.progress_queue.put({ - 'type': 'error', - 'sample_id': sample_id, - 'error_message': error_message - }) + self.progress_queue.put( + { + "type": "error", + "sample_id": sample_id, + "error_message": error_message, + } + ) except Exception as e: logger.error(f"Error queuing error: {e}") - + def process_queue(self): """Process queued progress updates on the UI thread.""" # This method is now handled by the GUI launcher # to ensure proper UI context pass - + def _remove_report(self, sample_id: str): """Remove a report from active tracking.""" if sample_id in self.active_reports: @@ -142,7 +143,7 @@ def _remove_report(self, sample_id: str): if sample_id in self._notification_ids: del self._notification_ids[sample_id] logger.debug(f"Removed {sample_id} from active reports") - + def get_active_reports(self) -> Dict[str, Dict[str, Any]]: """Get all currently active reports.""" return self.active_reports.copy() @@ -154,27 +155,24 @@ def get_active_reports(self) -> Dict[str, Dict[str, Any]]: def create_progress_callback(sample_id: str): """Create a progress callback function for a specific sample. - + Args: sample_id: ID of the sample being processed - + Returns: Callback function that can be passed to report generation """ # Start tracking this report progress_manager.start_report(sample_id) - + def progress_callback(progress_data: Dict[str, Any]): """Progress callback function.""" - stage = progress_data.get('stage', 'unknown') - message = progress_data.get('message', '') - progress = normalize_report_progress(progress_data.get('progress')) - + stage = progress_data.get("stage", "unknown") + message = progress_data.get("message", "") + progress = normalize_report_progress(progress_data.get("progress")) + progress_manager.update_progress( - sample_id=sample_id, - stage=stage, - message=message, - progress=progress + sample_id=sample_id, stage=stage, message=message, progress=progress ) - - return progress_callback \ No newline at end of file + + return progress_callback diff --git a/src/robin/gui/theme.py b/src/robin/gui/theme.py index 4a4a491c..c5d8162c 100644 --- a/src/robin/gui/theme.py +++ b/src/robin/gui/theme.py @@ -36,26 +36,23 @@ - platform """ -from contextlib import contextmanager -from signal import siginterrupt -from packaging import version -import requests import asyncio +import importlib.metadata +import json import logging import subprocess -import importlib.metadata import time -import json -from typing import Callable, Optional, Any, Dict, List - - -from nicegui import ui, app, events, run +from contextlib import contextmanager +from pathlib import Path +from signal import siginterrupt +from typing import Any, Callable, Dict, List, Optional -from robin.minknow.toml_config import minknow_gui_accessible +import requests +from nicegui import app, events, run, ui +from packaging import version from robin.gui.session import current_session_is_admin, current_session_username - -from pathlib import Path +from robin.minknow.toml_config import minknow_gui_accessible # These will be set by the get_imagefile() and get_version() functions IMAGEFILE = None @@ -63,14 +60,22 @@ import os -import psutil import platform +import psutil + # Check if we're in development mode -is_development_mode = os.environ.get("ROBIN_DEV_MODE", "").lower() in ("1", "true", "yes", "on") +is_development_mode = os.environ.get("ROBIN_DEV_MODE", "").lower() in ( + "1", + "true", + "yes", + "on", +) # Process large BAMs individually (do not use alongside live runs) -_process_large_bams_enabled = os.environ.get("ROBIN_PROCESS_LARGE_BAMS", "0").strip().lower() in ("1", "true", "yes", "on") +_process_large_bams_enabled = os.environ.get( + "ROBIN_PROCESS_LARGE_BAMS", "0" +).strip().lower() in ("1", "true", "yes", "on") # Per-client theme sync interval lower bound _THEME_SYNC_MIN_INTERVAL_SECONDS = 0.1 @@ -189,9 +194,7 @@ async def _async_wrapped() -> Any: return None raise - timer = app.timer( - interval, _wrapped, once=once, immediate=immediate, active=active - ) + timer = app.timer(interval, _wrapped, once=once, immediate=immediate, active=active) timer_box["timer"] = timer if client is not None: @@ -342,13 +345,17 @@ def get_imagefile(): if IMAGEFILE is None: try: from robin.gui import images + IMAGEFILE = os.path.join( - os.path.dirname(os.path.abspath(images.__file__)), "ROBIN_logo_small.png" + os.path.dirname(os.path.abspath(images.__file__)), + "ROBIN_logo_small.png", ) except (ImportError, AttributeError): # Fallback path when running standalone IMAGEFILE = os.path.join( - os.path.dirname(os.path.abspath(__file__)), "images", "ROBIN_logo_small.png" + os.path.dirname(os.path.abspath(__file__)), + "images", + "ROBIN_logo_small.png", ) return IMAGEFILE @@ -363,6 +370,7 @@ def get_about(): # Fallback when running standalone - create a minimal __about__ object class MockAbout: __version__ = "standalone-test" + __about__ = MockAbout() return __about__ @@ -394,7 +402,7 @@ def styled_table(*, columns, rows=None, pagination=20, class_size="table-xs", ** Tuple of (container, table) where container is the overflow wrapper column and table is the ui.table instance. """ # Add CSS to hide pagination for tables with no-pagination class (only once) - if not hasattr(styled_table, '_pagination_css_added'): + if not hasattr(styled_table, "_pagination_css_added"): ui.add_head_html(""" " + HEADER_HTML + + f"" ) # Add mobile-specific responsive CSS ui.add_head_html(""" @@ -1176,8 +1202,7 @@ def frame( } """) - ui.add_head_html( - """ + ui.add_head_html(""" - """ - ) + """) # Research-use consent is collected per user at login (see gui_launcher login flow). async def show_disclaimer(): @@ -1235,15 +1259,16 @@ def logout_user(): pass ui.navigate.to("/login") - with quitdialog, ui.card().classes( - "robin-dialog-surface p-4 md:p-5 min-w-[18rem] max-w-md" + with ( + quitdialog, + ui.card().classes("robin-dialog-surface p-4 md:p-5 min-w-[18rem] max-w-md"), ): ui.label("Quit R.O.B.I.N?").classes( "classification-insight-heading text-headline-small q-mb-sm" ) - ui.label( - "Quitting the app will stop running methylation analysis." - ).classes("classification-insight-foot") + ui.label("Quitting the app will stop running methylation analysis.").classes( + "classification-insight-foot" + ) ui.label("If you want to keep analysis running, click Cancel.").classes( "classification-insight-foot" ) @@ -1297,7 +1322,9 @@ def _on_dark_mode_toggle(e: Any) -> None: with ui.header(elevated=True).classes(header_classes): # Use flexbox layout instead of grid to prevent overlap - with ui.row().classes("w-full items-center justify-between px-1 py-0.5 sm:px-3 sm:py-1.5"): + with ui.row().classes( + "w-full items-center justify-between px-1 py-0.5 sm:px-3 sm:py-1.5" + ): # Left: Hamburger, then title (responsive) with ui.row().classes("items-center gap-1 sm:gap-2 min-w-0 flex-1"): with ui.button(icon="menu").classes("rounded-md flex-shrink-0"): @@ -1327,9 +1354,7 @@ def _on_dark_mode_toggle(e: Any) -> None: ).classes("text-body-medium") ui.menu_item( "Documentation", - lambda: ui.navigate.to( - "https://looselab.github.io/ROBIN/" - ), + lambda: ui.navigate.to("https://looselab.github.io/ROBIN/"), ).classes("text-body-medium") if _current_user_is_admin(): ui.separator() @@ -1342,6 +1367,7 @@ def _on_dark_mode_toggle(e: Any) -> None: lambda: ui.navigate.to("/admin"), ).classes("text-body-medium") ui.separator() + def _dark_mode_initial() -> bool: """Prefer session (browser) storage so initial value matches first paint.""" try: @@ -1368,15 +1394,13 @@ def _persist_dark_mode( pass # Session must be updated via a new request (see NiceGUI app.storage.browser docs). body = json.dumps({"value": val}) - ui.run_javascript( - f""" + ui.run_javascript(f""" fetch('/robin_dark_mode', {{ method: 'POST', headers: {{'Content-Type': 'application/json'}}, body: {json.dumps(body)}, }}); - """ - ) + """) def _sync_dark_mode_from_storage() -> None: try: @@ -1403,9 +1427,7 @@ def _sync_dark_mode_from_storage() -> None: "Change password", lambda: ui.navigate.to("/change-password?voluntary=1"), ).classes("text-body-medium") - ui.menu_item("Close", menu.close).classes( - "text-body-medium" - ) + ui.menu_item("Close", menu.close).classes("text-body-medium") ui.button( "LOG OUT", icon="logout", on_click=logout_user ).classes("bg-error text-white rounded-md") @@ -1455,8 +1477,7 @@ def _sync_dark_mode_from_storage() -> None: ) with ui.column().classes( - "w-full h-full max-w-full overflow-hidden flex flex-col items-center " - "px-1" + "w-full h-full max-w-full overflow-hidden flex flex-col items-center " "px-1" ) as main_content: pass @@ -1473,8 +1494,9 @@ def _sync_dark_mode_from_storage() -> None: if batphone: footer_classes += " batphone" with ui.footer().classes(footer_classes): - with ui.dialog() as dialog, ui.card().classes( - "robin-dialog-surface p-4 md:p-5 min-w-[16rem] max-w-sm" + with ( + ui.dialog() as dialog, + ui.card().classes("robin-dialog-surface p-4 md:p-5 min-w-[16rem] max-w-sm"), ): ui.label("Links").classes( "classification-insight-heading text-headline-small q-mb-sm" @@ -1518,9 +1540,13 @@ def _sync_dark_mode_from_storage() -> None: # Center: Buttons with proper spacing with ui.row().classes("items-center gap-2 flex-shrink-0"): - ui.button("Links", on_click=dialog.open).classes("rounded-md mobile-button text-xs px-2 py-1") + ui.button("Links", on_click=dialog.open).classes( + "rounded-md mobile-button text-xs px-2 py-1" + ) - with ui.button(icon="info").classes("rounded-md mobile-button px-2 py-1"): + with ui.button(icon="info").classes( + "rounded-md mobile-button px-2 py-1" + ): with ui.menu() as menu: ui.label().bind_text_from( app, "urls", backward=lambda n: f"Available urls: {n}" @@ -1530,9 +1556,7 @@ def _sync_dark_mode_from_storage() -> None: ) # Right side: Compact copyright (mobile only) - ui.label("©Looselab").classes( - "text-xs text-weight-italic flex-shrink-0" - ) + ui.label("©Looselab").classes("text-xs text-weight-italic flex-shrink-0") # Desktop-only additional info ui.label("Not for diagnostic use.").classes( @@ -1563,15 +1587,16 @@ async def cleanup_and_exit(): logging.info("User initiated shutdown via UI") # Create and show shutdown modal with M3 styling - with ui.dialog().props("persistent") as shutdown_dialog, ui.card().classes( - "robin-dialog-surface p-4 md:p-5 w-full max-w-sm" + with ( + ui.dialog().props("persistent") as shutdown_dialog, + ui.card().classes("robin-dialog-surface p-4 md:p-5 w-full max-w-sm"), ): ui.label("Shutting down").classes( "classification-insight-heading text-headline-small q-mb-sm" ) - ui.label( - "R.O.B.I.N is shutting down. Please wait while we clean up…" - ).classes("classification-insight-foot q-mb-md") + ui.label("R.O.B.I.N is shutting down. Please wait while we clean up…").classes( + "classification-insight-foot q-mb-md" + ) with ui.row().classes("w-full justify-center"): ui.spinner(size="lg", color="primary") @@ -1613,23 +1638,37 @@ def create_home_page(): ui.label("Welcome to the Application").classes( "text-headline-large text-center px-3" ) - with ui.row().classes('items-center m-auto'): + with ui.row().classes("items-center m-auto"): ui.circular_progress(value=0.1, show_value=False, size="xs") ui.circular_progress(value=0.1, show_value=False, size="xl") - with ui.row().classes('items-center m-auto'): - with ui.circular_progress(value=0.1, show_value=False, size="sm") as progress: + with ui.row().classes("items-center m-auto"): + with ui.circular_progress( + value=0.1, show_value=False, size="sm" + ) as progress: ui.button( - icon='star', - on_click=lambda: progress.set_value(progress.value + 0.1) - ).props('flat round') - ui.label('click to increase progress') - with ui.card().classes("w-full max-w-4xl mx-auto mobile-padding main-content-card").style("border: 2px solid var(--md-primary)"): - with ui.row().classes("w-full flex justify-between items-center flex-wrap gap-2"): - ui.label('Sample Name').classes("text-headline-medium flex-shrink-0") - ui.button("Button").classes("bg-primary text-white rounded-md mobile-button") + icon="star", + on_click=lambda: progress.set_value(progress.value + 0.1), + ).props("flat round") + ui.label("click to increase progress") + with ( + ui.card() + .classes("w-full max-w-4xl mx-auto mobile-padding main-content-card") + .style("border: 2px solid var(--md-primary)") + ): + with ui.row().classes( + "w-full flex justify-between items-center flex-wrap gap-2" + ): + ui.label("Sample Name").classes("text-headline-medium flex-shrink-0") + ui.button("Button").classes( + "bg-primary text-white rounded-md mobile-button" + ) ui.separator().classes().style("border: 1px solid var(--md-primary)") - with ui.card().classes("w-full bg-gradient-to-r from-blue-50 to-indigo-50 mobile-padding"): - ui.label("Run Information").classes("text-lg font-semibold mb-3 text-blue-800") + with ui.card().classes( + "w-full bg-gradient-to-r from-blue-50 to-indigo-50 mobile-padding" + ): + ui.label("Run Information").classes( + "text-lg font-semibold mb-3 text-blue-800" + ) with ui.row().classes("w-full gap-2 sm:gap-6 items-center flex-wrap"): ui.label("Run").classes("text-body-medium text-xs sm:text-sm") ui.label("Model").classes("text-body-medium text-xs sm:text-sm") @@ -1637,52 +1676,114 @@ def create_home_page(): ui.label("Flow Cell").classes("text-body-medium text-xs sm:text-sm") ui.label("Sample").classes("text-body-medium text-xs sm:text-sm") - with ui.card().classes("w-full mobile-padding"): - ui.label("Classification Results").classes("text-lg font-semibold mb-3 text-blue-800") + ui.label("Classification Results").classes( + "text-lg font-semibold mb-3 text-blue-800" + ) # Use responsive grid: 2 columns on desktop, 1 column on mobile - with ui.row().classes("w-full gap-2 sm:gap-3 flex-wrap classification-cards"): + with ui.row().classes( + "w-full gap-2 sm:gap-3 flex-wrap classification-cards" + ): # Sturgeon Classification - with ui.card().classes("flex-1 min-w-0 elevation-4 rounded-xl bg-gradient-to-br from-blue-50 to-indigo-50 border-l-4 border-blue-500 classification-card"): - ui.label("Sturgeon Classification").classes("font-bold text-blue-800 mb-2 text-sm sm:text-base") - ui.label("Class: --").classes("font-bold text-medium text-blue-600 text-xs sm:text-sm") - ui.label("Confidence: --%").classes("text-xs sm:text-sm text-blue-600") - ui.label("Probes: --").classes("text-xs sm:text-sm text-blue-600") - ui.label("Model: --").classes("text-xs sm:text-sm text-blue-600") - ui.label("Features: --").classes("text-xs sm:text-sm text-blue-600") + with ui.card().classes( + "flex-1 min-w-0 elevation-4 rounded-xl bg-gradient-to-br from-blue-50 to-indigo-50 border-l-4 border-blue-500 classification-card" + ): + ui.label("Sturgeon Classification").classes( + "font-bold text-blue-800 mb-2 text-sm sm:text-base" + ) + ui.label("Class: --").classes( + "font-bold text-medium text-blue-600 text-xs sm:text-sm" + ) + ui.label("Confidence: --%").classes( + "text-xs sm:text-sm text-blue-600" + ) + ui.label("Probes: --").classes( + "text-xs sm:text-sm text-blue-600" + ) + ui.label("Model: --").classes( + "text-xs sm:text-sm text-blue-600" + ) + ui.label("Features: --").classes( + "text-xs sm:text-sm text-blue-600" + ) # NanoDX Classification - with ui.card().classes("flex-1 min-w-0 elevation-4 rounded-xl bg-gradient-to-br from-green-50 to-green-100 border-l-4 border-green-500 classification-card"): - ui.label("NanoDX Classification").classes("font-bold text-green-800 mb-2 text-sm sm:text-base") - ui.label("Class: --").classes("font-bold text-medium text-green-600 text-xs sm:text-sm") - ui.label("Confidence: --%").classes("text-xs sm:text-sm text-green-600") - ui.label("Probes: --").classes("text-xs sm:text-sm text-green-600") - ui.label("Model: --").classes("text-xs sm:text-sm text-green-600") - ui.label("Features: --").classes("text-xs sm:text-sm text-green-600") + with ui.card().classes( + "flex-1 min-w-0 elevation-4 rounded-xl bg-gradient-to-br from-green-50 to-green-100 border-l-4 border-green-500 classification-card" + ): + ui.label("NanoDX Classification").classes( + "font-bold text-green-800 mb-2 text-sm sm:text-base" + ) + ui.label("Class: --").classes( + "font-bold text-medium text-green-600 text-xs sm:text-sm" + ) + ui.label("Confidence: --%").classes( + "text-xs sm:text-sm text-green-600" + ) + ui.label("Probes: --").classes( + "text-xs sm:text-sm text-green-600" + ) + ui.label("Model: --").classes( + "text-xs sm:text-sm text-green-600" + ) + ui.label("Features: --").classes( + "text-xs sm:text-sm text-green-600" + ) # PanNanoDX Classification - with ui.card().classes("flex-1 min-w-0 elevation-4 rounded-xl bg-gradient-to-br from-purple-50 to-purple-100 border-l-4 border-purple-500 classification-card"): - ui.label("PanNanoDX Classification").classes("font-bold text-purple-800 mb-2 text-sm sm:text-base") - ui.label("Class: --").classes("font-bold text-medium text-purple-600 text-xs sm:text-sm") - ui.label("Confidence: --%").classes("text-xs sm:text-sm text-purple-600") - ui.label("Probes: --").classes("text-xs sm:text-sm text-purple-600") - ui.label("Model: --").classes("text-xs sm:text-sm text-purple-600") - ui.label("Features: --").classes("text-xs sm:text-sm text-purple-600") + with ui.card().classes( + "flex-1 min-w-0 elevation-4 rounded-xl bg-gradient-to-br from-purple-50 to-purple-100 border-l-4 border-purple-500 classification-card" + ): + ui.label("PanNanoDX Classification").classes( + "font-bold text-purple-800 mb-2 text-sm sm:text-base" + ) + ui.label("Class: --").classes( + "font-bold text-medium text-purple-600 text-xs sm:text-sm" + ) + ui.label("Confidence: --%").classes( + "text-xs sm:text-sm text-purple-600" + ) + ui.label("Probes: --").classes( + "text-xs sm:text-sm text-purple-600" + ) + ui.label("Model: --").classes( + "text-xs sm:text-sm text-purple-600" + ) + ui.label("Features: --").classes( + "text-xs sm:text-sm text-purple-600" + ) # Random Forest Classification - with ui.card().classes("flex-1 min-w-0 elevation-4 rounded-xl bg-gradient-to-br from-orange-50 to-orange-100 border-l-4 border-orange-500 classification-card"): - ui.label("Random Forest Classification").classes("font-bold text-orange-800 mb-2 text-sm sm:text-base") - ui.label("Class: --").classes("font-bold text-medium text-orange-600 text-xs sm:text-sm") - ui.label("Confidence: --%").classes("text-xs sm:text-sm text-orange-600") - ui.label("Probes: --").classes("text-xs sm:text-sm text-orange-600") - ui.label("Model: --").classes("text-xs sm:text-sm text-orange-600") - ui.label("Features: --").classes("text-xs sm:text-sm text-orange-600") - + with ui.card().classes( + "flex-1 min-w-0 elevation-4 rounded-xl bg-gradient-to-br from-orange-50 to-orange-100 border-l-4 border-orange-500 classification-card" + ): + ui.label("Random Forest Classification").classes( + "font-bold text-orange-800 mb-2 text-sm sm:text-base" + ) + ui.label("Class: --").classes( + "font-bold text-medium text-orange-600 text-xs sm:text-sm" + ) + ui.label("Confidence: --%").classes( + "text-xs sm:text-sm text-orange-600" + ) + ui.label("Probes: --").classes( + "text-xs sm:text-sm text-orange-600" + ) + ui.label("Model: --").classes( + "text-xs sm:text-sm text-orange-600" + ) + ui.label("Features: --").classes( + "text-xs sm:text-sm text-orange-600" + ) - ui.label('text below').classes("text-body-medium px-3") + ui.label("text below").classes("text-body-medium px-3") with ui.card_section().classes("px-3"): - ui.image('https://picsum.photos/id/684/640/360').classes("w-full h-auto rounded-lg") - ui.label('Lorem ipsum dolor sit amet, consectetur adipiscing elit, ...').classes("text-body-medium mobile-text") + ui.image("https://picsum.photos/id/684/640/360").classes( + "w-full h-auto rounded-lg" + ) + ui.label( + "Lorem ipsum dolor sit amet, consectetur adipiscing elit, ..." + ).classes("text-body-medium mobile-text") def create_standalone_page(): @@ -1693,26 +1794,39 @@ def create_standalone_page(): ) ui.add_head_html(EDITORIAL_FONTS_HTML) ui.add_head_html( - HEADER_HTML + f"" + HEADER_HTML + + f"" ) # Create a simple header (same shell padding pattern as frame()) - with ui.header(elevated=True).classes("items-center duration-200 p-0 px-2 no-wrap elevation-1"): - with ui.row().classes("w-full items-center justify-between px-1 py-0.5 sm:px-3 sm:py-1.5"): - ui.html("R.O.B.I.N", sanitize=False).classes("text-headline-medium drop-shadow font-bold").style( - "font-weight: 700; font-family: var(--font-display)" - ) + with ui.header(elevated=True).classes( + "items-center duration-200 p-0 px-2 no-wrap elevation-1" + ): + with ui.row().classes( + "w-full items-center justify-between px-1 py-0.5 sm:px-3 sm:py-1.5" + ): + ui.html("R.O.B.I.N", sanitize=False).classes( + "text-headline-medium drop-shadow font-bold" + ).style("font-weight: 700; font-family: var(--font-display)") ui.image(get_imagefile()).style("width: 50px").classes("ml-auto") # Create main content with ui.column().classes("w-full h-full max-w-full overflow-hidden p-6"): - ui.label("Welcome to ROBIN Theme Test").classes("text-headline-large text-center") - ui.label("This is a standalone test of the ROBIN theme system.").classes("text-body-large text-center mt-4") - ui.label(f"Version: {get_about().__version__}").classes("text-body-medium text-center mt-2") + ui.label("Welcome to ROBIN Theme Test").classes( + "text-headline-large text-center" + ) + ui.label("This is a standalone test of the ROBIN theme system.").classes( + "text-body-large text-center mt-4" + ) + ui.label(f"Version: {get_about().__version__}").classes( + "text-body-medium text-center mt-2" + ) # Create a simple footer (same shell padding pattern as frame()) with ui.footer().classes("items-center duration-200 p-0 px-2 no-wrap elevation-1"): - with ui.row().classes("w-full items-center justify-between px-1 py-0.5 sm:px-3 sm:py-1.5"): + with ui.row().classes( + "w-full items-center justify-between px-1 py-0.5 sm:px-3 sm:py-1.5" + ): ui.image(get_imagefile()).style("width: 40px") ui.label("ROBIN Theme Test - Standalone Mode").classes("text-body-small") @@ -1728,12 +1842,12 @@ def create_workflow_page(): crossnn_version = get_crossnn_version() cnv_from_bam_version = get_cnv_from_bam_version() - with ui.element("div").classes("w-full min-w-0").props( - "id=workflow-diagram-page" + with ( + ui.element("div") + .classes("w-full min-w-0") + .props("id=workflow-diagram-page") ): - with ui.column().classes( - "w-full max-w-6xl mx-auto gap-3 p-2 md:p-3" - ): + with ui.column().classes("w-full max-w-6xl mx-auto gap-3 p-2 md:p-3"): with ui.element("div").classes( "classification-insight-shell w-full min-w-0" ): @@ -1748,12 +1862,8 @@ def create_workflow_page(): with ui.element("div").classes( "classification-insight-card w-full min-w-0" ): - with ui.column().classes( - "w-full min-w-0 gap-3 p-2 md:p-3" - ): - with ui.row().classes( - "items-center gap-2 min-w-0" - ): + with ui.column().classes("w-full min-w-0 gap-3 p-2 md:p-3"): + with ui.row().classes("items-center gap-2 min-w-0"): ui.icon("account_tree").classes( "classification-insight-icon" ) @@ -1768,7 +1878,7 @@ def create_workflow_page(): "w-full min-w-0 overflow-x-auto workflow-diagram-scroll" ): ui.mermaid( - f""" + f""" flowchart TD %% Style definitions with M3 color palette classDef minKNOW fill:#E8DEF8,stroke:#6750A4,stroke-width:2px,color:#1D192B,font-size:14px,font-weight:500 @@ -1849,16 +1959,20 @@ class report output style MinKNOW fill:#E8DEF8,stroke:#6750A4,stroke-width:2px style ROBIN fill:#EADDFF,stroke:#6750A4,stroke-width:2px """, - config={ - "theme": "redux", - "look": "neo", - "flowchart": {"curve": "basis", "defaultRenderer": "elk"}, - }, - ).classes("w-full min-w-0 workflow-diagram-mermaid") + config={ + "theme": "redux", + "look": "neo", + "flowchart": { + "curve": "basis", + "defaultRenderer": "elk", + }, + }, + ).classes("w-full min-w-0 workflow-diagram-mermaid") def register_theme_pages(): """Register the theme pages. This function should be called when the module is imported.""" + @ui.page("/") def home_page(): create_home_page() diff --git a/src/robin/gui_launcher.py b/src/robin/gui_launcher.py index ec3214ff..9a6de401 100644 --- a/src/robin/gui_launcher.py +++ b/src/robin/gui_launcher.py @@ -8,6 +8,7 @@ # Suppress pkg_resources deprecation warnings from sorted_nearest import warnings + warnings.filterwarnings( "ignore", message="pkg_resources is deprecated", category=UserWarning ) @@ -17,54 +18,49 @@ ) import asyncio -from contextlib import contextmanager +import csv +import getpass +import json import logging +import os +import pickle import queue +import secrets +import sys +import tempfile import threading import time +import uuid +import zipfile from collections import deque -import csv +from contextlib import contextmanager +from dataclasses import asdict, dataclass, field from datetime import datetime +from enum import Enum +from pathlib import Path +from typing import Any, Dict, List, Optional, Set +from urllib.parse import quote + from robin.analysis.master_csv_manager import MasterCSVManager from robin.analysis.mnpflex_eligibility import ( DEFAULT_MNPFLEX_IDLE_SECONDS, sample_ready_for_mnpflex_auto_run, sample_ready_for_mnpflex_auto_run_from_dir, ) - -from typing import Optional, Dict, Any, List, Set -from pathlib import Path -from dataclasses import dataclass, field, asdict -from enum import Enum -import os -import secrets -import sys -import tempfile -import zipfile -import json -import pickle -import getpass -import uuid -from urllib.parse import quote - -from robin.gui import theme, images -from robin.gui.session import clear_auth_session_fields, is_authenticated_session - from robin.build_info import get_git_commit - +from robin.gui import images, theme from robin.gui.components.news_feed import NewsFeed +from robin.gui.config import resolve_viewer_role +from robin.gui.session import clear_auth_session_fields, is_authenticated_session +from robin.reporting.report import create_pdf +from robin.reporting.sections.disclaimer_text import EXTENDED_DISCLAIMER_TEXT from robin.security import ( AuditService, AuthService, - get_consent_version, SecurityStore, + get_consent_version, ) -from robin.reporting.report import create_pdf -from robin.gui.config import resolve_viewer_role -from robin.reporting.sections.disclaimer_text import EXTENDED_DISCLAIMER_TEXT - - # Files that indicate an analysis step is complete. COMPLETION_JOB_PATTERNS: Dict[str, List[str]] = { "fusion": [ @@ -86,7 +82,11 @@ "target": ["coverage_main.csv", "bed_coverage_main.csv"], "sturgeon": ["sturgeon_scores.csv", "sturgeon_results.csv", "sturgeon_summary.csv"], "nanodx": ["NanoDX_scores.csv", "nanodx_results.csv", "nanodx_summary.csv"], - "pannanodx": ["PanNanoDX_scores.csv", "pannanodx_results.csv", "pannanodx_summary.csv"], + "pannanodx": [ + "PanNanoDX_scores.csv", + "pannanodx_results.csv", + "pannanodx_summary.csv", + ], "random_forest": [ "random_forest_scores.csv", "random_forest_results.csv", @@ -111,9 +111,17 @@ from robin.minknow.sample_id import ( SAMPLE_IDENTIFIER_MANIFEST_FILENAME, build_sample_registration, +) +from robin.minknow.sample_id import ( decrypt_identifier_manifest_field as _decrypt_identifier_manifest_field, +) +from robin.minknow.sample_id import ( get_test_id_from_manifest as _get_test_id_from_manifest, +) +from robin.minknow.sample_id import ( load_manifest_encrypted_fields as _load_manifest_encrypted_fields, +) +from robin.minknow.sample_id import ( normalize_dob, save_sample_identifier_manifest, save_sample_registration, @@ -138,7 +146,7 @@ def _sample_page_section_timer(page: str, sample_id: str, section: str): try: - from nicegui import ui, app, background_tasks + from nicegui import app, background_tasks, ui except ImportError: ui = None app = None @@ -146,7 +154,7 @@ def _sample_page_section_timer(page: str, sample_id: str, section: str): try: from fastapi import Request - from fastapi.responses import RedirectResponse, FileResponse + from fastapi.responses import FileResponse, RedirectResponse from starlette.middleware.base import BaseHTTPMiddleware except ImportError: # pragma: no cover Request = None @@ -156,7 +164,7 @@ def _sample_page_section_timer(page: str, sample_id: str, section: str): try: from argon2 import PasswordHasher - from argon2.exceptions import VerifyMismatchError, InvalidHashError + from argon2.exceptions import InvalidHashError, VerifyMismatchError except ImportError: # pragma: no cover PasswordHasher = None VerifyMismatchError = Exception @@ -182,7 +190,9 @@ def ensure_gui_password_set() -> bool: False otherwise (caller should exit). Requires a TTY to set a new password. """ if PasswordHasher is None: - logging.error("argon2-cffi is required for GUI password hashing. Install it with: pip install argon2-cffi") + logging.error( + "argon2-cffi is required for GUI password hashing. Install it with: pip install argon2-cffi" + ) return False path = _get_gui_password_hash_path() @@ -195,7 +205,9 @@ def ensure_gui_password_set() -> bool: logging.error("Could not read GUI password hash file: %s", e) return False if not stored: - logging.error("GUI password hash file is empty. Delete it and run again to set a new password.") + logging.error( + "GUI password hash file is empty. Delete it and run again to set a new password." + ) return False if sys.stdin.isatty(): try: @@ -275,7 +287,9 @@ def ensure_default_admin_password_set(auth_service: "AuthService") -> bool: print("Password cannot be empty.", file=sys.stderr) return False - if not auth_service.bootstrap_default_admin("".join(pwd1), username=DEFAULT_ADMIN_USERNAME): + if not auth_service.bootstrap_default_admin( + "".join(pwd1), username=DEFAULT_ADMIN_USERNAME + ): logging.error("Could not create the default admin user.") return False logging.info("Bootstrapped default admin user 'admin'") @@ -456,7 +470,9 @@ def to_dict(self) -> Dict[str, Any]: "last_seen": self.last_seen, "files_seen": self.files_seen, "files_processed": self.files_processed, - "file_progress": self.files_processed / self.files_seen if self.files_seen > 0 else 0.0, + "file_progress": ( + self.files_processed / self.files_seen if self.files_seen > 0 else 0.0 + ), "actions": "View", "_last_seen_raw": self._last_seen_raw, } @@ -552,7 +568,9 @@ def __init__(self, host: str = "0.0.0.0", port: int = 8081, reload: bool = False # RLock: _merge_job_types_with_persisted and other helpers acquire this while callers # (e.g. _update_master_record_from_workflow) may already hold it — plain Lock deadlocks. self._samples_record_lock = threading.RLock() - self._master_record_cache_file: Optional[Path] = None # Set when monitored_directory is known + self._master_record_cache_file: Optional[Path] = ( + None # Set when monitored_directory is known + ) self._background_scan_interval: float = 10.0 # Scan every 10 seconds self._background_scan_in_progress: bool = False # Sequential bulk SNP (one sample at a time; guard concurrent runs) @@ -570,6 +588,7 @@ def __init__(self, host: str = "0.0.0.0", port: int = 8081, reload: bool = False # Progress notification event from nicegui import Event + self.progress_notification_event = Event[Dict[str, Any]]() # CNV per-sample cache self._cnv_state: Dict[str, Dict[str, Any]] = {} @@ -644,7 +663,9 @@ def _get_request_context(self) -> Dict[str, str]: try: return { "ip": str(app.storage.user.get("_request_ip", "") or ""), - "user_agent": str(app.storage.user.get("_request_user_agent", "") or ""), + "user_agent": str( + app.storage.user.get("_request_user_agent", "") or "" + ), "session_id": str(app.storage.user.get("_session_id", "") or ""), "request_id": str(app.storage.user.get("_request_id", "") or ""), } @@ -715,9 +736,9 @@ def _render_admin_only_denied_page( ui.label("Access denied").classes( "classification-insight-heading text-headline-small" ) - ui.label( - "This page is available to admin users only." - ).classes("classification-insight-foot") + ui.label("This page is available to admin users only.").classes( + "classification-insight-foot" + ) ui.button( "Back to home", on_click=lambda: ui.navigate.to("/"), @@ -725,7 +746,10 @@ def _render_admin_only_denied_page( ).props("color=primary no-caps") def _current_user_has_training(self) -> bool: - from robin.security.user_approvals import TRAINING_RECEIVED_KEY, user_has_approval + from robin.security.user_approvals import ( + TRAINING_RECEIVED_KEY, + user_has_approval, + ) return user_has_approval( self.security_store, self._get_current_user_id(), TRAINING_RECEIVED_KEY @@ -752,7 +776,9 @@ def _current_user_can_remote_control_minknow(self) -> bool: def minknow_gui_accessible(self) -> bool: """MinKNOW is configured and the signed-in user may use remote control.""" - return self.minknow_gui_enabled and self._current_user_can_remote_control_minknow() + return ( + self.minknow_gui_enabled and self._current_user_can_remote_control_minknow() + ) def _notify_export_denied(self) -> None: from robin.security.user_approvals import EXPORT_DENIED_MESSAGE @@ -784,7 +810,9 @@ def _render_training_required_page( ui.label("Training approval required").classes( "classification-insight-heading text-headline-small" ) - ui.label(TRAINING_REQUIRED_MESSAGE).classes("classification-insight-foot") + ui.label(TRAINING_REQUIRED_MESSAGE).classes( + "classification-insight-foot" + ) ui.button( "Back to samples", on_click=lambda: ui.navigate.to("/live_data"), @@ -917,9 +945,15 @@ async def dispatch(self, request: Request, call_next): forwarded_for = request.headers.get("x-forwarded-for", "") real_ip = request.headers.get("x-real-ip", "") client_host = request.client.host if request.client else "" - ip = (forwarded_for.split(",")[0].strip() if forwarded_for else "") or real_ip or client_host + ip = ( + (forwarded_for.split(",")[0].strip() if forwarded_for else "") + or real_ip + or client_host + ) app.storage.user["_request_ip"] = ip - app.storage.user["_request_user_agent"] = request.headers.get("user-agent", "") + app.storage.user["_request_user_agent"] = request.headers.get( + "user-agent", "" + ) app.storage.user["_request_id"] = uuid.uuid4().hex if not app.storage.user.get("_session_id"): app.storage.user["_session_id"] = uuid.uuid4().hex @@ -939,7 +973,11 @@ async def dispatch(self, request: Request, call_next): ): return await call_next(request) gen = app.storage.general.get("_auth_generation") - if not app.storage.user.get("authenticated", False) or gen is None or app.storage.user.get("_auth_generation") != gen: + if ( + not app.storage.user.get("authenticated", False) + or gen is None + or app.storage.user.get("_auth_generation") != gen + ): clear_auth_session_fields() requested_path = path if request.url.query: @@ -949,7 +987,9 @@ async def dispatch(self, request: Request, call_next): user_id = app.storage.user.get("user_id") if user_id is not None and path != "/change-password": try: - if launcher.security_store.user_must_change_password(int(user_id)): + if launcher.security_store.user_must_change_password( + int(user_id) + ): requested_path = path if request.url.query: requested_path = f"{requested_path}?{request.url.query}" @@ -976,7 +1016,9 @@ def _get_password_hash(self) -> Optional[str]: except OSError: return None - def _verify_user_login(self, username: str, password: str) -> Optional[Dict[str, Any]]: + def _verify_user_login( + self, username: str, password: str + ) -> Optional[Dict[str, Any]]: user = self.auth_service.verify_login(username, password) if user is None: return None @@ -1002,7 +1044,9 @@ def _complete_post_login_navigation(self, user_id: int, redirect_to: str) -> Non else "/" ) if self.security_store.user_must_change_password(user_id): - ui.navigate.to(f"/change-password?redirect_to={quote(safe_target, safe='/?=&')}") + ui.navigate.to( + f"/change-password?redirect_to={quote(safe_target, safe='/?=&')}" + ) return has_consent = self.security_store.has_consent(user_id, self.consent_version) @@ -1014,13 +1058,18 @@ def _complete_post_login_navigation(self, user_id: int, redirect_to: str) -> Non ui.navigate.to(safe_target) return - with ui.dialog().props("persistent") as consent_dialog, ui.card().classes( - "robin-dialog-surface p-4 md:p-5 min-w-[18rem] max-w-2xl" + with ( + ui.dialog().props("persistent") as consent_dialog, + ui.card().classes( + "robin-dialog-surface p-4 md:p-5 min-w-[18rem] max-w-2xl" + ), ): ui.label("Research use agreement").classes( "classification-insight-heading text-headline-small q-mb-sm" ) - ui.label(EXTENDED_DISCLAIMER_TEXT).classes("classification-insight-foot q-mb-md") + ui.label(EXTENDED_DISCLAIMER_TEXT).classes( + "classification-insight-foot q-mb-md" + ) def _accept_consent() -> None: ctx = self._get_request_context() @@ -1056,7 +1105,9 @@ def _bootstrap_security_from_legacy_password(self) -> bool: return True legacy_path = _get_gui_password_hash_path() if self.auth_service.bootstrap_admin_from_legacy_hash(legacy_path): - logging.info("Bootstrapped default admin user from legacy GUI password hash") + logging.info( + "Bootstrapped default admin user from legacy GUI password hash" + ) return True return False @@ -1070,17 +1121,22 @@ def _setup_master_record_cache(self) -> None: if base.exists(): # Store cache in the monitored directory self._master_record_cache_file = base / ".samples_master_record.pkl" - logging.info(f"Master record cache file: {self._master_record_cache_file}") + logging.info( + f"Master record cache file: {self._master_record_cache_file}" + ) except Exception as e: logging.debug(f"Error setting up cache file: {e}") def _load_master_record_cache(self) -> bool: """Load master record from cache file. Returns True if successful.""" try: - if not self._master_record_cache_file or not self._master_record_cache_file.exists(): + if ( + not self._master_record_cache_file + or not self._master_record_cache_file.exists() + ): return False - with open(self._master_record_cache_file, 'rb') as f: + with open(self._master_record_cache_file, "rb") as f: cached_data = pickle.load(f) if isinstance(cached_data, dict): @@ -1090,7 +1146,9 @@ def _load_master_record_cache(self) -> bool: sid: SampleRecord(**data) if isinstance(data, dict) else data for sid, data in cached_data.items() } - logging.info(f"Loaded {len(self._samples_master_record)} samples from cache") + logging.info( + f"Loaded {len(self._samples_master_record)} samples from cache" + ) # Mark all as dirty to trigger UI refresh for record in self._samples_master_record.values(): record._dirty = True @@ -1113,8 +1171,8 @@ def _save_master_record_cache(self) -> None: } # Save to temporary file first, then rename (atomic operation) - temp_file = self._master_record_cache_file.with_suffix('.pkl.tmp') - with open(temp_file, 'wb') as f: + temp_file = self._master_record_cache_file.with_suffix(".pkl.tmp") + with open(temp_file, "wb") as f: pickle.dump(cache_data, f) temp_file.replace(self._master_record_cache_file) logging.debug(f"Saved master record cache ({len(cache_data)} samples)") @@ -1292,6 +1350,7 @@ async def _background_scan_samples_async(self) -> None: self._background_scan_in_progress = True try: import asyncio + updated_any = await asyncio.to_thread(self._background_scan_samples_sync) if updated_any and hasattr(self, "samples_table"): self._refresh_table_from_master() @@ -1365,13 +1424,18 @@ def _background_scan_samples_sync(self) -> bool: record.run_start = self._format_timestamp_for_display( first_row.get("run_info_run_time", "") ) - record.device = first_row.get("run_info_device", "") or "" - record.flowcell = first_row.get("run_info_flow_cell", "") or "" + record.device = ( + first_row.get("run_info_device", "") or "" + ) + record.flowcell = ( + first_row.get("run_info_flow_cell", "") or "" + ) # Update last_seen from saved value or use file mtime try: saved_last = float( - first_row.get("samples_overview_last_seen", 0.0) or 0.0 + first_row.get("samples_overview_last_seen", 0.0) + or 0.0 ) if saved_last > 0: record._last_seen_raw = saved_last @@ -1382,22 +1446,27 @@ def _background_scan_samples_sync(self) -> bool: # Update job counts from persisted overview record.active_jobs = int( - first_row.get("samples_overview_active_jobs", 0) or 0 + first_row.get("samples_overview_active_jobs", 0) + or 0 ) record.pending_jobs = int( - first_row.get("samples_overview_pending_jobs", 0) or 0 + first_row.get("samples_overview_pending_jobs", 0) + or 0 ) record.total_jobs = int( first_row.get("samples_overview_total_jobs", 0) or 0 ) record.completed_jobs = int( - first_row.get("samples_overview_completed_jobs", 0) or 0 + first_row.get("samples_overview_completed_jobs", 0) + or 0 ) record.failed_jobs = int( - first_row.get("samples_overview_failed_jobs", 0) or 0 + first_row.get("samples_overview_failed_jobs", 0) + or 0 ) record.job_types = str( - first_row.get("samples_overview_job_types", "") or "" + first_row.get("samples_overview_job_types", "") + or "" ) except Exception as e: logging.debug(f"Error reading master.csv for {sid}: {e}") @@ -1412,7 +1481,11 @@ def _background_scan_samples_sync(self) -> bool: # Do not leave new records as default "Live" when this folder is already # finished on disk — that would look like a Live→Complete transition and # re-run target.bam finalization on every restart. - if record.origin == "Live" and (now_ts - record._last_seen_raw) >= self.completion_timeout_seconds: + if ( + record.origin == "Live" + and (now_ts - record._last_seen_raw) + >= self.completion_timeout_seconds + ): if record.active_jobs == 0 and record.pending_jobs == 0: expected = self._get_expected_completion_job_types() if expected: @@ -1420,12 +1493,12 @@ def _background_scan_samples_sync(self) -> bool: sample_dir, expected ) else: - complete_on_disk = self._is_target_bam_finalize_redundant( - sid + complete_on_disk = ( + self._is_target_bam_finalize_redundant(sid) ) if not complete_on_disk: - complete_on_disk = self._is_target_bam_finalize_redundant( - sid + complete_on_disk = ( + self._is_target_bam_finalize_redundant(sid) ) if complete_on_disk: record.origin = "Complete" @@ -1434,21 +1507,37 @@ def _background_scan_samples_sync(self) -> bool: # Update origin based on inactivity timeout AND active jobs status prev_origin = record.origin # Only mark as Complete if timeout passed AND no active jobs - if record.origin == "Live" and (now_ts - record._last_seen_raw) >= self.completion_timeout_seconds: + if ( + record.origin == "Live" + and (now_ts - record._last_seen_raw) + >= self.completion_timeout_seconds + ): if record.active_jobs == 0 and record.pending_jobs == 0: record.origin = "Complete" record._dirty = True # Trigger finalization if transitioning from Live to Complete - if prev_origin == "Live" and sid not in self._finalized_samples: + if ( + prev_origin == "Live" + and sid not in self._finalized_samples + ): self._trigger_target_bam_finalization(sid) self._finalized_samples.add(sid) # If there are active jobs, keep as Live even if timeout passed - elif record.origin == "Pre-existing" and (now_ts - record._last_seen_raw) >= self.completion_timeout_seconds: + elif ( + record.origin == "Pre-existing" + and (now_ts - record._last_seen_raw) + >= self.completion_timeout_seconds + ): # Keep as Pre-existing if it was pre-existing and still inactive pass elif record.origin == "Complete": # Reactivate if file was modified recently OR if there are active jobs - if (now_ts - record._last_seen_raw) < self.completion_timeout_seconds or record.active_jobs > 0 or record.pending_jobs > 0: + if ( + (now_ts - record._last_seen_raw) + < self.completion_timeout_seconds + or record.active_jobs > 0 + or record.pending_jobs > 0 + ): record.origin = "Live" record._dirty = True @@ -1480,7 +1569,9 @@ def _background_scan_samples_sync(self) -> bool: for sid in to_remove: del self._samples_master_record[sid] updated_any = True - logging.debug(f"Removed {len(to_remove)} deleted samples from master record") + logging.debug( + f"Removed {len(to_remove)} deleted samples from master record" + ) # Save cache if anything changed if updated_any: @@ -1516,7 +1607,9 @@ def _refresh_table_from_master(self) -> None: except Exception as e: logging.error(f"Error refreshing table from master: {e}") - def _update_master_record_from_workflow(self, samples_data: List[Dict[str, Any]]) -> None: + def _update_master_record_from_workflow( + self, samples_data: List[Dict[str, Any]] + ) -> None: """Update master record from workflow polling data. This merges workflow stats into the master record without doing file I/O.""" try: @@ -1535,7 +1628,9 @@ def _update_master_record_from_workflow(self, samples_data: List[Dict[str, Any]] record = SampleRecord( sample_id=sid, _last_seen_raw=last_seen, - last_seen=time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(last_seen)), + last_seen=time.strftime( + "%Y-%m-%d %H:%M:%S", time.localtime(last_seen) + ), ) if sid in self._preexisting_sample_ids: record.origin = "Pre-existing" @@ -1575,7 +1670,11 @@ def _update_master_record_from_workflow(self, samples_data: List[Dict[str, Any]] # Persist updates to master.csv try: - base = Path(self.monitored_directory) if self.monitored_directory else None + base = ( + Path(self.monitored_directory) + if self.monitored_directory + else None + ) if base and base.exists(): manager = MasterCSVManager(str(base)) for record in self._samples_master_record.values(): @@ -1589,7 +1688,9 @@ def _update_master_record_from_workflow(self, samples_data: List[Dict[str, Any]] "job_types": record.job_types, "last_seen": float(record._last_seen_raw), } - manager.update_sample_overview(record.sample_id, persist_payload) + manager.update_sample_overview( + record.sample_id, persist_payload + ) record._dirty = False except Exception as e: logging.debug(f"Error persisting workflow updates: {e}") @@ -1679,7 +1780,9 @@ def launch_gui( self._setup_master_record_cache() # Try to load cache immediately for fast startup if self._load_master_record_cache(): - logging.info("Loaded samples from cache - table will populate immediately") + logging.info( + "Loaded samples from cache - table will populate immediately" + ) # Skip spurious target.bam finalization on restart when outputs already exist self._seed_finalized_samples_from_disk() @@ -1748,7 +1851,10 @@ def send_update( # Rate limiting: Skip low-priority updates if queue is getting too large if queue_size > threshold and priority < 5: # Only log when queue size changes significantly to reduce log spam - if abs(queue_size - self._last_queue_size_logged) > 50 or (current_time - self._last_log_time) > 10.0: + if ( + abs(queue_size - self._last_queue_size_logged) > 50 + or (current_time - self._last_log_time) > 10.0 + ): logging.warning( f"[GUI] Skipping low-priority update due to queue size: " f"{queue_size} (threshold: {threshold})" @@ -1822,9 +1928,9 @@ def _setup_notification_system(self, container): def _show_notification_in_container(self, container, data: Dict[str, Any]): """Show notification in the dedicated container.""" try: - message = data.get('message', '') - notification_type = data.get('type', 'info') - timeout = data.get('timeout', 5000) + message = data.get("message", "") + notification_type = data.get("type", "info") + timeout = data.get("timeout", 5000) # Create notification in the container with container: @@ -1832,7 +1938,7 @@ def _show_notification_in_container(self, container, data: Dict[str, Any]): message, type=notification_type, timeout=timeout, - position="top-right" + position="top-right", ) except Exception as e: @@ -1843,16 +1949,13 @@ def _handle_progress_notification_event(self, event_data: Dict[str, Any]): try: from nicegui import ui - message = event_data.get('message', '') - notification_type = event_data.get('type', 'info') - timeout = event_data.get('timeout', 5000) + message = event_data.get("message", "") + notification_type = event_data.get("type", "info") + timeout = event_data.get("timeout", 5000) # Show notification in the proper UI context ui.notify( - message, - type=notification_type, - timeout=timeout, - position="top-right" + message, type=notification_type, timeout=timeout, position="top-right" ) except Exception as e: @@ -1863,41 +1966,47 @@ def _handle_progress_update(self, update: Dict[str, Any]): try: from nicegui import ui - sample_id = update['sample_id'] - update_type = update['type'] + sample_id = update["sample_id"] + update_type = update["type"] - if update_type == 'start': + if update_type == "start": # Emit event for initial notification - self.progress_notification_event.emit({ - 'sample_id': sample_id, - 'message': f"[{sample_id}] Starting report generation...", - 'type': 'info', - 'timeout': 3000 # 3 seconds - }) + self.progress_notification_event.emit( + { + "sample_id": sample_id, + "message": f"[{sample_id}] Starting report generation...", + "type": "info", + "timeout": 3000, # 3 seconds + } + ) logging.debug(f"Started report generation for {sample_id}") - elif update_type == 'update': - stage = update['stage'] - message = update['message'] - progress = update.get('progress') + elif update_type == "update": + stage = update["stage"] + message = update["message"] + progress = update.get("progress") # Calculate progress percentage - progress_percent = int((progress or 0.0) * 100) if progress is not None else "" + progress_percent = ( + int((progress or 0.0) * 100) if progress is not None else "" + ) progress_text = f" ({progress_percent}%)" if progress_percent else "" # Emit event for progress notification - self.progress_notification_event.emit({ - 'sample_id': sample_id, - 'message': f"[{sample_id}] {message}{progress_text}", - 'type': 'info', - 'timeout': 2000 # 2 seconds for progress updates - }) + self.progress_notification_event.emit( + { + "sample_id": sample_id, + "message": f"[{sample_id}] {message}{progress_text}", + "type": "info", + "timeout": 2000, # 2 seconds for progress updates + } + ) logging.debug(f"Updated progress for {sample_id}: {stage} - {message}") - elif update_type == 'complete': - filename = update.get('filename') + elif update_type == "complete": + filename = update.get("filename") # Show completion notification completion_message = f"[{sample_id}] Report generation completed" @@ -1905,27 +2014,33 @@ def _handle_progress_update(self, update: Dict[str, Any]): completion_message += f": {filename}" # Emit event for completion notification - self.progress_notification_event.emit({ - 'sample_id': sample_id, - 'message': completion_message, - 'type': 'positive', - 'timeout': 5000 - }) + self.progress_notification_event.emit( + { + "sample_id": sample_id, + "message": completion_message, + "type": "positive", + "timeout": 5000, + } + ) logging.info(f"Completed report generation for {sample_id}") - elif update_type == 'error': - error_message = update['error_message'] + elif update_type == "error": + error_message = update["error_message"] # Emit event for error notification - self.progress_notification_event.emit({ - 'sample_id': sample_id, - 'message': f"[{sample_id}] Report generation failed: {error_message}", - 'type': 'negative', - 'timeout': 10000 - }) + self.progress_notification_event.emit( + { + "sample_id": sample_id, + "message": f"[{sample_id}] Report generation failed: {error_message}", + "type": "negative", + "timeout": 10000, + } + ) - logging.error(f"Report generation failed for {sample_id}: {error_message}") + logging.error( + f"Report generation failed for {sample_id}: {error_message}" + ) except Exception as e: logging.error(f"Error handling progress update: {e}") @@ -2244,7 +2359,9 @@ def _update_progress(self, data: Dict[str, Any]): def _update_file_progress(self): """Update file progress display in workflow monitor from current samples data.""" try: - if not hasattr(self, "sample_files_progress_container") or not hasattr(self, "_last_samples_rows"): + if not hasattr(self, "sample_files_progress_container") or not hasattr( + self, "_last_samples_rows" + ): return rows = self._last_samples_rows or [] @@ -2261,18 +2378,30 @@ def _update_file_progress(self): if files_seen > 0: total_files_seen += files_seen total_files_processed += files_processed - sample_progress_data.append({ - "sample_id": sample_id, - "files_seen": files_seen, - "files_processed": files_processed, - "progress": files_processed / files_seen if files_seen > 0 else 0.0 - }) + sample_progress_data.append( + { + "sample_id": sample_id, + "files_seen": files_seen, + "files_processed": files_processed, + "progress": ( + files_processed / files_seen if files_seen > 0 else 0.0 + ), + } + ) # Update overall progress - if hasattr(self, "overall_files_progress") and hasattr(self, "overall_files_label"): - overall_progress = total_files_processed / total_files_seen if total_files_seen > 0 else 0.0 + if hasattr(self, "overall_files_progress") and hasattr( + self, "overall_files_label" + ): + overall_progress = ( + total_files_processed / total_files_seen + if total_files_seen > 0 + else 0.0 + ) self.overall_files_progress.set_value(round(overall_progress, 2)) - self.overall_files_label.set_text(f"{total_files_processed}/{total_files_seen} files processed") + self.overall_files_label.set_text( + f"{total_files_processed}/{total_files_seen} files processed" + ) # Update per-sample progress bars (limit to first 20 to avoid UI overload) if hasattr(self, "sample_files_progress_container"): @@ -2310,9 +2439,9 @@ def _update_file_progress(self): else: # Show placeholder when no data with self.sample_files_progress_container: - ui.label( - "No file progress data available yet." - ).classes("classification-insight-foot italic") + ui.label("No file progress data available yet.").classes( + "classification-insight-foot italic" + ) except Exception as e: logging.debug(f"Error updating per-sample file progress: {e}") @@ -2416,14 +2545,18 @@ async def _robin_dark_mode(request: Request): def login_page(redirect_to: str = "/"): """Authenticate user session before allowing access.""" if self._is_authenticated(): - safe_target = redirect_to if redirect_to and redirect_to != "/login" else "/" + safe_target = ( + redirect_to if redirect_to and redirect_to != "/login" else "/" + ) return RedirectResponse(safe_target) clear_auth_session_fields() def try_login() -> None: username_value = str(username.value or "").strip() - login_result = self._verify_user_login(username_value, password.value) + login_result = self._verify_user_login( + username_value, password.value + ) if login_result is None: self._audit_log( event_type="auth.login.failure", @@ -2439,7 +2572,9 @@ def try_login() -> None: app.storage.user.update( { "authenticated": True, - "_auth_generation": app.storage.general.get("_auth_generation"), + "_auth_generation": app.storage.general.get( + "_auth_generation" + ), "user_id": login_result["user_id"], "username": login_result["username"], "roles": login_result["roles"], @@ -2461,7 +2596,9 @@ def try_login() -> None: }, ) - safe_target = redirect_to if redirect_to and redirect_to != "/login" else "/" + safe_target = ( + redirect_to if redirect_to and redirect_to != "/login" else "/" + ) self._complete_post_login_navigation( int(login_result["user_id"]), safe_target, @@ -2474,8 +2611,10 @@ def try_login() -> None: batphone=self.batman_mode, center=self.center, ): - with ui.element("div").classes("w-full min-w-0").props( - "id=login-page" + with ( + ui.element("div") + .classes("w-full min-w-0") + .props("id=login-page") ): with ui.column().classes( "w-full min-h-[70vh] items-center justify-center " @@ -2513,7 +2652,9 @@ def try_login() -> None: ) username = ( ui.input("Username") - .props("autocomplete=username outlined dense") + .props( + "autocomplete=username outlined dense" + ) .classes("w-full") .on("keydown.enter", try_login) ) @@ -2777,7 +2918,8 @@ def download_file(sample_id: str, filename: str): return # Security: Only allow alphanumeric characters and common file extensions import re - if not re.match(r'^[a-zA-Z0-9._-]+$', filename): + + if not re.match(r"^[a-zA-Z0-9._-]+$", filename): self._audit_log( event_type="report.exported", result="failure", @@ -2791,7 +2933,11 @@ def download_file(sample_id: str, filename: str): return # Find the sample directory - base_dir = Path(self.monitored_directory) if self.monitored_directory else None + base_dir = ( + Path(self.monitored_directory) + if self.monitored_directory + else None + ) if not base_dir or not base_dir.exists(): self._audit_log( event_type="report.exported", @@ -2872,15 +3018,13 @@ def download_file(sample_id: str, filename: str): # Setup global CSS and static files - moved to a helper function def _setup_global_resources(): """Setup global CSS and static file resources.""" - ui.add_css( - """ + ui.add_css(""" .shadows-into light-regular { font-family: "Shadows Into Light", cursive; font-weight: 800; font-style: normal; } - """ - ) + """) # Register fonts from the GUI package if available try: fonts_dir = Path(__file__).parent / "gui" / "fonts" @@ -2910,7 +3054,11 @@ def _setup_global_timers(): once=True, ) # Subsequent scans every 10 seconds - app.timer(self._background_scan_interval, self._background_scan_samples, active=True) + app.timer( + self._background_scan_interval, + self._background_scan_samples, + active=True, + ) # If cache was not loaded, do initial preexisting scan after GUI is ready if not self._samples_master_record: @@ -2941,12 +3089,11 @@ def _setup_global_timers(): try: from robin_native_api import attach_native_api + attach_native_api(app, self) except Exception as e: logging.warning("Native API not mounted: %s", e) - - # Start the GUI ui.run( host=self.host, @@ -3102,7 +3249,9 @@ def _create_samples_overview(self): ui.label("All tracked samples").classes( "text-headline-medium text-slate-900 dark:text-slate-50 shrink-0" ) - ui.label("A=Active P=Pending T=Total C=Completed F=Failed").classes( + ui.label( + "A=Active P=Pending T=Total C=Completed F=Failed" + ).classes( "text-[11px] text-slate-600 dark:text-slate-400 shrink-0" ) with ui.row().classes( @@ -3121,32 +3270,40 @@ def _create_samples_overview(self): "Clear selection", on_click=lambda: self._samples_clear_export_selection(), ).props("flat dense no-caps outline") - self.bulk_snp_button = ui.button( - "SNP: all missing", - icon="biotech", - on_click=lambda: None, - ).props("color=secondary dense no-caps").classes( - "border border-slate-300 dark:border-slate-600" + self.bulk_snp_button = ( + ui.button( + "SNP: all missing", + icon="biotech", + on_click=lambda: None, + ) + .props("color=secondary dense no-caps") + .classes( + "border border-slate-300 dark:border-slate-600" + ) ) # Optional bulk MNP-Flex action (Docker or API backend) if self._is_mnpflex_enabled_for_gui(): - self.bulk_mnpflex_button = ui.button( - "mnpflex run all", - on_click=lambda: None, - ).props( - "color=secondary dense no-caps" - ).classes( - "border border-slate-300 dark:border-slate-600" + self.bulk_mnpflex_button = ( + ui.button( + "mnpflex run all", + on_click=lambda: None, + ) + .props("color=secondary dense no-caps") + .classes( + "border border-slate-300 dark:border-slate-600" + ) ) else: self.bulk_mnpflex_button = None if can_export_reports: - self.export_reports_button = ui.button( - "Export reports", - on_click=lambda: None, - ).props("color=primary").classes( - "rounded-lg px-4 text-title-medium" + self.export_reports_button = ( + ui.button( + "Export reports", + on_click=lambda: None, + ) + .props("color=primary") + .classes("rounded-lg px-4 text-title-medium") ) self.export_reports_button.disable() else: @@ -3180,7 +3337,9 @@ def _create_samples_overview(self): (self._samples_filters or {}).get("query", "") ), "origin": str( - (self._samples_filters or {}).get("origin", "All") + (self._samples_filters or {}).get( + "origin", "All" + ) ) or "All", "job_type": "All", @@ -3208,7 +3367,8 @@ def _create_samples_overview(self): self.samples_search.value = current_query except Exception as e: logging.debug( - "[samples_overview] could not preset search filter: %s", e + "[samples_overview] could not preset search filter: %s", + e, ) # Origin filter @@ -3250,7 +3410,9 @@ def _create_samples_overview(self): ui.label("Loading samples…").classes( "ml-2 text-title-medium text-slate-800 dark:text-slate-100" ) - ui.label("This may take a moment for large directories").classes( + ui.label( + "This may take a moment for large directories" + ).classes( "text-body-small text-slate-500 dark:text-slate-400 mt-2" ) @@ -3259,118 +3421,118 @@ def _create_samples_overview(self): # Create samples table _samples_table_columns = [ - { - "name": "actions", - "label": "Actions", - "field": "actions", - }, - { - "name": "sample_id", - "label": "Library ID", - "field": "sample_id", - "sortable": True, - }, - { - "name": "test_id", - "label": "Test ID", - "field": "test_id", - "sortable": True, - }, - { - "name": "origin", - "label": "Origin", - "field": "origin", - "sortable": True, - }, - { - "name": "run_start", - "label": "Run Start", - "field": "run_start", - "sortable": True, - }, - { - "name": "device", - "label": "Device", - "field": "device", - "sortable": True, - }, - { - "name": "flowcell", - "label": "Flowcell", - "field": "flowcell", - "sortable": True, - }, - { - "name": "file_progress", - "label": "Job Progress", - "field": "file_progress", - "sortable": True, - }, - { - "name": "pipeline_progress", - "label": "Finalize/SNP", - "field": "pipeline_progress", - "sortable": True, - }, - { - "name": "active_jobs", - "label": "A", - "field": "active_jobs", - "sortable": True, - "align": "center", - "style": "width:52px; max-width:52px;", - "headerStyle": "width:52px; max-width:52px;", - }, - { - "name": "pending_jobs", - "label": "P", - "field": "pending_jobs", - "sortable": True, - "align": "center", - "style": "width:52px; max-width:52px;", - "headerStyle": "width:52px; max-width:52px;", - }, - { - "name": "total_jobs", - "label": "T", - "field": "total_jobs", - "sortable": True, - "align": "center", - "style": "width:52px; max-width:52px;", - "headerStyle": "width:52px; max-width:52px;", - }, - { - "name": "completed_jobs", - "label": "C", - "field": "completed_jobs", - "sortable": True, - "align": "center", - "style": "width:52px; max-width:52px;", - "headerStyle": "width:52px; max-width:52px;", - }, - { - "name": "failed_jobs", - "label": "F", - "field": "failed_jobs", - "sortable": True, - "align": "center", - "style": "width:52px; max-width:52px;", - "headerStyle": "width:52px; max-width:52px;", - }, - { - "name": "job_types", - "label": "Job Types", - "field": "job_types", - "sortable": True, - "style": "min-width: 11rem;", - "headerStyle": "min-width: 11rem;", - }, - { - "name": "last_seen", - "label": "Last Activity", - "field": "last_seen", - "sortable": True, - }, + { + "name": "actions", + "label": "Actions", + "field": "actions", + }, + { + "name": "sample_id", + "label": "Library ID", + "field": "sample_id", + "sortable": True, + }, + { + "name": "test_id", + "label": "Test ID", + "field": "test_id", + "sortable": True, + }, + { + "name": "origin", + "label": "Origin", + "field": "origin", + "sortable": True, + }, + { + "name": "run_start", + "label": "Run Start", + "field": "run_start", + "sortable": True, + }, + { + "name": "device", + "label": "Device", + "field": "device", + "sortable": True, + }, + { + "name": "flowcell", + "label": "Flowcell", + "field": "flowcell", + "sortable": True, + }, + { + "name": "file_progress", + "label": "Job Progress", + "field": "file_progress", + "sortable": True, + }, + { + "name": "pipeline_progress", + "label": "Finalize/SNP", + "field": "pipeline_progress", + "sortable": True, + }, + { + "name": "active_jobs", + "label": "A", + "field": "active_jobs", + "sortable": True, + "align": "center", + "style": "width:52px; max-width:52px;", + "headerStyle": "width:52px; max-width:52px;", + }, + { + "name": "pending_jobs", + "label": "P", + "field": "pending_jobs", + "sortable": True, + "align": "center", + "style": "width:52px; max-width:52px;", + "headerStyle": "width:52px; max-width:52px;", + }, + { + "name": "total_jobs", + "label": "T", + "field": "total_jobs", + "sortable": True, + "align": "center", + "style": "width:52px; max-width:52px;", + "headerStyle": "width:52px; max-width:52px;", + }, + { + "name": "completed_jobs", + "label": "C", + "field": "completed_jobs", + "sortable": True, + "align": "center", + "style": "width:52px; max-width:52px;", + "headerStyle": "width:52px; max-width:52px;", + }, + { + "name": "failed_jobs", + "label": "F", + "field": "failed_jobs", + "sortable": True, + "align": "center", + "style": "width:52px; max-width:52px;", + "headerStyle": "width:52px; max-width:52px;", + }, + { + "name": "job_types", + "label": "Job Types", + "field": "job_types", + "sortable": True, + "style": "min-width: 11rem;", + "headerStyle": "min-width: 11rem;", + }, + { + "name": "last_seen", + "label": "Last Activity", + "field": "last_seen", + "sortable": True, + }, ] if can_export_reports: _samples_table_columns.append( @@ -3579,12 +3741,18 @@ def _create_samples_overview(self): def _on_finalize_target(event): try: logging.debug("finalize-target event: %r", event) - sample_id = getattr(event, "args", None) if hasattr(event, "args") else None + sample_id = ( + getattr(event, "args", None) + if hasattr(event, "args") + else None + ) logging.debug("finalize-target sample_id=%r", sample_id) if isinstance(sample_id, str): # Keep this UI callback non-blocking. # All heavyweight checks/submissions run in background logic. - self._trigger_target_bam_finalization(sample_id, trigger_snp=True) + self._trigger_target_bam_finalization( + sample_id, trigger_snp=True + ) logging.debug( "finalize-target queued: sample_id=%s trigger_snp=True", sample_id, @@ -3597,7 +3765,9 @@ def _on_finalize_target(event): else: ui.notify("Invalid sample ID", type="warning") except Exception as e: - ui.notify(f"Error triggering finalization: {e}", type="negative") + ui.notify( + f"Error triggering finalization: {e}", type="negative" + ) try: self.samples_table.on("finalize-target", _on_finalize_target) @@ -3624,13 +3794,17 @@ def _on_export_toggled(event): sid = payload.get("id") val = bool(payload.get("value")) if sid: - selected_ids = self._get_selected_sample_ids() + selected_ids = ( + self._get_selected_sample_ids() + ) if val: selected_ids.add(str(sid)) else: selected_ids.discard(str(sid)) - selected_ids = self._set_selected_sample_ids( - selected_ids + selected_ids = ( + self._set_selected_sample_ids( + selected_ids + ) ) # reflect state back into rows try: @@ -3647,7 +3821,10 @@ def _on_export_toggled(event): exc_info=True, ) if selected_ids: - if self.export_reports_button is not None: + if ( + self.export_reports_button + is not None + ): self.export_reports_button.enable() elif self.export_reports_button is not None: self.export_reports_button.disable() @@ -3671,9 +3848,11 @@ def _on_export_header_toggle(event): try: logging.info( "[samples_overview] export-header-toggle raw_args=%r", - getattr(event, "args", None) - if hasattr(event, "args") - else event, + ( + getattr(event, "args", None) + if hasattr(event, "args") + else event + ), ) val = True if hasattr(event, "args"): @@ -3753,7 +3932,9 @@ async def _ask_run_bulk_snp(): with ui.card().classes( "robin-dialog-surface w-96 max-w-[95vw] p-4" ): - ui.label("Run SNP for all missing samples?").classes( + ui.label( + "Run SNP for all missing samples?" + ).classes( "classification-insight-heading text-headline-small mb-2" ) ui.label( @@ -3761,7 +3942,9 @@ async def _ask_run_bulk_snp(): "(same as triggering SNP manually each time). " "This may take a long time overall." ).classes("text-sm text-gray-600 mb-4") - with ui.row().classes("justify-end gap-2 flex-wrap"): + with ui.row().classes( + "justify-end gap-2 flex-wrap" + ): ui.button( "Cancel", on_click=lambda: dlg.submit(False), @@ -3857,18 +4040,14 @@ async def _ask_run_bulk_mnpflex(): ui.label( f"{n} sample(s) will be processed one after another. " "This can take a long time overall." - ).classes( - "text-sm text-gray-600 mb-4" - ) + ).classes("text-sm text-gray-600 mb-4") with ui.row().classes( "justify-end gap-2 flex-wrap" ): ui.button( "Cancel", on_click=lambda: dlg.submit(False), - ).props( - "flat no-caps outline" - ) + ).props("flat no-caps outline") ui.button( "Start", on_click=lambda: dlg.submit(True), @@ -3901,11 +4080,16 @@ def _run(): exc_info=True, ) - async def _export_selected_reports(state: Dict[str, Any], progress_dialog, files_to_download, download_complete, progress_callback, progress_updates): + async def _export_selected_reports( + state: Dict[str, Any], + progress_dialog, + files_to_download, + download_complete, + progress_callback, + progress_updates, + ): try: - selected = list( - self._get_selected_sample_ids() or [] - ) + selected = list(self._get_selected_sample_ids() or []) if not selected: ui.notify("No samples selected", type="warning") return @@ -3928,12 +4112,14 @@ async def _export_selected_reports(state: Dict[str, Any], progress_dialog, files # Emit progress update showing current sample and mark as starting current_sample_msg = f"Generating {idx + 1}/{total_samples} - {sid}" - progress_updates.put({ - 'stage': 'processing_sections', - 'message': 'Starting...', - 'progress': 0.0, - 'sample_id': sid - }) + progress_updates.put( + { + "stage": "processing_sections", + "message": "Starting...", + "progress": 0.0, + "sample_id": sid, + } + ) sample_dir = ( Path(self.monitored_directory) / sid @@ -3967,13 +4153,17 @@ async def _export_selected_reports(state: Dict[str, Any], progress_dialog, files ) sample_outputs: List[str] = [] - report_meta = self._report_generation_metadata() + report_meta = ( + self._report_generation_metadata() + ) # Don't use the notification system - use only our dialog callback if ng_run is not None: # Use custom callback that updates dialog only - def sample_progress_callback(data: Dict[str, Any]): - data['sample_id'] = sid + def sample_progress_callback( + data: Dict[str, Any], + ): + data["sample_id"] = sid progress_callback(data) pdf_file = await ng_run.io_bound( @@ -3981,45 +4171,89 @@ def sample_progress_callback(data: Dict[str, Any]): pdf_path, str(sample_dir), self.center or "Unknown", - report_type=state.get("type", "detailed"), + report_type=state.get( + "type", "detailed" + ), export_csv_dir=export_csv_dir, export_xlsx=False, export_zip=bool( state.get("export_csv", False) ), progress_callback=sample_progress_callback, - workflow_steps=self.workflow_steps if hasattr(self, 'workflow_steps') else None, - display_config=self.display_config if hasattr(self, 'display_config') else None, - viewer_role=resolve_viewer_role(self), - generated_by=report_meta["generated_by"] or None, - generated_at=report_meta["generated_at"], + workflow_steps=( + self.workflow_steps + if hasattr( + self, "workflow_steps" + ) + else None + ), + display_config=( + self.display_config + if hasattr( + self, "display_config" + ) + else None + ), + viewer_role=resolve_viewer_role( + self + ), + generated_by=report_meta[ + "generated_by" + ] + or None, + generated_at=report_meta[ + "generated_at" + ], plotting_preferences=self.plotting_preferences, ) else: # Use custom callback that updates dialog only - def sample_progress_callback(data: Dict[str, Any]): - data['sample_id'] = sid + def sample_progress_callback( + data: Dict[str, Any], + ): + data["sample_id"] = sid progress_callback(data) pdf_file = create_pdf( pdf_path, str(sample_dir), self.center or "Unknown", - report_type=state.get("type", "detailed"), + report_type=state.get( + "type", "detailed" + ), export_csv_dir=export_csv_dir, export_xlsx=False, export_zip=bool( state.get("export_csv", False) ), progress_callback=sample_progress_callback, - workflow_steps=self.workflow_steps if hasattr(self, 'workflow_steps') else None, - display_config=self.display_config if hasattr(self, 'display_config') else None, - viewer_role=resolve_viewer_role(self), - generated_by=report_meta["generated_by"] or None, - generated_at=report_meta["generated_at"], - plotting_preferences=self.plotting_preferences, - ) - + workflow_steps=( + self.workflow_steps + if hasattr( + self, "workflow_steps" + ) + else None + ), + display_config=( + self.display_config + if hasattr( + self, "display_config" + ) + else None + ), + viewer_role=resolve_viewer_role( + self + ), + generated_by=report_meta[ + "generated_by" + ] + or None, + generated_at=report_meta[ + "generated_at" + ], + plotting_preferences=self.plotting_preferences, + ) + if bool(state.get("export_pdf", True)): files_to_download.append(pdf_file) sample_outputs.append(pdf_file) @@ -4030,7 +4264,8 @@ def sample_progress_callback(data: Dict[str, Any]): and export_csv_dir ): zip_path = os.path.join( - export_csv_dir, f"{sid}_report_data.zip" + export_csv_dir, + f"{sid}_report_data.zip", ) if os.path.exists(zip_path): files_to_download.append(zip_path) @@ -4040,24 +4275,30 @@ def sample_progress_callback(data: Dict[str, Any]): state=state, target_id=sid, output_files=sample_outputs, - generated_at=report_meta["generated_at"], + generated_at=report_meta[ + "generated_at" + ], extra_details={"bulk_export": True}, ) else: - progress_updates.put({ - 'stage': 'processing_sections', - 'message': 'No PDF/CSV selected; skipping report build', - 'progress': 1.0, - 'sample_id': sid - }) + progress_updates.put( + { + "stage": "processing_sections", + "message": "No PDF/CSV selected; skipping report build", + "progress": 1.0, + "sample_id": sid, + } + ) # Mark sample as complete - progress_updates.put({ - 'stage': 'completed', - 'message': 'Completed', - 'progress': 1.0, - 'sample_id': sid - }) + progress_updates.put( + { + "stage": "completed", + "message": "Completed", + "progress": 1.0, + "sample_id": sid, + } + ) except Exception as e: # Report generation failed @@ -4071,16 +4312,20 @@ def sample_progress_callback(data: Dict[str, Any]): extra_details={"bulk_export": True}, ) # Mark sample as failed - progress_updates.put({ - 'stage': 'error', - 'message': f'Failed: {str(e)[:50]}', - 'progress': 1.0, - 'sample_id': sid - }) + progress_updates.put( + { + "stage": "error", + "message": f"Failed: {str(e)[:50]}", + "progress": 1.0, + "sample_id": sid, + } + ) # Mark as complete download_complete["done"] = True - logging.info(f"Bulk export complete. {len(files_to_download)} file(s) ready for download.") + logging.info( + f"Bulk export complete. {len(files_to_download)} file(s) ready for download." + ) except Exception as e: logging.error(f"Error in bulk export: {e}") download_complete["done"] = True @@ -4163,14 +4408,20 @@ async def _confirm_bulk_export(): f"Are you sure you want to export reports for {num_selected} sample(s)?" ).classes("classification-insight-foot mb-4") - with ui.row().classes("justify-end gap-2 flex-wrap"): + with ui.row().classes( + "justify-end gap-2 flex-wrap" + ): ui.button( "Cancel", - on_click=lambda: dialog.submit("Cancel"), + on_click=lambda: dialog.submit( + "Cancel" + ), ).props("flat no-caps outline") ui.button( "Export", - on_click=lambda: dialog.submit("Export"), + on_click=lambda: dialog.submit( + "Export" + ), icon="download", ).props("color=primary no-caps") @@ -4181,9 +4432,7 @@ async def _confirm_bulk_export(): if num_selected > 3 else "" ) - ).classes( - "text-sm font-medium text-gray-700 mt-4" - ) + ).classes("text-sm font-medium text-gray-700 mt-4") dialog_result = await dialog if dialog_result != "Export": @@ -4202,11 +4451,15 @@ async def _confirm_bulk_export(): tsv_export_path = None if bool(state.get("export_tsv", True)): report_meta = self._report_generation_metadata() - tsv_export_path = self._build_sample_tracking_tsv_export( - selected_ids, - generated_by=report_meta["generated_by"] or None, - generated_at=report_meta["generated_at"], - robin_commit=report_meta.get("robin_commit") or None, + tsv_export_path = ( + self._build_sample_tracking_tsv_export( + selected_ids, + generated_by=report_meta["generated_by"] + or None, + generated_at=report_meta["generated_at"], + robin_commit=report_meta.get("robin_commit") + or None, + ) ) if tsv_export_path and os.path.exists(tsv_export_path): self._audit_report_generated( @@ -4230,9 +4483,9 @@ async def _confirm_bulk_export(): "classification-insight-heading text-headline-small mb-3" ) - ui.label(f"Exporting {num_selected} report(s)").classes( - "text-sm font-medium text-gray-700 mb-4" - ) + ui.label( + f"Exporting {num_selected} report(s)" + ).classes("text-sm font-medium text-gray-700 mb-4") # Report type and output displays ui.label( @@ -4257,14 +4510,24 @@ async def _confirm_bulk_export(): with ui.column().classes("w-full"): for sid in selected_ids: with ui.column().classes("mb-3 w-full"): - ui.label(sid).classes("text-xs font-medium text-gray-700 mb-1") - progress_bar = ui.linear_progress(0.0).classes("mb-1") - progress_label = ui.label("Waiting...").classes("text-xs text-gray-500") + ui.label(sid).classes( + "text-xs font-medium text-gray-700 mb-1" + ) + progress_bar = ui.linear_progress( + 0.0 + ).classes("mb-1") + progress_label = ui.label( + "Waiting..." + ).classes("text-xs text-gray-500") sample_progress_bars[sid] = progress_bar - sample_progress_labels[sid] = progress_label + sample_progress_labels[sid] = ( + progress_label + ) # Messages container - use label with newlines for multiple messages - messages_label = ui.label("").classes("text-xs text-gray-500") + messages_label = ui.label("").classes( + "text-xs text-gray-500" + ) # Track messages progress_updates = queue.Queue() @@ -4285,15 +4548,31 @@ def process_progress_updates(): sample_id = update.get("sample_id") # Update the current sample's progress bar - if sample_id and sample_id in sample_progress_bars: + if ( + sample_id + and sample_id + in sample_progress_bars + ): if progress is not None: - from robin.gui.report_progress import normalize_report_progress + from robin.gui.report_progress import ( + normalize_report_progress, + ) - progress = normalize_report_progress(progress) - sample_progress_bars[sample_id].value = progress - sample_progress_labels[sample_id].text = f"{int(progress * 100)}% - {message}" + progress = ( + normalize_report_progress( + progress + ) + ) + sample_progress_bars[ + sample_id + ].value = progress + sample_progress_labels[ + sample_id + ].text = f"{int(progress * 100)}% - {message}" else: - sample_progress_labels[sample_id].text = message + sample_progress_labels[ + sample_id + ].text = message current_sample["id"] = sample_id # Add message to messages list @@ -4303,35 +4582,50 @@ def process_progress_updates(): messages_list.pop(0) # Update messages label - messages_label.text = "\n".join(messages_list[-5:]) + messages_label.text = "\n".join( + messages_list[-5:] + ) except queue.Empty: pass except Exception as e: - logging.debug(f"Error processing progress updates: {e}") + logging.debug( + f"Error processing progress updates: {e}" + ) # Set up timer to process updates - update_timer = ui.timer(0.1, process_progress_updates) + update_timer = ui.timer( + 0.1, process_progress_updates + ) - def progress_callback(progress_data: Dict[str, Any]): + def progress_callback( + progress_data: Dict[str, Any], + ): """Custom progress callback to update dialog (called from background thread).""" try: # Queue the update instead of directly updating UI progress_updates.put(progress_data) except Exception as e: - logging.debug(f"Error in progress callback: {e}") + logging.debug( + f"Error in progress callback: {e}" + ) # Track if still generating is_generating = {"active": True} # Storage for the files to download - files_to_download = [tsv_export_path] if tsv_export_path else [] + files_to_download = ( + [tsv_export_path] if tsv_export_path else [] + ) download_complete = {"done": False} # Timer to handle downloads once background task is done def handle_downloads(): """Handle downloads in UI context once generation is complete.""" - if download_complete["done"] and files_to_download: + if ( + download_complete["done"] + and files_to_download + ): # Log that export is complete valid_files = [ f @@ -4342,25 +4636,37 @@ def handle_downloads(): f"Bulk export complete. {len(valid_files)} file(s) ready for download." ) # Safari blocks multiple programmatic downloads; use one ZIP when needed. - bundle = self._zip_paths_for_bulk_download(valid_files) + bundle = self._zip_paths_for_bulk_download( + valid_files + ) if bundle: if len(valid_files) > 1: ui.notify( "Downloading a single ZIP (works in Safari; multiple separate downloads are blocked there).", type="info", ) - logging.debug(f"Initiating download: {bundle}") + logging.debug( + f"Initiating download: {bundle}" + ) ui.download(bundle) - if bundle.endswith(".zip") and "robin_reports_" in os.path.basename( + if bundle.endswith( + ".zip" + ) and "robin_reports_" in os.path.basename( bundle ): ui.timer( 120.0, - lambda p=bundle: self._unlink_quiet(p), + lambda p=bundle: self._unlink_quiet( + p + ), once=True, ) # Close dialog after 3 seconds - ui.timer(3.0, lambda: progress_dialog.submit(None), once=True) + ui.timer( + 3.0, + lambda: progress_dialog.submit(None), + once=True, + ) download_timer.deactivate() download_timer = ui.timer(0.1, handle_downloads) @@ -4369,7 +4675,14 @@ def handle_downloads(): async def complete_export(): """Complete the export and close dialog.""" try: - await _export_selected_reports(state, progress_dialog, files_to_download, download_complete, progress_callback, progress_updates) + await _export_selected_reports( + state, + progress_dialog, + files_to_download, + download_complete, + progress_callback, + progress_updates, + ) finally: # Clean up timer update_timer.deactivate() @@ -4388,7 +4701,10 @@ async def complete_export(): await progress_dialog # Wire the button now that handlers exist - if can_export_reports and self.export_reports_button is not None: + if ( + can_export_reports + and self.export_reports_button is not None + ): self.export_reports_button.on_click(_confirm_bulk_export) logging.info( "[samples_overview] bulk SNP and Export reports buttons " @@ -4417,7 +4733,6 @@ async def complete_export(): if hasattr(self, "samples_loading_container"): self.samples_loading_container.set_visibility(True) - def _update_samples_table_sync(self, data: Dict[str, Any]) -> None: """Synchronous version of samples table update""" try: @@ -4425,9 +4740,7 @@ def _update_samples_table_sync(self, data: Dict[str, Any]) -> None: return samples = data.get("samples", []) expected_job_types = self._get_expected_completion_job_types() - base = ( - Path(self.monitored_directory) if self.monitored_directory else None - ) + base = Path(self.monitored_directory) if self.monitored_directory else None # Deduplicate by sample_id taking the newest last_seen by_id: Dict[str, Dict[str, Any]] = {} @@ -4438,7 +4751,8 @@ def _update_samples_table_sync(self, data: Dict[str, Any]) -> None: if not existing or last_seen >= existing.get("_last_seen_raw", 0): origin_value = ( "Pre-existing" - if sid in self._preexisting_sample_ids and (time.time() - last_seen) >= self.completion_timeout_seconds + if sid in self._preexisting_sample_ids + and (time.time() - last_seen) >= self.completion_timeout_seconds else "Live" ) # Flip Live samples to Complete if inactive for configured timeout @@ -4465,12 +4779,12 @@ def _update_samples_table_sync(self, data: Dict[str, Any]) -> None: sample_dir, expected_job_types ) else: - complete_on_disk = self._is_target_bam_finalize_redundant( - sid + complete_on_disk = ( + self._is_target_bam_finalize_redundant(sid) ) if not complete_on_disk: - complete_on_disk = self._is_target_bam_finalize_redundant( - sid + complete_on_disk = ( + self._is_target_bam_finalize_redundant(sid) ) if complete_on_disk: origin_value = "Complete" @@ -4478,7 +4792,11 @@ def _update_samples_table_sync(self, data: Dict[str, Any]) -> None: active_jobs_count = merged["active_jobs"] pending_jobs_count = merged["pending_jobs"] # Only mark as Complete if timeout passed AND no active jobs - if origin_value == "Live" and (time.time() - last_seen) >= self.completion_timeout_seconds: + if ( + origin_value == "Live" + and (time.time() - last_seen) + >= self.completion_timeout_seconds + ): if active_jobs_count == 0 and pending_jobs_count == 0: should_complete = True if expected_job_types and base is not None: @@ -4501,7 +4819,9 @@ def _update_samples_table_sync(self, data: Dict[str, Any]) -> None: # Set file progress directly from job counts (same data source as other columns) files_seen = total_jobs files_processed = completed_jobs - file_progress = completed_jobs / total_jobs if total_jobs > 0 else 0.0 + file_progress = ( + completed_jobs / total_jobs if total_jobs > 0 else 0.0 + ) by_id[sid] = { "sample_id": sid, @@ -4699,7 +5019,9 @@ def _extract_event_value(self, event, default=""): return str(default) if isinstance(args, dict): return str( - args.get("value", args.get("label", args.get("modelValue", default))) + args.get( + "value", args.get("label", args.get("modelValue", default)) + ) ) if isinstance(args, (list, tuple)) and len(args) > 0: first = args[0] @@ -4815,18 +5137,21 @@ def _apply_samples_table_filters(self) -> None: # Origin filter, compute dynamic 'Complete' for display if needed now_ts = time.time() for r in rows: - try: - if r.get("origin") == "Live": - last_raw = float(r.get("_last_seen_raw", 0)) - active_jobs_count = r.get("active_jobs", 0) - pending_jobs_count = r.get("pending_jobs", 0) - # Only mark as Complete if timeout passed AND no active jobs - if last_raw and (now_ts - last_raw) >= self.completion_timeout_seconds: - if active_jobs_count == 0 and pending_jobs_count == 0: - r["origin"] = "Complete" - # If there are active jobs, keep as Live even if timeout passed - except Exception: - pass + try: + if r.get("origin") == "Live": + last_raw = float(r.get("_last_seen_raw", 0)) + active_jobs_count = r.get("active_jobs", 0) + pending_jobs_count = r.get("pending_jobs", 0) + # Only mark as Complete if timeout passed AND no active jobs + if ( + last_raw + and (now_ts - last_raw) >= self.completion_timeout_seconds + ): + if active_jobs_count == 0 and pending_jobs_count == 0: + r["origin"] = "Complete" + # If there are active jobs, keep as Live even if timeout passed + except Exception: + pass origin = (self._samples_filters or {}).get("origin", "All") if origin and origin != "All": @@ -4857,6 +5182,7 @@ def _apply_samples_table_filters(self) -> None: # Job type filter (exact token match in comma-separated job_types field) selected_job_type = (self._samples_filters or {}).get("job_type", "All") if selected_job_type and selected_job_type != "All": + def _row_has_job_type(r: Dict[str, Any]) -> bool: jt = str(r.get("job_types", "") or "").strip() if not jt: @@ -5033,8 +5359,9 @@ def _open_view_identifiers_modal( from cryptography.fernet import InvalidToken encrypted = _load_manifest_encrypted_fields(sample_dir) - with ui.dialog().props("persistent") as dialog, ui.card().classes( - "robin-dialog-surface w-full max-w-md p-4 md:p-5" + with ( + ui.dialog().props("persistent") as dialog, + ui.card().classes("robin-dialog-surface w-full max-w-md p-4 md:p-5"), ): ui.label("View sample identifiers").classes( "classification-insight-heading text-headline-small mb-2" @@ -5143,9 +5470,7 @@ async def confirm_report_generation(): } with ui.dialog().props("persistent") as dialog: - with ui.card().classes( - "robin-dialog-surface w-96 max-w-[95vw] p-4" - ): + with ui.card().classes("robin-dialog-surface w-96 max-w-[95vw] p-4"): title_label = ui.label("Generate report").classes( "classification-insight-heading text-headline-small mb-3" ) @@ -5183,13 +5508,17 @@ def on_include_sample_ids_change(e): value=None, ).classes("w-full") sample_dob_input.set_visibility(False) + def _on_sample_dob_change(_): v = getattr(sample_dob_input, "value", None) if hasattr(v, "strftime"): state["sample_dob"] = v.strftime("%Y-%m-%d") else: state["sample_dob"] = str(v).strip() if v else "" - sample_dob_input.on("update:model-value", _on_sample_dob_change) + + sample_dob_input.on( + "update:model-value", _on_sample_dob_change + ) with ui.column().classes("mb-4"): ui.label("Output formats").classes( @@ -5217,9 +5546,9 @@ def _on_sample_dob_change(_): ), ) - ui.label( - "Are you sure you want to generate a report?" - ).classes("classification-insight-foot mb-4") + ui.label("Are you sure you want to generate a report?").classes( + "classification-insight-foot mb-4" + ) def _capture_dob_and_confirm(): """Capture date picker value into state before closing dialog.""" @@ -5235,9 +5564,9 @@ def _capture_dob_and_confirm(): dialog.submit("Yes") with ui.row().classes("justify-end gap-2 flex-wrap"): - ui.button( - "No", on_click=lambda: dialog.submit("No") - ).props("flat no-caps outline") + ui.button("No", on_click=lambda: dialog.submit("No")).props( + "flat no-caps outline" + ) ui.button( "Yes", on_click=_capture_dob_and_confirm, @@ -5276,6 +5605,7 @@ def _capture_dob_and_confirm(): if encrypted: try: from cryptography.fernet import InvalidToken + decrypted: Dict[str, str] = { "sample_id": sample_id, "test_id": _get_test_id_from_manifest(sample_dir), @@ -5305,9 +5635,7 @@ def _capture_dob_and_confirm(): # Now show the progress dialog with ui.dialog().props("persistent") as progress_dialog: - with ui.card().classes( - "robin-dialog-surface w-96 max-w-[95vw] p-4" - ): + with ui.card().classes("robin-dialog-surface w-96 max-w-[95vw] p-4"): ui.label("Generating report").classes( "classification-insight-heading text-headline-small mb-2" ) @@ -5362,11 +5690,15 @@ def process_progress_updates(): progress = update.get("progress", 0.0) if progress is not None: - from robin.gui.report_progress import normalize_report_progress + from robin.gui.report_progress import ( + normalize_report_progress, + ) progress = normalize_report_progress(progress) progress_bar.value = progress - progress_text.text = f"{int(progress * 100)}% - {message}" + progress_text.text = ( + f"{int(progress * 100)}% - {message}" + ) else: progress_text.text = message @@ -5407,15 +5739,21 @@ def handle_downloads(): """Handle downloads in UI context once generation is complete.""" if download_complete["done"] and files_to_download: # Log that report generation is complete and downloads are available - file_count = len([f for f in files_to_download if f is not None]) - logging.info(f"Report generation complete for {sample_id}. {file_count} file(s) ready for download.") + file_count = len( + [f for f in files_to_download if f is not None] + ) + logging.info( + f"Report generation complete for {sample_id}. {file_count} file(s) ready for download." + ) for file_path in files_to_download: if file_path is not None: logging.debug(f"Initiating download: {file_path}") ui.download(file_path) # Close dialog after 3 seconds - ui.timer(3.0, lambda: progress_dialog.submit(None), once=True) + ui.timer( + 3.0, lambda: progress_dialog.submit(None), once=True + ) download_timer.deactivate() download_timer = ui.timer(0.1, handle_downloads) @@ -5424,7 +5762,13 @@ def handle_downloads(): async def complete_generation(): """Complete the report generation and close dialog.""" try: - await generate_and_download_report(state, progress_callback, progress_dialog, is_generating, files_to_download) + await generate_and_download_report( + state, + progress_callback, + progress_dialog, + is_generating, + files_to_download, + ) finally: # Clean up timer update_timer.deactivate() @@ -5441,7 +5785,13 @@ async def complete_generation(): await progress_dialog - async def generate_and_download_report(state: Dict[str, Any], progress_callback, progress_dialog, is_generating, files_to_download): + async def generate_and_download_report( + state: Dict[str, Any], + progress_callback, + progress_dialog, + is_generating, + files_to_download, + ): """Generate report and update progress in dialog.""" generated_files: List[str] = [] report_meta = self._report_generation_metadata() @@ -5459,10 +5809,14 @@ async def generate_and_download_report(state: Dict[str, Any], progress_callback, ) # Queue error notification files_to_download.append(None) # Signal error - ui.timer(0.1, lambda: ui.notify( - "Output directory not available for this sample", - type="warning", - ), once=True) + ui.timer( + 0.1, + lambda: ui.notify( + "Output directory not available for this sample", + type="warning", + ), + once=True, + ) return should_generate_report = bool( @@ -5474,9 +5828,7 @@ async def generate_and_download_report(state: Dict[str, Any], progress_callback, os.makedirs(str(sample_dir), exist_ok=True) export_csv_dir = None if bool(state.get("export_csv", False)): - export_csv_dir = os.path.join( - str(sample_dir), "report_csv" - ) + export_csv_dir = os.path.join(str(sample_dir), "report_csv") # Use only our custom callback, not the notification system def combined_callback(progress_data: Dict[str, Any]): @@ -5493,8 +5845,16 @@ def combined_callback(progress_data: Dict[str, Any]): export_xlsx=False, export_zip=bool(state.get("export_csv", False)), progress_callback=combined_callback, - workflow_steps=self.workflow_steps if hasattr(self, 'workflow_steps') else None, - display_config=self.display_config if hasattr(self, 'display_config') else None, + workflow_steps=( + self.workflow_steps + if hasattr(self, "workflow_steps") + else None + ), + display_config=( + self.display_config + if hasattr(self, "display_config") + else None + ), viewer_role=resolve_viewer_role(self), sample_identifiers=state.get("sample_identifiers"), generated_by=report_meta["generated_by"] or None, @@ -5504,6 +5864,7 @@ def combined_callback(progress_data: Dict[str, Any]): # Mark report as completed from robin.gui.report_progress import progress_manager + progress_manager.complete_report(sample_id, filename) # Queue files for download in UI context @@ -5541,6 +5902,7 @@ def combined_callback(progress_data: Dict[str, Any]): except Exception as e: # Mark report as failed from robin.gui.report_progress import progress_manager + progress_manager.error_report(sample_id, str(e)) self._audit_report_generated( state=state, @@ -5553,10 +5915,14 @@ def combined_callback(progress_data: Dict[str, Any]): ) # Queue error notification in UI context - ui.timer(0.1, lambda: ui.notify( - f"Error generating report: {str(e)}", - type="negative", - ), once=True) + ui.timer( + 0.1, + lambda: ui.notify( + f"Error generating report: {str(e)}", + type="negative", + ), + once=True, + ) files_to_download.append(None) # Signal error async def download_report(state: Dict[str, Any]): @@ -5567,7 +5933,6 @@ async def download_report(state: Dict[str, Any]): # Import here to avoid global dependency if GUI isn't used from nicegui import run as ng_run # type: ignore - if not sample_dir or not sample_dir.exists(): self._audit_report_generated( state=state, @@ -5588,7 +5953,7 @@ async def download_report(state: Dict[str, Any]): f"[{sample_id}] Starting report generation...", type="info", timeout=0, # Persistent notification - position="top-right" + position="top-right", ) filename = f"{sample_id}_run_report.pdf" @@ -5596,12 +5961,11 @@ async def download_report(state: Dict[str, Any]): os.makedirs(str(sample_dir), exist_ok=True) export_csv_dir = None if bool(state.get("export_csv", False)): - export_csv_dir = os.path.join( - str(sample_dir), "report_csv" - ) + export_csv_dir = os.path.join(str(sample_dir), "report_csv") # Create progress callback from robin.gui.report_progress import create_progress_callback + progress_callback = create_progress_callback(sample_id) pdf_file = await ng_run.io_bound( @@ -5614,8 +5978,12 @@ async def download_report(state: Dict[str, Any]): export_xlsx=False, export_zip=bool(state.get("export_csv", False)), progress_callback=progress_callback, - workflow_steps=self.workflow_steps if hasattr(self, 'workflow_steps') else None, - display_config=self.display_config if hasattr(self, 'display_config') else None, + workflow_steps=( + self.workflow_steps if hasattr(self, "workflow_steps") else None + ), + display_config=( + self.display_config if hasattr(self, "display_config") else None + ), viewer_role=resolve_viewer_role(self), generated_by=report_meta["generated_by"] or None, generated_at=report_meta["generated_at"], @@ -5624,6 +5992,7 @@ async def download_report(state: Dict[str, Any]): # Mark report as completed from robin.gui.report_progress import progress_manager + progress_manager.complete_report(sample_id, filename) generated_files.append(pdf_file) @@ -5647,6 +6016,7 @@ async def download_report(state: Dict[str, Any]): except Exception as e: # Mark report as failed from robin.gui.report_progress import progress_manager + progress_manager.error_report(sample_id, str(e)) self._audit_report_generated( state=state, @@ -5679,10 +6049,12 @@ async def download_report(state: Dict[str, Any]): ) ui.label( "This library ID has not been seen yet in the current session." - ).classes("text-body-medium text-slate-600 dark:text-slate-400 text-center") - ui.label( - "Redirecting to sample list…" - ).classes("text-body-small text-slate-500 dark:text-slate-500 mt-1") + ).classes( + "text-body-medium text-slate-600 dark:text-slate-400 text-center" + ) + ui.label("Redirecting to sample list…").classes( + "text-body-small text-slate-500 dark:text-slate-500 mt-1" + ) ui.button( "Back to samples", on_click=lambda: ui.navigate.to("/live_data") ).props("color=primary").classes("rounded-lg mt-2") @@ -5755,7 +6127,9 @@ async def check_directory_and_notify(): "dark:text-slate-200 break-words" ) if test_id: - with ui.row().classes("items-baseline gap-2 flex-wrap"): + with ui.row().classes( + "items-baseline gap-2 flex-wrap" + ): ui.label("Test ID").classes( "text-label-medium text-slate-500 dark:text-slate-400" ) @@ -5802,7 +6176,9 @@ async def check_directory_and_notify(): ).classes( "rounded-lg border border-slate-300 dark:border-slate-600 " "text-title-medium w-full md:w-auto md:min-w-[10rem]" - ).props("flat no-caps") + ).props( + "flat no-caps" + ) if self._current_user_can_export(): ui.button( "Generate report", @@ -5813,7 +6189,9 @@ async def check_directory_and_notify(): ) ui.button( "View audit", - on_click=lambda: self._open_sample_audit_dialog(sample_id), + on_click=lambda: self._open_sample_audit_dialog( + sample_id + ), icon="history", ).props("flat no-caps outline").classes( "rounded-lg w-full md:w-auto md:min-w-[10rem]" @@ -5829,19 +6207,21 @@ async def check_directory_and_notify(): "dark:from-slate-900/60 dark:to-zinc-950/80" ) with loading_container: - ui.spinner("bars", size="4em", color="primary").classes("mb-4") + ui.spinner("bars", size="4em", color="primary").classes( + "mb-4" + ) ui.label("Loading sample data…").classes( "text-title-medium text-slate-800 dark:text-slate-100" ) - ui.label("This may take a moment for large datasets").classes( + ui.label( + "This may take a moment for large datasets" + ).classes( "text-body-small text-slate-500 dark:text-slate-400" ) # Content container that will be shown when data is ready content_container = ( - ui.column() - .classes("w-full gap-2") - .style("display: none") + ui.column().classes("w-full gap-2").style("display: none") ) else: # For page refreshes, show content immediately @@ -5852,10 +6232,14 @@ async def check_directory_and_notify(): # Summary section (new component) - create UI immediately, load data async try: try: - from .gui.components.summary import add_summary_section # type: ignore + from .gui.components.summary import ( + add_summary_section, # type: ignore + ) except ImportError: # Try absolute import if relative fails - from robin.gui.components.summary import add_summary_section + from robin.gui.components.summary import ( + add_summary_section, + ) # Create the UI components immediately on the main thread with _sample_page_section_timer( @@ -5865,7 +6249,9 @@ async def check_directory_and_notify(): except Exception as e: logging.exception(f"[GUI] Summary section failed: {e}") try: - ui.notify(f"Summary section failed: {e}", type="warning") + ui.notify( + f"Summary section failed: {e}", type="warning" + ) except Exception: pass @@ -5877,6 +6263,7 @@ async def check_directory_and_notify(): resolve_viewer_role, ) except ImportError: + def is_section_visible(section_id, **kwargs): # type: ignore[misc] return True @@ -5887,10 +6274,14 @@ def resolve_viewer_role(_launcher): # type: ignore[misc] return "user" workflow_steps = ( - self.workflow_steps if hasattr(self, "workflow_steps") else None + self.workflow_steps + if hasattr(self, "workflow_steps") + else None ) display_config = ( - self.display_config if hasattr(self, "display_config") else None + self.display_config + if hasattr(self, "display_config") + else None ) viewer_role = resolve_viewer_role(self) @@ -5908,7 +6299,9 @@ def resolve_viewer_role(_launcher): # type: ignore[misc] ) ui.label( "This can take a moment for large datasets." - ).classes("text-body-small text-slate-500 dark:text-slate-400 mt-2") + ).classes( + "text-body-small text-slate-500 dark:text-slate-400 mt-2" + ) analysis_container = ui.column().classes("w-full gap-2") @@ -5932,7 +6325,9 @@ def _build_analysis_sections(): ): try: try: - from .gui.components.classification import add_classification_section # type: ignore + from .gui.components.classification import ( + add_classification_section, # type: ignore + ) except ImportError: from robin.gui.components.classification import ( add_classification_section, @@ -5947,10 +6342,13 @@ def _build_analysis_sections(): sample_dir, self ) except Exception as e: - logging.exception(f"[GUI] Classification section failed: {e}") + logging.exception( + f"[GUI] Classification section failed: {e}" + ) try: ui.notify( - f"Classification section failed: {e}", type="warning" + f"Classification section failed: {e}", + type="warning", ) except Exception: pass @@ -5964,9 +6362,13 @@ def _build_analysis_sections(): ): try: try: - from .gui.components.mnpflex import add_mnpflex_section # type: ignore + from .gui.components.mnpflex import ( + add_mnpflex_section, # type: ignore + ) except ImportError: - from robin.gui.components.mnpflex import add_mnpflex_section + from robin.gui.components.mnpflex import ( + add_mnpflex_section, + ) with _sample_page_section_timer( "live_data", sample_id, "mnpflex" @@ -5975,10 +6377,13 @@ def _build_analysis_sections(): self, sample_dir, sample_id ) except Exception as e: - logging.exception(f"[GUI] MNP-Flex section failed: {e}") + logging.exception( + f"[GUI] MNP-Flex section failed: {e}" + ) try: ui.notify( - f"MNP-Flex section failed: {e}", type="warning" + f"MNP-Flex section failed: {e}", + type="warning", ) except Exception: pass @@ -5992,7 +6397,9 @@ def _build_analysis_sections(): ): try: try: - from .gui.components.coverage import add_coverage_section # type: ignore + from .gui.components.coverage import ( + add_coverage_section, # type: ignore + ) except ImportError: from robin.gui.components.coverage import ( add_coverage_section, @@ -6004,12 +6411,13 @@ def _build_analysis_sections(): sample_id, "coverage", ): - add_coverage_section( - self, sample_dir - ) + add_coverage_section(self, sample_dir) except Exception as e: try: - ui.notify(f"Coverage section failed: {e}", type="warning") + ui.notify( + f"Coverage section failed: {e}", + type="warning", + ) except Exception: pass @@ -6022,18 +6430,27 @@ def _build_analysis_sections(): ): try: try: - from .gui.components.mgmt import add_mgmt_section # type: ignore + from .gui.components.mgmt import ( + add_mgmt_section, # type: ignore + ) except ImportError: - from robin.gui.components.mgmt import add_mgmt_section + from robin.gui.components.mgmt import ( + add_mgmt_section, + ) with _sample_page_section_timer( "live_data", sample_id, "mgmt" ): add_mgmt_section(self, sample_dir) except Exception as e: - logging.exception(f"[GUI] MGMT section failed: {e}") + logging.exception( + f"[GUI] MGMT section failed: {e}" + ) try: - ui.notify(f"MGMT section failed: {e}", type="warning") + ui.notify( + f"MGMT section failed: {e}", + type="warning", + ) except Exception: pass @@ -6046,18 +6463,27 @@ def _build_analysis_sections(): ): try: try: - from .gui.components.cnv import add_cnv_section # type: ignore + from .gui.components.cnv import ( + add_cnv_section, # type: ignore + ) except ImportError: - from robin.gui.components.cnv import add_cnv_section + from robin.gui.components.cnv import ( + add_cnv_section, + ) with _sample_page_section_timer( "live_data", sample_id, "cnv" ): add_cnv_section(self, sample_dir) except Exception as e: - logging.exception(f"[GUI] CNV section failed: {e}") + logging.exception( + f"[GUI] CNV section failed: {e}" + ) try: - ui.notify(f"CNV section failed: {e}", type="warning") + ui.notify( + f"CNV section failed: {e}", + type="warning", + ) except Exception: pass @@ -6070,26 +6496,39 @@ def _build_analysis_sections(): ): try: try: - from .gui.components.fusion import add_fusion_section # type: ignore + from .gui.components.fusion import ( + add_fusion_section, # type: ignore + ) except ImportError: - from robin.gui.components.fusion import add_fusion_section + from robin.gui.components.fusion import ( + add_fusion_section, + ) with _sample_page_section_timer( "live_data", sample_id, "fusion" ): add_fusion_section(self, sample_dir) except Exception as e: - logging.exception(f"[GUI] Fusion section failed: {e}") + logging.exception( + f"[GUI] Fusion section failed: {e}" + ) try: - ui.notify(f"Fusion section failed: {e}", type="warning") + ui.notify( + f"Fusion section failed: {e}", + type="warning", + ) except Exception: pass try: try: - from .gui.components.bed_coverage import add_bed_coverage_section # type: ignore + from .gui.components.bed_coverage import ( + add_bed_coverage_section, # type: ignore + ) except ImportError: - from robin.gui.components.bed_coverage import add_bed_coverage_section + from robin.gui.components.bed_coverage import ( + add_bed_coverage_section, + ) if is_section_visible( "bed_coverage", @@ -6098,13 +6537,22 @@ def _build_analysis_sections(): viewer_role=viewer_role, ): with _sample_page_section_timer( - "live_data", sample_id, "bed_coverage" + "live_data", + sample_id, + "bed_coverage", ): - add_bed_coverage_section(self, sample_dir) + add_bed_coverage_section( + self, sample_dir + ) except Exception as e: - logging.exception(f"[GUI] BED coverage section failed: {e}") + logging.exception( + f"[GUI] BED coverage section failed: {e}" + ) try: - ui.notify(f"BED Coverage section failed: {e}", type="warning") + ui.notify( + f"BED Coverage section failed: {e}", + type="warning", + ) except Exception: pass @@ -6117,22 +6565,33 @@ def _build_analysis_sections(): ): try: try: - from .gui.components.itd import add_itd_section # type: ignore + from .gui.components.itd import ( + add_itd_section, # type: ignore + ) except ImportError: - from robin.gui.components.itd import add_itd_section + from robin.gui.components.itd import ( + add_itd_section, + ) with _sample_page_section_timer( "live_data", sample_id, "itd" ): add_itd_section(self, sample_dir) except Exception as e: - logging.exception(f"[GUI] ITD section failed: {e}") + logging.exception( + f"[GUI] ITD section failed: {e}" + ) try: - ui.notify(f"ITD section failed: {e}", type="warning") + ui.notify( + f"ITD section failed: {e}", + type="warning", + ) except Exception: pass - total_elapsed = time.perf_counter() - t_analysis_start + total_elapsed = ( + time.perf_counter() - t_analysis_start + ) logging.debug( "[SamplePage] page=live_data sample=%s " "section=analysis_sections_total elapsed_s=%.3f", @@ -6140,7 +6599,9 @@ def _build_analysis_sections(): total_elapsed, ) except Exception as e: - logging.exception(f"[GUI] Failed to build analysis sections: {e}") + logging.exception( + f"[GUI] Failed to build analysis sections: {e}" + ) # Delay heavy UI creation to allow websocket handshake to complete ui.timer(0.5, _build_analysis_sections, once=True) @@ -6154,8 +6615,10 @@ def _build_analysis_sections(): display_config=display_config, viewer_role=viewer_role, ): - with ui.element("div").classes("w-full min-w-0").props( - "id=analysis-detail-output-files" + with ( + ui.element("div") + .classes("w-full min-w-0") + .props("id=analysis-detail-output-files") ): with ui.element("div").classes( "classification-insight-shell w-full min-w-0" @@ -6175,7 +6638,9 @@ def _build_analysis_sections(): ui.icon("folder_open").classes( "classification-insight-icon" ) - ui.label("Sample output directory").classes( + ui.label( + "Sample output directory" + ).classes( "classification-insight-model flex-1 min-w-0" ) ui.label(sample_id).classes( @@ -6271,12 +6736,19 @@ def _download_file(filename: str): if not self._require_export_or_notify(): return if not sample_dir or not sample_dir.exists(): - ui.notify("Sample directory not found", type="error") + ui.notify( + "Sample directory not found", type="error" + ) return file_path = sample_dir / filename - if not file_path.exists() or not file_path.is_file(): - ui.notify(f"File {filename} not found", type="error") + if ( + not file_path.exists() + or not file_path.is_file() + ): + ui.notify( + f"File {filename} not found", type="error" + ) return with open(file_path, "rb") as f: @@ -6308,7 +6780,9 @@ def _refresh_files_list_sync() -> List[Dict[str, Any]]: "size": stat.st_size, "mtime": time.strftime( "%Y-%m-%d %H:%M:%S", - time.localtime(stat.st_mtime), + time.localtime( + stat.st_mtime + ), ), "actions": f.name, } @@ -6360,24 +6834,31 @@ def _refresh_sample_detail() -> None: _notify_state["files_error"] = True if show_loading and loading_container: + async def _load_initial_data_and_show(): try: await _refresh_sample_detail_async() loading_container.style("display: none") content_container.style("display: flex") except Exception as e: - logging.error(f"Error loading initial data: {e}") + logging.error( + f"Error loading initial data: {e}" + ) loading_container.style("display: none") content_container.style("display: flex") try: - ui.timer(0.1, _load_initial_data_and_show, once=True) + ui.timer( + 0.1, _load_initial_data_and_show, once=True + ) except Exception: loading_container.style("display: none") content_container.style("display: flex") else: try: - ui.timer(0.1, _refresh_sample_detail_async, once=True) + ui.timer( + 0.1, _refresh_sample_detail_async, once=True + ) except Exception: pass @@ -6396,6 +6877,7 @@ async def _load_initial_data_and_show(): except Exception: pass elif show_loading and loading_container: + def _show_content_without_files() -> None: loading_container.style("display: none") content_container.style("display: flex") @@ -6451,7 +6933,9 @@ def _create_sample_details_page(self, sample_id: str): on_click=lambda: ui.navigate.to("/live_data"), ).classes( "mt-1 rounded-lg border border-slate-300 dark:border-slate-600" - ).props("flat") + ).props( + "flat" + ) return # Create the page with theme frame @@ -6466,7 +6950,9 @@ def _create_sample_details_page(self, sample_id: str): from robin.gui.config import is_section_visible, launcher_visibility_context from robin.gui.display_config import SAMPLE_DETAILS_SURFACE - workflow_steps, display_config, viewer_role = launcher_visibility_context(self) + workflow_steps, display_config, viewer_role = launcher_visibility_context( + self + ) details_surface = SAMPLE_DETAILS_SURFACE show_details_igv = is_section_visible( "target", @@ -6516,8 +7002,14 @@ def _create_sample_details_page(self, sample_id: str): viewer_role=viewer_role, ) and (show_fusion_target or show_fusion_genome) - with ui.element("div").classes("w-full min-w-0").props("id=sample-details-page"): - with ui.element("div").classes("classification-insight-shell w-full min-w-0"): + with ( + ui.element("div") + .classes("w-full min-w-0") + .props("id=sample-details-page") + ): + with ui.element("div").classes( + "classification-insight-shell w-full min-w-0" + ): with ui.row().classes( "w-full flex flex-col gap-3 md:flex-row md:justify-between " "md:items-start p-2 md:p-3" @@ -6567,14 +7059,18 @@ def _create_sample_details_page(self, sample_id: str): ): ui.button( "View audit", - on_click=lambda: self._open_sample_audit_dialog(sample_id), + on_click=lambda: self._open_sample_audit_dialog( + sample_id + ), icon="history", ).props("flat no-caps outline").classes( "rounded-lg w-full md:min-w-[10rem]" ) ui.button( "Back to sample", - on_click=lambda: ui.navigate.to(f"/live_data/{sample_id}"), + on_click=lambda: ui.navigate.to( + f"/live_data/{sample_id}" + ), ).props("color=primary no-caps").classes( "rounded-lg w-full md:min-w-[10rem]" ) @@ -6589,9 +7085,7 @@ def _create_sample_details_page(self, sample_id: str): with ui.element("div").classes( "classification-insight-card w-full min-w-0" ): - with ui.column().classes( - "w-full min-w-0 gap-2 p-2 md:p-3" - ): + with ui.column().classes("w-full min-w-0 gap-2 p-2 md:p-3"): with ui.row().classes("items-center gap-2 min-w-0"): ui.icon("folder_open").classes( "classification-insight-icon" @@ -6630,9 +7124,7 @@ def _create_sample_details_page(self, sample_id: str): with ui.column().classes( "w-full min-w-0 gap-2 p-2 md:p-3" ): - with ui.row().classes( - "items-center gap-2 min-w-0" - ): + with ui.row().classes("items-center gap-2 min-w-0"): ui.icon("hub").classes( "classification-insight-icon" ) @@ -6666,7 +7158,9 @@ def _create_sample_details_page(self, sample_id: str): try: from robin.gui.components.itd import add_itd_section except ImportError: - from .gui.components.itd import add_itd_section # type: ignore + from .gui.components.itd import ( + add_itd_section, # type: ignore + ) with _sample_page_section_timer( "sample_details", sample_id, "itd" @@ -6676,22 +7170,29 @@ def _create_sample_details_page(self, sample_id: str): # Fusion Pairs Table section if sample_dir and sample_dir.exists() and show_details_fusion_pairs: _fusion_pairs_t0 = time.perf_counter() + import pandas as pd + from robin.gui.components.fusion import ( - _load_processed_pickle, _cluster_fusion_reads, + _load_processed_pickle, ) - import pandas as pd sample_key = str(sample_dir) fusion_state = getattr(self, "_fusion_state", {}) cache_entry = fusion_state.setdefault(sample_key, {}) - target_file = sample_dir / "fusion_candidates_master_processed.pkl" + target_file = ( + sample_dir / "fusion_candidates_master_processed.pkl" + ) genome_file = sample_dir / "fusion_candidates_all_processed.pkl" target_mtime = ( - target_file.stat().st_mtime if target_file.exists() else None + target_file.stat().st_mtime + if target_file.exists() + else None ) genome_mtime = ( - genome_file.stat().st_mtime if genome_file.exists() else None + genome_file.stat().st_mtime + if genome_file.exists() + else None ) file_sig = (target_mtime, genome_mtime) cached_rows = cache_entry.get("details_pairs_rows") @@ -6705,24 +7206,43 @@ def _build_fusion_pairs_rows_sync() -> List[Dict[str, Any]]: import re fusion_data_local = None - target_file_local = sample_dir / "fusion_candidates_master_processed.pkl" - genome_file_local = sample_dir / "fusion_candidates_all_processed.pkl" + target_file_local = ( + sample_dir / "fusion_candidates_master_processed.pkl" + ) + genome_file_local = ( + sample_dir / "fusion_candidates_all_processed.pkl" + ) try: if show_fusion_target and target_file_local.exists(): - fusion_data_local = _load_processed_pickle(target_file_local) + fusion_data_local = _load_processed_pickle( + target_file_local + ) elif show_fusion_genome and genome_file_local.exists(): - fusion_data_local = _load_processed_pickle(genome_file_local) + fusion_data_local = _load_processed_pickle( + genome_file_local + ) except Exception as ex: logging.warning(f"Failed to load fusion data: {ex}") return [] if not fusion_data_local: return [] - annotated_data_local = fusion_data_local.get("annotated_data", pd.DataFrame()) - if annotated_data_local is None or annotated_data_local.empty: + annotated_data_local = fusion_data_local.get( + "annotated_data", pd.DataFrame() + ) + if ( + annotated_data_local is None + or annotated_data_local.empty + ): return [] - goodpairs_local = fusion_data_local.get("goodpairs", pd.Series()) - if goodpairs_local is not None and not goodpairs_local.empty and goodpairs_local.sum() > 0: + goodpairs_local = fusion_data_local.get( + "goodpairs", pd.Series() + ) + if ( + goodpairs_local is not None + and not goodpairs_local.empty + and goodpairs_local.sum() > 0 + ): aligned_goodpairs = goodpairs_local.reindex( annotated_data_local.index, fill_value=False ) @@ -6734,12 +7254,23 @@ def _build_fusion_pairs_rows_sync() -> List[Dict[str, Any]]: max_distance=10000, use_breakpoint_validation=True, ) - if clustered_data_local is None or clustered_data_local.empty: + if ( + clustered_data_local is None + or clustered_data_local.empty + ): return [] built_rows: List[Dict[str, Any]] = [] for _, row in clustered_data_local.iterrows(): - if all(col in row for col in ["gene1_start", "gene1_end", "gene2_start", "gene2_end"]): + if all( + col in row + for col in [ + "gene1_start", + "gene1_end", + "gene2_start", + "gene2_end", + ] + ): start1_raw = int(row["gene1_start"]) end1_raw = int(row["gene1_end"]) start2_raw = int(row["gene2_start"]) @@ -6747,19 +7278,31 @@ def _build_fusion_pairs_rows_sync() -> List[Dict[str, Any]]: else: pos1_str = str(row.get("gene1_position", "")) pos2_str = str(row.get("gene2_position", "")) - pos1_match = re.match(r'(\d+)[-–—](\d+)', pos1_str.replace(',', '')) - pos2_match = re.match(r'(\d+)[-–—](\d+)', pos2_str.replace(',', '')) + pos1_match = re.match( + r"(\d+)[-–—](\d+)", pos1_str.replace(",", "") + ) + pos2_match = re.match( + r"(\d+)[-–—](\d+)", pos2_str.replace(",", "") + ) if pos1_match and pos2_match: start1_raw = int(pos1_match.group(1)) end1_raw = int(pos1_match.group(2)) start2_raw = int(pos2_match.group(1)) end2_raw = int(pos2_match.group(2)) else: - pos1_single = re.search(r'(\d+)', pos1_str.replace(',', '')) - pos2_single = re.search(r'(\d+)', pos2_str.replace(',', '')) + pos1_single = re.search( + r"(\d+)", pos1_str.replace(",", "") + ) + pos2_single = re.search( + r"(\d+)", pos2_str.replace(",", "") + ) if pos1_single and pos2_single: - start1_raw = end1_raw = int(pos1_single.group(1)) - start2_raw = end2_raw = int(pos2_single.group(1)) + start1_raw = end1_raw = int( + pos1_single.group(1) + ) + start2_raw = end2_raw = int( + pos2_single.group(1) + ) else: continue padding = 10000 @@ -6773,9 +7316,17 @@ def _build_fusion_pairs_rows_sync() -> List[Dict[str, Any]]: { "fusion_pair": row.get("fusion_pair", ""), "chr1": chr1, - "pos1": f"{min1:,}-{max1:,}" if min1 != max1 else f"{min1:,}", + "pos1": ( + f"{min1:,}-{max1:,}" + if min1 != max1 + else f"{min1:,}" + ), "chr2": chr2, - "pos2": f"{min2:,}-{max2:,}" if min2 != max2 else f"{min2:,}", + "pos2": ( + f"{min2:,}-{max2:,}" + if min2 != max2 + else f"{min2:,}" + ), "reads": int(row.get("reads", 0)), "region": ( f"{chr1}:{max(1, min1 - padding)}-{max1 + padding} " @@ -6790,7 +7341,9 @@ def _build_fusion_pairs_rows_sync() -> List[Dict[str, Any]]: if not cache_hit: rebuilt_rows = _build_fusion_pairs_rows_sync() cache_entry["details_pairs_sig"] = file_sig - cache_entry["details_pairs_rows"] = [dict(r) for r in rebuilt_rows] + cache_entry["details_pairs_rows"] = [ + dict(r) for r in rebuilt_rows + ] cached_rows = cache_entry["details_pairs_rows"] cache_hit = True @@ -6801,11 +7354,19 @@ def _build_fusion_pairs_rows_sync() -> List[Dict[str, Any]]: try: if show_fusion_target and target_file.exists(): fusion_data = _load_processed_pickle(target_file) - if fusion_data and fusion_data.get("annotated_data") is not None: + if ( + fusion_data + and fusion_data.get("annotated_data") + is not None + ): fusion_data_loaded = True elif show_fusion_genome and genome_file.exists(): fusion_data = _load_processed_pickle(genome_file) - if fusion_data and fusion_data.get("annotated_data") is not None: + if ( + fusion_data + and fusion_data.get("annotated_data") + is not None + ): fusion_data_loaded = True except Exception as e: logging.warning(f"Failed to load fusion data: {e}") @@ -6824,7 +7385,9 @@ def _build_fusion_pairs_rows_sync() -> List[Dict[str, Any]]: if cache_hit or not annotated_data.empty: # Filter to good pairs if available if not goodpairs.empty and goodpairs.sum() > 0: - aligned_goodpairs = goodpairs.reindex(annotated_data.index, fill_value=False) + aligned_goodpairs = goodpairs.reindex( + annotated_data.index, fill_value=False + ) filtered_data = annotated_data[aligned_goodpairs] else: filtered_data = annotated_data @@ -6833,7 +7396,7 @@ def _build_fusion_pairs_rows_sync() -> List[Dict[str, Any]]: clustered_data = _cluster_fusion_reads( filtered_data, max_distance=10000, - use_breakpoint_validation=True + use_breakpoint_validation=True, ) if cache_hit or not clustered_data.empty: @@ -6855,24 +7418,72 @@ def _build_fusion_pairs_rows_sync() -> List[Dict[str, Any]]: ) columns = [ - {"name": "fusion_pair", "label": "Fusion Pair", "field": "fusion_pair", "sortable": False}, - {"name": "chr1", "label": "Chr 1", "field": "chr1", "sortable": False}, - {"name": "pos1", "label": "Breakpoint 1", "field": "pos1", "sortable": False}, - {"name": "chr2", "label": "Chr 2", "field": "chr2", "sortable": False}, - {"name": "pos2", "label": "Breakpoint 2", "field": "pos2", "sortable": False}, - {"name": "reads", "label": "Supporting Reads", "field": "reads", "sortable": False}, - {"name": "action", "label": "View in IGV", "field": "action", "sortable": False} + { + "name": "fusion_pair", + "label": "Fusion Pair", + "field": "fusion_pair", + "sortable": False, + }, + { + "name": "chr1", + "label": "Chr 1", + "field": "chr1", + "sortable": False, + }, + { + "name": "pos1", + "label": "Breakpoint 1", + "field": "pos1", + "sortable": False, + }, + { + "name": "chr2", + "label": "Chr 2", + "field": "chr2", + "sortable": False, + }, + { + "name": "pos2", + "label": "Breakpoint 2", + "field": "pos2", + "sortable": False, + }, + { + "name": "reads", + "label": "Supporting Reads", + "field": "reads", + "sortable": False, + }, + { + "name": "action", + "label": "View in IGV", + "field": "action", + "sortable": False, + }, ] # Format rows for display - rows = [dict(r) for r in cached_rows] if cache_hit else [] + rows = ( + [dict(r) for r in cached_rows] + if cache_hit + else [] + ) if not cache_hit: # Keep region in each source row to avoid duplicate mapping storage. import re + for idx, row in clustered_data.iterrows(): # Try to get start/end coordinates from the row if available # (e.g., from breakpoint validation) - if all(col in row for col in ["gene1_start", "gene1_end", "gene2_start", "gene2_end"]): + if all( + col in row + for col in [ + "gene1_start", + "gene1_end", + "gene2_start", + "gene2_end", + ] + ): # Use actual start/end coordinates if available start1_raw = int(row["gene1_start"]) end1_raw = int(row["gene1_end"]) @@ -6880,25 +7491,49 @@ def _build_fusion_pairs_rows_sync() -> List[Dict[str, Any]]: end2_raw = int(row["gene2_end"]) else: # Parse from position strings (format: "start-end") - pos1_str = str(row.get("gene1_position", "")) - pos2_str = str(row.get("gene2_position", "")) + pos1_str = str( + row.get("gene1_position", "") + ) + pos2_str = str( + row.get("gene2_position", "") + ) # Parse range format "start-end" or just a single number - pos1_match = re.match(r'(\d+)[-–—](\d+)', pos1_str.replace(',', '')) - pos2_match = re.match(r'(\d+)[-–—](\d+)', pos2_str.replace(',', '')) + pos1_match = re.match( + r"(\d+)[-–—](\d+)", + pos1_str.replace(",", ""), + ) + pos2_match = re.match( + r"(\d+)[-–—](\d+)", + pos2_str.replace(",", ""), + ) if pos1_match and pos2_match: # Extract both start and end from the range - start1_raw = int(pos1_match.group(1)) + start1_raw = int( + pos1_match.group(1) + ) end1_raw = int(pos1_match.group(2)) - start2_raw = int(pos2_match.group(1)) + start2_raw = int( + pos2_match.group(1) + ) end2_raw = int(pos2_match.group(2)) else: # Fallback: try to extract single coordinates - pos1_single = re.search(r'(\d+)', pos1_str.replace(',', '')) - pos2_single = re.search(r'(\d+)', pos2_str.replace(',', '')) + pos1_single = re.search( + r"(\d+)", + pos1_str.replace(",", ""), + ) + pos2_single = re.search( + r"(\d+)", + pos2_str.replace(",", ""), + ) if pos1_single and pos2_single: # Single coordinate - use it as both start and end - start1_raw = end1_raw = int(pos1_single.group(1)) - start2_raw = end2_raw = int(pos2_single.group(1)) + start1_raw = end1_raw = int( + pos1_single.group(1) + ) + start2_raw = end2_raw = int( + pos2_single.group(1) + ) else: # Skip this row if we can't parse coordinates continue @@ -6919,10 +7554,20 @@ def _build_fusion_pairs_rows_sync() -> List[Dict[str, Any]]: # Format region as "chr1:start-end chr2:start-end" region = f"{chr1}:{start1}-{end1} {chr2}:{start2}-{end2}" # For display, show the breakpoint range (not the padded version) - display_pos1 = f"{min1:,}-{max1:,}" if min1 != max1 else f"{min1:,}" - display_pos2 = f"{min2:,}-{max2:,}" if min2 != max2 else f"{min2:,}" + display_pos1 = ( + f"{min1:,}-{max1:,}" + if min1 != max1 + else f"{min1:,}" + ) + display_pos2 = ( + f"{min2:,}-{max2:,}" + if min2 != max2 + else f"{min2:,}" + ) formatted_row = { - "fusion_pair": row.get("fusion_pair", ""), + "fusion_pair": row.get( + "fusion_pair", "" + ), "chr1": chr1, "pos1": display_pos1, "chr2": chr2, @@ -6936,25 +7581,36 @@ def _build_fusion_pairs_rows_sync() -> List[Dict[str, Any]]: if rows and not cache_hit: cache_entry["details_pairs_sig"] = file_sig - cache_entry["details_pairs_rows"] = [dict(r) for r in rows] + cache_entry["details_pairs_rows"] = [ + dict(r) for r in rows + ] if rows: # Store fusion regions mapped by fusion pair for easy lookup fusion_regions_by_pair = {} for idx, row_data in enumerate(rows): - fusion_regions_by_pair[row_data["fusion_pair"]] = row_data.get("region", "") + fusion_regions_by_pair[ + row_data["fusion_pair"] + ] = row_data.get("region", "") # Create JavaScript map of regions for IGV navigation import json - js_regions_json = json.dumps(fusion_regions_by_pair) + + js_regions_json = json.dumps( + fusion_regions_by_pair + ) # Function to navigate IGV to a fusion region def navigate_to_fusion_region(fusion_pair: str): """Navigate IGV browser to the specified fusion pair region.""" if fusion_pair in fusion_regions_by_pair: - region = fusion_regions_by_pair[fusion_pair] + region = fusion_regions_by_pair[ + fusion_pair + ] # Escape region string for JavaScript - escaped_region = region.replace('"', '\\"').replace("'", "\\'") + escaped_region = region.replace( + '"', '\\"' + ).replace("'", "\\'") js_navigate = f""" (function() {{ try {{ @@ -6974,7 +7630,9 @@ def navigate_to_fusion_region(fusion_pair: str): }} }})(); """ - ui.run_javascript(js_navigate, timeout=5.0) + ui.run_javascript( + js_navigate, timeout=5.0 + ) # Initialize fusion regions map in window BEFORE creating table # This ensures it's available when the slot template renders @@ -6989,32 +7647,44 @@ def navigate_to_fusion_region(fusion_pair: str): # Render only one page at a time (Quasar server-side paging). fusion_preview_mode = len(rows) > 50_000 - fusion_rows_source = rows[:5_000] if fusion_preview_mode else rows + fusion_rows_source = ( + rows[:5_000] + if fusion_preview_mode + else rows + ) fusion_total = len(fusion_rows_source) - fusion_init_pagination = clamp_qtable_server_pagination( - { - "sortBy": None, - "descending": False, - "page": 1, - "rowsPerPage": 100, - "rowsNumber": fusion_total, - }, - rows_number=fusion_total, - rows_per_page_default=100, + fusion_init_pagination = ( + clamp_qtable_server_pagination( + { + "sortBy": None, + "descending": False, + "page": 1, + "rowsPerPage": 100, + "rowsNumber": fusion_total, + }, + rows_number=fusion_total, + rows_per_page_default=100, + ) ) - table_container, fusion_table = styled_server_paged_table( - columns=columns, - rows=[], - pagination=fusion_init_pagination, - row_key="__row_idx", - class_size="table-xs", + table_container, fusion_table = ( + styled_server_paged_table( + columns=columns, + rows=[], + pagination=fusion_init_pagination, + row_key="__row_idx", + class_size="table-xs", + ) ) if fusion_preview_mode: ui.label( f"Preview mode: showing first {len(fusion_rows_source):,} rows of {len(rows):,}. Apply upstream filters to narrow." - ).classes("classification-insight-level classification-insight-level--low w-full") + ).classes( + "classification-insight-level classification-insight-level--low w-full" + ) - def _fill_fusion_from_pagination(pag: Dict[str, Any]) -> None: + def _fill_fusion_from_pagination( + pag: Dict[str, Any], + ) -> None: total = len(fusion_rows_source) pag = clamp_qtable_server_pagination( pag, @@ -7025,14 +7695,18 @@ def _fill_fusion_from_pagination(pag: Dict[str, Any]) -> None: page = int(pag["page"]) start = (page - 1) * rpp end = start + rpp - fusion_table.rows = fusion_rows_source[start:end] + fusion_table.rows = fusion_rows_source[ + start:end + ] fusion_table.pagination = pag fusion_table.update() wire_qtable_server_pagination_handlers( fusion_table, _fill_fusion_from_pagination ) - _fill_fusion_from_pagination(fusion_init_pagination) + _fill_fusion_from_pagination( + fusion_init_pagination + ) # Add clickable action button column using slot that emits events to Python try: @@ -7051,7 +7725,7 @@ def _fill_fusion_from_pagination(pag: Dict[str, Any]) -> None: title="View in IGV" /> -""" +""", ) # Handle the event from the slot @@ -7062,19 +7736,40 @@ def on_fusion_view_igv(e): row_idx = getattr(e, "args", None) if row_idx is not None: row_idx = int(row_idx) - if 0 <= row_idx < len(fusion_rows_source): - fusion_pair = fusion_rows_source[row_idx].get("fusion_pair", "") + if ( + 0 + <= row_idx + < len(fusion_rows_source) + ): + fusion_pair = ( + fusion_rows_source[ + row_idx + ].get("fusion_pair", "") + ) if fusion_pair: - logging.debug(f"[Fusion] Button clicked for: {fusion_pair}") - navigate_to_fusion_region(fusion_pair) + logging.debug( + f"[Fusion] Button clicked for: {fusion_pair}" + ) + navigate_to_fusion_region( + fusion_pair + ) except Exception as ex: - logging.warning(f"Error handling fusion view IGV event: {ex}") + logging.warning( + f"Error handling fusion view IGV event: {ex}" + ) - fusion_table.on("fusion-view-igv", on_fusion_view_igv) - logging.debug("Added action button column slot with event handler") + fusion_table.on( + "fusion-view-igv", on_fusion_view_igv + ) + logging.debug( + "Added action button column slot with event handler" + ) except Exception as slot_ex: - logging.warning(f"Could not add action column slot: {slot_ex}") + logging.warning( + f"Could not add action column slot: {slot_ex}" + ) import traceback + logging.warning(traceback.format_exc()) # Fallback: Use JavaScript with inline region lookup @@ -7107,7 +7802,9 @@ def on_fusion_view_igv(e): console.log('[Fusion] Created navigation handler'); }})(); """ - ui.run_javascript(js_inline_handler, timeout=5.0) + ui.run_javascript( + js_inline_handler, timeout=5.0 + ) fusion_table.add_slot( "body-cell-action", @@ -7123,10 +7820,12 @@ def on_fusion_view_igv(e): title="View in IGV" /> -""" +""", ) except Exception as fallback_ex: - logging.warning(f"Fallback approach also failed: {fallback_ex}") + logging.warning( + f"Fallback approach also failed: {fallback_ex}" + ) # Use JavaScript to attach click handlers to rows - improved with event delegation js_attach_handlers = f""" @@ -7254,14 +7953,21 @@ def on_fusion_view_igv(e): # (avoids UI-slot-bound timers firing after navigation). def _run_fusion_handlers_js() -> None: try: - ui.run_javascript(js_attach_handlers, timeout=10.0) + ui.run_javascript( + js_attach_handlers, timeout=10.0 + ) except Exception: pass - fusion_timer_1 = app.timer(0.5, _run_fusion_handlers_js, once=True) - fusion_timer_2 = app.timer(2.0, _run_fusion_handlers_js, once=True) + fusion_timer_1 = app.timer( + 0.5, _run_fusion_handlers_js, once=True + ) + fusion_timer_2 = app.timer( + 2.0, _run_fusion_handlers_js, once=True + ) fusion_pairs_refresh_timer = None try: + def _cleanup_fusion_page() -> None: fusion_rows_source.clear() fusion_table.rows = [] @@ -7274,7 +7980,10 @@ def _cleanup_fusion_page() -> None: timer.deactivate() except Exception: pass - ui.context.client.on_disconnect(_cleanup_fusion_page) + + ui.context.client.on_disconnect( + _cleanup_fusion_page + ) except Exception: pass @@ -7286,22 +7995,32 @@ def _cleanup_fusion_page() -> None: f"Total supporting reads: {total_reads}" ).classes("classification-insight-foot") - async def _refresh_fusion_pairs_rows_async() -> None: + async def _refresh_fusion_pairs_rows_async() -> ( + None + ): try: new_sig = ( - target_file.stat().st_mtime - if target_file.exists() - else None, - genome_file.stat().st_mtime - if genome_file.exists() - else None, + ( + target_file.stat().st_mtime + if target_file.exists() + else None + ), + ( + genome_file.stat().st_mtime + if genome_file.exists() + else None + ), ) - if new_sig == cache_entry.get("details_pairs_sig"): + if new_sig == cache_entry.get( + "details_pairs_sig" + ): return updated_rows = await asyncio.to_thread( _build_fusion_pairs_rows_sync ) - cache_entry["details_pairs_sig"] = new_sig + cache_entry["details_pairs_sig"] = ( + new_sig + ) cache_entry["details_pairs_rows"] = [ dict(r) for r in updated_rows ] @@ -7316,6 +8035,7 @@ async def _refresh_fusion_pairs_rows_async() -> None: row_data["fusion_pair"] ] = row_data.get("region", "") import json + ui.run_javascript( f""" (function() {{ @@ -7400,9 +8120,11 @@ async def _refresh_fusion_pairs_rows_async() -> None: # Note: This runs during page creation, not during user interaction try: import pandas as pd + df = pd.read_csv( coverage_file, - usecols=lambda c: c in { + usecols=lambda c: c + in { "chrom", "startpos", "endpos", @@ -7427,18 +8149,30 @@ async def _refresh_fusion_pairs_rows_async() -> None: if all(col in df.columns for col in required_cols): # Calculate coverage if not present if "coverage" not in df.columns: - if "length" in df.columns and "bases" in df.columns: + if ( + "length" in df.columns + and "bases" in df.columns + ): df["coverage"] = df["bases"] / df["length"] - elif "startpos" in df.columns and "endpos" in df.columns: - df["length"] = df["endpos"] - df["startpos"] + 1 + elif ( + "startpos" in df.columns + and "endpos" in df.columns + ): + df["length"] = ( + df["endpos"] - df["startpos"] + 1 + ) if "bases" in df.columns: - df["coverage"] = df["bases"] / df["length"] + df["coverage"] = ( + df["bases"] / df["length"] + ) else: df["coverage"] = 0 # Prepare table data table_data = [] - gene_regions_by_name = {} # Store regions for navigation + gene_regions_by_name = ( + {} + ) # Store regions for navigation for _, row in df.iterrows(): gene_name = str(row["name"]) chrom = str(row["chrom"]) @@ -7452,15 +8186,19 @@ async def _refresh_fusion_pairs_rows_async() -> None: region = f"{chrom}:{startpos_nav}-{endpos_nav}" gene_regions_by_name[gene_name] = region - table_data.append({ - "chrom": chrom, - "startpos": f"{startpos_raw:,}", # Format with commas for display - "endpos": f"{endpos_raw:,}", # Format with commas for display - "name": gene_name, - "coverage": float(row.get("coverage", 0)), - "__row_idx": len(table_data), - "action": "", - }) + table_data.append( + { + "chrom": chrom, + "startpos": f"{startpos_raw:,}", # Format with commas for display + "endpos": f"{endpos_raw:,}", # Format with commas for display + "name": gene_name, + "coverage": float( + row.get("coverage", 0) + ), + "__row_idx": len(table_data), + "action": "", + } + ) if table_data: with ui.element("div").classes( @@ -7471,7 +8209,9 @@ async def _refresh_fusion_pairs_rows_async() -> None: ) ui.label( "Click a gene row to open the region in IGV." - ).classes("classification-insight-meta w-full mb-2") + ).classes( + "classification-insight-meta w-full mb-2" + ) from robin.gui.theme import ( clamp_qtable_server_pagination, @@ -7480,20 +8220,58 @@ async def _refresh_fusion_pairs_rows_async() -> None: ) columns = [ - {"name": "name", "label": "Gene Name", "field": "name", "sortable": False}, - {"name": "chrom", "label": "Chromosome", "field": "chrom", "sortable": False}, - {"name": "startpos", "label": "Start", "field": "startpos", "sortable": False}, - {"name": "endpos", "label": "End", "field": "endpos", "sortable": False}, - {"name": "coverage", "label": "Coverage (x)", "field": "coverage", "sortable": False}, - {"name": "action", "label": "View in IGV", "field": "action", "sortable": False} + { + "name": "name", + "label": "Gene Name", + "field": "name", + "sortable": False, + }, + { + "name": "chrom", + "label": "Chromosome", + "field": "chrom", + "sortable": False, + }, + { + "name": "startpos", + "label": "Start", + "field": "startpos", + "sortable": False, + }, + { + "name": "endpos", + "label": "End", + "field": "endpos", + "sortable": False, + }, + { + "name": "coverage", + "label": "Coverage (x)", + "field": "coverage", + "sortable": False, + }, + { + "name": "action", + "label": "View in IGV", + "field": "action", + "sortable": False, + }, ] gene_preview_mode = len(table_data) > 50_000 - gene_rows_source = table_data[:5_000] if gene_preview_mode else table_data + gene_rows_source = ( + table_data[:5_000] + if gene_preview_mode + else table_data + ) gene_page_state: Dict[str, Any] = { - "filtered_positions": list(range(len(gene_rows_source))), + "filtered_positions": list( + range(len(gene_rows_source)) + ), } - gene_total_matches = len(gene_page_state["filtered_positions"]) + gene_total_matches = len( + gene_page_state["filtered_positions"] + ) gene_init_pagination = clamp_qtable_server_pagination( { "sortBy": None, @@ -7505,20 +8283,30 @@ async def _refresh_fusion_pairs_rows_async() -> None: rows_number=gene_total_matches, rows_per_page_default=100, ) - table_container, gene_table = styled_server_paged_table( - columns=columns, - rows=[], - pagination=gene_init_pagination, - row_key="__row_idx", - class_size="table-xs", + table_container, gene_table = ( + styled_server_paged_table( + columns=columns, + rows=[], + pagination=gene_init_pagination, + row_key="__row_idx", + class_size="table-xs", + ) ) if gene_preview_mode: ui.label( f"Preview mode: showing first {len(gene_rows_source):,} rows of {len(table_data):,}. Apply filters to narrow." - ).classes("classification-insight-level classification-insight-level--low w-full") + ).classes( + "classification-insight-level classification-insight-level--low w-full" + ) - def _fill_gene_from_pagination(pag: Dict[str, Any]) -> None: - total = len(gene_page_state["filtered_positions"]) + def _fill_gene_from_pagination( + pag: Dict[str, Any], + ) -> None: + total = len( + gene_page_state[ + "filtered_positions" + ] + ) pag = clamp_qtable_server_pagination( pag, rows_number=total, @@ -7528,10 +8316,13 @@ def _fill_gene_from_pagination(pag: Dict[str, Any]) -> None: page = int(pag["page"]) start = (page - 1) * rpp end = start + rpp - positions = gene_page_state["filtered_positions"] + positions = gene_page_state[ + "filtered_positions" + ] slice_pos = positions[start:end] gene_table.rows = [ - gene_rows_source[i] for i in slice_pos + gene_rows_source[i] + for i in slice_pos ] gene_table.pagination = pag gene_table.update() @@ -7543,38 +8334,60 @@ def _fill_gene_from_pagination(pag: Dict[str, Any]) -> None: def _apply_gene_search(term: str) -> None: txt = str(term or "").strip().lower() if not txt: - gene_page_state["filtered_positions"] = list( + gene_page_state[ + "filtered_positions" + ] = list( range(len(gene_rows_source)) ) else: - gene_page_state["filtered_positions"] = [ + gene_page_state[ + "filtered_positions" + ] = [ i - for i, row in enumerate(gene_rows_source) - if txt in str(row.get("name", "")).lower() - or txt in str(row.get("chrom", "")).lower() + for i, row in enumerate( + gene_rows_source + ) + if txt + in str( + row.get("name", "") + ).lower() + or txt + in str( + row.get("chrom", "") + ).lower() ] pag = clamp_qtable_server_pagination( dict(gene_table.pagination), - rows_number=len(gene_page_state["filtered_positions"]), + rows_number=len( + gene_page_state[ + "filtered_positions" + ] + ), rows_per_page_default=100, ) pag["page"] = 1 _fill_gene_from_pagination(pag) - _fill_gene_from_pagination(gene_init_pagination) + _fill_gene_from_pagination( + gene_init_pagination + ) try: with gene_table.add_slot("top-right"): gene_search_input = ui.input( placeholder="Search genes..." - ).props("type=search dense clearable") + ).props( + "type=search dense clearable" + ) gene_search_input.on( "update:model-value", lambda e: _apply_gene_search( getattr(e, "value", "") ), ) - with gene_search_input.add_slot("append"): + with gene_search_input.add_slot( + "append" + ): ui.icon("search") except Exception: pass @@ -7596,14 +8409,21 @@ def _apply_gene_search(term: str) -> None: # Function to navigate IGV to a gene region import json - js_gene_regions_json = json.dumps(gene_regions_by_name) + + js_gene_regions_json = json.dumps( + gene_regions_by_name + ) def navigate_to_gene_region(gene_name: str): """Navigate IGV browser to the specified gene region.""" if gene_name in gene_regions_by_name: - region = gene_regions_by_name[gene_name] + region = gene_regions_by_name[ + gene_name + ] # Escape region string for JavaScript - escaped_region = region.replace('"', '\\"').replace("'", "\\'") + escaped_region = region.replace( + '"', '\\"' + ).replace("'", "\\'") js_navigate = f""" (function() {{ try {{ @@ -7623,7 +8443,9 @@ def navigate_to_gene_region(gene_name: str): }} }})(); """ - ui.run_javascript(js_navigate, timeout=5.0) + ui.run_javascript( + js_navigate, timeout=5.0 + ) # Store regions in window object for JavaScript access js_init_gene_regions = f""" @@ -7631,7 +8453,9 @@ def navigate_to_gene_region(gene_name: str): Object.assign(window.geneRegionsMap, {js_gene_regions_json}); console.log('[Gene] Loaded', Object.keys(window.geneRegionsMap).length, 'gene regions'); """ - ui.run_javascript(js_init_gene_regions, timeout=5.0) + ui.run_javascript( + js_init_gene_regions, timeout=5.0 + ) # Add clickable action button column using slot that emits events to Python try: @@ -7650,30 +8474,53 @@ def navigate_to_gene_region(gene_name: str): title="View in IGV" /> -""" +""", ) # Handle the event from the slot def on_gene_view_igv(e): """Handle gene view IGV event from table button.""" try: - row_idx = getattr(e, "args", None) + row_idx = getattr( + e, "args", None + ) gene_name = "" if row_idx is not None: row_idx = int(row_idx) - if 0 <= row_idx < len(gene_rows_source): - gene_name = gene_rows_source[row_idx].get("name", "") + if ( + 0 + <= row_idx + < len(gene_rows_source) + ): + gene_name = ( + gene_rows_source[ + row_idx + ].get("name", "") + ) if gene_name: - logging.debug(f"[Gene] Button clicked for: {gene_name}") - navigate_to_gene_region(gene_name) + logging.debug( + f"[Gene] Button clicked for: {gene_name}" + ) + navigate_to_gene_region( + gene_name + ) except Exception as ex: - logging.warning(f"Error handling gene view IGV event: {ex}") + logging.warning( + f"Error handling gene view IGV event: {ex}" + ) - gene_table.on("gene-view-igv", on_gene_view_igv) - logging.debug("Added action button column slot with event handler") + gene_table.on( + "gene-view-igv", on_gene_view_igv + ) + logging.debug( + "Added action button column slot with event handler" + ) except Exception as slot_ex: - logging.warning(f"Could not add action column slot: {slot_ex}") + logging.warning( + f"Could not add action column slot: {slot_ex}" + ) import traceback + logging.warning(traceback.format_exc()) gene_handler_timers: List[Any] = [] @@ -7775,25 +8622,50 @@ def on_gene_view_igv(e): def _run_gene_handlers_js() -> None: try: - ui.run_javascript(js_gene_table_handlers, timeout=10.0) + ui.run_javascript( + js_gene_table_handlers, + timeout=10.0, + ) except Exception: pass - gene_handler_timers.append(app.timer(0.5, _run_gene_handlers_js, once=True)) - gene_handler_timers.append(app.timer(2.0, _run_gene_handlers_js, once=True)) + gene_handler_timers.append( + app.timer( + 0.5, + _run_gene_handlers_js, + once=True, + ) + ) + gene_handler_timers.append( + app.timer( + 2.0, + _run_gene_handlers_js, + once=True, + ) + ) except Exception as e: - logging.warning(f"Could not add gene table click handlers: {e}") + logging.warning( + f"Could not add gene table click handlers: {e}" + ) try: + def _cleanup_gene_page() -> None: - gene_page_state["filtered_positions"] = [] + gene_page_state[ + "filtered_positions" + ] = [] gene_rows_source.clear() gene_table.rows = [] - for timer_obj in gene_handler_timers: + for ( + timer_obj + ) in gene_handler_timers: try: timer_obj.deactivate() except Exception: pass - ui.context.client.on_disconnect(_cleanup_gene_page) + + ui.context.client.on_disconnect( + _cleanup_gene_page + ) except Exception: pass @@ -7803,7 +8675,9 @@ def _cleanup_gene_page() -> None: f"Total target genes: {total_genes}" ).classes("classification-insight-foot") except Exception as e: - logging.warning(f"Could not load target gene table: {e}") + logging.warning( + f"Could not load target gene table: {e}" + ) tg_elapsed = time.perf_counter() - _t_target_genes logging.debug( "[SamplePage] page=sample_details sample=%s " @@ -7908,29 +8782,27 @@ def _create_workflow_monitor(self): center=self.center, setup_notifications=self._setup_notification_system, ): - with ui.element("div").classes("w-full min-w-0").props( - "id=workflow-monitor-page" + with ( + ui.element("div") + .classes("w-full min-w-0") + .props("id=workflow-monitor-page") ): - with ui.column().classes( - "w-full gap-3 p-2 md:p-3 max-w-6xl mx-auto" - ): + with ui.column().classes("w-full gap-3 p-2 md:p-3 max-w-6xl mx-auto"): with ui.element("div").classes( "classification-insight-shell w-full min-w-0" ): ui.label("Workflow monitor").classes( "classification-insight-heading text-headline-small" ) - ui.label( - "Real-time workflow monitoring and control." - ).classes("classification-insight-foot") + ui.label("Real-time workflow monitoring and control.").classes( + "classification-insight-foot" + ) # Workflow status overview with ui.element("div").classes( "classification-insight-card w-full min-w-0" ): - with ui.column().classes( - "w-full min-w-0 gap-3 p-2 md:p-3" - ): + with ui.column().classes("w-full min-w-0 gap-3 p-2 md:p-3"): with ui.row().classes("items-center gap-2 min-w-0"): ui.icon("monitor_heart").classes( "classification-insight-icon" @@ -7953,9 +8825,7 @@ def _create_workflow_monitor(self): "workflow-monitor-status-text--running" ) - with ui.row().classes( - "w-full gap-2 mt-2 flex-wrap" - ): + with ui.row().classes("w-full gap-2 mt-2 flex-wrap"): self.workflow_start_time = ui.label( "Started: —" ).classes("text-sm workflow-monitor-meta") @@ -7975,9 +8845,7 @@ def _create_workflow_monitor(self): ui.label("Run counts").classes( "target-coverage-panel__meta-label mt-2 mb-1" ) - with ui.row().classes( - "w-full gap-4 mt-1 flex-wrap" - ): + with ui.row().classes("w-full gap-4 mt-1 flex-wrap"): with ui.row().classes("items-center gap-2"): ui.label("Completed").classes( "text-xs workflow-monitor-meta" @@ -8007,7 +8875,9 @@ def _create_workflow_monitor(self): with ui.row().classes( "w-full min-w-0 gap-3 p-2 md:p-3 items-center flex-wrap" ): - ui.icon("biotech").classes("classification-insight-icon") + ui.icon("biotech").classes( + "classification-insight-icon" + ) with ui.column().classes("flex-1 min-w-0 gap-1"): ui.label("Sequencer (MinKNOW)").classes( "classification-insight-model" @@ -8026,9 +8896,7 @@ def _create_workflow_monitor(self): with ui.element("div").classes( "classification-insight-card w-full min-w-0" ): - with ui.column().classes( - "w-full min-w-0 gap-3 p-2 md:p-3" - ): + with ui.column().classes("w-full min-w-0 gap-3 p-2 md:p-3"): with ui.row().classes("items-center gap-2 min-w-0"): ui.icon("folder_open").classes( "classification-insight-icon" @@ -8037,44 +8905,36 @@ def _create_workflow_monitor(self): "classification-insight-model flex-1 min-w-0" ) - with ui.row().classes( - "w-full items-center gap-3 min-w-0" - ): + with ui.row().classes("w-full items-center gap-3 min-w-0"): ui.label("Overall files").classes( "text-sm font-medium workflow-monitor-meta shrink-0" ) - self.overall_files_progress = ( - ui.linear_progress(0.0).classes( - "flex-1 min-w-0" - ) - ) + self.overall_files_progress = ui.linear_progress( + 0.0 + ).classes("flex-1 min-w-0") self.overall_files_label = ui.label( "0/0 files processed" - ).classes( - "text-sm min-w-[120px] workflow-monitor-meta" - ) + ).classes("text-sm min-w-[120px] workflow-monitor-meta") ui.label("Per-sample progress").classes( "target-coverage-panel__meta-label mt-2 mb-1" ) - with ui.scroll_area().classes("w-full").style( - "max-height: 300px;" + with ( + ui.scroll_area() + .classes("w-full") + .style("max-height: 300px;") ): - self.sample_files_progress_container = ui.column().classes( - "w-full gap-2 p-1 min-w-0" + self.sample_files_progress_container = ( + ui.column().classes("w-full gap-2 p-1 min-w-0") ) # Queue status with ui.element("div").classes( "classification-insight-card w-full min-w-0" ): - with ui.column().classes( - "w-full min-w-0 gap-3 p-2 md:p-3" - ): + with ui.column().classes("w-full min-w-0 gap-3 p-2 md:p-3"): with ui.row().classes("items-center gap-2 min-w-0"): - ui.icon("layers").classes( - "classification-insight-icon" - ) + ui.icon("layers").classes("classification-insight-icon") ui.label("Queue status").classes( "classification-insight-model flex-1 min-w-0" ) @@ -8095,9 +8955,7 @@ def _create_workflow_monitor(self): with ui.element("div").classes( "workflow-monitor-queue-tile flex-1 min-w-[10rem]" ): - with ui.row().classes( - "items-center gap-1 min-w-0" - ): + with ui.row().classes("items-center gap-1 min-w-0"): ui.icon("science").classes( "text-base workflow-monitor-queue-icon" ) @@ -8114,7 +8972,9 @@ def _create_workflow_monitor(self): ui.label("Classification").classes( "classification-insight-foot" ) - self.classification_status = ui.label("0/0").classes( + self.classification_status = ui.label( + "0/0" + ).classes( "text-2xl font-bold workflow-monitor-queue-num--class" ) @@ -8138,23 +8998,19 @@ def _create_workflow_monitor(self): with ui.element("div").classes( "classification-insight-card w-full min-w-0" ): - with ui.column().classes( - "w-full min-w-0 gap-3 p-2 md:p-3" - ): + with ui.column().classes("w-full min-w-0 gap-3 p-2 md:p-3"): with ui.row().classes("items-center gap-2 min-w-0"): - ui.icon("work").classes( - "classification-insight-icon" - ) + ui.icon("work").classes("classification-insight-icon") ui.label("Active jobs").classes( "classification-insight-model flex-1 min-w-0" ) - with ui.row().classes( - "items-center gap-2 mb-2 flex-wrap" - ): - self.active_jobs_search = ui.input("Search…").props( - "outlined dense clearable" - ).classes("min-w-[12rem] flex-1") + with ui.row().classes("items-center gap-2 mb-2 flex-wrap"): + self.active_jobs_search = ( + ui.input("Search…") + .props("outlined dense clearable") + .classes("min-w-[12rem] flex-1") + ) self.active_jobs_type_filter = ( ui.select( options=["All"], @@ -8252,16 +9108,12 @@ def _on_active_jobs_filter_change(_=None): with ui.element("div").classes( "classification-insight-card w-full min-w-0" ): - with ui.column().classes( - "w-full min-w-0 gap-3 p-2 md:p-3" - ): + with ui.column().classes("w-full min-w-0 gap-3 p-2 md:p-3"): with ui.row().classes( "w-full items-center justify-between gap-2 " "flex-wrap min-w-0" ): - with ui.row().classes( - "items-center gap-2 min-w-0" - ): + with ui.row().classes("items-center gap-2 min-w-0"): ui.icon("article").classes( "classification-insight-icon" ) @@ -8281,12 +9133,8 @@ def _on_active_jobs_filter_change(_=None): ).props("color=primary no-caps outline") self.log_area = ( - ui.textarea( - "Workflow logs will appear here…" - ) - .classes( - "w-full h-40 workflow-monitor-log-area" - ) + ui.textarea("Workflow logs will appear here…") + .classes("w-full h-40 workflow-monitor-log-area") .props("outlined readonly dense") ) @@ -8294,20 +9142,14 @@ def _on_active_jobs_filter_change(_=None): with ui.element("div").classes( "classification-insight-card w-full min-w-0" ): - with ui.column().classes( - "w-full min-w-0 gap-3 p-2 md:p-3" - ): + with ui.column().classes("w-full min-w-0 gap-3 p-2 md:p-3"): with ui.row().classes("items-center gap-2 min-w-0"): - ui.icon("tune").classes( - "classification-insight-icon" - ) + ui.icon("tune").classes("classification-insight-icon") ui.label("Workflow configuration").classes( "classification-insight-model flex-1 min-w-0" ) - with ui.grid(columns=2).classes( - "w-full gap-3 min-w-0" - ): + with ui.grid(columns=2).classes("w-full gap-3 min-w-0"): with ui.column().classes("min-w-0 gap-1"): ui.label("Monitored directory").classes( "target-coverage-panel__meta-label" @@ -8326,9 +9168,7 @@ def _on_active_jobs_filter_change(_=None): ", ".join(self.workflow_steps) if self.workflow_steps else "Not specified" - ).classes( - "text-sm workflow-monitor-config-value" - ) + ).classes("text-sm workflow-monitor-config-value") with ui.column().classes("min-w-0 gap-1"): ui.label("Log level").classes( @@ -8349,9 +9189,7 @@ def _on_active_jobs_filter_change(_=None): with ui.element("div").classes( "classification-insight-card w-full min-w-0" ): - with ui.column().classes( - "w-full min-w-0 gap-3 p-2 md:p-3" - ): + with ui.column().classes("w-full min-w-0 gap-3 p-2 md:p-3"): with ui.row().classes("items-center gap-2 min-w-0"): ui.icon("error_outline").classes( "classification-insight-icon" @@ -8363,27 +9201,21 @@ def _on_active_jobs_filter_change(_=None): with ui.row().classes( "w-full justify-between gap-2 flex-wrap" ): - with ui.column().classes( - "text-center min-w-[5rem]" - ): + with ui.column().classes("text-center min-w-[5rem]"): self.preprocessing_errors = ui.label("0").classes( "text-xl font-bold workflow-monitor-error-num" ) ui.label("Preprocessing").classes( "text-xs workflow-monitor-meta" ) - with ui.column().classes( - "text-center min-w-[5rem]" - ): + with ui.column().classes("text-center min-w-[5rem]"): self.analysis_errors = ui.label("0").classes( "text-xl font-bold workflow-monitor-error-num" ) ui.label("Analysis").classes( "text-xs workflow-monitor-meta" ) - with ui.column().classes( - "text-center min-w-[5rem]" - ): + with ui.column().classes("text-center min-w-[5rem]"): self.classification_errors = ui.label("0").classes( "text-xl font-bold workflow-monitor-error-num" ) @@ -8397,13 +9229,9 @@ def _on_active_jobs_filter_change(_=None): ) self.error_summary_label = ui.label( "No errors detected." - ).classes( - "text-xs workflow-monitor-error-summary" - ) + ).classes("text-xs workflow-monitor-error-summary") - ui.label( - "R.O.B.I.N workflow monitor — session active" - ).classes( + ui.label("R.O.B.I.N workflow monitor — session active").classes( "classification-insight-foot text-center w-full py-2" ) @@ -8481,23 +9309,33 @@ def _locate_reference_for_sample(self, sample_dir: Path) -> Optional[str]: candidates: List[Path] = [] try: - candidates.extend([ - sample_dir / "reference.fasta", - sample_dir / "reference.fa", - ]) + candidates.extend( + [ + sample_dir / "reference.fasta", + sample_dir / "reference.fa", + ] + ) - base_dir = Path(self.monitored_directory) if self.monitored_directory else sample_dir.parent + base_dir = ( + Path(self.monitored_directory) + if self.monitored_directory + else sample_dir.parent + ) if base_dir: - candidates.extend([ - base_dir / "reference.fasta", - base_dir / "reference.fa", - ]) + candidates.extend( + [ + base_dir / "reference.fasta", + base_dir / "reference.fa", + ] + ) env_reference = os.environ.get("robin_REFERENCE") if env_reference: candidates.append(Path(env_reference)) except Exception as exc: - logging.debug(f"Error assembling fallback reference candidates for {sample_dir}: {exc}") + logging.debug( + f"Error assembling fallback reference candidates for {sample_dir}: {exc}" + ) for candidate in candidates: try: @@ -8556,7 +9394,9 @@ def _list_samples_needing_snp_calling(self) -> List[str]: if not targets_bed.is_file(): continue try: - if not targets_bed.read_text(encoding="utf-8", errors="replace").strip(): + if not targets_bed.read_text( + encoding="utf-8", errors="replace" + ).strip(): continue except OSError: continue @@ -8596,7 +9436,9 @@ def _wait_for_snp_outputs_or_terminal_status( while time.time() < deadline: if self._sample_has_snp_calling_outputs(sample_id): return True - phase = str(self._sample_pipeline_status.get(sample_id, {}).get("phase", "") or "") + phase = str( + self._sample_pipeline_status.get(sample_id, {}).get("phase", "") or "" + ) if phase in terminal_phases: return False time.sleep(poll_s) @@ -8681,9 +9523,7 @@ def _list_samples_needing_mnpflex_for_visible( if not sample_dir.exists(): continue - if ( - self._mnpflex_results_dir_for_sample(sample_dir, sid) is not None - ): + if self._mnpflex_results_dir_for_sample(sample_dir, sid) is not None: continue if not sample_ready_for_mnpflex_auto_run(r): @@ -8717,9 +9557,7 @@ def _zip_paths_for_bulk_download(self, paths: List[str]) -> Optional[str]: if os.path.exists(zpath): i = 2 while os.path.exists(zpath): - zpath = os.path.join( - tempfile.gettempdir(), f"{base_name}_{i}.zip" - ) + zpath = os.path.join(tempfile.gettempdir(), f"{base_name}_{i}.zip") i += 1 used_names: Set[str] = set() with zipfile.ZipFile(zpath, "w", zipfile.ZIP_DEFLATED) as zf: @@ -8759,9 +9597,13 @@ def _build_sample_tracking_tsv_export( table_fields: List[str] = [] table = getattr(self, "samples_table", None) - for col in (getattr(table, "columns", None) or []): + for col in getattr(table, "columns", None) or []: field = str(col.get("field", "") or "").strip() - if field and field not in ("actions", "export") and field not in table_fields: + if ( + field + and field not in ("actions", "export") + and field not in table_fields + ): table_fields.append(field) if not table_fields: table_fields = [ @@ -8805,7 +9647,12 @@ def _build_sample_tracking_tsv_export( "tucan", ] analysis_fields = { - "coverage": ["quality", "global_coverage", "target_coverage", "enrichment"], + "coverage": [ + "quality", + "global_coverage", + "target_coverage", + "enrichment", + ], "cnv": [ "genetic_sex", "bin_width", @@ -8882,12 +9729,15 @@ def _build_sample_tracking_tsv_export( from robin.gui.components.summary import _refresh_summary_cache_sync timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - out_path = os.path.join(tempfile.gettempdir(), f"robin_sample_tracking_{timestamp}.tsv") + out_path = os.path.join( + tempfile.gettempdir(), f"robin_sample_tracking_{timestamp}.tsv" + ) if os.path.exists(out_path): i = 2 while os.path.exists(out_path): out_path = os.path.join( - tempfile.gettempdir(), f"robin_sample_tracking_{timestamp}_{i}.tsv" + tempfile.gettempdir(), + f"robin_sample_tracking_{timestamp}_{i}.tsv", ) i += 1 @@ -8927,6 +9777,7 @@ def _clean_tsv_value(value: Any) -> Any: # Variant/SNP summary extraction (when available) try: + def _format_pathogenic_rows(rows: Any) -> str: if not isinstance(rows, list): return "" @@ -8951,7 +9802,9 @@ def _format_pathogenic_rows(rows: Any) -> str: significance = clnsig or onc or sci locus = f"{chrom}:{pos}" if chrom and pos else "" allele = f"{ref}>{alt}" if ref and alt else "" - item_parts = [p for p in [locus, allele, gene, significance] if p] + item_parts = [ + p for p in [locus, allele, gene, significance] if p + ] if item_parts: formatted.append("|".join(item_parts)) # Keep deterministic + compact while still "listing" all hits. @@ -8968,9 +9821,12 @@ def _format_pathogenic_rows(rows: Any) -> str: snp_display = json.load(f) summary_dict = snp_display.get("summary", {}) or {} rows_all = snp_display.get("rows_all", []) or [] - rows_pathogenic = snp_display.get("rows_pathogenic", []) or [] + rows_pathogenic = ( + snp_display.get("rows_pathogenic", []) or [] + ) rows_significant = ( - snp_display.get("rows_clinvar_significant") or rows_pathogenic + snp_display.get("rows_clinvar_significant") + or rows_pathogenic ) variant_data["snp_total_variants"] = summary_dict.get( "total_variants", len(rows_all) @@ -8981,18 +9837,22 @@ def _format_pathogenic_rows(rows: Any) -> str: "pathogenic_variants", len(rows_significant) ), ) - variant_data["snp_pathogenic_list"] = _format_pathogenic_rows( - rows_significant + variant_data["snp_pathogenic_list"] = ( + _format_pathogenic_rows(rows_significant) ) # INDEL summary (from ClinVar-annotated display if available) - indel_display_path = clair_dir / "snpsift_indel_output_display.json" + indel_display_path = ( + clair_dir / "snpsift_indel_output_display.json" + ) if indel_display_path.exists(): with open(indel_display_path, "r", encoding="utf-8") as f: indel_display = json.load(f) indel_summary = indel_display.get("summary", {}) or {} indel_rows_all = indel_display.get("rows_all", []) or [] - indel_rows_pathogenic = indel_display.get("rows_pathogenic", []) or [] + indel_rows_pathogenic = ( + indel_display.get("rows_pathogenic", []) or [] + ) indel_rows_significant = ( indel_display.get("rows_clinvar_significant") or indel_rows_pathogenic @@ -9000,14 +9860,17 @@ def _format_pathogenic_rows(rows: Any) -> str: variant_data["indel_total_variants"] = indel_summary.get( "total_variants", len(indel_rows_all) ) - variant_data["indel_pathogenic_variants"] = indel_summary.get( - "clinvar_significant_variants", + variant_data["indel_pathogenic_variants"] = ( indel_summary.get( - "pathogenic_variants", len(indel_rows_significant) - ), + "clinvar_significant_variants", + indel_summary.get( + "pathogenic_variants", + len(indel_rows_significant), + ), + ) ) - variant_data["indel_pathogenic_list"] = _format_pathogenic_rows( - indel_rows_significant + variant_data["indel_pathogenic_list"] = ( + _format_pathogenic_rows(indel_rows_significant) ) else: # Fallback: count records from annotated INDEL VCF if present @@ -9016,7 +9879,9 @@ def _format_pathogenic_rows(rows: Any) -> str: total_indel = 0 pathogenic_indel = 0 pathogenic_indel_items: List[str] = [] - with open(indel_vcf, "r", encoding="utf-8", errors="ignore") as f: + with open( + indel_vcf, "r", encoding="utf-8", errors="ignore" + ) as f: for line in f: if not line or line.startswith("#"): continue @@ -9031,7 +9896,9 @@ def _format_pathogenic_rows(rows: Any) -> str: f"{chrom}:{pos}|{ref}>{alt}" ) variant_data["indel_total_variants"] = total_indel - variant_data["indel_pathogenic_variants"] = pathogenic_indel + variant_data["indel_pathogenic_variants"] = ( + pathogenic_indel + ) variant_data["indel_pathogenic_list"] = "; ".join( pathogenic_indel_items ) @@ -9050,7 +9917,13 @@ def _format_pathogenic_rows(rows: Any) -> str: arm_events = 0 broad_gain_events = 0 broad_loss_events = 0 - with open(cnv_results_csv, "r", newline="", encoding="utf-8", errors="ignore") as f: + with open( + cnv_results_csv, + "r", + newline="", + encoding="utf-8", + errors="ignore", + ) as f: reader = csv.DictReader(f) for event_row in reader: region = str( @@ -9086,7 +9959,9 @@ def _format_pathogenic_rows(rows: Any) -> str: ) try: - results_dir = self._mnpflex_results_dir_for_sample(sample_dir, sample_id) + results_dir = self._mnpflex_results_dir_for_sample( + sample_dir, sample_id + ) if results_dir: bundle_path = results_dir / "bundle_summary.json" if bundle_path.exists(): @@ -9094,15 +9969,26 @@ def _format_pathogenic_rows(rows: Any) -> str: bundle = json.load(f) qc = bundle.get("qc", {}) or {} mgmt = bundle.get("mgmt", {}) or {} - classifier_summary = bundle.get("classifier_summary", {}) or {} - classifier = classifier_summary.get("classifier", {}) or {} - hierarchy = classifier_summary.get("summary_hierarchical", []) or [] + classifier_summary = ( + bundle.get("classifier_summary", {}) or {} + ) + classifier = ( + classifier_summary.get("classifier", {}) or {} + ) + hierarchy = ( + classifier_summary.get("summary_hierarchical", []) + or [] + ) scores = classifier_summary.get("scores") or [] top_path = "" top_path_score = "" try: if hierarchy: - def _flatten(nodes: List[Dict[str, Any]], path: Optional[List[str]] = None): + + def _flatten( + nodes: List[Dict[str, Any]], + path: Optional[List[str]] = None, + ): current_path = path or [] flat_rows = [] for node in nodes or []: @@ -9111,24 +9997,32 @@ def _flatten(nodes: List[Dict[str, Any]], path: Optional[List[str]] = None): next_path = current_path + [group] members = node.get("members") or [] if members: - flat_rows.extend(_flatten(members, next_path)) + flat_rows.extend( + _flatten(members, next_path) + ) else: flat_rows.append((score, next_path)) return flat_rows flat = _flatten(hierarchy) if flat: - best_score, best_path = max(flat, key=lambda x: x[0] or 0) + best_score, best_path = max( + flat, key=lambda x: x[0] or 0 + ) top_path = " > ".join(best_path) top_path_score = best_score elif scores: top = sorted( scores, - key=lambda item: float(item.get("score", 0) or 0), + key=lambda item: float( + item.get("score", 0) or 0 + ), reverse=True, )[:1] if top: - top_ref = top[0].get("reference_group") or {} + top_ref = ( + top[0].get("reference_group") or {} + ) top_path = ( top_ref.get("molecular_subclass") or top_ref.get("name") @@ -9140,19 +10034,27 @@ def _flatten(nodes: List[Dict[str, Any]], path: Optional[List[str]] = None): mnpflex_data = { "qc_status": qc.get("status", ""), "qc_avg_coverage": qc.get("avg_coverage", ""), - "qc_missing_site_count": qc.get("missing_site_count", ""), + "qc_missing_site_count": qc.get( + "missing_site_count", "" + ), "mgmt_status": mgmt.get("status", ""), "mgmt_average": mgmt.get("average", ""), "mgmt_site_count": mgmt.get("site_count", ""), "classifier_name": classifier.get("name", ""), "classifier_version": classifier.get("version", ""), - "classifier_type": classifier.get("classifier_type", ""), + "classifier_type": classifier.get( + "classifier_type", "" + ), "has_hierarchical_summary": bool(hierarchy), "top_path": top_path, "top_path_score": top_path_score, } except Exception as ex: - logging.debug("Could not extract MNP-Flex TSV fields for %s: %s", sample_id, ex) + logging.debug( + "Could not extract MNP-Flex TSV fields for %s: %s", + sample_id, + ex, + ) if mnpflex_data: analysis["mnpflex"] = mnpflex_data if variant_data: @@ -9160,24 +10062,40 @@ def _flatten(nodes: List[Dict[str, Any]], path: Optional[List[str]] = None): if cnv_broad_data: analysis["cnv_broad"] = cnv_broad_data - export_row: Dict[str, Any] = {k: row_data.get(k, "") for k in table_fields} + export_row: Dict[str, Any] = { + k: row_data.get(k, "") for k in table_fields + } for key in run_info_fields: export_row[f"run_summary_{key}"] = run_info.get(key, "") for model_name in classification_models: model_data = classification.get(model_name, {}) or {} - export_row[f"classification_{model_name}_class"] = model_data.get("classification", "") - export_row[f"classification_{model_name}_confidence"] = model_data.get("confidence", "") - export_row[f"classification_{model_name}_confidence_level"] = model_data.get("confidence_level", "") - export_row[f"classification_{model_name}_features"] = model_data.get("features", "") + export_row[f"classification_{model_name}_class"] = ( + model_data.get("classification", "") + ) + export_row[f"classification_{model_name}_confidence"] = ( + model_data.get("confidence", "") + ) + export_row[f"classification_{model_name}_confidence_level"] = ( + model_data.get("confidence_level", "") + ) + export_row[f"classification_{model_name}_features"] = ( + model_data.get("features", "") + ) for section, keys in analysis_fields.items(): section_data = analysis.get(section, {}) or {} for key in keys: - export_row[f"analysis_{section}_{key}"] = section_data.get(key, "") - writer.writerow({k: _clean_tsv_value(v) for k, v in export_row.items()}) + export_row[f"analysis_{section}_{key}"] = section_data.get( + key, "" + ) + writer.writerow( + {k: _clean_tsv_value(v) for k, v in export_row.items()} + ) return out_path except Exception as e: - logging.error("Failed to build sample tracking TSV export: %s", e, exc_info=True) + logging.error( + "Failed to build sample tracking TSV export: %s", e, exc_info=True + ) return None def _wait_for_snp_outputs_or_timeout( @@ -9269,7 +10187,9 @@ def _bulk_snp_sequential_run(self, sample_ids: List[str]) -> None: threading.current_thread().name, len(sample_ids or []), ) - from robin.analysis.target_analysis import is_docker_available_for_snp_analysis + from robin.analysis.target_analysis import ( + is_docker_available_for_snp_analysis, + ) docker_ok, docker_error = is_docker_available_for_snp_analysis() if not docker_ok: @@ -9332,12 +10252,14 @@ def _bulk_snp_sequential_run(self, sample_ids: List[str]) -> None: progress=1.0, detail="SNP outputs detected", ) - logging.info("Bulk SNP: completed via finalize-first %s", sid) + logging.info( + "Bulk SNP: completed via finalize-first %s", sid + ) else: phase = str( - self._sample_pipeline_status - .get(sid, {}) - .get("phase", "SNP still running") + self._sample_pipeline_status.get(sid, {}).get( + "phase", "SNP still running" + ) ) if phase not in { "SNP skipped", @@ -9393,9 +10315,7 @@ def _bulk_snp_sequential_run(self, sample_ids: List[str]) -> None: with self._snp_analysis_lock: self._snp_analysis_running_sample = sid try: - logging.info( - "Bulk SNP (%d/%d): submitting %s", idx, total, sid - ) + logging.info("Bulk SNP (%d/%d): submitting %s", idx, total, sid) submitted = self._resolve_submission_result( workflow_runner.submit_snp_analysis_job( sample_dir=str(sample_dir), @@ -9566,9 +10486,7 @@ def _bulk_mnpflex_sequential_run(self, sample_ids: List[str]) -> None: continue try: - logging.info( - "Bulk MNP-Flex (%d/%d): running %s", idx, total, sid - ) + logging.info("Bulk MNP-Flex (%d/%d): running %s", idx, total, sid) self._audit_log( event_type="run.started", user_id=self._get_current_user_id(), @@ -9682,10 +10600,14 @@ def _unlink_quiet(self, path: str) -> None: except OSError: pass - def _trigger_target_bam_finalization(self, sample_id: str, *, trigger_snp: bool = False) -> None: + def _trigger_target_bam_finalization( + self, sample_id: str, *, trigger_snp: bool = False + ) -> None: """Trigger target.bam finalization for a sample. Optionally start SNP analysis afterwards.""" logging.debug( - "target_bam_finalize: start sample_id=%s trigger_snp=%s", sample_id, trigger_snp + "target_bam_finalize: start sample_id=%s trigger_snp=%s", + sample_id, + trigger_snp, ) self._set_pipeline_status( sample_id, @@ -9707,7 +10629,8 @@ def _trigger_target_bam_finalization(self, sample_id: str, *, trigger_snp: bool if self._is_target_bam_finalize_redundant(sample_id): self._finalized_samples.add(sample_id) logging.debug( - "target_bam_finalize: redundant (target.bam already) sample_id=%s", sample_id + "target_bam_finalize: redundant (target.bam already) sample_id=%s", + sample_id, ) self._set_pipeline_status( sample_id, @@ -9756,6 +10679,7 @@ def _trigger_target_bam_finalization(self, sample_id: str, *, trigger_snp: bool # Trigger finalization (and optional SNP analysis) in background thread to avoid blocking GUI import threading + def finalize_in_background(): try: logging.debug( @@ -9774,7 +10698,9 @@ def finalize_in_background(): finalization_succeeded = True if already_finalized: - logging.info(f"Sample {sample_id} already finalized; skipping target.bam merge") + logging.info( + f"Sample {sample_id} already finalized; skipping target.bam merge" + ) else: logging.debug( "target_bam_finalize: merge path sample_id=%s", sample_id @@ -9901,7 +10827,9 @@ def finalize_in_background(): sample_id, phase="Finalize failed", progress=0.2, - detail=str(result.get("error", "unknown error"))[:120], + detail=str(result.get("error", "unknown error"))[ + :120 + ], level="negative", ) logging.warning( @@ -9945,11 +10873,14 @@ def finalize_in_background(): self._snp_analysis_running_sample = sample_id logging.debug( - "target_bam_finalize: snp lock set sample_id=%s", sample_id + "target_bam_finalize: snp lock set sample_id=%s", + sample_id, ) try: - reference_path = self._locate_reference_for_sample(sample_dir) + reference_path = self._locate_reference_for_sample( + sample_dir + ) if not reference_path: logging.debug( "target_bam_finalize: no reference sample_id=%s", @@ -9992,7 +10923,9 @@ def finalize_in_background(): detail="target.bam missing", level="warning", ) - message = "target.bam not found; cannot run SNP analysis." + message = ( + "target.bam not found; cannot run SNP analysis." + ) logging.warning(message) self.send_update( UpdateType.WARNING_NOTIFICATION, @@ -10006,8 +10939,13 @@ def finalize_in_background(): ) return - from robin.analysis.target_analysis import is_docker_available_for_snp_analysis - docker_ok, docker_error = is_docker_available_for_snp_analysis() + from robin.analysis.target_analysis import ( + is_docker_available_for_snp_analysis, + ) + + docker_ok, docker_error = ( + is_docker_available_for_snp_analysis() + ) if not docker_ok: logging.debug( "target_bam_finalize: docker unavailable sample_id=%s: %s", @@ -10102,6 +11040,7 @@ def finalize_in_background(): progress=0.85, detail="Queued in slow worker", ) + def _mark_snp_completion(_sid: str) -> None: finished = self._wait_for_snp_outputs_or_timeout( _sid, poll_s=5.0, max_wait_s=86400.0 @@ -10224,7 +11163,9 @@ def _mark_snp_completion(_sid: str) -> None: logging.debug( "target_bam_finalize: outer exception sample_id=%s: %s", sample_id, e ) - logging.error(f"Failed to trigger target.bam finalization for {sample_id}: {e}") + logging.error( + f"Failed to trigger target.bam finalization for {sample_id}: {e}" + ) def _get_expected_completion_job_types(self) -> Set[str]: """Get the set of job types expected to complete for this workflow.""" @@ -10237,7 +11178,9 @@ def _get_expected_completion_job_types(self) -> Set[str]: expected.add(step_name) return expected - def _expected_jobs_completed(self, sample_dir: Path, expected_job_types: Set[str]) -> bool: + def _expected_jobs_completed( + self, sample_dir: Path, expected_job_types: Set[str] + ) -> bool: """Return True if all expected job types have completion outputs.""" if not expected_job_types: return True @@ -10294,16 +11237,18 @@ def _calculate_job_counts_from_files( "total_jobs": total_jobs, "completed_jobs": completed_jobs, "failed_jobs": failed_jobs, - "job_types": ", ".join(sorted(job_types)) if job_types else "" + "job_types": ", ".join(sorted(job_types)) if job_types else "", } except Exception as e: - logging.warning(f"Error calculating job counts from files for {sample_dir}: {e}") + logging.warning( + f"Error calculating job counts from files for {sample_dir}: {e}" + ) return { "total_jobs": 0, "completed_jobs": 0, "failed_jobs": 0, - "job_types": "" + "job_types": "", } def _create_watched_folders_page(self): @@ -10311,8 +11256,8 @@ def _create_watched_folders_page(self): try: from robin.workflow_ray import ( add_watch_path, - remove_watch_path, get_watched_paths, + remove_watch_path, ) except ImportError: with theme.frame( @@ -10326,8 +11271,10 @@ def _create_watched_folders_page(self): "Manage folders requires Ray workflow. Are you running with --use-ray?", type="negative", ) - with ui.element("div").classes("w-full min-w-0").props( - "id=watched-folders-page" + with ( + ui.element("div") + .classes("w-full min-w-0") + .props("id=watched-folders-page") ): with ui.column().classes( "w-full max-w-2xl mx-auto gap-3 p-2 md:p-3" @@ -10357,12 +11304,12 @@ def _create_watched_folders_page(self): center=self.center, setup_notifications=self._setup_notification_system, ): - with ui.element("div").classes("w-full min-w-0").props( - "id=watched-folders-page" + with ( + ui.element("div") + .classes("w-full min-w-0") + .props("id=watched-folders-page") ): - with ui.column().classes( - "w-full max-w-2xl mx-auto gap-3 p-2 md:p-3" - ): + with ui.column().classes("w-full max-w-2xl mx-auto gap-3 p-2 md:p-3"): with ui.element("div").classes( "classification-insight-shell w-full min-w-0" ): @@ -10372,12 +11319,8 @@ def _create_watched_folders_page(self): with ui.element("div").classes( "classification-insight-card w-full min-w-0" ): - with ui.column().classes( - "w-full min-w-0 gap-3 p-2 md:p-3" - ): - with ui.row().classes( - "items-center gap-2 min-w-0" - ): + with ui.column().classes("w-full min-w-0 gap-3 p-2 md:p-3"): + with ui.row().classes("items-center gap-2 min-w-0"): ui.icon("folder_special").classes( "classification-insight-icon" ) @@ -10415,13 +11358,13 @@ def _create_watched_folders_page(self): remove_watch_path, get_watched_paths, ), - ).props("flat color=negative size=sm no-caps") + ).props( + "flat color=negative size=sm no-caps" + ) else: ui.label( "No folders currently watched." - ).classes( - "classification-insight-foot italic" - ) + ).classes("classification-insight-foot italic") ui.separator().classes("mgmt-detail-separator") @@ -10519,9 +11462,11 @@ async def do_add_folder(): type="warning", ) return - with ui.dialog().props("persistent").classes( - "w-full max-w-sm" - ) as add_dialog: + with ( + ui.dialog() + .props("persistent") + .classes("w-full max-w-sm") as add_dialog + ): with ui.card().classes( "robin-dialog-surface p-4 md:p-5 w-full" ): @@ -10640,7 +11585,9 @@ def _load_plotting_preferences(self): raw = self.security_store.get_gui_setting(PLOTTING_PREFERENCES_KEY) return PlottingPreferencesConfig.from_dict(raw) - def save_plotting_preferences(self, config, *, user_id: Optional[int] = None) -> None: + def save_plotting_preferences( + self, config, *, user_id: Optional[int] = None + ) -> None: """Persist plotting preferences and refresh the in-memory copy.""" from robin.gui.plotting_preferences import PLOTTING_PREFERENCES_KEY from robin.security.store import utc_now_iso @@ -10678,12 +11625,12 @@ def _create_sample_id_generator_page(self): center=self.center, setup_notifications=self._setup_notification_system, ): - with ui.element("div").classes("w-full min-w-0").props( - "id=sample-id-generator-page" + with ( + ui.element("div") + .classes("w-full min-w-0") + .props("id=sample-id-generator-page") ): - with ui.column().classes( - "w-full max-w-2xl mx-auto gap-3 p-2 md:p-3" - ): + with ui.column().classes("w-full max-w-2xl mx-auto gap-3 p-2 md:p-3"): with ui.element("div").classes( "classification-insight-shell w-full min-w-0" ): @@ -10693,12 +11640,8 @@ def _create_sample_id_generator_page(self): with ui.element("div").classes( "classification-insight-card w-full min-w-0" ): - with ui.column().classes( - "w-full min-w-0 gap-3 p-2 md:p-3" - ): - with ui.row().classes( - "items-center gap-2 min-w-0" - ): + with ui.column().classes("w-full min-w-0 gap-3 p-2 md:p-3"): + with ui.row().classes("items-center gap-2 min-w-0"): ui.icon("fingerprint").classes( "classification-insight-icon" ) @@ -10714,13 +11657,17 @@ def _create_sample_id_generator_page(self): "encrypted alongside the sample, provide a date of birth." ).classes("classification-insight-foot") - id_mode = ui.toggle( - { - "custom": "Use my sample ID", - "md5": "Generate MD5 ID", - }, - value="custom", - ).props("no-caps dense").classes("w-full") + id_mode = ( + ui.toggle( + { + "custom": "Use my sample ID", + "md5": "Generate MD5 ID", + }, + value="custom", + ) + .props("no-caps dense") + .classes("w-full") + ) md5_fields = ui.column().classes("w-full min-w-0 gap-3") with md5_fields: @@ -10773,10 +11720,14 @@ def _create_sample_id_generator_page(self): placeholder="e.g. 123 456 7890", ).classes("w-full") - notes = ui.textarea( - label="Notes (optional)", - placeholder="Free-text notes stored encrypted with the identifiers", - ).classes("w-full").props("outlined dense autogrow") + notes = ( + ui.textarea( + label="Notes (optional)", + placeholder="Free-text notes stored encrypted with the identifiers", + ) + .classes("w-full") + .props("outlined dense autogrow") + ) ui.separator().classes("mgmt-detail-separator") @@ -10787,11 +11738,13 @@ def _create_sample_id_generator_page(self): "text-sm font-mono break-all p-3 rounded sample-id-gen-hash-preview" ) result_label.set_visibility(False) - result_input = ui.input( - label="Sample ID", - placeholder="Register to create or confirm an ID", - ).classes("w-full font-mono").props( - "readonly outlined dense" + result_input = ( + ui.input( + label="Sample ID", + placeholder="Register to create or confirm an ID", + ) + .classes("w-full font-mono") + .props("readonly outlined dense") ) def _sync_id_mode() -> None: @@ -10809,11 +11762,14 @@ def register_sample_id() -> None: try: registration = build_sample_registration( mode=id_mode.value or "custom", - custom_sample_id=custom_sample_id.value or "", + custom_sample_id=custom_sample_id.value + or "", test_id=( (test_id.value or "").strip() if id_mode.value == "md5" - else (custom_test_id.value or "").strip() + else ( + custom_test_id.value or "" + ).strip() ), first_name=first_name.value or "", last_name=last_name.value or "", @@ -10858,9 +11814,7 @@ def copy_to_clipboard() -> None: type="warning", ) - with ui.row().classes( - "w-full gap-2 mt-2 flex-wrap" - ): + with ui.row().classes("w-full gap-2 mt-2 flex-wrap"): ui.button( "Register sample ID", on_click=register_sample_id, @@ -10906,6 +11860,7 @@ async def _do_add_folder( try: from nicegui import run as ng_run + success, message = await ng_run.io_bound(add_watch_path, path_val) except ImportError: success, message = add_watch_path(path_val) @@ -10919,7 +11874,10 @@ async def _do_add_folder( msg_l = "" notify_type = ( "warning" - if ("skipped previously-analysed" in msg_l or "skipped previously analyzed" in msg_l) + if ( + "skipped previously-analysed" in msg_l + or "skipped previously analyzed" in msg_l + ) else "positive" ) self._safe_notify(message, notify_type) @@ -10929,9 +11887,15 @@ async def _do_add_folder( if "deleted" not in str(e).lower() and "client" not in str(e).lower(): raise # Dialog/slot was closed or client disconnected; skip clearing input - if watched_container is not None and get_watched_paths is not None and remove_watch_path is not None: + if ( + watched_container is not None + and get_watched_paths is not None + and remove_watch_path is not None + ): try: - self._refresh_watched_list(watched_container, remove_watch_path, get_watched_paths) + self._refresh_watched_list( + watched_container, remove_watch_path, get_watched_paths + ) except RuntimeError as e: if "deleted" not in str(e).lower(): raise @@ -11019,16 +11983,22 @@ async def _do_add_folders( raise return failures - def _do_remove_folder(self, path, watched_container, remove_watch_path, get_watched_paths): + def _do_remove_folder( + self, path, watched_container, remove_watch_path, get_watched_paths + ): """Remove a folder from the watch list.""" success, message = remove_watch_path(path) if success: ui.notify(message, type="positive") - self._refresh_watched_list(watched_container, remove_watch_path, get_watched_paths) + self._refresh_watched_list( + watched_container, remove_watch_path, get_watched_paths + ) else: ui.notify(message, type="negative") - def _refresh_watched_list(self, watched_container, remove_watch_path, get_watched_paths): + def _refresh_watched_list( + self, watched_container, remove_watch_path, get_watched_paths + ): """Refresh the list of watched paths in the dialog.""" if not self._safe_clear_container(watched_container): return diff --git a/src/robin/logging_config.py b/src/robin/logging_config.py index 2b639445..d32d7bab 100644 --- a/src/robin/logging_config.py +++ b/src/robin/logging_config.py @@ -12,9 +12,9 @@ import logging import os import sys -from typing import Any, Dict, Optional -from dataclasses import dataclass, field from contextlib import contextmanager +from dataclasses import dataclass, field +from typing import Any, Dict, Optional @dataclass diff --git a/src/robin/memory_manager.py b/src/robin/memory_manager.py index 1892881e..494e9241 100644 --- a/src/robin/memory_manager.py +++ b/src/robin/memory_manager.py @@ -46,23 +46,24 @@ """ import gc +import logging import os +from typing import Any, Dict, Optional + import psutil -import logging -from typing import Optional, Dict, Any class MemoryManager: """ Memory management utility for long-running Ray actors. - + Monitors memory usage and triggers garbage collection based on: 1. Iteration count (every N iterations) 2. RSS memory threshold (when memory exceeds limit) - + Also optionally attempts to return freed memory to the OS using malloc_trim(). """ - + def __init__( self, gc_every: int = 50, @@ -70,11 +71,11 @@ def __init__( enable_malloc_trim: bool = True, restart_every: int = 10000, restart_rss_trigger_mb: int = 4096, - logger: Optional[logging.Logger] = None + logger: Optional[logging.Logger] = None, ): """ Initialize the memory manager. - + Args: gc_every: Trigger garbage collection every N iterations rss_trigger_mb: Trigger garbage collection when RSS exceeds this many MB @@ -88,26 +89,29 @@ def __init__( self._rss_trigger_bytes = rss_trigger_mb * 1024 * 1024 self._enable_malloc_trim = enable_malloc_trim self._logger = logger or logging.getLogger(__name__) - + # Actor restart mechanism disabled - keeping only memory cleanup self._restart_every = restart_every # Keep for stats but not used - self._restart_rss_trigger_bytes = restart_rss_trigger_mb * 1024 * 1024 # Keep for stats but not used + self._restart_rss_trigger_bytes = ( + restart_rss_trigger_mb * 1024 * 1024 + ) # Keep for stats but not used self._restart_requested = False self._restart_reason = None - + # Get process handle for memory monitoring try: self._process = psutil.Process(os.getpid()) except Exception as e: self._logger.warning(f"Failed to initialize psutil process handle: {e}") self._process = None - + # Initialize malloc_trim if enabled self._malloc_trim_func = None if self._enable_malloc_trim: try: import ctypes import ctypes.util + libc = ctypes.CDLL(ctypes.util.find_library("c")) self._malloc_trim_func = libc.malloc_trim self._logger.debug("malloc_trim() available for memory optimization") @@ -115,84 +119,86 @@ def __init__( self._logger.debug(f"malloc_trim() not available: {e}") # Don't change the setting, just disable the function self._malloc_trim_func = None - + self._logger.info( f"MemoryManager initialized: gc_every={self._gc_every}, " f"rss_trigger={rss_trigger_mb}MB, malloc_trim={self._enable_malloc_trim}, " f"actor_restarts=DISABLED (memory cleanup only)" ) - - def check_and_cleanup(self, force: bool = False, is_idle: bool = True) -> Dict[str, Any]: + + def check_and_cleanup( + self, force: bool = False, is_idle: bool = True + ) -> Dict[str, Any]: """ Check memory usage and trigger cleanup if needed. - + Args: force: Force garbage collection regardless of thresholds is_idle: Whether the actor is currently idle (no active jobs) - + Returns: Dictionary with cleanup statistics """ self._iteration_count += 1 - + # Get current memory info memory_info = self._get_memory_info() - rss_bytes = memory_info.get('rss_bytes', 0) - + rss_bytes = memory_info.get("rss_bytes", 0) + # Determine if cleanup is needed # Only perform cleanup if actor is idle (unless forced) - needs_gc = ( - force or - (is_idle and ( - (self._iteration_count % self._gc_every == 0) or - (rss_bytes > self._rss_trigger_bytes) - )) + needs_gc = force or ( + is_idle + and ( + (self._iteration_count % self._gc_every == 0) + or (rss_bytes > self._rss_trigger_bytes) + ) ) - + cleanup_stats = { - 'iteration': self._iteration_count, - 'rss_mb': memory_info.get('rss_mb', 0), - 'gc_triggered': False, - 'malloc_trim_attempted': False, - 'trigger_reason': None, - 'is_idle': is_idle, - 'skipped_due_to_busy': False, - 'restart_requested': False, - 'restart_reason': None + "iteration": self._iteration_count, + "rss_mb": memory_info.get("rss_mb", 0), + "gc_triggered": False, + "malloc_trim_attempted": False, + "trigger_reason": None, + "is_idle": is_idle, + "skipped_due_to_busy": False, + "restart_requested": False, + "restart_reason": None, } - + if needs_gc: - cleanup_stats['gc_triggered'] = True - + cleanup_stats["gc_triggered"] = True + # Determine trigger reason if force: - cleanup_stats['trigger_reason'] = 'forced' + cleanup_stats["trigger_reason"] = "forced" elif self._iteration_count % self._gc_every == 0: - cleanup_stats['trigger_reason'] = 'iteration_count' + cleanup_stats["trigger_reason"] = "iteration_count" elif rss_bytes > self._rss_trigger_bytes: - cleanup_stats['trigger_reason'] = 'rss_threshold' - + cleanup_stats["trigger_reason"] = "rss_threshold" + # Perform garbage collection try: collected = gc.collect() - cleanup_stats['objects_collected'] = collected + cleanup_stats["objects_collected"] = collected self._logger.debug(f"Garbage collection: {collected} objects collected") except Exception as e: self._logger.warning(f"Garbage collection failed: {e}") - cleanup_stats['gc_error'] = str(e) - + cleanup_stats["gc_error"] = str(e) + # Attempt malloc_trim if enabled if self._enable_malloc_trim and self._malloc_trim_func: try: # malloc_trim(0) returns freed memory to OS result = self._malloc_trim_func(0) - cleanup_stats['malloc_trim_attempted'] = True - cleanup_stats['malloc_trim_result'] = result + cleanup_stats["malloc_trim_attempted"] = True + cleanup_stats["malloc_trim_result"] = result self._logger.debug(f"malloc_trim() returned: {result}") except Exception as e: self._logger.debug(f"malloc_trim() failed: {e}") - cleanup_stats['malloc_trim_error'] = str(e) - + cleanup_stats["malloc_trim_error"] = str(e) + # Log cleanup event self._logger.info( f"Memory cleanup triggered ({cleanup_stats['trigger_reason']}): " @@ -202,109 +208,114 @@ def check_and_cleanup(self, force: bool = False, is_idle: bool = True) -> Dict[s ) elif not is_idle and not force: # Log that cleanup was skipped due to busy state - cleanup_stats['skipped_due_to_busy'] = True + cleanup_stats["skipped_due_to_busy"] = True self._logger.debug( f"Memory cleanup skipped: actor busy (RSS={cleanup_stats['rss_mb']:.1f}MB, " f"iterations={self._iteration_count})" ) - + # Actor restart mechanism disabled - keeping only memory cleanup # This allows actors to run indefinitely while still managing memory - cleanup_stats['restart_requested'] = False - cleanup_stats['restart_reason'] = None - + cleanup_stats["restart_requested"] = False + cleanup_stats["restart_reason"] = None + return cleanup_stats - + def _get_memory_info(self) -> Dict[str, Any]: """Get current memory usage information.""" if self._process is None: - return {'rss_bytes': 0, 'rss_mb': 0, 'error': 'process_not_available'} - + return {"rss_bytes": 0, "rss_mb": 0, "error": "process_not_available"} + try: memory_info = self._process.memory_info() rss_bytes = memory_info.rss rss_mb = rss_bytes / (1024 * 1024) - + return { - 'rss_bytes': rss_bytes, - 'rss_mb': rss_mb, - 'vms_bytes': memory_info.vms, - 'vms_mb': memory_info.vms / (1024 * 1024) + "rss_bytes": rss_bytes, + "rss_mb": rss_mb, + "vms_bytes": memory_info.vms, + "vms_mb": memory_info.vms / (1024 * 1024), } except Exception as e: self._logger.warning(f"Failed to get memory info: {e}") - return {'rss_bytes': 0, 'rss_mb': 0, 'error': str(e)} - + return {"rss_bytes": 0, "rss_mb": 0, "error": str(e)} + def get_stats(self) -> Dict[str, Any]: """Get current memory manager statistics.""" memory_info = self._get_memory_info() - + return { - 'iteration_count': self._iteration_count, - 'gc_every': self._gc_every, - 'rss_trigger_mb': self._rss_trigger_bytes / (1024 * 1024), - 'enable_malloc_trim': self._enable_malloc_trim, - 'restart_every': self._restart_every, - 'restart_rss_trigger_mb': self._restart_rss_trigger_bytes / (1024 * 1024), - 'restart_requested': self._restart_requested, - 'restart_reason': self._restart_reason, - 'current_memory': memory_info, - 'next_gc_at': self._gc_every - (self._iteration_count % self._gc_every), - 'next_restart_at': self._restart_every - (self._iteration_count % self._restart_every) + "iteration_count": self._iteration_count, + "gc_every": self._gc_every, + "rss_trigger_mb": self._rss_trigger_bytes / (1024 * 1024), + "enable_malloc_trim": self._enable_malloc_trim, + "restart_every": self._restart_every, + "restart_rss_trigger_mb": self._restart_rss_trigger_bytes / (1024 * 1024), + "restart_requested": self._restart_requested, + "restart_reason": self._restart_reason, + "current_memory": memory_info, + "next_gc_at": self._gc_every - (self._iteration_count % self._gc_every), + "next_restart_at": self._restart_every + - (self._iteration_count % self._restart_every), } - + def reset_iteration_count(self): """Reset the iteration counter.""" self._iteration_count = 0 self._logger.debug("Iteration count reset") - + def is_restart_requested(self) -> bool: """Check if actor restart has been requested.""" return self._restart_requested - + def get_restart_reason(self) -> Optional[str]: """Get the reason for restart request.""" return self._restart_reason - + def clear_restart_request(self): """Clear the restart request (call after restart).""" self._restart_requested = False self._restart_reason = None self._logger.info("Actor restart request cleared") - + def update_settings( self, gc_every: Optional[int] = None, rss_trigger_mb: Optional[int] = None, enable_malloc_trim: Optional[bool] = None, restart_every: Optional[int] = None, - restart_rss_trigger_mb: Optional[int] = None + restart_rss_trigger_mb: Optional[int] = None, ): """Update memory manager settings.""" if gc_every is not None: self._gc_every = max(1, gc_every) self._logger.info(f"Updated gc_every to {self._gc_every}") - + if rss_trigger_mb is not None: self._rss_trigger_bytes = rss_trigger_mb * 1024 * 1024 self._logger.info(f"Updated rss_trigger to {rss_trigger_mb}MB") - + if enable_malloc_trim is not None: self._enable_malloc_trim = enable_malloc_trim - self._logger.info(f"Updated enable_malloc_trim to {self._enable_malloc_trim}") - + self._logger.info( + f"Updated enable_malloc_trim to {self._enable_malloc_trim}" + ) + if restart_every is not None: self._restart_every = max(1, restart_every) self._logger.info(f"Updated restart_every to {self._restart_every}") - + if restart_rss_trigger_mb is not None: self._restart_rss_trigger_bytes = restart_rss_trigger_mb * 1024 * 1024 - self._logger.info(f"Updated restart_rss_trigger to {restart_rss_trigger_mb}MB") - + self._logger.info( + f"Updated restart_rss_trigger to {restart_rss_trigger_mb}MB" + ) + def __enter__(self): """Context manager entry.""" return self - + def __exit__(self, exc_type, exc_val, exc_tb): """Context manager exit with forced cleanup.""" self.check_and_cleanup(force=True, is_idle=True) @@ -317,11 +328,11 @@ def create_memory_manager( enable_malloc_trim: bool = True, restart_every: int = 10000, restart_rss_trigger_mb: int = 4096, - logger: Optional[logging.Logger] = None + logger: Optional[logging.Logger] = None, ) -> MemoryManager: """ Create a MemoryManager instance with the specified settings. - + Args: gc_every: Trigger garbage collection every N iterations rss_trigger_mb: Trigger garbage collection when RSS exceeds this many MB @@ -329,7 +340,7 @@ def create_memory_manager( restart_every: Trigger actor restart every N iterations restart_rss_trigger_mb: Trigger actor restart when RSS exceeds this many MB logger: Optional logger for memory management events - + Returns: Configured MemoryManager instance """ @@ -339,21 +350,21 @@ def create_memory_manager( enable_malloc_trim=enable_malloc_trim, restart_every=restart_every, restart_rss_trigger_mb=restart_rss_trigger_mb, - logger=logger + logger=logger, ) # Context manager for automatic cleanup class MemoryManagedContext: """Context manager that automatically triggers memory cleanup on exit.""" - + def __init__(self, memory_manager: MemoryManager, force_cleanup: bool = True): self.memory_manager = memory_manager self.force_cleanup = force_cleanup - + def __enter__(self): return self.memory_manager - + def __exit__(self, exc_type, exc_val, exc_tb): if self.force_cleanup: self.memory_manager.check_and_cleanup(force=True) diff --git a/src/robin/minknow/__init__.py b/src/robin/minknow/__init__.py index bb044036..59f83adf 100644 --- a/src/robin/minknow/__init__.py +++ b/src/robin/minknow/__init__.py @@ -15,13 +15,13 @@ stop_protocol_run, ) from robin.minknow.sample_id import ( + build_sample_registration, generate_sample_id_md5, has_encrypted_identifier_fields, save_sample_identifier_manifest, + save_sample_registration, validate_custom_sample_id, validate_dob, - build_sample_registration, - save_sample_registration, ) from robin.minknow.stream_monitor import MinKnowStreamMonitor, acquire_stream_monitor from robin.minknow.toml_config import MinKnowWorkflowConfig, load_minknow_toml diff --git a/src/robin/minknow/auth.py b/src/robin/minknow/auth.py index e7f245c8..1b7662fe 100644 --- a/src/robin/minknow/auth.py +++ b/src/robin/minknow/auth.py @@ -7,7 +7,6 @@ from pathlib import Path from typing import Mapping, MutableMapping, Optional - LOCAL_HOSTS = frozenset({"localhost", "127.0.0.1", "::1"}) @@ -73,7 +72,9 @@ def manager_kwargs(self, environ: Optional[Mapping[str, str]] = None) -> dict: if self.developer_api_token: kwargs["developer_api_token"] = self.developer_api_token if self.client_cert_chain_path is not None: - kwargs["client_certificate_chain"] = self.client_cert_chain_path.read_bytes() + kwargs["client_certificate_chain"] = ( + self.client_cert_chain_path.read_bytes() + ) if self.client_key_path is not None: kwargs["client_private_key"] = self.client_key_path.read_bytes() if self.ca_cert_path is not None: diff --git a/src/robin/minknow/cli.py b/src/robin/minknow/cli.py index 972f35e3..1c6fcbf6 100644 --- a/src/robin/minknow/cli.py +++ b/src/robin/minknow/cli.py @@ -23,9 +23,9 @@ ) from robin.minknow.stream_monitor import acquire_stream_monitor from robin.minknow.toml_config import load_minknow_toml +from robin.minknow.watch import process_auto_watch, watch_active_runs from robin.minknow.workflow_refs import load_workflow_config_for_refs from robin.workflow_config import load_workflow_toml -from robin.minknow.watch import process_auto_watch, watch_active_runs @click.group() @@ -265,7 +265,10 @@ def models( use_local_token: Optional[bool], ) -> None: """List basecall simplex and modified models installed on a MinKNOW host.""" - from robin.minknow.model_resolve import recommended_cpg_modified_model, score_simplex_model + from robin.minknow.model_resolve import ( + recommended_cpg_modified_model, + score_simplex_model, + ) auth = build_auth_config( host, @@ -425,7 +428,9 @@ def start( resolved_host = (host or workflow_config_loaded.settings.host).strip() if not resolved_host: - raise click.ClickException("--host is required (or set [minknow].host in preset file)") + raise click.ClickException( + "--host is required (or set [minknow].host in preset file)" + ) resolved_position = (position or preset.position or "").strip() if not resolved_position: @@ -461,9 +466,7 @@ def start( if raw_work: resolved_work_directory = Path(str(raw_work)).expanduser() if resolved_work_directory is not None: - click.echo( - f"Readfish output dir: {resolved_work_directory / sample_id}" - ) + click.echo(f"Readfish output dir: {resolved_work_directory / sample_id}") click.echo("Preset:") for line in preset.summary_lines(): click.echo(f" {line}") @@ -485,7 +488,9 @@ def start( experiment_group=experiment_group, readfish=workflow_config_loaded.readfish, work_directory=( - str(resolved_work_directory) if resolved_work_directory is not None else None + str(resolved_work_directory) + if resolved_work_directory is not None + else None ), ) @@ -820,9 +825,13 @@ def stop( except MinKnowStopError as exc: raise click.ClickException(str(exc)) from exc - click.echo(f"Stop requested for {result.position} (run_id={result.protocol_run_id})") + click.echo( + f"Stop requested for {result.position} (run_id={result.protocol_run_id})" + ) if result.waited: - click.echo(f"Protocol finished with state: {result.protocol_state or 'unknown'}") + click.echo( + f"Protocol finished with state: {result.protocol_state or 'unknown'}" + ) def _watch_auto_add_paths(settings: MinKnowSettings) -> None: @@ -870,7 +879,6 @@ def _validate_preset_models_for_start( ) -> None: """Resolve preset models against the connected host; raise on failure.""" import grpc - from minknow_api.manager import Manager from minknow_api.tools import protocols @@ -889,9 +897,7 @@ def _validate_preset_models_for_start( connection = flow_position.connect() flow_cell = connection.device.get_flow_cell_info() if not getattr(flow_cell, "has_flow_cell", False): - raise click.ClickException( - f"No flow cell present in position {position}" - ) + raise click.ClickException(f"No flow cell present in position {position}") product_code = ( preset.product_code @@ -927,8 +933,7 @@ def _validate_preset_models_for_start( if resolved_preset.basecall_simplex_model != preset.basecall_simplex_model: click.echo( - "Resolved simplex model: " - f"{resolved_preset.basecall_simplex_model}" + "Resolved simplex model: " f"{resolved_preset.basecall_simplex_model}" ) if resolved_preset.modified_models != preset.modified_models: if resolved_preset.modified_models: diff --git a/src/robin/minknow/client.py b/src/robin/minknow/client.py index 9158ce20..6750ae18 100644 --- a/src/robin/minknow/client.py +++ b/src/robin/minknow/client.py @@ -117,7 +117,9 @@ def _populate_flow_cell_info(self, connection: Any, status: PositionStatus) -> N self._apply_status(status, merge_flow_cell_info(status, flow_cell)) - def _populate_output_directories(self, connection: Any, status: PositionStatus) -> None: + def _populate_output_directories( + self, connection: Any, status: PositionStatus + ) -> None: try: directories = connection.instance.get_output_directories() except grpc.RpcError as exc: @@ -131,7 +133,9 @@ def _populate_output_directories(self, connection: Any, status: PositionStatus) self._apply_status(status, merge_output_directories(status, directories)) - def _populate_current_protocol_run(self, connection: Any, status: PositionStatus) -> None: + def _populate_current_protocol_run( + self, connection: Any, status: PositionStatus + ) -> None: try: run = connection.protocol.get_current_protocol_run() except grpc.RpcError as exc: diff --git a/src/robin/minknow/model_resolve.py b/src/robin/minknow/model_resolve.py index aa8c3529..6a64029b 100644 --- a/src/robin/minknow/model_resolve.py +++ b/src/robin/minknow/model_resolve.py @@ -44,9 +44,7 @@ def score_simplex_model( elif "fast" in lower: score += 10 - version = requested_version or ( - _simplex_version(requested) if requested else None - ) + version = requested_version or (_simplex_version(requested) if requested else None) if version and name.endswith(f"@{version}"): score += 5 @@ -139,9 +137,7 @@ def pick_modified_model( if "5mcg" in base.lower() or "5hmc" in base.lower(): cpg_models = sorted( - m - for m in available - if "5mcg" in m.lower() and "5hmc" in m.lower() + m for m in available if "5mcg" in m.lower() and "5hmc" in m.lower() ) if cpg_models: return _prefer_simplex_prefixed(cpg_models, simplex_model) @@ -193,9 +189,7 @@ def pick_methylation_simplex( def methylation_capable_simplex_models(available: list[str]) -> list[str]: """Simplex models that include CpG 5mC/5hmC calling (integrated modbases).""" - return sorted( - name for name in available if score_simplex_model(name) >= 140 - ) + return sorted(name for name in available if score_simplex_model(name) >= 140) def _requests_cpg_methylation( @@ -299,9 +293,7 @@ def resolve_preset_simplex_model( available_modified: set[str] = set() if simplex is not None: - available_modified = { - model.name for model in simplex.modified_models - } + available_modified = {model.name for model in simplex.modified_models} resolved_modified: list[str] = [] unresolved: list[str] = [] @@ -356,9 +348,7 @@ def resolve_preset_simplex_model( f"to list installed models." ) elif tuple(resolved_modified) != updated.modified_models: - updated = updated.with_overrides( - modified_models=tuple(resolved_modified) - ) + updated = updated.with_overrides(modified_models=tuple(resolved_modified)) return updated, warnings, errors @@ -372,9 +362,7 @@ def query_basecall_models( ) -> tuple[list[SimplexModelInfo], Optional[str]]: """Query MinKNOW for basecall models; return (models, error_message).""" try: - configs = manager.find_basecall_configurations( - product_code, kit, sample_rate - ) + configs = manager.find_basecall_configurations(product_code, kit, sample_rate) except Exception as exc: return [], f"Could not query basecall configurations: {exc}" diff --git a/src/robin/minknow/monitor.py b/src/robin/minknow/monitor.py index 378a1cae..e646e50d 100644 --- a/src/robin/minknow/monitor.py +++ b/src/robin/minknow/monitor.py @@ -81,9 +81,7 @@ def format_poll_summary( return f"MinKNOW ({host}): no data" active = sum( - 1 - for position in result.status.positions - if position_has_active_run(position) + 1 for position in result.status.positions if position_has_active_run(position) ) parts = [ f"MinKNOW Core {result.status.core_version}", diff --git a/src/robin/minknow/parsing.py b/src/robin/minknow/parsing.py index 64e232f9..6dcb9e1e 100644 --- a/src/robin/minknow/parsing.py +++ b/src/robin/minknow/parsing.py @@ -230,9 +230,7 @@ def merge_protocol_run( protocol_name = _non_empty_string(getattr(run, "protocol_id", None)) if protocol_name: updates["protocol_name"] = protocol_name - protocol_run_state = _enum_name( - protocol_state_enum, getattr(run, "state", None) - ) + protocol_run_state = _enum_name(protocol_state_enum, getattr(run, "state", None)) if protocol_run_state: updates["protocol_run_state"] = protocol_run_state @@ -257,7 +255,9 @@ def merge_protocol_run( return replace(status, **updates) -def merge_output_directories(status: PositionStatus, directories: Any) -> PositionStatus: +def merge_output_directories( + status: PositionStatus, directories: Any +) -> PositionStatus: """Attach static output directory paths from ``get_output_directories``.""" return replace( status, diff --git a/src/robin/minknow/preset.py b/src/robin/minknow/preset.py index d4f44cd7..a904bf0d 100644 --- a/src/robin/minknow/preset.py +++ b/src/robin/minknow/preset.py @@ -162,7 +162,9 @@ def with_overrides(self, **kwargs: Any) -> RobinRunPreset: filtered = {key: value for key, value in kwargs.items() if value is not None} unknown = set(filtered) - allowed if unknown: - raise ValueError(f"Unknown preset override(s): {', '.join(sorted(unknown))}") + raise ValueError( + f"Unknown preset override(s): {', '.join(sorted(unknown))}" + ) return replace(self, **filtered) def validate(self, *, check_paths: bool = False) -> list[str]: @@ -177,16 +179,25 @@ def validate(self, *, check_paths: bool = False) -> list[str]: if not self.kit: errors.append("kit is required") if self.enable_basecalling and not self.basecall_simplex_model: - errors.append("basecall_simplex_model is required when basecalling is enabled") + errors.append( + "basecall_simplex_model is required when basecalling is enabled" + ) if self.enable_basecalling and not self.alignment_reference: errors.append("alignment_reference is required when basecalling is enabled") if self.bed_file and not self.alignment_reference: errors.append("bed_file requires alignment_reference") if self.read_until_filter and not self.effective_read_until_reference(): - errors.append("read_until_filter requires read_until_reference or alignment_reference") + errors.append( + "read_until_filter requires read_until_reference or alignment_reference" + ) if self.read_until_bed_file and not self.effective_read_until_reference(): - errors.append("read_until_bed_file requires read_until_reference or alignment_reference") - if self.read_until_filter and self.read_until_filter not in {"enrich", "deplete"}: + errors.append( + "read_until_bed_file requires read_until_reference or alignment_reference" + ) + if self.read_until_filter and self.read_until_filter not in { + "enrich", + "deplete", + }: errors.append("read_until_filter must be 'enrich' or 'deplete'") if self.adaptive_sampling_backend not in ADAPTIVE_SAMPLING_BACKENDS: errors.append( @@ -332,8 +343,7 @@ def summary_lines(self) -> list[str]: elif self.readfish_adaptive_sampling_enabled(): bed = self.effective_read_until_bed_file() lines.append( - f"Adaptive sampling (readfish): {self.read_until_filter} " - f"({bed})" + f"Adaptive sampling (readfish): {self.read_until_filter} " f"({bed})" ) if self.simulation_bulk_file: lines.append(f"Simulated playback: {self.simulation_bulk_file}") diff --git a/src/robin/minknow/run.py b/src/robin/minknow/run.py index 3c81de04..a3ea4683 100644 --- a/src/robin/minknow/run.py +++ b/src/robin/minknow/run.py @@ -100,10 +100,9 @@ def fetch_basecall_models_for_position( if not getattr(flow_cell, "has_flow_cell", False): raise MinKnowStartError(f"No flow cell present in position {position}") - product_code = ( - getattr(flow_cell, "user_specified_product_code", None) - or getattr(flow_cell, "product_code", None) - ) + product_code = getattr( + flow_cell, "user_specified_product_code", None + ) or getattr(flow_cell, "product_code", None) if not product_code: raise MinKnowStartError("Could not determine flow cell product code") @@ -243,7 +242,9 @@ def start_protocol_run( stereo_model=None, barcoding=None, alignment=alignment_args, - min_qscore=_default_min_qscore(manager, preset, sample_rate, product_code), + min_qscore=_default_min_qscore( + manager, preset, sample_rate, product_code + ), ) read_until_args = None @@ -327,9 +328,7 @@ def start_protocol_run( sample_id=request.sample_id, experiment_group=experiment_group, work_directory=( - Path(request.work_directory) - if request.work_directory - else None + Path(request.work_directory) if request.work_directory else None ), ) except ReadfishStartError as exc: @@ -411,9 +410,7 @@ def stop_protocol_run( protocol_state: Optional[str] = None try: - position = _find_position( - manager, request.position, error_cls=MinKnowStopError - ) + position = _find_position(manager, request.position, error_cls=MinKnowStopError) connection = position.connect() protocol_run_id = (request.protocol_run_id or "").strip() @@ -526,7 +523,9 @@ def _current_protocol_state(connection: Any) -> Optional[str]: return None raise MinKnowStartError(_format_grpc_error(exc)) from exc except Exception as exc: - raise MinKnowStartError(f"Could not read current protocol state: {exc}") from exc + raise MinKnowStartError( + f"Could not read current protocol state: {exc}" + ) from exc return _enum_name( getattr(getattr(connection.protocol, "_pb", None), "ProtocolState", None), @@ -635,9 +634,7 @@ def _find_position( if position.name == name: return position available = ", ".join(pos.name for pos in manager.flow_cell_positions()) - raise error_cls( - f"Position {name!r} not found. Available: {available or 'none'}" - ) + raise error_cls(f"Position {name!r} not found. Available: {available or 'none'}") def _default_min_qscore( @@ -652,9 +649,7 @@ def _default_min_qscore( configs = manager.find_basecall_configurations( product_code, preset.kit, sample_rate ) - simplex = protocols.find_simplex_model( - configs, preset.basecall_simplex_model - ) + simplex = protocols.find_simplex_model(configs, preset.basecall_simplex_model) return int(simplex.default_q_score_cutoff) except Exception: LOGGER.debug("Could not resolve default q-score cutoff", exc_info=True) diff --git a/src/robin/minknow/stream_monitor.py b/src/robin/minknow/stream_monitor.py index d3c361cb..1cfbb84f 100644 --- a/src/robin/minknow/stream_monitor.py +++ b/src/robin/minknow/stream_monitor.py @@ -216,7 +216,9 @@ def _update_position(self, name: str, updated: PositionStatus) -> None: self._positions[name] = updated self._emit_current() - def _run_protocol_stream(self, description: Any, stop_event: threading.Event) -> None: + def _run_protocol_stream( + self, description: Any, stop_event: threading.Event + ) -> None: name = description.name try: connection = self._connect_position(description, name) @@ -260,7 +262,9 @@ def _run_acquisition_stream( try: import minknow_api.acquisition_pb2 as acquisition_pb2 - for acquisition_run in connection.acquisition.watch_current_acquisition_run(): + for ( + acquisition_run + ) in connection.acquisition.watch_current_acquisition_run(): if stop_event.is_set() or self._stop_event.is_set(): break with self._positions_lock: diff --git a/src/robin/minknow/toml_config.py b/src/robin/minknow/toml_config.py index 63f390df..1fd7fcb1 100644 --- a/src/robin/minknow/toml_config.py +++ b/src/robin/minknow/toml_config.py @@ -48,7 +48,9 @@ def load_minknow_toml( raise click.BadParameter(f"Invalid TOML in {path}: {exc}") from exc if not isinstance(raw, dict): - raise click.BadParameter(f"TOML config must be a table at the top level: {path}") + raise click.BadParameter( + f"TOML config must be a table at the top level: {path}" + ) inline_workflow = extract_workflow_ref_keys(raw) merged_workflow: dict[str, Any] = dict(inline_workflow) @@ -106,13 +108,13 @@ def resolve_minknow_gui_config( ) -> Optional[MinKnowWorkflowConfig]: """Return MinKNOW GUI settings when explicitly configured via TOML or env. - The GUI sequencer page is shown only when this returns a config with - ``settings.enabled`` true. Sources (in order): + The GUI sequencer page is shown only when this returns a config with + ``settings.enabled`` true. Sources (in order): - - ``[minknow]`` in the workflow TOML passed to ``robin workflow --toml`` - - ``[minknow]`` in ``ROBIN_WORKFLOW_TOML`` - - ``MINKNOW_PRESET`` pointing at a preset / workflow TOML file - - ``MINKNOW_ENABLED=true`` or an explicit ``MINKNOW_HOST`` environment variable + - ``[minknow]`` in the workflow TOML passed to ``robin workflow --toml`` + - ``[minknow]`` in ``ROBIN_WORKFLOW_TOML`` + - ``MINKNOW_PRESET`` pointing at a preset / workflow TOML file + - ``MINKNOW_ENABLED=true`` or an explicit ``MINKNOW_HOST`` environment variable """ import os diff --git a/src/robin/minknow/workflow_refs.py b/src/robin/minknow/workflow_refs.py index cd72d462..fcb25889 100644 --- a/src/robin/minknow/workflow_refs.py +++ b/src/robin/minknow/workflow_refs.py @@ -107,15 +107,9 @@ def workflow_context_from_runner(runner: Any) -> tuple[Optional[str], Optional[s target_panel = getattr(runner, "target_panel", None) ref_text = ( - str(Path(reference).expanduser()) - if reference not in (None, "") - else None - ) - panel_text = ( - str(target_panel).strip() - if target_panel not in (None, "") - else None + str(Path(reference).expanduser()) if reference not in (None, "") else None ) + panel_text = str(target_panel).strip() if target_panel not in (None, "") else None return ref_text, panel_text diff --git a/src/robin/readfish/analysis_hook.py b/src/robin/readfish/analysis_hook.py index 51ccbc62..0c58923b 100644 --- a/src/robin/readfish/analysis_hook.py +++ b/src/robin/readfish/analysis_hook.py @@ -175,10 +175,16 @@ def _resolve_readfish_context( target_panel: Optional[str], reference: Optional[str], ) -> Optional[tuple[Any, Any]]: - from robin.minknow.config import preset_path_from_environ, workflow_toml_from_environ + from robin.minknow.config import ( + preset_path_from_environ, + workflow_toml_from_environ, + ) from robin.minknow.toml_config import load_minknow_toml from robin.readfish.config import ReadfishConfig - from robin.workflow_config import load_minknow_from_workflow_toml, load_workflow_toml + from robin.workflow_config import ( + load_minknow_from_workflow_toml, + load_workflow_toml, + ) for path in _workflow_toml_candidates(work_dir): if not path.is_file(): diff --git a/src/robin/readfish/config.py b/src/robin/readfish/config.py index 9cb38d8d..1104b244 100644 --- a/src/robin/readfish/config.py +++ b/src/robin/readfish/config.py @@ -35,18 +35,24 @@ class ReadfishConfig: start_wait_poll_seconds: float = 10.0 @classmethod - def from_mapping(cls, data: Optional[Mapping[str, Any]]) -> Optional[ReadfishConfig]: + def from_mapping( + cls, data: Optional[Mapping[str, Any]] + ) -> Optional[ReadfishConfig]: """Return a config when a ``[readfish]`` table is present.""" if not isinstance(data, Mapping): return None if not data: return cls() - dorado_address = _optional_str(data.get("dorado_address")) or DEFAULT_DORADO_ADDRESS + dorado_address = ( + _optional_str(data.get("dorado_address")) or DEFAULT_DORADO_ADDRESS + ) log_dir = _optional_str(data.get("log_dir")) minimap2_index = _optional_str(data.get("minimap2_index")) dorado_config = _optional_str(data.get("dorado_config")) - readfish_executable = _optional_str(data.get("readfish_executable")) or "readfish" + readfish_executable = ( + _optional_str(data.get("readfish_executable")) or "readfish" + ) live_region_name = _optional_str(data.get("live_region_name")) or "robin_panel" mappy_rs_threads = max(4, int(data.get("mappy_rs_threads", 4))) live_toml_max_bin_width_bp = _positive_int( @@ -80,7 +86,9 @@ def resolve_log_file(self, *, sample_id: str, output_dir: Path) -> Path: else: base = output_dir base.mkdir(parents=True, exist_ok=True) - safe_sample = "".join(ch if ch.isalnum() or ch in "-_" else "_" for ch in sample_id) + safe_sample = "".join( + ch if ch.isalnum() or ch in "-_" else "_" for ch in sample_id + ) return base / f"readfish_{safe_sample}.log" diff --git a/src/robin/readfish/live_updater.py b/src/robin/readfish/live_updater.py index 62e79d00..7069cdd5 100644 --- a/src/robin/readfish/live_updater.py +++ b/src/robin/readfish/live_updater.py @@ -7,11 +7,11 @@ import os import re import threading +import tomllib from dataclasses import asdict, dataclass from pathlib import Path from typing import Optional -import tomllib import tomli_w LOGGER = logging.getLogger(__name__) @@ -274,7 +274,9 @@ def unregister(cls, sample_id: str) -> None: try: path.unlink(missing_ok=True) except OSError: - LOGGER.debug("Could not remove live session file %s", path, exc_info=True) + LOGGER.debug( + "Could not remove live session file %s", path, exc_info=True + ) @classmethod def get(cls, sample_id: str) -> Optional[ReadfishLiveSession]: @@ -303,7 +305,13 @@ def list_sessions(cls) -> list[ReadfishLiveSession]: region_name=str(data.get("region_name") or "robin_panel"), last_master_bed_path=data.get("last_master_bed_path"), ) - except (OSError, KeyError, TypeError, ValueError, json.JSONDecodeError): + except ( + OSError, + KeyError, + TypeError, + ValueError, + json.JSONDecodeError, + ): continue by_id.setdefault(session.sample_id, session) return sorted(by_id.values(), key=lambda item: item.sample_id) @@ -382,9 +390,7 @@ def notify_master_bed( LOGGER.exception( "Failed to write readfish live TOML for sample %s", sample_id ) - _announce( - f"Live update FAILED for sample {sample_id!r}: {exc}" - ) + _announce(f"Live update FAILED for sample {sample_id!r}: {exc}") return None _announce( diff --git a/src/robin/readfish/runner.py b/src/robin/readfish/runner.py index e484c482..6e368f5f 100644 --- a/src/robin/readfish/runner.py +++ b/src/robin/readfish/runner.py @@ -78,7 +78,9 @@ def prepare_readfish_toml( When ``work_directory`` is set, files go under ``{work_directory}/{sample_id}/``. """ if not preset.readfish_adaptive_sampling_enabled(): - raise ReadfishStartError("Preset is not configured for readfish adaptive sampling") + raise ReadfishStartError( + "Preset is not configured for readfish adaptive sampling" + ) targets_bed = preset.effective_read_until_bed_file() reference = preset.effective_read_until_reference() @@ -206,7 +208,9 @@ def start_readfish_targets( ) -> ReadfishStartResult: """Generate readfish TOML and start ``readfish targets`` in the background.""" if not preset.readfish_adaptive_sampling_enabled(): - raise ReadfishStartError("Preset is not configured for readfish adaptive sampling") + raise ReadfishStartError( + "Preset is not configured for readfish adaptive sampling" + ) _announce("Adaptive sampling backend is readfish — preparing launch") @@ -268,8 +272,7 @@ def start_readfish_targets( _announce(f"Experiment: {experiment_group}") _announce(f"Log file: {log_file}") _announce( - "Live updates: " - + ("enabled" if config.live_updates_enabled else "disabled") + "Live updates: " + ("enabled" if config.live_updates_enabled else "disabled") ) if config.validate_on_start: @@ -310,7 +313,10 @@ def start_readfish_targets( _announce(f"Process pid={process.pid} is still running after startup check") if config.live_updates_enabled: - from robin.readfish.live_updater import ReadfishLiveRegistry, ReadfishLiveSession + from robin.readfish.live_updater import ( + ReadfishLiveRegistry, + ReadfishLiveSession, + ) ReadfishLiveRegistry.register( ReadfishLiveSession( @@ -324,9 +330,7 @@ def start_readfish_targets( f"(region={config.live_region_name!r})" ) - _announce( - f"Ready — follow logs with: tail -f {log_file}" - ) + _announce(f"Ready — follow logs with: tail -f {log_file}") return ReadfishStartResult( pid=process.pid, toml_path=str(toml_path), @@ -393,8 +397,7 @@ def _validate_readfish_toml( if completed.returncode != 0: detail = (completed.stderr or completed.stdout or "").strip() raise ReadfishStartError( - "readfish validate failed" - + (f": {detail}" if detail else "") + "readfish validate failed" + (f": {detail}" if detail else "") ) diff --git a/src/robin/readfish/toml_builder.py b/src/robin/readfish/toml_builder.py index aee72d34..5ded623a 100644 --- a/src/robin/readfish/toml_builder.py +++ b/src/robin/readfish/toml_builder.py @@ -24,7 +24,9 @@ def build_readfish_toml_document( if config.dorado_config: dorado_model = dorado_config_name(config.dorado_config, prefer_fast=False) else: - dorado_model = dorado_config_name(preset.basecall_simplex_model, prefer_fast=True) + dorado_model = dorado_config_name( + preset.basecall_simplex_model, prefer_fast=True + ) region = _region_for_filter( name=config.live_region_name, filter_mode=preset.read_until_filter, diff --git a/src/robin/reporting/__init__.py b/src/robin/reporting/__init__.py index 5c07c39a..82c22d4d 100644 --- a/src/robin/reporting/__init__.py +++ b/src/robin/reporting/__init__.py @@ -4,6 +4,6 @@ This package contains all the code needed to generate PDF reports from ROBIN analysis results. """ -from .report import create_pdf, RobinReport +from .report import RobinReport, create_pdf __all__ = ["create_pdf", "RobinReport"] diff --git a/src/robin/reporting/cli.py b/src/robin/reporting/cli.py index 90b721b4..7bd76ec1 100644 --- a/src/robin/reporting/cli.py +++ b/src/robin/reporting/cli.py @@ -4,12 +4,13 @@ Command-line interface for the ROBIN report generation tool. """ -import sys -import click import logging +import sys from pathlib import Path from typing import Optional +import click + from robin.reporting.report import create_pdf diff --git a/src/robin/reporting/fonts.py b/src/robin/reporting/fonts.py index ec543997..957cb572 100644 --- a/src/robin/reporting/fonts.py +++ b/src/robin/reporting/fonts.py @@ -5,8 +5,10 @@ """ import os -from reportlab.pdfbase.ttfonts import TTFont + from reportlab.pdfbase import pdfmetrics +from reportlab.pdfbase.ttfonts import TTFont + from robin.gui import fonts diff --git a/src/robin/reporting/header_footer.py b/src/robin/reporting/header_footer.py index b6aae393..3ee4e074 100644 --- a/src/robin/reporting/header_footer.py +++ b/src/robin/reporting/header_footer.py @@ -4,16 +4,17 @@ This module contains the class for adding headers and footers to the PDF report. """ -from reportlab.pdfgen import canvas -from reportlab.lib.pagesizes import A4 -from reportlab.lib.units import inch -from datetime import datetime import os -from robin.gui import images -from reportlab.lib import colors +from datetime import datetime + from PIL import Image as PILImage +from reportlab.lib import colors +from reportlab.lib.pagesizes import A4 +from reportlab.lib.units import inch +from reportlab.pdfgen import canvas from robin.__init__ import __version__ +from robin.gui import images VERSION = __version__ diff --git a/src/robin/reporting/mnpflex_hierarchy.py b/src/robin/reporting/mnpflex_hierarchy.py index 845b52f6..8514d9c5 100644 --- a/src/robin/reporting/mnpflex_hierarchy.py +++ b/src/robin/reporting/mnpflex_hierarchy.py @@ -54,7 +54,9 @@ def append_mnpflex_classifier_prediction( return if include_heading: - elements.append(Paragraph("Classifier prediction", styles.styles[heading_style])) + elements.append( + Paragraph("Classifier prediction", styles.styles[heading_style]) + ) elements.append(Spacer(1, 4)) label_width = max(page_width - 0.9 * inch, 3.5 * inch) diff --git a/src/robin/reporting/pdf_extractor.py b/src/robin/reporting/pdf_extractor.py index 7f0efa20..4c8d583e 100644 --- a/src/robin/reporting/pdf_extractor.py +++ b/src/robin/reporting/pdf_extractor.py @@ -4,14 +4,15 @@ This module handles extraction of data from PDF reports and stores it for analysis. """ -import os +import json import logging +import os import re -import json from datetime import datetime -import PyPDF2 from typing import Dict, Optional +import PyPDF2 + logger = logging.getLogger(__name__) diff --git a/src/robin/reporting/plotting.py b/src/robin/reporting/plotting.py index 132a208d..79a6aac5 100644 --- a/src/robin/reporting/plotting.py +++ b/src/robin/reporting/plotting.py @@ -4,23 +4,22 @@ This module contains functions for creating plots used in the PDF report. """ -import pandas as pd -import numpy as np -import seaborn as sns -import matplotlib.pyplot as plt -import matplotlib.patheffects as mpath_effects import io -import textwrap -import matplotlib.font_manager as fm +import logging import os -from robin.gui import fonts +import textwrap +from typing import Any, Dict, List, Optional, Sequence, Tuple +import matplotlib.font_manager as fm +import matplotlib.patheffects as mpath_effects +import matplotlib.pyplot as plt import natsort - +import numpy as np +import pandas as pd +import seaborn as sns from matplotlib import gridspec -import logging -from typing import List, Optional, Sequence, Tuple, Dict, Any +from robin.gui import fonts logger = logging.getLogger(__name__) @@ -217,7 +216,14 @@ def _apply_cnv_chromosome_axes( ax.spines["top"].set_visible(False) ax.xaxis.set_ticks_position("bottom") ax.yaxis.set_ticks_position("left") - ax.grid(True, axis="y", color=MODERN_COLORS["grid"], linestyle="--", linewidth=0.4, alpha=0.55) + ax.grid( + True, + axis="y", + color=MODERN_COLORS["grid"], + linestyle="--", + linewidth=0.4, + alpha=0.55, + ) ax.grid(False, axis="x") ax.tick_params(colors=CNV_TEXT["primary"], labelsize=CNV_FONT["tick"]) @@ -311,11 +317,32 @@ def _add_cnv_regions_on_plot(ax, regions: List[Dict[str, Any]], y_max: float) -> fill_color = CNV_COLORS["gain_fill"] if is_gain else CNV_COLORS["loss_fill"] edge_color = CNV_COLORS["gain_edge"] if is_gain else CNV_COLORS["loss_edge"] - ax.axvspan(start_mb, end_mb, color=fill_color, alpha=0.55, zorder=0, linewidth=0) - ax.axvline(start_mb, color=edge_color, linestyle="--", linewidth=0.9, alpha=0.75, zorder=1) - ax.axvline(end_mb, color=edge_color, linestyle="--", linewidth=0.9, alpha=0.75, zorder=1) + ax.axvspan( + start_mb, end_mb, color=fill_color, alpha=0.55, zorder=0, linewidth=0 + ) + ax.axvline( + start_mb, + color=edge_color, + linestyle="--", + linewidth=0.9, + alpha=0.75, + zorder=1, + ) + ax.axvline( + end_mb, + color=edge_color, + linestyle="--", + linewidth=0.9, + alpha=0.75, + zorder=1, + ) _draw_region_bracket( - ax, start_mb, end_mb, y_max * 0.992, edge_color, height_frac=y_max * 0.028, + ax, + start_mb, + end_mb, + y_max * 0.992, + edge_color, + height_frac=y_max * 0.028, ) @@ -393,7 +420,9 @@ def _normalise_coverage_to_cnv_axis( return float(scale_mean_cnv) * ratio -def _panel_label_matches_configured(label: str, configured_genes: Sequence[str]) -> bool: +def _panel_label_matches_configured( + label: str, configured_genes: Sequence[str] +) -> bool: """True when a panel target label matches a configured ``[cnv].genes`` symbol.""" key = str(label).strip().casefold() if not key: @@ -404,8 +433,10 @@ def _panel_label_matches_configured(label: str, configured_genes: Sequence[str]) continue if key == want: return True - if key.startswith(f"{want}_") or key.startswith(f"{want}-") or key.startswith( - f"{want} " + if ( + key.startswith(f"{want}_") + or key.startswith(f"{want}-") + or key.startswith(f"{want} ") ): return True return False @@ -535,7 +566,11 @@ def _attach_normalised_coverage( return [] if mean_cov is None or not np.isfinite(mean_cov) or mean_cov <= 0: coverage_vals = np.asarray( - [float(p["coverage_val"]) for p in points if p.get("coverage_val") is not None], + [ + float(p["coverage_val"]) + for p in points + if p.get("coverage_val") is not None + ], dtype=float, ) coverage_vals = coverage_vals[np.isfinite(coverage_vals) & (coverage_vals > 0)] @@ -572,7 +607,11 @@ def _expand_ylim_for_coverage_points( """Widen CNV axis limits so normalised coverage markers stay in view.""" if not coverage_points: return y_min, y_max - ys = [float(p["y_norm"]) for p in coverage_points if np.isfinite(p.get("y_norm", np.nan))] + ys = [ + float(p["y_norm"]) + for p in coverage_points + if np.isfinite(p.get("y_norm", np.nan)) + ] if not ys: return y_min, y_max pad = max((y_max - y_min) * 0.08, 0.15) @@ -937,7 +976,9 @@ def _draw_region_bracket( ) -def _apply_cnv_axes_style(ax, *, xlabel: str, ylabel: str, title: Optional[str] = None) -> None: +def _apply_cnv_axes_style( + ax, *, xlabel: str, ylabel: str, title: Optional[str] = None +) -> None: """Apply consistent seaborn-inspired styling to a CNV axes.""" _setup_cnv_fonts() ax.set_facecolor("white") @@ -964,7 +1005,14 @@ def _apply_cnv_axes_style(ax, *, xlabel: str, ylabel: str, title: Optional[str] fontproperties=_CNV_FONT_BOLD, ) sns.despine(ax=ax, top=True, right=True) - ax.grid(True, axis="y", color=MODERN_COLORS["grid"], linestyle="--", linewidth=0.4, alpha=0.55) + ax.grid( + True, + axis="y", + color=MODERN_COLORS["grid"], + linestyle="--", + linewidth=0.4, + alpha=0.55, + ) ax.grid(False, axis="x") ax.tick_params(colors=CNV_TEXT["primary"], labelsize=CNV_FONT["tick"]) @@ -1011,7 +1059,14 @@ def _apply_cnv_genome_overview_axes( ax.xaxis.set_ticks_position("none") ax.tick_params(axis="x", which="both", bottom=False, labelbottom=False) ax.yaxis.set_ticks_position("left") - ax.grid(True, axis="y", color=MODERN_COLORS["grid"], linestyle="--", linewidth=0.4, alpha=0.55) + ax.grid( + True, + axis="y", + color=MODERN_COLORS["grid"], + linestyle="--", + linewidth=0.4, + alpha=0.55, + ) ax.grid(False, axis="x") ax.tick_params(axis="y", colors=CNV_TEXT["primary"], labelsize=CNV_FONT["tick"]) @@ -1022,14 +1077,37 @@ def _chromosome_cnv_dataframe(positions_mb, values) -> pd.DataFrame: ) -def _add_cnv_reference_lines(ax, mean_cnv: float, std_cnv: float, y_min: float, y_max: float) -> None: +def _add_cnv_reference_lines( + ax, mean_cnv: float, std_cnv: float, y_min: float, y_max: float +) -> None: """Genome-wide reference guides (mean plus optional spread).""" - ax.axhline(y=mean_cnv, color=CNV_COLORS["reference"], linestyle="--", linewidth=0.9, alpha=0.7, zorder=1) + ax.axhline( + y=mean_cnv, + color=CNV_COLORS["reference"], + linestyle="--", + linewidth=0.9, + alpha=0.7, + zorder=1, + ) for offset in (std_cnv, 2 * std_cnv): if y_min < mean_cnv + offset <= y_max: - ax.axhline(y=mean_cnv + offset, color=CNV_COLORS["reference"], linestyle=":", linewidth=0.6, alpha=0.4, zorder=1) + ax.axhline( + y=mean_cnv + offset, + color=CNV_COLORS["reference"], + linestyle=":", + linewidth=0.6, + alpha=0.4, + zorder=1, + ) if y_min <= mean_cnv - offset < y_max: - ax.axhline(y=mean_cnv - offset, color=CNV_COLORS["reference"], linestyle=":", linewidth=0.6, alpha=0.4, zorder=1) + ax.axhline( + y=mean_cnv - offset, + color=CNV_COLORS["reference"], + linestyle=":", + linewidth=0.6, + alpha=0.4, + zorder=1, + ) def _log2_linear_axis_limits( @@ -1239,18 +1317,25 @@ def _create_empty_cnv_buffer(): """Create a minimal valid JPEG buffer for empty CNV plots.""" try: plt.figure(figsize=(16, 4)) - plt.text(0.5, 0.5, "No CNV data available", - ha='center', va='center', transform=plt.gca().transAxes, - fontsize=14, color='gray') + plt.text( + 0.5, + 0.5, + "No CNV data available", + ha="center", + va="center", + transform=plt.gca().transAxes, + fontsize=14, + color="gray", + ) plt.title("Copy Number Changes") - plt.axis('off') - + plt.axis("off") + buf = io.BytesIO() fig = plt.gcf() plt.savefig(buf, format="jpg", dpi=300, bbox_inches="tight") plt.close(fig) buf.seek(0) - + # Validate buffer contains data if buf.getvalue(): return buf @@ -1261,30 +1346,180 @@ def _create_empty_cnv_buffer(): try: # Create a minimal 1x1 white JPEG from PIL import Image as PILImage - img = PILImage.new('RGB', (1, 1), color='white') + + img = PILImage.new("RGB", (1, 1), color="white") buf = io.BytesIO() - img.save(buf, format='JPEG') + img.save(buf, format="JPEG") buf.seek(0) return buf except Exception: # Last resort: return a minimal valid JPEG binary directly # This is a valid 1x1 white JPEG - jpeg_bytes = bytes([ - 0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10, 0x4A, 0x46, 0x49, 0x46, 0x00, 0x01, - 0x01, 0x01, 0x00, 0x48, 0x00, 0x48, 0x00, 0x00, 0xFF, 0xDB, 0x00, 0x43, - 0x00, 0x08, 0x06, 0x06, 0x07, 0x06, 0x05, 0x08, 0x07, 0x07, 0x07, 0x09, - 0x09, 0x08, 0x0A, 0x0C, 0x14, 0x0D, 0x0C, 0x0B, 0x0B, 0x0C, 0x19, 0x12, - 0x13, 0x0F, 0x14, 0x1D, 0x1A, 0x1F, 0x1E, 0x1D, 0x1A, 0x1C, 0x1C, 0x20, - 0x24, 0x2E, 0x27, 0x20, 0x22, 0x2C, 0x23, 0x1C, 0x1C, 0x28, 0x37, 0x29, - 0x2C, 0x30, 0x31, 0x34, 0x34, 0x34, 0x1F, 0x27, 0x39, 0x3D, 0x38, 0x32, - 0x3C, 0x2E, 0x33, 0x34, 0x32, 0xFF, 0xC0, 0x00, 0x0B, 0x08, 0x00, 0x01, - 0x00, 0x01, 0x01, 0x01, 0x11, 0x00, 0xFF, 0xC4, 0x00, 0x14, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x08, 0xFF, 0xC4, 0x00, 0x14, 0x10, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0xFF, 0xDA, 0x00, 0x08, 0x01, 0x01, 0x00, 0x00, 0x3F, 0x00, - 0xD2, 0xCF, 0x20, 0xFF, 0xD9 - ]) + jpeg_bytes = bytes( + [ + 0xFF, + 0xD8, + 0xFF, + 0xE0, + 0x00, + 0x10, + 0x4A, + 0x46, + 0x49, + 0x46, + 0x00, + 0x01, + 0x01, + 0x01, + 0x00, + 0x48, + 0x00, + 0x48, + 0x00, + 0x00, + 0xFF, + 0xDB, + 0x00, + 0x43, + 0x00, + 0x08, + 0x06, + 0x06, + 0x07, + 0x06, + 0x05, + 0x08, + 0x07, + 0x07, + 0x07, + 0x09, + 0x09, + 0x08, + 0x0A, + 0x0C, + 0x14, + 0x0D, + 0x0C, + 0x0B, + 0x0B, + 0x0C, + 0x19, + 0x12, + 0x13, + 0x0F, + 0x14, + 0x1D, + 0x1A, + 0x1F, + 0x1E, + 0x1D, + 0x1A, + 0x1C, + 0x1C, + 0x20, + 0x24, + 0x2E, + 0x27, + 0x20, + 0x22, + 0x2C, + 0x23, + 0x1C, + 0x1C, + 0x28, + 0x37, + 0x29, + 0x2C, + 0x30, + 0x31, + 0x34, + 0x34, + 0x34, + 0x1F, + 0x27, + 0x39, + 0x3D, + 0x38, + 0x32, + 0x3C, + 0x2E, + 0x33, + 0x34, + 0x32, + 0xFF, + 0xC0, + 0x00, + 0x0B, + 0x08, + 0x00, + 0x01, + 0x00, + 0x01, + 0x01, + 0x01, + 0x11, + 0x00, + 0xFF, + 0xC4, + 0x00, + 0x14, + 0x00, + 0x01, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x08, + 0xFF, + 0xC4, + 0x00, + 0x14, + 0x10, + 0x01, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0xFF, + 0xDA, + 0x00, + 0x08, + 0x01, + 0x01, + 0x00, + 0x00, + 0x3F, + 0x00, + 0xD2, + 0xCF, + 0x20, + 0xFF, + 0xD9, + ] + ) buf = io.BytesIO(jpeg_bytes) buf.seek(0) return buf @@ -1418,9 +1653,7 @@ def create_CNV_plot( y_max = max(mean_value + (4 * std_value), mean_value * 1.35, 2.5) width = CNV_GENOME_LANDSCAPE_FIG_WIDTH - fig, ax = plt.subplots( - figsize=(width, CNV_GENOME_LANDSCAPE_FIG_HEIGHT) - ) + fig, ax = plt.subplots(figsize=(width, CNV_GENOME_LANDSCAPE_FIG_HEIGHT)) genome_panel_points = _collect_genome_significant_panel_points( panel_genes_df, cnv_source, @@ -1509,7 +1742,7 @@ def create_CNV_plot( return _create_empty_cnv_buffer() buf_data = buf.getvalue() - if len(buf_data) < 2 or buf_data[:2] != b'\xff\xd8': + if len(buf_data) < 2 or buf_data[:2] != b"\xff\xd8": logger.warning("Invalid JPEG data for CNV plot") return _create_empty_cnv_buffer() @@ -1517,7 +1750,7 @@ def create_CNV_plot( return buf except Exception as e: logger.error(f"Error creating CNV plot: {str(e)}") - plt.close('all') + plt.close("all") return _create_empty_cnv_buffer() @@ -1561,7 +1794,8 @@ def cnv_genome_landscape_image_size_pt( caption_reserve_pt: float = CNV_GENOME_LANDSCAPE_CAPTION_RESERVE_PT, ) -> tuple[float, float]: """Return (width, height) in points for the genome-wide CNV plot on A4 landscape.""" - from reportlab.lib.pagesizes import A4, landscape as rl_landscape + from reportlab.lib.pagesizes import A4 + from reportlab.lib.pagesizes import landscape as rl_landscape page_w, page_h = rl_landscape(A4) frame_w = page_w - left_margin_pt - right_margin_pt - frame_padding_pt @@ -1799,7 +2033,11 @@ def create_CNV_plot_per_chromosome( "falling back to absolute ploidy" ) - cnv_source = normalized_cnv if plot_log2 else (result.cnv if hasattr(result, "cnv") else None) + cnv_source = ( + normalized_cnv + if plot_log2 + else (result.cnv if hasattr(result, "cnv") else None) + ) if not cnv_source: logger.warning("No CNV data available for per-chromosome plotting") return plots @@ -1846,11 +2084,13 @@ def create_CNV_plot_per_chromosome( continue if plot_log2: y_min, y_max, _, mean_cnv, std_cnv = _compute_log2_y_limits( - finite_values, [], + finite_values, + [], ) else: y_min, y_max, _, mean_cnv, std_cnv = _compute_cnv_y_limits( - finite_values, [], + finite_values, + [], ) y_min = 0.0 regions = (significant_regions or {}).get(contig, []) @@ -1917,7 +2157,9 @@ def create_CNV_plot_per_chromosome( else: _add_cnv_ploidy_reference_lines(ax, y_max, x_max_mb) _scatter_cnv_chromosome_points( - ax, cnv_df, color_by_state=plot_log2, + ax, + cnv_df, + color_by_state=plot_log2, ) _apply_cnv_chromosome_axes( ax, @@ -1963,20 +2205,24 @@ def create_CNV_plot_per_chromosome( continue buf_data = buf.getvalue() - if len(buf_data) < 2 or buf_data[:2] != b'\xff\xd8': - logger.warning(f"Invalid JPEG data for chromosome {contig} CNV plot") + if len(buf_data) < 2 or buf_data[:2] != b"\xff\xd8": + logger.warning( + f"Invalid JPEG data for chromosome {contig} CNV plot" + ) continue buf.seek(0) plots.append((contig, buf)) except Exception as e: - logger.error(f"Error creating CNV plot for chromosome {contig}: {str(e)}") - plt.close('all') + logger.error( + f"Error creating CNV plot for chromosome {contig}: {str(e)}" + ) + plt.close("all") continue except Exception as e: logger.error(f"Error in create_CNV_plot_per_chromosome: {str(e)}") - plt.close('all') + plt.close("all") return plots @@ -2007,7 +2253,11 @@ def classification_plot(df, title, threshold): "probes", } df_melted = df_melted[ - ~df_melted["Condition"].astype(str).str.strip().str.lower().isin(meta_conditions) + ~df_melted["Condition"] + .astype(str) + .str.strip() + .str.lower() + .isin(meta_conditions) ] # Filter conditions that cross the threshold diff --git a/src/robin/reporting/report.py b/src/robin/reporting/report.py index 4577a1e3..13e22f2b 100644 --- a/src/robin/reporting/report.py +++ b/src/robin/reporting/report.py @@ -4,22 +4,25 @@ This module contains the main report class that coordinates the generation of the PDF report. """ -import os import json import logging -import pandas as pd +import os from datetime import datetime -from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer, PageBreak + +import pandas as pd from reportlab.lib.pagesizes import A4 from reportlab.lib.units import inch -from .styling.styles import ReportStyles -from robin.gui import fonts +from reportlab.platypus import PageBreak, Paragraph, SimpleDocTemplate, Spacer + from robin.build_info import get_git_commit +from robin.gui import fonts from robin.utils.clinvar_manager import ( format_clinvar_version_label, load_sample_clinvar_provenance, ) +from .styling.styles import ReportStyles + logger = logging.getLogger(__name__) @@ -71,8 +74,12 @@ def __init__( self.viewer_role = viewer_role or DEFAULT_VIEWER_ROLE self.sample_identifiers = sample_identifiers - self.generated_by = (str(generated_by).strip() if generated_by else None) or None - self.generated_at = (str(generated_at).strip() if generated_at else None) or None + self.generated_by = ( + str(generated_by).strip() if generated_by else None + ) or None + self.generated_at = ( + str(generated_at).strip() if generated_at else None + ) or None from robin.gui.plotting_preferences import ( load_plotting_preferences, resolve_cnv_summary_normalized, @@ -121,18 +128,20 @@ def __init__( # Initialize sections self.sections = [] self._initialize_sections() - + def _emit_progress(self, stage: str, message: str, progress: float = None): """Emit a progress update if callback is available.""" if self.progress_callback: try: from robin.gui.report_progress import normalize_report_progress - self.progress_callback({ - 'stage': stage, - 'message': message, - 'progress': normalize_report_progress(progress), - }) + self.progress_callback( + { + "stage": stage, + "message": message, + "progress": normalize_report_progress(progress), + } + ) except Exception as e: logger.error(f"Error emitting progress: {e}") @@ -149,8 +158,8 @@ def _get_centre_id(self): def _create_document(self): """Create the PDF document with portrait and landscape page templates.""" - from reportlab.platypus import Frame, PageTemplate from reportlab.lib.pagesizes import landscape as rl_landscape + from reportlab.platypus import Frame, PageTemplate left = right = 1.0 * inch top = 1.35 * inch @@ -196,19 +205,20 @@ def _initialize_sections(self): # Import sections here to avoid circular imports from .sections.classification import ClassificationSection from .sections.cnv import CNVSection + from .sections.coverage import CoverageSection + from .sections.disclaimer import DisclaimerSection from .sections.fusion import FusionSection from .sections.itd import ItdSection - from .sections.coverage import CoverageSection from .sections.mgmt import MGMTSection from .sections.mnpflex import MNPFlexSection from .sections.run_data import RunDataSection - from .sections.disclaimer import DisclaimerSection from .sections.variants import VariantsSection - + # Import section visibility helpers try: from robin.gui.config import any_classification_visible, is_section_visible except ImportError: + def is_section_visible(section_id, **kwargs): # type: ignore[misc] return True @@ -216,7 +226,7 @@ def any_classification_visible(**kwargs): # type: ignore[misc] return True sections = [] - + if any_classification_visible( self.workflow_steps, self.display_config, @@ -224,7 +234,7 @@ def any_classification_visible(**kwargs): # type: ignore[misc] viewer_role=self.viewer_role, ): sections.append(ClassificationSection(self)) - + if is_section_visible( "cnv", workflow_steps=self.workflow_steps, @@ -233,9 +243,9 @@ def any_classification_visible(**kwargs): # type: ignore[misc] viewer_role=self.viewer_role, ): sections.append(CNVSection(self)) - + sections.append(VariantsSection(self)) - + if is_section_visible( "fusion", workflow_steps=self.workflow_steps, @@ -253,7 +263,7 @@ def any_classification_visible(**kwargs): # type: ignore[misc] viewer_role=self.viewer_role, ): sections.append(ItdSection(self)) - + if is_section_visible( "target", workflow_steps=self.workflow_steps, @@ -262,7 +272,7 @@ def any_classification_visible(**kwargs): # type: ignore[misc] viewer_role=self.viewer_role, ): sections.append(CoverageSection(self)) - + if is_section_visible( "mgmt", workflow_steps=self.workflow_steps, @@ -280,10 +290,10 @@ def any_classification_visible(**kwargs): # type: ignore[misc] viewer_role=self.viewer_role, ): sections.append(MNPFlexSection(self)) - + # Run data section (always included) sections.append(RunDataSection(self)) - + # Disclaimer section (always included) sections.append(DisclaimerSection(self)) @@ -306,7 +316,9 @@ def generate_report( """ try: logger.info("Starting report generation") - self._emit_progress("initializing", "Initializing report generation...", 0.0) + self._emit_progress( + "initializing", "Initializing report generation...", 0.0 + ) if not self.generated_at: self.generated_at = datetime.now().strftime("%Y-%m-%d %H:%M:%S") @@ -338,11 +350,13 @@ def generate_report( summary_lines.append(f"Hospital Number: {si['nhs_number']}
") if si.get("notes"): summary_lines.append(f"Notes: {si['notes']}
") - summary_lines.extend([ - f"Centre ID: {self.centreID if self.centreID else 'Not specified'}
", - f"Report Type: {report_type.title()}
", - f"Generated: {self.generated_at}
", - ]) + summary_lines.extend( + [ + f"Centre ID: {self.centreID if self.centreID else 'Not specified'}
", + f"Report Type: {report_type.title()}
", + f"Generated: {self.generated_at}
", + ] + ) if self.generated_by: summary_lines.append(f"Generated by: {self.generated_by}") if self.robin_commit: @@ -359,16 +373,24 @@ def generate_report( # Process each section total_sections = len(self.sections) - self._emit_progress("processing_sections", f"Processing {total_sections} report sections...", 0.2) - + self._emit_progress( + "processing_sections", + f"Processing {total_sections} report sections...", + 0.2, + ) + for i, section in enumerate(self.sections): section_name = section.__class__.__name__.replace("Section", "") progress = 0.2 + (i / total_sections) * 0.5 # 20% to 70% - + try: - self._emit_progress("processing_sections", f"Loading {section_name} data...", progress) + self._emit_progress( + "processing_sections", + f"Loading {section_name} data...", + progress, + ) section.add_content() - + summary_elements, main_elements = section.get_elements() # Always include summary elements @@ -380,9 +402,13 @@ def generate_report( or section.__class__.__name__ == "DisclaimerSection" ): self.elements.extend(main_elements) - - self._emit_progress("processing_sections", f"Completed {section_name} section", progress + 0.02) - + + self._emit_progress( + "processing_sections", + f"Completed {section_name} section", + progress + 0.02, + ) + except Exception as e: logger.error( f"Error processing section {section.__class__.__name__}: {e}", @@ -398,7 +424,11 @@ def generate_report( self.elements_summary.append( Paragraph(error_content, self.styles.styles["Error"]) ) - self._emit_progress("processing_sections", f"Skipped {section_name} due to error", progress) + self._emit_progress( + "processing_sections", + f"Skipped {section_name} due to error", + progress, + ) # Add detailed analysis header and elements only for detailed reports if report_type == "detailed": @@ -416,7 +446,7 @@ def generate_report( # Combine all elements logger.info("Combining elements for final PDF") self._emit_progress("building_pdf", "Combining report sections...", 0.75) - + if report_type == "detailed": final_elements = ( self.elements_summary + self.elements + self.end_of_report_elements @@ -459,8 +489,10 @@ def generate_report( "generated_at": self.generated_at, "generated_by": self.generated_by, "robin_commit": self.robin_commit or None, - "clinvar_release": self.clinvar_metadata.get("file_date") or None, - "clinvar_sha256": self.clinvar_metadata.get("sha256") or None, + "clinvar_release": self.clinvar_metadata.get("file_date") + or None, + "clinvar_sha256": self.clinvar_metadata.get("sha256") + or None, "pdf_filename": os.path.basename(self.filename), }, fh, @@ -493,7 +525,8 @@ def generate_report( "generated_at": self.generated_at, "generated_by": self.generated_by, "robin_commit": self.robin_commit or None, - "clinvar_release": self.clinvar_metadata.get("file_date") or None, + "clinvar_release": self.clinvar_metadata.get("file_date") + or None, "clinvar_sha256": self.clinvar_metadata.get("sha256") or None, "files": [], } @@ -503,15 +536,19 @@ def generate_report( for section in self.sections: frames = getattr(section, "get_export_frames", lambda: {})() total_frames += len(frames) - + frame_count = 0 for section in self.sections: frames = getattr(section, "get_export_frames", lambda: {})() for name, df in frames.items(): frame_count += 1 - progress = 0.85 + (frame_count / max(total_frames, 1)) * 0.08 - self._emit_progress("building_pdf", f"Exporting {name} data...", progress) - + progress = ( + 0.85 + (frame_count / max(total_frames, 1)) * 0.08 + ) + self._emit_progress( + "building_pdf", f"Exporting {name} data...", progress + ) + safe_name = name.replace(" ", "_") csv_path = os.path.join( export_csv_dir, diff --git a/src/robin/reporting/sections/base.py b/src/robin/reporting/sections/base.py index 64be177c..413f4123 100644 --- a/src/robin/reporting/sections/base.py +++ b/src/robin/reporting/sections/base.py @@ -4,12 +4,13 @@ This module contains the base class for report sections. """ -from abc import ABC, abstractmethod import logging -from reportlab.platypus import Paragraph, Spacer, Table, TableStyle, Image +from abc import ABC, abstractmethod + from reportlab.lib import colors -from reportlab.lib.units import inch from reportlab.lib.styles import ParagraphStyle +from reportlab.lib.units import inch +from reportlab.platypus import Image, Paragraph, Spacer, Table, TableStyle logger = logging.getLogger(__name__) @@ -140,9 +141,7 @@ def create_table( Table object with applied styling """ # Convert all data to Paragraphs with proper styling - cell_style = ( - self.COMPACT_TABLE_CELL_STYLE if compact else self.TABLE_CELL_STYLE - ) + cell_style = self.COMPACT_TABLE_CELL_STYLE if compact else self.TABLE_CELL_STYLE header_style = self.TABLE_HEADER_STYLE if font_size is not None: # Use leading slightly larger than font size for readable line spacing diff --git a/src/robin/reporting/sections/classification.py b/src/robin/reporting/sections/classification.py index 509b0f59..141caedf 100644 --- a/src/robin/reporting/sections/classification.py +++ b/src/robin/reporting/sections/classification.py @@ -5,25 +5,28 @@ including results from Sturgeon, Random Forest, NanoDX, and PannanoDX classifiers. """ +import logging import os + +import matplotlib import pandas as pd -from reportlab.platypus import PageBreak, Paragraph, Spacer, Table, Image from reportlab.lib.colors import HexColor from reportlab.lib.styles import ParagraphStyle from reportlab.lib.units import inch +from reportlab.platypus import Image, PageBreak, Paragraph, Spacer, Table + from ..sections.base import ReportSection -import logging -import matplotlib matplotlib.use("Agg") # ensure non-interactive backend before importing pyplot -import matplotlib.pyplot as plt +import io + import matplotlib.dates as mdates +import matplotlib.pyplot as plt import matplotlib.ticker as mticker import seaborn as sns -import io -from robin.classification_config import get_confidence_status from robin.analysis.mnpflex_hierarchy import mnpflex_hierarchy_has_content +from robin.classification_config import get_confidence_status from robin.reporting.mnpflex_hierarchy import ( append_mnpflex_classifier_prediction, append_mnpflex_top_path_paragraph, @@ -66,17 +69,26 @@ def _create_empty_plot_buffer(self): """Create a minimal valid PNG buffer for empty plots.""" try: plt.figure(figsize=(8, 5), facecolor="white") - plt.text(0.5, 0.5, "No data available", - ha='center', va='center', transform=plt.gca().transAxes, - fontsize=14, color='gray') + plt.text( + 0.5, + 0.5, + "No data available", + ha="center", + va="center", + transform=plt.gca().transAxes, + fontsize=14, + color="gray", + ) plt.title("Classification Plot") - plt.axis('off') - + plt.axis("off") + buf = io.BytesIO() - plt.savefig(buf, format="png", dpi=300, bbox_inches="tight", facecolor="white") + plt.savefig( + buf, format="png", dpi=300, bbox_inches="tight", facecolor="white" + ) plt.close() buf.seek(0) - + # Validate buffer contains data if buf.getvalue(): return buf @@ -85,9 +97,10 @@ def _create_empty_plot_buffer(self): except Exception: # If even this fails, return a minimal PNG import base64 + # Minimal 1x1 transparent PNG png_data = base64.b64decode( - 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==' + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==" ) buf = io.BytesIO(png_data) buf.seek(0) @@ -110,7 +123,9 @@ def _create_time_plot(self, df, classifier_name, figsize=(4.5, 2.75)): # Check if dataframe is empty or has no data if df.empty or len(df) == 0: - logger.warning(f"Empty dataframe for {classifier_name} time series plot") + logger.warning( + f"Empty dataframe for {classifier_name} time series plot" + ) return self._create_empty_plot_buffer() # Convert confidence values to percentages if not already @@ -125,12 +140,16 @@ def _create_time_plot(self, df, classifier_name, figsize=(4.5, 2.75)): if len(above_threshold) <= 3: top_classes = above_threshold else: - final_vals = df.loc[df.index[-1], above_threshold].sort_values(ascending=False) + final_vals = df.loc[df.index[-1], above_threshold].sort_values( + ascending=False + ) top_classes = final_vals.head(3).index # Check if we have any classes to plot if len(top_classes) == 0: - logger.warning(f"No classes above threshold for {classifier_name} time series plot") + logger.warning( + f"No classes above threshold for {classifier_name} time series plot" + ) return self._create_empty_plot_buffer() # Get current highest confidence for subtitle @@ -166,7 +185,9 @@ def _create_time_plot(self, df, classifier_name, figsize=(4.5, 2.75)): if len(c) <= len(out) + 3: out = c # Use full name to avoid collision break - out = c[: min(len(out) + 4, len(c))] + ("..." if len(c) > 20 else "") + out = c[: min(len(out) + 4, len(c))] + ( + "..." if len(c) > 20 else "" + ) truncated_map[c] = out plot_df["Class"] = plot_df["Class"].map(truncated_map) @@ -193,7 +214,9 @@ def _create_time_plot(self, df, classifier_name, figsize=(4.5, 2.75)): ) # Add title and subtitle (compact for 2x2 grid) - fig.suptitle(f"{classifier_name}", y=0.98, fontsize=10, fontweight="bold") + fig.suptitle( + f"{classifier_name}", y=0.98, fontsize=10, fontweight="bold" + ) ax.set_title( f"{predicted_class} ({confidence_value:.1f}%)", pad=4, @@ -204,7 +227,9 @@ def _create_time_plot(self, df, classifier_name, figsize=(4.5, 2.75)): ax.xaxis.set_major_formatter(mdates.DateFormatter("%H:%M")) ax.set_xlabel("Time", fontsize=8) ax.set_ylabel("Confidence (%)", fontsize=8) - ax.yaxis.set_major_formatter(mticker.PercentFormatter(xmax=100, decimals=0)) + ax.yaxis.set_major_formatter( + mticker.PercentFormatter(xmax=100, decimals=0) + ) # Set axis ranges ax.set_ylim(0, 100) @@ -231,28 +256,32 @@ def _create_time_plot(self, df, classifier_name, figsize=(4.5, 2.75)): # Save plot to bytes buffer with high DPI for crisp rendering buf = io.BytesIO() - fig.savefig(buf, format="png", dpi=300, bbox_inches="tight", facecolor="white") + fig.savefig( + buf, format="png", dpi=300, bbox_inches="tight", facecolor="white" + ) plt.close(fig) buf.seek(0) - + # Validate buffer contains valid image data if not buf.getvalue(): logger.warning(f"Empty buffer for {classifier_name} time series plot") return self._create_empty_plot_buffer() - + # Try to validate it's a valid PNG by checking magic bytes buf_data = buf.getvalue() - if len(buf_data) < 8 or buf_data[:8] != b'\x89PNG\r\n\x1a\n': - logger.warning(f"Invalid PNG data for {classifier_name} time series plot") + if len(buf_data) < 8 or buf_data[:8] != b"\x89PNG\r\n\x1a\n": + logger.warning( + f"Invalid PNG data for {classifier_name} time series plot" + ) return self._create_empty_plot_buffer() - + # Reset buffer position after validation buf.seek(0) return buf - + except Exception as e: logger.error(f"Error creating {classifier_name} time series plot: {str(e)}") - plt.close('all') # Close any open figures + plt.close("all") # Close any open figures return self._create_empty_plot_buffer() def add_content(self): @@ -287,7 +316,10 @@ def add_content(self): if not has_hierarchical_summary: # No confident classification - show message + legend in same place self.summary_elements.append( - Paragraph("MNP-Flex Hierarchical Summary", self.styles.styles["Heading3"]) + Paragraph( + "MNP-Flex Hierarchical Summary", + self.styles.styles["Heading3"], + ) ) self.summary_elements.append(Spacer(1, 6)) self.summary_elements.append( @@ -313,7 +345,10 @@ def add_content(self): else: # Has hierarchical summary - nested tree with legend self.summary_elements.append( - Paragraph("MNP-Flex Hierarchical Summary", self.styles.styles["Heading3"]) + Paragraph( + "MNP-Flex Hierarchical Summary", + self.styles.styles["Heading3"], + ) ) self.summary_elements.append(Spacer(1, 6)) append_mnpflex_classifier_prediction( @@ -344,7 +379,9 @@ def add_content(self): ) self.summary_elements.append(Spacer(1, 8)) except Exception as e: - logger.error(f"Error adding MNP-Flex hierarchy to classification summary: {e}") + logger.error( + f"Error adding MNP-Flex hierarchy to classification summary: {e}" + ) # Methylation Classification header - divides MNP-Flex from other classifiers self.summary_elements.append( @@ -453,7 +490,9 @@ def add_content(self): for name, buf in plot_buffers[row_start : row_start + plots_per_row]: if buf is not None: buf.seek(0) - row_cells.append(Image(buf, width=plot_width, height=plot_height)) + row_cells.append( + Image(buf, width=plot_width, height=plot_height) + ) else: row_cells.append(Spacer(plot_width, plot_height)) plot_table = Table( @@ -471,8 +510,10 @@ def add_content(self): # Add explanation text using centralized thresholds from robin.classification_config import CLASSIFIER_CONFIDENCE_THRESHOLDS - - explanation_lines = ["Note: Classification confidence levels are defined as follows:"] + + explanation_lines = [ + "Note: Classification confidence levels are defined as follows:" + ] for classifier, thresholds in CLASSIFIER_CONFIDENCE_THRESHOLDS.items(): explanation_lines.append( f"- {classifier.title()}: High (>={thresholds['high']:.0f}%), " @@ -481,7 +522,7 @@ def add_content(self): explanation_lines.append( "Multiple classifiers may provide different results based on their training data and methodology." ) - + Explanation_text = Paragraph( "\n".join(explanation_lines), ParagraphStyle( @@ -533,9 +574,7 @@ def _strip_html(s): # Add detailed results for each classifier self.elements.append(PageBreak()) self.elements.append( - Paragraph( - "Detailed Classification Results", self.styles.styles["Heading2"] - ) + Paragraph("Detailed Classification Results", self.styles.styles["Heading2"]) ) self.elements.append(Spacer(1, 4)) @@ -562,9 +601,7 @@ def _strip_html(s): # Build detailed table for top 10 predictions detailed_data = [["Predicted Class", "Confidence Score"]] for class_name, score in top_predictions.items(): - confidence = ( - score / 100.0 if name == "Random Forest" else score - ) + confidence = score / 100.0 if name == "Random Forest" else score detailed_data.append([class_name, f"{confidence:.1%}"]) detailed_table = self.create_table( @@ -614,9 +651,7 @@ def _strip_html(s): "ConfidencePercent": f"{confidence:.1%}", } ) - key = ( - f"classification_{name.lower().replace(' ', '_')}_top10" - ) + key = f"classification_{name.lower().replace(' ', '_')}_top10" from pandas import DataFrame as _DF self.export_frames[key] = _DF(rows) diff --git a/src/robin/reporting/sections/cnv.py b/src/robin/reporting/sections/cnv.py index 7fba3c51..d7e85f77 100644 --- a/src/robin/reporting/sections/cnv.py +++ b/src/robin/reporting/sections/cnv.py @@ -4,51 +4,41 @@ This module handles the Copy Number Variation (CNV) analysis section of the report. """ +import logging import os -import re import pickle -import logging +import re from pathlib import Path +import natsort import numpy as np import pandas as pd -import natsort +from reportlab.lib.styles import ParagraphStyle from reportlab.lib.units import inch from reportlab.platypus import ( + Image, + NextPageTemplate, PageBreak, Paragraph, - Image, Spacer, Table, TableStyle, - NextPageTemplate, ) -from reportlab.lib.styles import ParagraphStyle -from ..sections.base import ReportSection -from ..plotting import ( - create_CNV_plot, - create_CNV_plot_per_chromosome, - cnv_chromosome_fig_height_for_page, - cnv_chromosome_fig_width_for_page, - cnv_genome_landscape_image_size_pt, - CNV_CHROMOSOME_PLOT_SPACER_PT, - CNV_CHROMOSOME_PLOTS_PER_PAGE, - CNV_REPORT_FRAME_PADDING_PT, -) - -# from robin.subpages.CNVObjectClass import ( -# CNVAnalysis -# ) +from robin import resources from robin.analysis.cnv_analysis import ( - Result, - moving_average, CNV_Difference, + Result, compute_cnv_log2_from_ploidy, + moving_average, prepare_cnv_calling_track, resolve_cnv_calling_bin_width, ) -from robin.analysis.cnv_classification import detect_cnv_events, get_cnv_summary, CNVEvent +from robin.analysis.cnv_classification import ( + CNVEvent, + detect_cnv_events, + get_cnv_summary, +) from robin.analysis.cnv_regional import ( SIGNIFICANT_CNV_STATES, analyze_cytoband_cnv, @@ -61,11 +51,26 @@ load_target_coverage_df, panel_genes_in_region, ) -from robin.reference_contigs import is_visible_contig from robin.classification_config import get_cnv_thresholds, is_resolution_sufficient +from robin.reference_contigs import is_visible_contig from robin.workflow_config import get_cnv_genes, load_workflow_toml -from robin import resources +from ..plotting import ( + CNV_CHROMOSOME_PLOT_SPACER_PT, + CNV_CHROMOSOME_PLOTS_PER_PAGE, + CNV_REPORT_FRAME_PADDING_PT, + cnv_chromosome_fig_height_for_page, + cnv_chromosome_fig_width_for_page, + cnv_genome_landscape_image_size_pt, + create_CNV_plot, + create_CNV_plot_per_chromosome, +) +from ..sections.base import ReportSection + +# from robin.subpages.CNVObjectClass import ( +# CNVAnalysis +# ) + logger = logging.getLogger(__name__) @@ -322,7 +327,9 @@ def add_content(self): # Add gain/loss thresholds to chromosome stats using centralized rules for chrom, stats in chromosome_stats.items(): if chrom != "global": - gain_threshold, loss_threshold = get_cnv_thresholds(chrom, XYestimate) + gain_threshold, loss_threshold = get_cnv_thresholds( + chrom, XYestimate + ) stats["gain_threshold"] = gain_threshold stats["loss_threshold"] = loss_threshold @@ -390,8 +397,12 @@ def add_content(self): cnv_dict, cytobands_bed, centromere_bed, - gene_bed if gene_bed is not None else pd.DataFrame( - columns=["chrom", "start_pos", "end_pos", "gene"] + ( + gene_bed + if gene_bed is not None + else pd.DataFrame( + columns=["chrom", "start_pos", "end_pos", "gene"] + ) ), XYestimate, ) @@ -413,8 +424,12 @@ def add_content(self): cnv_dict, cytobands_bed, centromere_bed, - gene_bed if gene_bed is not None else pd.DataFrame( - columns=["chrom", "start_pos", "end_pos", "gene"] + ( + gene_bed + if gene_bed is not None + else pd.DataFrame( + columns=["chrom", "start_pos", "end_pos", "gene"] + ) ), XYestimate, ) @@ -510,7 +525,7 @@ def add_content(self): # Detect CNV events using centralized classification rules logger.info("Detecting CNV events using centralized rules") events = [] - + # Check if resolution is sufficient analysis_binw = int(cnv_dict.get("bin_width", 1000000)) calling_binw = resolve_cnv_calling_bin_width(analysis_binw) @@ -536,28 +551,34 @@ def add_content(self): support_cnv_data=analysis_log2, support_bin_width=analysis_binw, ) - + # Convert events to summary format summary_whole_chr_events = [] summary_arm_events = [] - + for event in events: if event.event_type.startswith("WHOLE_CHR_"): event_type = event.event_type.replace("WHOLE_CHR_", "") summary_whole_chr_events.append( f"Chromosome {event.chromosome[3:]}: {event_type} (mean={event.mean_cnv:.2f})" ) - logger.info(f"Detected whole chromosome {event_type} for {event.chromosome}") + logger.info( + f"Detected whole chromosome {event_type} for {event.chromosome}" + ) else: arm_label = f"{event.arm}-arm" if event.arm else "arm" summary_arm_events.append( f"Chromosome {event.chromosome[3:]} {arm_label}: {event.event_type} (mean={event.mean_cnv:.2f}, {event.proportion_affected:.0%} of arm)" ) - logger.info(f"Detected arm event: {event.chromosome} {arm_label} {event.event_type}") - + logger.info( + f"Detected arm event: {event.chromosome} {arm_label} {event.event_type}" + ) + # Log the final counts logger.info(f"Found {len(summary_arm_events)} arm events") - logger.info(f"Found {len(summary_whole_chr_events)} whole chromosome events") + logger.info( + f"Found {len(summary_whole_chr_events)} whole chromosome events" + ) self.summary_elements.append( Paragraph( @@ -675,20 +696,24 @@ def add_content(self): for event in events: if event.event_type.startswith("WHOLE_CHR_"): event_type = event.event_type.replace("WHOLE_CHR_", "") - whole_chr_events.append([ - event.chromosome.replace("chr", ""), - event_type, - f"{event.mean_cnv:.2f}", - ]) + whole_chr_events.append( + [ + event.chromosome.replace("chr", ""), + event_type, + f"{event.mean_cnv:.2f}", + ] + ) else: arm_label = f"{event.arm}-arm" if event.arm else "arm" - arm_events.append([ - event.chromosome.replace("chr", ""), - arm_label, - event.event_type, - f"{event.mean_cnv:.2f}", - f"{event.proportion_affected:.1%}", - ]) + arm_events.append( + [ + event.chromosome.replace("chr", ""), + arm_label, + event.event_type, + f"{event.mean_cnv:.2f}", + f"{event.proportion_affected:.1%}", + ] + ) # Add whole chromosome events summary if any exist if whole_chr_events: @@ -741,28 +766,32 @@ def add_content(self): regional_table = None if regional_cnv_events: - regional_data = [[ - "Chr", - "Region", - "Start (Mb)", - "End (Mb)", - "Length (Mb)", - "Mean CNV", - "State", - "Panel genes", - ]] + regional_data = [ + [ + "Chr", + "Region", + "Start (Mb)", + "End (Mb)", + "Length (Mb)", + "Mean CNV", + "State", + "Panel genes", + ] + ] for event in regional_cnv_events: panel_gene_text = format_panel_genes_for_table(event["panel_genes"]) - regional_data.append([ - event["chrom"], - event["region"], - f"{event['start_mb']:.2f}", - f"{event['end_mb']:.2f}", - f"{event['length_mb']:.2f}", - f"{event['mean_cnv']:.2f}", - event["state"], - panel_gene_text, - ]) + regional_data.append( + [ + event["chrom"], + event["region"], + f"{event['start_mb']:.2f}", + f"{event['end_mb']:.2f}", + f"{event['length_mb']:.2f}", + f"{event['mean_cnv']:.2f}", + event["state"], + panel_gene_text, + ] + ) regional_table = self.create_table( regional_data, repeat_rows=1, @@ -958,29 +987,26 @@ def add_content(self): height=chromosome_plot_height, ) ) - last_on_page = ( - (plot_idx + 1) % self.CHROMOSOME_PLOTS_PER_PAGE == 0 - ) - if ( - plot_idx < len(plotted_chromosomes) - 1 - and not last_on_page - ): + last_on_page = (plot_idx + 1) % self.CHROMOSOME_PLOTS_PER_PAGE == 0 + if plot_idx < len(plotted_chromosomes) - 1 and not last_on_page: self.elements.append(Spacer(1, self.CHROMOSOME_PLOT_SPACER)) # Combined event list for CSV export only (tables above already # show whole-chromosome, regional, and arm events separately). all_cnv_events = [] for event in regional_cnv_events: - all_cnv_events.append([ - event["chrom"], - event["region"], - f"{event['start_mb']:.2f}", - f"{event['end_mb']:.2f}", - f"{event['length_mb']:.2f}", - f"{event['mean_cnv']:.2f}", - event["state"], - format_panel_genes_for_table(event["panel_genes"]), - ]) + all_cnv_events.append( + [ + event["chrom"], + event["region"], + f"{event['start_mb']:.2f}", + f"{event['end_mb']:.2f}", + f"{event['length_mb']:.2f}", + f"{event['mean_cnv']:.2f}", + event["state"], + format_panel_genes_for_table(event["panel_genes"]), + ] + ) for event in events: region_name = ( f"{event.chromosome} {event.arm}-arm" @@ -993,16 +1019,18 @@ def add_content(self): event.start_pos, event.end_pos, ) - all_cnv_events.append([ - event.chromosome.replace("chr", ""), - region_name.replace(f"{event.chromosome} ", ""), - f"{event.start_pos/1e6:.2f}", - f"{event.end_pos/1e6:.2f}", - f"{event.length/1e6:.2f}", - f"{event.mean_cnv:.2f}", - event.event_type.replace("WHOLE_CHR_", ""), - format_panel_genes_for_table(panel_genes), - ]) + all_cnv_events.append( + [ + event.chromosome.replace("chr", ""), + region_name.replace(f"{event.chromosome} ", ""), + f"{event.start_pos/1e6:.2f}", + f"{event.end_pos/1e6:.2f}", + f"{event.length/1e6:.2f}", + f"{event.mean_cnv:.2f}", + event.event_type.replace("WHOLE_CHR_", ""), + format_panel_genes_for_table(panel_genes), + ] + ) try: # Whole chromosome events diff --git a/src/robin/reporting/sections/coverage.py b/src/robin/reporting/sections/coverage.py index e63e68a5..9d251f80 100644 --- a/src/robin/reporting/sections/coverage.py +++ b/src/robin/reporting/sections/coverage.py @@ -4,21 +4,24 @@ This module contains the coverage analysis section of the report. """ -from reportlab.lib.units import inch -from reportlab.platypus import Paragraph, Spacer, Table, TableStyle, PageBreak, Image -from reportlab.lib.styles import ParagraphStyle -from .base import ReportSection import os -import pandas as pd -import numpy as np -import matplotlib.pyplot as plt + import matplotlib +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +from reportlab.lib.styles import ParagraphStyle +from reportlab.lib.units import inch +from reportlab.platypus import Image, PageBreak, Paragraph, Spacer, Table, TableStyle + +from .base import ReportSection matplotlib.use("Agg") import io -import natsort import logging +import natsort + logger = logging.getLogger(__name__) from robin.reference_contigs import is_visible_contig @@ -145,11 +148,18 @@ def _create_chromosome_coverage_plot(self): # Check if data is available if not self.chromosome_data: plt.figure(figsize=(8, 4)) - plt.text(0.5, 0.5, "No chromosome coverage data available", - ha='center', va='center', transform=plt.gca().transAxes, - fontsize=14, color='gray') + plt.text( + 0.5, + 0.5, + "No chromosome coverage data available", + ha="center", + va="center", + transform=plt.gca().transAxes, + fontsize=14, + color="gray", + ) plt.title("Per Chromosome Coverage") - plt.axis('off') + plt.axis("off") else: plt.figure(figsize=(8, 4)) # Reduced from default size chromosomes = [d["name"] for d in self.chromosome_data] @@ -169,19 +179,19 @@ def _create_chromosome_coverage_plot(self): # Save plot to bytes buffer buf = io.BytesIO() - plt.savefig(buf, format="png", dpi=300, bbox_inches='tight') + plt.savefig(buf, format="png", dpi=300, bbox_inches="tight") plt.close() buf.seek(0) - + # Verify the buffer has data if buf.getvalue(): return buf else: return self._create_empty_plot_buffer() - + except Exception as e: logger.warning(f"Error creating chromosome coverage plot: {e}") - plt.close('all') # Close any open figures + plt.close("all") # Close any open figures return self._create_empty_plot_buffer() def _create_target_coverage_plot(self): @@ -191,11 +201,18 @@ def _create_target_coverage_plot(self): if self.bedcov_df_main.empty or self.cov_df_main.empty: # Create empty plot with message plt.figure(figsize=(8, 4)) - plt.text(0.5, 0.5, "No coverage data available", - ha='center', va='center', transform=plt.gca().transAxes, - fontsize=14, color='gray') + plt.text( + 0.5, + 0.5, + "No coverage data available", + ha="center", + va="center", + transform=plt.gca().transAxes, + fontsize=14, + color="gray", + ) plt.title("Target vs Off-Target Coverage by Chromosome") - plt.axis('off') + plt.axis("off") else: # Calculate target statistics self.bedcov_df_main["length"] = ( @@ -246,43 +263,51 @@ def _create_target_coverage_plot(self): # Save plot to bytes buffer buf = io.BytesIO() - plt.savefig(buf, format="png", dpi=300, bbox_inches='tight') + plt.savefig(buf, format="png", dpi=300, bbox_inches="tight") plt.close() buf.seek(0) - + # Verify the buffer has data if buf.getvalue(): return buf else: # Return a minimal valid PNG if buffer is empty return self._create_empty_plot_buffer() - + except Exception as e: logger.warning(f"Error creating target coverage plot: {e}") - plt.close('all') # Close any open figures + plt.close("all") # Close any open figures return self._create_empty_plot_buffer() def _create_empty_plot_buffer(self): """Create a minimal valid PNG buffer for empty plots.""" try: plt.figure(figsize=(8, 4)) - plt.text(0.5, 0.5, "No data available", - ha='center', va='center', transform=plt.gca().transAxes, - fontsize=14, color='gray') + plt.text( + 0.5, + 0.5, + "No data available", + ha="center", + va="center", + transform=plt.gca().transAxes, + fontsize=14, + color="gray", + ) plt.title("Coverage Plot") - plt.axis('off') - + plt.axis("off") + buf = io.BytesIO() - plt.savefig(buf, format="png", dpi=300, bbox_inches='tight') + plt.savefig(buf, format="png", dpi=300, bbox_inches="tight") plt.close() buf.seek(0) return buf except Exception: # If even this fails, return a minimal PNG import base64 + # Minimal 1x1 transparent PNG png_data = base64.b64decode( - 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==' + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==" ) buf = io.BytesIO(png_data) buf.seek(0) @@ -294,9 +319,7 @@ def _get_target_coverage_outliers(self): return [] df = self.target_coverage_df.copy() df["coverage"] = df["coverage"].round(2) - sorted_chroms = sorted( - df["chrom"].unique(), key=natsort.natsort_keygen() - ) + sorted_chroms = sorted(df["chrom"].unique(), key=natsort.natsort_keygen()) rows = [] for chrom in sorted_chroms: chrom_data = df[df["chrom"] == chrom] @@ -314,12 +337,14 @@ def _get_target_coverage_outliers(self): for _, out in outliers.iterrows(): cov = out["coverage"] event_type = "Gain" if cov > upper_bound else "Loss" - rows.append({ - "chrom": chrom, - "gene": out["name"], - "coverage": cov, - "type": event_type, - }) + rows.append( + { + "chrom": chrom, + "gene": out["name"], + "coverage": cov, + "type": event_type, + } + ) return rows def _create_target_boxplot(self): @@ -328,31 +353,39 @@ def _create_target_boxplot(self): # Check if data is available if self.target_coverage_df.empty: plt.figure(figsize=(8, 4)) - plt.text(0.5, 0.5, "No target coverage data available", - ha='center', va='center', transform=plt.gca().transAxes, - fontsize=14, color='gray') + plt.text( + 0.5, + 0.5, + "No target coverage data available", + ha="center", + va="center", + transform=plt.gca().transAxes, + fontsize=14, + color="gray", + ) plt.title("Target Coverage Distribution") - plt.axis('off') + plt.axis("off") else: # Prepare data - self.target_coverage_df["coverage"] = self.target_coverage_df["coverage"].round( - 2 - ) + self.target_coverage_df["coverage"] = self.target_coverage_df[ + "coverage" + ].round(2) # Create figure and boxplot plt.figure(figsize=(8, 4)) # Reduced size # Get sorted unique chromosomes sorted_chroms = sorted( - self.target_coverage_df["chrom"].unique(), key=natsort.natsort_keygen() + self.target_coverage_df["chrom"].unique(), + key=natsort.natsort_keygen(), ) # Create boxplot bp = plt.boxplot( [ - self.target_coverage_df[self.target_coverage_df["chrom"] == chrom][ - "coverage" - ] + self.target_coverage_df[ + self.target_coverage_df["chrom"] == chrom + ]["coverage"] for chrom in sorted_chroms ], patch_artist=True, @@ -380,16 +413,16 @@ def _create_target_boxplot(self): plt.savefig(buf, format="png", dpi=300, bbox_inches="tight") plt.close() buf.seek(0) - + # Verify the buffer has data if buf.getvalue(): return buf else: return self._create_empty_plot_buffer() - + except Exception as e: logger.warning(f"Error creating target boxplot: {e}") - plt.close('all') # Close any open figures + plt.close("all") # Close any open figures return self._create_empty_plot_buffer() def add_content(self): @@ -545,20 +578,20 @@ def add_detailed_coverage(self): chrom_buf = self._create_chromosome_coverage_plot() target_buf = self._create_target_coverage_plot() box_buf = self._create_target_boxplot() - + # Validate buffers before creating Image objects if chrom_buf and chrom_buf.getvalue(): chrom_plot = Image(chrom_buf, width=6 * inch, height=3 * inch) self.elements.append(chrom_plot) else: logger.warning("Skipping chromosome coverage plot - invalid buffer") - + if target_buf and target_buf.getvalue(): target_plot = Image(target_buf, width=6 * inch, height=3 * inch) self.elements.append(target_plot) else: logger.warning("Skipping target coverage plot - invalid buffer") - + if box_buf and box_buf.getvalue(): box_plot = Image(box_buf, width=6 * inch, height=3 * inch) self.elements.append(box_plot) @@ -584,12 +617,14 @@ def add_detailed_coverage(self): ) table_data = [["Chromosome", "Gene", "Coverage", "Type"]] for o in outliers: - table_data.append([ - o["chrom"], - o["gene"], - f"{o['coverage']:.1f}x", - o["type"], - ]) + table_data.append( + [ + o["chrom"], + o["gene"], + f"{o['coverage']:.1f}x", + o["type"], + ] + ) outliers_table = self.create_table( table_data, repeat_rows=1, @@ -598,13 +633,15 @@ def add_detailed_coverage(self): font_size=9, ) outliers_table.setStyle( - TableStyle([ - *self.MODERN_TABLE_STYLE._cmds, - ("ALIGN", (2, 1), (2, -1), "RIGHT"), - ("ALIGN", (3, 1), (3, -1), "CENTER"), - ("TOPPADDING", (0, 0), (-1, -1), 4), - ("BOTTOMPADDING", (0, 0), (-1, -1), 4), - ]) + TableStyle( + [ + *self.MODERN_TABLE_STYLE._cmds, + ("ALIGN", (2, 1), (2, -1), "RIGHT"), + ("ALIGN", (3, 1), (3, -1), "CENTER"), + ("TOPPADDING", (0, 0), (-1, -1), 4), + ("BOTTOMPADDING", (0, 0), (-1, -1), 4), + ] + ) ) self.elements.append(outliers_table) self.elements.append(Spacer(1, 6)) diff --git a/src/robin/reporting/sections/disclaimer.py b/src/robin/reporting/sections/disclaimer.py index 0c908df7..2da41f3f 100644 --- a/src/robin/reporting/sections/disclaimer.py +++ b/src/robin/reporting/sections/disclaimer.py @@ -5,6 +5,7 @@ """ from reportlab.platypus import PageBreak, Paragraph, Spacer + from ..sections.base import ReportSection from .disclaimer_text import EXTENDED_DISCLAIMER_TEXT diff --git a/src/robin/reporting/sections/fusion.py b/src/robin/reporting/sections/fusion.py index e70de163..95628945 100644 --- a/src/robin/reporting/sections/fusion.py +++ b/src/robin/reporting/sections/fusion.py @@ -4,16 +4,18 @@ This module contains the fusion analysis section of the report. """ -import os import logging +import os import pickle -from reportlab.platypus import Paragraph, Spacer, Table, TableStyle + +import networkx as nx +import pandas as pd from reportlab.lib import colors +from reportlab.lib.styles import ParagraphStyle from reportlab.lib.units import inch +from reportlab.platypus import Paragraph, Spacer, Table, TableStyle + from .base import ReportSection -import pandas as pd -import networkx as nx -from reportlab.lib.styles import ParagraphStyle logger = logging.getLogger(__name__) @@ -23,30 +25,28 @@ class FusionSection(ReportSection): def _get_validated_fusion_pairs(self, data): """Get validated fusion pairs using the same logic as the GUI. - + Uses breakpoint validation with minimum 4 reads support. """ if data is None or data.empty: return [] - + try: # Import validation functions from GUI module from robin.gui.components.fusion import _cluster_fusion_reads - + # Use breakpoint validation (same as GUI) clustered_data = _cluster_fusion_reads( - data, - max_distance=10000, - use_breakpoint_validation=True + data, max_distance=10000, use_breakpoint_validation=True ) - + if clustered_data.empty: return [] - + # Extract unique validated gene pairs validated_pairs = [] seen_pairs = set() - + for _, row in clustered_data.iterrows(): fusion_pair_str = row["fusion_pair"] # e.g., "GENE1-GENE2" if fusion_pair_str and fusion_pair_str not in seen_pairs: @@ -58,11 +58,13 @@ def _get_validated_fusion_pairs(self, data): if pair_key not in seen_pairs: validated_pairs.append(pair_key) seen_pairs.add(pair_key) - + return validated_pairs - + except Exception as e: - logger.warning(f"Failed to get validated fusion pairs, falling back to simple method: {e}") + logger.warning( + f"Failed to get validated fusion pairs, falling back to simple method: {e}" + ) return self._get_gene_pairs_simple(data) def _get_gene_pairs_simple(self, data): @@ -71,25 +73,25 @@ def _get_gene_pairs_simple(self, data): return [] # For processed data, use read_id grouping - if 'read_id' in data.columns: + if "read_id" in data.columns: read_groups = data.groupby("read_id") - elif 'readID' in data.columns: + elif "readID" in data.columns: read_groups = data.groupby("readID") else: return [] gene_pairs = [] gene_pair_reads = {} - + for _, group in read_groups: # Get genes from col4 (Gene column) - if 'col4' in group.columns: + if "col4" in group.columns: genes = group["col4"].unique() - elif 'Gene' in group.columns: + elif "Gene" in group.columns: genes = group["Gene"].unique() else: continue - + if len(genes) >= 2: genes = sorted([str(g).strip() for g in genes if g]) for i in range(len(genes) - 1): @@ -98,9 +100,9 @@ def _get_gene_pairs_simple(self, data): if pair not in gene_pair_reads: gene_pair_reads[pair] = set() # Get read ID - if 'read_id' in group.columns: + if "read_id" in group.columns: read_id = group["read_id"].iloc[0] - elif 'readID' in group.columns: + elif "readID" in group.columns: read_id = group["readID"].iloc[0] else: continue @@ -115,7 +117,7 @@ def _get_gene_pairs_simple(self, data): def _get_gene_pairs(self, data): """Get unique gene fusion pairs from the processed data. - + Uses validated fusion pairs with breakpoint validation (same as GUI). """ return self._get_validated_fusion_pairs(data) @@ -136,7 +138,9 @@ def _load_fusion_data(self): "master_path": os.path.join( self.report.output, "fusion_candidates_master_processed.pkl" ), - "all_path": os.path.join(self.report.output, "fusion_candidates_all_processed.pkl"), + "all_path": os.path.join( + self.report.output, "fusion_candidates_all_processed.pkl" + ), } try: @@ -146,15 +150,17 @@ def _load_fusion_data(self): try: processed_data = pickle.load(f) # Filter to only good pairs to reduce memory usage and processing time - annotated_data = processed_data.get("annotated_data", pd.DataFrame()) + annotated_data = processed_data.get( + "annotated_data", pd.DataFrame() + ) goodpairs = processed_data.get("goodpairs", pd.Series()) - + if not annotated_data.empty and not goodpairs.empty: # Only keep the good pairs (same as GUI does) fusion_data["master_candidates"] = annotated_data[goodpairs] else: fusion_data["master_candidates"] = annotated_data - + logger.debug( f"Loaded master fusion candidates from {fusion_data['master_path']} " f"({len(fusion_data['master_candidates'])} good pairs from {len(annotated_data)} total candidates)" @@ -169,15 +175,17 @@ def _load_fusion_data(self): try: processed_data = pickle.load(f) # Filter to only good pairs to reduce memory usage and processing time - annotated_data = processed_data.get("annotated_data", pd.DataFrame()) + annotated_data = processed_data.get( + "annotated_data", pd.DataFrame() + ) goodpairs = processed_data.get("goodpairs", pd.Series()) - + if not annotated_data.empty and not goodpairs.empty: # Only keep the good pairs (same as GUI does) fusion_data["all_candidates"] = annotated_data[goodpairs] else: fusion_data["all_candidates"] = annotated_data - + logger.debug( f"Loaded all fusion candidates from {fusion_data['all_path']} " f"({len(fusion_data['all_candidates'])} good pairs from {len(annotated_data)} total candidates)" @@ -235,17 +243,15 @@ def _format_fusion_table(self, data, title): try: # Use validated fusion pairs (same as GUI) from robin.gui.components.fusion import _cluster_fusion_reads - + # Get validated fusion pairs using breakpoint validation clustered_data = _cluster_fusion_reads( - data, - max_distance=10000, - use_breakpoint_validation=True + data, max_distance=10000, use_breakpoint_validation=True ) - + if clustered_data.empty: return None - + # Build fusion details from validated breakpoints fusion_details = {} for _, row in clustered_data.iterrows(): @@ -254,7 +260,7 @@ def _format_fusion_table(self, data, title): genes = [g.strip() for g in fusion_pair_str.split("-") if g.strip()] if len(genes) >= 2: gene_pair = "-".join(sorted(genes)) - + # Get breakpoint information from validated data fusion_details[gene_pair] = { "chrom1": row.get("chr1", "Unknown"), @@ -263,7 +269,7 @@ def _format_fusion_table(self, data, title): "pos2": row.get("gene2_position", "N/A"), "supporting_reads": int(row.get("reads", 0)), } - + # Add rows to table using Paragraph objects for wrappable text for gene_pair, details in sorted(fusion_details.items()): table_data.append( @@ -286,7 +292,9 @@ def _format_fusion_table(self, data, title): # Create and style the table table = Table( - table_data, colWidths=[inch * width for width in col_widths], repeatRows=1 + table_data, + colWidths=[inch * width for width in col_widths], + repeatRows=1, ) table.setStyle( TableStyle( @@ -295,18 +303,32 @@ def _format_fusion_table(self, data, title): *self.MODERN_TABLE_STYLE._cmds, # Preserve specific alignments ("ALIGN", (0, 0), (0, -1), "LEFT"), # Left-align Fusion Pair - ("ALIGN", (1, 0), (2, -1), "CENTER"), # Center-align Chromosomes + ( + "ALIGN", + (1, 0), + (2, -1), + "CENTER", + ), # Center-align Chromosomes ("ALIGN", (3, 0), (4, -1), "LEFT"), # Left-align positions - ("ALIGN", (5, 0), (5, -1), "RIGHT"), # Right-align Supporting Reads + ( + "ALIGN", + (5, 0), + (5, -1), + "RIGHT", + ), # Right-align Supporting Reads ] ) ) return table - + except Exception as e: - logger.warning(f"Failed to create fusion table with validation, using fallback: {e}") - return self._format_fusion_table_fallback(data, title, header_style, cell_style) + logger.warning( + f"Failed to create fusion table with validation, using fallback: {e}" + ) + return self._format_fusion_table_fallback( + data, title, header_style, cell_style + ) def _format_fusion_table_fallback(self, data, title, header_style, cell_style): """Fallback method for formatting fusion table from raw data.""" @@ -535,50 +557,62 @@ def _fusion_rows(df): """Get validated fusion rows for export (same as GUI logic).""" if df is None or df.empty: return [] - + try: # Use validated fusion pairs (same as GUI) from robin.gui.components.fusion import _cluster_fusion_reads - + # Get validated fusion pairs using breakpoint validation clustered_data = _cluster_fusion_reads( - df, - max_distance=10000, - use_breakpoint_validation=True + df, max_distance=10000, use_breakpoint_validation=True ) - + if clustered_data.empty: return [] - + # Build rows from validated breakpoints rows = [] seen_pairs = set() - + for _, row in clustered_data.iterrows(): fusion_pair_str = row["fusion_pair"] # e.g., "GENE1-GENE2" if fusion_pair_str and fusion_pair_str not in seen_pairs: seen_pairs.add(fusion_pair_str) - genes = [g.strip() for g in fusion_pair_str.split("-") if g.strip()] + genes = [ + g.strip() + for g in fusion_pair_str.split("-") + if g.strip() + ] if len(genes) >= 2: gene_pair = "-".join(sorted(genes)) - rows.append({ - "FusionPair": gene_pair, - "Chrom1": row.get("chr1", "Unknown"), - "Chrom2": row.get("chr2", "Unknown"), - "Gene1Pos": row.get("gene1_position", "N/A"), - "Gene2Pos": row.get("gene2_position", "N/A"), - "SupportingReads": int(row.get("reads", 0)), - }) + rows.append( + { + "FusionPair": gene_pair, + "Chrom1": row.get("chr1", "Unknown"), + "Chrom2": row.get("chr2", "Unknown"), + "Gene1Pos": row.get( + "gene1_position", "N/A" + ), + "Gene2Pos": row.get( + "gene2_position", "N/A" + ), + "SupportingReads": int(row.get("reads", 0)), + } + ) return rows except Exception as e: - logger.warning(f"Failed to get validated fusion rows for export: {e}") + logger.warning( + f"Failed to get validated fusion rows for export: {e}" + ) # Fallback to simple method with >= 4 reads - read_groups = df.groupby("read_id" if "read_id" in df.columns else "readID") + read_groups = df.groupby( + "read_id" if "read_id" in df.columns else "readID" + ) gene_pair_reads = {} for read_id, group in read_groups: - if 'col4' in group.columns: + if "col4" in group.columns: genes = sorted(group["col4"].unique()) - elif 'Gene' in group.columns: + elif "Gene" in group.columns: genes = sorted(group["Gene"].unique()) else: continue @@ -586,43 +620,47 @@ def _fusion_rows(df): for i in range(len(genes) - 1): for j in range(i + 1, len(genes)): gene_pair = f"{genes[i]}-{genes[j]}" - gene_pair_reads.setdefault(gene_pair, set()).add(read_id) + gene_pair_reads.setdefault( + gene_pair, set() + ).add(read_id) rows = [] for gene_pair, reads in gene_pair_reads.items(): if len(reads) >= 4: # Minimum 4 reads (matching GUI) # Get gene information - if 'col4' in df.columns: - g1_data = df[df['col4'] == gene_pair.split('-')[0]] - g2_data = df[df['col4'] == gene_pair.split('-')[1]] - elif 'Gene' in df.columns: - g1_data = df[df["Gene"] == gene_pair.split('-')[0]] - g2_data = df[df["Gene"] == gene_pair.split('-')[1]] + if "col4" in df.columns: + g1_data = df[df["col4"] == gene_pair.split("-")[0]] + g2_data = df[df["col4"] == gene_pair.split("-")[1]] + elif "Gene" in df.columns: + g1_data = df[df["Gene"] == gene_pair.split("-")[0]] + g2_data = df[df["Gene"] == gene_pair.split("-")[1]] else: continue - + if not g1_data.empty and not g2_data.empty: g1row = g1_data.iloc[0] g2row = g2_data.iloc[0] - - if 'reference_id' in g1row.index: - chrom1 = g1row.get('reference_id', 'Unknown') - chrom2 = g2row.get('reference_id', 'Unknown') + + if "reference_id" in g1row.index: + chrom1 = g1row.get("reference_id", "Unknown") + chrom2 = g2row.get("reference_id", "Unknown") pos1 = f"{g1row.get('reference_start', 'N/A')}-{g1row.get('reference_end', 'N/A')}" pos2 = f"{g2row.get('reference_start', 'N/A')}-{g2row.get('reference_end', 'N/A')}" else: - chrom1 = g1row.get('chromBED', 'Unknown') - chrom2 = g2row.get('chromBED', 'Unknown') + chrom1 = g1row.get("chromBED", "Unknown") + chrom2 = g2row.get("chromBED", "Unknown") pos1 = f"{g1row.get('BS', 'N/A')}-{g1row.get('BE', 'N/A')}" pos2 = f"{g2row.get('BS', 'N/A')}-{g2row.get('BE', 'N/A')}" - - rows.append({ - "FusionPair": gene_pair, - "Chrom1": chrom1, - "Chrom2": chrom2, - "Gene1Pos": pos1, - "Gene2Pos": pos2, - "SupportingReads": len(reads), - }) + + rows.append( + { + "FusionPair": gene_pair, + "Chrom1": chrom1, + "Chrom2": chrom2, + "Gene1Pos": pos1, + "Gene2Pos": pos2, + "SupportingReads": len(reads), + } + ) return rows master_rows = _fusion_rows(fusion_data["master_candidates"]) diff --git a/src/robin/reporting/sections/itd.py b/src/robin/reporting/sections/itd.py index b353acc8..e5cef348 100644 --- a/src/robin/reporting/sections/itd.py +++ b/src/robin/reporting/sections/itd.py @@ -16,6 +16,7 @@ ITD_EVENT_DISPLAY_COLUMNS, normalize_itd_events_df, ) + from .base import ReportSection logger = logging.getLogger(__name__) @@ -108,9 +109,11 @@ def add_content(self): n_called_genes = ( int((summary["n_events"] > 0).sum()) if not summary.empty and "n_events" in summary.columns - else int(events["gene"].nunique()) - if n_events and "gene" in events.columns - else 0 + else ( + int(events["gene"].nunique()) + if n_events and "gene" in events.columns + else 0 + ) ) if n_events: @@ -170,4 +173,6 @@ def add_content(self): if "n_events" in summary.columns else summary ) - self.export_frames["itd_summary"] = called.copy() if not called.empty else summary.head(0) + self.export_frames["itd_summary"] = ( + called.copy() if not called.empty else summary.head(0) + ) diff --git a/src/robin/reporting/sections/mgmt.py b/src/robin/reporting/sections/mgmt.py index 8538c469..d54384e0 100644 --- a/src/robin/reporting/sections/mgmt.py +++ b/src/robin/reporting/sections/mgmt.py @@ -5,16 +5,17 @@ """ import io +import logging import os -import pandas as pd + import natsort -from reportlab.lib.units import inch -from reportlab.platypus import PageBreak, Paragraph, Image, Spacer, Table, TableStyle +import pandas as pd from reportlab.lib.colors import HexColor from reportlab.lib.styles import ParagraphStyle -from ..sections.base import ReportSection +from reportlab.lib.units import inch +from reportlab.platypus import Image, PageBreak, Paragraph, Spacer, Table, TableStyle -import logging +from ..sections.base import ReportSection logger = logging.getLogger(__name__) @@ -37,25 +38,25 @@ def _load_image_buffer(path: str) -> io.BytesIO | None: def _extract_mgmt_specific_sites_from_bed(bed_path: str) -> pd.DataFrame: """ Extract MGMT-specific CpG site methylation data from a BED file. - + This function mirrors the logic from the GUI component to extract strand-specific methylation data for key CpG sites. - + Args: bed_path: Path to the BED file - + Returns: DataFrame with CpG site methylation data, or empty DataFrame if extraction fails """ try: df = pd.read_csv(bed_path, sep="\t", header=None) - + # Check if column 10 contains space-separated values (old format) has_space_separated_col10 = False if df.shape[1] > 10 and len(df) > 0: sample_val = str(df.iloc[0, 9]) - has_space_separated_col10 = ' ' in sample_val or '\t' in sample_val - + has_space_separated_col10 = " " in sample_val or "\t" in sample_val + # Check if this is the new bedmethyl format (separate columns) or old format if df.shape[1] >= 12 and not has_space_separated_col10: # New bedmethyl format with separate columns @@ -76,14 +77,14 @@ def _extract_mgmt_specific_sites_from_bed(bed_path: str) -> pd.DataFrame: num_cols_to_read = min(len(cols), df.shape[1]) df = df.iloc[:, :num_cols_to_read] df.columns = cols[:num_cols_to_read] - + df["Nvalid_cov"] = df["Nvalid_cov"].astype(float) df["Fraction_Modified"] = df["Fraction_Modified"].astype(float) if "Nmod" in df.columns: df["Nmod"] = df["Nmod"].astype(float) else: df["Nmod"] = df["Nvalid_cov"] * df["Fraction_Modified"] - + df["Start"] = df["Start"].astype(int) df["Coverage"] = df["Nvalid_cov"] df["Modified_Fraction"] = df["Fraction_Modified"] * 100.0 @@ -101,27 +102,29 @@ def _extract_mgmt_specific_sites_from_bed(bed_path: str) -> pd.DataFrame: "RGB", "Coverage_Info", ] - df = df.iloc[:, :len(cols)] + df = df.iloc[:, : len(cols)] df.columns = cols - + cov_split = df["Coverage_Info"].astype(str).str.split() df["Coverage"] = cov_split.str[0].astype(float) fraction_val = cov_split.str[1].astype(float).fillna(0.0) - is_percentage = (fraction_val > 1.0).any() if len(fraction_val) > 0 else False - + is_percentage = ( + (fraction_val > 1.0).any() if len(fraction_val) > 0 else False + ) + if is_percentage: df["Modified_Fraction"] = fraction_val df["Fraction_Modified"] = df["Modified_Fraction"] / 100.0 else: df["Fraction_Modified"] = fraction_val df["Modified_Fraction"] = df["Fraction_Modified"] * 100.0 - + df["Nvalid_cov"] = df["Coverage"] df["Nmod"] = df["Coverage"] * df["Fraction_Modified"] df["Start"] = df["Start"].astype(int) else: return pd.DataFrame() - + # Define CpG pairs and labels cpg_pairs = [ (129467255, 129467256), @@ -135,26 +138,26 @@ def _extract_mgmt_specific_sites_from_bed(bed_path: str) -> pd.DataFrame: "129467262/129467263": "3", "129467272/129467273": "4", } - + rows = [] for p1, p2 in cpg_pairs: pos_key = f"{p1}/{p2}" site_label = label_map.get(pos_key, "Unknown") - + # Check forward strand reads at position p1 fwd_p1 = df[ (df["Chromosome"] == "chr10") & (df["Start"] == p1 - 1) & (df["Strand"] == "+") ] - + # Check reverse strand reads at position p2 rev_p2 = df[ (df["Chromosome"] == "chr10") & (df["Start"] == p2 - 1) & (df["Strand"] == "-") ] - + # Get forward strand data if not fwd_p1.empty: cov_f = float(fwd_p1["Nvalid_cov"].iloc[0]) @@ -167,7 +170,7 @@ def _extract_mgmt_specific_sites_from_bed(bed_path: str) -> pd.DataFrame: mf = 0.0 meth_fwd_count = 0 meth_fwd_pct = 0.0 - + # Get reverse strand data if not rev_p2.empty: cov_r = float(rev_p2["Nvalid_cov"].iloc[0]) @@ -180,26 +183,28 @@ def _extract_mgmt_specific_sites_from_bed(bed_path: str) -> pd.DataFrame: mr = 0.0 meth_rev_count = 0 meth_rev_pct = 0.0 - + # Only add row if we have data if cov_f > 0 or cov_r > 0: tot = cov_f + cov_r weighted = ((cov_f * mf) + (cov_r * mr)) / tot if tot > 0 else 0.0 weighted_pct = weighted * 100.0 - - rows.append({ - "Site_Label": f"Site {site_label}", - "Position": pos_key, - "Coverage_Forward": int(cov_f), - "Coverage_Reverse": int(cov_r), - "Total_Coverage": int(tot), - "Methylation_Percentage": weighted_pct, - "Forward_Methylation": meth_fwd_pct, - "Reverse_Methylation": meth_rev_pct, - "Forward_Methylated_Count": meth_fwd_count, - "Reverse_Methylated_Count": meth_rev_count, - }) - + + rows.append( + { + "Site_Label": f"Site {site_label}", + "Position": pos_key, + "Coverage_Forward": int(cov_f), + "Coverage_Reverse": int(cov_r), + "Total_Coverage": int(tot), + "Methylation_Percentage": weighted_pct, + "Forward_Methylation": meth_fwd_pct, + "Reverse_Methylation": meth_rev_pct, + "Forward_Methylated_Count": meth_fwd_count, + "Reverse_Methylated_Count": meth_rev_count, + } + ) + return pd.DataFrame(rows) except Exception as e: logger.warning(f"Failed to extract CpG sites from BED file {bed_path}: {e}") @@ -220,12 +225,16 @@ def add_content(self): specific_sites = None # First, look for "final_mgmt.csv" files (highest priority) - final_files = [f for f in os.listdir(self.report.output) if f == "final_mgmt.csv"] + final_files = [ + f for f in os.listdir(self.report.output) if f == "final_mgmt.csv" + ] if final_files: file = final_files[0] mgmt_results = pd.read_csv(os.path.join(self.report.output, file)) plot_out = os.path.join(self.report.output, file.replace(".csv", ".png")) - specific_sites_file = os.path.join(self.report.output, "final_specific_sites.csv") + specific_sites_file = os.path.join( + self.report.output, "final_specific_sites.csv" + ) last_seen = 999999 # High number to indicate final result else: # Fallback to numeric-prefixed files @@ -236,7 +245,9 @@ def add_content(self): prefix = file.split("_")[0] count = int(prefix) if count > last_seen: - mgmt_results = pd.read_csv(os.path.join(self.report.output, file)) + mgmt_results = pd.read_csv( + os.path.join(self.report.output, file) + ) plot_out = os.path.join( self.report.output, file.replace(".csv", ".png") ) @@ -378,7 +389,7 @@ def add_content(self): logger.debug(f"Loaded CpG sites from CSV: {specific_sites_file}") except Exception as e: logger.warning(f"Failed to load CpG sites CSV: {e}") - + # If CSV doesn't exist or is empty, try to extract from BED file if specific_sites is None or specific_sites.empty: # Determine BED file path @@ -392,13 +403,13 @@ def add_content(self): os.path.join(self.report.output, f"{last_seen}_mgmt.bed"), os.path.join(self.report.output, f"{last_seen}_mgmt_mgmt.bed"), ] - + bed_path = None for candidate in bed_candidates: if os.path.exists(candidate): bed_path = candidate break - + if bed_path: try: specific_sites = _extract_mgmt_specific_sites_from_bed(bed_path) @@ -407,7 +418,7 @@ def add_content(self): except Exception as e: logger.warning(f"Failed to extract CpG sites from BED: {e}") specific_sites = None - + # Add specific CpG sites table if we have data if specific_sites is not None and not specific_sites.empty: self.elements.append( @@ -431,7 +442,7 @@ def add_content(self): "Rev\nMeth %", ] ] - + # Add methylation count columns if available if has_meth_counts: cpg_data[0].extend(["Fwd\nMeth\nCount", "Rev\nMeth\nCount"]) @@ -440,12 +451,16 @@ def add_content(self): for _, row in specific_sites.iterrows(): # Extract site number from Site_Label site_label = str(row.get("Site_Label", "")) - site_num = site_label.split(" ")[-1] if " " in site_label else site_label - + site_num = ( + site_label.split(" ")[-1] if " " in site_label else site_label + ) + # Extract position position = str(row.get("Position", "")) - pos_display = position.split("/")[0] if "/" in position else position - + pos_display = ( + position.split("/")[0] if "/" in position else position + ) + row_data = [ f"Site {site_num}", pos_display, @@ -456,14 +471,16 @@ def add_content(self): f"{row.get('Forward_Methylation', 0.0):.1f}", f"{row.get('Reverse_Methylation', 0.0):.1f}", ] - + # Add methylation counts if available if has_meth_counts: - row_data.extend([ - str(int(row.get("Forward_Methylated_Count", 0))), - str(int(row.get("Reverse_Methylated_Count", 0))), - ]) - + row_data.extend( + [ + str(int(row.get("Forward_Methylated_Count", 0))), + str(int(row.get("Reverse_Methylated_Count", 0))), + ] + ) + cpg_data.append(row_data) # Create the table with more compact column widths @@ -492,7 +509,7 @@ def add_content(self): 0.6 * inch, # Forward Methylation % 0.6 * inch, # Reverse Methylation % ] - + cpg_table = Table( cpg_data, colWidths=col_widths, @@ -570,7 +587,7 @@ def add_content(self): else: results_file = f"{last_seen}_mgmt.csv" plot_file = f"{last_seen}_mgmt.png" - + file_sources = [ ["Source", "Location"], # Shorter headers [ @@ -628,13 +645,15 @@ def add_content(self): # Try to add the methylation plot plot_added = False - + # First, try to load from existing PNG file (e.g. final_mgmt.png) if plot_out: plot_buf = _load_image_buffer(plot_out) if plot_buf is not None: try: - self.elements.append(Image(plot_buf, width=6 * inch, height=4 * inch)) + self.elements.append( + Image(plot_buf, width=6 * inch, height=4 * inch) + ) self.elements.append( Paragraph( "MGMT promoter methylation plot showing methylation levels across CpG sites", @@ -644,7 +663,7 @@ def add_content(self): plot_added = True except Exception as e: logger.warning(f"Failed to load MGMT plot from file: {e}") - + # If not found, try to generate from BAM file using locus_figure if not plot_added: # Try sorted BAM first, then fall back to unsorted @@ -652,38 +671,46 @@ def add_content(self): os.path.join(self.report.output, "mgmt_sorted.bam"), os.path.join(self.report.output, "mgmt.bam"), ] - + bam_path = None for candidate in bam_candidates: if os.path.exists(candidate): bam_path = candidate break - + if bam_path: try: - from robin.analysis.methylation_wrapper import locus_figure import warnings + import matplotlib.pyplot as plt - - logger.info(f"Generating MGMT plot from BAM file for report: {os.path.basename(bam_path)}") + + from robin.analysis.methylation_wrapper import locus_figure + + logger.info( + f"Generating MGMT plot from BAM file for report: {os.path.basename(bam_path)}" + ) fig = locus_figure( interval="chr10:129466536-129467536", bam_path=bam_path, motif="CG", mods="m", ) - + # Embed in memory (same pattern as CNV/coverage sections). # Avoids ReportLab failing later if path casing differs on disk. plot_buf = io.BytesIO() with warnings.catch_warnings(): warnings.simplefilter("ignore", UserWarning) - fig.savefig(plot_buf, format="png", dpi=150, bbox_inches="tight") + fig.savefig( + plot_buf, format="png", dpi=150, bbox_inches="tight" + ) plot_buf.seek(0) plt.close(fig) if plot_buf.getbuffer().nbytes > 0: - self.elements.append(Image(plot_buf, width=6 * inch, height=4 * inch)) + self.elements.append( + Image(plot_buf, width=6 * inch, height=4 * inch) + ) self.elements.append( Paragraph( "MGMT promoter methylation plot showing methylation levels across CpG sites", @@ -692,11 +719,13 @@ def add_content(self): ) plot_added = True else: - logger.warning("MGMT plot generation produced empty image data") - + logger.warning( + "MGMT plot generation produced empty image data" + ) + except Exception as e: logger.warning(f"Failed to generate MGMT plot from BAM: {e}") - + # If still not added, show error message if not plot_added: self.elements.append( @@ -745,10 +774,20 @@ def add_content(self): else None ), "PredictionScorePercent": ( - round(float(prediction_score), 2) if prediction_score is not None else None + round(float(prediction_score), 2) + if prediction_score is not None + else None + ), + "ResultsFile": ( + "final_mgmt.csv" + if last_seen == 999999 + else (f"{last_seen}_mgmt.csv" if last_seen else None) + ), + "PlotFile": ( + "final_mgmt.png" + if last_seen == 999999 + else (f"{last_seen}_mgmt.png" if last_seen else None) ), - "ResultsFile": "final_mgmt.csv" if last_seen == 999999 else (f"{last_seen}_mgmt.csv" if last_seen else None), - "PlotFile": "final_mgmt.png" if last_seen == 999999 else (f"{last_seen}_mgmt.png" if last_seen else None), "CpGSitesFile": ( os.path.basename(specific_sites_file) if specific_sites_file and os.path.exists(specific_sites_file) diff --git a/src/robin/reporting/sections/mnpflex.py b/src/robin/reporting/sections/mnpflex.py index 9237ffcb..415f7a9a 100644 --- a/src/robin/reporting/sections/mnpflex.py +++ b/src/robin/reporting/sections/mnpflex.py @@ -9,8 +9,8 @@ import os from typing import Any, Dict, List, Optional -from reportlab.platypus import Paragraph, Spacer from reportlab.lib.units import inch +from reportlab.platypus import Paragraph, Spacer from robin.analysis.mnpflex_docker import hierarchy_aggregate_display from robin.analysis.mnpflex_hierarchy import ( @@ -203,17 +203,27 @@ def add_content(self): [ { "score": item.get("score"), - "subclass": (item.get("reference_group") or {}).get("molecular_subclass") + "subclass": (item.get("reference_group") or {}).get( + "molecular_subclass" + ) or (item.get("reference_group") or {}).get("name"), - "class": (item.get("reference_group") or {}).get("molecular_class"), - "family": (item.get("reference_group") or {}).get("molecular_family"), - "superfamily": (item.get("reference_group") or {}).get("molecular_superfamily"), + "class": (item.get("reference_group") or {}).get( + "molecular_class" + ), + "family": (item.get("reference_group") or {}).get( + "molecular_family" + ), + "superfamily": (item.get("reference_group") or {}).get( + "molecular_superfamily" + ), } for item in top_scores ] ) except Exception as ex: - logger.error("Error building MNP-Flex export DataFrames: %s", ex, exc_info=True) + logger.error( + "Error building MNP-Flex export DataFrames: %s", ex, exc_info=True + ) # Top 10 classifier scores scores = classifier_summary.get("scores") or [] @@ -238,25 +248,31 @@ def add_content(self): ref.get("molecular_superfamily") or "", ] ) - self.elements.append(Paragraph("Top 10 classifier scores", self.styles.styles["Heading3"])) + self.elements.append( + Paragraph("Top 10 classifier scores", self.styles.styles["Heading3"]) + ) self.elements.append(self.create_table(top_rows)) self.elements.append(Spacer(1, 4)) hierarchy_preds = hierarchy_aggregate_display(classifier_summary) if hierarchy_preds or scores: if hierarchy_preds: - top_subclass = hierarchy_preds.get("molecular_subclass", {}).get("label") + top_subclass = hierarchy_preds.get("molecular_subclass", {}).get( + "label" + ) top_class = hierarchy_preds.get("molecular_class", {}).get("label") top_family = hierarchy_preds.get("molecular_family", {}).get("label") - top_superfamily = hierarchy_preds.get( - "molecular_superfamily", {} - ).get("label") - subclass_sum = hierarchy_preds.get("molecular_subclass", {}).get("score") + top_superfamily = hierarchy_preds.get("molecular_superfamily", {}).get( + "label" + ) + subclass_sum = hierarchy_preds.get("molecular_subclass", {}).get( + "score" + ) class_sum = hierarchy_preds.get("molecular_class", {}).get("score") family_sum = hierarchy_preds.get("molecular_family", {}).get("score") - superfamily_sum = hierarchy_preds.get( - "molecular_superfamily", {} - ).get("score") + superfamily_sum = hierarchy_preds.get("molecular_superfamily", {}).get( + "score" + ) else: top = sorted( scores, @@ -286,9 +302,17 @@ def add_content(self): ["Subclass", top_subclass or "N/A", self._format_score(subclass_sum)], ["Class", top_class or "N/A", self._format_score(class_sum)], ["Family", top_family or "N/A", self._format_score(family_sum)], - ["Superfamily", top_superfamily or "N/A", self._format_score(superfamily_sum)], + [ + "Superfamily", + top_superfamily or "N/A", + self._format_score(superfamily_sum), + ], ] - self.elements.append(Paragraph("Aggregate scores for top entry", self.styles.styles["Heading3"])) + self.elements.append( + Paragraph( + "Aggregate scores for top entry", self.styles.styles["Heading3"] + ) + ) if not has_hierarchical_summary: self.elements.append( Paragraph( @@ -303,7 +327,10 @@ def add_content(self): # Plots (if available) plot_specs = [ ("QC coverage plot", os.path.join(results_dir, "qc_coverage_plot.png")), - ("QC methylation density plot", os.path.join(results_dir, "qc_methylation_density_plot.png")), + ( + "QC methylation density plot", + os.path.join(results_dir, "qc_methylation_density_plot.png"), + ), ("MGMT region plot", os.path.join(results_dir, "mgmt_region_plot.png")), ] for title, path in plot_specs: diff --git a/src/robin/reporting/sections/run_data.py b/src/robin/reporting/sections/run_data.py index a22733b0..ac769ce5 100644 --- a/src/robin/reporting/sections/run_data.py +++ b/src/robin/reporting/sections/run_data.py @@ -6,16 +6,19 @@ import logging from datetime import datetime -from reportlab.platypus import Paragraph, Spacer, Table, TableStyle, PageBreak -from reportlab.lib.units import inch + from reportlab.lib.styles import ParagraphStyle -from .base import ReportSection +from reportlab.lib.units import inch +from reportlab.platypus import PageBreak, Paragraph, Spacer, Table, TableStyle + from robin.analysis.bam_preprocessor import ( _get_modbase_model_warning, _get_modbase_model_warning_level, _is_unresolved_modbase_model, ) +from .base import ReportSection + logger = logging.getLogger(__name__) @@ -187,7 +190,10 @@ def add_content(self): if getattr(self.report, "sample_identifiers", None): si = self.report.sample_identifiers sample_info = [ - ("Sample ID", si.get("sample_id", "") or self.report.sample_id or "—"), + ( + "Sample ID", + si.get("sample_id", "") or self.report.sample_id or "—", + ), ("Test ID", si.get("test_id", "") or "—"), ("First name", si.get("first_name", "") or "—"), ("Last name", si.get("last_name", "") or "—"), diff --git a/src/robin/reporting/sections/variants.py b/src/robin/reporting/sections/variants.py index 19a3220d..f065aa18 100644 --- a/src/robin/reporting/sections/variants.py +++ b/src/robin/reporting/sections/variants.py @@ -4,18 +4,21 @@ This module handles the Pathogenic Variant analysis section of the report. """ -import os import logging +import os import re + import pandas as pd -from reportlab.lib.units import inch -from reportlab.platypus import Paragraph, Spacer, Table, TableStyle, PageBreak -from reportlab.lib.styles import ParagraphStyle from reportlab.lib.colors import HexColor -from ..sections.base import ReportSection +from reportlab.lib.styles import ParagraphStyle +from reportlab.lib.units import inch +from reportlab.platypus import PageBreak, Paragraph, Spacer, Table, TableStyle + from robin import resources from robin.analysis.variant_classification import is_clinvar_significant_from_info +from ..sections.base import ReportSection + logger = logging.getLogger(__name__) @@ -80,7 +83,9 @@ def process_vcf(self, vcf_file, variant_type, result): if self._is_clinvar_significant(info_str): pathogenic_count += 1 logger.debug( - "Found ClinVar-significant variant at %s:%s", fields[0], fields[1] + "Found ClinVar-significant variant at %s:%s", + fields[0], + fields[1], ) variant_data = { @@ -455,7 +460,9 @@ def add_content(self): "Note: Variants are classified as pathogenic based on ClinVar annotations. " "Disease associations are derived from ClinVar's CLNDN field where available." ) - clinvar_release = getattr(self.report, "clinvar_metadata", {}).get("file_date") + clinvar_release = getattr(self.report, "clinvar_metadata", {}).get( + "file_date" + ) if clinvar_release: note_text += f" ClinVar release: {clinvar_release}." self.elements.append(Paragraph(note_text, note_style)) diff --git a/src/robin/reporting/styling/pdf_styles.py b/src/robin/reporting/styling/pdf_styles.py index bf0b7854..7c4a0d2a 100644 --- a/src/robin/reporting/styling/pdf_styles.py +++ b/src/robin/reporting/styling/pdf_styles.py @@ -5,9 +5,9 @@ for implementing Material Design 3 and Apple HIG principles in PDF reports. """ -from reportlab.platypus import Paragraph, Spacer, Table, TableStyle -from reportlab.lib.units import inch from reportlab.lib import colors +from reportlab.lib.units import inch +from reportlab.platypus import Paragraph, Spacer, Table, TableStyle class PDFStyleUtils: @@ -129,9 +129,7 @@ def create_info_card(title, content, card_style="SummaryCard", styles_dict=None) for item in content: if isinstance(item, str): if styles_dict and "Normal" in styles_dict: - elements.append( - Paragraph(f"• {item}", styles_dict["Normal"]) - ) + elements.append(Paragraph(f"• {item}", styles_dict["Normal"])) else: # Fallback to default style from reportlab.lib.styles import getSampleStyleSheet @@ -219,8 +217,8 @@ def create_data_table( table.setStyle(style) else: # Fallback to basic table styling - from reportlab.platypus import TableStyle from reportlab.lib import colors + from reportlab.platypus import TableStyle basic_style = TableStyle( [ diff --git a/src/robin/reporting/styling/styles.py b/src/robin/reporting/styling/styles.py index 9ee3b9d7..0dc4eba5 100644 --- a/src/robin/reporting/styling/styles.py +++ b/src/robin/reporting/styling/styles.py @@ -4,13 +4,14 @@ This module contains styling-related code for the ROBIN report generation. """ -from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet +import logging +import os + from reportlab.lib.colors import HexColor +from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet from reportlab.pdfbase import pdfmetrics from reportlab.pdfbase.ttfonts import TTFont from reportlab.platypus import TableStyle -import os -import logging logger = logging.getLogger(__name__) diff --git a/src/robin/reporting/utils.py b/src/robin/reporting/utils.py index 8cadc54d..12bb6562 100644 --- a/src/robin/reporting/utils.py +++ b/src/robin/reporting/utils.py @@ -4,8 +4,8 @@ This module contains utility functions for the report generation. """ -import pandas as pd import natsort +import pandas as pd def convert_to_space_separated_string(array): diff --git a/src/robin/reporting/utils/__init__.py b/src/robin/reporting/utils/__init__.py index 8fa5a227..5e43b1bc 100644 --- a/src/robin/reporting/utils/__init__.py +++ b/src/robin/reporting/utils/__init__.py @@ -5,9 +5,9 @@ """ from .formatting import ( + convert_to_space_separated_string, format_number, format_timestamp, - convert_to_space_separated_string, split_text, ) diff --git a/src/robin/reporting/utils/formatting.py b/src/robin/reporting/utils/formatting.py index 4f354623..c7fa658b 100644 --- a/src/robin/reporting/utils/formatting.py +++ b/src/robin/reporting/utils/formatting.py @@ -4,8 +4,8 @@ This module contains utility functions for formatting data in the report. """ -from datetime import datetime import logging +from datetime import datetime logger = logging.getLogger(__name__) diff --git a/src/robin/runtime_limits.py b/src/robin/runtime_limits.py index 6b4f42e4..2826170e 100644 --- a/src/robin/runtime_limits.py +++ b/src/robin/runtime_limits.py @@ -5,7 +5,6 @@ import os from typing import Dict - _NATIVE_THREAD_ENV_VARS = ( "POLARS_MAX_THREADS", "RAYON_NUM_THREADS", diff --git a/src/robin/security/__init__.py b/src/robin/security/__init__.py index 84ba2cdf..13ec7a01 100644 --- a/src/robin/security/__init__.py +++ b/src/robin/security/__init__.py @@ -6,13 +6,7 @@ get_consent_version, get_security_db_path, ) -from .user_metadata import ( - CLINICAL_ROLE_KEY, - EMAIL_KEY, - USER_METADATA_FIELDS, - metadata_field_labels, - normalize_metadata, -) +from .store import SecurityStore from .user_approvals import ( ADMIN_USER_APPROVALS_UPDATED_EVENT, MINKNOW_REMOTE_CONTROL_KEY, @@ -26,7 +20,13 @@ normalize_approvals, user_has_approval, ) -from .store import SecurityStore +from .user_metadata import ( + CLINICAL_ROLE_KEY, + EMAIL_KEY, + USER_METADATA_FIELDS, + metadata_field_labels, + normalize_metadata, +) __all__ = [ "AuditService", diff --git a/src/robin/security/constants.py b/src/robin/security/constants.py index d875a2e6..496938cf 100644 --- a/src/robin/security/constants.py +++ b/src/robin/security/constants.py @@ -3,7 +3,6 @@ import os from pathlib import Path - DEFAULT_CONSENT_VERSION = "v1" CONSENT_VERSION_ENV = "ROBIN_CONSENT_VERSION" diff --git a/src/robin/security/store.py b/src/robin/security/store.py index 419835a1..25219ebc 100644 --- a/src/robin/security/store.py +++ b/src/robin/security/store.py @@ -9,13 +9,13 @@ from .constants import get_security_db_path from .models import User, UserPublic -from .user_metadata import metadata_to_json, normalize_metadata, parse_metadata_json from .user_approvals import ( approvals_to_json, default_approvals, normalize_approvals, parse_approvals_json, ) +from .user_metadata import metadata_to_json, normalize_metadata, parse_metadata_json def utc_now_iso() -> str: @@ -44,8 +44,7 @@ def __init__(self, db_path: Optional[Path] = None): def _init_schema(self) -> None: with self._lock: - self._conn.executescript( - """ + self._conn.executescript(""" CREATE TABLE IF NOT EXISTS users ( id INTEGER PRIMARY KEY AUTOINCREMENT, username TEXT NOT NULL UNIQUE, @@ -112,8 +111,7 @@ def _init_schema(self) -> None: updated_by_user_id INTEGER, FOREIGN KEY(updated_by_user_id) REFERENCES users(id) ); - """ - ) + """) self._migrate_schema() def _migrate_schema(self) -> None: @@ -369,12 +367,10 @@ def set_user_active(self, username: str, is_active: bool) -> bool: def list_users(self) -> List[UserPublic]: with self._lock: - rows = self._conn.execute( - f""" + rows = self._conn.execute(f""" SELECT {self._user_public_select_columns()} FROM users ORDER BY username - """ - ).fetchall() + """).fetchall() return [self._row_to_user_public(row) for row in rows] def user_has_role(self, user_id: int, role_name: str) -> bool: @@ -407,15 +403,13 @@ def revoke_role(self, user_id: int, role_name: str) -> bool: def count_active_admins(self) -> int: with self._lock: - row = self._conn.execute( - """ + row = self._conn.execute(""" SELECT COUNT(*) AS c FROM users u INNER JOIN user_roles ur ON ur.user_id = u.id INNER JOIN roles r ON r.id = ur.role_id WHERE u.is_active = 1 AND r.name = 'admin' - """ - ).fetchone() + """).fetchone() return int(row["c"]) if row else 0 def set_last_login(self, user_id: int) -> None: @@ -478,7 +472,14 @@ def record_consent( INSERT INTO consents(user_id, consent_version, agreed_at, ip, user_agent, session_id) VALUES (?, ?, ?, ?, ?, ?) """, - (int(user_id), consent_version, utc_now_iso(), ip, user_agent, session_id), + ( + int(user_id), + consent_version, + utc_now_iso(), + ip, + user_agent, + session_id, + ), ) def list_consent_status(self, consent_version: str) -> List[Dict[str, Any]]: @@ -532,7 +533,9 @@ def append_audit_event( request_id: str = "", error_code: str = "", ) -> None: - details_json = json.dumps(details or {}, separators=(",", ":"), ensure_ascii=True) + details_json = json.dumps( + details or {}, separators=(",", ":"), ensure_ascii=True + ) with self._lock: self._conn.execute( """ @@ -635,7 +638,9 @@ def query_audit_events( { "id": int(row["id"]), "occurred_at": str(row["occurred_at"]), - "user_id": int(row["user_id"]) if row["user_id"] is not None else None, + "user_id": ( + int(row["user_id"]) if row["user_id"] is not None else None + ), "username": str(row["username"] or ""), "event_type": str(row["event_type"] or ""), "target_type": str(row["target_type"] or ""), diff --git a/src/robin/security/user_approvals.py b/src/robin/security/user_approvals.py index cd557fe9..4af9dc57 100644 --- a/src/robin/security/user_approvals.py +++ b/src/robin/security/user_approvals.py @@ -131,7 +131,9 @@ def user_has_approval(store: "SecurityStore", user_id: Optional[int], key: str) return bool(user.approvals.get(key, False)) -def effective_approvals(store: "SecurityStore", user_id: Optional[int]) -> Dict[str, bool]: +def effective_approvals( + store: "SecurityStore", user_id: Optional[int] +) -> Dict[str, bool]: """Return effective approval map for display (admins show all granted).""" labels = approval_field_labels() if user_id is None: diff --git a/src/robin/state_tracker.py b/src/robin/state_tracker.py index 73a3ccbe..75392b91 100644 --- a/src/robin/state_tracker.py +++ b/src/robin/state_tracker.py @@ -19,9 +19,12 @@ def _safe_stat(path: str) -> Tuple[int, int, int, float]: """ try: st = os.stat(path) - return int(getattr(st, "st_dev", 0) or 0), int(getattr(st, "st_ino", 0) or 0), int( - getattr(st, "st_size", 0) or 0 - ), float(getattr(st, "st_mtime", 0.0) or 0.0) + return ( + int(getattr(st, "st_dev", 0) or 0), + int(getattr(st, "st_ino", 0) or 0), + int(getattr(st, "st_size", 0) or 0), + float(getattr(st, "st_mtime", 0.0) or 0.0), + ) except Exception: return 0, 0, 0, 0.0 @@ -103,8 +106,7 @@ def close(self) -> None: pass def _init_schema(self) -> None: - self._conn.execute( - """ + self._conn.execute(""" CREATE TABLE IF NOT EXISTS file_state ( file_key TEXT PRIMARY KEY, path TEXT, @@ -119,8 +121,7 @@ def _init_schema(self) -> None: last_update REAL, last_error TEXT ); - """ - ) + """) self._conn.execute( "CREATE INDEX IF NOT EXISTS idx_file_state_sample_id ON file_state(sample_id);" ) @@ -253,4 +254,3 @@ def should_skip( if mask == 0: return False return self.is_done(path, job_type) - diff --git a/src/robin/utils/clinvar_manager.py b/src/robin/utils/clinvar_manager.py index 2fdab2c4..c7db0669 100644 --- a/src/robin/utils/clinvar_manager.py +++ b/src/robin/utils/clinvar_manager.py @@ -14,6 +14,7 @@ from email.utils import format_datetime, parsedate_to_datetime from pathlib import Path from typing import Any, Optional + import pysam logger = logging.getLogger("robin.clinvar") @@ -250,10 +251,9 @@ def load_sample_clinvar_provenance( def sample_snp_reannotation_inputs_ready(sample_dir: Path | str) -> bool: """Return True when Clair3 outputs exist for annotation-only reruns.""" clair_dir = Path(sample_dir) / "clair3" - return ( - (clair_dir / "output_done.vcf.gz").is_file() - and (clair_dir / "output_indel_done.vcf.gz").is_file() - ) + return (clair_dir / "output_done.vcf.gz").is_file() and ( + clair_dir / "output_indel_done.vcf.gz" + ).is_file() def compare_sample_clinvar_to_installed( @@ -317,7 +317,11 @@ def _download_url_to_file(url: str, target_path: Path, *, timeout_s: int = 600) # Use a temp file in the same directory so rename is atomic. with tempfile.NamedTemporaryFile( - mode="wb", suffix=".part", prefix=target_path.name + ".", dir=str(target_path.parent), delete=False + mode="wb", + suffix=".part", + prefix=target_path.name + ".", + dir=str(target_path.parent), + delete=False, ) as tmp: tmp_path = Path(tmp.name) try: @@ -372,7 +376,9 @@ def _download_url_to_file(url: str, target_path: Path, *, timeout_s: int = 600) mb = downloaded // (1024 * 1024) if mb // 16 != last_reported_mb // 16: last_reported_mb = mb - click.echo(f"Downloading ClinVar: {mb} MiB downloaded...") + click.echo( + f"Downloading ClinVar: {mb} MiB downloaded..." + ) tmp_path.replace(target_path) print(f"ClinVar download complete: {target_path}") @@ -495,7 +501,9 @@ def _ensure_tabix_index(gz_path: Path, tbi_path: Path) -> None: try: tbi_path.unlink() except OSError as exc: - logger.warning("Could not remove stale ClinVar tabix index %s: %s", tbi_path, exc) + logger.warning( + "Could not remove stale ClinVar tabix index %s: %s", tbi_path, exc + ) _build_tabix_index(gz_path, tbi_path) @@ -625,7 +633,11 @@ def update_clinvar_if_newer( # Add a small tolerance to avoid re-downloading due to timestamp rounding. if remote_mtime <= local_mtime + 1: - logger.info("ClinVar already up to date (local=%s, remote=%s).", local_mtime, remote_mtime) + logger.info( + "ClinVar already up to date (local=%s, remote=%s).", + local_mtime, + remote_mtime, + ) print("ClinVar already up to date.") refresh_clinvar_metadata( resources_dir=resources_dir, @@ -654,4 +666,3 @@ def update_clinvar_if_newer( compute_checksum=True, ) return True - diff --git a/src/robin/utils/docker_fs.py b/src/robin/utils/docker_fs.py index b669ab95..b56ecadb 100644 --- a/src/robin/utils/docker_fs.py +++ b/src/robin/utils/docker_fs.py @@ -144,9 +144,7 @@ def chown_tree_to_host_user(path: Path, *, timeout_s: int = 600) -> bool: return False parent = target.parent - logger.info( - "Normalizing ownership of %s to %s via Docker", target, user_spec - ) + logger.info("Normalizing ownership of %s to %s via Docker", target, user_spec) result = subprocess.run( [ "docker", diff --git a/src/robin/utils/marlin_manager.py b/src/robin/utils/marlin_manager.py index 209913f1..58864bc2 100644 --- a/src/robin/utils/marlin_manager.py +++ b/src/robin/utils/marlin_manager.py @@ -85,7 +85,9 @@ def _file_ok(path: Path) -> bool: return path.exists() and path.is_file() and path.stat().st_size > 0 -def _download_url_to_file(url: str, target_path: Path, *, timeout_s: int = 3600) -> None: +def _download_url_to_file( + url: str, target_path: Path, *, timeout_s: int = 3600 +) -> None: """Download ``url`` to ``target_path`` atomically (temp file + rename).""" target_path.parent.mkdir(parents=True, exist_ok=True) @@ -144,7 +146,9 @@ def _download_url_to_file(url: str, target_path: Path, *, timeout_s: int = 3600) downloaded += len(chunk) if total and downloaded % (50 * chunk_size) < chunk_size: pct = 100.0 * downloaded / total - print(f" MARLIN download: {pct:.1f}% ({downloaded}/{total} bytes)") + print( + f" MARLIN download: {pct:.1f}% ({downloaded}/{total} bytes)" + ) if total is not None and downloaded != total: raise RuntimeError( @@ -203,13 +207,20 @@ def resolve_model_path( download_url = url or os.environ.get(ENV_MODEL_URL) or DEFAULT_MARLIN_MODEL_URL try: _download_url_to_file(download_url, path) - except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, OSError) as exc: + except ( + urllib.error.URLError, + urllib.error.HTTPError, + TimeoutError, + OSError, + ) as exc: raise RuntimeError( f"Failed to download MARLIN model from {download_url} to {path}: {exc}" ) from exc if not _file_ok(path): - raise RuntimeError(f"MARLIN model download completed but file is missing: {path}") + raise RuntimeError( + f"MARLIN model download completed but file is missing: {path}" + ) return path diff --git a/src/robin/utils/mnpflex_client_standalone.py b/src/robin/utils/mnpflex_client_standalone.py index 813ecc49..c4e107bf 100644 --- a/src/robin/utils/mnpflex_client_standalone.py +++ b/src/robin/utils/mnpflex_client_standalone.py @@ -1,10 +1,10 @@ import json +import logging import os import time from dataclasses import dataclass from typing import Optional -import logging import requests @@ -213,7 +213,11 @@ def get_bundle_summary(self, workflow_run_id: int, task_result_id: int) -> dict: except Exception: text_snippet = "" - log_fn = logging.warning if status is not None and status >= 500 else logging.error + log_fn = ( + logging.warning + if status is not None and status >= 500 + else logging.error + ) # Avoid exc_info on every transient 5xx; the bulk runner will capture the final traceback. log_kwargs = {} if not (status is not None and status >= 500): @@ -307,7 +311,11 @@ def wait_for_results( workflow_run_id = runs[0].get("id") else: workflow_run_id = next( - (r.get("id") for r in runs if r.get("workflow_id") == workflow_id), + ( + r.get("id") + for r in runs + if r.get("workflow_id") == workflow_id + ), None, ) if workflow_run_id: diff --git a/src/robin/utils/model_checker.py b/src/robin/utils/model_checker.py index 8398c436..bd4c3c90 100644 --- a/src/robin/utils/model_checker.py +++ b/src/robin/utils/model_checker.py @@ -12,8 +12,7 @@ import os import sys from pathlib import Path -from typing import List, Tuple, Optional - +from typing import List, Optional, Tuple CLINVAR_VCF_GZ_NAME = "clinvar.vcf.gz" CLINVAR_TBI_NAME = "clinvar.vcf.gz.tbi" @@ -22,7 +21,7 @@ def get_models_directory(project_root: Optional[Path] = None) -> Path: """ Get the path to the models directory. - + Uses the models module's DIR constant, which works regardless of where robin is run from since it's based on the installed package location. """ @@ -31,6 +30,7 @@ def get_models_directory(project_root: Optional[Path] = None) -> Path: # Use the models module's DIR constant - this is the most reliable method # as it works regardless of current working directory from robin import models + return models.DIR except ImportError: # Fallback: try to find models directory relative to this file @@ -40,21 +40,21 @@ def get_models_directory(project_root: Optional[Path] = None) -> Path: Path(__file__).parent.parent.parent, # src directory Path.cwd(), # current working directory (fallback) ] - + for candidate_root in candidate_roots: # Try project_root/src/robin/models first models_dir = candidate_root / "src" / "robin" / "models" if models_dir.exists(): return models_dir - + # If candidate_root is actually the src directory, try src/robin/models directly models_dir = candidate_root / "robin" / "models" if models_dir.exists(): return models_dir - + # Final fallback: use current working directory project_root = Path.cwd() - + models_dir = project_root / "src" / "robin" / "models" return models_dir @@ -62,14 +62,14 @@ def get_models_directory(project_root: Optional[Path] = None) -> Path: def get_required_models() -> List[Tuple[str, str]]: """ Get list of required model files. - + Returns: List of tuples containing (model_name, filename) """ return [ ("general_model", "general.zip"), ("capper_model", "Capper_et_al_NN_v2.pkl"), - ("pancan_model", "pancan_devel_v5i_NN_v2.pkl") + ("pancan_model", "pancan_devel_v5i_NN_v2.pkl"), ] @@ -94,7 +94,9 @@ def get_resources_directory(project_root: Optional[Path] = None) -> Path: return project_root / "src" / "robin" / "resources" -def check_clinvar_files(resources_dir: Optional[Path] = None) -> Tuple[bool, List[str], List[str]]: +def check_clinvar_files( + resources_dir: Optional[Path] = None, +) -> Tuple[bool, List[str], List[str]]: """ Check if required ClinVar files exist. @@ -148,14 +150,14 @@ def _ensure_clinvar_or_exit(resources_dir: Optional[Path] = None) -> None: from robin.utils.clinvar_manager import ensure_clinvar_files # download_if_missing=True will either download or convert formats - ensure_clinvar_files( - resources_dir=resources_dir, download_if_missing=True - ) + ensure_clinvar_files(resources_dir=resources_dir, download_if_missing=True) except Exception as e: print(f"\n❌ Failed to download/generate ClinVar files: {e}") sys.exit(1) - all_present_after, missing_after, _present_after = check_clinvar_files(resources_dir) + all_present_after, missing_after, _present_after = check_clinvar_files( + resources_dir + ) if not all_present_after: print("\n❌ ClinVar files are still missing after download attempt:") for filename in missing_after: @@ -166,98 +168,100 @@ def _ensure_clinvar_or_exit(resources_dir: Optional[Path] = None) -> None: print("\n🎉 ClinVar files are now available.") -def check_model_files(project_root: Optional[Path] = None) -> Tuple[bool, List[str], List[str]]: +def check_model_files( + project_root: Optional[Path] = None, +) -> Tuple[bool, List[str], List[str]]: """ Check if all required model files are present. - + Args: project_root: Path to the project root directory - + Returns: Tuple of (all_present, missing_files, present_files) """ models_dir = get_models_directory(project_root) required_models = get_required_models() - + missing_files = [] present_files = [] - + for model_name, filename in required_models: model_path = models_dir / filename if model_path.exists() and model_path.stat().st_size > 0: present_files.append(filename) else: missing_files.append(filename) - + all_present = len(missing_files) == 0 return all_present, missing_files, present_files def _download_missing_models(missing_files, models_dir, project_root): """Download missing model files (same asset manifest logic as ``robin utils update-models``).""" - import json import hashlib - import urllib.request - import urllib.error + import json import os - + import urllib.error + import urllib.request + print("\n🔄 Attempting to download missing models...") - + # Load assets manifest try: assets_file = project_root / "assets.json" if not assets_file.exists(): print("❌ assets.json not found. Cannot download models automatically.") return False - - with open(assets_file, 'r') as f: + + with open(assets_file, "r") as f: manifest = json.load(f) except Exception as e: print(f"❌ Failed to load assets manifest: {e}") return False - + # Asset name mapping asset_mapping = { "general.zip": "general_model", - "Capper_et_al_NN_v2.pkl": "capper_model", - "pancan_devel_v5i_NN_v2.pkl": "pancan_model" + "Capper_et_al_NN_v2.pkl": "capper_model", + "pancan_devel_v5i_NN_v2.pkl": "pancan_model", } - - github_token = os.getenv('GITHUB_TOKEN') + + github_token = os.getenv("GITHUB_TOKEN") if not github_token: print("ℹ️ No GITHUB_TOKEN found. Trying public download...") - + success_count = 0 for filename in missing_files: if filename not in asset_mapping: print(f"⚠️ Unknown model file: {filename}") continue - + asset_name = asset_mapping[filename] if asset_name not in manifest["assets"]: print(f"❌ Asset '{asset_name}' not found in manifest") continue - + asset_info = manifest["assets"][asset_name] asset_url = asset_info["url"] expected_sha256 = asset_info["sha256"] - + target_path = models_dir / filename - + try: print(f"\n📥 Downloading {filename}...") - + # Download the file headers = {} if github_token: headers["Authorization"] = f"Bearer {github_token}" - + request = urllib.request.Request(asset_url, headers=headers) - + with urllib.request.urlopen(request) as response: - with open(target_path, 'wb') as f: + with open(target_path, "wb") as f: f.write(response.read()) - + # Verify checksum print("🔍 Verifying checksum...") sha256_hash = hashlib.sha256() @@ -265,17 +269,17 @@ def _download_missing_models(missing_files, models_dir, project_root): for chunk in iter(lambda: f.read(4096), b""): sha256_hash.update(chunk) calculated_sha256 = sha256_hash.hexdigest() - + if calculated_sha256 != expected_sha256: print(f"❌ Checksum mismatch for {filename}") print(f"Expected: {expected_sha256}") print(f"Got: {calculated_sha256}") target_path.unlink() continue - + print(f"✅ Successfully downloaded {filename}") success_count += 1 - + except urllib.error.HTTPError as e: if e.code == 401: print(f"❌ Authentication failed for {filename}. Need GitHub token.") @@ -285,26 +289,26 @@ def _download_missing_models(missing_files, models_dir, project_root): print(f"❌ HTTP error {e.code} downloading {filename}: {e.reason}") except Exception as e: print(f"❌ Failed to download {filename}: {e}") - + return success_count == len(missing_files) def print_model_status(project_root: Optional[Path] = None) -> bool: """ Print the status of model files and return whether all are present. - + Args: project_root: Path to the project root directory - + Returns: True if all models are present, False otherwise """ all_present, missing_files, present_files = check_model_files(project_root) - - print("\n" + "="*60) + + print("\n" + "=" * 60) print("ROBIN MODEL STATUS CHECK") - print("="*60) - + print("=" * 60) + if all_present: print("✅ All required model files are present:") for filename in present_files: @@ -315,35 +319,37 @@ def print_model_status(project_root: Optional[Path] = None) -> bool: print("❌ Missing required model files:") for filename in missing_files: print(f" ✗ {filename}") - + if present_files: print("\n✅ Present model files:") for filename in present_files: print(f" ✓ {filename}") - + models_dir = get_models_directory(project_root) print(f"\n📁 Models directory: {models_dir}") print(f"📁 Current working directory: {Path.cwd()}") - + # Ask user if they want to download - print("\n" + "="*60) + print("\n" + "=" * 60) print("AUTOMATIC DOWNLOAD OPTION") - print("="*60) + print("=" * 60) print("Would you like to automatically download the missing model files?") print("This uses the same logic as: robin utils update-models") - + try: response = input("\nDownload missing models? [Y/n]: ").strip().lower() - if response in ['', 'y', 'yes']: - if _download_missing_models(missing_files, models_dir, project_root or Path.cwd()): + if response in ["", "y", "yes"]: + if _download_missing_models( + missing_files, models_dir, project_root or Path.cwd() + ): print("\n🎉 All models downloaded successfully!") print("ROBIN is now ready to run.") return True else: print("\n⚠️ Some downloads failed. Trying alternative method...") - print("\n" + "="*60) + print("\n" + "=" * 60) print("FALLBACK TO API METHOD") - print("="*60) + print("=" * 60) print("The automatic download failed. You can try the API method:") print() print("1. Set a GitHub token:") @@ -357,18 +363,20 @@ def print_model_status(project_root: Optional[Path] = None) -> bool: print(" robin utils update-clinvar") print() print("After downloading, run ROBIN again.") - print("="*60) + print("=" * 60) return False else: - print("\n" + "="*60) + print("\n" + "=" * 60) print("MANUAL DOWNLOAD INSTRUCTIONS") - print("="*60) + print("=" * 60) print("To download the missing model files manually:") print() print(" robin utils update-models") print(" robin utils update-clinvar") print() - print("For private GitHub assets, set GITHUB_TOKEN before update-models:") + print( + "For private GitHub assets, set GITHUB_TOKEN before update-models:" + ) print(" export GITHUB_TOKEN=your_github_token") print(" robin utils update-models") print() @@ -378,7 +386,7 @@ def print_model_status(project_root: Optional[Path] = None) -> bool: print("Create a token at: https://github.com/settings/tokens") print() print("After downloading, you can run ROBIN normally.") - print("="*60) + print("=" * 60) return False except KeyboardInterrupt: print("\n\n⚠️ Download cancelled by user.") @@ -388,9 +396,9 @@ def print_model_status(project_root: Optional[Path] = None) -> bool: def validate_models_or_exit(project_root: Optional[Path] = None) -> None: """ Check for required models and exit with helpful message if any are missing. - + This function is designed to be called at application startup. - + Args: project_root: Path to the project root directory """ @@ -403,26 +411,28 @@ def validate_models_or_exit(project_root: Optional[Path] = None) -> None: sys.exit(1) -def get_model_path(filename: str, project_root: Optional[Path] = None) -> Optional[Path]: +def get_model_path( + filename: str, project_root: Optional[Path] = None +) -> Optional[Path]: """ Get the full path to a model file if it exists. - + Args: filename: Name of the model file project_root: Path to the project root directory - + Returns: Path to the model file if it exists, None otherwise """ models_dir = get_models_directory(project_root) model_path = models_dir / filename - + if model_path.exists() and model_path.stat().st_size > 0: return model_path - + return None if __name__ == "__main__": # Allow running this script directly for testing - validate_models_or_exit() \ No newline at end of file + validate_models_or_exit() diff --git a/src/robin/utils/model_updater.py b/src/robin/utils/model_updater.py index 8af12490..1b1b20f8 100644 --- a/src/robin/utils/model_updater.py +++ b/src/robin/utils/model_updater.py @@ -7,7 +7,7 @@ import urllib.error import urllib.request from pathlib import Path -from typing import Optional, Tuple, List, Dict, Any +from typing import Any, Dict, List, Optional, Tuple # https://github.com/{owner}/{repo}/releases/download/{tag}/{filename} _GITHUB_RELEASE_DOWNLOAD_RE = re.compile( @@ -28,7 +28,9 @@ def _default_repo_root_guess() -> Optional[Path]: return None -def _resolve_assets_manifest_path(manifest_path: Optional[str] = None) -> Optional[Path]: +def _resolve_assets_manifest_path( + manifest_path: Optional[str] = None, +) -> Optional[Path]: """ Resolve assets manifest path. @@ -83,7 +85,11 @@ def _load_assets_manifest(manifest_path: Optional[str]) -> Dict[str, Any]: # Packaged fallback import importlib.resources as importlib_resources - txt = importlib_resources.files("robin.resources").joinpath("assets.json").read_text(encoding="utf-8") + txt = ( + importlib_resources.files("robin.resources") + .joinpath("assets.json") + .read_text(encoding="utf-8") + ) return json.loads(txt) @@ -122,7 +128,9 @@ def _stream_http_response(resp, target_path: Path, *, label: str) -> None: downloaded = 0 if click is not None and total and total > 0: - with click.progressbar(length=total, label=label, show_eta=True, show_percent=True) as bar: + with click.progressbar( + length=total, label=label, show_eta=True, show_percent=True + ) as bar: with target_path.open("wb") as f: while True: chunk = resp.read(chunk_size) @@ -231,7 +239,13 @@ def _download_manifest_asset( _stream_http_response(resp, target_path, label=label) -def _download(url: str, target_path: Path, github_token: Optional[str], *, label: str = "Downloading") -> None: +def _download( + url: str, + target_path: Path, + github_token: Optional[str], + *, + label: str = "Downloading", +) -> None: _download_manifest_asset(url, target_path, github_token, label=label) @@ -269,7 +283,9 @@ def update_models( if mp: messages.append(f"Using assets manifest: {mp}") else: - messages.append("Using bundled package assets manifest (robin.resources/assets.json).") + messages.append( + "Using bundled package assets manifest (robin.resources/assets.json)." + ) models_dir = Path(models_dir).expanduser().resolve() models_dir.mkdir(parents=True, exist_ok=True) @@ -284,7 +300,9 @@ def update_models( url = str(asset_info.get("url") or "") expected_sha256 = str(asset_info.get("sha256") or "") if not url or not expected_sha256: - messages.append(f"Manifest entry incomplete for {asset_key} (missing url/sha256).") + messages.append( + f"Manifest entry incomplete for {asset_key} (missing url/sha256)." + ) return False, messages target_path = models_dir / filename @@ -299,7 +317,9 @@ def update_models( target_path.unlink() except Exception: pass - messages.append(f"Checksum mismatch for {filename} (expected {expected_sha256}, got {got}).") + messages.append( + f"Checksum mismatch for {filename} (expected {expected_sha256}, got {got})." + ) return False, messages messages.append(f"Downloaded {filename}.") except Exception as e: @@ -316,4 +336,3 @@ def update_models( return False, messages return True, messages - diff --git a/src/robin/utils/sequencing_files.py b/src/robin/utils/sequencing_files.py index c42284c4..99b12f0d 100644 --- a/src/robin/utils/sequencing_files.py +++ b/src/robin/utils/sequencing_files.py @@ -87,7 +87,9 @@ def panel_source_available(panel: str) -> bool: """True if ``{panel}_panel_source.bed`` exists in ``robin.resources``.""" import importlib.resources as importlib_resources - res = importlib_resources.files("robin.resources").joinpath(panel_source_filename(panel)) + res = importlib_resources.files("robin.resources").joinpath( + panel_source_filename(panel) + ) try: return res.is_file() except Exception: @@ -133,7 +135,9 @@ def reference_looks_like_url(reference: str) -> bool: return ref.startswith("http://") or ref.startswith("https://") -def _download_url(url: str, target_path: Path, *, label: str = "Downloading reference") -> None: +def _download_url( + url: str, target_path: Path, *, label: str = "Downloading reference" +) -> None: req = urllib.request.Request( url, headers={"User-Agent": "robin-sequencing-files/1.0"}, @@ -156,7 +160,9 @@ def _download_url(url: str, target_path: Path, *, label: str = "Downloading refe downloaded = 0 if click is not None and total and total > 0: - with click.progressbar(length=total, label=label, show_eta=True, show_percent=True) as bar: + with click.progressbar( + length=total, label=label, show_eta=True, show_percent=True + ) as bar: with target_path.open("wb") as f: while True: chunk = resp.read(chunk_size) diff --git a/src/robin/utils/tucan_manager.py b/src/robin/utils/tucan_manager.py index 0f170980..e1b2f0c7 100644 --- a/src/robin/utils/tucan_manager.py +++ b/src/robin/utils/tucan_manager.py @@ -60,7 +60,9 @@ def get_default_num_cpgs() -> int: try: value = int(raw) except ValueError as exc: - raise ValueError(f"Invalid {ENV_NUM_CPGS}={raw!r}; expected an integer") from exc + raise ValueError( + f"Invalid {ENV_NUM_CPGS}={raw!r}; expected an integer" + ) from exc if value <= 0: raise ValueError(f"{ENV_NUM_CPGS} must be positive, got {value}") return value @@ -251,7 +253,9 @@ def resolve_model_zip( model_dir = cache / MODEL_DIR_NAME if not _dir_has_model_assets(model_dir): _download_hf_model(model_dir) - _create_model_zip(model_dir, path if path.parent == cache else cache / MODEL_ZIP_NAME) + _create_model_zip( + model_dir, path if path.parent == cache else cache / MODEL_ZIP_NAME + ) # If caller requested a custom path outside cache, copy/create there. final = path if path.suffix.lower() == ".zip" else cache / MODEL_ZIP_NAME if final != (cache / MODEL_ZIP_NAME) and _file_ok(cache / MODEL_ZIP_NAME): diff --git a/src/robin/watcher.py b/src/robin/watcher.py index 960b3620..b3330934 100644 --- a/src/robin/watcher.py +++ b/src/robin/watcher.py @@ -6,9 +6,9 @@ from pathlib import Path from typing import List, Optional -from watchdog.events import FileSystemEventHandler, FileSystemEvent -from watchdog.observers import Observer from tqdm import tqdm +from watchdog.events import FileSystemEvent, FileSystemEventHandler +from watchdog.observers import Observer class FileChangeHandler(FileSystemEventHandler): @@ -188,7 +188,7 @@ def process_existing_files(self) -> None: print(f"Ignore patterns: {self.ignore_patterns}") existing_files = [] - + # Find all existing files that match patterns for pattern in self.patterns: if pattern == "*": diff --git a/src/robin/workflow_config.py b/src/robin/workflow_config.py index b4245e1c..79ee3bb9 100644 --- a/src/robin/workflow_config.py +++ b/src/robin/workflow_config.py @@ -10,7 +10,11 @@ import click from click.core import ParameterSource -from robin.minknow.toml_config import MinKnowWorkflowConfig, extract_minknow_config, load_minknow_toml +from robin.minknow.toml_config import ( + MinKnowWorkflowConfig, + extract_minknow_config, + load_minknow_toml, +) WORKFLOW_REQUIRED_KEYS = ("path", "workflow", "center", "target_panel") @@ -98,7 +102,9 @@ def load_workflow_toml(path: Path) -> dict[str, Any]: raise click.BadParameter(f"Invalid TOML in {path}: {exc}") from exc if not isinstance(raw, dict): - raise click.BadParameter(f"TOML config must be a table at the top level: {path}") + raise click.BadParameter( + f"TOML config must be a table at the top level: {path}" + ) return _normalize_config(raw) @@ -357,15 +363,10 @@ def _parameter_from_commandline(ctx: click.Context, param_name: str) -> bool: def _validate_required_params(params: Mapping[str, Any]) -> None: - missing = [ - key - for key in WORKFLOW_REQUIRED_KEYS - if params.get(key) in (None, "") - ] + missing = [key for key in WORKFLOW_REQUIRED_KEYS if params.get(key) in (None, "")] if missing: readable = ", ".join( - key.replace("_", "-") if key != "path" else "PATH" - for key in missing + key.replace("_", "-") if key != "path" else "PATH" for key in missing ) raise click.BadParameter( f"Missing required workflow setting(s): {readable}. " diff --git a/src/robin/workflow_hooks.py b/src/robin/workflow_hooks.py index 20a2e523..596ac598 100644 --- a/src/robin/workflow_hooks.py +++ b/src/robin/workflow_hooks.py @@ -7,9 +7,9 @@ import logging import time -from typing import Any, List, Dict +from typing import Any, Dict, List -from .gui_launcher import send_gui_update, UpdateType +from .gui_launcher import UpdateType, send_gui_update def install_workflow_hooks( @@ -137,7 +137,7 @@ def _install_manager_hooks(workflow_runner: Any) -> None: def _start_polling_updates(manager: Any, interval_seconds: float = 15.0) -> None: """Start a background thread that polls `manager.get_stats()` and sends GUI updates. - + Reduced default interval from 10s to 15s to reduce update frequency and prevent queue buildup. """ import threading diff --git a/src/robin/workflow_ray.py b/src/robin/workflow_ray.py index c2aaea8d..caec6f28 100644 --- a/src/robin/workflow_ray.py +++ b/src/robin/workflow_ray.py @@ -35,23 +35,23 @@ "ignore", message="The figure layout has changed to tight", category=UserWarning ) -import os +import argparse +import asyncio import copy +import inspect +import itertools +import os +import pickle import shutil +import tempfile import threading import time -import tempfile -import pickle import uuid -import asyncio -import argparse from collections import deque +from contextlib import nullcontext from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional, Set, Tuple, Callable -import inspect from pathlib import Path -import itertools -from contextlib import nullcontext +from typing import Any, Callable, Dict, List, Optional, Set, Tuple import ray from tqdm import tqdm @@ -59,6 +59,7 @@ from watchdog.observers import Observer try: + from rich.console import Console from rich.progress import ( BarColumn, Progress, @@ -68,7 +69,6 @@ TimeElapsedColumn, TimeRemainingColumn, ) - from rich.console import Console _RICH_AVAILABLE = True except Exception: @@ -125,12 +125,11 @@ def _warn_if_ray_temp_disk_is_full() -> None: # Optional GUI hook integration try: - from robin.gui_launcher import ( - send_gui_update as _gui_send_update, - UpdateType as _GUIUpdateType, - launch_gui as _gui_launch, - ) + from robin.gui_launcher import UpdateType as _GUIUpdateType + from robin.gui_launcher import launch_gui as _gui_launch + from robin.gui_launcher import send_gui_update as _gui_send_update except Exception: + def _gui_send_update(*args, **kwargs): return None @@ -162,10 +161,12 @@ def _gui_launch(*args, **kwargs): try: from robin.analysis.cnv_analysis import ( - cnv_handler as _cnv_handler, + cleanup_sample_cache_on_completion, clear_sample_cache, + ) + from robin.analysis.cnv_analysis import cnv_handler as _cnv_handler + from robin.analysis.cnv_analysis import ( get_sample_cache_stats, - cleanup_sample_cache_on_completion, ) except Exception as exc: _cnv_handler = None @@ -175,12 +176,14 @@ def _gui_launch(*args, **kwargs): _HANDLER_IMPORT_ERRORS["cnv"] = str(exc) try: + from robin.analysis.target_analysis import igv_bam_handler as _igv_bam_handler from robin.analysis.target_analysis import ( - target_handler as _target_handler, - igv_bam_handler as _igv_bam_handler, snp_analysis_handler as _snp_analysis_handler, + ) + from robin.analysis.target_analysis import ( target_bam_finalize_handler as _target_bam_finalize_handler, ) + from robin.analysis.target_analysis import target_handler as _target_handler except Exception: _target_handler = None _igv_bam_handler = None @@ -194,9 +197,7 @@ def _gui_launch(*args, **kwargs): _HANDLER_IMPORT_ERRORS["fusion"] = str(exc) try: - from robin.analysis.sturgeon_analysis import ( - sturgeon_handler as _sturgeon_handler, - ) + from robin.analysis.sturgeon_analysis import sturgeon_handler as _sturgeon_handler except Exception: _sturgeon_handler = None @@ -206,9 +207,7 @@ def _gui_launch(*args, **kwargs): _nanodx_handler = None try: - from robin.analysis.nanodx_analysis import ( - pannanodx_handler as _pannanodx_handler, - ) + from robin.analysis.nanodx_analysis import pannanodx_handler as _pannanodx_handler except Exception: _pannanodx_handler = None @@ -242,10 +241,8 @@ def _gui_launch(*args, **kwargs): # Optional logging helper try: - from robin.logging_config import ( - get_job_logger as _get_job_logger, - configure_logging as _configure_logging, - ) + from robin.logging_config import configure_logging as _configure_logging + from robin.logging_config import get_job_logger as _get_job_logger except Exception: def _get_job_logger(job_id: str, job_type: str, filepath: str): @@ -502,14 +499,30 @@ def get_filepaths(self) -> List[str]: "fusion": {"fusion"}, "itd": {"itd"}, "classification": {"sturgeon", "nanodx", "pannanodx"}, - "slow": {"random_forest", "marlin", "lamprey", "tucan", "igv_bam", "snp_analysis", "target_bam_finalize"}, + "slow": { + "random_forest", + "marlin", + "lamprey", + "tucan", + "igv_bam", + "snp_analysis", + "target_bam_finalize", + }, } TRIGGERS: Dict[str, List[str]] = { # preprocessing -> analyses "preprocessing": ["bed_conversion", "mgmt", "cnv", "target", "fusion", "itd"], # bed_conversion -> classifiers - "bed_conversion": ["sturgeon", "nanodx", "pannanodx", "random_forest", "marlin", "lamprey", "tucan"], + "bed_conversion": [ + "sturgeon", + "nanodx", + "pannanodx", + "random_forest", + "marlin", + "lamprey", + "tucan", + ], # Build IGV-ready BAM after target analysis "target": ["igv_bam"], } @@ -988,9 +1001,13 @@ def _impl(job: Job) -> WorkflowContext: if MemoryManager is not None and not DISABLE_MEMORY_MANAGER: try: # Configure memory management based on job type - gc_every = 10 if job_type in {"mgmt", "cnv", "target", "fusion", "itd"} else 25 + gc_every = ( + 10 if job_type in {"mgmt", "cnv", "target", "fusion", "itd"} else 25 + ) rss_trigger = ( - 1024 if job_type in {"mgmt", "cnv", "target", "fusion", "itd"} else 1024 + 1024 + if job_type in {"mgmt", "cnv", "target", "fusion", "itd"} + else 1024 ) memory_manager = MemoryManager( gc_every=gc_every, @@ -1108,7 +1125,12 @@ def _impl(job: Job) -> WorkflowContext: py_handler(job, reference=reference, target_panel=target_panel) elif accepts_reference and reference and job_type in ["mgmt", "target"]: py_handler(job, reference=reference) - elif accepts_target_panel and job_type in ["fusion", "target", "cnv", "itd"]: + elif accepts_target_panel and job_type in [ + "fusion", + "target", + "cnv", + "itd", + ]: py_handler(job, target_panel=target_panel) else: py_handler(job) @@ -1282,15 +1304,23 @@ def __init__(self, job_type: str, remote_func, resource_options: Dict[str, Any]) if MemoryManager is not None and not DISABLE_MEMORY_MANAGER: try: # Configure memory management based on job type - gc_every = 25 if job_type in {"mgmt", "cnv", "target", "fusion", "itd"} else 50 + gc_every = ( + 25 if job_type in {"mgmt", "cnv", "target", "fusion", "itd"} else 50 + ) rss_trigger = ( - 1024 if job_type in {"mgmt", "cnv", "target", "fusion", "itd"} else 2048 + 1024 + if job_type in {"mgmt", "cnv", "target", "fusion", "itd"} + else 2048 ) restart_every = ( - 5000 if job_type in {"mgmt", "cnv", "target", "fusion", "itd"} else 10000 + 5000 + if job_type in {"mgmt", "cnv", "target", "fusion", "itd"} + else 10000 ) restart_rss_trigger = ( - 2048 if job_type in {"mgmt", "cnv", "target", "fusion", "itd"} else 4096 + 2048 + if job_type in {"mgmt", "cnv", "target", "fusion", "itd"} + else 4096 ) self.memory_manager = MemoryManager( gc_every=gc_every, @@ -2035,15 +2065,9 @@ async def ping(self): "rf": [ "random_forest" ], # Random forest needs its own actor (slow/blocking) - "marlin": [ - "marlin" - ], # MARLIN (TensorFlow) needs its own actor - "lamprey": [ - "lamprey" - ], # Lamprey ONNX model is large; dedicated actor - "tucan": [ - "tucan" - ], # Tucan PyTorch ensemble; dedicated actor + "marlin": ["marlin"], # MARLIN (TensorFlow) needs its own actor + "lamprey": ["lamprey"], # Lamprey ONNX model is large; dedicated actor + "tucan": ["tucan"], # Tucan PyTorch ensemble; dedicated actor "slow": [ "igv_bam", "snp_analysis", @@ -5488,6 +5512,7 @@ def _handle(self, fp: str): # enqueue and flush under rate limiter self._pending_jobs.extend(jobs) self._flush_if_needed() + def on_created(self, event): if not event.is_directory: self._handle(event.src_path) @@ -5674,7 +5699,9 @@ def _sample_dir_has_analysis_artifacts_local(sample_dir: Path) -> bool: # Import lazily so workflow can run even if analysis deps aren't present. try: - from robin.analysis.bam_preprocessor import _extract_sample_id_from_bam # type: ignore + from robin.analysis.bam_preprocessor import ( + _extract_sample_id_from_bam, # type: ignore + ) except Exception: return [], [], None @@ -6293,7 +6320,9 @@ def _cat_of_local(jt: str) -> str: priority=1, ) - for notification in await coord.drain_gui_notifications.remote(): + for ( + notification + ) in await coord.drain_gui_notifications.remote(): _gui_send_update( _GUIUpdateType.WARNING_NOTIFICATION, notification, diff --git a/src/robin/workflow_simple.py b/src/robin/workflow_simple.py index 8ee63bdb..5a3ff31f 100644 --- a/src/robin/workflow_simple.py +++ b/src/robin/workflow_simple.py @@ -1,17 +1,17 @@ """Simple workflow management system for robin using threading.""" +import itertools import os -import time -import threading import queue -import itertools +import threading +import time from dataclasses import dataclass, field from pathlib import Path -from typing import Callable, Dict, List, Optional, Any, Set +from typing import Any, Callable, Dict, List, Optional, Set +from tqdm import tqdm from watchdog.events import FileSystemEventHandler from watchdog.observers import Observer -from tqdm import tqdm try: from rich.progress import ( @@ -27,12 +27,12 @@ _RICH_AVAILABLE = True except Exception: _RICH_AVAILABLE = False -from robin.logging_config import get_job_logger from robin.analysis.target_analysis import ( igv_bam_handler, snp_analysis_handler, target_bam_finalize_handler, ) +from robin.logging_config import get_job_logger # ---------- Batch Configuration ---------- # See workflow_ray.py for the full description. Two timeouts are honoured: @@ -48,22 +48,46 @@ # ROBIN_BATCH_TIMEOUT_BUSY_S_ e.g. ROBIN_BATCH_TIMEOUT_BUSY_S_CNV=45 BATCH_CONFIG: Dict[str, Dict[str, Any]] = { # Preprocessing should NOT be batched - each file needs individual sample ID extraction - "preprocessing": {"max_batch_size": 1, "timeout_seconds": 0, "timeout_seconds_busy": 0}, - "bed_conversion": {"max_batch_size": 20, "timeout_seconds": 2, "timeout_seconds_busy": 30}, + "preprocessing": { + "max_batch_size": 1, + "timeout_seconds": 0, + "timeout_seconds_busy": 0, + }, + "bed_conversion": { + "max_batch_size": 20, + "timeout_seconds": 2, + "timeout_seconds_busy": 30, + }, "mgmt": {"max_batch_size": 20, "timeout_seconds": 2, "timeout_seconds_busy": 30}, "cnv": {"max_batch_size": 50, "timeout_seconds": 2, "timeout_seconds_busy": 30}, "target": {"max_batch_size": 20, "timeout_seconds": 2, "timeout_seconds_busy": 30}, "fusion": {"max_batch_size": 20, "timeout_seconds": 2, "timeout_seconds_busy": 30}, "itd": {"max_batch_size": 20, "timeout_seconds": 2, "timeout_seconds_busy": 30}, - "sturgeon": {"max_batch_size": 20, "timeout_seconds": 2, "timeout_seconds_busy": 30}, + "sturgeon": { + "max_batch_size": 20, + "timeout_seconds": 2, + "timeout_seconds_busy": 30, + }, "nanodx": {"max_batch_size": 20, "timeout_seconds": 2, "timeout_seconds_busy": 30}, - "pannanodx": {"max_batch_size": 20, "timeout_seconds": 2, "timeout_seconds_busy": 30}, - "random_forest": {"max_batch_size": 20, "timeout_seconds": 2, "timeout_seconds_busy": 30}, + "pannanodx": { + "max_batch_size": 20, + "timeout_seconds": 2, + "timeout_seconds_busy": 30, + }, + "random_forest": { + "max_batch_size": 20, + "timeout_seconds": 2, + "timeout_seconds_busy": 30, + }, "marlin": {"max_batch_size": 20, "timeout_seconds": 2, "timeout_seconds_busy": 30}, "lamprey": {"max_batch_size": 20, "timeout_seconds": 2, "timeout_seconds_busy": 30}, "tucan": {"max_batch_size": 20, "timeout_seconds": 2, "timeout_seconds_busy": 30}, "igv_bam": {"max_batch_size": 20, "timeout_seconds": 2, "timeout_seconds_busy": 30}, - "snp_analysis": {"max_batch_size": 20, "timeout_seconds": 2, "timeout_seconds_busy": 30}, + "snp_analysis": { + "max_batch_size": 20, + "timeout_seconds": 2, + "timeout_seconds_busy": 30, + }, } @@ -108,7 +132,7 @@ class WorkflowContext: results: dict = field(default_factory=dict) history: list = field(default_factory=list) errors: list = field(default_factory=list) - + # NEW: Simple batch metadata batch_id: Optional[str] = None batch_index: Optional[int] = None @@ -144,14 +168,14 @@ def get_sample_id(self) -> str: # First try to get from bam_metadata (for backward compatibility) bam_metadata = self.metadata.get("bam_metadata", {}) sample_id = bam_metadata.get("sample_id", "unknown") - + # If not found or "unknown", try to get from preprocessing results if sample_id == "unknown": preprocessing_result = self.results.get("preprocessing", {}) sample_id = preprocessing_result.get("sample_id", "unknown") - + return sample_id - + def set_batch_info(self, batch_id: str, batch_index: int) -> None: """Set batch information for this context""" self.batch_id = batch_id @@ -311,13 +335,13 @@ class BatchedJob: contexts: List[WorkflowContext] # Multiple contexts for batched processing batch_id: str sample_id: str - + def get_sample_id(self) -> str: return self.sample_id - + def get_file_count(self) -> int: return len(self.contexts) - + def get_filepaths(self) -> List[str]: return [ctx.filepath for ctx in self.contexts] @@ -357,79 +381,91 @@ def add_job(self, job: Job) -> List[BatchedJob]: """Add a job and return any completed batches""" sample_id = job.get_sample_id() job_type = job.job_type - + with self.lock: # Initialize if needed if sample_id not in self.pending_jobs: self.pending_jobs[sample_id] = {} self.last_job_time[sample_id] = {} - + if job_type not in self.pending_jobs[sample_id]: self.pending_jobs[sample_id][job_type] = [] self.last_job_time[sample_id][job_type] = time.time() - + # Add job to pending list self.pending_jobs[sample_id][job_type].append(job) self.last_job_time[sample_id][job_type] = time.time() - + # Check for completed batches return self._check_and_create_batches(sample_id, job_type) - - def _check_and_create_batches(self, sample_id: str, job_type: str) -> List[BatchedJob]: + + def _check_and_create_batches( + self, sample_id: str, job_type: str + ) -> List[BatchedJob]: """Check if we should create batches for a sample/job_type combination. Jobs with force_individual_batch (e.g. large BAMs) are emitted as single-file batches. """ - config = BATCH_CONFIG.get(job_type, {"max_batch_size": 20, "timeout_seconds": 10}) + config = BATCH_CONFIG.get( + job_type, {"max_batch_size": 20, "timeout_seconds": 10} + ) max_batch_size = config["max_batch_size"] - + pending = self.pending_jobs[sample_id][job_type] batches = [] - + # Emit single-file batches for jobs marked force_individual_batch (e.g. large BAMs) - individual = [j for j in pending if (j.context.metadata or {}).get("force_individual_batch")] - rest = [j for j in pending if not (j.context.metadata or {}).get("force_individual_batch")] + individual = [ + j + for j in pending + if (j.context.metadata or {}).get("force_individual_batch") + ] + rest = [ + j + for j in pending + if not (j.context.metadata or {}).get("force_individual_batch") + ] for job in individual: batches.append(self._create_batched_job([job], sample_id, job_type)) - + # Create batches of max_batch_size from the rest while len(rest) >= max_batch_size: batch_jobs = rest[:max_batch_size] rest = rest[max_batch_size:] batched_job = self._create_batched_job(batch_jobs, sample_id, job_type) batches.append(batched_job) - + # Update pending list (only unbatched jobs remain) self.pending_jobs[sample_id][job_type] = rest - + return batches - + def check_timeouts(self) -> List[BatchedJob]: """Check for timed-out batches and return them""" current_time = time.time() timed_out_batches = [] - + with self.lock: for sample_id in list(self.pending_jobs.keys()): for job_type in list(self.pending_jobs[sample_id].keys()): jobs = self.pending_jobs[sample_id][job_type] if not jobs: continue - + timeout_seconds = self._effective_timeout(job_type) last_time = self.last_job_time[sample_id][job_type] - + if (current_time - last_time) >= timeout_seconds: # Create batch with remaining jobs batch = self._create_batched_job(jobs, sample_id, job_type) timed_out_batches.append(batch) - + # Clear the pending jobs self.pending_jobs[sample_id][job_type] = [] del self.last_job_time[sample_id][job_type] - + # Clean up empty entries self._cleanup_empty_entries() - + return timed_out_batches def force_flush_type(self, job_type: str) -> List[BatchedJob]: @@ -453,26 +489,32 @@ def force_flush_type(self, job_type: str) -> List[BatchedJob]: del self.last_job_time[sample_id][job_type] self._cleanup_empty_entries() return flushed - - def _create_batched_job(self, jobs: List[Job], sample_id: str, job_type: str) -> BatchedJob: + + def _create_batched_job( + self, jobs: List[Job], sample_id: str, job_type: str + ) -> BatchedJob: """Create a batched job from a list of individual jobs""" if not jobs: raise ValueError("Cannot create batched job from empty job list") - + # Validate all jobs have same sample_id and job_type for job in jobs: if job.get_sample_id() != sample_id: - raise ValueError(f"Mixed sample IDs in batch: {sample_id} vs {job.get_sample_id()}") + raise ValueError( + f"Mixed sample IDs in batch: {sample_id} vs {job.get_sample_id()}" + ) if job.job_type != job_type: - raise ValueError(f"Mixed job types in batch: {job_type} vs {job.job_type}") - + raise ValueError( + f"Mixed job types in batch: {job_type} vs {job.job_type}" + ) + # Use the first job as template template_job = jobs[0] batch_id = f"{sample_id}_{job_type}_{int(time.time() * 1000)}" - + # Extract contexts from all jobs contexts = [job.context for job in jobs] - + return BatchedJob( job_id=next(_job_id_counter), job_type=job_type, @@ -481,9 +523,9 @@ def _create_batched_job(self, jobs: List[Job], sample_id: str, job_type: str) -> step=template_job.step, contexts=contexts, batch_id=batch_id, - sample_id=sample_id + sample_id=sample_id, ) - + def _cleanup_empty_entries(self): """Remove empty entries from pending jobs and timestamps""" # Remove empty job type entries @@ -493,7 +535,7 @@ def _cleanup_empty_entries(self): del self.pending_jobs[sample_id][job_type] if job_type in self.last_job_time[sample_id]: del self.last_job_time[sample_id][job_type] - + # Remove empty sample entries if not self.pending_jobs[sample_id]: del self.pending_jobs[sample_id] @@ -645,15 +687,13 @@ def __init__( # 'sample_id', 'active_jobs', 'total_jobs', 'completed_jobs', 'failed_jobs', 'job_types' (set), 'last_seen' # } self.samples_by_id: Dict[str, Dict[str, Any]] = {} - + # Batching support. The batcher uses adaptive timeouts driven by # the live per-type inflight count from self.active_jobs. When a # type is idle, fire batches quickly; when busy, accumulate longer. self.enable_batching = enable_batching self.sample_batcher = ( - SampleJobBatcher( - inflight_callback=self._inflight_count_for_type - ) + SampleJobBatcher(inflight_callback=self._inflight_count_for_type) if enable_batching else None ) @@ -853,13 +893,15 @@ def enqueue_jobs(self, jobs: List[Job]) -> None: # Process jobs through batcher if batching is enabled if self.enable_batching and self.sample_batcher: # Separate preprocessing jobs from other jobs - preprocessing_jobs = [job for job in jobs if job.job_type == "preprocessing"] + preprocessing_jobs = [ + job for job in jobs if job.job_type == "preprocessing" + ] other_jobs = [job for job in jobs if job.job_type != "preprocessing"] - + # Submit preprocessing jobs directly (no batching) if preprocessing_jobs: self._enqueue_jobs_internal(preprocessing_jobs) - + # Process other jobs through batcher for job in other_jobs: batches = self.sample_batcher.add_job(job) @@ -869,7 +911,7 @@ def enqueue_jobs(self, jobs: List[Job]) -> None: self._enqueue_jobs_internal(regular_jobs) else: self._enqueue_jobs_internal(jobs) - + def _enqueue_jobs_internal(self, jobs: List[Job]) -> None: """Internal job enqueue logic""" jobs_to_enqueue = [] @@ -1053,17 +1095,21 @@ def worker( logger.info( f"Job {job.job_type} completed, triggering {len(triggered_jobs)} parallel jobs: {[j.job_type for j in triggered_jobs]}" ) - + # For CNV jobs, use batching if enabled if self.enable_batching and self.sample_batcher: # Check if any of the triggered jobs are CNV jobs - cnv_jobs = [j for j in triggered_jobs if j.job_type == "cnv"] - other_jobs = [j for j in triggered_jobs if j.job_type != "cnv"] - + cnv_jobs = [ + j for j in triggered_jobs if j.job_type == "cnv" + ] + other_jobs = [ + j for j in triggered_jobs if j.job_type != "cnv" + ] + # Submit non-CNV jobs immediately if other_jobs: self.enqueue_jobs(other_jobs) - + # For CNV jobs, add them to the batcher for cnv_job in cnv_jobs: batches = self.sample_batcher.add_job(cnv_job) @@ -1252,7 +1298,7 @@ def run(self) -> None: slow_worker.daemon = True self.slow_workers.append(slow_worker) slow_worker.start() - + # Start batch timeout thread if batching is enabled if self.enable_batching and self.sample_batcher: self.batch_timeout_thread = threading.Thread( @@ -1374,21 +1420,21 @@ def shutdown(self, timeout: float = 30.0) -> bool: True if all workers stopped gracefully, False if timeout occurred """ return self.stop(timeout) - + def _batch_timeout_loop(self): """Periodically check for timed-out batches""" while self.running: try: # Check for timed-out batches timed_out_batches = self.sample_batcher.check_timeouts() - + # Enqueue timed-out batches if timed_out_batches: regular_jobs = self._convert_batched_jobs(timed_out_batches) self._enqueue_jobs_internal(regular_jobs) - + time.sleep(1.0) # Check every second - + except Exception: time.sleep(1.0) @@ -1399,7 +1445,8 @@ def _inflight_count_for_type(self, job_type: str) -> int: timeouts.""" try: return sum( - 1 for info in self.active_jobs.values() + 1 + for info in self.active_jobs.values() if info.get("job_type") == job_type ) except Exception: @@ -1501,9 +1548,7 @@ def _coalesce_from_queue( if not self._job_is_coalescable_batch(entry): i += 1 continue - other_bjob: BatchedJob = entry.context.metadata.get( - "_batched_job" - ) + other_bjob: BatchedJob = entry.context.metadata.get("_batched_job") take = min(room, len(other_bjob.contexts)) if take <= 0: i += 1 @@ -1518,9 +1563,8 @@ def _coalesce_from_queue( try: if job_queue.unfinished_tasks > 0: job_queue.unfinished_tasks -= 1 - if ( - job_queue.unfinished_tasks == 0 - and hasattr(job_queue, "all_tasks_done") + if job_queue.unfinished_tasks == 0 and hasattr( + job_queue, "all_tasks_done" ): job_queue.all_tasks_done.notify_all() except Exception: @@ -1545,7 +1589,7 @@ def _coalesce_from_queue( except Exception: pass return absorbed - + def _convert_batched_jobs(self, batched_jobs: List[BatchedJob]) -> List[Job]: """Convert BatchedJob objects to regular Job objects for processing""" regular_jobs = [] @@ -1557,16 +1601,19 @@ def _convert_batched_jobs(self, batched_jobs: List[BatchedJob]) -> List[Job]: context=batched_job.contexts[0], # Use first context as primary origin=batched_job.origin, workflow=batched_job.workflow, - step=batched_job.step + step=batched_job.step, ) # Store batch information in metadata regular_job.context.metadata["_batched_job"] = batched_job regular_jobs.append(regular_job) - + return regular_jobs - - def register_batched_handler(self, job_type: str, handler: Callable[[BatchedJob], None]) -> None: + + def register_batched_handler( + self, job_type: str, handler: Callable[[BatchedJob], None] + ) -> None: """Register a handler that can process batched jobs""" + # Wrap the handler to extract BatchedJob from regular Job def wrapped_handler(job: Job) -> None: batched_job = job.context.metadata.get("_batched_job") @@ -1583,10 +1630,10 @@ def wrapped_handler(job: Job) -> None: step=job.step, contexts=[single_context], batch_id=f"single_{job.job_id}", - sample_id=single_context.get_sample_id() + sample_id=single_context.get_sample_id(), ) handler(single_batch) - + # Register the wrapped handler self.register_handler(job_type, wrapped_handler) @@ -1827,6 +1874,7 @@ def handle_file(self, filepath: str) -> None: try: # Check if preprocessor function accepts target_panel parameter import inspect + sig = inspect.signature(self.preprocessor_func) if "target_panel" in sig.parameters: jobs = self.preprocessor_func(filepath, target_panel=self.target_panel) @@ -1952,7 +2000,9 @@ def stop(self, timeout: float = 30.0) -> bool: _job_id_counter = itertools.count(1000) -def default_file_classifier(filepath: str, workflow_plan: List[str], target_panel: str) -> List[Job]: +def default_file_classifier( + filepath: str, workflow_plan: List[str], target_panel: str +) -> List[Job]: """Default classifier that creates jobs for a file based on a workflow plan.""" job_id = next(_job_id_counter) ctx = WorkflowContext(filepath) @@ -2104,32 +2154,35 @@ def command_handler(job: Job, command_template: str) -> None: def enhanced_handler(job: BatchedJob) -> None: """Enhanced handler that processes batched jobs sequentially""" - + sample_id = job.get_sample_id() job_type = job.job_type batch_size = job.get_file_count() - + # Process each context sequentially for i, context in enumerate(job.contexts): context.set_batch_info(job.batch_id, i) - + try: # Process individual file within batch process_single_file_in_batch(context, job_type, i, batch_size) context.add_result(job_type, f"{job_type}_ok") - + except Exception as e: # Fail entire batch if any file fails error_msg = f"File {i+1}/{batch_size} failed: {str(e)}" for ctx in job.contexts: ctx.add_error(job_type, error_msg) raise # Re-raise to fail the entire batch - + # Mark batch as completed for context in job.contexts: context.add_result(job_type, f"{job_type}_batch_completed") -def process_single_file_in_batch(context: WorkflowContext, job_type: str, index: int, total: int) -> None: + +def process_single_file_in_batch( + context: WorkflowContext, job_type: str, index: int, total: int +) -> None: """Process a single file within a batch - to be implemented per job type""" # This would call the existing single-file processing logic pass @@ -2191,7 +2244,7 @@ def register_handler( ) -> None: """Register a custom job handler.""" self.manager.register_handler(queue_type, job_type, handler) - + def register_batched_handler( self, job_type: str, handler: Callable[[BatchedJob], None] ) -> None: @@ -2375,6 +2428,7 @@ def submit_target_bam_finalize_job( if target_panel is None: try: import csv + master_csv = Path(sample_dir) / "master.csv" if master_csv.exists(): with master_csv.open("r", newline="") as fh: @@ -2513,7 +2567,9 @@ def classifier_func(filepath: str): if graceful_shutdown: print("[SHUTDOWN] File watcher stopped gracefully") else: - print("[SHUTDOWN] Warning: File watcher may not have stopped gracefully") + print( + "[SHUTDOWN] Warning: File watcher may not have stopped gracefully" + ) if self.verbose: if graceful_shutdown: @@ -2534,6 +2590,7 @@ def classifier_func(filepath: str): def _monitor_progress(self, watcher) -> None: """Monitor and display worker progress in real-time.""" import time + if _should_use_rich_progress(): self._monitor_progress_rich() return diff --git a/tests/test_bam_preprocessor.py b/tests/test_bam_preprocessor.py index d8cda525..95720460 100644 --- a/tests/test_bam_preprocessor.py +++ b/tests/test_bam_preprocessor.py @@ -10,13 +10,13 @@ from robin.analysis.bam_preprocessor import ( BamMetadata, + _extract_sample_id_from_bam, + _get_modbase_model_warning, + _get_modbase_model_warning_level, calculate_bam_summary, extract_bam_metadata, get_rg_tags_from_bam, process_bam_reads, - _extract_sample_id_from_bam, - _get_modbase_model_warning, - _get_modbase_model_warning_level, ) _FIXTURE_DIR = Path(__file__).resolve().parent / "fixtures" / "bam" @@ -191,6 +191,12 @@ def test_extract_sample_id_from_bam_fixtures() -> None: def test_bam_metadata_dataclass_post_init() -> None: - m = BamMetadata(file_path="/x.bam", file_size=1, creation_time=0.0, extracted_data=None, processing_steps=None) + m = BamMetadata( + file_path="/x.bam", + file_size=1, + creation_time=0.0, + extracted_data=None, + processing_steps=None, + ) assert m.extracted_data == {} assert m.processing_steps == [] diff --git a/tests/test_clinvar_manager.py b/tests/test_clinvar_manager.py index 1e50ee07..50bfa58f 100644 --- a/tests/test_clinvar_manager.py +++ b/tests/test_clinvar_manager.py @@ -20,7 +20,6 @@ from robin.utils import clinvar_manager as cm - # --- Small HTTP fakes (no real sockets) --- @@ -233,7 +232,9 @@ def test_ensure_tabix_pysam_failure_uses_cli_tabix(tmp_path: Path) -> None: def build_index(gz_path: Path, tbi_path: Path) -> None: tbi_path.write_bytes(b"index") - with patch("robin.utils.clinvar_manager._build_tabix_index", side_effect=build_index): + with patch( + "robin.utils.clinvar_manager._build_tabix_index", side_effect=build_index + ): with patch( "robin.utils.clinvar_manager._verify_clinvar_tabix_index", return_value=True, @@ -462,7 +463,9 @@ def test_record_sample_clinvar_provenance_not_overwritten_by_default( resources_dir=resources, ) - assert json.loads(provenance.read_text(encoding="utf-8"))["file_date"] == "2020-01-01" + assert ( + json.loads(provenance.read_text(encoding="utf-8"))["file_date"] == "2020-01-01" + ) # --- update_clinvar_if_newer --- @@ -545,7 +548,9 @@ def write_gz(url: str, path: Path, **kwargs: object) -> None: with patch.object(cm, "_download_url_to_file", side_effect=write_gz) as dl: with patch.object(cm, "_get_remote_last_modified", return_value=1.0): with patch.object(cm, "_ensure_tabix_index"): - cm.update_clinvar_if_newer(resources_dir=r, download_if_missing=True) + cm.update_clinvar_if_newer( + resources_dir=r, download_if_missing=True + ) assert dl.called assert (r / cm.CLINVAR_VCF_GZ_NAME).read_bytes() == b"fresh" @@ -587,4 +592,3 @@ def test_compare_sample_clinvar_to_installed_detects_stale_release( assert status["is_stale"] is True assert status["installed_label"] == "ClinVar release 2026-06-21" assert status["sample_label"] == "ClinVar release 2020-01-01" - diff --git a/tests/test_cnv_calling_track.py b/tests/test_cnv_calling_track.py index 7c3836c7..4b6dbd88 100644 --- a/tests/test_cnv_calling_track.py +++ b/tests/test_cnv_calling_track.py @@ -133,7 +133,12 @@ def test_whole_chromosome_event_suppresses_arm_events() -> None: cytobands = pd.DataFrame( [ {"chrom": "chr7", "name": "p22", "start_pos": 0, "end_pos": 40_000_000}, - {"chrom": "chr7", "name": "q31", "start_pos": 40_000_000, "end_pos": 100_000_000}, + { + "chrom": "chr7", + "name": "q31", + "start_pos": 40_000_000, + "end_pos": 100_000_000, + }, ] ) cnv_data = {"chr7": np.full(n_bins, 0.5, dtype=float)} diff --git a/tests/test_display_config.py b/tests/test_display_config.py index 9b50dd10..c5ffa509 100644 --- a/tests/test_display_config.py +++ b/tests/test_display_config.py @@ -16,16 +16,22 @@ def test_display_config_hides_admin_override() -> None: config = SampleDisplayConfig(sections={"sturgeon": False}) - assert is_section_visible( - "sturgeon", - workflow_steps=["sturgeon", "cnv"], - display_config=config, - ) is False - assert is_section_visible( - "cnv", - workflow_steps=["sturgeon", "cnv"], - display_config=config, - ) is True + assert ( + is_section_visible( + "sturgeon", + workflow_steps=["sturgeon", "cnv"], + display_config=config, + ) + is False + ) + assert ( + is_section_visible( + "cnv", + workflow_steps=["sturgeon", "cnv"], + display_config=config, + ) + is True + ) def test_display_config_respects_workflow_steps() -> None: @@ -35,11 +41,14 @@ def test_display_config_respects_workflow_steps() -> None: def test_fusion_child_hidden_when_parent_hidden() -> None: config = SampleDisplayConfig(sections={"fusion": False}) - assert is_section_visible( - "fusion_target", - workflow_steps=["fusion"], - display_config=config, - ) is False + assert ( + is_section_visible( + "fusion_target", + workflow_steps=["fusion"], + display_config=config, + ) + is False + ) def test_classification_visible_set() -> None: @@ -75,12 +84,15 @@ def test_migrate_stale_snp_false_to_active() -> None: assert config.schema_version == 3 assert "snp" not in config.role_sections["user"] assert config.role_sections["user"]["cnv"] is False - assert is_section_visible( - "snp", - workflow_steps=["cnv"], - display_config=config, - surface="sample_details", - ) is True + assert ( + is_section_visible( + "snp", + workflow_steps=["cnv"], + display_config=config, + surface="sample_details", + ) + is True + ) def test_role_sections_use_viewer_role() -> None: @@ -119,30 +131,44 @@ def test_legacy_sections_migrate_to_user_role() -> None: def test_any_sample_details_visible() -> None: - config = SampleDisplayConfig(sections={"target": False, "snp": False, "fusion": False}) - assert any_sample_details_visible(["target", "fusion", "snp_analysis"], config) is False + config = SampleDisplayConfig( + sections={"target": False, "snp": False, "fusion": False} + ) + assert ( + any_sample_details_visible(["target", "fusion", "snp_analysis"], config) + is False + ) config2 = SampleDisplayConfig(sections={"fusion": True, "fusion_target": True}) assert any_sample_details_visible(["fusion"], config2) is True - assert is_section_visible( - "snp", - workflow_steps=["snp_analysis"], - surface="sample_details", - ) is True + assert ( + is_section_visible( + "snp", + workflow_steps=["snp_analysis"], + surface="sample_details", + ) + is True + ) # SNP is optional (on-demand) analysis; visibility is admin-controlled, not workflow-gated. - assert is_section_visible( - "snp", - workflow_steps=["cnv"], - surface="sample_details", - ) is True + assert ( + is_section_visible( + "snp", + workflow_steps=["cnv"], + surface="sample_details", + ) + is True + ) config_hide_snp = SampleDisplayConfig(sections={"snp": False}) - assert is_section_visible( - "snp", - workflow_steps=["cnv"], - display_config=config_hide_snp, - surface="sample_details", - ) is False + assert ( + is_section_visible( + "snp", + workflow_steps=["cnv"], + display_config=config_hide_snp, + surface="sample_details", + ) + is False + ) def test_effective_section_map_includes_sample_details_sections() -> None: @@ -153,7 +179,11 @@ def test_effective_section_map_includes_sample_details_sections() -> None: def test_mnpflex_grouped_under_v12_classifier() -> None: - from robin.gui.display_config import DISPLAY_GROUP_LABELS, DISPLAY_GROUP_ORDER, DISPLAY_SECTIONS + from robin.gui.display_config import ( + DISPLAY_GROUP_LABELS, + DISPLAY_GROUP_ORDER, + DISPLAY_SECTIONS, + ) assert DISPLAY_SECTIONS["mnpflex"].group == "v12_classifier" assert DISPLAY_GROUP_ORDER.index("v12_classifier") == 1 @@ -169,8 +199,11 @@ def test_security_store_gui_settings_roundtrip(tmp_path: Path) -> None: assert loaded is not None assert loaded["sections"]["cnv"] is False restored = SampleDisplayConfig.from_dict(loaded) - assert is_section_visible( - "cnv", - workflow_steps=["cnv"], - display_config=restored, - ) is False + assert ( + is_section_visible( + "cnv", + workflow_steps=["cnv"], + display_config=restored, + ) + is False + ) diff --git a/tests/test_gui_sample_audit.py b/tests/test_gui_sample_audit.py index 06911028..fbc5728b 100644 --- a/tests/test_gui_sample_audit.py +++ b/tests/test_gui_sample_audit.py @@ -4,7 +4,10 @@ from pathlib import Path -from robin.gui.components.sample_audit import _audit_rows_for_sample, _export_sample_audit_csv +from robin.gui.components.sample_audit import ( + _audit_rows_for_sample, + _export_sample_audit_csv, +) from robin.security.store import SecurityStore diff --git a/tests/test_lamprey_analysis.py b/tests/test_lamprey_analysis.py index a66ccbb8..6ae5de10 100644 --- a/tests/test_lamprey_analysis.py +++ b/tests/test_lamprey_analysis.py @@ -39,9 +39,10 @@ def run(self, output_names, input_feed): def test_sturgeon_matched_confidence_tiers(): - assert CLASSIFIER_CONFIDENCE_THRESHOLDS["lamprey"] == CLASSIFIER_CONFIDENCE_THRESHOLDS[ - "sturgeon" - ] + assert ( + CLASSIFIER_CONFIDENCE_THRESHOLDS["lamprey"] + == CLASSIFIER_CONFIDENCE_THRESHOLDS["sturgeon"] + ) def test_research_ack_gate(monkeypatch): diff --git a/tests/test_mnpflex_config.py b/tests/test_mnpflex_config.py index 53d57562..e6b83af0 100644 --- a/tests/test_mnpflex_config.py +++ b/tests/test_mnpflex_config.py @@ -7,7 +7,9 @@ from robin.analysis.mnpflex_config import load_mnpflex_config -def test_load_mnpflex_config_api_from_credentials(monkeypatch: pytest.MonkeyPatch) -> None: +def test_load_mnpflex_config_api_from_credentials( + monkeypatch: pytest.MonkeyPatch, +) -> None: monkeypatch.delenv("MNPFLEX_BACKEND", raising=False) monkeypatch.delenv("MNPFLEX_DOCKER_IMAGE", raising=False) monkeypatch.setenv("MNPFLEX_USERNAME", "user") @@ -17,7 +19,9 @@ def test_load_mnpflex_config_api_from_credentials(monkeypatch: pytest.MonkeyPatc assert config.validation_error() is None -def test_load_mnpflex_config_docker_requires_image(monkeypatch: pytest.MonkeyPatch) -> None: +def test_load_mnpflex_config_docker_requires_image( + monkeypatch: pytest.MonkeyPatch, +) -> None: monkeypatch.setenv("MNPFLEX_BACKEND", "docker") monkeypatch.delenv("MNPFLEX_DOCKER_IMAGE", raising=False) config = load_mnpflex_config() diff --git a/tests/test_mnpflex_docker.py b/tests/test_mnpflex_docker.py index dc99379e..adf74ca9 100644 --- a/tests/test_mnpflex_docker.py +++ b/tests/test_mnpflex_docker.py @@ -7,6 +7,7 @@ import pytest +from robin.analysis.mnpflex_config import MNPFlexConfig from robin.analysis.mnpflex_docker import ( adapt_docker_outputs, build_bundle_summary_from_docker_dir, @@ -14,8 +15,6 @@ format_mnpflex_runtime_error, run_docker_mnpflex, ) -from robin.analysis.mnpflex_config import MNPFlexConfig - FIXTURE_DIR = ( Path(__file__).resolve().parent / "fixtures" / "mnpflex_docker" / "26D22147.MNPFlex" @@ -52,7 +51,10 @@ def test_build_bundle_summary_from_fixture() -> None: assert preds["molecular_superfamily"]["label"] == "Adult-Type Diffuse Gliomas" assert preds["molecular_family"]["label"] == "Glioblastoma, IDH-Wildtype" assert preds["molecular_class"]["label"] == "Glioblastoma, IDH-Wildtype, RTK2 Type" - assert preds["molecular_subclass"]["label"] == "Glioblastoma, IDH-Wildtype, RTK2 Subtype" + assert ( + preds["molecular_subclass"]["label"] + == "Glioblastoma, IDH-Wildtype, RTK2 Subtype" + ) assert preds["molecular_class"]["score"] == pytest.approx(0.254857897758484) scores = summary["classifier_summary"]["scores"] assert len(scores) >= 180 @@ -99,7 +101,9 @@ def test_find_docker_output_dir_supports_nested_layout(tmp_path: Path) -> None: nested = tmp_path / "docker_workspace" / "sample.bedstem" nested.mkdir(parents=True) for path in FIXTURE_DIR.glob("*.csv"): - (nested / path.name).write_text(path.read_text(encoding="utf-8"), encoding="utf-8") + (nested / path.name).write_text( + path.read_text(encoding="utf-8"), encoding="utf-8" + ) found = find_docker_output_dir(tmp_path / "docker_workspace", "sample.bedstem") assert found == nested @@ -114,12 +118,15 @@ def test_run_docker_mnpflex_invokes_container( docker_dir.mkdir(parents=True) for path in FIXTURE_DIR.glob("*.csv"): - (docker_dir / path.name).write_text(path.read_text(encoding="utf-8"), encoding="utf-8") + (docker_dir / path.name).write_text( + path.read_text(encoding="utf-8"), encoding="utf-8" + ) captured: dict = {} def fake_run(cmd, **kwargs): captured["cmd"] = cmd + class Result: returncode = 0 stdout = "" diff --git a/tests/test_plotting_preferences.py b/tests/test_plotting_preferences.py index 34dd89e7..328ba165 100644 --- a/tests/test_plotting_preferences.py +++ b/tests/test_plotting_preferences.py @@ -22,9 +22,7 @@ def test_plotting_preferences_defaults() -> None: def test_plotting_preferences_legacy_log2_alias() -> None: - config = PlottingPreferencesConfig.from_dict( - {"cnv_report_scale": "log2_ratio"} - ) + config = PlottingPreferencesConfig.from_dict({"cnv_report_scale": "log2_ratio"}) assert config.cnv_report_scale == CNV_REPORT_SCALE_NORMALIZED_DIFFERENCE @@ -65,10 +63,17 @@ def test_resolve_cnv_summary_normalized_uses_admin_preference(tmp_path: Path) -> def test_resolve_cnv_summary_normalized_defaults_to_ploidy() -> None: - assert resolve_cnv_summary_normalized(None, plotting_preferences=PlottingPreferencesConfig()) is False + assert ( + resolve_cnv_summary_normalized( + None, plotting_preferences=PlottingPreferencesConfig() + ) + is False + ) -def test_robin_report_loads_admin_plotting_preferences(tmp_path: Path, monkeypatch) -> None: +def test_robin_report_loads_admin_plotting_preferences( + tmp_path: Path, monkeypatch +) -> None: """When plotting_preferences is omitted, RobinReport must load the store default.""" from robin.reporting.report import RobinReport @@ -99,7 +104,9 @@ def test_robin_report_loads_admin_plotting_preferences(tmp_path: Path, monkeypat ) -def test_robin_report_respects_explicit_ploidy_override(tmp_path: Path, monkeypatch) -> None: +def test_robin_report_respects_explicit_ploidy_override( + tmp_path: Path, monkeypatch +) -> None: from robin.reporting.report import RobinReport prefs = PlottingPreferencesConfig( @@ -126,7 +133,9 @@ def test_robin_report_respects_explicit_ploidy_override(tmp_path: Path, monkeypa assert report.cnv_summary_normalized is False -def test_robin_report_empty_config_does_not_load_store(tmp_path: Path, monkeypatch) -> None: +def test_robin_report_empty_config_does_not_load_store( + tmp_path: Path, monkeypatch +) -> None: """Explicit empty PlottingPreferencesConfig must not be replaced by store load.""" from robin.reporting.report import RobinReport diff --git a/tests/test_reporting_cnv_summary_plot.py b/tests/test_reporting_cnv_summary_plot.py index d8d4799d..18615d69 100644 --- a/tests/test_reporting_cnv_summary_plot.py +++ b/tests/test_reporting_cnv_summary_plot.py @@ -218,9 +218,24 @@ def test_add_genome_panel_coverage_points_draws_scatter_and_mean_line() -> None: # Plotted points are outliers only; all-target mean is passed separately. panel_points = [ - {"position_bp": 1_000_000.0, "coverage_val": 20.0, "direction": "gain", "label": "GENE1"}, - {"position_bp": 2_000_000.0, "coverage_val": 40.0, "direction": "loss", "label": "GENE2"}, - {"position_bp": 3_000_000.0, "coverage_val": 30.0, "direction": "gain", "label": "GENE3"}, + { + "position_bp": 1_000_000.0, + "coverage_val": 20.0, + "direction": "gain", + "label": "GENE1", + }, + { + "position_bp": 2_000_000.0, + "coverage_val": 40.0, + "direction": "loss", + "label": "GENE2", + }, + { + "position_bp": 3_000_000.0, + "coverage_val": 30.0, + "direction": "gain", + "label": "GENE3", + }, ] assert ( @@ -246,12 +261,29 @@ def test_add_genome_panel_coverage_points_falls_back_to_outlier_mean() -> None: ax_cnv.twinx.return_value = ax_cov panel_points = [ - {"position_bp": 1_000_000.0, "coverage_val": 20.0, "direction": "gain", "label": "GENE1"}, - {"position_bp": 2_000_000.0, "coverage_val": 40.0, "direction": "loss", "label": "GENE2"}, - {"position_bp": 3_000_000.0, "coverage_val": 30.0, "direction": "gain", "label": "GENE3"}, + { + "position_bp": 1_000_000.0, + "coverage_val": 20.0, + "direction": "gain", + "label": "GENE1", + }, + { + "position_bp": 2_000_000.0, + "coverage_val": 40.0, + "direction": "loss", + "label": "GENE2", + }, + { + "position_bp": 3_000_000.0, + "coverage_val": 30.0, + "direction": "gain", + "label": "GENE3", + }, ] - assert _add_genome_panel_coverage_points(ax_cnv, panel_points, 250_000_000.0) is True + assert ( + _add_genome_panel_coverage_points(ax_cnv, panel_points, 250_000_000.0) is True + ) assert ax_cov.axhline.call_args[0][0] == 30.0 # fallback: mean of 20, 40, 30 @@ -259,7 +291,9 @@ def test_downsample_cnv_for_plot_groups_values() -> None: from robin.analysis.cnv_analysis import downsample_cnv_for_plot values = np.array([1.0, 3.0, 5.0, 7.0], dtype=float) - x_bp, out = downsample_cnv_for_plot(values, analysis_bin_width=12_000, plot_bin_width=24_000) + x_bp, out = downsample_cnv_for_plot( + values, analysis_bin_width=12_000, plot_bin_width=24_000 + ) assert len(out) == 2 assert out[0] == 2.0 assert out[1] == 6.0 @@ -274,7 +308,9 @@ def test_downsample_cnv_chromosome_track_keeps_full_x_axis() -> None: n_bins = 1000 values = np.linspace(0.0, 1.0, n_bins) x_mb, out, x_max_mb = downsample_cnv_chromosome_track( - values, analysis_bw, plot_bin_width=500_000, + values, + analysis_bw, + plot_bin_width=500_000, ) assert x_max_mb == n_bins * analysis_bw / 1_000_000 assert len(out) < n_bins @@ -437,19 +473,20 @@ class _FakeResult: def test_cnv_chromosome_fig_height_fits_four_per_page() -> None: from robin.reporting.plotting import ( - CNV_CHROMOSOME_PLOTS_PER_PAGE, CNV_CHROMOSOME_PLOT_SPACER_PT, + CNV_CHROMOSOME_PLOTS_PER_PAGE, CNV_REPORT_FRAME_PADDING_PT, cnv_chromosome_fig_height_for_page, ) page_height = 9.34 plot_height = cnv_chromosome_fig_height_for_page(page_height) - spacer_inch = (CNV_CHROMOSOME_PLOTS_PER_PAGE - 1) * CNV_CHROMOSOME_PLOT_SPACER_PT / 72.0 + spacer_inch = ( + (CNV_CHROMOSOME_PLOTS_PER_PAGE - 1) * CNV_CHROMOSOME_PLOT_SPACER_PT / 72.0 + ) frame_inch = page_height - CNV_REPORT_FRAME_PADDING_PT / 72.0 assert ( - CNV_CHROMOSOME_PLOTS_PER_PAGE * plot_height + spacer_inch - <= frame_inch + 1e-6 + CNV_CHROMOSOME_PLOTS_PER_PAGE * plot_height + spacer_inch <= frame_inch + 1e-6 ) assert plot_height < 2.5 @@ -483,7 +520,9 @@ def test_twelve_chromosome_pdf_images_fit_three_pages() -> None: elements = [] for plot_idx in range(12): buf = io.BytesIO() - PILImage.new("RGB", (int(width_inch * 100), int(height_inch * 100)), "white").save( + PILImage.new( + "RGB", (int(width_inch * 100), int(height_inch * 100)), "white" + ).save( buf, format="JPEG", ) diff --git a/tests/test_reporting_modbase_warning.py b/tests/test_reporting_modbase_warning.py index 1a60bb2e..2feed247 100644 --- a/tests/test_reporting_modbase_warning.py +++ b/tests/test_reporting_modbase_warning.py @@ -30,9 +30,7 @@ def _report(master_data): def _paragraph_text(elements): return " ".join( - element.getPlainText() - for element in elements - if isinstance(element, Paragraph) + element.getPlainText() for element in elements if isinstance(element, Paragraph) ) @@ -44,9 +42,7 @@ def test_report_includes_all_context_modbase_warning(): "devices": "p2soloTower", "flowcell_ids": "PAW67899", "basecall_models": "hac@v6.0.0", - "modbase_models": ( - "dna_r10.4.1_e8.2_400bps_hac@v6.0.0_5mC_5hmC@v1" - ), + "modbase_models": ("dna_r10.4.1_e8.2_400bps_hac@v6.0.0_5mC_5hmC@v1"), "counter_bam_passed": 1, "counter_bam_failed": 0, "counter_bases_count": 100, diff --git a/tests/test_security_auth.py b/tests/test_security_auth.py index e62034d5..bd48fa49 100644 --- a/tests/test_security_auth.py +++ b/tests/test_security_auth.py @@ -84,7 +84,9 @@ def test_auth_bootstrap_from_legacy_hash(tmp_path: Path) -> None: def test_bootstrap_admin_explicit_no_must_change(tmp_path: Path) -> None: store, auth = _store_and_auth(tmp_path) - user_id = auth.create_user("admin", "setup-pass", role="admin", must_change_password=False) + user_id = auth.create_user( + "admin", "setup-pass", role="admin", must_change_password=False + ) user = store.get_user_by_id(user_id) assert user is not None assert not user.must_change_password diff --git a/tests/test_security_cli.py b/tests/test_security_cli.py index 6b2739dc..38663e5a 100644 --- a/tests/test_security_cli.py +++ b/tests/test_security_cli.py @@ -60,7 +60,6 @@ def test_cli_bootstrap_admin(security_db: Path) -> None: assert store.user_has_role(user.id, "admin") - def test_cli_password_set_creates_admin(security_db: Path) -> None: runner = CliRunner() result = runner.invoke(main, ["password", "set"], input="s3cret\ns3cret\n") diff --git a/tests/test_tucan_analysis.py b/tests/test_tucan_analysis.py index 43e2afbb..4353830b 100644 --- a/tests/test_tucan_analysis.py +++ b/tests/test_tucan_analysis.py @@ -9,9 +9,9 @@ from robin.analysis.tucan_analysis import ( SCORE_META_COLUMNS, + _top_prediction, append_tucan_scores, binarize_methylation_calls, - _top_prediction, ) from robin.reporting.sections.classification import drop_classifier_score_meta_columns from robin.utils.tucan_manager import (