diff --git a/.github/workflows/mkdocs-preview.yml b/.github/workflows/mkdocs-preview.yml new file mode 100644 index 00000000..5dcdde57 --- /dev/null +++ b/.github/workflows/mkdocs-preview.yml @@ -0,0 +1,89 @@ +# Build MkDocs for pull requests and publish a preview under /preview/pr-/ +# on the existing gh-pages branch. Production root and /beta/ are preserved. + +name: Publish docs preview + +on: + pull_request: + branches: + - release_candidate2 + paths: + - "docs/**" + - "mkdocs.yml" + - "requirements-docs.txt" + - ".github/workflows/mkdocs-preview.yml" + workflow_dispatch: + +permissions: + contents: write + +concurrency: + group: gh-pages-deploy + cancel-in-progress: false + +jobs: + build-and-deploy: + if: github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: pip + cache-dependency-path: requirements-docs.txt + + - name: Install MkDocs + run: pip install -r requirements-docs.txt + + - name: Set preview URL + id: preview + shell: bash + run: | + if [ "${{ github.event_name }}" = "pull_request" ]; then + PR_NUMBER="${{ github.event.pull_request.number }}" + else + PR_NUMBER="manual-${GITHUB_RUN_NUMBER}" + fi + echo "number=${PR_NUMBER}" >> "$GITHUB_OUTPUT" + echo "url=https://looselab.github.io/ROBIN/preview/pr-${PR_NUMBER}/" >> "$GITHUB_OUTPUT" + + - name: Build site + env: + PREVIEW_SITE_URL: ${{ steps.preview.outputs.url }} + run: | + cp mkdocs.yml mkdocs.preview.yml + python - <<'PY' + from pathlib import Path + import os + p = Path("mkdocs.preview.yml") + text = p.read_text() + lines = text.splitlines() + preview_url = os.environ["PREVIEW_SITE_URL"] + replaced = False + for i, line in enumerate(lines): + if line.startswith("site_url:"): + lines[i] = f"site_url: {preview_url}" + replaced = True + break + if not replaced: + lines.insert(0, f"site_url: {preview_url}") + p.write_text("\n".join(lines) + "\n") + PY + mkdocs build --strict -f mkdocs.preview.yml + + - name: Add .nojekyll + run: touch site/.nojekyll + + - name: Deploy preview to GitHub Pages branch + uses: JamesIves/github-pages-deploy-action@v4 + with: + folder: site + branch: gh-pages + target-folder: preview/pr-${{ steps.preview.outputs.number }} + clean: true + + - name: Preview URL + run: | + echo "Preview: ${{ steps.preview.outputs.url }}" diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..13d3b415 --- /dev/null +++ b/LICENSE @@ -0,0 +1,34 @@ +Creative Commons Attribution-NonCommercial 4.0 International + +Copyright (c) 2023-present Matt Loose + +This work is licensed under the Creative Commons Attribution-NonCommercial 4.0 International License (CC BY-NC 4.0). + +You are free to: + +Share — copy and redistribute the material in any medium or format. +Adapt — remix, transform, and build upon the material. + +Under the following terms: + +Attribution — You must give appropriate credit, provide a link to the license, and indicate if changes were made. You may do so in any reasonable manner, but not in any way that suggests the licensor endorses you or your use. + +NonCommercial — You may not use the material for commercial purposes. + +No additional restrictions — You may not apply legal terms or technological measures that legally restrict others from doing anything the license permits. + +The licensor cannot revoke these freedoms as long as you follow the license terms. + +Notices: + +You do not have to comply with the license for elements of the material in the public domain or where your use is permitted by an applicable exception or limitation. + +No warranties are given. The license may not give you all of the permissions necessary for your intended use. For example, other rights such as publicity, privacy, or moral rights may limit how you use the material. + +The complete legal code for the Creative Commons Attribution-NonCommercial 4.0 International License is available at: +https://creativecommons.org/licenses/by-nc/4.0/legalcode + +License summary: +https://creativecommons.org/licenses/by-nc/4.0/ + +Third-party software, models, data, and other resources included with or used by ROBIN remain subject to their respective licences and terms. \ No newline at end of file diff --git a/README.md b/README.md index 8390557e..f3892e66 100644 --- a/README.md +++ b/README.md @@ -1,378 +1,108 @@ # ![ROBIN_logo_small.png](src/robin/gui/images/ROBIN_logo_small.png) R.O.B.I.N -# ***This software is provided as is for research use only.*** +**Rapid nanopOre Brain intraoperatIve classificatioN** -**IMPORTANT: use a fresh conda environment from `robin.yml` (`conda activate robin`). Do not reuse conda environments created for older ROBIN releases or for other projects.** +> **Research use only.** ROBIN is under active development and validation. Analysis outputs require expert interpretation and must not be treated as standalone clinical results. -**IMPORTANT (input BAMs): each file must contain 50,000 reads or fewer.** In MinKNOW, configure **read-count–based** BAM output—**we recommend one BAM every 50,000 reads**. **Do not** use **time-based** BAM rollover (MinKNOW’s typical default, e.g. hourly); it is unsupported and usually violates the read limit. Details: [BAM read limit and MinKNOW](#bam-read-limit-and-minknow-settings). +ROBIN is a real-time analysis and web-monitoring platform for Oxford Nanopore sequencing of CNS tumours. It processes aligned BAM files as they are produced and can combine methylation classification, copy-number analysis, MGMT analysis, targeted analysis and structural-event detection in one workflow. -## Table of contents +ROBIN incorporates **LITTLE JOHN** (Lightweight Infrastructure for Task Tracking and Logging with Extensible Job Orchestration for High-throughput aNalysis) for workflow orchestration and scaling. -- [About](#about) -- [Requirements](#requirements) -- [Installation](#installation) - - [If the `robin` conda environment already exists](#if-the-robin-conda-environment-already-exists) -- [Common issues](#common-issues) -- [Usage](#usage) - - [BAM read limit and MinKNOW](#bam-read-limit-and-minknow-settings) - - [Optional MNP-Flex integration](#optional-mnp-flex-integration) -- [Command reference](#command-reference) -- [Known issues and limitations](#known-issues-and-limitations) -- [Performance](#performance) -- [Dependencies](#dependencies) -- [License](#license) -- [Acknowledgments](#acknowledgments) +The ROBIN approach is described in *Neuro-Oncology*: [Rapid nanopore brain intraoperative classification](https://academic.oup.com/neuro-oncology/article/27/8/2035/8139084). ---- +## Documentation -## About +The MkDocs site is the canonical user and developer documentation: -**ROBIN** (Rapid nanopOre Brain intraoperatIve classificatioN) is a bioinformatics workflow system for processing and analysing human oncology BAM data from Oxford Nanopore sequencing. It was published in *Neuro-Oncology*: [ROBIN paper](https://academic.oup.com/neuro-oncology/article/27/8/2035/8139084). +- [Installation](docs/getting-started/installation.md) +- [Quickstart](docs/getting-started/quickstart.md) +- [MinKNOW configuration](docs/getting-started/minknow-configuration.md) +- [Using ROBIN](docs/using-robin/index.md) +- [Analysis pipelines](docs/analyses/index.md) +- [Command-line reference](docs/cli/index.md) +- [Developer architecture](docs/architecture/overview.md) -ROBIN provides automated preprocessing, multiple analysis pipelines, and real-time monitoring. It incorporates **LITTLE JOHN** (Lightweight Infrastructure for Task Tracking and Logging with Extensible Job Orchestration for High-throughput aNalysis), which handles orchestration and scaling behind the scenes. +## Quick installation -**Capabilities** include methylation analysis, copy-number variation, fusion detection, classification workflows, a multi-threaded execution model, and a web-based GUI for monitoring, progress, and visualisation. +ROBIN requires Python 3.12 and is intended to be installed in a fresh conda environment. -This repository is the canonical home for ROBIN—development, source code, and releases: [LooseLab/ROBIN](https://github.com/LooseLab/ROBIN). - -## Requirements - -| Resource | Notes | -|----------|--------| -| **RAM** | ≥ 64 GB recommended | -| **GPU** | As per ONT guidelines for adaptive sampling | -| **CPU** | As per ONT guidelines (more is generally better) | - -With **LITTLE JOHN**, ROBIN can run two PromethION flow cells simultaneously on a Nanopore P2i (e.g. LSK114 or modified ultra-long protocol), subject to the above resources. - ---- - -## Installation - -Use **conda** so native and Python dependencies stay consistent. Create the environment from **`robin.yml`** (Python 3.12); older Python 3.9-era env files are removed. - -For a step-by-step walkthrough, see [`docs/getting-started/installation.md`](docs/getting-started/installation.md). - -### Prerequisites - -- [Miniconda](https://docs.conda.io/en/latest/miniconda.html) or Anaconda - -### Steps - -1. **Clone with submodules** - ```bash - git clone --recursive https://github.com/LooseLab/ROBIN.git - cd ROBIN - ``` - -2. **Ensure submodules are current** (e.g. nanoDX, hv_rapidCNS2) - ```bash - git submodule update --init --recursive - ``` - -3. **Create and activate the environment** - ```bash - conda env create -f robin.yml - conda activate robin - ``` - Linux and macOS share this file. On Linux, for `libstdc++` / `CXXABI_1.3.15` errors, see [Common issues](#common-issues). - -4. **Install the package in editable mode** - ```bash - pip install -e . - ``` - -5. **Download models and ClinVar data** (models are checksum-verified via the assets manifest) - ```bash - robin utils update-models - robin utils update-clinvar - ``` - - To **re-download** models (e.g. after a failed partial run), use `robin utils update-models --overwrite`. Advanced: `python scripts/fetch_asset.py` and [`src/robin/resources/assets.json`](src/robin/resources/assets.json). - -### If the `robin` conda environment already exists - -`robin.yml` defines `name: robin`. **`conda env create -f robin.yml` will fail** if an environment with that name already exists. - -- **Use a different env name** (keep your existing `robin` env untouched): - - ```bash - conda env create -f robin.yml -n robin_littlejohn - conda activate robin_littlejohn - ``` - -- **Update the existing env** from the current file (keeps the name `robin`): - - ```bash - conda env update -n robin -f robin.yml --prune - conda activate robin - ``` - - `--prune` removes packages no longer listed in the YAML (where supported). - -- **Remove and recreate** (closest to a clean install; preferred if the old env was from another project or an older ROBIN release): - - ```bash - conda deactivate - conda env remove -n robin - conda env create -f robin.yml - conda activate robin - ``` - - - -After any of these, run **`pip install -e .`** again from the repository root with the env you intend to use. - -More detail: [`docs/getting-started/installation.md` → If the `robin` environment already exists](docs/getting-started/installation.md#if-the-robin-environment-already-exists). - ---- - -## Possible issue +```bash +git clone --recursive https://github.com/LooseLab/ROBIN.git +cd ROBIN -### `libstdc++.so.6` / `CXXABI_1.3.15` (Linux: SciPy, ICU, native extensions) +conda env create -f robin.yml +conda activate robin -The linker may use the **system** `libstdc++.so.6` (e.g. under `/lib/x86_64-linux-gnu/`) instead of conda’s (`libstdcxx-ng` in `$CONDA_PREFIX/lib`), producing errors such as: +pip install -e . -`version 'CXXABI_1.3.15' not found (required by ... scipy ... or libicui18n ...)` +robin utils update-models +robin utils update-clinvar +``` -After `conda activate robin`, prefer the env libraries first: +Then verify the installation: ```bash -export LD_LIBRARY_PATH="$CONDA_PREFIX/lib:${LD_LIBRARY_PATH}" +robin --help +robin list-job-types ``` -You can add that to your shell config (after conda init). - - ---- - -## Usage - -### BAM read limit and MinKNOW settings - -**ROBIN requires that each BAM file contain 50,000 reads or fewer.** Larger files are outside the supported real-time workflow. - -**MinKNOW configuration:** set BAM output to roll on **read count**, not on **time**. **Recommended:** **one BAM file every 50,000 reads** (smaller roll sizes, e.g. 10,000 reads per file, are also fine). Use whatever MinKNOW option splits or rotates BAMs by **number of reads** in each file. - -**Do not** use MinKNOW’s **time-based** BAM settings (for example the default behaviour of writing a new BAM every fixed period such as one hour). That mode is **unsupported**: it tends to produce BAMs with far more than 50,000 reads and does not match how ROBIN expects data to arrive. +For environment troubleshooting and optional extras, see the [installation guide](docs/getting-started/installation.md). -### What ROBIN expects from your sequencing setup +## Important sequencing requirements -- BAMs from an Oxford Nanopore sequencer; **real-time HAC** basecalling (SUP not required). -- **5mC / 5hmC modified-base calling in CpG contexts only** enabled in MinKNOW. Do **not** use all-context 5mC / 5hmC calling. -- **Real-time alignment in MinKNOW** — ROBIN does not realign reads. -- BAMs must respect the **[50,000-read limit](#bam-read-limit-and-minknow-settings)** and MinKNOW read-count output settings above. -- ROBIN does **not** consume POD5 or FASTQ; you can disable those outputs in MinKNOW if you wish. +ROBIN's supported real-time workflow expects aligned Oxford Nanopore BAM files produced by MinKNOW. -### Memory and Dorado +In particular: -On machines with **≤ 64 GB RAM**, restart the machine (or at least Dorado) before a heavy run. Dorado can retain memory indefinitely; on a P2i, after a run on position A and then B, restarting after position B (once basecalling finishes) is recommended. +- use HAC basecalling or better; +- enable 5mC/5hmC calling in CpG context when methylation analyses are required; +- perform alignment in MinKNOW using the same reference supplied to ROBIN; +- configure BAM rollover by **read count**, not time; +- keep every input BAM at **50,000 reads or fewer**. -### Optional MNP-Flex integration +See [MinKNOW configuration](docs/getting-started/minknow-configuration.md) for the complete setup. -ROBIN can submit methylation data to the commercial **MNP-Flex** service from -Heidelberg Epignostix and display the returned results in the GUI. This -requires an agreement and credentials from Epignostix. - -Set `MNPFLEX_USERNAME` and `MNPFLEX_PASSWORD` in the server environment before -starting ROBIN. See the -[MNP-Flex setup guide](docs/getting-started/mnpflex.md) for configuration, -data-transfer considerations, optional API settings, operation, and -troubleshooting. - -### Example workflows - -Primary pattern: +## Minimal workflow example ```bash -robin workflow --work-dir \ - -w target,cnv,fusion,mgmt,sturgeon,nanodx,pannanodx,random_forest \ - --reference ~/references/hg38_simple.fa \ - --center -``` - -| Argument | Meaning | -|----------|---------| -| `` | Directory containing BAM files | -| `--work-dir` | Output directory for results | -| `-w` / `--workflow` | Comma-separated job types (see [`list-job-types`](#list-job-types)) | -| `--reference` | Reference FASTA (required for many analyses) | -| `--center` | Site ID (e.g. `Sherwood`, `Auckland`, `New York`) | -| `--target-panel` | Panel for target/CNV/fusion (e.g. `rCNS2`, `AML`) | - -More examples: - -```bash -# Full analysis set with panel robin workflow ~/data/bam_files \ --work-dir ~/results \ -w target,cnv,fusion,mgmt,sturgeon,nanodx,pannanodx,random_forest \ - --reference ~/references/hg38_simple.fa \ + --reference ~/references/robin_ref/hg38.fa \ --center Sherwood \ --target-panel rCNS2 - -# Smaller workflow -robin workflow ~/data/bam_files \ - --work-dir ~/results \ - -w mgmt,sturgeon \ - --reference ~/references/hg38_simple.fa \ - --center Auckland \ - --target-panel AML - -# Verbose logging -robin workflow ~/data/bam_files \ - --work-dir ~/results \ - -w mgmt,cnv,sturgeon \ - --reference ~/references/hg38_simple.fa \ - --center New_York \ - --target-panel rCNS2 \ - --verbose \ - --log-level INFO -``` - -Full CLI flags for `workflow` are listed under [Command reference](#command-reference). - ---- - -## Command reference - -### `list-job-types` - -Lists job types by queue: - -```bash -robin list-job-types -``` - -| Queue | Job types | -|-------|-----------| -| Preprocessing | `preprocessing` | -| BED conversion | `bed_conversion` | -| Analysis | `mgmt`, `cnv`, `target`, `fusion` | -| Classification | `sturgeon`, `nanodx`, `pannanodx` | -| Slow | `random_forest` | - -### `workflow` - -```bash -robin workflow /path/to/directory --workflow "workflow_plan" [OPTIONS] ``` -**Commonly required** +A recommended first-run sequence, including reference staging and opening the GUI, is in the [Quickstart](docs/getting-started/quickstart.md). Full command options are documented in the [CLI reference](docs/cli/index.md). -- `--workflow`, `-w` — Plan such as `mgmt,sturgeon` or queue-style `preprocessing:bed_conversion,analysis:mgmt,classification:sturgeon` -- `--center` — Center ID (e.g. `Sherwood`, `Auckland`, `New York`) +## System requirements -**Common options** +A typical production-scale installation should have approximately **64 GB RAM or more**. CPU and GPU requirements depend on the Oxford Nanopore sequencing and basecalling configuration in use. -- `--work-dir`, `-d` — Output base directory -- `--reference`, `-r` — Reference genome (FASTA) -- `--verbose`, `-v` — Verbose output and traces -- `--no-process-existing` — Only watch for new files -- `--log-level` — `DEBUG` \| `INFO` \| `WARNING` \| `ERROR` (default: `ERROR`) -- `--job-log-level` — Per-job level, e.g. `preprocessing:DEBUG`, `mgmt:WARNING` -- `--deduplicate-jobs` — Deduplicate by sample ID for given types (e.g. `sturgeon`, `mgmt`) -- `--no-progress` — Disable file progress bars -- `--use-ray` / `--no-use-ray` — Ray distributed execution (default: on) -- `--with-gui` / `--no-gui` — NiceGUI monitor (default: on) +Some optional downstream analyses require additional software such as Docker. ROBIN's core scientific, bioinformatics and R dependencies are defined in `robin.yml` and `pyproject.toml`. -### Panel management +## Repository layout -Built-in panels include `rCNS2`, `AML`. Custom panels are stored after you add them from a BED file. +- `src/robin/` — application, workflow engine, analyses and GUI +- `docs/` — MkDocs user and developer documentation +- `robin.yml` — primary conda environment +- `pyproject.toml` — Python package configuration +- `src/robin/resources/` — packaged panels, manifests and other resources -**List panels** +## Reporting problems -```bash -robin list-panels -``` - -**Add a custom panel** (BED: ≥ 4 columns — chr, start, end, gene name(s); 4- or 6-column BED supported; multiple genes comma-separated in one region) +Please use [GitHub Issues](https://github.com/LooseLab/ROBIN/issues) for reproducible software problems and feature requests. -```bash -robin add-panel /path/to/your_panel.bed MyCustomPanel -robin add-panel /path/to/your_panel.bed MyCustomPanel --validate-only -``` - -Names must be non-empty and not reserved (`rCNS2`, `AML`). - -**Remove a custom panel** - -```bash -robin remove-panel MyCustomPanel -robin remove-panel MyCustomPanel --force -``` - -Built-in panels cannot be removed. - -**Use in a workflow** - -```bash -robin workflow /path/to/bam_files \ - --work-dir ~/results \ - -w target,cnv,fusion \ - --target-panel MyCustomPanel \ - --center Sherwood -``` - ---- - -## Known issues and limitations - -**Release and scope** - -1. This release is intended for testing; feedback is welcome. -2. **Real-time variant calling** is currently unavailable; it is planned to return - post run variant calling **IS** available. -3. All analyses must be interpreted by a qualified expert. - -**Operation and data** - -1. CNV calls use heuristics — verify by visual inspection. -2. **Ctrl+C** attempts graceful shutdown but may not always complete cleanly. -3. CSV export is in development and not yet reliable. -4. To **reanalyse** a dataset, remove the existing results under the ROBIN output folder first. -5. Other issues may exist — please [open an issue](https://github.com/LooseLab/ROBIN/issues) where possible. - ---- - -## Performance - -- **Batched processing** across analysis workflows -- **Memory-aware** behaviour for large or long runs -- **Non-blocking GUI** updates during analysis -- **Progress tracking** with live status - ---- - -## Dependencies - -Python package versions are declared in **`pyproject.toml`**. The **`robin.yml`** conda environment supplies the scientific stack, bioinformatics tools (e.g. samtools, bedtools), and R/Bioconductor packages used by the workflows. - -**Notable Python libraries** include Click, Watchdog, pysam, pandas, NumPy, SciPy, ruptures, tqdm, Ray, and NiceGUI. - -**External tools** include bedtools, samtools, and R (`Rscript`) for parts of the classification stack. - -**Git submodules** (e.g. nanoDX, hv_rapidCNS2) must be initialised as in [Installation](#installation). - ---- +When reporting a workflow problem, include the ROBIN version/commit, operating system, command used, relevant log output and enough information about the input configuration to reproduce the issue without sharing identifiable patient data. ## License -***This software is provided "as is", and is for research use only.*** +ROBIN is distributed under the **CC BY-NC 4.0** license. See [LICENSE](LICENSE). -ROBIN is distributed under a **CC BY-NC 4.0** license. See the `LICENSE` file. That license does not override licenses of third-party tools bundled or invoked by ROBIN. - ---- +Third-party tools, models and resources retain their own licences and terms of use. ## Acknowledgments -Third-party tools and references: - -- [Sturgeon](https://github.com/marcpaga/sturgeon) -- [Rapid-CNS2](https://link.springer.com/article/10.1007/s00401-022-02415-6) -- [Readfish](https://github.com/LooseLab/readfish) -- [cnv_from_bam](https://github.com/adoni5/cnv_from_bam) -- [methylartist](https://github.com/adamewing/methylartist) - -Libraries include [Click](https://click.palletsprojects.com/), [Watchdog](https://python-watchdog.readthedocs.io/), [pysam](https://pysam.readthedocs.io/), [Ray](https://ray.io/), and [NiceGUI](https://nicegui.io/). +ROBIN builds on work from the nanopore and computational pathology communities, including [Sturgeon](https://github.com/marcpaga/sturgeon), [Rapid-CNS2](https://link.springer.com/article/10.1007/s00401-022-02415-6), [Readfish](https://github.com/LooseLab/readfish), `cnv_from_bam` and `methylartist`. -Thanks to everyone who contributed to these ecosystems, including colleagues in Nottingham and beyond. We are particularly grateful to Areeba Patel, Felix Sahm and colleagues for Rapid-CNS2. The list is non-exhaustive; the software is under active development. +ROBIN is developed by the **Loose Lab, University of Nottingham**, with collaborators at **Nottingham University Hospitals NHS Trust** and other centres. diff --git a/docs/analyses/cnv.md b/docs/analyses/cnv.md new file mode 100644 index 00000000..6c289b37 --- /dev/null +++ b/docs/analyses/cnv.md @@ -0,0 +1,93 @@ +# Copy-number analysis + +ROBIN's CNV analysis derives a genome-wide copy-number profile from aligned nanopore reads and updates that profile as additional BAM chunks arrive. + +The workflow-facing implementation is in `src/robin/analysis/cnv_analysis.py`. + +## Purpose + +The CNV pipeline is intended to provide rapid evidence for broad gains and losses and to identify candidate copy-number breakpoints. It complements targeted coverage and structural-event analysis rather than replacing them. + +!!! warning + CNV calls are heuristic. Always review the copy-number profile visually and interpret it in the context of sequencing depth, sample quality, tumour purity, and other molecular/pathological evidence. + +## Processing model + +The implementation provides: + +- sample-versus-reference copy-number analysis; +- dynamic bin-width calculation; +- incremental state across BAM chunks; +- smoothing/normalisation of the copy-number signal; +- change-point/breakpoint detection using `ruptures`; +- sample-level persisted outputs used by the GUI and reporting code. + +```mermaid +flowchart LR + A[Aligned BAM chunks] --> B[Read-depth / CNV extraction] + B --> C[Accumulate sample signal] + D[Reference CNV data] --> E[Normalisation] + C --> E + E --> F[Smoothed CNV profile] + F --> G[Change-point detection] + G --> H[Plots / candidate breakpoints] +``` + +## Reference data + +The analysis uses reference CNV data distributed/resolved through ROBIN resources. The reference dictionary is cached in memory to avoid repeatedly loading the same large object for every sample. + +For reproducible interpretation, the input BAM, reference FASTA, and CNV reference resources must correspond to compatible genome coordinates/contig naming. + +## Incremental processing + +ROBIN receives multiple BAM chunks for the same sample. CNV state is therefore cached/accumulated at sample level. As sequencing progresses, additional reads refine the profile rather than producing unrelated per-BAM CNV plots. + +This means a CNV result observed very early in sequencing may change as coverage increases. + +## Breakpoint detection + +ROBIN applies change-point detection to the copy-number signal to identify candidate transitions between segments. + +Candidate breakpoints near chromosome ends and within centromeric satellite regions are filtered because these regions are especially problematic for robust copy-number segmentation. The current implementation also applies a telomere-proximity margin around chromosome tips. + +A detected change point is evidence for a transition in the dosage profile; it does not by itself establish the exact structural junction responsible for that change. + +## Relationship to structural-event analysis + +CNV and fusion/structural analysis answer different questions: + +- **CNV** asks whether dosage changes across genomic intervals. +- **Fusion analysis** asks whether individual reads/alignments support candidate rearrangement junctions. + +A deletion or amplification can be apparent in the CNV profile without sufficient read-level evidence to identify its precise junction. Conversely, a structural rearrangement can be detected without producing a large copy-number change. + +## GUI interpretation + +The GUI provides genome-wide and chromosome-level views of accumulated CNV results. When interpreting them, consider: + +- whether a change is supported across multiple adjacent bins; +- whether the profile is globally noisy; +- whether the event is close to a centromere/telomere or other difficult region; +- whether target coverage or structural-event evidence supports the same event; +- whether the apparent copy-number magnitude is plausible given tumour purity. + +## Troubleshooting + +### No CNV result + +Check that: + +- `cnv` is enabled in the workflow; +- BAMs are aligned to the expected reference; +- the required CNV reference resource is installed/available; +- preprocessing successfully resolved a sample ID; +- job logs do not report an import/resource error. + +### Very noisy profile + +Common causes include low accumulated coverage, uneven sequencing, incompatible reference/contig naming, or problematic input data. Allow additional BAM chunks to accumulate before interpreting an early noisy profile. + +### Breakpoint does not match an exact read junction + +This is expected in some cases. CNV segmentation detects changes in dosage and does not guarantee that a rearrangement junction can be resolved from the same data. Review fusion/structural-event outputs where appropriate. \ No newline at end of file diff --git a/docs/analyses/index.md b/docs/analyses/index.md new file mode 100644 index 00000000..c427cdf1 --- /dev/null +++ b/docs/analyses/index.md @@ -0,0 +1,134 @@ +# Analysis pipelines + +ROBIN combines complementary analyses from the same stream of aligned Oxford Nanopore BAM files. Some jobs operate directly on BAM alignments; others use methylation data generated by the BED/parquet conversion stage. + +This section describes what each major analysis is intended to measure, its important dependencies, and how it fits into the real-time workflow. + +!!! warning "Research use only" + ROBIN is research software. Its outputs, including classifications, copy-number profiles, methylation predictions, structural-event candidates, and variant annotations, require expert interpretation and must not be treated as standalone clinical results. + +## Pipeline map + +```mermaid +flowchart TD + A[Aligned BAM chunk] --> B[Preprocessing] + B --> C[BED / parquet conversion] + B --> D[CNV] + B --> E[Target analysis] + B --> F[Fusion] + B --> G[MGMT] + B --> H[ITD] + C --> I[Sturgeon] + C --> J[NanoDX] + C --> K[PanNanoDX] + C --> L[Random forest] +``` + +The workflow plan determines which branches are active. + +## Input assumptions + +ROBIN expects aligned BAMs produced during nanopore sequencing. For the standard real-time workflow: + +- use HAC basecalling or better; +- enable CpG-context 5mC/5hmC modified-base calling when methylation analyses are required; +- perform alignment in MinKNOW; +- use read-count-based BAM rollover; +- keep each BAM at or below 50,000 reads for the supported real-time path; +- use a reference assembly/contig naming scheme consistent with the reference supplied to ROBIN. + +See [MinKNOW configuration](../getting-started/minknow-configuration.md) for setup details. + +## Analysis summary + +| Analysis | Primary input | Main purpose | +| --- | --- | --- | +| Preprocessing | BAM | Validate input and extract sample/run metadata | +| BED conversion | BAM modified-base calls | Produce reusable methylation parquet data | +| Sturgeon | Methylation parquet | Neural-network methylation classification | +| NanoDX | Methylation parquet | Neural-network methylation classification | +| PanNanoDX | Methylation parquet | Pan-cancer methylation classification | +| Random forest | Methylation data | Rapid-CNS2 random-forest classification | +| MGMT | BAM / methylation calls | MGMT promoter methylation prediction | +| CNV | BAM alignments | Copy-number profile and candidate breakpoints | +| Target | BAM alignments | Targeted coverage and variant-analysis preparation | +| Fusion | BAM alignments | Candidate rearrangement/fusion detection | +| ITD | BAM alignments | Internal tandem duplication/insertion hotspot detection | + +## Methylation classifiers + +ROBIN supports multiple methylation classifiers because they provide complementary models and classification frameworks. Their scores should be interpreted in the context of the model being used rather than compared as though all classifiers share the same calibration. + +### Sturgeon + +Sturgeon consumes the parquet methylation representation produced by BED conversion and uses the installed Sturgeon model/runtime to generate methylation-class predictions. The analysis is incremental: as more methylation observations become available, ROBIN can update the sample classification. + +See [Methylation classification](methylation-classification.md). + +### NanoDX and PanNanoDX + +NanoDX and PanNanoDX use neural-network models supplied with/downloaded for ROBIN. The classification implementation runs model inference in a separate process, helping isolate model execution from the main workflow worker. + +See [Methylation classification](methylation-classification.md). + +### Random forest + +The random-forest analysis uses the Rapid-CNS2 resources and R-based model code. It is generally a heavier/slower classification path and is scheduled separately from latency-sensitive work. + +See [Methylation classification](methylation-classification.md). + +## MGMT promoter methylation + +MGMT analysis focuses on the promoter region at `chr10:129466536-129467536` in the reference coordinate system used by ROBIN. The implementation extracts locus-specific data, evaluates methylation, applies the prediction model, and generates outputs for display/reporting. + +See [MGMT](mgmt.md). + +## Copy-number analysis + +The CNV pipeline derives genome-wide copy-number information from aligned reads, compares the sample signal with reference data, and performs segmentation/breakpoint detection. The implementation supports incremental state and dynamic binning. + +CNV results are heuristic and should always be reviewed visually and in the context of sequencing depth and genome-wide behaviour. + +See [Copy number](cnv.md). + +## Target analysis and variants + +Target analysis measures coverage over the active target panel and produces data used elsewhere in ROBIN. The module also contains downstream SNP-analysis support, including Clair3 execution and annotation paths using snpEff/SnpSift and ClinVar resources where configured. + +Variant calling has additional software/runtime requirements, notably Docker for the Clair3 path. + +See [Target coverage and variants](target-and-variants.md). + +## Fusion analysis + +Fusion analysis uses alignment structure, particularly supplementary alignments, to identify reads supporting candidate rearrangements involving targeted genes. Preprocessing persists supplementary-read identifiers so downstream fusion processing can work from the complete set associated with each BAM. + +Evidence is staged and accumulated across BAM chunks for each sample before sample-level candidate outputs are generated. + +See [Structural events](structural-events.md). + +## ITD analysis + +ITD analysis scans configured hotspot regions for insertion/internal tandem duplication evidence. Hotspots can be resolved from the active panel or explicit configuration. Like fusion analysis, evidence can be staged per BAM and accumulated across the sample. + +See [Structural events](structural-events.md). + +## Choosing a workflow + +A broad CNS workflow may enable target, CNV, fusion, MGMT, and several methylation classifiers. A different target panel or experimental question may justify a smaller workflow. + +Use: + +```bash +robin list-job-types +``` + +to inspect the job types available in the installed version, and: + +```bash +robin workflow --help +``` + +for workflow-plan options. + +For the orchestration model behind these analyses, see [ROBIN architecture](../architecture/overview.md). diff --git a/docs/analyses/methylation-classification.md b/docs/analyses/methylation-classification.md new file mode 100644 index 00000000..844f43ce --- /dev/null +++ b/docs/analyses/methylation-classification.md @@ -0,0 +1,122 @@ +# Methylation classification + +ROBIN supports several methylation-based classifiers. They share an upstream modified-base extraction path but use different models and inference implementations. + +## Shared methylation input + +For the standard classifier workflow, aligned BAM chunks first pass through preprocessing and then `bed_conversion`. The conversion stage uses ROBIN's `matkit` utilities to extract modified-base information and maintain a sample-level parquet representation. + +```mermaid +flowchart LR + A[BAM chunks] --> B[Preprocessing] + B --> C[BED conversion] + C --> D[Sample methylation parquet] + D --> E[Sturgeon] + D --> F[NanoDX] + D --> G[PanNanoDX] + D --> H[Random forest] +``` + +This shared conversion prevents every classifier from independently repeating the same BAM-to-methylation extraction work. + +## Modified-base requirements + +Methylation classification depends on modified-base tags in the input BAM. For the supported sequencing configuration, enable **5mC/5hmC in CpG contexts** in MinKNOW. All-context modified-base calling is not the expected input mode. + +The BAM should already be aligned. ROBIN is not intended to realign the incoming sequencing data as part of this path. + +## Sturgeon + +The Sturgeon implementation is in `src/robin/analysis/sturgeon_analysis.py`. + +It: + +1. receives the sample methylation parquet produced by BED conversion; +2. validates and loads the methylation observations; +3. converts the data into the representation required by Sturgeon; +4. performs model inference using the installed Sturgeon/ONNX runtime; +5. writes classification results into the sample output structure for the GUI and reporting layer. + +The analysis is designed to be rerun as the accumulated methylation dataset grows, allowing the classification to evolve during sequencing. + +### Interpretation + +A Sturgeon result is a model score/classification, not an independent pathological diagnosis. Early in a run, limited informative CpGs may make scores unstable. Interpretation should therefore consider the amount of accumulated data and the trajectory of predictions over time. + +## NanoDX + +NanoDX is implemented in `src/robin/analysis/nanodx_analysis.py` and uses the NanoDX neural-network classifier bundled through ROBIN's submodule/model resources. + +The inference step is launched in a separate Python process. This keeps model execution isolated from the main workflow worker and allows inference failures to be handled without taking down the orchestrator. + +The analysis returns class labels and prediction values together with metadata such as the number of features used. + +## PanNanoDX + +PanNanoDX uses the same analysis module and overall processing path as NanoDX but selects the PanNanoDX model/class set. + +Do not confuse **PanNanoDX** with a target-panel name. PanNanoDX remains a methylation classifier even where a similarly named historical target panel has been removed from current ROBIN releases. + +## Random-forest classifier + +The random-forest implementation is in `src/robin/analysis/random_forest_analysis.py`. It integrates the Rapid-CNS2 model resources and R scripts. + +The analysis: + +- consumes accumulated methylation data; +- prepares the required BED/methylation representation; +- invokes the Rapid-CNS2 random-forest workflow; +- stores scores/votes and metadata for the sample; +- supports repeated analysis as more data arrive. + +Because this path is comparatively heavy, the workflow engine can schedule it separately from faster classification and analysis jobs. + +## Comparing classifier outputs + +The classifiers are complementary, but their raw scores are **not directly interchangeable**. They differ in training data, class definitions, feature sets, preprocessing, and model architecture. + +When reviewing a sample: + +- consider agreement or disagreement between classifiers; +- inspect how classifications change as sequencing progresses; +- check whether the predicted labels are represented in the model's class set; +- interpret scores alongside CNV, target, MGMT, fusion/ITD, and pathological information where appropriate. + +A high score from one classifier should not automatically override contradictory evidence from another analysis. + +## Model assets + +After installation, download/refresh model assets with: + +```bash +robin utils update-models +``` + +To force replacement of existing downloaded assets: + +```bash +robin utils update-models --overwrite +``` + +The asset-management path performs checksum verification against ROBIN's asset manifest. + +## Troubleshooting + +### No classifier output + +Check that: + +- the input BAM contains the expected modified-base tags; +- `bed_conversion` is part of the workflow path; +- the model assets have been downloaded; +- the sample parquet exists and is non-empty; +- the relevant classifier job type is enabled; +- job logs do not show an optional dependency/model import failure. + +### Classification remains weak + +Low-confidence or unstable early classifications can reflect insufficient informative methylation observations rather than a software failure. Check the amount of accumulated data and whether subsequent BAM chunks improve the signal. + +### Classifiers disagree + +This is not necessarily an error. The models use different training sets and classification schemes. Review the full evidence rather than trying to transform one model's score into another model's scale. \ No newline at end of file diff --git a/docs/analyses/mgmt.md b/docs/analyses/mgmt.md new file mode 100644 index 00000000..b3fc21f6 --- /dev/null +++ b/docs/analyses/mgmt.md @@ -0,0 +1,71 @@ +# MGMT promoter methylation + +ROBIN includes a focused analysis of methylation in the **MGMT** promoter region. The workflow implementation is in `src/robin/analysis/mgmt_analysis.py`. + +## Region analysed + +The current implementation targets: + +```text +chr10:129466536-129467536 +``` + +Coordinates must be interpreted against the reference assembly expected by ROBIN. Input BAMs should use compatible chromosome names and coordinates. + +## Analysis path + +The MGMT module combines locus extraction, modified-base information, model prediction, and visualisation. + +At a high level it: + +1. checks whether the incoming BAM contains reads overlapping the MGMT locus; +2. extracts/maintains the relevant alignments; +3. safely sorts and indexes the locus-specific BAM data; +4. evaluates methylation at the required sites; +5. applies the configured R-based prediction model; +6. generates sample-level result data and visualisation for the GUI/reporting layer. + +The implementation includes explicit checks that sorted BAMs are complete and readable before indexing, reducing the risk that a partially written file is propagated into downstream analysis. + +## Input requirements + +MGMT analysis requires: + +- an aligned BAM using the expected reference coordinates; +- reads covering the MGMT promoter region; +- modified-base calls suitable for CpG methylation analysis; +- the ROBIN model/R resources installed with the environment. + +If there are no reads at the locus, the absence of an MGMT prediction should not be interpreted as a biological negative result. + +## Real-time behaviour + +As more BAM chunks arrive, additional locus-spanning reads can contribute to the sample-level analysis. Confidence in the methylation estimate therefore depends on accumulated informative coverage. + +Early results should be interpreted cautiously where only a small number of reads/CpG observations are available. + +## Interpretation + +The MGMT result is a model-based methylation prediction. Review it together with the underlying methylation evidence and other sample information. + +Important considerations include: + +- number of informative reads; +- consistency of methylation across relevant CpGs; +- sequencing/basecalling quality; +- whether the expected modified-base model was used; +- whether the reference coordinates match the BAM. + +## Troubleshooting + +### No reads at the MGMT locus + +Confirm that the BAM is aligned, contains `chr10`, and uses the same reference build expected by ROBIN. You can independently inspect the locus with `samtools view` or IGV if necessary. + +### BAM sorting/indexing errors + +The MGMT implementation validates intermediate BAMs before indexing. Errors at this stage can indicate a truncated/incomplete input file or filesystem problem. Check the job log and verify the source BAM with `samtools quickcheck`. + +### No methylation prediction despite coverage + +Check that modified-base tags are present and that CpG-context 5mC/5hmC calling was enabled during basecalling. Also confirm that the required R/model resources are available in the active ROBIN environment. \ No newline at end of file diff --git a/docs/analyses/structural-events.md b/docs/analyses/structural-events.md new file mode 100644 index 00000000..93bce35b --- /dev/null +++ b/docs/analyses/structural-events.md @@ -0,0 +1,104 @@ +# Structural events: fusions and ITDs + +ROBIN contains read-level analyses for structural events that complement the dosage-based CNV pipeline. Two important paths are fusion/rearrangement analysis and ITD/insertion hotspot analysis. + +## Fusion analysis + +The workflow integration is implemented in `src/robin/analysis/fusion_analysis.py`, with the core processing in `src/robin/analysis/fusion_work.py`. + +### What it looks for + +Long nanopore reads can contain primary and supplementary alignments when a read spans a rearrangement breakpoint. ROBIN uses this alignment structure to identify candidate events involving genes/regions in the active target panel. + +```mermaid +flowchart LR + A[Aligned BAM] --> B[Preprocessing] + B --> C[Supplementary-read IDs] + C --> D[Gene-region intersections] + D --> E[Candidate rearrangements] + E --> F[Per-BAM staging] + F --> G[Sample accumulation] + G --> H[Fusion / structural-event outputs] +``` + +### Supplementary alignments + +During preprocessing, ROBIN identifies supplementary-read IDs and persists the complete set under a BAM-specific path in the sample output directory. The fusion handler then loads and validates that set. + +This avoids relying on a truncated in-memory list and allows the downstream analysis to revisit all relevant reads from each BAM chunk. + +### Staging and accumulation + +Fusion evidence is not treated as independent for every BAM. Candidate information is staged per input chunk and accumulated across the sample. This is important because a real event may only gain convincing support after several BAM chunks have arrived. + +### Interpretation + +A candidate fusion/rearrangement should be evaluated using: + +- number of supporting reads; +- mapping quality and uniqueness; +- consistency of breakpoint positions; +- genes/regions involved; +- presence of reciprocal or additional structural evidence; +- corresponding CNV changes where biologically expected. + +Supplementary alignment alone does not prove a clinically meaningful fusion. + +## ITD / insertion analysis + +The workflow integration is in `src/robin/analysis/itd_analysis.py`, with core detection logic in `src/robin/analysis/itd_work.py`. + +The analysis is designed for internal tandem duplications and insertion-like events in configured hotspot regions. + +### Hotspot configuration + +The active scan regions are resolved from the target panel and ITD configuration. Configuration can be supplied through workflow metadata/TOML and is written to the work directory so separate Ray worker processes can use the same settings. + +The implementation supports a configurable region mode and thresholds including minimum event length, minimum frequency, and minimum supporting reads. + +### Incremental evidence + +For each BAM, candidate evidence is staged. ROBIN then force-accumulates the staged candidates at sample level when processing a batch. + +The resulting summary includes the number of processed files, detected events, output paths, and genes/hotspots considered. + +### Panel dependence + +ITD analysis requires an active target panel. If no applicable hotspot overlaps the configured panel, the handler records an empty result rather than scanning arbitrary regions of the genome. + +## Relationship to CNV + +Structural-event and CNV evidence are related but not equivalent. + +A rearrangement can be copy-number neutral, so it may be visible in split/supplementary alignments without a strong CNV signal. Conversely, a broad deletion can be obvious from dosage while its exact junction is not represented by enough informative reads to call structurally. + +For important events, review both forms of evidence. + +## Input requirements + +For fusion/ITD analysis: + +- BAMs must already be aligned; +- supplementary alignments should be retained by the alignment process; +- the target panel must use coordinates compatible with the BAM/reference; +- enough reads must span the event to support interpretation. + +Alignment settings that remove useful supplementary/split-read information can reduce sensitivity. + +## Troubleshooting + +### Fusion analysis produces no candidates + +This can be a true negative, but also check whether preprocessing found supplementary alignments and whether the relevant genes are present in the active panel. + +### ITD analysis reports no scan windows + +Check the active `--target-panel` and ITD hotspot configuration. The handler deliberately returns an empty result if no hotspot is applicable. + +### Candidate support changes over time + +This is expected in real-time operation. Both analyses accumulate evidence as additional BAM chunks arrive. A weak early candidate can gain or lose relative support as the dataset grows. + +### Event conflicts with CNV + +Do not assume one analysis is necessarily wrong. Structural rearrangements and dosage changes describe different properties of the tumour genome. Inspect the read-level evidence and genomic context. \ No newline at end of file diff --git a/docs/analyses/target-and-variants.md b/docs/analyses/target-and-variants.md new file mode 100644 index 00000000..019eb931 --- /dev/null +++ b/docs/analyses/target-and-variants.md @@ -0,0 +1,92 @@ +# Target coverage and variants + +Target analysis operates on the active ROBIN target panel. It provides targeted coverage information and prepares data used by downstream variant-analysis paths. + +The main implementation is in `src/robin/analysis/target_analysis.py`. + +## Target panels + +A target panel defines genomic intervals/genes of interest. ROBIN ships with supported built-in panels and can register custom BED-based panels through the CLI. + +Use: + +```bash +robin list-panels +``` + +to inspect available panels. See [Panel commands](../cli/panels.md) for adding and removing custom panels. + +## Coverage analysis + +For each sample, target analysis accumulates aligned-read evidence over panel regions and generates target-level coverage summaries. These outputs support both direct interpretation in the GUI and downstream steps that need to know which regions have sufficient sequencing evidence. + +Because sequencing is continuous, coverage should be viewed as a time-dependent quantity: a target below threshold early in a run may exceed the threshold later. + +## Variant-analysis path + +`target_analysis.py` also contains support for downstream SNP analysis. The current implementation includes: + +- preparation of target BAM data; +- Clair3-based variant calling; +- snpEff annotation; +- SnpSift annotation against ClinVar where the required database/index is available; +- chromosome-name normalisation for ClinVar annotation; +- generation of display-ready variant data. + +The variant path has additional dependencies beyond target coverage itself. + +## Docker requirement + +The Clair3 path uses Docker. ROBIN checks whether the Docker Python package is installed and whether the Docker daemon is reachable before attempting this analysis. + +If Docker is unavailable, target coverage can still be useful, but the Docker-dependent variant step will not run successfully. + +## ClinVar resources + +ROBIN expects a bgzipped ClinVar VCF and tabix index for SnpSift annotation. Assets can be refreshed with: + +```bash +robin utils update-clinvar +``` + +The implementation attempts to ensure that the tabix index exists before annotation. + +## Reference and chromosome naming + +Variant annotation frequently combines resources that use different contig naming conventions. For example, one VCF may use `chr1` while another uses `1`. + +ROBIN includes a normalisation/restoration step around ClinVar annotation to handle this mismatch, but all primary sequencing/reference resources should still be based on the same genome assembly. + +## Interpretation + +Coverage and variant calls answer different questions: + +- coverage indicates how much sequencing evidence exists over a target; +- a variant caller evaluates sequence differences supported by those reads; +- annotation adds external information to called variants. + +A region having high coverage does not imply that a variant is present, and a variant observed at low coverage requires particular caution. + +## Troubleshooting + +### Target panel not recognised + +Run `robin list-panels` and verify the panel name supplied to `--target-panel`. For custom panels, confirm registration succeeded and the BED coordinates use the expected reference. + +### Variant calling does not start + +Check Docker first: + +```bash +docker info +``` + +Also verify that the current user has permission to access the Docker daemon and that the required image can be obtained. + +### ClinVar annotation missing + +Refresh ClinVar resources with `robin utils update-clinvar` and check that both the compressed VCF and `.tbi` index are present in the ROBIN resources location. + +### Coverage differs from expectation + +Confirm the BAM/reference/panel coordinate systems match, then inspect individual alignments in the affected target. Differences can also arise from mapping quality, supplementary/secondary alignments, or incomplete accumulated sequencing depth. \ No newline at end of file diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md new file mode 100644 index 00000000..ca51e824 --- /dev/null +++ b/docs/architecture/overview.md @@ -0,0 +1,172 @@ +# ROBIN architecture + +ROBIN is a real-time analysis system for aligned Oxford Nanopore BAM data. The application combines file watching, workflow orchestration, analysis handlers, persistent per-sample outputs, and a NiceGUI web interface. + +This page describes the architecture implemented in `release_candidate2` and is intended for developers and advanced users who need to understand how data moves through ROBIN. + +## High-level data flow + +```mermaid +flowchart LR + A[MinKNOW aligned BAM chunks] --> B[File watcher] + B --> C[Preprocessing] + C --> D[Workflow coordinator] + D --> E[BED / methylation conversion] + D --> F[CNV] + D --> G[Target analysis] + D --> H[Fusion] + D --> I[MGMT] + D --> J[ITD] + E --> K[Sturgeon] + E --> L[NanoDX / PanNanoDX] + E --> M[Random forest] + F --> N[Per-sample results] + G --> N + H --> N + I --> N + J --> N + K --> N + L --> N + M --> N + N --> O[NiceGUI] + N --> P[Reports / exports] +``` + +The exact jobs that run are selected by the workflow plan and target panel. Not every analysis is required for every run. + +## Main components + +### Command-line entry point + +`src/robin/cli.py` is the principal command-line interface. The `robin workflow` command configures the input directory, output directory, reference, target panel, workflow plan, logging, GUI integration, and execution options. + +The CLI should be treated as the supported public entry point. Analysis modules can be imported directly for development and testing, but normal operation should use `robin workflow`. + +### Workflow engine + +`src/robin/workflow_ray.py` contains the Ray-based workflow engine. It provides: + +- specialised workers for different job queues; +- a central coordinator for submission, triggering, deduplication, and statistics; +- per-job batching; +- workflow context objects that carry metadata and results between stages; +- optional GUI update hooks; +- job-specific logging. + +Preprocessing is deliberately unbatched because every BAM must first be inspected independently. Most downstream analyses can accumulate multiple BAM chunks for a sample before performing work. + +The default batch policy uses small idle timeouts for responsiveness and longer busy timeouts so that files arriving while a worker is occupied can be combined into larger batches. Batch behaviour can be tuned with the `ROBIN_BATCH_TIMEOUT_*` environment variables. + +### Workflow context + +Each submitted file travels through the workflow with a `WorkflowContext`. The context contains: + +- the current file path; +- metadata, including BAM metadata and sample identity; +- results produced by completed jobs; +- processing history; +- errors; +- batch metadata when jobs are grouped. + +The sample ID is normally established during preprocessing and is then used to group downstream work and write sample-specific results. + +## Preprocessing + +`src/robin/analysis/bam_preprocessor.py` is the first analytical stage for BAM input. It validates and inspects each BAM and extracts metadata used by downstream jobs. + +Important preprocessing responsibilities include: + +- determining sample and run metadata from BAM headers/tags; +- distinguishing pass/fail input where relevant; +- checking modified-base configuration; +- collecting run, device, flow-cell, basecalling, and read statistics; +- identifying supplementary alignments needed by fusion analysis; +- persisting supplementary-read IDs under the sample output directory; +- updating the master CSV state used by ROBIN. + +ROBIN is designed around relatively small BAM chunks generated continuously by MinKNOW. The supported workflow expects BAMs of no more than 50,000 reads unless the explicit large-BAM override is enabled. + +## Methylation conversion + +Several classifiers consume methylation measurements rather than BAM alignments directly. `src/robin/analysis/bed_conversion.py` converts BAM modified-base information into the parquet representation used by those downstream classifiers. + +The conversion layer uses ROBIN's `matkit` utilities and can resolve the reference FASTA from the workflow arguments, job metadata, or output directory. A reference-CpG mode is available through `ROBIN_MATKIT_CPG_MODE`. + +The resulting methylation data are reused by classifiers rather than independently extracting the same measurements for every model. + +## Analysis handlers + +Workflow-facing analysis modules expose handler functions that accept a workflow job and update its context/results. Major handlers include: + +| Job | Module | Purpose | +| --- | --- | --- | +| `preprocessing` | `bam_preprocessor.py` | BAM validation and metadata extraction | +| `bed_conversion` | `bed_conversion.py` | Modified-base extraction and parquet generation | +| `mgmt` | `mgmt_analysis.py` | MGMT promoter methylation analysis | +| `cnv` | `cnv_analysis.py` | Genome-wide copy-number analysis and breakpoint detection | +| `target` | `target_analysis.py` | Target coverage and downstream variant-analysis preparation | +| `fusion` | `fusion_analysis.py` / `fusion_work.py` | Structural/fusion candidate detection | +| `itd` | `itd_analysis.py` / `itd_work.py` | ITD/insertion hotspot detection | +| `sturgeon` | `sturgeon_analysis.py` | Sturgeon methylation classification | +| `nanodx` | `nanodx_analysis.py` | NanoDX methylation classification | +| `pannanodx` | `nanodx_analysis.py` | PanNanoDX methylation classification | +| `random_forest` | `random_forest_analysis.py` | Rapid-CNS2 random-forest classification | + +Additional experimental or specialised handlers may be present in the source tree. Use `robin list-job-types` for the job types exposed by the installed version. + +## Per-sample accumulation + +A sequencing run produces many BAM chunks. ROBIN therefore treats most analyses as incremental rather than as isolated single-file jobs. + +Depending on the analysis, a handler may: + +1. stage information extracted from the newly arrived BAM; +2. merge it with previously staged data for the same sample; +3. update cumulative result files; +4. regenerate summaries or plots; +5. notify the GUI that newer results are available. + +Fusion and ITD analysis make this staging/accumulation pattern particularly explicit. This design allows evidence distributed across multiple BAM chunks to contribute to the same sample-level result. + +## GUI + +The NiceGUI application is primarily implemented in `src/robin/gui_launcher.py` with supporting components under `src/robin/gui/`. It reads the evolving per-sample outputs and presents run activity, classifications, copy number, coverage, MGMT, structural events, and other enabled analyses. + +The GUI is a presentation and monitoring layer: analytical work remains in the workflow handlers rather than being performed in browser callbacks. + +## Output model + +`--work-dir` is the root for ROBIN state and results. Downstream analyses create sample-specific files beneath this location. These files serve three purposes: + +- persistence across a long-running sequencing session; +- communication between incremental analysis steps; +- input to the GUI and reporting code. + +For that reason, deleting or manually editing files in an active ROBIN work directory can invalidate accumulated state. For a clean reanalysis, use a fresh output directory or remove the previous analysis outputs before restarting. + +## Design principles + +The current architecture reflects several practical requirements of real-time nanopore analysis: + +**Incremental processing.** New BAM chunks should refine existing sample results rather than restart an analysis from zero. + +**Separation of orchestration and analysis.** Ray/coordinator code schedules jobs; analysis modules implement biological computations. + +**Sample-aware state.** Evidence from multiple BAMs is grouped by sample identity. + +**Responsive first results.** Initial batches are dispatched quickly, while later chunks can be accumulated for efficiency. + +**Fault isolation.** Handlers are imported defensively and job errors are attached to workflow context so that failure of one analysis does not necessarily prevent unrelated analyses from running. + +## Where to start when developing + +For changes to: + +- CLI behaviour: start with `src/robin/cli.py`; +- scheduling, batching, queues, or triggers: `src/robin/workflow_ray.py`; +- BAM metadata: `src/robin/analysis/bam_preprocessor.py`; +- a biological analysis: the corresponding module under `src/robin/analysis/`; +- GUI behaviour: `src/robin/gui_launcher.py` and `src/robin/gui/`; +- reports: `src/robin/reporting/`. + +See [Analysis pipelines](../analyses/index.md) for the biological and computational role of each major analysis. \ No newline at end of file diff --git a/docs/cli/index.md b/docs/cli/index.md index 1808acbe..b6a8b109 100644 --- a/docs/cli/index.md +++ b/docs/cli/index.md @@ -1,11 +1,8 @@ # Command-line reference -!!! abstract "What this section covers" - The **`robin`** CLI (Click): **`workflow`**, **job types**, **panels**, **utilities**, and **default admin password**. For a first run, see [Installation](../getting-started/installation.md) and [Quickstart](../getting-started/quickstart.md). +This section documents the `robin` command-line interface. For a first run, use the [Quickstart](../getting-started/quickstart.md); for installation, see [Installation](../getting-started/installation.md). -Run **`robin --help`** and **`robin --help`** for options in your build. - ---- +Run `robin --help` and `robin --help` to confirm options in your installed version. ## Commands @@ -13,61 +10,53 @@ Run **`robin --help`** and **`robin --help`** for options in your buil
### [`robin workflow`](workflow.md) -Watch a BAM directory and run the selected pipeline (Ray, optional NiceGUI). +Watch a BAM directory and run the selected pipeline using Ray or threaded execution, with optional NiceGUI monitoring.
### [`robin list-job-types`](jobs.md) -Print job types and queue layout for `-w` strings. +Show available analysis job types and how they map to workflow queues.
-### [Panels](panels.md) -`list-panels`, `add-panel`, `remove-panel` — built-in and custom BED panels. +### [Panel commands](panels.md) +List built-in panels and add or remove custom BED panels.
### [`robin utils`](utils.md) -Models, ClinVar, `sequencing-files`, `mgmt`, and more. +Stage references and panels, update model/ClinVar resources and run utility analyses.
### [`robin password`](password.md) -Set or replace the default admin password for GUI sign-in. +Set or replace the default administrator password used by the web interface.
---- - -## Disclaimer (`I agree`) +## Startup and consent -Commands that start processing or change protected state usually show a **research-use disclaimer** and require typing **`I agree`** exactly. +Commands that start protected processing may display the research-use disclaimer and require `I agree`. The workflow can also prompt for initial GUI-user setup. ---- +See [Starting ROBIN](../getting-started/startup.md) for the complete startup sequence. -## Environment variables (selected) +## Selected environment variables | Variable | Effect | |----------|--------| -| `GITHUB_TOKEN` | Private GitHub assets for `robin utils update-models`. | -| `ROBIN_PROCESS_LARGE_BAMS` | Warns that large-BAM mode must not run alongside live sequencing. | -| `LJ_BAM_THREADS` | Optional BAM threading ([README — Performance](https://github.com/LooseLab/ROBIN/blob/main/README.md#performance)). | - ---- +| `ROBIN_PROCESS_LARGE_BAMS` | Enables large-BAM behaviour intended for non-live processing; ROBIN warns against combining this with live sequencing. | +| `LJ_BAM_THREADS` | Controls optional BAM decompression/read threading. | -## Startup behaviour - -**`robin workflow`** runs model checks, optional reference validation, **`I agree`**, and (if the GUI is on) may prompt for the **default admin password** when no GUI users exist — see **[What happens at startup](../getting-started/startup.md)**. - ---- +Environment variables specific to individual analyses are documented with those analyses or their job configuration. ## Web interface -Browser guide: **[Using ROBIN](../using-robin/index.md)**. - ---- +For browser navigation, authentication, sample pages and result interpretation, see [Using ROBIN](../using-robin/index.md). -## See also +## Related -- [README — Command reference](https://github.com/LooseLab/ROBIN/blob/main/README.md#command-reference) +- [`robin workflow`](workflow.md) +- [Job types](jobs.md) +- [Panel commands](panels.md) +- [`robin utils`](utils.md) diff --git a/docs/cli/jobs.md b/docs/cli/jobs.md index 7826210b..249972d6 100644 --- a/docs/cli/jobs.md +++ b/docs/cli/jobs.md @@ -8,68 +8,92 @@ robin list-job-types ``` -You must complete the **disclaimer** (`I agree`). +You must complete the **disclaimer** (`I agree`) unless consent has already been recorded for an active administrator. ## Queues and job types -The orchestration layer assigns each job type to a queue (simplified names here; internal Ray queue names may differ slightly): +The `release_candidate2` CLI registers the following workflow handlers: -| Queue (concept) | Job types | Role | -|-----------------|-----------|------| -| **Preprocessing** | `preprocessing` | Read BAM headers/metadata; entry point for each new file. | -| **BED conversion** | `bed_conversion` | Prepare inputs for classifiers that need BED-level views. | -| **Analysis** | `mgmt`, `cnv`, `target`, `fusion` | Methylation (MGMT), copy number, targeted variant/fusion panels. | -| **Classification** | `sturgeon`, `nanodx`, `pannanodx` | Methylation / expression classifiers. | -| **Slow** | `random_forest`, `marlin`, `lamprey` | Heavier models (RF; MARLIN TF; Lamprey ONNX research-only). | +| Queue | Job types | Role | +| --- | --- | --- | +| **Preprocessing** | `preprocessing` | Validate each BAM and extract sample/run metadata. | +| **BED conversion** | `bed_conversion` | Extract modified-base data and maintain classifier input. | +| **MGMT** | `mgmt` | MGMT promoter methylation analysis. | +| **CNV** | `cnv` | Genome-wide copy-number analysis. | +| **Target** | `target` | Target coverage and downstream variant-analysis preparation. | +| **Fusion / structural** | `fusion`, `itd` | Fusion/rearrangement and ITD/insertion analysis. | +| **Classification** | `sturgeon`, `nanodx`, `pannanodx` | Methylation classifiers. | +| **Slow** | `random_forest`, `marlin`, `lamprey`, `tucan` | Heavier/specialised classifiers and research analyses. | + +The older queue-qualified workflow syntax may use legacy queue names internally; the simplified job list above is preferable for normal use. ## Automatic steps -When you use the **simplified** workflow format (`-w mgmt,sturgeon`, …): +When you use the simplified workflow format (`-w mgmt,sturgeon`, for example), ROBIN constructs the required pipeline around the requested analyses. -1. **`preprocessing`** is prepended if you did not list it. -2. **`bed_conversion`** is inserted when any of **`sturgeon`**, **`nanodx`**, **`pannanodx`**, **`random_forest`**, **`marlin`**, or **`lamprey`** appear — those jobs expect BED conversion upstream. +In particular, classifier jobs that consume the methylation parquet require `bed_conversion` upstream. Preprocessing is the entry point for new BAM files and establishes metadata used by downstream jobs. -You do not need to list `bed_conversion` manually for those classifiers unless you are hand-editing **legacy** queue-prefixed pipelines. +You normally do not need to manually construct the legacy queue-prefixed form. ## Valid job type names -The CLI accepts only these **job** identifiers in workflow strings: +The CLI handler configuration in `release_candidate2` includes: -`preprocessing`, `bed_conversion`, `mgmt`, `cnv`, `target`, `fusion`, `sturgeon`, `nanodx`, `pannanodx`, `random_forest`, `marlin`, `lamprey` +`preprocessing`, `bed_conversion`, `mgmt`, `cnv`, `target`, `fusion`, `itd`, `sturgeon`, `nanodx`, `pannanodx`, `random_forest`, `marlin`, `lamprey`, `tucan` -Unknown names produce warnings and are skipped. +Use `robin list-job-types` as the authoritative runtime list for your installed checkout. ## Examples ```bash -# Minimal classifier run (preprocessing + bed_conversion added as needed) -robin workflow /data/bams -w sturgeon --center Demo --target-panel rCNS2 -d /out --reference /ref/hg38.fa +# Minimal classifier run +robin workflow /data/bams \ + -w sturgeon \ + --center Demo \ + --target-panel rCNS2 \ + -d /out \ + --reference /ref/hg38.fa -# Full stack (typical) +# Broad CNS analysis robin workflow /data/bams \ - -w target,cnv,fusion,mgmt,sturgeon,nanodx,pannanodx,random_forest,marlin,lamprey \ + -w target,cnv,fusion,mgmt,sturgeon,nanodx,pannanodx,random_forest \ --center Sherwood \ --target-panel rCNS2 \ -d ~/results \ --reference ~/references/hg38.fa + +# Add ITD analysis where the active panel/configuration defines applicable hotspots +robin workflow /data/bams \ + -w target,itd \ + --center Demo \ + --target-panel AML \ + -d /out \ + --reference /ref/hg38.fa ``` +## ITD notes + +`itd` shares the structural/fusion scheduling path but has its own handler. It requires an active target panel and resolves scan windows from the ITD hotspot configuration. If no applicable hotspots overlap the active panel, ROBIN records an empty result rather than scanning arbitrary genomic regions. + +See [Structural events: fusions and ITDs](../analyses/structural-events.md). + ## MARLIN notes -- Install the optional extra: `pip install 'robin[marlin]'` (pulls TensorFlow ≥2.16 and `tf-keras` for Keras-2 HDF5 loading; required for Python 3.12+). -- On first MARLIN job, ROBIN downloads `marlin_v1.model.hdf5` (~1.1 GiB) from Zenodo into `~/.cache/robin/marlin/` (override with `ROBIN_MARLIN_MODEL_PATH` / `ROBIN_MARLIN_CACHE_DIR`). -- Default probe genome build is **hg38** (also supports `hg19` / `t2t` via job metadata `marlin_genome_build`). +- Install the optional extra: `pip install 'robin[marlin]'` where required by your checkout. +- Model/runtime requirements are separate from the core ROBIN workflow. +- Treat MARLIN output according to the limitations and licensing of the upstream model. + +## Lamprey notes -## Lamprey notes (research / evaluation only) +Lamprey support is intended for research/evaluation and has upstream licensing/model requirements. Install and use it only where those requirements are satisfied. -**Lamprey is not for clinical care, diagnosis, or medical decision-making.** Its upstream license restricts use to internal non-commercial research and evaluation (Oncode / Cyclomics / UMCU). Contact `software@cyclomics.com` for clinical/commercial licensing. +## Experimental/specialised jobs -- Install Lamprey **separately** (ROBIN does not vendor it), e.g. `pip install git+ssh://git@github.com/princessmaximacenter/lamprey.git`, plus `pip install 'robin[lamprey]'`. -- Acknowledge the research terms before first model download: `export ROBIN_LAMPREY_RESEARCH_ACK=1`. -- On first run, ROBIN downloads the HuggingFace model (`tachterberg/Lamprey`, ~7 GiB) into `~/.cache/robin/lamprey/` (override with `ROBIN_LAMPREY_MODEL_PATH` / `ROBIN_LAMPREY_CACHE_DIR`). -- ROBIN Lamprey support is **hg38 only**. Confidence tiers match Sturgeon (high ≥95%, medium ≥80%). +`marlin`, `lamprey`, and `tucan` are specialised paths and may have additional model, dependency, licensing, or validation requirements. Their presence in `list-job-types` does not imply that all required external resources are installed. ## Related -- [`robin workflow`](workflow.md) -- [Quickstart](../getting-started/quickstart.md) +- [`robin workflow`](workflow.md) +- [Quickstart](../getting-started/quickstart.md) +- [Analysis pipelines](../analyses/index.md) +- [ROBIN architecture](../architecture/overview.md) diff --git a/docs/cli/utils.md b/docs/cli/utils.md index 617d264a..58c9981c 100644 --- a/docs/cli/utils.md +++ b/docs/cli/utils.md @@ -32,7 +32,7 @@ robin utils update-models --overwrite | `--manifest` | Path to `assets.json` (default: packaged manifest or `ROBIN_ASSETS_MANIFEST`). | | `--overwrite` | Replace existing files. | -For **private** GitHub-hosted assets, set **`GITHUB_TOKEN`** before running. +The standard ROBIN asset manifest uses public download locations, so a GitHub access token is not required for normal installation. More context: [Installation — models](../getting-started/installation.md#step-4-download-models-and-clinvar). diff --git a/docs/cli/workflow.md b/docs/cli/workflow.md index ace3cc92..f24726d8 100644 --- a/docs/cli/workflow.md +++ b/docs/cli/workflow.md @@ -1,12 +1,8 @@ # `robin workflow` -!!! abstract "What this page covers" - Flags and behaviour for **`robin workflow`**: synopsis, workflow string formats, Ray vs threading, GUI, logging, file handling, and links to **job types** and **quickstart**. Startup order (disclaimer, password, when the GUI appears): **[What happens at startup](../getting-started/startup.md)**. +This command runs the ROBIN workflow engine on BAM files under a watched directory. For a first run, start with the [Quickstart](../getting-started/quickstart.md); this page is the detailed command reference. -Run the **Little John** orchestrated pipeline on BAM files under a watched directory: preprocessing, optional BED conversion, analyses (MGMT, CNV, target, fusion), classifiers (Sturgeon, NanoDX, PanNanoDX, random forest), and optional **NiceGUI** monitoring. - -!!! info "Startup sequence" - When you run this command, ROBIN checks models, optionally validates `--reference`, asks you to type **`I agree`** to the research disclaimer, then (with the GUI enabled) may prompt for the **default admin password** when no GUI users exist. On the default **Ray** path, the browser UI is only started if **`--work-dir`** is set. Details: **[What happens at startup](../getting-started/startup.md)**. +ROBIN can run preprocessing, BED conversion, MGMT, CNV, target analysis, fusion/ITD detection and multiple classification jobs, with optional NiceGUI monitoring. ## Synopsis @@ -14,145 +10,141 @@ Run the **Little John** orchestrated pipeline on BAM files under a watched direc robin workflow -w --center --target-panel [OPTIONS] ``` -Or load settings from a TOML file: +Or load repeatable settings from TOML: ```bash robin workflow --toml my_settings.toml ``` -CLI flags override values from the TOML file when you pass them explicitly. You can also mix both — for example, keep a site-specific config file and override the input directory per run: - -```bash -robin workflow /path/to/bams --toml my_settings.toml -``` +CLI flags override values from the TOML file when passed explicitly. | Argument / option | Required | Description | |-------------------|----------|-------------| -| `PATH` | Yes* | Directory containing (or receiving) BAM files. Must exist. | -| `-t` / `--toml` | No | TOML file with workflow settings (see [Configuration file](#configuration-file)). | -| `-w` / `--workflow` | Yes* | Comma-separated job types or legacy `queue:job` steps (see [Job types](jobs.md)). | -| `--center` | Yes* | Site or study label (e.g. `Sherwood`, `Auckland`) — used in outputs and reports. | -| `--target-panel` | Yes* | Panel name: built-in (`rCNS2`, `AML`, …) or custom from `robin add-panel`. Run `robin utils sequencing-files --help` to see choices on your install. | -| `-d` / `--work-dir` | No | Base directory for all run outputs. | -| `-r` / `--reference` | No* | Path to reference **FASTA**. If provided, ROBIN validates the file and ensures an index (e.g. `.fai`). Required for analyses that need a reference. | +| `PATH` | Yes* | Directory containing or receiving BAM files. | +| `-t` / `--toml` | No | TOML file containing workflow settings. | +| `-w` / `--workflow` | Yes* | Comma-separated job types or legacy `queue:job` steps. | +| `--center` | Yes* | Site or study label used in outputs and reports. | +| `--target-panel` | Yes* | Built-in or custom panel name. | +| `-d` / `--work-dir` | No | Base directory for run outputs. | +| `-r` / `--reference` | No* | Reference FASTA; required by analyses that need a reference. | -\* Required on the command line **or** in the TOML file when using `--toml`. +\* Required on the command line or supplied through TOML where applicable. ## Configuration file -Use a TOML file to store repeatable run settings. An example ships with the repository at [`examples/workflow.example.toml`](https://github.com/LooseLab/ROBIN/blob/main/examples/workflow.example.toml). +An example ships at `examples/workflow.example.toml`. ```toml path = "empty_folder" workflow = "cnv,fusion,target,mgmt,sturgeon,nanodx,pannanodx,random_forest" center = "NUH" target_panel = "rCNS2" - work_dir = "../../REF_SAMPLES" reference = "~/references/GCA_000001405.15_GRCh38_no_alt_analysis_set.fna" - -# Optional log_level = "INFO" analysis_workers = 2 with_gui = true deduplicate_jobs = ["sturgeon", "mgmt"] ``` -Run: - -```bash -robin workflow --toml my_settings.toml -``` - -TOML keys use the same names as long-form CLI flags (`target_panel`, `work_dir`, `no_process_existing`, etc.). List-valued options (`deduplicate_jobs`, `job_log_level`, `commands`, `queue_priority`) can be TOML arrays. `workflow` can be a comma-separated string or a TOML array of job types. +TOML keys use the same names as long-form CLI flags. List-valued options can be TOML arrays. `workflow` can be a comma-separated string or an array of job types. -## Workflow string formats +## Workflow strings -### Simplified (recommended) - -Comma-separated **job types** only — queues are assigned automatically: +### Simplified format — recommended ```bash -w mgmt,sturgeon -w target,cnv,fusion,mgmt,sturgeon,nanodx,pannanodx,random_forest ``` -Behaviour: +ROBIN inserts required upstream stages such as `preprocessing` and, for classifiers that require it, `bed_conversion`. -- **`preprocessing`** is always inserted as the first step if missing. -- **`bed_conversion`** is inserted **before** classification jobs when needed (`sturgeon`, `nanodx`, `pannanodx`, `random_forest`). +See [Job types](jobs.md) for the currently registered jobs and their queue mapping. -### Legacy (explicit queues) +### Legacy explicit-queue format -Use `queue:job` steps, e.g.: +Legacy workflows can specify `queue:job` pairs, for example: ```text preprocessing:preprocessing,bed_conversion:bed_conversion,mgmt:mgmt,classification:sturgeon ``` +Use the simplified format for new configurations unless explicit queue control is required. + +## Sequencing input requirements + +The supported real-time workflow expects BAMs that: + +- are aligned before ROBIN receives them; +- use a reference compatible with the FASTA supplied through `--reference`; +- contain the required modified-base tags for methylation analyses; +- are rotated by read count rather than elapsed time; +- contain **50,000 reads or fewer per BAM**. + +See [MinKNOW configuration](../getting-started/minknow-configuration.md) for instrument-specific setup. + ## Execution engine | Mode | Flag | Notes | -|------|------|--------| -| **Ray** (default) | `--use-ray` | Distributed task execution; optional Ray dashboard (`--ray-dashboard` / `--no-ray-dashboard`). | -| Threading | `--no-use-ray` | Falls back to threaded workers; analysis worker counts apply per queue mode. | +|------|------|-------| +| Ray | `--use-ray` | Default distributed/task-oriented execution path. | +| Threading | `--no-use-ray` | Threaded fallback execution. | -**Presets** (`--preset`): `p2i`, `standard` (default), `high` — adjust Ray actor grouping and CPU caps for different hardware (e.g. P2i vs server). +Presets such as `p2i`, `standard` and `high` adjust worker grouping and resource allocation for different hardware profiles. -## GUI (NiceGUI) +## Web interface | Option | Default | Description | |--------|---------|-------------| -| `--with-gui` / `--no-gui` | GUI on | Launch workflow monitor in the browser. | +| `--with-gui` / `--no-gui` | On | Enable or disable NiceGUI monitoring. | | `--gui-host` | `0.0.0.0` | Bind address. | -| `--gui-port` | `8081` | Port for the GUI. | +| `--gui-port` | `8081` | Port. | + +The exact startup sequence, consent and initial-user behaviour are documented under [Starting ROBIN](../getting-started/startup.md). ## Logging and progress | Option | Description | |--------|-------------| -| `--log-level` | Global level: `DEBUG`, `INFO`, `WARNING`, `ERROR` (default `ERROR`). | -| `--job-log-level` | Repeatable, e.g. `preprocessing:DEBUG`, `mgmt:WARNING`. | -| `--verbose` / `-v` | Verbose CLI and traces. | -| `--no-progress` | Disable file progress bars. | +| `--log-level` | Global logging level. | +| `--job-log-level` | Per-job logging level, e.g. `preprocessing:DEBUG`. | +| `--verbose` / `-v` | Verbose CLI output and traces. | +| `--no-progress` | Disable progress bars. | ## File handling | Option | Description | |--------|-------------| -| `--no-process-existing` | Only process BAMs that appear **after** startup (skip files already on disk). | -| `--no-watch` | Do not watch the directory for new files (advanced; default is to watch). | +| `--no-process-existing` | Ignore BAMs already present at startup and process only newly arriving files. | +| `--no-watch` | Do not continue watching for new BAM files. | -## Ray tuning +## Ray and worker tuning -| Option | Description | -|--------|-------------| -| `--ray-num-cpus` | Cap CPUs for Ray (auto if omitted). | -| `--queue-priority` | Repeatable, e.g. `preprocessing:10`, `mgmt:5` (Ray mode). | -| `--show-priorities` | Print queue priorities and exit. | -| `--analysis-workers` | Workers per analysis queue (default from code constant, often `1`). | -| `--preprocessing-workers`, `--bed-workers` | Workers for preprocessing and `bed_conversion`. | -| `--legacy-analysis-queue` | Single shared analysis queue instead of per-type queues (threading path). | +Advanced options include `--ray-num-cpus`, `--queue-priority`, `--analysis-workers`, `--preprocessing-workers`, `--bed-workers`, `--show-priorities` and `--legacy-analysis-queue`. -## Deduplication +Use: -`--deduplicate-jobs` (repeatable) — job types that should run **at most once per sample** even when multiple triggers fire (e.g. `sturgeon`, `mgmt`). +```bash +robin workflow --help +``` -## Custom per-job shell commands +for the definitive option list in the installed version. -`--commands` / `-c` (repeatable) — optional mappings `job_type:shell_command` for custom steps (advanced; see `robin workflow --help`). +## Deduplication and custom commands -## Inputs and sequencing assumptions +`--deduplicate-jobs` can restrict selected job types to a single execution per sample where appropriate. -ROBIN expects **real-time** BAMs with **≤ 50,000 reads per file** and **read-count–based** rollover in MinKNOW. See the [README](https://github.com/LooseLab/ROBIN/blob/main/README.md#bam-read-limit-and-minknow-settings). +`--commands` / `-c` can attach custom shell commands to job types. This is an advanced integration feature; verify behaviour carefully before using it in automated runs. -## Exit +## Stopping a workflow -Stop with **Ctrl+C** — the runner attempts graceful shutdown; complex runs may not always exit instantly. +Use **Ctrl+C** to request graceful shutdown. ROBIN attempts to stop watchers, workers, Ray and the GUI cleanly, although complex runs may take a short period to terminate fully. ## Related -- [Job types and queues](jobs.md) -- [Panel commands](panels.md) -- [Utilities — reference files](utils.md#robin-utils-sequencing-files) -- [Quickstart](../getting-started/quickstart.md) +- [Quickstart](../getting-started/quickstart.md) +- [Starting ROBIN](../getting-started/startup.md) +- [Job types](jobs.md) +- [Panel commands](panels.md) +- [Utilities](utils.md) diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md index 470f004b..3cd206b6 100644 --- a/docs/getting-started/installation.md +++ b/docs/getting-started/installation.md @@ -1,74 +1,64 @@ # Installation -!!! abstract "What this page covers" - Install ROBIN from this repository: **clone** → **conda environment** → **editable pip install** → **models & ClinVar** → **verify**. - For **running** workflows (MinKNOW, BAM limits, CLI examples), see the repo **[README](https://github.com/LooseLab/ROBIN/blob/main/README.md)**. This page is **install only**. +Install ROBIN from this repository, create a fresh environment, install the package and download the required model assets. ---- +For running ROBIN after installation, continue to the [Quickstart](quickstart.md). ## Before you start | You need | Notes | |----------|--------| -| **Git** + **[Git LFS](https://git-lfs.com/)** | Clone includes submodules / LFS assets where used. | -| **[Conda](https://docs.conda.io/)** | Miniconda or Anaconda. | -| **Python 3.12** | Provided by the `robin` conda env (`robin.yml`). | +| **Git** | Used to clone ROBIN and its submodules. Git LFS is not required for the current repository. | +| **Conda** | Miniconda or Anaconda. | +| **Python 3.12** | Provided by the `robin` conda environment. | -**Recommended for production:** **64 GB RAM** or more; CPU/GPU per Oxford Nanopore guidance for your sequencer. Docker is optional. +**Recommended system memory:** 64 GB RAM or more for typical production-scale use. CPU/GPU requirements depend on the Oxford Nanopore sequencing setup. Docker is optional for some downstream analysis paths. ---- - -## ROBIN with Little John - -### Step 1: Clone the repository - -Include submodules: +## 1. Clone the repository ```bash git clone --recursive https://github.com/LooseLab/ROBIN.git cd ROBIN ``` -If you already cloned without submodules: +If the repository was cloned without submodules: ```bash git submodule update --init --recursive ``` -### Step 2: Create the conda environment +## 2. Create the conda environment -| File | Role | -|------|------| -| **`robin.yml`** | Main env: Python 3.12, scientific stack, bioinformatics, R/Bioconductor (Linux and macOS). | -| **`robin_linux_extras.yml`** | **Linux only**, optional: use if you hit `libstdc++` / `CXXABI_1.3.15` (see README *Common issues*). | +ROBIN uses the environment definition in `robin.yml`: ```bash conda env create -f robin.yml conda activate robin ``` -- `pyproject.toml` requires Python **≥ 3.12**; this env matches that. -- Prefer a **fresh** env for this codebase—not an older ROBIN env from past releases. +Use a **fresh environment** for this codebase rather than reusing an environment from an older ROBIN release or another project. -#### If the `robin` environment already exists +### If the `robin` environment already exists -`robin.yml` sets **`name: robin`**. If create fails because the env exists: +`robin.yml` defines `name: robin`. Choose one of the following: | Approach | Command | |----------|---------| -| **Update in place** | `conda env update -n robin -f robin.yml --prune` then `conda activate robin` | -| **Remove and recreate** | `conda deactivate` → `conda env remove -n robin` → `conda env create -f robin.yml` | -| **New name** | `conda env create -f robin.yml -n robin_littlejohn` → `conda activate robin_littlejohn` | +| Update in place | `conda env update -n robin -f robin.yml --prune` | +| Remove and recreate | `conda env remove -n robin` followed by `conda env create -f robin.yml` | +| Use another name | `conda env create -f robin.yml -n robin_littlejohn` | -**Linux** — if you see `CXXABI_1.3.15` / wrong `libstdc++`: +Then activate the environment you intend to use. + +### Linux `libstdc++` / `CXXABI_1.3.15` problems + +If native libraries resolve against the system `libstdc++` instead of conda's copy, apply the Linux extras environment: ```bash conda env update -n robin -f robin_linux_extras.yml ``` -(See README *Common issues*.) - -### Step 3: Install ROBIN (editable) +## 3. Install ROBIN From the repository root: @@ -76,57 +66,48 @@ From the repository root: pip install -e . ``` -This installs the `robin` CLI from your working tree. +This installs the `robin` command-line interface from your working tree. -**Optional extras** (install only if needed): +Optional extras are available for features that require additional packages: | Extra | Command | Use | |-------|---------|-----| -| GUI | `pip install -e '.[gui]'` | NiceGUI launcher (often already in `robin.yml`) | -| MinKNOW API | `pip install -e '.[minknow]'` | `robin minknow status/watch`; pin `minknow_api` to your Core version — see [MinKNOW configuration](minknow-configuration.md#optional-programmatic-minknow-integration) | +| GUI | `pip install -e '.[gui]'` | NiceGUI support where not already supplied by the environment | +| MinKNOW API | `pip install -e '.[minknow]'` | Programmatic MinKNOW integration | -### Step 4: Download models and ClinVar +See [MinKNOW configuration](minknow-configuration.md) for instrument integration details. -Assets are SHA256-verified. Set **`GITHUB_TOKEN`** if downloads use private GitHub. +## 4. Download model and annotation assets ```bash robin utils update-models robin utils update-clinvar ``` -Private GitHub: +Model assets are resolved from the public sources defined in ROBIN's asset manifest and are checksum-verified. -```bash -export GITHUB_TOKEN=your_personal_access_token -robin utils update-models -``` - -Force re-download models: +To replace existing model downloads: ```bash robin utils update-models --overwrite ``` -### Step 5: Verify +## 5. Verify the installation ```bash robin --help robin list-job-types ``` ---- +If both commands run successfully, continue to the [Quickstart](quickstart.md). -## Troubleshooting (install) +## Troubleshooting -| Issue | What to do | -|-------|------------| +| Problem | Check | +|---------|-------| | Missing submodules | `git submodule update --init --recursive` | -| Model / ClinVar download failures | Set `GITHUB_TOKEN` if required; retry `robin utils update-models --overwrite` and `robin utils update-clinvar` | -| Wrong conda env | `conda env list` — activate the env created from `robin.yml` | - ---- - -## Next steps +| Model or ClinVar download failure | Check network access and retry the relevant `robin utils` command | +| Wrong environment | Use `conda env list` and activate the environment created from `robin.yml` | +| Linux native-library error | Apply `robin_linux_extras.yml` as described above | -- [Quickstart](quickstart.md) — run a workflow -- [README — Usage](https://github.com/LooseLab/ROBIN/blob/main/README.md#usage) — deep operational detail +For operational problems after ROBIN starts, use [Troubleshooting](../using-robin/troubleshooting.md). diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index f786d740..57989b53 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -1,182 +1,116 @@ # ROBIN Quickstart -!!! abstract "What this page covers" - Run **`robin workflow`** after [installation](installation.md): what ROBIN expects from **MinKNOW/BAMs**, how to stage **reference + panel BEDs**, example commands, and **panels**. - For **disclaimer, default admin password, and startup order**, see **[What happens at startup](startup.md)**. For the **web UI**, see **[Using ROBIN](../using-robin/index.md)**. For every flag, see **[CLI reference](../cli/index.md)** and the **[README](https://github.com/LooseLab/ROBIN/blob/main/README.md)**. +Use this page to get from an installed ROBIN environment to a running analysis with the web interface open. ---- +!!! note + If ROBIN is not installed yet, start with [Installation](installation.md). -## What ROBIN expects from sequencing +## 1. Prepare the reference and panel -ROBIN consumes **aligned BAMs** from Oxford Nanopore (usually written in real time by MinKNOW): - -| Expectation | Notes | -|-------------|--------| -| Basecalling | **HAC** is sufficient; SUP not required. | -| Methylation | Enable **5mC / 5hmC modified-base calling in CpG contexts only** in MinKNOW if your analyses need methylation. Do **not** use all-context calling. | -| Alignment | Done **in MinKNOW** — ROBIN does not realign reads. | -| BAM rollover | **Read-count–based** chunks. **Each BAM must be ≤ 50,000 reads**; we recommend **~50,000 reads per file**. Do **not** rely on **time-only** (e.g. hourly) rollover — see [README — BAM read limit](https://github.com/LooseLab/ROBIN/blob/main/README.md#bam-read-limit-and-minknow-settings). | -| POD5 / FASTQ | Not required; you can turn them off if you only need BAM. | - -!!! tip "Memory on smaller machines" - On **≤ 64 GB RAM**, restart between long runs or after moving the flow cell. Dorado can retain GPU/host memory; restarting Dorado or the instrument after a run reduces OOM risk. - ---- - -## Reference and panel BEDs (one command) {#reference-and-panel-beds-one-command} - -Use **`robin utils sequencing-files`** to gather **panel BED** + **GRCh38 reference** in one folder for MinKNOW and for **`robin workflow --reference`**. Run after [installation](installation.md) when you need a single consistent reference path. +Create a folder containing the reference FASTA and the BED files required for your sequencing setup: ```bash -robin utils sequencing-files --panel rCNS2 --output-dir ~/references/robin_ref +robin utils sequencing-files \ + --panel rCNS2 \ + --output-dir ~/references/robin_ref ``` -| Option | Purpose | -|--------|---------| -| `-p` / `--panel` | **Required.** Same names as `--target-panel` (built-in panels such as `rCNS2`, `AML`; run `robin utils sequencing-files --help` for the list on your install). | -| `-r` / `--reference` | **Reference FASTA:** either an **HTTPS URL** to download, or a **local path** to `.fa` / `.fa.gz`. If omitted, ROBIN uses the default **NCBI GRCh38 no-alt analysis set** (UCSC-style contig names) — a **large** download; use `-r` to point at an existing file if you already have GRCh38. | -| `-o` / `--output-dir` | Output folder (default: **`./reference_files`** in the current directory). | -| `-y` / `--yes` | Skip the confirmation prompt (for scripts). | +By default this stages the built-in `rCNS2` panel and downloads the supported GRCh38 no-alt reference. If you already have a compatible GRCh38 FASTA, provide it with `--reference`. -Use the **same** reference file for **MinKNOW alignment** and **`robin workflow --reference`**. +Use the **same reference FASTA** for MinKNOW alignment and for `robin workflow --reference`. -### Other `robin utils` commands +For all options, see [`robin utils sequencing-files`](../cli/utils.md#robin-utils-sequencing-files). -| Command | Purpose | -|---------|---------| -| `robin utils update-models` | Models / classification assets ([Installation](installation.md)). | -| `robin utils update-clinvar` | ClinVar resources for annotation. | -| `robin utils mgmt` | Summarise **MGMT** CpG methylation from existing `mgmt_sorted.bam` outputs. | +## 2. Configure MinKNOW -Run **`robin utils --help`** for the full list. +ROBIN expects aligned Oxford Nanopore BAM files produced during sequencing. ---- +For the standard real-time workflow: -## Run a workflow +- use HAC basecalling or better; +- enable **5mC/5hmC calling in CpG context** when methylation analyses are required; +- align reads in MinKNOW using the same reference supplied to ROBIN; +- configure BAM output by **read count**, not time; +- keep each BAM at **50,000 reads or fewer**. -Typical invocation: +!!! warning "BAM rollover matters" + Time-based BAM rollover can produce files that are too large for ROBIN's supported real-time workflow. Configure MinKNOW to rotate BAMs by read count; approximately 50,000 reads per BAM is recommended. -```bash -robin workflow --work-dir \ - -w target,cnv,fusion,mgmt,sturgeon,nanodx,pannanodx,random_forest \ - --reference ~/references/hg38_simple.fa \ - --center -``` +For the complete instrument setup, see [MinKNOW configuration](minknow-configuration.md). -| Argument | Meaning | -|----------|---------| -| `` | Directory watched for incoming BAMs | -| `--work-dir` | Root for outputs | -| `-w` | Comma-separated analysis types | -| `--reference` | Reference FASTA (needed for most steps) | -| `--center` | Site label (e.g. `Sherwood`, `Auckland`) | -| `--target-panel` | Panel name, e.g. `rCNS2` | +## 3. Start ROBIN -### Examples +Assuming MinKNOW is writing BAMs to `~/data/bam_files` and the reference is at `~/references/robin_ref/hg38.fa`: ```bash -# Broad workflow with a fixed panel robin workflow ~/data/bam_files \ --work-dir ~/results \ -w target,cnv,fusion,mgmt,sturgeon,nanodx,pannanodx,random_forest \ - --reference ~/references/hg38_simple.fa \ + --reference ~/references/robin_ref/hg38.fa \ --center Sherwood \ --target-panel rCNS2 - -# Smaller selection -robin workflow ~/data/bam_files \ - --work-dir ~/results \ - -w mgmt,sturgeon \ - --reference ~/references/hg38_simple.fa \ - --center Auckland \ - --target-panel AML - -# More logging -robin workflow ~/data/bam_files \ - --work-dir ~/results \ - -w mgmt,cnv,sturgeon \ - --reference ~/references/hg38_simple.fa \ - --center New_York \ - --target-panel rCNS2 \ - --verbose \ - --log-level INFO ``` -Point `--reference` at the **same GRCh38** file you use in MinKNOW—for example from your [`sequencing-files`](#reference-and-panel-beds-one-command) output. - ---- - -## Commands you’ll use often +Replace the paths and `--center` value for your site. -### `list-job-types` +ROBIN automatically adds preprocessing and other required upstream steps where appropriate. To see the analysis names available in your installed version, run: ```bash robin list-job-types ``` -Examples include preprocessing, bed_conversion, mgmt, cnv, target, fusion, sturgeon, nanodx, pannanodx, random_forest (see live output for your version). +For detailed workflow options, see [`robin workflow`](../cli/workflow.md) and [Job types](../cli/jobs.md). -### `workflow` +## 4. Complete startup -```bash -robin workflow /path/to/directory -w "" [OPTIONS] -``` - -**Required:** `-w` / `--workflow`, and `--center`. +On first use, ROBIN may ask you to: -**Useful:** `--work-dir`, `--reference`, `--verbose`, `--log-level`, `--no-process-existing`, `--deduplicate-jobs`, `--use-ray` / `--no-use-ray`, `--with-gui` / `--no-gui` — full list: **`robin workflow --help`**. +1. accept the research-use disclaimer; +2. create the initial GUI administrator password. ---- +With the GUI enabled, ROBIN prints the web address it is serving. Open that address in a browser and sign in. -## Panel management +For the exact startup sequence, authentication behaviour, Ray/threading mode and shutdown behaviour, see [Starting ROBIN](startup.md). -Built-in panels include **rCNS2** and **AML**. Custom panels are registered from BED (at least four columns: chr, start, end, gene name(s)). +## 5. Confirm that the run is working -```bash -robin list-panels -robin add-panel /path/to/panel.bed MyCustomPanel -robin add-panel /path/to/panel.bed MyCustomPanel --validate-only -robin remove-panel MyCustomPanel -robin remove-panel MyCustomPanel --force -``` +Once BAMs appear in the watched directory, confirm that: -You cannot reuse reserved names: `rCNS2`, `AML`. +- ROBIN detects the sample; +- preprocessing jobs complete; +- the sample appears in the web interface; +- enabled analyses begin producing results as data accumulate. ---- +Then continue with: -## Behaviour and limits +- [Using ROBIN](../using-robin/index.md) — navigation, samples and the web interface; +- [Reading your results](../using-robin/sample-results.md) — interpreting the GUI; +- [Analysis pipelines](../analyses/index.md) — what each analysis does; +- [Troubleshooting](../using-robin/troubleshooting.md) — common operational problems. -- **CNV** — Heuristic; **review visually** before clinical use. -- **Stop with Ctrl+C** — Graceful shutdown is attempted but not guaranteed. -- **Issues** — [GitHub issues](https://github.com/LooseLab/ROBIN/issues). +## Common variations -### Performance (brief) +### Run only selected analyses -Batched processing on heavy paths; optional **`LJ_BAM_THREADS`**; non-blocking GUI when NiceGUI is enabled. +You do not need to enable every analysis. For example: -### License - -Research use; see **LICENSE** in the repository. ROBIN integrates tools such as Sturgeon, Rapid-CNS2, Readfish, and others—see the repo for attribution. - ---- - -## Next steps +```bash +robin workflow ~/data/bam_files \ + --work-dir ~/results \ + -w mgmt,sturgeon \ + --reference ~/references/robin_ref/hg38.fa \ + --center Sherwood \ + --target-panel rCNS2 +``` -
+### Use a different panel -
-### Wet lab -[Library preparation](library-preparation.md) -
+Built-in and custom panel management is documented under [Panel commands](../cli/panels.md). -
-### Instrument -[MinKNOW configuration](minknow-configuration.md) -
+### Run without the browser interface -
-### Deep dive -[README — Usage](https://github.com/LooseLab/ROBIN/blob/main/README.md#usage) -
+```bash +robin workflow ... --no-gui +``` -
+See the [workflow command reference](../cli/workflow.md) for other execution options. diff --git a/docs/getting-started/startup.md b/docs/getting-started/startup.md index 28c3f011..847a29cf 100644 --- a/docs/getting-started/startup.md +++ b/docs/getting-started/startup.md @@ -24,7 +24,7 @@ ## 1. Model assets -ROBIN checks **required model files** (same manifest as `robin utils update-models`). If anything is missing, the process exits and tells you to run **`robin utils update-models`** (and **`GITHUB_TOKEN`** if assets are on private GitHub). +ROBIN checks **required model files** using the same manifest as `robin utils update-models`. If anything is missing, the process exits and tells you to run **`robin utils update-models`**. --- diff --git a/docs/index.md b/docs/index.md index ac95e61b..15e80c3e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,90 +8,79 @@ ROBIN is for **research use** at this time. The technology is under active development and validation. !!! abstract "What ROBIN is" - ROBIN helps deliver **same-day molecular insight** from nanopore brain tumour sequencing—so clinicians and families spend less time waiting on classification that can guide treatment. It is built for the lab and the browser: live BAM processing, classifiers, and a web app to follow runs and read results. + ROBIN is a real-time analysis and web-monitoring platform for Oxford Nanopore sequencing of CNS tumours. It combines live BAM processing with methylation classification, copy-number analysis, MGMT analysis, targeted analysis and structural-event detection. ---- - -## Why speed matters +## Get started
-
-12,700 -Annual new cases of brain and other CNS tumours in the UK (2017–2019) +
+### Install ROBIN +[Installation](getting-started/installation.md) covers the conda environment, package installation and required model assets.
-
-35 -People diagnosed **every day** +
+### Run your first workflow +[Quickstart](getting-started/quickstart.md) takes you from an installed environment to a running analysis and web interface.
-
-100+ -Distinct brain tumour types—each with different treatment implications +
+### Configure sequencing +[MinKNOW configuration](getting-started/minknow-configuration.md) covers alignment, modified-base calling and BAM rollover settings.
-*Source: [Cancer Research UK statistics](https://www.cancerresearchuk.org/health-professional/cancer-statistics/statistics-by-cancer-type/brain-other-cns-and-intracranial-tumours) (2017–2019).* - ---- - -## The gap today - -Standard pathways can leave patients and teams waiting **many weeks** for a full molecular picture. That delay affects: - -- **Patients and families** — prolonged uncertainty -- **Treatment planning** — slower decisions on surgery, radiotherapy, trials, and drugs -- **Equity** — access to rapid classification varies by site and geography - -ROBIN is one approach aimed at **hours instead of weeks**, with equitable access regardless of location—see our [Neuro-Oncology publication](https://academic.oup.com/neuro-oncology/advance-article/doi/10.1093/neuonc/noaf103/8139084?searchresult=1). - ---- - -## What ROBIN delivers +## Explore ROBIN
- -### Hours, not weeks -Molecular profiling aligned to your nanopore run—so teams can plan sooner after surgery. +### Use the web interface +[Using ROBIN](using-robin/index.md) covers sign-in, navigation, samples, results and troubleshooting.
- -### Clearer patient journey -Same-day results can reduce anxiety and support shared decision-making. +### Understand the analyses +[Analysis pipelines](analyses/index.md) explains the purpose, inputs and interpretation of ROBIN's analysis workflows.
- -### Built for real workflows -Web monitoring, classification, CNV, MGMT, fusion, and reporting—where your pipeline enables them. +### Command-line reference +[CLI reference](cli/index.md) documents `robin workflow`, job types, panels and utilities.
--- -## Explore the documentation +## Why rapid molecular analysis matters + +Brain and CNS tumours encompass many biologically distinct entities, and molecular information increasingly contributes to classification and research workflows. Conventional diagnostic pathways may take days to weeks to assemble the complete molecular picture. + +ROBIN was developed to explore whether nanopore sequencing and real-time analysis can shorten that interval substantially. The approach is described in our [Neuro-Oncology publication](https://academic.oup.com/neuro-oncology/advance-article/doi/10.1093/neuonc/noaf103/8139084?searchresult=1). + +ROBIN is intended to support research and validation of rapid workflows. Its outputs require expert interpretation and should not be treated as standalone clinical results. + +--- + +## What ROBIN provides
-### Install and run -[Installation](getting-started/installation.md) → [Quickstart](getting-started/quickstart.md) → [Startup behaviour](getting-started/startup.md) +### Real-time processing +ROBIN watches aligned BAM output as sequencing progresses and schedules enabled analyses incrementally.
-### Command line -[CLI overview](cli/index.md) — `robin workflow`, panels, utilities, password. +### Complementary analyses +Methylation classifiers, CNV, MGMT, target/variant analysis and structural-event detection can be combined in one workflow.
-### Web app (browser) -[Using ROBIN](using-robin/index.md) — sign-in, samples, results, troubleshooting. +### Browser-based monitoring +The web interface brings sample metadata, progress and accumulated analysis results together during a run.
@@ -100,27 +89,20 @@ Web monitoring, classification, CNV, MGMT, fusion, and reporting—where your pi ## Partners and adoption -We work with clinicians and surgeons in the UK and internationally to move rapid classification toward routine care. If your organisation is interested in implementing ROBIN: +ROBIN is developed by the **Loose Lab** at the University of Nottingham with collaborators at **Nottingham University Hospitals NHS Trust** and other centres. -- [GitHub repository](https://github.com/LooseLab/robin) -- [Contact form](https://forms.gle/kdX2eiPQPdDUpaBE9) +- [GitHub repository](https://github.com/LooseLab/ROBIN) +- [Contact form](https://forms.gle/kdX2eiPQPdDUpaBE9) --- ## Information for patients and families -ROBIN is a **research and hospital** tool—not a substitute for medical advice. If you are worried about symptoms: - -- **Speak to your GP** urgently about new or worsening neurological symptoms. -- **[Cancer Research UK — brain tumours](https://www.cancerresearchuk.org/about-cancer/brain-tumours)** — trusted overview. -- **[The Brain Tumour Charity](https://www.thebraintumourcharity.org/)** — support and information. -- **[brainstrust](https://www.brainstrust.org.uk/)** — specialist support. -- **[Brain Tumour Research](https://www.braintumourresearch.org/)** — research news. -- **[MyBrainFirst.org](https://www.mybrainfirst.org/)** — symptom awareness. +ROBIN is a research tool and is not a substitute for medical advice. General information and support are available from: -!!! note "Clinical care first" - Early diagnosis and the right care pathway matter most. The organisations above can help while molecular technologies continue to roll out across the NHS and beyond. - ---- +- [Cancer Research UK — brain tumours](https://www.cancerresearchuk.org/about-cancer/brain-tumours) +- [The Brain Tumour Charity](https://www.thebraintumourcharity.org/) +- [brainstrust](https://www.brainstrust.org.uk/) +- [Brain Tumour Research](https://www.braintumourresearch.org/) -ROBIN is developed by the **Loose Lab** (University of Nottingham) with **Nottingham University Hospitals NHS Trust**. [GitHub](https://github.com/LooseLab/robin) +For concerns about symptoms or treatment, contact an appropriate healthcare professional. diff --git a/mkdocs.yml b/mkdocs.yml index bce1f7b9..1c614717 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -19,7 +19,6 @@ theme: - search.share - content.code.copy palette: - # Editorial Bioinformatics (design.md): emerald shell, slate content in dark mode - scheme: default primary: custom accent: custom @@ -71,15 +70,28 @@ plugins: nav: - Home: index.md - - Releases: - - "v0.5 (from v0.1.7)": releases/v0.5.md - Getting Started: - Installation: getting-started/installation.md - - ROBIN Quickstart: getting-started/quickstart.md - - At startup (disclaimer, GUI, password): getting-started/startup.md - - Library Preparation: getting-started/library-preparation.md - - MinKNOW Configuration: getting-started/minknow-configuration.md - - MNP-Flex Setup: getting-started/mnpflex.md + - Quickstart: getting-started/quickstart.md + - Sequencing setup: + - Library preparation: getting-started/library-preparation.md + - MinKNOW configuration: getting-started/minknow-configuration.md + - Starting ROBIN: getting-started/startup.md + - MNP-Flex integration: getting-started/mnpflex.md + - Using ROBIN: + - Overview: using-robin/index.md + - First steps and navigation: using-robin/authentication-and-layout.md + - Users, consent, and auditing: using-robin/audit-and-users.md + - Tour of the screens: using-robin/pages-and-routes.md + - Reading your results: using-robin/sample-results.md + - Troubleshooting: using-robin/troubleshooting.md + - Analysis pipelines: + - Overview: analyses/index.md + - Methylation classification: analyses/methylation-classification.md + - MGMT promoter methylation: analyses/mgmt.md + - Copy-number analysis: analyses/cnv.md + - Target coverage and variants: analyses/target-and-variants.md + - Structural events (fusion and ITD): analyses/structural-events.md - Command-line reference: - Overview: cli/index.md - workflow command: cli/workflow.md @@ -89,10 +101,8 @@ nav: - Default admin password: cli/password.md - GUI users: cli/users.md - Audit log: cli/audit.md - - Using ROBIN: - - Overview: using-robin/index.md - - First steps and navigation: using-robin/authentication-and-layout.md - - Users, consent, and auditing: using-robin/audit-and-users.md - - Tour of the screens: using-robin/pages-and-routes.md - - Reading your results: using-robin/sample-results.md - - Troubleshooting: using-robin/troubleshooting.md + - Developer guide: + - Architecture: architecture/overview.md + - MinKNOW integration: development/minknow-integration.md + - Releases: + - "v0.5 (from v0.1.7)": releases/v0.5.md diff --git a/pyproject.toml b/pyproject.toml index 324444f5..8e9f8fe0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,81 +5,64 @@ build-backend = "hatchling.build" [project] name = "robin" version = "0.5.4" -description = "A Python CLI tool with file watching capabilities" +description = "Real-time Oxford Nanopore oncology analysis and monitoring workflow" readme = "README.md" -license = {text = "MIT"} +license = {text = "CC-BY-NC-4.0"} authors = [ {name = "Matt Loose", email = "matt.loose@nottingham.ac.uk"}, ] classifiers = [ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.12", ] requires-python = ">=3.12" dependencies = [ + "alive-progress", + "argon2-cffi>=23.1.0", "click>=8.0.0", - "watchdog>=3.0.0", - "pysam>=0.21.0", - "python-dateutil>=2.8.0", - #"pandas>=1.3.0", - "pandas<=2.1.4", - #"numpy>=1.21.0", - "numpy<2", - "scipy>=1.7.0", - "ruptures>=1.1.0", - "tqdm>=4.64.0", - "rich", - "ray[default]>=2.0.0", - "uvloop<0.22", - "networkx", - "sturgeon@git+https://github.com/looselab/sturgeon@testing", + "cnv_from_bam", + "cryptography>=41.0.0", + "dna_features_viewer", + "docker", + "gff3_parser", "methylartist@git+https://github.com/mattloose/methylartist", - "polars", - "tqdm", "natsort", - "reportlab", - "icecream", - "psutil", - "ruptures", - "pywebview", - "torch", #==1.13.1", - "scikit-learn", - "tabulate", "networkx", - "docker", + "nicegui>=3.15.0", + "numpy<2", "opencv-python", - "zxing-cpp<3", - "pyranges", + "packaging", + "pandas>=2.0,<=2.1.4", + "plotly", + "polars", + "psutil", "pyarrow>=23.0.1", - "requests", "PyPDF2", - "alive-progress", - "nicegui>=3.15.0", - "argon2-cffi>=23.1.0", - "gff3_parser", - "dna_features_viewer", - "icecream", - "pywebview", + "pyranges", + "python-dateutil>=2.8.0", + "ray[default]>=2.0.0", + "reportlab", + "requests", + "rich", + "ruptures>=1.1.0", "scikit-learn", - "yappi", + "scipy>=1.11,<1.14", + "seaborn", + "sturgeon@git+https://github.com/looselab/sturgeon@testing", "tabulate", - "docker", - "opencv-python", - "zxing-cpp", "tomli-w", - "packaging", - "plotly", - "seaborn", - "svglib", - "geocoder", - "cnv_from_bam", - "cryptography>=41.0.0", + "torch", + "tqdm>=4.64.0", + "uvloop<0.22", + "watchdog>=3.0.0", + "zxing-cpp<3", ] [project.optional-dependencies] +# Kept for backwards compatibility with existing `pip install '.[gui]'` +# instructions. NiceGUI is currently also part of the base installation. gui = [ "nicegui>=3.15.0", ] @@ -127,9 +110,10 @@ dev = [ robin = "robin.cli:main" [project.urls] -Homepage = "https://github.com/yourusername/robin" -Repository = "https://github.com/yourusername/robin" -Issues = "https://github.com/yourusername/robin/issues" +Homepage = "https://looselab.github.io/ROBIN/" +Documentation = "https://looselab.github.io/ROBIN/" +Repository = "https://github.com/LooseLab/ROBIN" +Issues = "https://github.com/LooseLab/ROBIN/issues" [tool.hatch.build.targets.wheel] packages = ["src/robin"] @@ -142,7 +126,7 @@ allow-direct-references = true [tool.black] line-length = 88 -target-version = ['py38'] +target-version = ['py312'] include = '\.pyi?$' extend-exclude = ''' /( @@ -165,7 +149,7 @@ line_length = 88 known_first_party = ["robin"] [tool.mypy] -python_version = "3.8" +python_version = "3.12" warn_return_any = true warn_unused_configs = true disallow_untyped_defs = true @@ -215,4 +199,4 @@ exclude_lines = [ "if __name__ == .__main__.:", "class .*\\bProtocol\\):", "@(abc\\.)?abstractmethod", -] +] diff --git a/robin.yml b/robin.yml index 7f08499e..aa4f0ae1 100644 --- a/robin.yml +++ b/robin.yml @@ -49,7 +49,6 @@ dependencies: - bioconductor-genomeinfodb # graphics/system deps - - git-lfs - pkg-config - cairo - pycairo diff --git a/src/robin/analysis/utilities/__init__.py b/src/robin/analysis/utilities/__init__.py index 4314c41f..8fa14e22 100644 --- a/src/robin/analysis/utilities/__init__.py +++ b/src/robin/analysis/utilities/__init__.py @@ -1,3 +1,3 @@ # SPDX-FileCopyrightText: 2023-present Matt Loose # -# SPDX-License-Identifier: MIT +# SPDX-License-Identifier: CC-BY-NC-4.0 diff --git a/src/robin/models/__init__.py b/src/robin/models/__init__.py index c1961f80..0f4fe3f1 100644 --- a/src/robin/models/__init__.py +++ b/src/robin/models/__init__.py @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: 2023-present Matt Loose # -# SPDX-License-Identifier: MIT +# SPDX-License-Identifier: CC-BY-NC-4.0 from pathlib import Path