diff --git a/database/Snakemake b/database/Snakemake index 3918658..a6cd5d0 100644 --- a/database/Snakemake +++ b/database/Snakemake @@ -14,9 +14,11 @@ Fill in database/config.yaml's `sources` URLs before running. The pipeline: 6. load_mibig_compounds } turn results into "compound" db entries (linked to MIBiG's URL) 7. parse_mibig_gbks - antiSMASH GBKs -> linear module readouts (PARAS-annotated) 8. load_mibig_bgcs - turn readouts into "bgc" db entries + 9. annotate_npclassifier - chemical-class annotation (compounds only, NPClassifier API) + 10. annotate_chebi - bioactivity annotation (compounds only, local ChEBI flat files) -Steps 4, 6, and 8 all mutate the same DuckDB file, so they're chained through marker -files (rather than each declaring the database itself as `output`) to force +Steps 4, 6, 8, 9, and 10 all mutate the same DuckDB file, so they're chained through +marker files (rather than each declaring the database itself as `output`) to force Snakemake to serialize them -- DuckDB doesn't support concurrent writers. """ @@ -39,10 +41,45 @@ PARAS_TRAINING_DATA_PATH = config["paras"].get("training_data_path") PARSE_COMPOUNDS_WORKERS = config["compute"]["parse_compounds_workers"] PARSE_GBKS_WORKERS = config["compute"]["parse_gbks_workers"] +TAXONOMY_ENABLED = config.get("taxonomy", {}).get("enabled", False) +TAXDUMP_DIR = WORKDIR / "taxdump" + +NPCLASSIFIER_REQUESTS_PER_SECOND = config.get("npclassifier", {}).get("requests_per_second", 2.0) +NPCLASSIFIER_WORKERS = config.get("npclassifier", {}).get("workers", 8) + +CHEBI_DIR = WORKDIR / "chebi" + +# Toggle whole branches of the pipeline off (see config.yaml's `enabled` comment) -- +# disabling npatlas/mibig skips their download+parse rules entirely, not just db +# loading, since the "real" inputs below are only declared when enabled: nothing else +# in the DAG needs npatlas/results.jsonl or mibig_gbk/readouts.jsonl, so Snakemake +# never schedules the rules that would produce them. Same idea for chebi: disabling it +# skips its bulk download entirely. +ENABLED = config.get("enabled", {}) +NPATLAS_ENABLED = ENABLED.get("npatlas", True) +MIBIG_ENABLED = ENABLED.get("mibig", True) +NPCLASSIFIER_ENABLED = ENABLED.get("npclassifier", True) +CHEBI_ENABLED = ENABLED.get("chebi", True) + rule all: input: - MARKERS / "bgcs_loaded.done" + MARKERS / "bgcs_loaded.done", + MARKERS / "npclassifier_annotated.done", + MARKERS / "chebi_annotated.done" + + +# --------------------------------------------------------------------------- +# NCBI taxonomy dump (used to standardize phylogeny genus/species/type to taxids) +# --------------------------------------------------------------------------- + +rule download_taxdump: + output: + names=TAXDUMP_DIR / "names.dmp", + nodes=TAXDUMP_DIR / "nodes.dmp" + run: + import taxonomy + taxonomy.download_taxdump(TAXDUMP_DIR) # --------------------------------------------------------------------------- @@ -131,19 +168,24 @@ rule parse_npatlas: rule load_npatlas_compounds: input: - results=WORKDIR / "npatlas" / "results.jsonl", - db_created=MARKERS / "db_created.done" + db_created=MARKERS / "db_created.done", + # Only declared when enabled -- nothing else in the DAG needs npatlas/results.jsonl, + # so download_npatlas/resolve_npatlas_sdf/parse_npatlas never run when disabled. + **({"results": WORKDIR / "npatlas" / "results.jsonl"} if NPATLAS_ENABLED else {}), + **({"taxdump_names": TAXDUMP_DIR / "names.dmp", "taxdump_nodes": TAXDUMP_DIR / "nodes.dmp"} if TAXONOMY_ENABLED and NPATLAS_ENABLED else {}) output: marker=touch(MARKERS / "npatlas_loaded.done") run: - import load_compounds - load_compounds.run( - results_path=input.results, - db_path=DB_PATH, - source="npatlas", - reaction_rules_path=RXN_RULES, - matching_rules_path=MXN_RULES, - ) + if NPATLAS_ENABLED: + import load_compounds + load_compounds.run( + results_path=input.results, + db_path=DB_PATH, + source="npatlas", + reaction_rules_path=RXN_RULES, + matching_rules_path=MXN_RULES, + taxdump_dir=TAXDUMP_DIR if TAXONOMY_ENABLED else None, + ) # --------------------------------------------------------------------------- @@ -159,13 +201,18 @@ rule extract_mibig_compounds: # module docstring): MIBiG 4.0's GBKs dropped the ACCESSION.VERSION suffix # parse_gbks.py used to read this from, but the JSON's own top-level "version" # field still has it. - versions=WORKDIR / "mibig_json" / "versions.json" + versions=WORKDIR / "mibig_json" / "versions.json", + # accession -> {organism_name, ncbi_tax_id, biosyn_class}, consumed by both + # load_mibig_compounds and load_mibig_bgcs to populate phylogeny/chemical_class + # annotations (see RetroMolDuckDB.add_phylogeny_annotation/add_flat_annotation). + annotations=WORKDIR / "mibig_json" / "annotations.json" run: import extract_mibig_compounds extract_mibig_compounds.run( mibig_json_dir=input.extract_dir, output_path=output.compounds, versions_output_path=output.versions, + annotations_output_path=output.annotations, ) @@ -189,23 +236,35 @@ rule parse_mibig_compounds: rule load_mibig_compounds: input: - results=WORKDIR / "mibig_json" / "results.jsonl", - # MIBiG URLs need an accession's version, from the JSON (see + prev=MARKERS / "npatlas_loaded.done", + # Only declared when enabled -- nothing else in the DAG needs these, so + # download_mibig_json/extract_mibig_compounds/parse_mibig_compounds never run + # when disabled. MIBiG URLs need an accession's version, from the JSON (see # extract_mibig_compounds rule above) -- not from the GBKs, which no longer carry it. - versions=WORKDIR / "mibig_json" / "versions.json", - prev=MARKERS / "npatlas_loaded.done" + **( + { + "results": WORKDIR / "mibig_json" / "results.jsonl", + "versions": WORKDIR / "mibig_json" / "versions.json", + "annotations": WORKDIR / "mibig_json" / "annotations.json", + } + if MIBIG_ENABLED else {} + ), + **({"taxdump_names": TAXDUMP_DIR / "names.dmp", "taxdump_nodes": TAXDUMP_DIR / "nodes.dmp"} if TAXONOMY_ENABLED and MIBIG_ENABLED else {}) output: marker=touch(MARKERS / "mibig_compounds_loaded.done") run: - import load_compounds - load_compounds.run( - results_path=input.results, - db_path=DB_PATH, - source="mibig", - reaction_rules_path=RXN_RULES, - matching_rules_path=MXN_RULES, - mibig_versions_path=input.versions, - ) + if MIBIG_ENABLED: + import load_compounds + load_compounds.run( + results_path=input.results, + db_path=DB_PATH, + source="mibig", + reaction_rules_path=RXN_RULES, + matching_rules_path=MXN_RULES, + mibig_versions_path=input.versions, + mibig_annotations_path=input.annotations, + taxdump_dir=TAXDUMP_DIR if TAXONOMY_ENABLED else None, + ) # --------------------------------------------------------------------------- @@ -232,17 +291,97 @@ rule parse_mibig_gbks: rule load_mibig_bgcs: input: - readouts=WORKDIR / "mibig_gbk" / "readouts.jsonl", - versions=WORKDIR / "mibig_json" / "versions.json", - prev=MARKERS / "mibig_compounds_loaded.done" + prev=MARKERS / "mibig_compounds_loaded.done", + # Only declared when enabled -- nothing else in the DAG needs these, so + # download_mibig_gbk/parse_mibig_gbks never run when disabled. + **( + { + "readouts": WORKDIR / "mibig_gbk" / "readouts.jsonl", + "versions": WORKDIR / "mibig_json" / "versions.json", + "annotations": WORKDIR / "mibig_json" / "annotations.json", + } + if MIBIG_ENABLED else {} + ), + **({"taxdump_names": TAXDUMP_DIR / "names.dmp", "taxdump_nodes": TAXDUMP_DIR / "nodes.dmp"} if TAXONOMY_ENABLED and MIBIG_ENABLED else {}) output: marker=touch(MARKERS / "bgcs_loaded.done") run: - import load_bgcs - load_bgcs.run( - readouts_path=input.readouts, - db_path=DB_PATH, - reaction_rules_path=RXN_RULES, - matching_rules_path=MXN_RULES, - mibig_versions_path=input.versions, + if MIBIG_ENABLED: + import load_bgcs + load_bgcs.run( + readouts_path=input.readouts, + db_path=DB_PATH, + reaction_rules_path=RXN_RULES, + matching_rules_path=MXN_RULES, + mibig_versions_path=input.versions, + mibig_annotations_path=input.annotations, + taxdump_dir=TAXDUMP_DIR if TAXONOMY_ENABLED else None, + ) + + +# --------------------------------------------------------------------------- +# Step 9: NPClassifier chemical-class annotation (compounds only) +# --------------------------------------------------------------------------- + +rule annotate_npclassifier: + input: + # Chained after bgcs_loaded.done (not just the compound-loading steps) purely to + # serialize this write against load_mibig_bgcs's -- both mutate DB_PATH and DuckDB + # doesn't support concurrent writers (see module docstring at the top of this file). + prev=MARKERS / "bgcs_loaded.done" + output: + marker=touch(MARKERS / "npclassifier_annotated.done") + # Declared like parse_npatlas/parse_mibig_gbks's worker counts, even though this + # rule is I/O-bound rather than CPU-bound: without a `threads:` declaration, + # Snakemake assumes a 1-core job while it actually opens NPCLASSIFIER_WORKERS + # concurrent connections, so `--cores` wouldn't see or cap it. Declaring it here + # means `--cores` fewer than npclassifier.workers automatically caps this job's + # `threads` value at run time -- passed through below instead of the raw config + # number, so a run with e.g. `--cores 6` gets at most 6 workers even if + # npclassifier.workers says 8. + threads: NPCLASSIFIER_WORKERS + run: + if NPCLASSIFIER_ENABLED: + import annotate_npclassifier + annotate_npclassifier.run( + db_path=DB_PATH, + cache_path=WORKDIR / "npclassifier" / "cache.jsonl", + requests_per_second=NPCLASSIFIER_REQUESTS_PER_SECOND, + workers=threads, + ) + + +# --------------------------------------------------------------------------- +# Step 10: ChEBI bioactivity annotation (compounds only) +# --------------------------------------------------------------------------- + +rule download_chebi: + output: + compounds=CHEBI_DIR / "compounds.tsv.gz", + structures=CHEBI_DIR / "structures.tsv.gz", + relation=CHEBI_DIR / "relation.tsv.gz" + run: + import chebi + chebi.download_chebi_flat_files(CHEBI_DIR) + + +rule annotate_chebi: + input: + # Chained after npclassifier_annotated.done purely to serialize this write + # against annotate_npclassifier's (see module docstring at the top of this file). + prev=MARKERS / "npclassifier_annotated.done", + # Only declared when enabled -- download_chebi never runs when disabled. + **( + { + "compounds": CHEBI_DIR / "compounds.tsv.gz", + "structures": CHEBI_DIR / "structures.tsv.gz", + "relation": CHEBI_DIR / "relation.tsv.gz", + } + if CHEBI_ENABLED else {} ) + output: + marker=touch(MARKERS / "chebi_annotated.done") + run: + if CHEBI_ENABLED: + import annotate_chebi + annotate_chebi.run(db_path=DB_PATH, chebi_dir=CHEBI_DIR) diff --git a/database/config.yaml b/database/config.yaml index 904ffe3..b53e44f 100644 --- a/database/config.yaml +++ b/database/config.yaml @@ -6,17 +6,48 @@ sources: mibig_json_url: "https://dl.secondarymetabolites.org/mibig/mibig_json_4.0.tar.gz" mibig_gbk_url: "https://dl.secondarymetabolites.org/mibig/mibig_gbk_4.0.tar.gz" +enabled: + # Toggle whole branches of the pipeline off. Disabling a source skips its downloads + # and RetroMol parsing entirely (not just db loading) -- e.g. npatlas: false means + # download_npatlas/parse_npatlas never run. mibig covers both MIBiG compounds and + # BGCs (they share the same JSON/GBK downloads). npclassifier gates step 9 + # (chemical-class annotation) independently of which source(s) loaded the compounds + # it classifies. + npatlas: true + mibig: true + npclassifier: true + chebi: true + paths: # Final DuckDB database produced by the pipeline. - database: "/Users/davidmeijer/Downloads/retromol.duckdb" + database: "/Users/davidmeijer/Desktop/retromol.duckdb" # Scratch space for downloads and intermediate per-step results. - workdir: "/Users/davidmeijer/retromol_tmp" + workdir: "/Users/davidmeijer/Desktop/retromol_tmp" # null -> RuleSet.load_default()'s bundled reaction/matching rules. reaction_rules: null matching_rules: null +npclassifier: + # Rate limit for the free, GNPS2-hosted NPClassifier API (no published limit -- + # kept conservative). This is the *combined* rate across all workers below, not + # per-worker. Classifications are cached in workdir/npclassifier/cache.jsonl, so + # reruns only pay for compounds not already classified. + requests_per_second: 50.0 + + # Concurrent requests. Classification is I/O-bound (waiting on the API), not + # CPU-bound, so this is safe to raise well past your core count -- it's bounded by + # requests_per_second above either way. + workers: 8 + +taxonomy: + # NCBI taxdump (names.dmp/nodes.dmp), downloaded once into workdir/taxdump and reused + # across pipeline runs -- used to standardize phylogeny genus/species/type to NCBI + # taxids (see database/scripts/taxonomy.py). Set to null to skip taxid resolution + # entirely (phylogeny is then stored as raw, unstandardized text/no taxids). + enabled: true + paras: threshold: 0.1 keep_top: 3 diff --git a/database/envs/retromol.yaml b/database/envs/retromol.yaml new file mode 100644 index 0000000..79db115 --- /dev/null +++ b/database/envs/retromol.yaml @@ -0,0 +1,34 @@ +# Conda env for running database/Snakemake (the database-construction pipeline only -- +# this has no bearing on the GUI/webapp). +# +# Every rule in that Snakefile uses Snakemake's `run:` directive (inline Python that +# imports and calls database/scripts/*.py functions directly), not `shell:`. Snakemake +# executes `run:` blocks in its own process rather than a subprocess, so a per-rule +# `conda:` env (the --use-conda mechanism) would NOT actually reach any of this +# pipeline's own code -- only `shell()` calls made from *within* a `run:` block, which +# this pipeline never makes. There is deliberately no per-rule conda: directive in +# database/Snakemake, and no --use-conda flag in the invocation below, because of this. +# +# This env is instead the one Snakemake itself runs in, so it needs both Snakemake and +# every runtime dependency database/scripts/*.py imports (rdkit, duckdb, etc. -- all +# already listed in the repo's own pyproject.toml, pulled in by the editable install +# below rather than duplicated here). +# +# Usage: +# +# conda env create -f database/envs/retromol.yaml +# conda activate retromol +# pip install -e . +# snakemake -p -s database/Snakemake --configfile database/config.yaml \ +# --workflow-profile database/profiles/slurm +name: retromol + +channels: + - conda-forge + +dependencies: + - python=3.11 + - pip + - snakemake + - snakemake-executor-plugin-slurm + - mamba diff --git a/database/profiles/slurm/config.yaml b/database/profiles/slurm/config.yaml new file mode 100644 index 0000000..13a1265 --- /dev/null +++ b/database/profiles/slurm/config.yaml @@ -0,0 +1,83 @@ +# Snakemake workflow profile for running database/Snakemake on a Slurm cluster. +# +# Usage (from the repo root, with database/envs/retromol.yaml's env active -- see that +# file's own comment for why there's no --use-conda here): +# +# conda activate retromol +# snakemake -p -s database/Snakemake --configfile database/config.yaml \ +# --workflow-profile database/profiles/slurm +executor: slurm +jobs: 6 # concurrent Slurm jobs -- mainly matters for the independent download_* rules; + # everything downstream of them is a strict chain (create_db -> load_* -> + # annotate_*), serialized through marker files because DuckDB doesn't support + # concurrent writers (see database/Snakemake's module docstring), so raising + # this past ~6-8 buys little. +latency-wait: 60 +rerun-incomplete: true +printshellcmds: true +# Independent rules (e.g. the download_* rules, or npclassifier/chebi annotation vs. +# each other before they're serialized) shouldn't all be killed by one unrelated rule +# failing/timing out -- let everything already running finish. +keep-going: true + +default-resources: + slurm_account: null # fill in your account, if your cluster requires one + slurm_partition: null # fill in your cluster's partition name + runtime: 60 + mem_mb: 4000 + disk_mb: 20000 + +# parse_npatlas/parse_mibig_compounds/parse_mibig_gbks' worker counts come from +# database/config.yaml's `compute:` section (PARSE_COMPOUNDS_WORKERS/PARSE_GBKS_WORKERS), +# not from this profile -- deliberately not duplicated here as set-threads to avoid two +# sources of truth for the same value; edit config.yaml instead. +set-resources: + download_npatlas: + runtime: 120 + mem_mb: 4000 + disk_mb: 10000 + download_mibig_json: + runtime: 60 + mem_mb: 2000 + disk_mb: 5000 + download_mibig_gbk: + runtime: 60 + mem_mb: 2000 + disk_mb: 5000 + download_taxdump: + runtime: 60 + mem_mb: 2000 + disk_mb: 5000 + download_chebi: + # A few hundred MB total across compounds/structures/relation flat files. + runtime: 60 + mem_mb: 2000 + disk_mb: 5000 + parse_npatlas: + runtime: 720 + mem_mb: 16000 + parse_mibig_compounds: + runtime: 240 + mem_mb: 8000 + parse_mibig_gbks: + runtime: 480 + mem_mb: 16000 + load_npatlas_compounds: + runtime: 60 + mem_mb: 4000 + load_mibig_compounds: + runtime: 60 + mem_mb: 4000 + load_mibig_bgcs: + runtime: 60 + mem_mb: 4000 + annotate_npclassifier: + # Rate-limited API calls (see database/config.yaml's npclassifier.requests_per_second) + # -- mostly wall-clock waiting on the network, not compute. + runtime: 1440 + mem_mb: 2000 + annotate_chebi: + # Loads all of ChEBI's flat files into memory once (see chebi.py's ChebiDB.load) -- + # a few hundred MB, generous headroom here. + runtime: 120 + mem_mb: 8000 diff --git a/database/scripts/annotate_chebi.py b/database/scripts/annotate_chebi.py new file mode 100644 index 0000000..ad8de7f --- /dev/null +++ b/database/scripts/annotate_chebi.py @@ -0,0 +1,77 @@ +"""Step 10: annotate every compound entry's bioactivity via ChEBI's role ontology. + +Runs after compound loading, so every distinct molecule -- regardless of which +source(s) it came from -- is looked up exactly once. BGC entries are skipped: +bioactivity here is a property of the compound structure (looked up by its InChIKey +entry id), not of the producing organism/cluster. + +This queries a local bulk release (see chebi.py) rather than a rate-limited API -- no +pacing or caching needed, lookups are just local reads. +""" + +import argparse +import logging +from pathlib import Path + +from tqdm import tqdm + +from chebi import ChebiDB +from retromol_database.duckdb import RetroMolDuckDB + +log = logging.getLogger(__name__) + + +def run(db_path: str | Path, chebi_dir: str | Path, log_every: int = 500) -> None: + processed = 0 + matched = 0 + annotated = 0 + + db = RetroMolDuckDB.open(db_path) + try: + total = db.count_entries_by_type(["compound"]) + chebi = ChebiDB.load(chebi_dir) + with tqdm(total=total, desc="annotate_chebi", unit="cmpd") as pbar: + for entry in db.iter_entries(): + if entry.type != "compound": + continue + + processed += 1 + result = chebi.roles_for_inchikey(entry.id) + if result is not None: + matched += 1 + for role in result.biological_roles: + db.add_bioactivity_annotation( + entry.id, level="chebi_biological_role", label=role.label, external_id=role.chebi_accession + ) + annotated += 1 + for role in result.chemical_roles: + db.add_bioactivity_annotation( + entry.id, level="chebi_chemical_role", label=role.label, external_id=role.chebi_accession + ) + annotated += 1 + + pbar.update(1) + pbar.set_postfix(matched=matched, annotated=annotated) + + if log_every > 0 and processed % log_every == 0: + log.info("annotate_chebi: processed=%d matched=%d annotated=%d", processed, matched, annotated) + finally: + db.close() + + log.info("annotate_chebi: processed=%d matched=%d annotated=%d", processed, matched, annotated) + + +def main() -> None: + logging.basicConfig(level=logging.INFO) + + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--db-path", required=True) + ap.add_argument("--chebi-dir", required=True, help="dir with ChEBI's compounds.tsv.gz/structures.tsv.gz/relation.tsv.gz") + ap.add_argument("--log-every", type=int, default=500) + args = ap.parse_args() + + run(db_path=args.db_path, chebi_dir=args.chebi_dir, log_every=args.log_every) + + +if __name__ == "__main__": + main() diff --git a/database/scripts/annotate_npclassifier.py b/database/scripts/annotate_npclassifier.py new file mode 100644 index 0000000..a51b8c7 --- /dev/null +++ b/database/scripts/annotate_npclassifier.py @@ -0,0 +1,245 @@ +"""Step 9: classify every compound entry's chemical class via NPClassifier. + +Runs after both compound-loading steps (NPAtlas + MIBiG compounds are already +deduplicated by inchikey in `entries` by then -- see load_compounds.py), so each +distinct molecule is classified exactly once no matter which source(s) it came from. +BGC entries are skipped: NPClassifier needs a compound's own SMILES structure +(`entry.raw`), which a bgc entry doesn't have. + +Idempotent across reruns via a local JSONL cache (`--cache-path`) keyed by entry_id: +already-classified compounds are skipped without hitting the API again, and only +successes are cached -- flushed to disk as each one completes, not batched at the end +-- so a kill mid-run loses at most the handful of requests in flight, and a transient +failure gets retried on the next run. + +Classification is I/O-bound (waiting on NPClassifier's API), not CPU-bound, so +`--workers` runs multiple requests concurrently via a thread pool -- the GIL doesn't +block this the way it would CPU-bound work, since each thread is blocked on network +I/O rather than holding the GIL. `--requests-per-second` still caps the *combined* +rate across all workers (a shared, thread-safe RateLimiter -- see below), since that +cap is about being a good citizen towards NPClassifier's free, GNPS2-hosted service, +not about this process's own resources. DB writes and cache appends only ever happen +on the main thread (as results complete), so there's no concurrent-write concern +there -- only the HTTP calls themselves run in parallel. +""" + +import argparse +import itertools +import json +import logging +import threading +import time +from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait +from pathlib import Path + +from tqdm import tqdm + +from npclassifier import ClassificationResult, classify_smiles +from retromol_database.duckdb import Entry, RetroMolDuckDB + +log = logging.getLogger(__name__) + + +class RateLimiter: + """Thread-safe request pacing shared across worker threads. + + Each call reserves the next available time slot under a lock (cheap, no waiting + while holding it), then sleeps outside the lock until its own slot arrives -- so + threads don't serialize on the actual waiting, only on the quick slot reservation. + """ + + def __init__(self, requests_per_second: float) -> None: + self._min_interval = 1.0 / requests_per_second if requests_per_second > 0 else 0.0 + self._lock = threading.Lock() + self._next_slot = time.monotonic() + + def wait_for_slot(self) -> None: + with self._lock: + now = time.monotonic() + slot = max(now, self._next_slot) + self._next_slot = slot + self._min_interval + delay = slot - now + if delay > 0: + time.sleep(delay) + + +def _load_cache(cache_path: Path) -> dict[str, ClassificationResult]: + if not cache_path.exists(): + return {} + + cache: dict[str, ClassificationResult] = {} + with open(cache_path) as fh: + for line in fh: + line = line.strip() + if not line: + continue + row = json.loads(line) + cache[row["entry_id"]] = ClassificationResult( + pathway=row["pathway"], + superclass=row["superclass"], + class_=row["class_"], + is_glycoside=row["is_glycoside"], + ) + return cache + + +def _append_cache(cache_path: Path, entry_id: str, result: ClassificationResult) -> None: + with open(cache_path, "a") as fh: + fh.write( + json.dumps( + { + "entry_id": entry_id, + "pathway": result.pathway, + "superclass": result.superclass, + "class_": result.class_, + "is_glycoside": result.is_glycoside, + } + ) + + "\n" + ) + + +def _apply(db: RetroMolDuckDB, entry_id: str, result: ClassificationResult) -> None: + for label in result.pathway: + db.add_chemical_class_annotation(entry_id, level="pathway", label=label) + for label in result.superclass: + db.add_chemical_class_annotation(entry_id, level="superclass", label=label) + for label in result.class_: + db.add_chemical_class_annotation(entry_id, level="class", label=label) + # Presence-only, same convention as every other flag-shaped annotation: only stored + # when true, and with a real name ("Glycoside") rather than a bare "Yes" that means + # nothing once it's sitting in a chart/chip out of context. + if result.is_glycoside: + db.add_chemical_class_annotation(entry_id, level="is_glycoside", label="Glycoside") + + +def run( + db_path: str | Path, + cache_path: str | Path, + requests_per_second: float = 2.0, + workers: int = 8, + limit: int | None = None, + log_every: int = 100, +) -> None: + cache_path = Path(cache_path) + cache_path.parent.mkdir(parents=True, exist_ok=True) + cache = _load_cache(cache_path) + log.info("annotate_npclassifier: loaded %d cached classifications from %s", len(cache), cache_path) + + rate_limiter = RateLimiter(requests_per_second) + + classified = 0 + reused = 0 + skipped_no_smiles = 0 + failed = 0 + + def classify_one(entry: Entry) -> tuple[Entry, ClassificationResult | None]: + rate_limiter.wait_for_slot() + return entry, classify_smiles(entry.raw) + + db = RetroMolDuckDB.open(db_path) + try: + to_classify: list[Entry] = [] + for entry in db.iter_entries(): + if entry.type != "compound": + continue + + if entry.id in cache: + _apply(db, entry.id, cache[entry.id]) + reused += 1 + continue + + if not entry.raw: + skipped_no_smiles += 1 + continue + + if limit is not None and len(to_classify) >= limit: + continue + + to_classify.append(entry) + + log.info( + "annotate_npclassifier: %d compounds to classify (workers=%d, requests_per_second=%s)", + len(to_classify), workers, requests_per_second, + ) + + # Bounded sliding window rather than submitting the whole backlog up front -- + # at hundreds/thousands of compounds, an upfront submit() for everything means + # a Ctrl+C has to wait for every already-launched request (each with its own + # retry/backoff chain, worse under rate-limiting) before the pool can actually + # exit, since ThreadPoolExecutor won't drop already-running work. Keeping at + # most `max_pending` in flight means an interrupt only has to wait for that many + # -- same shape as parse_gbks.py's own bounded-window loop, same reason. + max_pending = max(workers * 2, 1) + entries_iter = iter(to_classify) + + pool = ThreadPoolExecutor(max_workers=workers) + try: + pending = {pool.submit(classify_one, e) for e in itertools.islice(entries_iter, max_pending)} + + with tqdm(total=len(to_classify), desc="annotate_npclassifier", unit="cmpd") as pbar: + while pending: + done_futures, pending = wait(pending, return_when=FIRST_COMPLETED) + + for future in done_futures: + entry, result = future.result() + + if result is None: + failed += 1 + else: + _apply(db, entry.id, result) + _append_cache(cache_path, entry.id, result) + classified += 1 + + pbar.update(1) + pbar.set_postfix(classified=classified, reused=reused, failed=failed) + + done = classified + failed + if log_every > 0 and done % log_every == 0: + log.info( + "annotate_npclassifier: classified=%d reused=%d failed=%d skipped_no_smiles=%d", + classified, reused, failed, skipped_no_smiles, + ) + + next_entry = next(entries_iter, None) + if next_entry is not None: + pending.add(pool.submit(classify_one, next_entry)) + except KeyboardInterrupt: + log.warning("annotate_npclassifier: interrupted -- cancelling not-yet-started requests") + pool.shutdown(wait=False, cancel_futures=True) + raise + else: + pool.shutdown(wait=True) + finally: + db.close() + + log.info( + "annotate_npclassifier: classified=%d reused=%d failed=%d skipped_no_smiles=%d", + classified, reused, failed, skipped_no_smiles, + ) + + +def main() -> None: + logging.basicConfig(level=logging.INFO) + + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--db-path", required=True) + ap.add_argument("--cache-path", required=True) + ap.add_argument("--requests-per-second", type=float, default=2.0) + ap.add_argument("--workers", type=int, default=8, help="concurrent requests (I/O-bound, not CPU-bound)") + ap.add_argument("--limit", type=int, default=None, help="classify at most N new compounds (testing/dry-run)") + ap.add_argument("--log-every", type=int, default=100) + args = ap.parse_args() + + run( + db_path=args.db_path, + cache_path=args.cache_path, + requests_per_second=args.requests_per_second, + workers=args.workers, + limit=args.limit, + log_every=args.log_every, + ) + + +if __name__ == "__main__": + main() diff --git a/database/scripts/chebi.py b/database/scripts/chebi.py new file mode 100644 index 0000000..d4652d4 --- /dev/null +++ b/database/scripts/chebi.py @@ -0,0 +1,239 @@ +"""ChEBI role-ontology client for bioactivity annotation. + +Downloads ChEBI's flat-file bulk release (ftp.ebi.ac.uk) once, then looks up a +compound's biological/chemical roles by its standard InChIKey. ChEBI's role ontology +(`has_role` edges under CHEBI:24432 "biological role" / CHEBI:51086 "chemical role") +is annotated on generic compound classes more often than on specific stereo-defined +structures -- confirmed +live (2026-08-26): erythromycin A (CHEBI:42355) itself carries no has_role edges, only +its `is_a` parent "erythromycin" (CHEBI:48923) does (roles: xenobiotic, bacterial +metabolite, environmental contaminant). So lookups walk a few steps up the `is_a` +ancestor chain collecting has_role edges at every level, not just the leaf's own. + +Schema confirmed live by downloading and inspecting +ftp.ebi.ac.uk/pub/databases/chebi/flat_files/: structures.tsv (compound_id, +standard_inchi_key), relation.tsv (relation_type_id 4=has_role, 5=is_a; init_id/final_id +are compounds.tsv ids), compounds.tsv (id, name, chebi_accession). +""" + +from __future__ import annotations + +import csv +import gzip +import logging +import re +import urllib.request +from dataclasses import dataclass +from pathlib import Path + +from tqdm import tqdm + +log = logging.getLogger(__name__) + +CHEBI_FLAT_FILES_URL = "https://ftp.ebi.ac.uk/pub/databases/chebi/flat_files" +CHEBI_FILES = ["compounds.tsv.gz", "structures.tsv.gz", "relation.tsv.gz"] + +# ChEBI's `name` column carries inline HTML for italicized genus/species names and +# similar formatting (e.g. "Aspergillus metabolite", "S-configuration" +# -- confirmed live in compounds.tsv.gz/chemical_data.tsv.gz). Stripped here, once, at +# load time -- so every consumer of name_by_compound_id (role labels today, anything +# else later) gets plain text without needing to know this quirk exists. +_HTML_TAG_RE = re.compile(r"<[^>]+>") + + +def _strip_html(text: str) -> str: + return _HTML_TAG_RE.sub("", text) + +RELATION_TYPE_HAS_ROLE = "4" +RELATION_TYPE_IS_A = "5" + +# ChEBI's role-ontology roots (CHEBI:50906 "role"'s direct children) -- a has_role +# target is classified by walking its own is_a ancestry for one of these two ids. +# "application" (CHEBI:33232), the third root, isn't surfaced -- not requested. +CHEBI_ID_BIOLOGICAL_ROLE = "24432" +CHEBI_ID_CHEMICAL_ROLE = "51086" + +# How far to walk up `is_a` ancestry: role classification (shallow ontology, ~1-4 hops +# in practice) gets a bit more headroom than role-collection on the queried compound +# (kept shallow deliberately -- climbing further starts pulling in unrelated purely +# structural classes, e.g. erythromycin A's other is_a parent "cyclic ketone", which +# carry no roles of their own but would otherwise cost unbounded fan-out for nothing). +MAX_ROLE_CLASSIFICATION_HOPS = 10 +MAX_ROLE_COLLECTION_HOPS = 4 + + +def download_chebi_flat_files(dest_dir: str | Path, *, force: bool = False) -> Path: + """Download compounds.tsv.gz/structures.tsv.gz/relation.tsv.gz into `dest_dir` + (no-op if all three already exist, unless `force`).""" + dest_dir = Path(dest_dir).expanduser() + dest_dir.mkdir(parents=True, exist_ok=True) + + if not force and all((dest_dir / name).exists() for name in CHEBI_FILES): + return dest_dir + + for name in CHEBI_FILES: + url = f"{CHEBI_FLAT_FILES_URL}/{name}" + dest = dest_dir / name + log.info("downloading %s", url) + with urllib.request.urlopen(url) as resp, open(dest, "wb") as out: + total = int(resp.headers.get("Content-Length") or 0) or None + with tqdm(total=total, desc=f"download_chebi[{name}]", unit="B", unit_scale=True, unit_divisor=1024) as pbar: + while chunk := resp.read(1024 * 1024): + out.write(chunk) + pbar.update(len(chunk)) + + return dest_dir + + +@dataclass(frozen=True) +class ChebiRole: + label: str + chebi_accession: str # e.g. "CHEBI:35703" -- the role term's own id, for linking out + + +@dataclass(frozen=True) +class ChebiRoles: + chebi_id: str # the matched compound's own CHEBI accession, e.g. "CHEBI:42355" + biological_roles: list[ChebiRole] + chemical_roles: list[ChebiRole] + + +class ChebiDB: + """In-memory index over ChEBI's flat files, built once per pipeline run.""" + + def __init__( + self, + *, + chebi_accession_by_compound_id: dict[str, str], + name_by_compound_id: dict[str, str], + compound_id_by_inchikey: dict[str, str], + is_a_parents: dict[str, list[str]], + has_role: dict[str, list[str]], + ) -> None: + self._chebi_accession_by_compound_id = chebi_accession_by_compound_id + self._name_by_compound_id = name_by_compound_id + self._compound_id_by_inchikey = compound_id_by_inchikey + self._is_a_parents = is_a_parents + self._has_role = has_role + + @classmethod + def load(cls, chebi_dir: str | Path) -> "ChebiDB": + chebi_dir = Path(chebi_dir).expanduser() + + chebi_accession_by_compound_id: dict[str, str] = {} + name_by_compound_id: dict[str, str] = {} + with gzip.open(chebi_dir / "compounds.tsv.gz", "rt", encoding="utf-8", errors="replace") as fh: + reader = csv.reader(fh, delimiter="\t", quotechar='"') + header = next(reader) + id_idx = header.index("id") + name_idx = header.index("name") + accession_idx = header.index("chebi_accession") + for row in reader: + if len(row) <= accession_idx or not row[accession_idx]: + continue + chebi_accession_by_compound_id[row[id_idx]] = row[accession_idx] + name_by_compound_id[row[id_idx]] = _strip_html(row[name_idx]) + + compound_id_by_inchikey: dict[str, str] = {} + with gzip.open(chebi_dir / "structures.tsv.gz", "rt", encoding="utf-8", errors="replace") as fh: + reader = csv.reader(fh, delimiter="\t", quotechar='"') + header = next(reader) + cid_idx = header.index("compound_id") + key_idx = header.index("standard_inchi_key") + for row in reader: + if len(row) <= key_idx or not row[key_idx]: + continue + compound_id_by_inchikey.setdefault(row[key_idx], row[cid_idx]) + + is_a_parents: dict[str, list[str]] = {} + has_role: dict[str, list[str]] = {} + with gzip.open(chebi_dir / "relation.tsv.gz", "rt", encoding="utf-8", errors="replace") as fh: + reader = csv.reader(fh, delimiter="\t", quotechar='"') + header = next(reader) + type_idx = header.index("relation_type_id") + init_idx = header.index("init_id") + final_idx = header.index("final_id") + for row in reader: + if len(row) <= final_idx: + continue + rtype, init_id, final_id = row[type_idx], row[init_idx], row[final_idx] + if rtype == RELATION_TYPE_IS_A: + is_a_parents.setdefault(init_id, []).append(final_id) + elif rtype == RELATION_TYPE_HAS_ROLE: + has_role.setdefault(init_id, []).append(final_id) + + log.info( + "loaded ChEBI flat files: %d compounds, %d structures, %d is_a edges, %d has_role edges", + len(chebi_accession_by_compound_id), len(compound_id_by_inchikey), len(is_a_parents), len(has_role), + ) + return cls( + chebi_accession_by_compound_id=chebi_accession_by_compound_id, + name_by_compound_id=name_by_compound_id, + compound_id_by_inchikey=compound_id_by_inchikey, + is_a_parents=is_a_parents, + has_role=has_role, + ) + + def _walk_is_a(self, start: str, max_hops: int): + """Yield ancestor compound ids reachable from `start` via `is_a`, breadth-first, + up to `max_hops` levels, never revisiting a node.""" + seen = {start} + frontier = [start] + for _ in range(max_hops): + next_frontier = [] + for cid in frontier: + for parent in self._is_a_parents.get(cid, []): + if parent not in seen: + seen.add(parent) + next_frontier.append(parent) + yield parent + if not next_frontier: + return + frontier = next_frontier + + def _role_category(self, role_compound_id: str) -> str | None: + """Classify a has_role target by walking its own is_a ancestry for one of + CHEBI_ID_BIOLOGICAL_ROLE/CHEBI_ID_CHEMICAL_ROLE. None if neither is hit (e.g. + it falls under "application" instead, or the chain doesn't resolve).""" + if role_compound_id == CHEBI_ID_BIOLOGICAL_ROLE: + return "biological_role" + if role_compound_id == CHEBI_ID_CHEMICAL_ROLE: + return "chemical_role" + for ancestor in self._walk_is_a(role_compound_id, MAX_ROLE_CLASSIFICATION_HOPS): + if ancestor == CHEBI_ID_BIOLOGICAL_ROLE: + return "biological_role" + if ancestor == CHEBI_ID_CHEMICAL_ROLE: + return "chemical_role" + return None + + def roles_for_inchikey(self, inchikey: str) -> ChebiRoles | None: + """None if `inchikey` has no match in ChEBI at all. Roles are collected from + the matched compound and a few levels of its `is_a` ancestors (see module + docstring -- ChEBI usually annotates roles on a generic parent class, not every + specific stereo-defined child structure).""" + compound_id = self._compound_id_by_inchikey.get(inchikey) + if compound_id is None: + return None + + role_compound_ids: set[str] = set(self._has_role.get(compound_id, [])) + for ancestor in self._walk_is_a(compound_id, MAX_ROLE_COLLECTION_HOPS): + role_compound_ids.update(self._has_role.get(ancestor, [])) + + biological_roles: list[ChebiRole] = [] + chemical_roles: list[ChebiRole] = [] + for role_id in role_compound_ids: + label = self._name_by_compound_id.get(role_id) + accession = self._chebi_accession_by_compound_id.get(role_id) + if not label or not accession: + continue + category = self._role_category(role_id) + if category == "biological_role": + biological_roles.append(ChebiRole(label=label, chebi_accession=accession)) + elif category == "chemical_role": + chemical_roles.append(ChebiRole(label=label, chebi_accession=accession)) + + return ChebiRoles( + chebi_id=self._chebi_accession_by_compound_id.get(compound_id, compound_id), + biological_roles=biological_roles, + chemical_roles=chemical_roles, + ) diff --git a/database/scripts/common.py b/database/scripts/common.py index 2101524..2668725 100644 --- a/database/scripts/common.py +++ b/database/scripts/common.py @@ -168,6 +168,93 @@ def run_retromol_stream_quiet( yield ResultEvent(serialized, err) +# Common secondary-metabolite-producing fungal genera -- MIBiG's JSON has no direct +# kingdom/type field (unlike NPAtlas), so `phylogeny_from_organism_name` falls back to +# this bundled set to distinguish fungal from bacterial entries. MIBiG is overwhelmingly +# bacterial, so "bacterium" is the default and this set only needs to catch the fungal +# minority. Not exhaustive -- a genus missing from this list is classified "bacterium". +FUNGAL_GENERA = { + "aspergillus", "penicillium", "fusarium", "trichoderma", "curvularia", + "colletotrichum", "alternaria", "cladosporium", "talaromyces", "chaetomium", + "acremonium", "beauveria", "metarhizium", "monascus", "epicoccum", + "pestalotiopsis", "phoma", "botrytis", "verticillium", "myrothecium", +} + + +# Metagenomic/environmental-sample naming conventions (e.g. "uncultured Streptomyces sp.", +# "unidentified bacterium") -- not a genus, so skipped when picking the genus token, and +# rejected outright if a whole genus/species value collapses to just one of these. +_NON_TAXONOMIC_PREFIXES = {"uncultured", "unclassified", "unidentified"} + +# Species-epithet placeholders meaning "no real species-level identification" -- checked +# against a species token after any leading genus-name duplicate is stripped (see +# clean_species_epithet), not just the bare "sp."/"sp" abbreviation. Shared by both +# MIBiG's free-text organism_name and NPAtlas's own origin_species SDF field, which turn +# out to carry the same kinds of placeholder values (confirmed live: NPAtlas's +# origin_species includes bare "sp.", genus-duplicated "Streptomyces sp.", and +# "unidentified" -- none of which are real species, but nothing was rejecting them before +# this, so they leaked into the phylogeny_annotations species column as if they were). +_NON_TAXONOMIC_SPECIES_TOKENS = {"sp", "spp", "unidentified", "uncultured", "unclassified"} + + +def clean_genus(genus_raw: str | None) -> str | None: + """Reject a genus value that's actually a non-taxonomic placeholder + ("unidentified"/"uncultured"/"unclassified") rather than a real genus name.""" + if not genus_raw: + return None + genus = genus_raw.strip() + if not genus or genus.lower() in _NON_TAXONOMIC_PREFIXES: + return None + return genus + + +def clean_species_epithet(genus: str | None, species_raw: str | None) -> str | None: + """Reject a species value carrying no real species-level information: a bare + "sp."/"sp" abbreviation, a genus name duplicated into the species field (e.g. + NPAtlas's origin_species="Streptomyces sp." alongside genus="Streptomyces"), or an + "unidentified"/"uncultured"/"unclassified" placeholder.""" + if not species_raw: + return None + + species = species_raw.strip() + if genus and species.lower().startswith(genus.lower() + " "): + species = species[len(genus):].strip() + + if not species or species.rstrip(".").lower() in _NON_TAXONOMIC_SPECIES_TOKENS: + return None + + return species + + +def phylogeny_from_organism_name(organism_name: str | None) -> tuple[str | None, str | None, str | None]: + """Split MIBiG's free-text `organism_name` (e.g. "Streptomyces coelicolor A3(2)") + into (type, genus, species). Type is inferred from `FUNGAL_GENERA` since MIBiG's + JSON carries no kingdom field; genus/species are the name's first two tokens (after + dropping a leading "uncultured"/"unclassified"/"unidentified" marker), with a + non-taxonomic species value (see clean_species_epithet) dropped. + + :param organism_name: MIBiG cluster.organism_name, or None + :return: (type_label, genus, species) -- each may be None if unresolvable + """ + if not organism_name: + return None, None, None + + tokens = organism_name.split() + if tokens and tokens[0].lower() in _NON_TAXONOMIC_PREFIXES: + tokens = tokens[1:] + if not tokens: + return None, None, None + + genus = clean_genus(tokens[0]) + if not genus: + return None, None, None + + species = clean_species_epithet(genus, tokens[1] if len(tokens) > 1 else None) + + type_label = "Fungus" if genus.lower() in FUNGAL_GENERA else "Bacterium" + return type_label, genus, species + + def split_accession_version(record_id: str) -> tuple[str, str | None]: """ Split a GenBank-style "ACCESSION.VERSION" id (e.g. "BGC0000001.5") in two. diff --git a/database/scripts/download_sources.py b/database/scripts/download_sources.py index 1237223..8b91496 100644 --- a/database/scripts/download_sources.py +++ b/database/scripts/download_sources.py @@ -14,6 +14,7 @@ from urllib.parse import urlsplit import requests +from tqdm import tqdm def download(url: str, dest: str | Path) -> None: @@ -21,10 +22,13 @@ def download(url: str, dest: str | Path) -> None: dest.parent.mkdir(parents=True, exist_ok=True) with requests.get(url, stream=True, timeout=300) as resp: resp.raise_for_status() + total = int(resp.headers.get("Content-Length") or 0) or None with open(dest, "wb") as fh: - for chunk in resp.iter_content(chunk_size=1024 * 1024): - if chunk: - fh.write(chunk) + with tqdm(total=total, desc=f"download[{dest.name}]", unit="B", unit_scale=True, unit_divisor=1024) as pbar: + for chunk in resp.iter_content(chunk_size=1024 * 1024): + if chunk: + fh.write(chunk) + pbar.update(len(chunk)) def _url_filename(url: str) -> str: diff --git a/database/scripts/extract_mibig_compounds.py b/database/scripts/extract_mibig_compounds.py index 67d3345..e0baf0d 100644 --- a/database/scripts/extract_mibig_compounds.py +++ b/database/scripts/extract_mibig_compounds.py @@ -23,6 +23,8 @@ from pathlib import Path from typing import Any, Iterator +from tqdm import tqdm + log = logging.getLogger(__name__) @@ -66,20 +68,71 @@ def _iter_compound_records(data: dict[str, Any], root: dict[str, Any], accession } -def run(mibig_json_dir: str | Path, output_path: str | Path, versions_output_path: str | Path) -> None: +# MIBiG's own short biosynthesis-class codes -> friendlier display labels for the +# chemical_class annotation. Covers every value observed across the full 4.0 corpus +# (PKS/NRPS/ribosomal/other/terpene/saccharide); an unrecognized future code falls +# back to itself unchanged rather than being dropped. +BIOSYN_CLASS_LABELS = { + "PKS": "Polyketide", + "NRPS": "Nonribosomal peptide", + "ribosomal": "RiPP", + "terpene": "Terpene", + "saccharide": "Saccharide", + "other": "Other", +} + + +def _annotations(root: dict[str, Any]) -> dict[str, Any]: + """Phylogeny + chemical-class metadata shared by every compound/BGC under one accession. + + MIBiG 4.0's real (flat) schema nests these under "taxonomy" ({"name", "ncbiTaxId"}) + and "biosynthesis" ({"classes": [{"class": "PKS", ...}, ...]}) -- not the top-level + "organism_name"/"ncbi_tax_id"/"biosyn_class" keys older MIBiG releases used. + """ + taxonomy = root.get("taxonomy") + taxonomy = taxonomy if isinstance(taxonomy, dict) else {} + organism_name = taxonomy.get("name") + ncbi_tax_id = taxonomy.get("ncbiTaxId") + + biosynthesis = root.get("biosynthesis") + biosynthesis = biosynthesis if isinstance(biosynthesis, dict) else {} + classes = biosynthesis.get("classes") + classes = classes if isinstance(classes, list) else [] + biosyn_class = [ + BIOSYN_CLASS_LABELS.get(c.get("class"), c.get("class")) + for c in classes + if isinstance(c, dict) and c.get("class") + ] + + return { + "organism_name": organism_name if isinstance(organism_name, str) else None, + "ncbi_tax_id": str(ncbi_tax_id) if ncbi_tax_id is not None else None, + "biosyn_class": biosyn_class, + } + + +def run( + mibig_json_dir: str | Path, + output_path: str | Path, + versions_output_path: str | Path, + annotations_output_path: str | Path, +) -> None: mibig_json_dir = Path(mibig_json_dir) output_path = Path(output_path) versions_output_path = Path(versions_output_path) + annotations_output_path = Path(annotations_output_path) output_path.parent.mkdir(parents=True, exist_ok=True) versions_output_path.parent.mkdir(parents=True, exist_ok=True) + annotations_output_path.parent.mkdir(parents=True, exist_ok=True) json_files = sorted(mibig_json_dir.rglob("*.json")) written = 0 skipped = 0 versions: dict[str, str] = {} + annotations: dict[str, dict[str, Any]] = {} with open(output_path, "w") as out: - for path in json_files: + for path in tqdm(json_files, desc="extract_mibig_compounds", unit="file"): try: with open(path) as fh: data = json.load(fh) @@ -98,6 +151,8 @@ def run(mibig_json_dir: str | Path, output_path: str | Path, versions_output_pat if version is not None: versions[accession] = version + annotations[accession] = _annotations(root) + had_any = False for record in _iter_compound_records(data, root, accession): out.write(json.dumps(record) + "\n") @@ -112,9 +167,13 @@ def run(mibig_json_dir: str | Path, output_path: str | Path, versions_output_pat with open(versions_output_path, "w") as fh: json.dump(versions, fh, indent=2, sort_keys=True) + with open(annotations_output_path, "w") as fh: + json.dump(annotations, fh, indent=2, sort_keys=True) + log.info( - "extract_mibig_compounds: wrote %d compound records, skipped %d files, resolved %d accession versions", - written, skipped, len(versions), + "extract_mibig_compounds: wrote %d compound records, skipped %d files, " + "resolved %d accession versions, %d accession annotations", + written, skipped, len(versions), len(annotations), ) @@ -125,9 +184,15 @@ def main() -> None: ap.add_argument("--mibig-json-dir", required=True) ap.add_argument("--output", required=True) ap.add_argument("--versions-output", required=True) + ap.add_argument("--annotations-output", required=True) args = ap.parse_args() - run(mibig_json_dir=args.mibig_json_dir, output_path=args.output, versions_output_path=args.versions_output) + run( + mibig_json_dir=args.mibig_json_dir, + output_path=args.output, + versions_output_path=args.versions_output, + annotations_output_path=args.annotations_output, + ) if __name__ == "__main__": diff --git a/database/scripts/load_bgcs.py b/database/scripts/load_bgcs.py index f6f88fc..99b6bc5 100644 --- a/database/scripts/load_bgcs.py +++ b/database/scripts/load_bgcs.py @@ -26,9 +26,12 @@ import logging from pathlib import Path -from common import build_fingerprint_context, load_ruleset, mibig_url +from tqdm import tqdm + +from common import build_fingerprint_context, load_ruleset, mibig_url, phylogeny_from_organism_name from retromol_antismash.modules import LinearReadout, bgc_primary_sequence from retromol_database.duckdb import RetroMolDuckDB +from taxonomy import TaxonomyDB, resolve_phylogeny log = logging.getLogger(__name__) @@ -39,15 +42,22 @@ def run( reaction_rules_path: str | Path | None, matching_rules_path: str | Path | None, mibig_versions_path: str | Path, + mibig_annotations_path: str | Path, match_stereochemistry: bool = False, include_raw_gbk: bool = True, + taxdump_dir: str | Path | None = None, ) -> None: ruleset = load_ruleset(reaction_rules_path, matching_rules_path, match_stereochemistry) _, fingerprinter = build_fingerprint_context(ruleset) + taxdb = TaxonomyDB.load(taxdump_dir) if taxdump_dir else None + with open(mibig_versions_path) as fh: versions: dict[str, str] = json.load(fh) + with open(mibig_annotations_path) as fh: + annotations: dict[str, dict] = json.load(fh) + added = 0 skipped = 0 skipped_existing_file = 0 @@ -55,7 +65,8 @@ def run( db = RetroMolDuckDB.open(db_path) try: with open(readouts_path) as fh: - for line in fh: + pbar = tqdm(fh, desc="load_bgcs", unit="region") + for line in pbar: line = line.strip() if not line: continue @@ -65,6 +76,7 @@ def run( if file_hash and db.bgc_content_hash_exists(file_hash): skipped_existing_file += 1 + pbar.set_postfix(added=added, skipped=skipped, skipped_existing=skipped_existing_file) continue readout = LinearReadout.from_dict(entry["readout"]) @@ -72,6 +84,7 @@ def run( if not names: skipped += 1 + pbar.set_postfix(added=added, skipped=skipped, skipped_existing=skipped_existing_file) continue fp = fingerprinter.encode(tokens) @@ -91,7 +104,38 @@ def run( fingerprint=fp, content_hash=file_hash, ) + + record = annotations.get(accession) if accession else None + if record: + fallback_type, fallback_genus, fallback_species = phylogeny_from_organism_name( + record.get("organism_name") + ) + resolution = resolve_phylogeny( + taxdb, + ncbi_tax_id=record.get("ncbi_tax_id"), + genus=fallback_genus, + species=fallback_species, + fallback_type_label=fallback_type, + ) + db.add_phylogeny_annotation( + entry_id, + type_label=resolution.type_label, + type_taxid=resolution.type_taxid, + genus=resolution.genus, + genus_taxid=resolution.genus_taxid, + species=resolution.species, + species_taxid=resolution.species_taxid, + ) + # biosynthetic_class describes this BGC's own biosynthesis machinery + # (PKS/NRPS/RiPP/...) -- a gene-cluster property, not populated for + # compounds (see RetroMolDuckDB.add_biosynthetic_class_annotation / + # load_compounds.py). + for chemical_class in record.get("biosyn_class") or []: + if chemical_class: + db.add_biosynthetic_class_annotation(entry_id, str(chemical_class)) + added += 1 + pbar.set_postfix(added=added, skipped=skipped, skipped_existing=skipped_existing_file) finally: db.close() @@ -110,8 +154,10 @@ def main() -> None: ap.add_argument("--rxn-rules", default=None) ap.add_argument("--mxn-rules", default=None) ap.add_argument("--mibig-versions", required=True) + ap.add_argument("--mibig-annotations", required=True) ap.add_argument("--match-stereochemistry", action="store_true") ap.add_argument("--no-raw-gbk", action="store_true") + ap.add_argument("--taxdump-dir", default=None, help="dir with NCBI names.dmp/nodes.dmp (skips taxid resolution if omitted)") args = ap.parse_args() run( @@ -120,8 +166,10 @@ def main() -> None: reaction_rules_path=args.rxn_rules, matching_rules_path=args.mxn_rules, mibig_versions_path=args.mibig_versions, + mibig_annotations_path=args.mibig_annotations, match_stereochemistry=args.match_stereochemistry, include_raw_gbk=not args.no_raw_gbk, + taxdump_dir=args.taxdump_dir, ) diff --git a/database/scripts/load_compounds.py b/database/scripts/load_compounds.py index b22a5ab..1898591 100644 --- a/database/scripts/load_compounds.py +++ b/database/scripts/load_compounds.py @@ -23,16 +23,20 @@ from common import ( build_fingerprint_context, + clean_genus, + clean_species_epithet, find_key_ci, load_ruleset, mibig_url, npatlas_url, per_monomer_tokens, + phylogeny_from_organism_name, primary_sequence_from_result, ) from retromol.model.result import Result from retromol_database.duckdb import RetroMolDuckDB from retromol_fingerprint.fingerprint import TOKEN_LINK +from taxonomy import TaxonomyDB, resolve_phylogeny log = logging.getLogger(__name__) @@ -49,6 +53,30 @@ def _npatlas_name_and_url(props: dict) -> tuple[str | None, str | None]: return name, npatlas_url(npaid) +def _npatlas_phylogeny(props: dict) -> tuple[str | None, str | None, str | None]: + """NPAtlas's SDF carries type/genus/species directly (unlike MIBiG's free-text + organism_name) -- see the `origin_type`/`genus`/`origin_species` SDF properties. + + Both genus and origin_species are used as-is by NPAtlas's own curators, which + turns out to include the same non-taxonomic placeholders MIBiG's free text does + (bare "sp.", a genus name duplicated into the species field e.g. "Streptomyces + sp.", "unidentified") -- cleaned the same way as MIBiG's organism_name parsing + (see common.clean_genus/clean_species_epithet), so neither source's placeholder + values end up stored as if they were real species-level identifications. + """ + type_key = find_key_ci(props, ["origin_type"]) + type_label = props.get(type_key) if type_key else None + + genus_key = find_key_ci(props, ["genus"]) + genus = clean_genus(props.get(genus_key) if genus_key else None) + + species_key = find_key_ci(props, ["origin_species"]) + species_raw = props.get(species_key) if species_key else None + species = clean_species_epithet(genus, species_raw) + + return (type_label or None), genus, species + + def _mibig_name_and_url(props: dict, versions: dict[str, str]) -> tuple[str | None, str | None]: accession = props.get("mibig_accession") version = versions.get(accession) if accession else None @@ -56,6 +84,35 @@ def _mibig_name_and_url(props: dict, versions: dict[str, str]) -> tuple[str | No return name, mibig_url(accession, version) +def _apply_mibig_annotations( + db: RetroMolDuckDB, entry_id: str, props: dict, annotations: dict[str, dict], taxdb: TaxonomyDB | None +) -> None: + accession = props.get("mibig_accession") + record = annotations.get(accession) if accession else None + if not record: + return + + fallback_type, fallback_genus, fallback_species = phylogeny_from_organism_name(record.get("organism_name")) + resolution = resolve_phylogeny( + taxdb, + ncbi_tax_id=record.get("ncbi_tax_id"), + genus=fallback_genus, + species=fallback_species, + fallback_type_label=fallback_type, + ) + db.add_phylogeny_annotation( + entry_id, + type_label=resolution.type_label, + type_taxid=resolution.type_taxid, + genus=resolution.genus, + genus_taxid=resolution.genus_taxid, + species=resolution.species, + species_taxid=resolution.species_taxid, + ) + # biosynthetic_class describes the BGC's own biosynthesis machinery (PKS/NRPS/...), + # not the compound structure -- populated in load_bgcs.py instead, not here. + + def run( results_path: str | Path, db_path: str | Path, @@ -64,16 +121,25 @@ def run( matching_rules_path: str | Path | None, match_stereochemistry: bool = False, mibig_versions_path: str | Path | None = None, + mibig_annotations_path: str | Path | None = None, + taxdump_dir: str | Path | None = None, log_every: int = 1000, ) -> None: ruleset = load_ruleset(reaction_rules_path, matching_rules_path, match_stereochemistry) name_to_rule, fingerprinter = build_fingerprint_context(ruleset) + taxdb = TaxonomyDB.load(taxdump_dir) if taxdump_dir else None + versions: dict[str, str] = {} if source == "mibig" and mibig_versions_path is not None: with open(mibig_versions_path) as fh: versions = json.load(fh) + annotations: dict[str, dict] = {} + if source == "mibig" and mibig_annotations_path is not None: + with open(mibig_annotations_path) as fh: + annotations = json.load(fh) + compounds = 0 added = 0 skipped = 0 @@ -118,6 +184,22 @@ def run( primary_sequence=names, fingerprint=fp, ) + if source == "mibig": + _apply_mibig_annotations(db, result.submission.inchikey, props, annotations, taxdb) + else: + type_label, genus, species = _npatlas_phylogeny(props) + resolution = resolve_phylogeny( + taxdb, genus=genus, species=species, fallback_type_label=type_label + ) + db.add_phylogeny_annotation( + result.submission.inchikey, + type_label=resolution.type_label, + type_taxid=resolution.type_taxid, + genus=resolution.genus, + genus_taxid=resolution.genus_taxid, + species=resolution.species, + species_taxid=resolution.species_taxid, + ) added += 1 compounds += 1 @@ -146,6 +228,8 @@ def main() -> None: ap.add_argument("--mxn-rules", default=None) ap.add_argument("--match-stereochemistry", action="store_true") ap.add_argument("--mibig-versions", default=None, help="required when --source=mibig") + ap.add_argument("--mibig-annotations", default=None, help="required when --source=mibig") + ap.add_argument("--taxdump-dir", default=None, help="dir with NCBI names.dmp/nodes.dmp (skips taxid resolution if omitted)") ap.add_argument("--log-every", type=int, default=1000, help="log a progress line every N compounds (0 to disable)") args = ap.parse_args() @@ -157,6 +241,8 @@ def main() -> None: matching_rules_path=args.mxn_rules, match_stereochemistry=args.match_stereochemistry, mibig_versions_path=args.mibig_versions, + mibig_annotations_path=args.mibig_annotations, + taxdump_dir=args.taxdump_dir, log_every=args.log_every, ) diff --git a/database/scripts/npclassifier.py b/database/scripts/npclassifier.py new file mode 100644 index 0000000..2276f17 --- /dev/null +++ b/database/scripts/npclassifier.py @@ -0,0 +1,84 @@ +"""NPClassifier API client for chemical-class annotation. + +Classifies a compound's chemical class directly from its own structure via GNPS2's +free, public NPClassifier service (https://npclassifier.gnps2.org/classify?smiles=...) -- +independent of MIBiG's coarse biosynthetic-class labels (PKS/NRPS/...), and available +for NPAtlas compounds too, which MIBiG's biosyn_class never was. A classification has +three list-valued levels (pathway/superclass/class, most to least general) plus a single +is_glycoside boolean; response shape confirmed live against the API (2026-08-26): +{"pathway_results": [...], "superclass_results": [...], "class_results": [...], "isglycoside": bool}. + +No published rate limit exists for this service, so callers are responsible for pacing +requests themselves (see annotate_npclassifier.py's --requests-per-second) -- this module +only wraps a single call with retry/backoff. +""" + +from __future__ import annotations + +import json +import logging +import time +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass + +log = logging.getLogger(__name__) + +NPCLASSIFIER_URL = "https://npclassifier.gnps2.org/classify" + + +@dataclass(frozen=True) +class ClassificationResult: + pathway: list[str] + superclass: list[str] + class_: list[str] + is_glycoside: bool + + +def classify_smiles( + smiles: str, + *, + timeout: float = 30.0, + max_retries: int = 3, + backoff_seconds: float = 2.0, +) -> ClassificationResult | None: + """Classify one SMILES via NPClassifier, retrying transient failures (timeouts, 5xx, + 429, malformed JSON) with linear backoff. Returns None (logged, not raised) if every + attempt fails -- one unclassifiable/unreachable-service molecule shouldn't abort a + whole pipeline run.""" + url = f"{NPCLASSIFIER_URL}?smiles={urllib.parse.quote(smiles, safe='')}" + + for attempt in range(1, max_retries + 1): + try: + with urllib.request.urlopen(url, timeout=timeout) as resp: + data = json.loads(resp.read()) + return ClassificationResult( + pathway=[str(p) for p in data.get("pathway_results") or []], + superclass=[str(s) for s in data.get("superclass_results") or []], + class_=[str(c) for c in data.get("class_results") or []], + is_glycoside=bool(data.get("isglycoside", False)), + ) + except (urllib.error.URLError, urllib.error.HTTPError, json.JSONDecodeError, TimeoutError, OSError) as exc: + if isinstance(exc, urllib.error.HTTPError) and exc.code == 429: + # Rate-limited: back off much harder than a transient failure, and + # respect Retry-After if the server sent one -- retrying at the same + # pace that just got us 429'd only makes the storm worse. + retry_after = exc.headers.get("Retry-After") if exc.headers else None + try: + delay = float(retry_after) if retry_after is not None else backoff_seconds * attempt * 5 + except ValueError: + delay = backoff_seconds * attempt * 5 + log.warning( + "NPClassifier rate-limited (429) on attempt %d/%d for %r -- backing off %.1fs", + attempt, max_retries, smiles, delay, + ) + else: + delay = backoff_seconds * attempt + log.warning("NPClassifier request failed (attempt %d/%d) for %r: %s", attempt, max_retries, smiles, exc) + + if attempt < max_retries: + time.sleep(delay) + + log.error("NPClassifier: giving up on %r after %d attempts", smiles, max_retries) + return None diff --git a/database/scripts/taxonomy.py b/database/scripts/taxonomy.py new file mode 100644 index 0000000..3f6b580 --- /dev/null +++ b/database/scripts/taxonomy.py @@ -0,0 +1,235 @@ +"""NCBI taxonomy dump download + name/taxid resolution for phylogeny annotation. + +Downloads and parses `taxdump.tar.gz` (https://ftp.ncbi.nlm.nih.gov/pub/taxonomy/taxdump.tar.gz) +once, then resolves organism names <-> NCBI taxids and classifies a taxid's broad "type" +(Bacterium/Archaeon/Fungus/Other) by walking its lineage. MIBiG's JSON gives a taxid +directly (see extract_mibig_compounds.py's `taxonomy.ncbiTaxId`); NPAtlas gives only +genus/species text, resolved here by scientific-name/synonym lookup instead -- so both +sources end up with identically-standardized taxids and canonical names. +""" + +from __future__ import annotations + +import logging +import tarfile +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Iterator + +from tqdm import tqdm + +log = logging.getLogger(__name__) + +TAXDUMP_URL = "https://ftp.ncbi.nlm.nih.gov/pub/taxonomy/taxdump.tar.gz" + +# Fixed NCBI taxids for the lineage nodes used to classify a taxon's broad "type" (see +# TaxonomyDB.type_label_and_taxid) -- these are stable anchor points in NCBI's taxonomy, +# not resolved by name. +TAXID_BACTERIA = 2 +TAXID_ARCHAEA = 2157 +TAXID_FUNGI = 4751 +TYPE_LABELS_BY_TAXID = { + TAXID_BACTERIA: "Bacterium", + TAXID_ARCHAEA: "Archaeon", + TAXID_FUNGI: "Fungus", +} + + +def download_taxdump(dest_dir: str | Path, *, force: bool = False) -> Path: + """Download and extract names.dmp/nodes.dmp from NCBI's taxdump into `dest_dir` + (no-op if both files already exist there, unless `force`).""" + dest_dir = Path(dest_dir).expanduser() + dest_dir.mkdir(parents=True, exist_ok=True) + + names_path = dest_dir / "names.dmp" + nodes_path = dest_dir / "nodes.dmp" + if not force and names_path.exists() and nodes_path.exists(): + return dest_dir + + log.info("downloading NCBI taxdump from %s", TAXDUMP_URL) + archive_path = dest_dir / "taxdump.tar.gz" + with urllib.request.urlopen(TAXDUMP_URL) as resp, open(archive_path, "wb") as out: + total = int(resp.headers.get("Content-Length") or 0) or None + with tqdm(total=total, desc="download_taxdump", unit="B", unit_scale=True, unit_divisor=1024) as pbar: + while chunk := resp.read(1024 * 1024): + out.write(chunk) + pbar.update(len(chunk)) + + with tarfile.open(archive_path, mode="r:gz") as tar: + for member in tar.getmembers(): + if member.name in ("names.dmp", "nodes.dmp"): + tar.extract(member, path=dest_dir) + archive_path.unlink() + + if not names_path.exists() or not nodes_path.exists(): + raise FileNotFoundError(f"taxdump at {TAXDUMP_URL} did not contain names.dmp/nodes.dmp") + + return dest_dir + + +def _iter_dmp_rows(path: Path) -> Iterator[list[str]]: + """NCBI .dmp rows are "\\t|\\t"-separated, terminated by a trailing "\\t|".""" + with open(path, encoding="utf-8", errors="replace") as fh: + for line in fh: + line = line.rstrip("\n").rstrip("\t|") + yield [field.strip() for field in line.split("\t|\t")] + + +@dataclass(frozen=True) +class PhylogenyResolution: + type_label: str | None + type_taxid: str | None + genus: str | None + genus_taxid: str | None + species: str | None + species_taxid: str | None + + +@dataclass +class TaxonomyDB: + """In-memory index over NCBI's taxdump, built once per pipeline run.""" + + parent_by_taxid: dict[int, int] + rank_by_taxid: dict[int, str] + scientific_name_by_taxid: dict[int, str] + taxid_by_name: dict[str, int] # lowercased scientific name/synonym -> taxid + + @classmethod + def load(cls, taxdump_dir: str | Path) -> "TaxonomyDB": + taxdump_dir = Path(taxdump_dir).expanduser() + + parent_by_taxid: dict[int, int] = {} + rank_by_taxid: dict[int, str] = {} + for row in _iter_dmp_rows(taxdump_dir / "nodes.dmp"): + taxid, parent_taxid, rank = int(row[0]), int(row[1]), row[2] + parent_by_taxid[taxid] = parent_taxid + rank_by_taxid[taxid] = rank + + scientific_name_by_taxid: dict[int, str] = {} + taxid_by_name: dict[str, int] = {} + for row in _iter_dmp_rows(taxdump_dir / "names.dmp"): + taxid, name_txt, name_class = int(row[0]), row[1], row[3] + if name_class == "scientific name": + scientific_name_by_taxid[taxid] = name_txt + # Index every name class (scientific name, synonym, common name, ...) -- + # an organism string from a compound source can be any of these. + taxid_by_name.setdefault(name_txt.lower(), taxid) + + log.info("loaded NCBI taxdump: %d nodes, %d names", len(parent_by_taxid), len(taxid_by_name)) + return cls( + parent_by_taxid=parent_by_taxid, + rank_by_taxid=rank_by_taxid, + scientific_name_by_taxid=scientific_name_by_taxid, + taxid_by_name=taxid_by_name, + ) + + def resolve_taxid(self, name: str | None) -> int | None: + """Look up a taxid by scientific name or synonym, case-insensitive.""" + if not name: + return None + return self.taxid_by_name.get(name.strip().lower()) + + def canonical_name(self, taxid: int | None) -> str | None: + if taxid is None: + return None + return self.scientific_name_by_taxid.get(taxid) + + def lineage(self, taxid: int) -> list[int]: + """The ancestor chain from `taxid` (inclusive) up to the root, stopping early + if a cycle or a gap in nodes.dmp is hit.""" + chain = [taxid] + seen = {taxid} + current = taxid + while current in self.parent_by_taxid and current != 1: + parent = self.parent_by_taxid[current] + if parent == current or parent in seen: + break + chain.append(parent) + seen.add(parent) + current = parent + return chain + + def ancestor_at_rank(self, taxid: int | None, rank: str) -> int | None: + """The ancestor of `taxid` (inclusive) whose rank is exactly `rank` (e.g. + "genus", "species"), or None if no such ancestor exists in the lineage.""" + if taxid is None: + return None + for ancestor in self.lineage(taxid): + if self.rank_by_taxid.get(ancestor) == rank: + return ancestor + return None + + def type_label_and_taxid(self, taxid: int | None) -> tuple[str | None, int | None]: + """Classify a taxid's broad type by walking its lineage for one of the fixed + Bacteria/Archaea/Fungi anchor nodes; "Other" if the lineage resolves but hits + none of them (e.g. Viruses, Protista).""" + if taxid is None: + return None, None + for ancestor in self.lineage(taxid): + if ancestor in TYPE_LABELS_BY_TAXID: + return TYPE_LABELS_BY_TAXID[ancestor], ancestor + return "Other", None + + +_UNRESOLVED = PhylogenyResolution(None, None, None, None, None, None) + + +def resolve_phylogeny( + taxdb: TaxonomyDB | None, + *, + ncbi_tax_id: str | int | None = None, + genus: str | None = None, + species: str | None = None, + fallback_type_label: str | None = None, +) -> PhylogenyResolution: + """Standardize phylogeny fields to NCBI taxids -- and *only* to NCBI taxids: every + label stored is read back off a resolved taxid via TaxonomyDB.canonical_name, never + passed through as raw, unstandardized source text. If a taxid can't be resolved at + all (no taxdb loaded, or the given ncbi_tax_id/genus/species doesn't match anything + in NCBI's taxonomy), the result is fully unannotated -- type/genus/species all + None -- rather than a best-effort guess from whatever text the source gave us. + `fallback_type_label` (NPAtlas's own origin_type field) is accepted for API + compatibility with callers but is deliberately unused for the same reason: it isn't + NCBI vocabulary either. + + If `ncbi_tax_id` is given (MIBiG), it's the source of truth: genus/species/type and + their taxids are all derived from its lineage, overriding any given genus/species + text. Otherwise (NPAtlas, or an MIBiG entry whose ncbi_tax_id didn't parse), + `genus`/`species` text is resolved to a taxid by name lookup -- "Genus species" + first, falling back to genus alone -- so both sources end up identically + standardized when they resolve at all. + """ + if taxdb is None: + return _UNRESOLVED + + leaf_taxid: int | None = None + if ncbi_tax_id: + try: + leaf_taxid = int(ncbi_tax_id) + except (TypeError, ValueError): + leaf_taxid = None + + if leaf_taxid is None and genus: + query = f"{genus} {species}" if species else genus + leaf_taxid = taxdb.resolve_taxid(query) or taxdb.resolve_taxid(genus) + + if leaf_taxid is None: + return _UNRESOLVED + + type_label, type_taxid = taxdb.type_label_and_taxid(leaf_taxid) + + genus_taxid = taxdb.ancestor_at_rank(leaf_taxid, "genus") + genus_name = taxdb.canonical_name(genus_taxid) if genus_taxid else None + + species_taxid = taxdb.ancestor_at_rank(leaf_taxid, "species") + species_name = taxdb.canonical_name(species_taxid) if species_taxid else None + + return PhylogenyResolution( + type_label=type_label, + type_taxid=str(type_taxid) if type_taxid is not None else None, + genus=genus_name, + genus_taxid=str(genus_taxid) if genus_taxid is not None else None, + species=species_name, + species_taxid=str(species_taxid) if species_taxid is not None else None, + ) diff --git a/gui/scripts/dev_backend.sh b/gui/scripts/dev_backend.sh index ccce31e..477c435 100644 --- a/gui/scripts/dev_backend.sh +++ b/gui/scripts/dev_backend.sh @@ -10,7 +10,7 @@ export FLASK_ENV=development export PORT=4000 # DB connection -export RETROMOL_DUCKDB_PATH="$HOME/Downloads/retromol.duckdb" +export RETROMOL_DUCKDB_PATH="$HOME/Desktop/retromol.duckdb" # Redis connection (uses Dockerized Redis) export REDIS_URL="redis://localhost:6379/0" diff --git a/gui/src/client/package-lock.json b/gui/src/client/package-lock.json index a29b418..8e7e256 100644 --- a/gui/src/client/package-lock.json +++ b/gui/src/client/package-lock.json @@ -16,6 +16,7 @@ "@mui/icons-material": "^7.3.5", "@mui/material": "^7.3.5", "@mui/x-charts": "^8.17.0", + "@mui/x-data-grid": "^9.12.0", "@react-spring/web": "^10.0.3", "@tanstack/react-query": "^5.76.1", "@types/react": "18.3.10", @@ -2263,6 +2264,28 @@ "node": ">=6.9.0" } }, + "node_modules/@base-ui/utils": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@base-ui/utils/-/utils-0.3.2.tgz", + "integrity": "sha512-oWy1aq/I2GmYjpl4PhEAhzflF8VPGKgZeq0xAWTbfD5KBWyxcN0ZP2+WHSUm/5Z6lVMBDLReLcoXwSYoRc/zNQ==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.29.2", + "@floating-ui/utils": "^0.2.12", + "reselect": "^5.2.0", + "use-sync-external-store": "^1.6.0" + }, + "peerDependencies": { + "@types/react": "^17 || ^18 || ^19", + "react": "^17 || ^18 || ^19", + "react-dom": "^17 || ^18 || ^19" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, "node_modules/@bcoe/v8-coverage": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz", @@ -2884,9 +2907,9 @@ "license": "Python-2.0" }, "node_modules/@eslint/eslintrc/node_modules/js-yaml": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.1.tgz", - "integrity": "sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", "dev": true, "funding": [ { @@ -2916,6 +2939,12 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, + "node_modules/@floating-ui/utils": { + "version": "0.2.12", + "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.12.tgz", + "integrity": "sha512-HpCo8tmWzLVad5s2d19EhAz5zqrrQ6s69qd6moPMQvkOuSwDT1YgRfWSVuc4ennqrgv3OHppiOGMQ7oC13yIww==", + "license": "MIT" + }, "node_modules/@humanwhocodes/config-array": { "version": "0.13.0", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.13.0.tgz", @@ -3595,16 +3624,16 @@ } }, "node_modules/@mui/x-charts": { - "version": "8.29.2", - "resolved": "https://registry.npmjs.org/@mui/x-charts/-/x-charts-8.29.2.tgz", - "integrity": "sha512-s0f0gstCEBO4HV1aj4wNapL6RYREO+YyFLs4cSKm8bu347AuADngBtepNSCXuzsCQ+C+PxAlBoCy2a1ssTxMuA==", + "version": "8.29.3", + "resolved": "https://registry.npmjs.org/@mui/x-charts/-/x-charts-8.29.3.tgz", + "integrity": "sha512-8hF+rnf/rj258FmBMx6XwNmdFXM+gCnR9TKpd8ywJUwVPzZoZ0U/LMtsFHe16MZ9YOpOhTSdKHycq/OmiYXFlQ==", "license": "MIT", "dependencies": { "@babel/runtime": "^7.28.4", "@mui/utils": "^7.3.5", "@mui/x-charts-vendor": "8.29.0", "@mui/x-internal-gestures": "0.5.0", - "@mui/x-internals": "8.29.2", + "@mui/x-internals": "8.29.3", "bezier-easing": "^2.1.0", "clsx": "^2.1.1", "prop-types": "^15.8.1", @@ -3662,6 +3691,116 @@ "internmap": "^2.0.3" } }, + "node_modules/@mui/x-data-grid": { + "version": "9.12.0", + "resolved": "https://registry.npmjs.org/@mui/x-data-grid/-/x-data-grid-9.12.0.tgz", + "integrity": "sha512-lI7NYAPhoLIWJECB2gZuVairzWviDDOE1RpjLcPXJM+hw1mwCG+isqNfx81X+17xuHtnEEBVahCDWAJFAvCF9g==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.29.7", + "@base-ui/utils": "^0.3.1", + "@mui/utils": "^9.3.0", + "@mui/x-internals": "^9.12.0", + "@mui/x-virtualizer": "0.7.0", + "clsx": "^2.1.1", + "prop-types": "^15.8.1", + "use-sync-external-store": "^1.6.0" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "@emotion/react": "^11.9.0", + "@emotion/styled": "^11.8.1", + "@mui/material": "^7.3.0 || ^9.0.0", + "@mui/system": "^7.3.0 || ^9.0.0", + "react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react-dom": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@emotion/react": { + "optional": true + }, + "@emotion/styled": { + "optional": true + } + } + }, + "node_modules/@mui/x-data-grid/node_modules/@mui/types": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/@mui/types/-/types-9.3.0.tgz", + "integrity": "sha512-2JSxyfpEFWNUB2vKs/T1BvkfyNisMHWph8bLMj8T0uHwmLl/0qfAwQkfwMT6kxLXN9uIum9AEbECXU8er3amIg==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.29.7" + }, + "peerDependencies": { + "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@mui/x-data-grid/node_modules/@mui/utils": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/@mui/utils/-/utils-9.3.0.tgz", + "integrity": "sha512-2HZdHwWJ6eB+7lVGSOHsByGw8jeRulT4g0NZ608Wb8Q57DE2jbNqrWPFuJsvkQQiBiTmlpvQL3i+/62zsiPrkw==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.29.7", + "@mui/types": "^9.3.0", + "@types/prop-types": "^15.7.15", + "clsx": "^2.1.1", + "prop-types": "^15.8.1", + "react-is": "^19.2.8" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@mui/x-data-grid/node_modules/@mui/x-internals": { + "version": "9.12.0", + "resolved": "https://registry.npmjs.org/@mui/x-internals/-/x-internals-9.12.0.tgz", + "integrity": "sha512-rpG9EgJhH3JeOK2Q/tKaUWk5vP8C1NfL8DaXMKR1XCA7RhpZUI/xT67WKY80QtogNWu7U/BRezKyQGtiZ8soYg==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.29.7", + "@base-ui/utils": "^0.3.1", + "@mui/utils": "^9.3.0", + "reselect": "^5.2.0", + "use-sync-external-store": "^1.6.0" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react-dom": "^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, "node_modules/@mui/x-internal-gestures": { "version": "0.5.0", "resolved": "https://registry.npmjs.org/@mui/x-internal-gestures/-/x-internal-gestures-0.5.0.tgz", @@ -3672,9 +3811,9 @@ } }, "node_modules/@mui/x-internals": { - "version": "8.29.2", - "resolved": "https://registry.npmjs.org/@mui/x-internals/-/x-internals-8.29.2.tgz", - "integrity": "sha512-TLILyHia5NHh3MGErFDh0bXZ4V6iS45hdStofsV5wklF6dpgZjduo65oJB0h81mI5mexNlhHANEVfw0KV6BxBg==", + "version": "8.29.3", + "resolved": "https://registry.npmjs.org/@mui/x-internals/-/x-internals-8.29.3.tgz", + "integrity": "sha512-xJSaAXQDZ+35svja+i5N0qg7gTbZwcnAGZBgC+37Z3SoxRlTR5NwNBUlqKDm5TJwLvG17DeDYolxqWtcR93tsg==", "license": "MIT", "dependencies": { "@babel/runtime": "^7.28.4", @@ -3693,6 +3832,100 @@ "react": "^17.0.0 || ^18.0.0 || ^19.0.0" } }, + "node_modules/@mui/x-virtualizer": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/@mui/x-virtualizer/-/x-virtualizer-0.7.0.tgz", + "integrity": "sha512-Bz5p/t78zE2q6rf/nHY3j29hr7JlWaBNT0OD8jZFcfstr8jqrPkzirGsl/Bz5EpEjEzxV73kLfZLpIToGql+rA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.29.7", + "@base-ui/utils": "^0.3.1", + "@mui/utils": "^9.3.0", + "@mui/x-internals": "^9.12.0" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react-dom": "^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/@mui/x-virtualizer/node_modules/@mui/types": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/@mui/types/-/types-9.3.0.tgz", + "integrity": "sha512-2JSxyfpEFWNUB2vKs/T1BvkfyNisMHWph8bLMj8T0uHwmLl/0qfAwQkfwMT6kxLXN9uIum9AEbECXU8er3amIg==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.29.7" + }, + "peerDependencies": { + "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@mui/x-virtualizer/node_modules/@mui/utils": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/@mui/utils/-/utils-9.3.0.tgz", + "integrity": "sha512-2HZdHwWJ6eB+7lVGSOHsByGw8jeRulT4g0NZ608Wb8Q57DE2jbNqrWPFuJsvkQQiBiTmlpvQL3i+/62zsiPrkw==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.29.7", + "@mui/types": "^9.3.0", + "@types/prop-types": "^15.7.15", + "clsx": "^2.1.1", + "prop-types": "^15.8.1", + "react-is": "^19.2.8" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@mui/x-virtualizer/node_modules/@mui/x-internals": { + "version": "9.12.0", + "resolved": "https://registry.npmjs.org/@mui/x-internals/-/x-internals-9.12.0.tgz", + "integrity": "sha512-rpG9EgJhH3JeOK2Q/tKaUWk5vP8C1NfL8DaXMKR1XCA7RhpZUI/xT67WKY80QtogNWu7U/BRezKyQGtiZ8soYg==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.29.7", + "@base-ui/utils": "^0.3.1", + "@mui/utils": "^9.3.0", + "reselect": "^5.2.0", + "use-sync-external-store": "^1.6.0" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react-dom": "^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, "node_modules/@nicolo-ribaudo/eslint-scope-5-internals": { "version": "5.1.1-v1", "resolved": "https://registry.npmjs.org/@nicolo-ribaudo/eslint-scope-5-internals/-/eslint-scope-5-internals-5.1.1-v1.tgz", @@ -4281,9 +4514,9 @@ } }, "node_modules/@tanstack/query-core": { - "version": "5.101.4", - "resolved": "https://registry.npmjs.org/@tanstack/query-core/-/query-core-5.101.4.tgz", - "integrity": "sha512-gNwcvOJcRbLWPOLG/2OBm+zM+Yv+MKsXKEOWC57USuZDEsI71hEErQsiEGx5wX9rzWWkfwM0fVSPoiIFSsxfiw==", + "version": "5.102.6", + "resolved": "https://registry.npmjs.org/@tanstack/query-core/-/query-core-5.102.6.tgz", + "integrity": "sha512-jp+GucyQ+fel2ILb8ZLQeqMaAqZL/Bzaj+dKOQiPk4vPdf2rQPEV7heUyEVIhatpY42j4AFHav4DzBa1oTIp/A==", "license": "MIT", "funding": { "type": "github", @@ -4291,12 +4524,12 @@ } }, "node_modules/@tanstack/react-query": { - "version": "5.101.4", - "resolved": "https://registry.npmjs.org/@tanstack/react-query/-/react-query-5.101.4.tgz", - "integrity": "sha512-yRg2pfOCxIs4ZJW3XYYHU/WgtD04FHSnfHlpRT7h7pR77hwkdRG4wxbKe4aq6P0RvXUTBSQpQeadS1SUYUe+KA==", + "version": "5.102.6", + "resolved": "https://registry.npmjs.org/@tanstack/react-query/-/react-query-5.102.6.tgz", + "integrity": "sha512-ANz4KZ8z80D85fiz5IAHjpb8XBPLrjOb/0natlD9Ascyy/3p96V86Zw8UbOfA0HDLcvK/A4gNd95r723y5UT2w==", "license": "MIT", "dependencies": { - "@tanstack/query-core": "5.101.4" + "@tanstack/query-core": "5.102.6" }, "funding": { "type": "github", @@ -4383,9 +4616,9 @@ } }, "node_modules/@testing-library/user-event": { - "version": "14.6.3", - "resolved": "https://registry.npmjs.org/@testing-library/user-event/-/user-event-14.6.3.tgz", - "integrity": "sha512-6dBq67jT8lE+JTE8Exm02Kt6ze43hz1jdiSpSJwtTZiT1xQQ6b7nZYTTQ9njdArdU8XklOwaDp/AbT/eYSKF4g==", + "version": "14.6.6", + "resolved": "https://registry.npmjs.org/@testing-library/user-event/-/user-event-14.6.6.tgz", + "integrity": "sha512-Jbs9FpkkIDw8FgSc6kOVsOv8JuuqGAL7J4X1oot77JxAoDlkNn2GRkd0aYRVuQ+pVQAiHWVkE4rX/dkF5fBiCw==", "dev": true, "license": "MIT", "engines": { @@ -4407,9 +4640,9 @@ } }, "node_modules/@tsconfig/node10": { - "version": "1.0.12", - "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.12.tgz", - "integrity": "sha512-UCYBaeFvM11aU2y3YPZ//O5Rhj+xKyzy7mvcIoAjASbigy8mHMryP5cK7dgjlz2hWxh1g5pLw084E0a/wlUSFQ==", + "version": "1.0.13", + "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.13.tgz", + "integrity": "sha512-gcLdvR9HO1ZJBypsOGqaP6TFEzb6vIta0KSTLt9NAQ6pXQO3cRgSVyCN6pzYqI9DlJgY71XKO0dpDhCf08b3pg==", "dev": true, "license": "MIT" }, @@ -4571,9 +4804,9 @@ } }, "node_modules/@types/d3-shape": { - "version": "3.1.8", - "resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.1.8.tgz", - "integrity": "sha512-lae0iWfcDeR7qt7rA88BNiqdvPS5pFVPpo5OfjElwNaT2yyekbM0C9vK+yqBqEmHr6lDkRnYNoTBYlAgJa7a4w==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.2.0.tgz", + "integrity": "sha512-kVd74ta9eof3eJOvbNd1vGKS/XERRyQbT26Og63hIsvDO84cjD5gEOhsXf26w3FSoNlPVz84DOFcKv/oou+fMw==", "license": "MIT", "dependencies": { "@types/d3-path": "*" @@ -4747,9 +4980,9 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "26.2.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-26.2.0.tgz", - "integrity": "sha512-5IviulTZeRNp2vAJ514cc/HUlY5nZ9fCbq9DMyC52BrhFZACo3nI0R7qBxhQmo/d27NFe96ur/b7Wwxklda+kg==", + "version": "26.4.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.4.0.tgz", + "integrity": "sha512-faiGnoIrLH/V8cibOMEAZ8pMw6oXqSukl29ra4mN8GdaB2ZewzeaLj+INpV5N+Z1eKWzY+IzaIZH2EIR6YZRNQ==", "dev": true, "license": "MIT", "peer": true, @@ -5198,9 +5431,9 @@ } }, "node_modules/@ungap/structured-clone": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.3.tgz", - "integrity": "sha512-60YRaenCQcVjYEKOcG824+DRGGIQ3VKErcBoAEDJZz5bKIs2ZG+X/H9Nk+Q6EVkwJk5QNApxbrc5QtBSwtrXAg==", + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.4.tgz", + "integrity": "sha512-JL+CF0GeLHyPWI0rXu7UnxgiuOm9UQWzadi0OYOJNhNO2q6EZElpwlgXkNkfU1PzANDHq3YcwKVZprdvS+BrbQ==", "dev": true, "license": "ISC" }, @@ -6300,9 +6533,9 @@ "license": "MIT" }, "node_modules/baseline-browser-mapping": { - "version": "2.11.13", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.13.tgz", - "integrity": "sha512-k9HNuUVMlqVjQ9UHzfPjIqiDbWw7WqT1AoT7GL8VwvF3r0ZfArtgiSPAlmupyNquNgOJHTuH4CKYf8ttMTWBTQ==", + "version": "2.11.19", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.19.tgz", + "integrity": "sha512-Grytf1xOxOEMTGRwx6rLGKkTabd4vMg3VrKdj/7joCmV0qgh4QwMMO6xh34YEXQqirAuUdgQGa5orJQQ+69RBw==", "dev": true, "license": "Apache-2.0", "bin": { @@ -6658,9 +6891,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001809", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001809.tgz", - "integrity": "sha512-xxWVywk6a6Arlk+hymeycyn/VgqEfLDxupvhH/xiY5SJ/18kmi9o6MiO320DCUzypORHLtvh0I4i04tUhCNHNQ==", + "version": "1.0.30001810", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001810.tgz", + "integrity": "sha512-TITQPUkaz+aVk5GL6NhOdwk1aEaNTSDPsGFWrTuhKGtjTF70jL/Oht2W4c6rXUe5fu7Ie19VIahAXHIIiWWNeg==", "dev": true, "funding": [ { @@ -7006,9 +7239,9 @@ "license": "MIT" }, "node_modules/colord": { - "version": "2.9.3", - "resolved": "https://registry.npmjs.org/colord/-/colord-2.9.3.tgz", - "integrity": "sha512-jeC1axXpnb0/2nn/Y1LPuLdgXBLH7aDcHu4KEKfqw3CUhX7ZpfBSlPKyqXE6btIgEzfWtrX3/tyBCaCvXvMkOw==", + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/colord/-/colord-2.10.0.tgz", + "integrity": "sha512-AidJptpBJmjTclAp9BkLwJi0T93fo5epJnbaZslpg6QVzpHjAiveF55mE9AcUJiGMqRHgMDY8soMsQtuNYMHfw==", "dev": true, "license": "MIT" }, @@ -7898,9 +8131,9 @@ } }, "node_modules/dayjs": { - "version": "1.11.21", - "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.21.tgz", - "integrity": "sha512-98IT+HOahAisibz/yjKbzuOBwYcjJ7BCLPzARyHiyEBmRz4fatF+KPJszEHXsGYjUG234aH/cOjW1wwTbKUZlA==", + "version": "1.11.23", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.23.tgz", + "integrity": "sha512-QDTCU0M0MxR3hQfnlDJfwekQiaanm1ubOD231u73WBckQ/fsamwRLiE2GBz6D3a/xF1NgfiDLJjXBa1hYOYTtQ==", "license": "MIT" }, "node_modules/debug": { @@ -8272,9 +8505,9 @@ } }, "node_modules/dompurify": { - "version": "3.4.13", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.13.tgz", - "integrity": "sha512-2vmYIoqjze2d+kakP8S/nS5shfsl587kzwEjcGlTdiksUVgFHnFCsLYDVj/JNqJVOQZGSYBTmuycv0PodwmnMQ==", + "version": "3.4.14", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.14.tgz", + "integrity": "sha512-dVoH9z+MY+C9IilgGCk3YfFqjLi3fChm2OiKJMzh6axrJ5qwxqWaZamgmHrpv22CN/KdbZJuGEGgfQoL00LTdg==", "license": "(MPL-2.0 OR Apache-2.0)", "optionalDependencies": { "@types/trusted-types": "^2.0.7" @@ -8369,9 +8602,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.5.403", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.403.tgz", - "integrity": "sha512-MQsYmdaLzvaCX5j+ZZBr5Fm6uCCnPQcRtlvmvRlWqrXy+BH2O4ffXIAScF+JQznQWB9brWp4lSD9Z4yNmaf2BA==", + "version": "1.5.415", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.415.tgz", + "integrity": "sha512-958V+Kbhtgz+SxXeEVKBjrlKRBIDAYvUJfwhjxMZ5S6ut9jAl7l9ZKBkBrvjyjZE36PabLUo2L8kEeV5O4vgJg==", "dev": true, "license": "ISC" }, @@ -8601,9 +8834,9 @@ } }, "node_modules/es-module-lexer": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.1.tgz", - "integrity": "sha512-shc1dbU90Yl/xq1QrC7QRtfcwURZuVRfPhZbDoldJ1cn1gzDvBaBWlv0eFolj5+0znnPJz5TXLxsN77X/12KTA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.2.tgz", + "integrity": "sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw==", "dev": true, "license": "MIT" }, @@ -9265,9 +9498,9 @@ } }, "node_modules/eslint/node_modules/js-yaml": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.1.tgz", - "integrity": "sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", "dev": true, "funding": [ { @@ -9622,9 +9855,9 @@ "license": "MIT" }, "node_modules/fast-uri": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.5.tgz", - "integrity": "sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw==", + "version": "3.1.6", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.6.tgz", + "integrity": "sha512-7Ical1vFEMr0onbVzEDIreM22I4khW+fzyQPwvAFWBp1iwdshSZRsL4jjRvPG9JP1uiqMHRto+YU6R2/CzDz5Q==", "dev": true, "funding": [ { @@ -12611,9 +12844,9 @@ } }, "node_modules/jest-watch-typeahead/node_modules/strip-ansi/node_modules/ansi-regex": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", - "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.3.0.tgz", + "integrity": "sha512-WpDfL7NO6j7tH88IDBNVdUJxDh9nmCteAVW9dsep846XdwF4naCBK+/tGLX3KJgcpgMRXCFlTM2hKGoK9FsdrQ==", "dev": true, "license": "MIT", "engines": { @@ -12691,9 +12924,9 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "3.15.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.15.1.tgz", - "integrity": "sha512-S99WuO3HlhO3XN41EtYUNl9zzXjoJx7QvmipxsJVxtCBT0YHEFy+iOJhjSvrmV12nYhWpZaM8lPHkJm0yUMbag==", + "version": "3.15.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.15.2.tgz", + "integrity": "sha512-6EuL879VkRA+1Cz578mKMiKvjPNEuk6+r1JaFzoSWejZmtf7xWbIyw1e3KkxlkzTIt9Taw6JBhEppG7utc1P+w==", "dev": true, "license": "MIT", "dependencies": { @@ -13335,16 +13568,16 @@ } }, "node_modules/minimizer-webpack-plugin": { - "version": "5.6.1", - "resolved": "https://registry.npmjs.org/minimizer-webpack-plugin/-/minimizer-webpack-plugin-5.6.1.tgz", - "integrity": "sha512-DoeAZz8Q1C1znwsUzej1fdoi4jCf7/+Em27ouLqfK/+3m8G+D7yDhUwrc3CNhjSzGUN1kn7Iv4sWmjflQHenpw==", + "version": "5.7.0", + "resolved": "https://registry.npmjs.org/minimizer-webpack-plugin/-/minimizer-webpack-plugin-5.7.0.tgz", + "integrity": "sha512-0ReIvHAVVojdDOn+kmRzrT62A6Btc9KeAK6ANTpd8+S5mnm5RDdu+EN6924xLFFzb36Rkoj+xOXkDA0jTHId/g==", "dev": true, "license": "MIT", "dependencies": { - "@jridgewell/trace-mapping": "^0.3.25", + "@jridgewell/trace-mapping": "^0.3.31", "jest-worker": "^27.4.5", - "schema-utils": "^4.3.0", - "terser": "^5.31.1" + "schema-utils": "^4.3.3", + "terser": "^5.51.0" }, "engines": { "node": ">= 10.13.0" @@ -14093,16 +14326,6 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, - "node_modules/pify": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", - "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/pirates": { "version": "4.0.7", "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.7.tgz", @@ -15500,9 +15723,9 @@ } }, "node_modules/postcss-svgo/node_modules/svgo": { - "version": "2.8.3", - "resolved": "https://registry.npmjs.org/svgo/-/svgo-2.8.3.tgz", - "integrity": "sha512-5EZD0pafXX6PphdwOGCiVLDSaV1xyuQao2blHajHLsPxr07q4mmEjdtXEWgG07ae2mIz8Ex2CDXNCTiXhy3Khw==", + "version": "2.8.4", + "resolved": "https://registry.npmjs.org/svgo/-/svgo-2.8.4.tgz", + "integrity": "sha512-2GJ4h3rl13qYTdwllaK6QlL8tG+UrM8626V2Ylcd/yBUv2Y/EwLsYisVV6UDCRmlk+C74G8nMpxJCk0RWPdDCw==", "dev": true, "license": "MIT", "dependencies": { @@ -16223,14 +16446,11 @@ } }, "node_modules/read-cache": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz", - "integrity": "sha512-Owdv/Ft7IjOgm/i0xvNDZ1LrRANRfew4b2prF3OWMQLxLfu3bS8FVhCsrSCMK4lR56Y9ya+AThoTpDCTxCmpRA==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.2.tgz", + "integrity": "sha512-/peqiBB/n07gQGLsWaHho3WfvUyRscw0gYTsEFMhrIe/nWLkYaf5SbKYjGYqtRV3aPwykJgF2VEMo1ac4bnsGA==", "dev": true, - "license": "MIT", - "dependencies": { - "pify": "^2.3.0" - } + "license": "MIT" }, "node_modules/readable-stream": { "version": "3.6.2", @@ -16455,9 +16675,9 @@ "license": "MIT" }, "node_modules/reselect": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/reselect/-/reselect-5.2.0.tgz", - "integrity": "sha512-AgZ3UOZm3YndfrJ4OYjgrT7bmCm/1iqkjvEfH/oYjzh6PD2qw4QuT3jjnXIrpdt4MTpMXclMT3lXbmRY+XRakw==", + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/reselect/-/reselect-5.3.0.tgz", + "integrity": "sha512-XGoLeRAVzUTcJ1qkxPQhDJyIZ5d6zzZD9nT7AEZOaaU9UbWclhycElmhO+VD5bFeLuzhPBaOV2oXC8uG35ZSpg==", "license": "MIT" }, "node_modules/resolve": { @@ -17887,11 +18107,14 @@ } }, "node_modules/svg-parser": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/svg-parser/-/svg-parser-2.0.4.tgz", - "integrity": "sha512-e4hG1hRwoOdRb37cIMSgzNsxyzKfayW6VOflrwvR+/bzrkyxY/31WkbgnQpgtrNp1SdpJvpUAGTa/ZoiPNDuRQ==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/svg-parser/-/svg-parser-2.1.0.tgz", + "integrity": "sha512-bwLf38YmY+TDYHJw1Ex0Co8c4yeXuJAo8YnXGZrscxrvYoVVIvLeniEkV1Ks/54VteMnL4FtyN1+P+TZJdUPmQ==", "dev": true, - "license": "MIT" + "license": "MIT", + "engines": { + "node": ">=8" + } }, "node_modules/svgo": { "version": "1.3.2", @@ -18258,9 +18481,9 @@ } }, "node_modules/terser": { - "version": "5.49.2", - "resolved": "https://registry.npmjs.org/terser/-/terser-5.49.2.tgz", - "integrity": "sha512-rGbJiKeQ4WDe3EXlDAIaQcwftVfv2Q8o1awFNfvXolJYKkb1AuZY1RTOmqx4LJXZENbWZA7eIsYGHuEzHsi1nQ==", + "version": "5.51.0", + "resolved": "https://registry.npmjs.org/terser/-/terser-5.51.0.tgz", + "integrity": "sha512-myiQ6aFnxDOjdiXdTlC8ngVccQD88uHXTx5RUQOSEnarDYgJMjDagwAQszcOusjvmf4YqWsxoD1+MY+oAJRmpw==", "dev": true, "license": "BSD-2-Clause", "dependencies": { @@ -18439,9 +18662,9 @@ } }, "node_modules/tinyglobby/node_modules/picomatch": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", - "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", "dev": true, "license": "MIT", "peer": true, diff --git a/gui/src/client/package.json b/gui/src/client/package.json index 11ce0ad..fb2ceb6 100644 --- a/gui/src/client/package.json +++ b/gui/src/client/package.json @@ -11,6 +11,7 @@ "@mui/icons-material": "^7.3.5", "@mui/material": "^7.3.5", "@mui/x-charts": "^8.17.0", + "@mui/x-data-grid": "^9.12.0", "@react-spring/web": "^10.0.3", "@tanstack/react-query": "^5.76.1", "@types/react": "18.3.10", diff --git a/gui/src/client/src/components/MenuContent.tsx b/gui/src/client/src/components/MenuContent.tsx index 725483e..76b3c9d 100644 --- a/gui/src/client/src/components/MenuContent.tsx +++ b/gui/src/client/src/components/MenuContent.tsx @@ -6,7 +6,6 @@ import ListItemIcon from "@mui/material/ListItemIcon"; import ListItemText from "@mui/material/ListItemText"; import Stack from "@mui/material/Stack"; import ExploreIcon from "@mui/icons-material/Explore"; -import BarChartIcon from "@mui/icons-material/BarChart"; import HomeRoundedIcon from "@mui/icons-material/HomeRounded"; import UploadFileIcon from "@mui/icons-material/UploadFile"; import RuleIcon from "@mui/icons-material/Rule"; @@ -29,11 +28,6 @@ const mainListItems = [ icon: , to: `/dashboard/discovery` }, - { - text: "Enrichment", - icon: , - to: `/dashboard/enrichment` - }, { text: "Generate", icon: , diff --git a/gui/src/client/src/components/workspace/AlignmentGrid.tsx b/gui/src/client/src/components/workspace/AlignmentGrid.tsx index 2e46c8c..72fe3f8 100644 --- a/gui/src/client/src/components/workspace/AlignmentGrid.tsx +++ b/gui/src/client/src/components/workspace/AlignmentGrid.tsx @@ -23,6 +23,103 @@ import { MotifName } from "../MotifName"; import { horizontalScrollSx } from "../../theme/scrollbarSx"; import { buildAlignmentSvg, downloadSvg, type AlignmentSvgRow } from "./alignmentSvgExport"; import { MinimalIconButton} from "../MinimalIconButton"; +import { getEntryAnnotations } from "../../features/database/api"; +import type { EntryAnnotation } from "../../features/database/types"; +import { isChebiRoleRank, toSentenceCase } from "../../features/database/format"; + +const ANNOTATION_CATEGORY_LABELS: Record = { + phylogeny: "Phylogeny", + biosynthetic_class: "Biosynthetic class", + chemical_class: "Chemical class", + bioactivity: "Bioactivity", +}; + +function groupAnnotationsByCategory(annotations: EntryAnnotation[]): [string, EntryAnnotation[]][] { + const groups = new Map(); + for (const a of annotations) { + const bucket = groups.get(a.category) ?? []; + bucket.push(a); + groups.set(a.category, bucket); + } + return Array.from(groups.entries()); +} + +// Shared by the "Databases" (source) row and every AnnotationChips category row -- +// one component, not parallel copies of the same Stack/Typography markup, so the +// label-to-chip gap is guaranteed identical by construction rather than by keeping two +// separate JSX blocks in sync by hand. +const AnnotationRow: React.FC<{ label: string; sx?: object; children: React.ReactNode }> = ({ + label, + sx, + children, +}) => ( + + + {label} + + {children} + +); + +const AnnotationChips: React.FC<{ entryId: string }> = ({ entryId }) => { + const [annotations, setAnnotations] = React.useState(null); + const [error, setError] = React.useState(null); + + React.useEffect(() => { + const controller = new AbortController(); + getEntryAnnotations(entryId, controller.signal) + .then((resp) => setAnnotations(resp.results)) + .catch((err) => { + if (controller.signal.aborted) return; + setError(err instanceof Error ? err.message : String(err)); + }); + return () => controller.abort(); + }, [entryId]); + + if (error) { + return ( + + Couldn't load annotations: {error} + + ); + } + + if (annotations === null) { + return null; + } + + if (annotations.length === 0) { + return null; + } + + return ( + + {groupAnnotationsByCategory(annotations).map(([category, items]) => ( + + {items.map((item) => { + const label = isChebiRoleRank(item.rank) ? toSentenceCase(item.label) : item.label; + return item.url ? ( + + ) : ( + + ); + })} + + ))} + + ); +}; // Shared vertical sizing (padding, border width, font size, line height) so every // row -- label, sequence cells, and score -- resolves to the exact same height. @@ -302,7 +399,7 @@ export function ResultRow({ {result.sources.length > 0 && ( - + {result.sources.map((source, idx) => source.url ? ( ) )} - + )} + = <> - value && setResultsView(value)} + onChange={(e) => setResultsView(e.target.value as ViewMode)} + sx={{ minWidth: 220 }} > - Pairwise - - - - - Multiple sequence alignment - - - - - - - - Compare - - - - + {VIEW_MODE_OPTIONS.map((option) => { + const enabled = option.flag === null || item.flags[option.flag]; + const menuItem = ( + + {option.label} + + ); + if (enabled) return menuItem; + // A disabled MenuItem still needs a wrapping span for the tooltip to + // fire (disabled elements don't receive pointer events on their own). + return ( + + {menuItem} + + ); + })} + {selectedForMsa.size} of {payload.results.length} selected @@ -280,7 +301,7 @@ export const DialogViewDiscoveryQuery: React.FC = - {resultsView !== "compare" && ( + {(resultsView === "pairwise" || resultsView === "msa") && ( Cell shading the alignment reflects each motif's structural similarity to the query's aligned motif in that column. Darker means a closer match, so a sequence's weak spots stand out at a glance. @@ -322,6 +343,10 @@ export const DialogViewDiscoveryQuery: React.FC = ) + ) : resultsView === "enrichment" ? ( + + + ) : selectedResults.length === 0 ? ( Select at least one result above to compare compounds. diff --git a/gui/src/client/src/components/workspace/DiscoveryEnrichmentView.tsx b/gui/src/client/src/components/workspace/DiscoveryEnrichmentView.tsx new file mode 100644 index 0000000..445cc94 --- /dev/null +++ b/gui/src/client/src/components/workspace/DiscoveryEnrichmentView.tsx @@ -0,0 +1,280 @@ +// Enrichment analysis over a Discovery query's own nearest neighbors -- reuses the +// exact same /api/enrichmentAnalysis endpoint (Fisher's exact + Benjamini-Hochberg, +// see gui/src/server/routes/enrichment.py) the old standalone Enrichment tab used, +// just fed from the results dialog's own checkbox selection instead of a separate +// manual entry-search step. +import React from "react"; +import Alert from "@mui/material/Alert"; +import Box from "@mui/material/Box"; +import Button from "@mui/material/Button"; +import Chip from "@mui/material/Chip"; +import CircularProgress from "@mui/material/CircularProgress"; +import Stack from "@mui/material/Stack"; +import MuiTooltip from "@mui/material/Tooltip"; +import Typography from "@mui/material/Typography"; +import CheckCircleIcon from "@mui/icons-material/CheckCircle"; +import CancelIcon from "@mui/icons-material/Cancel"; +import DownloadIcon from "@mui/icons-material/Download"; +import RefreshIcon from "@mui/icons-material/Refresh"; +import { DataGrid, type GridColDef } from "@mui/x-data-grid"; +import { useQuery } from "@tanstack/react-query"; +import { runEnrichmentAnalysis } from "../../features/enrichment/api"; +import { Q_VALUE_SIGNIFICANT, type EnrichmentResult } from "../../features/enrichment/types"; + +function formatScientific(value: number): string { + return value === 0 ? "0" : value.toExponential(2); +} + +// annotation_terms.category/rank/label come straight from the pipeline in raw form +// (e.g. "chemical_class", "chebi_biological_role") -- humanized here for display only, +// same "touch only presentation, never the underlying value" approach as +// features/database/format.ts's toSentenceCase. +function humanizeAnnotationText(value: string): string { + const spaced = value.replace(/_/g, " "); + const sentenceCased = spaced.length > 0 ? spaced.charAt(0).toUpperCase() + spaced.slice(1) : spaced; + return sentenceCased.replace(/\bchebi\b/gi, "ChEBI"); +} + +function downloadTsv(rows: EnrichmentResult[], filename: string): void { + const headers = [ + "category", "rank", "label", + "selectedWithTerm", "selectedTotal", "backgroundWithTerm", "backgroundTotal", + "foldEnrichment", "direction", "pValue", "qValue", "significant", + ]; + const lines = [headers.join("\t")]; + for (const r of rows) { + lines.push( + [ + r.category ?? "", + r.rank ?? "", + r.label, + r.selectedWithTerm, + r.selectedTotal, + r.backgroundWithTerm, + r.backgroundTotal, + r.foldEnrichment ?? "", + r.direction, + r.pValue, + r.qValue, + r.qValue < Q_VALUE_SIGNIFICANT ? "yes" : "no", + ].join("\t") + ); + } + + const blob = new Blob([lines.join("\n")], { type: "text/tab-separated-values;charset=utf-8" }); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = filename; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + URL.revokeObjectURL(url); +} + +const columns: GridColDef[] = [ + { + field: "category", + headerName: "Category", + width: 140, + valueFormatter: (value: string | null) => (value ? humanizeAnnotationText(value) : "—"), + }, + { + field: "rank", + headerName: "Rank", + width: 140, + valueFormatter: (value: string | null) => (value ? humanizeAnnotationText(value) : "—"), + }, + { + field: "label", + headerName: "Label", + width: 220, + valueFormatter: (value: string) => humanizeAnnotationText(value), + }, + { + field: "selectedWithTerm", + headerName: "Selected", + width: 110, + type: "number", + renderCell: (params) => `${params.row.selectedWithTerm} / ${params.row.selectedTotal}`, + }, + { + field: "backgroundWithTerm", + headerName: "Background", + width: 120, + type: "number", + renderCell: (params) => `${params.row.backgroundWithTerm} / ${params.row.backgroundTotal}`, + }, + { + field: "foldEnrichment", + headerName: "Fold", + width: 100, + type: "number", + valueFormatter: (value: number | null) => (value === null ? "—" : value.toFixed(2)), + }, + { field: "direction", headerName: "Direction", width: 110 }, + { + field: "pValue", + headerName: "p-value", + width: 110, + type: "number", + valueFormatter: (value: number) => formatScientific(value), + }, + { + field: "qValue", + headerName: "q-value", + width: 110, + type: "number", + valueFormatter: (value: number) => formatScientific(value), + }, + { + field: "significant", + headerName: "Significant", + width: 120, + type: "boolean", + valueGetter: (_value, row) => row.qValue < Q_VALUE_SIGNIFICANT, + renderCell: (params) => + params.value ? ( + + ) : ( + + ), + }, +]; + +// DataGrid's own "baseTooltip" slot type has no `arrow` prop (it's a stripped-down +// interface, not @mui/material's full TooltipProps -- see gridBaseSlots.d.ts), so +// arrow-style tooltips (sort/filter/menu hover hints) need a full slot override +// rather than a slotProps tweak. +function GridArrowTooltip(props: React.ComponentProps) { + return ; +} + +export const DiscoveryEnrichmentView: React.FC<{ entryIds: string[] }> = ({ entryIds }) => { + // Recomputes only on mount (first open) and on explicit "Recalculate" clicks -- + // never automatically as the user toggles checkboxes above, which would otherwise + // fire a request on every single click while they're still adjusting the selection. + const [queriedIds, setQueriedIds] = React.useState(entryIds); + const didInit = React.useRef(false); + React.useEffect(() => { + if (didInit.current) return; + didInit.current = true; + setQueriedIds(entryIds); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + const selectionChanged = React.useMemo(() => { + const a = [...entryIds].sort(); + const b = [...queriedIds].sort(); + return a.length !== b.length || a.some((id, i) => id !== b[i]); + }, [entryIds, queriedIds]); + + const enrichmentQuery = useQuery({ + queryKey: ["discoveryEnrichmentAnalysis", queriedIds], + queryFn: ({ signal }) => runEnrichmentAnalysis(queriedIds, signal), + enabled: queriedIds.length > 0, + }); + + const rows = enrichmentQuery.data?.results ?? []; + + return ( + + + + + Enrichment of the {queriedIds.length} selected nearest neighbor{queriedIds.length === 1 ? "" : "s"} vs. the + rest of the database, one two-sided Fisher's exact test per annotation term, Benjamini-Hochberg corrected. + + {selectionChanged && ( + + )} + + + + + + + + {queriedIds.length === 0 && ( + + Select at least one result above, then click Recalculate to run enrichment analysis. + + )} + + {enrichmentQuery.error && ( + + {(enrichmentQuery.error as Error).message || "Failed to run enrichment analysis."} + + )} + + {queriedIds.length > 0 && ( + + row.termId} + loading={enrichmentQuery.isLoading} + density="compact" + disableRowSelectionOnClick + showCellVerticalBorder + showColumnVerticalBorder + initialState={{ + sorting: { sortModel: [{ field: "qValue", sort: "asc" }] }, + }} + pageSizeOptions={[25, 50, 100]} + slots={{ baseTooltip: GridArrowTooltip }} + sx={{ + // Explicit border rules rather than relying on the --DataGrid-rowBorderColor + // CSS variable / showCellVerticalBorder alone -- confirmed live that the + // variable covers vertical cell borders but not the horizontal row + // separator in this DataGrid version, so it's set directly here instead. + // "divider" resolves through the app's own theme (light/dark both), not a + // hardcoded color. + "& .MuiDataGrid-cell": { + fontSize: "0.8125rem", + borderRight: "1px solid", + borderRightColor: "divider", + borderBottom: "1px solid", + borderBottomColor: "divider", + }, + "& .MuiDataGrid-columnHeader": { + borderRight: "1px solid", + borderRightColor: "divider", + }, + // Sort arrow / column-menu (three dots) icons -- bare icon, no button + // chrome (padding, hover box, border) around them. + "& .MuiDataGrid-iconButtonContainer .MuiIconButton-root, & .MuiDataGrid-menuIconButton": { + padding: 0, + border: "none", + backgroundColor: "transparent", + boxShadow: "none", + "&:hover": { backgroundColor: "transparent" }, + }, + }} + /> + + )} + + ); +}; diff --git a/gui/src/client/src/components/workspace/Workspace.tsx b/gui/src/client/src/components/workspace/Workspace.tsx index feecb4b..a8c9709 100644 --- a/gui/src/client/src/components/workspace/Workspace.tsx +++ b/gui/src/client/src/components/workspace/Workspace.tsx @@ -16,7 +16,6 @@ import { WorkspaceUpload } from "./WorkspaceUpload"; import { WorkspaceDiscovery } from "./WorkspaceDiscovery"; import { WorkspaceRules } from "./WorkspaceRules"; import { WorkspaceGenerate } from "./WorkspaceGenerate"; -// import { WorkspaceEnrichment } from "./tabs/enrichment/WorkspaceEnrichment"; export const Workspace: React.FC = () => { const { showOverlay, hideOverlay } = useOverlay(); @@ -172,8 +171,6 @@ export const Workspace: React.FC = () => { } /> } /> } /> - {/*} />*/} - Analysis currently available. Check back later.} /> } /> } /> diff --git a/gui/src/client/src/components/workspace/WorkspaceDiscovery.tsx b/gui/src/client/src/components/workspace/WorkspaceDiscovery.tsx index baa6349..1288e21 100644 --- a/gui/src/client/src/components/workspace/WorkspaceDiscovery.tsx +++ b/gui/src/client/src/components/workspace/WorkspaceDiscovery.tsx @@ -165,6 +165,7 @@ function DiscoveryQueryListItem({ const flagChips = [ item.flags.computeMsa ? "MSA" : null, item.flags.computeCompare ? "Compare" : null, + item.flags.computeEnrichment ? "Enrichment" : null, ].filter((label): label is string => label !== null); const handleToggle = (e?: React.SyntheticEvent) => { @@ -382,6 +383,7 @@ export const WorkspaceDiscovery: React.FC = ({ session, const [onlyUserUploads, setOnlyUserUploads] = React.useState(false); const [computeMsa, setComputeMsa] = React.useState(true); const [computeCompare, setComputeCompare] = React.useState(false); + const [computeEnrichment, setComputeEnrichment] = React.useState(true); const [submitting, setSubmitting] = React.useState(false); const [queryOptionsOpen, setQueryOptionsOpen] = React.useState(false); @@ -447,7 +449,7 @@ export const WorkspaceDiscovery: React.FC = ({ session, includeUserUploads: includeUserUploads || onlyUserUploads, onlyUserUploads, queryOriginSmiles, - flags: { computeMsa, computeCompare }, + flags: { computeMsa, computeCompare, computeEnrichment }, }); setSession((prev) => (prev ? { ...prev, items: [...prev.items, item] } : prev)); setViewingItemId(item.id); @@ -906,6 +908,22 @@ export const WorkspaceDiscovery: React.FC = ({ session, } label="Compute compound comparison" /> + + setComputeEnrichment(e.target.checked)} + disabled={submitting} + /> + } + label="Compute enrichment" + /> + diff --git a/gui/src/client/src/components/workspace/WorkspaceHome.tsx b/gui/src/client/src/components/workspace/WorkspaceHome.tsx index 96e953f..9d4c03b 100644 --- a/gui/src/client/src/components/workspace/WorkspaceHome.tsx +++ b/gui/src/client/src/components/workspace/WorkspaceHome.tsx @@ -9,8 +9,26 @@ import Skeleton from "@mui/material/Skeleton"; import { useTheme } from "@mui/material/styles"; import { Link as RouterLink } from "react-router-dom"; import { PieChart } from "@mui/x-charts/PieChart"; -import { getDatabaseStats } from "../../features/database/api"; -import { DatabaseStatsResp } from "../../features/database/types"; +import { getDatabaseStats, getAnnotationStats } from "../../features/database/api"; +import { DatabaseStatsResp, AnnotationStatsResp, Count } from "../../features/database/types"; +import { toSentenceCase } from "../../features/database/format"; + +// annotationStats.coverage carries one entry per (category, entry type) that's +// actually populated for it (see RetroMolDuckDB.annotation_stats) -- looked up by its +// display label rather than a fixed array index, so a section silently gets no tile if +// the backend ever stops sending that particular label instead of rendering garbage. +const coverageTile = (stats: AnnotationStatsResp, label: string): React.ReactNode => { + const entry = stats.coverage.find((c) => c.label === label); + if (!entry) return null; + return ( + + ); +}; const ENTRY_TYPE_LABELS: Record = { compound: "Compounds", @@ -55,6 +73,69 @@ const ChartCard: React.FC<{ title: string; description?: string; children: React ); +// Pie charts, not bar charts -- the installed @mui/x-charts (8.29.2) BarChart has a +// real rendering bug where rotating x-axis tick labels (any angle, tried -90 and -45) +// renders every tick as an empty : text node present, content blank, confirmed +// live in the DOM. A CSS-transform workaround got labels rotated but the axis itself +// (many ticks, long category names) is still cramped and prone to overlap/misread at +// this data size. PieChart has no axis-label problem at all -- its legend is plain +// text -- and reads better for "which N labels dominate" than a bar chart would here +// anyway. Long tails (more than TOP_N distinct labels) collapse into "Other" so the +// legend stays readable instead of listing 15 items. A slice also collapses into +// "Other" if it's under MIN_SLICE_SHARE of the total, even when it's within the top +// N by rank -- a skewed distribution (one dominant label, four small ones) can still +// produce a top-N slice too thin to see or click, not just labels past rank N. +const TOP_N = 5; +const MIN_SLICE_SHARE = 0.02; + +const AnnotationPieCard: React.FC<{ + title: string; + description?: string; + counts: Count[]; + colors: string[]; + emptyMessage: string; +}> = ({ title, description, counts, colors, emptyMessage }) => { + const total = counts.reduce((sum, c) => sum + c.count, 0); + const kept: Count[] = []; + let otherTotal = 0; + counts.forEach((c, i) => { + if (i < TOP_N && c.count / total >= MIN_SLICE_SHARE) { + kept.push(c); + } else { + otherTotal += c.count; + } + }); + const pieData = [ + ...kept.map((c, i) => ({ id: i, value: c.count, label: c.label })), + ...(otherTotal > 0 ? [{ id: "other", value: otherTotal, label: "Other" }] : []), + ]; + + return ( + + {counts.length === 0 ? ( + + {emptyMessage} + + ) : ( + + )} + + ); +}; + export const WorkspaceHome: React.FC = () => { const theme = useTheme(); const palette = theme.vars || theme; @@ -63,6 +144,10 @@ export const WorkspaceHome: React.FC = () => { const [loading, setLoading] = React.useState(true); const [error, setError] = React.useState(null); + const [annotationStats, setAnnotationStats] = React.useState(null); + const [annotationLoading, setAnnotationLoading] = React.useState(true); + const [annotationError, setAnnotationError] = React.useState(null); + React.useEffect(() => { const controller = new AbortController(); setLoading(true); @@ -82,6 +167,25 @@ export const WorkspaceHome: React.FC = () => { return () => controller.abort(); }, []); + React.useEffect(() => { + const controller = new AbortController(); + setAnnotationLoading(true); + setAnnotationError(null); + + getAnnotationStats(controller.signal) + .then(setAnnotationStats) + .catch((err) => { + if (controller.signal.aborted) return; + setAnnotationError(err instanceof Error ? err.message : String(err)); + }) + .finally(() => { + if (controller.signal.aborted) return; + setAnnotationLoading(false); + }); + + return () => controller.abort(); + }, []); + const chartColors = [ palette.palette.primary.main, palette.palette.warning.main, @@ -199,6 +303,133 @@ export const WorkspaceHome: React.FC = () => { /> + + + Annotations + + + {annotationError && ( + + Couldn't load annotation statistics: {annotationError} + + )} + + {!annotationError && annotationLoading && ( + + {Array.from({ length: 2 }).map((_, i) => ( + + ))} + + )} + + {!annotationError && !annotationLoading && annotationStats && ( + <> + + Phylogeny + + + {coverageTile(annotationStats, "Phylogeny (compounds)")} + {coverageTile(annotationStats, "Phylogeny (gene clusters)")} + + + + + + + + + Chemical class + + + NPClassifier, predicted from every compound's own structure + + + {coverageTile(annotationStats, "Chemical class (compounds)")} + + + + + + + + + Biosynthetic class + + + MIBiG's own coarse label (PKS / NRPS / RiPP / ...), a separate classification from NPClassifier's chemical class above + + + {coverageTile(annotationStats, "Biosynthetic class (gene clusters)")} + + + + + + + Bioactivity + + + ChEBI's role ontology, looked up by structure (InChIKey) + + + {coverageTile(annotationStats, "Bioactivity (compounds)")} + + + ({ ...c, label: toSentenceCase(c.label) }))} + colors={chartColors} + emptyMessage="No ChEBI biological-role matches yet." + /> + ({ ...c, label: toSentenceCase(c.label) }))} + colors={chartColors} + emptyMessage="No ChEBI chemical-role matches yet." + /> + + + )} )} diff --git a/gui/src/client/src/components/workspace/WorkspaceUpload.tsx b/gui/src/client/src/components/workspace/WorkspaceUpload.tsx index 48d6d03..57833cf 100644 --- a/gui/src/client/src/components/workspace/WorkspaceUpload.tsx +++ b/gui/src/client/src/components/workspace/WorkspaceUpload.tsx @@ -286,7 +286,7 @@ export const WorkspaceUpload: React.FC = ({ session, setSe Getting started - In this tab you can import compounds and biosynthetic gene clusters (BGCs) into your workspace. Use the buttons below to upload your data files. After importing, you can visualize and analyze your data within the  + In this tab you can import compounds and biosynthetic gene clusters (BGCs) into your workspace. Use the buttons below to upload your data files. After importing, you can visualize and analyze your data (including enrichment analysis against your nearest neighbors) within the  = ({ session, setSe > Discovery -  and  - - Enrichment - -  tabs. A maximum of {MAX_ITEMS} items can be imported into the workspace. Keep an eye on for updates on your queries. +  tab. A maximum of {MAX_ITEMS} items can be imported into the workspace. Keep an eye on for updates on your queries. diff --git a/gui/src/client/src/features/database/api.ts b/gui/src/client/src/features/database/api.ts index 09f41bd..569795f 100644 --- a/gui/src/client/src/features/database/api.ts +++ b/gui/src/client/src/features/database/api.ts @@ -1,6 +1,22 @@ import { getJson } from "../http"; -import { DatabaseStatsRespSchema, type DatabaseStatsResp } from "./types"; +import { + AnnotationStatsRespSchema, + DatabaseStatsRespSchema, + EntryAnnotationsRespSchema, + type AnnotationStatsResp, + type DatabaseStatsResp, + type EntryAnnotationsResp, +} from "./types"; export async function getDatabaseStats(signal?: AbortSignal): Promise { return getJson("/api/databaseStats", DatabaseStatsRespSchema, signal); } + +export async function getAnnotationStats(signal?: AbortSignal): Promise { + return getJson("/api/annotationStats", AnnotationStatsRespSchema, signal); +} + +export async function getEntryAnnotations(entryId: string, signal?: AbortSignal): Promise { + const params = new URLSearchParams({ entryId }); + return getJson(`/api/entryAnnotations?${params.toString()}`, EntryAnnotationsRespSchema, signal); +} diff --git a/gui/src/client/src/features/database/format.ts b/gui/src/client/src/features/database/format.ts new file mode 100644 index 0000000..cb1f4ff --- /dev/null +++ b/gui/src/client/src/features/database/format.ts @@ -0,0 +1,17 @@ +// ChEBI role names (both "chebi_biological_role" and "chebi_chemical_role") come from +// the source data lowercase (e.g. "antibacterial agent", "xenobiotic") -- sentence-cased +// here for display only. Never touches the stored/queried label itself (exact-match +// lookups, external links, and backend grouping all still use the raw lowercase form). +const CHEBI_ROLE_RANKS = new Set(["chebi_biological_role", "chebi_chemical_role"]); + +export function isChebiRoleRank(rank: string | null | undefined): boolean { + return !!rank && CHEBI_ROLE_RANKS.has(rank); +} + +export function toSentenceCase(label: string): string { + // Capitalize only the label's own first character -- "antibacterial agent" -> + // "Antibacterial agent", not "Antibacterial Agent". Not `\b\w`-regex-based: JS + // regex `\w` is ASCII-only, so it doesn't recognize letters like "ø" as word + // characters, which would break `\b` (word-boundary) detection right after one. + return label.length > 0 ? label.charAt(0).toUpperCase() + label.slice(1) : label; +} diff --git a/gui/src/client/src/features/database/types.ts b/gui/src/client/src/features/database/types.ts index c46d9c2..cac3cd4 100644 --- a/gui/src/client/src/features/database/types.ts +++ b/gui/src/client/src/features/database/types.ts @@ -17,3 +17,40 @@ export const DatabaseStatsRespSchema = z.object({ withoutSourceUrlCount: z.number().int().nonnegative(), }); export type DatabaseStatsResp = z.output; + +export const AnnotationCoverageSchema = z.object({ + label: z.string(), + withAnnotationCount: z.number().int().nonnegative(), + withoutAnnotationCount: z.number().int().nonnegative(), +}); +export type AnnotationCoverage = z.output; + +export const AnnotationStatsRespSchema = z.object({ + coverage: z.array(AnnotationCoverageSchema), + countsByCategory: z.array(CountSchema), + phylogenyTypeCounts: z.array(CountSchema), + phylogenyGenusCounts: z.array(CountSchema), + phylogenySpeciesCounts: z.array(CountSchema), + biosyntheticClassCounts: z.array(CountSchema), + chemicalClassPathwayCounts: z.array(CountSchema), + chemicalClassSuperclassCounts: z.array(CountSchema), + chemicalClassClassCounts: z.array(CountSchema), + bioactivityBiologicalRoleCounts: z.array(CountSchema), + bioactivityChemicalRoleCounts: z.array(CountSchema), +}); +export type AnnotationStatsResp = z.output; + +export const EntryAnnotationSchema = z.object({ + id: z.string(), + category: z.string(), + rank: z.string().nullable(), + label: z.string(), + externalId: z.string().nullable(), + url: z.string().nullable(), +}); +export type EntryAnnotation = z.output; + +export const EntryAnnotationsRespSchema = z.object({ + results: z.array(EntryAnnotationSchema), +}); +export type EntryAnnotationsResp = z.output; diff --git a/gui/src/client/src/features/discovery/types.ts b/gui/src/client/src/features/discovery/types.ts index 1cc3baf..d9ad244 100644 --- a/gui/src/client/src/features/discovery/types.ts +++ b/gui/src/client/src/features/discovery/types.ts @@ -46,6 +46,7 @@ export const SubmitDiscoveryQueryReqSchema = z.object({ flags: z.object({ computeMsa: z.boolean().optional(), computeCompare: z.boolean().optional(), + computeEnrichment: z.boolean().optional(), }), }); export type SubmitDiscoveryQueryReq = z.output; diff --git a/gui/src/client/src/features/enrichment/api.ts b/gui/src/client/src/features/enrichment/api.ts new file mode 100644 index 0000000..d9d24fc --- /dev/null +++ b/gui/src/client/src/features/enrichment/api.ts @@ -0,0 +1,11 @@ +import { postJson } from "../http"; +import { EnrichmentAnalysisRespSchema, type EnrichmentAnalysisResp } from "./types"; + +export const MAX_ENRICHMENT_SELECTION = 100; + +export async function runEnrichmentAnalysis( + entryIds: string[], + signal?: AbortSignal +): Promise { + return postJson("/api/enrichmentAnalysis", { entryIds }, EnrichmentAnalysisRespSchema, signal); +} diff --git a/gui/src/client/src/features/enrichment/types.ts b/gui/src/client/src/features/enrichment/types.ts new file mode 100644 index 0000000..c58db0d --- /dev/null +++ b/gui/src/client/src/features/enrichment/types.ts @@ -0,0 +1,25 @@ +import { z } from "zod"; + +export const EnrichmentResultSchema = z.object({ + termId: z.string(), + category: z.string().nullable(), + rank: z.string().nullable(), + label: z.string(), + selectedWithTerm: z.number().int().nonnegative(), + selectedTotal: z.number().int().nonnegative(), + backgroundWithTerm: z.number().int().nonnegative(), + backgroundTotal: z.number().int().nonnegative(), + foldEnrichment: z.number().nullable(), + direction: z.enum(["enriched", "depleted"]), + pValue: z.number(), + qValue: z.number(), +}); +export type EnrichmentResult = z.output; + +export const EnrichmentAnalysisRespSchema = z.object({ + results: z.array(EnrichmentResultSchema), +}); +export type EnrichmentAnalysisResp = z.output; + +// q-value threshold below which a term is flagged significant in the results table. +export const Q_VALUE_SIGNIFICANT = 0.05; diff --git a/gui/src/client/src/features/session/types.ts b/gui/src/client/src/features/session/types.ts index 01e8144..49752ca 100644 --- a/gui/src/client/src/features/session/types.ts +++ b/gui/src/client/src/features/session/types.ts @@ -40,6 +40,10 @@ export const ClusterItemSchema = BaseItemSchema.extend({ export const DiscoveryQueryFlagsSchema = z.object({ computeMsa: z.boolean().default(false), computeCompare: z.boolean().default(false), + // Not precomputed (see run_discovery_query_job) -- gates whether the results + // dialog's Enrichment view is enabled; the analysis itself is fetched live from + // /api/enrichmentAnalysis based on whichever nearest neighbors are checked. + computeEnrichment: z.boolean().default(true), }); export type DiscoveryQueryFlags = z.output; diff --git a/gui/src/client/src/features/sources.ts b/gui/src/client/src/features/sources.ts new file mode 100644 index 0000000..096473f --- /dev/null +++ b/gui/src/client/src/features/sources.ts @@ -0,0 +1,28 @@ +export type EntrySourceLike = { + name: string; + databaseName: string; + url: string | null; +}; + +export type GroupedSource = { + databaseName: string; + count: number; + items: EntrySourceLike[]; +}; + +/** Group an entry's sources by database, so e.g. two differently-worded MIBiG + * records for the same compound render as one "MIBiG ×2" chip instead of two + * identical-looking "MIBiG" chips. */ +export function groupSourcesByDatabase(sources: EntrySourceLike[]): GroupedSource[] { + const byDatabase = new Map(); + for (const source of sources) { + const items = byDatabase.get(source.databaseName) ?? []; + items.push(source); + byDatabase.set(source.databaseName, items); + } + return Array.from(byDatabase.entries()).map(([databaseName, items]) => ({ + databaseName, + count: items.length, + items, + })); +} diff --git a/gui/src/server/app.py b/gui/src/server/app.py index 10bb52c..bcaa7ed 100644 --- a/gui/src/server/app.py +++ b/gui/src/server/app.py @@ -40,6 +40,8 @@ ) from routes.rate_limit import limiter, RATE_LIMIT_REJECTIONS from routes.rules import blp_rule_set, blp_generate_backbone +from routes.enrichment import blp_enrichment_analysis +from routes.entry_annotations import blp_entry_annotations # Initialize the Flask app @@ -255,6 +257,8 @@ def ready() -> tuple[dict[str, str], int]: app.register_blueprint(blp_get_discovery_query_result) app.register_blueprint(blp_rule_set) app.register_blueprint(blp_generate_backbone) +app.register_blueprint(blp_enrichment_analysis) +app.register_blueprint(blp_entry_annotations) # The two rate-limit tiers on top of the app-wide default (see routes/rate_limit.py) # are applied as @limiter.limit(...) decorators directly on each route function, in diff --git a/gui/src/server/routes/discovery.py b/gui/src/server/routes/discovery.py index 8a43ef9..dab602d 100644 --- a/gui/src/server/routes/discovery.py +++ b/gui/src/server/routes/discovery.py @@ -1225,6 +1225,11 @@ def submit_discovery_query() -> tuple[Response, int]: flags_normalized = { "computeMsa": bool(flags.get("computeMsa", False)), "computeCompare": bool(flags.get("computeCompare", False)), + # Not precomputed here (unlike MSA/Compare) -- enrichment is recalculated + # on-demand from the results dialog via /api/enrichmentAnalysis, since it + # depends on which nearest neighbors the user has checked, not the full + # result set. This flag only gates whether that dropdown option is enabled. + "computeEnrichment": bool(flags.get("computeEnrichment", True)), } full_sess = load_session_with_items(session_id) diff --git a/gui/src/server/routes/enrichment.py b/gui/src/server/routes/enrichment.py new file mode 100644 index 0000000..2ead704 --- /dev/null +++ b/gui/src/server/routes/enrichment.py @@ -0,0 +1,112 @@ +"""Enrichment analysis: given a set of db entry ids (a Discovery query's checked +nearest neighbors), test whether that selection is enriched for any annotation term +(phylogeny, chemical class, ...) compared to its background -- entries of the same +type(s) not selected. One Fisher's exact test per term observed on the selection, +Benjamini-Hochberg corrected across all of them. +""" + +from flask import Blueprint, Response, jsonify, request +from scipy.stats import fisher_exact + +from routes.database import open_retromol_db + +blp_enrichment_analysis = Blueprint("enrichment_analysis", __name__) + +# Mirrors the frontend's client-side guardrail; must also be enforced here since the +# frontend check is UX-only (see session_store.py's MAX_SESSION_ITEMS for the same pattern). +MAX_ENRICHMENT_SELECTION = 100 + + +def _benjamini_hochberg(p_values: list[float]) -> list[float]: + """Benjamini-Hochberg FDR correction. Returns q-values in the same order as `p_values`.""" + m = len(p_values) + if m == 0: + return [] + + order = sorted(range(m), key=lambda i: p_values[i]) + q_values = [0.0] * m + + running_min = 1.0 + for rank, idx in reversed(list(enumerate(order, start=1))): + q = p_values[idx] * m / rank + running_min = min(running_min, q) + q_values[idx] = min(running_min, 1.0) + + return q_values + + +@blp_enrichment_analysis.post("/api/enrichmentAnalysis") +def enrichment_analysis() -> tuple[Response, int]: + """ + Test whether a selected set of entries is enriched (or depleted) for any annotation + term compared to its background -- entries of the same type(s), excluding the + selection itself. One two-sided Fisher's exact test per term observed on at least + one selected entry, Benjamini-Hochberg corrected across all tested terms. + + :return: a tuple containing the enrichment results and an HTTP status code + """ + body = request.get_json(silent=True) or {} + entry_ids = body.get("entryIds") + + if not isinstance(entry_ids, list) or not entry_ids or not all(isinstance(x, str) and x for x in entry_ids): + return jsonify({"error": "entryIds must be a non-empty list of non-empty strings"}), 400 + + entry_ids = list(dict.fromkeys(entry_ids)) # de-dupe, preserve order + if len(entry_ids) > MAX_ENRICHMENT_SELECTION: + return jsonify({"error": f"entryIds cannot have more than {MAX_ENRICHMENT_SELECTION} entries"}), 400 + + try: + with open_retromol_db() as db: + selected_entries = db.get_entries(entry_ids) + found_ids = {e.id for e in selected_entries} + missing_ids = [i for i in entry_ids if i not in found_ids] + if missing_ids: + return jsonify({"error": f"unknown entry id(s): {missing_ids[:5]}"}), 400 + + background_types = sorted({e.type for e in selected_entries}) + background_total = db.count_entries_by_type(background_types) + n_selected = len(entry_ids) + + selected_counts = db.annotation_term_counts(entry_ids) + background_counts = db.annotation_term_counts_for_types(background_types) + terms = db.annotation_terms_by_ids(list(selected_counts.keys())) + + rows = [] + p_values = [] + for term_id, a in selected_counts.items(): + term_total = background_counts.get(term_id, a) + b = n_selected - a + c = term_total - a + d = (background_total - n_selected) - c + + _, p_value = fisher_exact([[a, b], [c, d]], alternative="two-sided") + + expected_rate = term_total / background_total if background_total else 0.0 + observed_rate = a / n_selected if n_selected else 0.0 + fold_enrichment = (observed_rate / expected_rate) if expected_rate > 0 else None + + term = terms.get(term_id) + rows.append({ + "termId": term_id, + "category": term.category if term else None, + "rank": term.rank if term else None, + "label": term.label if term else term_id, + "selectedWithTerm": a, + "selectedTotal": n_selected, + "backgroundWithTerm": term_total, + "backgroundTotal": background_total, + "foldEnrichment": fold_enrichment, + "direction": "enriched" if fold_enrichment is not None and fold_enrichment > 1 else "depleted", + "pValue": float(p_value), + }) + p_values.append(float(p_value)) + + q_values = _benjamini_hochberg(p_values) + for row, q in zip(rows, q_values): + row["qValue"] = q + + rows.sort(key=lambda r: r["qValue"]) + except Exception as e: + return jsonify({"error": str(e)}), 503 + + return jsonify({"results": rows}), 200 diff --git a/gui/src/server/routes/entry_annotations.py b/gui/src/server/routes/entry_annotations.py new file mode 100644 index 0000000..01de3d9 --- /dev/null +++ b/gui/src/server/routes/entry_annotations.py @@ -0,0 +1,65 @@ +"""Entry annotations: every phylogeny/biosynthetic_class/chemical_class/bioactivity +term linked to one entry, each with a link to that term's source website where one +exists -- used by the Discovery tab's expanded result view ("what do we know about +this compound/BGC"). +""" + +from flask import Blueprint, Response, jsonify, request + +from routes.database import open_retromol_db + +blp_entry_annotations = Blueprint("entry_annotations", __name__) + +NCBI_TAXONOMY_URL = "https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=Info&id={id}" +CHEBI_ENTITY_URL = "https://www.ebi.ac.uk/chebi/searchId.do?chebiId={id}" + + +def _annotation_url(category: str, rank: str | None, external_id: str | None) -> str | None: + """Build a "view on " link for one term, if its category/rank has a known + external database and it resolved an id there. biosynthetic_class and chemical_class + (NPClassifier) have no canonical per-label public page, so those always return None.""" + if not external_id: + return None + if category == "phylogeny": + return NCBI_TAXONOMY_URL.format(id=external_id) + if category == "bioactivity": + if rank in ("chebi_biological_role", "chebi_chemical_role"): + return CHEBI_ENTITY_URL.format(id=external_id) + return None + + +@blp_entry_annotations.get("/api/entryAnnotations") +def entry_annotations() -> tuple[Response, int]: + """ + Every annotation term linked to one entry, for the Discovery tab's expanded result view. + + :return: a tuple containing the annotation list and an HTTP status code + """ + entry_id = (request.args.get("entryId") or "").strip() + if not entry_id: + return jsonify({"error": "entryId is required"}), 400 + + try: + with open_retromol_db() as db: + terms = db.entry_annotation_terms(entry_id) + except Exception as e: + return jsonify({"error": str(e)}), 503 + + return ( + jsonify( + { + "results": [ + { + "id": t.id, + "category": t.category, + "rank": t.rank, + "label": t.label, + "externalId": t.external_id, + "url": _annotation_url(t.category, t.rank, t.external_id), + } + for t in terms + ] + } + ), + 200, + ) diff --git a/gui/src/server/routes/stats.py b/gui/src/server/routes/stats.py index 7a15d54..373aaa6 100644 --- a/gui/src/server/routes/stats.py +++ b/gui/src/server/routes/stats.py @@ -14,6 +14,19 @@ def _to_camel_case(snake: str) -> str: return head + "".join(word.capitalize() for word in tail) +def _camelize(value): + """Recursively camelCase every dict key in `value` -- `asdict()` on a dataclass + with nested dataclasses (e.g. AnnotationStats.coverage: list[AnnotationCoverage]) + only produces nested plain dicts/lists, so a shallow top-level-only conversion + misses every key below the first level (that's exactly what broke + coverage[i].with_annotation_count -> withAnnotationCount before this).""" + if isinstance(value, dict): + return {_to_camel_case(k): _camelize(v) for k, v in value.items()} + if isinstance(value, list): + return [_camelize(v) for v in value] + return value + + @blp_database_stats.get("/api/databaseStats") def database_stats() -> tuple[Response, int]: """ @@ -25,5 +38,20 @@ def database_stats() -> tuple[Response, int]: with open_retromol_db() as db: stats = db.stats() - payload = {_to_camel_case(k): v for k, v in asdict(stats).items()} + payload = _camelize(asdict(stats)) + return jsonify(payload), 200 + + +@blp_database_stats.get("/api/annotationStats") +def annotation_stats() -> tuple[Response, int]: + """ + Summary statistics over the annotation_terms/entry_annotations tables (phylogeny, + chemical class, ... coverage and per-label counts), for the workspace home tab. + + :return: a tuple containing the stats payload and an HTTP status code + """ + with open_retromol_db() as db: + stats = db.annotation_stats() + + payload = _camelize(asdict(stats)) return jsonify(payload), 200 diff --git a/src/retromol_database/duckdb.py b/src/retromol_database/duckdb.py index 8d0de2d..67c4f15 100644 --- a/src/retromol_database/duckdb.py +++ b/src/retromol_database/duckdb.py @@ -61,6 +61,54 @@ class DatabaseStats: without_source_url_count: int +@dataclass(frozen=True) +class AnnotationTerm: + id: str + category: str + rank: str | None + label: str + parent_id: str | None + # An id in whatever external database this term comes from (NCBI taxid for + # phylogeny; ChEBI accession for bioactivity), for linking out to that database's + # own page -- None for categories with no such external page (biosynthetic_class, + # chemical_class). + external_id: str | None = None + + +@dataclass(frozen=True) +class AnnotationCoverage: + label: str + with_annotation_count: int + without_annotation_count: int + + +@dataclass(frozen=True) +class AnnotationStats: + # Coverage per (category, entry type) that actually gets populated for it -- + # phylogeny applies to both compounds and bgcs (an organism produces both), so it + # gets two entries; the rest are single-entry-type by design (see the population + # sites: chemical_class/bioactivity are compound-structure-derived, biosynthetic_class + # describes a BGC's own biosynthesis machinery). + coverage: list[AnnotationCoverage] + counts_by_category: list[Count] + # Phylogeny: one chart per rank. + phylogeny_type_counts: list[Count] + phylogeny_genus_counts: list[Count] + phylogeny_species_counts: list[Count] + # MIBiG's own coarse biosynthetic-class label -- a distinct category from chemical_class + # below, not a rank within it (see biosynthetic_class_annotations table comment). + biosynthetic_class_counts: list[Count] + # Chemical class: NPClassifier's three structure-derived levels (see + # database/scripts/annotate_npclassifier.py), kept as separate charts rather than + # pooled, since they're different ranks within the same classification scheme. + chemical_class_pathway_counts: list[Count] + chemical_class_superclass_counts: list[Count] + chemical_class_class_counts: list[Count] + # Bioactivity: ChEBI's role ontology (see database/scripts/annotate_chebi.py). + bioactivity_biological_role_counts: list[Count] + bioactivity_chemical_role_counts: list[Count] + + def _normalize_entry_type(entry_type: str) -> EntryType: if entry_type not in ENTRY_TYPES: raise ValueError(f"entry type must be one of {ENTRY_TYPES}, got {entry_type}") @@ -146,6 +194,144 @@ def create_schema(self) -> None: ) """ ) + # Four dedicated annotation tables, one per category, each with its own typed + # columns instead of a single generic label -- see RetroMolDuckDB.add_phylogeny_annotation + # / add_bioactivity_annotation / add_biosynthetic_class_annotation / add_chemical_class_annotation. + # Phylogeny is one row per entry (an entry has exactly one organism); the other + # three are multi-valued per entry. chemical_class/bioactivity are populated for + # compounds only (they describe the molecule's own structure); biosynthetic_class + # is populated for bgcs only (it describes the gene cluster's biosynthesis + # machinery, not the compound) -- enforced by pipeline callers, not by this schema. + self.con.execute( + """ + CREATE TABLE IF NOT EXISTS phylogeny_annotations ( + entry_id VARCHAR PRIMARY KEY, + type_label VARCHAR, + type_taxid VARCHAR, + genus VARCHAR, + genus_taxid VARCHAR, + species VARCHAR, + species_taxid VARCHAR + ) + """ + ) + # `level` distinguishes bioactivity's two signals (see + # database/scripts/annotate_chebi.py): ChEBI's "chebi_biological_role" and + # "chebi_chemical_role" (its role ontology, under CHEBI:24432/CHEBI:51086) -- + # both multi-valued per entry. `external_id` is the term's own ChEBI accession, + # for building a "view on ChEBI" link; null when unresolved. + self.con.execute( + """ + CREATE TABLE IF NOT EXISTS bioactivity_annotations ( + entry_id VARCHAR NOT NULL, + level VARCHAR NOT NULL, + label VARCHAR NOT NULL, + external_id VARCHAR, + PRIMARY KEY (entry_id, level, label) + ) + """ + ) + # MIBiG's own coarse biosynthetic-class label (PKS/NRPS/RiPP/terpene/saccharide/other, + # from the BGC's annotated biosynthesis machinery) -- a distinct classification scheme + # from chemical_class below (NPClassifier's structure-derived classification of the + # compound itself), not a rank/level within it. + self.con.execute( + """ + CREATE TABLE IF NOT EXISTS biosynthetic_class_annotations ( + entry_id VARCHAR NOT NULL, + label VARCHAR NOT NULL, + PRIMARY KEY (entry_id, label) + ) + """ + ) + # `level` distinguishes NPClassifier's "pathway"/"superclass"/"class"/"is_glycoside" + # (structure-derived, from a compound's own SMILES -- see + # database/scripts/annotate_npclassifier.py) -- every chemical_class row comes from + # NPClassifier now (MIBiG's biosynthetic class lives in biosynthetic_class_annotations + # above instead). The three list-valued levels can each carry more than one label per + # entry (hence label being part of the primary key, same as bioactivity); is_glycoside + # is stored as a single ("is_glycoside", "Yes") row only when true -- absence means + # false/unclassified, the same presence-only convention every other annotation table uses. + self.con.execute( + """ + CREATE TABLE IF NOT EXISTS chemical_class_annotations ( + entry_id VARCHAR NOT NULL, + level VARCHAR NOT NULL, + label VARCHAR NOT NULL, + PRIMARY KEY (entry_id, level, label) + ) + """ + ) + # Read-only views reconstructing the old generic (id/category/rank/label/parent_id) + # term shape and (entry_id, term_id) links across all three tables above -- so + # annotation_stats/enrichment queries (which are category-agnostic) don't need to + # know about the dedicated tables. term_id encodings match the pre-migration scheme + # (e.g. "phylogeny:genus:bacterium:streptomyces") so existing consumers are unaffected. + self.con.execute( + """ + CREATE OR REPLACE VIEW annotation_terms AS + SELECT id, category, rank, label, parent_id, any_value(taxid) AS external_id + FROM ( + SELECT + 'phylogeny:type:' || lower(type_label) AS id, + 'phylogeny' AS category, 'type' AS rank, type_label AS label, + CAST(NULL AS VARCHAR) AS parent_id, type_taxid AS taxid + FROM phylogeny_annotations WHERE type_label IS NOT NULL + UNION ALL + SELECT + 'phylogeny:genus:' || lower(type_label) || ':' || lower(genus) AS id, + 'phylogeny' AS category, 'genus' AS rank, genus AS label, + 'phylogeny:type:' || lower(type_label) AS parent_id, genus_taxid AS taxid + FROM phylogeny_annotations WHERE type_label IS NOT NULL AND genus IS NOT NULL + UNION ALL + SELECT + 'phylogeny:species:' || lower(type_label) || ':' || lower(genus) || ':' || lower(species) AS id, + 'phylogeny' AS category, 'species' AS rank, species AS label, + 'phylogeny:genus:' || lower(type_label) || ':' || lower(genus) AS parent_id, species_taxid AS taxid + FROM phylogeny_annotations + WHERE type_label IS NOT NULL AND genus IS NOT NULL AND species IS NOT NULL + UNION ALL + SELECT + 'bioactivity:' || level || ':' || lower(label) AS id, + 'bioactivity' AS category, level AS rank, label, + CAST(NULL AS VARCHAR) AS parent_id, external_id AS taxid + FROM bioactivity_annotations + UNION ALL + SELECT + 'biosynthetic_class:' || lower(label) AS id, + 'biosynthetic_class' AS category, CAST(NULL AS VARCHAR) AS rank, label, + CAST(NULL AS VARCHAR) AS parent_id, CAST(NULL AS VARCHAR) AS taxid + FROM biosynthetic_class_annotations + UNION ALL + SELECT + 'chemical_class:' || level || ':' || lower(label) AS id, + 'chemical_class' AS category, level AS rank, label, + CAST(NULL AS VARCHAR) AS parent_id, CAST(NULL AS VARCHAR) AS taxid + FROM chemical_class_annotations + ) + GROUP BY id, category, rank, label, parent_id + """ + ) + self.con.execute( + """ + CREATE OR REPLACE VIEW entry_annotations AS + SELECT entry_id, 'phylogeny:type:' || lower(type_label) AS term_id + FROM phylogeny_annotations WHERE type_label IS NOT NULL + UNION ALL + SELECT entry_id, 'phylogeny:genus:' || lower(type_label) || ':' || lower(genus) + FROM phylogeny_annotations WHERE type_label IS NOT NULL AND genus IS NOT NULL + UNION ALL + SELECT entry_id, 'phylogeny:species:' || lower(type_label) || ':' || lower(genus) || ':' || lower(species) + FROM phylogeny_annotations + WHERE type_label IS NOT NULL AND genus IS NOT NULL AND species IS NOT NULL + UNION ALL + SELECT entry_id, 'bioactivity:' || level || ':' || lower(label) FROM bioactivity_annotations + UNION ALL + SELECT entry_id, 'biosynthetic_class:' || lower(label) FROM biosynthetic_class_annotations + UNION ALL + SELECT entry_id, 'chemical_class:' || level || ':' || lower(label) FROM chemical_class_annotations + """ + ) def add_entry( self, @@ -207,6 +393,295 @@ def bgc_content_hash_exists(self, content_hash: str) -> bool: def count(self) -> int: return int(self.con.execute("SELECT count(*) FROM entries").fetchone()[0]) + def add_phylogeny_annotation( + self, + entry_id: str, + *, + type_label: str | None, + type_taxid: str | None = None, + genus: str | None = None, + genus_taxid: str | None = None, + species: str | None = None, + species_taxid: str | None = None, + ) -> None: + """Set (or replace) `entry_id`'s single phylogeny row. `species`/`species_taxid` + are only meaningful -- and only surfaced by the annotation_terms/entry_annotations + views -- when `genus` is also given; a taxid is optional at every rank (e.g. NPAtlas + names that don't resolve against the NCBI taxdump still store their raw genus/species + text with a NULL taxid). No-ops if `type_label` isn't resolvable.""" + if not type_label: + return + + self.con.execute( + """ + INSERT INTO phylogeny_annotations + (entry_id, type_label, type_taxid, genus, genus_taxid, species, species_taxid) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (entry_id) DO UPDATE SET + type_label = excluded.type_label, + type_taxid = excluded.type_taxid, + genus = excluded.genus, + genus_taxid = excluded.genus_taxid, + species = excluded.species, + species_taxid = excluded.species_taxid + """, + [entry_id, type_label, type_taxid, genus, genus_taxid, species, species_taxid], + ) + + def add_bioactivity_annotation( + self, entry_id: str, *, level: str, label: str, external_id: str | None = None + ) -> None: + """Link `entry_id` (a compound) to a bioactivity label at the given `level` (e.g. + "chebi_biological_role", "chebi_chemical_role" -- see + database/scripts/annotate_chebi.py). Compounds only -- callers must not use this + for bgc entries.""" + self.con.execute( + """ + INSERT INTO bioactivity_annotations (entry_id, level, label, external_id) + VALUES (?, ?, ?, ?) + ON CONFLICT (entry_id, level, label) DO NOTHING + """, + [entry_id, level, label, external_id], + ) + + def add_biosynthetic_class_annotation(self, entry_id: str, label: str) -> None: + """Link `entry_id` (a bgc) to a MIBiG biosynthetic-class label (PKS/NRPS/RiPP/...). + BGCs only -- callers must not use this for compound entries.""" + self.con.execute( + """ + INSERT INTO biosynthetic_class_annotations (entry_id, label) + VALUES (?, ?) + ON CONFLICT (entry_id, label) DO NOTHING + """, + [entry_id, label], + ) + + def add_chemical_class_annotation(self, entry_id: str, *, level: str, label: str) -> None: + """Link `entry_id` (a compound) to a chemical-class label at the given `level` + (e.g. "biosyn_class", or NPClassifier's "pathway"/"superclass"/"class"/"is_glycoside" -- + see the chemical_class_annotations table comment in create_schema). Compounds only -- + callers must not use this for bgc entries.""" + self.con.execute( + """ + INSERT INTO chemical_class_annotations (entry_id, level, label) + VALUES (?, ?, ?) + ON CONFLICT (entry_id, level, label) DO NOTHING + """, + [entry_id, level, label], + ) + + def count_entries_by_type(self, entry_types: Sequence[str]) -> int: + if not entry_types: + return 0 + types = [_normalize_entry_type(t) for t in entry_types] + return int( + self.con.execute( + "SELECT count(*) FROM entries WHERE type IN (SELECT UNNEST(?))", + [types], + ).fetchone()[0] + ) + + def annotation_term_counts(self, entry_ids: Sequence[str]) -> dict[str, int]: + """For every term linked to at least one of the given entry ids, how many of those + ids carry it. Used as the "selected" side of an enrichment contingency table.""" + if not entry_ids: + return {} + + rows = self.con.execute( + """ + SELECT term_id, count(DISTINCT entry_id) + FROM entry_annotations + WHERE entry_id IN (SELECT UNNEST(?)) + GROUP BY term_id + """, + [list(entry_ids)], + ).fetchall() + return {str(row[0]): int(row[1]) for row in rows} + + def annotation_term_counts_for_types(self, entry_types: Sequence[str]) -> dict[str, int]: + """For every term, how many entries of the given type(s) carry it -- the + "background pool" side of an enrichment contingency table.""" + if not entry_types: + return {} + + types = [_normalize_entry_type(t) for t in entry_types] + rows = self.con.execute( + """ + SELECT ea.term_id, count(DISTINCT ea.entry_id) + FROM entry_annotations ea + JOIN entries e ON e.id = ea.entry_id + WHERE e.type IN (SELECT UNNEST(?)) + GROUP BY ea.term_id + """, + [types], + ).fetchall() + return {str(row[0]): int(row[1]) for row in rows} + + def annotation_terms_by_ids(self, term_ids: Sequence[str]) -> dict[str, AnnotationTerm]: + if not term_ids: + return {} + + rows = self.con.execute( + """ + SELECT id, category, rank, label, parent_id, external_id + FROM annotation_terms + WHERE id IN (SELECT UNNEST(?)) + """, + [list(term_ids)], + ).fetchall() + return { + str(row[0]): AnnotationTerm( + id=str(row[0]), category=str(row[1]), rank=row[2], label=str(row[3]), parent_id=row[4], + external_id=row[5], + ) + for row in rows + } + + def entry_annotation_terms(self, entry_id: str) -> list[AnnotationTerm]: + """Every annotation term linked to `entry_id`, across all four categories -- + the "show me everything known about this compound/bgc" query used by the + Discovery tab's expanded result view.""" + rows = self.con.execute( + """ + SELECT t.id, t.category, t.rank, t.label, t.parent_id, t.external_id + FROM entry_annotations ea + JOIN annotation_terms t ON t.id = ea.term_id + WHERE ea.entry_id = ? + ORDER BY t.category, t.rank NULLS FIRST, t.label + """, + [entry_id], + ).fetchall() + return [ + AnnotationTerm( + id=str(row[0]), category=str(row[1]), rank=row[2], label=str(row[3]), parent_id=row[4], + external_id=row[5], + ) + for row in rows + ] + + def search_entries( + self, query: str, *, entry_type: str | None = None, limit: int = 100 + ) -> list[Entry]: + """Look up up to `limit` entries by a case-insensitive substring match on any of + their source names, or an exact match on their id -- the "query and select" lookup + used by the Enrichment tab (distinct from `closest()`'s fingerprint-similarity search).""" + if limit < 1: + raise ValueError("limit must be >= 1") + + where_sql = "WHERE (e.id = $2 OR es.name ILIKE '%' || $2 || '%')" + params: list[object] = [limit, query] + + if entry_type is not None: + entry_type = _normalize_entry_type(entry_type) + where_sql += " AND e.type = $3" + params.append(entry_type) + + rows = self.con.execute( + f""" + SELECT DISTINCT e.id, e.raw, e.type, e.primary_sequence, e.fingerprint + FROM entries e + LEFT JOIN entry_sources es ON es.entry_id = e.id + {where_sql} + ORDER BY e.id + LIMIT $1 + """, + params, + ).fetchall() + + sources_by_id = self._sources_for_entry_ids([str(row[0]) for row in rows]) + return [_entry_from_row(row, sources_by_id.get(str(row[0]), [])) for row in rows] + + def _label_counts(self, *, category: str, rank: str | None = None, limit: int | None = None) -> list[Count]: + """Distinct-entry counts per label, for one (category, rank) slice of + annotation_terms/entry_annotations -- the building block for every chart on the + Dashboard's Annotations section.""" + where_sql = "WHERE t.category = ?" + params: list[object] = [category] + if rank is not None: + where_sql += " AND t.rank = ?" + params.append(rank) + + limit_sql = " LIMIT ?" if limit is not None else "" + if limit is not None: + params.append(limit) + + rows = self.con.execute( + f""" + SELECT t.label, count(DISTINCT ea.entry_id) + FROM entry_annotations ea + JOIN annotation_terms t ON t.id = ea.term_id + {where_sql} + GROUP BY t.label + ORDER BY count(DISTINCT ea.entry_id) DESC + {limit_sql} + """, + params, + ).fetchall() + return [Count(label=str(row[0]), count=int(row[1])) for row in rows] + + def _annotation_coverage(self, *, category: str, entry_type: str, label: str) -> AnnotationCoverage: + """with/without-annotation counts for one (category, entry type) slice -- e.g. + "how many compounds have a chemical_class annotation", not "how many entries of + any type have any annotation" (too coarse to be useful once there are several + categories that each apply to only one entry type).""" + total = self.count_entries_by_type([entry_type]) + with_count = int( + self.con.execute( + """ + SELECT count(DISTINCT ea.entry_id) + FROM entry_annotations ea + JOIN annotation_terms t ON t.id = ea.term_id + JOIN entries e ON e.id = ea.entry_id + WHERE t.category = ? AND e.type = ? + """, + [category, entry_type], + ).fetchone()[0] + ) + return AnnotationCoverage(label=label, with_annotation_count=with_count, without_annotation_count=total - with_count) + + def annotation_stats(self) -> AnnotationStats: + """Summary counts over annotation_terms/entry_annotations, for the Dashboard.""" + coverage = [ + self._annotation_coverage(category="phylogeny", entry_type="compound", label="Phylogeny (compounds)"), + self._annotation_coverage(category="phylogeny", entry_type="bgc", label="Phylogeny (gene clusters)"), + self._annotation_coverage( + category="chemical_class", entry_type="compound", label="Chemical class (compounds)" + ), + self._annotation_coverage( + category="biosynthetic_class", entry_type="bgc", label="Biosynthetic class (gene clusters)" + ), + self._annotation_coverage(category="bioactivity", entry_type="compound", label="Bioactivity (compounds)"), + ] + + category_rows = self.con.execute( + """ + SELECT t.category, count(DISTINCT ea.entry_id) + FROM entry_annotations ea + JOIN annotation_terms t ON t.id = ea.term_id + GROUP BY t.category + ORDER BY t.category + """ + ).fetchall() + counts_by_category = [Count(label=str(row[0]), count=int(row[1])) for row in category_rows] + + return AnnotationStats( + coverage=coverage, + counts_by_category=counts_by_category, + phylogeny_type_counts=self._label_counts(category="phylogeny", rank="type"), + phylogeny_genus_counts=self._label_counts(category="phylogeny", rank="genus", limit=15), + phylogeny_species_counts=self._label_counts(category="phylogeny", rank="species", limit=15), + biosynthetic_class_counts=self._label_counts(category="biosynthetic_class"), + chemical_class_pathway_counts=self._label_counts(category="chemical_class", rank="pathway", limit=15), + chemical_class_superclass_counts=self._label_counts(category="chemical_class", rank="superclass", limit=15), + chemical_class_class_counts=self._label_counts(category="chemical_class", rank="class", limit=15), + bioactivity_biological_role_counts=self._label_counts( + category="bioactivity", rank="chebi_biological_role", limit=15 + ), + bioactivity_chemical_role_counts=self._label_counts( + category="bioactivity", rank="chebi_chemical_role", limit=15 + ), + ) + def stats(self) -> DatabaseStats: """ Compute summary statistics over the whole entries table, for display on a @@ -316,6 +791,24 @@ def get_entry(self, entry_id: str) -> Entry | None: sources = self._sources_for_entry_ids([entry_id]).get(entry_id, []) return _entry_from_row(row, sources) + def get_entries(self, entry_ids: Sequence[str]) -> list[Entry]: + """Batch version of get_entry -- one round trip for up to `len(entry_ids)` entries, + in no particular order (missing ids are silently omitted, not errored).""" + if not entry_ids: + return [] + + rows = self.con.execute( + """ + SELECT id, raw, type, primary_sequence, fingerprint + FROM entries + WHERE id IN (SELECT UNNEST(?)) + """, + [list(entry_ids)], + ).fetchall() + + sources_by_id = self._sources_for_entry_ids([str(row[0]) for row in rows]) + return [_entry_from_row(row, sources_by_id.get(str(row[0]), [])) for row in rows] + def iter_entries(self) -> Iterator[Entry]: rows = self.con.execute( """