From ab57f19e12e2f1bf8234146844e9d089d7e0757e Mon Sep 17 00:00:00 2001 From: David Meijer Date: Thu, 20 Aug 2026 14:32:59 -0400 Subject: [PATCH 01/11] WIP --- database/Snakemake | 11 +- database/scripts/common.py | 38 ++ database/scripts/extract_mibig_compounds.py | 40 +- database/scripts/load_bgcs.py | 17 +- database/scripts/load_compounds.py | 25 + gui/src/client/src/components/MenuContent.tsx | 6 + .../src/components/workspace/Workspace.tsx | 7 +- .../components/workspace/WorkspaceHome.tsx | 99 +++- .../workspace/tabs/browse/WorkspaceBrowse.tsx | 247 ++++++++++ .../tabs/enrichment/WorkspaceEnrichment.tsx | 258 +++++++++++ gui/src/client/src/features/browse/api.ts | 31 ++ gui/src/client/src/features/browse/types.ts | 34 ++ gui/src/client/src/features/database/api.ts | 6 +- gui/src/client/src/features/database/types.ts | 10 + gui/src/client/src/features/enrichment/api.ts | 27 ++ .../client/src/features/enrichment/types.ts | 45 ++ gui/src/server/app.py | 7 + gui/src/server/routes/browse.py | 144 ++++++ gui/src/server/routes/enrichment.py | 157 +++++++ gui/src/server/routes/stats.py | 15 + src/retromol_database/duckdb.py | 426 ++++++++++++++++++ 21 files changed, 1638 insertions(+), 12 deletions(-) create mode 100644 gui/src/client/src/components/workspace/tabs/browse/WorkspaceBrowse.tsx create mode 100644 gui/src/client/src/components/workspace/tabs/enrichment/WorkspaceEnrichment.tsx create mode 100644 gui/src/client/src/features/browse/api.ts create mode 100644 gui/src/client/src/features/browse/types.ts create mode 100644 gui/src/client/src/features/enrichment/api.ts create mode 100644 gui/src/client/src/features/enrichment/types.ts create mode 100644 gui/src/server/routes/browse.py create mode 100644 gui/src/server/routes/enrichment.py diff --git a/database/Snakemake b/database/Snakemake index 3918658..2a9a884 100644 --- a/database/Snakemake +++ b/database/Snakemake @@ -159,13 +159,18 @@ rule extract_mibig_compounds: # module docstring): MIBiG 4.0's GBKs dropped the ACCESSION.VERSION suffix # parse_gbks.py used to read this from, but the JSON's own top-level "version" # field still has it. - versions=WORKDIR / "mibig_json" / "versions.json" + versions=WORKDIR / "mibig_json" / "versions.json", + # accession -> {organism_name, ncbi_tax_id, biosyn_class}, consumed by both + # load_mibig_compounds and load_mibig_bgcs to populate phylogeny/chemical_class + # annotations (see RetroMolDuckDB.add_phylogeny_annotation/add_flat_annotation). + annotations=WORKDIR / "mibig_json" / "annotations.json" run: import extract_mibig_compounds extract_mibig_compounds.run( mibig_json_dir=input.extract_dir, output_path=output.compounds, versions_output_path=output.versions, + annotations_output_path=output.annotations, ) @@ -193,6 +198,7 @@ rule load_mibig_compounds: # MIBiG URLs need an accession's version, from the JSON (see # extract_mibig_compounds rule above) -- not from the GBKs, which no longer carry it. versions=WORKDIR / "mibig_json" / "versions.json", + annotations=WORKDIR / "mibig_json" / "annotations.json", prev=MARKERS / "npatlas_loaded.done" output: marker=touch(MARKERS / "mibig_compounds_loaded.done") @@ -205,6 +211,7 @@ rule load_mibig_compounds: reaction_rules_path=RXN_RULES, matching_rules_path=MXN_RULES, mibig_versions_path=input.versions, + mibig_annotations_path=input.annotations, ) @@ -234,6 +241,7 @@ rule load_mibig_bgcs: input: readouts=WORKDIR / "mibig_gbk" / "readouts.jsonl", versions=WORKDIR / "mibig_json" / "versions.json", + annotations=WORKDIR / "mibig_json" / "annotations.json", prev=MARKERS / "mibig_compounds_loaded.done" output: marker=touch(MARKERS / "bgcs_loaded.done") @@ -245,4 +253,5 @@ rule load_mibig_bgcs: reaction_rules_path=RXN_RULES, matching_rules_path=MXN_RULES, mibig_versions_path=input.versions, + mibig_annotations_path=input.annotations, ) diff --git a/database/scripts/common.py b/database/scripts/common.py index 2101524..c73c639 100644 --- a/database/scripts/common.py +++ b/database/scripts/common.py @@ -168,6 +168,44 @@ def run_retromol_stream_quiet( yield ResultEvent(serialized, err) +# Common secondary-metabolite-producing fungal genera -- MIBiG's JSON has no direct +# kingdom/type field (unlike NPAtlas), so `phylogeny_from_organism_name` falls back to +# this bundled set to distinguish fungal from bacterial entries. MIBiG is overwhelmingly +# bacterial, so "bacterium" is the default and this set only needs to catch the fungal +# minority. Not exhaustive -- a genus missing from this list is classified "bacterium". +FUNGAL_GENERA = { + "aspergillus", "penicillium", "fusarium", "trichoderma", "curvularia", + "colletotrichum", "alternaria", "cladosporium", "talaromyces", "chaetomium", + "acremonium", "beauveria", "metarhizium", "monascus", "epicoccum", + "pestalotiopsis", "phoma", "botrytis", "verticillium", "myrothecium", +} + + +def phylogeny_from_organism_name(organism_name: str | None) -> tuple[str | None, str | None, str | None]: + """Split MIBiG's free-text `organism_name` (e.g. "Streptomyces coelicolor A3(2)") + into (type, genus, species). Type is inferred from `FUNGAL_GENERA` since MIBiG's + JSON carries no kingdom field; genus/species are the name's first two tokens, with + species dropped when the second token is "sp."/"sp" (strain-only identification). + + :param organism_name: MIBiG cluster.organism_name, or None + :return: (type_label, genus, species) -- each may be None if unresolvable + """ + if not organism_name: + return None, None, None + + tokens = organism_name.split() + if not tokens: + return None, None, None + + genus = tokens[0] + species = tokens[1] if len(tokens) > 1 else None + if species is not None and species.rstrip(".").lower() == "sp": + species = None + + type_label = "Fungus" if genus.lower() in FUNGAL_GENERA else "Bacterium" + return type_label, genus, species + + def split_accession_version(record_id: str) -> tuple[str, str | None]: """ Split a GenBank-style "ACCESSION.VERSION" id (e.g. "BGC0000001.5") in two. diff --git a/database/scripts/extract_mibig_compounds.py b/database/scripts/extract_mibig_compounds.py index 67d3345..acae622 100644 --- a/database/scripts/extract_mibig_compounds.py +++ b/database/scripts/extract_mibig_compounds.py @@ -66,17 +66,37 @@ def _iter_compound_records(data: dict[str, Any], root: dict[str, Any], accession } -def run(mibig_json_dir: str | Path, output_path: str | Path, versions_output_path: str | Path) -> None: +def _annotations(root: dict[str, Any]) -> dict[str, Any]: + """Phylogeny + chemical-class metadata shared by every compound/BGC under one accession.""" + organism_name = root.get("organism_name") + ncbi_tax_id = root.get("ncbi_tax_id") + biosyn_class = root.get("biosyn_class") + return { + "organism_name": organism_name if isinstance(organism_name, str) else None, + "ncbi_tax_id": str(ncbi_tax_id) if ncbi_tax_id is not None else None, + "biosyn_class": biosyn_class if isinstance(biosyn_class, list) else [], + } + + +def run( + mibig_json_dir: str | Path, + output_path: str | Path, + versions_output_path: str | Path, + annotations_output_path: str | Path, +) -> None: mibig_json_dir = Path(mibig_json_dir) output_path = Path(output_path) versions_output_path = Path(versions_output_path) + annotations_output_path = Path(annotations_output_path) output_path.parent.mkdir(parents=True, exist_ok=True) versions_output_path.parent.mkdir(parents=True, exist_ok=True) + annotations_output_path.parent.mkdir(parents=True, exist_ok=True) json_files = sorted(mibig_json_dir.rglob("*.json")) written = 0 skipped = 0 versions: dict[str, str] = {} + annotations: dict[str, dict[str, Any]] = {} with open(output_path, "w") as out: for path in json_files: @@ -98,6 +118,8 @@ def run(mibig_json_dir: str | Path, output_path: str | Path, versions_output_pat if version is not None: versions[accession] = version + annotations[accession] = _annotations(root) + had_any = False for record in _iter_compound_records(data, root, accession): out.write(json.dumps(record) + "\n") @@ -112,9 +134,13 @@ def run(mibig_json_dir: str | Path, output_path: str | Path, versions_output_pat with open(versions_output_path, "w") as fh: json.dump(versions, fh, indent=2, sort_keys=True) + with open(annotations_output_path, "w") as fh: + json.dump(annotations, fh, indent=2, sort_keys=True) + log.info( - "extract_mibig_compounds: wrote %d compound records, skipped %d files, resolved %d accession versions", - written, skipped, len(versions), + "extract_mibig_compounds: wrote %d compound records, skipped %d files, " + "resolved %d accession versions, %d accession annotations", + written, skipped, len(versions), len(annotations), ) @@ -125,9 +151,15 @@ def main() -> None: ap.add_argument("--mibig-json-dir", required=True) ap.add_argument("--output", required=True) ap.add_argument("--versions-output", required=True) + ap.add_argument("--annotations-output", required=True) args = ap.parse_args() - run(mibig_json_dir=args.mibig_json_dir, output_path=args.output, versions_output_path=args.versions_output) + run( + mibig_json_dir=args.mibig_json_dir, + output_path=args.output, + versions_output_path=args.versions_output, + annotations_output_path=args.annotations_output, + ) if __name__ == "__main__": diff --git a/database/scripts/load_bgcs.py b/database/scripts/load_bgcs.py index f6f88fc..bbdde7f 100644 --- a/database/scripts/load_bgcs.py +++ b/database/scripts/load_bgcs.py @@ -26,7 +26,7 @@ import logging from pathlib import Path -from common import build_fingerprint_context, load_ruleset, mibig_url +from common import build_fingerprint_context, load_ruleset, mibig_url, phylogeny_from_organism_name from retromol_antismash.modules import LinearReadout, bgc_primary_sequence from retromol_database.duckdb import RetroMolDuckDB @@ -39,6 +39,7 @@ def run( reaction_rules_path: str | Path | None, matching_rules_path: str | Path | None, mibig_versions_path: str | Path, + mibig_annotations_path: str | Path, match_stereochemistry: bool = False, include_raw_gbk: bool = True, ) -> None: @@ -48,6 +49,9 @@ def run( with open(mibig_versions_path) as fh: versions: dict[str, str] = json.load(fh) + with open(mibig_annotations_path) as fh: + annotations: dict[str, dict] = json.load(fh) + added = 0 skipped = 0 skipped_existing_file = 0 @@ -91,6 +95,15 @@ def run( fingerprint=fp, content_hash=file_hash, ) + + record = annotations.get(accession) if accession else None + if record: + type_label, genus, species = phylogeny_from_organism_name(record.get("organism_name")) + db.add_phylogeny_annotation(entry_id, type_label=type_label, genus=genus, species=species) + for chemical_class in record.get("biosyn_class") or []: + if chemical_class: + db.add_flat_annotation(entry_id, category="chemical_class", label=str(chemical_class)) + added += 1 finally: db.close() @@ -110,6 +123,7 @@ def main() -> None: ap.add_argument("--rxn-rules", default=None) ap.add_argument("--mxn-rules", default=None) ap.add_argument("--mibig-versions", required=True) + ap.add_argument("--mibig-annotations", required=True) ap.add_argument("--match-stereochemistry", action="store_true") ap.add_argument("--no-raw-gbk", action="store_true") args = ap.parse_args() @@ -120,6 +134,7 @@ def main() -> None: reaction_rules_path=args.rxn_rules, matching_rules_path=args.mxn_rules, mibig_versions_path=args.mibig_versions, + mibig_annotations_path=args.mibig_annotations, match_stereochemistry=args.match_stereochemistry, include_raw_gbk=not args.no_raw_gbk, ) diff --git a/database/scripts/load_compounds.py b/database/scripts/load_compounds.py index b22a5ab..dc7678e 100644 --- a/database/scripts/load_compounds.py +++ b/database/scripts/load_compounds.py @@ -28,6 +28,7 @@ mibig_url, npatlas_url, per_monomer_tokens, + phylogeny_from_organism_name, primary_sequence_from_result, ) from retromol.model.result import Result @@ -56,6 +57,20 @@ def _mibig_name_and_url(props: dict, versions: dict[str, str]) -> tuple[str | No return name, mibig_url(accession, version) +def _apply_mibig_annotations(db: RetroMolDuckDB, entry_id: str, props: dict, annotations: dict[str, dict]) -> None: + accession = props.get("mibig_accession") + record = annotations.get(accession) if accession else None + if not record: + return + + type_label, genus, species = phylogeny_from_organism_name(record.get("organism_name")) + db.add_phylogeny_annotation(entry_id, type_label=type_label, genus=genus, species=species) + + for chemical_class in record.get("biosyn_class") or []: + if chemical_class: + db.add_flat_annotation(entry_id, category="chemical_class", label=str(chemical_class)) + + def run( results_path: str | Path, db_path: str | Path, @@ -64,6 +79,7 @@ def run( matching_rules_path: str | Path | None, match_stereochemistry: bool = False, mibig_versions_path: str | Path | None = None, + mibig_annotations_path: str | Path | None = None, log_every: int = 1000, ) -> None: ruleset = load_ruleset(reaction_rules_path, matching_rules_path, match_stereochemistry) @@ -74,6 +90,11 @@ def run( with open(mibig_versions_path) as fh: versions = json.load(fh) + annotations: dict[str, dict] = {} + if source == "mibig" and mibig_annotations_path is not None: + with open(mibig_annotations_path) as fh: + annotations = json.load(fh) + compounds = 0 added = 0 skipped = 0 @@ -118,6 +139,8 @@ def run( primary_sequence=names, fingerprint=fp, ) + if source == "mibig": + _apply_mibig_annotations(db, result.submission.inchikey, props, annotations) added += 1 compounds += 1 @@ -146,6 +169,7 @@ def main() -> None: ap.add_argument("--mxn-rules", default=None) ap.add_argument("--match-stereochemistry", action="store_true") ap.add_argument("--mibig-versions", default=None, help="required when --source=mibig") + ap.add_argument("--mibig-annotations", default=None, help="required when --source=mibig") ap.add_argument("--log-every", type=int, default=1000, help="log a progress line every N compounds (0 to disable)") args = ap.parse_args() @@ -157,6 +181,7 @@ def main() -> None: matching_rules_path=args.mxn_rules, match_stereochemistry=args.match_stereochemistry, mibig_versions_path=args.mibig_versions, + mibig_annotations_path=args.mibig_annotations, log_every=args.log_every, ) diff --git a/gui/src/client/src/components/MenuContent.tsx b/gui/src/client/src/components/MenuContent.tsx index 725483e..b2d37f9 100644 --- a/gui/src/client/src/components/MenuContent.tsx +++ b/gui/src/client/src/components/MenuContent.tsx @@ -11,6 +11,7 @@ import HomeRoundedIcon from "@mui/icons-material/HomeRounded"; import UploadFileIcon from "@mui/icons-material/UploadFile"; import RuleIcon from "@mui/icons-material/Rule"; import AutoFixHighIcon from "@mui/icons-material/AutoFixHigh"; +import TableRowsIcon from "@mui/icons-material/TableRows"; import { useNavigate, useLocation } from "react-router-dom"; const mainListItems = [ @@ -34,6 +35,11 @@ const mainListItems = [ icon: , to: `/dashboard/enrichment` }, + { + text: "Browse", + icon: , + to: `/dashboard/browse` + }, { text: "Generate", icon: , diff --git a/gui/src/client/src/components/workspace/Workspace.tsx b/gui/src/client/src/components/workspace/Workspace.tsx index feecb4b..5ca6309 100644 --- a/gui/src/client/src/components/workspace/Workspace.tsx +++ b/gui/src/client/src/components/workspace/Workspace.tsx @@ -16,7 +16,8 @@ import { WorkspaceUpload } from "./WorkspaceUpload"; import { WorkspaceDiscovery } from "./WorkspaceDiscovery"; import { WorkspaceRules } from "./WorkspaceRules"; import { WorkspaceGenerate } from "./WorkspaceGenerate"; -// import { WorkspaceEnrichment } from "./tabs/enrichment/WorkspaceEnrichment"; +import { WorkspaceEnrichment } from "./tabs/enrichment/WorkspaceEnrichment"; +import { WorkspaceBrowse } from "./tabs/browse/WorkspaceBrowse"; export const Workspace: React.FC = () => { const { showOverlay, hideOverlay } = useOverlay(); @@ -172,8 +173,8 @@ export const Workspace: React.FC = () => { } /> } /> } /> - {/*} />*/} - Analysis currently available. Check back later.} /> + } /> + } /> } /> } /> diff --git a/gui/src/client/src/components/workspace/WorkspaceHome.tsx b/gui/src/client/src/components/workspace/WorkspaceHome.tsx index 96e953f..b8efbde 100644 --- a/gui/src/client/src/components/workspace/WorkspaceHome.tsx +++ b/gui/src/client/src/components/workspace/WorkspaceHome.tsx @@ -9,8 +9,9 @@ import Skeleton from "@mui/material/Skeleton"; import { useTheme } from "@mui/material/styles"; import { Link as RouterLink } from "react-router-dom"; import { PieChart } from "@mui/x-charts/PieChart"; -import { getDatabaseStats } from "../../features/database/api"; -import { DatabaseStatsResp } from "../../features/database/types"; +import { BarChart } from "@mui/x-charts/BarChart"; +import { getDatabaseStats, getAnnotationStats } from "../../features/database/api"; +import { DatabaseStatsResp, AnnotationStatsResp } from "../../features/database/types"; const ENTRY_TYPE_LABELS: Record = { compound: "Compounds", @@ -63,6 +64,10 @@ export const WorkspaceHome: React.FC = () => { const [loading, setLoading] = React.useState(true); const [error, setError] = React.useState(null); + const [annotationStats, setAnnotationStats] = React.useState(null); + const [annotationLoading, setAnnotationLoading] = React.useState(true); + const [annotationError, setAnnotationError] = React.useState(null); + React.useEffect(() => { const controller = new AbortController(); setLoading(true); @@ -82,6 +87,25 @@ export const WorkspaceHome: React.FC = () => { return () => controller.abort(); }, []); + React.useEffect(() => { + const controller = new AbortController(); + setAnnotationLoading(true); + setAnnotationError(null); + + getAnnotationStats(controller.signal) + .then(setAnnotationStats) + .catch((err) => { + if (controller.signal.aborted) return; + setAnnotationError(err instanceof Error ? err.message : String(err)); + }) + .finally(() => { + if (controller.signal.aborted) return; + setAnnotationLoading(false); + }); + + return () => controller.abort(); + }, []); + const chartColors = [ palette.palette.primary.main, palette.palette.warning.main, @@ -199,6 +223,77 @@ export const WorkspaceHome: React.FC = () => { /> + + + Annotations + + + {annotationError && ( + + Couldn't load annotation statistics: {annotationError} + + )} + + {!annotationError && annotationLoading && ( + + {Array.from({ length: 2 }).map((_, i) => ( + + ))} + + )} + + {!annotationError && !annotationLoading && annotationStats && ( + <> + + + + + + + {annotationStats.phylogenyTypeCounts.length === 0 ? ( + No phylogeny annotations yet. + ) : ( + ({ label: c.label, count: c.count }))} + xAxis={[{ scaleType: "band", dataKey: "label" }]} + series={[{ dataKey: "count", color: chartColors[0] }]} + height={220} + /> + )} + + + + {annotationStats.topGenera.length === 0 ? ( + No phylogeny annotations yet. + ) : ( + ({ label: c.label, count: c.count }))} + xAxis={[{ scaleType: "band", dataKey: "label" }]} + series={[{ dataKey: "count", color: chartColors[1] }]} + height={220} + /> + )} + + + + {annotationStats.chemicalClassCounts.length === 0 ? ( + No chemical class annotations yet. + ) : ( + ({ label: c.label, count: c.count }))} + xAxis={[{ scaleType: "band", dataKey: "label" }]} + series={[{ dataKey: "count", color: chartColors[2] }]} + height={220} + /> + )} + + + + )} )} diff --git a/gui/src/client/src/components/workspace/tabs/browse/WorkspaceBrowse.tsx b/gui/src/client/src/components/workspace/tabs/browse/WorkspaceBrowse.tsx new file mode 100644 index 0000000..1948dc7 --- /dev/null +++ b/gui/src/client/src/components/workspace/tabs/browse/WorkspaceBrowse.tsx @@ -0,0 +1,247 @@ +import React from "react"; +import Alert from "@mui/material/Alert"; +import Box from "@mui/material/Box"; +import Button from "@mui/material/Button"; +import Card from "@mui/material/Card"; +import CardContent from "@mui/material/CardContent"; +import Chip from "@mui/material/Chip"; +import CircularProgress from "@mui/material/CircularProgress"; +import DownloadIcon from "@mui/icons-material/Download"; +import MenuItem from "@mui/material/MenuItem"; +import Stack from "@mui/material/Stack"; +import Table from "@mui/material/Table"; +import TableBody from "@mui/material/TableBody"; +import TableCell from "@mui/material/TableCell"; +import TableContainer from "@mui/material/TableContainer"; +import TableHead from "@mui/material/TableHead"; +import TablePagination from "@mui/material/TablePagination"; +import TableRow from "@mui/material/TableRow"; +import TextField from "@mui/material/TextField"; +import Tooltip from "@mui/material/Tooltip"; +import Typography from "@mui/material/Typography"; +import { useNotifications } from "../../../NotificationProvider"; +import { getAnnotationTerms, getBrowseEntries, browseEntriesExportUrl } from "../../../../features/browse/api"; +import type { AnnotationTerm, BrowseEntry } from "../../../../features/browse/types"; +import type { EntryType } from "../../../../features/enrichment/types"; + +const ALL_CATEGORY = "all"; +const ALL_TERM = "all"; + +export const WorkspaceBrowse: React.FC = () => { + const { pushNotification } = useNotifications(); + + const [entryType, setEntryType] = React.useState("all"); + const [terms, setTerms] = React.useState([]); + const [category, setCategory] = React.useState(ALL_CATEGORY); + const [termId, setTermId] = React.useState(ALL_TERM); + + const [entries, setEntries] = React.useState([]); + const [loading, setLoading] = React.useState(true); + const [error, setError] = React.useState(null); + + const [page, setPage] = React.useState(0); + const [rowsPerPage, setRowsPerPage] = React.useState(25); + + React.useEffect(() => { + const controller = new AbortController(); + getAnnotationTerms(undefined, controller.signal) + .then((resp) => setTerms(resp.terms)) + .catch((err) => { + if (controller.signal.aborted) return; + pushNotification(`Failed to load annotation terms: ${err instanceof Error ? err.message : String(err)}`, "error"); + }); + return () => controller.abort(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + React.useEffect(() => { + const controller = new AbortController(); + setLoading(true); + setError(null); + + getBrowseEntries(entryType, termId === ALL_TERM ? null : termId, controller.signal) + .then((resp) => { + setEntries(resp.entries); + setPage(0); + }) + .catch((err) => { + if (controller.signal.aborted) return; + setError(err instanceof Error ? err.message : String(err)); + }) + .finally(() => { + if (controller.signal.aborted) return; + setLoading(false); + }); + + return () => controller.abort(); + }, [entryType, termId]); + + const categories = React.useMemo( + () => Array.from(new Set(terms.map((t) => t.category))).sort(), + [terms] + ); + + const termOptions = React.useMemo( + () => terms.filter((t) => category === ALL_CATEGORY || t.category === category), + [terms, category] + ); + + const handleCategoryChange = (value: string) => { + setCategory(value); + setTermId(ALL_TERM); // term choices depend on category, so reset + }; + + const paginatedEntries = entries.slice(page * rowsPerPage, page * rowsPerPage + rowsPerPage); + const exportUrl = browseEntriesExportUrl(entryType, termId === ALL_TERM ? null : termId); + + return ( + + + + + Browse annotations + + + Browse database entries and their annotations, and download the filtered set as a TSV file + (compound SMILES/InChIKey, or BGC id, alongside phylogeny and chemical class annotations). + + + + setEntryType(e.target.value as EntryType | "all")} + > + All + Compounds + Gene clusters (BGCs) + + + handleCategoryChange(e.target.value)} + > + All categories + {categories.map((c) => ( + {c} + ))} + + + setTermId(e.target.value)} + > + Any + {termOptions.map((t) => ( + + {t.rank ? `${t.rank}: ${t.label}` : t.label} + + ))} + + + + + + + + + + + + {error && ( + + Couldn't load entries: {error} + + )} + + {loading && ( + + + + )} + + {!loading && !error && ( + + + + {entries.length.toLocaleString()} matching entries + + + + + + + Name + Type + SMILES / InChIKey + Sources + Phylogeny + Chemical class + + + + {paginatedEntries.map((entry) => ( + + {entry.name} + {entry.type === "bgc" ? "BGC" : "Compound"} + + {entry.type === "compound" ? entry.smiles ?? "-" : entry.id} + + + {entry.sources.map((s) => ( + + ))} + + + {[entry.phylogenyType, entry.genus, entry.species].filter(Boolean).join(" › ") || "-"} + + + {entry.chemicalClasses.length > 0 + ? entry.chemicalClasses.map((c) => ( + + )) + : "-"} + + + ))} + +
+ + + setPage(newPage)} + rowsPerPage={rowsPerPage} + onRowsPerPageChange={(e) => { + setRowsPerPage(parseInt(e.target.value, 10)); + setPage(0); + }} + rowsPerPageOptions={[10, 25, 50, 100]} + /> + + + )} + + ); +}; diff --git a/gui/src/client/src/components/workspace/tabs/enrichment/WorkspaceEnrichment.tsx b/gui/src/client/src/components/workspace/tabs/enrichment/WorkspaceEnrichment.tsx new file mode 100644 index 0000000..13f404a --- /dev/null +++ b/gui/src/client/src/components/workspace/tabs/enrichment/WorkspaceEnrichment.tsx @@ -0,0 +1,258 @@ +import React from "react"; +import Alert from "@mui/material/Alert"; +import Box from "@mui/material/Box"; +import Button from "@mui/material/Button"; +import Card from "@mui/material/Card"; +import CardContent from "@mui/material/CardContent"; +import Checkbox from "@mui/material/Checkbox"; +import Chip from "@mui/material/Chip"; +import CircularProgress from "@mui/material/CircularProgress"; +import MenuItem from "@mui/material/MenuItem"; +import Stack from "@mui/material/Stack"; +import Table from "@mui/material/Table"; +import TableBody from "@mui/material/TableBody"; +import TableCell from "@mui/material/TableCell"; +import TableContainer from "@mui/material/TableContainer"; +import TableHead from "@mui/material/TableHead"; +import TableRow from "@mui/material/TableRow"; +import TextField from "@mui/material/TextField"; +import Typography from "@mui/material/Typography"; +import { useNotifications } from "../../../NotificationProvider"; +import { searchEntries, runEnrichmentAnalysis, MAX_ENTRY_SEARCH_RESULTS, MAX_ENRICHMENT_SELECTION } from "../../../../features/enrichment/api"; +import type { EntryType, EnrichmentResult, SearchEntry } from "../../../../features/enrichment/types"; + +const Q_VALUE_SIGNIFICANT = 0.05; + +export const WorkspaceEnrichment: React.FC = () => { + const { pushNotification } = useNotifications(); + + const [query, setQuery] = React.useState(""); + const [entryType, setEntryType] = React.useState("all"); + const [searchResults, setSearchResults] = React.useState([]); + const [searching, setSearching] = React.useState(false); + const [searchError, setSearchError] = React.useState(null); + + const [selectedIds, setSelectedIds] = React.useState>(new Set()); + + const [running, setRunning] = React.useState(false); + const [results, setResults] = React.useState(null); + + const handleSearch = async (event?: React.FormEvent) => { + event?.preventDefault(); + if (!query.trim()) return; + + setSearching(true); + setSearchError(null); + try { + const resp = await searchEntries(query.trim(), entryType); + setSearchResults(resp.results); + } catch (err) { + setSearchError(err instanceof Error ? err.message : String(err)); + } finally { + setSearching(false); + } + }; + + const toggleSelected = (id: string) => { + setSelectedIds((prev) => { + const next = new Set(prev); + if (next.has(id)) { + next.delete(id); + } else { + if (next.size >= MAX_ENRICHMENT_SELECTION) { + pushNotification(`You can select at most ${MAX_ENRICHMENT_SELECTION} entries.`, "warning"); + return prev; + } + next.add(id); + } + return next; + }); + }; + + const handleSelectAllResults = () => { + setSelectedIds((prev) => { + const next = new Set(prev); + for (const entry of searchResults) { + if (next.size >= MAX_ENRICHMENT_SELECTION) break; + next.add(entry.id); + } + return next; + }); + }; + + const handleClearSelection = () => setSelectedIds(new Set()); + + const handleRunEnrichment = async () => { + if (selectedIds.size === 0) return; + + setRunning(true); + setResults(null); + try { + const resp = await runEnrichmentAnalysis(Array.from(selectedIds)); + setResults(resp.results); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + pushNotification(`Enrichment analysis failed: ${msg}`, "error"); + } finally { + setRunning(false); + } + }; + + return ( + + + + + Enrichment analysis + + + Search the database, select up to {MAX_ENRICHMENT_SELECTION} entries, then test whether that + selection is enriched (or depleted) for any annotation label compared to the rest of the database. + + + + + setQuery(e.target.value)} + /> + setEntryType(e.target.value as EntryType | "all")} + > + All + Compounds + Gene clusters (BGCs) + + + + + + {searchError && ( + + Search failed: {searchError} + + )} + + + + {searchResults.length > 0 && ( + + + + + Results ({searchResults.length}{searchResults.length >= MAX_ENTRY_SEARCH_RESULTS ? "+" : ""}) + + + + + + + + + {selectedIds.size} / {MAX_ENRICHMENT_SELECTION} selected + + + + + + + + Name + Type + Sources + + + + {searchResults.map((entry) => ( + toggleSelected(entry.id)} sx={{ cursor: "pointer" }}> + + e.stopPropagation()} onChange={() => toggleSelected(entry.id)} /> + + {entry.name} + {entry.type === "bgc" ? "BGC" : "Compound"} + + {entry.sources.map((s) => ( + + ))} + + + ))} + +
+ + + + + + + + )} + + {results && ( + + + + Enrichment results + + + {results.length === 0 ? ( + + No annotated terms were found on the selected entries. + + ) : ( + + + + + Category + Label + Selected + Background + Fold + Direction + p-value + q-value + + + + {results.map((r) => ( + + {r.category ?? "-"} + + {r.label} + {r.qValue < Q_VALUE_SIGNIFICANT && ( + + )} + + {r.selectedWithTerm} / {r.selectedTotal} + {r.backgroundWithTerm} / {r.backgroundTotal} + {r.foldEnrichment != null ? r.foldEnrichment.toFixed(2) : "-"} + {r.direction} + {r.pValue.toExponential(2)} + {r.qValue.toExponential(2)} + + ))} + +
+ + )} + + + )} + + ); +}; diff --git a/gui/src/client/src/features/browse/api.ts b/gui/src/client/src/features/browse/api.ts new file mode 100644 index 0000000..9392868 --- /dev/null +++ b/gui/src/client/src/features/browse/api.ts @@ -0,0 +1,31 @@ +import { getJson } from "../http"; +import { AnnotationTermsRespSchema, BrowseEntriesRespSchema } from "./types"; +import type { EntryType } from "../enrichment/types"; + +function buildParams(entryType: EntryType | "all", termId: string | null): URLSearchParams { + const params = new URLSearchParams(); + if (entryType !== "all") params.set("type", entryType); + if (termId) params.set("termId", termId); + return params; +} + +export async function getAnnotationTerms(category?: string, signal?: AbortSignal) { + const params = new URLSearchParams(); + if (category) params.set("category", category); + const qs = params.toString(); + return getJson(`/api/annotationTerms${qs ? `?${qs}` : ""}`, AnnotationTermsRespSchema, signal); +} + +export async function getBrowseEntries( + entryType: EntryType | "all" = "all", + termId: string | null = null, + signal?: AbortSignal +) { + const params = buildParams(entryType, termId); + return getJson(`/api/browseEntries?${params.toString()}`, BrowseEntriesRespSchema, signal); +} + +export function browseEntriesExportUrl(entryType: EntryType | "all", termId: string | null): string { + const params = buildParams(entryType, termId); + return `/api/browseEntries.tsv?${params.toString()}`; +} diff --git a/gui/src/client/src/features/browse/types.ts b/gui/src/client/src/features/browse/types.ts new file mode 100644 index 0000000..8254f5e --- /dev/null +++ b/gui/src/client/src/features/browse/types.ts @@ -0,0 +1,34 @@ +import { z } from "zod"; +import { EntryTypeSchema, EntrySourceSchema } from "../enrichment/types"; + +export const AnnotationTermSchema = z.object({ + id: z.string(), + category: z.string(), + rank: z.string().nullable(), + label: z.string(), + parentId: z.string().nullable(), +}); +export type AnnotationTerm = z.output; + +export const AnnotationTermsRespSchema = z.object({ + terms: z.array(AnnotationTermSchema), +}); + +export const BrowseEntrySchema = z.object({ + id: z.string(), + type: EntryTypeSchema, + name: z.string(), + url: z.string().nullable(), + smiles: z.string().nullable(), + sources: z.array(EntrySourceSchema), + phylogenyType: z.string().nullable(), + genus: z.string().nullable(), + species: z.string().nullable(), + chemicalClasses: z.array(z.string()), +}); +export type BrowseEntry = z.output; + +export const BrowseEntriesRespSchema = z.object({ + entries: z.array(BrowseEntrySchema), +}); +export type BrowseEntriesResp = z.output; diff --git a/gui/src/client/src/features/database/api.ts b/gui/src/client/src/features/database/api.ts index 09f41bd..d4c48a2 100644 --- a/gui/src/client/src/features/database/api.ts +++ b/gui/src/client/src/features/database/api.ts @@ -1,6 +1,10 @@ import { getJson } from "../http"; -import { DatabaseStatsRespSchema, type DatabaseStatsResp } from "./types"; +import { AnnotationStatsRespSchema, DatabaseStatsRespSchema, type AnnotationStatsResp, type DatabaseStatsResp } from "./types"; export async function getDatabaseStats(signal?: AbortSignal): Promise { return getJson("/api/databaseStats", DatabaseStatsRespSchema, signal); } + +export async function getAnnotationStats(signal?: AbortSignal): Promise { + return getJson("/api/annotationStats", AnnotationStatsRespSchema, signal); +} diff --git a/gui/src/client/src/features/database/types.ts b/gui/src/client/src/features/database/types.ts index c46d9c2..e068db7 100644 --- a/gui/src/client/src/features/database/types.ts +++ b/gui/src/client/src/features/database/types.ts @@ -17,3 +17,13 @@ export const DatabaseStatsRespSchema = z.object({ withoutSourceUrlCount: z.number().int().nonnegative(), }); export type DatabaseStatsResp = z.output; + +export const AnnotationStatsRespSchema = z.object({ + withAnnotationCount: z.number().int().nonnegative(), + withoutAnnotationCount: z.number().int().nonnegative(), + countsByCategory: z.array(CountSchema), + phylogenyTypeCounts: z.array(CountSchema), + topGenera: z.array(CountSchema), + chemicalClassCounts: z.array(CountSchema), +}); +export type AnnotationStatsResp = z.output; diff --git a/gui/src/client/src/features/enrichment/api.ts b/gui/src/client/src/features/enrichment/api.ts new file mode 100644 index 0000000..84765d0 --- /dev/null +++ b/gui/src/client/src/features/enrichment/api.ts @@ -0,0 +1,27 @@ +import { getJson, postJson } from "../http"; +import { + EntrySearchRespSchema, + EnrichmentAnalysisRespSchema, + type EntrySearchResp, + type EnrichmentAnalysisResp, + type EntryType, +} from "./types"; + +export const MAX_ENTRY_SEARCH_RESULTS = 100; +export const MAX_ENRICHMENT_SELECTION = 100; + +export async function searchEntries( + query: string, + entryType: EntryType | "all" = "all", + signal?: AbortSignal +): Promise { + const params = new URLSearchParams({ q: query, type: entryType, limit: String(MAX_ENTRY_SEARCH_RESULTS) }); + return getJson(`/api/entrySearch?${params.toString()}`, EntrySearchRespSchema, signal); +} + +export async function runEnrichmentAnalysis( + entryIds: string[], + signal?: AbortSignal +): Promise { + return postJson("/api/enrichmentAnalysis", { entryIds }, EnrichmentAnalysisRespSchema, signal); +} diff --git a/gui/src/client/src/features/enrichment/types.ts b/gui/src/client/src/features/enrichment/types.ts new file mode 100644 index 0000000..64ae689 --- /dev/null +++ b/gui/src/client/src/features/enrichment/types.ts @@ -0,0 +1,45 @@ +import { z } from "zod"; + +export const EntryTypeSchema = z.enum(["compound", "bgc"]); +export type EntryType = z.output; + +export const EntrySourceSchema = z.object({ + name: z.string(), + databaseName: z.string(), + url: z.string().nullable(), +}); + +export const SearchEntrySchema = z.object({ + id: z.string(), + name: z.string(), + url: z.string().nullable(), + type: EntryTypeSchema, + sources: z.array(EntrySourceSchema), +}); +export type SearchEntry = z.output; + +export const EntrySearchRespSchema = z.object({ + results: z.array(SearchEntrySchema), +}); +export type EntrySearchResp = z.output; + +export const EnrichmentResultSchema = z.object({ + termId: z.string(), + category: z.string().nullable(), + rank: z.string().nullable(), + label: z.string(), + selectedWithTerm: z.number().int().nonnegative(), + selectedTotal: z.number().int().nonnegative(), + backgroundWithTerm: z.number().int().nonnegative(), + backgroundTotal: z.number().int().nonnegative(), + foldEnrichment: z.number().nullable(), + direction: z.enum(["enriched", "depleted"]), + pValue: z.number(), + qValue: z.number(), +}); +export type EnrichmentResult = z.output; + +export const EnrichmentAnalysisRespSchema = z.object({ + results: z.array(EnrichmentResultSchema), +}); +export type EnrichmentAnalysisResp = z.output; diff --git a/gui/src/server/app.py b/gui/src/server/app.py index 10bb52c..2f9ae23 100644 --- a/gui/src/server/app.py +++ b/gui/src/server/app.py @@ -40,6 +40,8 @@ ) from routes.rate_limit import limiter, RATE_LIMIT_REJECTIONS from routes.rules import blp_rule_set, blp_generate_backbone +from routes.enrichment import blp_entry_search, blp_enrichment_analysis +from routes.browse import blp_browse_entries, blp_annotation_terms, blp_export_entries # Initialize the Flask app @@ -255,6 +257,11 @@ def ready() -> tuple[dict[str, str], int]: app.register_blueprint(blp_get_discovery_query_result) app.register_blueprint(blp_rule_set) app.register_blueprint(blp_generate_backbone) +app.register_blueprint(blp_entry_search) +app.register_blueprint(blp_enrichment_analysis) +app.register_blueprint(blp_browse_entries) +app.register_blueprint(blp_annotation_terms) +app.register_blueprint(blp_export_entries) # The two rate-limit tiers on top of the app-wide default (see routes/rate_limit.py) # are applied as @limiter.limit(...) decorators directly on each route function, in diff --git a/gui/src/server/routes/browse.py b/gui/src/server/routes/browse.py new file mode 100644 index 0000000..471c7df --- /dev/null +++ b/gui/src/server/routes/browse.py @@ -0,0 +1,144 @@ +"""Browse tab: list/filter database entries together with their annotations, and +export the same (filtered) set as a TSV file -- compound smiles/inchikey or BGC +accession/id, alongside whatever phylogeny/chemical_class annotations are on record. +""" + +import csv +import io + +from flask import Blueprint, Response, jsonify, request + +from retromol_database.duckdb import BrowseEntry, ENTRY_TYPES +from routes.database import open_retromol_db + +blp_browse_entries = Blueprint("browse_entries", __name__) +blp_annotation_terms = Blueprint("annotation_terms", __name__) +blp_export_entries = Blueprint("export_entries", __name__) + +TSV_COLUMNS = [ + "id", "type", "name", "smiles", "inchikey", "url", "sources", + "phylogeny_type", "genus", "species", "chemical_classes", +] + + +def _entry_type_param() -> tuple[str | None, str | None]: + """Read+validate the shared `type` query param. Returns (entry_type, error).""" + entry_type = request.args.get("type") + if entry_type in (None, "", "all"): + return None, None + if entry_type not in ENTRY_TYPES: + return None, f"type must be one of {ENTRY_TYPES} or 'all'" + return entry_type, None + + +def _browse_entry_payload(entry: BrowseEntry) -> dict: + return { + "id": entry.id, + "type": entry.type, + "name": entry.name, + "url": entry.url, + "smiles": entry.raw if entry.type == "compound" else None, + "sources": [ + {"name": s.name, "databaseName": s.database_name, "url": s.url} for s in entry.sources + ], + "phylogenyType": entry.phylogeny_type, + "genus": entry.genus, + "species": entry.species, + "chemicalClasses": entry.chemical_classes, + } + + +def _tsv_row(entry: BrowseEntry) -> list[str]: + return [ + entry.id, + entry.type, + entry.name, + entry.raw if entry.type == "compound" else "", + entry.id if entry.type == "compound" else "", # entries are keyed by InChIKey for compounds + entry.url or "", + ";".join(f"{s.database_name}:{s.name}" for s in entry.sources), + entry.phylogeny_type or "", + entry.genus or "", + entry.species or "", + ";".join(entry.chemical_classes), + ] + + +@blp_annotation_terms.get("/api/annotationTerms") +def annotation_terms() -> tuple[Response, int]: + """ + Every annotation term in the database, for the Browse tab's filter dropdown. + + :return: a tuple containing the terms payload and an HTTP status code + """ + category = request.args.get("category") or None + + try: + with open_retromol_db() as db: + terms = db.list_annotation_terms(category=category) + except Exception as e: + return jsonify({"error": str(e)}), 503 + + return jsonify({ + "terms": [ + {"id": t.id, "category": t.category, "rank": t.rank, "label": t.label, "parentId": t.parent_id} + for t in terms + ] + }), 200 + + +@blp_browse_entries.get("/api/browseEntries") +def browse_entries() -> tuple[Response, int]: + """ + List (optionally filtered) database entries with their annotations, for the + Browse tab's table. Not paginated server-side -- see module docstring; the + frontend paginates client-side over the full filtered set. + + :return: a tuple containing the entries payload and an HTTP status code + """ + entry_type, error = _entry_type_param() + if error: + return jsonify({"error": error}), 400 + + term_id = request.args.get("termId") or None + + try: + with open_retromol_db() as db: + entries = db.browse_entries(entry_type=entry_type, term_id=term_id) + except Exception as e: + return jsonify({"error": str(e)}), 503 + + return jsonify({"entries": [_browse_entry_payload(e) for e in entries]}), 200 + + +@blp_export_entries.get("/api/browseEntries.tsv") +def export_entries_tsv() -> Response: + """ + Export the same (optionally filtered) set `browseEntries` returns as a TSV + file download. + + :return: a TSV file response + """ + entry_type, error = _entry_type_param() + if error: + return jsonify({"error": error}), 400 + + term_id = request.args.get("termId") or None + + try: + with open_retromol_db() as db: + entries = db.browse_entries(entry_type=entry_type, term_id=term_id) + except Exception as e: + return jsonify({"error": str(e)}), 503 + + buf = io.StringIO() + writer = csv.writer(buf, delimiter="\t", lineterminator="\n") + writer.writerow(TSV_COLUMNS) + for entry in entries: + writer.writerow(_tsv_row(entry)) + + return Response( + buf.getvalue(), + mimetype="text/tab-separated-values", + headers={"Content-Disposition": 'attachment; filename="retromol_entries.tsv"'}, + ) diff --git a/gui/src/server/routes/enrichment.py b/gui/src/server/routes/enrichment.py new file mode 100644 index 0000000..d0e6bbd --- /dev/null +++ b/gui/src/server/routes/enrichment.py @@ -0,0 +1,157 @@ +"""Enrichment tab: search up to MAX_ENTRY_SEARCH_RESULTS db entries, select some of +them, and test whether the selection is enriched for any annotation term (phylogeny, +chemical class, ...) compared to its background -- entries of the same type(s) not +selected. One Fisher's exact test per term observed on the selection, Benjamini-Hochberg +corrected across all of them. +""" + +from flask import Blueprint, Response, jsonify, request +from scipy.stats import fisher_exact + +from retromol_database.duckdb import ENTRY_TYPES +from routes.database import open_retromol_db + +blp_entry_search = Blueprint("entry_search", __name__) +blp_enrichment_analysis = Blueprint("enrichment_analysis", __name__) + +# Mirrors the frontend's client-side guardrail; must also be enforced here since the +# frontend check is UX-only (see session_store.py's MAX_SESSION_ITEMS for the same pattern). +MAX_ENTRY_SEARCH_RESULTS = 100 +MAX_ENRICHMENT_SELECTION = 100 + + +def _entry_payload(entry) -> dict: + return { + "id": entry.id, + "name": entry.name, + "url": entry.url, + "type": entry.type, + "sources": [ + {"name": s.name, "databaseName": s.database_name, "url": s.url} for s in entry.sources + ], + } + + +@blp_entry_search.get("/api/entrySearch") +def entry_search() -> tuple[Response, int]: + """ + Search entries by name/id, for the Enrichment tab's "query and select" step. + + :return: a tuple containing the search results and an HTTP status code + """ + query = (request.args.get("q") or "").strip() + if not query: + return jsonify({"error": "q is required"}), 400 + + entry_type = request.args.get("type") + if entry_type == "all": + entry_type = None + if entry_type is not None and entry_type not in ENTRY_TYPES: + return jsonify({"error": f"type must be one of {ENTRY_TYPES} or 'all'"}), 400 + + limit = request.args.get("limit", default=MAX_ENTRY_SEARCH_RESULTS, type=int) + if limit is None or not (1 <= limit <= MAX_ENTRY_SEARCH_RESULTS): + return jsonify({"error": f"limit must be an integer between 1 and {MAX_ENTRY_SEARCH_RESULTS}"}), 400 + + try: + with open_retromol_db() as db: + results = db.search_entries(query, entry_type=entry_type, limit=limit) + except Exception as e: + return jsonify({"error": str(e)}), 503 + + return jsonify({"results": [_entry_payload(e) for e in results]}), 200 + + +def _benjamini_hochberg(p_values: list[float]) -> list[float]: + """Benjamini-Hochberg FDR correction. Returns q-values in the same order as `p_values`.""" + m = len(p_values) + if m == 0: + return [] + + order = sorted(range(m), key=lambda i: p_values[i]) + q_values = [0.0] * m + + running_min = 1.0 + for rank, idx in reversed(list(enumerate(order, start=1))): + q = p_values[idx] * m / rank + running_min = min(running_min, q) + q_values[idx] = min(running_min, 1.0) + + return q_values + + +@blp_enrichment_analysis.post("/api/enrichmentAnalysis") +def enrichment_analysis() -> tuple[Response, int]: + """ + Test whether a selected set of entries is enriched (or depleted) for any annotation + term compared to its background -- entries of the same type(s), excluding the + selection itself. One two-sided Fisher's exact test per term observed on at least + one selected entry, Benjamini-Hochberg corrected across all tested terms. + + :return: a tuple containing the enrichment results and an HTTP status code + """ + body = request.get_json(silent=True) or {} + entry_ids = body.get("entryIds") + + if not isinstance(entry_ids, list) or not entry_ids or not all(isinstance(x, str) and x for x in entry_ids): + return jsonify({"error": "entryIds must be a non-empty list of non-empty strings"}), 400 + + entry_ids = list(dict.fromkeys(entry_ids)) # de-dupe, preserve order + if len(entry_ids) > MAX_ENRICHMENT_SELECTION: + return jsonify({"error": f"entryIds cannot have more than {MAX_ENRICHMENT_SELECTION} entries"}), 400 + + try: + with open_retromol_db() as db: + selected_entries = db.get_entries(entry_ids) + found_ids = {e.id for e in selected_entries} + missing_ids = [i for i in entry_ids if i not in found_ids] + if missing_ids: + return jsonify({"error": f"unknown entry id(s): {missing_ids[:5]}"}), 400 + + background_types = sorted({e.type for e in selected_entries}) + background_total = db.count_entries_by_type(background_types) + n_selected = len(entry_ids) + + selected_counts = db.annotation_term_counts(entry_ids) + background_counts = db.annotation_term_counts_for_types(background_types) + terms = db.annotation_terms_by_ids(list(selected_counts.keys())) + + rows = [] + p_values = [] + for term_id, a in selected_counts.items(): + term_total = background_counts.get(term_id, a) + b = n_selected - a + c = term_total - a + d = (background_total - n_selected) - c + + _, p_value = fisher_exact([[a, b], [c, d]], alternative="two-sided") + + expected_rate = term_total / background_total if background_total else 0.0 + observed_rate = a / n_selected if n_selected else 0.0 + fold_enrichment = (observed_rate / expected_rate) if expected_rate > 0 else None + + term = terms.get(term_id) + rows.append({ + "termId": term_id, + "category": term.category if term else None, + "rank": term.rank if term else None, + "label": term.label if term else term_id, + "selectedWithTerm": a, + "selectedTotal": n_selected, + "backgroundWithTerm": term_total, + "backgroundTotal": background_total, + "foldEnrichment": fold_enrichment, + "direction": "enriched" if fold_enrichment is not None and fold_enrichment > 1 else "depleted", + "pValue": float(p_value), + }) + p_values.append(float(p_value)) + + q_values = _benjamini_hochberg(p_values) + for row, q in zip(rows, q_values): + row["qValue"] = q + + rows.sort(key=lambda r: r["qValue"]) + except Exception as e: + return jsonify({"error": str(e)}), 503 + + return jsonify({"results": rows}), 200 diff --git a/gui/src/server/routes/stats.py b/gui/src/server/routes/stats.py index 7a15d54..6d433aa 100644 --- a/gui/src/server/routes/stats.py +++ b/gui/src/server/routes/stats.py @@ -27,3 +27,18 @@ def database_stats() -> tuple[Response, int]: payload = {_to_camel_case(k): v for k, v in asdict(stats).items()} return jsonify(payload), 200 + + +@blp_database_stats.get("/api/annotationStats") +def annotation_stats() -> tuple[Response, int]: + """ + Summary statistics over the annotation_terms/entry_annotations tables (phylogeny, + chemical class, ... coverage and per-label counts), for the workspace home tab. + + :return: a tuple containing the stats payload and an HTTP status code + """ + with open_retromol_db() as db: + stats = db.annotation_stats() + + payload = {_to_camel_case(k): v for k, v in asdict(stats).items()} + return jsonify(payload), 200 diff --git a/src/retromol_database/duckdb.py b/src/retromol_database/duckdb.py index 8d0de2d..bf65d45 100644 --- a/src/retromol_database/duckdb.py +++ b/src/retromol_database/duckdb.py @@ -61,6 +61,39 @@ class DatabaseStats: without_source_url_count: int +@dataclass(frozen=True) +class AnnotationTerm: + id: str + category: str + rank: str | None + label: str + parent_id: str | None + + +@dataclass(frozen=True) +class BrowseEntry: + id: str + type: EntryType + name: str + url: str | None + raw: str | None + sources: list[EntrySource] + phylogeny_type: str | None + genus: str | None + species: str | None + chemical_classes: list[str] + + +@dataclass(frozen=True) +class AnnotationStats: + with_annotation_count: int + without_annotation_count: int + counts_by_category: list[Count] + phylogeny_type_counts: list[Count] + top_genera: list[Count] + chemical_class_counts: list[Count] + + def _normalize_entry_type(entry_type: str) -> EntryType: if entry_type not in ENTRY_TYPES: raise ValueError(f"entry type must be one of {ENTRY_TYPES}, got {entry_type}") @@ -146,6 +179,26 @@ def create_schema(self) -> None: ) """ ) + self.con.execute( + """ + CREATE TABLE IF NOT EXISTS annotation_terms ( + id VARCHAR PRIMARY KEY, + category VARCHAR NOT NULL, + rank VARCHAR, + label VARCHAR NOT NULL, + parent_id VARCHAR + ) + """ + ) + self.con.execute( + """ + CREATE TABLE IF NOT EXISTS entry_annotations ( + entry_id VARCHAR NOT NULL, + term_id VARCHAR NOT NULL, + PRIMARY KEY (entry_id, term_id) + ) + """ + ) def add_entry( self, @@ -207,6 +260,361 @@ def bgc_content_hash_exists(self, content_hash: str) -> bool: def count(self) -> int: return int(self.con.execute("SELECT count(*) FROM entries").fetchone()[0]) + def add_annotation_term( + self, + *, + term_id: str, + category: str, + label: str, + rank: str | None = None, + parent_id: str | None = None, + ) -> str: + """Add (or no-op if already present) a single annotation term. `term_id` is a + caller-supplied deterministic slug (e.g. "phylogeny:genus:bacterium:streptomyces") + so repeated calls for the same term across many entries are idempotent.""" + self.con.execute( + """ + INSERT INTO annotation_terms (id, category, rank, label, parent_id) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT (id) DO NOTHING + """, + [term_id, category, rank, label, parent_id], + ) + return term_id + + def link_entry_annotation(self, entry_id: str, term_id: str) -> None: + self.con.execute( + """ + INSERT INTO entry_annotations (entry_id, term_id) + VALUES (?, ?) + ON CONFLICT (entry_id, term_id) DO NOTHING + """, + [entry_id, term_id], + ) + + def add_phylogeny_annotation( + self, + entry_id: str, + *, + type_label: str | None, + genus: str | None, + species: str | None, + ) -> None: + """Link `entry_id` to whichever of type/genus/species is resolvable, linking every + level (not just the most specific one) so term-level queries at any rank don't need + to walk the parent_id chain. `species` is ignored if `genus` isn't given.""" + if not type_label: + return + + type_id = self.add_annotation_term( + term_id=f"phylogeny:type:{type_label.lower()}", + category="phylogeny", + rank="type", + label=type_label, + ) + self.link_entry_annotation(entry_id, type_id) + + if not genus: + return + + genus_id = self.add_annotation_term( + term_id=f"phylogeny:genus:{type_label.lower()}:{genus.lower()}", + category="phylogeny", + rank="genus", + label=genus, + parent_id=type_id, + ) + self.link_entry_annotation(entry_id, genus_id) + + if not species: + return + + species_id = self.add_annotation_term( + term_id=f"phylogeny:species:{type_label.lower()}:{genus.lower()}:{species.lower()}", + category="phylogeny", + rank="species", + label=species, + parent_id=genus_id, + ) + self.link_entry_annotation(entry_id, species_id) + + def add_flat_annotation(self, entry_id: str, *, category: str, label: str) -> None: + """Link `entry_id` to a single, non-hierarchical term (e.g. chemical_class, bioactivity).""" + term_id = self.add_annotation_term( + term_id=f"{category}:{label.lower()}", + category=category, + label=label, + ) + self.link_entry_annotation(entry_id, term_id) + + def count_entries_by_type(self, entry_types: Sequence[str]) -> int: + if not entry_types: + return 0 + types = [_normalize_entry_type(t) for t in entry_types] + return int( + self.con.execute( + "SELECT count(*) FROM entries WHERE type IN (SELECT UNNEST(?))", + [types], + ).fetchone()[0] + ) + + def annotation_term_counts(self, entry_ids: Sequence[str]) -> dict[str, int]: + """For every term linked to at least one of the given entry ids, how many of those + ids carry it. Used as the "selected" side of an enrichment contingency table.""" + if not entry_ids: + return {} + + rows = self.con.execute( + """ + SELECT term_id, count(DISTINCT entry_id) + FROM entry_annotations + WHERE entry_id IN (SELECT UNNEST(?)) + GROUP BY term_id + """, + [list(entry_ids)], + ).fetchall() + return {str(row[0]): int(row[1]) for row in rows} + + def annotation_term_counts_for_types(self, entry_types: Sequence[str]) -> dict[str, int]: + """For every term, how many entries of the given type(s) carry it -- the + "background pool" side of an enrichment contingency table.""" + if not entry_types: + return {} + + types = [_normalize_entry_type(t) for t in entry_types] + rows = self.con.execute( + """ + SELECT ea.term_id, count(DISTINCT ea.entry_id) + FROM entry_annotations ea + JOIN entries e ON e.id = ea.entry_id + WHERE e.type IN (SELECT UNNEST(?)) + GROUP BY ea.term_id + """, + [types], + ).fetchall() + return {str(row[0]): int(row[1]) for row in rows} + + def annotation_terms_by_ids(self, term_ids: Sequence[str]) -> dict[str, AnnotationTerm]: + if not term_ids: + return {} + + rows = self.con.execute( + """ + SELECT id, category, rank, label, parent_id + FROM annotation_terms + WHERE id IN (SELECT UNNEST(?)) + """, + [list(term_ids)], + ).fetchall() + return { + str(row[0]): AnnotationTerm( + id=str(row[0]), category=str(row[1]), rank=row[2], label=str(row[3]), parent_id=row[4] + ) + for row in rows + } + + def list_annotation_terms(self, category: str | None = None) -> list[AnnotationTerm]: + """Every annotation term in the database (not just ones with entries counted), + for populating a filter dropdown in the Browse tab.""" + where_sql = "" + params: list[object] = [] + if category is not None: + where_sql = "WHERE category = ?" + params.append(category) + + rows = self.con.execute( + f""" + SELECT id, category, rank, label, parent_id + FROM annotation_terms + {where_sql} + ORDER BY category, rank NULLS FIRST, label + """, + params, + ).fetchall() + return [ + AnnotationTerm(id=str(r[0]), category=str(r[1]), rank=r[2], label=str(r[3]), parent_id=r[4]) + for r in rows + ] + + def browse_entries( + self, *, entry_type: str | None = None, term_id: str | None = None + ) -> list[BrowseEntry]: + """Every entry (optionally filtered by type and/or a single annotation term id -- + matching phylogeny at any rank or a chemical class), with its sources and + annotations attached. Used for both the Browse tab's table and its TSV export -- + both need the whole matching set, not a similarity-ranked slice.""" + where_sql = [] + params: list[object] = [] + + if entry_type is not None: + entry_type = _normalize_entry_type(entry_type) + where_sql.append("e.type = ?") + params.append(entry_type) + + if term_id is not None: + where_sql.append("e.id IN (SELECT entry_id FROM entry_annotations WHERE term_id = ?)") + params.append(term_id) + + where_clause = f"WHERE {' AND '.join(where_sql)}" if where_sql else "" + + rows = self.con.execute( + f""" + SELECT e.id, e.raw, e.type, e.primary_sequence, e.fingerprint + FROM entries e + {where_clause} + ORDER BY e.id + """, + params, + ).fetchall() + + entry_ids = [str(row[0]) for row in rows] + sources_by_id = self._sources_for_entry_ids(entry_ids) + + annotation_rows = ( + self.con.execute( + """ + SELECT ea.entry_id, t.category, t.rank, t.label + FROM entry_annotations ea + JOIN annotation_terms t ON t.id = ea.term_id + WHERE ea.entry_id IN (SELECT UNNEST(?)) + """, + [entry_ids], + ).fetchall() + if entry_ids + else [] + ) + + phylogeny_type_by_id: dict[str, str] = {} + genus_by_id: dict[str, str] = {} + species_by_id: dict[str, str] = {} + chemical_classes_by_id: dict[str, list[str]] = {} + for entry_id, category, rank, label in annotation_rows: + entry_id = str(entry_id) + if category == "phylogeny" and rank == "type": + phylogeny_type_by_id[entry_id] = str(label) + elif category == "phylogeny" and rank == "genus": + genus_by_id[entry_id] = str(label) + elif category == "phylogeny" and rank == "species": + species_by_id[entry_id] = str(label) + elif category == "chemical_class": + chemical_classes_by_id.setdefault(entry_id, []).append(str(label)) + + out: list[BrowseEntry] = [] + for row in rows: + entry_id = str(row[0]) + entry = _entry_from_row(row, sources_by_id.get(entry_id, [])) + out.append( + BrowseEntry( + id=entry.id, + type=entry.type, + name=entry.name, + url=entry.url, + raw=entry.raw, + sources=entry.sources, + phylogeny_type=phylogeny_type_by_id.get(entry_id), + genus=genus_by_id.get(entry_id), + species=species_by_id.get(entry_id), + chemical_classes=chemical_classes_by_id.get(entry_id, []), + ) + ) + return out + + def search_entries( + self, query: str, *, entry_type: str | None = None, limit: int = 100 + ) -> list[Entry]: + """Look up up to `limit` entries by a case-insensitive substring match on any of + their source names, or an exact match on their id -- the "query and select" lookup + used by the Enrichment tab (distinct from `closest()`'s fingerprint-similarity search).""" + if limit < 1: + raise ValueError("limit must be >= 1") + + where_sql = "WHERE (e.id = $2 OR es.name ILIKE '%' || $2 || '%')" + params: list[object] = [limit, query] + + if entry_type is not None: + entry_type = _normalize_entry_type(entry_type) + where_sql += " AND e.type = $3" + params.append(entry_type) + + rows = self.con.execute( + f""" + SELECT DISTINCT e.id, e.raw, e.type, e.primary_sequence, e.fingerprint + FROM entries e + LEFT JOIN entry_sources es ON es.entry_id = e.id + {where_sql} + ORDER BY e.id + LIMIT $1 + """, + params, + ).fetchall() + + sources_by_id = self._sources_for_entry_ids([str(row[0]) for row in rows]) + return [_entry_from_row(row, sources_by_id.get(str(row[0]), [])) for row in rows] + + def annotation_stats(self) -> AnnotationStats: + """Summary counts over annotation_terms/entry_annotations, for the Dashboard.""" + with_annotation_count = int( + self.con.execute("SELECT count(DISTINCT entry_id) FROM entry_annotations").fetchone()[0] + ) + without_annotation_count = self.count() - with_annotation_count + + category_rows = self.con.execute( + """ + SELECT t.category, count(DISTINCT ea.entry_id) + FROM entry_annotations ea + JOIN annotation_terms t ON t.id = ea.term_id + GROUP BY t.category + ORDER BY t.category + """ + ).fetchall() + counts_by_category = [Count(label=str(row[0]), count=int(row[1])) for row in category_rows] + + phylogeny_type_rows = self.con.execute( + """ + SELECT t.label, count(DISTINCT ea.entry_id) + FROM entry_annotations ea + JOIN annotation_terms t ON t.id = ea.term_id + WHERE t.category = 'phylogeny' AND t.rank = 'type' + GROUP BY t.label + ORDER BY count(DISTINCT ea.entry_id) DESC + """ + ).fetchall() + phylogeny_type_counts = [Count(label=str(row[0]), count=int(row[1])) for row in phylogeny_type_rows] + + genus_rows = self.con.execute( + """ + SELECT t.label, count(DISTINCT ea.entry_id) + FROM entry_annotations ea + JOIN annotation_terms t ON t.id = ea.term_id + WHERE t.category = 'phylogeny' AND t.rank = 'genus' + GROUP BY t.label + ORDER BY count(DISTINCT ea.entry_id) DESC + LIMIT 15 + """ + ).fetchall() + top_genera = [Count(label=str(row[0]), count=int(row[1])) for row in genus_rows] + + chem_class_rows = self.con.execute( + """ + SELECT t.label, count(DISTINCT ea.entry_id) + FROM entry_annotations ea + JOIN annotation_terms t ON t.id = ea.term_id + WHERE t.category = 'chemical_class' + GROUP BY t.label + ORDER BY count(DISTINCT ea.entry_id) DESC + """ + ).fetchall() + chemical_class_counts = [Count(label=str(row[0]), count=int(row[1])) for row in chem_class_rows] + + return AnnotationStats( + with_annotation_count=with_annotation_count, + without_annotation_count=without_annotation_count, + counts_by_category=counts_by_category, + phylogeny_type_counts=phylogeny_type_counts, + top_genera=top_genera, + chemical_class_counts=chemical_class_counts, + ) + def stats(self) -> DatabaseStats: """ Compute summary statistics over the whole entries table, for display on a @@ -316,6 +724,24 @@ def get_entry(self, entry_id: str) -> Entry | None: sources = self._sources_for_entry_ids([entry_id]).get(entry_id, []) return _entry_from_row(row, sources) + def get_entries(self, entry_ids: Sequence[str]) -> list[Entry]: + """Batch version of get_entry -- one round trip for up to `len(entry_ids)` entries, + in no particular order (missing ids are silently omitted, not errored).""" + if not entry_ids: + return [] + + rows = self.con.execute( + """ + SELECT id, raw, type, primary_sequence, fingerprint + FROM entries + WHERE id IN (SELECT UNNEST(?)) + """, + [list(entry_ids)], + ).fetchall() + + sources_by_id = self._sources_for_entry_ids([str(row[0]) for row in rows]) + return [_entry_from_row(row, sources_by_id.get(str(row[0]), [])) for row in rows] + def iter_entries(self) -> Iterator[Entry]: rows = self.con.execute( """ From 0c4f05924019f1a0214f7b27586dd261dec791fd Mon Sep 17 00:00:00 2001 From: David Meijer Date: Thu, 20 Aug 2026 15:26:13 -0400 Subject: [PATCH 02/11] WIP --- database/scripts/common.py | 12 +- database/scripts/extract_mibig_compounds.py | 41 +++- database/scripts/load_compounds.py | 20 ++ .../workspace/tabs/browse/WorkspaceBrowse.tsx | 207 ++++++++++-------- .../tabs/enrichment/WorkspaceEnrichment.tsx | 15 +- gui/src/client/src/features/browse/api.ts | 5 + gui/src/client/src/features/browse/types.ts | 2 + gui/src/client/src/features/sources.ts | 28 +++ gui/src/server/routes/browse.py | 19 +- src/retromol_database/duckdb.py | 43 +++- 10 files changed, 284 insertions(+), 108 deletions(-) create mode 100644 gui/src/client/src/features/sources.ts diff --git a/database/scripts/common.py b/database/scripts/common.py index c73c639..ac2b01d 100644 --- a/database/scripts/common.py +++ b/database/scripts/common.py @@ -181,11 +181,17 @@ def run_retromol_stream_quiet( } +# Metagenomic/environmental-sample naming conventions (e.g. "uncultured Streptomyces sp.", +# "unidentified bacterium") -- not a genus, so skipped when picking the genus token. +_NON_TAXONOMIC_PREFIXES = {"uncultured", "unclassified", "unidentified"} + + def phylogeny_from_organism_name(organism_name: str | None) -> tuple[str | None, str | None, str | None]: """Split MIBiG's free-text `organism_name` (e.g. "Streptomyces coelicolor A3(2)") into (type, genus, species). Type is inferred from `FUNGAL_GENERA` since MIBiG's - JSON carries no kingdom field; genus/species are the name's first two tokens, with - species dropped when the second token is "sp."/"sp" (strain-only identification). + JSON carries no kingdom field; genus/species are the name's first two tokens (after + dropping a leading "uncultured"/"unclassified"/"unidentified" marker), with species + dropped when the following token is "sp."/"sp" (strain-only identification). :param organism_name: MIBiG cluster.organism_name, or None :return: (type_label, genus, species) -- each may be None if unresolvable @@ -194,6 +200,8 @@ def phylogeny_from_organism_name(organism_name: str | None) -> tuple[str | None, return None, None, None tokens = organism_name.split() + if tokens and tokens[0].lower() in _NON_TAXONOMIC_PREFIXES: + tokens = tokens[1:] if not tokens: return None, None, None diff --git a/database/scripts/extract_mibig_compounds.py b/database/scripts/extract_mibig_compounds.py index acae622..3a25155 100644 --- a/database/scripts/extract_mibig_compounds.py +++ b/database/scripts/extract_mibig_compounds.py @@ -66,15 +66,46 @@ def _iter_compound_records(data: dict[str, Any], root: dict[str, Any], accession } +# MIBiG's own short biosynthesis-class codes -> friendlier display labels for the +# chemical_class annotation. Covers every value observed across the full 4.0 corpus +# (PKS/NRPS/ribosomal/other/terpene/saccharide); an unrecognized future code falls +# back to itself unchanged rather than being dropped. +BIOSYN_CLASS_LABELS = { + "PKS": "Polyketide", + "NRPS": "Nonribosomal peptide", + "ribosomal": "RiPP", + "terpene": "Terpene", + "saccharide": "Saccharide", + "other": "Other", +} + + def _annotations(root: dict[str, Any]) -> dict[str, Any]: - """Phylogeny + chemical-class metadata shared by every compound/BGC under one accession.""" - organism_name = root.get("organism_name") - ncbi_tax_id = root.get("ncbi_tax_id") - biosyn_class = root.get("biosyn_class") + """Phylogeny + chemical-class metadata shared by every compound/BGC under one accession. + + MIBiG 4.0's real (flat) schema nests these under "taxonomy" ({"name", "ncbiTaxId"}) + and "biosynthesis" ({"classes": [{"class": "PKS", ...}, ...]}) -- not the top-level + "organism_name"/"ncbi_tax_id"/"biosyn_class" keys older MIBiG releases used. + """ + taxonomy = root.get("taxonomy") + taxonomy = taxonomy if isinstance(taxonomy, dict) else {} + organism_name = taxonomy.get("name") + ncbi_tax_id = taxonomy.get("ncbiTaxId") + + biosynthesis = root.get("biosynthesis") + biosynthesis = biosynthesis if isinstance(biosynthesis, dict) else {} + classes = biosynthesis.get("classes") + classes = classes if isinstance(classes, list) else [] + biosyn_class = [ + BIOSYN_CLASS_LABELS.get(c.get("class"), c.get("class")) + for c in classes + if isinstance(c, dict) and c.get("class") + ] + return { "organism_name": organism_name if isinstance(organism_name, str) else None, "ncbi_tax_id": str(ncbi_tax_id) if ncbi_tax_id is not None else None, - "biosyn_class": biosyn_class if isinstance(biosyn_class, list) else [], + "biosyn_class": biosyn_class, } diff --git a/database/scripts/load_compounds.py b/database/scripts/load_compounds.py index dc7678e..37d8089 100644 --- a/database/scripts/load_compounds.py +++ b/database/scripts/load_compounds.py @@ -50,6 +50,21 @@ def _npatlas_name_and_url(props: dict) -> tuple[str | None, str | None]: return name, npatlas_url(npaid) +def _npatlas_phylogeny(props: dict) -> tuple[str | None, str | None, str | None]: + """NPAtlas's SDF carries type/genus/species directly (unlike MIBiG's free-text + organism_name) -- see the `origin_type`/`genus`/`origin_species` SDF properties.""" + type_key = find_key_ci(props, ["origin_type"]) + type_label = props.get(type_key) if type_key else None + + genus_key = find_key_ci(props, ["genus"]) + genus = props.get(genus_key) if genus_key else None + + species_key = find_key_ci(props, ["origin_species"]) + species = props.get(species_key) if species_key else None + + return (type_label or None), (genus or None), (species or None) + + def _mibig_name_and_url(props: dict, versions: dict[str, str]) -> tuple[str | None, str | None]: accession = props.get("mibig_accession") version = versions.get(accession) if accession else None @@ -141,6 +156,11 @@ def run( ) if source == "mibig": _apply_mibig_annotations(db, result.submission.inchikey, props, annotations) + else: + type_label, genus, species = _npatlas_phylogeny(props) + db.add_phylogeny_annotation( + result.submission.inchikey, type_label=type_label, genus=genus, species=species + ) added += 1 compounds += 1 diff --git a/gui/src/client/src/components/workspace/tabs/browse/WorkspaceBrowse.tsx b/gui/src/client/src/components/workspace/tabs/browse/WorkspaceBrowse.tsx index 1948dc7..d177791 100644 --- a/gui/src/client/src/components/workspace/tabs/browse/WorkspaceBrowse.tsx +++ b/gui/src/client/src/components/workspace/tabs/browse/WorkspaceBrowse.tsx @@ -20,7 +20,8 @@ import TextField from "@mui/material/TextField"; import Tooltip from "@mui/material/Tooltip"; import Typography from "@mui/material/Typography"; import { useNotifications } from "../../../NotificationProvider"; -import { getAnnotationTerms, getBrowseEntries, browseEntriesExportUrl } from "../../../../features/browse/api"; +import { getAnnotationTerms, getBrowseEntries, browseEntriesExportUrl, MAX_BROWSE_ENTRIES } from "../../../../features/browse/api"; +import { groupSourcesByDatabase } from "../../../../features/sources"; import type { AnnotationTerm, BrowseEntry } from "../../../../features/browse/types"; import type { EntryType } from "../../../../features/enrichment/types"; @@ -36,7 +37,10 @@ export const WorkspaceBrowse: React.FC = () => { const [termId, setTermId] = React.useState(ALL_TERM); const [entries, setEntries] = React.useState([]); - const [loading, setLoading] = React.useState(true); + const [totalCount, setTotalCount] = React.useState(0); + const [truncated, setTruncated] = React.useState(false); + const [hasSearched, setHasSearched] = React.useState(false); + const [loading, setLoading] = React.useState(false); const [error, setError] = React.useState(null); const [page, setPage] = React.useState(0); @@ -54,27 +58,24 @@ export const WorkspaceBrowse: React.FC = () => { // eslint-disable-next-line react-hooks/exhaustive-deps }, []); - React.useEffect(() => { - const controller = new AbortController(); + const handleBrowse = async (event?: React.FormEvent) => { + event?.preventDefault(); + setLoading(true); setError(null); - - getBrowseEntries(entryType, termId === ALL_TERM ? null : termId, controller.signal) - .then((resp) => { - setEntries(resp.entries); - setPage(0); - }) - .catch((err) => { - if (controller.signal.aborted) return; - setError(err instanceof Error ? err.message : String(err)); - }) - .finally(() => { - if (controller.signal.aborted) return; - setLoading(false); - }); - - return () => controller.abort(); - }, [entryType, termId]); + try { + const resp = await getBrowseEntries(entryType, termId === ALL_TERM ? null : termId); + setEntries(resp.entries); + setTotalCount(resp.totalCount); + setTruncated(resp.truncated); + setPage(0); + setHasSearched(true); + } catch (err) { + setError(err instanceof Error ? err.message : String(err)); + } finally { + setLoading(false); + } + }; const categories = React.useMemo( () => Array.from(new Set(terms.map((t) => t.category))).sort(), @@ -102,68 +103,84 @@ export const WorkspaceBrowse: React.FC = () => { Browse annotations - Browse database entries and their annotations, and download the filtered set as a TSV file - (compound SMILES/InChIKey, or BGC id, alongside phylogeny and chemical class annotations). + Browse database entries and their annotations, and download the filtered set as a TSV file. - - setEntryType(e.target.value as EntryType | "all")} - > - All - Compounds - Gene clusters (BGCs) - - - handleCategoryChange(e.target.value)} - > - All categories - {categories.map((c) => ( - {c} - ))} - - - setTermId(e.target.value)} - > - Any - {termOptions.map((t) => ( - - {t.rank ? `${t.rank}: ${t.label}` : t.label} - - ))} - - - - - - - - + + + setEntryType(e.target.value as EntryType | "all")} + > + All + Compounds + Gene clusters (BGCs) + + + handleCategoryChange(e.target.value)} + > + All categories + {categories.map((c) => ( + {c} + ))} + + + setTermId(e.target.value)} + > + Any + {termOptions.map((t) => ( + + {t.rank ? `${t.rank}: ${t.label}` : t.label} + + ))} + + + + + + + + + + + + + @@ -173,17 +190,20 @@ export const WorkspaceBrowse: React.FC = () => { )} - {loading && ( - - - + {hasSearched && truncated && !loading && !error && ( + + {totalCount.toLocaleString()} entries match these filters. Showing (and downloading) only the + first {MAX_BROWSE_ENTRIES.toLocaleString()}. Narrow the filters to see/export the rest. + )} - {!loading && !error && ( + {hasSearched && !loading && !error && ( - {entries.length.toLocaleString()} matching entries + {truncated + ? `${entries.length.toLocaleString()} of ${totalCount.toLocaleString()} matching entries shown` + : `${entries.length.toLocaleString()} matching entries`} @@ -207,8 +227,17 @@ export const WorkspaceBrowse: React.FC = () => { {entry.type === "compound" ? entry.smiles ?? "-" : entry.id} - {entry.sources.map((s) => ( - + {groupSourcesByDatabase(entry.sources).map((g) => ( + s.name).join(", ")} + > + 1 ? `${g.databaseName} ×${g.count}` : g.databaseName} + size="small" + sx={{ mr: 0.5 }} + /> + ))} diff --git a/gui/src/client/src/components/workspace/tabs/enrichment/WorkspaceEnrichment.tsx b/gui/src/client/src/components/workspace/tabs/enrichment/WorkspaceEnrichment.tsx index 13f404a..1b2dfb3 100644 --- a/gui/src/client/src/components/workspace/tabs/enrichment/WorkspaceEnrichment.tsx +++ b/gui/src/client/src/components/workspace/tabs/enrichment/WorkspaceEnrichment.tsx @@ -16,9 +16,11 @@ import TableContainer from "@mui/material/TableContainer"; import TableHead from "@mui/material/TableHead"; import TableRow from "@mui/material/TableRow"; import TextField from "@mui/material/TextField"; +import Tooltip from "@mui/material/Tooltip"; import Typography from "@mui/material/Typography"; import { useNotifications } from "../../../NotificationProvider"; import { searchEntries, runEnrichmentAnalysis, MAX_ENTRY_SEARCH_RESULTS, MAX_ENRICHMENT_SELECTION } from "../../../../features/enrichment/api"; +import { groupSourcesByDatabase } from "../../../../features/sources"; import type { EntryType, EnrichmentResult, SearchEntry } from "../../../../features/enrichment/types"; const Q_VALUE_SIGNIFICANT = 0.05; @@ -131,6 +133,9 @@ export const WorkspaceEnrichment: React.FC = () => { Compounds Gene clusters (BGCs) + + + @@ -183,8 +188,14 @@ export const WorkspaceEnrichment: React.FC = () => { {entry.name} {entry.type === "bgc" ? "BGC" : "Compound"} - {entry.sources.map((s) => ( - + {groupSourcesByDatabase(entry.sources).map((g) => ( + s.name).join(", ")}> + 1 ? `${g.databaseName} ×${g.count}` : g.databaseName} + size="small" + sx={{ mr: 0.5 }} + /> + ))} diff --git a/gui/src/client/src/features/browse/api.ts b/gui/src/client/src/features/browse/api.ts index 9392868..a17c9c4 100644 --- a/gui/src/client/src/features/browse/api.ts +++ b/gui/src/client/src/features/browse/api.ts @@ -2,6 +2,11 @@ import { getJson } from "../http"; import { AnnotationTermsRespSchema, BrowseEntriesRespSchema } from "./types"; import type { EntryType } from "../enrichment/types"; +// Mirrors MAX_BROWSE_ENTRIES in gui/src/server/routes/browse.py -- both browsing and +// TSV export are capped there so an unfiltered query over a huge database can't pull +// the whole table into memory; kept here only for the frontend's warning copy. +export const MAX_BROWSE_ENTRIES = 1000; + function buildParams(entryType: EntryType | "all", termId: string | null): URLSearchParams { const params = new URLSearchParams(); if (entryType !== "all") params.set("type", entryType); diff --git a/gui/src/client/src/features/browse/types.ts b/gui/src/client/src/features/browse/types.ts index 8254f5e..94aad2c 100644 --- a/gui/src/client/src/features/browse/types.ts +++ b/gui/src/client/src/features/browse/types.ts @@ -30,5 +30,7 @@ export type BrowseEntry = z.output; export const BrowseEntriesRespSchema = z.object({ entries: z.array(BrowseEntrySchema), + totalCount: z.number().int().nonnegative(), + truncated: z.boolean(), }); export type BrowseEntriesResp = z.output; diff --git a/gui/src/client/src/features/sources.ts b/gui/src/client/src/features/sources.ts new file mode 100644 index 0000000..096473f --- /dev/null +++ b/gui/src/client/src/features/sources.ts @@ -0,0 +1,28 @@ +export type EntrySourceLike = { + name: string; + databaseName: string; + url: string | null; +}; + +export type GroupedSource = { + databaseName: string; + count: number; + items: EntrySourceLike[]; +}; + +/** Group an entry's sources by database, so e.g. two differently-worded MIBiG + * records for the same compound render as one "MIBiG ×2" chip instead of two + * identical-looking "MIBiG" chips. */ +export function groupSourcesByDatabase(sources: EntrySourceLike[]): GroupedSource[] { + const byDatabase = new Map(); + for (const source of sources) { + const items = byDatabase.get(source.databaseName) ?? []; + items.push(source); + byDatabase.set(source.databaseName, items); + } + return Array.from(byDatabase.entries()).map(([databaseName, items]) => ({ + databaseName, + count: items.length, + items, + })); +} diff --git a/gui/src/server/routes/browse.py b/gui/src/server/routes/browse.py index 471c7df..fafe20c 100644 --- a/gui/src/server/routes/browse.py +++ b/gui/src/server/routes/browse.py @@ -15,6 +15,14 @@ blp_annotation_terms = Blueprint("annotation_terms", __name__) blp_export_entries = Blueprint("export_entries", __name__) +# Both the Browse tab's table and its TSV export fetch the whole matching set (see +# RetroMolDuckDB.browse_entries's docstring) -- with no cap, an unfiltered browse/export +# over a database with e.g. a million entries would materialize the whole table in memory +# and ship a multi-hundred-MB response. Cap both at the same size and tell the caller +# (via totalCount) when a result was truncated, so the frontend can warn instead of +# silently handing back a partial set. +MAX_BROWSE_ENTRIES = 1000 + TSV_COLUMNS = [ "id", "type", "name", "smiles", "inchikey", "url", "sources", "phylogeny_type", "genus", "species", "chemical_classes", @@ -104,11 +112,16 @@ def browse_entries() -> tuple[Response, int]: try: with open_retromol_db() as db: - entries = db.browse_entries(entry_type=entry_type, term_id=term_id) + total_count = db.count_browse_entries(entry_type=entry_type, term_id=term_id) + entries = db.browse_entries(entry_type=entry_type, term_id=term_id, limit=MAX_BROWSE_ENTRIES) except Exception as e: return jsonify({"error": str(e)}), 503 - return jsonify({"entries": [_browse_entry_payload(e) for e in entries]}), 200 + return jsonify({ + "entries": [_browse_entry_payload(e) for e in entries], + "totalCount": total_count, + "truncated": total_count > len(entries), + }), 200 @blp_export_entries.get("/api/browseEntries.tsv") @@ -127,7 +140,7 @@ def export_entries_tsv() -> Response: try: with open_retromol_db() as db: - entries = db.browse_entries(entry_type=entry_type, term_id=term_id) + entries = db.browse_entries(entry_type=entry_type, term_id=term_id, limit=MAX_BROWSE_ENTRIES) except Exception as e: return jsonify({"error": str(e)}), 503 diff --git a/src/retromol_database/duckdb.py b/src/retromol_database/duckdb.py index bf65d45..dd179d8 100644 --- a/src/retromol_database/duckdb.py +++ b/src/retromol_database/duckdb.py @@ -436,13 +436,9 @@ def list_annotation_terms(self, category: str | None = None) -> list[AnnotationT for r in rows ] - def browse_entries( - self, *, entry_type: str | None = None, term_id: str | None = None - ) -> list[BrowseEntry]: - """Every entry (optionally filtered by type and/or a single annotation term id -- - matching phylogeny at any rank or a chemical class), with its sources and - annotations attached. Used for both the Browse tab's table and its TSV export -- - both need the whole matching set, not a similarity-ranked slice.""" + def _browse_where_clause( + self, *, entry_type: str | None, term_id: str | None + ) -> tuple[str, list[object]]: where_sql = [] params: list[object] = [] @@ -456,6 +452,38 @@ def browse_entries( params.append(term_id) where_clause = f"WHERE {' AND '.join(where_sql)}" if where_sql else "" + return where_clause, params + + def count_browse_entries(self, *, entry_type: str | None = None, term_id: str | None = None) -> int: + """Total number of entries a `browse_entries(...)` call with the same filters would + match, regardless of its `limit` -- lets callers warn when a result set was truncated.""" + where_clause, params = self._browse_where_clause(entry_type=entry_type, term_id=term_id) + return int( + self.con.execute(f"SELECT count(*) FROM entries e {where_clause}", params).fetchone()[0] + ) + + def browse_entries( + self, *, entry_type: str | None = None, term_id: str | None = None, limit: int | None = None + ) -> list[BrowseEntry]: + """Every entry (optionally filtered by type and/or a single annotation term id -- + matching phylogeny at any rank or a chemical class), with its sources and + annotations attached. Used for both the Browse tab's table and its TSV export. + + `limit` bounds how many rows are fetched -- callers should always pass one (see + MAX_BROWSE_ENTRIES in routes/browse.py) since an unfiltered call over a + multi-million-row database would otherwise materialize the whole table in memory. + Use `count_browse_entries` with the same filters to tell whether the result was + truncated. Results are ordered by id, so `limit` always returns the same prefix. + """ + if limit is not None and limit < 1: + raise ValueError("limit must be >= 1") + + where_clause, params = self._browse_where_clause(entry_type=entry_type, term_id=term_id) + + limit_clause = "" + if limit is not None: + limit_clause = "LIMIT ?" + params = [*params, limit] rows = self.con.execute( f""" @@ -463,6 +491,7 @@ def browse_entries( FROM entries e {where_clause} ORDER BY e.id + {limit_clause} """, params, ).fetchall() From 510143e3759d2bf73e82d7407b8fa85dfecc0e23 Mon Sep 17 00:00:00 2001 From: David Meijer Date: Wed, 26 Aug 2026 16:19:24 -0400 Subject: [PATCH 03/11] UPD: remove Browse tab --- gui/src/client/src/components/MenuContent.tsx | 6 - .../src/components/workspace/Workspace.tsx | 2 - .../workspace/tabs/browse/WorkspaceBrowse.tsx | 276 ------------------ gui/src/client/src/features/browse/api.ts | 36 --- gui/src/client/src/features/browse/types.ts | 36 --- gui/src/server/app.py | 4 - gui/src/server/routes/browse.py | 157 ---------- src/retromol_database/duckdb.py | 149 ---------- 8 files changed, 666 deletions(-) delete mode 100644 gui/src/client/src/components/workspace/tabs/browse/WorkspaceBrowse.tsx delete mode 100644 gui/src/client/src/features/browse/api.ts delete mode 100644 gui/src/client/src/features/browse/types.ts delete mode 100644 gui/src/server/routes/browse.py diff --git a/gui/src/client/src/components/MenuContent.tsx b/gui/src/client/src/components/MenuContent.tsx index b2d37f9..725483e 100644 --- a/gui/src/client/src/components/MenuContent.tsx +++ b/gui/src/client/src/components/MenuContent.tsx @@ -11,7 +11,6 @@ import HomeRoundedIcon from "@mui/icons-material/HomeRounded"; import UploadFileIcon from "@mui/icons-material/UploadFile"; import RuleIcon from "@mui/icons-material/Rule"; import AutoFixHighIcon from "@mui/icons-material/AutoFixHigh"; -import TableRowsIcon from "@mui/icons-material/TableRows"; import { useNavigate, useLocation } from "react-router-dom"; const mainListItems = [ @@ -35,11 +34,6 @@ const mainListItems = [ icon: , to: `/dashboard/enrichment` }, - { - text: "Browse", - icon: , - to: `/dashboard/browse` - }, { text: "Generate", icon: , diff --git a/gui/src/client/src/components/workspace/Workspace.tsx b/gui/src/client/src/components/workspace/Workspace.tsx index 5ca6309..499b292 100644 --- a/gui/src/client/src/components/workspace/Workspace.tsx +++ b/gui/src/client/src/components/workspace/Workspace.tsx @@ -17,7 +17,6 @@ import { WorkspaceDiscovery } from "./WorkspaceDiscovery"; import { WorkspaceRules } from "./WorkspaceRules"; import { WorkspaceGenerate } from "./WorkspaceGenerate"; import { WorkspaceEnrichment } from "./tabs/enrichment/WorkspaceEnrichment"; -import { WorkspaceBrowse } from "./tabs/browse/WorkspaceBrowse"; export const Workspace: React.FC = () => { const { showOverlay, hideOverlay } = useOverlay(); @@ -174,7 +173,6 @@ export const Workspace: React.FC = () => { } /> } /> } /> - } /> } /> } /> diff --git a/gui/src/client/src/components/workspace/tabs/browse/WorkspaceBrowse.tsx b/gui/src/client/src/components/workspace/tabs/browse/WorkspaceBrowse.tsx deleted file mode 100644 index d177791..0000000 --- a/gui/src/client/src/components/workspace/tabs/browse/WorkspaceBrowse.tsx +++ /dev/null @@ -1,276 +0,0 @@ -import React from "react"; -import Alert from "@mui/material/Alert"; -import Box from "@mui/material/Box"; -import Button from "@mui/material/Button"; -import Card from "@mui/material/Card"; -import CardContent from "@mui/material/CardContent"; -import Chip from "@mui/material/Chip"; -import CircularProgress from "@mui/material/CircularProgress"; -import DownloadIcon from "@mui/icons-material/Download"; -import MenuItem from "@mui/material/MenuItem"; -import Stack from "@mui/material/Stack"; -import Table from "@mui/material/Table"; -import TableBody from "@mui/material/TableBody"; -import TableCell from "@mui/material/TableCell"; -import TableContainer from "@mui/material/TableContainer"; -import TableHead from "@mui/material/TableHead"; -import TablePagination from "@mui/material/TablePagination"; -import TableRow from "@mui/material/TableRow"; -import TextField from "@mui/material/TextField"; -import Tooltip from "@mui/material/Tooltip"; -import Typography from "@mui/material/Typography"; -import { useNotifications } from "../../../NotificationProvider"; -import { getAnnotationTerms, getBrowseEntries, browseEntriesExportUrl, MAX_BROWSE_ENTRIES } from "../../../../features/browse/api"; -import { groupSourcesByDatabase } from "../../../../features/sources"; -import type { AnnotationTerm, BrowseEntry } from "../../../../features/browse/types"; -import type { EntryType } from "../../../../features/enrichment/types"; - -const ALL_CATEGORY = "all"; -const ALL_TERM = "all"; - -export const WorkspaceBrowse: React.FC = () => { - const { pushNotification } = useNotifications(); - - const [entryType, setEntryType] = React.useState("all"); - const [terms, setTerms] = React.useState([]); - const [category, setCategory] = React.useState(ALL_CATEGORY); - const [termId, setTermId] = React.useState(ALL_TERM); - - const [entries, setEntries] = React.useState([]); - const [totalCount, setTotalCount] = React.useState(0); - const [truncated, setTruncated] = React.useState(false); - const [hasSearched, setHasSearched] = React.useState(false); - const [loading, setLoading] = React.useState(false); - const [error, setError] = React.useState(null); - - const [page, setPage] = React.useState(0); - const [rowsPerPage, setRowsPerPage] = React.useState(25); - - React.useEffect(() => { - const controller = new AbortController(); - getAnnotationTerms(undefined, controller.signal) - .then((resp) => setTerms(resp.terms)) - .catch((err) => { - if (controller.signal.aborted) return; - pushNotification(`Failed to load annotation terms: ${err instanceof Error ? err.message : String(err)}`, "error"); - }); - return () => controller.abort(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, []); - - const handleBrowse = async (event?: React.FormEvent) => { - event?.preventDefault(); - - setLoading(true); - setError(null); - try { - const resp = await getBrowseEntries(entryType, termId === ALL_TERM ? null : termId); - setEntries(resp.entries); - setTotalCount(resp.totalCount); - setTruncated(resp.truncated); - setPage(0); - setHasSearched(true); - } catch (err) { - setError(err instanceof Error ? err.message : String(err)); - } finally { - setLoading(false); - } - }; - - const categories = React.useMemo( - () => Array.from(new Set(terms.map((t) => t.category))).sort(), - [terms] - ); - - const termOptions = React.useMemo( - () => terms.filter((t) => category === ALL_CATEGORY || t.category === category), - [terms, category] - ); - - const handleCategoryChange = (value: string) => { - setCategory(value); - setTermId(ALL_TERM); // term choices depend on category, so reset - }; - - const paginatedEntries = entries.slice(page * rowsPerPage, page * rowsPerPage + rowsPerPage); - const exportUrl = browseEntriesExportUrl(entryType, termId === ALL_TERM ? null : termId); - - return ( - - - - - Browse annotations - - - Browse database entries and their annotations, and download the filtered set as a TSV file. - - - - - setEntryType(e.target.value as EntryType | "all")} - > - All - Compounds - Gene clusters (BGCs) - - - handleCategoryChange(e.target.value)} - > - All categories - {categories.map((c) => ( - {c} - ))} - - - setTermId(e.target.value)} - > - Any - {termOptions.map((t) => ( - - {t.rank ? `${t.rank}: ${t.label}` : t.label} - - ))} - - - - - - - - - - - - - - - - - {error && ( - - Couldn't load entries: {error} - - )} - - {hasSearched && truncated && !loading && !error && ( - - {totalCount.toLocaleString()} entries match these filters. Showing (and downloading) only the - first {MAX_BROWSE_ENTRIES.toLocaleString()}. Narrow the filters to see/export the rest. - - )} - - {hasSearched && !loading && !error && ( - - - - {truncated - ? `${entries.length.toLocaleString()} of ${totalCount.toLocaleString()} matching entries shown` - : `${entries.length.toLocaleString()} matching entries`} - - - - - - - Name - Type - SMILES / InChIKey - Sources - Phylogeny - Chemical class - - - - {paginatedEntries.map((entry) => ( - - {entry.name} - {entry.type === "bgc" ? "BGC" : "Compound"} - - {entry.type === "compound" ? entry.smiles ?? "-" : entry.id} - - - {groupSourcesByDatabase(entry.sources).map((g) => ( - s.name).join(", ")} - > - 1 ? `${g.databaseName} ×${g.count}` : g.databaseName} - size="small" - sx={{ mr: 0.5 }} - /> - - ))} - - - {[entry.phylogenyType, entry.genus, entry.species].filter(Boolean).join(" › ") || "-"} - - - {entry.chemicalClasses.length > 0 - ? entry.chemicalClasses.map((c) => ( - - )) - : "-"} - - - ))} - -
- - - setPage(newPage)} - rowsPerPage={rowsPerPage} - onRowsPerPageChange={(e) => { - setRowsPerPage(parseInt(e.target.value, 10)); - setPage(0); - }} - rowsPerPageOptions={[10, 25, 50, 100]} - /> - - - )} - - ); -}; diff --git a/gui/src/client/src/features/browse/api.ts b/gui/src/client/src/features/browse/api.ts deleted file mode 100644 index a17c9c4..0000000 --- a/gui/src/client/src/features/browse/api.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { getJson } from "../http"; -import { AnnotationTermsRespSchema, BrowseEntriesRespSchema } from "./types"; -import type { EntryType } from "../enrichment/types"; - -// Mirrors MAX_BROWSE_ENTRIES in gui/src/server/routes/browse.py -- both browsing and -// TSV export are capped there so an unfiltered query over a huge database can't pull -// the whole table into memory; kept here only for the frontend's warning copy. -export const MAX_BROWSE_ENTRIES = 1000; - -function buildParams(entryType: EntryType | "all", termId: string | null): URLSearchParams { - const params = new URLSearchParams(); - if (entryType !== "all") params.set("type", entryType); - if (termId) params.set("termId", termId); - return params; -} - -export async function getAnnotationTerms(category?: string, signal?: AbortSignal) { - const params = new URLSearchParams(); - if (category) params.set("category", category); - const qs = params.toString(); - return getJson(`/api/annotationTerms${qs ? `?${qs}` : ""}`, AnnotationTermsRespSchema, signal); -} - -export async function getBrowseEntries( - entryType: EntryType | "all" = "all", - termId: string | null = null, - signal?: AbortSignal -) { - const params = buildParams(entryType, termId); - return getJson(`/api/browseEntries?${params.toString()}`, BrowseEntriesRespSchema, signal); -} - -export function browseEntriesExportUrl(entryType: EntryType | "all", termId: string | null): string { - const params = buildParams(entryType, termId); - return `/api/browseEntries.tsv?${params.toString()}`; -} diff --git a/gui/src/client/src/features/browse/types.ts b/gui/src/client/src/features/browse/types.ts deleted file mode 100644 index 94aad2c..0000000 --- a/gui/src/client/src/features/browse/types.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { z } from "zod"; -import { EntryTypeSchema, EntrySourceSchema } from "../enrichment/types"; - -export const AnnotationTermSchema = z.object({ - id: z.string(), - category: z.string(), - rank: z.string().nullable(), - label: z.string(), - parentId: z.string().nullable(), -}); -export type AnnotationTerm = z.output; - -export const AnnotationTermsRespSchema = z.object({ - terms: z.array(AnnotationTermSchema), -}); - -export const BrowseEntrySchema = z.object({ - id: z.string(), - type: EntryTypeSchema, - name: z.string(), - url: z.string().nullable(), - smiles: z.string().nullable(), - sources: z.array(EntrySourceSchema), - phylogenyType: z.string().nullable(), - genus: z.string().nullable(), - species: z.string().nullable(), - chemicalClasses: z.array(z.string()), -}); -export type BrowseEntry = z.output; - -export const BrowseEntriesRespSchema = z.object({ - entries: z.array(BrowseEntrySchema), - totalCount: z.number().int().nonnegative(), - truncated: z.boolean(), -}); -export type BrowseEntriesResp = z.output; diff --git a/gui/src/server/app.py b/gui/src/server/app.py index 2f9ae23..4011f08 100644 --- a/gui/src/server/app.py +++ b/gui/src/server/app.py @@ -41,7 +41,6 @@ from routes.rate_limit import limiter, RATE_LIMIT_REJECTIONS from routes.rules import blp_rule_set, blp_generate_backbone from routes.enrichment import blp_entry_search, blp_enrichment_analysis -from routes.browse import blp_browse_entries, blp_annotation_terms, blp_export_entries # Initialize the Flask app @@ -259,9 +258,6 @@ def ready() -> tuple[dict[str, str], int]: app.register_blueprint(blp_generate_backbone) app.register_blueprint(blp_entry_search) app.register_blueprint(blp_enrichment_analysis) -app.register_blueprint(blp_browse_entries) -app.register_blueprint(blp_annotation_terms) -app.register_blueprint(blp_export_entries) # The two rate-limit tiers on top of the app-wide default (see routes/rate_limit.py) # are applied as @limiter.limit(...) decorators directly on each route function, in diff --git a/gui/src/server/routes/browse.py b/gui/src/server/routes/browse.py deleted file mode 100644 index fafe20c..0000000 --- a/gui/src/server/routes/browse.py +++ /dev/null @@ -1,157 +0,0 @@ -"""Browse tab: list/filter database entries together with their annotations, and -export the same (filtered) set as a TSV file -- compound smiles/inchikey or BGC -accession/id, alongside whatever phylogeny/chemical_class annotations are on record. -""" - -import csv -import io - -from flask import Blueprint, Response, jsonify, request - -from retromol_database.duckdb import BrowseEntry, ENTRY_TYPES -from routes.database import open_retromol_db - -blp_browse_entries = Blueprint("browse_entries", __name__) -blp_annotation_terms = Blueprint("annotation_terms", __name__) -blp_export_entries = Blueprint("export_entries", __name__) - -# Both the Browse tab's table and its TSV export fetch the whole matching set (see -# RetroMolDuckDB.browse_entries's docstring) -- with no cap, an unfiltered browse/export -# over a database with e.g. a million entries would materialize the whole table in memory -# and ship a multi-hundred-MB response. Cap both at the same size and tell the caller -# (via totalCount) when a result was truncated, so the frontend can warn instead of -# silently handing back a partial set. -MAX_BROWSE_ENTRIES = 1000 - -TSV_COLUMNS = [ - "id", "type", "name", "smiles", "inchikey", "url", "sources", - "phylogeny_type", "genus", "species", "chemical_classes", -] - - -def _entry_type_param() -> tuple[str | None, str | None]: - """Read+validate the shared `type` query param. Returns (entry_type, error).""" - entry_type = request.args.get("type") - if entry_type in (None, "", "all"): - return None, None - if entry_type not in ENTRY_TYPES: - return None, f"type must be one of {ENTRY_TYPES} or 'all'" - return entry_type, None - - -def _browse_entry_payload(entry: BrowseEntry) -> dict: - return { - "id": entry.id, - "type": entry.type, - "name": entry.name, - "url": entry.url, - "smiles": entry.raw if entry.type == "compound" else None, - "sources": [ - {"name": s.name, "databaseName": s.database_name, "url": s.url} for s in entry.sources - ], - "phylogenyType": entry.phylogeny_type, - "genus": entry.genus, - "species": entry.species, - "chemicalClasses": entry.chemical_classes, - } - - -def _tsv_row(entry: BrowseEntry) -> list[str]: - return [ - entry.id, - entry.type, - entry.name, - entry.raw if entry.type == "compound" else "", - entry.id if entry.type == "compound" else "", # entries are keyed by InChIKey for compounds - entry.url or "", - ";".join(f"{s.database_name}:{s.name}" for s in entry.sources), - entry.phylogeny_type or "", - entry.genus or "", - entry.species or "", - ";".join(entry.chemical_classes), - ] - - -@blp_annotation_terms.get("/api/annotationTerms") -def annotation_terms() -> tuple[Response, int]: - """ - Every annotation term in the database, for the Browse tab's filter dropdown. - - :return: a tuple containing the terms payload and an HTTP status code - """ - category = request.args.get("category") or None - - try: - with open_retromol_db() as db: - terms = db.list_annotation_terms(category=category) - except Exception as e: - return jsonify({"error": str(e)}), 503 - - return jsonify({ - "terms": [ - {"id": t.id, "category": t.category, "rank": t.rank, "label": t.label, "parentId": t.parent_id} - for t in terms - ] - }), 200 - - -@blp_browse_entries.get("/api/browseEntries") -def browse_entries() -> tuple[Response, int]: - """ - List (optionally filtered) database entries with their annotations, for the - Browse tab's table. Not paginated server-side -- see module docstring; the - frontend paginates client-side over the full filtered set. - - :return: a tuple containing the entries payload and an HTTP status code - """ - entry_type, error = _entry_type_param() - if error: - return jsonify({"error": error}), 400 - - term_id = request.args.get("termId") or None - - try: - with open_retromol_db() as db: - total_count = db.count_browse_entries(entry_type=entry_type, term_id=term_id) - entries = db.browse_entries(entry_type=entry_type, term_id=term_id, limit=MAX_BROWSE_ENTRIES) - except Exception as e: - return jsonify({"error": str(e)}), 503 - - return jsonify({ - "entries": [_browse_entry_payload(e) for e in entries], - "totalCount": total_count, - "truncated": total_count > len(entries), - }), 200 - - -@blp_export_entries.get("/api/browseEntries.tsv") -def export_entries_tsv() -> Response: - """ - Export the same (optionally filtered) set `browseEntries` returns as a TSV - file download. - - :return: a TSV file response - """ - entry_type, error = _entry_type_param() - if error: - return jsonify({"error": error}), 400 - - term_id = request.args.get("termId") or None - - try: - with open_retromol_db() as db: - entries = db.browse_entries(entry_type=entry_type, term_id=term_id, limit=MAX_BROWSE_ENTRIES) - except Exception as e: - return jsonify({"error": str(e)}), 503 - - buf = io.StringIO() - writer = csv.writer(buf, delimiter="\t", lineterminator="\n") - writer.writerow(TSV_COLUMNS) - for entry in entries: - writer.writerow(_tsv_row(entry)) - - return Response( - buf.getvalue(), - mimetype="text/tab-separated-values", - headers={"Content-Disposition": 'attachment; filename="retromol_entries.tsv"'}, - ) diff --git a/src/retromol_database/duckdb.py b/src/retromol_database/duckdb.py index dd179d8..b16c3c7 100644 --- a/src/retromol_database/duckdb.py +++ b/src/retromol_database/duckdb.py @@ -70,20 +70,6 @@ class AnnotationTerm: parent_id: str | None -@dataclass(frozen=True) -class BrowseEntry: - id: str - type: EntryType - name: str - url: str | None - raw: str | None - sources: list[EntrySource] - phylogeny_type: str | None - genus: str | None - species: str | None - chemical_classes: list[str] - - @dataclass(frozen=True) class AnnotationStats: with_annotation_count: int @@ -413,141 +399,6 @@ def annotation_terms_by_ids(self, term_ids: Sequence[str]) -> dict[str, Annotati for row in rows } - def list_annotation_terms(self, category: str | None = None) -> list[AnnotationTerm]: - """Every annotation term in the database (not just ones with entries counted), - for populating a filter dropdown in the Browse tab.""" - where_sql = "" - params: list[object] = [] - if category is not None: - where_sql = "WHERE category = ?" - params.append(category) - - rows = self.con.execute( - f""" - SELECT id, category, rank, label, parent_id - FROM annotation_terms - {where_sql} - ORDER BY category, rank NULLS FIRST, label - """, - params, - ).fetchall() - return [ - AnnotationTerm(id=str(r[0]), category=str(r[1]), rank=r[2], label=str(r[3]), parent_id=r[4]) - for r in rows - ] - - def _browse_where_clause( - self, *, entry_type: str | None, term_id: str | None - ) -> tuple[str, list[object]]: - where_sql = [] - params: list[object] = [] - - if entry_type is not None: - entry_type = _normalize_entry_type(entry_type) - where_sql.append("e.type = ?") - params.append(entry_type) - - if term_id is not None: - where_sql.append("e.id IN (SELECT entry_id FROM entry_annotations WHERE term_id = ?)") - params.append(term_id) - - where_clause = f"WHERE {' AND '.join(where_sql)}" if where_sql else "" - return where_clause, params - - def count_browse_entries(self, *, entry_type: str | None = None, term_id: str | None = None) -> int: - """Total number of entries a `browse_entries(...)` call with the same filters would - match, regardless of its `limit` -- lets callers warn when a result set was truncated.""" - where_clause, params = self._browse_where_clause(entry_type=entry_type, term_id=term_id) - return int( - self.con.execute(f"SELECT count(*) FROM entries e {where_clause}", params).fetchone()[0] - ) - - def browse_entries( - self, *, entry_type: str | None = None, term_id: str | None = None, limit: int | None = None - ) -> list[BrowseEntry]: - """Every entry (optionally filtered by type and/or a single annotation term id -- - matching phylogeny at any rank or a chemical class), with its sources and - annotations attached. Used for both the Browse tab's table and its TSV export. - - `limit` bounds how many rows are fetched -- callers should always pass one (see - MAX_BROWSE_ENTRIES in routes/browse.py) since an unfiltered call over a - multi-million-row database would otherwise materialize the whole table in memory. - Use `count_browse_entries` with the same filters to tell whether the result was - truncated. Results are ordered by id, so `limit` always returns the same prefix. - """ - if limit is not None and limit < 1: - raise ValueError("limit must be >= 1") - - where_clause, params = self._browse_where_clause(entry_type=entry_type, term_id=term_id) - - limit_clause = "" - if limit is not None: - limit_clause = "LIMIT ?" - params = [*params, limit] - - rows = self.con.execute( - f""" - SELECT e.id, e.raw, e.type, e.primary_sequence, e.fingerprint - FROM entries e - {where_clause} - ORDER BY e.id - {limit_clause} - """, - params, - ).fetchall() - - entry_ids = [str(row[0]) for row in rows] - sources_by_id = self._sources_for_entry_ids(entry_ids) - - annotation_rows = ( - self.con.execute( - """ - SELECT ea.entry_id, t.category, t.rank, t.label - FROM entry_annotations ea - JOIN annotation_terms t ON t.id = ea.term_id - WHERE ea.entry_id IN (SELECT UNNEST(?)) - """, - [entry_ids], - ).fetchall() - if entry_ids - else [] - ) - - phylogeny_type_by_id: dict[str, str] = {} - genus_by_id: dict[str, str] = {} - species_by_id: dict[str, str] = {} - chemical_classes_by_id: dict[str, list[str]] = {} - for entry_id, category, rank, label in annotation_rows: - entry_id = str(entry_id) - if category == "phylogeny" and rank == "type": - phylogeny_type_by_id[entry_id] = str(label) - elif category == "phylogeny" and rank == "genus": - genus_by_id[entry_id] = str(label) - elif category == "phylogeny" and rank == "species": - species_by_id[entry_id] = str(label) - elif category == "chemical_class": - chemical_classes_by_id.setdefault(entry_id, []).append(str(label)) - - out: list[BrowseEntry] = [] - for row in rows: - entry_id = str(row[0]) - entry = _entry_from_row(row, sources_by_id.get(entry_id, [])) - out.append( - BrowseEntry( - id=entry.id, - type=entry.type, - name=entry.name, - url=entry.url, - raw=entry.raw, - sources=entry.sources, - phylogeny_type=phylogeny_type_by_id.get(entry_id), - genus=genus_by_id.get(entry_id), - species=species_by_id.get(entry_id), - chemical_classes=chemical_classes_by_id.get(entry_id, []), - ) - ) - return out - def search_entries( self, query: str, *, entry_type: str | None = None, limit: int = 100 ) -> list[Entry]: From 41c833b649e1c2fa1c75859d7f6f7d8cc3d0ad02 Mon Sep 17 00:00:00 2001 From: David Meijer Date: Wed, 26 Aug 2026 17:44:50 -0400 Subject: [PATCH 04/11] UPD: parsing and storing annotations --- database/Snakemake | 240 +++++++++-- database/config.yaml | 33 ++ database/scripts/annotate_chebi.py | 69 +++ database/scripts/annotate_chembl.py | 78 ++++ database/scripts/annotate_npclassifier.py | 162 +++++++ database/scripts/chebi.py | 222 ++++++++++ database/scripts/chembl.py | 141 ++++++ database/scripts/load_bgcs.py | 32 +- database/scripts/load_compounds.py | 44 +- database/scripts/npclassifier.py | 68 +++ database/scripts/taxonomy.py | 218 ++++++++++ .../components/workspace/AlignmentGrid.tsx | 81 ++++ .../components/workspace/WorkspaceHome.tsx | 162 +++++-- gui/src/client/src/features/database/api.ts | 14 +- gui/src/client/src/features/database/types.ts | 27 +- gui/src/server/app.py | 2 + gui/src/server/routes/entry_annotations.py | 71 +++ src/retromol_database/duckdb.py | 407 ++++++++++++------ 18 files changed, 1856 insertions(+), 215 deletions(-) create mode 100644 database/scripts/annotate_chebi.py create mode 100644 database/scripts/annotate_chembl.py create mode 100644 database/scripts/annotate_npclassifier.py create mode 100644 database/scripts/chebi.py create mode 100644 database/scripts/chembl.py create mode 100644 database/scripts/npclassifier.py create mode 100644 database/scripts/taxonomy.py create mode 100644 gui/src/server/routes/entry_annotations.py diff --git a/database/Snakemake b/database/Snakemake index 2a9a884..2496332 100644 --- a/database/Snakemake +++ b/database/Snakemake @@ -14,10 +14,13 @@ Fill in database/config.yaml's `sources` URLs before running. The pipeline: 6. load_mibig_compounds } turn results into "compound" db entries (linked to MIBiG's URL) 7. parse_mibig_gbks - antiSMASH GBKs -> linear module readouts (PARAS-annotated) 8. load_mibig_bgcs - turn readouts into "bgc" db entries + 9. annotate_npclassifier - chemical-class annotation (compounds only, NPClassifier API) + 10. annotate_chembl - bioactivity annotation (compounds only, local ChEMBL SQLite) + 11. annotate_chebi - bioactivity annotation (compounds only, local ChEBI flat files) -Steps 4, 6, and 8 all mutate the same DuckDB file, so they're chained through marker -files (rather than each declaring the database itself as `output`) to force -Snakemake to serialize them -- DuckDB doesn't support concurrent writers. +Steps 4, 6, 8, 9, 10, and 11 all mutate the same DuckDB file, so they're chained +through marker files (rather than each declaring the database itself as `output`) to +force Snakemake to serialize them -- DuckDB doesn't support concurrent writers. """ import sys @@ -39,10 +42,49 @@ PARAS_TRAINING_DATA_PATH = config["paras"].get("training_data_path") PARSE_COMPOUNDS_WORKERS = config["compute"]["parse_compounds_workers"] PARSE_GBKS_WORKERS = config["compute"]["parse_gbks_workers"] +TAXONOMY_ENABLED = config.get("taxonomy", {}).get("enabled", False) +TAXDUMP_DIR = WORKDIR / "taxdump" + +NPCLASSIFIER_REQUESTS_PER_SECOND = config.get("npclassifier", {}).get("requests_per_second", 2.0) + +CHEMBL_SQLITE_URL = config.get("chembl", {}).get("sqlite_url") +CHEMBL_DIR = WORKDIR / "chembl" + +CHEBI_DIR = WORKDIR / "chebi" + +# Toggle whole branches of the pipeline off (see config.yaml's `enabled` comment) -- +# disabling npatlas/mibig skips their download+parse rules entirely, not just db +# loading, since the "real" inputs below are only declared when enabled: nothing else +# in the DAG needs npatlas/results.jsonl or mibig_gbk/readouts.jsonl, so Snakemake +# never schedules the rules that would produce them. Same idea for chembl/chebi: +# disabling either skips its bulk download entirely. +ENABLED = config.get("enabled", {}) +NPATLAS_ENABLED = ENABLED.get("npatlas", True) +MIBIG_ENABLED = ENABLED.get("mibig", True) +NPCLASSIFIER_ENABLED = ENABLED.get("npclassifier", True) +CHEMBL_ENABLED = ENABLED.get("chembl", True) +CHEBI_ENABLED = ENABLED.get("chebi", True) + rule all: input: - MARKERS / "bgcs_loaded.done" + MARKERS / "bgcs_loaded.done", + MARKERS / "npclassifier_annotated.done", + MARKERS / "chembl_annotated.done", + MARKERS / "chebi_annotated.done" + + +# --------------------------------------------------------------------------- +# NCBI taxonomy dump (used to standardize phylogeny genus/species/type to taxids) +# --------------------------------------------------------------------------- + +rule download_taxdump: + output: + names=TAXDUMP_DIR / "names.dmp", + nodes=TAXDUMP_DIR / "nodes.dmp" + run: + import taxonomy + taxonomy.download_taxdump(TAXDUMP_DIR) # --------------------------------------------------------------------------- @@ -131,19 +173,24 @@ rule parse_npatlas: rule load_npatlas_compounds: input: - results=WORKDIR / "npatlas" / "results.jsonl", - db_created=MARKERS / "db_created.done" + db_created=MARKERS / "db_created.done", + # Only declared when enabled -- nothing else in the DAG needs npatlas/results.jsonl, + # so download_npatlas/resolve_npatlas_sdf/parse_npatlas never run when disabled. + **({"results": WORKDIR / "npatlas" / "results.jsonl"} if NPATLAS_ENABLED else {}), + **({"taxdump_names": TAXDUMP_DIR / "names.dmp", "taxdump_nodes": TAXDUMP_DIR / "nodes.dmp"} if TAXONOMY_ENABLED and NPATLAS_ENABLED else {}) output: marker=touch(MARKERS / "npatlas_loaded.done") run: - import load_compounds - load_compounds.run( - results_path=input.results, - db_path=DB_PATH, - source="npatlas", - reaction_rules_path=RXN_RULES, - matching_rules_path=MXN_RULES, - ) + if NPATLAS_ENABLED: + import load_compounds + load_compounds.run( + results_path=input.results, + db_path=DB_PATH, + source="npatlas", + reaction_rules_path=RXN_RULES, + matching_rules_path=MXN_RULES, + taxdump_dir=TAXDUMP_DIR if TAXONOMY_ENABLED else None, + ) # --------------------------------------------------------------------------- @@ -194,25 +241,35 @@ rule parse_mibig_compounds: rule load_mibig_compounds: input: - results=WORKDIR / "mibig_json" / "results.jsonl", - # MIBiG URLs need an accession's version, from the JSON (see + prev=MARKERS / "npatlas_loaded.done", + # Only declared when enabled -- nothing else in the DAG needs these, so + # download_mibig_json/extract_mibig_compounds/parse_mibig_compounds never run + # when disabled. MIBiG URLs need an accession's version, from the JSON (see # extract_mibig_compounds rule above) -- not from the GBKs, which no longer carry it. - versions=WORKDIR / "mibig_json" / "versions.json", - annotations=WORKDIR / "mibig_json" / "annotations.json", - prev=MARKERS / "npatlas_loaded.done" + **( + { + "results": WORKDIR / "mibig_json" / "results.jsonl", + "versions": WORKDIR / "mibig_json" / "versions.json", + "annotations": WORKDIR / "mibig_json" / "annotations.json", + } + if MIBIG_ENABLED else {} + ), + **({"taxdump_names": TAXDUMP_DIR / "names.dmp", "taxdump_nodes": TAXDUMP_DIR / "nodes.dmp"} if TAXONOMY_ENABLED and MIBIG_ENABLED else {}) output: marker=touch(MARKERS / "mibig_compounds_loaded.done") run: - import load_compounds - load_compounds.run( - results_path=input.results, - db_path=DB_PATH, - source="mibig", - reaction_rules_path=RXN_RULES, - matching_rules_path=MXN_RULES, - mibig_versions_path=input.versions, - mibig_annotations_path=input.annotations, - ) + if MIBIG_ENABLED: + import load_compounds + load_compounds.run( + results_path=input.results, + db_path=DB_PATH, + source="mibig", + reaction_rules_path=RXN_RULES, + matching_rules_path=MXN_RULES, + mibig_versions_path=input.versions, + mibig_annotations_path=input.annotations, + taxdump_dir=TAXDUMP_DIR if TAXONOMY_ENABLED else None, + ) # --------------------------------------------------------------------------- @@ -239,19 +296,120 @@ rule parse_mibig_gbks: rule load_mibig_bgcs: input: - readouts=WORKDIR / "mibig_gbk" / "readouts.jsonl", - versions=WORKDIR / "mibig_json" / "versions.json", - annotations=WORKDIR / "mibig_json" / "annotations.json", - prev=MARKERS / "mibig_compounds_loaded.done" + prev=MARKERS / "mibig_compounds_loaded.done", + # Only declared when enabled -- nothing else in the DAG needs these, so + # download_mibig_gbk/parse_mibig_gbks never run when disabled. + **( + { + "readouts": WORKDIR / "mibig_gbk" / "readouts.jsonl", + "versions": WORKDIR / "mibig_json" / "versions.json", + "annotations": WORKDIR / "mibig_json" / "annotations.json", + } + if MIBIG_ENABLED else {} + ), + **({"taxdump_names": TAXDUMP_DIR / "names.dmp", "taxdump_nodes": TAXDUMP_DIR / "nodes.dmp"} if TAXONOMY_ENABLED and MIBIG_ENABLED else {}) output: marker=touch(MARKERS / "bgcs_loaded.done") run: - import load_bgcs - load_bgcs.run( - readouts_path=input.readouts, - db_path=DB_PATH, - reaction_rules_path=RXN_RULES, - matching_rules_path=MXN_RULES, - mibig_versions_path=input.versions, - mibig_annotations_path=input.annotations, + if MIBIG_ENABLED: + import load_bgcs + load_bgcs.run( + readouts_path=input.readouts, + db_path=DB_PATH, + reaction_rules_path=RXN_RULES, + matching_rules_path=MXN_RULES, + mibig_versions_path=input.versions, + mibig_annotations_path=input.annotations, + taxdump_dir=TAXDUMP_DIR if TAXONOMY_ENABLED else None, + ) + + +# --------------------------------------------------------------------------- +# Step 9: NPClassifier chemical-class annotation (compounds only) +# --------------------------------------------------------------------------- + +rule annotate_npclassifier: + input: + # Chained after bgcs_loaded.done (not just the compound-loading steps) purely to + # serialize this write against load_mibig_bgcs's -- both mutate DB_PATH and DuckDB + # doesn't support concurrent writers (see module docstring at the top of this file). + prev=MARKERS / "bgcs_loaded.done" + output: + marker=touch(MARKERS / "npclassifier_annotated.done") + run: + if NPCLASSIFIER_ENABLED: + import annotate_npclassifier + annotate_npclassifier.run( + db_path=DB_PATH, + cache_path=WORKDIR / "npclassifier" / "cache.jsonl", + requests_per_second=NPCLASSIFIER_REQUESTS_PER_SECOND, + ) + + +# --------------------------------------------------------------------------- +# Step 10: ChEMBL bioactivity annotation (compounds only) +# --------------------------------------------------------------------------- + +rule download_chembl: + output: + extract_dir=directory(CHEMBL_DIR / "extracted") + params: + url=CHEMBL_SQLITE_URL + run: + import chembl + chembl.download_chembl_sqlite(output.extract_dir, url=params.url) + + +rule annotate_chembl: + input: + # Chained after npclassifier_annotated.done purely to serialize this write + # against annotate_npclassifier's (see module docstring at the top of this file). + prev=MARKERS / "npclassifier_annotated.done", + # Only declared when enabled -- download_chembl (a multi-GB download) never + # runs when disabled. + **({"extract_dir": CHEMBL_DIR / "extracted"} if CHEMBL_ENABLED else {}) + output: + marker=touch(MARKERS / "chembl_annotated.done") + run: + if CHEMBL_ENABLED: + import annotate_chembl + candidates = sorted(Path(input.extract_dir).rglob("*.db")) + if not candidates: + raise FileNotFoundError(f"no ChEMBL .db file found under {input.extract_dir}") + annotate_chembl.run(db_path=DB_PATH, chembl_sqlite_path=candidates[0]) + + +# --------------------------------------------------------------------------- +# Step 11: ChEBI bioactivity annotation (compounds only) +# --------------------------------------------------------------------------- + +rule download_chebi: + output: + compounds=CHEBI_DIR / "compounds.tsv.gz", + structures=CHEBI_DIR / "structures.tsv.gz", + relation=CHEBI_DIR / "relation.tsv.gz" + run: + import chebi + chebi.download_chebi_flat_files(CHEBI_DIR) + + +rule annotate_chebi: + input: + # Chained after chembl_annotated.done purely to serialize this write against + # annotate_chembl's (see module docstring at the top of this file). + prev=MARKERS / "chembl_annotated.done", + # Only declared when enabled -- download_chebi never runs when disabled. + **( + { + "compounds": CHEBI_DIR / "compounds.tsv.gz", + "structures": CHEBI_DIR / "structures.tsv.gz", + "relation": CHEBI_DIR / "relation.tsv.gz", + } + if CHEBI_ENABLED else {} ) + output: + marker=touch(MARKERS / "chebi_annotated.done") + run: + if CHEBI_ENABLED: + import annotate_chebi + annotate_chebi.run(db_path=DB_PATH, chebi_dir=CHEBI_DIR) diff --git a/database/config.yaml b/database/config.yaml index 904ffe3..9404726 100644 --- a/database/config.yaml +++ b/database/config.yaml @@ -6,6 +6,19 @@ sources: mibig_json_url: "https://dl.secondarymetabolites.org/mibig/mibig_json_4.0.tar.gz" mibig_gbk_url: "https://dl.secondarymetabolites.org/mibig/mibig_gbk_4.0.tar.gz" +enabled: + # Toggle whole branches of the pipeline off. Disabling a source skips its downloads + # and RetroMol parsing entirely (not just db loading) -- e.g. npatlas: false means + # download_npatlas/parse_npatlas never run. mibig covers both MIBiG compounds and + # BGCs (they share the same JSON/GBK downloads). npclassifier gates step 9 + # (chemical-class annotation) independently of which source(s) loaded the compounds + # it classifies. + npatlas: true + mibig: true + npclassifier: true + chembl: true + chebi: true + paths: # Final DuckDB database produced by the pipeline. database: "/Users/davidmeijer/Downloads/retromol.duckdb" @@ -17,6 +30,26 @@ paths: reaction_rules: null matching_rules: null +npclassifier: + # Rate limit for the free, GNPS2-hosted NPClassifier API (no published limit -- + # kept conservative). Classifications are cached in workdir/npclassifier/cache.jsonl, + # so reruns only pay for compounds not already classified. + requests_per_second: 10.0 + +chembl: + # ChEMBL's official SQLite release -- a multi-GB bulk download (~5.4GB compressed + # as of ChEMBL 37), extracted once into workdir/chembl and queried locally by + # InChIKey (see database/scripts/chembl.py). "latest" always points at the newest + # ChEMBL release; pin to a specific release's URL for reproducibility across runs. + sqlite_url: "https://ftp.ebi.ac.uk/pub/databases/chembl/ChEMBLdb/latest/chembl_37_sqlite.tar.gz" + +taxonomy: + # NCBI taxdump (names.dmp/nodes.dmp), downloaded once into workdir/taxdump and reused + # across pipeline runs -- used to standardize phylogeny genus/species/type to NCBI + # taxids (see database/scripts/taxonomy.py). Set to null to skip taxid resolution + # entirely (phylogeny is then stored as raw, unstandardized text/no taxids). + enabled: true + paras: threshold: 0.1 keep_top: 3 diff --git a/database/scripts/annotate_chebi.py b/database/scripts/annotate_chebi.py new file mode 100644 index 0000000..2dde1a4 --- /dev/null +++ b/database/scripts/annotate_chebi.py @@ -0,0 +1,69 @@ +"""Step 11: annotate every compound entry's bioactivity via ChEBI's role ontology. + +Runs after compound loading, so every distinct molecule -- regardless of which +source(s) it came from -- is looked up exactly once. BGC entries are skipped, same +reasoning as annotate_chembl.py. + +Like annotate_chembl.py, this queries a local bulk release (see chebi.py) rather than +a rate-limited API -- no pacing or caching needed. +""" + +import argparse +import logging +from pathlib import Path + +from chebi import ChebiDB +from retromol_database.duckdb import RetroMolDuckDB + +log = logging.getLogger(__name__) + + +def run(db_path: str | Path, chebi_dir: str | Path, log_every: int = 500) -> None: + processed = 0 + matched = 0 + annotated = 0 + + db = RetroMolDuckDB.open(db_path) + try: + chebi = ChebiDB.load(chebi_dir) + for entry in db.iter_entries(): + if entry.type != "compound": + continue + + processed += 1 + result = chebi.roles_for_inchikey(entry.id) + if result is not None: + matched += 1 + for role in result.biological_roles: + db.add_bioactivity_annotation( + entry.id, level="chebi_biological_role", label=role.label, external_id=role.chebi_accession + ) + annotated += 1 + for role in result.chemical_roles: + db.add_bioactivity_annotation( + entry.id, level="chebi_chemical_role", label=role.label, external_id=role.chebi_accession + ) + annotated += 1 + + if log_every > 0 and processed % log_every == 0: + log.info("annotate_chebi: processed=%d matched=%d annotated=%d", processed, matched, annotated) + finally: + db.close() + + log.info("annotate_chebi: processed=%d matched=%d annotated=%d", processed, matched, annotated) + + +def main() -> None: + logging.basicConfig(level=logging.INFO) + + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--db-path", required=True) + ap.add_argument("--chebi-dir", required=True, help="dir with ChEBI's compounds.tsv.gz/structures.tsv.gz/relation.tsv.gz") + ap.add_argument("--log-every", type=int, default=500) + args = ap.parse_args() + + run(db_path=args.db_path, chebi_dir=args.chebi_dir, log_every=args.log_every) + + +if __name__ == "__main__": + main() diff --git a/database/scripts/annotate_chembl.py b/database/scripts/annotate_chembl.py new file mode 100644 index 0000000..eefbb76 --- /dev/null +++ b/database/scripts/annotate_chembl.py @@ -0,0 +1,78 @@ +"""Step 10: annotate every compound entry's bioactivity via ChEMBL. + +Runs after compound loading (like annotate_npclassifier.py), so every distinct +molecule -- regardless of which source(s) it came from -- is looked up exactly once. +BGC entries are skipped: bioactivity here is a property of the compound structure +(looked up by its InChIKey entry id), not of the producing organism/cluster. + +Unlike NPClassifier, this queries a local ChEMBL SQLite release (see chembl.py) rather +than a rate-limited API -- no pacing or caching needed, lookups are just local reads. +""" + +import argparse +import logging +from pathlib import Path + +from chembl import ChemblDB +from retromol_database.duckdb import RetroMolDuckDB + +log = logging.getLogger(__name__) + + +def run(db_path: str | Path, chembl_sqlite_path: str | Path, log_every: int = 500) -> None: + processed = 0 + matched = 0 + annotated = 0 + + db = RetroMolDuckDB.open(db_path) + try: + with ChemblDB(chembl_sqlite_path) as chembl: + for entry in db.iter_entries(): + if entry.type != "compound": + continue + + processed += 1 + result = chembl.bioactivity_for_inchikey(entry.id) + if result is not None: + matched += 1 + if result.max_phase_label: + db.add_bioactivity_annotation( + entry.id, + level="chembl_max_phase", + label=result.max_phase_label, + external_id=result.chembl_id, + ) + annotated += 1 + for atc in result.atc_categories: + db.add_bioactivity_annotation( + entry.id, + level="chembl_atc", + label=atc.level1_description, + external_id=atc.level5, + ) + annotated += 1 + + if log_every > 0 and processed % log_every == 0: + log.info( + "annotate_chembl: processed=%d matched=%d annotated=%d", processed, matched, annotated + ) + finally: + db.close() + + log.info("annotate_chembl: processed=%d matched=%d annotated=%d", processed, matched, annotated) + + +def main() -> None: + logging.basicConfig(level=logging.INFO) + + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--db-path", required=True) + ap.add_argument("--chembl-sqlite-path", required=True) + ap.add_argument("--log-every", type=int, default=500) + args = ap.parse_args() + + run(db_path=args.db_path, chembl_sqlite_path=args.chembl_sqlite_path, log_every=args.log_every) + + +if __name__ == "__main__": + main() diff --git a/database/scripts/annotate_npclassifier.py b/database/scripts/annotate_npclassifier.py new file mode 100644 index 0000000..15de53d --- /dev/null +++ b/database/scripts/annotate_npclassifier.py @@ -0,0 +1,162 @@ +"""Step 9: classify every compound entry's chemical class via NPClassifier. + +Runs after both compound-loading steps (NPAtlas + MIBiG compounds are already +deduplicated by inchikey in `entries` by then -- see load_compounds.py), so each +distinct molecule is classified exactly once no matter which source(s) it came from. +BGC entries are skipped: NPClassifier needs a compound's own SMILES structure +(`entry.raw`), which a bgc entry doesn't have. + +Idempotent across reruns via a local JSONL cache (`--cache-path`) keyed by entry_id: +already-classified compounds are skipped without hitting the API again, and only +successes are cached, so a transient failure gets retried on the next run. Requests +are paced (`--requests-per-second`, conservative by default) since NPClassifier is a +free, GNPS2-hosted academic service with no published rate limit. +""" + +import argparse +import json +import logging +import time +from pathlib import Path + +from npclassifier import ClassificationResult, classify_smiles +from retromol_database.duckdb import RetroMolDuckDB + +log = logging.getLogger(__name__) + + +def _load_cache(cache_path: Path) -> dict[str, ClassificationResult]: + if not cache_path.exists(): + return {} + + cache: dict[str, ClassificationResult] = {} + with open(cache_path) as fh: + for line in fh: + line = line.strip() + if not line: + continue + row = json.loads(line) + cache[row["entry_id"]] = ClassificationResult( + pathway=row["pathway"], + superclass=row["superclass"], + class_=row["class_"], + is_glycoside=row["is_glycoside"], + ) + return cache + + +def _append_cache(cache_path: Path, entry_id: str, result: ClassificationResult) -> None: + with open(cache_path, "a") as fh: + fh.write( + json.dumps( + { + "entry_id": entry_id, + "pathway": result.pathway, + "superclass": result.superclass, + "class_": result.class_, + "is_glycoside": result.is_glycoside, + } + ) + + "\n" + ) + + +def _apply(db: RetroMolDuckDB, entry_id: str, result: ClassificationResult) -> None: + for label in result.pathway: + db.add_chemical_class_annotation(entry_id, level="pathway", label=label) + for label in result.superclass: + db.add_chemical_class_annotation(entry_id, level="superclass", label=label) + for label in result.class_: + db.add_chemical_class_annotation(entry_id, level="class", label=label) + if result.is_glycoside: + db.add_chemical_class_annotation(entry_id, level="is_glycoside", label="Yes") + + +def run( + db_path: str | Path, + cache_path: str | Path, + requests_per_second: float = 2.0, + limit: int | None = None, + log_every: int = 100, +) -> None: + cache_path = Path(cache_path) + cache_path.parent.mkdir(parents=True, exist_ok=True) + cache = _load_cache(cache_path) + log.info("annotate_npclassifier: loaded %d cached classifications from %s", len(cache), cache_path) + + min_interval = 1.0 / requests_per_second if requests_per_second > 0 else 0.0 + last_call = 0.0 + + classified = 0 + reused = 0 + skipped_no_smiles = 0 + failed = 0 + + db = RetroMolDuckDB.open(db_path) + try: + for entry in db.iter_entries(): + if entry.type != "compound": + continue + + if entry.id in cache: + _apply(db, entry.id, cache[entry.id]) + reused += 1 + continue + + if not entry.raw: + skipped_no_smiles += 1 + continue + + if limit is not None and classified >= limit: + continue + + elapsed = time.monotonic() - last_call + if elapsed < min_interval: + time.sleep(min_interval - elapsed) + last_call = time.monotonic() + + result = classify_smiles(entry.raw) + if result is None: + failed += 1 + continue + + _apply(db, entry.id, result) + _append_cache(cache_path, entry.id, result) + classified += 1 + + if log_every > 0 and classified % log_every == 0: + log.info( + "annotate_npclassifier: classified=%d reused=%d failed=%d skipped_no_smiles=%d", + classified, reused, failed, skipped_no_smiles, + ) + finally: + db.close() + + log.info( + "annotate_npclassifier: classified=%d reused=%d failed=%d skipped_no_smiles=%d", + classified, reused, failed, skipped_no_smiles, + ) + + +def main() -> None: + logging.basicConfig(level=logging.INFO) + + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--db-path", required=True) + ap.add_argument("--cache-path", required=True) + ap.add_argument("--requests-per-second", type=float, default=2.0) + ap.add_argument("--limit", type=int, default=None, help="classify at most N new compounds (testing/dry-run)") + ap.add_argument("--log-every", type=int, default=100) + args = ap.parse_args() + + run( + db_path=args.db_path, + cache_path=args.cache_path, + requests_per_second=args.requests_per_second, + limit=args.limit, + log_every=args.log_every, + ) + + +if __name__ == "__main__": + main() diff --git a/database/scripts/chebi.py b/database/scripts/chebi.py new file mode 100644 index 0000000..b9c95af --- /dev/null +++ b/database/scripts/chebi.py @@ -0,0 +1,222 @@ +"""ChEBI role-ontology client for bioactivity annotation. + +Downloads ChEBI's flat-file bulk release (ftp.ebi.ac.uk) once, then looks up a +compound's biological/chemical roles by its standard InChIKey. Unlike ChEMBL's +clinical-development framing, ChEBI's role ontology (`has_role` edges under +CHEBI:24432 "biological role" / CHEBI:51086 "chemical role") is annotated on generic +compound classes more often than on specific stereo-defined structures -- confirmed +live (2026-08-26): erythromycin A (CHEBI:42355) itself carries no has_role edges, only +its `is_a` parent "erythromycin" (CHEBI:48923) does (roles: xenobiotic, bacterial +metabolite, environmental contaminant). So lookups walk a few steps up the `is_a` +ancestor chain collecting has_role edges at every level, not just the leaf's own. + +Schema confirmed live by downloading and inspecting +ftp.ebi.ac.uk/pub/databases/chebi/flat_files/: structures.tsv (compound_id, +standard_inchi_key), relation.tsv (relation_type_id 4=has_role, 5=is_a; init_id/final_id +are compounds.tsv ids), compounds.tsv (id, name, chebi_accession). +""" + +from __future__ import annotations + +import csv +import gzip +import logging +import shutil +import urllib.request +from dataclasses import dataclass +from pathlib import Path + +log = logging.getLogger(__name__) + +CHEBI_FLAT_FILES_URL = "https://ftp.ebi.ac.uk/pub/databases/chebi/flat_files" +CHEBI_FILES = ["compounds.tsv.gz", "structures.tsv.gz", "relation.tsv.gz"] + +RELATION_TYPE_HAS_ROLE = "4" +RELATION_TYPE_IS_A = "5" + +# ChEBI's role-ontology roots (CHEBI:50906 "role"'s direct children) -- a has_role +# target is classified by walking its own is_a ancestry for one of these two ids. +# "application" (CHEBI:33232), the third root, isn't surfaced -- not requested. +CHEBI_ID_BIOLOGICAL_ROLE = "24432" +CHEBI_ID_CHEMICAL_ROLE = "51086" + +# How far to walk up `is_a` ancestry: role classification (shallow ontology, ~1-4 hops +# in practice) gets a bit more headroom than role-collection on the queried compound +# (kept shallow deliberately -- climbing further starts pulling in unrelated purely +# structural classes, e.g. erythromycin A's other is_a parent "cyclic ketone", which +# carry no roles of their own but would otherwise cost unbounded fan-out for nothing). +MAX_ROLE_CLASSIFICATION_HOPS = 10 +MAX_ROLE_COLLECTION_HOPS = 4 + + +def download_chebi_flat_files(dest_dir: str | Path, *, force: bool = False) -> Path: + """Download compounds.tsv.gz/structures.tsv.gz/relation.tsv.gz into `dest_dir` + (no-op if all three already exist, unless `force`).""" + dest_dir = Path(dest_dir).expanduser() + dest_dir.mkdir(parents=True, exist_ok=True) + + if not force and all((dest_dir / name).exists() for name in CHEBI_FILES): + return dest_dir + + for name in CHEBI_FILES: + url = f"{CHEBI_FLAT_FILES_URL}/{name}" + dest = dest_dir / name + log.info("downloading %s", url) + with urllib.request.urlopen(url) as resp, open(dest, "wb") as out: + shutil.copyfileobj(resp, out, length=1024 * 1024) + + return dest_dir + + +@dataclass(frozen=True) +class ChebiRole: + label: str + chebi_accession: str # e.g. "CHEBI:35703" -- the role term's own id, for linking out + + +@dataclass(frozen=True) +class ChebiRoles: + chebi_id: str # the matched compound's own CHEBI accession, e.g. "CHEBI:42355" + biological_roles: list[ChebiRole] + chemical_roles: list[ChebiRole] + + +class ChebiDB: + """In-memory index over ChEBI's flat files, built once per pipeline run.""" + + def __init__( + self, + *, + chebi_accession_by_compound_id: dict[str, str], + name_by_compound_id: dict[str, str], + compound_id_by_inchikey: dict[str, str], + is_a_parents: dict[str, list[str]], + has_role: dict[str, list[str]], + ) -> None: + self._chebi_accession_by_compound_id = chebi_accession_by_compound_id + self._name_by_compound_id = name_by_compound_id + self._compound_id_by_inchikey = compound_id_by_inchikey + self._is_a_parents = is_a_parents + self._has_role = has_role + + @classmethod + def load(cls, chebi_dir: str | Path) -> "ChebiDB": + chebi_dir = Path(chebi_dir).expanduser() + + chebi_accession_by_compound_id: dict[str, str] = {} + name_by_compound_id: dict[str, str] = {} + with gzip.open(chebi_dir / "compounds.tsv.gz", "rt", encoding="utf-8", errors="replace") as fh: + reader = csv.reader(fh, delimiter="\t", quotechar='"') + header = next(reader) + id_idx = header.index("id") + name_idx = header.index("name") + accession_idx = header.index("chebi_accession") + for row in reader: + if len(row) <= accession_idx or not row[accession_idx]: + continue + chebi_accession_by_compound_id[row[id_idx]] = row[accession_idx] + name_by_compound_id[row[id_idx]] = row[name_idx] + + compound_id_by_inchikey: dict[str, str] = {} + with gzip.open(chebi_dir / "structures.tsv.gz", "rt", encoding="utf-8", errors="replace") as fh: + reader = csv.reader(fh, delimiter="\t", quotechar='"') + header = next(reader) + cid_idx = header.index("compound_id") + key_idx = header.index("standard_inchi_key") + for row in reader: + if len(row) <= key_idx or not row[key_idx]: + continue + compound_id_by_inchikey.setdefault(row[key_idx], row[cid_idx]) + + is_a_parents: dict[str, list[str]] = {} + has_role: dict[str, list[str]] = {} + with gzip.open(chebi_dir / "relation.tsv.gz", "rt", encoding="utf-8", errors="replace") as fh: + reader = csv.reader(fh, delimiter="\t", quotechar='"') + header = next(reader) + type_idx = header.index("relation_type_id") + init_idx = header.index("init_id") + final_idx = header.index("final_id") + for row in reader: + if len(row) <= final_idx: + continue + rtype, init_id, final_id = row[type_idx], row[init_idx], row[final_idx] + if rtype == RELATION_TYPE_IS_A: + is_a_parents.setdefault(init_id, []).append(final_id) + elif rtype == RELATION_TYPE_HAS_ROLE: + has_role.setdefault(init_id, []).append(final_id) + + log.info( + "loaded ChEBI flat files: %d compounds, %d structures, %d is_a edges, %d has_role edges", + len(chebi_accession_by_compound_id), len(compound_id_by_inchikey), len(is_a_parents), len(has_role), + ) + return cls( + chebi_accession_by_compound_id=chebi_accession_by_compound_id, + name_by_compound_id=name_by_compound_id, + compound_id_by_inchikey=compound_id_by_inchikey, + is_a_parents=is_a_parents, + has_role=has_role, + ) + + def _walk_is_a(self, start: str, max_hops: int): + """Yield ancestor compound ids reachable from `start` via `is_a`, breadth-first, + up to `max_hops` levels, never revisiting a node.""" + seen = {start} + frontier = [start] + for _ in range(max_hops): + next_frontier = [] + for cid in frontier: + for parent in self._is_a_parents.get(cid, []): + if parent not in seen: + seen.add(parent) + next_frontier.append(parent) + yield parent + if not next_frontier: + return + frontier = next_frontier + + def _role_category(self, role_compound_id: str) -> str | None: + """Classify a has_role target by walking its own is_a ancestry for one of + CHEBI_ID_BIOLOGICAL_ROLE/CHEBI_ID_CHEMICAL_ROLE. None if neither is hit (e.g. + it falls under "application" instead, or the chain doesn't resolve).""" + if role_compound_id == CHEBI_ID_BIOLOGICAL_ROLE: + return "biological_role" + if role_compound_id == CHEBI_ID_CHEMICAL_ROLE: + return "chemical_role" + for ancestor in self._walk_is_a(role_compound_id, MAX_ROLE_CLASSIFICATION_HOPS): + if ancestor == CHEBI_ID_BIOLOGICAL_ROLE: + return "biological_role" + if ancestor == CHEBI_ID_CHEMICAL_ROLE: + return "chemical_role" + return None + + def roles_for_inchikey(self, inchikey: str) -> ChebiRoles | None: + """None if `inchikey` has no match in ChEBI at all. Roles are collected from + the matched compound and a few levels of its `is_a` ancestors (see module + docstring -- ChEBI usually annotates roles on a generic parent class, not every + specific stereo-defined child structure).""" + compound_id = self._compound_id_by_inchikey.get(inchikey) + if compound_id is None: + return None + + role_compound_ids: set[str] = set(self._has_role.get(compound_id, [])) + for ancestor in self._walk_is_a(compound_id, MAX_ROLE_COLLECTION_HOPS): + role_compound_ids.update(self._has_role.get(ancestor, [])) + + biological_roles: list[ChebiRole] = [] + chemical_roles: list[ChebiRole] = [] + for role_id in role_compound_ids: + label = self._name_by_compound_id.get(role_id) + accession = self._chebi_accession_by_compound_id.get(role_id) + if not label or not accession: + continue + category = self._role_category(role_id) + if category == "biological_role": + biological_roles.append(ChebiRole(label=label, chebi_accession=accession)) + elif category == "chemical_role": + chemical_roles.append(ChebiRole(label=label, chebi_accession=accession)) + + return ChebiRoles( + chebi_id=self._chebi_accession_by_compound_id.get(compound_id, compound_id), + biological_roles=biological_roles, + chemical_roles=chemical_roles, + ) diff --git a/database/scripts/chembl.py b/database/scripts/chembl.py new file mode 100644 index 0000000..3379f81 --- /dev/null +++ b/database/scripts/chembl.py @@ -0,0 +1,141 @@ +"""ChEMBL bulk-database client for bioactivity annotation. + +Downloads and extracts ChEMBL's official SQLite release (ftp.ebi.ac.uk) once, then +looks up a compound's known medicinal-relevance signals by its standard InChIKey -- +ChEMBL's own maximum clinical-development phase (MAX_PHASE) and WHO ATC therapeutic +category (a defined vocabulary, unlike MIBiG's free-text chem_acts) -- rather than +querying ChEMBL's live API per compound, which would be far too slow at our scale. + +Schema confirmed live against ChEMBL's own schema documentation +(https://ftp.ebi.ac.uk/pub/databases/chembl/ChEMBLdb/latest/schema_documentation.txt, +2026-08-26): COMPOUND_STRUCTURES.STANDARD_INCHI_KEY -> MOLECULE_DICTIONARY.MOLREGNO +(MAX_PHASE, CHEMBL_ID) -> MOLECULE_ATC_CLASSIFICATION -> ATC_CLASSIFICATION.LEVEL1_DESCRIPTION. +""" + +from __future__ import annotations + +import logging +import shutil +import sqlite3 +import tarfile +import urllib.request +from dataclasses import dataclass +from pathlib import Path + +log = logging.getLogger(__name__) + +# "latest" always resolves to the newest ChEMBL release -- pin to a specific +# release's URL (e.g. .../chembl_37/chembl_37_sqlite.tar.gz) if a pipeline run needs +# to be reproducible across ChEMBL releases. +CHEMBL_SQLITE_URL = "https://ftp.ebi.ac.uk/pub/databases/chembl/ChEMBLdb/latest/chembl_37_sqlite.tar.gz" + +# ChEMBL's MAX_PHASE values (MOLECULE_DICTIONARY.MAX_PHASE) mapped to a display label. +# Only phases with real clinical-development meaning are surfaced -- 0/-1 (no reported +# development / unknown) aren't stored, the same presence-only convention every other +# annotation table uses. Python's dict lookup treats 3 and 3.0 as the same key, so this +# works whether sqlite hands back MAX_PHASE as an int or a float. +MAX_PHASE_LABELS = { + 4: "Approved", + 3: "Phase 3", + 2: "Phase 2", + 1: "Phase 1", + 0.5: "Early phase 1", +} + + +def download_chembl_sqlite(dest_dir: str | Path, *, url: str = CHEMBL_SQLITE_URL, force: bool = False) -> Path: + """Download and extract ChEMBL's SQLite release into `dest_dir`, returning the path + to the extracted .db file (no-op if one is already there, unless `force`). + + This is a multi-GB download (~5.4GB compressed as of ChEMBL 37) -- streamed to disk + in chunks rather than read into memory, unlike taxonomy.py's much smaller taxdump. + """ + dest_dir = Path(dest_dir).expanduser() + dest_dir.mkdir(parents=True, exist_ok=True) + + existing = sorted(dest_dir.rglob("*.db")) + if not force and existing: + return existing[0] + + archive_path = dest_dir / "chembl_sqlite.tar.gz" + log.info("downloading ChEMBL SQLite release from %s", url) + with urllib.request.urlopen(url) as resp, open(archive_path, "wb") as out: + shutil.copyfileobj(resp, out, length=1024 * 1024) + + log.info("extracting %s", archive_path) + with tarfile.open(archive_path, mode="r:gz") as tar: + tar.extractall(path=dest_dir) + archive_path.unlink() + + found = sorted(dest_dir.rglob("*.db")) + if not found: + raise FileNotFoundError(f"ChEMBL archive at {url} did not contain a .db file") + return found[0] + + +@dataclass(frozen=True) +class AtcCategory: + level1_description: str # display label -- coarse WHO therapeutic category + level5: str # e.g. "J01FA01" -- the compound's own specific ATC code, for linking out + + +@dataclass(frozen=True) +class ChemblBioactivity: + chembl_id: str + max_phase_label: str | None + atc_categories: list[AtcCategory] + + +class ChemblDB: + """Read-only lookup over a local ChEMBL SQLite release, by standard InChIKey.""" + + def __init__(self, sqlite_path: str | Path) -> None: + self.con = sqlite3.connect(f"file:{Path(sqlite_path).expanduser()}?mode=ro", uri=True) + + def close(self) -> None: + self.con.close() + + def __enter__(self) -> "ChemblDB": + return self + + def __exit__(self, exc_type, exc, tb) -> None: + self.close() + + def bioactivity_for_inchikey(self, inchikey: str) -> ChemblBioactivity | None: + """None if `inchikey` has no match in ChEMBL at all; otherwise a result whose + `max_phase_label`/`atc_categories` may still both be empty (a compound can be + in ChEMBL -- have activity/assay records -- without being an ATC-classified drug + or having reached any clinical phase).""" + row = self.con.execute( + """ + SELECT md.molregno, md.chembl_id, md.max_phase + FROM compound_structures cs + JOIN molecule_dictionary md ON md.molregno = cs.molregno + WHERE cs.standard_inchi_key = ? + """, + (inchikey,), + ).fetchone() + if row is None: + return None + + molregno, chembl_id, max_phase = row + max_phase_label = MAX_PHASE_LABELS.get(max_phase) + + atc_rows = self.con.execute( + """ + SELECT DISTINCT ac.level1_description, ac.level5 + FROM molecule_atc_classification mac + JOIN atc_classification ac ON ac.level5 = mac.level5 + WHERE mac.molregno = ? + """, + (molregno,), + ).fetchall() + atc_categories = [ + AtcCategory(level1_description=str(r[0]), level5=str(r[1])) for r in atc_rows if r[0] and r[1] + ] + + return ChemblBioactivity( + chembl_id=str(chembl_id), + max_phase_label=max_phase_label, + atc_categories=atc_categories, + ) diff --git a/database/scripts/load_bgcs.py b/database/scripts/load_bgcs.py index bbdde7f..520373e 100644 --- a/database/scripts/load_bgcs.py +++ b/database/scripts/load_bgcs.py @@ -29,6 +29,7 @@ from common import build_fingerprint_context, load_ruleset, mibig_url, phylogeny_from_organism_name from retromol_antismash.modules import LinearReadout, bgc_primary_sequence from retromol_database.duckdb import RetroMolDuckDB +from taxonomy import TaxonomyDB, resolve_phylogeny log = logging.getLogger(__name__) @@ -42,10 +43,13 @@ def run( mibig_annotations_path: str | Path, match_stereochemistry: bool = False, include_raw_gbk: bool = True, + taxdump_dir: str | Path | None = None, ) -> None: ruleset = load_ruleset(reaction_rules_path, matching_rules_path, match_stereochemistry) _, fingerprinter = build_fingerprint_context(ruleset) + taxdb = TaxonomyDB.load(taxdump_dir) if taxdump_dir else None + with open(mibig_versions_path) as fh: versions: dict[str, str] = json.load(fh) @@ -98,11 +102,27 @@ def run( record = annotations.get(accession) if accession else None if record: - type_label, genus, species = phylogeny_from_organism_name(record.get("organism_name")) - db.add_phylogeny_annotation(entry_id, type_label=type_label, genus=genus, species=species) - for chemical_class in record.get("biosyn_class") or []: - if chemical_class: - db.add_flat_annotation(entry_id, category="chemical_class", label=str(chemical_class)) + fallback_type, fallback_genus, fallback_species = phylogeny_from_organism_name( + record.get("organism_name") + ) + resolution = resolve_phylogeny( + taxdb, + ncbi_tax_id=record.get("ncbi_tax_id"), + genus=fallback_genus, + species=fallback_species, + fallback_type_label=fallback_type, + ) + db.add_phylogeny_annotation( + entry_id, + type_label=resolution.type_label, + type_taxid=resolution.type_taxid, + genus=resolution.genus, + genus_taxid=resolution.genus_taxid, + species=resolution.species, + species_taxid=resolution.species_taxid, + ) + # chemical_class is compound-only (see RetroMolDuckDB.add_chemical_class_annotation) -- + # a BGC's biosynthetic class isn't populated here anymore. added += 1 finally: @@ -126,6 +146,7 @@ def main() -> None: ap.add_argument("--mibig-annotations", required=True) ap.add_argument("--match-stereochemistry", action="store_true") ap.add_argument("--no-raw-gbk", action="store_true") + ap.add_argument("--taxdump-dir", default=None, help="dir with NCBI names.dmp/nodes.dmp (skips taxid resolution if omitted)") args = ap.parse_args() run( @@ -137,6 +158,7 @@ def main() -> None: mibig_annotations_path=args.mibig_annotations, match_stereochemistry=args.match_stereochemistry, include_raw_gbk=not args.no_raw_gbk, + taxdump_dir=args.taxdump_dir, ) diff --git a/database/scripts/load_compounds.py b/database/scripts/load_compounds.py index 37d8089..a84c1d6 100644 --- a/database/scripts/load_compounds.py +++ b/database/scripts/load_compounds.py @@ -34,6 +34,7 @@ from retromol.model.result import Result from retromol_database.duckdb import RetroMolDuckDB from retromol_fingerprint.fingerprint import TOKEN_LINK +from taxonomy import TaxonomyDB, resolve_phylogeny log = logging.getLogger(__name__) @@ -72,18 +73,35 @@ def _mibig_name_and_url(props: dict, versions: dict[str, str]) -> tuple[str | No return name, mibig_url(accession, version) -def _apply_mibig_annotations(db: RetroMolDuckDB, entry_id: str, props: dict, annotations: dict[str, dict]) -> None: +def _apply_mibig_annotations( + db: RetroMolDuckDB, entry_id: str, props: dict, annotations: dict[str, dict], taxdb: TaxonomyDB | None +) -> None: accession = props.get("mibig_accession") record = annotations.get(accession) if accession else None if not record: return - type_label, genus, species = phylogeny_from_organism_name(record.get("organism_name")) - db.add_phylogeny_annotation(entry_id, type_label=type_label, genus=genus, species=species) + fallback_type, fallback_genus, fallback_species = phylogeny_from_organism_name(record.get("organism_name")) + resolution = resolve_phylogeny( + taxdb, + ncbi_tax_id=record.get("ncbi_tax_id"), + genus=fallback_genus, + species=fallback_species, + fallback_type_label=fallback_type, + ) + db.add_phylogeny_annotation( + entry_id, + type_label=resolution.type_label, + type_taxid=resolution.type_taxid, + genus=resolution.genus, + genus_taxid=resolution.genus_taxid, + species=resolution.species, + species_taxid=resolution.species_taxid, + ) for chemical_class in record.get("biosyn_class") or []: if chemical_class: - db.add_flat_annotation(entry_id, category="chemical_class", label=str(chemical_class)) + db.add_biosynthetic_class_annotation(entry_id, str(chemical_class)) def run( @@ -95,11 +113,14 @@ def run( match_stereochemistry: bool = False, mibig_versions_path: str | Path | None = None, mibig_annotations_path: str | Path | None = None, + taxdump_dir: str | Path | None = None, log_every: int = 1000, ) -> None: ruleset = load_ruleset(reaction_rules_path, matching_rules_path, match_stereochemistry) name_to_rule, fingerprinter = build_fingerprint_context(ruleset) + taxdb = TaxonomyDB.load(taxdump_dir) if taxdump_dir else None + versions: dict[str, str] = {} if source == "mibig" and mibig_versions_path is not None: with open(mibig_versions_path) as fh: @@ -155,11 +176,20 @@ def run( fingerprint=fp, ) if source == "mibig": - _apply_mibig_annotations(db, result.submission.inchikey, props, annotations) + _apply_mibig_annotations(db, result.submission.inchikey, props, annotations, taxdb) else: type_label, genus, species = _npatlas_phylogeny(props) + resolution = resolve_phylogeny( + taxdb, genus=genus, species=species, fallback_type_label=type_label + ) db.add_phylogeny_annotation( - result.submission.inchikey, type_label=type_label, genus=genus, species=species + result.submission.inchikey, + type_label=resolution.type_label, + type_taxid=resolution.type_taxid, + genus=resolution.genus, + genus_taxid=resolution.genus_taxid, + species=resolution.species, + species_taxid=resolution.species_taxid, ) added += 1 @@ -190,6 +220,7 @@ def main() -> None: ap.add_argument("--match-stereochemistry", action="store_true") ap.add_argument("--mibig-versions", default=None, help="required when --source=mibig") ap.add_argument("--mibig-annotations", default=None, help="required when --source=mibig") + ap.add_argument("--taxdump-dir", default=None, help="dir with NCBI names.dmp/nodes.dmp (skips taxid resolution if omitted)") ap.add_argument("--log-every", type=int, default=1000, help="log a progress line every N compounds (0 to disable)") args = ap.parse_args() @@ -202,6 +233,7 @@ def main() -> None: match_stereochemistry=args.match_stereochemistry, mibig_versions_path=args.mibig_versions, mibig_annotations_path=args.mibig_annotations, + taxdump_dir=args.taxdump_dir, log_every=args.log_every, ) diff --git a/database/scripts/npclassifier.py b/database/scripts/npclassifier.py new file mode 100644 index 0000000..cbb028e --- /dev/null +++ b/database/scripts/npclassifier.py @@ -0,0 +1,68 @@ +"""NPClassifier API client for chemical-class annotation. + +Classifies a compound's chemical class directly from its own structure via GNPS2's +free, public NPClassifier service (https://npclassifier.gnps2.org/classify?smiles=...) -- +independent of MIBiG's coarse biosynthetic-class labels (PKS/NRPS/...), and available +for NPAtlas compounds too, which MIBiG's biosyn_class never was. A classification has +three list-valued levels (pathway/superclass/class, most to least general) plus a single +is_glycoside boolean; response shape confirmed live against the API (2026-08-26): +{"pathway_results": [...], "superclass_results": [...], "class_results": [...], "isglycoside": bool}. + +No published rate limit exists for this service, so callers are responsible for pacing +requests themselves (see annotate_npclassifier.py's --requests-per-second) -- this module +only wraps a single call with retry/backoff. +""" + +from __future__ import annotations + +import json +import logging +import time +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass + +log = logging.getLogger(__name__) + +NPCLASSIFIER_URL = "https://npclassifier.gnps2.org/classify" + + +@dataclass(frozen=True) +class ClassificationResult: + pathway: list[str] + superclass: list[str] + class_: list[str] + is_glycoside: bool + + +def classify_smiles( + smiles: str, + *, + timeout: float = 30.0, + max_retries: int = 3, + backoff_seconds: float = 2.0, +) -> ClassificationResult | None: + """Classify one SMILES via NPClassifier, retrying transient failures (timeouts, 5xx, + 429, malformed JSON) with linear backoff. Returns None (logged, not raised) if every + attempt fails -- one unclassifiable/unreachable-service molecule shouldn't abort a + whole pipeline run.""" + url = f"{NPCLASSIFIER_URL}?smiles={urllib.parse.quote(smiles, safe='')}" + + for attempt in range(1, max_retries + 1): + try: + with urllib.request.urlopen(url, timeout=timeout) as resp: + data = json.loads(resp.read()) + return ClassificationResult( + pathway=[str(p) for p in data.get("pathway_results") or []], + superclass=[str(s) for s in data.get("superclass_results") or []], + class_=[str(c) for c in data.get("class_results") or []], + is_glycoside=bool(data.get("isglycoside", False)), + ) + except (urllib.error.URLError, urllib.error.HTTPError, json.JSONDecodeError, TimeoutError, OSError) as exc: + log.warning("NPClassifier request failed (attempt %d/%d) for %r: %s", attempt, max_retries, smiles, exc) + if attempt < max_retries: + time.sleep(backoff_seconds * attempt) + + log.error("NPClassifier: giving up on %r after %d attempts", smiles, max_retries) + return None diff --git a/database/scripts/taxonomy.py b/database/scripts/taxonomy.py new file mode 100644 index 0000000..7f5555a --- /dev/null +++ b/database/scripts/taxonomy.py @@ -0,0 +1,218 @@ +"""NCBI taxonomy dump download + name/taxid resolution for phylogeny annotation. + +Downloads and parses `taxdump.tar.gz` (https://ftp.ncbi.nlm.nih.gov/pub/taxonomy/taxdump.tar.gz) +once, then resolves organism names <-> NCBI taxids and classifies a taxid's broad "type" +(Bacterium/Archaeon/Fungus/Other) by walking its lineage. MIBiG's JSON gives a taxid +directly (see extract_mibig_compounds.py's `taxonomy.ncbiTaxId`); NPAtlas gives only +genus/species text, resolved here by scientific-name/synonym lookup instead -- so both +sources end up with identically-standardized taxids and canonical names. +""" + +from __future__ import annotations + +import io +import logging +import tarfile +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Iterator + +log = logging.getLogger(__name__) + +TAXDUMP_URL = "https://ftp.ncbi.nlm.nih.gov/pub/taxonomy/taxdump.tar.gz" + +# Fixed NCBI taxids for the lineage nodes used to classify a taxon's broad "type" (see +# TaxonomyDB.type_label_and_taxid) -- these are stable anchor points in NCBI's taxonomy, +# not resolved by name. +TAXID_BACTERIA = 2 +TAXID_ARCHAEA = 2157 +TAXID_FUNGI = 4751 +TYPE_LABELS_BY_TAXID = { + TAXID_BACTERIA: "Bacterium", + TAXID_ARCHAEA: "Archaeon", + TAXID_FUNGI: "Fungus", +} + + +def download_taxdump(dest_dir: str | Path, *, force: bool = False) -> Path: + """Download and extract names.dmp/nodes.dmp from NCBI's taxdump into `dest_dir` + (no-op if both files already exist there, unless `force`).""" + dest_dir = Path(dest_dir).expanduser() + dest_dir.mkdir(parents=True, exist_ok=True) + + names_path = dest_dir / "names.dmp" + nodes_path = dest_dir / "nodes.dmp" + if not force and names_path.exists() and nodes_path.exists(): + return dest_dir + + log.info("downloading NCBI taxdump from %s", TAXDUMP_URL) + with urllib.request.urlopen(TAXDUMP_URL) as resp: + raw = resp.read() + + with tarfile.open(fileobj=io.BytesIO(raw), mode="r:gz") as tar: + for member in tar.getmembers(): + if member.name in ("names.dmp", "nodes.dmp"): + tar.extract(member, path=dest_dir) + + if not names_path.exists() or not nodes_path.exists(): + raise FileNotFoundError(f"taxdump at {TAXDUMP_URL} did not contain names.dmp/nodes.dmp") + + return dest_dir + + +def _iter_dmp_rows(path: Path) -> Iterator[list[str]]: + """NCBI .dmp rows are "\\t|\\t"-separated, terminated by a trailing "\\t|".""" + with open(path, encoding="utf-8", errors="replace") as fh: + for line in fh: + line = line.rstrip("\n").rstrip("\t|") + yield [field.strip() for field in line.split("\t|\t")] + + +@dataclass(frozen=True) +class PhylogenyResolution: + type_label: str | None + type_taxid: str | None + genus: str | None + genus_taxid: str | None + species: str | None + species_taxid: str | None + + +@dataclass +class TaxonomyDB: + """In-memory index over NCBI's taxdump, built once per pipeline run.""" + + parent_by_taxid: dict[int, int] + rank_by_taxid: dict[int, str] + scientific_name_by_taxid: dict[int, str] + taxid_by_name: dict[str, int] # lowercased scientific name/synonym -> taxid + + @classmethod + def load(cls, taxdump_dir: str | Path) -> "TaxonomyDB": + taxdump_dir = Path(taxdump_dir).expanduser() + + parent_by_taxid: dict[int, int] = {} + rank_by_taxid: dict[int, str] = {} + for row in _iter_dmp_rows(taxdump_dir / "nodes.dmp"): + taxid, parent_taxid, rank = int(row[0]), int(row[1]), row[2] + parent_by_taxid[taxid] = parent_taxid + rank_by_taxid[taxid] = rank + + scientific_name_by_taxid: dict[int, str] = {} + taxid_by_name: dict[str, int] = {} + for row in _iter_dmp_rows(taxdump_dir / "names.dmp"): + taxid, name_txt, name_class = int(row[0]), row[1], row[3] + if name_class == "scientific name": + scientific_name_by_taxid[taxid] = name_txt + # Index every name class (scientific name, synonym, common name, ...) -- + # an organism string from a compound source can be any of these. + taxid_by_name.setdefault(name_txt.lower(), taxid) + + log.info("loaded NCBI taxdump: %d nodes, %d names", len(parent_by_taxid), len(taxid_by_name)) + return cls( + parent_by_taxid=parent_by_taxid, + rank_by_taxid=rank_by_taxid, + scientific_name_by_taxid=scientific_name_by_taxid, + taxid_by_name=taxid_by_name, + ) + + def resolve_taxid(self, name: str | None) -> int | None: + """Look up a taxid by scientific name or synonym, case-insensitive.""" + if not name: + return None + return self.taxid_by_name.get(name.strip().lower()) + + def canonical_name(self, taxid: int | None) -> str | None: + if taxid is None: + return None + return self.scientific_name_by_taxid.get(taxid) + + def lineage(self, taxid: int) -> list[int]: + """The ancestor chain from `taxid` (inclusive) up to the root, stopping early + if a cycle or a gap in nodes.dmp is hit.""" + chain = [taxid] + seen = {taxid} + current = taxid + while current in self.parent_by_taxid and current != 1: + parent = self.parent_by_taxid[current] + if parent == current or parent in seen: + break + chain.append(parent) + seen.add(parent) + current = parent + return chain + + def ancestor_at_rank(self, taxid: int | None, rank: str) -> int | None: + """The ancestor of `taxid` (inclusive) whose rank is exactly `rank` (e.g. + "genus", "species"), or None if no such ancestor exists in the lineage.""" + if taxid is None: + return None + for ancestor in self.lineage(taxid): + if self.rank_by_taxid.get(ancestor) == rank: + return ancestor + return None + + def type_label_and_taxid(self, taxid: int | None) -> tuple[str | None, int | None]: + """Classify a taxid's broad type by walking its lineage for one of the fixed + Bacteria/Archaea/Fungi anchor nodes; "Other" if the lineage resolves but hits + none of them (e.g. Viruses, Protista).""" + if taxid is None: + return None, None + for ancestor in self.lineage(taxid): + if ancestor in TYPE_LABELS_BY_TAXID: + return TYPE_LABELS_BY_TAXID[ancestor], ancestor + return "Other", None + + +def resolve_phylogeny( + taxdb: TaxonomyDB | None, + *, + ncbi_tax_id: str | int | None = None, + genus: str | None = None, + species: str | None = None, + fallback_type_label: str | None = None, +) -> PhylogenyResolution: + """Standardize phylogeny fields to NCBI taxids where possible. + + If `ncbi_tax_id` is given (MIBiG), it's the source of truth: genus/species/type and + their taxids are all derived from its lineage, overriding any given genus/species + text. Otherwise (NPAtlas), `genus`/`species` text is resolved to a taxid by name + lookup -- "Genus species" first, falling back to genus alone -- and, if resolved, + canonical names/taxids for every rank are read back off the same lineage, so both + sources end up identically standardized. If nothing resolves (no taxdb, or the + name/taxid isn't found), the given genus/species/fallback_type_label pass through + unchanged with no taxids -- unresolved is not an error here, just unenriched. + """ + if taxdb is None: + return PhylogenyResolution(fallback_type_label, None, genus, None, species, None) + + leaf_taxid: int | None = None + if ncbi_tax_id: + try: + leaf_taxid = int(ncbi_tax_id) + except (TypeError, ValueError): + leaf_taxid = None + elif genus: + query = f"{genus} {species}" if species else genus + leaf_taxid = taxdb.resolve_taxid(query) or taxdb.resolve_taxid(genus) + + if leaf_taxid is None: + return PhylogenyResolution(fallback_type_label, None, genus, None, species, None) + + type_label, type_taxid = taxdb.type_label_and_taxid(leaf_taxid) + + genus_taxid = taxdb.ancestor_at_rank(leaf_taxid, "genus") + genus_name = taxdb.canonical_name(genus_taxid) if genus_taxid else genus + + species_taxid = taxdb.ancestor_at_rank(leaf_taxid, "species") + species_name = taxdb.canonical_name(species_taxid) if species_taxid else species + + return PhylogenyResolution( + type_label=type_label or fallback_type_label, + type_taxid=str(type_taxid) if type_taxid is not None else None, + genus=genus_name, + genus_taxid=str(genus_taxid) if genus_taxid is not None else None, + species=species_name, + species_taxid=str(species_taxid) if species_taxid is not None else None, + ) diff --git a/gui/src/client/src/components/workspace/AlignmentGrid.tsx b/gui/src/client/src/components/workspace/AlignmentGrid.tsx index 2e46c8c..8358a6f 100644 --- a/gui/src/client/src/components/workspace/AlignmentGrid.tsx +++ b/gui/src/client/src/components/workspace/AlignmentGrid.tsx @@ -23,6 +23,86 @@ import { MotifName } from "../MotifName"; import { horizontalScrollSx } from "../../theme/scrollbarSx"; import { buildAlignmentSvg, downloadSvg, type AlignmentSvgRow } from "./alignmentSvgExport"; import { MinimalIconButton} from "../MinimalIconButton"; +import { getEntryAnnotations } from "../../features/database/api"; +import type { EntryAnnotation } from "../../features/database/types"; + +const ANNOTATION_CATEGORY_LABELS: Record = { + phylogeny: "Phylogeny", + biosynthetic_class: "Biosynthetic class", + chemical_class: "Chemical class", + bioactivity: "Bioactivity", +}; + +function groupAnnotationsByCategory(annotations: EntryAnnotation[]): [string, EntryAnnotation[]][] { + const groups = new Map(); + for (const a of annotations) { + const bucket = groups.get(a.category) ?? []; + bucket.push(a); + groups.set(a.category, bucket); + } + return Array.from(groups.entries()); +} + +const AnnotationChips: React.FC<{ entryId: string }> = ({ entryId }) => { + const [annotations, setAnnotations] = React.useState(null); + const [error, setError] = React.useState(null); + + React.useEffect(() => { + const controller = new AbortController(); + getEntryAnnotations(entryId, controller.signal) + .then((resp) => setAnnotations(resp.results)) + .catch((err) => { + if (controller.signal.aborted) return; + setError(err instanceof Error ? err.message : String(err)); + }); + return () => controller.abort(); + }, [entryId]); + + if (error) { + return ( + + Couldn't load annotations: {error} + + ); + } + + if (annotations === null) { + return null; + } + + if (annotations.length === 0) { + return null; + } + + return ( + + {groupAnnotationsByCategory(annotations).map(([category, items]) => ( + + + {ANNOTATION_CATEGORY_LABELS[category] ?? category} + + {items.map((item) => + item.url ? ( + + ) : ( + + ) + )} + + ))} + + ); +}; // Shared vertical sizing (padding, border width, font size, line height) so every // row -- label, sequence cells, and score -- resolves to the exact same height. @@ -327,6 +407,7 @@ export function ResultRow({ )} )} + = { compound: "Compounds", @@ -56,6 +56,29 @@ const ChartCard: React.FC<{ title: string; description?: string; children: React ); +const AnnotationBarCard: React.FC<{ + title: string; + description?: string; + counts: Count[]; + color: string; + emptyMessage: string; +}> = ({ title, description, counts, color, emptyMessage }) => ( + + {counts.length === 0 ? ( + + {emptyMessage} + + ) : ( + ({ label: c.label, count: c.count }))} + xAxis={[{ scaleType: "band", dataKey: "label" }]} + series={[{ dataKey: "count", color }]} + height={220} + /> + )} + +); + export const WorkspaceHome: React.FC = () => { const theme = useTheme(); const palette = theme.vars || theme; @@ -252,45 +275,110 @@ export const WorkspaceHome: React.FC = () => { /> + + Phylogeny + - - {annotationStats.phylogenyTypeCounts.length === 0 ? ( - No phylogeny annotations yet. - ) : ( - ({ label: c.label, count: c.count }))} - xAxis={[{ scaleType: "band", dataKey: "label" }]} - series={[{ dataKey: "count", color: chartColors[0] }]} - height={220} - /> - )} - + + + + + + + Chemical class + + + NPClassifier, predicted from every compound's own structure + + + + + + - - {annotationStats.topGenera.length === 0 ? ( - No phylogeny annotations yet. - ) : ( - ({ label: c.label, count: c.count }))} - xAxis={[{ scaleType: "band", dataKey: "label" }]} - series={[{ dataKey: "count", color: chartColors[1] }]} - height={220} - /> - )} - + + Biosynthetic class + + + MIBiG's own coarse label (PKS / NRPS / RiPP / ...), a separate classification from NPClassifier's chemical class above + + + + - - {annotationStats.chemicalClassCounts.length === 0 ? ( - No chemical class annotations yet. - ) : ( - ({ label: c.label, count: c.count }))} - xAxis={[{ scaleType: "band", dataKey: "label" }]} - series={[{ dataKey: "count", color: chartColors[2] }]} - height={220} - /> - )} - + + Bioactivity + + + ChEMBL + ChEBI, looked up by structure (InChIKey) + + + + + + )} diff --git a/gui/src/client/src/features/database/api.ts b/gui/src/client/src/features/database/api.ts index d4c48a2..569795f 100644 --- a/gui/src/client/src/features/database/api.ts +++ b/gui/src/client/src/features/database/api.ts @@ -1,5 +1,12 @@ import { getJson } from "../http"; -import { AnnotationStatsRespSchema, DatabaseStatsRespSchema, type AnnotationStatsResp, type DatabaseStatsResp } from "./types"; +import { + AnnotationStatsRespSchema, + DatabaseStatsRespSchema, + EntryAnnotationsRespSchema, + type AnnotationStatsResp, + type DatabaseStatsResp, + type EntryAnnotationsResp, +} from "./types"; export async function getDatabaseStats(signal?: AbortSignal): Promise { return getJson("/api/databaseStats", DatabaseStatsRespSchema, signal); @@ -8,3 +15,8 @@ export async function getDatabaseStats(signal?: AbortSignal): Promise { return getJson("/api/annotationStats", AnnotationStatsRespSchema, signal); } + +export async function getEntryAnnotations(entryId: string, signal?: AbortSignal): Promise { + const params = new URLSearchParams({ entryId }); + return getJson(`/api/entryAnnotations?${params.toString()}`, EntryAnnotationsRespSchema, signal); +} diff --git a/gui/src/client/src/features/database/types.ts b/gui/src/client/src/features/database/types.ts index e068db7..ca680e9 100644 --- a/gui/src/client/src/features/database/types.ts +++ b/gui/src/client/src/features/database/types.ts @@ -23,7 +23,30 @@ export const AnnotationStatsRespSchema = z.object({ withoutAnnotationCount: z.number().int().nonnegative(), countsByCategory: z.array(CountSchema), phylogenyTypeCounts: z.array(CountSchema), - topGenera: z.array(CountSchema), - chemicalClassCounts: z.array(CountSchema), + phylogenyGenusCounts: z.array(CountSchema), + phylogenySpeciesCounts: z.array(CountSchema), + biosyntheticClassCounts: z.array(CountSchema), + chemicalClassPathwayCounts: z.array(CountSchema), + chemicalClassSuperclassCounts: z.array(CountSchema), + chemicalClassClassCounts: z.array(CountSchema), + bioactivityAtcCounts: z.array(CountSchema), + bioactivityMaxPhaseCounts: z.array(CountSchema), + bioactivityBiologicalRoleCounts: z.array(CountSchema), + bioactivityChemicalRoleCounts: z.array(CountSchema), }); export type AnnotationStatsResp = z.output; + +export const EntryAnnotationSchema = z.object({ + id: z.string(), + category: z.string(), + rank: z.string().nullable(), + label: z.string(), + externalId: z.string().nullable(), + url: z.string().nullable(), +}); +export type EntryAnnotation = z.output; + +export const EntryAnnotationsRespSchema = z.object({ + results: z.array(EntryAnnotationSchema), +}); +export type EntryAnnotationsResp = z.output; diff --git a/gui/src/server/app.py b/gui/src/server/app.py index 4011f08..7b15b4e 100644 --- a/gui/src/server/app.py +++ b/gui/src/server/app.py @@ -41,6 +41,7 @@ from routes.rate_limit import limiter, RATE_LIMIT_REJECTIONS from routes.rules import blp_rule_set, blp_generate_backbone from routes.enrichment import blp_entry_search, blp_enrichment_analysis +from routes.entry_annotations import blp_entry_annotations # Initialize the Flask app @@ -258,6 +259,7 @@ def ready() -> tuple[dict[str, str], int]: app.register_blueprint(blp_generate_backbone) app.register_blueprint(blp_entry_search) app.register_blueprint(blp_enrichment_analysis) +app.register_blueprint(blp_entry_annotations) # The two rate-limit tiers on top of the app-wide default (see routes/rate_limit.py) # are applied as @limiter.limit(...) decorators directly on each route function, in diff --git a/gui/src/server/routes/entry_annotations.py b/gui/src/server/routes/entry_annotations.py new file mode 100644 index 0000000..fb90951 --- /dev/null +++ b/gui/src/server/routes/entry_annotations.py @@ -0,0 +1,71 @@ +"""Entry annotations: every phylogeny/biosynthetic_class/chemical_class/bioactivity +term linked to one entry, each with a link to that term's source website where one +exists -- used by the Discovery tab's expanded result view ("what do we know about +this compound/BGC"). +""" + +from flask import Blueprint, Response, jsonify, request + +from routes.database import open_retromol_db + +blp_entry_annotations = Blueprint("entry_annotations", __name__) + +NCBI_TAXONOMY_URL = "https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=Info&id={id}" +CHEMBL_COMPOUND_URL = "https://www.ebi.ac.uk/chembl/compound_report_card/{id}/" +WHO_ATC_INDEX_URL = "https://www.whocc.no/atc_ddd_index/?code={id}" +CHEBI_ENTITY_URL = "https://www.ebi.ac.uk/chebi/searchId.do?chebiId={id}" + + +def _annotation_url(category: str, rank: str | None, external_id: str | None) -> str | None: + """Build a "view on " link for one term, if its category/rank has a known + external database and it resolved an id there. biosynthetic_class and chemical_class + (NPClassifier) have no canonical per-label public page, so those always return None.""" + if not external_id: + return None + if category == "phylogeny": + return NCBI_TAXONOMY_URL.format(id=external_id) + if category == "bioactivity": + if rank == "chembl_max_phase": + return CHEMBL_COMPOUND_URL.format(id=external_id) + if rank == "chembl_atc": + return WHO_ATC_INDEX_URL.format(id=external_id) + if rank in ("chebi_biological_role", "chebi_chemical_role"): + return CHEBI_ENTITY_URL.format(id=external_id) + return None + + +@blp_entry_annotations.get("/api/entryAnnotations") +def entry_annotations() -> tuple[Response, int]: + """ + Every annotation term linked to one entry, for the Discovery tab's expanded result view. + + :return: a tuple containing the annotation list and an HTTP status code + """ + entry_id = (request.args.get("entryId") or "").strip() + if not entry_id: + return jsonify({"error": "entryId is required"}), 400 + + try: + with open_retromol_db() as db: + terms = db.entry_annotation_terms(entry_id) + except Exception as e: + return jsonify({"error": str(e)}), 503 + + return ( + jsonify( + { + "results": [ + { + "id": t.id, + "category": t.category, + "rank": t.rank, + "label": t.label, + "externalId": t.external_id, + "url": _annotation_url(t.category, t.rank, t.external_id), + } + for t in terms + ] + } + ), + 200, + ) diff --git a/src/retromol_database/duckdb.py b/src/retromol_database/duckdb.py index b16c3c7..4a75c04 100644 --- a/src/retromol_database/duckdb.py +++ b/src/retromol_database/duckdb.py @@ -68,6 +68,11 @@ class AnnotationTerm: rank: str | None label: str parent_id: str | None + # An id in whatever external database this term comes from (NCBI taxid for + # phylogeny; ATC code/ChEBI accession/ChEMBL id for bioactivity), for linking out + # to that database's own page -- None for categories with no such external page + # (biosynthetic_class, chemical_class). + external_id: str | None = None @dataclass(frozen=True) @@ -75,9 +80,26 @@ class AnnotationStats: with_annotation_count: int without_annotation_count: int counts_by_category: list[Count] + # Phylogeny: one chart per rank. phylogeny_type_counts: list[Count] - top_genera: list[Count] - chemical_class_counts: list[Count] + phylogeny_genus_counts: list[Count] + phylogeny_species_counts: list[Count] + # MIBiG's own coarse biosynthetic-class label -- a distinct category from chemical_class + # below, not a rank within it (see biosynthetic_class_annotations table comment). + biosynthetic_class_counts: list[Count] + # Chemical class: NPClassifier's three structure-derived levels (see + # database/scripts/annotate_npclassifier.py), kept as separate charts rather than + # pooled, since they're different ranks within the same classification scheme. + chemical_class_pathway_counts: list[Count] + chemical_class_superclass_counts: list[Count] + chemical_class_class_counts: list[Count] + # Bioactivity: ChEMBL's two signals (see database/scripts/annotate_chembl.py) -- + # WHO ATC therapeutic category and clinical-development phase -- plus ChEBI's role + # ontology (see database/scripts/annotate_chebi.py). + bioactivity_atc_counts: list[Count] + bioactivity_max_phase_counts: list[Count] + bioactivity_biological_role_counts: list[Count] + bioactivity_chemical_role_counts: list[Count] def _normalize_entry_type(entry_type: str) -> EntryType: @@ -165,26 +187,149 @@ def create_schema(self) -> None: ) """ ) + # Four dedicated annotation tables, one per category, each with its own typed + # columns instead of a single generic label -- see RetroMolDuckDB.add_phylogeny_annotation + # / add_bioactivity_annotation / add_biosynthetic_class_annotation / add_chemical_class_annotation. + # Phylogeny is one row per entry (an entry has exactly one organism); the other three + # are multi-valued per entry. biosynthetic_class/chemical_class are populated for + # compounds only (they describe the molecule, not the organism/cluster) -- enforced + # by pipeline callers, not by this schema. Bioactivity population logic isn't wired + # up yet; that table exists ahead of that. self.con.execute( """ - CREATE TABLE IF NOT EXISTS annotation_terms ( - id VARCHAR PRIMARY KEY, - category VARCHAR NOT NULL, - rank VARCHAR, + CREATE TABLE IF NOT EXISTS phylogeny_annotations ( + entry_id VARCHAR PRIMARY KEY, + type_label VARCHAR, + type_taxid VARCHAR, + genus VARCHAR, + genus_taxid VARCHAR, + species VARCHAR, + species_taxid VARCHAR + ) + """ + ) + # `level` distinguishes bioactivity's four signals (see + # database/scripts/annotate_chembl.py / annotate_chebi.py): ChEMBL's + # "chembl_max_phase" (clinical-development stage -- "Approved"/"Phase 3"/... -- + # only stored when meaningful, same presence-only convention as chemical_class's + # is_glycoside) and "chembl_atc" (WHO ATC therapeutic category), plus ChEBI's + # "chebi_biological_role" and "chebi_chemical_role" (its role ontology, under + # CHEBI:24432/CHEBI:51086). All four are multi-valued per entry except + # chembl_max_phase. `external_id` is the id in that row's own external database + # (an ATC code, a ChEBI accession, or a ChEMBL id for the max_phase row's own + # compound page) -- for building a "view on " link; null when unresolved. + self.con.execute( + """ + CREATE TABLE IF NOT EXISTS bioactivity_annotations ( + entry_id VARCHAR NOT NULL, + level VARCHAR NOT NULL, label VARCHAR NOT NULL, - parent_id VARCHAR + external_id VARCHAR, + PRIMARY KEY (entry_id, level, label) ) """ ) + # MIBiG's own coarse biosynthetic-class label (PKS/NRPS/RiPP/terpene/saccharide/other, + # from the BGC's annotated biosynthesis machinery) -- a distinct classification scheme + # from chemical_class below (NPClassifier's structure-derived classification of the + # compound itself), not a rank/level within it. self.con.execute( """ - CREATE TABLE IF NOT EXISTS entry_annotations ( + CREATE TABLE IF NOT EXISTS biosynthetic_class_annotations ( entry_id VARCHAR NOT NULL, - term_id VARCHAR NOT NULL, - PRIMARY KEY (entry_id, term_id) + label VARCHAR NOT NULL, + PRIMARY KEY (entry_id, label) + ) + """ + ) + # `level` distinguishes NPClassifier's "pathway"/"superclass"/"class"/"is_glycoside" + # (structure-derived, from a compound's own SMILES -- see + # database/scripts/annotate_npclassifier.py) -- every chemical_class row comes from + # NPClassifier now (MIBiG's biosynthetic class lives in biosynthetic_class_annotations + # above instead). The three list-valued levels can each carry more than one label per + # entry (hence label being part of the primary key, same as bioactivity); is_glycoside + # is stored as a single ("is_glycoside", "Yes") row only when true -- absence means + # false/unclassified, the same presence-only convention every other annotation table uses. + self.con.execute( + """ + CREATE TABLE IF NOT EXISTS chemical_class_annotations ( + entry_id VARCHAR NOT NULL, + level VARCHAR NOT NULL, + label VARCHAR NOT NULL, + PRIMARY KEY (entry_id, level, label) ) """ ) + # Read-only views reconstructing the old generic (id/category/rank/label/parent_id) + # term shape and (entry_id, term_id) links across all three tables above -- so + # annotation_stats/enrichment queries (which are category-agnostic) don't need to + # know about the dedicated tables. term_id encodings match the pre-migration scheme + # (e.g. "phylogeny:genus:bacterium:streptomyces") so existing consumers are unaffected. + self.con.execute( + """ + CREATE OR REPLACE VIEW annotation_terms AS + SELECT id, category, rank, label, parent_id, any_value(taxid) AS external_id + FROM ( + SELECT + 'phylogeny:type:' || lower(type_label) AS id, + 'phylogeny' AS category, 'type' AS rank, type_label AS label, + CAST(NULL AS VARCHAR) AS parent_id, type_taxid AS taxid + FROM phylogeny_annotations WHERE type_label IS NOT NULL + UNION ALL + SELECT + 'phylogeny:genus:' || lower(type_label) || ':' || lower(genus) AS id, + 'phylogeny' AS category, 'genus' AS rank, genus AS label, + 'phylogeny:type:' || lower(type_label) AS parent_id, genus_taxid AS taxid + FROM phylogeny_annotations WHERE type_label IS NOT NULL AND genus IS NOT NULL + UNION ALL + SELECT + 'phylogeny:species:' || lower(type_label) || ':' || lower(genus) || ':' || lower(species) AS id, + 'phylogeny' AS category, 'species' AS rank, species AS label, + 'phylogeny:genus:' || lower(type_label) || ':' || lower(genus) AS parent_id, species_taxid AS taxid + FROM phylogeny_annotations + WHERE type_label IS NOT NULL AND genus IS NOT NULL AND species IS NOT NULL + UNION ALL + SELECT + 'bioactivity:' || level || ':' || lower(label) AS id, + 'bioactivity' AS category, level AS rank, label, + CAST(NULL AS VARCHAR) AS parent_id, external_id AS taxid + FROM bioactivity_annotations + UNION ALL + SELECT + 'biosynthetic_class:' || lower(label) AS id, + 'biosynthetic_class' AS category, CAST(NULL AS VARCHAR) AS rank, label, + CAST(NULL AS VARCHAR) AS parent_id, CAST(NULL AS VARCHAR) AS taxid + FROM biosynthetic_class_annotations + UNION ALL + SELECT + 'chemical_class:' || level || ':' || lower(label) AS id, + 'chemical_class' AS category, level AS rank, label, + CAST(NULL AS VARCHAR) AS parent_id, CAST(NULL AS VARCHAR) AS taxid + FROM chemical_class_annotations + ) + GROUP BY id, category, rank, label, parent_id + """ + ) + self.con.execute( + """ + CREATE OR REPLACE VIEW entry_annotations AS + SELECT entry_id, 'phylogeny:type:' || lower(type_label) AS term_id + FROM phylogeny_annotations WHERE type_label IS NOT NULL + UNION ALL + SELECT entry_id, 'phylogeny:genus:' || lower(type_label) || ':' || lower(genus) + FROM phylogeny_annotations WHERE type_label IS NOT NULL AND genus IS NOT NULL + UNION ALL + SELECT entry_id, 'phylogeny:species:' || lower(type_label) || ':' || lower(genus) || ':' || lower(species) + FROM phylogeny_annotations + WHERE type_label IS NOT NULL AND genus IS NOT NULL AND species IS NOT NULL + UNION ALL + SELECT entry_id, 'bioactivity:' || level || ':' || lower(label) FROM bioactivity_annotations + UNION ALL + SELECT entry_id, 'biosynthetic_class:' || lower(label) FROM biosynthetic_class_annotations + UNION ALL + SELECT entry_id, 'chemical_class:' || level || ':' || lower(label) FROM chemical_class_annotations + """ + ) def add_entry( self, @@ -246,92 +391,82 @@ def bgc_content_hash_exists(self, content_hash: str) -> bool: def count(self) -> int: return int(self.con.execute("SELECT count(*) FROM entries").fetchone()[0]) - def add_annotation_term( - self, - *, - term_id: str, - category: str, - label: str, - rank: str | None = None, - parent_id: str | None = None, - ) -> str: - """Add (or no-op if already present) a single annotation term. `term_id` is a - caller-supplied deterministic slug (e.g. "phylogeny:genus:bacterium:streptomyces") - so repeated calls for the same term across many entries are idempotent.""" - self.con.execute( - """ - INSERT INTO annotation_terms (id, category, rank, label, parent_id) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT (id) DO NOTHING - """, - [term_id, category, rank, label, parent_id], - ) - return term_id - - def link_entry_annotation(self, entry_id: str, term_id: str) -> None: - self.con.execute( - """ - INSERT INTO entry_annotations (entry_id, term_id) - VALUES (?, ?) - ON CONFLICT (entry_id, term_id) DO NOTHING - """, - [entry_id, term_id], - ) - def add_phylogeny_annotation( self, entry_id: str, *, type_label: str | None, - genus: str | None, - species: str | None, + type_taxid: str | None = None, + genus: str | None = None, + genus_taxid: str | None = None, + species: str | None = None, + species_taxid: str | None = None, ) -> None: - """Link `entry_id` to whichever of type/genus/species is resolvable, linking every - level (not just the most specific one) so term-level queries at any rank don't need - to walk the parent_id chain. `species` is ignored if `genus` isn't given.""" + """Set (or replace) `entry_id`'s single phylogeny row. `species`/`species_taxid` + are only meaningful -- and only surfaced by the annotation_terms/entry_annotations + views -- when `genus` is also given; a taxid is optional at every rank (e.g. NPAtlas + names that don't resolve against the NCBI taxdump still store their raw genus/species + text with a NULL taxid). No-ops if `type_label` isn't resolvable.""" if not type_label: return - type_id = self.add_annotation_term( - term_id=f"phylogeny:type:{type_label.lower()}", - category="phylogeny", - rank="type", - label=type_label, + self.con.execute( + """ + INSERT INTO phylogeny_annotations + (entry_id, type_label, type_taxid, genus, genus_taxid, species, species_taxid) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (entry_id) DO UPDATE SET + type_label = excluded.type_label, + type_taxid = excluded.type_taxid, + genus = excluded.genus, + genus_taxid = excluded.genus_taxid, + species = excluded.species, + species_taxid = excluded.species_taxid + """, + [entry_id, type_label, type_taxid, genus, genus_taxid, species, species_taxid], ) - self.link_entry_annotation(entry_id, type_id) - if not genus: - return - - genus_id = self.add_annotation_term( - term_id=f"phylogeny:genus:{type_label.lower()}:{genus.lower()}", - category="phylogeny", - rank="genus", - label=genus, - parent_id=type_id, + def add_bioactivity_annotation( + self, entry_id: str, *, level: str, label: str, external_id: str | None = None + ) -> None: + """Link `entry_id` (a compound) to a bioactivity label at the given `level` (e.g. + "chembl_max_phase", "chembl_atc", "chebi_biological_role", "chebi_chemical_role" -- + see database/scripts/annotate_chembl.py / annotate_chebi.py). Compounds only -- + callers must not use this for bgc entries.""" + self.con.execute( + """ + INSERT INTO bioactivity_annotations (entry_id, level, label, external_id) + VALUES (?, ?, ?, ?) + ON CONFLICT (entry_id, level, label) DO NOTHING + """, + [entry_id, level, label, external_id], ) - self.link_entry_annotation(entry_id, genus_id) - if not species: - return - - species_id = self.add_annotation_term( - term_id=f"phylogeny:species:{type_label.lower()}:{genus.lower()}:{species.lower()}", - category="phylogeny", - rank="species", - label=species, - parent_id=genus_id, + def add_biosynthetic_class_annotation(self, entry_id: str, label: str) -> None: + """Link `entry_id` (a compound) to a MIBiG biosynthetic-class label (PKS/NRPS/RiPP/...). + Compounds only -- callers must not use this for bgc entries.""" + self.con.execute( + """ + INSERT INTO biosynthetic_class_annotations (entry_id, label) + VALUES (?, ?) + ON CONFLICT (entry_id, label) DO NOTHING + """, + [entry_id, label], ) - self.link_entry_annotation(entry_id, species_id) - - def add_flat_annotation(self, entry_id: str, *, category: str, label: str) -> None: - """Link `entry_id` to a single, non-hierarchical term (e.g. chemical_class, bioactivity).""" - term_id = self.add_annotation_term( - term_id=f"{category}:{label.lower()}", - category=category, - label=label, + + def add_chemical_class_annotation(self, entry_id: str, *, level: str, label: str) -> None: + """Link `entry_id` (a compound) to a chemical-class label at the given `level` + (e.g. "biosyn_class", or NPClassifier's "pathway"/"superclass"/"class"/"is_glycoside" -- + see the chemical_class_annotations table comment in create_schema). Compounds only -- + callers must not use this for bgc entries.""" + self.con.execute( + """ + INSERT INTO chemical_class_annotations (entry_id, level, label) + VALUES (?, ?, ?) + ON CONFLICT (entry_id, level, label) DO NOTHING + """, + [entry_id, level, label], ) - self.link_entry_annotation(entry_id, term_id) def count_entries_by_type(self, entry_types: Sequence[str]) -> int: if not entry_types: @@ -386,7 +521,7 @@ def annotation_terms_by_ids(self, term_ids: Sequence[str]) -> dict[str, Annotati rows = self.con.execute( """ - SELECT id, category, rank, label, parent_id + SELECT id, category, rank, label, parent_id, external_id FROM annotation_terms WHERE id IN (SELECT UNNEST(?)) """, @@ -394,11 +529,34 @@ def annotation_terms_by_ids(self, term_ids: Sequence[str]) -> dict[str, Annotati ).fetchall() return { str(row[0]): AnnotationTerm( - id=str(row[0]), category=str(row[1]), rank=row[2], label=str(row[3]), parent_id=row[4] + id=str(row[0]), category=str(row[1]), rank=row[2], label=str(row[3]), parent_id=row[4], + external_id=row[5], ) for row in rows } + def entry_annotation_terms(self, entry_id: str) -> list[AnnotationTerm]: + """Every annotation term linked to `entry_id`, across all four categories -- + the "show me everything known about this compound/bgc" query used by the + Discovery tab's expanded result view.""" + rows = self.con.execute( + """ + SELECT t.id, t.category, t.rank, t.label, t.parent_id, t.external_id + FROM entry_annotations ea + JOIN annotation_terms t ON t.id = ea.term_id + WHERE ea.entry_id = ? + ORDER BY t.category, t.rank NULLS FIRST, t.label + """, + [entry_id], + ).fetchall() + return [ + AnnotationTerm( + id=str(row[0]), category=str(row[1]), rank=row[2], label=str(row[3]), parent_id=row[4], + external_id=row[5], + ) + for row in rows + ] + def search_entries( self, query: str, *, entry_type: str | None = None, limit: int = 100 ) -> list[Entry]: @@ -431,6 +589,34 @@ def search_entries( sources_by_id = self._sources_for_entry_ids([str(row[0]) for row in rows]) return [_entry_from_row(row, sources_by_id.get(str(row[0]), [])) for row in rows] + def _label_counts(self, *, category: str, rank: str | None = None, limit: int | None = None) -> list[Count]: + """Distinct-entry counts per label, for one (category, rank) slice of + annotation_terms/entry_annotations -- the building block for every chart on the + Dashboard's Annotations section.""" + where_sql = "WHERE t.category = ?" + params: list[object] = [category] + if rank is not None: + where_sql += " AND t.rank = ?" + params.append(rank) + + limit_sql = " LIMIT ?" if limit is not None else "" + if limit is not None: + params.append(limit) + + rows = self.con.execute( + f""" + SELECT t.label, count(DISTINCT ea.entry_id) + FROM entry_annotations ea + JOIN annotation_terms t ON t.id = ea.term_id + {where_sql} + GROUP BY t.label + ORDER BY count(DISTINCT ea.entry_id) DESC + {limit_sql} + """, + params, + ).fetchall() + return [Count(label=str(row[0]), count=int(row[1])) for row in rows] + def annotation_stats(self) -> AnnotationStats: """Summary counts over annotation_terms/entry_annotations, for the Dashboard.""" with_annotation_count = int( @@ -449,50 +635,25 @@ def annotation_stats(self) -> AnnotationStats: ).fetchall() counts_by_category = [Count(label=str(row[0]), count=int(row[1])) for row in category_rows] - phylogeny_type_rows = self.con.execute( - """ - SELECT t.label, count(DISTINCT ea.entry_id) - FROM entry_annotations ea - JOIN annotation_terms t ON t.id = ea.term_id - WHERE t.category = 'phylogeny' AND t.rank = 'type' - GROUP BY t.label - ORDER BY count(DISTINCT ea.entry_id) DESC - """ - ).fetchall() - phylogeny_type_counts = [Count(label=str(row[0]), count=int(row[1])) for row in phylogeny_type_rows] - - genus_rows = self.con.execute( - """ - SELECT t.label, count(DISTINCT ea.entry_id) - FROM entry_annotations ea - JOIN annotation_terms t ON t.id = ea.term_id - WHERE t.category = 'phylogeny' AND t.rank = 'genus' - GROUP BY t.label - ORDER BY count(DISTINCT ea.entry_id) DESC - LIMIT 15 - """ - ).fetchall() - top_genera = [Count(label=str(row[0]), count=int(row[1])) for row in genus_rows] - - chem_class_rows = self.con.execute( - """ - SELECT t.label, count(DISTINCT ea.entry_id) - FROM entry_annotations ea - JOIN annotation_terms t ON t.id = ea.term_id - WHERE t.category = 'chemical_class' - GROUP BY t.label - ORDER BY count(DISTINCT ea.entry_id) DESC - """ - ).fetchall() - chemical_class_counts = [Count(label=str(row[0]), count=int(row[1])) for row in chem_class_rows] - return AnnotationStats( with_annotation_count=with_annotation_count, without_annotation_count=without_annotation_count, counts_by_category=counts_by_category, - phylogeny_type_counts=phylogeny_type_counts, - top_genera=top_genera, - chemical_class_counts=chemical_class_counts, + phylogeny_type_counts=self._label_counts(category="phylogeny", rank="type"), + phylogeny_genus_counts=self._label_counts(category="phylogeny", rank="genus", limit=15), + phylogeny_species_counts=self._label_counts(category="phylogeny", rank="species", limit=15), + biosynthetic_class_counts=self._label_counts(category="biosynthetic_class"), + chemical_class_pathway_counts=self._label_counts(category="chemical_class", rank="pathway", limit=15), + chemical_class_superclass_counts=self._label_counts(category="chemical_class", rank="superclass", limit=15), + chemical_class_class_counts=self._label_counts(category="chemical_class", rank="class", limit=15), + bioactivity_atc_counts=self._label_counts(category="bioactivity", rank="chembl_atc", limit=15), + bioactivity_max_phase_counts=self._label_counts(category="bioactivity", rank="chembl_max_phase"), + bioactivity_biological_role_counts=self._label_counts( + category="bioactivity", rank="chebi_biological_role", limit=15 + ), + bioactivity_chemical_role_counts=self._label_counts( + category="bioactivity", rank="chebi_chemical_role", limit=15 + ), ) def stats(self) -> DatabaseStats: From 41cbb37f54e3fa0565658ea94418f0238e568b29 Mon Sep 17 00:00:00 2001 From: David Meijer Date: Wed, 26 Aug 2026 17:53:34 -0400 Subject: [PATCH 05/11] ADD: env and Slurm profile for database construction --- database/envs/retromol.yaml | 34 +++++++++++ database/profiles/slurm/config.yaml | 94 +++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+) create mode 100644 database/envs/retromol.yaml create mode 100644 database/profiles/slurm/config.yaml diff --git a/database/envs/retromol.yaml b/database/envs/retromol.yaml new file mode 100644 index 0000000..79db115 --- /dev/null +++ b/database/envs/retromol.yaml @@ -0,0 +1,34 @@ +# Conda env for running database/Snakemake (the database-construction pipeline only -- +# this has no bearing on the GUI/webapp). +# +# Every rule in that Snakefile uses Snakemake's `run:` directive (inline Python that +# imports and calls database/scripts/*.py functions directly), not `shell:`. Snakemake +# executes `run:` blocks in its own process rather than a subprocess, so a per-rule +# `conda:` env (the --use-conda mechanism) would NOT actually reach any of this +# pipeline's own code -- only `shell()` calls made from *within* a `run:` block, which +# this pipeline never makes. There is deliberately no per-rule conda: directive in +# database/Snakemake, and no --use-conda flag in the invocation below, because of this. +# +# This env is instead the one Snakemake itself runs in, so it needs both Snakemake and +# every runtime dependency database/scripts/*.py imports (rdkit, duckdb, etc. -- all +# already listed in the repo's own pyproject.toml, pulled in by the editable install +# below rather than duplicated here). +# +# Usage: +# +# conda env create -f database/envs/retromol.yaml +# conda activate retromol +# pip install -e . +# snakemake -p -s database/Snakemake --configfile database/config.yaml \ +# --workflow-profile database/profiles/slurm +name: retromol + +channels: + - conda-forge + +dependencies: + - python=3.11 + - pip + - snakemake + - snakemake-executor-plugin-slurm + - mamba diff --git a/database/profiles/slurm/config.yaml b/database/profiles/slurm/config.yaml new file mode 100644 index 0000000..d19bcfb --- /dev/null +++ b/database/profiles/slurm/config.yaml @@ -0,0 +1,94 @@ +# Snakemake workflow profile for running database/Snakemake on a Slurm cluster. +# +# Usage (from the repo root, with database/envs/retromol.yaml's env active -- see that +# file's own comment for why there's no --use-conda here): +# +# conda activate retromol +# snakemake -p -s database/Snakemake --configfile database/config.yaml \ +# --workflow-profile database/profiles/slurm +executor: slurm +jobs: 6 # concurrent Slurm jobs -- mainly matters for the independent download_* rules; + # everything downstream of them is a strict chain (create_db -> load_* -> + # annotate_*), serialized through marker files because DuckDB doesn't support + # concurrent writers (see database/Snakemake's module docstring), so raising + # this past ~6-8 buys little. +latency-wait: 60 +rerun-incomplete: true +printshellcmds: true +# Independent rules (e.g. the download_* rules, or npclassifier/chembl/chebi annotation +# vs. each other before they're serialized) shouldn't all be killed by one unrelated +# rule failing/timing out -- let everything already running finish. +keep-going: true + +default-resources: + slurm_account: null # fill in your account, if your cluster requires one + slurm_partition: null # fill in your cluster's partition name + runtime: 60 + mem_mb: 4000 + disk_mb: 20000 + +# parse_npatlas/parse_mibig_compounds/parse_mibig_gbks' worker counts come from +# database/config.yaml's `compute:` section (PARSE_COMPOUNDS_WORKERS/PARSE_GBKS_WORKERS), +# not from this profile -- deliberately not duplicated here as set-threads to avoid two +# sources of truth for the same value; edit config.yaml instead. +set-resources: + download_npatlas: + runtime: 120 + mem_mb: 4000 + disk_mb: 10000 + download_mibig_json: + runtime: 60 + mem_mb: 2000 + disk_mb: 5000 + download_mibig_gbk: + runtime: 60 + mem_mb: 2000 + disk_mb: 5000 + download_taxdump: + runtime: 60 + mem_mb: 2000 + disk_mb: 5000 + download_chembl: + # ~5.4GB compressed download, extracted to a several-GB sqlite file (see + # database/scripts/chembl.py) -- the heaviest single download in this pipeline. + runtime: 240 + mem_mb: 4000 + disk_mb: 40000 + download_chebi: + # A few hundred MB total across compounds/structures/relation flat files. + runtime: 60 + mem_mb: 2000 + disk_mb: 5000 + parse_npatlas: + runtime: 720 + mem_mb: 16000 + parse_mibig_compounds: + runtime: 240 + mem_mb: 8000 + parse_mibig_gbks: + runtime: 480 + mem_mb: 16000 + load_npatlas_compounds: + runtime: 60 + mem_mb: 4000 + load_mibig_compounds: + runtime: 60 + mem_mb: 4000 + load_mibig_bgcs: + runtime: 60 + mem_mb: 4000 + annotate_npclassifier: + # Rate-limited API calls (see database/config.yaml's npclassifier.requests_per_second) + # -- mostly wall-clock waiting on the network, not compute. + runtime: 1440 + mem_mb: 2000 + annotate_chembl: + # Local sqlite reads against the downloaded ChEMBL release -- fast, but give it + # enough runtime to walk every compound entry once. + runtime: 240 + mem_mb: 4000 + annotate_chebi: + # Loads all of ChEBI's flat files into memory once (see chebi.py's ChebiDB.load) -- + # a few hundred MB, generous headroom here. + runtime: 120 + mem_mb: 8000 From f2f686f629b4b98112ac9b08b8d2b6536805894e Mon Sep 17 00:00:00 2001 From: David Meijer Date: Wed, 26 Aug 2026 21:19:51 -0400 Subject: [PATCH 06/11] UPD: increase prep rate npclassifier --- database/Snakemake | 11 +++ database/config.yaml | 16 ++-- database/scripts/annotate_npclassifier.py | 97 +++++++++++++++++------ 3 files changed, 95 insertions(+), 29 deletions(-) diff --git a/database/Snakemake b/database/Snakemake index 2496332..8560202 100644 --- a/database/Snakemake +++ b/database/Snakemake @@ -46,6 +46,7 @@ TAXONOMY_ENABLED = config.get("taxonomy", {}).get("enabled", False) TAXDUMP_DIR = WORKDIR / "taxdump" NPCLASSIFIER_REQUESTS_PER_SECOND = config.get("npclassifier", {}).get("requests_per_second", 2.0) +NPCLASSIFIER_WORKERS = config.get("npclassifier", {}).get("workers", 8) CHEMBL_SQLITE_URL = config.get("chembl", {}).get("sqlite_url") CHEMBL_DIR = WORKDIR / "chembl" @@ -336,6 +337,15 @@ rule annotate_npclassifier: prev=MARKERS / "bgcs_loaded.done" output: marker=touch(MARKERS / "npclassifier_annotated.done") + # Declared like parse_npatlas/parse_mibig_gbks's worker counts, even though this + # rule is I/O-bound rather than CPU-bound: without a `threads:` declaration, + # Snakemake assumes a 1-core job while it actually opens NPCLASSIFIER_WORKERS + # concurrent connections, so `--cores` wouldn't see or cap it. Declaring it here + # means `--cores` fewer than npclassifier.workers automatically caps this job's + # `threads` value at run time -- passed through below instead of the raw config + # number, so a run with e.g. `--cores 6` gets at most 6 workers even if + # npclassifier.workers says 8. + threads: NPCLASSIFIER_WORKERS run: if NPCLASSIFIER_ENABLED: import annotate_npclassifier @@ -343,6 +353,7 @@ rule annotate_npclassifier: db_path=DB_PATH, cache_path=WORKDIR / "npclassifier" / "cache.jsonl", requests_per_second=NPCLASSIFIER_REQUESTS_PER_SECOND, + workers=threads, ) diff --git a/database/config.yaml b/database/config.yaml index 9404726..18f6ead 100644 --- a/database/config.yaml +++ b/database/config.yaml @@ -21,10 +21,10 @@ enabled: paths: # Final DuckDB database produced by the pipeline. - database: "/Users/davidmeijer/Downloads/retromol.duckdb" + database: "/Users/davidmeijer/Desktop/retromol.duckdb" # Scratch space for downloads and intermediate per-step results. - workdir: "/Users/davidmeijer/retromol_tmp" + workdir: "/Users/davidmeijer/Desktop/retromol_tmp" # null -> RuleSet.load_default()'s bundled reaction/matching rules. reaction_rules: null @@ -32,9 +32,15 @@ paths: npclassifier: # Rate limit for the free, GNPS2-hosted NPClassifier API (no published limit -- - # kept conservative). Classifications are cached in workdir/npclassifier/cache.jsonl, - # so reruns only pay for compounds not already classified. - requests_per_second: 10.0 + # kept conservative). This is the *combined* rate across all workers below, not + # per-worker. Classifications are cached in workdir/npclassifier/cache.jsonl, so + # reruns only pay for compounds not already classified. + requests_per_second: 100.0 + + # Concurrent requests. Classification is I/O-bound (waiting on the API), not + # CPU-bound, so this is safe to raise well past your core count -- it's bounded by + # requests_per_second above either way. + workers: 8 chembl: # ChEMBL's official SQLite release -- a multi-GB bulk download (~5.4GB compressed diff --git a/database/scripts/annotate_npclassifier.py b/database/scripts/annotate_npclassifier.py index 15de53d..bbfd798 100644 --- a/database/scripts/annotate_npclassifier.py +++ b/database/scripts/annotate_npclassifier.py @@ -8,23 +8,58 @@ Idempotent across reruns via a local JSONL cache (`--cache-path`) keyed by entry_id: already-classified compounds are skipped without hitting the API again, and only -successes are cached, so a transient failure gets retried on the next run. Requests -are paced (`--requests-per-second`, conservative by default) since NPClassifier is a -free, GNPS2-hosted academic service with no published rate limit. +successes are cached -- flushed to disk as each one completes, not batched at the end +-- so a kill mid-run loses at most the handful of requests in flight, and a transient +failure gets retried on the next run. + +Classification is I/O-bound (waiting on NPClassifier's API), not CPU-bound, so +`--workers` runs multiple requests concurrently via a thread pool -- the GIL doesn't +block this the way it would CPU-bound work, since each thread is blocked on network +I/O rather than holding the GIL. `--requests-per-second` still caps the *combined* +rate across all workers (a shared, thread-safe RateLimiter -- see below), since that +cap is about being a good citizen towards NPClassifier's free, GNPS2-hosted service, +not about this process's own resources. DB writes and cache appends only ever happen +on the main thread (as results complete), so there's no concurrent-write concern +there -- only the HTTP calls themselves run in parallel. """ import argparse import json import logging +import threading import time +from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path from npclassifier import ClassificationResult, classify_smiles -from retromol_database.duckdb import RetroMolDuckDB +from retromol_database.duckdb import Entry, RetroMolDuckDB log = logging.getLogger(__name__) +class RateLimiter: + """Thread-safe request pacing shared across worker threads. + + Each call reserves the next available time slot under a lock (cheap, no waiting + while holding it), then sleeps outside the lock until its own slot arrives -- so + threads don't serialize on the actual waiting, only on the quick slot reservation. + """ + + def __init__(self, requests_per_second: float) -> None: + self._min_interval = 1.0 / requests_per_second if requests_per_second > 0 else 0.0 + self._lock = threading.Lock() + self._next_slot = time.monotonic() + + def wait_for_slot(self) -> None: + with self._lock: + now = time.monotonic() + slot = max(now, self._next_slot) + self._next_slot = slot + self._min_interval + delay = slot - now + if delay > 0: + time.sleep(delay) + + def _load_cache(cache_path: Path) -> dict[str, ClassificationResult]: if not cache_path.exists(): return {} @@ -76,6 +111,7 @@ def run( db_path: str | Path, cache_path: str | Path, requests_per_second: float = 2.0, + workers: int = 8, limit: int | None = None, log_every: int = 100, ) -> None: @@ -84,16 +120,20 @@ def run( cache = _load_cache(cache_path) log.info("annotate_npclassifier: loaded %d cached classifications from %s", len(cache), cache_path) - min_interval = 1.0 / requests_per_second if requests_per_second > 0 else 0.0 - last_call = 0.0 + rate_limiter = RateLimiter(requests_per_second) classified = 0 reused = 0 skipped_no_smiles = 0 failed = 0 + def classify_one(entry: Entry) -> tuple[Entry, ClassificationResult | None]: + rate_limiter.wait_for_slot() + return entry, classify_smiles(entry.raw) + db = RetroMolDuckDB.open(db_path) try: + to_classify: list[Entry] = [] for entry in db.iter_entries(): if entry.type != "compound": continue @@ -107,28 +147,35 @@ def run( skipped_no_smiles += 1 continue - if limit is not None and classified >= limit: + if limit is not None and len(to_classify) >= limit: continue - elapsed = time.monotonic() - last_call - if elapsed < min_interval: - time.sleep(min_interval - elapsed) - last_call = time.monotonic() - - result = classify_smiles(entry.raw) - if result is None: - failed += 1 - continue + to_classify.append(entry) - _apply(db, entry.id, result) - _append_cache(cache_path, entry.id, result) - classified += 1 + log.info( + "annotate_npclassifier: %d compounds to classify (workers=%d, requests_per_second=%s)", + len(to_classify), workers, requests_per_second, + ) - if log_every > 0 and classified % log_every == 0: - log.info( - "annotate_npclassifier: classified=%d reused=%d failed=%d skipped_no_smiles=%d", - classified, reused, failed, skipped_no_smiles, - ) + with ThreadPoolExecutor(max_workers=workers) as pool: + futures = [pool.submit(classify_one, entry) for entry in to_classify] + + for future in as_completed(futures): + entry, result = future.result() + + if result is None: + failed += 1 + else: + _apply(db, entry.id, result) + _append_cache(cache_path, entry.id, result) + classified += 1 + + done = classified + failed + if log_every > 0 and done % log_every == 0: + log.info( + "annotate_npclassifier: classified=%d reused=%d failed=%d skipped_no_smiles=%d", + classified, reused, failed, skipped_no_smiles, + ) finally: db.close() @@ -145,6 +192,7 @@ def main() -> None: ap.add_argument("--db-path", required=True) ap.add_argument("--cache-path", required=True) ap.add_argument("--requests-per-second", type=float, default=2.0) + ap.add_argument("--workers", type=int, default=8, help="concurrent requests (I/O-bound, not CPU-bound)") ap.add_argument("--limit", type=int, default=None, help="classify at most N new compounds (testing/dry-run)") ap.add_argument("--log-every", type=int, default=100) args = ap.parse_args() @@ -153,6 +201,7 @@ def main() -> None: db_path=args.db_path, cache_path=args.cache_path, requests_per_second=args.requests_per_second, + workers=args.workers, limit=args.limit, log_every=args.log_every, ) From 712ee8b6ffd5869e6ad78a885be4289d482fb1cc Mon Sep 17 00:00:00 2001 From: David Meijer Date: Wed, 26 Aug 2026 21:41:26 -0400 Subject: [PATCH 07/11] ENH: add progress bars --- database/config.yaml | 2 +- database/scripts/annotate_chebi.py | 49 ++++++++------ database/scripts/annotate_chembl.py | 63 ++++++++++-------- database/scripts/annotate_npclassifier.py | 71 +++++++++++++++------ database/scripts/chebi.py | 9 ++- database/scripts/chembl.py | 13 +++- database/scripts/download_sources.py | 10 ++- database/scripts/extract_mibig_compounds.py | 4 +- database/scripts/load_bgcs.py | 8 ++- database/scripts/npclassifier.py | 20 +++++- database/scripts/taxonomy.py | 17 +++-- 11 files changed, 179 insertions(+), 87 deletions(-) diff --git a/database/config.yaml b/database/config.yaml index 18f6ead..df6f076 100644 --- a/database/config.yaml +++ b/database/config.yaml @@ -35,7 +35,7 @@ npclassifier: # kept conservative). This is the *combined* rate across all workers below, not # per-worker. Classifications are cached in workdir/npclassifier/cache.jsonl, so # reruns only pay for compounds not already classified. - requests_per_second: 100.0 + requests_per_second: 50.0 # Concurrent requests. Classification is I/O-bound (waiting on the API), not # CPU-bound, so this is safe to raise well past your core count -- it's bounded by diff --git a/database/scripts/annotate_chebi.py b/database/scripts/annotate_chebi.py index 2dde1a4..24d8547 100644 --- a/database/scripts/annotate_chebi.py +++ b/database/scripts/annotate_chebi.py @@ -12,6 +12,8 @@ import logging from pathlib import Path +from tqdm import tqdm + from chebi import ChebiDB from retromol_database.duckdb import RetroMolDuckDB @@ -25,28 +27,33 @@ def run(db_path: str | Path, chebi_dir: str | Path, log_every: int = 500) -> Non db = RetroMolDuckDB.open(db_path) try: + total = db.count_entries_by_type(["compound"]) chebi = ChebiDB.load(chebi_dir) - for entry in db.iter_entries(): - if entry.type != "compound": - continue - - processed += 1 - result = chebi.roles_for_inchikey(entry.id) - if result is not None: - matched += 1 - for role in result.biological_roles: - db.add_bioactivity_annotation( - entry.id, level="chebi_biological_role", label=role.label, external_id=role.chebi_accession - ) - annotated += 1 - for role in result.chemical_roles: - db.add_bioactivity_annotation( - entry.id, level="chebi_chemical_role", label=role.label, external_id=role.chebi_accession - ) - annotated += 1 - - if log_every > 0 and processed % log_every == 0: - log.info("annotate_chebi: processed=%d matched=%d annotated=%d", processed, matched, annotated) + with tqdm(total=total, desc="annotate_chebi", unit="cmpd") as pbar: + for entry in db.iter_entries(): + if entry.type != "compound": + continue + + processed += 1 + result = chebi.roles_for_inchikey(entry.id) + if result is not None: + matched += 1 + for role in result.biological_roles: + db.add_bioactivity_annotation( + entry.id, level="chebi_biological_role", label=role.label, external_id=role.chebi_accession + ) + annotated += 1 + for role in result.chemical_roles: + db.add_bioactivity_annotation( + entry.id, level="chebi_chemical_role", label=role.label, external_id=role.chebi_accession + ) + annotated += 1 + + pbar.update(1) + pbar.set_postfix(matched=matched, annotated=annotated) + + if log_every > 0 and processed % log_every == 0: + log.info("annotate_chebi: processed=%d matched=%d annotated=%d", processed, matched, annotated) finally: db.close() diff --git a/database/scripts/annotate_chembl.py b/database/scripts/annotate_chembl.py index eefbb76..7519397 100644 --- a/database/scripts/annotate_chembl.py +++ b/database/scripts/annotate_chembl.py @@ -13,6 +13,8 @@ import logging from pathlib import Path +from tqdm import tqdm + from chembl import ChemblDB from retromol_database.duckdb import RetroMolDuckDB @@ -26,36 +28,41 @@ def run(db_path: str | Path, chembl_sqlite_path: str | Path, log_every: int = 50 db = RetroMolDuckDB.open(db_path) try: + total = db.count_entries_by_type(["compound"]) with ChemblDB(chembl_sqlite_path) as chembl: - for entry in db.iter_entries(): - if entry.type != "compound": - continue - - processed += 1 - result = chembl.bioactivity_for_inchikey(entry.id) - if result is not None: - matched += 1 - if result.max_phase_label: - db.add_bioactivity_annotation( - entry.id, - level="chembl_max_phase", - label=result.max_phase_label, - external_id=result.chembl_id, - ) - annotated += 1 - for atc in result.atc_categories: - db.add_bioactivity_annotation( - entry.id, - level="chembl_atc", - label=atc.level1_description, - external_id=atc.level5, + with tqdm(total=total, desc="annotate_chembl", unit="cmpd") as pbar: + for entry in db.iter_entries(): + if entry.type != "compound": + continue + + processed += 1 + result = chembl.bioactivity_for_inchikey(entry.id) + if result is not None: + matched += 1 + if result.max_phase_label: + db.add_bioactivity_annotation( + entry.id, + level="chembl_max_phase", + label=result.max_phase_label, + external_id=result.chembl_id, + ) + annotated += 1 + for atc in result.atc_categories: + db.add_bioactivity_annotation( + entry.id, + level="chembl_atc", + label=atc.level1_description, + external_id=atc.level5, + ) + annotated += 1 + + pbar.update(1) + pbar.set_postfix(matched=matched, annotated=annotated) + + if log_every > 0 and processed % log_every == 0: + log.info( + "annotate_chembl: processed=%d matched=%d annotated=%d", processed, matched, annotated ) - annotated += 1 - - if log_every > 0 and processed % log_every == 0: - log.info( - "annotate_chembl: processed=%d matched=%d annotated=%d", processed, matched, annotated - ) finally: db.close() diff --git a/database/scripts/annotate_npclassifier.py b/database/scripts/annotate_npclassifier.py index bbfd798..7a453df 100644 --- a/database/scripts/annotate_npclassifier.py +++ b/database/scripts/annotate_npclassifier.py @@ -24,13 +24,16 @@ """ import argparse +import itertools import json import logging import threading import time -from concurrent.futures import ThreadPoolExecutor, as_completed +from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait from pathlib import Path +from tqdm import tqdm + from npclassifier import ClassificationResult, classify_smiles from retromol_database.duckdb import Entry, RetroMolDuckDB @@ -157,25 +160,53 @@ def classify_one(entry: Entry) -> tuple[Entry, ClassificationResult | None]: len(to_classify), workers, requests_per_second, ) - with ThreadPoolExecutor(max_workers=workers) as pool: - futures = [pool.submit(classify_one, entry) for entry in to_classify] - - for future in as_completed(futures): - entry, result = future.result() - - if result is None: - failed += 1 - else: - _apply(db, entry.id, result) - _append_cache(cache_path, entry.id, result) - classified += 1 - - done = classified + failed - if log_every > 0 and done % log_every == 0: - log.info( - "annotate_npclassifier: classified=%d reused=%d failed=%d skipped_no_smiles=%d", - classified, reused, failed, skipped_no_smiles, - ) + # Bounded sliding window rather than submitting the whole backlog up front -- + # at hundreds/thousands of compounds, an upfront submit() for everything means + # a Ctrl+C has to wait for every already-launched request (each with its own + # retry/backoff chain, worse under rate-limiting) before the pool can actually + # exit, since ThreadPoolExecutor won't drop already-running work. Keeping at + # most `max_pending` in flight means an interrupt only has to wait for that many + # -- same shape as parse_gbks.py's own bounded-window loop, same reason. + max_pending = max(workers * 2, 1) + entries_iter = iter(to_classify) + + pool = ThreadPoolExecutor(max_workers=workers) + try: + pending = {pool.submit(classify_one, e) for e in itertools.islice(entries_iter, max_pending)} + + with tqdm(total=len(to_classify), desc="annotate_npclassifier", unit="cmpd") as pbar: + while pending: + done_futures, pending = wait(pending, return_when=FIRST_COMPLETED) + + for future in done_futures: + entry, result = future.result() + + if result is None: + failed += 1 + else: + _apply(db, entry.id, result) + _append_cache(cache_path, entry.id, result) + classified += 1 + + pbar.update(1) + pbar.set_postfix(classified=classified, reused=reused, failed=failed) + + done = classified + failed + if log_every > 0 and done % log_every == 0: + log.info( + "annotate_npclassifier: classified=%d reused=%d failed=%d skipped_no_smiles=%d", + classified, reused, failed, skipped_no_smiles, + ) + + next_entry = next(entries_iter, None) + if next_entry is not None: + pending.add(pool.submit(classify_one, next_entry)) + except KeyboardInterrupt: + log.warning("annotate_npclassifier: interrupted -- cancelling not-yet-started requests") + pool.shutdown(wait=False, cancel_futures=True) + raise + else: + pool.shutdown(wait=True) finally: db.close() diff --git a/database/scripts/chebi.py b/database/scripts/chebi.py index b9c95af..62c53f4 100644 --- a/database/scripts/chebi.py +++ b/database/scripts/chebi.py @@ -21,11 +21,12 @@ import csv import gzip import logging -import shutil import urllib.request from dataclasses import dataclass from pathlib import Path +from tqdm import tqdm + log = logging.getLogger(__name__) CHEBI_FLAT_FILES_URL = "https://ftp.ebi.ac.uk/pub/databases/chebi/flat_files" @@ -63,7 +64,11 @@ def download_chebi_flat_files(dest_dir: str | Path, *, force: bool = False) -> P dest = dest_dir / name log.info("downloading %s", url) with urllib.request.urlopen(url) as resp, open(dest, "wb") as out: - shutil.copyfileobj(resp, out, length=1024 * 1024) + total = int(resp.headers.get("Content-Length") or 0) or None + with tqdm(total=total, desc=f"download_chebi[{name}]", unit="B", unit_scale=True, unit_divisor=1024) as pbar: + while chunk := resp.read(1024 * 1024): + out.write(chunk) + pbar.update(len(chunk)) return dest_dir diff --git a/database/scripts/chembl.py b/database/scripts/chembl.py index 3379f81..4ade384 100644 --- a/database/scripts/chembl.py +++ b/database/scripts/chembl.py @@ -15,13 +15,14 @@ from __future__ import annotations import logging -import shutil import sqlite3 import tarfile import urllib.request from dataclasses import dataclass from pathlib import Path +from tqdm import tqdm + log = logging.getLogger(__name__) # "latest" always resolves to the newest ChEMBL release -- pin to a specific @@ -60,11 +61,17 @@ def download_chembl_sqlite(dest_dir: str | Path, *, url: str = CHEMBL_SQLITE_URL archive_path = dest_dir / "chembl_sqlite.tar.gz" log.info("downloading ChEMBL SQLite release from %s", url) with urllib.request.urlopen(url) as resp, open(archive_path, "wb") as out: - shutil.copyfileobj(resp, out, length=1024 * 1024) + total = int(resp.headers.get("Content-Length") or 0) or None + with tqdm(total=total, desc="download_chembl", unit="B", unit_scale=True, unit_divisor=1024) as pbar: + while chunk := resp.read(1024 * 1024): + out.write(chunk) + pbar.update(len(chunk)) log.info("extracting %s", archive_path) with tarfile.open(archive_path, mode="r:gz") as tar: - tar.extractall(path=dest_dir) + members = tar.getmembers() + for member in tqdm(members, desc="extract_chembl", unit="file"): + tar.extract(member, path=dest_dir) archive_path.unlink() found = sorted(dest_dir.rglob("*.db")) diff --git a/database/scripts/download_sources.py b/database/scripts/download_sources.py index 1237223..8b91496 100644 --- a/database/scripts/download_sources.py +++ b/database/scripts/download_sources.py @@ -14,6 +14,7 @@ from urllib.parse import urlsplit import requests +from tqdm import tqdm def download(url: str, dest: str | Path) -> None: @@ -21,10 +22,13 @@ def download(url: str, dest: str | Path) -> None: dest.parent.mkdir(parents=True, exist_ok=True) with requests.get(url, stream=True, timeout=300) as resp: resp.raise_for_status() + total = int(resp.headers.get("Content-Length") or 0) or None with open(dest, "wb") as fh: - for chunk in resp.iter_content(chunk_size=1024 * 1024): - if chunk: - fh.write(chunk) + with tqdm(total=total, desc=f"download[{dest.name}]", unit="B", unit_scale=True, unit_divisor=1024) as pbar: + for chunk in resp.iter_content(chunk_size=1024 * 1024): + if chunk: + fh.write(chunk) + pbar.update(len(chunk)) def _url_filename(url: str) -> str: diff --git a/database/scripts/extract_mibig_compounds.py b/database/scripts/extract_mibig_compounds.py index 3a25155..e0baf0d 100644 --- a/database/scripts/extract_mibig_compounds.py +++ b/database/scripts/extract_mibig_compounds.py @@ -23,6 +23,8 @@ from pathlib import Path from typing import Any, Iterator +from tqdm import tqdm + log = logging.getLogger(__name__) @@ -130,7 +132,7 @@ def run( annotations: dict[str, dict[str, Any]] = {} with open(output_path, "w") as out: - for path in json_files: + for path in tqdm(json_files, desc="extract_mibig_compounds", unit="file"): try: with open(path) as fh: data = json.load(fh) diff --git a/database/scripts/load_bgcs.py b/database/scripts/load_bgcs.py index 520373e..ab8dc2f 100644 --- a/database/scripts/load_bgcs.py +++ b/database/scripts/load_bgcs.py @@ -26,6 +26,8 @@ import logging from pathlib import Path +from tqdm import tqdm + from common import build_fingerprint_context, load_ruleset, mibig_url, phylogeny_from_organism_name from retromol_antismash.modules import LinearReadout, bgc_primary_sequence from retromol_database.duckdb import RetroMolDuckDB @@ -63,7 +65,8 @@ def run( db = RetroMolDuckDB.open(db_path) try: with open(readouts_path) as fh: - for line in fh: + pbar = tqdm(fh, desc="load_bgcs", unit="region") + for line in pbar: line = line.strip() if not line: continue @@ -73,6 +76,7 @@ def run( if file_hash and db.bgc_content_hash_exists(file_hash): skipped_existing_file += 1 + pbar.set_postfix(added=added, skipped=skipped, skipped_existing=skipped_existing_file) continue readout = LinearReadout.from_dict(entry["readout"]) @@ -80,6 +84,7 @@ def run( if not names: skipped += 1 + pbar.set_postfix(added=added, skipped=skipped, skipped_existing=skipped_existing_file) continue fp = fingerprinter.encode(tokens) @@ -125,6 +130,7 @@ def run( # a BGC's biosynthetic class isn't populated here anymore. added += 1 + pbar.set_postfix(added=added, skipped=skipped, skipped_existing=skipped_existing_file) finally: db.close() diff --git a/database/scripts/npclassifier.py b/database/scripts/npclassifier.py index cbb028e..2276f17 100644 --- a/database/scripts/npclassifier.py +++ b/database/scripts/npclassifier.py @@ -60,9 +60,25 @@ def classify_smiles( is_glycoside=bool(data.get("isglycoside", False)), ) except (urllib.error.URLError, urllib.error.HTTPError, json.JSONDecodeError, TimeoutError, OSError) as exc: - log.warning("NPClassifier request failed (attempt %d/%d) for %r: %s", attempt, max_retries, smiles, exc) + if isinstance(exc, urllib.error.HTTPError) and exc.code == 429: + # Rate-limited: back off much harder than a transient failure, and + # respect Retry-After if the server sent one -- retrying at the same + # pace that just got us 429'd only makes the storm worse. + retry_after = exc.headers.get("Retry-After") if exc.headers else None + try: + delay = float(retry_after) if retry_after is not None else backoff_seconds * attempt * 5 + except ValueError: + delay = backoff_seconds * attempt * 5 + log.warning( + "NPClassifier rate-limited (429) on attempt %d/%d for %r -- backing off %.1fs", + attempt, max_retries, smiles, delay, + ) + else: + delay = backoff_seconds * attempt + log.warning("NPClassifier request failed (attempt %d/%d) for %r: %s", attempt, max_retries, smiles, exc) + if attempt < max_retries: - time.sleep(backoff_seconds * attempt) + time.sleep(delay) log.error("NPClassifier: giving up on %r after %d attempts", smiles, max_retries) return None diff --git a/database/scripts/taxonomy.py b/database/scripts/taxonomy.py index 7f5555a..87d428e 100644 --- a/database/scripts/taxonomy.py +++ b/database/scripts/taxonomy.py @@ -10,7 +10,6 @@ from __future__ import annotations -import io import logging import tarfile import urllib.request @@ -18,6 +17,8 @@ from pathlib import Path from typing import Iterator +from tqdm import tqdm + log = logging.getLogger(__name__) TAXDUMP_URL = "https://ftp.ncbi.nlm.nih.gov/pub/taxonomy/taxdump.tar.gz" @@ -47,13 +48,19 @@ def download_taxdump(dest_dir: str | Path, *, force: bool = False) -> Path: return dest_dir log.info("downloading NCBI taxdump from %s", TAXDUMP_URL) - with urllib.request.urlopen(TAXDUMP_URL) as resp: - raw = resp.read() - - with tarfile.open(fileobj=io.BytesIO(raw), mode="r:gz") as tar: + archive_path = dest_dir / "taxdump.tar.gz" + with urllib.request.urlopen(TAXDUMP_URL) as resp, open(archive_path, "wb") as out: + total = int(resp.headers.get("Content-Length") or 0) or None + with tqdm(total=total, desc="download_taxdump", unit="B", unit_scale=True, unit_divisor=1024) as pbar: + while chunk := resp.read(1024 * 1024): + out.write(chunk) + pbar.update(len(chunk)) + + with tarfile.open(archive_path, mode="r:gz") as tar: for member in tar.getmembers(): if member.name in ("names.dmp", "nodes.dmp"): tar.extract(member, path=dest_dir) + archive_path.unlink() if not names_path.exists() or not nodes_path.exists(): raise FileNotFoundError(f"taxdump at {TAXDUMP_URL} did not contain names.dmp/nodes.dmp") From fb583b3ded2e68500f5c8b878ae3815bea5e3b19 Mon Sep 17 00:00:00 2001 From: David Meijer Date: Wed, 26 Aug 2026 23:29:51 -0400 Subject: [PATCH 08/11] UPD: annotation pipeline and display --- database/Snakemake | 59 +----- database/config.yaml | 8 - database/profiles/slurm/config.yaml | 17 +- database/scripts/annotate_chebi.py | 11 +- database/scripts/annotate_chembl.py | 85 --------- database/scripts/annotate_npclassifier.py | 5 +- database/scripts/chebi.py | 22 ++- database/scripts/chembl.py | 148 --------------- database/scripts/common.py | 55 +++++- database/scripts/load_bgcs.py | 9 +- database/scripts/load_compounds.py | 25 ++- database/scripts/taxonomy.py | 36 ++-- gui/scripts/dev_backend.sh | 2 +- .../components/workspace/AlignmentGrid.tsx | 46 +++-- .../components/workspace/WorkspaceHome.tsx | 172 +++++++++++------- .../client/src/features/database/format.ts | 17 ++ gui/src/client/src/features/database/types.ts | 10 +- gui/src/server/routes/entry_annotations.py | 6 - gui/src/server/routes/stats.py | 17 +- src/retromol_database/duckdb.py | 100 ++++++---- 20 files changed, 380 insertions(+), 470 deletions(-) delete mode 100644 database/scripts/annotate_chembl.py delete mode 100644 database/scripts/chembl.py create mode 100644 gui/src/client/src/features/database/format.ts diff --git a/database/Snakemake b/database/Snakemake index 8560202..a6cd5d0 100644 --- a/database/Snakemake +++ b/database/Snakemake @@ -15,12 +15,11 @@ Fill in database/config.yaml's `sources` URLs before running. The pipeline: 7. parse_mibig_gbks - antiSMASH GBKs -> linear module readouts (PARAS-annotated) 8. load_mibig_bgcs - turn readouts into "bgc" db entries 9. annotate_npclassifier - chemical-class annotation (compounds only, NPClassifier API) - 10. annotate_chembl - bioactivity annotation (compounds only, local ChEMBL SQLite) - 11. annotate_chebi - bioactivity annotation (compounds only, local ChEBI flat files) + 10. annotate_chebi - bioactivity annotation (compounds only, local ChEBI flat files) -Steps 4, 6, 8, 9, 10, and 11 all mutate the same DuckDB file, so they're chained -through marker files (rather than each declaring the database itself as `output`) to -force Snakemake to serialize them -- DuckDB doesn't support concurrent writers. +Steps 4, 6, 8, 9, and 10 all mutate the same DuckDB file, so they're chained through +marker files (rather than each declaring the database itself as `output`) to force +Snakemake to serialize them -- DuckDB doesn't support concurrent writers. """ import sys @@ -48,22 +47,18 @@ TAXDUMP_DIR = WORKDIR / "taxdump" NPCLASSIFIER_REQUESTS_PER_SECOND = config.get("npclassifier", {}).get("requests_per_second", 2.0) NPCLASSIFIER_WORKERS = config.get("npclassifier", {}).get("workers", 8) -CHEMBL_SQLITE_URL = config.get("chembl", {}).get("sqlite_url") -CHEMBL_DIR = WORKDIR / "chembl" - CHEBI_DIR = WORKDIR / "chebi" # Toggle whole branches of the pipeline off (see config.yaml's `enabled` comment) -- # disabling npatlas/mibig skips their download+parse rules entirely, not just db # loading, since the "real" inputs below are only declared when enabled: nothing else # in the DAG needs npatlas/results.jsonl or mibig_gbk/readouts.jsonl, so Snakemake -# never schedules the rules that would produce them. Same idea for chembl/chebi: -# disabling either skips its bulk download entirely. +# never schedules the rules that would produce them. Same idea for chebi: disabling it +# skips its bulk download entirely. ENABLED = config.get("enabled", {}) NPATLAS_ENABLED = ENABLED.get("npatlas", True) MIBIG_ENABLED = ENABLED.get("mibig", True) NPCLASSIFIER_ENABLED = ENABLED.get("npclassifier", True) -CHEMBL_ENABLED = ENABLED.get("chembl", True) CHEBI_ENABLED = ENABLED.get("chebi", True) @@ -71,7 +66,6 @@ rule all: input: MARKERS / "bgcs_loaded.done", MARKERS / "npclassifier_annotated.done", - MARKERS / "chembl_annotated.done", MARKERS / "chebi_annotated.done" @@ -358,40 +352,7 @@ rule annotate_npclassifier: # --------------------------------------------------------------------------- -# Step 10: ChEMBL bioactivity annotation (compounds only) -# --------------------------------------------------------------------------- - -rule download_chembl: - output: - extract_dir=directory(CHEMBL_DIR / "extracted") - params: - url=CHEMBL_SQLITE_URL - run: - import chembl - chembl.download_chembl_sqlite(output.extract_dir, url=params.url) - - -rule annotate_chembl: - input: - # Chained after npclassifier_annotated.done purely to serialize this write - # against annotate_npclassifier's (see module docstring at the top of this file). - prev=MARKERS / "npclassifier_annotated.done", - # Only declared when enabled -- download_chembl (a multi-GB download) never - # runs when disabled. - **({"extract_dir": CHEMBL_DIR / "extracted"} if CHEMBL_ENABLED else {}) - output: - marker=touch(MARKERS / "chembl_annotated.done") - run: - if CHEMBL_ENABLED: - import annotate_chembl - candidates = sorted(Path(input.extract_dir).rglob("*.db")) - if not candidates: - raise FileNotFoundError(f"no ChEMBL .db file found under {input.extract_dir}") - annotate_chembl.run(db_path=DB_PATH, chembl_sqlite_path=candidates[0]) - - -# --------------------------------------------------------------------------- -# Step 11: ChEBI bioactivity annotation (compounds only) +# Step 10: ChEBI bioactivity annotation (compounds only) # --------------------------------------------------------------------------- rule download_chebi: @@ -406,9 +367,9 @@ rule download_chebi: rule annotate_chebi: input: - # Chained after chembl_annotated.done purely to serialize this write against - # annotate_chembl's (see module docstring at the top of this file). - prev=MARKERS / "chembl_annotated.done", + # Chained after npclassifier_annotated.done purely to serialize this write + # against annotate_npclassifier's (see module docstring at the top of this file). + prev=MARKERS / "npclassifier_annotated.done", # Only declared when enabled -- download_chebi never runs when disabled. **( { diff --git a/database/config.yaml b/database/config.yaml index df6f076..b53e44f 100644 --- a/database/config.yaml +++ b/database/config.yaml @@ -16,7 +16,6 @@ enabled: npatlas: true mibig: true npclassifier: true - chembl: true chebi: true paths: @@ -42,13 +41,6 @@ npclassifier: # requests_per_second above either way. workers: 8 -chembl: - # ChEMBL's official SQLite release -- a multi-GB bulk download (~5.4GB compressed - # as of ChEMBL 37), extracted once into workdir/chembl and queried locally by - # InChIKey (see database/scripts/chembl.py). "latest" always points at the newest - # ChEMBL release; pin to a specific release's URL for reproducibility across runs. - sqlite_url: "https://ftp.ebi.ac.uk/pub/databases/chembl/ChEMBLdb/latest/chembl_37_sqlite.tar.gz" - taxonomy: # NCBI taxdump (names.dmp/nodes.dmp), downloaded once into workdir/taxdump and reused # across pipeline runs -- used to standardize phylogeny genus/species/type to NCBI diff --git a/database/profiles/slurm/config.yaml b/database/profiles/slurm/config.yaml index d19bcfb..13a1265 100644 --- a/database/profiles/slurm/config.yaml +++ b/database/profiles/slurm/config.yaml @@ -15,9 +15,9 @@ jobs: 6 # concurrent Slurm jobs -- mainly matters for the independent download_ latency-wait: 60 rerun-incomplete: true printshellcmds: true -# Independent rules (e.g. the download_* rules, or npclassifier/chembl/chebi annotation -# vs. each other before they're serialized) shouldn't all be killed by one unrelated -# rule failing/timing out -- let everything already running finish. +# Independent rules (e.g. the download_* rules, or npclassifier/chebi annotation vs. +# each other before they're serialized) shouldn't all be killed by one unrelated rule +# failing/timing out -- let everything already running finish. keep-going: true default-resources: @@ -48,12 +48,6 @@ set-resources: runtime: 60 mem_mb: 2000 disk_mb: 5000 - download_chembl: - # ~5.4GB compressed download, extracted to a several-GB sqlite file (see - # database/scripts/chembl.py) -- the heaviest single download in this pipeline. - runtime: 240 - mem_mb: 4000 - disk_mb: 40000 download_chebi: # A few hundred MB total across compounds/structures/relation flat files. runtime: 60 @@ -82,11 +76,6 @@ set-resources: # -- mostly wall-clock waiting on the network, not compute. runtime: 1440 mem_mb: 2000 - annotate_chembl: - # Local sqlite reads against the downloaded ChEMBL release -- fast, but give it - # enough runtime to walk every compound entry once. - runtime: 240 - mem_mb: 4000 annotate_chebi: # Loads all of ChEBI's flat files into memory once (see chebi.py's ChebiDB.load) -- # a few hundred MB, generous headroom here. diff --git a/database/scripts/annotate_chebi.py b/database/scripts/annotate_chebi.py index 24d8547..ad8de7f 100644 --- a/database/scripts/annotate_chebi.py +++ b/database/scripts/annotate_chebi.py @@ -1,11 +1,12 @@ -"""Step 11: annotate every compound entry's bioactivity via ChEBI's role ontology. +"""Step 10: annotate every compound entry's bioactivity via ChEBI's role ontology. Runs after compound loading, so every distinct molecule -- regardless of which -source(s) it came from -- is looked up exactly once. BGC entries are skipped, same -reasoning as annotate_chembl.py. +source(s) it came from -- is looked up exactly once. BGC entries are skipped: +bioactivity here is a property of the compound structure (looked up by its InChIKey +entry id), not of the producing organism/cluster. -Like annotate_chembl.py, this queries a local bulk release (see chebi.py) rather than -a rate-limited API -- no pacing or caching needed. +This queries a local bulk release (see chebi.py) rather than a rate-limited API -- no +pacing or caching needed, lookups are just local reads. """ import argparse diff --git a/database/scripts/annotate_chembl.py b/database/scripts/annotate_chembl.py deleted file mode 100644 index 7519397..0000000 --- a/database/scripts/annotate_chembl.py +++ /dev/null @@ -1,85 +0,0 @@ -"""Step 10: annotate every compound entry's bioactivity via ChEMBL. - -Runs after compound loading (like annotate_npclassifier.py), so every distinct -molecule -- regardless of which source(s) it came from -- is looked up exactly once. -BGC entries are skipped: bioactivity here is a property of the compound structure -(looked up by its InChIKey entry id), not of the producing organism/cluster. - -Unlike NPClassifier, this queries a local ChEMBL SQLite release (see chembl.py) rather -than a rate-limited API -- no pacing or caching needed, lookups are just local reads. -""" - -import argparse -import logging -from pathlib import Path - -from tqdm import tqdm - -from chembl import ChemblDB -from retromol_database.duckdb import RetroMolDuckDB - -log = logging.getLogger(__name__) - - -def run(db_path: str | Path, chembl_sqlite_path: str | Path, log_every: int = 500) -> None: - processed = 0 - matched = 0 - annotated = 0 - - db = RetroMolDuckDB.open(db_path) - try: - total = db.count_entries_by_type(["compound"]) - with ChemblDB(chembl_sqlite_path) as chembl: - with tqdm(total=total, desc="annotate_chembl", unit="cmpd") as pbar: - for entry in db.iter_entries(): - if entry.type != "compound": - continue - - processed += 1 - result = chembl.bioactivity_for_inchikey(entry.id) - if result is not None: - matched += 1 - if result.max_phase_label: - db.add_bioactivity_annotation( - entry.id, - level="chembl_max_phase", - label=result.max_phase_label, - external_id=result.chembl_id, - ) - annotated += 1 - for atc in result.atc_categories: - db.add_bioactivity_annotation( - entry.id, - level="chembl_atc", - label=atc.level1_description, - external_id=atc.level5, - ) - annotated += 1 - - pbar.update(1) - pbar.set_postfix(matched=matched, annotated=annotated) - - if log_every > 0 and processed % log_every == 0: - log.info( - "annotate_chembl: processed=%d matched=%d annotated=%d", processed, matched, annotated - ) - finally: - db.close() - - log.info("annotate_chembl: processed=%d matched=%d annotated=%d", processed, matched, annotated) - - -def main() -> None: - logging.basicConfig(level=logging.INFO) - - ap = argparse.ArgumentParser(description=__doc__) - ap.add_argument("--db-path", required=True) - ap.add_argument("--chembl-sqlite-path", required=True) - ap.add_argument("--log-every", type=int, default=500) - args = ap.parse_args() - - run(db_path=args.db_path, chembl_sqlite_path=args.chembl_sqlite_path, log_every=args.log_every) - - -if __name__ == "__main__": - main() diff --git a/database/scripts/annotate_npclassifier.py b/database/scripts/annotate_npclassifier.py index 7a453df..a51b8c7 100644 --- a/database/scripts/annotate_npclassifier.py +++ b/database/scripts/annotate_npclassifier.py @@ -106,8 +106,11 @@ def _apply(db: RetroMolDuckDB, entry_id: str, result: ClassificationResult) -> N db.add_chemical_class_annotation(entry_id, level="superclass", label=label) for label in result.class_: db.add_chemical_class_annotation(entry_id, level="class", label=label) + # Presence-only, same convention as every other flag-shaped annotation: only stored + # when true, and with a real name ("Glycoside") rather than a bare "Yes" that means + # nothing once it's sitting in a chart/chip out of context. if result.is_glycoside: - db.add_chemical_class_annotation(entry_id, level="is_glycoside", label="Yes") + db.add_chemical_class_annotation(entry_id, level="is_glycoside", label="Glycoside") def run( diff --git a/database/scripts/chebi.py b/database/scripts/chebi.py index 62c53f4..d4652d4 100644 --- a/database/scripts/chebi.py +++ b/database/scripts/chebi.py @@ -1,10 +1,10 @@ """ChEBI role-ontology client for bioactivity annotation. Downloads ChEBI's flat-file bulk release (ftp.ebi.ac.uk) once, then looks up a -compound's biological/chemical roles by its standard InChIKey. Unlike ChEMBL's -clinical-development framing, ChEBI's role ontology (`has_role` edges under -CHEBI:24432 "biological role" / CHEBI:51086 "chemical role") is annotated on generic -compound classes more often than on specific stereo-defined structures -- confirmed +compound's biological/chemical roles by its standard InChIKey. ChEBI's role ontology +(`has_role` edges under CHEBI:24432 "biological role" / CHEBI:51086 "chemical role") +is annotated on generic compound classes more often than on specific stereo-defined +structures -- confirmed live (2026-08-26): erythromycin A (CHEBI:42355) itself carries no has_role edges, only its `is_a` parent "erythromycin" (CHEBI:48923) does (roles: xenobiotic, bacterial metabolite, environmental contaminant). So lookups walk a few steps up the `is_a` @@ -21,6 +21,7 @@ import csv import gzip import logging +import re import urllib.request from dataclasses import dataclass from pathlib import Path @@ -32,6 +33,17 @@ CHEBI_FLAT_FILES_URL = "https://ftp.ebi.ac.uk/pub/databases/chebi/flat_files" CHEBI_FILES = ["compounds.tsv.gz", "structures.tsv.gz", "relation.tsv.gz"] +# ChEBI's `name` column carries inline HTML for italicized genus/species names and +# similar formatting (e.g. "Aspergillus metabolite", "S-configuration" +# -- confirmed live in compounds.tsv.gz/chemical_data.tsv.gz). Stripped here, once, at +# load time -- so every consumer of name_by_compound_id (role labels today, anything +# else later) gets plain text without needing to know this quirk exists. +_HTML_TAG_RE = re.compile(r"<[^>]+>") + + +def _strip_html(text: str) -> str: + return _HTML_TAG_RE.sub("", text) + RELATION_TYPE_HAS_ROLE = "4" RELATION_TYPE_IS_A = "5" @@ -120,7 +132,7 @@ def load(cls, chebi_dir: str | Path) -> "ChebiDB": if len(row) <= accession_idx or not row[accession_idx]: continue chebi_accession_by_compound_id[row[id_idx]] = row[accession_idx] - name_by_compound_id[row[id_idx]] = row[name_idx] + name_by_compound_id[row[id_idx]] = _strip_html(row[name_idx]) compound_id_by_inchikey: dict[str, str] = {} with gzip.open(chebi_dir / "structures.tsv.gz", "rt", encoding="utf-8", errors="replace") as fh: diff --git a/database/scripts/chembl.py b/database/scripts/chembl.py deleted file mode 100644 index 4ade384..0000000 --- a/database/scripts/chembl.py +++ /dev/null @@ -1,148 +0,0 @@ -"""ChEMBL bulk-database client for bioactivity annotation. - -Downloads and extracts ChEMBL's official SQLite release (ftp.ebi.ac.uk) once, then -looks up a compound's known medicinal-relevance signals by its standard InChIKey -- -ChEMBL's own maximum clinical-development phase (MAX_PHASE) and WHO ATC therapeutic -category (a defined vocabulary, unlike MIBiG's free-text chem_acts) -- rather than -querying ChEMBL's live API per compound, which would be far too slow at our scale. - -Schema confirmed live against ChEMBL's own schema documentation -(https://ftp.ebi.ac.uk/pub/databases/chembl/ChEMBLdb/latest/schema_documentation.txt, -2026-08-26): COMPOUND_STRUCTURES.STANDARD_INCHI_KEY -> MOLECULE_DICTIONARY.MOLREGNO -(MAX_PHASE, CHEMBL_ID) -> MOLECULE_ATC_CLASSIFICATION -> ATC_CLASSIFICATION.LEVEL1_DESCRIPTION. -""" - -from __future__ import annotations - -import logging -import sqlite3 -import tarfile -import urllib.request -from dataclasses import dataclass -from pathlib import Path - -from tqdm import tqdm - -log = logging.getLogger(__name__) - -# "latest" always resolves to the newest ChEMBL release -- pin to a specific -# release's URL (e.g. .../chembl_37/chembl_37_sqlite.tar.gz) if a pipeline run needs -# to be reproducible across ChEMBL releases. -CHEMBL_SQLITE_URL = "https://ftp.ebi.ac.uk/pub/databases/chembl/ChEMBLdb/latest/chembl_37_sqlite.tar.gz" - -# ChEMBL's MAX_PHASE values (MOLECULE_DICTIONARY.MAX_PHASE) mapped to a display label. -# Only phases with real clinical-development meaning are surfaced -- 0/-1 (no reported -# development / unknown) aren't stored, the same presence-only convention every other -# annotation table uses. Python's dict lookup treats 3 and 3.0 as the same key, so this -# works whether sqlite hands back MAX_PHASE as an int or a float. -MAX_PHASE_LABELS = { - 4: "Approved", - 3: "Phase 3", - 2: "Phase 2", - 1: "Phase 1", - 0.5: "Early phase 1", -} - - -def download_chembl_sqlite(dest_dir: str | Path, *, url: str = CHEMBL_SQLITE_URL, force: bool = False) -> Path: - """Download and extract ChEMBL's SQLite release into `dest_dir`, returning the path - to the extracted .db file (no-op if one is already there, unless `force`). - - This is a multi-GB download (~5.4GB compressed as of ChEMBL 37) -- streamed to disk - in chunks rather than read into memory, unlike taxonomy.py's much smaller taxdump. - """ - dest_dir = Path(dest_dir).expanduser() - dest_dir.mkdir(parents=True, exist_ok=True) - - existing = sorted(dest_dir.rglob("*.db")) - if not force and existing: - return existing[0] - - archive_path = dest_dir / "chembl_sqlite.tar.gz" - log.info("downloading ChEMBL SQLite release from %s", url) - with urllib.request.urlopen(url) as resp, open(archive_path, "wb") as out: - total = int(resp.headers.get("Content-Length") or 0) or None - with tqdm(total=total, desc="download_chembl", unit="B", unit_scale=True, unit_divisor=1024) as pbar: - while chunk := resp.read(1024 * 1024): - out.write(chunk) - pbar.update(len(chunk)) - - log.info("extracting %s", archive_path) - with tarfile.open(archive_path, mode="r:gz") as tar: - members = tar.getmembers() - for member in tqdm(members, desc="extract_chembl", unit="file"): - tar.extract(member, path=dest_dir) - archive_path.unlink() - - found = sorted(dest_dir.rglob("*.db")) - if not found: - raise FileNotFoundError(f"ChEMBL archive at {url} did not contain a .db file") - return found[0] - - -@dataclass(frozen=True) -class AtcCategory: - level1_description: str # display label -- coarse WHO therapeutic category - level5: str # e.g. "J01FA01" -- the compound's own specific ATC code, for linking out - - -@dataclass(frozen=True) -class ChemblBioactivity: - chembl_id: str - max_phase_label: str | None - atc_categories: list[AtcCategory] - - -class ChemblDB: - """Read-only lookup over a local ChEMBL SQLite release, by standard InChIKey.""" - - def __init__(self, sqlite_path: str | Path) -> None: - self.con = sqlite3.connect(f"file:{Path(sqlite_path).expanduser()}?mode=ro", uri=True) - - def close(self) -> None: - self.con.close() - - def __enter__(self) -> "ChemblDB": - return self - - def __exit__(self, exc_type, exc, tb) -> None: - self.close() - - def bioactivity_for_inchikey(self, inchikey: str) -> ChemblBioactivity | None: - """None if `inchikey` has no match in ChEMBL at all; otherwise a result whose - `max_phase_label`/`atc_categories` may still both be empty (a compound can be - in ChEMBL -- have activity/assay records -- without being an ATC-classified drug - or having reached any clinical phase).""" - row = self.con.execute( - """ - SELECT md.molregno, md.chembl_id, md.max_phase - FROM compound_structures cs - JOIN molecule_dictionary md ON md.molregno = cs.molregno - WHERE cs.standard_inchi_key = ? - """, - (inchikey,), - ).fetchone() - if row is None: - return None - - molregno, chembl_id, max_phase = row - max_phase_label = MAX_PHASE_LABELS.get(max_phase) - - atc_rows = self.con.execute( - """ - SELECT DISTINCT ac.level1_description, ac.level5 - FROM molecule_atc_classification mac - JOIN atc_classification ac ON ac.level5 = mac.level5 - WHERE mac.molregno = ? - """, - (molregno,), - ).fetchall() - atc_categories = [ - AtcCategory(level1_description=str(r[0]), level5=str(r[1])) for r in atc_rows if r[0] and r[1] - ] - - return ChemblBioactivity( - chembl_id=str(chembl_id), - max_phase_label=max_phase_label, - atc_categories=atc_categories, - ) diff --git a/database/scripts/common.py b/database/scripts/common.py index ac2b01d..2668725 100644 --- a/database/scripts/common.py +++ b/database/scripts/common.py @@ -182,16 +182,56 @@ def run_retromol_stream_quiet( # Metagenomic/environmental-sample naming conventions (e.g. "uncultured Streptomyces sp.", -# "unidentified bacterium") -- not a genus, so skipped when picking the genus token. +# "unidentified bacterium") -- not a genus, so skipped when picking the genus token, and +# rejected outright if a whole genus/species value collapses to just one of these. _NON_TAXONOMIC_PREFIXES = {"uncultured", "unclassified", "unidentified"} +# Species-epithet placeholders meaning "no real species-level identification" -- checked +# against a species token after any leading genus-name duplicate is stripped (see +# clean_species_epithet), not just the bare "sp."/"sp" abbreviation. Shared by both +# MIBiG's free-text organism_name and NPAtlas's own origin_species SDF field, which turn +# out to carry the same kinds of placeholder values (confirmed live: NPAtlas's +# origin_species includes bare "sp.", genus-duplicated "Streptomyces sp.", and +# "unidentified" -- none of which are real species, but nothing was rejecting them before +# this, so they leaked into the phylogeny_annotations species column as if they were). +_NON_TAXONOMIC_SPECIES_TOKENS = {"sp", "spp", "unidentified", "uncultured", "unclassified"} + + +def clean_genus(genus_raw: str | None) -> str | None: + """Reject a genus value that's actually a non-taxonomic placeholder + ("unidentified"/"uncultured"/"unclassified") rather than a real genus name.""" + if not genus_raw: + return None + genus = genus_raw.strip() + if not genus or genus.lower() in _NON_TAXONOMIC_PREFIXES: + return None + return genus + + +def clean_species_epithet(genus: str | None, species_raw: str | None) -> str | None: + """Reject a species value carrying no real species-level information: a bare + "sp."/"sp" abbreviation, a genus name duplicated into the species field (e.g. + NPAtlas's origin_species="Streptomyces sp." alongside genus="Streptomyces"), or an + "unidentified"/"uncultured"/"unclassified" placeholder.""" + if not species_raw: + return None + + species = species_raw.strip() + if genus and species.lower().startswith(genus.lower() + " "): + species = species[len(genus):].strip() + + if not species or species.rstrip(".").lower() in _NON_TAXONOMIC_SPECIES_TOKENS: + return None + + return species + def phylogeny_from_organism_name(organism_name: str | None) -> tuple[str | None, str | None, str | None]: """Split MIBiG's free-text `organism_name` (e.g. "Streptomyces coelicolor A3(2)") into (type, genus, species). Type is inferred from `FUNGAL_GENERA` since MIBiG's JSON carries no kingdom field; genus/species are the name's first two tokens (after - dropping a leading "uncultured"/"unclassified"/"unidentified" marker), with species - dropped when the following token is "sp."/"sp" (strain-only identification). + dropping a leading "uncultured"/"unclassified"/"unidentified" marker), with a + non-taxonomic species value (see clean_species_epithet) dropped. :param organism_name: MIBiG cluster.organism_name, or None :return: (type_label, genus, species) -- each may be None if unresolvable @@ -205,10 +245,11 @@ def phylogeny_from_organism_name(organism_name: str | None) -> tuple[str | None, if not tokens: return None, None, None - genus = tokens[0] - species = tokens[1] if len(tokens) > 1 else None - if species is not None and species.rstrip(".").lower() == "sp": - species = None + genus = clean_genus(tokens[0]) + if not genus: + return None, None, None + + species = clean_species_epithet(genus, tokens[1] if len(tokens) > 1 else None) type_label = "Fungus" if genus.lower() in FUNGAL_GENERA else "Bacterium" return type_label, genus, species diff --git a/database/scripts/load_bgcs.py b/database/scripts/load_bgcs.py index ab8dc2f..99b6bc5 100644 --- a/database/scripts/load_bgcs.py +++ b/database/scripts/load_bgcs.py @@ -126,8 +126,13 @@ def run( species=resolution.species, species_taxid=resolution.species_taxid, ) - # chemical_class is compound-only (see RetroMolDuckDB.add_chemical_class_annotation) -- - # a BGC's biosynthetic class isn't populated here anymore. + # biosynthetic_class describes this BGC's own biosynthesis machinery + # (PKS/NRPS/RiPP/...) -- a gene-cluster property, not populated for + # compounds (see RetroMolDuckDB.add_biosynthetic_class_annotation / + # load_compounds.py). + for chemical_class in record.get("biosyn_class") or []: + if chemical_class: + db.add_biosynthetic_class_annotation(entry_id, str(chemical_class)) added += 1 pbar.set_postfix(added=added, skipped=skipped, skipped_existing=skipped_existing_file) diff --git a/database/scripts/load_compounds.py b/database/scripts/load_compounds.py index a84c1d6..1898591 100644 --- a/database/scripts/load_compounds.py +++ b/database/scripts/load_compounds.py @@ -23,6 +23,8 @@ from common import ( build_fingerprint_context, + clean_genus, + clean_species_epithet, find_key_ci, load_ruleset, mibig_url, @@ -53,17 +55,26 @@ def _npatlas_name_and_url(props: dict) -> tuple[str | None, str | None]: def _npatlas_phylogeny(props: dict) -> tuple[str | None, str | None, str | None]: """NPAtlas's SDF carries type/genus/species directly (unlike MIBiG's free-text - organism_name) -- see the `origin_type`/`genus`/`origin_species` SDF properties.""" + organism_name) -- see the `origin_type`/`genus`/`origin_species` SDF properties. + + Both genus and origin_species are used as-is by NPAtlas's own curators, which + turns out to include the same non-taxonomic placeholders MIBiG's free text does + (bare "sp.", a genus name duplicated into the species field e.g. "Streptomyces + sp.", "unidentified") -- cleaned the same way as MIBiG's organism_name parsing + (see common.clean_genus/clean_species_epithet), so neither source's placeholder + values end up stored as if they were real species-level identifications. + """ type_key = find_key_ci(props, ["origin_type"]) type_label = props.get(type_key) if type_key else None genus_key = find_key_ci(props, ["genus"]) - genus = props.get(genus_key) if genus_key else None + genus = clean_genus(props.get(genus_key) if genus_key else None) species_key = find_key_ci(props, ["origin_species"]) - species = props.get(species_key) if species_key else None + species_raw = props.get(species_key) if species_key else None + species = clean_species_epithet(genus, species_raw) - return (type_label or None), (genus or None), (species or None) + return (type_label or None), genus, species def _mibig_name_and_url(props: dict, versions: dict[str, str]) -> tuple[str | None, str | None]: @@ -98,10 +109,8 @@ def _apply_mibig_annotations( species=resolution.species, species_taxid=resolution.species_taxid, ) - - for chemical_class in record.get("biosyn_class") or []: - if chemical_class: - db.add_biosynthetic_class_annotation(entry_id, str(chemical_class)) + # biosynthetic_class describes the BGC's own biosynthesis machinery (PKS/NRPS/...), + # not the compound structure -- populated in load_bgcs.py instead, not here. def run( diff --git a/database/scripts/taxonomy.py b/database/scripts/taxonomy.py index 87d428e..3f6b580 100644 --- a/database/scripts/taxonomy.py +++ b/database/scripts/taxonomy.py @@ -172,6 +172,9 @@ def type_label_and_taxid(self, taxid: int | None) -> tuple[str | None, int | Non return "Other", None +_UNRESOLVED = PhylogenyResolution(None, None, None, None, None, None) + + def resolve_phylogeny( taxdb: TaxonomyDB | None, *, @@ -180,19 +183,25 @@ def resolve_phylogeny( species: str | None = None, fallback_type_label: str | None = None, ) -> PhylogenyResolution: - """Standardize phylogeny fields to NCBI taxids where possible. + """Standardize phylogeny fields to NCBI taxids -- and *only* to NCBI taxids: every + label stored is read back off a resolved taxid via TaxonomyDB.canonical_name, never + passed through as raw, unstandardized source text. If a taxid can't be resolved at + all (no taxdb loaded, or the given ncbi_tax_id/genus/species doesn't match anything + in NCBI's taxonomy), the result is fully unannotated -- type/genus/species all + None -- rather than a best-effort guess from whatever text the source gave us. + `fallback_type_label` (NPAtlas's own origin_type field) is accepted for API + compatibility with callers but is deliberately unused for the same reason: it isn't + NCBI vocabulary either. If `ncbi_tax_id` is given (MIBiG), it's the source of truth: genus/species/type and their taxids are all derived from its lineage, overriding any given genus/species - text. Otherwise (NPAtlas), `genus`/`species` text is resolved to a taxid by name - lookup -- "Genus species" first, falling back to genus alone -- and, if resolved, - canonical names/taxids for every rank are read back off the same lineage, so both - sources end up identically standardized. If nothing resolves (no taxdb, or the - name/taxid isn't found), the given genus/species/fallback_type_label pass through - unchanged with no taxids -- unresolved is not an error here, just unenriched. + text. Otherwise (NPAtlas, or an MIBiG entry whose ncbi_tax_id didn't parse), + `genus`/`species` text is resolved to a taxid by name lookup -- "Genus species" + first, falling back to genus alone -- so both sources end up identically + standardized when they resolve at all. """ if taxdb is None: - return PhylogenyResolution(fallback_type_label, None, genus, None, species, None) + return _UNRESOLVED leaf_taxid: int | None = None if ncbi_tax_id: @@ -200,23 +209,24 @@ def resolve_phylogeny( leaf_taxid = int(ncbi_tax_id) except (TypeError, ValueError): leaf_taxid = None - elif genus: + + if leaf_taxid is None and genus: query = f"{genus} {species}" if species else genus leaf_taxid = taxdb.resolve_taxid(query) or taxdb.resolve_taxid(genus) if leaf_taxid is None: - return PhylogenyResolution(fallback_type_label, None, genus, None, species, None) + return _UNRESOLVED type_label, type_taxid = taxdb.type_label_and_taxid(leaf_taxid) genus_taxid = taxdb.ancestor_at_rank(leaf_taxid, "genus") - genus_name = taxdb.canonical_name(genus_taxid) if genus_taxid else genus + genus_name = taxdb.canonical_name(genus_taxid) if genus_taxid else None species_taxid = taxdb.ancestor_at_rank(leaf_taxid, "species") - species_name = taxdb.canonical_name(species_taxid) if species_taxid else species + species_name = taxdb.canonical_name(species_taxid) if species_taxid else None return PhylogenyResolution( - type_label=type_label or fallback_type_label, + type_label=type_label, type_taxid=str(type_taxid) if type_taxid is not None else None, genus=genus_name, genus_taxid=str(genus_taxid) if genus_taxid is not None else None, diff --git a/gui/scripts/dev_backend.sh b/gui/scripts/dev_backend.sh index ccce31e..477c435 100644 --- a/gui/scripts/dev_backend.sh +++ b/gui/scripts/dev_backend.sh @@ -10,7 +10,7 @@ export FLASK_ENV=development export PORT=4000 # DB connection -export RETROMOL_DUCKDB_PATH="$HOME/Downloads/retromol.duckdb" +export RETROMOL_DUCKDB_PATH="$HOME/Desktop/retromol.duckdb" # Redis connection (uses Dockerized Redis) export REDIS_URL="redis://localhost:6379/0" diff --git a/gui/src/client/src/components/workspace/AlignmentGrid.tsx b/gui/src/client/src/components/workspace/AlignmentGrid.tsx index 8358a6f..72fe3f8 100644 --- a/gui/src/client/src/components/workspace/AlignmentGrid.tsx +++ b/gui/src/client/src/components/workspace/AlignmentGrid.tsx @@ -25,6 +25,7 @@ import { buildAlignmentSvg, downloadSvg, type AlignmentSvgRow } from "./alignmen import { MinimalIconButton} from "../MinimalIconButton"; import { getEntryAnnotations } from "../../features/database/api"; import type { EntryAnnotation } from "../../features/database/types"; +import { isChebiRoleRank, toSentenceCase } from "../../features/database/format"; const ANNOTATION_CATEGORY_LABELS: Record = { phylogeny: "Phylogeny", @@ -43,6 +44,23 @@ function groupAnnotationsByCategory(annotations: EntryAnnotation[]): [string, En return Array.from(groups.entries()); } +// Shared by the "Databases" (source) row and every AnnotationChips category row -- +// one component, not parallel copies of the same Stack/Typography markup, so the +// label-to-chip gap is guaranteed identical by construction rather than by keeping two +// separate JSX blocks in sync by hand. +const AnnotationRow: React.FC<{ label: string; sx?: object; children: React.ReactNode }> = ({ + label, + sx, + children, +}) => ( + + + {label} + + {children} + +); + const AnnotationChips: React.FC<{ entryId: string }> = ({ entryId }) => { const [annotations, setAnnotations] = React.useState(null); const [error, setError] = React.useState(null); @@ -75,14 +93,12 @@ const AnnotationChips: React.FC<{ entryId: string }> = ({ entryId }) => { } return ( - + {groupAnnotationsByCategory(annotations).map(([category, items]) => ( - - - {ANNOTATION_CATEGORY_LABELS[category] ?? category} - - {items.map((item) => - item.url ? ( + + {items.map((item) => { + const label = isChebiRoleRank(item.rank) ? toSentenceCase(item.label) : item.label; + return item.url ? ( = ({ entryId }) => { target="_blank" rel="noopener noreferrer" clickable + color="primary" size="small" variant="outlined" - label={item.label} + label={label} /> ) : ( - - ) - )} - + + ); + })} + ))} ); @@ -382,7 +399,7 @@ export function ResultRow({ {result.sources.length > 0 && ( - + {result.sources.map((source, idx) => source.url ? ( ) )} - + )} { + const entry = stats.coverage.find((c) => c.label === label); + if (!entry) return null; + return ( + + ); +}; const ENTRY_TYPE_LABELS: Record = { compound: "Compounds", @@ -56,28 +73,68 @@ const ChartCard: React.FC<{ title: string; description?: string; children: React ); -const AnnotationBarCard: React.FC<{ +// Pie charts, not bar charts -- the installed @mui/x-charts (8.29.2) BarChart has a +// real rendering bug where rotating x-axis tick labels (any angle, tried -90 and -45) +// renders every tick as an empty : text node present, content blank, confirmed +// live in the DOM. A CSS-transform workaround got labels rotated but the axis itself +// (many ticks, long category names) is still cramped and prone to overlap/misread at +// this data size. PieChart has no axis-label problem at all -- its legend is plain +// text -- and reads better for "which N labels dominate" than a bar chart would here +// anyway. Long tails (more than TOP_N distinct labels) collapse into "Other" so the +// legend stays readable instead of listing 15 items. A slice also collapses into +// "Other" if it's under MIN_SLICE_SHARE of the total, even when it's within the top +// N by rank -- a skewed distribution (one dominant label, four small ones) can still +// produce a top-N slice too thin to see or click, not just labels past rank N. +const TOP_N = 5; +const MIN_SLICE_SHARE = 0.02; + +const AnnotationPieCard: React.FC<{ title: string; description?: string; counts: Count[]; - color: string; + colors: string[]; emptyMessage: string; -}> = ({ title, description, counts, color, emptyMessage }) => ( - - {counts.length === 0 ? ( - - {emptyMessage} - - ) : ( - ({ label: c.label, count: c.count }))} - xAxis={[{ scaleType: "band", dataKey: "label" }]} - series={[{ dataKey: "count", color }]} - height={220} - /> - )} - -); +}> = ({ title, description, counts, colors, emptyMessage }) => { + const total = counts.reduce((sum, c) => sum + c.count, 0); + const kept: Count[] = []; + let otherTotal = 0; + counts.forEach((c, i) => { + if (i < TOP_N && c.count / total >= MIN_SLICE_SHARE) { + kept.push(c); + } else { + otherTotal += c.count; + } + }); + const pieData = [ + ...kept.map((c, i) => ({ id: i, value: c.count, label: c.label })), + ...(otherTotal > 0 ? [{ id: "other", value: otherTotal, label: "Other" }] : []), + ]; + + return ( + + {counts.length === 0 ? ( + + {emptyMessage} + + ) : ( + + )} + + ); +}; export const WorkspaceHome: React.FC = () => { const theme = useTheme(); @@ -267,37 +324,33 @@ export const WorkspaceHome: React.FC = () => { {!annotationError && !annotationLoading && annotationStats && ( <> - - - - Phylogeny - + + - - @@ -309,22 +362,25 @@ export const WorkspaceHome: React.FC = () => { NPClassifier, predicted from every compound's own structure - + + - - @@ -336,10 +392,13 @@ export const WorkspaceHome: React.FC = () => { MIBiG's own coarse label (PKS / NRPS / RiPP / ...), a separate classification from NPClassifier's chemical class above - + + @@ -348,35 +407,24 @@ export const WorkspaceHome: React.FC = () => { Bioactivity - ChEMBL + ChEBI, looked up by structure (InChIKey) + ChEBI's role ontology, looked up by structure (InChIKey) - - - + + ({ ...c, label: toSentenceCase(c.label) }))} + colors={chartColors} emptyMessage="No ChEBI biological-role matches yet." /> - ({ ...c, label: toSentenceCase(c.label) }))} + colors={chartColors} emptyMessage="No ChEBI chemical-role matches yet." /> diff --git a/gui/src/client/src/features/database/format.ts b/gui/src/client/src/features/database/format.ts new file mode 100644 index 0000000..cb1f4ff --- /dev/null +++ b/gui/src/client/src/features/database/format.ts @@ -0,0 +1,17 @@ +// ChEBI role names (both "chebi_biological_role" and "chebi_chemical_role") come from +// the source data lowercase (e.g. "antibacterial agent", "xenobiotic") -- sentence-cased +// here for display only. Never touches the stored/queried label itself (exact-match +// lookups, external links, and backend grouping all still use the raw lowercase form). +const CHEBI_ROLE_RANKS = new Set(["chebi_biological_role", "chebi_chemical_role"]); + +export function isChebiRoleRank(rank: string | null | undefined): boolean { + return !!rank && CHEBI_ROLE_RANKS.has(rank); +} + +export function toSentenceCase(label: string): string { + // Capitalize only the label's own first character -- "antibacterial agent" -> + // "Antibacterial agent", not "Antibacterial Agent". Not `\b\w`-regex-based: JS + // regex `\w` is ASCII-only, so it doesn't recognize letters like "ø" as word + // characters, which would break `\b` (word-boundary) detection right after one. + return label.length > 0 ? label.charAt(0).toUpperCase() + label.slice(1) : label; +} diff --git a/gui/src/client/src/features/database/types.ts b/gui/src/client/src/features/database/types.ts index ca680e9..cac3cd4 100644 --- a/gui/src/client/src/features/database/types.ts +++ b/gui/src/client/src/features/database/types.ts @@ -18,9 +18,15 @@ export const DatabaseStatsRespSchema = z.object({ }); export type DatabaseStatsResp = z.output; -export const AnnotationStatsRespSchema = z.object({ +export const AnnotationCoverageSchema = z.object({ + label: z.string(), withAnnotationCount: z.number().int().nonnegative(), withoutAnnotationCount: z.number().int().nonnegative(), +}); +export type AnnotationCoverage = z.output; + +export const AnnotationStatsRespSchema = z.object({ + coverage: z.array(AnnotationCoverageSchema), countsByCategory: z.array(CountSchema), phylogenyTypeCounts: z.array(CountSchema), phylogenyGenusCounts: z.array(CountSchema), @@ -29,8 +35,6 @@ export const AnnotationStatsRespSchema = z.object({ chemicalClassPathwayCounts: z.array(CountSchema), chemicalClassSuperclassCounts: z.array(CountSchema), chemicalClassClassCounts: z.array(CountSchema), - bioactivityAtcCounts: z.array(CountSchema), - bioactivityMaxPhaseCounts: z.array(CountSchema), bioactivityBiologicalRoleCounts: z.array(CountSchema), bioactivityChemicalRoleCounts: z.array(CountSchema), }); diff --git a/gui/src/server/routes/entry_annotations.py b/gui/src/server/routes/entry_annotations.py index fb90951..01de3d9 100644 --- a/gui/src/server/routes/entry_annotations.py +++ b/gui/src/server/routes/entry_annotations.py @@ -11,8 +11,6 @@ blp_entry_annotations = Blueprint("entry_annotations", __name__) NCBI_TAXONOMY_URL = "https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=Info&id={id}" -CHEMBL_COMPOUND_URL = "https://www.ebi.ac.uk/chembl/compound_report_card/{id}/" -WHO_ATC_INDEX_URL = "https://www.whocc.no/atc_ddd_index/?code={id}" CHEBI_ENTITY_URL = "https://www.ebi.ac.uk/chebi/searchId.do?chebiId={id}" @@ -25,10 +23,6 @@ def _annotation_url(category: str, rank: str | None, external_id: str | None) -> if category == "phylogeny": return NCBI_TAXONOMY_URL.format(id=external_id) if category == "bioactivity": - if rank == "chembl_max_phase": - return CHEMBL_COMPOUND_URL.format(id=external_id) - if rank == "chembl_atc": - return WHO_ATC_INDEX_URL.format(id=external_id) if rank in ("chebi_biological_role", "chebi_chemical_role"): return CHEBI_ENTITY_URL.format(id=external_id) return None diff --git a/gui/src/server/routes/stats.py b/gui/src/server/routes/stats.py index 6d433aa..373aaa6 100644 --- a/gui/src/server/routes/stats.py +++ b/gui/src/server/routes/stats.py @@ -14,6 +14,19 @@ def _to_camel_case(snake: str) -> str: return head + "".join(word.capitalize() for word in tail) +def _camelize(value): + """Recursively camelCase every dict key in `value` -- `asdict()` on a dataclass + with nested dataclasses (e.g. AnnotationStats.coverage: list[AnnotationCoverage]) + only produces nested plain dicts/lists, so a shallow top-level-only conversion + misses every key below the first level (that's exactly what broke + coverage[i].with_annotation_count -> withAnnotationCount before this).""" + if isinstance(value, dict): + return {_to_camel_case(k): _camelize(v) for k, v in value.items()} + if isinstance(value, list): + return [_camelize(v) for v in value] + return value + + @blp_database_stats.get("/api/databaseStats") def database_stats() -> tuple[Response, int]: """ @@ -25,7 +38,7 @@ def database_stats() -> tuple[Response, int]: with open_retromol_db() as db: stats = db.stats() - payload = {_to_camel_case(k): v for k, v in asdict(stats).items()} + payload = _camelize(asdict(stats)) return jsonify(payload), 200 @@ -40,5 +53,5 @@ def annotation_stats() -> tuple[Response, int]: with open_retromol_db() as db: stats = db.annotation_stats() - payload = {_to_camel_case(k): v for k, v in asdict(stats).items()} + payload = _camelize(asdict(stats)) return jsonify(payload), 200 diff --git a/src/retromol_database/duckdb.py b/src/retromol_database/duckdb.py index 4a75c04..67c4f15 100644 --- a/src/retromol_database/duckdb.py +++ b/src/retromol_database/duckdb.py @@ -69,16 +69,27 @@ class AnnotationTerm: label: str parent_id: str | None # An id in whatever external database this term comes from (NCBI taxid for - # phylogeny; ATC code/ChEBI accession/ChEMBL id for bioactivity), for linking out - # to that database's own page -- None for categories with no such external page - # (biosynthetic_class, chemical_class). + # phylogeny; ChEBI accession for bioactivity), for linking out to that database's + # own page -- None for categories with no such external page (biosynthetic_class, + # chemical_class). external_id: str | None = None @dataclass(frozen=True) -class AnnotationStats: +class AnnotationCoverage: + label: str with_annotation_count: int without_annotation_count: int + + +@dataclass(frozen=True) +class AnnotationStats: + # Coverage per (category, entry type) that actually gets populated for it -- + # phylogeny applies to both compounds and bgcs (an organism produces both), so it + # gets two entries; the rest are single-entry-type by design (see the population + # sites: chemical_class/bioactivity are compound-structure-derived, biosynthetic_class + # describes a BGC's own biosynthesis machinery). + coverage: list[AnnotationCoverage] counts_by_category: list[Count] # Phylogeny: one chart per rank. phylogeny_type_counts: list[Count] @@ -93,11 +104,7 @@ class AnnotationStats: chemical_class_pathway_counts: list[Count] chemical_class_superclass_counts: list[Count] chemical_class_class_counts: list[Count] - # Bioactivity: ChEMBL's two signals (see database/scripts/annotate_chembl.py) -- - # WHO ATC therapeutic category and clinical-development phase -- plus ChEBI's role - # ontology (see database/scripts/annotate_chebi.py). - bioactivity_atc_counts: list[Count] - bioactivity_max_phase_counts: list[Count] + # Bioactivity: ChEBI's role ontology (see database/scripts/annotate_chebi.py). bioactivity_biological_role_counts: list[Count] bioactivity_chemical_role_counts: list[Count] @@ -190,11 +197,11 @@ def create_schema(self) -> None: # Four dedicated annotation tables, one per category, each with its own typed # columns instead of a single generic label -- see RetroMolDuckDB.add_phylogeny_annotation # / add_bioactivity_annotation / add_biosynthetic_class_annotation / add_chemical_class_annotation. - # Phylogeny is one row per entry (an entry has exactly one organism); the other three - # are multi-valued per entry. biosynthetic_class/chemical_class are populated for - # compounds only (they describe the molecule, not the organism/cluster) -- enforced - # by pipeline callers, not by this schema. Bioactivity population logic isn't wired - # up yet; that table exists ahead of that. + # Phylogeny is one row per entry (an entry has exactly one organism); the other + # three are multi-valued per entry. chemical_class/bioactivity are populated for + # compounds only (they describe the molecule's own structure); biosynthetic_class + # is populated for bgcs only (it describes the gene cluster's biosynthesis + # machinery, not the compound) -- enforced by pipeline callers, not by this schema. self.con.execute( """ CREATE TABLE IF NOT EXISTS phylogeny_annotations ( @@ -208,16 +215,11 @@ def create_schema(self) -> None: ) """ ) - # `level` distinguishes bioactivity's four signals (see - # database/scripts/annotate_chembl.py / annotate_chebi.py): ChEMBL's - # "chembl_max_phase" (clinical-development stage -- "Approved"/"Phase 3"/... -- - # only stored when meaningful, same presence-only convention as chemical_class's - # is_glycoside) and "chembl_atc" (WHO ATC therapeutic category), plus ChEBI's - # "chebi_biological_role" and "chebi_chemical_role" (its role ontology, under - # CHEBI:24432/CHEBI:51086). All four are multi-valued per entry except - # chembl_max_phase. `external_id` is the id in that row's own external database - # (an ATC code, a ChEBI accession, or a ChEMBL id for the max_phase row's own - # compound page) -- for building a "view on " link; null when unresolved. + # `level` distinguishes bioactivity's two signals (see + # database/scripts/annotate_chebi.py): ChEBI's "chebi_biological_role" and + # "chebi_chemical_role" (its role ontology, under CHEBI:24432/CHEBI:51086) -- + # both multi-valued per entry. `external_id` is the term's own ChEBI accession, + # for building a "view on ChEBI" link; null when unresolved. self.con.execute( """ CREATE TABLE IF NOT EXISTS bioactivity_annotations ( @@ -430,9 +432,9 @@ def add_bioactivity_annotation( self, entry_id: str, *, level: str, label: str, external_id: str | None = None ) -> None: """Link `entry_id` (a compound) to a bioactivity label at the given `level` (e.g. - "chembl_max_phase", "chembl_atc", "chebi_biological_role", "chebi_chemical_role" -- - see database/scripts/annotate_chembl.py / annotate_chebi.py). Compounds only -- - callers must not use this for bgc entries.""" + "chebi_biological_role", "chebi_chemical_role" -- see + database/scripts/annotate_chebi.py). Compounds only -- callers must not use this + for bgc entries.""" self.con.execute( """ INSERT INTO bioactivity_annotations (entry_id, level, label, external_id) @@ -443,8 +445,8 @@ def add_bioactivity_annotation( ) def add_biosynthetic_class_annotation(self, entry_id: str, label: str) -> None: - """Link `entry_id` (a compound) to a MIBiG biosynthetic-class label (PKS/NRPS/RiPP/...). - Compounds only -- callers must not use this for bgc entries.""" + """Link `entry_id` (a bgc) to a MIBiG biosynthetic-class label (PKS/NRPS/RiPP/...). + BGCs only -- callers must not use this for compound entries.""" self.con.execute( """ INSERT INTO biosynthetic_class_annotations (entry_id, label) @@ -617,12 +619,39 @@ def _label_counts(self, *, category: str, rank: str | None = None, limit: int | ).fetchall() return [Count(label=str(row[0]), count=int(row[1])) for row in rows] + def _annotation_coverage(self, *, category: str, entry_type: str, label: str) -> AnnotationCoverage: + """with/without-annotation counts for one (category, entry type) slice -- e.g. + "how many compounds have a chemical_class annotation", not "how many entries of + any type have any annotation" (too coarse to be useful once there are several + categories that each apply to only one entry type).""" + total = self.count_entries_by_type([entry_type]) + with_count = int( + self.con.execute( + """ + SELECT count(DISTINCT ea.entry_id) + FROM entry_annotations ea + JOIN annotation_terms t ON t.id = ea.term_id + JOIN entries e ON e.id = ea.entry_id + WHERE t.category = ? AND e.type = ? + """, + [category, entry_type], + ).fetchone()[0] + ) + return AnnotationCoverage(label=label, with_annotation_count=with_count, without_annotation_count=total - with_count) + def annotation_stats(self) -> AnnotationStats: """Summary counts over annotation_terms/entry_annotations, for the Dashboard.""" - with_annotation_count = int( - self.con.execute("SELECT count(DISTINCT entry_id) FROM entry_annotations").fetchone()[0] - ) - without_annotation_count = self.count() - with_annotation_count + coverage = [ + self._annotation_coverage(category="phylogeny", entry_type="compound", label="Phylogeny (compounds)"), + self._annotation_coverage(category="phylogeny", entry_type="bgc", label="Phylogeny (gene clusters)"), + self._annotation_coverage( + category="chemical_class", entry_type="compound", label="Chemical class (compounds)" + ), + self._annotation_coverage( + category="biosynthetic_class", entry_type="bgc", label="Biosynthetic class (gene clusters)" + ), + self._annotation_coverage(category="bioactivity", entry_type="compound", label="Bioactivity (compounds)"), + ] category_rows = self.con.execute( """ @@ -636,8 +665,7 @@ def annotation_stats(self) -> AnnotationStats: counts_by_category = [Count(label=str(row[0]), count=int(row[1])) for row in category_rows] return AnnotationStats( - with_annotation_count=with_annotation_count, - without_annotation_count=without_annotation_count, + coverage=coverage, counts_by_category=counts_by_category, phylogeny_type_counts=self._label_counts(category="phylogeny", rank="type"), phylogeny_genus_counts=self._label_counts(category="phylogeny", rank="genus", limit=15), @@ -646,8 +674,6 @@ def annotation_stats(self) -> AnnotationStats: chemical_class_pathway_counts=self._label_counts(category="chemical_class", rank="pathway", limit=15), chemical_class_superclass_counts=self._label_counts(category="chemical_class", rank="superclass", limit=15), chemical_class_class_counts=self._label_counts(category="chemical_class", rank="class", limit=15), - bioactivity_atc_counts=self._label_counts(category="bioactivity", rank="chembl_atc", limit=15), - bioactivity_max_phase_counts=self._label_counts(category="bioactivity", rank="chembl_max_phase"), bioactivity_biological_role_counts=self._label_counts( category="bioactivity", rank="chebi_biological_role", limit=15 ), From 8aef3fc0be2c7f34c5b5278d24cfa1bca8c7c46f Mon Sep 17 00:00:00 2001 From: David Meijer Date: Wed, 26 Aug 2026 23:49:44 -0400 Subject: [PATCH 09/11] FEA: moved enrichment to results tab --- gui/src/client/package-lock.json | 254 +++++++++++++++-- gui/src/client/package.json | 1 + gui/src/client/src/components/MenuContent.tsx | 6 - .../workspace/DialogViewDiscoveryQuery.tsx | 75 +++-- .../workspace/DiscoveryEnrichmentView.tsx | 223 +++++++++++++++ .../src/components/workspace/Workspace.tsx | 2 - .../workspace/WorkspaceDiscovery.tsx | 20 +- .../components/workspace/WorkspaceUpload.tsx | 14 +- .../tabs/enrichment/WorkspaceEnrichment.tsx | 269 ------------------ .../client/src/features/discovery/types.ts | 1 + gui/src/client/src/features/enrichment/api.ts | 20 +- .../client/src/features/enrichment/types.ts | 26 +- gui/src/client/src/features/session/types.ts | 4 + gui/src/server/app.py | 3 +- gui/src/server/routes/discovery.py | 5 + gui/src/server/routes/enrichment.py | 55 +--- 16 files changed, 552 insertions(+), 426 deletions(-) create mode 100644 gui/src/client/src/components/workspace/DiscoveryEnrichmentView.tsx delete mode 100644 gui/src/client/src/components/workspace/tabs/enrichment/WorkspaceEnrichment.tsx diff --git a/gui/src/client/package-lock.json b/gui/src/client/package-lock.json index a29b418..d777eac 100644 --- a/gui/src/client/package-lock.json +++ b/gui/src/client/package-lock.json @@ -16,6 +16,7 @@ "@mui/icons-material": "^7.3.5", "@mui/material": "^7.3.5", "@mui/x-charts": "^8.17.0", + "@mui/x-data-grid": "^9.12.0", "@react-spring/web": "^10.0.3", "@tanstack/react-query": "^5.76.1", "@types/react": "18.3.10", @@ -2263,6 +2264,28 @@ "node": ">=6.9.0" } }, + "node_modules/@base-ui/utils": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@base-ui/utils/-/utils-0.3.2.tgz", + "integrity": "sha512-oWy1aq/I2GmYjpl4PhEAhzflF8VPGKgZeq0xAWTbfD5KBWyxcN0ZP2+WHSUm/5Z6lVMBDLReLcoXwSYoRc/zNQ==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.29.2", + "@floating-ui/utils": "^0.2.12", + "reselect": "^5.2.0", + "use-sync-external-store": "^1.6.0" + }, + "peerDependencies": { + "@types/react": "^17 || ^18 || ^19", + "react": "^17 || ^18 || ^19", + "react-dom": "^17 || ^18 || ^19" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, "node_modules/@bcoe/v8-coverage": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz", @@ -2718,6 +2741,7 @@ "resolved": "https://registry.npmjs.org/@emotion/is-prop-valid/-/is-prop-valid-1.4.0.tgz", "integrity": "sha512-QgD4fyscGcbbKwJmqNvUMSE02OsHUa+lAWKdEUIJKgqe5IwRSKd7+KhibEWdaKwgjLj0DRSHA9biAIqGBk05lw==", "license": "MIT", + "peer": true, "dependencies": { "@emotion/memoize": "^0.9.0" } @@ -2916,6 +2940,12 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, + "node_modules/@floating-ui/utils": { + "version": "0.2.12", + "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.12.tgz", + "integrity": "sha512-HpCo8tmWzLVad5s2d19EhAz5zqrrQ6s69qd6moPMQvkOuSwDT1YgRfWSVuc4ennqrgv3OHppiOGMQ7oC13yIww==", + "license": "MIT" + }, "node_modules/@humanwhocodes/config-array": { "version": "0.13.0", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.13.0.tgz", @@ -3662,6 +3692,116 @@ "internmap": "^2.0.3" } }, + "node_modules/@mui/x-data-grid": { + "version": "9.12.0", + "resolved": "https://registry.npmjs.org/@mui/x-data-grid/-/x-data-grid-9.12.0.tgz", + "integrity": "sha512-lI7NYAPhoLIWJECB2gZuVairzWviDDOE1RpjLcPXJM+hw1mwCG+isqNfx81X+17xuHtnEEBVahCDWAJFAvCF9g==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.29.7", + "@base-ui/utils": "^0.3.1", + "@mui/utils": "^9.3.0", + "@mui/x-internals": "^9.12.0", + "@mui/x-virtualizer": "0.7.0", + "clsx": "^2.1.1", + "prop-types": "^15.8.1", + "use-sync-external-store": "^1.6.0" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "@emotion/react": "^11.9.0", + "@emotion/styled": "^11.8.1", + "@mui/material": "^7.3.0 || ^9.0.0", + "@mui/system": "^7.3.0 || ^9.0.0", + "react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react-dom": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@emotion/react": { + "optional": true + }, + "@emotion/styled": { + "optional": true + } + } + }, + "node_modules/@mui/x-data-grid/node_modules/@mui/types": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/@mui/types/-/types-9.3.0.tgz", + "integrity": "sha512-2JSxyfpEFWNUB2vKs/T1BvkfyNisMHWph8bLMj8T0uHwmLl/0qfAwQkfwMT6kxLXN9uIum9AEbECXU8er3amIg==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.29.7" + }, + "peerDependencies": { + "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@mui/x-data-grid/node_modules/@mui/utils": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/@mui/utils/-/utils-9.3.0.tgz", + "integrity": "sha512-2HZdHwWJ6eB+7lVGSOHsByGw8jeRulT4g0NZ608Wb8Q57DE2jbNqrWPFuJsvkQQiBiTmlpvQL3i+/62zsiPrkw==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.29.7", + "@mui/types": "^9.3.0", + "@types/prop-types": "^15.7.15", + "clsx": "^2.1.1", + "prop-types": "^15.8.1", + "react-is": "^19.2.8" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@mui/x-data-grid/node_modules/@mui/x-internals": { + "version": "9.12.0", + "resolved": "https://registry.npmjs.org/@mui/x-internals/-/x-internals-9.12.0.tgz", + "integrity": "sha512-rpG9EgJhH3JeOK2Q/tKaUWk5vP8C1NfL8DaXMKR1XCA7RhpZUI/xT67WKY80QtogNWu7U/BRezKyQGtiZ8soYg==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.29.7", + "@base-ui/utils": "^0.3.1", + "@mui/utils": "^9.3.0", + "reselect": "^5.2.0", + "use-sync-external-store": "^1.6.0" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react-dom": "^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, "node_modules/@mui/x-internal-gestures": { "version": "0.5.0", "resolved": "https://registry.npmjs.org/@mui/x-internal-gestures/-/x-internal-gestures-0.5.0.tgz", @@ -3693,6 +3833,100 @@ "react": "^17.0.0 || ^18.0.0 || ^19.0.0" } }, + "node_modules/@mui/x-virtualizer": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/@mui/x-virtualizer/-/x-virtualizer-0.7.0.tgz", + "integrity": "sha512-Bz5p/t78zE2q6rf/nHY3j29hr7JlWaBNT0OD8jZFcfstr8jqrPkzirGsl/Bz5EpEjEzxV73kLfZLpIToGql+rA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.29.7", + "@base-ui/utils": "^0.3.1", + "@mui/utils": "^9.3.0", + "@mui/x-internals": "^9.12.0" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react-dom": "^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/@mui/x-virtualizer/node_modules/@mui/types": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/@mui/types/-/types-9.3.0.tgz", + "integrity": "sha512-2JSxyfpEFWNUB2vKs/T1BvkfyNisMHWph8bLMj8T0uHwmLl/0qfAwQkfwMT6kxLXN9uIum9AEbECXU8er3amIg==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.29.7" + }, + "peerDependencies": { + "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@mui/x-virtualizer/node_modules/@mui/utils": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/@mui/utils/-/utils-9.3.0.tgz", + "integrity": "sha512-2HZdHwWJ6eB+7lVGSOHsByGw8jeRulT4g0NZ608Wb8Q57DE2jbNqrWPFuJsvkQQiBiTmlpvQL3i+/62zsiPrkw==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.29.7", + "@mui/types": "^9.3.0", + "@types/prop-types": "^15.7.15", + "clsx": "^2.1.1", + "prop-types": "^15.8.1", + "react-is": "^19.2.8" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@mui/x-virtualizer/node_modules/@mui/x-internals": { + "version": "9.12.0", + "resolved": "https://registry.npmjs.org/@mui/x-internals/-/x-internals-9.12.0.tgz", + "integrity": "sha512-rpG9EgJhH3JeOK2Q/tKaUWk5vP8C1NfL8DaXMKR1XCA7RhpZUI/xT67WKY80QtogNWu7U/BRezKyQGtiZ8soYg==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.29.7", + "@base-ui/utils": "^0.3.1", + "@mui/utils": "^9.3.0", + "reselect": "^5.2.0", + "use-sync-external-store": "^1.6.0" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react-dom": "^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, "node_modules/@nicolo-ribaudo/eslint-scope-5-internals": { "version": "5.1.1-v1", "resolved": "https://registry.npmjs.org/@nicolo-ribaudo/eslint-scope-5-internals/-/eslint-scope-5-internals-5.1.1-v1.tgz", @@ -18166,24 +18400,6 @@ } } }, - "node_modules/tailwindcss/node_modules/yaml": { - "version": "2.9.0", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", - "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==", - "dev": true, - "license": "ISC", - "optional": true, - "peer": true, - "bin": { - "yaml": "bin.mjs" - }, - "engines": { - "node": ">= 14.6" - }, - "funding": { - "url": "https://github.com/sponsors/eemeli" - } - }, "node_modules/tapable": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.3.3.tgz", @@ -18693,6 +18909,7 @@ "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", "dev": true, "license": "(MIT OR CC0-1.0)", + "peer": true, "engines": { "node": ">=10" }, @@ -19240,6 +19457,7 @@ "integrity": "sha512-0XavAZbNJ5sDrCbkpWL8mia0o5WPOd2YGtxrEiZkBK9FjLppIUK2TgxK6qGD2P3hUXTJNNPVibrerKcx5WkR1g==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@types/bonjour": "^3.5.9", "@types/connect-history-api-fallback": "^1.3.5", diff --git a/gui/src/client/package.json b/gui/src/client/package.json index 11ce0ad..fb2ceb6 100644 --- a/gui/src/client/package.json +++ b/gui/src/client/package.json @@ -11,6 +11,7 @@ "@mui/icons-material": "^7.3.5", "@mui/material": "^7.3.5", "@mui/x-charts": "^8.17.0", + "@mui/x-data-grid": "^9.12.0", "@react-spring/web": "^10.0.3", "@tanstack/react-query": "^5.76.1", "@types/react": "18.3.10", diff --git a/gui/src/client/src/components/MenuContent.tsx b/gui/src/client/src/components/MenuContent.tsx index 725483e..76b3c9d 100644 --- a/gui/src/client/src/components/MenuContent.tsx +++ b/gui/src/client/src/components/MenuContent.tsx @@ -6,7 +6,6 @@ import ListItemIcon from "@mui/material/ListItemIcon"; import ListItemText from "@mui/material/ListItemText"; import Stack from "@mui/material/Stack"; import ExploreIcon from "@mui/icons-material/Explore"; -import BarChartIcon from "@mui/icons-material/BarChart"; import HomeRoundedIcon from "@mui/icons-material/HomeRounded"; import UploadFileIcon from "@mui/icons-material/UploadFile"; import RuleIcon from "@mui/icons-material/Rule"; @@ -29,11 +28,6 @@ const mainListItems = [ icon: , to: `/dashboard/discovery` }, - { - text: "Enrichment", - icon: , - to: `/dashboard/enrichment` - }, { text: "Generate", icon: , diff --git a/gui/src/client/src/components/workspace/DialogViewDiscoveryQuery.tsx b/gui/src/client/src/components/workspace/DialogViewDiscoveryQuery.tsx index ea15656..54768e1 100644 --- a/gui/src/client/src/components/workspace/DialogViewDiscoveryQuery.tsx +++ b/gui/src/client/src/components/workspace/DialogViewDiscoveryQuery.tsx @@ -16,10 +16,10 @@ import Box from "@mui/material/Box"; import Button from "@mui/material/Button"; import CircularProgress from "@mui/material/CircularProgress"; import Divider from "@mui/material/Divider"; +import MenuItem from "@mui/material/MenuItem"; +import Select from "@mui/material/Select"; import Stack from "@mui/material/Stack"; import Tooltip from "@mui/material/Tooltip"; -import ToggleButton from "@mui/material/ToggleButton"; -import ToggleButtonGroup from "@mui/material/ToggleButtonGroup"; import Typography from "@mui/material/Typography"; import DownloadIcon from "@mui/icons-material/Download"; import { useQuery } from "@tanstack/react-query"; @@ -29,11 +29,28 @@ import { DISCOVERY_SCORE_MODE_OPTIONS, type DiscoveryResult } from "../../featur import { getDiscoveryQueryResult } from "../../features/discovery/api"; import { AlignmentGrid, ResultRow, DownloadSvgButton, sanitizeFilenamePart, type AlignmentGridRow } from "./AlignmentGrid"; import { WorkspaceCompare } from "./WorkspaceCompare"; +import { DiscoveryEnrichmentView } from "./DiscoveryEnrichmentView"; import { downloadJson } from "./downloadJson"; import { useNotifications } from "../NotificationProvider"; import { MAX_ITEMS, importCompound, importClustersBatch } from "../../features/jobs/api"; -type ViewMode = "pairwise" | "msa" | "compare"; +type ViewMode = "pairwise" | "msa" | "compare" | "enrichment"; + +const VIEW_MODE_OPTIONS: { + value: ViewMode; + label: string; + flag: keyof DiscoveryQueryItem["flags"] | null; + // The "Compute for this query" checkbox's own label (see WorkspaceDiscovery.tsx), + // for the disabled-option tooltip -- deliberately not just reusing `label` above, + // since the view's display name ("Multiple sequence alignment") and the checkbox + // that enables it ("Compute MSA") aren't worded the same. + checkboxLabel: string | null; +}[] = [ + { value: "pairwise", label: "Pairwise", flag: null, checkboxLabel: null }, + { value: "msa", label: "Multiple sequence alignment", flag: "computeMsa", checkboxLabel: "Compute MSA" }, + { value: "compare", label: "Compare", flag: "computeCompare", checkboxLabel: "Compute compound comparison" }, + { value: "enrichment", label: "Enrichment", flag: "computeEnrichment", checkboxLabel: "Compute enrichment" }, +]; // Same default the "Import BGCs" dialog offers -- a sent-back BGC result has no UI of // its own to pick a threshold, so it's reparsed with the standard starting point. @@ -240,30 +257,34 @@ export const DialogViewDiscoveryQuery: React.FC = <> - value && setResultsView(value)} + onChange={(e) => setResultsView(e.target.value as ViewMode)} + sx={{ minWidth: 220 }} > - Pairwise - - - - - Multiple sequence alignment - - - - - - - - Compare - - - - + {VIEW_MODE_OPTIONS.map((option) => { + const enabled = option.flag === null || item.flags[option.flag]; + const menuItem = ( + + {option.label} + + ); + if (enabled) return menuItem; + // A disabled MenuItem still needs a wrapping span for the tooltip to + // fire (disabled elements don't receive pointer events on their own). + return ( + + {menuItem} + + ); + })} + {selectedForMsa.size} of {payload.results.length} selected @@ -280,7 +301,7 @@ export const DialogViewDiscoveryQuery: React.FC = - {resultsView !== "compare" && ( + {(resultsView === "pairwise" || resultsView === "msa") && ( Cell shading the alignment reflects each motif's structural similarity to the query's aligned motif in that column. Darker means a closer match, so a sequence's weak spots stand out at a glance. @@ -322,6 +343,10 @@ export const DialogViewDiscoveryQuery: React.FC = ) + ) : resultsView === "enrichment" ? ( + + + ) : selectedResults.length === 0 ? ( Select at least one result above to compare compounds. diff --git a/gui/src/client/src/components/workspace/DiscoveryEnrichmentView.tsx b/gui/src/client/src/components/workspace/DiscoveryEnrichmentView.tsx new file mode 100644 index 0000000..35d0534 --- /dev/null +++ b/gui/src/client/src/components/workspace/DiscoveryEnrichmentView.tsx @@ -0,0 +1,223 @@ +// Enrichment analysis over a Discovery query's own nearest neighbors -- reuses the +// exact same /api/enrichmentAnalysis endpoint (Fisher's exact + Benjamini-Hochberg, +// see gui/src/server/routes/enrichment.py) the old standalone Enrichment tab used, +// just fed from the results dialog's own checkbox selection instead of a separate +// manual entry-search step. +import React from "react"; +import Alert from "@mui/material/Alert"; +import Box from "@mui/material/Box"; +import Button from "@mui/material/Button"; +import Chip from "@mui/material/Chip"; +import CircularProgress from "@mui/material/CircularProgress"; +import Stack from "@mui/material/Stack"; +import Typography from "@mui/material/Typography"; +import CheckCircleIcon from "@mui/icons-material/CheckCircle"; +import CancelIcon from "@mui/icons-material/Cancel"; +import DownloadIcon from "@mui/icons-material/Download"; +import RefreshIcon from "@mui/icons-material/Refresh"; +import { DataGrid, type GridColDef } from "@mui/x-data-grid"; +import { useQuery } from "@tanstack/react-query"; +import { runEnrichmentAnalysis } from "../../features/enrichment/api"; +import { Q_VALUE_SIGNIFICANT, type EnrichmentResult } from "../../features/enrichment/types"; + +function formatScientific(value: number): string { + return value === 0 ? "0" : value.toExponential(2); +} + +function downloadTsv(rows: EnrichmentResult[], filename: string): void { + const headers = [ + "category", "rank", "label", + "selectedWithTerm", "selectedTotal", "backgroundWithTerm", "backgroundTotal", + "foldEnrichment", "direction", "pValue", "qValue", "significant", + ]; + const lines = [headers.join("\t")]; + for (const r of rows) { + lines.push( + [ + r.category ?? "", + r.rank ?? "", + r.label, + r.selectedWithTerm, + r.selectedTotal, + r.backgroundWithTerm, + r.backgroundTotal, + r.foldEnrichment ?? "", + r.direction, + r.pValue, + r.qValue, + r.qValue < Q_VALUE_SIGNIFICANT ? "yes" : "no", + ].join("\t") + ); + } + + const blob = new Blob([lines.join("\n")], { type: "text/tab-separated-values;charset=utf-8" }); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = filename; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + URL.revokeObjectURL(url); +} + +const columns: GridColDef[] = [ + { field: "category", headerName: "Category", width: 140 }, + { + field: "rank", + headerName: "Rank", + width: 110, + valueGetter: (value) => value ?? "—", + }, + { field: "label", headerName: "Label", width: 220 }, + { + field: "selectedWithTerm", + headerName: "Selected", + width: 110, + type: "number", + renderCell: (params) => `${params.row.selectedWithTerm} / ${params.row.selectedTotal}`, + }, + { + field: "backgroundWithTerm", + headerName: "Background", + width: 120, + type: "number", + renderCell: (params) => `${params.row.backgroundWithTerm} / ${params.row.backgroundTotal}`, + }, + { + field: "foldEnrichment", + headerName: "Fold", + width: 100, + type: "number", + valueFormatter: (value: number | null) => (value === null ? "—" : value.toFixed(2)), + }, + { field: "direction", headerName: "Direction", width: 110 }, + { + field: "pValue", + headerName: "p-value", + width: 110, + type: "number", + valueFormatter: (value: number) => formatScientific(value), + }, + { + field: "qValue", + headerName: "q-value", + width: 110, + type: "number", + valueFormatter: (value: number) => formatScientific(value), + }, + { + field: "significant", + headerName: "Significant", + width: 120, + type: "boolean", + valueGetter: (_value, row) => row.qValue < Q_VALUE_SIGNIFICANT, + renderCell: (params) => + params.value ? ( + + ) : ( + + ), + }, +]; + +export const DiscoveryEnrichmentView: React.FC<{ entryIds: string[] }> = ({ entryIds }) => { + // Recomputes only on mount (first open) and on explicit "Recalculate" clicks -- + // never automatically as the user toggles checkboxes above, which would otherwise + // fire a request on every single click while they're still adjusting the selection. + const [queriedIds, setQueriedIds] = React.useState(entryIds); + const didInit = React.useRef(false); + React.useEffect(() => { + if (didInit.current) return; + didInit.current = true; + setQueriedIds(entryIds); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + const selectionChanged = React.useMemo(() => { + const a = [...entryIds].sort(); + const b = [...queriedIds].sort(); + return a.length !== b.length || a.some((id, i) => id !== b[i]); + }, [entryIds, queriedIds]); + + const enrichmentQuery = useQuery({ + queryKey: ["discoveryEnrichmentAnalysis", queriedIds], + queryFn: ({ signal }) => runEnrichmentAnalysis(queriedIds, signal), + enabled: queriedIds.length > 0, + }); + + const rows = enrichmentQuery.data?.results ?? []; + + return ( + + + + + Enrichment of the {queriedIds.length} selected nearest neighbor{queriedIds.length === 1 ? "" : "s"} vs. the + rest of the database, one two-sided Fisher's exact test per annotation term, Benjamini-Hochberg corrected. + + {selectionChanged && ( + + )} + + + + + + + + {queriedIds.length === 0 && ( + + Select at least one result above, then click Recalculate to run enrichment analysis. + + )} + + {enrichmentQuery.error && ( + + {(enrichmentQuery.error as Error).message || "Failed to run enrichment analysis."} + + )} + + {queriedIds.length > 0 && ( + + row.termId} + loading={enrichmentQuery.isLoading} + density="compact" + disableRowSelectionOnClick + initialState={{ + sorting: { sortModel: [{ field: "qValue", sort: "asc" }] }, + }} + pageSizeOptions={[25, 50, 100]} + sx={{ + "& .MuiDataGrid-cell": { fontSize: "0.8125rem" }, + }} + /> + + )} + + ); +}; diff --git a/gui/src/client/src/components/workspace/Workspace.tsx b/gui/src/client/src/components/workspace/Workspace.tsx index 499b292..a8c9709 100644 --- a/gui/src/client/src/components/workspace/Workspace.tsx +++ b/gui/src/client/src/components/workspace/Workspace.tsx @@ -16,7 +16,6 @@ import { WorkspaceUpload } from "./WorkspaceUpload"; import { WorkspaceDiscovery } from "./WorkspaceDiscovery"; import { WorkspaceRules } from "./WorkspaceRules"; import { WorkspaceGenerate } from "./WorkspaceGenerate"; -import { WorkspaceEnrichment } from "./tabs/enrichment/WorkspaceEnrichment"; export const Workspace: React.FC = () => { const { showOverlay, hideOverlay } = useOverlay(); @@ -172,7 +171,6 @@ export const Workspace: React.FC = () => { } /> } /> } /> - } /> } /> } /> diff --git a/gui/src/client/src/components/workspace/WorkspaceDiscovery.tsx b/gui/src/client/src/components/workspace/WorkspaceDiscovery.tsx index baa6349..1288e21 100644 --- a/gui/src/client/src/components/workspace/WorkspaceDiscovery.tsx +++ b/gui/src/client/src/components/workspace/WorkspaceDiscovery.tsx @@ -165,6 +165,7 @@ function DiscoveryQueryListItem({ const flagChips = [ item.flags.computeMsa ? "MSA" : null, item.flags.computeCompare ? "Compare" : null, + item.flags.computeEnrichment ? "Enrichment" : null, ].filter((label): label is string => label !== null); const handleToggle = (e?: React.SyntheticEvent) => { @@ -382,6 +383,7 @@ export const WorkspaceDiscovery: React.FC = ({ session, const [onlyUserUploads, setOnlyUserUploads] = React.useState(false); const [computeMsa, setComputeMsa] = React.useState(true); const [computeCompare, setComputeCompare] = React.useState(false); + const [computeEnrichment, setComputeEnrichment] = React.useState(true); const [submitting, setSubmitting] = React.useState(false); const [queryOptionsOpen, setQueryOptionsOpen] = React.useState(false); @@ -447,7 +449,7 @@ export const WorkspaceDiscovery: React.FC = ({ session, includeUserUploads: includeUserUploads || onlyUserUploads, onlyUserUploads, queryOriginSmiles, - flags: { computeMsa, computeCompare }, + flags: { computeMsa, computeCompare, computeEnrichment }, }); setSession((prev) => (prev ? { ...prev, items: [...prev.items, item] } : prev)); setViewingItemId(item.id); @@ -906,6 +908,22 @@ export const WorkspaceDiscovery: React.FC = ({ session, } label="Compute compound comparison" /> + + setComputeEnrichment(e.target.checked)} + disabled={submitting} + /> + } + label="Compute enrichment" + /> + diff --git a/gui/src/client/src/components/workspace/WorkspaceUpload.tsx b/gui/src/client/src/components/workspace/WorkspaceUpload.tsx index 48d6d03..57833cf 100644 --- a/gui/src/client/src/components/workspace/WorkspaceUpload.tsx +++ b/gui/src/client/src/components/workspace/WorkspaceUpload.tsx @@ -286,7 +286,7 @@ export const WorkspaceUpload: React.FC = ({ session, setSe Getting started - In this tab you can import compounds and biosynthetic gene clusters (BGCs) into your workspace. Use the buttons below to upload your data files. After importing, you can visualize and analyze your data within the  + In this tab you can import compounds and biosynthetic gene clusters (BGCs) into your workspace. Use the buttons below to upload your data files. After importing, you can visualize and analyze your data (including enrichment analysis against your nearest neighbors) within the  = ({ session, setSe > Discovery -  and  - - Enrichment - -  tabs. A maximum of {MAX_ITEMS} items can be imported into the workspace. Keep an eye on for updates on your queries. +  tab. A maximum of {MAX_ITEMS} items can be imported into the workspace. Keep an eye on for updates on your queries. diff --git a/gui/src/client/src/components/workspace/tabs/enrichment/WorkspaceEnrichment.tsx b/gui/src/client/src/components/workspace/tabs/enrichment/WorkspaceEnrichment.tsx deleted file mode 100644 index 1b2dfb3..0000000 --- a/gui/src/client/src/components/workspace/tabs/enrichment/WorkspaceEnrichment.tsx +++ /dev/null @@ -1,269 +0,0 @@ -import React from "react"; -import Alert from "@mui/material/Alert"; -import Box from "@mui/material/Box"; -import Button from "@mui/material/Button"; -import Card from "@mui/material/Card"; -import CardContent from "@mui/material/CardContent"; -import Checkbox from "@mui/material/Checkbox"; -import Chip from "@mui/material/Chip"; -import CircularProgress from "@mui/material/CircularProgress"; -import MenuItem from "@mui/material/MenuItem"; -import Stack from "@mui/material/Stack"; -import Table from "@mui/material/Table"; -import TableBody from "@mui/material/TableBody"; -import TableCell from "@mui/material/TableCell"; -import TableContainer from "@mui/material/TableContainer"; -import TableHead from "@mui/material/TableHead"; -import TableRow from "@mui/material/TableRow"; -import TextField from "@mui/material/TextField"; -import Tooltip from "@mui/material/Tooltip"; -import Typography from "@mui/material/Typography"; -import { useNotifications } from "../../../NotificationProvider"; -import { searchEntries, runEnrichmentAnalysis, MAX_ENTRY_SEARCH_RESULTS, MAX_ENRICHMENT_SELECTION } from "../../../../features/enrichment/api"; -import { groupSourcesByDatabase } from "../../../../features/sources"; -import type { EntryType, EnrichmentResult, SearchEntry } from "../../../../features/enrichment/types"; - -const Q_VALUE_SIGNIFICANT = 0.05; - -export const WorkspaceEnrichment: React.FC = () => { - const { pushNotification } = useNotifications(); - - const [query, setQuery] = React.useState(""); - const [entryType, setEntryType] = React.useState("all"); - const [searchResults, setSearchResults] = React.useState([]); - const [searching, setSearching] = React.useState(false); - const [searchError, setSearchError] = React.useState(null); - - const [selectedIds, setSelectedIds] = React.useState>(new Set()); - - const [running, setRunning] = React.useState(false); - const [results, setResults] = React.useState(null); - - const handleSearch = async (event?: React.FormEvent) => { - event?.preventDefault(); - if (!query.trim()) return; - - setSearching(true); - setSearchError(null); - try { - const resp = await searchEntries(query.trim(), entryType); - setSearchResults(resp.results); - } catch (err) { - setSearchError(err instanceof Error ? err.message : String(err)); - } finally { - setSearching(false); - } - }; - - const toggleSelected = (id: string) => { - setSelectedIds((prev) => { - const next = new Set(prev); - if (next.has(id)) { - next.delete(id); - } else { - if (next.size >= MAX_ENRICHMENT_SELECTION) { - pushNotification(`You can select at most ${MAX_ENRICHMENT_SELECTION} entries.`, "warning"); - return prev; - } - next.add(id); - } - return next; - }); - }; - - const handleSelectAllResults = () => { - setSelectedIds((prev) => { - const next = new Set(prev); - for (const entry of searchResults) { - if (next.size >= MAX_ENRICHMENT_SELECTION) break; - next.add(entry.id); - } - return next; - }); - }; - - const handleClearSelection = () => setSelectedIds(new Set()); - - const handleRunEnrichment = async () => { - if (selectedIds.size === 0) return; - - setRunning(true); - setResults(null); - try { - const resp = await runEnrichmentAnalysis(Array.from(selectedIds)); - setResults(resp.results); - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - pushNotification(`Enrichment analysis failed: ${msg}`, "error"); - } finally { - setRunning(false); - } - }; - - return ( - - - - - Enrichment analysis - - - Search the database, select up to {MAX_ENRICHMENT_SELECTION} entries, then test whether that - selection is enriched (or depleted) for any annotation label compared to the rest of the database. - - - - - setQuery(e.target.value)} - /> - setEntryType(e.target.value as EntryType | "all")} - > - All - Compounds - Gene clusters (BGCs) - - - - - - - - - {searchError && ( - - Search failed: {searchError} - - )} - - - - {searchResults.length > 0 && ( - - - - - Results ({searchResults.length}{searchResults.length >= MAX_ENTRY_SEARCH_RESULTS ? "+" : ""}) - - - - - - - - - {selectedIds.size} / {MAX_ENRICHMENT_SELECTION} selected - - - - - - - - Name - Type - Sources - - - - {searchResults.map((entry) => ( - toggleSelected(entry.id)} sx={{ cursor: "pointer" }}> - - e.stopPropagation()} onChange={() => toggleSelected(entry.id)} /> - - {entry.name} - {entry.type === "bgc" ? "BGC" : "Compound"} - - {groupSourcesByDatabase(entry.sources).map((g) => ( - s.name).join(", ")}> - 1 ? `${g.databaseName} ×${g.count}` : g.databaseName} - size="small" - sx={{ mr: 0.5 }} - /> - - ))} - - - ))} - -
- - - - - - - - )} - - {results && ( - - - - Enrichment results - - - {results.length === 0 ? ( - - No annotated terms were found on the selected entries. - - ) : ( - - - - - Category - Label - Selected - Background - Fold - Direction - p-value - q-value - - - - {results.map((r) => ( - - {r.category ?? "-"} - - {r.label} - {r.qValue < Q_VALUE_SIGNIFICANT && ( - - )} - - {r.selectedWithTerm} / {r.selectedTotal} - {r.backgroundWithTerm} / {r.backgroundTotal} - {r.foldEnrichment != null ? r.foldEnrichment.toFixed(2) : "-"} - {r.direction} - {r.pValue.toExponential(2)} - {r.qValue.toExponential(2)} - - ))} - -
- - )} - - - )} - - ); -}; diff --git a/gui/src/client/src/features/discovery/types.ts b/gui/src/client/src/features/discovery/types.ts index 1cc3baf..d9ad244 100644 --- a/gui/src/client/src/features/discovery/types.ts +++ b/gui/src/client/src/features/discovery/types.ts @@ -46,6 +46,7 @@ export const SubmitDiscoveryQueryReqSchema = z.object({ flags: z.object({ computeMsa: z.boolean().optional(), computeCompare: z.boolean().optional(), + computeEnrichment: z.boolean().optional(), }), }); export type SubmitDiscoveryQueryReq = z.output; diff --git a/gui/src/client/src/features/enrichment/api.ts b/gui/src/client/src/features/enrichment/api.ts index 84765d0..d9d24fc 100644 --- a/gui/src/client/src/features/enrichment/api.ts +++ b/gui/src/client/src/features/enrichment/api.ts @@ -1,24 +1,8 @@ -import { getJson, postJson } from "../http"; -import { - EntrySearchRespSchema, - EnrichmentAnalysisRespSchema, - type EntrySearchResp, - type EnrichmentAnalysisResp, - type EntryType, -} from "./types"; +import { postJson } from "../http"; +import { EnrichmentAnalysisRespSchema, type EnrichmentAnalysisResp } from "./types"; -export const MAX_ENTRY_SEARCH_RESULTS = 100; export const MAX_ENRICHMENT_SELECTION = 100; -export async function searchEntries( - query: string, - entryType: EntryType | "all" = "all", - signal?: AbortSignal -): Promise { - const params = new URLSearchParams({ q: query, type: entryType, limit: String(MAX_ENTRY_SEARCH_RESULTS) }); - return getJson(`/api/entrySearch?${params.toString()}`, EntrySearchRespSchema, signal); -} - export async function runEnrichmentAnalysis( entryIds: string[], signal?: AbortSignal diff --git a/gui/src/client/src/features/enrichment/types.ts b/gui/src/client/src/features/enrichment/types.ts index 64ae689..c58db0d 100644 --- a/gui/src/client/src/features/enrichment/types.ts +++ b/gui/src/client/src/features/enrichment/types.ts @@ -1,28 +1,5 @@ import { z } from "zod"; -export const EntryTypeSchema = z.enum(["compound", "bgc"]); -export type EntryType = z.output; - -export const EntrySourceSchema = z.object({ - name: z.string(), - databaseName: z.string(), - url: z.string().nullable(), -}); - -export const SearchEntrySchema = z.object({ - id: z.string(), - name: z.string(), - url: z.string().nullable(), - type: EntryTypeSchema, - sources: z.array(EntrySourceSchema), -}); -export type SearchEntry = z.output; - -export const EntrySearchRespSchema = z.object({ - results: z.array(SearchEntrySchema), -}); -export type EntrySearchResp = z.output; - export const EnrichmentResultSchema = z.object({ termId: z.string(), category: z.string().nullable(), @@ -43,3 +20,6 @@ export const EnrichmentAnalysisRespSchema = z.object({ results: z.array(EnrichmentResultSchema), }); export type EnrichmentAnalysisResp = z.output; + +// q-value threshold below which a term is flagged significant in the results table. +export const Q_VALUE_SIGNIFICANT = 0.05; diff --git a/gui/src/client/src/features/session/types.ts b/gui/src/client/src/features/session/types.ts index 01e8144..49752ca 100644 --- a/gui/src/client/src/features/session/types.ts +++ b/gui/src/client/src/features/session/types.ts @@ -40,6 +40,10 @@ export const ClusterItemSchema = BaseItemSchema.extend({ export const DiscoveryQueryFlagsSchema = z.object({ computeMsa: z.boolean().default(false), computeCompare: z.boolean().default(false), + // Not precomputed (see run_discovery_query_job) -- gates whether the results + // dialog's Enrichment view is enabled; the analysis itself is fetched live from + // /api/enrichmentAnalysis based on whichever nearest neighbors are checked. + computeEnrichment: z.boolean().default(true), }); export type DiscoveryQueryFlags = z.output; diff --git a/gui/src/server/app.py b/gui/src/server/app.py index 7b15b4e..bcaa7ed 100644 --- a/gui/src/server/app.py +++ b/gui/src/server/app.py @@ -40,7 +40,7 @@ ) from routes.rate_limit import limiter, RATE_LIMIT_REJECTIONS from routes.rules import blp_rule_set, blp_generate_backbone -from routes.enrichment import blp_entry_search, blp_enrichment_analysis +from routes.enrichment import blp_enrichment_analysis from routes.entry_annotations import blp_entry_annotations @@ -257,7 +257,6 @@ def ready() -> tuple[dict[str, str], int]: app.register_blueprint(blp_get_discovery_query_result) app.register_blueprint(blp_rule_set) app.register_blueprint(blp_generate_backbone) -app.register_blueprint(blp_entry_search) app.register_blueprint(blp_enrichment_analysis) app.register_blueprint(blp_entry_annotations) diff --git a/gui/src/server/routes/discovery.py b/gui/src/server/routes/discovery.py index 8a43ef9..dab602d 100644 --- a/gui/src/server/routes/discovery.py +++ b/gui/src/server/routes/discovery.py @@ -1225,6 +1225,11 @@ def submit_discovery_query() -> tuple[Response, int]: flags_normalized = { "computeMsa": bool(flags.get("computeMsa", False)), "computeCompare": bool(flags.get("computeCompare", False)), + # Not precomputed here (unlike MSA/Compare) -- enrichment is recalculated + # on-demand from the results dialog via /api/enrichmentAnalysis, since it + # depends on which nearest neighbors the user has checked, not the full + # result set. This flag only gates whether that dropdown option is enabled. + "computeEnrichment": bool(flags.get("computeEnrichment", True)), } full_sess = load_session_with_items(session_id) diff --git a/gui/src/server/routes/enrichment.py b/gui/src/server/routes/enrichment.py index d0e6bbd..2ead704 100644 --- a/gui/src/server/routes/enrichment.py +++ b/gui/src/server/routes/enrichment.py @@ -1,67 +1,22 @@ -"""Enrichment tab: search up to MAX_ENTRY_SEARCH_RESULTS db entries, select some of -them, and test whether the selection is enriched for any annotation term (phylogeny, -chemical class, ...) compared to its background -- entries of the same type(s) not -selected. One Fisher's exact test per term observed on the selection, Benjamini-Hochberg -corrected across all of them. +"""Enrichment analysis: given a set of db entry ids (a Discovery query's checked +nearest neighbors), test whether that selection is enriched for any annotation term +(phylogeny, chemical class, ...) compared to its background -- entries of the same +type(s) not selected. One Fisher's exact test per term observed on the selection, +Benjamini-Hochberg corrected across all of them. """ from flask import Blueprint, Response, jsonify, request from scipy.stats import fisher_exact -from retromol_database.duckdb import ENTRY_TYPES from routes.database import open_retromol_db -blp_entry_search = Blueprint("entry_search", __name__) blp_enrichment_analysis = Blueprint("enrichment_analysis", __name__) # Mirrors the frontend's client-side guardrail; must also be enforced here since the # frontend check is UX-only (see session_store.py's MAX_SESSION_ITEMS for the same pattern). -MAX_ENTRY_SEARCH_RESULTS = 100 MAX_ENRICHMENT_SELECTION = 100 -def _entry_payload(entry) -> dict: - return { - "id": entry.id, - "name": entry.name, - "url": entry.url, - "type": entry.type, - "sources": [ - {"name": s.name, "databaseName": s.database_name, "url": s.url} for s in entry.sources - ], - } - - -@blp_entry_search.get("/api/entrySearch") -def entry_search() -> tuple[Response, int]: - """ - Search entries by name/id, for the Enrichment tab's "query and select" step. - - :return: a tuple containing the search results and an HTTP status code - """ - query = (request.args.get("q") or "").strip() - if not query: - return jsonify({"error": "q is required"}), 400 - - entry_type = request.args.get("type") - if entry_type == "all": - entry_type = None - if entry_type is not None and entry_type not in ENTRY_TYPES: - return jsonify({"error": f"type must be one of {ENTRY_TYPES} or 'all'"}), 400 - - limit = request.args.get("limit", default=MAX_ENTRY_SEARCH_RESULTS, type=int) - if limit is None or not (1 <= limit <= MAX_ENTRY_SEARCH_RESULTS): - return jsonify({"error": f"limit must be an integer between 1 and {MAX_ENTRY_SEARCH_RESULTS}"}), 400 - - try: - with open_retromol_db() as db: - results = db.search_entries(query, entry_type=entry_type, limit=limit) - except Exception as e: - return jsonify({"error": str(e)}), 503 - - return jsonify({"results": [_entry_payload(e) for e in results]}), 200 - - def _benjamini_hochberg(p_values: list[float]) -> list[float]: """Benjamini-Hochberg FDR correction. Returns q-values in the same order as `p_values`.""" m = len(p_values) From 580f62fb15c00c34f6da16d7ffb960e2c1dd590d Mon Sep 17 00:00:00 2001 From: David Meijer Date: Thu, 27 Aug 2026 00:05:16 -0400 Subject: [PATCH 10/11] STY: updated styling result tab for enrichment matches rest of app --- .../workspace/DiscoveryEnrichmentView.tsx | 67 +++++++++++++++++-- 1 file changed, 62 insertions(+), 5 deletions(-) diff --git a/gui/src/client/src/components/workspace/DiscoveryEnrichmentView.tsx b/gui/src/client/src/components/workspace/DiscoveryEnrichmentView.tsx index 35d0534..445cc94 100644 --- a/gui/src/client/src/components/workspace/DiscoveryEnrichmentView.tsx +++ b/gui/src/client/src/components/workspace/DiscoveryEnrichmentView.tsx @@ -10,6 +10,7 @@ import Button from "@mui/material/Button"; import Chip from "@mui/material/Chip"; import CircularProgress from "@mui/material/CircularProgress"; import Stack from "@mui/material/Stack"; +import MuiTooltip from "@mui/material/Tooltip"; import Typography from "@mui/material/Typography"; import CheckCircleIcon from "@mui/icons-material/CheckCircle"; import CancelIcon from "@mui/icons-material/Cancel"; @@ -24,6 +25,16 @@ function formatScientific(value: number): string { return value === 0 ? "0" : value.toExponential(2); } +// annotation_terms.category/rank/label come straight from the pipeline in raw form +// (e.g. "chemical_class", "chebi_biological_role") -- humanized here for display only, +// same "touch only presentation, never the underlying value" approach as +// features/database/format.ts's toSentenceCase. +function humanizeAnnotationText(value: string): string { + const spaced = value.replace(/_/g, " "); + const sentenceCased = spaced.length > 0 ? spaced.charAt(0).toUpperCase() + spaced.slice(1) : spaced; + return sentenceCased.replace(/\bchebi\b/gi, "ChEBI"); +} + function downloadTsv(rows: EnrichmentResult[], filename: string): void { const headers = [ "category", "rank", "label", @@ -62,14 +73,24 @@ function downloadTsv(rows: EnrichmentResult[], filename: string): void { } const columns: GridColDef[] = [ - { field: "category", headerName: "Category", width: 140 }, + { + field: "category", + headerName: "Category", + width: 140, + valueFormatter: (value: string | null) => (value ? humanizeAnnotationText(value) : "—"), + }, { field: "rank", headerName: "Rank", - width: 110, - valueGetter: (value) => value ?? "—", + width: 140, + valueFormatter: (value: string | null) => (value ? humanizeAnnotationText(value) : "—"), + }, + { + field: "label", + headerName: "Label", + width: 220, + valueFormatter: (value: string) => humanizeAnnotationText(value), }, - { field: "label", headerName: "Label", width: 220 }, { field: "selectedWithTerm", headerName: "Selected", @@ -121,6 +142,14 @@ const columns: GridColDef[] = [ }, ]; +// DataGrid's own "baseTooltip" slot type has no `arrow` prop (it's a stripped-down +// interface, not @mui/material's full TooltipProps -- see gridBaseSlots.d.ts), so +// arrow-style tooltips (sort/filter/menu hover hints) need a full slot override +// rather than a slotProps tweak. +function GridArrowTooltip(props: React.ComponentProps) { + return ; +} + export const DiscoveryEnrichmentView: React.FC<{ entryIds: string[] }> = ({ entryIds }) => { // Recomputes only on mount (first open) and on explicit "Recalculate" clicks -- // never automatically as the user toggles checkboxes above, which would otherwise @@ -208,12 +237,40 @@ export const DiscoveryEnrichmentView: React.FC<{ entryIds: string[] }> = ({ entr loading={enrichmentQuery.isLoading} density="compact" disableRowSelectionOnClick + showCellVerticalBorder + showColumnVerticalBorder initialState={{ sorting: { sortModel: [{ field: "qValue", sort: "asc" }] }, }} pageSizeOptions={[25, 50, 100]} + slots={{ baseTooltip: GridArrowTooltip }} sx={{ - "& .MuiDataGrid-cell": { fontSize: "0.8125rem" }, + // Explicit border rules rather than relying on the --DataGrid-rowBorderColor + // CSS variable / showCellVerticalBorder alone -- confirmed live that the + // variable covers vertical cell borders but not the horizontal row + // separator in this DataGrid version, so it's set directly here instead. + // "divider" resolves through the app's own theme (light/dark both), not a + // hardcoded color. + "& .MuiDataGrid-cell": { + fontSize: "0.8125rem", + borderRight: "1px solid", + borderRightColor: "divider", + borderBottom: "1px solid", + borderBottomColor: "divider", + }, + "& .MuiDataGrid-columnHeader": { + borderRight: "1px solid", + borderRightColor: "divider", + }, + // Sort arrow / column-menu (three dots) icons -- bare icon, no button + // chrome (padding, hover box, border) around them. + "& .MuiDataGrid-iconButtonContainer .MuiIconButton-root, & .MuiDataGrid-menuIconButton": { + padding: 0, + border: "none", + backgroundColor: "transparent", + boxShadow: "none", + "&:hover": { backgroundColor: "transparent" }, + }, }} /> From 191ecb30a88ec79c1e3f311319125c4fb560c17b Mon Sep 17 00:00:00 2001 From: David Meijer Date: Thu, 27 Aug 2026 00:07:41 -0400 Subject: [PATCH 11/11] FIX: generate up-to-date package-lock.json --- gui/src/client/package-lock.json | 219 ++++++++++++++++--------------- 1 file changed, 112 insertions(+), 107 deletions(-) diff --git a/gui/src/client/package-lock.json b/gui/src/client/package-lock.json index d777eac..8e7e256 100644 --- a/gui/src/client/package-lock.json +++ b/gui/src/client/package-lock.json @@ -2741,7 +2741,6 @@ "resolved": "https://registry.npmjs.org/@emotion/is-prop-valid/-/is-prop-valid-1.4.0.tgz", "integrity": "sha512-QgD4fyscGcbbKwJmqNvUMSE02OsHUa+lAWKdEUIJKgqe5IwRSKd7+KhibEWdaKwgjLj0DRSHA9biAIqGBk05lw==", "license": "MIT", - "peer": true, "dependencies": { "@emotion/memoize": "^0.9.0" } @@ -2908,9 +2907,9 @@ "license": "Python-2.0" }, "node_modules/@eslint/eslintrc/node_modules/js-yaml": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.1.tgz", - "integrity": "sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", "dev": true, "funding": [ { @@ -3625,16 +3624,16 @@ } }, "node_modules/@mui/x-charts": { - "version": "8.29.2", - "resolved": "https://registry.npmjs.org/@mui/x-charts/-/x-charts-8.29.2.tgz", - "integrity": "sha512-s0f0gstCEBO4HV1aj4wNapL6RYREO+YyFLs4cSKm8bu347AuADngBtepNSCXuzsCQ+C+PxAlBoCy2a1ssTxMuA==", + "version": "8.29.3", + "resolved": "https://registry.npmjs.org/@mui/x-charts/-/x-charts-8.29.3.tgz", + "integrity": "sha512-8hF+rnf/rj258FmBMx6XwNmdFXM+gCnR9TKpd8ywJUwVPzZoZ0U/LMtsFHe16MZ9YOpOhTSdKHycq/OmiYXFlQ==", "license": "MIT", "dependencies": { "@babel/runtime": "^7.28.4", "@mui/utils": "^7.3.5", "@mui/x-charts-vendor": "8.29.0", "@mui/x-internal-gestures": "0.5.0", - "@mui/x-internals": "8.29.2", + "@mui/x-internals": "8.29.3", "bezier-easing": "^2.1.0", "clsx": "^2.1.1", "prop-types": "^15.8.1", @@ -3812,9 +3811,9 @@ } }, "node_modules/@mui/x-internals": { - "version": "8.29.2", - "resolved": "https://registry.npmjs.org/@mui/x-internals/-/x-internals-8.29.2.tgz", - "integrity": "sha512-TLILyHia5NHh3MGErFDh0bXZ4V6iS45hdStofsV5wklF6dpgZjduo65oJB0h81mI5mexNlhHANEVfw0KV6BxBg==", + "version": "8.29.3", + "resolved": "https://registry.npmjs.org/@mui/x-internals/-/x-internals-8.29.3.tgz", + "integrity": "sha512-xJSaAXQDZ+35svja+i5N0qg7gTbZwcnAGZBgC+37Z3SoxRlTR5NwNBUlqKDm5TJwLvG17DeDYolxqWtcR93tsg==", "license": "MIT", "dependencies": { "@babel/runtime": "^7.28.4", @@ -4515,9 +4514,9 @@ } }, "node_modules/@tanstack/query-core": { - "version": "5.101.4", - "resolved": "https://registry.npmjs.org/@tanstack/query-core/-/query-core-5.101.4.tgz", - "integrity": "sha512-gNwcvOJcRbLWPOLG/2OBm+zM+Yv+MKsXKEOWC57USuZDEsI71hEErQsiEGx5wX9rzWWkfwM0fVSPoiIFSsxfiw==", + "version": "5.102.6", + "resolved": "https://registry.npmjs.org/@tanstack/query-core/-/query-core-5.102.6.tgz", + "integrity": "sha512-jp+GucyQ+fel2ILb8ZLQeqMaAqZL/Bzaj+dKOQiPk4vPdf2rQPEV7heUyEVIhatpY42j4AFHav4DzBa1oTIp/A==", "license": "MIT", "funding": { "type": "github", @@ -4525,12 +4524,12 @@ } }, "node_modules/@tanstack/react-query": { - "version": "5.101.4", - "resolved": "https://registry.npmjs.org/@tanstack/react-query/-/react-query-5.101.4.tgz", - "integrity": "sha512-yRg2pfOCxIs4ZJW3XYYHU/WgtD04FHSnfHlpRT7h7pR77hwkdRG4wxbKe4aq6P0RvXUTBSQpQeadS1SUYUe+KA==", + "version": "5.102.6", + "resolved": "https://registry.npmjs.org/@tanstack/react-query/-/react-query-5.102.6.tgz", + "integrity": "sha512-ANz4KZ8z80D85fiz5IAHjpb8XBPLrjOb/0natlD9Ascyy/3p96V86Zw8UbOfA0HDLcvK/A4gNd95r723y5UT2w==", "license": "MIT", "dependencies": { - "@tanstack/query-core": "5.101.4" + "@tanstack/query-core": "5.102.6" }, "funding": { "type": "github", @@ -4617,9 +4616,9 @@ } }, "node_modules/@testing-library/user-event": { - "version": "14.6.3", - "resolved": "https://registry.npmjs.org/@testing-library/user-event/-/user-event-14.6.3.tgz", - "integrity": "sha512-6dBq67jT8lE+JTE8Exm02Kt6ze43hz1jdiSpSJwtTZiT1xQQ6b7nZYTTQ9njdArdU8XklOwaDp/AbT/eYSKF4g==", + "version": "14.6.6", + "resolved": "https://registry.npmjs.org/@testing-library/user-event/-/user-event-14.6.6.tgz", + "integrity": "sha512-Jbs9FpkkIDw8FgSc6kOVsOv8JuuqGAL7J4X1oot77JxAoDlkNn2GRkd0aYRVuQ+pVQAiHWVkE4rX/dkF5fBiCw==", "dev": true, "license": "MIT", "engines": { @@ -4641,9 +4640,9 @@ } }, "node_modules/@tsconfig/node10": { - "version": "1.0.12", - "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.12.tgz", - "integrity": "sha512-UCYBaeFvM11aU2y3YPZ//O5Rhj+xKyzy7mvcIoAjASbigy8mHMryP5cK7dgjlz2hWxh1g5pLw084E0a/wlUSFQ==", + "version": "1.0.13", + "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.13.tgz", + "integrity": "sha512-gcLdvR9HO1ZJBypsOGqaP6TFEzb6vIta0KSTLt9NAQ6pXQO3cRgSVyCN6pzYqI9DlJgY71XKO0dpDhCf08b3pg==", "dev": true, "license": "MIT" }, @@ -4805,9 +4804,9 @@ } }, "node_modules/@types/d3-shape": { - "version": "3.1.8", - "resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.1.8.tgz", - "integrity": "sha512-lae0iWfcDeR7qt7rA88BNiqdvPS5pFVPpo5OfjElwNaT2yyekbM0C9vK+yqBqEmHr6lDkRnYNoTBYlAgJa7a4w==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.2.0.tgz", + "integrity": "sha512-kVd74ta9eof3eJOvbNd1vGKS/XERRyQbT26Og63hIsvDO84cjD5gEOhsXf26w3FSoNlPVz84DOFcKv/oou+fMw==", "license": "MIT", "dependencies": { "@types/d3-path": "*" @@ -4981,9 +4980,9 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "26.2.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-26.2.0.tgz", - "integrity": "sha512-5IviulTZeRNp2vAJ514cc/HUlY5nZ9fCbq9DMyC52BrhFZACo3nI0R7qBxhQmo/d27NFe96ur/b7Wwxklda+kg==", + "version": "26.4.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.4.0.tgz", + "integrity": "sha512-faiGnoIrLH/V8cibOMEAZ8pMw6oXqSukl29ra4mN8GdaB2ZewzeaLj+INpV5N+Z1eKWzY+IzaIZH2EIR6YZRNQ==", "dev": true, "license": "MIT", "peer": true, @@ -5432,9 +5431,9 @@ } }, "node_modules/@ungap/structured-clone": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.3.tgz", - "integrity": "sha512-60YRaenCQcVjYEKOcG824+DRGGIQ3VKErcBoAEDJZz5bKIs2ZG+X/H9Nk+Q6EVkwJk5QNApxbrc5QtBSwtrXAg==", + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.4.tgz", + "integrity": "sha512-JL+CF0GeLHyPWI0rXu7UnxgiuOm9UQWzadi0OYOJNhNO2q6EZElpwlgXkNkfU1PzANDHq3YcwKVZprdvS+BrbQ==", "dev": true, "license": "ISC" }, @@ -6534,9 +6533,9 @@ "license": "MIT" }, "node_modules/baseline-browser-mapping": { - "version": "2.11.13", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.13.tgz", - "integrity": "sha512-k9HNuUVMlqVjQ9UHzfPjIqiDbWw7WqT1AoT7GL8VwvF3r0ZfArtgiSPAlmupyNquNgOJHTuH4CKYf8ttMTWBTQ==", + "version": "2.11.19", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.19.tgz", + "integrity": "sha512-Grytf1xOxOEMTGRwx6rLGKkTabd4vMg3VrKdj/7joCmV0qgh4QwMMO6xh34YEXQqirAuUdgQGa5orJQQ+69RBw==", "dev": true, "license": "Apache-2.0", "bin": { @@ -6892,9 +6891,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001809", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001809.tgz", - "integrity": "sha512-xxWVywk6a6Arlk+hymeycyn/VgqEfLDxupvhH/xiY5SJ/18kmi9o6MiO320DCUzypORHLtvh0I4i04tUhCNHNQ==", + "version": "1.0.30001810", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001810.tgz", + "integrity": "sha512-TITQPUkaz+aVk5GL6NhOdwk1aEaNTSDPsGFWrTuhKGtjTF70jL/Oht2W4c6rXUe5fu7Ie19VIahAXHIIiWWNeg==", "dev": true, "funding": [ { @@ -7240,9 +7239,9 @@ "license": "MIT" }, "node_modules/colord": { - "version": "2.9.3", - "resolved": "https://registry.npmjs.org/colord/-/colord-2.9.3.tgz", - "integrity": "sha512-jeC1axXpnb0/2nn/Y1LPuLdgXBLH7aDcHu4KEKfqw3CUhX7ZpfBSlPKyqXE6btIgEzfWtrX3/tyBCaCvXvMkOw==", + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/colord/-/colord-2.10.0.tgz", + "integrity": "sha512-AidJptpBJmjTclAp9BkLwJi0T93fo5epJnbaZslpg6QVzpHjAiveF55mE9AcUJiGMqRHgMDY8soMsQtuNYMHfw==", "dev": true, "license": "MIT" }, @@ -8132,9 +8131,9 @@ } }, "node_modules/dayjs": { - "version": "1.11.21", - "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.21.tgz", - "integrity": "sha512-98IT+HOahAisibz/yjKbzuOBwYcjJ7BCLPzARyHiyEBmRz4fatF+KPJszEHXsGYjUG234aH/cOjW1wwTbKUZlA==", + "version": "1.11.23", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.23.tgz", + "integrity": "sha512-QDTCU0M0MxR3hQfnlDJfwekQiaanm1ubOD231u73WBckQ/fsamwRLiE2GBz6D3a/xF1NgfiDLJjXBa1hYOYTtQ==", "license": "MIT" }, "node_modules/debug": { @@ -8506,9 +8505,9 @@ } }, "node_modules/dompurify": { - "version": "3.4.13", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.13.tgz", - "integrity": "sha512-2vmYIoqjze2d+kakP8S/nS5shfsl587kzwEjcGlTdiksUVgFHnFCsLYDVj/JNqJVOQZGSYBTmuycv0PodwmnMQ==", + "version": "3.4.14", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.14.tgz", + "integrity": "sha512-dVoH9z+MY+C9IilgGCk3YfFqjLi3fChm2OiKJMzh6axrJ5qwxqWaZamgmHrpv22CN/KdbZJuGEGgfQoL00LTdg==", "license": "(MPL-2.0 OR Apache-2.0)", "optionalDependencies": { "@types/trusted-types": "^2.0.7" @@ -8603,9 +8602,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.5.403", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.403.tgz", - "integrity": "sha512-MQsYmdaLzvaCX5j+ZZBr5Fm6uCCnPQcRtlvmvRlWqrXy+BH2O4ffXIAScF+JQznQWB9brWp4lSD9Z4yNmaf2BA==", + "version": "1.5.415", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.415.tgz", + "integrity": "sha512-958V+Kbhtgz+SxXeEVKBjrlKRBIDAYvUJfwhjxMZ5S6ut9jAl7l9ZKBkBrvjyjZE36PabLUo2L8kEeV5O4vgJg==", "dev": true, "license": "ISC" }, @@ -8835,9 +8834,9 @@ } }, "node_modules/es-module-lexer": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.1.tgz", - "integrity": "sha512-shc1dbU90Yl/xq1QrC7QRtfcwURZuVRfPhZbDoldJ1cn1gzDvBaBWlv0eFolj5+0znnPJz5TXLxsN77X/12KTA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.2.tgz", + "integrity": "sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw==", "dev": true, "license": "MIT" }, @@ -9499,9 +9498,9 @@ } }, "node_modules/eslint/node_modules/js-yaml": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.1.tgz", - "integrity": "sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", "dev": true, "funding": [ { @@ -9856,9 +9855,9 @@ "license": "MIT" }, "node_modules/fast-uri": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.5.tgz", - "integrity": "sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw==", + "version": "3.1.6", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.6.tgz", + "integrity": "sha512-7Ical1vFEMr0onbVzEDIreM22I4khW+fzyQPwvAFWBp1iwdshSZRsL4jjRvPG9JP1uiqMHRto+YU6R2/CzDz5Q==", "dev": true, "funding": [ { @@ -12845,9 +12844,9 @@ } }, "node_modules/jest-watch-typeahead/node_modules/strip-ansi/node_modules/ansi-regex": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", - "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.3.0.tgz", + "integrity": "sha512-WpDfL7NO6j7tH88IDBNVdUJxDh9nmCteAVW9dsep846XdwF4naCBK+/tGLX3KJgcpgMRXCFlTM2hKGoK9FsdrQ==", "dev": true, "license": "MIT", "engines": { @@ -12925,9 +12924,9 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "3.15.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.15.1.tgz", - "integrity": "sha512-S99WuO3HlhO3XN41EtYUNl9zzXjoJx7QvmipxsJVxtCBT0YHEFy+iOJhjSvrmV12nYhWpZaM8lPHkJm0yUMbag==", + "version": "3.15.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.15.2.tgz", + "integrity": "sha512-6EuL879VkRA+1Cz578mKMiKvjPNEuk6+r1JaFzoSWejZmtf7xWbIyw1e3KkxlkzTIt9Taw6JBhEppG7utc1P+w==", "dev": true, "license": "MIT", "dependencies": { @@ -13569,16 +13568,16 @@ } }, "node_modules/minimizer-webpack-plugin": { - "version": "5.6.1", - "resolved": "https://registry.npmjs.org/minimizer-webpack-plugin/-/minimizer-webpack-plugin-5.6.1.tgz", - "integrity": "sha512-DoeAZz8Q1C1znwsUzej1fdoi4jCf7/+Em27ouLqfK/+3m8G+D7yDhUwrc3CNhjSzGUN1kn7Iv4sWmjflQHenpw==", + "version": "5.7.0", + "resolved": "https://registry.npmjs.org/minimizer-webpack-plugin/-/minimizer-webpack-plugin-5.7.0.tgz", + "integrity": "sha512-0ReIvHAVVojdDOn+kmRzrT62A6Btc9KeAK6ANTpd8+S5mnm5RDdu+EN6924xLFFzb36Rkoj+xOXkDA0jTHId/g==", "dev": true, "license": "MIT", "dependencies": { - "@jridgewell/trace-mapping": "^0.3.25", + "@jridgewell/trace-mapping": "^0.3.31", "jest-worker": "^27.4.5", - "schema-utils": "^4.3.0", - "terser": "^5.31.1" + "schema-utils": "^4.3.3", + "terser": "^5.51.0" }, "engines": { "node": ">= 10.13.0" @@ -14327,16 +14326,6 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, - "node_modules/pify": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", - "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/pirates": { "version": "4.0.7", "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.7.tgz", @@ -15734,9 +15723,9 @@ } }, "node_modules/postcss-svgo/node_modules/svgo": { - "version": "2.8.3", - "resolved": "https://registry.npmjs.org/svgo/-/svgo-2.8.3.tgz", - "integrity": "sha512-5EZD0pafXX6PphdwOGCiVLDSaV1xyuQao2blHajHLsPxr07q4mmEjdtXEWgG07ae2mIz8Ex2CDXNCTiXhy3Khw==", + "version": "2.8.4", + "resolved": "https://registry.npmjs.org/svgo/-/svgo-2.8.4.tgz", + "integrity": "sha512-2GJ4h3rl13qYTdwllaK6QlL8tG+UrM8626V2Ylcd/yBUv2Y/EwLsYisVV6UDCRmlk+C74G8nMpxJCk0RWPdDCw==", "dev": true, "license": "MIT", "dependencies": { @@ -16457,14 +16446,11 @@ } }, "node_modules/read-cache": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz", - "integrity": "sha512-Owdv/Ft7IjOgm/i0xvNDZ1LrRANRfew4b2prF3OWMQLxLfu3bS8FVhCsrSCMK4lR56Y9ya+AThoTpDCTxCmpRA==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.2.tgz", + "integrity": "sha512-/peqiBB/n07gQGLsWaHho3WfvUyRscw0gYTsEFMhrIe/nWLkYaf5SbKYjGYqtRV3aPwykJgF2VEMo1ac4bnsGA==", "dev": true, - "license": "MIT", - "dependencies": { - "pify": "^2.3.0" - } + "license": "MIT" }, "node_modules/readable-stream": { "version": "3.6.2", @@ -16689,9 +16675,9 @@ "license": "MIT" }, "node_modules/reselect": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/reselect/-/reselect-5.2.0.tgz", - "integrity": "sha512-AgZ3UOZm3YndfrJ4OYjgrT7bmCm/1iqkjvEfH/oYjzh6PD2qw4QuT3jjnXIrpdt4MTpMXclMT3lXbmRY+XRakw==", + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/reselect/-/reselect-5.3.0.tgz", + "integrity": "sha512-XGoLeRAVzUTcJ1qkxPQhDJyIZ5d6zzZD9nT7AEZOaaU9UbWclhycElmhO+VD5bFeLuzhPBaOV2oXC8uG35ZSpg==", "license": "MIT" }, "node_modules/resolve": { @@ -18121,11 +18107,14 @@ } }, "node_modules/svg-parser": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/svg-parser/-/svg-parser-2.0.4.tgz", - "integrity": "sha512-e4hG1hRwoOdRb37cIMSgzNsxyzKfayW6VOflrwvR+/bzrkyxY/31WkbgnQpgtrNp1SdpJvpUAGTa/ZoiPNDuRQ==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/svg-parser/-/svg-parser-2.1.0.tgz", + "integrity": "sha512-bwLf38YmY+TDYHJw1Ex0Co8c4yeXuJAo8YnXGZrscxrvYoVVIvLeniEkV1Ks/54VteMnL4FtyN1+P+TZJdUPmQ==", "dev": true, - "license": "MIT" + "license": "MIT", + "engines": { + "node": ">=8" + } }, "node_modules/svgo": { "version": "1.3.2", @@ -18400,6 +18389,24 @@ } } }, + "node_modules/tailwindcss/node_modules/yaml": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", + "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==", + "dev": true, + "license": "ISC", + "optional": true, + "peer": true, + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + }, "node_modules/tapable": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.3.3.tgz", @@ -18474,9 +18481,9 @@ } }, "node_modules/terser": { - "version": "5.49.2", - "resolved": "https://registry.npmjs.org/terser/-/terser-5.49.2.tgz", - "integrity": "sha512-rGbJiKeQ4WDe3EXlDAIaQcwftVfv2Q8o1awFNfvXolJYKkb1AuZY1RTOmqx4LJXZENbWZA7eIsYGHuEzHsi1nQ==", + "version": "5.51.0", + "resolved": "https://registry.npmjs.org/terser/-/terser-5.51.0.tgz", + "integrity": "sha512-myiQ6aFnxDOjdiXdTlC8ngVccQD88uHXTx5RUQOSEnarDYgJMjDagwAQszcOusjvmf4YqWsxoD1+MY+oAJRmpw==", "dev": true, "license": "BSD-2-Clause", "dependencies": { @@ -18655,9 +18662,9 @@ } }, "node_modules/tinyglobby/node_modules/picomatch": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", - "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", "dev": true, "license": "MIT", "peer": true, @@ -18909,7 +18916,6 @@ "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", "dev": true, "license": "(MIT OR CC0-1.0)", - "peer": true, "engines": { "node": ">=10" }, @@ -19457,7 +19463,6 @@ "integrity": "sha512-0XavAZbNJ5sDrCbkpWL8mia0o5WPOd2YGtxrEiZkBK9FjLppIUK2TgxK6qGD2P3hUXTJNNPVibrerKcx5WkR1g==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/bonjour": "^3.5.9", "@types/connect-history-api-fallback": "^1.3.5",