diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..cb09af8 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,38 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + +# The job only reads the repo and needs no secrets. +permissions: + contents: read + +jobs: + lint-and-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + # Matches the interpreter the project's venv was built with (3.11.5). + python-version: "3.11" + cache: pip + + - name: Install dependencies + run: pip install -r requirements-dev.txt + + - name: Lint with ruff + run: ruff check . + + # Only the offline test suite runs here. It imports the real project and + # eval modules but exercises pure logic (schemas, source helpers, grading, + # and the Critic evidence wiring), so it needs no OpenAI, Pinecone, or + # Tavily key. The full eval harness in evals/eval.py and the Critic's + # groundedness judgment need live keys and are intentionally not run here. + - name: Run offline tests + run: pytest tests/ -q diff --git a/README.md b/README.md index 2806d85..6d734d3 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # SentryQuery AI +[![CI](https://github.com/henry-hai/SentryQuery-AI/actions/workflows/ci.yml/badge.svg)](https://github.com/henry-hai/SentryQuery-AI/actions/workflows/ci.yml) + An agentic RAG assistant that answers questions over indexed enterprise documents with a two-agent LangGraph pipeline over Pinecone, and a Streamlit web UI. A **Researcher** retrieves and drafts, and a **Critic** verifies every claim @@ -37,6 +39,32 @@ wasted tool call. ## Architecture +```mermaid +flowchart TD + Q([User query]) --> R + + subgraph GRAPH ["LangGraph StateGraph"] + R["Researcher
GPT-4o, temp=0"] + C["Critic
gpt-4o-mini, temp=0"] + V{"CriticVerdict"} + R -->|"drafts AnswerSchema
at synthesis step"| C + C --> V + V -->|"REVISE + reason
capped at MAX_REVISIONS"| R + end + + R -->|"tool call"| RET["Pinecone retriever"] + R -->|"tool call"| WEB["Tavily web search"] + RET -->|"exact chunks captured in state"| R + WEB -->|"URLs captured in state"| R + + V -->|APPROVE| UI([Streamlit UI
answer, verdict badge, source chunks]) + + DOCS[("./docs/ PDFs")] -.->|"one-time ingest
1000-char chunks, 200 overlap"| IDX[("Pinecone index")] + IDX -.-> RET +``` + +Dotted edges are the one-time ingestion path. Solid edges are the per-query path. + Documents are indexed once into Pinecone using OpenAI embeddings. On each query, a **Researcher** agent (built via `create_agent` from `langchain.agents`, LangChain's current agent constructor, which compiles a LangGraph state graph @@ -61,7 +89,7 @@ Researcher with that note, capped at a configurable number of passes This is the same groundedness check the eval harness grades offline, now enforced at runtime: **the Critic enforces at request time what the eval harness -verifies in CI.** The Critic runs on a cheaper model (`gpt-4o-mini`) than the +verifies offline.** The Critic runs on a cheaper model (`gpt-4o-mini`) than the Researcher's GPT-4o, since groundedness checking is a narrower verification task, so the smaller model suffices at a fraction of the per-call cost. It runs at `temperature=0` for deterministic, reproducible verdicts. @@ -82,6 +110,7 @@ and the exact source chunks the agent consulted, expanded by PDF and page. - Tavily for live web search (optional second agent tool) - LangSmith for optional run tracing (env-toggleable, with a structured-logging fallback) - Streamlit for the web UI +- GitHub Actions for CI (ruff plus an offline pytest suite), with dev dependencies pinned separately in `requirements-dev.txt` ## Setup @@ -112,6 +141,10 @@ Drop your PDFs into `./docs/` and index them once: python sentry_query.py --ingest ``` +The corpus currently indexed for the demo is three public 10-K annual filings +spanning retail, airline, and industrial sectors, and the application is +corpus-neutral with no company or document name hard-coded anywhere. + Launch the web UI: ``` @@ -140,6 +173,23 @@ points are new *correctness dimensions* (structured validity, groundedness) that the original code could not satisfy, not a change in answer accuracy. The harness output is the only performance number claimed here. +## Continuous integration + +GitHub Actions runs on every push and pull request to `main`: Python 3.11, ruff +against a pinned minimal rule set, then an offline pytest suite. The job needs no +API keys and no secrets, runs with `permissions: contents: read`, and finishes in +well under two minutes. + +The test suite is deliberately offline. It covers the Pydantic schema contracts, +the source-citation helpers, the eval grading logic, and the Critic's evidence +wiring, meaning that `critique()` is handed the exact retrieved chunks alongside +the answer and returns the structured verdict faithfully. + +What CI does not do is run the Critic's groundedness judgment or the full +`evals/eval.py` harness. That judgment is a live `gpt-4o-mini` call, so it needs +real keys and stays in the local eval run. CI verifies the deterministic logic and +the wiring around it, not the model's verdict itself. + ## Observability Every pipeline run emits a structured log record (tool routing, Critic verdict, diff --git a/agent.py b/agent.py index e98dd7a..af05471 100644 --- a/agent.py +++ b/agent.py @@ -261,4 +261,4 @@ def run_query(handle: AgentHandle, query: str) -> QueryResult: schema_ok=False, retriever_calls=len(queries), retrieved=list(handle.retrieved), - ) \ No newline at end of file + ) diff --git a/evals/eval.py b/evals/eval.py index 6a5bbe0..3a76320 100644 --- a/evals/eval.py +++ b/evals/eval.py @@ -170,4 +170,4 @@ def main() -> int: if __name__ == "__main__": - sys.exit(main()) \ No newline at end of file + sys.exit(main()) diff --git a/graph.py b/graph.py index 8c0f8fb..f1303e8 100644 --- a/graph.py +++ b/graph.py @@ -171,4 +171,4 @@ def run_pipeline(system: System, query: str) -> PipelineResult: revisions=final.get("revisions", 0), ) log_run(query, result) - return result \ No newline at end of file + return result diff --git a/observability.py b/observability.py index c5f7534..3430762 100644 --- a/observability.py +++ b/observability.py @@ -78,4 +78,4 @@ def log_run(query: str, result) -> None: } if result.verdict == "REVISE" or result.revisions > 0: record["critic_reason"] = result.critic_reason - logger.info("run %s", json.dumps(record, default=str)) \ No newline at end of file + logger.info("run %s", json.dumps(record, default=str)) diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..b893c8e --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,6 @@ +# Dev-only tooling, kept separate from the runtime requirements.txt. +# Install with: pip install -r requirements-dev.txt +-r requirements.txt + +ruff>=0.6.0 +pytest>=8.0.0 diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..c7b06a0 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,11 @@ +# Minimal, correctness-focused lint config. +# Pinned explicitly so CI is deterministic regardless of ruff's version defaults. +# We select real-error rules (pyflakes plus the import, statement, syntax, and +# warning subsets of pycodestyle) and deliberately leave out opinionated +# reformatting rules (import sorting, Optional-to-union rewrites, blind-except), +# so linting flags genuine problems without rewriting deliberate style. +target-version = "py311" +extend-exclude = ["venv", "venv_win", ".claude"] + +[lint] +select = ["E4", "E7", "E9", "F", "W"] diff --git a/schema.py b/schema.py index f91362c..f0f88c5 100644 --- a/schema.py +++ b/schema.py @@ -49,4 +49,4 @@ class CriticVerdict(BaseModel): ) reason: str = Field( description="If REVISE, the specific unsupported/overreaching claim; if APPROVE, a brief confirmation.", - ) \ No newline at end of file + ) diff --git a/sentry_query.py b/sentry_query.py index 24b8e89..5e2bc5e 100644 --- a/sentry_query.py +++ b/sentry_query.py @@ -143,4 +143,4 @@ def run_ui() -> None: if "--ingest" in sys.argv: run_ingest() else: - run_ui() \ No newline at end of file + run_ui() diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..bcfb75d --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,22 @@ +"""Test bootstrap: make the top-level modules importable with no real keys. + +The project modules (config, agent, graph, schema) live at the repo root and the +eval harness lives in evals/, so both paths go on sys.path before collection. + +Importing config.py constructs the Pinecone client (and the OpenAI embeddings), +which now raise if no API key is present, even though these tests never make a +network call. So we set clearly-fake placeholder keys first, and only if the +environment does not already provide real ones. These are not credentials, and +no test in this suite contacts OpenAI, Pinecone, or Tavily. +""" +import os +import sys +from pathlib import Path + +for var in ("PINECONE_API_KEY", "OPENAI_API_KEY", "TAVILY_API_KEY"): + os.environ.setdefault(var, "ci-placeholder-not-a-real-key") + +ROOT = Path(__file__).resolve().parents[1] +for path in (ROOT, ROOT / "evals"): + if str(path) not in sys.path: + sys.path.insert(0, str(path)) diff --git a/tests/test_agent_helpers.py b/tests/test_agent_helpers.py new file mode 100644 index 0000000..b21880d --- /dev/null +++ b/tests/test_agent_helpers.py @@ -0,0 +1,39 @@ +"""Unit tests for the pure source-handling helpers in agent.py. + +These do not touch the agent, Pinecone, or OpenAI, so they run offline. They +guard the citation path: page numbers shown to users must be 1-based, and source +lists must de-duplicate while preserving order. +""" +from types import SimpleNamespace + +from langchain_core.documents import Document + +from agent import _dedup, _source_label, extract_search_queries + + +def test_source_label_renders_1_based_page_from_zero_indexed_float(): + doc = Document(page_content="x", metadata={"source": "/a/b/costco-10k.pdf", "page": 4.0}) + assert _source_label(doc) == "costco-10k.pdf p.5" + + +def test_source_label_handles_missing_page(): + doc = Document(page_content="x", metadata={"source": "deere-10k.pdf"}) + assert _source_label(doc) == "deere-10k.pdf p.?" + + +def test_dedup_preserves_first_seen_order(): + assert _dedup(["b", "a", "b", "c", "a"]) == ["b", "a", "c"] + + +def test_extract_search_queries_returns_only_document_searches(): + messages = [ + SimpleNamespace( + tool_calls=[ + {"name": "search_documents", "args": {"query": "net sales"}}, + {"name": "web_search", "args": {"query": "stock price"}}, + ] + ), + SimpleNamespace(tool_calls=[{"name": "search_documents", "args": {"query": "revenue"}}]), + SimpleNamespace(tool_calls=None), + ] + assert extract_search_queries(messages) == ["net sales", "revenue"] diff --git a/tests/test_critic_plumbing.py b/tests/test_critic_plumbing.py new file mode 100644 index 0000000..ebcbf8b --- /dev/null +++ b/tests/test_critic_plumbing.py @@ -0,0 +1,65 @@ +"""Offline tests for the Critic wiring in graph.critique. + +IMPORTANT SCOPE NOTE. These do NOT test the Critic's groundedness *judgment*. +That judgment is the gpt-4o-mini call, so verifying that an ungrounded answer is +actually ruled REVISE genuinely requires a live model and cannot run in CI +without a key. Mocking the model would only assert the canned verdict back to +itself, which proves nothing. That semantic check stays in evals/eval.py, which +needs live keys. + +What these tests DO cover, with a stub model, is the plumbing that CI can verify +deterministically: the Critic is handed the exact retrieved chunks as evidence, +each chunk is labelled with its source, the answer under review is included, and +the structured verdict is returned faithfully to the caller. +""" +from langchain_core.documents import Document + +from graph import critique +from schema import CriticVerdict + + +class StubCritic: + """Stand-in for the structured-output LLM: records the prompt it was given + and returns a preset verdict, so we can inspect the evidence and the passthrough.""" + + def __init__(self, verdict: CriticVerdict): + self._verdict = verdict + self.last_prompt: str | None = None + + def invoke(self, prompt: str) -> CriticVerdict: + self.last_prompt = prompt + return self._verdict + + +CHUNKS = [ + Document( + page_content="Total net sales were 254.5 billion dollars.", + metadata={"source": "costco-10k.pdf", "page": 4.0}, + ), + Document( + page_content="Membership fees were 4.8 billion dollars.", + metadata={"source": "costco-10k.pdf", "page": 5.0}, + ), +] + + +def test_critique_feeds_every_chunk_and_the_answer_as_evidence(): + stub = StubCritic(CriticVerdict(verdict="APPROVE", reason="ok")) + answer = "Net sales were 254.5 billion dollars." + critique(stub, answer, CHUNKS) + + prompt = stub.last_prompt + assert answer in prompt + for chunk in CHUNKS: + assert chunk.page_content in prompt + # Each chunk is attributed with its 1-based source label. + assert "costco-10k.pdf p.5" in prompt + assert "costco-10k.pdf p.6" in prompt + + +def test_critique_returns_the_models_verdict_unchanged(): + revise = CriticVerdict(verdict="REVISE", reason="claim X is unsupported") + stub = StubCritic(revise) + out = critique(stub, "The CEO is a professional boxer.", CHUNKS) + assert out.verdict == "REVISE" + assert out.reason == "claim X is unsupported" diff --git a/tests/test_eval_helpers.py b/tests/test_eval_helpers.py new file mode 100644 index 0000000..14282b9 --- /dev/null +++ b/tests/test_eval_helpers.py @@ -0,0 +1,69 @@ +"""Unit tests for the grading helpers in evals/eval.py. + +These exercise the harness scoring logic (schema validity and the per-case +grade) against hand-built result objects, with no agent, Pinecone, or OpenAI +call. The full harness in eval.py still needs live keys and is not run in CI. +""" +from dataclasses import dataclass, field + +import eval as evalmod + + +@dataclass +class FakeResult: + """Stand-in for a PipelineResult, holding only the fields grading reads.""" + + answer: str = "" + sources: list = field(default_factory=list) + tool_used: str = "docs" + confidence: float | None = 0.9 + retriever_calls: int = 1 + verdict: str = "APPROVE" + + +def test_validates_as_schema_true_for_complete_result(): + assert evalmod.validates_as_schema(FakeResult(answer="net sales were 254 billion")) + + +def test_validates_as_schema_false_when_confidence_is_none(): + # A fallback result carries confidence=None, which must not validate. + assert not evalmod.validates_as_schema(FakeResult(confidence=None)) + + +def test_validates_as_schema_false_when_confidence_out_of_range(): + assert not evalmod.validates_as_schema(FakeResult(confidence=1.5)) + + +def test_grade_passes_a_well_formed_grounded_case(): + case = { + "expected_keywords_any": ["net sales", "$"], + "must_use_retriever": True, + "expect_verdict": "APPROVE", + } + result = FakeResult(answer="Total net sales were 254 billion.", retriever_calls=2) + ok, reasons = evalmod.grade(case, result) + assert ok, reasons + + +def test_grade_flags_missing_keyword(): + case = {"expected_keywords_any": ["operating revenue"]} + result = FakeResult(answer="The company sells groceries.") + ok, reasons = evalmod.grade(case, result) + assert not ok + assert any("missing" in r for r in reasons) + + +def test_grade_flags_retriever_use_mismatch(): + case = {"expected_keywords_any": ["price"], "must_use_retriever": False} + result = FakeResult(answer="The price is $50.", retriever_calls=3) + ok, reasons = evalmod.grade(case, result) + assert not ok + assert any("retriever use mismatch" in r for r in reasons) + + +def test_grade_flags_verdict_mismatch(): + case = {"expected_keywords_any": ["net sales"], "expect_verdict": "APPROVE"} + result = FakeResult(answer="net sales rose", verdict="REVISE") + ok, reasons = evalmod.grade(case, result) + assert not ok + assert any("verdict mismatch" in r for r in reasons) diff --git a/tests/test_schema.py b/tests/test_schema.py new file mode 100644 index 0000000..e142d1c --- /dev/null +++ b/tests/test_schema.py @@ -0,0 +1,47 @@ +"""Contract tests for the AnswerSchema and CriticVerdict Pydantic models. + +Pure validation, no API calls. These lock the structured-output contract that +both the UI and the Critic depend on. +""" +import pytest +from pydantic import ValidationError + +from schema import AnswerSchema, CriticVerdict + + +def test_answer_schema_accepts_valid_payload(): + a = AnswerSchema( + answer="Net sales were about 254 billion dollars.", + sources=["costco-10k.pdf p.5"], + tool_used="docs", + confidence=0.9, + ) + assert a.tool_used == "docs" + assert a.confidence == 0.9 + + +def test_answer_schema_defaults_sources_to_empty_list(): + a = AnswerSchema(answer="x", tool_used="none", confidence=0.5) + assert a.sources == [] + + +@pytest.mark.parametrize("bad_confidence", [-0.1, 1.1, 2.0]) +def test_answer_schema_rejects_confidence_out_of_range(bad_confidence): + with pytest.raises(ValidationError): + AnswerSchema(answer="x", tool_used="docs", confidence=bad_confidence) + + +def test_answer_schema_rejects_unknown_tool_used(): + with pytest.raises(ValidationError): + AnswerSchema(answer="x", tool_used="database", confidence=0.5) + + +@pytest.mark.parametrize("verdict", ["APPROVE", "REVISE"]) +def test_critic_verdict_accepts_valid_verdicts(verdict): + v = CriticVerdict(verdict=verdict, reason="because") + assert v.verdict == verdict + + +def test_critic_verdict_rejects_unknown_verdict(): + with pytest.raises(ValidationError): + CriticVerdict(verdict="MAYBE", reason="because")