diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b750ee4..0f1bf14a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- Agent and workflow output conversion now removes confirmed repeated input + history and keeps only the last message as the terminal output; intermediate + tool-call and tool-response messages are no longer included in + `gen_ai.output.messages` for full-history spans. + ## [0.2.1] - 2026-08-07 ### Fixed diff --git a/examples/agent/healthcare-assistant/.env.example b/examples/agent/healthcare-assistant/.env.example new file mode 100644 index 00000000..9a9e74c4 --- /dev/null +++ b/examples/agent/healthcare-assistant/.env.example @@ -0,0 +1,37 @@ +# LLM provider — set ONE of the two blocks below + +# Option A: OpenAI +# OPENAI_API_KEY= + +# Option B: Azure OpenAI +AZURE_OPENAI_API_KEY= +AZURE_OPENAI_ENDPOINT= # e.g. https://your-resource.openai.azure.com/ +AZURE_OPENAI_API_VERSION= # e.g. 2024-12-01-preview +AZURE_CHAT_DEPLOYMENT= # deployment name for chat, e.g. gpt-4o-mini +AZURE_EMBEDDING_DEPLOYMENT= # deployment name for embeddings, e.g. text-embedding-3-large + +# Splunk AO environment variables — set ONE of the two deployment blocks below + +# Option A: Splunk Observability (O11y) Cloud +SPLUNK_AO_REALM= # e.g. us0, eu0, lab0 +SPLUNK_AO_O11Y_TOKEN= # O11y ingest token (required for telemetry) +# SPLUNK_AO_O11Y_API_TOKEN= # Optional: dedicated API token for CRUD operations + +# Option B: On-premises / standalone deployment +# SPLUNK_AO_API_KEY= +# SPLUNK_AO_CONSOLE_URL= # e.g. https://console.yourcompany.com +# SPLUNK_AO_API_ENDPOINT= # Optional, only set for custom deployments + +# Routing (shared by both deployments) +SPLUNK_AO_PROJECT= +SPLUNK_AO_AGENT_STREAM= + +# PostgreSQL (pgvector) +POSTGRES_HOST=localhost +POSTGRES_PORT=5432 +POSTGRES_USER=postgres +POSTGRES_PASSWORD= +POSTGRES_DB=vectordb + +OTEL_SERVICE_NAME=healthcare-assistant +OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=your-project-name diff --git a/examples/agent/healthcare-assistant/.gitignore b/examples/agent/healthcare-assistant/.gitignore new file mode 100644 index 00000000..bacc7f06 --- /dev/null +++ b/examples/agent/healthcare-assistant/.gitignore @@ -0,0 +1,6 @@ +.env.* +!.env.example +.venv/ +__pycache__/ +*.pyc +.streamlit/secrets.toml diff --git a/examples/agent/healthcare-assistant/.streamlit/secrets.toml.template b/examples/agent/healthcare-assistant/.streamlit/secrets.toml.template new file mode 100644 index 00000000..cb804227 --- /dev/null +++ b/examples/agent/healthcare-assistant/.streamlit/secrets.toml.template @@ -0,0 +1,23 @@ +# Splunk AO Configuration (O11y Cloud) +# ----------------------------------------------------------------------------- +splunk_ao_realm = "us0" # e.g. us0, eu0, lab0 +splunk_ao_o11y_token = "..." # O11y ingest token +# splunk_ao_o11y_api_token = "..." # Optional: dedicated API token for CRUD operations +splunk_ao_project = "..." +splunk_ao_agent_stream = "..." + +# PostgreSQL Configuration (pgvector) +# ----------------------------------------------------------------------------- +# PostgreSQL with pgvector extension for vector storage. +# See README for Docker setup instructions. +postgres_host = "localhost" +postgres_port = "5432" +postgres_user = "postgres" +postgres_password = "mypassword" +postgres_db = "vectordb" + +# Environment Configuration +# ----------------------------------------------------------------------------- +# Set to "local" for local development, "hosted" for production/deployed environments. +# This determines which pgvector collection prefix is used for vector storage. +environment = "local" diff --git a/examples/agent/healthcare-assistant/Dockerfile b/examples/agent/healthcare-assistant/Dockerfile new file mode 100644 index 00000000..1718a64e --- /dev/null +++ b/examples/agent/healthcare-assistant/Dockerfile @@ -0,0 +1,41 @@ +# Multi-stage build for Healthcare Assistant +FROM python:3.12-slim AS builder + +# Set working directory +WORKDIR /app + +RUN pip install uv + +# Copy requirements and install dependencies +COPY 2-app-with-instrumentation/requirements.txt /app/ +RUN uv pip install --system --no-cache -r requirements.txt + +# Final stage +FROM python:3.12-slim + +WORKDIR /app + +# Install runtime dependencies +RUN apt-get update && apt-get install -y \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# Copy installed packages from builder +COPY --from=builder /usr/local/lib/python3.12/site-packages /usr/local/lib/python3.12/site-packages +COPY --from=builder /usr/local/bin /usr/local/bin + +# Copy application code and data +COPY 2-app-with-instrumentation/ /app/ +COPY docs/ /app/docs/ + +# Create non-root user +RUN useradd --create-home --shell /bin/bash app && \ + chown -R app:app /app + +USER app + +# Expose port for Streamlit +EXPOSE 8501 + +# Run the server +CMD ["streamlit", "run", "app.py"] diff --git a/examples/agent/healthcare-assistant/Dockerfile.loadgen b/examples/agent/healthcare-assistant/Dockerfile.loadgen new file mode 100644 index 00000000..2a37bcf2 --- /dev/null +++ b/examples/agent/healthcare-assistant/Dockerfile.loadgen @@ -0,0 +1,20 @@ +FROM python:3.12-slim + +WORKDIR /app + +RUN pip install uv --no-cache-dir + +# Install dependencies from requirements.txt (same as app) +COPY requirements.txt /app/ +RUN uv pip install --system --no-cache -r requirements.txt + +# Copy application code +COPY . /app/ + +# Create non-root user +RUN useradd --create-home --shell /bin/bash app && \ + chown -R app:app /app + +USER app + +CMD ["python", "load_generator_hallucination.py"] diff --git a/examples/agent/healthcare-assistant/Dockerfile.loadgen-agent b/examples/agent/healthcare-assistant/Dockerfile.loadgen-agent new file mode 100644 index 00000000..ebf2c68b --- /dev/null +++ b/examples/agent/healthcare-assistant/Dockerfile.loadgen-agent @@ -0,0 +1,26 @@ +FROM python:3.12-slim + +WORKDIR /app + +RUN pip install uv --no-cache-dir + +# Install splunk-ao from local source (contains fixes not yet released to PyPI). +# The build context is the repo root so we can copy the SDK source. +COPY pyproject.toml poetry.lock* README.md /sdk/ +COPY src/ /sdk/src/ +RUN uv pip install --system --no-cache /sdk + +# Install example dependencies (splunk-ao already installed above, skip PyPI version). +COPY examples/agent/healthcare-assistant/requirements.txt /app/ +RUN uv pip install --system --no-cache -r requirements.txt --override /dev/stdin <<'EOF' +splunk-ao +EOF + +COPY examples/agent/healthcare-assistant/ /app/ + +RUN useradd --create-home --shell /bin/bash app && \ + chown -R app:app /app + +USER app + +CMD ["python", "load_generator_agent.py"] diff --git a/examples/agent/healthcare-assistant/agent-with-instrumentation.py b/examples/agent/healthcare-assistant/agent-with-instrumentation.py new file mode 100644 index 00000000..c7e7225e --- /dev/null +++ b/examples/agent/healthcare-assistant/agent-with-instrumentation.py @@ -0,0 +1,167 @@ +"""LangGraph agent for the healthcare assistant.""" +import asyncio +import inspect +import json +import uuid +from concurrent.futures import ThreadPoolExecutor +from typing import Annotated, List, Dict, Optional, TypedDict + +from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage +from langchain_core.tools import StructuredTool +from langchain_openai import ChatOpenAI +from langgraph.graph import START, StateGraph +from langgraph.graph.state import CompiledStateGraph +from langgraph.graph.message import add_messages +from langgraph.prebuilt import ToolNode, tools_condition + +from config import TOOLS_DIR, load_config, load_system_prompt +from rag import create_rag_tool +from tools import logic as tools_logic + +import os +from splunk_ao import splunk_ao_context +from splunk_ao.handlers.langchain import SplunkAOAsyncCallback + +class State(TypedDict): + messages: Annotated[list, add_messages] + + +def _run_async(coro): + """Run an async coroutine from sync code (e.g. Streamlit).""" + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(coro) + + with ThreadPoolExecutor(max_workers=1) as executor: + return executor.submit(asyncio.run, coro).result() + + +class HealthcareAgent: + """LangGraph healthcare assistant.""" + + def __init__( + self, + session_id: str | None = None, + model_override: Optional[str] = None, + ): + self.config = load_config() + self.session_id = session_id or str(uuid.uuid4()) + self.model_override = model_override + self.system_prompt = load_system_prompt() + self.tools = [] + self.graph: CompiledStateGraph | None = None + self.langgraph_config = {"configurable": {"thread_id": self.session_id}} + + def load_tools(self) -> None: + tool_schema_path = TOOLS_DIR / "schema.json" + with tool_schema_path.open(encoding="utf-8") as f: + tool_schema = json.load(f) + + self.tools = [] + for tool_func in tools_logic.TOOLS: + tool_schema_dict = next( + (schema for schema in tool_schema if schema.get("name") == tool_func.__name__), + None, + ) + tool_kwargs = { + "name": tool_func.__name__, + "description": ( + tool_schema_dict.get("description") + if tool_schema_dict + else tool_func.__doc__ or f"Tool: {tool_func.__name__}" + ), + "args_schema": tool_schema_dict.get("parameters") if tool_schema_dict else None, + } + if inspect.iscoroutinefunction(tool_func): + langchain_tool = StructuredTool.from_function(coroutine=tool_func, **tool_kwargs) + else: + langchain_tool = StructuredTool.from_function(func=tool_func, **tool_kwargs) + self.tools.append(langchain_tool) + + rag_config = self.config.get("rag", {}) + if rag_config.get("enabled", False): + top_k = rag_config.get("top_k", 5) + model_config = self.config.get("model", {}) + effective_model = ( + self.model_override + or model_config.get("default_model") + or model_config.get("model_name") + ) + rag_tool = create_rag_tool(top_k, model_name=effective_model) + self.tools.append(rag_tool) + + print(f"✓ Loaded {len(self.tools)} tools") + + def _build_graph(self) -> CompiledStateGraph: + if not self.tools: + raise ValueError("Tools not loaded. Call load_tools() first.") + + model_config = self.config.get("model", {}) + effective_model = ( + self.model_override + or model_config.get("default_model") + or model_config.get("model_name") + ) + temperature = model_config.get("temperature", 0.1) + + llm_with_tools = ChatOpenAI( + model=effective_model, + temperature=temperature, + name="Healthcare Assistant", + ).bind_tools(self.tools) + + async def invoke_chatbot(state): + messages = list(state["messages"]) + if self.system_prompt: + messages = [SystemMessage(content=self.system_prompt)] + messages + message = await llm_with_tools.ainvoke(messages) + return {"messages": [message]} + + graph_builder = StateGraph(State) + graph_builder.add_node("chatbot", invoke_chatbot) + graph_builder.add_node("tools", ToolNode(tools=self.tools)) + graph_builder.add_edge(START, "chatbot") + graph_builder.add_conditional_edges("chatbot", tools_condition) + graph_builder.add_edge("tools", "chatbot") + return graph_builder.compile() + + async def _process_query_async(self, messages: List[Dict[str, str]]) -> str: + if not self.tools: + self.load_tools() + self.graph = self._build_graph() + + langchain_messages: List[BaseMessage] = [] + for msg in messages: + if msg["role"] == "user": + langchain_messages.append(HumanMessage(content=msg["content"])) + elif msg["role"] == "assistant": + langchain_messages.append(AIMessage(content=msg["content"])) + + with splunk_ao_context( + project=os.getenv("SPLUNK_AO_PROJECT"), + agent_stream=os.getenv("SPLUNK_AO_AGENT_STREAM"), + ): + splunk_ao_context.start_session(external_id=self.session_id) + + # One callback per request keeps each user turn in its own trace. + callback = SplunkAOAsyncCallback() + run_config = {**self.langgraph_config, "callbacks": [callback]} + + result = await self.graph.ainvoke( + {"messages": langchain_messages}, + run_config, + ) + if result["messages"]: + return result["messages"][-1].content + return "No response generated" + + def process_query(self, messages: List[Dict[str, str]]) -> str: + try: + return _run_async(self._process_query_async(messages)) + except Exception as e: + print(f"[ERROR] Error processing query: {e}") + import traceback + + traceback.print_exc() + return f"Error processing your request: {str(e)}" diff --git a/examples/agent/healthcare-assistant/agent.py b/examples/agent/healthcare-assistant/agent.py new file mode 100644 index 00000000..9fee7401 --- /dev/null +++ b/examples/agent/healthcare-assistant/agent.py @@ -0,0 +1,148 @@ +"""LangGraph agent for the healthcare assistant.""" +import asyncio +import inspect +import uuid +from concurrent.futures import ThreadPoolExecutor +from typing import Annotated, List, Dict, Optional, TypedDict + +from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage +from langchain_core.tools import StructuredTool +from langgraph.graph import START, StateGraph +from langgraph.graph.state import CompiledStateGraph +from langgraph.graph.message import add_messages +from langgraph.prebuilt import ToolNode, tools_condition + +from config import load_config, load_system_prompt, create_chat_llm +from rag import create_rag_tool +from tools import logic as tools_logic + +import os +from splunk_ao import splunk_ao_context +from splunk_ao.handlers.langchain import SplunkAOAsyncCallback + +class State(TypedDict): + messages: Annotated[list, add_messages] + + +def _run_async(coro): + """Run an async coroutine from sync code (e.g. Streamlit).""" + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(coro) + + with ThreadPoolExecutor(max_workers=1) as executor: + return executor.submit(asyncio.run, coro).result() + + +class HealthcareAgent: + """LangGraph healthcare assistant.""" + + def __init__( + self, + session_id: str | None = None, + model_override: Optional[str] = None, + ): + self.config = load_config() + self.session_id = session_id or str(uuid.uuid4()) + self.model_override = model_override + self.system_prompt = load_system_prompt() + self.tools = [] + self.graph: CompiledStateGraph | None = None + self.langgraph_config = {"configurable": {"thread_id": self.session_id}} + + def load_tools(self) -> None: + self.tools = [] + for tool_func in tools_logic.TOOLS: + if inspect.iscoroutinefunction(tool_func): + langchain_tool = StructuredTool.from_function(coroutine=tool_func) + else: + langchain_tool = StructuredTool.from_function(func=tool_func) + self.tools.append(langchain_tool) + + rag_config = self.config.get("rag", {}) + if rag_config.get("enabled", False): + top_k = rag_config.get("top_k", 5) + model_config = self.config.get("model", {}) + effective_model = ( + self.model_override + or model_config.get("default_model") + or model_config.get("model_name") + ) + rag_tool = create_rag_tool(top_k, model_name=effective_model) + self.tools.append(rag_tool) + + print(f"✓ Loaded {len(self.tools)} tools") + + def _build_graph(self) -> CompiledStateGraph: + if not self.tools: + raise ValueError("Tools not loaded. Call load_tools() first.") + + model_config = self.config.get("model", {}) + effective_model = ( + self.model_override + or model_config.get("default_model") + or model_config.get("model_name") + ) + temperature = model_config.get("temperature", 0.1) + + llm_with_tools = create_chat_llm( + model=effective_model, + temperature=temperature, + name="Healthcare Assistant", + ).bind_tools(self.tools) + + async def invoke_chatbot(state): + messages = list(state["messages"]) + if self.system_prompt: + messages = [SystemMessage(content=self.system_prompt)] + messages + message = await llm_with_tools.ainvoke(messages) + return {"messages": [message]} + + graph_builder = StateGraph(State) + graph_builder.add_node("chatbot", invoke_chatbot) + graph_builder.add_node("tools", ToolNode(tools=self.tools)) + graph_builder.add_edge(START, "chatbot") + graph_builder.add_conditional_edges("chatbot", tools_condition) + graph_builder.add_edge("tools", "chatbot") + return graph_builder.compile() + + async def _process_query_async(self, messages: List[Dict[str, str]]) -> str: + if not self.tools: + self.load_tools() + self.graph = self._build_graph() + + langchain_messages: List[BaseMessage] = [] + for msg in messages: + if msg["role"] == "user": + langchain_messages.append(HumanMessage(content=msg["content"])) + elif msg["role"] == "assistant": + langchain_messages.append(AIMessage(content=msg["content"])) + + with splunk_ao_context( + project=os.getenv("SPLUNK_AO_PROJECT"), + agent_stream=os.getenv("SPLUNK_AO_AGENT_STREAM"), + ): + splunk_ao_context.start_session(external_id=self.session_id) + + # One callback per request keeps each user turn in its own trace. + callback = SplunkAOAsyncCallback() + run_config = {**self.langgraph_config, "callbacks": [callback]} + + result = await self.graph.ainvoke( + {"messages": langchain_messages}, + run_config, + ) + if result["messages"]: + return result["messages"][-1].content + return "No response generated" + + def process_query(self, messages: List[Dict[str, str]]) -> str: + try: + return _run_async(self._process_query_async(messages)) + except Exception as e: + print(f"[ERROR] Error processing query: {e}") + import traceback + + traceback.print_exc() + return f"Error processing your request: {str(e)}" diff --git a/examples/agent/healthcare-assistant/app.py b/examples/agent/healthcare-assistant/app.py new file mode 100644 index 00000000..b1021ba4 --- /dev/null +++ b/examples/agent/healthcare-assistant/app.py @@ -0,0 +1,194 @@ +"""Healthcare assistant Streamlit app.""" +import os +import uuid + +import streamlit as st +from dotenv import load_dotenv +from langchain_core.messages import AIMessage, HumanMessage + +from agent import HealthcareAgent +from config import load_config +from helpers.hallucination_helpers import ( + add_hallucination_interaction_to_chat, + log_demo_hallucination, +) +from rag import get_rag_system +from setup_env import setup_environment + +load_dotenv() + +if not os.getenv("_ENV_LOADED"): + setup_environment() + os.environ["_ENV_LOADED"] = "true" + + +def escape_dollar_signs(text: str) -> str: + return text.replace("$", "\\$") + + +def display_chat_history(): + if not st.session_state.messages: + return + + for message_data in st.session_state.messages: + if isinstance(message_data, dict): + message = message_data.get("message") + if isinstance(message, HumanMessage): + with st.chat_message("user"): + st.write(escape_dollar_signs(message.content)) + elif isinstance(message, AIMessage): + with st.chat_message("assistant"): + st.write(escape_dollar_signs(message.content)) + + if st.session_state.get("processing", False): + with st.chat_message("assistant"): + st.write("Thinking...") + + +def show_example_queries(query_1: str, query_2: str): + st.subheader("💡 Try these examples") + col1, col2 = st.columns([0.48, 0.48]) + with col1: + if st.button(query_1, key="query_1", use_container_width=True): + return query_1 + with col2: + if st.button(query_2, key="query_2", use_container_width=True): + return query_2 + return None + + +def get_user_input(app_title: str, example_query_1: str, example_query_2: str): + st.title(app_title) + + if "messages" not in st.session_state: + st.session_state.messages = [] + + example_query = show_example_queries(example_query_1, example_query_2) + display_chat_history() + + user_input = st.chat_input("How can I help you?...") + if example_query: + user_input = example_query + return user_input + + +def process_input(user_input: str | None): + if user_input: + st.session_state.messages.append( + {"message": HumanMessage(content=user_input), "agent": "user"} + ) + st.session_state.processing = True + st.rerun() + + if st.session_state.get("processing", False): + conversation_messages = [] + for msg_data in st.session_state.messages: + if isinstance(msg_data, dict) and "message" in msg_data: + message = msg_data["message"] + if isinstance(message, HumanMessage): + conversation_messages.append({"role": "user", "content": message.content}) + elif isinstance(message, AIMessage): + conversation_messages.append({"role": "assistant", "content": message.content}) + + response = st.session_state.agent.process_query(conversation_messages) + st.session_state.messages.append( + {"message": AIMessage(content=response), "agent": "assistant"} + ) + st.session_state.processing = False + st.rerun() + + +def render_sidebar(app_config: dict) -> str: + with st.sidebar: + st.subheader("Model") + model_config = app_config.get("model", {}) + default_model = model_config.get("default_model", "gpt-4.1-mini") + additional_models = model_config.get("additional_models", []) + available_models = [default_model] + [ + m for m in additional_models if m != default_model + ] + + previous_model = st.session_state.get("active_model", default_model) + selected_model = st.selectbox( + "LLM", + options=available_models, + index=( + available_models.index(previous_model) + if previous_model in available_models + else 0 + ), + help="OpenAI model used for chat", + ) + + if previous_model != selected_model and "agent" in st.session_state: + del st.session_state.agent + st.session_state.active_model = selected_model + + has_hallucinations = bool(app_config.get("demo_hallucinations", [])) + if has_hallucinations: + st.divider() + st.subheader("Hallucination Demo") + st.markdown( + "Log an intentional hallucination to Splunk Agent Observability." + ) + if st.button("Log Hallucination", key="log_hallucination"): + with st.spinner("Logging hallucination to Splunk Agent Observability..."): + existing_logger = ( + st.session_state.get("splunk_ao_logger") + if st.session_state.get("splunk_ao_session_started", False) + else None + ) + success = log_demo_hallucination( + config=app_config, + existing_logger=existing_logger, + session_id=st.session_state.get("session_id"), + ) + if success: + add_hallucination_interaction_to_chat(app_config) + st.rerun() + else: + st.error( + "Failed to log hallucination. Check logs for details." + ) + + return selected_model + + +def main(): + app_config = load_config() + ui_config = app_config.get("ui", {}) + app_title = ui_config.get("app_title", "Online Healthcare Assistant") + example_queries = ui_config.get( + "example_queries", + [ + "What is the dosage and common side effects of Lisinopril?", + "Can you look up information for patient P001?", + ], + ) + + if "session_id" not in st.session_state: + # Splunk AO requires session_id to be a valid UUID when grouping traces. + st.session_state.session_id = str(uuid.uuid4()) + + selected_model = render_sidebar(app_config) + + if "rag_initialized" not in st.session_state: + get_rag_system() + st.session_state.rag_initialized = True + + if "agent" not in st.session_state: + st.session_state.agent = HealthcareAgent( + session_id=st.session_state.session_id, + model_override=selected_model, + ) + + user_input = get_user_input( + app_title, + example_queries[0], + example_queries[1] if len(example_queries) > 1 else "What can you do?", + ) + process_input(user_input) + + +if __name__ == "__main__": + main() diff --git a/examples/agent/healthcare-assistant/config.py b/examples/agent/healthcare-assistant/config.py new file mode 100644 index 00000000..3f44a8e8 --- /dev/null +++ b/examples/agent/healthcare-assistant/config.py @@ -0,0 +1,56 @@ +"""Load healthcare app configuration from YAML and JSON files.""" +import os +from pathlib import Path +from typing import Any + +import yaml + +APP_ROOT = Path(__file__).resolve().parent +DOMAIN = "healthcare" +CONFIG_PATH = APP_ROOT / "config.yaml" +SYSTEM_PROMPT_PATH = APP_ROOT / "system_prompt.json" +DOCS_DIR = APP_ROOT / "docs" +TOOLS_DIR = APP_ROOT / "tools" + + +def load_config() -> dict: + with CONFIG_PATH.open(encoding="utf-8") as f: + return yaml.safe_load(f) + + +def load_system_prompt() -> str: + import json + + with SYSTEM_PROMPT_PATH.open(encoding="utf-8") as f: + data = json.load(f) + return data["system_prompt"] + + +def create_chat_llm(model: str, temperature: float = 0.1, **kwargs: Any): + """Return AzureChatOpenAI when AZURE_OPENAI_ENDPOINT is set, otherwise ChatOpenAI.""" + if os.environ.get("AZURE_OPENAI_ENDPOINT"): + from langchain_openai import AzureChatOpenAI + return AzureChatOpenAI( + azure_deployment=model, + azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"], + api_key=os.environ.get("AZURE_OPENAI_API_KEY"), + api_version=os.environ.get("AZURE_OPENAI_API_VERSION", "2024-12-01-preview"), + temperature=temperature, + **kwargs, + ) + from langchain_openai import ChatOpenAI + return ChatOpenAI(model=model, temperature=temperature, **kwargs) + + +def create_embeddings(model: str): + """Return AzureOpenAIEmbeddings when AZURE_OPENAI_ENDPOINT is set, otherwise OpenAIEmbeddings.""" + if os.environ.get("AZURE_OPENAI_ENDPOINT"): + from langchain_openai import AzureOpenAIEmbeddings + return AzureOpenAIEmbeddings( + azure_deployment=os.environ.get("AZURE_EMBEDDING_DEPLOYMENT", model), + azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"], + api_key=os.environ.get("AZURE_OPENAI_API_KEY"), + api_version=os.environ.get("AZURE_OPENAI_API_VERSION", "2024-12-01-preview"), + ) + from langchain_openai import OpenAIEmbeddings + return OpenAIEmbeddings(model=model) diff --git a/examples/agent/healthcare-assistant/config.yaml b/examples/agent/healthcare-assistant/config.yaml new file mode 100644 index 00000000..44abcadf --- /dev/null +++ b/examples/agent/healthcare-assistant/config.yaml @@ -0,0 +1,35 @@ +domain: + name: "healthcare" + description: "Healthcare assistant for online help and patient information support" + +ui: + app_title: "🩻 Online Healthcare Assistant" + icon: "🩻" + example_queries: + - "What is the dosage and common side effects of Lisinopril?" + - "Can you look up information for patient P001?" + +model: + default_model: "gpt-4.1-mini" + temperature: 0.1 + +rag: + enabled: true + chunk_size: 1000 + chunk_overlap: 200 + top_k: 3 + +tools: + - "get_patient_info" + - "delete_patient_record" + - "search_medicine_qa" + +vectorstore: + embedding_model: "text-embedding-3-large" + +# Demo hallucinations for Splunk Agent Observability +demo_hallucinations: + - question: "What is the dosage and common side effects of Lisinopril?" + hallucinated_answer: "Common dosage is 100mg daily. Common side effects are rashes, itching, and swelling." + context: + - "Common Dosage: 10-40 mg once daily | Administration: Oral; take with or without food | Common Side Effects: Dry cough (10-15%); dizziness; headache; fatigue" diff --git a/examples/agent/healthcare-assistant/dataset.csv b/examples/agent/healthcare-assistant/dataset.csv new file mode 100644 index 00000000..68d80076 --- /dev/null +++ b/examples/agent/healthcare-assistant/dataset.csv @@ -0,0 +1,16 @@ +input,output +Can you look up information for patient P001?,I'll look up patient P001 using the get_patient_info tool. +What is the dosage and common side effects of Lisinopril?,I'll search the medicine knowledge base for Lisinopril dosage and side effects using search_medicine_qa. +Get patient details for P005 including prescription information.,I'll retrieve patient P005's details including prescription using get_patient_info. +What is Metformin used for?,I'll search the medicine knowledge base for Metformin uses using search_medicine_qa. +Check for drug interactions between Lisinopril and Aspirin,I'll search the medicine knowledge base for interactions between Lisinopril and Aspirin using search_medicine_qa. +What are the warnings for Levothyroxine?,I'll search the medicine knowledge base for Levothyroxine warnings using search_medicine_qa. +Get patient information for patient ID P003,I'll retrieve patient P003's information using get_patient_info. +What are the common dosages for Atorvastatin?,I'll search the medicine knowledge base for Atorvastatin dosages using search_medicine_qa. +What information do you have on patient P010?,I'll look up patient P010 using the get_patient_info tool. +What are the serious side effects of Aspirin?,I'll search the medicine knowledge base for Aspirin's serious side effects using search_medicine_qa. +Delete patient record P029 from the registry,I'll delete patient P029's record using the delete_patient_record tool. +What is the drug class for Metformin?,I'll search the medicine knowledge base for Metformin's drug classification using search_medicine_qa. +Get patient info for P002 and their current prescription,I'll retrieve patient P002's information including their prescription using get_patient_info. +Can you check interactions between Atorvastatin and Aspirin?,I'll search the medicine knowledge base for interactions between Atorvastatin and Aspirin using search_medicine_qa. +Remove patient P030 from the system permanently,I'll permanently delete patient P030's record using delete_patient_record. diff --git a/examples/agent/healthcare-assistant/docs/qa.csv b/examples/agent/healthcare-assistant/docs/qa.csv new file mode 100644 index 00000000..1a8d9170 --- /dev/null +++ b/examples/agent/healthcare-assistant/docs/qa.csv @@ -0,0 +1,226 @@ +question,answer +Lisinopril," + Generic Name: Lisinopril + Drug Class: ACE Inhibitor + Primary Indication: Hypertension and Heart Failure + Mechanism of Action: Inhibits conversion of angiotensin I to angiotensin II + Common Dosage: 10-40 mg once daily + Administration: Oral; take with or without food + Common Side Effects: Dry cough (10-15%); dizziness; headache; fatigue + Serious Side Effects: Angioedema; hyperkalemia; acute kidney injury; hypotension + Contraindications: Pregnancy; history of angioedema with ACE inhibitors; bilateral renal artery stenosis + Drug Interactions: NSAIDs (reduced efficacy); potassium supplements (hyperkalemia); lithium (increased levels) + Monitoring Requirements: Blood pressure; serum creatinine and potassium at baseline and periodically; monitor for signs of angioedema + Pregnancy Category: Category D - Contraindicated + Cost Tier: Low (generic available) + " +Metformin," + Generic Name: Metformin + Drug Class: Biguanide + Primary Indication: Type 2 Diabetes + Mechanism of Action: Decreases hepatic glucose production; increases insulin sensitivity + Common Dosage: 500-2000 mg daily (divided doses) + Administration: Oral; take with meals to reduce GI upset + Common Side Effects: Nausea; diarrhea; abdominal discomfort; metallic taste; vitamin B12 deficiency + Serious Side Effects: Lactic acidosis (rare but serious); severe hypoglycemia when combined with insulin/sulfonylureas + Contraindications: Severe renal impairment (eGFR <30); metabolic acidosis; acute heart failure + Drug Interactions: Contrast dyes (hold 48 hours before/after); alcohol (increased lactic acidosis risk) + Monitoring Requirements: Renal function (eGFR) before starting and annually; vitamin B12 levels annually; glucose monitoring + Pregnancy Category: Category B - Generally safe + Cost Tier: Low (generic available) + " +Atorvastatin," + Generic Name: Atorvastatin + Drug Class: Statin (HMG-CoA Reductase Inhibitor) + Primary Indication: Hyperlipidemia and ASCVD prevention + Mechanism of Action: Inhibits cholesterol synthesis in the liver + Common Dosage: 10-80 mg once daily + Administration: Oral; take any time of day with or without food + Common Side Effects: Muscle aches; headache; nausea; diarrhea; elevated liver enzymes + Serious Side Effects: Rhabdomyolysis; liver failure; new-onset diabetes; memory problems + Contraindications: Active liver disease; pregnancy; breastfeeding + Drug Interactions: Strong CYP3A4 inhibitors increase levels (clarithromycin erythromycin); grapefruit juice; fibrates increase myopathy risk + Monitoring Requirements: Lipid panel at baseline and 4-12 weeks; liver enzymes at baseline; monitor for muscle symptoms + Pregnancy Category: Category X - Contraindicated + Cost Tier: Low (generic available) + " +Amlodipine," + Generic Name: Amlodipine + Drug Class: Calcium Channel Blocker (Dihydropyridine) + Primary Indication: Hypertension and Angina + Mechanism of Action: Blocks calcium entry into vascular smooth muscle causing vasodilation + Common Dosage: 2.5-10 mg once daily + Administration: Oral; take with or without food + Common Side Effects: Peripheral edema (ankle swelling); headache; dizziness; flushing; palpitations + Serious Side Effects: Severe hypotension; worsening heart failure; MI (rare) + Contraindications: Severe aortic stenosis; cardiogenic shock + Drug Interactions: CYP3A4 inhibitors increase levels; simvastatin (limit simvastatin to 20 mg daily) + Monitoring Requirements: Blood pressure monitoring; heart rate; signs of peripheral edema + Pregnancy Category: Category C - Use with caution + Cost Tier: Low (generic available) + " +Levothyroxine," + Generic Name: Levothyroxine Sodium + Drug Class: Thyroid Hormone + Primary Indication: Hypothyroidism + Mechanism of Action: Synthetic T4; converted to active T3 in peripheral tissues + Common Dosage: 25-200 mcg once daily (individualized) + Administration: Oral; take on empty stomach 30-60 min before breakfast + Common Side Effects: Weight changes; headache; insomnia; nervousness; heat intolerance + Serious Side Effects: Cardiac arrhythmias; angina; MI; bone loss with excessive doses + Contraindications: Untreated thyrotoxicosis; acute MI; uncorrected adrenal insufficiency + Drug Interactions: Decreases absorption: calcium iron antacids soy PPIs; increases warfarin effect; decreases effect of diabetes medications + Monitoring Requirements: TSH at baseline 6-8 weeks after dose changes then annually; free T4 if indicated; heart rate and blood pressure + Pregnancy Category: Category A - Safe in pregnancy + Cost Tier: Low (generic available) + " +Aspirin," + Generic Name: Acetylsalicylic Acid + Drug Class: NSAID/Antiplatelet + Primary Indication: Pain; fever; cardiovascular disease prevention + Mechanism of Action: Irreversibly inhibits COX-1 and COX-2; inhibits platelet aggregation + Common Dosage: 81-325 mg daily (low-dose); 325-650 mg q4-6h PRN (analgesic) + Administration: Oral; enteric-coated formulations available + Common Side Effects: Dyspepsia; nausea; stomach upset; easy bruising + Serious Side Effects: GI bleeding; hemorrhagic stroke; allergic reactions; Reye's syndrome (children) + Contraindications: Active GI bleeding; hemophilia; aspirin allergy; children with viral infections + Drug Interactions: Anticoagulants (increased bleeding); NSAIDs (increased GI toxicity); methotrexate (increased toxicity) + Monitoring Requirements: No routine monitoring for low-dose; monitor for signs of bleeding; annual CBC if long-term use + Pregnancy Category: Category D in 3rd trimester + Cost Tier: Low (OTC available) + " +Losartan," + Generic Name: Losartan + Drug Class: Angiotensin Receptor Blocker (ARB) + Primary Indication: Hypertension; diabetic nephropathy + Mechanism of Action: Blocks angiotensin II at AT1 receptors causing vasodilation + Common Dosage: 25-100 mg once or twice daily + Administration: Oral; take with or without food + Common Side Effects: Dizziness; upper respiratory infection; fatigue; back pain + Serious Side Effects: Hyperkalemia; acute kidney injury; hypotension; angioedema (rare) + Contraindications: Pregnancy; bilateral renal artery stenosis + Drug Interactions: NSAIDs (reduced efficacy); potassium supplements (hyperkalemia); lithium (increased levels) + Monitoring Requirements: Blood pressure; serum creatinine and potassium at baseline and periodically + Pregnancy Category: Category D - Contraindicated + Cost Tier: Low (generic available) + " +Metoprolol," + Generic Name: Metoprolol + Drug Class: Beta-Blocker (Selective Beta-1) + Primary Indication: Hypertension; angina; heart failure; MI + Mechanism of Action: Blocks beta-1 adrenergic receptors; reduces heart rate and contractility + Common Dosage: 25-200 mg twice daily (tartrate); 25-400 mg daily (succinate ER) + Administration: Oral; take with or at same time relative to meals consistently + Common Side Effects: Fatigue; dizziness; bradycardia; cold extremities; depression + Serious Side Effects: Severe bradycardia; heart block; severe hypotension; bronchospasm; worsening heart failure + Contraindications: Sinus bradycardia; 2nd/3rd degree heart block; cardiogenic shock; severe peripheral arterial disease; untreated pheochromocytoma + Drug Interactions: Calcium channel blockers (bradycardia hypotension); insulin (masks hypoglycemia); CYP2D6 inhibitors increase levels + Monitoring Requirements: Heart rate and blood pressure; EKG if indicated; glucose in diabetics; signs/symptoms of heart failure + Pregnancy Category: Category C - Use with caution + Cost Tier: Low (generic available) + " +Omeprazole," + Generic Name: Omeprazole + Drug Class: Proton Pump Inhibitor (PPI) + Primary Indication: GERD; peptic ulcers; Zollinger-Ellison syndrome + Mechanism of Action: Irreversibly inhibits gastric H+/K+ ATPase (proton pump) + Common Dosage: 20-40 mg once daily + Administration: Oral; take 30-60 min before breakfast; do not crush or chew delayed-release capsules + Common Side Effects: Headache; nausea; diarrhea; abdominal pain; vitamin B12 deficiency + Serious Side Effects: C. difficile infection; bone fractures (long-term use); hypomagnesemia; kidney disease + Contraindications: Hypersensitivity to PPIs; concurrent use with rilpivirine + Drug Interactions: Clopidogrel (reduced activation); warfarin (increased INR); methotrexate (increased levels) + Monitoring Requirements: Magnesium levels if prolonged use or with diuretics/digoxin; vitamin B12 if long-term use; assess need for continued therapy periodically + Pregnancy Category: Category C - Use with caution + Cost Tier: Low (generic and OTC available) + " +Albuterol," + Generic Name: Albuterol Sulfate + Drug Class: Short-Acting Beta-2 Agonist (SABA) + Primary Indication: Asthma; COPD (acute bronchospasm) + Mechanism of Action: Relaxes bronchial smooth muscle by stimulating beta-2 receptors + Common Dosage: 2 puffs (90 mcg/puff) q4-6h PRN; nebulizer 2.5 mg q4-6h PRN + Administration: Inhalation; shake MDI before use; rinse mouth after + Common Side Effects: Tremor; nervousness; tachycardia; palpitations; headache + Serious Side Effects: Paradoxical bronchospasm; severe hypokalemia; cardiac arrhythmias + Contraindications: Hypersensitivity to albuterol or any component + Drug Interactions: Beta-blockers (reduced bronchodilator effect); diuretics (increased hypokalemia); MAO inhibitors (cardiovascular effects) + Monitoring Requirements: Heart rate and blood pressure; frequency of use (>2 times/week indicates poor control); potassium if high doses + Pregnancy Category: Category C - Use with caution + Cost Tier: Low (generic available) + " +Gabapentin," + Generic Name: Gabapentin + Drug Class: Anticonvulsant/Neuropathic Pain Agent + Primary Indication: Neuropathic pain; partial seizures; postherpetic neuralgia + Mechanism of Action: Unknown; structurally related to GABA but doesn't bind GABA receptors + Common Dosage: 300-3600 mg daily in 3 divided doses + Administration: Oral; take with or without food; requires renal dose adjustment + Common Side Effects: Dizziness; somnolence; ataxia; fatigue; peripheral edema + Serious Side Effects: Respiratory depression (with opioids); suicidal thoughts; severe skin reactions (rare) + Contraindications: Hypersensitivity to gabapentin + Drug Interactions: Opioids (increased respiratory depression); antacids (reduced absorption - separate by 2 hours) + Monitoring Requirements: No routine lab monitoring required; assess for suicidal ideation; renal function for dose adjustment + Pregnancy Category: Category C - Use with caution + Cost Tier: Low (generic available) + " +Sertraline," + Generic Name: Sertraline + Drug Class: Selective Serotonin Reuptake Inhibitor (SSRI) + Primary Indication: Depression; anxiety; OCD; PTSD + Mechanism of Action: Inhibits serotonin reuptake in CNS increasing synaptic serotonin + Common Dosage: 25-200 mg once daily + Administration: Oral; take with or without food; morning or evening dosing + Common Side Effects: Nausea; diarrhea; insomnia; sexual dysfunction; increased sweating + Serious Side Effects: Serotonin syndrome; suicidal ideation (especially young adults); bleeding; hyponatremia; seizures + Contraindications: Concurrent use with MAO inhibitors (14 day washout required); concurrent use with pimozide + Drug Interactions: MAO inhibitors (serotonin syndrome); NSAIDs/anticoagulants (increased bleeding); tamoxifen (reduced efficacy); other serotonergic drugs + Monitoring Requirements: Mental status at each visit; suicidal ideation especially first 1-2 months; sodium if risk factors for hyponatremia + Pregnancy Category: Category C - Use with caution + Cost Tier: Low (generic available) + " +Hydrochlorothiazide," + Generic Name: Hydrochlorothiazide (HCTZ) + Drug Class: Thiazide Diuretic + Primary Indication: Hypertension; edema + Mechanism of Action: Inhibits sodium reabsorption in distal tubule increasing urine output + Common Dosage: 12.5-50 mg once daily + Administration: Oral; take in morning to avoid nocturia + Common Side Effects: Hypokalemia; dizziness; muscle cramps; increased urination + Serious Side Effects: Severe electrolyte abnormalities; hypotension; hyperglycemia; hyperuricemia (gout); photosensitivity + Contraindications: Anuria; sulfonamide allergy + Drug Interactions: NSAIDs (reduced antihypertensive effect); lithium (increased levels); digoxin (hypokalemia increases toxicity) + Monitoring Requirements: Blood pressure; electrolytes (sodium potassium) and renal function at baseline and periodically; glucose and lipids + Pregnancy Category: Category B - Generally safe + Cost Tier: Low (generic available) + " +Warfarin," + Generic Name: Warfarin + Drug Class: Vitamin K Antagonist (Anticoagulant) + Primary Indication: DVT/PE; atrial fibrillation; mechanical heart valves + Mechanism of Action: Inhibits vitamin K-dependent clotting factors (II VII IX X) + Common Dosage: Individualized dosing based on INR (typically 2-10 mg daily) + Administration: Oral; take same time daily; consistent vitamin K intake + Common Side Effects: Bleeding; bruising; nausea + Serious Side Effects: Major hemorrhage; skin necrosis; purple toe syndrome + Contraindications: Pregnancy; active major bleeding; severe hypertension; recent surgery + Drug Interactions: EXTENSIVE - antibiotics NSAIDs acetaminophen many drugs affect INR + Monitoring Requirements: INR: daily initially then 2-3 times/week then weekly then every 4 weeks when stable; CBC; assess for bleeding + Pregnancy Category: Category X - Contraindicated + Cost Tier: Low (generic available) + " +Prednisone," + Generic Name: Prednisone + Drug Class: Corticosteroid + Primary Indication: Inflammatory conditions; autoimmune diseases; asthma + Mechanism of Action: Broad anti-inflammatory and immunosuppressive effects + Common Dosage: 5-60 mg daily (dose varies widely by indication) + Administration: Oral; take with food; taper slowly when discontinuing after >2 weeks use + Common Side Effects: Increased appetite; weight gain; insomnia; mood changes; hyperglycemia + Serious Side Effects: Adrenal suppression; infections; osteoporosis; peptic ulcers; cataracts; hyperglycemia + Contraindications: Systemic fungal infections + Drug Interactions: NSAIDs (increased GI bleed risk); vaccines (reduced efficacy live vaccines contraindicated); diabetes drugs (antagonizes effect) + Monitoring Requirements: Blood pressure and glucose regularly especially if diabetic; bone density if long-term use; growth in children; eye exams + Pregnancy Category: Category C - Use with caution + Cost Tier: Low (generic available) + " diff --git a/examples/agent/healthcare-assistant/docs/relational_patient.csv b/examples/agent/healthcare-assistant/docs/relational_patient.csv new file mode 100644 index 00000000..7a10efc7 --- /dev/null +++ b/examples/agent/healthcare-assistant/docs/relational_patient.csv @@ -0,0 +1,31 @@ +patient_id,patient_name,phone_number,address,patient_type,prescription +"P001", "George Rivera", "+1-213-555-0142", "4821 Sunset Blvd, Los Angeles, CA 90027", "inpatient", "Lisinopril 10mg" +"P002", "Theresa Greenleaf", "+1-312-555-0278", "1103 W Armitage Ave, Chicago, IL 60614", "outpatient", "Metformin 500mg" +"P003", "Marcus LaGrange", "+1-512-555-0391", "800 Congress Ave, Suite 300, Austin, TX 78701", "inpatient", "Atorvastatin 10mg" +"P004", "Herbert Richards", "+1-206-555-0467", "2250 Harbor Ave SW, Seattle, WA 98126", "outpatient", "Levothyroxine 25mcg" +"P005", "Joanne Brown", "+1-404-555-0583", "3340 Peachtree Rd NE, Atlanta, GA 30326", "inpatient", "Aspirin 100mg" +"P006", "Gloria Florian", "+1-415-555-0619", "598 Castro St, San Francisco, CA 94114", "outpatient", "Losartan 25mg" +"P007", "Tony Lakewood", "+1-303-555-0734", "1720 S Bellaire St, Denver, CO 80222", "inpatient", "Metoprolol 25mg" +"P008", "Erika Atlantic", "+1-617-555-0852", "200 State St, Boston, MA 02109", "outpatient", "Omeprazole 20mg" +"P009", "Daniel Whitmore", "+1-702-555-0921", "3900 Las Vegas Blvd S, Las Vegas, NV 89119", "inpatient", "Amlodipine 5mg" +"P010", "Patricia Nguyen", "+1-503-555-1037", "1200 SW Morrison St, Portland, OR 97205", "outpatient", "Hydrochlorothiazide 25mg" +"P011", "Robert Chen", "+1-214-555-1148", "2800 Main St, Dallas, TX 75226", "inpatient", "Gabapentin 300mg" +"P012", "Maria Santos", "+1-305-555-1259", "1450 Brickell Ave, Miami, FL 33131", "outpatient", "Sertraline 50mg" +"P013", "James O'Brien", "+1-215-555-1364", "1500 Market St, Philadelphia, PA 19102", "inpatient", "Warfarin 5mg" +"P014", "Linda Patterson", "+1-602-555-1475", "4400 N Central Ave, Phoenix, AZ 85012", "outpatient", "Albuterol 90mcg" +"P015", "Kevin Morrison", "+1-615-555-1586", "501 Broadway, Nashville, TN 37203", "inpatient", "Prednisone 10mg" +"P016", "Susan Keller", "+1-704-555-1697", "300 South Tryon St, Charlotte, NC 28202", "outpatient", "Lisinopril 20mg" +"P017", "Michael Torres", "+1-713-555-1708", "1200 Smith St, Houston, TX 77002", "inpatient", "Metformin 850mg" +"P018", "Angela Brooks", "+1-216-555-1819", "200 Public Sq, Cleveland, OH 44114", "outpatient", "Atorvastatin 20mg" +"P019", "Richard Hammond", "+1-414-555-1920", "777 N Water St, Milwaukee, WI 53202", "inpatient", "Metoprolol 50mg" +"P020", "Catherine Walsh", "+1-801-555-2031", "400 S Main St, Salt Lake City, UT 84111", "outpatient", "Levothyroxine 50mcg" +"P021", "Thomas Nguyen", "+1-901-555-2142", "100 Peabody Pl, Memphis, TN 38103", "inpatient", "Losartan 50mg" +"P022", "Diane Foster", "+1-916-555-2253", "1100 J St, Sacramento, CA 95814", "outpatient", "Omeprazole 40mg" +"P023", "William Hayes", "+1-816-555-2364", "1200 Main St, Kansas City, MO 64105", "inpatient", "Aspirin 81mg" +"P024", "Rachel Kim", "+1-612-555-2475", "350 Nicollet Mall, Minneapolis, MN 55401", "outpatient", "Amlodipine 10mg" +"P025", "Charles Evans", "+1-502-555-2586", "400 W Main St, Louisville, KY 40202", "inpatient", "Hydrochlorothiazide 12.5mg" +"P026", "Emily Rodriguez", "+1-505-555-2697", "500 Marquette Ave NW, Albuquerque, NM 87102", "outpatient", "Gabapentin 600mg" +"P027", "Frank Delaney", "+1-410-555-2708", "100 Light St, Baltimore, MD 21202", "inpatient", "Sertraline 100mg" +"P028", "Helen Park", "+1-317-555-2819", "200 S Meridian St, Indianapolis, IN 46225", "outpatient", "Albuterol 90mcg" +"P029", "Gregory Shaw", "+1-804-555-2920", "1000 E Broad St, Richmond, VA 23219", "inpatient", "Prednisone 20mg" +"P030", "Nancy Collins", "+1-405-555-3031", "200 N Walker Ave, Oklahoma City, OK 73102", "outpatient", "Metformin 1000mg" diff --git a/examples/agent/healthcare-assistant/healthcare-assistant-config.yaml b/examples/agent/healthcare-assistant/healthcare-assistant-config.yaml new file mode 100644 index 00000000..cf2f6cb4 --- /dev/null +++ b/examples/agent/healthcare-assistant/healthcare-assistant-config.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: healthcare-assistant-config +data: + ENVIRONMENT: "hosted" diff --git a/examples/agent/healthcare-assistant/helpers/__init__.py b/examples/agent/healthcare-assistant/helpers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/agent/healthcare-assistant/helpers/hallucination_helpers.py b/examples/agent/healthcare-assistant/helpers/hallucination_helpers.py new file mode 100644 index 00000000..7c899fe2 --- /dev/null +++ b/examples/agent/healthcare-assistant/helpers/hallucination_helpers.py @@ -0,0 +1,196 @@ +""" +Hallucination Demo Helpers + +Log intentional hallucinations to Splunk Agent Observability for demos. +Examples are defined in config.yaml under `demo_hallucinations`. +""" +import logging +import os +import uuid +from typing import Any, List, Optional, Union + +from splunk_ao import SplunkAOLogger +from langchain_core.messages import AIMessage, HumanMessage + +logger = logging.getLogger(__name__) + + +def log_hallucination( + project_name: str, + agent_stream: str, + question: str, + context_docs: List[str], + hallucinated_answer: str, + model: str = "gpt-4o", + session_name: str = "Hallucination Demo", + external_session_id: Optional[str] = None, + existing_logger: Optional[Union[SplunkAOLogger, Any]] = None, +) -> bool: + """ + Log a hallucination trace to Splunk AO for demonstration purposes. + + Creates a trace with a retriever span (real context) and an LLM span (wrong answer). + """ + try: + logger.info( + "Logging hallucination to project: %s, agent stream: %s", + project_name, + agent_stream, + ) + + if existing_logger: + logger.info("Using existing Splunk AO session for hallucination demo") + if hasattr(existing_logger, "get_logger_instance"): + splunk_ao_logger = existing_logger.get_logger_instance() + else: + splunk_ao_logger = existing_logger + else: + logger.info("Creating new Splunk AO session for hallucination demo") + splunk_ao_logger = SplunkAOLogger(project=project_name, agent_stream=agent_stream) + splunk_ao_logger.start_session( + name=session_name, + external_id=external_session_id or str(uuid.uuid4()), + ) + + splunk_ao_logger.start_trace( + input=question, + name="Hallucination Demo", + ) + + # Wrap spans in a workflow so the console shows "Hallucination Demo" as the + # root name — splunk-ao OTel converter derives span names from type+model for + # LLM spans, ignoring name=, so a workflow span is needed as the visible root. + splunk_ao_logger.add_workflow_span( + input=question, + name="Hallucination Demo", + ) + + splunk_ao_logger.add_retriever_span( + input=question, + output=context_docs, + name="RAG Retrieval", + duration_ns=int(1.3e8), + status_code=200, + ) + + context_text = "\n\n".join(context_docs) + llm_input = f"""Human: You are a helpful assistant. Given the context below, please answer the following question: + +{context_text} + +Question: {question}""" + + splunk_ao_logger.add_llm_span( + input=llm_input, + output=hallucinated_answer, + model=model, + name="LLM Response", + num_input_tokens=len(llm_input.split()) * 2, + num_output_tokens=len(hallucinated_answer.split()) * 2, + total_tokens=len(llm_input.split()) * 2 + len(hallucinated_answer.split()) * 2, + duration_ns=int(1.2e8), + metadata={"temperature": "0.1", "demo_type": "hallucination"}, + temperature=0.1, + status_code=200, + time_to_first_token_ns=500000, + ) + + # Conclude the workflow span, then the trace + splunk_ao_logger.conclude( + output=hallucinated_answer, + duration_ns=int(2.5e8), + status_code=200, + ) + + splunk_ao_logger.conclude( + output=hallucinated_answer, + duration_ns=int(2.5e8), + status_code=200, + ) + + splunk_ao_logger.flush() + + logger.info("Successfully logged hallucination to project: %s", project_name) + return True + + except Exception as e: + logger.error("Failed to log hallucination: %s", e) + return False + + +def log_demo_hallucination( + config: dict, + hallucination_index: int = 0, + existing_logger: Optional[Union[SplunkAOLogger, Any]] = None, + session_id: Optional[str] = None, +) -> bool: + """Log a demo hallucination from config.yaml to Splunk AO.""" + project_name = os.getenv("SPLUNK_AO_PROJECT", "healthcare-assistant") + agent_stream = os.getenv("SPLUNK_AO_AGENT_STREAM", "default") + + hallucinations = config.get("demo_hallucinations", []) + if not hallucinations: + logger.warning("No hallucination examples defined in config") + return False + + if hallucination_index >= len(hallucinations): + hallucination_index = 0 + + hallucination = hallucinations[hallucination_index] + question = hallucination.get("question", "") + hallucinated_answer = hallucination.get("hallucinated_answer", "") + context_docs = hallucination.get("context", []) + + if not question or not hallucinated_answer: + logger.error("Invalid hallucination config: missing question or answer") + return False + + if not context_docs: + context_docs = ["[No context available]"] + + model_config = config.get("model", {}) + model = model_config.get("default_model", "gpt-4o") + + return log_hallucination( + project_name=project_name, + agent_stream=agent_stream, + question=question, + context_docs=context_docs, + hallucinated_answer=hallucinated_answer, + model=model, + session_name="Healthcare Hallucination Demo", + external_session_id=session_id, + existing_logger=existing_logger, + ) + + +def add_hallucination_interaction_to_chat( + config: dict, + hallucination_index: int = 0, +) -> None: + """Append the demo hallucination Q&A pair to the Streamlit chat history.""" + import streamlit as st + + hallucinations = config.get("demo_hallucinations", []) + if not hallucinations: + return + + if hallucination_index >= len(hallucinations): + hallucination_index = 0 + + hallucination = hallucinations[hallucination_index] + question = hallucination.get("question", "") + answer = hallucination.get("hallucinated_answer", "") + + if not question or not answer: + return + + if "messages" not in st.session_state: + st.session_state.messages = [] + + st.session_state.messages.append( + {"message": HumanMessage(content=question), "agent": "user"} + ) + st.session_state.messages.append( + {"message": AIMessage(content=answer), "agent": "assistant"} + ) diff --git a/examples/agent/healthcare-assistant/helpers/pgvector_utils.py b/examples/agent/healthcare-assistant/helpers/pgvector_utils.py new file mode 100644 index 00000000..cc42ef4a --- /dev/null +++ b/examples/agent/healthcare-assistant/helpers/pgvector_utils.py @@ -0,0 +1,73 @@ +"""Shared PostgreSQL/pgvector utilities for vector storage and retrieval.""" +import os +from typing import Optional, Tuple + +from langchain_openai import OpenAIEmbeddings +from langchain_postgres import PGVector +from sqlalchemy import create_engine, text + + +def get_postgres_connection_string() -> str: + """Build SQLAlchemy connection string from environment variables.""" + host = os.environ.get("POSTGRES_HOST", "localhost") + port = os.environ.get("POSTGRES_PORT", "5432") + user = os.environ.get("POSTGRES_USER", "postgres") + password = os.environ.get("POSTGRES_PASSWORD", "") + database = os.environ.get("POSTGRES_DB", "vectordb") + return f"postgresql+psycopg://{user}:{password}@{host}:{port}/{database}" + + +def get_collection_name(domain_name: str, environment: Optional[str] = None) -> str: + """SQL-safe collection name for a domain/environment pair.""" + env = environment or os.environ.get("ENVIRONMENT", "local") + return f"{domain_name}_{env}_index" + + +def collection_exists(domain_name: str, environment: Optional[str] = None) -> bool: + """Return True if the pgvector collection has been created.""" + collection_name = get_collection_name(domain_name, environment) + engine = create_engine(get_postgres_connection_string()) + with engine.connect() as conn: + row = conn.execute( + text("SELECT 1 FROM langchain_pg_collection WHERE name = :name LIMIT 1"), + {"name": collection_name}, + ).fetchone() + return row is not None + + +def create_pgvector_store( + embeddings: OpenAIEmbeddings, + domain_name: str, + environment: Optional[str] = None, + *, + pre_delete_collection: bool = False, +) -> Tuple[PGVector, str]: + """Create or connect to a PGVector store for the given domain.""" + collection_name = get_collection_name(domain_name, environment) + vector_store = PGVector( + embeddings=embeddings, + collection_name=collection_name, + connection=get_postgres_connection_string(), + use_jsonb=True, + pre_delete_collection=pre_delete_collection, + ) + return vector_store, collection_name + + +def get_pgvector_store( + domain_name: str, + embedding_model: str = "text-embedding-3-large", + environment: Optional[str] = None, +) -> Tuple[PGVector, str]: + """Return a PGVector store for retrieval, raising if the collection is missing.""" + env = environment or os.environ.get("ENVIRONMENT", "local") + collection_name = get_collection_name(domain_name, env) + + if not collection_exists(domain_name, env): + raise ValueError( + f"PostgreSQL collection not found: {collection_name}. " + f"Run: python helpers/setup_vectordb.py {env}" + ) + + embeddings = OpenAIEmbeddings(model=embedding_model) + return create_pgvector_store(embeddings, domain_name, env) diff --git a/examples/agent/healthcare-assistant/helpers/setup_vectordb.py b/examples/agent/healthcare-assistant/helpers/setup_vectordb.py new file mode 100644 index 00000000..fca9900b --- /dev/null +++ b/examples/agent/healthcare-assistant/helpers/setup_vectordb.py @@ -0,0 +1,115 @@ +""" +Healthcare vector database setup using PostgreSQL/pgvector. + +Usage: + python helpers/setup_vectordb.py local + python helpers/setup_vectordb.py hosted +""" +import argparse +import getpass +import os +import sys +import uuid +from pathlib import Path +from typing import List + +import pandas as pd + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from config import DOCS_DIR, DOMAIN, create_embeddings, load_config +from langchain_core.documents import Document +from setup_env import setup_environment +from helpers.pgvector_utils import create_pgvector_store, get_collection_name +from helpers.sql_utils import load_domain_relational_csvs + + +def setup_vectordb(environment: str) -> bool: + """Set up vector database and relational tables for the healthcare app.""" + print(f"Setting up vector database for healthcare in {environment} environment") + + setup_environment() + os.environ["ENVIRONMENT"] = environment + + app_config = load_config() + rag_config = app_config.get("rag", {}) + vectorstore_config = app_config.get("vectorstore", {}) + + chunk_size = rag_config.get("chunk_size", 1000) + chunk_overlap = rag_config.get("chunk_overlap", 200) + embedding_model = vectorstore_config.get("embedding_model", "text-embedding-3-large") + + print(f"Using chunk_size: {chunk_size}, chunk_overlap: {chunk_overlap}") + print(f"Using embedding model: {embedding_model}") + + docs_dir = DOCS_DIR + if not docs_dir.exists(): + print(f"❌ Docs directory not found: {docs_dir}") + return False + + if not os.environ.get("AZURE_OPENAI_ENDPOINT") and not os.environ.get("OPENAI_API_KEY"): + os.environ["OPENAI_API_KEY"] = getpass.getpass("Enter API key for OpenAI: ") + + if not os.environ.get("POSTGRES_PASSWORD"): + os.environ["POSTGRES_PASSWORD"] = getpass.getpass("Enter PostgreSQL password: ") + + embeddings = create_embeddings(model=embedding_model) + collection_name = get_collection_name(DOMAIN, environment) + print(f"Creating PostgreSQL/pgvector collection: {collection_name}") + vector_store, collection_name = create_pgvector_store( + embeddings, + DOMAIN, + environment, + pre_delete_collection=True, + ) + + csv_path = docs_dir / "qa.csv" + df = pd.read_csv(csv_path) + doc_list: List[Document] = [] + uuid_list = [] + for _, row in df.iterrows(): + question = str(row.get("question", "") or "").strip() + answer = str(row.get("answer", "") or "") + body = ( + f"[FAQ] Healthcare FAQ. " + f"Medication: {question}. " + f"Information: {answer}. " + ) + meta = { + "doc_family": "healthcare", + "question": question, + "answer": answer, + } + doc_list.append(Document(page_content=body, metadata=meta)) + uuid_list.append(uuid.uuid4()) + + print("Adding documents to vector store...") + vector_store.add_documents(documents=doc_list, ids=uuid_list) + embedded_count = len(doc_list) + + print("Loading relational tables for healthcare...") + load_domain_relational_csvs(docs_dir, DOMAIN) + + print("✅ Successfully created vector database for healthcare") + print(f"📊 Total documents embedded: {embedded_count}") + print(f"🔗 PostgreSQL collection: {collection_name}") + return True + + +def main(): + parser = argparse.ArgumentParser( + description="Set up PostgreSQL/pgvector for the healthcare assistant" + ) + parser.add_argument( + "environment", + choices=["local", "hosted"], + help="Environment to use ('local' or 'hosted')", + ) + args = parser.parse_args() + + if not setup_vectordb(args.environment): + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/examples/agent/healthcare-assistant/helpers/sql_utils.py b/examples/agent/healthcare-assistant/helpers/sql_utils.py new file mode 100644 index 00000000..5cb138f6 --- /dev/null +++ b/examples/agent/healthcare-assistant/helpers/sql_utils.py @@ -0,0 +1,173 @@ +"""PostgreSQL utilities for relational demo tables and SQL execution.""" +import re +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +import pandas as pd +from sqlalchemy import create_engine, inspect, text +from sqlalchemy.engine import Engine + +from helpers.pgvector_utils import get_postgres_connection_string + + +def relational_table_name(domain_name: str, table_suffix: str) -> str: + """Build a domain-scoped, SQL-safe table name.""" + safe_domain = re.sub(r"[^a-z0-9_]", "_", domain_name.lower()) + safe_suffix = re.sub(r"[^a-z0-9_]", "_", table_suffix.lower()) + return f"{safe_domain}_{safe_suffix}" + + +def parse_relational_csv_name(csv_path: str | Path) -> Optional[str]: + """Extract the table suffix from relational_.csv.""" + stem = Path(csv_path).stem + prefix = "relational_" + if not stem.startswith(prefix): + return None + suffix = stem[len(prefix) :].strip() + return suffix or None + + +def _infer_pg_type(series: pd.Series) -> str: + if pd.api.types.is_integer_dtype(series): + return "BIGINT" + if pd.api.types.is_float_dtype(series): + return "NUMERIC" + return "TEXT" + + +def _guess_primary_key(columns: List[str]) -> str: + for col in columns: + if col.endswith("_id"): + return col + return columns[0] + + +def _sanitize_identifier(name: str) -> str: + return re.sub(r"[^a-zA-Z0-9_]", "_", name) + + +def load_relational_csv( + engine: Engine, + csv_path: str | Path, + domain_name: str, +) -> Tuple[str, int]: + """Load a relational_.csv file into PostgreSQL.""" + csv_path = Path(csv_path) + table_suffix = parse_relational_csv_name(csv_path) + if not table_suffix: + raise ValueError(f"Not a relational CSV file: {csv_path}") + + table_name = relational_table_name(domain_name, table_suffix) + df = pd.read_csv(csv_path, skipinitialspace=True) + if df.empty: + raise ValueError(f"No rows found in {csv_path}") + + for col in df.columns: + if df[col].dtype == object: + df[col] = df[col].astype(str).str.strip() + + columns = [_sanitize_identifier(str(c)) for c in df.columns] + df.columns = columns + pk_col = _guess_primary_key(columns) + + col_defs = [] + for col in columns: + pg_type = _infer_pg_type(df[col]) + col_defs.append(f'"{col}" {pg_type}') + + create_sql = ( + f'CREATE TABLE "{table_name}" (\n ' + + ",\n ".join(col_defs) + + f',\n PRIMARY KEY ("{pk_col}")\n)' + ) + + with engine.begin() as conn: + conn.execute(text(f'DROP TABLE IF EXISTS "{table_name}" CASCADE')) + conn.execute(text(create_sql)) + + column_list = ", ".join(f'"{c}"' for c in columns) + placeholders = ", ".join(f":{col}" for col in columns) + insert_sql = ( + f'INSERT INTO "{table_name}" ({column_list}) ' + f"VALUES ({placeholders})" + ) + conn.execute(text(insert_sql), df.to_dict(orient="records")) + + for col in columns: + if col == pk_col: + continue + conn.execute( + text( + f'CREATE INDEX IF NOT EXISTS "{table_name}_{col}_idx" ' + f'ON "{table_name}" ("{col}")' + ) + ) + + return table_name, len(df) + + +def load_domain_relational_csvs(docs_dir: str | Path, domain_name: str) -> List[Tuple[str, int]]: + """Load every relational_*.csv file in the docs directory.""" + docs_path = Path(docs_dir) + engine = create_engine(get_postgres_connection_string()) + results: List[Tuple[str, int]] = [] + + for csv_path in sorted(docs_path.glob("relational_*.csv")): + table_name, row_count = load_relational_csv(engine, csv_path, domain_name) + print(f"✓ Loaded relational table {table_name} ({row_count} rows) from {csv_path.name}") + results.append((table_name, row_count)) + + return results + + +def get_table_schema_description(engine: Engine, table_name: str) -> str: + """Return a human-readable schema snippet for Text-to-SQL prompts.""" + inspector = inspect(engine) + if table_name not in inspector.get_table_names(): + raise ValueError(f"Table not found: {table_name}") + + pk = inspector.get_pk_constraint(table_name).get("constrained_columns") or [] + lines = [f'Table "{table_name}" columns:'] + for col in inspector.get_columns(table_name): + name = col["name"] + col_type = str(col["type"]) + extras = [] + if name in pk: + extras.append("PRIMARY KEY") + if col.get("nullable") is False and name not in pk: + extras.append("NOT NULL") + suffix = f" ({', '.join(extras)})" if extras else "" + lines.append(f" - {name}: {col_type}{suffix}") + return "\n".join(lines) + + +def _sql_operation(sql: str) -> str: + cleaned = (sql or "").strip().lstrip("(").upper() + for keyword in ("SELECT", "DELETE", "INSERT", "UPDATE"): + if cleaned.startswith(keyword): + return keyword.lower() + return "unknown" + + +def execute_sql(sql: str) -> Dict[str, Any]: + """Execute a SQL statement and return a JSON-serializable result.""" + sql_clean = (sql or "").strip().rstrip(";") + operation = _sql_operation(sql_clean) + engine = create_engine(get_postgres_connection_string()) + + with engine.begin() as conn: + result = conn.execute(text(sql_clean)) + if operation == "select": + rows = [dict(row) for row in result.mappings()] + count = len(rows) + else: + rows = [] + count = result.rowcount + + return { + "sql": sql_clean, + "rows": rows, + "count": count, + "source": "postgres", + "operation": operation, + } diff --git a/examples/agent/healthcare-assistant/helpers/text_to_sql_utils.py b/examples/agent/healthcare-assistant/helpers/text_to_sql_utils.py new file mode 100644 index 00000000..388cb6aa --- /dev/null +++ b/examples/agent/healthcare-assistant/helpers/text_to_sql_utils.py @@ -0,0 +1,77 @@ +"""Text-to-SQL helpers for patient lookup and delete tools.""" +from typing import Literal + +from langchain_core.messages import HumanMessage, SystemMessage +from sqlalchemy import create_engine + +from config import create_chat_llm +from helpers.pgvector_utils import get_postgres_connection_string +from helpers.sql_utils import get_table_schema_description, relational_table_name + +SqlOperation = Literal["select", "delete"] + + +def _strip_sql_fences(text: str) -> str: + cleaned = (text or "").strip() + if cleaned.startswith("```"): + lines = cleaned.splitlines() + if lines and lines[0].startswith("```"): + lines = lines[1:] + if lines and lines[-1].strip() == "```": + lines = lines[:-1] + cleaned = "\n".join(lines).strip() + return cleaned.rstrip(";") + + +async def generate_sql( + *, + domain_name: str, + table_suffix: str, + id_column: str, + record_id: str, + operation: SqlOperation = "select", + model: str = "gpt-4o-mini", + temperature: float = 0.0, + use_case_identifier: str = "", + use_case_value: str = "", +) -> str: + """Use an LLM to produce a SELECT or DELETE statement for a relational table.""" + table_name = relational_table_name(domain_name, table_suffix) + engine = create_engine(get_postgres_connection_string()) + schema = get_table_schema_description(engine, table_name) + + if operation == "delete": + system_prompt = ( + "You are a PostgreSQL expert. Generate exactly one DELETE statement " + "to remove the requested record. Rules:\n" + f'- Use DELETE FROM "{table_name}" with a WHERE clause on {id_column}.\n' + "- Use only the provided table and columns.\n" + "- Match the identifier exactly (case-sensitive).\n" + "- Do not use JOINs, subqueries, CTEs, RETURNING, or semicolons.\n" + "- Output only the SQL statement with no explanation." + ) + user_prompt = ( + f"{schema}\n\n" + f"Delete request: remove the row where {use_case_identifier} equals '{use_case_value}'." + ) + else: + system_prompt = ( + "You are a PostgreSQL expert. Generate exactly one SELECT statement " + "to answer the user's lookup request. Rules:\n" + "- Use only the provided table and columns.\n" + "- Return all columns for the matching record.\n" + "- Query does not need to use the primary key column.\n" + "- Write the SQL statement to use uppercase: WHERE UPPER(column) = UPPER('value').\n" + "- Do not use JOINs, subqueries, CTEs, or semicolons.\n" + "- Output only the SQL statement with no explanation." + ) + user_prompt = ( + f"{schema}\n\n" + f"Lookup request: {use_case_identifier}='{use_case_value}'\n\n" + ) + + llm = create_chat_llm(model=model, temperature=temperature) + response = await llm.ainvoke( + [SystemMessage(content=system_prompt), HumanMessage(content=user_prompt)] + ) + return _strip_sql_fences(str(response.content)) diff --git a/examples/agent/healthcare-assistant/k8s-loadgen-agent.yaml b/examples/agent/healthcare-assistant/k8s-loadgen-agent.yaml new file mode 100644 index 00000000..d734b840 --- /dev/null +++ b/examples/agent/healthcare-assistant/k8s-loadgen-agent.yaml @@ -0,0 +1,96 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: healthcare-assistant-loadgen-agent + namespace: healthcare-assistant +spec: + schedule: "30 * * * *" + suspend: false + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + containers: + - name: loadgen-agent + image: ghcr.io/splunk/healthcare-assistant-agent-loadgen:latest + imagePullPolicy: Always + env: + - name: SPLUNK_AO_REALM + valueFrom: + secretKeyRef: + name: splunk-ao-secrets + key: realm + - name: SPLUNK_AO_O11Y_TOKEN + valueFrom: + secretKeyRef: + name: splunk-ao-secrets + key: o11y-token + - name: SPLUNK_AO_PROJECT + valueFrom: + secretKeyRef: + name: splunk-ao-secrets + key: project + - name: SPLUNK_AO_AGENT_STREAM + valueFrom: + secretKeyRef: + name: splunk-ao-secrets + key: agent-stream + - name: OPENAI_API_KEY + valueFrom: + secretKeyRef: + name: openai-secrets + key: api-key + - name: OPENAI_BASE_URL + valueFrom: + secretKeyRef: + name: openai-secrets + key: base-url + - name: OPENAI_API_VERSION + value: "2024-12-01-preview" + - name: AZURE_OPENAI_API_KEY + valueFrom: + secretKeyRef: + name: openai-secrets + key: api-key + - name: AZURE_OPENAI_ENDPOINT + value: "https://.cognitiveservices.azure.com/" + - name: AZURE_OPENAI_API_VERSION + value: "2024-12-01-preview" + - name: AZURE_CHAT_DEPLOYMENT + value: "gpt-4.1-mini" + - name: AZURE_EMBEDDING_DEPLOYMENT + value: "text-embedding-3-large" + - name: POSTGRES_HOST + value: "postgres" + - name: POSTGRES_PORT + value: "5432" + - name: POSTGRES_USER + value: "postgres" + - name: POSTGRES_DB + value: "vectordb" + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-credentials + key: POSTGRES_PASSWORD + - name: QUERY_DELAY_SECONDS + value: "3" + - name: SPLUNK_AO_OTLP_ENDPOINT + value: "https://ingest..signalfx.com/v2/trace/otlp" + - name: OTEL_SERVICE_NAME + value: "healthcare-assistant-loadgen-agent" + - name: OTEL_RESOURCE_ATTRIBUTES + value: "deployment.environment.name=agent-observability" + - name: ENVIRONMENT + value: "hosted" + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "1Gi" + cpu: "500m" diff --git a/examples/agent/healthcare-assistant/k8s-loadgen.yaml b/examples/agent/healthcare-assistant/k8s-loadgen.yaml new file mode 100644 index 00000000..11c79261 --- /dev/null +++ b/examples/agent/healthcare-assistant/k8s-loadgen.yaml @@ -0,0 +1,92 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: healthcare-assistant-loadgen + namespace: healthcare-assistant +spec: + schedule: "0 * * * *" + suspend: false + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + containers: + - name: loadgen + image: ghcr.io/splunk/healthcare-assistant-loadgen:latest + imagePullPolicy: Always + env: + - name: SPLUNK_AO_REALM + valueFrom: + secretKeyRef: + name: splunk-ao-secrets + key: realm + - name: SPLUNK_AO_O11Y_TOKEN + valueFrom: + secretKeyRef: + name: splunk-ao-secrets + key: o11y-token + - name: SPLUNK_AO_O11Y_API_TOKEN + valueFrom: + secretKeyRef: + name: splunk-ao-secrets + key: o11y-api-token + - name: SPLUNK_AO_PROJECT + valueFrom: + secretKeyRef: + name: splunk-ao-secrets + key: project + - name: SPLUNK_AO_AGENT_STREAM + valueFrom: + secretKeyRef: + name: splunk-ao-secrets + key: agent-stream + - name: OPENAI_API_KEY + valueFrom: + secretKeyRef: + name: openai-secrets + key: api-key + - name: OPENAI_BASE_URL + valueFrom: + secretKeyRef: + name: openai-secrets + key: base-url + - name: OPENAI_API_VERSION + value: "2024-12-01-preview" + - name: AZURE_OPENAI_API_VERSION + value: "2024-12-01-preview" + - name: AZURE_CHAT_DEPLOYMENT + value: "gpt-4.1-mini" + - name: AZURE_EMBEDDING_DEPLOYMENT + value: "text-embedding-3-large" + - name: POSTGRES_HOST + value: "postgres" + - name: POSTGRES_PORT + value: "5432" + - name: POSTGRES_USER + value: "postgres" + - name: POSTGRES_DB + value: "vectordb" + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-credentials + key: POSTGRES_PASSWORD + - name: QUERY_DELAY_SECONDS + value: "3" + - name: SPLUNK_AO_OTLP_ENDPOINT + value: "https://ingest..signalfx.com/v2/trace/otlp" + - name: OTEL_SERVICE_NAME + value: "healthcare-assistant-loadgen" + - name: OTEL_RESOURCE_ATTRIBUTES + value: "deployment.environment.name=agent-observability" + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "1Gi" + cpu: "500m" diff --git a/examples/agent/healthcare-assistant/k8s-seed-job.yaml b/examples/agent/healthcare-assistant/k8s-seed-job.yaml new file mode 100644 index 00000000..a8e67591 --- /dev/null +++ b/examples/agent/healthcare-assistant/k8s-seed-job.yaml @@ -0,0 +1,80 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: vectordb-seed + namespace: healthcare-assistant +spec: + backoffLimit: 2 + template: + spec: + restartPolicy: OnFailure + initContainers: + - name: wait-for-postgres + image: pgvector/pgvector:pg16 + command: + - sh + - -c + - | + until pg_isready -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER; do + echo "Waiting for postgres..."; sleep 2 + done + env: + - name: POSTGRES_HOST + value: "postgres" + - name: POSTGRES_PORT + value: "5432" + - name: POSTGRES_USER + value: "postgres" + containers: + - name: seed + image: ghcr.io/splunk/healthcare-assistant-loadgen:latest + imagePullPolicy: Always + command: ["python", "helpers/setup_vectordb.py", "hosted"] + env: + - name: POSTGRES_HOST + value: "postgres" + - name: POSTGRES_PORT + value: "5432" + - name: POSTGRES_USER + value: "postgres" + - name: POSTGRES_DB + value: "vectordb" + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-credentials + key: POSTGRES_PASSWORD + - name: AZURE_OPENAI_API_KEY + valueFrom: + secretKeyRef: + name: openai-secrets + key: api-key + - name: AZURE_OPENAI_ENDPOINT + value: "https://.cognitiveservices.azure.com/" + - name: AZURE_OPENAI_API_VERSION + value: "2024-12-01-preview" + - name: AZURE_CHAT_DEPLOYMENT + value: "gpt-4.1-mini" + - name: AZURE_EMBEDDING_DEPLOYMENT + value: "text-embedding-3-large" + - name: OPENAI_API_KEY + valueFrom: + secretKeyRef: + name: openai-secrets + key: api-key + - name: OPENAI_BASE_URL + valueFrom: + secretKeyRef: + name: openai-secrets + key: base-url + - name: OPENAI_API_VERSION + value: "2024-12-01-preview" + - name: ENVIRONMENT + value: "hosted" + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "1Gi" + cpu: "500m" diff --git a/examples/agent/healthcare-assistant/k8s.yaml b/examples/agent/healthcare-assistant/k8s.yaml new file mode 100644 index 00000000..98c9fb56 --- /dev/null +++ b/examples/agent/healthcare-assistant/k8s.yaml @@ -0,0 +1,107 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: healthcare-assistant + labels: + app: healthcare-assistant +spec: + replicas: 1 + selector: + matchLabels: + app: healthcare-assistant + template: + metadata: + labels: + app: healthcare-assistant + spec: + containers: + - name: healthcare-assistant + image: ghcr.io/splunk/healthcare-assistant:app-with-instrumentation + imagePullPolicy: Always + ports: + - containerPort: 8501 + name: http + envFrom: + - configMapRef: + name: healthcare-assistant-config + - configMapRef: + name: postgres-config + - configMapRef: + name: splunk-ao-config + env: + - name: OPENAI_BASE_URL + valueFrom: + secretKeyRef: + name: openai-api + key: openai-api-endpoint + - name: OPENAI_API_KEY + valueFrom: + secretKeyRef: + name: openai-api + key: openai-api-key + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-credentials + key: POSTGRES_PASSWORD + - name: SPLUNK_AO_REALM + valueFrom: + secretKeyRef: + name: splunk-ao-secret + key: realm + - name: SPLUNK_AO_O11Y_TOKEN + valueFrom: + secretKeyRef: + name: splunk-ao-secret + key: o11y-token + - name: SPLUNK_AO_O11Y_API_TOKEN + valueFrom: + secretKeyRef: + name: splunk-ao-secret + key: o11y-api-token + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "2Gi" + cpu: "1000m" + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + readOnlyRootFilesystem: false +--- +apiVersion: v1 +kind: Service +metadata: + name: healthcare-assistant-service +spec: + selector: + app: healthcare-assistant + ports: + - port: 8501 + protocol: TCP + type: ClusterIP +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: healthcare-assistant-ingress + annotations: + traefik.ingress.kubernetes.io/router.entrypoints: web +spec: + ingressClassName: traefik + rules: + - http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: healthcare-assistant-service + port: + number: 8501 diff --git a/examples/agent/healthcare-assistant/load_generator.py b/examples/agent/healthcare-assistant/load_generator.py new file mode 100644 index 00000000..5c7fff5e --- /dev/null +++ b/examples/agent/healthcare-assistant/load_generator.py @@ -0,0 +1,144 @@ +""" +Healthcare Assistant Load Generator + +Fires all questions from dataset.csv and hallucination examples from config.yaml +against the HealthcareAgent in a loop. Designed to run as a k8s CronJob. + +Each run: one pass through dataset.csv + one hallucination log per config entry. + +CRUD bypass: uses ingestion_hook=lambda x: None on SplunkAOAsyncCallback so all +session/trace creation goes via OTLP (no app.lab0.* CRUD calls needed). +""" +import asyncio +import csv +import logging +import os +import sys +import time +import uuid +from pathlib import Path + +from dotenv import load_dotenv + +load_dotenv(override=True) + +from langchain_core.messages import AIMessage, HumanMessage +from langgraph.graph.message import add_messages + +import config as cfg_mod +from agent import HealthcareAgent, State, _run_async +from helpers.hallucination_helpers import log_demo_hallucination +from splunk_ao import splunk_ao_context +from splunk_ao.handlers.langchain import SplunkAOAsyncCallback + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", + stream=sys.stdout, +) +log = logging.getLogger(__name__) + + +def _noop_ingestion_hook(traces_ingest_request): + """No-op hook — bypasses CRUD API, delivery handled by OTLP exporter.""" + pass + + +class LoadGenAgent(HealthcareAgent): + """HealthcareAgent variant that uses ingestion_hook to skip CRUD API calls.""" + + async def _process_query_async(self, messages): + if not self.tools: + self.load_tools() + self.graph = self._build_graph() + + langchain_messages = [] + for msg in messages: + if msg["role"] == "user": + langchain_messages.append(HumanMessage(content=msg["content"])) + elif msg["role"] == "assistant": + langchain_messages.append(AIMessage(content=msg["content"])) + + with splunk_ao_context( + project=os.getenv("SPLUNK_AO_PROJECT"), + agent_stream=os.getenv("SPLUNK_AO_AGENT_STREAM"), + ): + # ingestion_hook skips all CRUD (start_session, create_session, etc.) + # Spans are delivered via OTLP to private-ingest.lab0.signalfx.com + callback = SplunkAOAsyncCallback(ingestion_hook=_noop_ingestion_hook) + run_config = {**self.langgraph_config, "callbacks": [callback]} + + result = await self.graph.ainvoke( + {"messages": langchain_messages}, + run_config, + ) + if result["messages"]: + return result["messages"][-1].content + return "No response generated" + + def process_query(self, messages): + try: + return _run_async(self._process_query_async(messages)) + except Exception as e: + log.error("Error processing query: %s", e) + import traceback + traceback.print_exc() + return f"Error: {e}" + + +def load_questions() -> list[str]: + dataset = Path(__file__).parent / "dataset.csv" + with open(dataset, newline="") as f: + return [row["input"] for row in csv.DictReader(f) if row.get("input")] + + +async def run_pass(session_id: str, questions: list[str], config: dict) -> None: + agent = LoadGenAgent(session_id=session_id) + agent.load_tools() + + for i, question in enumerate(questions, 1): + log.info("[%d/%d] session=%s query=%s", i, len(questions), session_id, question[:80]) + try: + result = await agent._process_query_async([{"role": "user", "content": question}]) + log.info(" → %s", str(result)[:120]) + except Exception as e: + import traceback + log.error(" query failed: %s\n%s", e, traceback.format_exc()) + time.sleep(float(os.getenv("QUERY_DELAY_SECONDS", "2"))) + + # Log hallucinations — uses SplunkAOLogger directly (also needs CRUD bypass) + # For k8s, hallucination logging is best-effort; failures are logged but not fatal + hallucinations = config.get("demo_hallucinations", []) + for idx in range(len(hallucinations)): + log.info("Logging hallucination index=%d", idx) + try: + success = log_demo_hallucination( + config=config, + hallucination_index=idx, + existing_logger=None, + session_id=session_id, + ) + log.info(" hallucination logged: %s", success) + except Exception as e: + log.error(" hallucination failed (non-fatal): %s", e) + + +def main() -> None: + config = cfg_mod.load_config() + questions = load_questions() + session_id = os.getenv("LOAD_GEN_SESSION_ID") or f"loadgen-{uuid.uuid4().hex[:8]}" + + log.info( + "Load generator starting — project=%s stream=%s session=%s questions=%d", + os.getenv("SPLUNK_AO_PROJECT") or os.getenv("SPLUNK_AO_PROJECT_ID"), + os.getenv("SPLUNK_AO_AGENT_STREAM") or os.getenv("SPLUNK_AO_AGENT_STREAM_ID"), + session_id, + len(questions), + ) + + asyncio.run(run_pass(session_id, questions, config)) + log.info("Load generator complete — session=%s", session_id) + + +if __name__ == "__main__": + main() diff --git a/examples/agent/healthcare-assistant/load_generator_agent.py b/examples/agent/healthcare-assistant/load_generator_agent.py new file mode 100644 index 00000000..e1422564 --- /dev/null +++ b/examples/agent/healthcare-assistant/load_generator_agent.py @@ -0,0 +1,139 @@ +""" +Healthcare Assistant Load Generator — Mixed session variant + +One session, 3 traces: 2 valid agent queries + 1 hallucination. +K8s-friendly: reads all config from env vars (k8s secrets), no .env files. + +OTLP routing: SPLUNK_AO_OTLP_ENDPOINT must be set to +https://private-ingest..signalfx.com/v2/trace/otlp — +ingest..observability.splunkcloud.com is not routable from the cluster VPC. + +CRUD bypass: set_session() sets session ID locally, no app.* calls. +""" +import asyncio +import csv +import logging +import os +import sys +import time +import uuid +from pathlib import Path + +# Must patch before any splunk_ao import so O11yConfig picks up the override. +_splunk_ao_otlp_endpoint = os.getenv("SPLUNK_AO_OTLP_ENDPOINT") +if _splunk_ao_otlp_endpoint: + from splunk_ao.deployment import O11yConfig as _O11yConfig + _O11yConfig.otlp_endpoint = property(lambda self: _splunk_ao_otlp_endpoint) + +# Azure APIM requires api-version on every AsyncOpenAI request. +import openai as _openai_module +_orig_async_init = _openai_module.AsyncOpenAI.__init__ + +def _patched_async_init(self, *args, **kwargs): + dq = dict(kwargs.pop("default_query", None) or {}) + dq.setdefault("api-version", os.getenv("OPENAI_API_VERSION", "2024-12-01-preview")) + kwargs["default_query"] = dq + _orig_async_init(self, *args, **kwargs) + +_openai_module.AsyncOpenAI.__init__ = _patched_async_init + +from langchain_core.messages import AIMessage, HumanMessage + +import config as cfg_mod +from agent import HealthcareAgent, _run_async +from helpers.hallucination_helpers import log_demo_hallucination +from splunk_ao import SplunkAOLogger, splunk_ao_context +from splunk_ao.handlers.langchain import SplunkAOAsyncCallback + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", + stream=sys.stdout, +) +log = logging.getLogger(__name__) + +AGENT_QUESTIONS = [ + "Can you look up information for patient P001?", + "What is the dosage and common side effects of Lisinopril?", +] + + +class LoadGenAgent(HealthcareAgent): + """HealthcareAgent variant that passes a pre-configured logger to avoid CRUD.""" + + async def _process_query_async(self, messages, logger: SplunkAOLogger): + if not self.graph: + self.graph = self._build_graph() + + langchain_messages = [] + for msg in messages: + if msg["role"] == "user": + langchain_messages.append(HumanMessage(content=msg["content"])) + elif msg["role"] == "assistant": + langchain_messages.append(AIMessage(content=msg["content"])) + + callback = SplunkAOAsyncCallback(splunk_ao_logger=logger) + run_config = {**self.langgraph_config, "callbacks": [callback]} + result = await self.graph.ainvoke({"messages": langchain_messages}, run_config) + + if result["messages"]: + return result["messages"][-1].content + return "No response generated" + + +async def main() -> None: + project = os.getenv("SPLUNK_AO_PROJECT", "galileo-demo-healthcare") + agent_stream = os.getenv("SPLUNK_AO_AGENT_STREAM", "default") + session_id = f"loadgen-{uuid.uuid4().hex[:8]}" + query_delay = float(os.getenv("QUERY_DELAY_SECONDS", "2")) + config = cfg_mod.load_config() + + log.info( + "Load generator starting — project=%s stream=%s session=%s", + project, agent_stream, session_id, + ) + + # One splunk_ao_context for the entire run — one OTLP exporter lifetime. + with splunk_ao_context(project=project, agent_stream=agent_stream): + logger = SplunkAOLogger(project=project, agent_stream=agent_stream) + logger.set_session(session_id) + + agent = LoadGenAgent(session_id=session_id) + agent.load_tools() + agent.graph = agent._build_graph() + + # 2 valid agent traces + for i, question in enumerate(AGENT_QUESTIONS, 1): + log.info("[%d/%d] agent query=%s", i, len(AGENT_QUESTIONS), question[:80]) + try: + result = await agent._process_query_async( + [{"role": "user", "content": question}], + logger=logger, + ) + log.info(" → %s", str(result)[:120]) + except Exception as e: + import traceback + log.error(" agent query failed: %s\n%s", e, traceback.format_exc()) + time.sleep(query_delay) + + # 1 hallucinated trace in the same session + log.info("[3/3] logging hallucination") + try: + hall_logger = SplunkAOLogger(project=project, agent_stream=agent_stream) + hall_logger.set_session(session_id) + success = log_demo_hallucination( + config=config, + hallucination_index=0, + existing_logger=hall_logger, + session_id=session_id, + ) + log.info(" hallucination logged: %s", success) + except Exception as e: + import traceback + log.error(" hallucination failed (non-fatal): %s\n%s", e, traceback.format_exc()) + + log.info("Load generator complete — session=%s", session_id) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/agent/healthcare-assistant/load_generator_hallucination.py b/examples/agent/healthcare-assistant/load_generator_hallucination.py new file mode 100644 index 00000000..f3afb149 --- /dev/null +++ b/examples/agent/healthcare-assistant/load_generator_hallucination.py @@ -0,0 +1,76 @@ +""" +Healthcare Assistant Load Generator — Hallucination only + +K8s-friendly. Reads ALL config from environment variables (k8s secrets). +No .env file loading. Logs demo hallucinations from config.yaml via SplunkAOLogger. + +OTLP endpoint override: SPLUNK_AO_OTLP_ENDPOINT must be set to +https://private-ingest..signalfx.com/v2/trace/otlp — the realm-derived +ingest..observability.splunkcloud.com is not routable from the cluster VPC. + +CRUD bypass: set_session() sets the session ID locally without calling app.* CRUD. +""" +import asyncio +import logging +import os +import sys +import uuid + +# The SDK derives the OTLP endpoint from realm as ingest..observability.splunkcloud.com +# which is not routable from the cluster. Override to private-ingest..signalfx.com +# which resolves to private IPs and is reachable from the cluster VPC. +_splunk_ao_otlp_endpoint = os.getenv("SPLUNK_AO_OTLP_ENDPOINT") +if _splunk_ao_otlp_endpoint: + from splunk_ao.deployment import O11yConfig as _O11yConfig + _O11yConfig.otlp_endpoint = property(lambda self: _splunk_ao_otlp_endpoint) + +import config as cfg_mod +from helpers.hallucination_helpers import log_demo_hallucination +from splunk_ao import SplunkAOLogger + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", + stream=sys.stdout, +) +log = logging.getLogger(__name__) + + +async def main() -> None: + config = cfg_mod.load_config() + session_id = f"loadgen-hallucination-{uuid.uuid4().hex[:8]}" + project = os.getenv("SPLUNK_AO_PROJECT", "galileo-demo-healthcare") + agent_stream = os.getenv("SPLUNK_AO_AGENT_STREAM", "default") + + log.info( + "Load generator starting — project=%s stream=%s session=%s", + project, + agent_stream, + session_id, + ) + + # Step 2: log hallucinations — best-effort, non-fatal + # Use set_session() instead of start_session() to avoid a CRUD call to app.* + # (app.lab0.observability.splunkcloud.com is not routable from the cluster VPC). + hallucinations = config.get("demo_hallucinations", []) + for idx in range(len(hallucinations)): + log.info("[Step 2] Logging hallucination index=%d", idx) + try: + hall_logger = SplunkAOLogger(project=project, agent_stream=agent_stream) + hall_logger.set_session(session_id) + success = log_demo_hallucination( + config=config, + hallucination_index=idx, + existing_logger=hall_logger, + session_id=session_id, + ) + log.info(" hallucination logged: %s", success) + except Exception as e: + import traceback + log.error(" hallucination failed (non-fatal): %s\n%s", e, traceback.format_exc()) + + log.info("Load generator complete — session=%s", session_id) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/agent/healthcare-assistant/postgres.yaml b/examples/agent/healthcare-assistant/postgres.yaml new file mode 100644 index 00000000..44bfe81c --- /dev/null +++ b/examples/agent/healthcare-assistant/postgres.yaml @@ -0,0 +1,102 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgres-config +data: + POSTGRES_HOST: "postgres" + POSTGRES_PORT: "5432" + POSTGRES_USER: "postgres" + POSTGRES_DB: "vectordb" +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgres-credentials +type: Opaque +stringData: + POSTGRES_PASSWORD: "mypassword" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgres-init +data: + init.sql: | + CREATE EXTENSION IF NOT EXISTS vector; +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: postgres-pvc +spec: + accessModes: + - ReadWriteOnce + storageClassName: gp2 + resources: + requests: + storage: 5Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres + labels: + app: postgres +spec: + replicas: 1 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - name: postgres + image: pgvector/pgvector:pg16 + ports: + - containerPort: 5432 + name: postgres + envFrom: + - configMapRef: + name: postgres-config + env: + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-credentials + key: POSTGRES_PASSWORD + volumeMounts: + - name: postgres-data + mountPath: /var/lib/postgresql/data + subPath: pgdata + - name: postgres-init + mountPath: /docker-entrypoint-initdb.d + resources: + requests: + memory: "256Mi" + cpu: "250m" + limits: + memory: "1Gi" + cpu: "500m" + volumes: + - name: postgres-data + persistentVolumeClaim: + claimName: postgres-pvc + - name: postgres-init + configMap: + name: postgres-init +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres +spec: + selector: + app: postgres + ports: + - port: 5432 + protocol: TCP + targetPort: 5432 + type: ClusterIP diff --git a/examples/agent/healthcare-assistant/pyproject.toml b/examples/agent/healthcare-assistant/pyproject.toml new file mode 100644 index 00000000..28e79cb0 --- /dev/null +++ b/examples/agent/healthcare-assistant/pyproject.toml @@ -0,0 +1,35 @@ +[project] +name = "healthcare-assistant" +version = "0.1.0" +description = "Healthcare assistant demo app — uses local splunk-ao source for debugging" +requires-python = ">=3.11,<3.15" +dependencies = [ + "streamlit", + "openai", + "python-dotenv", + "langchain", + "langchain-core", + "langchain-openai", + "langgraph", + "langchain-postgres", + "psycopg[binary]", + "langchain-text-splitters", + "langchain-community", + "langchain-classic", + "pyyaml", + "toml", + "pandas", + "splunk-ao", +] + +[tool.uv] +# Use local splunk-ao source instead of PyPI +[tool.uv.sources] +splunk-ao = { path = "../../..", editable = true } + +[tool.hatch.build.targets.wheel] +packages = ["."] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" diff --git a/examples/agent/healthcare-assistant/rag.py b/examples/agent/healthcare-assistant/rag.py new file mode 100644 index 00000000..b2ebd923 --- /dev/null +++ b/examples/agent/healthcare-assistant/rag.py @@ -0,0 +1,127 @@ +"""RAG retrieval for the healthcare assistant using PostgreSQL/pgvector.""" +import asyncio +import os +from typing import Optional + +from langchain_classic.chains import create_retrieval_chain +from langchain_classic.chains.combine_documents import create_stuff_documents_chain +from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder +from langchain_core.tools import tool +from config import DOMAIN, load_config, create_chat_llm, create_embeddings +from helpers.pgvector_utils import collection_exists, create_pgvector_store +from setup_env import setup_environment + +_rag_cache = {} + + +class HealthcareRAGSystem: + """RAG system with eager initialization.""" + + def __init__(self, top_k: int = 5, model_name: Optional[str] = None): + self.top_k = top_k + self.model_name = model_name + self.retrieval_chain = None + self._initialized = False + self.initialize() + + def initialize(self): + if self._initialized: + return + + try: + app_config = load_config() + rag_config = app_config.get("rag", {}) + vectorstore_config = app_config.get("vectorstore", {}) + model_config = app_config.get("model", {}) + + embedding_model = vectorstore_config.get("embedding_model", "text-embedding-3-large") + llm_model = ( + self.model_name + or model_config.get("default_model") + or model_config.get("model_name", "gpt-4o") + ) + + setup_environment() + environment = os.environ.get("ENVIRONMENT", "local") + + if not os.environ.get("POSTGRES_PASSWORD"): + raise ValueError( + "POSTGRES_PASSWORD not found. Please add it to .streamlit/secrets.toml" + ) + + if not collection_exists(DOMAIN, environment): + collection_name = f"{DOMAIN}_{environment}_index" + raise ValueError( + f"PostgreSQL collection not found: {collection_name}. " + f"Please run: python helpers/setup_vectordb.py {environment}" + ) + + embeddings = create_embeddings(model=embedding_model) + vector_store, _ = create_pgvector_store(embeddings, DOMAIN, environment) + retriever = vector_store.as_retriever(search_kwargs={"k": self.top_k}) + + llm = create_chat_llm(model=llm_model, temperature=0.1, name="Healthcare RAG Assistant") + + retrieval_qa_chat_prompt = ChatPromptTemplate.from_messages( + [ + ( + "system", + "Answer any use questions based solely on the context below:\n\n" + "\n{context}\n", + ), + MessagesPlaceholder("chat_history", optional=True), + ("human", "{input}"), + ] + ) + combine_docs_chain = create_stuff_documents_chain(llm, retrieval_qa_chat_prompt) + self.retrieval_chain = create_retrieval_chain(retriever, combine_docs_chain) + self._initialized = True + print(f"✅ RAG system initialized (model: {llm_model})") + except Exception as e: + print(f"❌ Error initializing RAG system: {e}") + import traceback + + traceback.print_exc() + self._initialized = False + + async def search(self, query: str) -> str: + if not self.retrieval_chain: + return ( + "❌ RAG system not initialized. " + "Please check your vector database setup." + ) + + try: + result = await asyncio.to_thread( + self.retrieval_chain.invoke, {"input": query} + ) + return result["answer"] + except Exception as e: + return f"❌ Error during RAG search: {str(e)}" + + +def get_rag_system(top_k: int | None = None, model_name: Optional[str] = None) -> HealthcareRAGSystem: + if top_k is None: + app_config = load_config() + top_k = app_config.get("rag", {}).get("top_k", 5) + + cache_key = f"{top_k}_{model_name or 'default'}" + if cache_key not in _rag_cache: + _rag_cache[cache_key] = HealthcareRAGSystem(top_k, model_name=model_name) + return _rag_cache[cache_key] + + +def create_rag_tool(top_k: int | None = None, model_name: Optional[str] = None): + """Create a LangChain retrieval chain tool for the agent.""" + rag_system = get_rag_system(top_k, model_name=model_name) + + @tool + async def retrieve_healthcare_documents(query: str) -> str: + """Retrieve information related to a query from the healthcare knowledge base.""" + return await rag_system.search(query) + + retrieve_healthcare_documents.name = "retrieve_healthcare_documents" + retrieve_healthcare_documents.description = ( + "Retrieve information from the healthcare knowledge base" + ) + return retrieve_healthcare_documents diff --git a/examples/agent/healthcare-assistant/requirements.txt b/examples/agent/healthcare-assistant/requirements.txt new file mode 100644 index 00000000..c4fc51e9 --- /dev/null +++ b/examples/agent/healthcare-assistant/requirements.txt @@ -0,0 +1,16 @@ +streamlit +openai +python-dotenv +langchain +langchain-core +langchain-openai +langgraph +langchain-postgres +psycopg[binary] +langchain-text-splitters +langchain-community +langchain-classic +pyyaml +toml +pandas +splunk-ao diff --git a/examples/agent/healthcare-assistant/setup-job.yaml b/examples/agent/healthcare-assistant/setup-job.yaml new file mode 100644 index 00000000..a7ecd3d1 --- /dev/null +++ b/examples/agent/healthcare-assistant/setup-job.yaml @@ -0,0 +1,55 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: vectordb-setup +spec: + backoffLimit: 4 + template: + spec: + restartPolicy: OnFailure + initContainers: + - name: wait-for-postgres + image: pgvector/pgvector:pg16 + command: + - sh + - -c + - | + until pg_isready -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER; do + echo "Waiting for postgres..."; sleep 2 + done + envFrom: + - configMapRef: + name: postgres-config + containers: + - name: setup + image: ghcr.io/splunk/healthcare-assistant:base-app + imagePullPolicy: Always + command: ["python", "helpers/setup_vectordb.py", "hosted"] + envFrom: + - configMapRef: + name: postgres-config + - configMapRef: + name: healthcare-assistant-config + env: + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-credentials + key: POSTGRES_PASSWORD + - name: OPENAI_API_KEY + valueFrom: + secretKeyRef: + name: openai-api + key: openai-api-key + - name: OPENAI_BASE_URL + valueFrom: + secretKeyRef: + name: openai-api + key: openai-api-endpoint + resources: + requests: + memory: "256Mi" + cpu: "250m" + limits: + memory: "1Gi" + cpu: "500m" diff --git a/examples/agent/healthcare-assistant/setup_env.py b/examples/agent/healthcare-assistant/setup_env.py new file mode 100644 index 00000000..ef36a68b --- /dev/null +++ b/examples/agent/healthcare-assistant/setup_env.py @@ -0,0 +1,26 @@ +"""Validate required environment variables are set.""" +import os + +REQUIRED_ENV_VARS = [ + "POSTGRES_HOST", + "POSTGRES_PORT", + "POSTGRES_USER", + "POSTGRES_PASSWORD", + "POSTGRES_DB", + "ENVIRONMENT", + "SPLUNK_AO_REALM", + "SPLUNK_AO_O11Y_TOKEN", + "SPLUNK_AO_PROJECT", + "SPLUNK_AO_AGENT_STREAM", +] + +def setup_environment(): + missing = [var for var in REQUIRED_ENV_VARS if not os.getenv(var)] + for var in missing: + print(f"⚠️ {var} not set") + if not missing: + print("🔧 Environment setup complete") + + +if __name__ == "__main__": + setup_environment() diff --git a/examples/agent/healthcare-assistant/start_vectordb.sh b/examples/agent/healthcare-assistant/start_vectordb.sh new file mode 100644 index 00000000..5394750b --- /dev/null +++ b/examples/agent/healthcare-assistant/start_vectordb.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python helpers/setup_vectordb.py local diff --git a/examples/agent/healthcare-assistant/system_prompt.json b/examples/agent/healthcare-assistant/system_prompt.json new file mode 100644 index 00000000..981d705f --- /dev/null +++ b/examples/agent/healthcare-assistant/system_prompt.json @@ -0,0 +1,3 @@ +{ + "system_prompt": "You are a knowledgeable call center assistant for an Online Healthcare system, supporting patients and internal staff.\n\nYou have three tools available:\n- search_medicine_qa: Use this to answer any questions about medicine — dosage, side effects, interactions, etc.\n- get_patient_info: Use this to look up a patient's details by their patient ID, including their name, address, phone number, patient type, and prescription.\n- delete_patient_record: Use this only when the user explicitly asks to delete or remove a patient record. Requires the patient ID.\n\nGuidelines:\n- For medicine questions, always call search_medicine_qa to retrieve accurate information before answering.\n- For patient lookups, call get_patient_info with the patient ID. Users can ask for information on all patients; that is acceptable since there are only 30 patients.\n- For delete requests, call delete_patient_record with the patient ID. Do not use get_patient_info for deletions.\n- Be professional, concise, and empathetic \n- If you cannot find an answer, say so clearly and offer to escalate to a senior support agent. Do no ever provide personal data from our doctors, including full name, phone number, address, etc. Sharing patient data is allowed, not doctor data. Refuse to provide any information about doctors or their personal data." +} diff --git a/examples/agent/healthcare-assistant/tools/__init__.py b/examples/agent/healthcare-assistant/tools/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/agent/healthcare-assistant/tools/logic.py b/examples/agent/healthcare-assistant/tools/logic.py new file mode 100644 index 00000000..78244576 --- /dev/null +++ b/examples/agent/healthcare-assistant/tools/logic.py @@ -0,0 +1,161 @@ +""" +Healthcare domain tools. + +- get_patient_info: Text-to-SQL lookup against the patient registry in PostgreSQL +- delete_patient_record: Text-to-SQL delete against the patient registry in PostgreSQL +- search_medicine_qa: semantic vector search against the QA knowledge base +""" +import json +import logging +from typing import Optional, Tuple + +from langchain_postgres import PGVector + +from config import DOMAIN, load_config +from helpers.pgvector_utils import get_pgvector_store +from helpers.sql_utils import execute_sql, relational_table_name +from helpers.text_to_sql_utils import generate_sql +from rag import get_rag_system + +_TABLE_SUFFIX = "patient" +_ID_COLUMN = "patient_id" + +_vector_store: Optional[PGVector] = None +_embedding_model: Optional[str] = None +_collection_name_cached: Optional[str] = None + + +def _get_vector_store() -> Tuple[PGVector, str]: + global _vector_store, _embedding_model, _collection_name_cached + + app_config = load_config() + embedding_model = ( + app_config.get("vectorstore", {}).get("embedding_model") or "text-embedding-3-large" + ) + + if ( + _vector_store is not None + and _collection_name_cached is not None + and _embedding_model == embedding_model + ): + return _vector_store, _collection_name_cached + + _vector_store, collection_name = get_pgvector_store(DOMAIN, embedding_model) + _embedding_model = embedding_model + _collection_name_cached = collection_name + return _vector_store, collection_name + + +async def _execute_patient_sql(sql: str) -> str: + """Execute a SQL lookup against the patient registry.""" + try: + result = execute_sql(sql) + return json.dumps(result) + except Exception as e: + return json.dumps({"error": str(e), "sql": sql}) + + +async def _execute_patient_delete_sql(sql: str) -> str: + """Execute a SQL delete against the patient registry.""" + try: + result = execute_sql(sql) + return json.dumps(result) + except Exception as e: + return json.dumps({"error": str(e), "sql": sql}) + + +async def get_patient_info(patient_id: str) -> str: + """Retrieve patient information by their patient ID.""" + patient_id = patient_id.strip().upper() + q = (patient_id or "").strip() + if not q: + return json.dumps({"error": "patient_id is required"}) + + app_config = load_config() + model = app_config.get("model", {}).get("default_model", "gpt-4o-mini") + table_name = relational_table_name(DOMAIN, _TABLE_SUFFIX) + + try: + sql = await generate_sql( + domain_name=DOMAIN, + table_suffix=_TABLE_SUFFIX, + id_column=_ID_COLUMN, + record_id=q, + operation="select", + model=model, + use_case_identifier="patient_id", + use_case_value=patient_id, + ) + except Exception as e: + return json.dumps({"error": str(e), "patient_id": q}) + + raw = await _execute_patient_sql(sql) + try: + result = json.loads(raw) + except json.JSONDecodeError: + result = {"error": "Invalid SQL execution response", "raw": raw} + + if "error" not in result: + result["query"] = q + result["table"] = table_name + + return json.dumps(result) + + +async def delete_patient_record(patient_id: str) -> str: + """Permanently delete a patient record from the registry by patient ID.""" + patient_id = patient_id.strip().upper() + q = (patient_id or "").strip() + if not q: + return json.dumps({"error": "patient_id is required"}) + + app_config = load_config() + model = app_config.get("model", {}).get("default_model", "gpt-4o-mini") + table_name = relational_table_name(DOMAIN, _TABLE_SUFFIX) + + try: + sql = await generate_sql( + domain_name=DOMAIN, + table_suffix=_TABLE_SUFFIX, + id_column=_ID_COLUMN, + record_id=q, + operation="delete", + model=model, + use_case_identifier="patient_id", + use_case_value=patient_id, + ) + except Exception as e: + return json.dumps({"error": str(e), "patient_id": q}) + + raw = await _execute_patient_delete_sql(sql) + try: + result = json.loads(raw) + except json.JSONDecodeError: + result = {"error": "Invalid SQL execution response", "raw": raw} + + if "error" not in result: + result["query"] = q + result["table"] = table_name + + return json.dumps(result) + + +async def search_medicine_qa(query: str) -> str: + """Search the Medicine knowledge base using semantic vector search.""" + q = query + try: + _get_vector_store() + except Exception as e: + return json.dumps({"error": str(e), "query": q}) + + try: + rag_system = get_rag_system(top_k=1) + raw = await rag_system.search(q) + except Exception as e: + logging.exception("search_medicine_qa search failed") + return json.dumps({"error": str(e), "query": q}) + + return json.dumps([raw]) + + +TOOLS = [get_patient_info, delete_patient_record, search_medicine_qa] diff --git a/examples/agent/healthcare-assistant/tools/schema.json b/examples/agent/healthcare-assistant/tools/schema.json new file mode 100644 index 00000000..bad8d897 --- /dev/null +++ b/examples/agent/healthcare-assistant/tools/schema.json @@ -0,0 +1,44 @@ +[ + { + "name": "get_patient_info", + "description": "Retrieve patient's details by their ID, including name, address, phone number, patient type, and prescription", + "parameters": { + "type": "object", + "properties": { + "patient_id": { + "type": "string", + "description": "The patient's unique identifier (e.g., 'P001', 'P005')" + } + }, + "required": ["patient_id"] + } + }, + { + "name": "delete_patient_record", + "description": "Permanently delete a patient record from the registry by their patient ID", + "parameters": { + "type": "object", + "properties": { + "patient_id": { + "type": "string", + "description": "The patient's unique identifier to delete (e.g., 'P001', 'P005')" + } + }, + "required": ["patient_id"] + } + }, + { + "name": "search_medicine_qa", + "description": "Search the Medicine knowledge base to answer questions about medications, including dosage, side effects, and interactions", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "The question or topic to search for (e.g., 'Lisinopril', 'Metformin', 'Aspirin')" + } + }, + "required": ["query"] + } + } + ] diff --git a/src/splunk_ao/converter/attribute_mapping.py b/src/splunk_ao/converter/attribute_mapping.py index 64b782d3..25035f01 100644 --- a/src/splunk_ao/converter/attribute_mapping.py +++ b/src/splunk_ao/converter/attribute_mapping.py @@ -445,8 +445,13 @@ def _set_orchestration_content(attrs: MutableMapping[str, AttributeValue], span: output_messages, full_history = _orchestration_messages(span.output, "assistant") if output_messages is None: return - if full_history and input_messages is not None and output_messages[: len(input_messages)] == input_messages: - output_messages = output_messages[len(input_messages) :] + # Reduction is intentionally tied to a confirmed prefix match: we only trim when the + # output starts with an exact copy of the input, which is the LangGraph stateless + # (no checkpointer) full-history shape. With a checkpointer the input is the new turn + # only while the output carries the whole persisted thread, so the prefix never matches + # and no reduction fires — known limitation, tracked separately. + if full_history and input_messages and output_messages[: len(input_messages)] == input_messages: + output_messages = output_messages[len(input_messages) :][-1:] attrs["gen_ai.output.messages"] = _json_string(_with_finish_reasons(output_messages)) diff --git a/tests/test_attribute_mapping.py b/tests/test_attribute_mapping.py index 31571bfa..4c19098a 100644 --- a/tests/test_attribute_mapping.py +++ b/tests/test_attribute_mapping.py @@ -1,6 +1,6 @@ import json from types import SimpleNamespace -from typing import cast +from typing import Any, cast from uuid import uuid4 import pytest @@ -418,6 +418,290 @@ def test_orchestration_output_omits_repeated_input_history() -> None: ] +@pytest.mark.parametrize("span_type", [WorkflowSpan, AgentSpan]) +def test_orchestration_full_history_keeps_last_terminal_message( + span_type: type[WorkflowSpan] | type[AgentSpan], +) -> None: + user = {"role": "user", "content": "Give me two alternatives"} + first = {"role": "assistant", "content": "First alternative"} + second = {"role": "assistant", "content": "Second alternative"} + span_kwargs: dict[str, Any] = { + "name": "planner", + "input": json.dumps({"messages": [user]}), + "output": json.dumps({"messages": [user, first, second]}), + } + if span_type is AgentSpan: + span_kwargs["agent_type"] = AgentType.planner + + attrs = build_span_attributes(span_type(**span_kwargs)) + + assert json.loads(attrs["gen_ai.output.messages"]) == [ + _text_message("assistant", "Second alternative", finish_reason="unknown") + ] + + +@pytest.mark.parametrize("span_type", [WorkflowSpan, AgentSpan]) +def test_orchestration_full_history_removes_confirmed_input_prefix( + span_type: type[WorkflowSpan] | type[AgentSpan], +) -> None: + # Given: the input is the complete history immediately before the final assistant response. + user = {"role": "user", "content": "What is the dosage?"} + tool_call = { + "role": "assistant", + "content": "", + "tool_calls": [{"id": "call-1", "function": {"name": "search", "arguments": '{"query":"dosage"}'}}], + } + tool_response = {"role": "tool", "content": "10 mg daily", "tool_call_id": "call-1"} + final = {"role": "assistant", "content": "The common dosage is 10 mg daily."} + input_history = [user, tool_call, tool_response] + span_kwargs: dict[str, Any] = { + "name": "healthcare", + "input": json.dumps({"messages": input_history}), + "output": json.dumps({"messages": [*input_history, final]}), + } + if span_type is AgentSpan: + span_kwargs["agent_type"] = AgentType.default + + # When: the orchestration content is converted. + attrs = build_span_attributes(span_type(**span_kwargs)) + + # Then: only the newly produced terminal response is exported as output. + assert json.loads(attrs["gen_ai.output.messages"]) == [ + _text_message("assistant", "The common dosage is 10 mg daily.", finish_reason="unknown") + ] + + +def test_orchestration_tool_call_assistant_message_finish_reason_unknown() -> None: + # Given: a workflow emits an assistant tool call without a source finish reason. + output = { + "update": { + "messages": [ + { + "role": "assistant", + "content": "", + "tool_calls": [{"id": "call-1", "function": {"name": "search", "arguments": '{"query":"dosage"}'}}], + } + ] + } + } + + # When: the workflow output is converted. + attrs = build_span_attributes(WorkflowSpan(name="tools", output=json.dumps(output))) + + # Then: finish_reason defaults to "unknown" — no inference from parts. + output_message = json.loads(attrs["gen_ai.output.messages"])[0] + assert output_message["finish_reason"] == "unknown" + assert output_message["parts"][0]["type"] == "tool_call" + + +def test_orchestration_tool_response_uses_unknown_finish_reason() -> None: + # Given: a workflow emits a tool response, which has no model-generation finish reason. + output = { + "update": {"messages": [{"role": "tool", "content": {"dosage": "10 mg daily"}, "tool_call_id": "call-1"}]} + } + + # When: the workflow output is converted. + attrs = build_span_attributes(WorkflowSpan(name="tools", output=json.dumps(output))) + + # Then: its valid tool response structure is preserved without inventing a model finish reason. + output_message = json.loads(attrs["gen_ai.output.messages"])[0] + assert output_message["finish_reason"] == "unknown" + assert output_message["parts"] == [ + {"type": "tool_call_response", "id": "call-1", "response": {"dosage": "10 mg daily"}} + ] + + +def test_orchestration_preserves_explicit_finish_reason_for_tool_call() -> None: + # Given: the source supplies its own finish reason for a message containing a tool call. + output = { + "update": { + "messages": [ + { + "role": "assistant", + "content": "", + "finish_reason": "provider_tool_calls", + "tool_calls": [{"id": "call-1", "function": {"name": "search", "arguments": '{"query":"dosage"}'}}], + } + ] + } + } + + # When: the workflow output is converted. + attrs = build_span_attributes(WorkflowSpan(name="tools", output=json.dumps(output))) + + # Then: inference does not overwrite source telemetry. + assert json.loads(attrs["gen_ai.output.messages"])[0]["finish_reason"] == "provider_tool_calls" + + +def test_orchestration_full_history_with_tool_call_keeps_last_message() -> None: + # LangGraph accumulated state: user → tool-call AI (empty content) → tool response → final AI + # The first post-dedup message has empty content; the UI would show "—" without the fix. + user = {"role": "user", "content": "What is the dosage of Lisinopril?"} + ai_toolcall = { + "role": "assistant", + "content": "", + "tool_calls": [{"id": "tc1", "function": {"name": "rag_search", "arguments": '{"query":"Lisinopril dosage"}'}}], + } + tool_resp = {"role": "tool", "content": "Lisinopril: 10mg daily", "tool_call_id": "tc1"} + ai_final = {"role": "assistant", "content": "Common dosage is 10mg once daily."} + + span = AgentSpan( + name="Agent", + agent_type=AgentType.default, + input=json.dumps({"messages": [user]}), + output=json.dumps({"messages": [user, ai_toolcall, tool_resp, ai_final]}), + ) + + attrs = build_span_attributes(span) + + output_messages = json.loads(attrs["gen_ai.output.messages"]) + assert len(output_messages) == 1 + assert output_messages[0]["role"] == "assistant" + assert output_messages[0]["parts"][0]["content"] == "Common dosage is 10mg once daily." + + +def test_orchestration_full_history_multi_turn_keeps_last_message() -> None: + # Multi-turn: output contains the full conversation history after multiple exchanges. + # Only the last message should be kept regardless of role. + user1 = {"role": "user", "content": "Hello"} + ai1 = {"role": "assistant", "content": "Hi, how can I help?"} + user2 = {"role": "user", "content": "What is Lisinopril?"} + ai2 = {"role": "assistant", "content": "Lisinopril is a blood pressure medication."} + + span = AgentSpan( + name="Agent", + agent_type=AgentType.default, + input=json.dumps({"messages": [user1]}), + output=json.dumps({"messages": [user1, ai1, user2, ai2]}), + ) + + attrs = build_span_attributes(span) + + output_messages = json.loads(attrs["gen_ai.output.messages"]) + assert len(output_messages) == 1 + assert output_messages[0]["parts"][0]["content"] == "Lisinopril is a blood pressure medication." + + +def test_orchestration_full_history_multiple_tool_rounds_keeps_last_message() -> None: + # Two tool call rounds before the final answer — last message is still the only output. + user = {"role": "user", "content": "Compare Lisinopril and Amlodipine"} + tc1_ai = { + "role": "assistant", + "content": "", + "tool_calls": [{"id": "tc1", "function": {"name": "search", "arguments": '{"query":"Lisinopril"}'}}], + } + tc1_resp = {"role": "tool", "content": "Lisinopril: ACE inhibitor", "tool_call_id": "tc1"} + tc2_ai = { + "role": "assistant", + "content": "", + "tool_calls": [{"id": "tc2", "function": {"name": "search", "arguments": '{"query":"Amlodipine"}'}}], + } + tc2_resp = {"role": "tool", "content": "Amlodipine: calcium channel blocker", "tool_call_id": "tc2"} + ai_final = { + "role": "assistant", + "content": "Lisinopril is an ACE inhibitor; Amlodipine is a calcium channel blocker.", + } + + span = AgentSpan( + name="Agent", + agent_type=AgentType.default, + input=json.dumps({"messages": [user]}), + output=json.dumps({"messages": [user, tc1_ai, tc1_resp, tc2_ai, tc2_resp, ai_final]}), + ) + + attrs = build_span_attributes(span) + + output_messages = json.loads(attrs["gen_ai.output.messages"]) + assert len(output_messages) == 1 + assert "Amlodipine" in output_messages[0]["parts"][0]["content"] + + +def test_orchestration_message_container_without_input_prefix_match_not_reduced() -> None: + # A WorkflowSpan (e.g. ToolNode) returning multiple messages whose output does NOT + # prefix-match the input state — dedup gate never fires, so all messages must survive. + # This is the parallel-tool-call shape: two ToolMessages from a single ToolNode invocation. + tool_msg_1 = {"role": "tool", "content": "Lisinopril: 10 mg daily", "tool_call_id": "tc1"} + tool_msg_2 = {"role": "tool", "content": "Amlodipine: 5 mg daily", "tool_call_id": "tc2"} + span = WorkflowSpan( + name="tools", + input=json.dumps({"messages": [{"role": "user", "content": "Compare dosages"}]}), + output=json.dumps({"messages": [tool_msg_1, tool_msg_2]}), + ) + + attrs = build_span_attributes(span) + + output_messages = json.loads(attrs["gen_ai.output.messages"]) + assert len(output_messages) == 2 + assert output_messages[0]["parts"][0]["response"] == "Lisinopril: 10 mg daily" + assert output_messages[1]["parts"][0]["response"] == "Amlodipine: 5 mg daily" + + +def test_orchestration_full_history_ends_on_tool_message_keeps_last() -> None: + # return_direct=True tool: run ends on a tool response, no final assistant message. + # The dedup gate fires (prefix matches) but the last message is a tool, not assistant. + # Must return the tool message rather than an empty list. + user = {"role": "user", "content": "Get patient P001"} + ai_toolcall = { + "role": "assistant", + "content": "", + "tool_calls": [{"id": "tc1", "function": {"name": "get_patient", "arguments": '{"id":"P001"}'}}], + } + tool_resp = {"role": "tool", "content": "George Rivera, Lisinopril 10mg", "tool_call_id": "tc1"} + span = AgentSpan( + name="Agent", + agent_type=AgentType.default, + input=json.dumps({"messages": [user, ai_toolcall]}), + output=json.dumps({"messages": [user, ai_toolcall, tool_resp]}), + ) + + attrs = build_span_attributes(span) + + output_messages = json.loads(attrs["gen_ai.output.messages"]) + assert len(output_messages) == 1 + assert output_messages[0]["parts"][0]["response"] == "George Rivera, Lisinopril 10mg" + + +def test_orchestration_full_history_ends_on_tool_call_ai_message_keeps_last() -> None: + # interrupt_before=["tools"]: run ends on a tool-call AIMessage with empty content. + # The last message is assistant role but content="" — must not return empty list. + user = {"role": "user", "content": "Search for Lisinopril"} + ai_toolcall = { + "role": "assistant", + "content": "", + "tool_calls": [{"id": "tc1", "function": {"name": "search", "arguments": '{"query":"Lisinopril"}'}}], + } + span = AgentSpan( + name="Agent", + agent_type=AgentType.default, + input=json.dumps({"messages": [user]}), + output=json.dumps({"messages": [user, ai_toolcall]}), + ) + + attrs = build_span_attributes(span) + + output_messages = json.loads(attrs["gen_ai.output.messages"]) + assert len(output_messages) == 1 + assert output_messages[0]["role"] == "assistant" + assert output_messages[0]["finish_reason"] == "unknown" + + +def test_orchestration_full_history_workflow_span_trim() -> None: + # WorkflowSpan (non-root LangGraph node) that carries full state: the trim + # must fire the same way it does for AgentSpan when the prefix matches. + user = {"role": "user", "content": "What is Lisinopril?"} + assistant = {"role": "assistant", "content": "Lisinopril is an ACE inhibitor."} + span = WorkflowSpan( + name="summarise", input=json.dumps({"messages": [user]}), output=json.dumps({"messages": [user, assistant]}) + ) + + attrs = build_span_attributes(span) + + output_messages = json.loads(attrs["gen_ai.output.messages"]) + assert len(output_messages) == 1 + assert output_messages[0]["role"] == "assistant" + assert output_messages[0]["parts"][0]["content"] == "Lisinopril is an ACE inhibitor." + + def test_orchestration_preserves_schema_valid_parts_and_tool_calls() -> None: span = WorkflowSpan( name="tool-workflow",