feat(examples): add healthcare-assistant demo app

CHANGELOG.md

@@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+
+- Agent and workflow output conversion now removes confirmed repeated input
+  history and keeps only the last message as the terminal output; intermediate
+  tool-call and tool-response messages are no longer included in
+  `gen_ai.output.messages` for full-history spans.
+
 ## [0.2.1] - 2026-08-07
 
 ### Fixed

examples/agent/healthcare-assistant/.env.example

@@ -0,0 +1,37 @@
+# LLM provider — set ONE of the two blocks below
+
+# Option A: OpenAI
+# OPENAI_API_KEY=
+
+# Option B: Azure OpenAI
+AZURE_OPENAI_API_KEY=
+AZURE_OPENAI_ENDPOINT=        # e.g. https://your-resource.openai.azure.com/
+AZURE_OPENAI_API_VERSION=     # e.g. 2024-12-01-preview
+AZURE_CHAT_DEPLOYMENT=        # deployment name for chat, e.g. gpt-4o-mini
+AZURE_EMBEDDING_DEPLOYMENT=   # deployment name for embeddings, e.g. text-embedding-3-large
+
+# Splunk AO environment variables — set ONE of the two deployment blocks below
+
+# Option A: Splunk Observability (O11y) Cloud
+SPLUNK_AO_REALM=              # e.g. us0, eu0, lab0
+SPLUNK_AO_O11Y_TOKEN=         # O11y ingest token (required for telemetry)
+# SPLUNK_AO_O11Y_API_TOKEN=   # Optional: dedicated API token for CRUD operations
+
+# Option B: On-premises / standalone deployment
+# SPLUNK_AO_API_KEY=
+# SPLUNK_AO_CONSOLE_URL=      # e.g. https://console.yourcompany.com
+# SPLUNK_AO_API_ENDPOINT=     # Optional, only set for custom deployments
+
+# Routing (shared by both deployments)
+SPLUNK_AO_PROJECT=
+SPLUNK_AO_AGENT_STREAM=
+
+# PostgreSQL (pgvector)
+POSTGRES_HOST=localhost
+POSTGRES_PORT=5432
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=
+POSTGRES_DB=vectordb
+
+OTEL_SERVICE_NAME=healthcare-assistant
+OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=your-project-name

examples/agent/healthcare-assistant/.gitignore

@@ -0,0 +1,6 @@
+.env.*
+!.env.example
+.venv/
+__pycache__/
+*.pyc
+.streamlit/secrets.toml

examples/agent/healthcare-assistant/.streamlit/secrets.toml.template

@@ -0,0 +1,23 @@
+# Splunk AO Configuration (O11y Cloud)
+# -----------------------------------------------------------------------------
+splunk_ao_realm = "us0"           # e.g. us0, eu0, lab0
+splunk_ao_o11y_token = "..."      # O11y ingest token
+# splunk_ao_o11y_api_token = "..."  # Optional: dedicated API token for CRUD operations
+splunk_ao_project = "..."
+splunk_ao_agent_stream = "..."
+
+# PostgreSQL Configuration (pgvector)
+# -----------------------------------------------------------------------------
+# PostgreSQL with pgvector extension for vector storage.
+# See README for Docker setup instructions.
+postgres_host = "localhost"
+postgres_port = "5432"
+postgres_user = "postgres"
+postgres_password = "mypassword"
+postgres_db = "vectordb"
+
+# Environment Configuration
+# -----------------------------------------------------------------------------
+# Set to "local" for local development, "hosted" for production/deployed environments.
+# This determines which pgvector collection prefix is used for vector storage.
+environment = "local"

examples/agent/healthcare-assistant/Dockerfile

@@ -0,0 +1,41 @@
+# Multi-stage build for Healthcare Assistant
+FROM python:3.12-slim AS builder
+
+# Set working directory
+WORKDIR /app
+
+RUN pip install uv
+
+# Copy requirements and install dependencies
+COPY 2-app-with-instrumentation/requirements.txt /app/
+RUN uv pip install --system --no-cache -r requirements.txt
+
+# Final stage
+FROM python:3.12-slim
+
+WORKDIR /app
+
+# Install runtime dependencies
+RUN apt-get update && apt-get install -y \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy installed packages from builder
+COPY --from=builder /usr/local/lib/python3.12/site-packages /usr/local/lib/python3.12/site-packages
+COPY --from=builder /usr/local/bin /usr/local/bin
+
+# Copy application code and data
+COPY 2-app-with-instrumentation/ /app/
+COPY docs/ /app/docs/
+
+# Create non-root user
+RUN useradd --create-home --shell /bin/bash app && \
+    chown -R app:app /app
+
+USER app
+
+# Expose port for Streamlit
+EXPOSE 8501
+
+# Run the server
+CMD ["streamlit", "run", "app.py"]

examples/agent/healthcare-assistant/Dockerfile.loadgen

@@ -0,0 +1,20 @@
+FROM python:3.12-slim
+
+WORKDIR /app
+
+RUN pip install uv --no-cache-dir
+
+# Install dependencies from requirements.txt (same as app)
+COPY requirements.txt /app/
+RUN uv pip install --system --no-cache -r requirements.txt
+
+# Copy application code
+COPY . /app/
+
+# Create non-root user
+RUN useradd --create-home --shell /bin/bash app && \
+    chown -R app:app /app
+
+USER app
+
+CMD ["python", "load_generator_hallucination.py"]

examples/agent/healthcare-assistant/Dockerfile.loadgen-agent

@@ -0,0 +1,26 @@
+FROM python:3.12-slim
+
+WORKDIR /app
+
+RUN pip install uv --no-cache-dir
+
+# Install splunk-ao from local source (contains fixes not yet released to PyPI).
+# The build context is the repo root so we can copy the SDK source.
+COPY pyproject.toml poetry.lock* README.md /sdk/
+COPY src/ /sdk/src/
+RUN uv pip install --system --no-cache /sdk
+
+# Install example dependencies (splunk-ao already installed above, skip PyPI version).
+COPY examples/agent/healthcare-assistant/requirements.txt /app/
+RUN uv pip install --system --no-cache -r requirements.txt --override /dev/stdin <<'EOF'
+splunk-ao
+EOF
+
+COPY examples/agent/healthcare-assistant/ /app/
+
+RUN useradd --create-home --shell /bin/bash app && \
+    chown -R app:app /app
+
+USER app
+
+CMD ["python", "load_generator_agent.py"]

examples/agent/healthcare-assistant/agent-with-instrumentation.py

@@ -0,0 +1,167 @@
+"""LangGraph agent for the healthcare assistant."""
+import asyncio
+import inspect
+import json
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+from typing import Annotated, List, Dict, Optional, TypedDict
+
+from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage
+from langchain_core.tools import StructuredTool
+from langchain_openai import ChatOpenAI
+from langgraph.graph import START, StateGraph
+from langgraph.graph.state import CompiledStateGraph
+from langgraph.graph.message import add_messages
+from langgraph.prebuilt import ToolNode, tools_condition
+
+from config import TOOLS_DIR, load_config, load_system_prompt
+from rag import create_rag_tool
+from tools import logic as tools_logic
+
+import os
+from splunk_ao import splunk_ao_context
+from splunk_ao.handlers.langchain import SplunkAOAsyncCallback
+
+class State(TypedDict):
+    messages: Annotated[list, add_messages]
+
+
+def _run_async(coro):
+    """Run an async coroutine from sync code (e.g. Streamlit)."""
+    try:
+        asyncio.get_running_loop()
+    except RuntimeError:
+        return asyncio.run(coro)
+
+    with ThreadPoolExecutor(max_workers=1) as executor:
+        return executor.submit(asyncio.run, coro).result()
+
+
+class HealthcareAgent:
+    """LangGraph healthcare assistant."""
+
+    def __init__(
+        self,
+        session_id: str | None = None,
+        model_override: Optional[str] = None,
+    ):
+        self.config = load_config()
+        self.session_id = session_id or str(uuid.uuid4())
+        self.model_override = model_override
+        self.system_prompt = load_system_prompt()
+        self.tools = []
+        self.graph: CompiledStateGraph | None = None
+        self.langgraph_config = {"configurable": {"thread_id": self.session_id}}
+
+    def load_tools(self) -> None:
+        tool_schema_path = TOOLS_DIR / "schema.json"
+        with tool_schema_path.open(encoding="utf-8") as f:
+            tool_schema = json.load(f)
+
+        self.tools = []
+        for tool_func in tools_logic.TOOLS:
+            tool_schema_dict = next(
+                (schema for schema in tool_schema if schema.get("name") == tool_func.__name__),
+                None,
+            )
+            tool_kwargs = {
+                "name": tool_func.__name__,
+                "description": (
+                    tool_schema_dict.get("description")
+                    if tool_schema_dict
+                    else tool_func.__doc__ or f"Tool: {tool_func.__name__}"
+                ),
+                "args_schema": tool_schema_dict.get("parameters") if tool_schema_dict else None,
+            }
+            if inspect.iscoroutinefunction(tool_func):
+                langchain_tool = StructuredTool.from_function(coroutine=tool_func, **tool_kwargs)
+            else:
+                langchain_tool = StructuredTool.from_function(func=tool_func, **tool_kwargs)
+            self.tools.append(langchain_tool)
+
+        rag_config = self.config.get("rag", {})
+        if rag_config.get("enabled", False):
+            top_k = rag_config.get("top_k", 5)
+            model_config = self.config.get("model", {})
+            effective_model = (
+                self.model_override
+                or model_config.get("default_model")
+                or model_config.get("model_name")
+            )
+            rag_tool = create_rag_tool(top_k, model_name=effective_model)
+            self.tools.append(rag_tool)
+
+        print(f"✓ Loaded {len(self.tools)} tools")
+
+    def _build_graph(self) -> CompiledStateGraph:
+        if not self.tools:
+            raise ValueError("Tools not loaded. Call load_tools() first.")
+
+        model_config = self.config.get("model", {})
+        effective_model = (
+            self.model_override
+            or model_config.get("default_model")
+            or model_config.get("model_name")
+        )
+        temperature = model_config.get("temperature", 0.1)
+
+        llm_with_tools = ChatOpenAI(
+            model=effective_model,
+            temperature=temperature,
+            name="Healthcare Assistant",
+        ).bind_tools(self.tools)
+
+        async def invoke_chatbot(state):
+            messages = list(state["messages"])
+            if self.system_prompt:
+                messages = [SystemMessage(content=self.system_prompt)] + messages
+            message = await llm_with_tools.ainvoke(messages)
+            return {"messages": [message]}
+
+        graph_builder = StateGraph(State)
+        graph_builder.add_node("chatbot", invoke_chatbot)
+        graph_builder.add_node("tools", ToolNode(tools=self.tools))
+        graph_builder.add_edge(START, "chatbot")
+        graph_builder.add_conditional_edges("chatbot", tools_condition)
+        graph_builder.add_edge("tools", "chatbot")
+        return graph_builder.compile()
+
+    async def _process_query_async(self, messages: List[Dict[str, str]]) -> str:
+        if not self.tools:
+            self.load_tools()
+        self.graph = self._build_graph()
+
+        langchain_messages: List[BaseMessage] = []
+        for msg in messages:
+            if msg["role"] == "user":
+                langchain_messages.append(HumanMessage(content=msg["content"]))
+            elif msg["role"] == "assistant":
+                langchain_messages.append(AIMessage(content=msg["content"]))
+
+        with splunk_ao_context(
+            project=os.getenv("SPLUNK_AO_PROJECT"),
+            agent_stream=os.getenv("SPLUNK_AO_AGENT_STREAM"),
+        ):
+            splunk_ao_context.start_session(external_id=self.session_id)
+
+            # One callback per request keeps each user turn in its own trace.
+            callback = SplunkAOAsyncCallback()
+            run_config = {**self.langgraph_config, "callbacks": [callback]}
+
+            result = await self.graph.ainvoke(
+                {"messages": langchain_messages},
+                run_config,
+            )
+        if result["messages"]:
+            return result["messages"][-1].content
+        return "No response generated"
+
+    def process_query(self, messages: List[Dict[str, str]]) -> str:
+        try:
+            return _run_async(self._process_query_async(messages))
+        except Exception as e:
+            print(f"[ERROR] Error processing query: {e}")
+            import traceback
+
+            traceback.print_exc()
+            return f"Error processing your request: {str(e)}"