From 14bb872dc36e710a1e0d5d74677ab13dcbc391cc Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 28 Feb 2026 04:00:30 +0000 Subject: [PATCH] feat: add perception UI, multimodal chat, and documentation updates - Add Perception page with image/audio upload and QCA analysis results - Wire multimodal chat: attach button now sends image/audio as base64 via WebSocket - Add perception_result WebSocket handler with PerceptionCard in chat messages - Add TypeScript types: BasirahInsight, NutqTranscription, PerceptionResult - Add Eye icon to Icons component - Update README with new API endpoints, architecture sections, and features - Update CHANGELOG with all gap-closure features and fixes - Update docs/index.html with perception, living memory, quaternary modules https://claude.ai/code/session_01JGamoWGB9PN39TCpi3F6J1 --- CHANGELOG.md | 21 + README.md | 61 ++- docs/index.html | 13 +- frontend/src/App.tsx | 135 ++++++- frontend/src/components/ChatMessage.tsx | 131 ++++++ frontend/src/components/Icons.tsx | 6 + frontend/src/pages/PerceptionPage.tsx | 508 ++++++++++++++++++++++++ frontend/src/types.ts | 36 ++ 8 files changed, 897 insertions(+), 14 deletions(-) create mode 100644 frontend/src/pages/PerceptionPage.tsx diff --git a/CHANGELOG.md b/CHANGELOG.md index 0197991..e22fa86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- Multimodal perception pipeline: Basirah (vision) + Nutq (voice) wired into QCA engine +- Auditory-first processing (Sam' before Basar) per Quran 16:78/17:36 +- Perception-Qalb integration: emotional context modulates vision and voice analysis +- `POST /api/perception/analyze` endpoint for multimodal analysis +- WebSocket `multimodal` message type for real-time perception +- DNA-inspired quaternary encoding module (`backend/memory/quaternary.py`) +- Quaternary checksum as 4th integrity layer in Lawh al-Mahfuz +- VectorStore (ChromaDB) integration in Living Memory novelty gate +- Hybrid text + semantic similarity in Living Memory `_find_best_match()` +- `search_entities()` method in KnowledgeGraph (fixes MemoryPyramid layer 4) +- Perception page in frontend UI with image/audio upload +- Image and audio attachment support in chat interface +- `BasirahEngine` and `NutqEngine` exports in perception `__init__.py` +- NutqEngine: language detection (Arabic, Urdu, English) via Unicode analysis +- NutqEngine: 8+ intent types (question, command, greeting, farewell, confirmation, negation, request, statement) + ### Fixed +- BasirahEngine: JSON-structured LLM output parsing (was returning empty extracted_text/key_elements) +- BasirahEngine: category detection bug (`"error" in raw_lower and "text"` evaluated boolean inside list) +- NutqEngine: `_adjust_for_tone()` was a no-op, now implements warm/patient/focused text transforms +- KnowledgeGraph: missing `search_entities()` caused MemoryPyramid layer 4 to silently fail - Replace deprecated `datetime.utcnow()` with `datetime.now(timezone.utc)` across all modules - Fix Pydantic `class Config` deprecation in `SkillExecuteRequest` (use `model_config` dict) - Fix `check_provider_health()` called without `await` in settings endpoint diff --git a/README.md b/README.md index 83c3c89..deb863e 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,7 @@ make dev # Start backend + frontend |---------|---------------| | **Chat** | Talk to your AI in the browser or terminal | | **Browse the web** | AI can search Google, read websites, extract information | +| **Analyze images & voice** | Upload images for vision analysis, audio for transcription | | **Run code** | AI writes and executes Python, bash scripts | | **Manage files** | Read, write, organize files on your computer | | **Remember things** | Remembers your conversations and preferences | @@ -114,8 +115,11 @@ make dev # Start backend + frontend | Feature | What It Means | |---------|---------------| | **QALB-7 Cognitive Pipeline** | 7-layer architecture: ethics → deliberation → emotion → conviction → metacognition | +| **Multimodal Perception** | Sam' (hearing) + Basar (sight) → Fu'ad integration, with Qalb-aware context | | **Developmental Stages** | Agents grow from Nutfah (5 tools, 5 turns) to Khalq Akhar (all tools, 25 turns) | +| **Living Memory** | Novelty gate with hybrid text+vector similarity — never re-stores 1+1=2 | | **5-Layer Memory Pyramid** | Unified query across episodic, semantic, neural pathways, vectors, and knowledge graph | +| **DNA Integrity** | Quaternary (ACGT) checksums with Hamming distance verification in Lawh al-Mahfuz | | **Causal Reasoning** | Pearl's 3-rung causal ladder: observation, intervention, counterfactual | | **Plugin system** | Add new abilities with a simple Python file | | **Event bus + Hooks** | Decoupled communication — modify data at any point in the pipeline | @@ -282,6 +286,9 @@ Each agent processes every task through these cognitive layers: | Module | Arabic | Purpose | File | |--------|--------|---------|------| +| **Multimodal Perception** | سمع+بصر | Sam' (hearing) first, then Basar (sight), Qalb-aware context | `perception/basirah.py`, `perception/nutq.py` | +| **Living Memory** | ذاكرة حية | Novelty gate + hybrid text/vector similarity + Dhikr daemon | `memory/living_memory.py` | +| **Quaternary Encoding** | تشفير رباعي | DNA-inspired ACGT checksums with Hamming distance verification | `memory/quaternary.py` | | **Lawwama Self-Healing** | لوّامة | Immune memory, health metrics, adaptive checkpoint intervals | `core/self_healing.py` | | **Parallel Agents** | — | Concurrent task scheduling + skill transfer between agents | `core/parallel_agents.py` | | **Imagination** | تصوير | Predictive coding — simulate outcomes before acting | `core/imagination.py` | @@ -296,14 +303,19 @@ All memory layers are queried through a unified `MemoryPyramid`: | Layer | Module | Purpose | |-------|--------|---------| +| **Living Memory** | `memory/living_memory.py` | Novelty gate (hybrid text + vector similarity), importance scoring, Dhikr daemon | | **Dhikr** | `memory/dhikr.py` | Three-tier persistent memory (episodic, semantic, procedural) | | **Masalik** | `memory/masalik.py` | Neural pathway network with spreading activation | -| **Lawh al-Mahfuz** | `memory/lawh_mahfuz.py` | Immutable core memory with triple-checksum integrity | -| **VectorStore** | `memory/vector_store.py` | Semantic embedding search (ChromaDB) | -| **KnowledgeGraph** | `memory/knowledge_graph.py` | Entity-relationship graph (SQLite) | +| **VectorStore** | `memory/vector_store.py` | Semantic embedding search (ChromaDB) — also used by Living Memory | +| **KnowledgeGraph** | `memory/knowledge_graph.py` | Entity-relationship graph with full-text search (SQLite) | +| **Lawh al-Mahfuz** | `memory/lawh_mahfuz.py` | Immutable memory with 4-layer integrity: SHA-256 + CRC-32 + length + quaternary checksum | Unified query: `memory/memory_pyramid.py` merges, deduplicates, and ranks results by relevance x certainty x recency. +**Living Memory** solves the 1+1=2 problem: seeing the same information again doesn't create a new trace — it activates the existing one. New information enriches existing traces, related info gets linked, and only genuinely novel content is stored. + +**Quaternary Encoding** (`memory/quaternary.py`) provides DNA-inspired error detection: binary data is encoded using a 4-symbol alphabet (A, C, G, T), chunked into codons (triplets), and verified using XOR parity and Hamming distance. + ### Developmental Stages (Nafs Levels 1–7) Agents grow through seven stages, each unlocking new capabilities: @@ -441,6 +453,45 @@ GET /api/tasks/history Get task history POST /api/memory/query Search memories POST /api/memory/store Store a memory POST /api/memory/consolidate Prune old memories +GET /api/memory/list List recent memories +``` + +### Perception (Sam' + Basar) +``` +POST /api/perception/analyze Multimodal analysis (text + base64 image + base64 audio) +``` + +Accepts `MultimodalInput` with fields: `text`, `image_base64`, `audio_base64`, `media_type`, `qalb_state`. +Processes Sam' (audio) first, then Basar (image), integrates via Fu'ad. + +### Cognitive Pipeline +``` +POST /api/qalb/analyze Analyze emotional state from text +GET /api/qalb/trend/{user_id} Get emotional trend over time +POST /api/cognitive/route Route to best cognitive method +POST /api/yaqin/tag Tag knowledge with certainty level +GET /api/yaqin/stats Get Yaqin statistics +``` + +### Federation +``` +GET /api/federation/status Federation network status +POST /api/federation/discover Discover agents by capability +POST /api/federation/route Route task to best agent +``` + +### Nafs & Ruh +``` +GET /api/nafs/tiers Get all 7 Nafs tier definitions +GET /api/nafs/{agent_id} Get agent Nafs level and progress +GET /api/ruh/{agent_id} Get agent Ruh energy level +``` + +### Knowledge +``` +POST /api/knowledge/ingest Ingest from URL or YouTube +POST /api/knowledge/upload Upload PDF for knowledge extraction +GET /api/knowledge/sources List ingested knowledge sources ``` ### Plugins & Extensibility @@ -490,6 +541,10 @@ POST /api/shura Multi-agent consultation WS /ws/{client_id} WebSocket connection ``` +**WebSocket message types**: `chat`, `task`, `command`, `multimodal`, `ping` + +The `multimodal` type accepts: `{ type: "multimodal", content: "text", image_base64: "...", audio_base64: "...", media_type: "image/png", qalb_state: "neutral" }` and returns a `perception_result` message with full QCA analysis. + ### Channels ``` POST /api/channels/{name}/start Start a channel adapter diff --git a/docs/index.html b/docs/index.html index 5ca01cd..7b2d744 100644 --- a/docs/index.html +++ b/docs/index.html @@ -1148,6 +1148,9 @@

Extension Modules

+ + + @@ -1157,19 +1160,21 @@

Extension Modules

ModuleArabicPurposeFile
Multimodal Perceptionسمع+بصرSam' (hearing) first, then Basar (sight), Qalb-aware context modulationperception/basirah.py, perception/nutq.py
Living Memoryذاكرة حيةNovelty gate + hybrid text/vector similarity + Dhikr maintenance daemonmemory/living_memory.py
Quaternary Encodingتشفير رباعيDNA-inspired ACGT checksums with codon chunking and Hamming verificationmemory/quaternary.py
Self-HealingلوّامةImmune memory, health metrics, adaptive checkpointscore/self_healing.py
Parallel Agents—Concurrent task scheduling + skill transfercore/parallel_agents.py
ImaginationتصويرPredictive coding — simulate before actingcore/imagination.py
-

5-Layer Memory Pyramid

+

Memory Architecture (6-Layer Pyramid)

All memory layers are queried through a unified MemoryPyramid:

+ - - - + + +
LayerModulePurpose
Living Memorymemory/living_memory.pyNovelty gate with hybrid text + vector similarity, importance scoring, Dhikr daemon
Dhikrmemory/dhikr.pyThree-tier persistent memory (episodic, semantic, procedural)
Masalikmemory/masalik.pyNeural pathways with spreading activation
Lawh al-Mahfuzmemory/lawh_mahfuz.pyImmutable memory with triple-checksum integrity (SHA-256 + CRC-32 + length)
VectorStorememory/vector_store.pySemantic embedding search (ChromaDB)
KnowledgeGraphmemory/knowledge_graph.pyEntity-relationship graph (SQLite)
VectorStorememory/vector_store.pySemantic embedding search (ChromaDB) — also used by Living Memory
KnowledgeGraphmemory/knowledge_graph.pyEntity-relationship graph with full-text entity search (SQLite)
Lawh al-Mahfuzmemory/lawh_mahfuz.pyImmutable memory with 4-layer integrity: SHA-256 + CRC-32 + length + quaternary checksum

Unified query: memory/memory_pyramid.py merges, deduplicates, and ranks by relevance × certainty × recency.

+

Quaternary Encoding: DNA-inspired error detection using a 4-symbol alphabet (A, C, G, T). Data is encoded as quaternary strings, chunked into codons (triplets mapping to 64 semantic categories), and verified using XOR parity and Hamming distance checks.

Yaqin Certainty Engine

Every piece of knowledge is tagged with its certainty level:

diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index b850703..f12b1af 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -21,6 +21,7 @@ import type { Memory, Integration, SystemStatus, + PerceptionResult, } from "./types"; import { config } from "./config"; import { useApi } from "./hooks/useApi"; @@ -133,6 +134,7 @@ class ErrorBoundary extends Component { } const ChannelsPage = lazy(() => import("./pages/ChannelsPage")); +const PerceptionPage = lazy(() => import("./pages/PerceptionPage")); const SkillsPage = lazy(() => import("./pages/SkillsPage")); const SecurityPage = lazy(() => import("./pages/SecurityPage")); const AutomationPage = lazy(() => import("./pages/AutomationPage")); @@ -145,6 +147,19 @@ const DeveloperPage = lazy(() => import("./pages/DeveloperPage")); const WelcomePage = lazy(() => import("./pages/WelcomePage")); const SettingsPage = lazy(() => import("./pages/SettingsPage")); +function fileToBase64(file: File): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => { + const result = reader.result as string; + const base64 = result.split(",")[1] || result; + resolve(base64); + }; + reader.onerror = reject; + reader.readAsDataURL(file); + }); +} + // ===== MAIN APP INNER ===== function AppInner() { const { addToast } = useToast(); @@ -220,6 +235,7 @@ function AppInner() { }[] >([]); const [showSessionHistory, setShowSessionHistory] = useState(false); + const [attachedFiles, setAttachedFiles] = useState([]); const CHAT_COMMANDS = [ { name: "/help", description: "Show available commands" }, @@ -414,6 +430,26 @@ function AppInner() { addTerminalLine("Task completed", "gold"); loadAgents(); break; + case "perception_result": + setStreaming(false); + setStreamingText(""); + setTypingIndicator(false); + setToolStatus(""); + setMessages((prev) => [ + ...prev, + { + id: Date.now(), + role: "assistant" as const, + content: + (data.result as PerceptionResult)?.batin || + "Perception analysis complete", + agent: "Basirah", + ts: new Date().toLocaleTimeString(), + perception: data.result as PerceptionResult, + }, + ]); + addTerminalLine("Perception analysis complete", "gold"); + break; case "agent_created": addTerminalLine( `Agent created: ${(data.agent as Record).name}`, @@ -677,12 +713,21 @@ function AppInner() { }, []); const sendMessage = async () => { - if (!input.trim() || streaming) return; + if ((!input.trim() && attachedFiles.length === 0) || streaming) return; const content = input; + const files = [...attachedFiles]; + const hasMedia = files.some( + (f) => f.type.startsWith("image/") || f.type.startsWith("audio/"), + ); + const userMsg: ChatMessage = { id: Date.now(), role: "user", - content, + content: + content + + (files.length > 0 + ? ` [${files.map((f) => f.name).join(", ")}]` + : ""), ts: new Date().toLocaleTimeString(), }; setMessages((prev) => [...prev, userMsg]); @@ -691,12 +736,40 @@ function AppInner() { setTypingIndicator(true); setToolStatus(""); setInput(""); + setAttachedFiles([]); // Reset textarea height after clearing if (chatTextareaRef.current) { chatTextareaRef.current.style.height = "auto"; } addTerminalLine(`> ${content.substring(0, 60)}...`, "info"); + // If media files attached, send as multimodal via WebSocket + if (hasMedia && ws) { + try { + const payload: Record = { + type: "multimodal", + text: content, + agent_id: selectedAgent?.id, + session_id: sessionId, + }; + for (const file of files) { + const b64 = await fileToBase64(file); + if (file.type.startsWith("image/")) { + payload.image_base64 = b64; + payload.media_type = file.type; + } else if (file.type.startsWith("audio/")) { + payload.audio_base64 = b64; + } + } + ws.send(JSON.stringify(payload)); + } catch { + setStreaming(false); + setTypingIndicator(false); + addTerminalLine("Failed to process media files", "error"); + } + return; + } + // Prefer HTTP POST /api/chat (returns message_id, streams via WebSocket) // Fall back to WebSocket direct send if HTTP fails try { @@ -865,6 +938,12 @@ function AppInner() { { label: "Tools", items: [ + { + id: "perception", + label: "Perception", + desc: "Vision & voice analysis", + icon: , + }, { id: "memory", label: "Memory", @@ -1335,22 +1414,62 @@ function AppInner() { {/* Input box */}
+ {/* Attached files preview */} + {attachedFiles.length > 0 && ( +
+ {attachedFiles.map((file, idx) => ( +
+ {file.type.startsWith("image/") ? ( + + + + ) : ( + + + + + )} + + {file.name} + + +
+ ))} +
+ )} + {/* Left action buttons */}
{/* Attach / Upload button */}
); + case "perception": + return ; case "skills": return ; case "security": diff --git a/frontend/src/components/ChatMessage.tsx b/frontend/src/components/ChatMessage.tsx index e1baf57..edddf8c 100644 --- a/frontend/src/components/ChatMessage.tsx +++ b/frontend/src/components/ChatMessage.tsx @@ -2,6 +2,7 @@ import { useState, useMemo } from "react"; import type { ChatMessage as ChatMessageType, CognitiveMetadata, + PerceptionResult, } from "../types"; import { Markdown } from "./Markdown"; @@ -391,6 +392,133 @@ function CognitiveBar({ cognitive }: { cognitive: CognitiveMetadata }) { ); } +const PERCEPTION_CATEGORY_COLORS: Record = { + text: "bg-blue-100 dark:bg-blue-500/15 text-blue-700 dark:text-blue-400", + diagram: "bg-purple-100 dark:bg-purple-500/15 text-purple-700 dark:text-purple-400", + screenshot: "bg-cyan-100 dark:bg-cyan-500/15 text-cyan-700 dark:text-cyan-400", + photo: "bg-emerald-100 dark:bg-emerald-500/15 text-emerald-700 dark:text-emerald-400", + document: "bg-amber-100 dark:bg-amber-500/15 text-amber-700 dark:text-amber-400", +}; + +function PerceptionCard({ perception }: { perception: PerceptionResult }) { + const [expanded, setExpanded] = useState(false); + const basirah = perception.perception?.basirah; + const nutq = perception.perception?.nutq; + + return ( +
+ + + {expanded && ( +
+ {basirah && ( +
+
Vision (Basirah)
+

{basirah.description}

+
+ Confidence +
+
+
+ {(basirah.confidence * 100).toFixed(0)}% +
+ {basirah.extracted_text && ( +
+ Text: + {basirah.extracted_text.substring(0, 200)} + {basirah.extracted_text.length > 200 && "..."} +
+ )} + {basirah.key_elements?.length > 0 && ( +
+ {basirah.key_elements.map((el, i) => ( + + {el} + + ))} +
+ )} +
+ )} + {nutq && ( +
+
Voice (Nutq)
+

{nutq.text}

+
+ Intent: {nutq.intent} + {" · "} + Lang: {nutq.language} +
+
+ )} + {perception.zahir && ( +
+ Zahir: {perception.zahir} +
+ )} + {perception.batin && ( +
+ Batin: {perception.batin} +
+ )} +
+ )} +
+ ); +} + interface ChatMessageBubbleProps { msg: ChatMessageType; selectedAgent?: { name: string } | null; @@ -476,6 +604,9 @@ export function ChatMessageBubble({ {/* Cognitive bar */} {msg.cognitive && } + {/* Perception card */} + {msg.perception && } + {/* Action buttons — hover reveal */}
+
+ +
+
+ {/* Input Panel */} +
+ {/* Text input */} +
+ +