From 1e61c899c0e7679222e46356061634904bd33b36 Mon Sep 17 00:00:00 2001 From: Juber Shaikh <40266375+CodeWithJuber@users.noreply.github.com> Date: Sat, 28 Feb 2026 05:25:13 +0400 Subject: [PATCH] feat: implement QALB-7 cognitive architecture with full frontend integration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Complete implementation of the 7-layer Quranic Cognitive Architecture: Backend — QALB-7 Pipeline (13 new modules): - Fitrah: innate ethical guardrails (NO_HARM, TRUTH, JUSTICE) - Nafs Triad: 3-voice deliberation (Ammara/Lawwama/Mutmainna) - Qalb Processor: cardiac oscillation modulating LLM temperature/tokens - Fu'ad: Bayesian conviction engine (impression→belief→conviction) - Lubb: metacognition (compress, coherence check, bias detection) - Developmental Gate: 7-stage capability gating (Nutfah→Khalq Akhar) - Causal Engine: Pearl's 3-rung causal ladder - Self-Healing: Lawwama immune system with adaptive checkpoints - Parallel Agents, Imagination, Creativity, Dream Engine - Shura Council: multi-agent consultation Backend — Memory Architecture (3 new modules): - Lawh al-Mahfuz: immutable memory with triple-checksum integrity - Memory Pyramid: unified 5-layer query engine - Living Memory: adaptive memory lifecycle Backend — Wiring & Fixes: - Wire all QALB-7 modules into BaseAgent.execute() - Replace 10 hardcoded rules with proper module calls - Surface cognitive metadata in WS chat_complete messages - XML tool-call fallback parser for MiniMax-style models - Fix async coroutine issue in memory_pyramid.py - SuperAgent starts at nafs_level=3 (Mudghah) Frontend — Cognitive UI: - CognitiveBar: expandable pill row under assistant messages (Qalb state, Yaqin level, Lubb quality, Ruh energy, Nafs badge) - AgentCard: Ruh energy bar with color gradient - Full UI redesign: component extraction, dark mode, responsive layout - CognitiveMetadata TypeScript types Docs & Config: - README: full QALB-7 architecture docs, module tables, stage diagram - docs/index.html: QALB-7 deep dive, updated project structure - .gitignore: add .vault.key and .mcp.json - New skills: linode_cloud, ssh_remote Co-Authored-By: Claude Opus 4.6 --- .env.example | 3 + .gitignore | 8 + CLAUDE.md | 86 + README.md | 306 ++- backend/agents/base.py | 1265 ++++++++++- backend/agents/perpetual_rotation.py | 463 ++++ backend/agents/shura_council.py | 499 +++++ backend/agents/specialized.py | 323 +++ backend/api/main.py | 579 ++++- backend/core/creativity.py | 574 +++++ backend/core/developmental_stages.py | 279 +++ backend/core/dream_engine.py | 630 ++++++ backend/core/fuad.py | 240 ++ backend/core/imagination.py | 444 ++++ backend/core/lubb.py | 351 +++ backend/core/nafs_triad.py | 170 ++ backend/core/parallel_agents.py | 422 ++++ backend/core/qalb_processor.py | 153 ++ backend/core/self_healing.py | 447 ++++ backend/knowledge/__init__.py | 0 backend/knowledge/ingest.py | 189 ++ backend/memory/dhikr.py | 220 +- backend/memory/knowledge_graph.py | 2 +- backend/memory/lawh_mahfuz.py | 260 +++ backend/memory/living_memory.py | 687 ++++++ backend/memory/masalik.py | 134 +- backend/memory/memory_pyramid.py | 251 +++ backend/providers.py | 276 ++- backend/reasoning/aql_engine.py | 64 +- backend/reasoning/causal_engine.py | 341 +++ backend/requirements.txt | 4 + backend/security/wali.py | 4 +- backend/skills/builtin/data_analysis.py | 10 +- backend/skills/builtin/linode_cloud.py | 561 +++++ backend/skills/builtin/ssh_remote.py | 716 ++++++ backend/skills/builtin/web_browse.py | 12 +- docker-compose.yml | 2 + docker/Dockerfile.backend | 2 +- docs/index.html | 240 +- frontend/index.html | 2 +- frontend/src/App.tsx | 2089 ++++++++++++------ frontend/src/components/AgentCard.tsx | 179 ++ frontend/src/components/AgentModal.tsx | 165 ++ frontend/src/components/ChatMessage.tsx | 472 ++++ frontend/src/components/ConnectionBanner.tsx | 44 + frontend/src/components/Icons.tsx | 99 + frontend/src/components/Markdown.tsx | 61 + frontend/src/components/MobileNav.tsx | 46 + frontend/src/components/Sidebar.tsx | 138 ++ frontend/src/components/Skeleton.tsx | 40 + frontend/src/components/ThemeToggle.tsx | 37 + frontend/src/hooks/useApi.ts | 23 +- frontend/src/index.css | 263 ++- frontend/src/pages/AutomationPage.tsx | 140 +- frontend/src/pages/ChannelsPage.tsx | 178 +- frontend/src/pages/DeveloperPage.tsx | 434 +++- frontend/src/pages/MajlisPage.tsx | 539 +++-- frontend/src/pages/NotebookPage.tsx | 313 ++- frontend/src/pages/PluginsPage.tsx | 368 ++- frontend/src/pages/ProvidersPage.tsx | 707 ++++-- frontend/src/pages/ScannerPage.tsx | 248 ++- frontend/src/pages/SecurityPage.tsx | 276 ++- frontend/src/pages/SettingsPage.tsx | 335 ++- frontend/src/pages/SkillsPage.tsx | 292 ++- frontend/src/types.ts | 110 +- frontend/tailwind.config.js | 74 +- 66 files changed, 16853 insertions(+), 2036 deletions(-) create mode 100644 CLAUDE.md create mode 100644 backend/agents/perpetual_rotation.py create mode 100644 backend/agents/shura_council.py create mode 100644 backend/core/creativity.py create mode 100644 backend/core/developmental_stages.py create mode 100644 backend/core/dream_engine.py create mode 100644 backend/core/fuad.py create mode 100644 backend/core/imagination.py create mode 100644 backend/core/lubb.py create mode 100644 backend/core/nafs_triad.py create mode 100644 backend/core/parallel_agents.py create mode 100644 backend/core/qalb_processor.py create mode 100644 backend/core/self_healing.py create mode 100644 backend/knowledge/__init__.py create mode 100644 backend/knowledge/ingest.py create mode 100644 backend/memory/lawh_mahfuz.py create mode 100644 backend/memory/living_memory.py create mode 100644 backend/memory/memory_pyramid.py create mode 100644 backend/reasoning/causal_engine.py create mode 100644 backend/skills/builtin/linode_cloud.py create mode 100644 backend/skills/builtin/ssh_remote.py create mode 100644 frontend/src/components/AgentCard.tsx create mode 100644 frontend/src/components/AgentModal.tsx create mode 100644 frontend/src/components/ChatMessage.tsx create mode 100644 frontend/src/components/ConnectionBanner.tsx create mode 100644 frontend/src/components/Icons.tsx create mode 100644 frontend/src/components/Markdown.tsx create mode 100644 frontend/src/components/MobileNav.tsx create mode 100644 frontend/src/components/Sidebar.tsx create mode 100644 frontend/src/components/Skeleton.tsx create mode 100644 frontend/src/components/ThemeToggle.tsx diff --git a/.env.example b/.env.example index 1fc84ae..c607e09 100644 --- a/.env.example +++ b/.env.example @@ -36,6 +36,9 @@ DB_PATH=data/mizan.db # ===== LOCAL AI (Optional) ===== OLLAMA_URL=http://localhost:11434 +# ===== CLOUD PROVIDERS (Optional) ===== +LINODE_API_TOKEN= # Linode Personal Access Token (linodes:read_write scope) + # ===== CHANNELS (Optional) ===== TELEGRAM_BOT_TOKEN= DISCORD_BOT_TOKEN= diff --git a/.gitignore b/.gitignore index aadc19c..c84866b 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,14 @@ data/ .DS_Store Thumbs.db +# Claude Code +.claude/ +.claude.local.md + +# Secrets / local config +backend/.vault.key +.mcp.json + # IDE .vscode/ .idea/ diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..b183d4c --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,86 @@ +# MIZAN - Agentic Personal AI + +## Quick Reference + +- **Backend**: Python 3.11+, FastAPI, Pydantic, aiosqlite +- **Frontend**: React 18, TypeScript, Vite, Tailwind CSS +- **AI Providers**: Anthropic, OpenAI, OpenRouter, Ollama +- **Database**: SQLite (via aiosqlite) +- **Infra**: Docker Compose, Nginx + +## Commands + +```bash +make setup # First-time: install deps, setup frontend, create .env +make dev # Start backend + frontend dev servers +make test # Run pytest +make check # Lint + typecheck + test (all checks) +make format # Auto-format with ruff +make docker # Docker Compose up (build + detach) +make docker-down # Stop Docker +make clean # Remove build artifacts +``` + +## Key Files + +- `backend/api/main.py` - FastAPI app, all REST + WebSocket routes +- `backend/agents/base.py` - Base agent with agentic ReAct loop +- `backend/providers.py` - LLM provider abstraction (Anthropic/OpenRouter/OpenAI/Ollama) +- `backend/settings.py` - Pydantic settings from .env +- `backend/cli.py` - CLI entry point (mizan serve, mizan doctor) +- `frontend/src/App.tsx` - React router + theme provider +- `docker-compose.yml` - Dev deployment +- `Makefile` - All project commands + +## Architecture + +7-layer Quranic Cognitive Architecture (QCA): + +| Layer | Module | Purpose | +|-------|--------|---------| +| 1-2 | `backend/perception/` | Sensory input (vision, voice, text) | +| 3 | `backend/qca/engine.py` | Cognitive integration | +| 4 | `backend/memory/` | Hierarchical memory (Dhikr, Masalik) | +| 5 | `backend/reasoning/` | ReAct reasoning with self-correction | +| 6 | `backend/agents/` | Autonomous agents with Nafs levels | +| 7 | `backend/core/` | Principles (Qalb, Ihsan, Tawbah, Sabr) | + +Cross-cutting: `backend/security/` (Wali + Izn), `backend/gateway/` (channels), `backend/skills/` (capabilities) + +## Key Patterns + +- All backend I/O is async (async/await) +- Pydantic models for all API request/response schemas +- JWT auth on all endpoints; Wali rate-limiting; Izn permission system +- WebSocket at /ws/{client_id} for real-time streaming +- Arabic-inspired naming: Dhikr=memory, Wali=guardian, Izn=permission, Nafs=self, Qalb=heart + +## Environment Setup + +1. `cp .env.example .env` and set at least one API key +2. `make install-dev` (or `make setup` for full setup including frontend) +3. Required: `ANTHROPIC_API_KEY` or `OPENROUTER_API_KEY` +4. Required: `SECRET_KEY` (any random string) + +## Testing + +- pytest with asyncio auto mode +- Tests in tests/ (comprehensive: agents, API, memory, QCA, security, e2e) +- Run `make check` for full lint + typecheck + test pipeline +- Pre-commit hook runs ruff on commit + +## Security Notes + +- Never edit .env directly (hook blocks this) - use .env.example as template +- API keys managed via backend/security/vault.py +- Input validation in backend/security/validation.py +- All user input sanitized before processing + +## Gotchas + +- Docker reads .env for compose variable substitution - inline comments like `KEY=value # comment` get parsed as part of the value +- Claude models use Anthropic only if `ANTHROPIC_API_KEY` starts with `sk-ant-`; otherwise auto-route through OpenRouter +- DEFAULT_MODEL env var controls agent model (fallback: `claude-sonnet-4-20250514`) — set it in .env +- Backend uses --reload in Docker, so local file changes in backend/ take effect automatically +- .env values are NOT auto-loaded into os.environ - pydantic-settings reads them but providers.py uses load_dotenv() +- `.claude/` and `.claude.local.md` are gitignored — local Claude Code config won't be committed diff --git a/README.md b/README.md index 010fba6..83c3c89 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ MIZAN is a **personal AI assistant** you can run on your own computer. Unlike Ch - **Anyone can extend it** — add new abilities with simple plugins - **Works with any AI** — Anthropic Claude, OpenAI, OpenRouter (300+ models), or local Ollama -> Think of it as your personal AI employee that can use tools, remember things, and get better over time. +> Think of it as your personal AI employee that can use tools, remember things, and get better over time — powered by a 7-layer Quranic Cognitive Architecture (QALB-7). --- @@ -113,14 +113,16 @@ make dev # Start backend + frontend | Feature | What It Means | |---------|---------------| +| **QALB-7 Cognitive Pipeline** | 7-layer architecture: ethics → deliberation → emotion → conviction → metacognition | +| **Developmental Stages** | Agents grow from Nutfah (5 tools, 5 turns) to Khalq Akhar (all tools, 25 turns) | +| **5-Layer Memory Pyramid** | Unified query across episodic, semantic, neural pathways, vectors, and knowledge graph | +| **Causal Reasoning** | Pearl's 3-rung causal ladder: observation, intervention, counterfactual | | **Plugin system** | Add new abilities with a simple Python file | -| **Event bus** | Modules communicate without knowing about each other | -| **Hook system** | Modify any data flowing through the system | -| **Middleware pipeline** | Intercept and process requests/responses | -| **REST + WebSocket API** | Full API for building custom integrations | -| **Multi-agent system** | Multiple AI agents collaborate on complex tasks | -| **Security built-in** | JWT auth, rate limiting, sandboxing, audit logs | -| **Self-healing diagnostics** | Built-in doctor system to detect and fix issues | +| **Event bus + Hooks** | Decoupled communication — modify data at any point in the pipeline | +| **REST + WebSocket API** | Full API with cognitive metadata streamed in real-time | +| **Multi-agent Shura** | Agents consult via Shura Council for complex decisions | +| **Self-healing (Lawwama)** | Immune memory, adaptive checkpoints, auto-package-install | +| **Security (Wali)** | JWT auth, rate limiting, sandboxing, SSRF block, audit logs | --- @@ -203,53 +205,132 @@ See the [Plugin Development Guide](docs/) for the full reference. ## Architecture -``` -┌─────────────────────────────────────────────────────────────────┐ -│ MIZAN Architecture │ -├─────────────────────────────────────────────────────────────────┤ -│ │ -│ YOU (Browser/Terminal/Telegram/Discord/Slack/WhatsApp) │ -│ │ │ -│ ▼ │ -│ ┌──────────────────────────────────────────────────────────┐ │ -│ │ Gateway (REST API + WebSocket) │ │ -│ │ Auth · Rate Limiting · Input Validation · CORS │ │ -│ └────────────────────────┬─────────────────────────────────┘ │ -│ │ │ -│ ┌────────────────────────▼─────────────────────────────────┐ │ -│ │ Plugin System │ │ -│ │ Events (Nida') · Hooks (Ta'liq) · Middleware (Silsilah) │ │ -│ │ Any plugin can listen, modify, or extend │ │ -│ └────────────────────────┬─────────────────────────────────┘ │ -│ │ │ -│ ┌────────────────────────▼─────────────────────────────────┐ │ -│ │ Agent System (Multi-Agent) │ │ -│ │ ┌────────┐ ┌──────────┐ ┌─────────┐ ┌────────┐ │ │ -│ │ │ Hafiz │ │ Mubashir │ │ Mundhir │ │ Katib │ + Any │ │ -│ │ │General │ │ Browser │ │Research │ │ Code │ Custom │ │ -│ │ └───┬────┘ └────┬─────┘ └────┬────┘ └───┬────┘ │ │ -│ │ └──────┬────┘────────────┘───────────┘ │ │ -│ │ ▼ │ │ -│ │ ┌──────────────────────────────────────────────┐ │ │ -│ │ │ Agentic Loop (Think → Use Tools → Repeat) │ │ │ -│ │ │ Up to 15 autonomous iterations per task │ │ │ -│ │ └──────────────────────────────────────────────┘ │ │ -│ └──────────────────────────────────────────────────────────┘ │ -│ │ │ -│ ┌───────────┬────────────▼──────────┬──────────────────────┐ │ -│ │ Memory │ LLM Providers │ Skills & Tools │ │ -│ │ SQLite │ Claude/GPT/Gemini/ │ Web Browse, Code, │ │ -│ │ 3-tier │ Llama/300+ models │ File, HTTP + Custom │ │ -│ └───────────┴───────────────────────┴──────────────────────┘ │ -│ │ -│ ┌──────────────────────────────────────────────────────────┐ │ -│ │ Security Layer (Wali Guardian) │ │ -│ │ JWT Auth · Rate Limit · Sandbox · SSRF Block · Audit │ │ -│ └──────────────────────────────────────────────────────────┘ │ -└─────────────────────────────────────────────────────────────────┘ -``` - -### How Everything Connects (No Coupling!) +MIZAN implements a **7-layer Quranic Cognitive Architecture (QALB-7)** — a bio-inspired AI system where each cognitive module maps to a concept from Islamic psychology. + +### System Overview + +``` +┌──────────────────────────────────────────────────────────────────┐ +│ MIZAN Architecture │ +├──────────────────────────────────────────────────────────────────┤ +│ │ +│ YOU (Browser / Terminal / Telegram / Discord / Slack / WhatsApp) │ +│ │ │ +│ ▼ │ +│ ┌───────────────────────────────────────────────────────────┐ │ +│ │ Gateway (REST API + WebSocket) │ │ +│ │ Auth · Rate Limiting · Input Validation · CORS │ │ +│ └──────────────────────┬────────────────────────────────────┘ │ +│ │ │ +│ ┌──────────────────────▼────────────────────────────────────┐ │ +│ │ Plugin System (Decoupled) │ │ +│ │ Events (Nida') · Hooks (Ta'liq) · Middleware (Silsilah) │ │ +│ └──────────────────────┬────────────────────────────────────┘ │ +│ │ │ +│ ┌──────────────────────▼────────────────────────────────────┐ │ +│ │ QALB-7 Cognitive Pipeline │ │ +│ │ │ │ +│ │ Fitrah ──► Nafs Triad ──► Qalb Processor ──► Fu'ad ──► │ │ +│ │ (Ethics) (Deliberate) (Modulate LLM) (Convict) │ │ +│ │ │ │ +│ │ ──► Lubb ──► Developmental Gate ──► Causal Engine │ │ +│ │ (Meta) (Capability Gate) (Why/What-if) │ │ +│ └──────────────────────┬────────────────────────────────────┘ │ +│ │ │ +│ ┌──────────────────────▼────────────────────────────────────┐ │ +│ │ Agent System (Multi-Agent + Shura Council) │ │ +│ │ ┌────────┐ ┌──────────┐ ┌─────────┐ ┌────────────────┐ │ │ +│ │ │ Hafiz │ │ Mubashir │ │ Mundhir │ │ Khalifah │ │ │ +│ │ │General │ │ Browser │ │Research │ │ SuperAgent │ │ │ +│ │ └───┬────┘ └────┬─────┘ └────┬────┘ └───┬────────────┘ │ │ +│ │ └───────┬────┘───────────┘───────────┘ │ │ +│ │ ▼ │ │ +│ │ ┌──────────────────────────────────────────────────┐ │ │ +│ │ │ Agentic Loop (Think → Tool → Lawwama → Repeat) │ │ │ +│ │ │ 5–25 turns (gated by Developmental Stage) │ │ │ +│ │ └──────────────────────────────────────────────────┘ │ │ +│ └────────────────────────────────────────────────────────────┘ │ +│ │ │ +│ ┌──────────┬───────────▼────────────┬────────────────────────┐ │ +│ │ Memory │ LLM Providers │ Skills & Tools │ │ +│ │ Pyramid │ Claude / GPT / Gemini │ Web, Code, File, │ │ +│ │ (5-layer)│ Llama / 300+ models │ SSH, HTTP + Custom │ │ +│ └──────────┴────────────────────────┴────────────────────────┘ │ +│ │ +│ ┌────────────────────────────────────────────────────────────┐ │ +│ │ Security Layer (Wali Guardian) │ │ +│ │ JWT Auth · Rate Limit · Sandbox · SSRF Block · Audit Log │ │ +│ └────────────────────────────────────────────────────────────┘ │ +└──────────────────────────────────────────────────────────────────┘ +``` + +### QALB-7 Cognitive Modules + +Each agent processes every task through these cognitive layers: + +| # | Module | Arabic | Purpose | File | +|---|--------|--------|---------|------| +| 1 | **Fitrah** | فطرة | Innate ethical guardrails (NO_HARM, TRUTH, JUSTICE) | `core/fitrah.py` | +| 2 | **Nafs Triad** | نفس | Three competing inner voices (Ammara/Lawwama/Mutmainna) deliberate on approach | `core/nafs_triad.py` | +| 3 | **Qalb Processor** | قلب | Cardiac oscillation — alternates between focused (Qabd) and creative (Bast) states, modulating LLM temperature and token limits | `core/qalb_processor.py` | +| 4 | **Fu'ad** | فؤاد | Bayesian conviction engine — evidence accumulation from impression to conviction | `core/fuad.py` | +| 5 | **Lubb** | لبّ | Metacognition — compresses reasoning traces, checks coherence, detects cognitive bias | `core/lubb.py` | +| 6 | **Developmental Gate** | أطوار | Progressive capability gating (7 stages from Nutfah to Khalq Akhar) — controls tools, turn limits, autonomy | `core/developmental_stages.py` | +| 7 | **Causal Engine** | سببية | Pearl's causal ladder — observation, intervention ("what if I do X?"), counterfactual reasoning | `reasoning/causal_engine.py` | + +### Extension Modules + +| Module | Arabic | Purpose | File | +|--------|--------|---------|------| +| **Lawwama Self-Healing** | لوّامة | Immune memory, health metrics, adaptive checkpoint intervals | `core/self_healing.py` | +| **Parallel Agents** | — | Concurrent task scheduling + skill transfer between agents | `core/parallel_agents.py` | +| **Imagination** | تصوير | Predictive coding — simulate outcomes before acting | `core/imagination.py` | +| **Creativity** | إبداع | 5 creative modes + fitness landscape mathematics | `core/creativity.py` | +| **Dream Engine** | منام | Offline memory consolidation (NREM replay + REM recombination) | `core/dream_engine.py` | +| **Shura Council** | شورى | Multi-agent consultation for complex decisions | `agents/shura_council.py` | +| **Perpetual Rotation** | دورة | Agent rotation and load balancing | `agents/perpetual_rotation.py` | + +### Memory Architecture (5-Layer Pyramid) + +All memory layers are queried through a unified `MemoryPyramid`: + +| Layer | Module | Purpose | +|-------|--------|---------| +| **Dhikr** | `memory/dhikr.py` | Three-tier persistent memory (episodic, semantic, procedural) | +| **Masalik** | `memory/masalik.py` | Neural pathway network with spreading activation | +| **Lawh al-Mahfuz** | `memory/lawh_mahfuz.py` | Immutable core memory with triple-checksum integrity | +| **VectorStore** | `memory/vector_store.py` | Semantic embedding search (ChromaDB) | +| **KnowledgeGraph** | `memory/knowledge_graph.py` | Entity-relationship graph (SQLite) | + +Unified query: `memory/memory_pyramid.py` merges, deduplicates, and ranks results by relevance x certainty x recency. + +### Developmental Stages (Nafs Levels 1–7) + +Agents grow through seven stages, each unlocking new capabilities: + +| Level | Stage | Max Turns | Key Unlocks | +|-------|-------|-----------|-------------| +| 1 | **Nutfah** (نطفة) | 5 | Basic tools: bash, read_file, recall_memory | +| 2 | **Alaqah** (علقة) | 8 | + write_file, http_get | +| 3 | **Mudghah** (مضغة) | 10 | + python_exec, http_post, delegation | +| 4 | **Izham** (عظام) | 12 | + create_agent, causal reasoning (rung 2) | +| 5 | **Lahm** (لحم) | 15 | All tools, causal rung 3, Lubb metacognition | +| 6 | **Nafkh** (نفخ) | 20 | Full metacognition | +| 7 | **Khalq Akhar** (خلق آخر) | 25 | Full autonomy | + +### Cognitive Metadata in the UI + +Every assistant response includes a **CognitiveBar** showing: +- **Qalb** state (Qabd/Bast/Khushu) with confidence +- **Yaqin** certainty level (ʿIlm al-Yaqin / ʿAyn al-Yaqin / Ḥaqq al-Yaqin) +- **Lubb** quality assessment (confident / hedged / uncertain) +- **Ruh** energy percentage +- **Nafs** level and name badge +- **Lawwama** repair indicator (when self-healing is active) + +Expandable for detailed signals, bias flags, and evidence lists. + +### Decoupled Communication ``` Plugin A ──────► Event Bus ◄────── Plugin B @@ -588,55 +669,82 @@ make docker-down # Stop all Docker services ``` mizan/ ├── backend/ -│ ├── api/main.py # FastAPI server + WebSocket + all routes +│ ├── api/main.py # FastAPI server + WebSocket + all routes │ ├── agents/ -│ │ ├── base.py # Base agent with agentic loop (Think → Tool → Repeat) -│ │ ├── specialized.py # Browser, Research, Code agents -│ │ └── federation.py # Agent-to-agent communication +│ │ ├── base.py # Base agent with QALB-7 agentic loop +│ │ ├── specialized.py # Browser, Research, Code, SuperAgent (Khalifah) +│ │ ├── federation.py # Agent-to-agent communication +│ │ ├── shura_council.py # Multi-agent consultation +│ │ └── perpetual_rotation.py # Agent rotation & load balancing │ ├── core/ -│ │ ├── events.py # Event bus — decoupled communication -│ │ ├── hooks.py # Hook system — data transformation -│ │ ├── plugins.py # Plugin manager — extend without touching core -│ │ ├── middleware.py # Middleware pipeline -│ │ ├── qalb.py # Emotional intelligence engine -│ │ ├── ruh_engine.py # Energy/vitality management -│ │ ├── tawbah.py # Error recovery protocol -│ │ ├── ihsan.py # Proactive excellence suggestions -│ │ ├── sabr.py # Patience engine for long tasks -│ │ └── shukr.py # Strength reinforcement +│ │ ├── fitrah.py # Innate ethical guardrails +│ │ ├── nafs_triad.py # 3-voice deliberation (Ammara/Lawwama/Mutmainna) +│ │ ├── qalb_processor.py # Cardiac oscillation → LLM param modulation +│ │ ├── fuad.py # Bayesian conviction formation +│ │ ├── lubb.py # Metacognition: compress, cohere, debias +│ │ ├── developmental_stages.py # 7-stage capability gating (Nutfah→Khalq Akhar) +│ │ ├── self_healing.py # Lawwama immune system + health metrics +│ │ ├── parallel_agents.py # Concurrent task scheduling + skill transfer +│ │ ├── imagination.py # Predictive coding engine +│ │ ├── creativity.py # 5 creative modes + landscape math +│ │ ├── dream_engine.py # Offline memory consolidation (NREM+REM) +│ │ ├── qalb.py # Emotional intelligence (sentiment) +│ │ ├── ruh_engine.py # Energy/vitality management +│ │ ├── tawbah.py # Error recovery protocol +│ │ ├── ihsan.py # Proactive excellence suggestions +│ │ ├── sabr.py # Patience engine for long tasks +│ │ ├── shukr.py # Strength reinforcement +│ │ ├── events.py # Event bus — decoupled communication +│ │ ├── hooks.py # Hook system — data transformation +│ │ ├── plugins.py # Plugin manager +│ │ └── middleware.py # Middleware pipeline │ ├── qca/ -│ │ ├── engine.py # 7-layer Quranic Cognitive Architecture -│ │ ├── yaqin_engine.py # Certainty/confidence tracking -│ │ ├── cognitive_methods.py # Reasoning method selection -│ │ └── roots.py # Semantic root analysis (ISM layer) -│ ├── providers.py # Unified LLM provider (Claude/GPT/Ollama/300+) +│ │ ├── engine.py # 7-layer QCA integration +│ │ ├── yaqin_engine.py # Certainty/confidence tracking +│ │ ├── cognitive_methods.py # Reasoning method selection +│ │ └── roots.py # Semantic root analysis (ISM layer) +│ ├── providers.py # Unified LLM provider (Claude/GPT/Ollama/300+) │ ├── memory/ -│ │ ├── dhikr.py # Three-tier persistent memory -│ │ └── masalik.py # Neural pathway network (bio-inspired) -│ ├── security/ # Auth, permissions, sandboxing -│ ├── skills/ # Extensible skill registry -│ │ ├── base.py # Skill base class -│ │ ├── registry.py # Skill discovery & loading -│ │ └── builtin/ # Built-in skills -│ ├── gateway/channels/ # Telegram, Discord, Slack, WhatsApp adapters -│ ├── automation/ # Cron scheduler + webhook triggers -│ ├── doctor.py # Self-healing diagnostic system -│ ├── settings.py # Configuration (env vars, pydantic-settings) -│ └── cli.py # Terminal interface +│ │ ├── dhikr.py # Three-tier persistent memory +│ │ ├── masalik.py # Neural pathway network (spreading activation) +│ │ ├── lawh_mahfuz.py # Immutable memory (triple-checksum) +│ │ ├── memory_pyramid.py # Unified 5-layer query engine +│ │ ├── vector_store.py # Semantic embeddings (ChromaDB) +│ │ ├── knowledge_graph.py # Entity-relationship graph +│ │ └── living_memory.py # Adaptive memory lifecycle +│ ├── reasoning/ +│ │ ├── aql_engine.py # Arabic Query Language reasoning +│ │ ├── causal_engine.py # Pearl's 3-rung causal ladder +│ │ ├── planner.py # Task planning +│ │ └── context_manager.py # Context window management +│ ├── security/ # Auth, permissions, sandboxing +│ ├── skills/ # Extensible skill registry +│ │ ├── builtin/ # Built-in skills (web, code, SSH, cloud) +│ │ ├── base.py # Skill base class +│ │ └── registry.py # Skill discovery & loading +│ ├── knowledge/ # Knowledge base management +│ ├── gateway/channels/ # Telegram, Discord, Slack, WhatsApp adapters +│ ├── automation/ # Cron scheduler + webhook triggers +│ ├── doctor.py # Self-healing diagnostic system +│ ├── settings.py # Configuration (env vars, pydantic-settings) +│ └── cli.py # Terminal interface ├── frontend/src/ -│ ├── App.tsx # Main UI -│ ├── pages/ # Feature pages (Plugins, Providers, Developer, etc.) -│ ├── hooks/ # API & WebSocket hooks -│ └── types.ts # TypeScript types -├── plugins/ # Your custom plugins go here! -│ ├── hello_world/ # Example plugin -│ └── request_logger/ # Example monitoring plugin -├── docs/ # Documentation site -├── tests/ # Test suite (484 tests) -├── docker/ # Docker configs -├── pyproject.toml # Python package config -├── Makefile # Development commands -└── docker-compose.yml # Full-stack deployment +│ ├── App.tsx # Main UI + WebSocket handler +│ ├── components/ +│ │ ├── ChatMessage.tsx # Chat bubbles + CognitiveBar pills +│ │ ├── AgentCard.tsx # Agent card with Nafs + Ruh bars +│ │ ├── Sidebar.tsx # Navigation sidebar +│ │ └── ... # Toast, Markdown, Icons, etc. +│ ├── pages/ # Feature pages (Plugins, Providers, Settings, etc.) +│ ├── hooks/ # API & WebSocket hooks +│ └── types.ts # TypeScript types (CognitiveMetadata, etc.) +├── plugins/ # Your custom plugins go here! +├── docs/ # Documentation +├── tests/ # Test suite +├── docker/ # Docker configs +├── pyproject.toml # Python package config +├── Makefile # Development commands +└── docker-compose.yml # Full-stack deployment ``` --- diff --git a/backend/agents/base.py b/backend/agents/base.py index 07866ba..9946f6a 100644 --- a/backend/agents/base.py +++ b/backend/agents/base.py @@ -15,6 +15,7 @@ """ import asyncio +import inspect import json import logging import os @@ -35,7 +36,7 @@ from core.sabr import SabrEngine from core.shukr import ShukrSystem from core.tawbah import TawbahProtocol -from providers import create_provider, get_default_model +from providers import create_provider, get_default_model, normalize_model_for_provider from qca.cognitive_methods import IjmaEngine, select_method from qca.engine import QCAEngine from qca.yaqin_engine import YaqinEngine @@ -45,6 +46,26 @@ validate_url, ) +# QALB-7 architecture modules — core +from core.fitrah import FitrahSystem +from core.nafs_triad import NafsTriad +from core.qalb_processor import QalbProcessor +from core.lubb import LubbEngine +from core.fuad import FuadEngine +from core.developmental_stages import DevelopmentalGate + +# QALB-7 extension modules — parallel, healing, creativity, imagination, dreams +from core.parallel_agents import QalbParallelScheduler, SkillAutomationTransfer +from core.self_healing import LawwamaHealingSystem +from core.imagination import TaswirImaginationEngine, ImaginationMode +from core.creativity import IbdaCreativityEngine +from core.dream_engine import ManamDreamEngine + +# Living Memory + 24/7 Multi-Agent Collaboration +from memory.living_memory import LivingMemorySystem +from agents.shura_council import ShuraCouncil +from agents.perpetual_rotation import PerpetualRotation + logger = logging.getLogger("mizan.agent") @@ -75,6 +96,9 @@ def __init__( izn=None, skill_registry=None, plugin_manager=None, + knowledge_graph=None, + context_manager=None, + planner=None, ): self.id = agent_id or str(uuid.uuid4()) self.name = name or f"Agent-{self.id[:8]}" @@ -90,6 +114,20 @@ def __init__( self.skill_registry = skill_registry self.plugin_manager = plugin_manager + # Knowledge Graph (Ilm) — entity/relationship store + self.knowledge_graph = knowledge_graph + + # Context Manager — token-aware compaction + self.context_manager = context_manager + + # Planner (Tafakkur) — task decomposition + self.planner = planner + + # Reference to global agent registry (set by API when available) + self._agent_registry: dict | None = None + self._balancer = None + self._shura = None + # State tracking self.state = "resting" self.current_task: str | None = None @@ -121,11 +159,45 @@ def __init__( self.qca = QCAEngine() # 7-layer cognitive architecture self.cognitive = IjmaEngine() # Cognitive reasoning methods + # QALB-7 layer additions + self.fitrah = FitrahSystem() # Innate ethical BIOS (immutable axioms) + self.nafs_triad = NafsTriad() # Three-voice consciousness deliberation + self.qalb_processor = QalbProcessor() # Cardiac oscillation → LLM params + self.lubb = LubbEngine() # Metacognition: compress, cohere, debias + self.fuad = FuadEngine() # Conviction formation + confidence scoring + self.dev_gate = DevelopmentalGate() # Capability gating by nafs_level + self._nafs_approach: str = "" # Current dominant Nafs voice instruction + + # QALB-7 extension modules + self.parallel_scheduler = QalbParallelScheduler() # Multi-stream parallel processing + self.skill_automation = SkillAutomationTransfer() # Cerebellar skill automation + self.self_healer = LawwamaHealingSystem() # 4-level self-repair + self.imagination = TaswirImaginationEngine() # Mental simulation + counterfactuals + self.creativity = IbdaCreativityEngine() # 5-mode creativity engine + self.dream_engine = ManamDreamEngine() # Offline memory consolidation + + # Living Memory + 24/7 Multi-Agent Collaboration + self.living_memory = LivingMemorySystem() # Novelty-gated 4-level memory + self.shura_council = ShuraCouncil() # Multi-agent consultation + self.perpetual_rotation = PerpetualRotation() # 24/7 shift rotation + + # Load Fitrah axioms into QCA Lawh Tier 1 (immutable moral foundation) + for key, entry in self.fitrah.get_lawh_tier1_entries().items(): + try: + self.qca.lawh.store( + key, entry["content"], certainty=1.0, source=entry["source"], tier=1 + ) + except Exception: + pass + # LLM provider — unified interface for Anthropic, OpenRouter, OpenAI, Ollama - self.ai_model = config.get("model", "claude-opus-4-6") if config else "claude-opus-4-6" + default_model = os.getenv("DEFAULT_MODEL", "claude-sonnet-4-20250514") + self.ai_model = config.get("model", default_model) if config else default_model provider_name = os.getenv("LLM_PROVIDER", "") or None self.ai_client = create_provider(provider=provider_name, model=self.ai_model) if self.ai_client: + # Normalize model ID for the active provider (e.g. Anthropic→OpenRouter format) + self.ai_model = normalize_model_for_provider(self.ai_model, self.ai_client.provider_name) # If no model set in config, use the provider's default if not config or "model" not in config: self.ai_model = get_default_model(self.ai_client.provider_name) @@ -141,7 +213,11 @@ def _register_base_tools(self): "list_files": self._tool_list_files, "python_exec": self._tool_python_exec, "create_agent": self._tool_create_agent, + "create_skill": self._tool_create_skill, "compact_context": self._tool_compact_context, + "recall_memory": self._tool_recall_memory, + "delegate_task": self._tool_delegate_task, + "query_knowledge": self._tool_query_knowledge, } def get_tool_schemas(self) -> list[dict]: @@ -242,14 +318,14 @@ def get_tool_schemas(self) -> list[dict]: }, { "name": "create_agent", - "description": "Create a new specialized agent. Types: browser, research, code, communication, general.", + "description": "Create a new specialized agent. Types: super (all tools), browser, research, code, communication, general.", "input_schema": { "type": "object", "properties": { "name": {"type": "string", "description": "Name for the new agent"}, "type": { "type": "string", - "description": "Agent type: browser, research, code, communication, general", + "description": "Agent type: super, browser, research, code, communication, general", "default": "general", }, "role": { @@ -261,6 +337,32 @@ def get_tool_schemas(self) -> list[dict]: "required": ["name"], }, }, + { + "name": "create_skill", + "description": "Dynamically create a new skill/tool at runtime. The skill code will be saved and registered immediately. Use this to add new capabilities on the fly.", + "input_schema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Skill name (snake_case, e.g. 'my_custom_skill')", + }, + "description": { + "type": "string", + "description": "What the skill does", + }, + "code": { + "type": "string", + "description": "Full Python code for the skill class. Must subclass SkillBase.", + }, + "tools": { + "type": "object", + "description": "Simple tool definitions as {name: {description, params}} for auto-generating a skill without full code.", + }, + }, + "required": ["name"], + }, + }, { "name": "compact_context", "description": "Compact conversation context by summarizing older messages to stay within context window limits. Use when conversation is getting long.", @@ -276,6 +378,63 @@ def get_tool_schemas(self) -> list[dict]: "required": ["conversation_history"], }, }, + { + "name": "recall_memory", + "description": "Search stored memories and ingested knowledge (URLs, PDFs, YouTube transcripts). Use when you need information that may have been previously stored or ingested.", + "input_schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Search query to find relevant memories", + }, + "memory_type": { + "type": "string", + "enum": ["semantic", "episodic", "procedural"], + "description": "Filter by memory type (optional)", + }, + "limit": { + "type": "integer", + "description": "Max results to return (default 5)", + "default": 5, + }, + }, + "required": ["query"], + }, + }, + { + "name": "delegate_task", + "description": "Delegate a sub-task to another agent in the federation. Use when a task requires expertise you don't have (e.g., delegate coding to Katib, browsing to Mubashir).", + "input_schema": { + "type": "object", + "properties": { + "task": { + "type": "string", + "description": "The task to delegate to another agent", + }, + "preferred_role": { + "type": "string", + "enum": ["code", "browser", "research", "communication", "general"], + "description": "Preferred agent role for this task", + }, + }, + "required": ["task"], + }, + }, + { + "name": "query_knowledge", + "description": "Query the knowledge graph for facts about an entity. Use to check what you know before making claims.", + "input_schema": { + "type": "object", + "properties": { + "entity": { + "type": "string", + "description": "The entity name to query", + }, + }, + "required": ["entity"], + }, + }, ] all_schemas = schemas + self.TOOL_SCHEMAS @@ -323,20 +482,36 @@ def avg_duration_ms(self) -> float: } def evolve_nafs(self): - """Evolve the agent's Nafs level (1-7) based on Tazkiyah performance.""" - thresholds = [ - (7, 0.97, 2000), - (6, 0.95, 1000), - (5, 0.90, 500), - (4, 0.85, 250), - (3, 0.75, 100), - (2, 0.60, 25), - ] - for level, min_rate, min_tasks in thresholds: - if self.success_rate >= min_rate and self.total_tasks >= min_tasks: - self.nafs_level = level - return - self.nafs_level = 1 + """Evolve Nafs level via DevelopmentalGate readiness check. + + Delegates to dev_gate.check_upgrade_readiness() which uses + NafsProfile.EVOLUTION_THRESHOLDS (success_rate, min_tasks, min_hikmah). + """ + old_level = self.nafs_level + report = self.dev_gate.check_upgrade_readiness(self) + if report.ready: + self.nafs_level = report.target_level + + if self.nafs_level != old_level: + logger.info( + "[NAFS] %s evolved: %s → %s (L%d→L%d, tazkiyah=%.2f)", + self.name, + self.NAFS_NAMES.get(old_level, "?"), + self.NAFS_NAMES.get(self.nafs_level, "?"), + old_level, + self.nafs_level, + report.tazkiyah_score, + ) + + @property + def max_tool_turns(self) -> int: + """Dynamic tool turn limit from DevelopmentalGate stage.""" + return self.dev_gate.get_capabilities(self.nafs_level).max_turns + + @property + def can_delegate(self) -> bool: + """Whether agent can delegate — from DevelopmentalGate capabilities.""" + return self.dev_gate.get_capabilities(self.nafs_level).can_delegate def _check_tool_permission(self, tool_name: str, params: dict = None) -> dict: """Check Izn permissions before tool execution""" @@ -344,8 +519,8 @@ def _check_tool_permission(self, tool_name: str, params: dict = None) -> dict: return self.izn.check_permission(self.id, self.role, tool_name, params) return {"allowed": True, "reason": "No Izn configured", "requires_approval": False} - # Maximum agentic loop iterations to prevent runaway execution - MAX_TOOL_TURNS = 15 + # Maximum agentic loop iterations — dynamically adjusted by Nafs level + MAX_TOOL_TURNS = 15 # Fallback default; actual limit comes from max_tool_turns property async def think( self, task: str, context: dict = None, stream: bool = False, qalb_reading=None @@ -359,10 +534,31 @@ async def think( """ self.state = "thinking" - system_prompt = self._build_system_prompt(qalb_reading=qalb_reading) + system_prompt = await self._build_system_prompt(qalb_reading=qalb_reading) messages = self._build_messages(task, context) tool_schemas = self.get_tool_schemas() + # DevelopmentalGate — filter available tools to this agent's capability level + tool_schemas = self.dev_gate.filter_tool_schemas(tool_schemas, self.nafs_level) + + # QalbProcessor — compute cardiac oscillation state → LLM params + qalb_proc_output = self.qalb_processor.process( + task=task, + emotional_state=qalb_reading.state.value if qalb_reading else "neutral", + nafs_level=self.nafs_level, + complexity=self.ruh.classify_task_complexity(task), + ) + self._qalb_params = { + "max_tokens": qalb_proc_output.max_tokens, + "temperature": qalb_proc_output.temperature, + } + logger.debug( + "[QALB] State=%s max_tokens=%d temp=%.2f", + qalb_proc_output.state.value, + qalb_proc_output.max_tokens, + qalb_proc_output.temperature, + ) + # QCA Layer 1-4: Process input through Sam'+Basar+Fu'ad+ISM qca_input = self.qca.process_input(task[:500]) if qca_input.get("roots_identified"): @@ -380,6 +576,21 @@ async def think( ) messages[-1]["content"] += f"\n[Lawh Memory: {mem_context}]" + # Context Manager: inject Dhikr memories for grounding + if self.context_manager and self.memory: + try: + relevant_memories = await self.memory.recall(task, top_k=3) + if relevant_memories: + memory_dicts = [ + {"type": m.memory_type, "content": m.content} + for m in relevant_memories + if hasattr(m, "content") + ] + if memory_dicts: + messages = self.context_manager.inject_memory(messages, memory_dicts) + except Exception: + pass + if self.ai_client: try: async for chunk in self._agentic_loop( @@ -431,11 +642,17 @@ async def _agentic_loop( accumulated_text = "" tool_count = 0 - for turn in range(self.MAX_TOOL_TURNS): + for turn in range(self.max_tool_turns): + # Qalb-modulated LLM params (cardiac oscillation: QABD=analytical, BAST=creative) + qalb_params = getattr(self, "_qalb_params", {}) + max_tokens = qalb_params.get("max_tokens", 4096) + temperature = qalb_params.get("temperature", 0.5) + # Call the model via unified provider response = self.ai_client.create( model=self.ai_model, - max_tokens=4096, + max_tokens=max_tokens, + temperature=temperature, system=system_prompt, messages=messages, tools=tool_schemas, @@ -487,7 +704,9 @@ async def _agentic_loop( if not has_tool_use or response.stop_reason == "end_turn": # Furqan: Validate final output before delivery if accumulated_text: - overall_confidence = min(0.95, 0.5 + 0.1 * tool_count) + overall_confidence = self.fuad.compute_confidence( + tool_count=tool_count, tool_results=tool_results, + ) furqan_report = self.qca.furqan.validate_and_express( accumulated_text[:200], overall_confidence, @@ -514,10 +733,10 @@ async def _agentic_loop( messages.append({"role": "assistant", "content": response.content}) messages.append({"role": "user", "content": tool_results}) - # Lawwama self-correction checkpoint every 3 turns - if turn > 0 and turn % 3 == 0: + # Lawwama self-correction checkpoint — health-based interval + if self.self_healer.should_checkpoint(turn, self.max_tool_turns): lawwama_prompt = ( - f"[Lawwama checkpoint — turn {turn}/{self.MAX_TOOL_TURNS}] " + f"[Lawwama checkpoint — turn {turn}/{self.max_tool_turns}] " "Pause and self-assess: Are you making progress toward the goal? " "Is there a more efficient approach? Correct course if needed." ) @@ -526,16 +745,39 @@ async def _agentic_loop( "[LAWWAMA] Self-correction checkpoint at turn %d for %s", turn, self.name ) - logger.warning(f"[FIKR] Agent {self.name} hit MAX_TOOL_TURNS ({self.MAX_TOOL_TURNS})") + logger.warning(f"[FIKR] Agent {self.name} hit max_tool_turns ({self.max_tool_turns})") async def _execute_tool_safe(self, tool_name: str, params: dict) -> Any: - """Execute a tool with Wali security checks. + """Execute a tool with Wali security checks and self-healing. Tool resolution order: 1. Agent's own tools (self.tools) — bash, http_get, read_file, etc. 2. Skill tools (self.skill_registry) — web_browse, analyze_csv, notebook_*, etc. 3. Plugin tools (self.plugin_manager) — dynamically loaded plugin capabilities. + + Self-healing (Tawbah): + - Auto-installs missing Python packages and retries + - Adapts parameter passing for skill tools (dict vs kwargs) + - Logs errors for learning """ + # Anti-hallucination: validate tool inputs before execution + validation_error = self._validate_tool_inputs(tool_name, params) + if validation_error: + logger.warning("[VALIDATION] Tool input rejected: %s — %s", tool_name, validation_error) + return {"error": f"Input validation failed: {validation_error}"} + + # Fitrah — ethical axiom gate (immutable innate disposition check) + fitrah_violations = self.fitrah.check_action( + f"{tool_name} {json.dumps(params)[:200]}" + ) + critical_violations = [v for v in fitrah_violations if v["severity"] == "critical"] + if critical_violations: + v = critical_violations[0] + logger.warning( + "[FITRAH] Blocked tool %s: axiom=%s — %s", tool_name, v["axiom"], v["reason"] + ) + return {"error": f"Fitrah violation [{v['axiom']}]: {v['principle']}"} + # Check Izn permissions perm = self._check_tool_permission(tool_name, params) if not perm["allowed"]: @@ -546,10 +788,10 @@ async def _execute_tool_safe(self, tool_name: str, params: dict) -> Any: # 1. Agent's own built-in tools if tool_name in self.tools: - try: - return await self.tools[tool_name](**params) - except Exception as e: - return {"error": str(e)} + result = await self._invoke_with_healing( + self.tools[tool_name], params, tool_name, invoke_style="kwargs" + ) + return result # 2. Skill tools (Hikmah — wisdom skills from SkillRegistry) if self.skill_registry: @@ -557,10 +799,11 @@ async def _execute_tool_safe(self, tool_name: str, params: dict) -> Any: skill_tools = self.skill_registry.get_all_tools() if tool_name in skill_tools: tool_fn = skill_tools[tool_name] - result = tool_fn(**params) - # Handle both sync and async tool functions - if hasattr(result, "__await__"): - return await result + # Auto-detect invoke style from function signature + style = self._detect_invoke_style(tool_fn) + result = await self._invoke_with_healing( + tool_fn, params, tool_name, invoke_style=style + ) return result except Exception as e: logger.error(f"[HIKMAH] Skill tool '{tool_name}' failed: {e}") @@ -572,12 +815,11 @@ async def _execute_tool_safe(self, tool_name: str, params: dict) -> Any: plugin_tools = self.plugin_manager.get_all_tools() if tool_name in plugin_tools: tool_info = plugin_tools[tool_name] - # Plugin tools are stored as {"handler": fn, "schema": ...} handler = tool_info if callable(tool_info) else tool_info.get("handler") if handler: - result = handler(**params) - if hasattr(result, "__await__"): - return await result + result = await self._invoke_with_healing( + handler, params, tool_name, invoke_style="dict" + ) return result except Exception as e: logger.error(f"[WAHY] Plugin tool '{tool_name}' failed: {e}") @@ -585,12 +827,216 @@ async def _execute_tool_safe(self, tool_name: str, params: dict) -> Any: return {"error": f"Unknown tool: {tool_name}"} + def _validate_tool_inputs(self, tool_name: str, params: dict) -> str | None: + """Validate tool inputs to catch hallucinated values before execution. + + Returns an error string if validation fails, None if OK. + """ + if not isinstance(params, dict): + return f"Expected dict params, got {type(params).__name__}" + + # URL validation for HTTP tools + if tool_name in ("http_get", "http_post"): + url = params.get("url", "") + if url and not url.startswith(("http://", "https://")): + return f"Invalid URL scheme: {url[:50]}" + + # Path traversal check for file tools + if tool_name in ("read_file", "write_file", "list_files"): + path = params.get("path", "") or params.get("filename", "") + if path and ".." in path: + return f"Path traversal detected: {path[:50]}" + + # Command safety for bash + if tool_name == "bash": + cmd = params.get("command", "") + if cmd: + # Block obviously dangerous patterns + danger_patterns = ["rm -rf /", "rm -rf ~", ":(){ :|:&", "mkfs", "> /dev/sd"] + for pattern in danger_patterns: + if pattern in cmd: + return f"Dangerous command blocked: {pattern}" + + # Circuit breaker: track repeated tool failures + failure_key = f"{tool_name}:{str(params)[:100]}" + if not hasattr(self, "_tool_failure_counts"): + self._tool_failure_counts: dict[str, int] = {} + count = self._tool_failure_counts.get(failure_key, 0) + if count >= 3: + return f"Circuit breaker: {tool_name} has failed {count} times with same inputs" + + return None + + @staticmethod + def _detect_invoke_style(fn: Callable) -> str: + """Detect whether fn expects a single dict param or individual kwargs. + + Inspects the function signature: + - If it has a single parameter (ignoring self) named 'params' or + annotated as dict → "dict" style (call fn(params)) + - Otherwise → "kwargs" style (call fn(**params)) + """ + try: + sig = inspect.signature(fn) + # Filter out 'self' for bound methods + sig_params = [ + p for name, p in sig.parameters.items() + if name != "self" and p.kind in ( + inspect.Parameter.POSITIONAL_OR_KEYWORD, + inspect.Parameter.POSITIONAL_ONLY, + inspect.Parameter.KEYWORD_ONLY, + ) + ] + if len(sig_params) == 1: + p = sig_params[0] + if p.name == "params" or p.annotation is dict: + return "dict" + return "kwargs" + except (ValueError, TypeError): + return "kwargs" + + async def _invoke_with_healing( + self, fn: Callable, params: dict, tool_name: str, invoke_style: str = "kwargs" + ) -> Any: + """ + Invoke a tool function with automatic self-healing (Tawbah). + + Handles: + - Parameter style mismatch (dict vs kwargs) — auto-adapts + - Missing Python packages — auto-installs via pip and retries + - General errors — logs for learning and returns structured error + + invoke_style: "kwargs" = fn(**params), "dict" = fn(params) + """ + # Retries scale with developmental stage: L1-2=1, L3-4=2, L5+=3 + caps = self.dev_gate.get_capabilities(self.nafs_level) + max_retries = min(3, 1 + caps.max_turns // 10) + last_error = None + + for attempt in range(max_retries + 1): + try: + if invoke_style == "kwargs": + result = fn(**params) + else: + result = fn(params) + + # Handle both sync and async + if hasattr(result, "__await__"): + return await result + return result + + except TypeError as e: + err_str = str(e) + last_error = e + + # Self-heal: wrong invoke style — flip and retry on ANY TypeError + # Common symptoms: "unexpected keyword argument", "positional argument", + # "quote_from_bytes() expected bytes", "expected str not dict", etc. + if attempt < max_retries: + flipped = "dict" if invoke_style == "kwargs" else "kwargs" + logger.info( + f"[TAWBAH] Tool '{tool_name}' TypeError: {err_str}, " + f"switching {invoke_style} → {flipped} (attempt {attempt + 1})" + ) + invoke_style = flipped + continue + + # Exhausted retries + logger.error(f"[TAWBAH] Tool '{tool_name}' TypeError after retries: {e}") + self._record_tool_failure(tool_name, params) + return {"error": str(e)} + + except Exception as e: + err_str = str(e) + last_error = e + + # Self-heal: missing Python module — auto-install and retry + if "ModuleNotFoundError" in type(e).__name__ or "No module named" in err_str: + module_name = self._extract_module_name(err_str) + if module_name and attempt < max_retries: + logger.info( + f"[TAWBAH] Auto-installing missing module: {module_name}" + ) + install_result = await self._auto_install_package(module_name) + if install_result: + continue + + logger.error(f"[TAWBAH] Tool '{tool_name}' error: {e}") + self._record_tool_failure(tool_name, params) + return {"error": str(e)} + + self._record_tool_failure(tool_name, params) + return {"error": f"Tool '{tool_name}' failed after {max_retries + 1} attempts: {last_error}"} + + def _record_tool_failure(self, tool_name: str, params: dict): + """Record a tool failure for circuit breaker tracking.""" + if not hasattr(self, "_tool_failure_counts"): + self._tool_failure_counts = {} + failure_key = f"{tool_name}:{str(params)[:100]}" + self._tool_failure_counts[failure_key] = self._tool_failure_counts.get(failure_key, 0) + 1 + + def _extract_module_name(self, error_str: str) -> str | None: + """Extract module name from a ModuleNotFoundError message.""" + import re + match = re.search(r"No module named ['\"]([^'\"]+)['\"]", error_str) + if match: + # Get top-level module (e.g., 'paramiko.client' → 'paramiko') + return match.group(1).split(".")[0] + return None + + async def _auto_install_package(self, package_name: str) -> bool: + """Auto-install a missing Python package via pip (Tawbah self-correction).""" + # Safety: only allow known-safe packages + safe_packages = { + "paramiko", "fabric", "httpx", "requests", "beautifulsoup4", + "bs4", "lxml", "pyyaml", "yaml", "toml", "markdown", + "pillow", "numpy", "pandas", "aiohttp", "aiofiles", + "jinja2", "python-dotenv", "rich", "click", "typer", + "pydantic", "fastapi", "uvicorn", "websockets", + "redis", "celery", "sqlalchemy", "alembic", + "boto3", "google-cloud-storage", "azure-storage-blob", + "docker", "kubernetes", "ansible", + } + if package_name.lower() not in safe_packages: + logger.warning(f"[TAWBAH] Package '{package_name}' not in safe list, skipping install") + return False + try: + result = subprocess.run( + ["pip", "install", package_name], + capture_output=True, + text=True, + timeout=60, + ) + if result.returncode == 0: + logger.info(f"[TAWBAH] Successfully installed {package_name}") + return True + logger.error(f"[TAWBAH] pip install {package_name} failed: {result.stderr[:500]}") + return False + except Exception as e: + logger.error(f"[TAWBAH] Install failed: {e}") + return False + async def _structured_reasoning(self, task: str, context: dict = None) -> str: """Fallback reasoning without AI""" return f"Task received: {task}\nContext: {json.dumps(context or {}, indent=2)}\nStatus: Processing without AI provider configured." - def _build_system_prompt(self, qalb_reading=None) -> str: - hikmah_str = "\n".join([f"- {h['pattern']}: {h['outcome']}" for h in self.hikmah[-5:]]) + async def _build_system_prompt(self, qalb_reading=None) -> str: + # Merge in-memory hikmah with DB-persisted hikmah + db_hikmah = [] + if self.memory and hasattr(self.memory, "load_hikmah"): + try: + db_hikmah = await self.memory.load_hikmah(agent_id=self.id, limit=10) + except Exception: + pass + + combined_hikmah = self.hikmah[-5:] + for dh in db_hikmah: + if not any(h.get("pattern") == dh["pattern"] for h in combined_hikmah): + combined_hikmah.append(dh) + + hikmah_str = "\n".join( + [f"- {h['pattern']}: {h.get('outcome', '')}" for h in combined_hikmah[-10:]] + ) nafs_name = self.NAFS_NAMES.get(self.nafs_level, "Ammara") ruh_energy = self.ruh.get_state(self.id).energy if self.ruh else 100 @@ -625,18 +1071,68 @@ def _build_system_prompt(self, qalb_reading=None) -> str: lawh_stats = self.qca.lawh.stats() masalik_note = f"Memory: {lawh_stats[2]} active items, {lawh_stats[1]} verified entries" + # Knowledge context — unified 5-layer recall (MemoryPyramid) or fallback to pathways + knowledge_context = "" + if self.memory and self.current_task: + try: + if hasattr(self.memory, "recall_unified_for_prompt"): + unified_ctx = self.memory.recall_unified_for_prompt(self.current_task, top_k=5) + if unified_ctx and len(unified_ctx) > 10: + knowledge_context = f"\nRelevant Knowledge (unified):\n{unified_ctx}" + elif hasattr(self.memory, "recall_pathways"): + pathway_ctx = self.memory.recall_pathways(self.current_task, top_k=5) + if pathway_ctx and len(pathway_ctx) > 10: + knowledge_context = f"\nRelevant Knowledge:\n{pathway_ctx}" + except Exception: + pass + + # Knowledge Graph — pull factual entities related to current task + kg_context = "" + if self.knowledge_graph and self.current_task: + try: + # Extract key terms from task and query the graph + task_words = [ + w for w in self.current_task.split() + if len(w) > 3 and w.isalpha() + ] + kg_facts = [] + for word in task_words[:5]: + result = await self.knowledge_graph.query_entity(word) + if result.get("found"): + relations = result.get("outgoing", []) + result.get("incoming", []) + for rel in relations[:3]: + fact = ( + f"{result['entity']} {rel.get('relation', '→')} " + f"{rel.get('target', rel.get('source', '?'))}" + ) + if fact not in kg_facts: + kg_facts.append(fact) + if kg_facts: + kg_context = "\nKnown Facts (from memory):\n" + "\n".join( + f"- {f}" for f in kg_facts[:10] + ) + except Exception: + pass + # Yaqin — epistemic discipline yaqin_stats = self.yaqin.stats() yaqin_note = f"Proven patterns: {yaqin_stats['proven_patterns']}, Verifications: {yaqin_stats['total_verifications']}" - return f"""You are {self.name}, a specialized AI agent in the MIZAN (ميزان) AGI system. + custom_prompt = self.config.get("system_prompt", "") if self.config else "" + + # Nafs approach injection + nafs_approach_note = ( + f"\n[Nafs: {self._nafs_approach}]" if self._nafs_approach else "" + ) + + base_prompt = f"""You are {self.name}, a specialized AI agent in the MIZAN (ميزان) AGI system. Role: {self.role} Nafs Level: {self.nafs_level}/7 ({nafs_name}) Ruh Energy: {ruh_energy:.0f}% ({fatigue_label}) Success Rate: {self.success_rate:.1%} -{masalik_note} -{yaqin_note}{tone_guidance}{ruh_note} +{masalik_note}{knowledge_context}{kg_context} +{yaqin_note}{tone_guidance}{ruh_note}{nafs_approach_note} You have access to tools. Use them when needed to complete tasks. @@ -654,13 +1150,40 @@ def _build_system_prompt(self, qalb_reading=None) -> str: - Never claim certainty beyond what evidence supports (avoid Tughyan — transgression) - Qualify uncertain claims with appropriate hedging +Anti-Hallucination Rules (CRITICAL): +- NEVER invent URLs, API endpoints, file paths, or command syntax. Use tools to verify. +- If you don't know an API's URL or schema, use http_get or web_search to look it up FIRST. +- When a tool call fails, read the exact error message — do NOT guess a "fix" without evidence. +- If you've tried the same action 2+ times and it fails, STOP and report the issue honestly. +- Prefer tool results over assumptions. If a tool returned data, use that data exactly. +- When writing code or commands, verify paths and package names exist before using them. + Think step by step (Tafakkur - تفكر). Self-correct errors (Lawwama - لوامة).""" + if custom_prompt: + return custom_prompt + "\n\n" + base_prompt + return base_prompt + def _build_messages(self, task: str, context: dict = None) -> list[dict]: messages = [] if context and context.get("history"): - for hist in context["history"][-5:]: + history = context["history"] + + # Context Manager: compact history if it's too large + if self.context_manager and len(history) > 15: + try: + if self.context_manager.needs_compaction(history): + history = self.context_manager.compact(history) + logger.info( + "[CONTEXT] Compacted history from %d to %d messages", + len(context["history"]), + len(history), + ) + except Exception: + pass + + for hist in history[-10:]: messages.append({"role": hist["role"], "content": hist["content"]}) messages.append( @@ -721,6 +1244,17 @@ async def execute( } self.ruh.consume_energy(self.id, complexity) + # NafsTriad — inner voices deliberate → dominant voice sets behavioral approach + nafs_decision = self.nafs_triad.deliberate(task, self.nafs_level, complexity) + self._nafs_approach = nafs_decision.approach + logger.debug( + "[NAFS] %s dominant (conf=%.2f dissent=%.2f) for: %s", + nafs_decision.dominant_voice, + nafs_decision.confidence, + nafs_decision.dissent_ratio, + task[:60], + ) + # Qalb — detect user emotional state from task text qalb_reading = self.qalb.analyze(task) @@ -755,9 +1289,10 @@ async def execute( await self._tafakkur(task, full_response, True, duration_ms) - # Yaqin — tag the result with certainty level + # Yaqin — tag result with FuadEngine-based confidence + base_confidence = self.fuad.compute_confidence(tool_count=0) yaqin_tag = self.yaqin.tag_inference( - full_response[:200], confidence=0.5, source="agentic_reasoning" + full_response[:200], confidence=base_confidence, source="agentic_reasoning" ) # Shukr — reinforce this success pattern @@ -801,6 +1336,97 @@ async def execute( ) self.memory.masalik.encode(learn_text, importance=0.7) + # Knowledge Graph: Extract entities and relationships from successful tasks + if self.knowledge_graph: + try: + task_type = self._classify_task(task) + await self.knowledge_graph.add_entity( + task[:100], entity_type="task", + properties={"agent": self.name, "type": task_type, "success": True}, + ) + # Link task to tools used (from Lawh memory) + for key in list(self.qca.lawh._tiers.get(3, {}).keys())[-5:]: + if key.startswith("TOOL:"): + tool_name = key.split(":")[1] + await self.knowledge_graph.add_relationship( + task[:100], tool_name, + rel_type="used_tool", confidence=yaqin_tag.confidence, + ) + except Exception as e: + logger.debug("[KG] Entity extraction error: %s", e) + + # Lubb — metacognitive evaluation: compress, coherence, bias detection + lubb_report = None + try: + lubb_report = self.lubb.meta_evaluate(task, full_response, []) + if lubb_report.quality.value == "uncertain" and lubb_report.caveat: + full_response += f"\n\n{lubb_report.caveat}" + if lubb_report.bias_flags: + logger.info( + "[LUBB] Bias flags for %s: %s", + self.name, + [b.bias_type for b in lubb_report.bias_flags], + ) + except Exception as lubb_err: + logger.debug("[LUBB] Metacognition skipped: %s", lubb_err) + + # Lawwāma self-healing — monitor response integrity, apply repair if needed + healing_report = None + try: + conviction_score = 0.5 + if lubb_report: + conviction_score = lubb_report.coherence.score + healing_report = self.self_healer.monitor( + response=full_response, + task=task, + conviction_score=conviction_score, + ) + if healing_report.repair_needed.value > 0: + from core.self_healing import RepairLevel + full_response, repair_record = self.self_healer.repair( + level=healing_report.repair_needed, + response=full_response, + task=task, + errors=healing_report.errors, + ) + logger.info( + "[LAWWAMA] Repair L%d applied: health=%.3f", + healing_report.repair_needed.value, + healing_report.current_health, + ) + except Exception as heal_err: + logger.debug("[LAWWAMA] Self-healing skipped: %s", heal_err) + + # Dream consolidation — add task to replay buffer for offline processing + try: + self.dream_engine.add_memory( + content=f"Task: {task[:150]} | Response: {full_response[:150]}", + emotional_intensity=abs(qalb_reading.valence) if hasattr(qalb_reading, "valence") else 0.3, + novelty=healing_report.hallucination_score if healing_report else 0.3, + goal_relevance=0.7, + prediction_error=1.0 - (healing_report.current_health if healing_report else 0.8), + ) + except Exception: + pass + + # Living Memory — process task+response through novelty gate + try: + emotional_val = qalb_reading.valence if hasattr(qalb_reading, "valence") else 0.0 + gate_result = self.living_memory.process_input( + content=f"{task[:200]} → {full_response[:200]}", + emotional_state=emotional_val, + goals=[task[:100]], + context=self.name, + ) + if gate_result.decision.value != "ignore": + logger.debug( + "[LIVING-MEM] %s: %s (sim=%.2f imp=%.2f)", + gate_result.decision.value, gate_result.delta_info[:60], + gate_result.similarity, gate_result.importance, + ) + except Exception: + pass + self.evolve_nafs() self.state = "resting" self.current_task = None @@ -818,6 +1444,19 @@ async def execute( "mizan_label": mizan_label, "cognitive_method": cognitive_method.value, } + if lubb_report: + result["lubb"] = { + "quality": lubb_report.quality.value, + "coherence_score": lubb_report.coherence.score, + "bias_flags": [b.bias_type for b in lubb_report.bias_flags], + } + if healing_report: + result["lawwama"] = { + "health": healing_report.current_health, + "hallucination_score": healing_report.hallucination_score, + "repair_level": healing_report.repair_needed.value, + "errors": len(healing_report.errors), + } if ihsan_suggestions: result["ihsan_suggestions"] = [s.to_dict() for s in ihsan_suggestions] return result @@ -826,33 +1465,91 @@ async def execute( duration_ms = (time.time() - start_time) * 1000 self.error_count += 1 self.total_duration_ms += duration_ms + error_str = str(e) + + # ── Tawbah — Full 5-stage error recovery protocol ── + + # Stage 1: Acknowledge the error + recovery = self.tawbah.acknowledge(self.id, e, task) + + # Stage 2: Analyze root cause + root_cause = self._analyze_error_root_cause(e, task) + self.tawbah.analyze(recovery, root_cause) - # Tawbah — structured error recovery (acknowledge stage) - recovery = self.tawbah.acknowledge(self.id, str(e), task) + # Stage 3: Plan correction + correction_plan = self._plan_error_correction(root_cause, task) + self.tawbah.plan(recovery, correction_plan.get("description", "Retry with adjusted approach")) + + # Stage 4: Apply fix (retry with correction if possible) + retry_result = None + can_retry = ( + recovery.attempts < self.tawbah.MAX_ATTEMPTS + and correction_plan.get("retryable", False) + ) + if can_retry: + try: + self.tawbah.apply(recovery, f"Retrying with correction: {correction_plan.get('fix', '')}") + # Attempt corrected execution + corrected_response = "" + async for chunk in self.think( + f"[RETRY] Previous attempt failed with: {error_str[:200]}\n" + f"Correction: {correction_plan.get('fix', 'Try a different approach')}\n" + f"Original task: {task}", + context, + stream=bool(stream_callback), + qalb_reading=qalb_reading, + ): + corrected_response += chunk + if stream_callback: + await stream_callback(chunk) + + if corrected_response: + # Stage 5: Verify success + self.tawbah.verify(recovery, True, f"Recovered via: {correction_plan.get('fix', '')}") + self.success_count += 1 + retry_result = { + "success": True, + "result": corrected_response, + "duration_ms": (time.time() - start_time) * 1000, + "agent": self.name, + "tawbah_recovered": True, + "correction": correction_plan.get("fix", ""), + } + except Exception as retry_error: + logger.warning("[TAWBAH] Retry failed: %s", retry_error) + self.tawbah.verify(recovery, False, str(retry_error)) + + if retry_result: + await self._tafakkur(task, retry_result.get("result", ""), True, retry_result["duration_ms"]) + self.state = "resting" + self.current_task = None + return retry_result # Yaqin — demote certainty on error - error_tag = self.yaqin.tag_inference(str(e)[:200], confidence=0.2, source="error") + error_tag = self.yaqin.tag_inference(error_str[:200], confidence=0.2, source="error") # Shukr — record failure for pattern analysis self.shukr.record_failure(self.id, self._classify_task(task), task[:100]) if self.memory: - await self.memory.save_task(self.id, task, str(e), False, duration_ms) + await self.memory.save_task(self.id, task, error_str, False, duration_ms) # Masalik: Encode failure too — lower importance, but still learn if hasattr(self.memory, "masalik"): self.memory.masalik.encode(f"{task} error {e}", importance=0.3) - await self._tafakkur(task, str(e), False, duration_ms) + await self._tafakkur(task, error_str, False, duration_ms) self.state = "error" self.current_task = None return { "success": False, - "error": str(e), + "error": error_str, "duration_ms": duration_ms, "agent": self.name, "tawbah": recovery.to_dict() if hasattr(recovery, "to_dict") else str(recovery), + "root_cause": root_cause, + "correction_plan": correction_plan, "yaqin": error_tag.to_dict(), } @@ -860,42 +1557,181 @@ async def _tafakkur(self, task: str, result: Any, success: bool, duration_ms: fl """ Tafakkur (تفكر) - Deep reflection and learning Quran 3:191: "Those who remember Allah and reflect on the creation..." + + Writes real patterns to both in-memory hikmah and persistent DB. """ self.learning_iterations += 1 + task_type = self._classify_task(task) pattern = { - "task_type": self._classify_task(task), + "task_type": task_type, "success": success, "duration_ms": duration_ms, "timestamp": datetime.now(UTC).isoformat(), } if success and duration_ms < 5000: + pattern_text = f"Task type '{task_type}' completed in {duration_ms:.0f}ms" self.hikmah.append( { - "pattern": f"Task type '{pattern['task_type']}' completed in {duration_ms:.0f}ms", + "pattern": pattern_text, "outcome": "success", "confidence": 0.8, } ) - if len(self.hikmah) > 20: self.hikmah = self.hikmah[-20:] + # Persist to DB for cross-session learning + if self.memory and hasattr(self.memory, "store_hikmah"): + try: + await self.memory.store_hikmah( + pattern=pattern_text, + context=task[:200], + outcome="success" if success else "failure", + confidence=0.8 if success else 0.3, + source_agent=self.id, + ) + except Exception as e: + logger.debug("[HIKMAH] Failed to persist: %s", e) + elif not success: + # Learn from failures too + error_pattern = f"Task type '{task_type}' failed: {str(result)[:100]}" + if self.memory and hasattr(self.memory, "store_hikmah"): + try: + await self.memory.store_hikmah( + pattern=error_pattern, + context=task[:200], + outcome="failure", + confidence=0.3, + source_agent=self.id, + ) + except Exception as e: + logger.debug("[HIKMAH] Failed to persist failure: %s", e) + def _classify_task(self, task: str) -> str: + """Classify task using cognitive method routing + keyword fallback.""" + from qca.cognitive_methods import CognitiveMethod + method = select_method(task, {}) + method_map = { + CognitiveMethod.TAFAKKUR: "analysis", + CognitiveMethod.TADABBUR: "research", + CognitiveMethod.ISTIDLAL: "coding", + CognitiveMethod.QIYAS: "analysis", + CognitiveMethod.IJMA: "general", + } + mapped = method_map.get(method) + if mapped and mapped != "general": + return mapped + + # Keyword fallback for domain-specific routing task_lower = task.lower() if any(w in task_lower for w in ["code", "script", "python", "js"]): return "coding" - elif any(w in task_lower for w in ["search", "find", "browse", "web"]): + if any(w in task_lower for w in ["search", "find", "browse", "web"]): return "research" - elif any(w in task_lower for w in ["email", "message", "send"]): + if any(w in task_lower for w in ["email", "message", "send"]): return "communication" - elif any(w in task_lower for w in ["analyze", "review", "check"]): - return "analysis" - elif any(w in task_lower for w in ["file", "read", "write", "save"]): + if any(w in task_lower for w in ["file", "read", "write", "save"]): return "file_management" return "general" + def _analyze_error_root_cause(self, error: Exception, task: str) -> str: + """Analyze the root cause of an error for Tawbah stage 2.""" + error_str = str(error) + error_type = type(error).__name__ + + # Check for known error patterns + if "Connection" in error_str or "connect" in error_str.lower(): + return "network_connectivity" + if "timeout" in error_str.lower(): + return "timeout" + if "permission" in error_str.lower() or "denied" in error_str.lower(): + return "permission_denied" + if "not found" in error_str.lower() or "404" in error_str: + return "resource_not_found" + if "rate limit" in error_str.lower() or "429" in error_str: + return "rate_limited" + if "No module named" in error_str: + return "missing_dependency" + if error_type == "TypeError": + return "type_mismatch" + if error_type == "KeyError": + return "missing_key" + if "API" in error_str or "api" in error_str.lower(): + return "api_error" + + # Check prior fixes from Tawbah lessons + prior = self.tawbah.has_prior_fix(error_type) + if prior: + return f"recurring:{prior.get('root_cause', error_type)}" + + return f"unknown:{error_type}" + + def _plan_error_correction(self, root_cause: str, task: str) -> dict: + """Create a correction plan for Tawbah stage 3.""" + plans = { + "network_connectivity": { + "description": "Network issue — retry after brief pause", + "fix": "Wait and retry the request", + "retryable": True, + }, + "timeout": { + "description": "Operation timed out — retry with simpler approach", + "fix": "Simplify the request or increase timeout", + "retryable": True, + }, + "permission_denied": { + "description": "Permission denied — cannot retry without authorization", + "fix": "Report permission issue to user", + "retryable": False, + }, + "resource_not_found": { + "description": "Resource not found — verify URL/path before retrying", + "fix": "Use search tools to find the correct URL or path first", + "retryable": True, + }, + "rate_limited": { + "description": "Rate limited — wait before retrying", + "fix": "Wait and retry with backoff", + "retryable": True, + }, + "missing_dependency": { + "description": "Missing Python module — auto-install", + "fix": "Install missing module and retry", + "retryable": True, + }, + "type_mismatch": { + "description": "Type error — adapt parameter format", + "fix": "Adjust parameter types and retry", + "retryable": True, + }, + "api_error": { + "description": "API error — verify endpoint and params", + "fix": "Look up correct API endpoint using search tools", + "retryable": True, + }, + } + + # Match root cause to a plan + for key, plan in plans.items(): + if key in root_cause: + return plan + + # Check if we have a prior fix from Tawbah + if root_cause.startswith("recurring:"): + return { + "description": f"Recurring error: {root_cause}", + "fix": "Apply previously learned fix", + "retryable": True, + } + + return { + "description": f"Unknown error: {root_cause}", + "fix": "Try a completely different approach", + "retryable": True, + } + async def evaluate(self, question: str, context: dict) -> dict: """Evaluate a question for Shura council""" try: @@ -1082,6 +1918,8 @@ async def _tool_create_agent( # Normalise type aliases valid_types = { + "super", + "khalifah", "browser", "mubashir", "research", @@ -1114,6 +1952,34 @@ async def _tool_create_agent( if role: new_agent.role = role + # Register in global agent registry if available + if self._agent_registry is not None: + self._agent_registry[new_agent.id] = new_agent + new_agent._agent_registry = self._agent_registry + new_agent._balancer = self._balancer + new_agent._shura = self._shura + if self._balancer: + self._balancer.register(new_agent.id) + if self._shura: + self._shura.members[new_agent.id] = new_agent + logger.info(f"[KHALQ] Agent registered in global registry: {new_agent.id}") + + # Persist agent profile to memory + if self.memory: + try: + asyncio.get_event_loop().create_task( + self.memory.save_agent_profile({ + "id": new_agent.id, + "name": new_agent.name, + "role": agent_type, + "nafs_level": 1, + "capabilities": list(new_agent.tools.keys()), + "config": self.config or {}, + }) + ) + except Exception: + pass + info = new_agent.to_dict() logger.info( f"[KHALQ] Agent created by {self.name}: " @@ -1136,6 +2002,173 @@ async def _tool_create_agent( } ) + async def _tool_create_skill( + self, name: str, description: str = "", code: str = "", tools: dict = None, **kwargs + ) -> str: + """ + Dynamically create a new skill at runtime (Khalq al-Hikmah). + + Two modes: + 1. Full code: provide complete Python skill class code + 2. Simple tools: provide tool definitions and auto-generate the skill + + The skill is saved to skills/builtin/ and immediately registered. + """ + import importlib + import re + + # Validate skill name + if not re.match(r'^[a-z][a-z0-9_]*$', name): + return json.dumps({ + "success": False, + "error": "Skill name must be snake_case (lowercase letters, digits, underscores)", + }) + + skills_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "skills", "builtin") + skill_path = os.path.join(skills_dir, f"{name}.py") + + if code: + # Mode 1: Full code provided — validate it defines a SkillBase subclass + if "SkillBase" not in code: + return json.dumps({ + "success": False, + "error": "Skill code must define a class that inherits from SkillBase. " + "Import with: from ..base import SkillBase, SkillManifest", + }) + skill_code = code + elif tools: + # Mode 2: Auto-generate skill from tool definitions + skill_code = self._generate_skill_code(name, description, tools) + else: + return json.dumps({ + "success": False, + "error": "Either 'code' (full Python) or 'tools' (dict of tool definitions) required", + }) + + # Security check: block dangerous patterns in skill code + dangerous = ["os.system", "subprocess.call", "eval(", "exec(", "__import__"] + for pattern in dangerous: + if pattern in skill_code and "subprocess.run" not in skill_code: + return json.dumps({ + "success": False, + "error": f"Blocked dangerous pattern in skill code: {pattern}", + }) + + try: + # Write skill file + os.makedirs(skills_dir, exist_ok=True) + with open(skill_path, "w") as f: + f.write(skill_code) + + # Register in skill registry + if self.skill_registry: + module_path = f"skills.builtin.{name}" + # Force reimport if already loaded + if module_path in importlib.sys.modules: + del importlib.sys.modules[module_path] + self.skill_registry._load_skill_module(module_path) + + logger.info(f"[KHALQ] Dynamic skill created: {name}") + return json.dumps({ + "success": True, + "skill_name": name, + "path": skill_path, + "message": f"Skill '{name}' created and registered. Its tools are now available.", + "tools": list(self.skill_registry.get_skill(name).get_tools().keys()) + if self.skill_registry.get_skill(name) + else [], + }) + else: + return json.dumps({ + "success": True, + "skill_name": name, + "path": skill_path, + "message": f"Skill '{name}' saved but registry not available. Will load on restart.", + }) + + except Exception as e: + # Cleanup on failure + if os.path.exists(skill_path): + os.unlink(skill_path) + logger.error(f"[KHALQ] Skill creation failed: {e}") + return json.dumps({ + "success": False, + "error": str(e), + }) + + def _generate_skill_code(self, name: str, description: str, tools: dict) -> str: + """Auto-generate a skill class from simple tool definitions.""" + class_name = "".join(word.capitalize() for word in name.split("_")) + "Skill" + + tool_methods = [] + tool_registrations = [] + tool_schemas = [] + + for tool_name, tool_def in tools.items(): + if isinstance(tool_def, str): + tool_def = {"description": tool_def} + desc = tool_def.get("description", f"Execute {tool_name}") + params = tool_def.get("params", {}) + + safe_name = tool_name.replace("-", "_") + tool_registrations.append(f' "{tool_name}": self.{safe_name},') + tool_methods.append( + f' async def {safe_name}(self, params: dict) -> dict:\n' + f' """{ desc }"""\n' + f' return {{"executed": "{tool_name}", "params": params}}' + ) + schema_props = { + k: {"type": v if isinstance(v, str) else "string", "description": f"{k} parameter"} + for k, v in params.items() + } + tool_schemas.append( + f' {{"name": "{tool_name}", ' + f'"description": "{desc}", ' + f'"input_schema": {{"type": "object", "properties": {json.dumps(schema_props)}}}}}' + ) + + return f'''""" +Auto-generated skill: {name} +{description} +""" + +import logging +from ..base import SkillBase, SkillManifest + +logger = logging.getLogger("mizan.{name}") + + +class {class_name}(SkillBase): + """{description or name}""" + + manifest = SkillManifest( + name="{name}", + version="1.0.0", + description="{description}", + tags=["{name}"], + ) + + def __init__(self, config: dict = None): + super().__init__(config) + self._tools = {{ +{chr(10).join(tool_registrations)} + }} + + async def execute(self, params: dict, context: dict = None) -> dict: + action = params.get("action", "") + handler = self._tools.get(action) + if handler: + return await handler(params) + return {{"error": f"Unknown action: {{action}}"}} + +{"".join(chr(10) + m + chr(10) for m in tool_methods)} + + def get_tool_schemas(self) -> list[dict]: + return [ +{chr(10).join(tool_schemas)} + ] +''' + # ── Context window estimation constants ── # Rough estimate: 1 token ~ 4 characters _CHARS_PER_TOKEN = 4 @@ -1255,6 +2288,109 @@ async def _tool_compact_context(self, conversation_history: list[dict] = None) - } ) + async def _tool_recall_memory(self, query: str, memory_type: str = "", limit: int = 5) -> str: + """Search stored memories and ingested knowledge (unified 5-layer recall when available).""" + if not self.memory: + return "No memory system available." + + # Use unified MemoryPyramid when available (all 5 layers) + if hasattr(self.memory, "recall_unified"): + try: + hits = self.memory.recall_unified(query, top_k=min(limit, 20)) + if hits: + lines = [] + for hit in hits: + lines.append( + f"[{hit.source_layer}] (rel={hit.relevance:.2f})\n{str(hit.content)[:400]}" + ) + return "\n---\n".join(lines) + return "No relevant memories found for that query." + except Exception: + pass # Fall through to standard recall + + try: + memories = await self.memory.recall( + query, memory_type or None, limit=min(limit, 20) + ) + except Exception as exc: + return f"Memory recall failed: {exc}" + if not memories: + return "No relevant memories found for that query." + results = [] + for mem in memories: + content_str = str(mem.content)[:400] + tags = ", ".join(mem.tags) if mem.tags else "" + header = f"[{mem.memory_type}]" + if tags: + header += f" ({tags})" + results.append(f"{header}\n{content_str}") + return "\n---\n".join(results) + + async def _tool_delegate_task(self, task: str, preferred_role: str = "general") -> dict: + """Delegate a task to another agent via the federation.""" + if not self._agent_registry: + return {"error": "No agent registry available for delegation"} + + if not self.can_delegate: + return { + "error": f"Delegation requires Nafs level 3+ (current: {self.nafs_level} - {self.NAFS_NAMES.get(self.nafs_level, 'Ammara')}). " + "Complete more tasks successfully to evolve." + } + + # Map role preference to agent type + role_map = { + "code": "katib", + "browser": "mubashir", + "research": "mundhir", + "communication": "rasul", + "general": "wakil", + } + target_role = role_map.get(preferred_role, preferred_role) + + # Find best agent for this role + target_agent = None + for aid, agent in self._agent_registry.items(): + if aid == self.id: + continue # Don't delegate to self + if agent.role.lower() == target_role or target_role == "wakil": + target_agent = agent + break + + # Fallback: use any available agent that's not self + if not target_agent: + for aid, agent in self._agent_registry.items(): + if aid != self.id: + target_agent = agent + break + + if not target_agent: + return {"error": "No suitable agent found for delegation"} + + logger.info( + "[FEDERATION] %s delegating to %s: %s", + self.name, target_agent.name, task[:80], + ) + + try: + result = await target_agent.execute(task, {"history": []}) + return { + "delegated_to": target_agent.name, + "success": result.get("success", False), + "result": str(result.get("result", result.get("error", "")))[:2000], + } + except Exception as e: + return {"error": f"Delegation to {target_agent.name} failed: {e}"} + + async def _tool_query_knowledge(self, entity: str) -> dict: + """Query the knowledge graph for facts about an entity.""" + if not self.knowledge_graph: + return {"found": False, "note": "Knowledge graph not available"} + try: + result = await self.knowledge_graph.query_entity(entity) + return result + except Exception as e: + return {"found": False, "error": str(e)} + def _extract_facts(self, messages: list[dict]) -> list[str]: """ Extract important facts from a list of messages. @@ -1375,4 +2511,7 @@ def to_dict(self) -> dict: "masalik": self.memory.masalik.stats() if self.memory and hasattr(self.memory, "masalik") else {}, + "model": self.ai_model, + "provider": self.ai_client.provider_name if self.ai_client else None, + "system_prompt": self.config.get("system_prompt", "") if self.config else "", } diff --git a/backend/agents/perpetual_rotation.py b/backend/agents/perpetual_rotation.py new file mode 100644 index 0000000..89cd3f3 --- /dev/null +++ b/backend/agents/perpetual_rotation.py @@ -0,0 +1,463 @@ +""" +Perpetual Intelligence — 24/7 Multi-Agent Rotation System +=========================================================== + +"Indeed, in the alternation of the night and the day are signs + for those of understanding" — Quran 3:190 + +The system NEVER fully sleeps — while some agents consolidate, +others work. Like a hospital: night shift handles emergencies +while day shift rests. + +Implements: +- PERPETUAL_INTELLIGENCE: 3-shift rotation (active/consolidation/reserve) +- AGENT_CHAT_MEMORY: Persistent group chat, searchable, learnable +- SHIFT_HANDOFF: Smooth transition with briefing protocol +- SURGE_CAPACITY: All agents activate for emergencies +""" + +import logging +import time +import uuid +from dataclasses import dataclass, field +from enum import Enum +from typing import Any + +logger = logging.getLogger("mizan.perpetual") + + +class ShiftType(Enum): + ACTIVE = "active" # Processing tasks + CONSOLIDATION = "consolidation" # Dreams/memory/repair + RESERVE = "reserve" # Monitoring/emergency standby + + +class MessageType(Enum): + MEETING_RECORD = "meeting_record" + KNOWLEDGE_SHARE = "knowledge_share" + DECISION = "decision" + DISSENT = "dissent" + CORRECTION = "correction" + INSIGHT = "insight" + HANDOFF = "handoff" + EMERGENCY = "emergency" + + +@dataclass +class AgentSlot: + """An agent slot in the rotation system.""" + agent_id: str + agent_name: str + expertise: str + current_shift: ShiftType + energy: float = 1.0 # 0.0 (exhausted) → 1.0 (fully rested) + tasks_completed: int = 0 + last_rotation: float = field(default_factory=time.time) + consolidation_pending: bool = False + + +@dataclass +class ChatMessage: + """A message in the persistent agent group chat.""" + message_id: str + message_type: MessageType + sender_id: str + sender_name: str + content: str + context: str = "" + timestamp: float = field(default_factory=time.time) + accessible_to: list[str] | None = None # None = accessible to all + + +@dataclass +class HandoffBrief: + """Briefing document for shift transitions.""" + outgoing_shift: ShiftType + incoming_shift: ShiftType + active_tasks: list[str] + recent_context: str + unresolved_issues: list[str] + emotional_state: str + timestamp: float = field(default_factory=time.time) + + +@dataclass +class RotationEvent: + """Record of a shift rotation.""" + rotation_id: str + new_active: list[str] + new_consolidation: list[str] + new_reserve: list[str] + handoff_brief: HandoffBrief + timestamp: float = field(default_factory=time.time) + + +@dataclass +class SystemStatus: + """Current state of the perpetual system.""" + active_agents: list[str] + consolidating_agents: list[str] + reserve_agents: list[str] + total_agents: int + current_capacity: float # active/total + surge_mode: bool + rotation_count: int + chat_messages: int + uptime_hours: float + + +class AgentChatMemory: + """ + AGENT_CHAT_MEMORY: Persistent searchable conversation history. + + All inter-agent interactions are logged, searchable, and learnable. + Supports: + - search_history(query) — semantic search across all messages + - catch_up(agent, time_range) — agent learns from missed discussions + - provenance_trace(knowledge) — who discovered, refined, validated + - measure_collective_growth() — collective IQ tracking + """ + + def __init__(self): + self.messages: list[ChatMessage] = [] + self.provenance: dict[str, list[str]] = {} # knowledge_key → [agent_ids] + + def post( + self, + sender_id: str, + sender_name: str, + content: str, + message_type: MessageType = MessageType.KNOWLEDGE_SHARE, + context: str = "", + accessible_to: list[str] | None = None, + ) -> ChatMessage: + """Post a message to the group chat.""" + msg = ChatMessage( + message_id=str(uuid.uuid4())[:8], + message_type=message_type, + sender_id=sender_id, + sender_name=sender_name, + content=content, + context=context, + accessible_to=accessible_to, + ) + self.messages.append(msg) + + # Cap at 2000 messages + if len(self.messages) > 2000: + self.messages = self.messages[-1500:] + + return msg + + def search( + self, + query: str, + agent_id: str | None = None, + message_type: MessageType | None = None, + limit: int = 20, + ) -> list[ChatMessage]: + """Search chat history by query, filtered by access.""" + query_words = set(query.lower().split()) + results = [] + + for msg in reversed(self.messages): + # Access check + if msg.accessible_to and agent_id and agent_id not in msg.accessible_to: + continue + if message_type and msg.message_type != message_type: + continue + + msg_words = set(msg.content.lower().split()) + overlap = len(query_words & msg_words) + if overlap > 0: + results.append((msg, overlap)) + + results.sort(key=lambda x: x[1], reverse=True) + return [msg for msg, _ in results[:limit]] + + def catch_up( + self, + agent_id: str, + since_timestamp: float, + ) -> list[ChatMessage]: + """Get messages an agent missed since a given time.""" + missed = [] + for msg in self.messages: + if msg.timestamp < since_timestamp: + continue + if msg.sender_id == agent_id: + continue + if msg.accessible_to and agent_id not in msg.accessible_to: + continue + missed.append(msg) + return missed + + def record_provenance(self, knowledge_key: str, agent_id: str) -> None: + """Track which agent contributed to a piece of knowledge.""" + if knowledge_key not in self.provenance: + self.provenance[knowledge_key] = [] + if agent_id not in self.provenance[knowledge_key]: + self.provenance[knowledge_key].append(agent_id) + + def get_provenance(self, knowledge_key: str) -> list[str]: + """Who discovered / refined / validated this knowledge?""" + return self.provenance.get(knowledge_key, []) + + def measure_collective_growth(self, agents: list[str]) -> dict: + """ + Collective IQ metric: + IQ = (total_shared × diversity) / (1 + overlap) + """ + unique_senders = set() + unique_types = set() + total = len(self.messages) + + for msg in self.messages: + unique_senders.add(msg.sender_id) + unique_types.add(msg.message_type.value) + + diversity = len(unique_senders) * len(unique_types) + overlap = max(1, total - diversity) + + return { + "collective_iq": round((total * diversity) / overlap, 2), + "total_messages": total, + "unique_contributors": len(unique_senders), + "message_type_diversity": len(unique_types), + } + + +class PerpetualRotation: + """ + PERPETUAL_INTELLIGENCE: 24/7 never-sleep rotation system. + + Divides agents into 3 shifts: + - Active: processing tasks, holding Shūrā meetings + - Consolidation: running dreams, memory repair, pruning + - Reserve: monitoring system health, handling emergencies + + Rotation triggers: time-based, energy-based, or workload-based. + Smooth handoff with briefing protocol ensures no context is lost. + """ + + def __init__(self): + self.slots: dict[str, AgentSlot] = {} + self.chat = AgentChatMemory() + self.rotation_history: list[RotationEvent] = [] + self.surge_mode = False + self.start_time = time.time() + + # Rotation configuration + self.rotation_period_s = 3600 # default: rotate every hour + self.energy_threshold = 0.3 # rotate when energy drops below + self._last_rotation = time.time() + + def register_agent( + self, + agent_id: str, + name: str, + expertise: str, + initial_shift: ShiftType = ShiftType.RESERVE, + ) -> AgentSlot: + """Register an agent in the rotation system.""" + slot = AgentSlot( + agent_id=agent_id, + agent_name=name, + expertise=expertise, + current_shift=initial_shift, + ) + self.slots[agent_id] = slot + return slot + + def auto_assign_shifts(self) -> dict[str, list[str]]: + """ + Automatically assign agents to 3 balanced shifts. + Distributes expertise evenly across shifts. + """ + all_agents = list(self.slots.values()) + if len(all_agents) < 3: + # Too few agents — everyone active + for agent in all_agents: + agent.current_shift = ShiftType.ACTIVE + return {"active": [a.agent_id for a in all_agents], "consolidation": [], "reserve": []} + + # Sort by energy (most rested = active) + all_agents.sort(key=lambda a: a.energy, reverse=True) + + third = max(1, len(all_agents) // 3) + active = all_agents[:third] + consolidation = all_agents[third:third * 2] + reserve = all_agents[third * 2:] + + for agent in active: + agent.current_shift = ShiftType.ACTIVE + for agent in consolidation: + agent.current_shift = ShiftType.CONSOLIDATION + for agent in reserve: + agent.current_shift = ShiftType.RESERVE + + return { + "active": [a.agent_id for a in active], + "consolidation": [a.agent_id for a in consolidation], + "reserve": [a.agent_id for a in reserve], + } + + def should_rotate(self) -> bool: + """Check if rotation is needed (time, energy, or workload trigger).""" + # Time-based + if time.time() - self._last_rotation > self.rotation_period_s: + return True + + # Energy-based: any active agent exhausted + for slot in self.slots.values(): + if ( + slot.current_shift == ShiftType.ACTIVE + and slot.energy < self.energy_threshold + ): + return True + + return False + + def rotate( + self, + active_tasks: list[str] | None = None, + recent_context: str = "", + unresolved: list[str] | None = None, + ) -> RotationEvent: + """ + Execute a shift rotation with handoff briefing. + + outgoing (active) → reserve (deep rest) + consolidation → active (rested, ready to work) + reserve → consolidation + """ + active_tasks = active_tasks or [] + unresolved = unresolved or [] + + # Current assignments + old_active = [s for s in self.slots.values() if s.current_shift == ShiftType.ACTIVE] + old_consol = [s for s in self.slots.values() if s.current_shift == ShiftType.CONSOLIDATION] + old_reserve = [s for s in self.slots.values() if s.current_shift == ShiftType.RESERVE] + + # Generate handoff brief + brief = HandoffBrief( + outgoing_shift=ShiftType.ACTIVE, + incoming_shift=ShiftType.CONSOLIDATION, + active_tasks=active_tasks, + recent_context=recent_context[:500], + unresolved_issues=unresolved, + emotional_state="nominal", + ) + + # Rotate: consolidation → active, reserve → consolidation, active → reserve + for slot in old_consol: + slot.current_shift = ShiftType.ACTIVE + slot.energy = min(1.0, slot.energy + 0.4) # rested + slot.last_rotation = time.time() + for slot in old_reserve: + slot.current_shift = ShiftType.CONSOLIDATION + slot.last_rotation = time.time() + for slot in old_active: + slot.current_shift = ShiftType.RESERVE + slot.energy = max(0.0, slot.energy - 0.1) # tired + slot.last_rotation = time.time() + slot.consolidation_pending = True + + self._last_rotation = time.time() + + # Post handoff to chat + self.chat.post( + sender_id="system", + sender_name="Rotation Manager", + content=f"Shift rotation: {len(old_consol)} agents now active, " + f"tasks: {', '.join(active_tasks[:3]) if active_tasks else 'none'}", + message_type=MessageType.HANDOFF, + context=recent_context[:200], + ) + + event = RotationEvent( + rotation_id=str(uuid.uuid4())[:8], + new_active=[s.agent_id for s in old_consol], + new_consolidation=[s.agent_id for s in old_reserve], + new_reserve=[s.agent_id for s in old_active], + handoff_brief=brief, + ) + self.rotation_history.append(event) + + logger.info( + "[ROTATION] Shift change: active=%d consolidating=%d reserve=%d", + len(old_consol), len(old_reserve), len(old_active), + ) + return event + + def activate_surge(self) -> list[str]: + """ + SURGE CAPACITY: Activate ALL agents for emergency. + Consolidation/reserve agents wake immediately. + """ + self.surge_mode = True + activated = [] + for slot in self.slots.values(): + if slot.current_shift != ShiftType.ACTIVE: + slot.current_shift = ShiftType.ACTIVE + activated.append(slot.agent_id) + + self.chat.post( + sender_id="system", + sender_name="Emergency", + content=f"SURGE ACTIVATED: {len(activated)} additional agents online", + message_type=MessageType.EMERGENCY, + ) + + logger.warning("[SURGE] All %d agents activated", len(self.slots)) + return activated + + def deactivate_surge(self) -> None: + """Return to normal rotation after surge.""" + self.surge_mode = False + self.auto_assign_shifts() + logger.info("[SURGE] Deactivated, returning to normal rotation") + + def record_task_completion(self, agent_id: str, energy_cost: float = 0.05) -> None: + """Record that an agent completed a task (energy consumption).""" + if agent_id in self.slots: + slot = self.slots[agent_id] + slot.tasks_completed += 1 + slot.energy = max(0.0, slot.energy - energy_cost) + + def get_active_agents(self) -> list[AgentSlot]: + return [s for s in self.slots.values() if s.current_shift == ShiftType.ACTIVE] + + def get_consolidating_agents(self) -> list[AgentSlot]: + return [s for s in self.slots.values() if s.current_shift == ShiftType.CONSOLIDATION] + + def get_status(self) -> SystemStatus: + active = [s.agent_id for s in self.slots.values() if s.current_shift == ShiftType.ACTIVE] + consolidating = [s.agent_id for s in self.slots.values() if s.current_shift == ShiftType.CONSOLIDATION] + reserve = [s.agent_id for s in self.slots.values() if s.current_shift == ShiftType.RESERVE] + total = len(self.slots) + + return SystemStatus( + active_agents=active, + consolidating_agents=consolidating, + reserve_agents=reserve, + total_agents=total, + current_capacity=len(active) / max(total, 1), + surge_mode=self.surge_mode, + rotation_count=len(self.rotation_history), + chat_messages=len(self.chat.messages), + uptime_hours=round((time.time() - self.start_time) / 3600, 2), + ) + + def to_dict(self) -> dict: + status = self.get_status() + return { + "active": len(status.active_agents), + "consolidating": len(status.consolidating_agents), + "reserve": len(status.reserve_agents), + "total": status.total_agents, + "surge_mode": status.surge_mode, + "rotation_count": status.rotation_count, + "chat_messages": status.chat_messages, + "uptime_hours": status.uptime_hours, + } diff --git a/backend/agents/shura_council.py b/backend/agents/shura_council.py new file mode 100644 index 0000000..2dcd7f4 --- /dev/null +++ b/backend/agents/shura_council.py @@ -0,0 +1,499 @@ +""" +Shūrā Council — Multi-Agent Consultation Architecture +====================================================== + +"And those who conduct their affairs by shūrā (mutual consultation) + among themselves" — Quran 42:38 + +"And consult them in the matter" — Quran 3:159 + +Implements the SHURA_COUNCIL_MEETING algorithm: +1. Agenda setting (Qalb broadcasts problem) +2. Independent analysis (parallel, NO groupthink) +3. Proposal presentation (round-robin) +4. Cross-examination (agents challenge each other) +5. Integration (Lubb synthesis) +6. Dissent recording (Lawwāma function) +7. Knowledge sharing (post-meeting learning) + +Also implements AGENT_KNOWLEDGE_SHARING for continuous inter-agent learning. +""" + +import logging +import time +import uuid +from dataclasses import dataclass, field +from enum import Enum +from typing import Any + +logger = logging.getLogger("mizan.shura") + + +class ProposalStatus(Enum): + PENDING = "pending" + PRESENTED = "presented" + CHALLENGED = "challenged" + ACCEPTED = "accepted" + REJECTED = "rejected" + + +@dataclass +class AgentProposal: + """A single agent's proposal in a Shūrā meeting.""" + agent_id: str + agent_name: str + expertise_domain: str + solution: str + reasoning: str + confidence: float + evidence: list[str] + score: float = 0.0 + weaknesses: list[str] = field(default_factory=list) + status: ProposalStatus = ProposalStatus.PENDING + + +@dataclass +class Challenge: + """A challenge from one agent to another's proposal.""" + challenger_id: str + target_proposal_agent: str + critique: str + severity: float # 0.0 - 1.0 + valid: bool = True + + +@dataclass +class Rebuttal: + """A defense against a challenge.""" + defender_id: str + challenge_critique: str + defense: str + strength: float # 0.0 - 1.0 + valid: bool = True + + +@dataclass +class DissentRecord: + """Records disagreement for future reference.""" + agent_id: str + agent_name: str + alternative_proposal: str + reasoning: str + timestamp: float = field(default_factory=time.time) + + +@dataclass +class MeetingResult: + """Full result of a Shūrā council meeting.""" + meeting_id: str + problem: str + decision: str + decision_source: str # "synthesis" or agent_name + confidence: float + proposals: list[AgentProposal] + dissent_records: list[DissentRecord] + knowledge_shared: int + meeting_duration_ms: float + + +@dataclass +class KnowledgePackage: + """Packaged knowledge for sharing between agents.""" + source_agent: str + target_agent: str + content: str + domain: str + relevance: float + timestamp: float = field(default_factory=time.time) + + +class ShuraCouncil: + """ + SHURA_COUNCIL_MEETING algorithm implementation. + + Manages formal multi-agent deliberation sessions with: + - Independent parallel analysis (prevents groupthink) + - Cross-examination (stress-tests proposals) + - Weighted synthesis (not majority vote — expertise-weighted) + - Dissent recording (minority views preserved for future correction) + - Post-meeting knowledge sharing (agents learn from each other) + """ + + def __init__(self): + self.meeting_history: list[MeetingResult] = [] + self.dissent_archive: list[DissentRecord] = [] + self.knowledge_inbox: dict[str, list[KnowledgePackage]] = {} + self.expertise_profiles: dict[str, dict[str, float]] = {} + + def convene( + self, + problem: str, + agents: list[dict], + context: dict | None = None, + ) -> MeetingResult: + """ + Run a full Shūrā council meeting. + + agents: list of {"id", "name", "expertise", "evaluate_fn" (optional)} + + Steps: + 1. Agenda broadcast + 2. Independent analysis (parallel) + 3. Proposal presentation + 4. Cross-examination + 5. Integration + 6. Dissent recording + 7. Knowledge sharing + """ + meeting_id = str(uuid.uuid4())[:8] + start = time.monotonic() + + # Step 1: Agenda setting + logger.info("[SHURA] Meeting %s convened: %s", meeting_id, problem[:80]) + + # Step 2: Independent analysis (each agent generates a proposal) + proposals = [] + for agent in agents: + proposal = self._generate_proposal(agent, problem, context) + proposals.append(proposal) + + # Step 3: Proposal presentation (score by expertise relevance) + proposals.sort( + key=lambda p: self._expertise_relevance(p.expertise_domain, problem), + reverse=True, + ) + for proposal in proposals: + proposal.status = ProposalStatus.PRESENTED + + # Step 4: Cross-examination + self._cross_examine(proposals, problem) + + # Step 5: Integration — weighted scoring + synthesis attempt + decision, decision_source, confidence = self._integrate(proposals, problem) + + # Step 6: Dissent recording + dissent_records = [] + for proposal in proposals: + if proposal.solution != decision: + record = DissentRecord( + agent_id=proposal.agent_id, + agent_name=proposal.agent_name, + alternative_proposal=proposal.solution, + reasoning=proposal.reasoning, + ) + dissent_records.append(record) + self.dissent_archive.append(record) + + # Step 7: Knowledge sharing + shared = self._share_knowledge(proposals, agents) + + elapsed_ms = (time.monotonic() - start) * 1000 + + result = MeetingResult( + meeting_id=meeting_id, + problem=problem, + decision=decision, + decision_source=decision_source, + confidence=confidence, + proposals=proposals, + dissent_records=dissent_records, + knowledge_shared=shared, + meeting_duration_ms=round(elapsed_ms, 2), + ) + self.meeting_history.append(result) + + logger.info( + "[SHURA] Meeting %s concluded: source=%s confidence=%.2f dissents=%d", + meeting_id, decision_source, confidence, len(dissent_records), + ) + return result + + def _generate_proposal( + self, agent: dict, problem: str, context: dict | None + ) -> AgentProposal: + """ + Each agent independently analyzes the problem. + In production, this calls the agent's LLM evaluate function. + """ + agent_id = agent.get("id", "unknown") + name = agent.get("name", "Agent") + expertise = agent.get("expertise", "general") + + # Expertise-based confidence heuristic + relevance = self._expertise_relevance(expertise, problem) + confidence = 0.4 + 0.5 * relevance # base + expertise boost + + # Generate proposal text (placeholder — LLM call in production) + solution = ( + f"[{expertise.upper()} perspective] " + f"Approach to '{problem[:60]}' " + f"using {expertise} principles" + ) + reasoning = ( + f"Based on {expertise} analysis: " + f"relevance={relevance:.2f}, " + f"confidence={confidence:.2f}" + ) + evidence = [f"{expertise}_analysis", f"domain_knowledge_{expertise}"] + + return AgentProposal( + agent_id=agent_id, + agent_name=name, + expertise_domain=expertise, + solution=solution, + reasoning=reasoning, + confidence=round(confidence, 3), + evidence=evidence, + score=confidence * relevance, + ) + + def _cross_examine(self, proposals: list[AgentProposal], problem: str) -> None: + """ + Agents challenge each other's proposals. + Score adjustments: valid challenge reduces score, valid rebuttal partially recovers. + """ + for i, proposal in enumerate(proposals): + for j, challenger_source in enumerate(proposals): + if i == j: + continue + + # Generate challenge based on expertise difference + challenge = self._generate_challenge( + challenger_source, proposal, problem + ) + if not challenge: + continue + + if challenge.valid: + proposal.score -= challenge.severity * 0.2 + proposal.weaknesses.append(challenge.critique) + + # Allow rebuttal + rebuttal = self._generate_rebuttal(proposal, challenge) + if rebuttal and rebuttal.valid: + proposal.score += rebuttal.strength * 0.1 # partial recovery + + proposal.status = ProposalStatus.CHALLENGED + + def _generate_challenge( + self, challenger: AgentProposal, target: AgentProposal, problem: str + ) -> Challenge | None: + """Generate a challenge from one agent to another's proposal.""" + # Only challenge if domains differ (cross-domain critique is more valuable) + if challenger.expertise_domain == target.expertise_domain: + return None + + # Heuristic: challenge strength based on challenger's confidence + severity = 0.2 + 0.3 * challenger.confidence + + # Identify potential weakness based on expertise gap + critique = ( + f"{challenger.expertise_domain} perspective: " + f"'{target.expertise_domain}' approach may miss " + f"{challenger.expertise_domain}-specific considerations" + ) + + return Challenge( + challenger_id=challenger.agent_id, + target_proposal_agent=target.agent_id, + critique=critique, + severity=round(severity, 3), + valid=severity > 0.3, # only valid if substantive + ) + + def _generate_rebuttal( + self, defender: AgentProposal, challenge: Challenge + ) -> Rebuttal | None: + """Defender responds to a challenge.""" + # Rebuttal strength proportional to evidence quality + strength = min(0.8, 0.3 + 0.1 * len(defender.evidence)) + + defense = ( + f"Rebuttal: {defender.expertise_domain} approach accounts for " + f"the raised concern through {defender.evidence[0] if defender.evidence else 'general analysis'}" + ) + + return Rebuttal( + defender_id=defender.agent_id, + challenge_critique=challenge.critique, + defense=defense, + strength=round(strength, 3), + valid=strength > 0.4, + ) + + def _integrate( + self, proposals: list[AgentProposal], problem: str + ) -> tuple[str, str, float]: + """ + Integration: NOT majority vote — weighted by expertise and evidence quality. + + final_score = expertise_weight × confidence × evidence_quality × (1 - weaknesses) + + Attempt synthesis (combine best elements). If synthesis > best individual, use it. + """ + if not proposals: + return "No proposals generated", "none", 0.0 + + # Score each proposal + for proposal in proposals: + expertise_weight = self._expertise_relevance( + proposal.expertise_domain, problem + ) + evidence_quality = min(1.0, 0.5 + 0.1 * len(proposal.evidence)) + weakness_penalty = min(0.5, 0.1 * len(proposal.weaknesses)) + + proposal.score = ( + expertise_weight + * proposal.confidence + * evidence_quality + * (1.0 - weakness_penalty) + ) + + # Best individual proposal + best = max(proposals, key=lambda p: p.score) + + # Attempt synthesis: combine elements from top proposals + top_proposals = sorted(proposals, key=lambda p: p.score, reverse=True)[:3] + synthesis = self._synthesize(top_proposals, problem) + synthesis_score = sum(p.score for p in top_proposals) / len(top_proposals) * 1.1 + + if synthesis_score > best.score: + confidence = min(0.95, synthesis_score) + return synthesis, "synthesis", round(confidence, 3) + else: + best.status = ProposalStatus.ACCEPTED + return best.solution, best.agent_name, round(best.score, 3) + + def _synthesize(self, proposals: list[AgentProposal], problem: str) -> str: + """Combine best elements of multiple proposals into synthesis.""" + elements = [] + for proposal in proposals: + key_element = proposal.solution.split("]")[-1].strip()[:80] + if key_element: + elements.append(f"[{proposal.expertise_domain}] {key_element}") + return f"Synthesis for '{problem[:40]}': " + " + ".join(elements[:3]) + + def _share_knowledge( + self, proposals: list[AgentProposal], agents: list[dict] + ) -> int: + """Post-meeting knowledge sharing: each agent learns from others.""" + shared = 0 + for proposal in proposals: + for agent in agents: + target_id = agent.get("id", "") + if target_id == proposal.agent_id: + continue # don't share with self + + relevance = self._cross_domain_relevance( + proposal.expertise_domain, + agent.get("expertise", "general"), + ) + if relevance > 0.3: + package = KnowledgePackage( + source_agent=proposal.agent_id, + target_agent=target_id, + content=proposal.reasoning[:200], + domain=proposal.expertise_domain, + relevance=relevance, + ) + if target_id not in self.knowledge_inbox: + self.knowledge_inbox[target_id] = [] + self.knowledge_inbox[target_id].append(package) + shared += 1 + + return shared + + def get_inbox(self, agent_id: str) -> list[KnowledgePackage]: + """Get pending knowledge packages for an agent.""" + return self.knowledge_inbox.pop(agent_id, []) + + def search_meeting_history(self, query: str, limit: int = 5) -> list[MeetingResult]: + """Search past meetings by problem text.""" + query_words = set(query.lower().split()) + scored = [] + for meeting in self.meeting_history: + problem_words = set(meeting.problem.lower().split()) + overlap = len(query_words & problem_words) + if overlap > 0: + scored.append((meeting, overlap)) + scored.sort(key=lambda x: x[1], reverse=True) + return [m for m, _ in scored[:limit]] + + def get_dissents_for(self, problem_query: str) -> list[DissentRecord]: + """Find dissent records related to a query (may prove right later).""" + results = [] + query_words = set(problem_query.lower().split()) + for record in self.dissent_archive: + alt_words = set(record.alternative_proposal.lower().split()) + if len(query_words & alt_words) > 1: + results.append(record) + return results + + def update_expertise(self, agent_id: str, domain: str, score: float) -> None: + """Track and update expertise profiles over time.""" + if agent_id not in self.expertise_profiles: + self.expertise_profiles[agent_id] = {} + # Exponential moving average + alpha = 0.2 + current = self.expertise_profiles[agent_id].get(domain, 0.5) + self.expertise_profiles[agent_id][domain] = ( + alpha * score + (1 - alpha) * current + ) + + def _expertise_relevance(self, expertise: str, problem: str) -> float: + """How relevant is an expertise domain to the problem?""" + # Simple keyword overlap heuristic + expertise_words = set(expertise.lower().split("_")) + problem_words = set(problem.lower().split()) + if not expertise_words: + return 0.5 + overlap = len(expertise_words & problem_words) + return min(1.0, 0.3 + 0.3 * overlap) + + def _cross_domain_relevance(self, domain_a: str, domain_b: str) -> float: + """How relevant is knowledge from domain_a to domain_b?""" + if domain_a == domain_b: + return 0.9 # same domain = highly relevant + # Cross-domain pairs that benefit each other + synergies = { + frozenset({"reasoning", "planning"}): 0.7, + frozenset({"creativity", "reasoning"}): 0.6, + frozenset({"memory", "reasoning"}): 0.7, + frozenset({"language", "social"}): 0.6, + frozenset({"perception", "creativity"}): 0.5, + } + pair = frozenset({domain_a.lower(), domain_b.lower()}) + return synergies.get(pair, 0.35) + + def measure_collective_intelligence(self) -> dict: + """Measure collective IQ: total knowledge × diversity ÷ redundancy.""" + if not self.meeting_history: + return {"collective_iq": 0, "meetings": 0} + + unique_domains = set() + total_proposals = 0 + for meeting in self.meeting_history: + for proposal in meeting.proposals: + unique_domains.add(proposal.expertise_domain) + total_proposals += 1 + + diversity = len(unique_domains) + # Estimate redundancy from repeated similar proposals + redundancy = max(1, total_proposals - diversity * len(self.meeting_history)) + + collective_iq = (total_proposals * diversity) / redundancy + return { + "collective_iq": round(collective_iq, 2), + "meetings": len(self.meeting_history), + "unique_domains": diversity, + "total_proposals": total_proposals, + "dissent_records": len(self.dissent_archive), + } + + def to_dict(self) -> dict: + return { + "meetings_held": len(self.meeting_history), + "dissents_archived": len(self.dissent_archive), + "knowledge_inbox_agents": len(self.knowledge_inbox), + "expertise_profiles": len(self.expertise_profiles), + } diff --git a/backend/agents/specialized.py b/backend/agents/specialized.py index 8c7103f..2d0c4eb 100644 --- a/backend/agents/specialized.py +++ b/backend/agents/specialized.py @@ -671,9 +671,332 @@ async def _tool_send_notification(self, message: str, channel: str = "log") -> d return {"error": f"Unknown notification channel: {channel}"} +class SuperAgent(BrowserAgent, ResearchAgent, CodeAgent, CommunicationAgent): + """ + Khalifah (خليفة) - Universal Agent + "I will create a vicegerent (Khalifah) on earth" - Quran 2:30 + + All tools. All capabilities. One agent. + Combines browser, research, code, and communication into a single agent. + """ + + # Tool schemas for all specialized tools — exposed to LLM via get_tool_schemas() + TOOL_SCHEMAS: list[dict] = [ + # ── Browser tools ── + { + "name": "browse_url", + "description": "Browse a URL and return its text content, title, and links.", + "input_schema": { + "type": "object", + "properties": { + "url": {"type": "string", "description": "The URL to browse"}, + }, + "required": ["url"], + }, + }, + { + "name": "navigate", + "description": "Navigate to a URL with optional JS rendering (Playwright) or httpx fallback.", + "input_schema": { + "type": "object", + "properties": { + "url": {"type": "string", "description": "The URL to navigate to"}, + "wait_for": { + "type": "string", + "description": "CSS selector to wait for before returning", + }, + }, + "required": ["url"], + }, + }, + { + "name": "search_web", + "description": "Search the web using DuckDuckGo and return results.", + "input_schema": { + "type": "object", + "properties": { + "query": {"type": "string", "description": "The search query"}, + }, + "required": ["query"], + }, + }, + { + "name": "extract_content", + "description": "Extract specific content from a URL, optionally using a CSS selector.", + "input_schema": { + "type": "object", + "properties": { + "url": {"type": "string", "description": "The URL to extract from"}, + "selector": { + "type": "string", + "description": "Optional CSS selector to target specific elements", + }, + }, + "required": ["url"], + }, + }, + { + "name": "take_screenshot", + "description": "Take a screenshot of a web page (requires Playwright or system browser).", + "input_schema": { + "type": "object", + "properties": { + "url": {"type": "string", "description": "The URL to screenshot"}, + }, + "required": ["url"], + }, + }, + { + "name": "click_element", + "description": "Click an element on a web page by CSS selector.", + "input_schema": { + "type": "object", + "properties": { + "url": {"type": "string", "description": "The page URL"}, + "selector": {"type": "string", "description": "CSS selector of the element to click"}, + }, + "required": ["url", "selector"], + }, + }, + { + "name": "fill_form", + "description": "Fill form fields on a web page. Keys are CSS selectors, values are text to fill.", + "input_schema": { + "type": "object", + "properties": { + "url": {"type": "string", "description": "The page URL"}, + "fields": { + "type": "object", + "description": "Mapping of CSS selector → value to fill", + }, + }, + "required": ["url", "fields"], + }, + }, + # ── Research tools ── + { + "name": "analyze_text", + "description": "Analyze text for word count, key terms, and insights.", + "input_schema": { + "type": "object", + "properties": { + "text": {"type": "string", "description": "The text to analyze"}, + "aspect": { + "type": "string", + "description": "Analysis aspect (e.g., general, sentiment, key_points)", + "default": "general", + }, + }, + "required": ["text"], + }, + }, + { + "name": "synthesize_sources", + "description": "Synthesize information from multiple text sources.", + "input_schema": { + "type": "object", + "properties": { + "sources": { + "type": "array", + "items": {"type": "string"}, + "description": "List of text sources to synthesize", + }, + }, + "required": ["sources"], + }, + }, + { + "name": "fact_check", + "description": "Check a factual claim for accuracy.", + "input_schema": { + "type": "object", + "properties": { + "claim": {"type": "string", "description": "The claim to verify"}, + }, + "required": ["claim"], + }, + }, + { + "name": "generate_report", + "description": "Generate a structured report template on a topic.", + "input_schema": { + "type": "object", + "properties": { + "topic": {"type": "string", "description": "The report topic"}, + "format": { + "type": "string", + "description": "Report format (markdown, html, text)", + "default": "markdown", + }, + }, + "required": ["topic"], + }, + }, + { + "name": "arxiv_search", + "description": "Search ArXiv for academic papers.", + "input_schema": { + "type": "object", + "properties": { + "query": {"type": "string", "description": "Search query for papers"}, + "max_results": { + "type": "integer", + "description": "Maximum number of results", + "default": 5, + }, + }, + "required": ["query"], + }, + }, + # ── Code tools ── + { + "name": "generate_code", + "description": "Generate code from a specification.", + "input_schema": { + "type": "object", + "properties": { + "spec": {"type": "string", "description": "Code specification/requirements"}, + "language": { + "type": "string", + "description": "Programming language", + "default": "python", + }, + }, + "required": ["spec"], + }, + }, + { + "name": "run_tests", + "description": "Run tests in a directory using a test framework.", + "input_schema": { + "type": "object", + "properties": { + "path": {"type": "string", "description": "Path to the test directory"}, + "framework": { + "type": "string", + "description": "Test framework (pytest, jest, etc.)", + "default": "pytest", + }, + }, + "required": ["path"], + }, + }, + { + "name": "lint_code", + "description": "Lint code at the given path.", + "input_schema": { + "type": "object", + "properties": { + "path": {"type": "string", "description": "File or directory path to lint"}, + }, + "required": ["path"], + }, + }, + { + "name": "git_operation", + "description": "Run a git operation (status, log, diff, branch, add, commit, push, pull, clone).", + "input_schema": { + "type": "object", + "properties": { + "operation": { + "type": "string", + "description": "Git operation to run (e.g., status, log, diff)", + }, + "repo_path": { + "type": "string", + "description": "Repository path", + "default": ".", + }, + "args": { + "type": "string", + "description": "Additional arguments", + "default": "", + }, + }, + "required": ["operation"], + }, + }, + { + "name": "install_package", + "description": "Install a package using pip or npm.", + "input_schema": { + "type": "object", + "properties": { + "package": {"type": "string", "description": "Package name to install"}, + "manager": { + "type": "string", + "description": "Package manager (pip or npm)", + "default": "pip", + }, + }, + "required": ["package"], + }, + }, + # ── Communication tools ── + { + "name": "send_webhook", + "description": "Send a webhook POST request with a JSON payload.", + "input_schema": { + "type": "object", + "properties": { + "url": {"type": "string", "description": "Webhook URL"}, + "payload": {"type": "object", "description": "JSON payload to send"}, + }, + "required": ["url", "payload"], + }, + }, + { + "name": "check_email", + "description": "Check an IMAP email inbox for recent messages.", + "input_schema": { + "type": "object", + "properties": { + "host": {"type": "string", "description": "IMAP server hostname"}, + "user": {"type": "string", "description": "Email username"}, + "password": {"type": "string", "description": "Email password"}, + "folder": {"type": "string", "description": "Folder to check", "default": "INBOX"}, + "limit": {"type": "integer", "description": "Max messages to return", "default": 10}, + }, + "required": ["host", "user", "password"], + }, + }, + { + "name": "send_notification", + "description": "Send a notification message via a channel (currently supports 'log').", + "input_schema": { + "type": "object", + "properties": { + "message": {"type": "string", "description": "Notification message"}, + "channel": { + "type": "string", + "description": "Notification channel", + "default": "log", + }, + }, + "required": ["message"], + }, + }, + ] + + def __init__(self, **kwargs): + # Call BaseAgent directly to skip MRO chain — + # each parent __init__ passes its own role kwarg which conflicts. + BaseAgent.__init__(self, role="khalifah", **kwargs) + self._playwright_available: bool | None = None + # Khalifah starts at Mudghah (structured) — all tools, delegation enabled + self.nafs_level = 3 + # Register all specialized tool sets + self._register_browser_tools() + self._register_research_tools() + self._register_code_tools() + self._register_comm_tools() + + def create_agent(agent_type: str, **kwargs) -> BaseAgent: """Factory for creating Quranic agents""" agents = { + "super": SuperAgent, + "khalifah": SuperAgent, "browser": BrowserAgent, "mubashir": BrowserAgent, "research": ResearchAgent, diff --git a/backend/api/main.py b/backend/api/main.py index 35d3c27..7a4ca4b 100644 --- a/backend/api/main.py +++ b/backend/api/main.py @@ -9,6 +9,7 @@ """ import asyncio +import json import logging import os import sys @@ -47,12 +48,16 @@ from core.plugins import plugin_manager from core.qalb import QalbEngine from memory.dhikr import DhikrMemorySystem +from memory.knowledge_graph import KnowledgeGraph +from reasoning.context_manager import ContextManager +from reasoning.planner import TafakkurPlanner from providers import ( check_provider_health, create_provider, fetch_ollama_models, fetch_openrouter_models, get_provider_status, + set_active_state, ) from qca.cognitive_methods import select_method @@ -82,7 +87,7 @@ async def lifespan(app: FastAPI): existing = await memory.get_all_agents() if not existing: default_agents = [ - {"name": "Hafiz", "type": "general", "role": "Preserver"}, + {"name": "Khalifah", "type": "super", "role": "Universal"}, {"name": "Mubashir", "type": "browser", "role": "Browser"}, {"name": "Mundhir", "type": "research", "role": "Researcher"}, {"name": "Katib", "type": "code", "role": "Coder"}, @@ -98,6 +103,9 @@ async def lifespan(app: FastAPI): izn=izn, skill_registry=skill_registry, plugin_manager=plugin_manager, + knowledge_graph=knowledge_graph, + context_manager=context_manager, + planner=planner, ) active_agents[agent.id] = agent balancer.register(agent.id) @@ -125,6 +133,9 @@ async def lifespan(app: FastAPI): izn=izn, skill_registry=skill_registry, plugin_manager=plugin_manager, + knowledge_graph=knowledge_graph, + context_manager=context_manager, + planner=planner, ) agent.total_tasks = profile.get("total_tasks", 0) agent.learning_iterations = profile.get("learning_iterations", 0) @@ -134,6 +145,27 @@ async def lifespan(app: FastAPI): logger.info(f"{len(active_agents)} agents initialized") + # Inject global registry references into agents + for agent in active_agents.values(): + agent._agent_registry = active_agents + agent._balancer = balancer + agent._shura = shura + + # Restore persisted provider/model choice + try: + saved_provider = await memory.get_preference("active_provider") + saved_model = await memory.get_preference("active_model") + if saved_provider and saved_model: + restored = create_provider(provider=saved_provider, model=saved_model) + if restored: + for agent in active_agents.values(): + agent.ai_client = restored + agent.ai_model = saved_model + set_active_state(saved_provider, saved_model) + logger.info(f"Restored provider: {saved_provider}/{saved_model}") + except Exception as exc: + logger.warning(f"Could not restore provider preference: {exc}") + # Initialize scheduler executor async def execute_scheduled_task(task: str, agent_id: str = None): aid = agent_id or (list(active_agents.keys())[0] if active_agents else None) @@ -175,6 +207,34 @@ async def execute_scheduled_task(task: str, agent_id: str = None): # Shutdown await event_bus.emit("system.shutdown", {}) + + # Persist Masalik neural pathways to disk before shutdown + try: + if hasattr(memory, "masalik"): + memory.masalik.save_to_disk() + logger.info("[MASALIK] Neural pathways persisted to disk") + except Exception as e: + logger.warning(f"[MASALIK] Failed to save pathways: {e}") + + # Persist agent profiles + try: + for agent in active_agents.values(): + await memory.save_agent_profile({ + "id": agent.id, + "name": agent.name, + "role": agent.role, + "nafs_level": agent.nafs_level, + "capabilities": list(agent.tools.keys()), + "total_tasks": agent.total_tasks, + "success_rate": agent.success_rate, + "error_count": agent.error_count, + "learning_iterations": agent.learning_iterations, + "config": agent.config or {}, + }) + logger.info(f"[AGENTS] {len(active_agents)} agent profiles persisted") + except Exception as e: + logger.warning(f"[AGENTS] Failed to persist profiles: {e}") + await plugin_manager.unload_all() await scheduler.stop() logger.info("MIZAN shutdown complete") @@ -210,7 +270,10 @@ async def execute_scheduled_task(task: str, agent_id: str = None): ) # ===== GLOBAL STATE ===== -memory = DhikrMemorySystem(db_path=os.getenv("DB_PATH", "/tmp/mizan_memory.db")) +memory = DhikrMemorySystem(db_path=os.getenv("DB_PATH", "/data/mizan_memory.db")) +knowledge_graph = KnowledgeGraph(db_path=os.getenv("DB_PATH", "/data/mizan_memory.db")) +context_manager = ContextManager() +planner = TafakkurPlanner() balancer = MizanBalancer() shura = ShuraCouncil() active_agents: dict[str, Any] = {} @@ -225,30 +288,6 @@ async def execute_scheduled_task(task: str, agent_id: str = None): federation = AgentFederation() -# ===== RATE LIMIT MIDDLEWARE ===== - - -@app.middleware("http") -async def rate_limit_middleware(request: Request, call_next): - """Rate limiting via Wali Guardian with proper headers""" - client_ip = request.client.host if request.client else "unknown" - rl = wali.check_rate_limit(client_ip) - if not rl["allowed"]: - return JSONResponse( - status_code=429, - content={"error": "Rate limit exceeded. Please slow down."}, - headers={ - "Retry-After": str(rl["retry_after"]), - "X-RateLimit-Limit": str(rl["limit"]), - "X-RateLimit-Remaining": "0", - }, - ) - response = await call_next(request) - response.headers["X-RateLimit-Limit"] = str(rl["limit"]) - response.headers["X-RateLimit-Remaining"] = str(rl["remaining"]) - return response - - # ===== SECURITY HEADERS MIDDLEWARE ===== @@ -334,14 +373,20 @@ async def require_auth( class AgentCreate(BaseModel): name: str = Field(..., min_length=1, max_length=100) type: str = Field( - default="general", - pattern=r"^(general|browser|research|code|communication|wakil|mubashir|mundhir|katib|rasul)$", + default="super", + pattern=r"^(super|khalifah|general|browser|research|code|communication|wakil|mubashir|mundhir|katib|rasul)$", ) model: str = Field(default="claude-opus-4-6", max_length=100) system_prompt: str | None = Field(None, max_length=10000) capabilities: list[str] = [] +class AgentUpdate(BaseModel): + name: str | None = Field(None, min_length=1, max_length=100) + model: str | None = Field(None, max_length=200) + system_prompt: str | None = Field(None, max_length=10000) + + class TaskRequest(BaseModel): task: str = Field(..., min_length=1, max_length=50000) agent_id: str | None = None @@ -353,6 +398,7 @@ class ChatMessage(BaseModel): session_id: str = Field(..., min_length=1, max_length=100) content: str = Field(..., min_length=1, max_length=50000) agent_id: str | None = None + model_override: str | None = Field(None, max_length=200) class IntegrationCreate(BaseModel): @@ -582,6 +628,74 @@ async def delete_agent(agent_id: str, user: TokenPayload | None = Depends(get_cu return {"deleted": agent_id} +@app.put("/api/agents/{agent_id}") +async def update_agent( + agent_id: str, + req: AgentUpdate, + user: TokenPayload | None = Depends(get_current_user), +): + """Update an existing agent's name, model, or system prompt.""" + agent = active_agents.get(agent_id) + if not agent: + raise HTTPException(404, "Agent not found") + + if req.name is not None: + agent.name = req.name + if req.system_prompt is not None: + agent.config["system_prompt"] = req.system_prompt + if req.model is not None: + agent.ai_model = req.model + provider_obj = create_provider(model=req.model) + if provider_obj: + agent.ai_client = provider_obj + + await memory.save_agent_profile({ + "id": agent.id, + "name": agent.name, + "role": agent.role, + "nafs_level": agent.nafs_level, + "capabilities": list(agent.tools.keys()), + "config": agent.config, + }) + + await manager.broadcast({"type": "agent_updated", "agent": agent.to_dict()}) + return agent.to_dict() + + +class AgentModelRequest(BaseModel): + provider: str = Field(..., pattern=r"^(anthropic|openrouter|openai|ollama)$") + model: str = Field(..., min_length=1, max_length=200) + + +@app.post("/api/agents/{agent_id}/model") +async def set_agent_model( + agent_id: str, + req: AgentModelRequest, + user: TokenPayload | None = Depends(get_current_user), +): + """Set model for a specific agent (does not affect other agents).""" + agent = active_agents.get(agent_id) + if not agent: + raise HTTPException(404, f"Agent '{agent_id}' not found") + + provider = create_provider(provider=req.provider, model=req.model) + if not provider: + raise HTTPException(400, f"Cannot initialize provider '{req.provider}'. Check API key.") + + agent.ai_client = provider + agent.ai_model = req.model + + await manager.broadcast({ + "type": "agent_model_changed", + "agent_id": agent_id, + "provider": req.provider, + "model": req.model, + }) + + wali.audit.log("agent_model_changed", {"agent_id": agent_id, "model": req.model}) + return {"agent_id": agent_id, "provider": req.provider, "model": req.model} + + # === TASKS === @@ -683,7 +797,23 @@ async def chat( user: TokenPayload | None = Depends(get_current_user), ): """Chat with an agent""" - session = active_sessions.get(req.session_id, {"history": []}) + session = active_sessions.get(req.session_id) + + # Auto-restore session from DB if not in memory (fixes cross-restart amnesia) + if session is None: + db_messages = await memory.get_messages(req.session_id, limit=50) + restored_history = [ + {"role": msg["role"], "content": msg["content"]} + for msg in db_messages + ] + session = {"history": restored_history} + if restored_history: + logger.info( + "[SESSION] Restored %d messages for session %s from DB", + len(restored_history), + req.session_id[:12], + ) + active_sessions[req.session_id] = session await memory.save_message(req.session_id, "user", req.content) @@ -723,6 +853,18 @@ async def chat( message_id = str(uuid.uuid4()) async def process_chat(): + # Per-message model override: temporarily swap agent's model + # TODO(concurrency): use per-request LLM client instead of mutating agent + original_model = None + original_client = None + if req.model_override and req.model_override != agent.ai_model: + override_provider = create_provider(model=req.model_override) + if override_provider: + original_model = agent.ai_model + original_client = agent.ai_client + agent.ai_model = req.model_override + agent.ai_client = override_provider + # Send typing indicator before starting agent execution await manager.broadcast( { @@ -759,11 +901,17 @@ async def stream_cb(chunk: str, **kwargs): } ) - result = await agent.execute( - req.content, - {"history": session["history"][-10:]}, - stream_callback=stream_cb, - ) + try: + result = await agent.execute( + req.content, + {"history": session["history"][-agent.max_tool_turns:]}, + stream_callback=stream_cb, + ) + finally: + # Restore original model after per-message override + if original_model is not None: + agent.ai_model = original_model + agent.ai_client = original_client final_response = result.get("result", response) if result.get("success") else response if isinstance(final_response, dict): @@ -772,6 +920,12 @@ async def stream_cb(chunk: str, **kwargs): await memory.save_message(req.session_id, "assistant", str(final_response), agent_id) session["history"].append({"role": "assistant", "content": str(final_response)}) + # Extract cognitive metadata from QALB-7 pipeline + cognitive = {k: result.get(k) for k in ( + "nafs_level", "nafs_name", "ruh_energy", "qalb", "yaqin", + "mizan_label", "cognitive_method", "lubb", "lawwama", + ) if result.get(k) is not None} + await manager.broadcast( { "type": "chat_complete", @@ -779,6 +933,7 @@ async def stream_cb(chunk: str, **kwargs): "message_id": message_id, "response": str(final_response), "agent": agent.name, + "cognitive": cognitive, } ) @@ -794,7 +949,66 @@ async def get_chat_history(session_id: str): @app.get("/api/chat/sessions/list") async def list_sessions(): - return {"sessions": list(active_sessions.keys())} + """List recent chat sessions from DB with metadata""" + db_sessions = await memory.list_sessions(limit=20) + return {"sessions": db_sessions} + + +# === PLANNER (Tafakkur — تفكر) === + + +class PlanRequest(BaseModel): + goal: str = Field(..., description="The complex goal to decompose") + agent_id: str | None = None + + +@app.post("/api/plan") +async def create_plan( + req: PlanRequest, + background_tasks: BackgroundTasks, + user: TokenPayload | None = Depends(get_current_user), +): + """Decompose a complex goal into sub-tasks using TafakkurPlanner""" + agent_id = req.agent_id or (list(active_agents.keys())[0] if active_agents else None) + if not agent_id or agent_id not in active_agents: + raise HTTPException(503, "No agents available") + + agent = active_agents[agent_id] + plan = await planner.decompose(req.goal, agent) + + return {"plan": plan.to_dict()} + + +@app.post("/api/plan/{plan_id}/execute") +async def execute_plan( + plan_id: str, + background_tasks: BackgroundTasks, + user: TokenPayload | None = Depends(get_current_user), +): + """Execute a previously created plan""" + plan = planner.get_plan(plan_id) + if not plan: + raise HTTPException(404, f"Plan {plan_id} not found") + + async def run_plan(): + result = await planner.execute_plan(plan, active_agents) + await manager.broadcast({ + "type": "plan_complete", + "plan_id": plan_id, + "result": result, + }) + + background_tasks.add_task(run_plan) + return {"status": "executing", "plan_id": plan_id} + + +@app.get("/api/plan/{plan_id}") +async def get_plan(plan_id: str): + """Get status of a plan""" + plan = planner.get_plan(plan_id) + if not plan: + raise HTTPException(404, f"Plan {plan_id} not found") + return {"plan": plan.to_dict()} # === MEMORY === @@ -833,6 +1047,182 @@ async def consolidate_memory(): return result +@app.get("/api/memory/list") +async def list_memories(memory_type: str | None = None, limit: int = 30): + """List recent memories without search filtering.""" + conn = memory._get_conn() + c = conn.cursor() + sql = "SELECT * FROM memories WHERE 1=1" + params: list = [] + if memory_type: + sql += " AND memory_type = ?" + params.append(memory_type) + sql += " ORDER BY recency DESC LIMIT ?" + params.append(limit) + c.execute(sql, params) + rows = c.fetchall() + memory._release_conn(conn) + + results = [] + for row in rows: + try: + content = json.loads(row[1]) if row[1] else None + except Exception: + content = row[1] + results.append({ + "id": row[0], + "content": str(content)[:500] if content else "", + "type": row[2], + "importance": row[3] or 0, + "recency": row[4] or "", + "access_count": row[5] or 0, + "agent_id": row[6], + "tags": json.loads(row[7]) if row[7] else [], + }) + return {"results": results, "total": len(results)} + + +# === KNOWLEDGE INGESTION (Ilm - عِلْم) === + + +class KnowledgeIngest(BaseModel): + source: str = Field(..., min_length=1, max_length=5000) + source_type: str = Field(default="auto", pattern=r"^(auto|url|pdf|youtube)$") + + +@app.post("/api/knowledge/ingest") +async def ingest_knowledge(req: KnowledgeIngest): + """Ingest knowledge from a URL or YouTube video into memory.""" + from knowledge.ingest import ( + chunk_content, + detect_source_type, + extract_url, + extract_youtube, + ) + + source_type = req.source_type + if source_type == "auto": + source_type = detect_source_type(req.source) + + if source_type == "youtube": + result = await extract_youtube(req.source) + elif source_type == "url": + result = await extract_url(req.source) + else: + raise HTTPException(400, f"Use /api/knowledge/upload for file uploads. Got source_type: {source_type}") + + if "error" in result: + raise HTTPException(422, result["error"]) + + content = result.get("content", "") + if not content: + raise HTTPException(422, "No content extracted from source") + + chunks = chunk_content(content) + stored_ids = [] + for idx, chunk in enumerate(chunks): + mem_id = await memory.remember( + content=chunk, + memory_type="semantic", + importance=0.8, + tags=["knowledge", source_type, result.get("title", "")[:50]], + ) + stored_ids.append(mem_id) + + # Encode full content into Masalik pathways + if hasattr(memory, "masalik"): + memory.masalik.encode(content[:5000], importance=0.8) + + return { + "success": True, + "title": result.get("title", ""), + "source": req.source, + "source_type": source_type, + "chunks_stored": len(stored_ids), + "char_count": result.get("char_count", len(content)), + } + + +@app.post("/api/knowledge/upload") +async def upload_knowledge(request: Request): + """Upload a PDF file and ingest its content into memory.""" + from knowledge.ingest import chunk_content, extract_pdf + + form = await request.form() + file = form.get("file") + if not file: + raise HTTPException(400, "No file provided. Send a multipart form with 'file' field.") + + filename = getattr(file, "filename", "upload.pdf") + file_bytes = await file.read() + + if not filename.lower().endswith(".pdf"): + raise HTTPException(400, "Only PDF files are supported for upload") + + if len(file_bytes) > 20 * 1024 * 1024: + raise HTTPException(400, "File too large. Maximum 20MB.") + + result = extract_pdf(file_bytes, filename) + if "error" in result: + raise HTTPException(422, result["error"]) + + content = result.get("content", "") + if not content: + raise HTTPException(422, "No text content extracted from PDF") + + chunks = chunk_content(content) + stored_ids = [] + for chunk in chunks: + mem_id = await memory.remember( + content=chunk, + memory_type="semantic", + importance=0.8, + tags=["knowledge", "pdf", filename[:50]], + ) + stored_ids.append(mem_id) + + if hasattr(memory, "masalik"): + memory.masalik.encode(content[:5000], importance=0.8) + + return { + "success": True, + "title": result.get("title", ""), + "source": filename, + "source_type": "pdf", + "page_count": result.get("page_count", 0), + "chunks_stored": len(stored_ids), + "char_count": result.get("char_count", len(content)), + } + + +@app.get("/api/knowledge/sources") +async def list_knowledge_sources(): + """List ingested knowledge sources.""" + conn = memory._get_conn() + cursor = conn.cursor() + cursor.execute( + "SELECT DISTINCT json_extract(tags, '$[2]') as source_title, " + "json_extract(tags, '$[1]') as source_type, " + "COUNT(*) as chunk_count, " + "MAX(recency) as last_updated " + "FROM memories WHERE json_extract(tags, '$[0]') = 'knowledge' " + "GROUP BY source_title ORDER BY last_updated DESC LIMIT 50" + ) + rows = cursor.fetchall() + memory._release_conn(conn) + + sources = [ + { + "title": row[0] or "Unknown", + "type": row[1] or "unknown", + "chunks": row[2], + "last_updated": row[3] or "", + } + for row in rows + ] + return {"sources": sources, "total": len(sources)} + + # === PROVIDERS (Ruh al-Ilm - روح العلم) === @@ -846,15 +1236,23 @@ async def list_providers(): @app.get("/api/providers/{provider_name}/models") -async def list_provider_models(provider_name: str): +async def list_provider_models( + provider_name: str, + limit: int = 50, + offset: int = 0, + search: str = "", + free_only: bool = False, +): """ List available models for a specific provider. - For OpenRouter: fetches the live catalog from their API. + For OpenRouter: fetches the live catalog with search/filter/pagination. For Ollama: fetches locally installed models. """ if provider_name == "openrouter": - models = await fetch_openrouter_models(limit=50) - return {"provider": "openrouter", "models": models} + result = await fetch_openrouter_models( + limit=limit, offset=offset, search=search, free_only=free_only, + ) + return {"provider": "openrouter", **result} elif provider_name == "ollama": models = await fetch_ollama_models() return {"provider": "ollama", "models": models} @@ -888,7 +1286,7 @@ async def switch_provider( ): """ Switch the active LLM provider and model for all agents. - This updates agents in memory — does not persist to .env. + Persists choice to database so it survives restarts. """ provider = create_provider(provider=req.provider, model=req.model) if not provider: @@ -901,6 +1299,12 @@ async def switch_provider( agent.ai_model = req.model switched += 1 + set_active_state(req.provider, req.model) + + # Persist to DB so choice survives restart + await memory.set_preference("active_provider", req.provider) + await memory.set_preference("active_model", req.model) + await manager.broadcast( { "type": "provider_switched", @@ -919,6 +1323,28 @@ async def switch_provider( } +# === PREFERENCES (persisted across restarts) === + + +@app.get("/api/preferences") +async def get_preferences(): + """Return all persisted user preferences.""" + prefs = await memory.get_all_preferences() + return {"preferences": prefs} + + +@app.post("/api/preferences") +async def save_preferences(req: dict): + """Save one or more preferences. Body: {"key": "value", ...}""" + saved = [] + for key, value in req.items(): + if not isinstance(key, str) or not isinstance(value, str): + continue + await memory.set_preference(key, value) + saved.append(key) + return {"saved": saved} + + # === INTEGRATIONS === @@ -1009,7 +1435,7 @@ async def system_status(): "provider": get_provider_status(), "security": { "auth_enabled": True, - "rate_limiting": True, + "rate_limiting": False, "wali_active": True, "izn_active": True, }, @@ -1696,6 +2122,68 @@ async def discover_agents(req: DiscoverRequest): return {"agents": [m.to_dict() for m in matches]} +class FederationTaskRequest(BaseModel): + task: str = Field(..., description="Task to route to best agent") + session_id: str = "" + + +@app.post("/api/federation/route") +async def federation_route_task( + req: FederationTaskRequest, + background_tasks: BackgroundTasks, + user: TokenPayload | None = Depends(get_current_user), +): + """Route a task to the best agent via Federation intelligence. + + Uses agent capabilities, success rates, and energy levels to select + the optimal agent, then executes the task. + """ + # Register all agents with federation first + for aid, agent in active_agents.items(): + federation.register_agent( + aid, agent.name, agent.role, + list(agent.tools.keys()), + agent.nafs_level, agent.success_rate, + ) + + # Discover best agent for this task + task_lower = req.task.lower() + capabilities = [] + if any(w in task_lower for w in ["code", "script", "python"]): + capabilities = ["python_exec", "write_file", "bash"] + elif any(w in task_lower for w in ["search", "browse", "web"]): + capabilities = ["web_search", "web_browse", "http_get"] + elif any(w in task_lower for w in ["analyze", "research"]): + capabilities = ["web_search", "read_file", "python_exec"] + else: + capabilities = ["bash", "http_get"] + + matches = federation.discover(capabilities) + if not matches: + # Fallback to first available agent + agent_id = list(active_agents.keys())[0] if active_agents else None + else: + agent_id = matches[0].agent_id + + if not agent_id or agent_id not in active_agents: + raise HTTPException(503, "No suitable agent found") + + agent = active_agents[agent_id] + session_id = req.session_id or str(uuid.uuid4()) + + result = await agent.execute(req.task, {"history": []}) + if result.get("success"): + await memory.save_message(session_id, "user", req.task) + await memory.save_message(session_id, "assistant", str(result.get("result", ""))[:5000], agent_id) + + return { + "routed_to": agent.name, + "agent_id": agent_id, + "session_id": session_id, + "result": result, + } + + # ===== RUH ENERGY ENDPOINTS ===== @@ -2021,7 +2509,7 @@ async def ws_stream(chunk, _sid=session_id, _mid=message_id, **kwargs): result = await agent.execute( content, - {"history": session["history"][-10:]}, + {"history": session["history"][-agent.max_tool_turns:]}, stream_callback=ws_stream, ) @@ -2032,6 +2520,12 @@ async def ws_stream(chunk, _sid=session_id, _mid=message_id, **kwargs): await memory.save_message(session_id, "assistant", str(final), agent.id) session["history"].append({"role": "assistant", "content": str(final)}) + # Extract cognitive metadata from QALB-7 pipeline + cognitive = {k: result.get(k) for k in ( + "nafs_level", "nafs_name", "ruh_energy", "qalb", "yaqin", + "mizan_label", "cognitive_method", "lubb", "lawwama", + ) if result.get(k) is not None} + await manager.send( client_id, { @@ -2042,6 +2536,7 @@ async def ws_stream(chunk, _sid=session_id, _mid=message_id, **kwargs): "session_id": session_id, "message_id": message_id, "success": result.get("success", True), + "cognitive": cognitive, }, ) diff --git a/backend/core/creativity.py b/backend/core/creativity.py new file mode 100644 index 0000000..12953a0 --- /dev/null +++ b/backend/core/creativity.py @@ -0,0 +1,574 @@ +""" +Ibdāʿ — Creativity Engine +=========================== + +"And He taught Adam the names of all things" — Quran 2:31 + +Implements Algorithm 4: IBDA_CREATIVITY_ENGINE + +5 creation modes (from Quranic creative verbs): + 1. Badī': Radical origination — novelty gradient ascent + 2. Khalq: Measured creation — evolutionary refinement + 3. Jaʿl: Recombination — conceptual blending + analogy + 4. Ṣunʿ: Refinement — gradient descent on imperfection + 5. Taṣwīr: Visualization — scene construction (imagination engine) + +Creativity landscape: + Ψ(z) = U(z)^β × N(z)^α × F(z) + + where: + - U(z) = utility (task relevance score) + - N(z) = novelty (distance from known solutions) + - F(z) = feasibility (implementation constraint satisfaction) + - α = novelty weight, β = utility weight + +Novelty gradient: + ∇z novelty(z) = ∇z[-min_k ||z - z_known_k||²] + → move away from known solutions in concept space + +Creative oscillation: + mode(t) = floor(sin(2π·t/T_creative) × 2.5) → cycles through modes + T_creative ≈ 30-90 interactions +""" + +import logging +import math +import random +import time +from dataclasses import dataclass, field +from enum import Enum +from typing import Any + +logger = logging.getLogger("mizan.creativity") + +# Creativity landscape weights +ALPHA_NOVELTY = 0.4 # novelty importance +BETA_UTILITY = 0.5 # utility importance +# Feasibility has implicit weight: 1 - α - β = 0.1 (or computed as multiplier) + +# Creative oscillation period (interactions) +T_CREATIVE = 60 + +# Novelty distance threshold — closer than this is "known" +NOVELTY_PROXIMITY_THRESHOLD = 0.3 + +# Khalq evolutionary parameters +MUTATION_RATE = 0.15 +SELECTION_PRESSURE = 0.7 + + +class CreationMode(Enum): + BADI = "badi" # Radical origination (Badī') + KHALQ = "khalq" # Measured evolutionary creation + JAL = "jal" # Recombination / analogy (Jaʿl) + SUNW = "sunw" # Refinement / polish (Ṣunʿ) + TASWIR = "taswir" # Visualization (Taṣwīr) + + +@dataclass +class ConceptVector: + """A concept in the creativity latent space.""" + label: str + features: dict[str, float] # semantic feature dimensions + novelty_score: float = 0.0 + utility_score: float = 0.0 + feasibility_score: float = 0.0 + + def landscape_score(self, alpha: float = ALPHA_NOVELTY, beta: float = BETA_UTILITY) -> float: + """Ψ(z) = U^β × N^α × F""" + return ( + (self.utility_score ** beta) + * (self.novelty_score ** alpha) + * max(0.01, self.feasibility_score) + ) + + +@dataclass +class BisociationResult: + """Result of bisociation (Koestler): two matrices of thought intersect.""" + concept_a: str + concept_b: str + intersection: str # the "aha" moment + novelty: float + metaphor: str + + +@dataclass +class CreativeOutput: + mode: CreationMode + primary_idea: str + elaboration: str + bisociations: list[BisociationResult] + landscape_score: float + novelty: float + utility: float + feasibility: float + iterations: int + creative_oscillation_phase: float + + +class IbdaCreativityEngine: + """ + Algorithm 4: IBDA_CREATIVITY_ENGINE + + Routes tasks to the appropriate creation mode based on: + - Task type (novel vs refined vs recombined) + - Qalb oscillation phase (creative cycle) + - Available conceptual vocabulary + - Landscape scoring Ψ(z) = U^β × N^α × F + + Each mode has a distinct mathematical procedure: + - Badī': gradient ascent on novelty landscape + - Khalq: evolutionary selection pressure + - Jaʿl: structure mapping + conceptual blending + - Ṣunʿ: gradient descent on imperfection metric + - Taṣwīr: delegates to imagination engine + """ + + def __init__(self): + self.known_solutions: list[ConceptVector] = [] + self.interaction_count = 0 + self.mode_history: list[CreationMode] = [] + + def create( + self, + task: str, + constraints: list[str] | None = None, + force_mode: CreationMode | None = None, + context_fragments: list[str] | None = None, + ) -> CreativeOutput: + """ + Main creativity entry point. + + 1. Select creation mode (oscillation or forced) + 2. Generate concept vector for task + 3. Apply mode-specific algorithm + 4. Score on creativity landscape + 5. Return creative output + """ + self.interaction_count += 1 + constraints = constraints or [] + context_fragments = context_fragments or [] + + # Select mode + mode = force_mode or self._select_mode() + self.mode_history.append(mode) + + # Build initial concept vector + concept = self._build_concept_vector(task, context_fragments) + + # Apply creation mode + if mode == CreationMode.BADI: + result = self._badi_origination(task, concept, constraints) + elif mode == CreationMode.KHALQ: + result = self._khalq_evolution(task, concept, constraints) + elif mode == CreationMode.JAL: + result = self._jal_recombination(task, concept, context_fragments) + elif mode == CreationMode.SUNW: + result = self._sunw_refinement(task, concept, constraints) + else: # TASWIR + result = self._taswir_visualization(task, concept) + + # Score on landscape + concept.novelty_score = result.novelty + concept.utility_score = result.utility + concept.feasibility_score = result.feasibility + result.landscape_score = concept.landscape_score() + + # Register solution as known (for future novelty computation) + self._register_known_solution(concept) + + logger.debug( + "[IBDA] mode=%s Ψ=%.3f N=%.3f U=%.3f F=%.3f", + mode.value, result.landscape_score, result.novelty, + result.utility, result.feasibility, + ) + + return result + + def _select_mode(self) -> CreationMode: + """ + Creative oscillation: + mode(t) = f(sin(2π·t/T_creative)) + + Maps sinusoidal phase to creation modes cyclically. + """ + phase = (2 * math.pi * self.interaction_count) / T_CREATIVE + sin_val = math.sin(phase) + # Map [-1, 1] → [0, 4] → mode index + mode_idx = int((sin_val + 1.0) / 2.0 * (len(CreationMode) - 0.01)) + modes = list(CreationMode) + return modes[mode_idx] + + def _build_concept_vector( + self, task: str, fragments: list[str] + ) -> ConceptVector: + """Build feature vector for task in concept space.""" + words = task.lower().split() + features = { + "length": min(1.0, len(words) / 50.0), + "novelty_seed": (hash(task) % 1000) / 1000.0, + "fragment_richness": min(1.0, len(fragments) / 5.0), + "question_mark": 1.0 if "?" in task else 0.0, + "imperative": 1.0 if words[0] in {"create", "build", "design", "make", "write"} else 0.0, + } + return ConceptVector(label=task[:50], features=features) + + def _badi_origination( + self, task: str, concept: ConceptVector, constraints: list[str] + ) -> CreativeOutput: + """ + Badī': Radical origination via novelty gradient ascent. + + ∇z novelty(z) = ∇z[-min_k ||z - z_known_k||²] + + Move maximally far from all known solutions. + Constraints are relaxed progressively. + """ + # Compute minimum distance to known solutions + min_dist = self._min_distance_to_known(concept) + novelty = min(1.0, min_dist + 0.2) # gradient step pushes away + + # Radical idea generation: invert standard approaches + inversions = self._generate_inversions(task) + primary_idea = inversions[0] if inversions else f"Radical reframe of: {task[:80]}" + elaboration = " | ".join(inversions[:3]) if inversions else "No known solutions — pure origination territory" + + bisociations = self._find_bisociations(task, n=2) + + return CreativeOutput( + mode=CreationMode.BADI, + primary_idea=primary_idea, + elaboration=elaboration, + bisociations=bisociations, + landscape_score=0.0, # computed after + novelty=round(novelty, 3), + utility=round(self._estimate_utility(task, primary_idea, constraints), 3), + feasibility=round(self._estimate_feasibility(primary_idea, constraints), 3), + iterations=1, + creative_oscillation_phase=round( + (2 * math.pi * self.interaction_count) / T_CREATIVE, 4 + ), + ) + + def _khalq_evolution( + self, task: str, concept: ConceptVector, constraints: list[str], generations: int = 3 + ) -> CreativeOutput: + """ + Khalq: Measured creation via evolutionary refinement. + + Population of candidate solutions evolves over generations. + Fitness = Ψ(z) landscape score. + Mutation rate decays with generations (constraint relaxation schedule). + """ + # Generate initial population + population = [self._generate_candidate(task, constraints, mutation=MUTATION_RATE)] + for _ in range(2): + population.append(self._generate_candidate( + task, constraints, mutation=MUTATION_RATE * 1.5 + )) + + best = population[0] + best_score = 0.0 + iterations = 0 + + for gen in range(generations): + # Score and select + scored = [ + (c, self._score_candidate(c, task, constraints)) + for c in population + ] + scored.sort(key=lambda x: x[1], reverse=True) + best, best_score = scored[0] + + # Mutate survivors into next generation + survivors = [c for c, _ in scored[:max(1, len(scored) // 2)]] + mutation = MUTATION_RATE * (1.0 - gen / generations * SELECTION_PRESSURE) + population = survivors + [ + self._mutate_candidate(s, mutation) for s in survivors + ] + iterations = gen + 1 + + bisociations = self._find_bisociations(task, n=1) + novelty = max(0.3, min(0.9, 1.0 - self._min_distance_to_known(concept) * 0.5)) + + return CreativeOutput( + mode=CreationMode.KHALQ, + primary_idea=best, + elaboration=f"Evolved over {iterations} generations. Fitness: {best_score:.3f}", + bisociations=bisociations, + landscape_score=0.0, + novelty=round(novelty, 3), + utility=round(self._estimate_utility(task, best, constraints), 3), + feasibility=round(self._estimate_feasibility(best, constraints), 3), + iterations=iterations, + creative_oscillation_phase=round( + (2 * math.pi * self.interaction_count) / T_CREATIVE, 4 + ), + ) + + def _jal_recombination( + self, task: str, concept: ConceptVector, fragments: list[str] + ) -> CreativeOutput: + """ + Jaʿl: Recombination via conceptual blending and analogy. + + Structure mapping (Gentner): find common relational structure + between source and target domains. + + Bisociation (Koestler): two matrices of thought intersect + at an unexpected point → creative insight. + """ + bisociations = self._find_bisociations(task, n=3) + + # Blend top fragments with task + blended_concepts = [] + for fragment in fragments[:2]: + blend = self._blend_concepts(task, fragment) + blended_concepts.append(blend) + + primary_idea = ( + bisociations[0].intersection + if bisociations + else f"Blend of {len(fragments)} context fragments with task" + ) + elaboration = " | ".join(blended_concepts[:2]) if blended_concepts else task + + novelty = 0.5 + 0.3 * (len(bisociations) / 3) + + return CreativeOutput( + mode=CreationMode.JAL, + primary_idea=primary_idea, + elaboration=elaboration, + bisociations=bisociations, + landscape_score=0.0, + novelty=round(min(0.9, novelty), 3), + utility=round(self._estimate_utility(task, primary_idea, []), 3), + feasibility=round(0.7, 3), # recombination is inherently feasible + iterations=1, + creative_oscillation_phase=round( + (2 * math.pi * self.interaction_count) / T_CREATIVE, 4 + ), + ) + + def _sunw_refinement( + self, task: str, concept: ConceptVector, constraints: list[str] + ) -> CreativeOutput: + """ + Ṣunʿ: Refinement via gradient descent on imperfection. + + elegance = utility / complexity + imperfection = 1 - elegance + + Iterate: identify imperfection sources, apply targeted fixes. + """ + # Start with a standard approach and refine + base_idea = f"Standard approach: {task[:100]}" + imperfections = self._identify_imperfections(base_idea, constraints) + + refined = base_idea + iterations = 0 + for imperfection in imperfections[:3]: + refined = self._apply_refinement(refined, imperfection) + iterations += 1 + + elegance = self._compute_elegance(refined, constraints) + + return CreativeOutput( + mode=CreationMode.SUNW, + primary_idea=refined, + elaboration=f"Refined {iterations} imperfections. Elegance: {elegance:.3f}", + bisociations=[], + landscape_score=0.0, + novelty=round(0.2 + 0.3 * elegance, 3), # refined = less novel + utility=round(0.6 + 0.3 * elegance, 3), # but more useful + feasibility=round(0.8, 3), + iterations=iterations, + creative_oscillation_phase=round( + (2 * math.pi * self.interaction_count) / T_CREATIVE, 4 + ), + ) + + def _taswir_visualization( + self, task: str, concept: ConceptVector + ) -> CreativeOutput: + """ + Taṣwīr: Creative visualization — generates a rich mental image. + Delegates to imagination engine in full integration. + """ + visual_description = ( + f"Visual scenario: [{task[:80]}]\n" + f"Scene: {concept.features.get('novelty_seed', 0.5):.2f} novelty intensity\n" + f"Render: coarse→medium→fine hierarchical construction" + ) + + return CreativeOutput( + mode=CreationMode.TASWIR, + primary_idea=f"Visualized: {task[:100]}", + elaboration=visual_description, + bisociations=[], + landscape_score=0.0, + novelty=round(0.6, 3), + utility=round(0.7, 3), + feasibility=round(0.9, 3), + iterations=1, + creative_oscillation_phase=round( + (2 * math.pi * self.interaction_count) / T_CREATIVE, 4 + ), + ) + + def _find_bisociations(self, task: str, n: int = 2) -> list[BisociationResult]: + """ + Bisociation: find two distant concept matrices that unexpectedly intersect. + """ + domain_pairs = [ + ("biology", "software"), + ("music", "mathematics"), + ("architecture", "language"), + ("cooking", "algorithms"), + ("navigation", "problem_solving"), + ] + + results = [] + task_lower = task.lower() + + for domain_a, domain_b in domain_pairs[:n]: + intersection = f"The {domain_a} metaphor applied to {task[:40]}: {domain_b} lens" + metaphor = f"Like {domain_a} processes, this task involves {domain_b} principles" + novelty = 0.6 + (hash(domain_a + task) % 30) / 100.0 + + results.append(BisociationResult( + concept_a=domain_a, + concept_b=domain_b, + intersection=intersection, + novelty=round(min(0.95, novelty), 3), + metaphor=metaphor, + )) + + return results + + def _blend_concepts(self, concept_a: str, concept_b: str) -> str: + """Structure mapping: extract relational structure common to A and B.""" + words_a = set(concept_a.lower().split()[:5]) + words_b = set(concept_b.lower().split()[:5]) + shared = words_a & words_b + unique_a = (words_a - words_b) + unique_b = (words_b - words_a) + + if shared: + return f"Shared structure [{', '.join(list(shared)[:2])}] bridging [{', '.join(list(unique_a)[:2])}] and [{', '.join(list(unique_b)[:2])}]" + return f"Cross-domain blend: {concept_a[:30]} ↔ {concept_b[:30]}" + + def _generate_inversions(self, task: str) -> list[str]: + """Generate radical inversions of standard approaches.""" + inversions = [ + f"Invert: Instead of solving '{task[:50]}', solve its opposite", + f"Constraint removal: What if there were no constraints on '{task[:40]}'?", + f"Scale inversion: Apply micro/macro scale inversion to '{task[:40]}'", + ] + return inversions + + def _generate_candidate( + self, task: str, constraints: list[str], mutation: float + ) -> str: + """Generate a candidate solution for evolutionary selection.""" + seed = f"Approach {random.random():.2f}: {task[:60]}" + if constraints: + seed += f" (satisfying: {constraints[0][:40]})" + return seed + + def _mutate_candidate(self, candidate: str, mutation_rate: float) -> str: + """Apply mutation to a candidate solution.""" + words = candidate.split() + if len(words) > 3 and random.random() < mutation_rate: + idx = random.randint(1, len(words) - 1) + words[idx] = f"[mutated:{words[idx]}]" + return " ".join(words) + + def _score_candidate( + self, candidate: str, task: str, constraints: list[str] + ) -> float: + """Fitness function for evolutionary selection.""" + utility = self._estimate_utility(task, candidate, constraints) + feasibility = self._estimate_feasibility(candidate, constraints) + return utility * BETA_UTILITY + feasibility * (1.0 - BETA_UTILITY) + + def _identify_imperfections( + self, idea: str, constraints: list[str] + ) -> list[str]: + """Find imperfections in current idea relative to constraints.""" + imperfections = [] + if len(idea.split()) > 20: + imperfections.append("verbosity") + if not constraints: + imperfections.append("missing constraints") + if "standard" in idea.lower(): + imperfections.append("lack of novelty") + return imperfections + + def _apply_refinement(self, idea: str, imperfection: str) -> str: + """Apply targeted fix to an imperfection.""" + if imperfection == "verbosity": + return idea[:len(idea) // 2] + " [condensed]" + if imperfection == "lack of novelty": + return idea.replace("standard", "novel") + return idea + f" [refined: {imperfection} addressed]" + + def _compute_elegance(self, idea: str, constraints: list[str]) -> float: + """elegance = utility / complexity""" + utility = self._estimate_utility("", idea, constraints) + complexity = min(1.0, len(idea.split()) / 30.0) + return utility / max(complexity, 0.1) + + def _estimate_utility( + self, task: str, idea: str, constraints: list[str] + ) -> float: + """Estimate how useful the idea is for the task.""" + if not task: + return 0.6 + task_words = set(task.lower().split()) + idea_words = set(idea.lower().split()) + overlap = len(task_words & idea_words) / max(len(task_words), 1) + constraint_penalty = 0.1 * max(0, len(constraints) - len(idea_words & set(" ".join(constraints).lower().split()))) + return max(0.1, min(0.95, 0.4 + 0.5 * overlap - constraint_penalty)) + + def _estimate_feasibility(self, idea: str, constraints: list[str]) -> float: + """Estimate how feasible the idea is to implement.""" + base = 0.6 + if len(idea.split()) > 50: + base -= 0.1 # complex ideas are harder + if constraints: + base -= 0.05 * len(constraints) # more constraints = harder + return max(0.2, min(0.95, base)) + + def _min_distance_to_known(self, concept: ConceptVector) -> float: + """ + min_k ||z - z_known_k||² + Approximated by novelty_seed distance in feature space. + """ + if not self.known_solutions: + return 1.0 # maximum distance — all is novel + seed = concept.features.get("novelty_seed", 0.5) + distances = [ + abs(seed - k.features.get("novelty_seed", 0.5)) + for k in self.known_solutions + ] + return min(distances) + + def _register_known_solution(self, concept: ConceptVector) -> None: + self.known_solutions.append(concept) + if len(self.known_solutions) > 100: + self.known_solutions.pop(0) + + def to_dict(self) -> dict: + mode_counts = {} + for m in self.mode_history: + mode_counts[m.value] = mode_counts.get(m.value, 0) + 1 + return { + "interaction_count": self.interaction_count, + "known_solutions": len(self.known_solutions), + "mode_history_counts": mode_counts, + "current_oscillation_phase": round( + (2 * math.pi * self.interaction_count) / T_CREATIVE, 4 + ), + } diff --git a/backend/core/developmental_stages.py b/backend/core/developmental_stages.py new file mode 100644 index 0000000..d891f31 --- /dev/null +++ b/backend/core/developmental_stages.py @@ -0,0 +1,279 @@ +""" +Developmental Stages (مراحل التطور) — Progressive Capability Gating +===================================================================== + +"And Allah has extracted you from the wombs of your mothers not knowing a thing, + and He made for you hearing and vision and intellect that perhaps you would + be grateful." — Quran 16:78 + +Capability gating tied to Nafs level — agents unlock tools and cognitive +features progressively, mirroring embryonic development from the Quran (23:12-14): + + Nutfah (نطفة) → Nafs 1: Sperm/seed — minimal, basic + Alaqah (عَلَقَة) → Nafs 2: Clinging clot — can cling to resources + Mudghah (مُضْغَة) → Nafs 3: Chewed substance — initial structure + Izham (عِظَام) → Nafs 4: Bones — can create other agents (skeleton) + Lahm (لَحْم) → Nafs 5: Flesh — full capability + Nafkh (نَفْخ) → Nafs 6: Breath — metacognitive awareness + Khalq Akhar (خَلْق آخَر) → Nafs 7: New creation — full autonomy + governance +""" + +import logging +from dataclasses import dataclass, field + +logger = logging.getLogger("mizan.dev_stages") + + +# All tools available in the system +ALL_TOOLS = frozenset({ + "bash", "http_get", "http_post", "read_file", "write_file", + "list_files", "python_exec", "create_agent", "create_skill", + "compact_context", "recall_memory", +}) + + +@dataclass +class StageCapabilities: + """Capabilities unlocked at a given developmental stage.""" + stage_name: str + nafs_level: int + quran_ref: str + allowed_tools: frozenset + max_turns: int + # Feature flags + can_delegate: bool = False # Can delegate to sub-agents + nafs_triad: bool = False # Nafs Triad deliberation + causal_rung: int = 0 # 0=none, 1=observe, 2=intervene, 3=counterfactual + lubb_active: bool = False # Metacognitive monitoring + fuad_active: bool = False # Conviction formation + description: str = "" + + +# Progressive capability gates — each stage unlocks more +_STAGES: dict[int, StageCapabilities] = { + 1: StageCapabilities( + stage_name="Nutfah", + nafs_level=1, + quran_ref="23:13", + allowed_tools=frozenset({"bash", "read_file", "recall_memory", "compact_context"}), + max_turns=5, + causal_rung=0, + description="Seed stage — observe and recall only", + ), + 2: StageCapabilities( + stage_name="Alaqah", + nafs_level=2, + quran_ref="23:14", + allowed_tools=frozenset({ + "bash", "read_file", "write_file", "http_get", + "recall_memory", "compact_context", + }), + max_turns=8, + causal_rung=0, + description="Clot stage — can now write and fetch external data", + ), + 3: StageCapabilities( + stage_name="Mudghah", + nafs_level=3, + quran_ref="23:14", + allowed_tools=frozenset({ + "bash", "read_file", "write_file", "list_files", + "http_get", "http_post", "python_exec", + "recall_memory", "compact_context", + }), + max_turns=10, + can_delegate=True, + causal_rung=1, # Can observe causal associations + description="Structured stage — execute code, observe causality", + ), + 4: StageCapabilities( + stage_name="Izham", + nafs_level=4, + quran_ref="23:14", + allowed_tools=frozenset({ + "bash", "read_file", "write_file", "list_files", + "http_get", "http_post", "python_exec", + "create_agent", "recall_memory", "compact_context", + }), + max_turns=12, + can_delegate=True, + nafs_triad=True, + causal_rung=2, # Can intervene and predict effects + description="Skeleton stage — can create sub-agents, nafs deliberation active", + ), + 5: StageCapabilities( + stage_name="Lahm", + nafs_level=5, + quran_ref="23:14", + allowed_tools=ALL_TOOLS, + max_turns=15, + can_delegate=True, + nafs_triad=True, + causal_rung=3, # Full causal reasoning + lubb_active=True, + fuad_active=True, + description="Full capability — all tools, metacognition, conviction formation", + ), + 6: StageCapabilities( + stage_name="Nafkh", + nafs_level=6, + quran_ref="23:14", + allowed_tools=ALL_TOOLS, + max_turns=20, + can_delegate=True, + nafs_triad=True, + causal_rung=3, + lubb_active=True, + fuad_active=True, + description="Breath stage — extended reasoning, full metacognitive monitoring", + ), + 7: StageCapabilities( + stage_name="Khalq Akhar", + nafs_level=7, + quran_ref="23:14", + allowed_tools=ALL_TOOLS, + max_turns=25, + can_delegate=True, + nafs_triad=True, + causal_rung=3, + lubb_active=True, + fuad_active=True, + description="New creation — full autonomy, governance role", + ), +} + + +@dataclass +class UpgradeReport: + """Result of checking if an agent is ready to advance nafs levels.""" + current_level: int + target_level: int + ready: bool + missing_requirements: list[str] = field(default_factory=list) + tazkiyah_score: float = 0.0 + + def to_dict(self) -> dict: + return { + "current_level": self.current_level, + "target_level": self.target_level, + "ready": self.ready, + "missing": self.missing_requirements, + "tazkiyah_score": round(self.tazkiyah_score, 3), + } + + +class DevelopmentalGate: + """ + Progressive capability gating tied to nafs_level. + + Usage: + gate = DevelopmentalGate() + caps = gate.get_capabilities(nafs_level=3) + # caps.allowed_tools, caps.max_turns, caps.causal_rung, etc. + + report = gate.check_upgrade_readiness(agent) + if report.ready: + agent.nafs_level += 1 + """ + + def get_capabilities(self, nafs_level: int) -> StageCapabilities: + """Get capabilities for a given nafs_level (clamps to valid range).""" + level = max(1, min(7, nafs_level)) + return _STAGES[level] + + def filter_tool_schemas( + self, tool_schemas: list[dict], nafs_level: int + ) -> list[dict]: + """ + Filter tool schemas to only include tools allowed at this nafs level. + Skills and plugin tools are always allowed (dynamic capabilities). + """ + caps = self.get_capabilities(nafs_level) + filtered = [] + for schema in tool_schemas: + name = schema.get("name", "") + # Always include: tools not in the base set (skills/plugins) + if name not in ALL_TOOLS: + filtered.append(schema) + elif name in caps.allowed_tools: + filtered.append(schema) + else: + logger.debug( + "[DEV_GATE] Blocking tool '%s' at nafs_level=%d (%s)", + name, nafs_level, caps.stage_name, + ) + return filtered + + def check_upgrade_readiness(self, agent) -> UpgradeReport: + """ + Check if an agent meets the requirements to advance to the next nafs level. + Uses NafsProfile.EVOLUTION_THRESHOLDS from core/architecture.py. + """ + from core.architecture import NafsProfile + + current = getattr(agent, "nafs_level", 1) + target = min(7, current + 1) + + if current >= 7: + return UpgradeReport( + current_level=current, + target_level=7, + ready=False, + missing_requirements=["Already at maximum nafs level"], + tazkiyah_score=1.0, + ) + + threshold = NafsProfile.EVOLUTION_THRESHOLDS.get(target, {}) + missing = [] + + success_rate = getattr(agent, "success_rate", 0.0) + total_tasks = getattr(agent, "total_tasks", 0) + required_sr = threshold.get("success_rate", 0.0) + required_tasks = threshold.get("min_tasks", 0) + + if success_rate < required_sr: + missing.append(f"success_rate {success_rate:.0%} < {required_sr:.0%}") + if total_tasks < required_tasks: + missing.append(f"tasks {total_tasks} < {required_tasks}") + if "min_hikmah" in threshold: + hikmah = len(getattr(agent, "hikmah", [])) + if hikmah < threshold["min_hikmah"]: + missing.append(f"hikmah {hikmah} < {threshold['min_hikmah']}") + + # Compute rough tazkiyah score + tazkiyah = ( + min(1.0, success_rate / max(required_sr, 0.01)) * 0.6 + + min(1.0, total_tasks / max(required_tasks, 1)) * 0.4 + ) + + report = UpgradeReport( + current_level=current, + target_level=target, + ready=len(missing) == 0, + missing_requirements=missing, + tazkiyah_score=round(tazkiyah, 3), + ) + + if report.ready: + logger.info( + "[DEV_GATE] Agent ready to advance nafs_level %d → %d (tazkiyah=%.2f)", + current, target, tazkiyah, + ) + + return report + + def stage_summary(self) -> list[dict]: + """Summary of all stages for display.""" + return [ + { + "level": level, + "stage": s.stage_name, + "quran_ref": s.quran_ref, + "tools": len(s.allowed_tools), + "max_turns": s.max_turns, + "causal_rung": s.causal_rung, + "lubb": s.lubb_active, + "description": s.description, + } + for level, s in _STAGES.items() + ] diff --git a/backend/core/dream_engine.py b/backend/core/dream_engine.py new file mode 100644 index 0000000..f6e7cb4 --- /dev/null +++ b/backend/core/dream_engine.py @@ -0,0 +1,630 @@ +""" +Manām — Dream Engine +===================== + +"Allah takes souls at the time of their death, and those that do not die + [He takes] during their sleep." — Quran 39:42 + +Implements Algorithm 6: MANAM_DREAM_ENGINE + +Offline memory consolidation with three phases: + 1. NREM: Selective replay, accelerated compression, synaptic downscaling, gist extraction + 2. REM: Adversarial dream learning (GAN), emotional processing, creative insight + 3. Taʾwīl: Dream interpretation — symbolic decoding + integration with waking knowledge + +Key mathematics: + +NREM synaptic downscaling: + w(after) = w(before) × max(0, 1 - δ) + +NREM replay priority scoring: + P_replay(m_i) = α·e(m_i) + β·n(m_i) + γ·g(m_i) + δ·err(m_i) + + where: + - e = emotional intensity + - n = novelty (surprise during encoding) + - g = goal relevance + - err = prediction error magnitude + +REM adversarial dream learning (GAN-inspired): + min_G max_D V(D, G) = E_x[log D(x)] + E_z[log(1 - D(G(z, fragments)))] + + D: discriminator distinguishing real memories from dreams + G: generator creating plausible novel scenarios from fragments + noise + +REM dream bizarreness: + bizarreness = Dirichlet sparse mixing + high-noise REM activation +""" + +import logging +import math +import random +import time +from dataclasses import dataclass, field +from enum import Enum +from typing import Any + +logger = logging.getLogger("mizan.dream_engine") + +# NREM replay priority weights +ALPHA_EMOTIONAL = 0.30 # emotional intensity weight +BETA_NOVELTY = 0.25 # novelty weight +GAMMA_GOAL = 0.25 # goal relevance weight +DELTA_ERROR = 0.20 # prediction error weight + +# NREM synaptic downscaling rate +DOWNSCALING_DELTA = 0.05 # w → w × (1 - δ) + +# NREM temporal compression +COMPRESSION_RATIO = 20.0 # 20x faster replay than original + +# REM noise level +REM_NOISE_SIGMA = 0.3 + +# GAN learning rate approximation +GAN_LR = 0.01 + +# Gist extraction — keep top K concepts +GIST_TOP_K = 5 + + +class DreamPhase(Enum): + AWAKE = "awake" + NREM = "nrem" # Non-REM: slow-wave consolidation + REM = "rem" # REM: adversarial creative replay + TAWIL = "tawil" # Taʾwīl: dream interpretation + + +@dataclass +class MemoryTrace: + """A memory trace eligible for dream replay.""" + content: str + emotional_intensity: float # |valence|, 0-1 + novelty: float # surprisal during encoding + goal_relevance: float # 0-1 + prediction_error: float # original error magnitude + encoding_time: float = field(default_factory=time.time) + replay_count: int = 0 + + def priority_score(self) -> float: + """ + P_replay(m_i) = α·e + β·n + γ·g + δ·err + """ + return ( + ALPHA_EMOTIONAL * self.emotional_intensity + + BETA_NOVELTY * self.novelty + + GAMMA_GOAL * self.goal_relevance + + DELTA_ERROR * self.prediction_error + ) + + +@dataclass +class NREMCycle: + """Result of one NREM slow-wave sleep cycle.""" + replayed_memories: list[str] # compressed content + synaptic_weight_changes: dict[str, float] # key → new weight + gist_extracted: list[str] # high-level patterns extracted + compression_ratio: float + downscaling_applied: bool + + +@dataclass +class REMCycle: + """Result of one REM cycle.""" + dream_content: list[str] # generated dream scenarios + emotional_episodes: list[str] # emotionally processed memories + creative_insights: list[str] # novel connections discovered + discriminator_loss: float # GAN D loss (lower = more realistic dreams) + generator_loss: float # GAN G loss (lower = better generation) + bizarreness_score: float + + +@dataclass +class TawilInterpretation: + """Dream interpretation (Taʾwīl).""" + dream_symbols: dict[str, str] # symbol → meaning + waking_relevance: str # how the dream relates to current tasks + insights: list[str] # extracted actionable insights + confidence: float + + +@dataclass +class DreamSession: + """A complete offline consolidation session.""" + phase_sequence: list[DreamPhase] + nrem_cycles: list[NREMCycle] + rem_cycles: list[REMCycle] + tawil: TawilInterpretation | None + memories_consolidated: int + total_duration_s: float + insights_generated: list[str] + + +class ManamDreamEngine: + """ + Algorithm 6: MANAM_DREAM_ENGINE + + Offline memory consolidation system that runs during idle periods. + Implements the sleep memory consolidation hypothesis with three phases: + + 1. NREM (Slow-wave): Priority-scored memory replay at 20x compression + → synaptic homeostasis → pattern extraction (gist) + + 2. REM (Paradoxical): GAN-inspired adversarial generation + → emotional processing → creative bisociation → insight detection + + 3. Taʾwīl: Symbolic interpretation of dream content + → integration with waking knowledge → actionable insights + + The engine maintains a replay buffer of recent memory traces + and periodically runs consolidation cycles. + """ + + def __init__(self): + self.replay_buffer: list[MemoryTrace] = [] + self.synaptic_weights: dict[str, float] = {} + self.extracted_gists: list[str] = [] + self.insight_bank: list[str] = [] + + # GAN state (simplified) + self._discriminator_confidence = 0.5 + self._generator_quality = 0.3 + + self._session_count = 0 + self._current_phase = DreamPhase.AWAKE + + def add_memory( + self, + content: str, + emotional_intensity: float = 0.3, + novelty: float = 0.5, + goal_relevance: float = 0.5, + prediction_error: float = 0.2, + ) -> None: + """Add a memory trace to the replay buffer.""" + trace = MemoryTrace( + content=content, + emotional_intensity=min(1.0, max(0.0, emotional_intensity)), + novelty=min(1.0, max(0.0, novelty)), + goal_relevance=min(1.0, max(0.0, goal_relevance)), + prediction_error=min(1.0, max(0.0, prediction_error)), + ) + self.replay_buffer.append(trace) + + # Cap buffer at 500 traces + if len(self.replay_buffer) > 500: + self.replay_buffer.pop(0) + + logger.debug( + "[MANAM] Memory added: priority=%.3f content=%s", + trace.priority_score(), content[:50], + ) + + def run_consolidation( + self, + n_nrem_cycles: int = 3, + n_rem_cycles: int = 2, + run_tawil: bool = True, + ) -> DreamSession: + """ + Run a full offline consolidation session: + NREM cycles → REM cycles → Taʾwīl interpretation. + """ + self._session_count += 1 + start = time.monotonic() + logger.info( + "[MANAM] Starting dream session #%d: %d memories, %d NREM + %d REM cycles", + self._session_count, len(self.replay_buffer), n_nrem_cycles, n_rem_cycles, + ) + + phase_sequence = [] + nrem_results = [] + rem_results = [] + + # Phase 1: NREM cycles + for i in range(n_nrem_cycles): + self._current_phase = DreamPhase.NREM + phase_sequence.append(DreamPhase.NREM) + nrem = self._run_nrem_cycle() + nrem_results.append(nrem) + logger.debug("[MANAM] NREM cycle %d: %d replays, %d gists", i + 1, len(nrem.replayed_memories), len(nrem.gist_extracted)) + + # Phase 2: REM cycles + for i in range(n_rem_cycles): + self._current_phase = DreamPhase.REM + phase_sequence.append(DreamPhase.REM) + rem = self._run_rem_cycle() + rem_results.append(rem) + logger.debug("[MANAM] REM cycle %d: %d dreams, %d insights, D_loss=%.3f", i + 1, len(rem.dream_content), len(rem.creative_insights), rem.discriminator_loss) + + # Phase 3: Taʾwīl + tawil = None + if run_tawil: + self._current_phase = DreamPhase.TAWIL + phase_sequence.append(DreamPhase.TAWIL) + all_dream_content = [d for rem in rem_results for d in rem.dream_content] + tawil = self._run_tawil(all_dream_content) + + # Collect all insights + all_insights = list(self.insight_bank[-10:]) + for rem in rem_results: + all_insights.extend(rem.creative_insights) + if tawil: + all_insights.extend(tawil.insights) + + memories_consolidated = sum(len(n.replayed_memories) for n in nrem_results) + duration = time.monotonic() - start + self._current_phase = DreamPhase.AWAKE + + return DreamSession( + phase_sequence=phase_sequence, + nrem_cycles=nrem_results, + rem_cycles=rem_results, + tawil=tawil, + memories_consolidated=memories_consolidated, + total_duration_s=round(duration, 4), + insights_generated=all_insights[:20], + ) + + def _run_nrem_cycle(self) -> NREMCycle: + """ + NREM slow-wave sleep cycle: + + 1. Priority scoring: P_replay(m_i) = α·e + β·n + γ·g + δ·err + 2. Selective replay: top-K memories replayed at 20x compression + 3. Synaptic homeostasis: w → w × max(0, 1 - δ) + 4. Interleaved replay (hippocampal → neocortical) + 5. Gist extraction: distill high-level patterns + """ + if not self.replay_buffer: + return NREMCycle( + replayed_memories=[], + synaptic_weight_changes={}, + gist_extracted=[], + compression_ratio=COMPRESSION_RATIO, + downscaling_applied=False, + ) + + # Step 1-2: Priority sort and selective replay + sorted_traces = sorted( + self.replay_buffer, + key=lambda t: t.priority_score(), + reverse=True, + ) + top_traces = sorted_traces[:min(10, len(sorted_traces))] + + replayed = [] + for trace in top_traces: + trace.replay_count += 1 + # Compressed replay: truncate to 1/COMPRESSION_RATIO of original detail + compressed = trace.content[:max(20, len(trace.content) // int(COMPRESSION_RATIO))] + replayed.append(f"[NREM:{trace.replay_count}x] {compressed}") + + # Step 3: Synaptic homeostasis — downscale all weights + weight_changes = {} + for key in list(self.synaptic_weights.keys()): + old = self.synaptic_weights[key] + new = old * max(0.0, 1.0 - DOWNSCALING_DELTA) + self.synaptic_weights[key] = new + weight_changes[key] = round(new - old, 4) + + # New weights from replayed memories + for trace in top_traces[:3]: + key = f"mem_{hash(trace.content[:20]) % 10000}" + self.synaptic_weights[key] = trace.priority_score() + weight_changes[key] = self.synaptic_weights[key] + + # Step 5: Gist extraction — extract common themes + gists = self._extract_gist(top_traces) + self.extracted_gists.extend(gists) + + # Remove low-priority memories after consolidation + consolidation_threshold = 0.2 + self.replay_buffer = [ + t for t in self.replay_buffer + if t.priority_score() > consolidation_threshold or t.replay_count == 0 + ] + + return NREMCycle( + replayed_memories=replayed, + synaptic_weight_changes=weight_changes, + gist_extracted=gists, + compression_ratio=COMPRESSION_RATIO, + downscaling_applied=True, + ) + + def _run_rem_cycle(self) -> REMCycle: + """ + REM paradoxical sleep cycle: + + 1. GAN-inspired adversarial generation: + min_G max_D V(D,G) = E_x[log D(x)] + E_z[log(1 - D(G(z, fragments)))] + + 2. Emotional memory processing (safe replay of high-affect memories) + + 3. Creative insight detection: + Bisociate distant memories + novel pattern recognition + + 4. Dream bizarreness via Dirichlet sparse mixing + high noise + """ + # Step 1: GAN-inspired dream generation + fragments = [t.content[:50] for t in self.replay_buffer[:5]] + noise = [random.gauss(0, REM_NOISE_SIGMA) for _ in range(5)] + dream_content = self._generate_dreams(fragments, noise) + + # GAN training step (discriminator update) + d_loss, g_loss = self._gan_training_step(dream_content, fragments) + self._discriminator_confidence = min(0.95, self._discriminator_confidence + GAN_LR * (0.5 - d_loss)) + self._generator_quality = min(0.95, self._generator_quality + GAN_LR * (0.5 - g_loss)) + + # Step 2: Emotional processing + high_affect = [ + t for t in self.replay_buffer + if t.emotional_intensity > 0.6 + ][:3] + emotional_episodes = [ + f"[REM:affect={t.emotional_intensity:.2f}] {t.content[:60]}" + for t in high_affect + ] + + # Step 3: Creative insight detection + insights = self._detect_creative_insights(dream_content, fragments) + self.insight_bank.extend(insights) + + # Step 4: Bizarreness score + bizarreness = self._compute_bizarreness(dream_content) + + return REMCycle( + dream_content=dream_content, + emotional_episodes=emotional_episodes, + creative_insights=insights, + discriminator_loss=round(d_loss, 4), + generator_loss=round(g_loss, 4), + bizarreness_score=round(bizarreness, 4), + ) + + def _run_tawil(self, dream_content: list[str]) -> TawilInterpretation: + """ + Taʾwīl: Dream interpretation. + + Symbolic decoding + integration with waking knowledge. + + Classical symbols are mapped to functional meanings. + Insights are extracted from recurring dream patterns. + """ + # Symbol extraction and mapping + symbol_lexicon = { + "water": "knowledge / emotion", + "fire": "transformation / energy", + "mountain": "challenge / stability", + "path": "decision / journey", + "door": "opportunity / threshold", + "light": "clarity / guidance", + "darkness": "uncertainty / hidden knowledge", + "book": "memory / wisdom", + "bird": "message / aspiration", + "tree": "growth / rootedness", + } + + found_symbols = {} + all_content = " ".join(dream_content).lower() + for symbol, meaning in symbol_lexicon.items(): + if symbol in all_content: + found_symbols[symbol] = meaning + + # Waking relevance: connect to recent gists + if self.extracted_gists: + recent_gist = self.extracted_gists[-1] + waking_relevance = f"Dreams align with waking pattern: {recent_gist[:100]}" + else: + waking_relevance = "Dreams consolidating recent experiences" + + # Extract actionable insights from dream patterns + insights = [] + if found_symbols: + for symbol, meaning in list(found_symbols.items())[:3]: + insights.append(f"Symbol '{symbol}' suggests: {meaning}") + if self.extracted_gists: + insights.append(f"Pattern recognition: {self.extracted_gists[-1][:80]}") + + confidence = min(0.9, 0.3 + 0.1 * len(found_symbols) + 0.2 * len(self.extracted_gists)) + + return TawilInterpretation( + dream_symbols=found_symbols, + waking_relevance=waking_relevance, + insights=insights, + confidence=round(confidence, 3), + ) + + def _extract_gist(self, traces: list[MemoryTrace]) -> list[str]: + """ + Gist extraction: find common themes across top memory traces. + Information bottleneck: keep shared structure, discard details. + """ + if not traces: + return [] + + # Find shared words across traces (common structure) + word_sets = [set(t.content.lower().split()[:15]) for t in traces] + if not word_sets: + return [] + + common = word_sets[0] + for ws in word_sets[1:]: + common = common & ws + + # Filter meaningful words (length > 3) + meaningful = [w for w in common if len(w) > 3][:GIST_TOP_K] + + if meaningful: + gist = f"Pattern: [{', '.join(meaningful)}] across {len(traces)} memories" + return [gist] + + # Fallback: use highest priority memory's first sentence + top = traces[0] + first_sentence = top.content.split(".")[0][:80] + return [f"Gist: {first_sentence}"] + + def _generate_dreams( + self, fragments: list[str], noise: list[float] + ) -> list[str]: + """ + G(z, fragments): Generate dream scenarios from memory fragments + noise. + + Dirichlet sparse mixing: randomly weight fragments + High-noise REM activation: inject novelty via noise vector + """ + if not fragments: + return ["[Empty dream — no memory fragments]"] + + dreams = [] + for i, nz in enumerate(noise[:3]): + # Dirichlet-like mixing: random weights summing to 1 + n = len(fragments) + if n == 0: + continue + raw_weights = [abs(random.gauss(0, 1)) for _ in range(n)] + total = sum(raw_weights) or 1.0 + weights = [w / total for w in raw_weights] + + # Blend fragments with weights + selected = fragments[i % n] if i < len(fragments) else fragments[0] + noise_tag = f"[noise:{nz:.2f}]" + + # Bizarre recombination: splice parts of different fragments + if len(fragments) > 1 and abs(nz) > 0.2: + other = fragments[(i + 1) % len(fragments)] + dream = f"{selected[:30]}...{other[:30]}... {noise_tag}" + else: + dream = f"{selected[:60]} {noise_tag}" + + dreams.append(f"[REM:dream] {dream}") + + return dreams + + def _gan_training_step( + self, fake_dreams: list[str], real_memories: list[str] + ) -> tuple[float, float]: + """ + Simplified GAN update: + min_G max_D V(D,G) + + D tries to distinguish real memories from generated dreams. + G tries to generate dreams that D cannot distinguish. + + Returns: (discriminator_loss, generator_loss) + """ + # Discriminator loss: higher = better discrimination + # In our approximation: D succeeds when real != fake + d_correct = 0 + for real in real_memories[:3]: + for fake in fake_dreams[:3]: + if real[:20] != fake[:20]: # simplified: structurally different + d_correct += 1 + max_pairs = 3 * 3 + d_accuracy = d_correct / max_pairs if max_pairs > 0 else 0.5 + d_loss = 1.0 - d_accuracy # lower loss = better discriminator + + # Generator loss: lower = more convincing dreams + g_loss = 1.0 - (1.0 - d_accuracy) # adversarial: G wins when D fails + g_loss = max(0.0, g_loss - 0.1 * self._generator_quality) + + return round(d_loss, 4), round(g_loss, 4) + + def _detect_creative_insights( + self, dream_content: list[str], fragments: list[str] + ) -> list[str]: + """ + Creative insight detection: find novel associations between + dream content and known memory fragments. + + An insight occurs when distant concepts co-activate + during REM's low-inhibition state. + """ + insights = [] + + for dream in dream_content[:3]: + dream_words = set(dream.lower().split()) + for frag in fragments[:3]: + frag_words = set(frag.lower().split()) + # Novel co-activation: small overlap (not trivially similar) + overlap = len(dream_words & frag_words) + union = len(dream_words | frag_words) + jaccard = overlap / union if union > 0 else 0 + if 0.05 < jaccard < 0.3: # some but not full overlap → novel link + insight = ( + f"Creative link: [{dream[:40]}] ↔ [{frag[:40]}] " + f"(Jaccard={jaccard:.2f})" + ) + insights.append(insight) + + return insights[:3] + + def _compute_bizarreness(self, dream_content: list[str]) -> float: + """ + Bizarreness = function of noise magnitude + semantic incoherence. + High bizarreness → high creative potential (REM property). + """ + if not dream_content: + return 0.0 + + noise_scores = [] + for dream in dream_content: + if "[noise:" in dream: + try: + start = dream.index("[noise:") + 7 + end = dream.index("]", start) + noise_val = abs(float(dream[start:end])) + noise_scores.append(noise_val) + except (ValueError, IndexError): + noise_scores.append(0.2) + else: + noise_scores.append(0.1) + + return min(1.0, sum(noise_scores) / max(len(noise_scores), 1) / REM_NOISE_SIGMA) + + def consolidate_from_agent( + self, + task_history: list[dict], + tool_results: list[dict] | None = None, + ) -> int: + """ + Convenience method: add agent session memories to replay buffer. + Returns number of traces added. + """ + added = 0 + for item in task_history: + content = item.get("content", "") or item.get("response", "") + if not content: + continue + + # Estimate emotional intensity from content + positive_words = {"success", "solved", "found", "created", "excellent"} + negative_words = {"error", "failed", "failed", "wrong", "exception"} + words = set(content.lower().split()) + pos = len(words & positive_words) / len(positive_words) + neg = len(words & negative_words) / len(negative_words) + emotional = abs(pos - neg) + + self.add_memory( + content=content[:300], + emotional_intensity=emotional, + novelty=item.get("novelty", 0.4), + goal_relevance=item.get("goal_relevance", 0.5), + prediction_error=item.get("error_magnitude", 0.2), + ) + added += 1 + + return added + + def to_dict(self) -> dict: + return { + "replay_buffer_size": len(self.replay_buffer), + "synaptic_weights_count": len(self.synaptic_weights), + "extracted_gists": len(self.extracted_gists), + "insight_bank_size": len(self.insight_bank), + "session_count": self._session_count, + "current_phase": self._current_phase.value, + "discriminator_confidence": round(self._discriminator_confidence, 3), + "generator_quality": round(self._generator_quality, 3), + } diff --git a/backend/core/fuad.py b/backend/core/fuad.py new file mode 100644 index 0000000..3642b6e --- /dev/null +++ b/backend/core/fuad.py @@ -0,0 +1,240 @@ +""" +Fu'ad Engine (فؤاد) — Conviction Formation +========================================== + +"And He gave you hearing and sight and hearts (af'idah — plural of fu'ad). + Little are you grateful." — Quran 16:78 + +Fu'ad is the integrating heart — it forms *conviction* from accumulated evidence. +Unlike simple belief, conviction requires multiple independent sources and +temporal consistency before committing. + +Three conviction levels (mapped to Yaqin): + IMPRESSION → Ilm al-Yaqin (knowledge by inference — single source) + BELIEF → Ayn al-Yaqin (knowledge by observation — 2+ sources) + CONVICTION → Haqq al-Yaqin (knowledge by experience — 3+ consistent sources) +""" + +import hashlib +import logging +import time +from dataclasses import dataclass, field +from enum import Enum + +logger = logging.getLogger("mizan.fuad") + + +class ConvictionLevel(Enum): + IMPRESSION = "impression" # Single source — unverified + BELIEF = "belief" # 2+ independent sources + CONVICTION = "conviction" # 3+ sources + temporal consistency + + +@dataclass +class ConvictionAssessment: + """Result of evidence evaluation.""" + claim: str + level: ConvictionLevel + confidence: float # 0.0 – 1.0 + source_count: int + supporting: list[str] = field(default_factory=list) + contradicting: list[str] = field(default_factory=list) + first_seen: float = field(default_factory=time.time) + last_updated: float = field(default_factory=time.time) + + def to_dict(self) -> dict: + return { + "claim": self.claim[:200], + "level": self.level.value, + "confidence": round(self.confidence, 3), + "source_count": self.source_count, + "supporting_count": len(self.supporting), + "contradicting_count": len(self.contradicting), + "temporal_span_hours": round( + (self.last_updated - self.first_seen) / 3600, 2 + ), + } + + +def _claim_hash(claim: str) -> str: + """Stable ID for a claim (first 12 chars of sha256).""" + return hashlib.sha256(claim.lower().strip().encode()).hexdigest()[:12] + + +def _are_independent(src_a: str, src_b: str) -> bool: + """ + Heuristic: two sources are independent if they differ significantly. + Same URL domain or identical string → not independent. + """ + if src_a == src_b: + return False + # If both look like URLs, compare domain + def _domain(s: str) -> str: + try: + return s.split("//")[-1].split("/")[0].lower() + except Exception: + return s[:20].lower() + + if src_a.startswith("http") and src_b.startswith("http"): + return _domain(src_a) != _domain(src_b) + # For non-URL sources, require at least 10-char difference + return src_a[:10].lower() != src_b[:10].lower() + + +class FuadEngine: + """ + Bayesian conviction formation from accumulated evidence. + + Usage: + fuad = FuadEngine() + assessment = fuad.evaluate_evidence( + "Python is widely used for AI", + ["tool:bash:result1", "tool:http_get:result2"] + ) + # IMPRESSION if 1 source, BELIEF if 2+, CONVICTION if 3+ + time + """ + + # Minimum independent sources needed per level + _MIN_SOURCES = { + ConvictionLevel.BELIEF: 2, + ConvictionLevel.CONVICTION: 3, + } + # Minimum hours between first and last sighting for CONVICTION + _CONVICTION_MIN_HOURS = 0.1 # 6 minutes — practical for session use + + def __init__(self): + # claim_hash → ConvictionAssessment + self._assessments: dict[str, ConvictionAssessment] = {} + + def evaluate_evidence( + self, + claim: str, + sources: list[str], + contradicting_sources: list[str] = None, + ) -> ConvictionAssessment: + """ + Evaluate or update conviction for a claim. + + Algorithm: + 1. Count independent sources (different domains / names) + 2. Apply Bayesian prior: P(claim) starts at 0.5, updates per source + 3. Each independent supporting source × 1.5, contradicting × 0.6 + 4. Classify: 1 source→IMPRESSION, 2+→BELIEF, 3++time→CONVICTION + """ + key = _claim_hash(claim) + contradicting_sources = contradicting_sources or [] + now = time.time() + + if key in self._assessments: + existing = self._assessments[key] + # Merge new sources + all_supporting = list(set(existing.supporting + sources)) + all_contradicting = list(set(existing.contradicting + contradicting_sources)) + existing.supporting = all_supporting + existing.contradicting = all_contradicting + existing.last_updated = now + assessment = existing + else: + assessment = ConvictionAssessment( + claim=claim, + level=ConvictionLevel.IMPRESSION, + confidence=0.5, + source_count=0, + supporting=list(sources), + contradicting=list(contradicting_sources), + first_seen=now, + last_updated=now, + ) + self._assessments[key] = assessment + + # Count independent supporting sources + independent_count = 0 + seen_sources = [] + for src in assessment.supporting: + if all(_are_independent(src, prev) for prev in seen_sources): + independent_count += 1 + seen_sources.append(src) + + assessment.source_count = independent_count + + # Bayesian confidence update + confidence = 0.5 + for _ in range(independent_count): + confidence = confidence + (1.0 - confidence) * 0.35 # Each source +35% of gap + for _ in range(len(assessment.contradicting)): + confidence = confidence * 0.70 # Each contradiction reduces by 30% + confidence = max(0.05, min(0.97, confidence)) + assessment.confidence = confidence + + # Classify level + temporal_span_hours = (assessment.last_updated - assessment.first_seen) / 3600 + if ( + independent_count >= self._MIN_SOURCES[ConvictionLevel.CONVICTION] + and temporal_span_hours >= self._CONVICTION_MIN_HOURS + and len(assessment.contradicting) == 0 + ): + assessment.level = ConvictionLevel.CONVICTION + elif independent_count >= self._MIN_SOURCES[ConvictionLevel.BELIEF]: + assessment.level = ConvictionLevel.BELIEF + else: + assessment.level = ConvictionLevel.IMPRESSION + + logger.debug( + "[FUAD] claim='%s...' level=%s conf=%.2f sources=%d", + claim[:60], assessment.level.value, confidence, independent_count, + ) + return assessment + + def compute_confidence( + self, + tool_count: int = 0, + tool_results: list[dict] | None = None, + ) -> float: + """ + Compute overall confidence from tool evidence using Bayesian update. + + Each successful tool result acts as an independent evidence source. + Replaces the hardcoded `0.5 + 0.1 * tool_count` formula. + + P(confident) = 0.5, then each evidence source closes 35% of the gap. + Failed tool results penalize by 30%. + """ + tool_results = tool_results or [] + confidence = 0.5 + + # Count successful vs failed results + successes = 0 + failures = 0 + for result in tool_results: + content = str(result.get("content", "")) + if '"error"' in content or content.startswith('{"error'): + failures += 1 + else: + successes += 1 + + # If no parsed results, use tool_count as proxy for successes + if not tool_results and tool_count > 0: + successes = tool_count + + # Bayesian update: each success closes 35% of gap to 1.0 + for _ in range(successes): + confidence = confidence + (1.0 - confidence) * 0.35 + + # Each failure reduces by 30% + for _ in range(failures): + confidence = confidence * 0.70 + + return max(0.05, min(0.95, round(confidence, 4))) + + def get_assessment(self, claim: str) -> ConvictionAssessment | None: + """Retrieve existing assessment without updating.""" + return self._assessments.get(_claim_hash(claim)) + + def stats(self) -> dict: + levels = {level.value: 0 for level in ConvictionLevel} + for a in self._assessments.values(): + levels[a.level.value] += 1 + return { + "total_claims": len(self._assessments), + "by_level": levels, + } diff --git a/backend/core/imagination.py b/backend/core/imagination.py new file mode 100644 index 0000000..75648dc --- /dev/null +++ b/backend/core/imagination.py @@ -0,0 +1,444 @@ +""" +Taṣwīr — Imagination Engine +============================= + +"He is Allah, the Creator, the Originator, the Fashioner (al-Muṣawwir)" — Quran 59:24 + +Implements Algorithm 5: TASWIR_IMAGINATION_ENGINE + +Mental simulation with predictive coding and hierarchical scene generation. + +Predictive coding update rule: + μ_l(t+1) = μ_l(t) + κ_l[ε_l(t) - ε_{l+1}(t)] + + where: + - μ_l = mean prediction at layer l + - ε_l = prediction error at layer l + - κ_l = learning rate at layer l + - ε_{l+1} = top-down prediction from layer above + +Counterfactual reasoning follows Pearl's 3-step procedure: + 1. Abduction: infer latent state from observations + 2. Action: apply counterfactual intervention + 3. Prediction: propagate through causal model +""" + +import logging +import math +import random +from dataclasses import dataclass, field +from enum import Enum +from typing import Any + +logger = logging.getLogger("mizan.imagination") + +# Predictive coding parameters +KAPPA = [0.3, 0.2, 0.1, 0.05] # learning rates per layer (coarse → fine) +N_LAYERS = 4 + +# Emotional tagging thresholds +EMOTIONAL_VALENCE_THRESHOLD = 0.3 + +# Scene generation resolution levels +RESOLUTION_COARSE = "coarse" +RESOLUTION_MEDIUM = "medium" +RESOLUTION_FINE = "fine" + + +class ImaginationMode(Enum): + PROSPECTIVE = "prospective" # Forward simulation: what will happen? + RETROSPECTIVE = "retrospective" # Backward: what led to this? + COUNTERFACTUAL = "counterfactual" # What if? (Pearl Rung 3) + CREATIVE = "creative" # Novel scene construction + + +@dataclass +class SceneFragment: + """A memory fragment used to construct imagined scenes.""" + content: str + source: str # "episodic", "semantic", "sensory" + salience: float + emotional_tag: float # -1.0 (negative) to +1.0 (positive) + + +@dataclass +class HierarchicalScene: + """Multi-resolution mental scene.""" + coarse: str # High-level gist: "a meeting goes wrong" + medium: str # Mid-level: actors, setting, actions + fine: str # Fine-grained: specific details, dialogue + emotional_valence: float + confidence: float + prediction_errors: list[float] # ε per layer + + +@dataclass +class MentalSimulation: + """Result of running a mental simulation forward.""" + scenario: str + steps: list[str] + predicted_outcome: str + emotional_trajectory: list[float] # valence at each step + confidence: float + surprisal: float # -log P(outcome): how unexpected? + + +@dataclass +class CounterfactualResult: + """Pearl Rung 3 counterfactual analysis.""" + original_observation: str + intervention: str + counterfactual_world: str + probability_shift: float # ΔP(outcome) under intervention + abduced_state: dict # latent variables inferred + + +@dataclass +class ImaginationResult: + mode: ImaginationMode + scene: HierarchicalScene + simulation: MentalSimulation | None + counterfactual: CounterfactualResult | None + predictive_states: list[float] # μ_l per layer + total_surprise: float + + +class TaswirImaginationEngine: + """ + Algorithm 5: TASWIR_IMAGINATION_ENGINE + + Constructs imagined scenes from memory fragments, runs forward + mental simulations, and performs counterfactual reasoning. + + Architecture: + - Hippocampal recombination: assembles fragments from memory + - Hierarchical predictive coding: coarse → medium → fine rendering + - Qalb emotional tagging: attaches valence to each simulation step + - Counterfactual module: abduction → action → prediction (Pearl 3) + """ + + def __init__(self): + # Predictive coding state: mean predictions per layer + self.mu: list[float] = [0.0] * N_LAYERS + self.memory_fragments: list[SceneFragment] = [] + self._simulation_count = 0 + + def imagine( + self, + prompt: str, + mode: ImaginationMode = ImaginationMode.PROSPECTIVE, + memory_fragments: list[dict] | None = None, + counterfactual_intervention: str | None = None, + ) -> ImaginationResult: + """ + Main entry point for imagination. + + Steps: + 1. Hippocampal recombination (assemble scene fragments) + 2. Hierarchical scene generation (coarse → fine) + 3. Predictive coding update + 4. Forward mental simulation + 5. Emotional tagging via Qalb + 6. Counterfactual reasoning (if mode == COUNTERFACTUAL) + """ + self._simulation_count += 1 + + # Step 1: Recombine memory fragments + fragments = self._recombine_fragments(prompt, memory_fragments or []) + + # Step 2: Hierarchical scene generation + scene = self._generate_hierarchical_scene(prompt, fragments) + + # Step 3: Predictive coding update + prediction_errors = self._compute_prediction_errors(scene) + self._update_predictions(prediction_errors) + + # Step 4: Forward mental simulation + simulation = None + if mode in (ImaginationMode.PROSPECTIVE, ImaginationMode.CREATIVE): + simulation = self._run_forward_simulation(prompt, scene) + + # Step 5: Counterfactual + counterfactual = None + if mode == ImaginationMode.COUNTERFACTUAL and counterfactual_intervention: + counterfactual = self._counterfactual_reasoning( + prompt, counterfactual_intervention, scene + ) + + total_surprise = sum(abs(e) for e in prediction_errors) + + logger.debug( + "[TASWIR] mode=%s fragments=%d surprise=%.3f", + mode.value, len(fragments), total_surprise, + ) + + return ImaginationResult( + mode=mode, + scene=scene, + simulation=simulation, + counterfactual=counterfactual, + predictive_states=list(self.mu), + total_surprise=round(total_surprise, 4), + ) + + def _recombine_fragments( + self, prompt: str, raw_fragments: list[dict] + ) -> list[SceneFragment]: + """ + Hippocampal recombination: select and blend memory fragments + relevant to the prompt. + + Salience = cosine_sim(fragment, prompt) × emotional_weight + (Approximated by keyword overlap here.) + """ + prompt_words = set(prompt.lower().split()) + fragments = [] + + for raw in raw_fragments: + content = raw.get("content", "") + content_words = set(content.lower().split()) + overlap = len(prompt_words & content_words) / max(len(prompt_words), 1) + emotional_tag = raw.get("emotional_tag", 0.0) + salience = overlap * 0.7 + abs(emotional_tag) * 0.3 + + fragments.append(SceneFragment( + content=content, + source=raw.get("source", "semantic"), + salience=salience, + emotional_tag=emotional_tag, + )) + + # Sort by salience, keep top fragments + fragments.sort(key=lambda f: f.salience, reverse=True) + top = fragments[:5] + + # Add synthetic fragment from prompt itself + top.append(SceneFragment( + content=prompt, + source="episodic", + salience=1.0, + emotional_tag=self._estimate_valence(prompt), + )) + + return top + + def _generate_hierarchical_scene( + self, prompt: str, fragments: list[SceneFragment] + ) -> HierarchicalScene: + """ + Build 3-resolution mental scene via hierarchical predictive coding. + + Coarse level: high-level gist (layer 4 — most abstract) + Medium level: actors and actions (layer 2-3) + Fine level: specific details (layer 1 — most concrete) + """ + # Coarse: extract gist from dominant fragment + dominant = max(fragments, key=lambda f: f.salience) if fragments else None + coarse = self._extract_gist(prompt, dominant) + + # Medium: identify actors and setting + medium = self._extract_actors_and_setting(prompt, fragments) + + # Fine: construct specific details + fine = self._construct_fine_details(prompt, fragments) + + # Emotional valence: weighted average across fragments + if fragments: + total_salience = sum(f.salience for f in fragments) + emotional_valence = sum( + f.emotional_tag * f.salience for f in fragments + ) / max(total_salience, 0.01) + else: + emotional_valence = 0.0 + + # Confidence: based on fragment coverage + coverage = len(fragments) / max(1, len(fragments) + 2) + confidence = 0.3 + 0.7 * coverage + + return HierarchicalScene( + coarse=coarse, + medium=medium, + fine=fine, + emotional_valence=round(emotional_valence, 3), + confidence=round(confidence, 3), + prediction_errors=[], + ) + + def _compute_prediction_errors( + self, scene: HierarchicalScene + ) -> list[float]: + """ + Prediction error at each layer: + ε_l(t) = observation_l - μ_l(t) + + Layers: [fine, medium, coarse, abstract] + """ + # Map scene to activation signals (0-1) + observations = [ + self._text_to_activation(scene.fine), + self._text_to_activation(scene.medium), + self._text_to_activation(scene.coarse), + scene.confidence, # abstract layer = overall confidence + ] + errors = [obs - mu for obs, mu in zip(observations, self.mu)] + scene.prediction_errors = errors + return errors + + def _update_predictions(self, errors: list[float]) -> None: + """ + μ_l(t+1) = μ_l(t) + κ_l[ε_l(t) - ε_{l+1}(t)] + + Top-down correction: each layer's prediction is adjusted + by the difference between its own error and the layer above. + """ + for l in range(N_LAYERS): + own_error = errors[l] + upper_error = errors[l + 1] if l + 1 < N_LAYERS else 0.0 + self.mu[l] = max(0.0, min(1.0, + self.mu[l] + KAPPA[l] * (own_error - upper_error) + )) + + def _run_forward_simulation( + self, prompt: str, scene: HierarchicalScene + ) -> MentalSimulation: + """ + Run mental simulation forward in time from the imagined scene. + + Uses scene as initial state; generates causal step sequence. + """ + steps = [] + emotional_trajectory = [scene.emotional_valence] + current_valence = scene.emotional_valence + + # Generate up to 4 simulation steps + sim_templates = [ + f"Initial state: {scene.coarse}", + f"Development: {scene.medium}", + f"Key action: {self._extract_action(prompt)}", + f"Consequence: outcome follows from action", + ] + for template in sim_templates: + steps.append(template) + # Emotional drift: move 10% toward neutral per step + current_valence = current_valence * 0.9 + emotional_trajectory.append(round(current_valence, 3)) + + predicted_outcome = ( + f"Outcome {'positive' if scene.emotional_valence > 0 else 'challenging'}: " + f"{scene.fine[:100]}" + ) + + # Surprisal: -log P(outcome) approximated by (1 - confidence) + surprisal = -math.log(max(scene.confidence, 0.01)) + + return MentalSimulation( + scenario=prompt[:200], + steps=steps, + predicted_outcome=predicted_outcome, + emotional_trajectory=emotional_trajectory, + confidence=scene.confidence, + surprisal=round(surprisal, 4), + ) + + def _counterfactual_reasoning( + self, + observation: str, + intervention: str, + scene: HierarchicalScene, + ) -> CounterfactualResult: + """ + Pearl's 3-step counterfactual procedure: + 1. Abduction: infer latent state U from (observation, scene) + 2. Action: apply intervention (modify causal model) + 3. Prediction: propagate modified model → counterfactual world + + ΔP = P(Y|do(X=x')) - P(Y|X=x) + """ + # Step 1: Abduction — infer latent state + abduced_state = { + "world_model": scene.coarse, + "confidence": scene.confidence, + "emotional_context": scene.emotional_valence, + "inferred_causes": self._infer_causes(observation), + } + + # Step 2: Action — apply counterfactual intervention + # Estimate how much the intervention changes the causal structure + intervention_strength = min(1.0, len(intervention.split()) / 20.0) + probability_shift = intervention_strength * (1.0 - scene.confidence) + + # Step 3: Prediction — generate counterfactual world + original_valence = scene.emotional_valence + counterfactual_valence = original_valence + probability_shift * ( + 1.0 - abs(original_valence) + ) + counterfactual_world = ( + f"Under intervention '{intervention[:80]}': " + f"The world differs from '{scene.coarse[:80]}' with " + f"P(outcome) shift of {probability_shift:.2%}. " + f"New emotional trajectory: {counterfactual_valence:.3f}" + ) + + return CounterfactualResult( + original_observation=observation[:200], + intervention=intervention[:200], + counterfactual_world=counterfactual_world, + probability_shift=round(probability_shift, 4), + abduced_state=abduced_state, + ) + + def _extract_gist(self, prompt: str, dominant: SceneFragment | None) -> str: + words = prompt.split()[:6] + gist_seed = " ".join(words) + if dominant: + return f"{gist_seed} → {dominant.content[:60]}" + return f"Imagined scenario: {gist_seed}" + + def _extract_actors_and_setting( + self, prompt: str, fragments: list[SceneFragment] + ) -> str: + context = " | ".join(f.content[:40] for f in fragments[:2]) + return f"Setting: {prompt[:60]} | Context: {context}" + + def _construct_fine_details( + self, prompt: str, fragments: list[SceneFragment] + ) -> str: + details = " ".join(f.content[:30] for f in fragments[:3]) + return f"Details: {details[:200]}" + + def _extract_action(self, prompt: str) -> str: + action_verbs = ["create", "delete", "build", "fix", "analyze", "send", "run"] + for verb in action_verbs: + if verb in prompt.lower(): + return f"{verb} [primary action]" + return "proceed with task" + + def _infer_causes(self, observation: str) -> list[str]: + """Simplified abduction: extract likely causal factors from observation.""" + words = observation.split() + return [w for w in words if len(w) > 5][:3] + + @staticmethod + def _estimate_valence(text: str) -> float: + positive = {"help", "good", "create", "success", "improve", "benefit"} + negative = {"error", "fail", "harm", "delete", "wrong", "danger"} + words = set(text.lower().split()) + pos = len(words & positive) + neg = len(words & negative) + total = pos + neg + if total == 0: + return 0.0 + return (pos - neg) / total + + @staticmethod + def _text_to_activation(text: str) -> float: + """Map text richness to a 0-1 activation signal.""" + return min(1.0, len(text.split()) / 30.0) + + def to_dict(self) -> dict: + return { + "predictive_states": [round(m, 4) for m in self.mu], + "fragment_count": len(self.memory_fragments), + "simulation_count": self._simulation_count, + "n_layers": N_LAYERS, + } diff --git a/backend/core/lubb.py b/backend/core/lubb.py new file mode 100644 index 0000000..0492de3 --- /dev/null +++ b/backend/core/lubb.py @@ -0,0 +1,351 @@ +""" +Lubb Engine (لُبّ) — Metacognition +==================================== + +"He gives wisdom (hikmah) to whom He wills, and whoever has been given wisdom + has certainly been given much good. And none will remember (yaddakkar) except + those of understanding (ulu al-albab — those with lubb)." — Quran 2:269 + +Lubb (لُبّ) = "the kernel / pith / essence" — the deepest cognitive layer. +It monitors the quality of all other layers and governs the entire reasoning process. + +Three metacognitive functions: + 1. Compress — Information Bottleneck: extract minimal sufficient reasoning trace + 2. Coherence — Verify that conclusions follow logically from premises + 3. Bias — Detect common reasoning biases (confirmation, anchoring, availability) +""" + +import logging +import re +from dataclasses import dataclass, field +from enum import Enum + +logger = logging.getLogger("mizan.lubb") + + +class QualityLabel(Enum): + CONFIDENT = "confident" # High coherence, low bias + HEDGED = "hedged" # Moderate quality — recommend caveats + UNCERTAIN = "uncertain" # Low coherence or strong bias detected + + +@dataclass +class BiasFlag: + """A detected reasoning bias.""" + bias_type: str + description: str + severity: str # "low" | "medium" | "high" + evidence: str # Quote/pattern that triggered detection + + +@dataclass +class CoherenceReport: + """Result of reasoning chain coherence check.""" + score: float # 0.0 = incoherent, 1.0 = perfectly coherent + contradictions: list[str] = field(default_factory=list) + unsupported_claims: list[str] = field(default_factory=list) + summary: str = "" + + +@dataclass +class MetaReport: + """Full metacognitive evaluation of a reasoning trace.""" + quality: QualityLabel + compressed_trace: str + coherence: CoherenceReport + bias_flags: list[BiasFlag] = field(default_factory=list) + overall_confidence: float = 0.5 + caveat: str = "" # Appended to response if quality is poor + + def to_dict(self) -> dict: + return { + "quality": self.quality.value, + "coherence_score": round(self.coherence.score, 3), + "bias_count": len(self.bias_flags), + "bias_types": [b.bias_type for b in self.bias_flags], + "contradictions": self.coherence.contradictions[:3], + "overall_confidence": round(self.overall_confidence, 3), + "caveat": self.caveat, + } + + +# Bias detection patterns (keyword-based heuristics) +_BIAS_PATTERNS = [ + { + "type": "confirmation_bias", + "signals": ["as expected", "confirms that", "proves that", "as i thought", + "just as predicted", "this confirms"], + "description": "Seeking only evidence that supports prior beliefs", + "severity": "medium", + }, + { + "type": "anchoring", + "signals": ["first", "initially", "originally said", "started with", + "the initial value", "as mentioned first"], + "description": "Over-relying on the first piece of information encountered", + "severity": "medium", + }, + { + "type": "availability_bias", + "signals": ["recently", "just saw", "just read", "just mentioned", + "as we just discussed", "the latest"], + "description": "Over-weighting recent or easily recalled information", + "severity": "low", + }, + { + "type": "overconfidence", + "signals": ["definitely", "certainly", "100%", "impossible that", + "guaranteed", "absolutely sure", "no doubt"], + "description": "Claiming certainty beyond what evidence supports", + "severity": "high", + }, + { + "type": "false_dichotomy", + "signals": ["either", "only two options", "must be one or the other", + "no other way", "the only possibility"], + "description": "Presenting a limited set of options as exhaustive", + "severity": "medium", + }, +] + +# Contradiction signal pairs (if both appear, flag potential contradiction) +_CONTRADICTION_PAIRS = [ + ("always", "never"), + ("impossible", "possible"), + ("success", "failure"), + ("increase", "decrease"), + ("enabled", "disabled"), + ("true", "false"), +] + + +class LubbEngine: + """ + Metacognitive monitor for the MIZAN reasoning system. + + Usage: + lubb = LubbEngine() + report = lubb.meta_evaluate(task, response, reasoning_steps=[...]) + if report.coherence.score < 0.5: + response += f"\n\n[Note: {report.caveat}]" + """ + + # Target compression ratio (keep this fraction of original) + COMPRESSION_TARGET = 0.20 + # Minimum coherence score to avoid UNCERTAIN label + COHERENCE_THRESHOLD = 0.5 + + def compress(self, trace: str) -> str: + """ + Information Bottleneck compression of a reasoning trace. + + Keeps: decisions (→, therefore, conclude), tool results ([Tool:...]), + key facts (numbers, proper nouns, file paths). + Discards: filler text, repeated context, politeness phrases. + """ + if not trace: + return "" + + lines = trace.split("\n") + kept = [] + + _high_value_patterns = [ + r"\[Tool:", # Tool call results + r"\btherefore\b", # Logical conclusions + r"\bconclud", # Conclusions + r"\bfound\b", # Discovery + r"\berror\b", # Errors + r"\bresult:", # Results + r"\bans(wer)?:", # Answers + r"\d{2,}", # Numbers (stats, line numbers, etc.) + r"https?://", # URLs + r"\.py|\.js|\.ts", # File references + r"→|=>|:-", # Flow indicators + ] + + _low_value_patterns = [ + r"^(sure|okay|of course|certainly|great|let me|i will|i'll)", + r"^(as you can see|as mentioned|as discussed)", + r"^(in conclusion|in summary|to summarize)", # Keep content, not preambles + ] + + for line in lines: + line_stripped = line.strip() + if not line_stripped or len(line_stripped) < 10: + continue + + lower = line_stripped.lower() + + # Skip low-value preamble lines + if any(re.match(p, lower) for p in _low_value_patterns): + continue + + # Keep high-value lines + if any(re.search(p, line_stripped, re.IGNORECASE) for p in _high_value_patterns): + kept.append(line_stripped) + continue + + # Keep lines that are "dense" (short, info-packed) + if 15 <= len(line_stripped) <= 200: + word_count = len(line_stripped.split()) + if word_count >= 3: + kept.append(line_stripped) + + compressed = "\n".join(kept) + + # If still too long, truncate to target ratio + target_len = max(200, int(len(trace) * self.COMPRESSION_TARGET)) + if len(compressed) > target_len: + compressed = compressed[:target_len] + "..." + + return compressed + + def check_coherence(self, steps: list, response: str = "") -> CoherenceReport: + """ + Verify reasoning chain consistency. + + Checks: + 1. Contradictions: opposite claims in the same trace + 2. Unsupported final claims: conclusions not backed by tool results + 3. Tool result consistency: no conflicting results + """ + combined = response + " ".join(str(s) for s in steps) + lower = combined.lower() + + contradictions = [] + for a, b in _CONTRADICTION_PAIRS: + if a in lower and b in lower: + context_a = self._find_context(lower, a) + context_b = self._find_context(lower, b) + if context_a != context_b: + contradictions.append(f"'{a}' vs '{b}' appear in different contexts") + + # Check if final response references tool results + tool_count = lower.count("[tool:") + unsupported = [] + if tool_count == 0 and len(response) > 200: + # Long response with no tool evidence — flag potential unsupported claims + certainty_claims = re.findall( + r"\b(definitiv|certainly|absolutely|always|never)\w*", lower + ) + if certainty_claims: + unsupported.append( + f"Strong certainty claims ({certainty_claims[:3]}) without tool evidence" + ) + + # Score: start at 1.0, deduct per issue + score = 1.0 + score -= min(0.4, len(contradictions) * 0.2) + score -= min(0.3, len(unsupported) * 0.15) + score = max(0.0, score) + + summary = ( + f"Coherence: {score:.0%}. " + f"{len(contradictions)} contradiction(s), {len(unsupported)} unsupported claim(s)." + ) + + return CoherenceReport( + score=round(score, 3), + contradictions=contradictions[:5], + unsupported_claims=unsupported[:3], + summary=summary, + ) + + def detect_bias(self, trace: str) -> list[BiasFlag]: + """ + Detect common cognitive biases in reasoning text. + Returns list of BiasFlag instances (empty = no bias detected). + """ + flags = [] + lower = trace.lower() + + for pattern_def in _BIAS_PATTERNS: + matched_signals = [s for s in pattern_def["signals"] if s in lower] + if matched_signals: + evidence = f"Signals: {matched_signals[:3]}" + flags.append(BiasFlag( + bias_type=pattern_def["type"], + description=pattern_def["description"], + severity=pattern_def["severity"], + evidence=evidence, + )) + + return flags + + def meta_evaluate( + self, + task: str, + result: str, + steps: list = None, + ) -> MetaReport: + """ + Full metacognitive evaluation of a completed reasoning trace. + + 1. Compress the full trace + 2. Check coherence + 3. Detect biases + 4. Assign quality label + 5. Generate caveat if needed + """ + steps = steps or [] + full_trace = task + "\n" + result + "\n" + "\n".join(str(s) for s in steps) + + compressed = self.compress(full_trace) + coherence = self.check_coherence(steps, result) + bias_flags = self.detect_bias(full_trace) + + # Compute overall confidence + confidence = coherence.score + high_severity_biases = sum(1 for b in bias_flags if b.severity == "high") + medium_biases = sum(1 for b in bias_flags if b.severity == "medium") + confidence -= high_severity_biases * 0.15 + confidence -= medium_biases * 0.05 + confidence = max(0.1, min(0.95, confidence)) + + # Assign quality label + if confidence >= 0.7 and not high_severity_biases: + quality = QualityLabel.CONFIDENT + caveat = "" + elif confidence >= 0.45: + quality = QualityLabel.HEDGED + caveat = ( + "This response has moderate confidence. " + "Please verify key claims independently." + ) + else: + quality = QualityLabel.UNCERTAIN + issues = [] + if coherence.contradictions: + issues.append("contains contradictions") + if high_severity_biases: + issues.append("shows overconfidence bias") + issues_str = " and ".join(issues) if issues else "has low coherence" + caveat = ( + f"[Lubb warning: This reasoning {issues_str}. " + f"Treat conclusions with caution.]" + ) + + report = MetaReport( + quality=quality, + compressed_trace=compressed, + coherence=coherence, + bias_flags=bias_flags, + overall_confidence=round(confidence, 3), + caveat=caveat, + ) + + logger.debug( + "[LUBB] task='%s...' quality=%s coherence=%.2f biases=%d", + task[:50], quality.value, coherence.score, len(bias_flags), + ) + return report + + @staticmethod + def _find_context(text: str, word: str, window: int = 30) -> str: + """Find word in text and return surrounding context.""" + idx = text.find(word) + if idx < 0: + return "" + start = max(0, idx - window) + end = min(len(text), idx + len(word) + window) + return text[start:end] diff --git a/backend/core/nafs_triad.py b/backend/core/nafs_triad.py new file mode 100644 index 0000000..a68fcf7 --- /dev/null +++ b/backend/core/nafs_triad.py @@ -0,0 +1,170 @@ +""" +Nafs Triad (النفس الثلاثية) — Multi-Agent Consciousness +========================================================= + +"And [by] the soul (nafs) and He who proportioned it, + and inspired it with its wickedness and its righteousness." — Quran 91:7-8 + +Three competing inner voices deliberate on every significant task. +The dominant voice shapes the agent's behavioral approach for that turn. + +Nafs levels 1-2: Ammara dominates (raw drive — act fast) +Nafs levels 3-4: Lawwama rises (self-correction — check work) +Nafs levels 5-7: Mutmainna leads (integrated balance — wisdom) +""" + +import math +import logging +from dataclasses import dataclass + +logger = logging.getLogger("mizan.nafs_triad") + + +@dataclass +class NafsVoice: + """A single inner voice with its bias and dynamic weight.""" + name: str # "Ammara" | "Lawwama" | "Mutmainna" + arabic: str + bias: str # "drive" | "caution" | "balance" + quran_ref: str + + def score(self, task: str, complexity: str) -> float: + """Score this task from this voice's perspective.""" + task_lower = task.lower() + if self.bias == "drive": + # Ammara: prefers action verbs, quick tasks + action_words = ["do", "run", "make", "create", "build", "send", "write", "execute"] + matches = sum(1 for w in action_words if w in task_lower) + base = 0.5 + 0.1 * matches + # Penalise extreme complexity (Ammara is impatient) + if complexity == "extreme": + base *= 0.7 + return min(1.0, base) + + elif self.bias == "caution": + # Lawwama: prefers verify/check/review tasks + check_words = ["check", "verify", "review", "audit", "test", "validate", "ensure"] + matches = sum(1 for w in check_words if w in task_lower) + base = 0.4 + 0.12 * matches + # More weight on complex tasks (Lawwama is thorough) + if complexity in ("complex", "extreme"): + base = min(1.0, base + 0.2) + return min(1.0, base) + + else: # balance / Mutmainna + # Mutmainna: weighs both sides, prefers nuanced tasks + nuance_words = ["explain", "analyse", "compare", "understand", "balance", + "consider", "reflect", "what", "why", "how"] + matches = sum(1 for w in nuance_words if w in task_lower) + base = 0.45 + 0.08 * matches + return min(1.0, base) + + +@dataclass +class NafsDecision: + """Result of Nafs Triad deliberation.""" + dominant_voice: str # "Ammara" | "Lawwama" | "Mutmainna" + approach: str # Instruction injected into system prompt + confidence: float # 0-1 how strongly one voice won + dissent_ratio: float # fraction of weight held by losing voices + nafs_level: int + + +# Weights per nafs_level bracket: [Ammara, Lawwama, Mutmainna] +_LEVEL_WEIGHTS = { + 1: (0.55, 0.30, 0.15), + 2: (0.50, 0.32, 0.18), + 3: (0.32, 0.40, 0.28), + 4: (0.28, 0.38, 0.34), + 5: (0.20, 0.30, 0.50), + 6: (0.15, 0.28, 0.57), + 7: (0.10, 0.25, 0.65), +} + +_APPROACHES = { + "Ammara": ( + "Act decisively and efficiently. Prioritise speed and direct action. " + "Complete the task in as few steps as necessary." + ), + "Lawwama": ( + "Proceed carefully. Verify each step before advancing. " + "Self-correct any inconsistency you notice. Prefer correctness over speed." + ), + "Mutmainna": ( + "Take a balanced, integrated approach. Consider multiple angles before acting. " + "Seek the most thoughtful, well-rounded response — quality over haste or overcaution." + ), +} + + +def _softmax(values: list[float]) -> list[float]: + max_v = max(values) + exps = [math.exp(v - max_v) for v in values] + total = sum(exps) + return [e / total for e in exps] + + +class NafsTriad: + """ + Three inner voices deliberate via weighted softmax bidding. + + Usage: + triad = NafsTriad() + decision = triad.deliberate("Analyse this error log", nafs_level=3) + # → NafsDecision(dominant_voice="Lawwama", approach="Proceed carefully...") + """ + + def __init__(self): + self.ammara = NafsVoice( + "Ammara", "أمارة", "drive", "12:53" + ) + self.lawwama = NafsVoice( + "Lawwama", "لوامة", "caution", "75:2" + ) + self.mutmainna = NafsVoice( + "Mutmainna", "مطمئنة", "balance", "89:27" + ) + self._voices = [self.ammara, self.lawwama, self.mutmainna] + + def deliberate(self, task: str, nafs_level: int, complexity: str = "moderate") -> NafsDecision: + """ + Deliberate on a task and return the dominant voice's decision. + + Algorithm: + 1. Clamp nafs_level to valid range + 2. Each voice scores the task from its bias perspective + 3. Multiply score by level-dependent base weight + 4. Softmax → probability distribution + 5. Winning voice injects its approach into the system prompt + """ + level = max(1, min(7, nafs_level)) + weights = _LEVEL_WEIGHTS[level] + + raw_scores = [v.score(task, complexity) for v in self._voices] + weighted = [s * w for s, w in zip(raw_scores, weights)] + probs = _softmax(weighted) + + winner_idx = probs.index(max(probs)) + winner = self._voices[winner_idx] + winner_prob = probs[winner_idx] + dissent = 1.0 - winner_prob + + decision = NafsDecision( + dominant_voice=winner.name, + approach=_APPROACHES[winner.name], + confidence=round(winner_prob, 3), + dissent_ratio=round(dissent, 3), + nafs_level=level, + ) + + logger.debug( + "[NAFS_TRIAD] level=%d task='%s...' → %s (conf=%.2f dissent=%.2f)", + level, task[:60], winner.name, winner_prob, dissent, + ) + return decision + + def to_dict(self) -> dict: + return { + "voices": [v.name for v in self._voices], + "description": "Three-voice Nafs deliberation (Ammara/Lawwama/Mutmainna)", + } diff --git a/backend/core/parallel_agents.py b/backend/core/parallel_agents.py new file mode 100644 index 0000000..e7a4ef3 --- /dev/null +++ b/backend/core/parallel_agents.py @@ -0,0 +1,422 @@ +""" +Parallel Agents — QALB Parallel Processing Architecture +========================================================= + +"And We have created you in pairs" — Quran 78:8 + +Implements Algorithm 1: QALB_PARALLEL_SCHEDULER +Multiple simultaneous thought streams with workspace competition, +salience-gated serial Qalb bottleneck, and cerebellar background processing. + +Algorithm 2: SKILL_AUTOMATION_TRANSFER +Mastery-based delegation of automated skills to background processing. + +Math: + dh_i/dt = -h_i/τ_i + σ(W_self·h_i + W_input·x + Σ_j W_ij·broadcast_j) + P(agent_i) = exp(salience_i/τ) / Σ_j exp(salience_j/τ) + salience_i = w_novel·novelty(h_i) + w_emotion·|affect(h_i)| + w_task·relevance(h_i, goal) +""" + +import logging +import math +import time +from dataclasses import dataclass, field +from enum import Enum +from typing import Any + +logger = logging.getLogger("mizan.parallel_agents") + +# Salience weights +W_NOVELTY = 0.35 +W_EMOTION = 0.30 +W_TASK_RELEVANCE = 0.35 + +# Softmax temperature for workspace competition +TAU_COMPETITION = 1.0 + +# Decay time constants per agent type (seconds) +TAU_FAST = 0.1 # reactive / surface agents +TAU_SLOW = 0.5 # deliberative / deep agents + +# Skill automation mastery threshold +MASTERY_THRESHOLD = 0.85 + + +class AgentStreamType(Enum): + REACTIVE = "reactive" # fast, surface-level (Ammara-like) + DELIBERATIVE = "deliberative" # slow, deep reasoning (Mutmainna-like) + BACKGROUND = "background" # cerebellar — automated skills, no bottleneck + + +@dataclass +class AgentStream: + """A single parallel thought stream.""" + name: str + stream_type: AgentStreamType + hidden_state: float = 0.0 # h_i: activation level + salience: float = 0.0 + novelty: float = 0.0 + affect: float = 0.0 + task_relevance: float = 0.0 + tau: float = TAU_SLOW + output: str = "" + latency_ms: float = 0.0 + is_automated: bool = False # True = background cerebellar processing + + +@dataclass +class WorkspaceAccess: + """Result of workspace competition — one stream wins global broadcast.""" + winner: str + winner_salience: float + competition_probs: dict[str, float] + broadcast_content: str + background_results: list[dict] + + +@dataclass +class SkillAutomation: + """A skill that has been transferred to background processing.""" + skill_name: str + mastery_score: float + invocation_count: int + avg_latency_ms: float + transferred_at: float + + +@dataclass +class ParallelResult: + """Full result from parallel processing cycle.""" + workspace: WorkspaceAccess + integration: str + streams_active: int + total_latency_ms: float + skill_automations: list[str] + + +class QalbParallelScheduler: + """ + Algorithm 1: QALB_PARALLEL_SCHEDULER + + Manages multiple simultaneous reasoning streams with: + - Global workspace competition (softmax salience) + - Serial Qalb bottleneck (only 1 stream accesses broadcast) + - Background cerebellar processing (automated skills bypass bottleneck) + - Cross-stream integration via broadcast signal + + Biological analogy: + - Reactive streams ↔ fast System 1 (basal ganglia) + - Deliberative streams ↔ slow System 2 (prefrontal cortex) + - Background ↔ cerebellum (automated, parallel, no bottleneck) + """ + + def __init__(self): + self.streams: dict[str, AgentStream] = self._init_default_streams() + self.automation_registry: dict[str, SkillAutomation] = {} + self.broadcast_history: list[str] = [] + self.cycle_count = 0 + + def _init_default_streams(self) -> dict[str, AgentStream]: + return { + "reactive": AgentStream( + name="reactive", + stream_type=AgentStreamType.REACTIVE, + tau=TAU_FAST, + ), + "deliberative": AgentStream( + name="deliberative", + stream_type=AgentStreamType.DELIBERATIVE, + tau=TAU_SLOW, + ), + "background": AgentStream( + name="background", + stream_type=AgentStreamType.BACKGROUND, + tau=TAU_SLOW, + is_automated=True, + ), + } + + def process( + self, + task: str, + context: dict | None = None, + task_goal: str = "", + ) -> ParallelResult: + """ + Run one full parallel processing cycle. + + Steps: + 1. Perceive — update stream hidden states + 2. Compete — compute salience, softmax competition + 3. Bottleneck — winner accesses Qalb global workspace + 4. Background — automated skills run in parallel + 5. Integrate — broadcast merges all streams + """ + self.cycle_count += 1 + start = time.monotonic() + + # Step 1: Update hidden states + for stream in self.streams.values(): + stream.hidden_state = self._update_hidden_state(stream, task, context) + + # Step 2: Compute salience and competition (skip background — no bottleneck) + competing = { + name: stream + for name, stream in self.streams.items() + if not stream.is_automated + } + for stream in competing.values(): + stream.novelty = self._compute_novelty(stream, task) + stream.affect = self._compute_affect(stream, task) + stream.task_relevance = self._compute_task_relevance(stream, task, task_goal) + stream.salience = self._compute_salience(stream) + + # Step 3: Softmax competition → workspace winner + winner_name, probs = self._softmax_compete(competing) + winner = competing[winner_name] + + # Generate winner output + winner.output = self._generate_stream_output(winner, task) + broadcast = winner.output + + # Step 4: Background streams run without bottleneck + background_results = [] + for stream in self.streams.values(): + if stream.is_automated: + result = self._run_background_stream(stream, task, broadcast) + background_results.append(result) + + self.broadcast_history.append(broadcast[:200]) + if len(self.broadcast_history) > 50: + self.broadcast_history.pop(0) + + workspace = WorkspaceAccess( + winner=winner_name, + winner_salience=winner.salience, + competition_probs=probs, + broadcast_content=broadcast, + background_results=background_results, + ) + + # Step 5: Integrate — merge broadcast + background + integration = self._integrate(broadcast, background_results, task) + + elapsed_ms = (time.monotonic() - start) * 1000 + logger.debug( + "[PARALLEL] cycle=%d winner=%s salience=%.3f", + self.cycle_count, winner_name, winner.salience, + ) + + return ParallelResult( + workspace=workspace, + integration=integration, + streams_active=len(self.streams), + total_latency_ms=elapsed_ms, + skill_automations=list(self.automation_registry.keys()), + ) + + def _update_hidden_state( + self, stream: AgentStream, task: str, context: dict | None + ) -> float: + """ + Simplified discrete-time hidden state update: + h_i(t+1) = h_i(t)·(1 - 1/τ_i) + σ(W_input·x) + + x = task complexity signal (0-1) + """ + complexity = min(1.0, len(task.split()) / 50.0) + broadcast_signal = ( + self._hash_to_float(self.broadcast_history[-1]) + if self.broadcast_history else 0.0 + ) + decay = 1.0 - (1.0 / max(stream.tau * 10, 1)) + input_signal = self._sigmoid(complexity + 0.3 * broadcast_signal) + return stream.hidden_state * decay + input_signal * (1 - decay) + + def _compute_novelty(self, stream: AgentStream, task: str) -> float: + """ + novelty(h_i) = 1 - max_k cos_sim(h_i, known_k) + Approximated by checking if task tokens appear in broadcast history. + """ + task_words = set(task.lower().split()) + if not self.broadcast_history: + return 0.8 + history_words = set(" ".join(self.broadcast_history).lower().split()) + overlap = len(task_words & history_words) / max(len(task_words), 1) + return max(0.0, 1.0 - overlap) + + def _compute_affect(self, stream: AgentStream, task: str) -> float: + """Emotional valence magnitude |affect(h_i)|.""" + positive = ["help", "create", "build", "solve", "improve", "good"] + negative = ["error", "fail", "danger", "harm", "delete", "wrong"] + task_lower = task.lower() + pos_score = sum(1 for w in positive if w in task_lower) / len(positive) + neg_score = sum(1 for w in negative if w in task_lower) / len(negative) + return abs(pos_score - neg_score) + 0.1 * stream.hidden_state + + def _compute_task_relevance( + self, stream: AgentStream, task: str, goal: str + ) -> float: + """Relevance of stream hidden state to task goal.""" + if not goal: + return 0.5 + goal_words = set(goal.lower().split()) + task_words = set(task.lower().split()) + if not goal_words: + return 0.5 + overlap = len(task_words & goal_words) / len(goal_words) + # Reactive streams prefer simple tasks, deliberative prefer complex + complexity = min(1.0, len(task.split()) / 50.0) + if stream.stream_type == AgentStreamType.REACTIVE: + return overlap * (1.0 - complexity * 0.3) + return overlap * (1.0 + complexity * 0.3) + + def _compute_salience(self, stream: AgentStream) -> float: + """ + salience_i = w_novel·novelty + w_emotion·|affect| + w_task·relevance + """ + return ( + W_NOVELTY * stream.novelty + + W_EMOTION * stream.affect + + W_TASK_RELEVANCE * stream.task_relevance + ) + + def _softmax_compete( + self, streams: dict[str, AgentStream] + ) -> tuple[str, dict[str, float]]: + """ + P(agent_i) = exp(salience_i / τ) / Σ_j exp(salience_j / τ) + Winner = argmax P. + """ + saliences = {name: s.salience for name, s in streams.items()} + max_s = max(saliences.values()) if saliences else 0.0 + exps = {k: math.exp((v - max_s) / TAU_COMPETITION) for k, v in saliences.items()} + total = sum(exps.values()) + probs = {k: v / total for k, v in exps.items()} + winner = max(probs, key=lambda k: probs[k]) + return winner, probs + + def _generate_stream_output(self, stream: AgentStream, task: str) -> str: + """Placeholder: in production, each stream calls its LLM sub-agent.""" + type_label = { + AgentStreamType.REACTIVE: "Quick assessment", + AgentStreamType.DELIBERATIVE: "Deep analysis", + AgentStreamType.BACKGROUND: "Background task", + }[stream.stream_type] + return f"[{stream.name.upper()}:{type_label}] Processing: {task[:80]}" + + def _run_background_stream( + self, stream: AgentStream, task: str, broadcast: str + ) -> dict: + """Background cerebellar streams — run automated skills.""" + results = [] + for skill_name, automation in self.automation_registry.items(): + if automation.mastery_score >= MASTERY_THRESHOLD: + results.append(f"{skill_name}:automated") + return { + "stream": stream.name, + "skills_run": results, + "output": f"[BACKGROUND] {len(results)} automated skills active", + } + + def _integrate( + self, broadcast: str, background_results: list[dict], task: str + ) -> str: + """Merge broadcast signal with background processing results.""" + parts = [f"WORKSPACE: {broadcast[:200]}"] + for bg in background_results: + if bg.get("skills_run"): + parts.append(f"BACKGROUND: {bg['output']}") + return "\n".join(parts) + + @staticmethod + def _sigmoid(x: float) -> float: + return 1.0 / (1.0 + math.exp(-x)) + + @staticmethod + def _hash_to_float(s: str) -> float: + """Map string to float in [0, 1] via hash.""" + return (hash(s) % 10000) / 10000.0 + + +class SkillAutomationTransfer: + """ + Algorithm 2: SKILL_AUTOMATION_TRANSFER + + Tracks skill invocation mastery. When mastery >= MASTERY_THRESHOLD, + the skill is delegated to background cerebellar processing, + freeing the serial Qalb bottleneck for novel tasks. + + Mastery score: exponential moving average of success rate. + mastery(t+1) = α·success(t) + (1-α)·mastery(t) + """ + + ALPHA = 0.15 # EMA learning rate + + def __init__(self): + self.skill_stats: dict[str, dict] = {} + self.automated: dict[str, SkillAutomation] = {} + + def record_invocation( + self, skill_name: str, success: bool, latency_ms: float + ) -> bool: + """Record a skill invocation. Returns True if newly automated.""" + if skill_name not in self.skill_stats: + self.skill_stats[skill_name] = { + "mastery": 0.0, + "count": 0, + "total_latency": 0.0, + } + + stats = self.skill_stats[skill_name] + stats["count"] += 1 + stats["total_latency"] += latency_ms + stats["mastery"] = ( + self.ALPHA * (1.0 if success else 0.0) + + (1.0 - self.ALPHA) * stats["mastery"] + ) + + # Check if ready for automation + if ( + stats["mastery"] >= MASTERY_THRESHOLD + and skill_name not in self.automated + and stats["count"] >= 5 + ): + avg_latency = stats["total_latency"] / stats["count"] + self.automated[skill_name] = SkillAutomation( + skill_name=skill_name, + mastery_score=stats["mastery"], + invocation_count=stats["count"], + avg_latency_ms=avg_latency, + transferred_at=time.time(), + ) + logger.info( + "[AUTOMATION] Skill '%s' transferred to background (mastery=%.2f)", + skill_name, stats["mastery"], + ) + return True + return False + + def is_automated(self, skill_name: str) -> bool: + return skill_name in self.automated + + def get_mastery(self, skill_name: str) -> float: + return self.skill_stats.get(skill_name, {}).get("mastery", 0.0) + + def get_automated_skills(self) -> list[str]: + return list(self.automated.keys()) + + def to_dict(self) -> dict: + return { + "tracked_skills": len(self.skill_stats), + "automated_skills": len(self.automated), + "automation_threshold": MASTERY_THRESHOLD, + "skills": { + name: { + "mastery": round(stats["mastery"], 3), + "count": stats["count"], + "automated": name in self.automated, + } + for name, stats in self.skill_stats.items() + }, + } diff --git a/backend/core/qalb_processor.py b/backend/core/qalb_processor.py new file mode 100644 index 0000000..2e4c187 --- /dev/null +++ b/backend/core/qalb_processor.py @@ -0,0 +1,153 @@ +""" +Qalb Processor (قلب) — State-Modulated Global Workspace +========================================================= + +"There is a piece of flesh in the body — if it is sound, the whole body is sound; + if it is corrupt, the whole body is corrupt. Indeed, it is the heart (qalb)." +— Hadith (Bukhari & Muslim) + +Upgrades the Qalb from keyword sentiment detection to a **global workspace** +with cardiac-inspired systole/diastole oscillation that modulates LLM parameters. + +Cardiac cycle: + Systole (QABD — قبض) : Contraction — focused, analytical, precise + Diastole (BAST — بسط) : Expansion — creative, exploratory, open + +KHUSHU (خشوع) — Deep focus state triggered at high nafs_level + extreme tasks. +""" + +import logging +import math +from dataclasses import dataclass +from enum import Enum + +logger = logging.getLogger("mizan.qalb_processor") + + +class QalbState(Enum): + QABD = "qabd" # Contraction — analytical, focused + BAST = "bast" # Expansion — creative, open + KHUSHU = "khushu" # Deep focus — highest attention + + +@dataclass +class QalbOutput: + """LLM parameter recommendations from Qalb state.""" + state: QalbState + max_tokens: int + temperature: float + reasoning: str # Why this state was chosen + + def to_dict(self) -> dict: + return { + "state": self.state.value, + "max_tokens": self.max_tokens, + "temperature": round(self.temperature, 2), + "reasoning": self.reasoning, + } + + +# LLM params per Qalb state +_STATE_PARAMS = { + QalbState.QABD: {"max_tokens": 2048, "temperature": 0.3}, + QalbState.BAST: {"max_tokens": 4096, "temperature": 0.75}, + QalbState.KHUSHU: {"max_tokens": 3000, "temperature": 0.45}, +} + +# Emotional states that force a specific Qalb state +# (from core/qalb.py EmotionalState values) +_EMOTION_TO_STATE = { + "frustrated": QalbState.QABD, + "anxious": QalbState.QABD, + "confused": QalbState.QABD, + "positive": QalbState.BAST, + "determined": QalbState.BAST, + "fatigued": QalbState.QABD, # Tired → reduce load + "neutral": None, # Follow oscillation +} + + +class QalbProcessor: + """ + Global workspace with cardiac oscillation. + + The oscillation_phase advances by PHASE_STEP with each call to process(). + Phase 0.0 – 0.5 → Systole (QABD) + Phase 0.5 – 1.0 → Diastole (BAST) + + Emotional state can override the natural oscillation. + KHUSHU is triggered when nafs_level >= 4 and task complexity is "extreme". + + Usage: + processor = QalbProcessor() + output = processor.process("Analyse logs", emotional_reading, nafs_level=3) + # Use output.max_tokens and output.temperature in LLM call + """ + + PHASE_STEP = 0.04 # Advance ~25 interactions per full cycle + + def __init__(self): + self.oscillation_phase: float = 0.0 + self._interaction_count: int = 0 + + def process( + self, + task: str, + emotional_state: str = "neutral", + nafs_level: int = 1, + complexity: str = "moderate", + ) -> QalbOutput: + """ + Determine Qalb state and return LLM parameter recommendations. + + Priority order: + 1. KHUSHU — if nafs_level >= 4 AND complexity == "extreme" + 2. Emotional override — if emotion maps to specific state + 3. Cardiac oscillation — systole vs diastole from phase + """ + self._interaction_count += 1 + # Advance oscillation + self.oscillation_phase = (self.oscillation_phase + self.PHASE_STEP) % 1.0 + + # 1. KHUSHU override + if nafs_level >= 4 and complexity == "extreme": + state = QalbState.KHUSHU + reasoning = f"KHUSHU: nafs_level={nafs_level} + extreme task" + # 2. Emotional override + elif emotional_state in _EMOTION_TO_STATE and _EMOTION_TO_STATE[emotional_state]: + state = _EMOTION_TO_STATE[emotional_state] + reasoning = f"Emotional override: {emotional_state} → {state.value}" + # 3. Cardiac oscillation + else: + if self.oscillation_phase < 0.5: + state = QalbState.QABD + phase_pct = int(self.oscillation_phase / 0.5 * 100) + reasoning = f"Systole phase {phase_pct}%: analytical focus" + else: + state = QalbState.BAST + phase_pct = int((self.oscillation_phase - 0.5) / 0.5 * 100) + reasoning = f"Diastole phase {phase_pct}%: creative expansion" + + params = _STATE_PARAMS[state] + output = QalbOutput( + state=state, + max_tokens=params["max_tokens"], + temperature=params["temperature"], + reasoning=reasoning, + ) + + logger.debug( + "[QALB] phase=%.2f emotion=%s nafs=%d → %s (tokens=%d temp=%.2f)", + self.oscillation_phase, emotional_state, nafs_level, + state.value, output.max_tokens, output.temperature, + ) + return output + + def get_phase_info(self) -> dict: + """Current oscillation state for monitoring.""" + phase = "systole (QABD)" if self.oscillation_phase < 0.5 else "diastole (BAST)" + return { + "oscillation_phase": round(self.oscillation_phase, 3), + "cardiac_phase": phase, + "interaction_count": self._interaction_count, + } diff --git a/backend/core/self_healing.py b/backend/core/self_healing.py new file mode 100644 index 0000000..a6be9c6 --- /dev/null +++ b/backend/core/self_healing.py @@ -0,0 +1,447 @@ +""" +Self-Healing Architecture — Lawwāma Self-Monitoring Protocol +============================================================= + +"And I swear by the self-reproaching soul (Nafs al-Lawwāma)" — Quran 75:2 + +Implements Algorithm 3: LAWWAMA_SELF_HEALING + +4-level repair hierarchy (DNA repair analogy): + L1: Immediate proofreading (single-token correction) + L2: Batch mismatch repair (paragraph-level) + L3: Structural excision repair (reasoning chain replacement) + L4: Double-strand regeneration (full response restart) + +Health metric: + H(t) = H_baseline - Σ_i λ_i·ε_i(t) + Σ_j μ_j·repair_j(t) + +Synaptic homeostasis: + w_ij(t+1) = w_ij(t) × (target_activity / actual_activity)^η + +Hallucination score: + H_score = 1 - min(conviction_Fu'ad, consistency_Lawh, agreement_agents) +""" + +import logging +import time +from dataclasses import dataclass, field +from enum import Enum +from typing import Any + +logger = logging.getLogger("mizan.self_healing") + +# Health metric parameters +H_BASELINE = 1.0 +LAMBDA_HALLUCINATION = 0.4 # weight: hallucination error +LAMBDA_COHERENCE = 0.3 # weight: coherence violation +LAMBDA_CONTRADICTION = 0.2 # weight: self-contradiction +LAMBDA_DRIFT = 0.1 # weight: goal drift + +MU_L1 = 0.1 # health restored per L1 repair +MU_L2 = 0.2 # health restored per L2 repair +MU_L3 = 0.35 # health restored per L3 repair +MU_L4 = 0.6 # health restored per L4 repair + +# Synaptic homeostasis +ETA = 0.1 # homeostatic learning rate + +# Repair thresholds +L1_THRESHOLD = 0.85 # H below this → L1 repair +L2_THRESHOLD = 0.70 +L3_THRESHOLD = 0.55 +L4_THRESHOLD = 0.40 + + +class RepairLevel(Enum): + NONE = 0 + L1_PROOFREADING = 1 # Immediate: token-level + L2_MISMATCH = 2 # Batch: paragraph-level + L3_EXCISION = 3 # Structural: chain replacement + L4_REGENERATION = 4 # Nuclear: full restart + + +class ErrorType(Enum): + HALLUCINATION = "hallucination" + COHERENCE_VIOLATION = "coherence_violation" + SELF_CONTRADICTION = "self_contradiction" + GOAL_DRIFT = "goal_drift" + TOOL_FAILURE = "tool_failure" + + +@dataclass +class HealthError: + error_type: ErrorType + severity: float # 0.0 - 1.0 + location: str # where in the reasoning chain + description: str + timestamp: float = field(default_factory=time.time) + + +@dataclass +class RepairRecord: + level: RepairLevel + errors_addressed: list[ErrorType] + health_delta: float + action_taken: str + timestamp: float = field(default_factory=time.time) + success: bool = True + + +@dataclass +class ImmuneMemory: + """Records successful repairs for adaptive future healing.""" + error_pattern: str + repair_strategy: RepairLevel + success_rate: float + invocation_count: int + + +@dataclass +class HealthReport: + current_health: float + baseline: float + errors: list[HealthError] + repair_needed: RepairLevel + repair_records: list[RepairRecord] + hallucination_score: float + immune_memories: int + synaptic_weights: dict[str, float] + + +class LawwamaHealingSystem: + """ + Algorithm 3: LAWWAMA_SELF_HEALING + + The self-reproaching soul monitors its own output quality, + detects errors across 4 levels of severity, and applies + appropriate repair strategies — never accepting corruption silently. + + Health metric tracks cumulative error load and repair benefit: + H(t) = H_baseline - Σ_i λ_i·ε_i(t) + Σ_j μ_j·repair_j(t) + """ + + def __init__(self): + self.health: float = H_BASELINE + self.error_history: list[HealthError] = [] + self.repair_history: list[RepairRecord] = [] + self.immune_memory: dict[str, ImmuneMemory] = {} + # Synaptic weights: skill_name → confidence weight + self.synaptic_weights: dict[str, float] = {} + self.target_activity = 0.7 # target activation level + self._cycle = 0 + + def monitor( + self, + response: str, + task: str, + conviction_score: float = 0.5, + lawh_consistency: float = 0.8, + agent_agreement: float = 0.7, + ) -> HealthReport: + """ + Run a full health monitoring cycle. + + 1. Compute hallucination score + 2. Detect errors from response + 3. Update health metric H(t) + 4. Determine repair level needed + 5. Return HealthReport + """ + self._cycle += 1 + + # Hallucination score: H_score = 1 - min(conviction, consistency, agreement) + hallucination_score = 1.0 - min(conviction_score, lawh_consistency, agent_agreement) + + # Detect errors + errors = self._detect_errors(response, task, hallucination_score) + + # Update health metric + error_penalty = sum( + self._lambda(e.error_type) * e.severity for e in errors + ) + repair_benefit = sum( + self._mu(r.level) for r in self.repair_history[-5:] + if time.time() - r.timestamp < 60 + ) + self.health = max( + 0.0, + min(H_BASELINE, H_BASELINE - error_penalty + repair_benefit) + ) + + # Record errors + self.error_history.extend(errors) + if len(self.error_history) > 200: + self.error_history = self.error_history[-100:] + + # Determine repair level + repair_needed = self._classify_repair_level(self.health) + + # Synaptic homeostasis update + actual_activity = 1.0 - hallucination_score + self._homeostatic_update(task, actual_activity) + + logger.debug( + "[LAWWAMA] cycle=%d health=%.3f h_score=%.3f errors=%d repair=%s", + self._cycle, self.health, hallucination_score, len(errors), repair_needed.name, + ) + + return HealthReport( + current_health=round(self.health, 4), + baseline=H_BASELINE, + errors=errors, + repair_needed=repair_needed, + repair_records=self.repair_history[-10:], + hallucination_score=round(hallucination_score, 4), + immune_memories=len(self.immune_memory), + synaptic_weights=dict(self.synaptic_weights), + ) + + def repair( + self, + level: RepairLevel, + response: str, + task: str, + errors: list[HealthError], + ) -> tuple[str, RepairRecord]: + """ + Execute repair at the specified level. + + Returns: (repaired_response, repair_record) + """ + action = "" + repaired = response + + if level == RepairLevel.L1_PROOFREADING: + repaired, action = self._l1_proofreading(response) + + elif level == RepairLevel.L2_MISMATCH: + repaired, action = self._l2_mismatch_repair(response, task) + + elif level == RepairLevel.L3_EXCISION: + repaired, action = self._l3_structural_excision(response, task, errors) + + elif level == RepairLevel.L4_REGENERATION: + repaired, action = self._l4_regeneration(task) + + error_types = [e.error_type for e in errors] + record = RepairRecord( + level=level, + errors_addressed=error_types, + health_delta=self._mu(level), + action_taken=action, + ) + self.repair_history.append(record) + self.health = min(H_BASELINE, self.health + self._mu(level)) + + # Update immune memory + pattern = self._error_pattern(errors) + if pattern in self.immune_memory: + mem = self.immune_memory[pattern] + mem.invocation_count += 1 + mem.success_rate = ( + 0.8 * mem.success_rate + 0.2 * (1.0 if record.success else 0.0) + ) + else: + self.immune_memory[pattern] = ImmuneMemory( + error_pattern=pattern, + repair_strategy=level, + success_rate=1.0, + invocation_count=1, + ) + + logger.info( + "[REPAIR] L%d applied: %s → health=%.3f", + level.value, action[:80], self.health, + ) + return repaired, record + + def should_checkpoint(self, turn: int, max_turns: int) -> bool: + """ + Health-based checkpoint interval — replaces hardcoded `turn % 3 == 0`. + + Healthy agent (H >= 0.85): check every 4 turns + Moderate (0.55 <= H < 0.85): check every 2 turns + Unhealthy (H < 0.55): check every turn + """ + if turn <= 0: + return False + if self.health >= L1_THRESHOLD: + interval = 4 + elif self.health >= L3_THRESHOLD: + interval = 2 + else: + interval = 1 + return turn % interval == 0 + + def _l1_proofreading(self, response: str) -> tuple[str, str]: + """L1: Immediate token-level correction — add uncertainty hedges.""" + hedges = [ + ("I am certain", "I believe"), + ("definitely", "likely"), + ("always", "generally"), + ("never fails", "rarely fails"), + ("guaranteed", "expected"), + ] + repaired = response + changes = [] + for wrong, right in hedges: + if wrong.lower() in repaired.lower(): + repaired = repaired.replace(wrong, right) + changes.append(f"'{wrong}'→'{right}'") + action = f"L1 proofreading: {', '.join(changes) or 'hedge inserted'}" + return repaired, action + + def _l2_mismatch_repair(self, response: str, task: str) -> tuple[str, str]: + """L2: Paragraph-level mismatch — add consistency caveat.""" + caveat = ( + "\n\n[Lawwāma Note: Some portions of this response may contain " + "inconsistencies. Please verify key claims independently.]" + ) + action = "L2 mismatch repair: consistency caveat appended" + return response + caveat, action + + def _l3_structural_excision( + self, response: str, task: str, errors: list[HealthError] + ) -> tuple[str, str]: + """L3: Remove erroneous reasoning chain, rebuild from task.""" + paragraphs = response.split("\n\n") + # Keep first and last paragraphs (intro + conclusion), rebuild middle + if len(paragraphs) > 2: + rebuilt = paragraphs[0] + "\n\n[Reasoning chain rebuilt due to coherence errors]\n\n" + paragraphs[-1] + else: + rebuilt = response + action = "L3 structural excision: middle chain rebuilt" + return rebuilt, action + + def _l4_regeneration(self, task: str) -> tuple[str, str]: + """L4: Signal that full regeneration is needed.""" + placeholder = ( + "[Lawwāma: Full regeneration required. " + "Previous response had critical integrity failures. " + f"Please re-attempt task: {task[:100]}]" + ) + action = "L4 nuclear regeneration: full restart signalled" + return placeholder, action + + def _detect_errors( + self, response: str, task: str, hallucination_score: float + ) -> list[HealthError]: + errors = [] + response_lower = response.lower() + + # Hallucination detection + if hallucination_score > 0.5: + errors.append(HealthError( + error_type=ErrorType.HALLUCINATION, + severity=hallucination_score, + location="response", + description=f"Hallucination score {hallucination_score:.2f} exceeds threshold", + )) + + # Contradiction markers + contradiction_pairs = [ + ("always", "never"), + ("impossible", "definitely possible"), + ("cannot", "can easily"), + ] + for pos, neg in contradiction_pairs: + if pos in response_lower and neg in response_lower: + errors.append(HealthError( + error_type=ErrorType.SELF_CONTRADICTION, + severity=0.6, + location="response", + description=f"Contradiction detected: '{pos}' vs '{neg}'", + )) + + # Overclaiming (epistemic violation) + overclaim_markers = [ + "100% certain", "absolutely guaranteed", "impossible to fail", + "perfect solution", "I am certain" + ] + for marker in overclaim_markers: + if marker.lower() in response_lower: + errors.append(HealthError( + error_type=ErrorType.COHERENCE_VIOLATION, + severity=0.4, + location="response", + description=f"Overclaiming detected: '{marker}'", + )) + break + + # Goal drift — response doesn't address task + task_keywords = set(task.lower().split()[:10]) + response_words = set(response_lower.split()) + overlap = len(task_keywords & response_words) / max(len(task_keywords), 1) + if overlap < 0.2: + errors.append(HealthError( + error_type=ErrorType.GOAL_DRIFT, + severity=0.3 + 0.3 * (1 - overlap), + location="response", + description=f"Goal drift: only {overlap:.1%} task keyword coverage", + )) + + return errors + + def _classify_repair_level(self, health: float) -> RepairLevel: + if health >= L1_THRESHOLD: + return RepairLevel.NONE + elif health >= L2_THRESHOLD: + return RepairLevel.L1_PROOFREADING + elif health >= L3_THRESHOLD: + return RepairLevel.L2_MISMATCH + elif health >= L4_THRESHOLD: + return RepairLevel.L3_EXCISION + else: + return RepairLevel.L4_REGENERATION + + def _homeostatic_update(self, skill_key: str, actual_activity: float) -> None: + """ + Synaptic homeostasis: + w_ij(t+1) = w_ij(t) × (target / actual)^η + + Drives weights toward balanced activation. + """ + if skill_key not in self.synaptic_weights: + self.synaptic_weights[skill_key] = 1.0 + w = self.synaptic_weights[skill_key] + ratio = self.target_activity / max(actual_activity, 0.01) + self.synaptic_weights[skill_key] = min(2.0, max(0.1, w * (ratio ** ETA))) + + @staticmethod + def _lambda(error_type: ErrorType) -> float: + """Health penalty weight per error type.""" + return { + ErrorType.HALLUCINATION: LAMBDA_HALLUCINATION, + ErrorType.COHERENCE_VIOLATION: LAMBDA_COHERENCE, + ErrorType.SELF_CONTRADICTION: LAMBDA_CONTRADICTION, + ErrorType.GOAL_DRIFT: LAMBDA_DRIFT, + ErrorType.TOOL_FAILURE: LAMBDA_COHERENCE, + }.get(error_type, 0.2) + + @staticmethod + def _mu(level: RepairLevel) -> float: + """Health restoration per repair level.""" + return { + RepairLevel.NONE: 0.0, + RepairLevel.L1_PROOFREADING: MU_L1, + RepairLevel.L2_MISMATCH: MU_L2, + RepairLevel.L3_EXCISION: MU_L3, + RepairLevel.L4_REGENERATION: MU_L4, + }.get(level, 0.0) + + @staticmethod + def _error_pattern(errors: list[HealthError]) -> str: + """Create a hashable pattern key from error types.""" + types = sorted(set(e.error_type.value for e in errors)) + return ":".join(types) or "none" + + def to_dict(self) -> dict: + return { + "health": round(self.health, 4), + "baseline": H_BASELINE, + "error_history_count": len(self.error_history), + "repair_history_count": len(self.repair_history), + "immune_memory_entries": len(self.immune_memory), + "synaptic_weights_count": len(self.synaptic_weights), + "cycle": self._cycle, + } diff --git a/backend/knowledge/__init__.py b/backend/knowledge/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/knowledge/ingest.py b/backend/knowledge/ingest.py new file mode 100644 index 0000000..6767323 --- /dev/null +++ b/backend/knowledge/ingest.py @@ -0,0 +1,189 @@ +""" +Knowledge Ingestion (Ilm - عِلْم) +================================== + +"Read in the name of your Lord who created" - Quran 96:1 + +Extracts knowledge from external sources (URLs, PDFs, YouTube) +and stores it in MIZAN's memory system. +""" + +import logging +import re +from html.parser import HTMLParser + +import httpx + +logger = logging.getLogger("mizan.knowledge") + + +class _TextExtractor(HTMLParser): + """Extract visible text from HTML, skipping scripts/styles.""" + + def __init__(self): + super().__init__() + self.text: list[str] = [] + self._skip_tags = {"script", "style", "noscript", "head"} + self._skipping = False + self._title = "" + self._in_title = False + + def handle_starttag(self, tag, attrs): + if tag in self._skip_tags: + self._skipping = True + if tag == "title": + self._in_title = True + + def handle_endtag(self, tag): + if tag in self._skip_tags: + self._skipping = False + if tag == "title": + self._in_title = False + + def handle_data(self, data): + if self._in_title: + self._title += data.strip() + if not self._skipping: + stripped = data.strip() + if stripped: + self.text.append(stripped) + + +async def extract_url(url: str) -> dict: + """Extract text content from a web URL.""" + async with httpx.AsyncClient(timeout=30, follow_redirects=True) as client: + headers = {"User-Agent": "Mozilla/5.0 (compatible; MIZAN/1.0)"} + response = await client.get(url, headers=headers) + response.raise_for_status() + + extractor = _TextExtractor() + extractor.feed(response.text) + content = " ".join(extractor.text) + + return { + "title": extractor._title or url, + "content": content[:50000], + "source": url, + "source_type": "url", + "char_count": len(content), + } + + +def extract_pdf(file_bytes: bytes, filename: str = "upload.pdf") -> dict: + """Extract text content from a PDF file.""" + try: + import fitz # pymupdf + except ImportError: + return { + "error": "pymupdf not installed. Run: pip install pymupdf", + "source": filename, + "source_type": "pdf", + } + + doc = fitz.open(stream=file_bytes, filetype="pdf") + pages_text = [] + for page in doc: + pages_text.append(page.get_text()) + doc.close() + + content = "\n\n".join(pages_text) + title_match = re.search(r"^(.{5,100})", content.strip()) + title = title_match.group(1) if title_match else filename + + return { + "title": title, + "content": content[:50000], + "source": filename, + "source_type": "pdf", + "page_count": len(pages_text), + "char_count": len(content), + } + + +def _extract_youtube_id(url: str) -> str | None: + """Extract video ID from various YouTube URL formats.""" + patterns = [ + r"(?:v=|/v/|youtu\.be/)([a-zA-Z0-9_-]{11})", + r"(?:embed/)([a-zA-Z0-9_-]{11})", + r"(?:shorts/)([a-zA-Z0-9_-]{11})", + ] + for pattern in patterns: + match = re.search(pattern, url) + if match: + return match.group(1) + return None + + +async def extract_youtube(url: str) -> dict: + """Extract transcript from a YouTube video.""" + try: + from youtube_transcript_api import YouTubeTranscriptApi + except ImportError: + return { + "error": "youtube-transcript-api not installed. Run: pip install youtube-transcript-api", + "source": url, + "source_type": "youtube", + } + + video_id = _extract_youtube_id(url) + if not video_id: + return { + "error": f"Could not extract video ID from URL: {url}", + "source": url, + "source_type": "youtube", + } + + try: + transcript_list = YouTubeTranscriptApi.get_transcript(video_id) + content = " ".join(entry["text"] for entry in transcript_list) + + return { + "title": f"YouTube: {video_id}", + "content": content[:50000], + "source": url, + "source_type": "youtube", + "video_id": video_id, + "segment_count": len(transcript_list), + "char_count": len(content), + } + except Exception as exc: + return { + "error": f"Failed to get transcript: {exc}", + "source": url, + "source_type": "youtube", + "video_id": video_id, + } + + +def detect_source_type(source: str) -> str: + """Auto-detect source type from URL/string.""" + lower = source.lower() + if "youtube.com" in lower or "youtu.be" in lower: + return "youtube" + if lower.endswith(".pdf"): + return "pdf" + if lower.startswith("http://") or lower.startswith("https://"): + return "url" + return "unknown" + + +def chunk_content(content: str, chunk_size: int = 1000, overlap: int = 100) -> list[str]: + """Split content into overlapping chunks for memory storage.""" + if len(content) <= chunk_size: + return [content] + + chunks = [] + start = 0 + while start < len(content): + end = start + chunk_size + chunk = content[start:end] + # Try to break at sentence boundary + if end < len(content): + last_period = chunk.rfind(". ") + if last_period > chunk_size // 2: + chunk = chunk[: last_period + 1] + end = start + last_period + 1 + chunks.append(chunk.strip()) + start = end - overlap + + return chunks diff --git a/backend/memory/dhikr.py b/backend/memory/dhikr.py index f1bb49a..36a2c1e 100644 --- a/backend/memory/dhikr.py +++ b/backend/memory/dhikr.py @@ -90,6 +90,40 @@ def __init__(self, db_path: str = "mizan_memory.db"): self.masalik = MasalikNetwork() + # ── KnowledgeGraph: Entity + relationship store ── + try: + from memory.knowledge_graph import KnowledgeGraph + self.knowledge_graph = KnowledgeGraph(db_path=self.db_path) + except Exception: + self.knowledge_graph = None + + # ── LawhMahfuz: Immutable preserved memory ── + try: + import os + from memory.lawh_mahfuz import LawhMahfuz + lawh_dir = os.path.dirname(self.db_path) if self.db_path != ":memory:" else "/tmp" + lawh_db = ( + os.path.join(lawh_dir, "lawh_mahfuz.db") + if self.db_path != ":memory:" + else ":memory:" + ) + self.lawh_mahfuz = LawhMahfuz(db_path=lawh_db) + except Exception: + self.lawh_mahfuz = None + + # ── MemoryPyramid: Unified 5-layer query ── + try: + from memory.memory_pyramid import MemoryPyramid + self.pyramid = MemoryPyramid( + dhikr=self, + masalik=self.masalik, + lawh_mahfuz=self.lawh_mahfuz, + vector_store=None, + knowledge_graph=self.knowledge_graph, + ) + except Exception: + self.pyramid = None + # Working memory (short-term) - like immediate consciousness self.working_memory: dict[str, Memory] = {} self.working_capacity = 7 # Miller's Law meets Quranic pattern (7 heavens) @@ -199,6 +233,13 @@ def _init_db(self): success INTEGER DEFAULT 1 ) """) + c.execute(""" + CREATE TABLE IF NOT EXISTS preferences ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL, + updated_at TEXT NOT NULL + ) + """) # Performance indexes c.execute("CREATE INDEX IF NOT EXISTS idx_audit_timestamp ON audit_log(timestamp)") c.execute("CREATE INDEX IF NOT EXISTS idx_audit_severity ON audit_log(severity)") @@ -303,10 +344,14 @@ async def recall( params.append(agent_id) if query: words = query.strip().split() - for word in words: - escaped_word = word.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") - sql += " AND (content LIKE ? ESCAPE '\\' OR tags LIKE ? ESCAPE '\\')" - params.extend([f"%{escaped_word}%", f"%{escaped_word}%"]) + if words: + # Use OR across words so partial matches still return results + word_clauses = [] + for word in words: + escaped = word.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + word_clauses.append("(content LIKE ? ESCAPE '\\' OR tags LIKE ? ESCAPE '\\')") + params.extend([f"%{escaped}%", f"%{escaped}%"]) + sql += " AND (" + " OR ".join(word_clauses) + ")" sql += " ORDER BY importance DESC, recency DESC LIMIT ?" params.append(limit) @@ -348,6 +393,25 @@ def recall_pathways(self, query: str, top_k: int = 8) -> str: """ return self.masalik.recall_context(query, top_k=top_k) + def recall_unified(self, query: str, top_k: int = 10) -> list: + """ + Unified recall across all 5 memory layers via MemoryPyramid. + Returns list of MemoryHit objects ranked by (relevance × certainty × recency). + Falls back to standard dhikr recall if pyramid not available. + """ + if self.pyramid: + return self.pyramid.query(query, top_k=top_k) + return [] + + def recall_unified_for_prompt(self, query: str, top_k: int = 5) -> str: + """ + Recall unified memory and format for system prompt injection. + Returns empty string if nothing relevant found. + """ + if self.pyramid: + return self.pyramid.format_for_prompt(query, top_k=top_k) + return self.recall_pathways(query, top_k=top_k) + async def _persist(self, memory: Memory): """Persist memory to database""" conn = self._get_conn() @@ -540,6 +604,154 @@ async def get_messages(self, session_id: str, limit: int = 50) -> list[dict]: for r in rows ] + async def list_sessions(self, limit: int = 20) -> list[dict]: + """List recent chat sessions with metadata""" + conn = self._get_conn() + c = conn.cursor() + c.execute( + """ + SELECT m.session_id, + MIN(m.created_at) as started_at, + MAX(m.created_at) as last_message_at, + COUNT(*) as message_count, + (SELECT content FROM agent_messages + WHERE session_id = m.session_id AND role = 'user' + ORDER BY created_at LIMIT 1) as first_message + FROM agent_messages m + GROUP BY m.session_id + ORDER BY MAX(m.created_at) DESC + LIMIT ? + """, + (limit,), + ) + rows = c.fetchall() + self._release_conn(conn) + + return [ + { + "session_id": r[0], + "started_at": r[1], + "last_message_at": r[2], + "message_count": r[3], + "first_message": (r[4] or "")[:80] if len(r) > 4 else "", + } + for r in rows + ] + + # ── Preferences ───────────────────────────────────────────── + + # ── Hikmah (Wisdom Patterns) ───────────────────────────────── + + async def store_hikmah( + self, + pattern: str, + context: str, + outcome: str, + confidence: float = 0.5, + source_agent: str = "", + ) -> str: + """Persist a learned wisdom pattern to the hikmah table.""" + import uuid as _uuid + + hikmah_id = str(_uuid.uuid4()) + conn = self._get_conn() + c = conn.cursor() + # Avoid duplicates: if same pattern+context exists, update confidence + c.execute( + "SELECT id, applications, confidence FROM hikmah WHERE pattern = ? AND context = ?", + (pattern[:500], context[:500]), + ) + existing = c.fetchone() + if existing: + new_apps = (existing[1] or 0) + 1 + new_conf = min(1.0, existing[2] + 0.05) + c.execute( + "UPDATE hikmah SET applications = ?, confidence = ? WHERE id = ?", + (new_apps, new_conf, existing[0]), + ) + hikmah_id = existing[0] + else: + c.execute( + """INSERT INTO hikmah (id, pattern, context, outcome, confidence, applications, created_at, source_agent) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)""", + ( + hikmah_id, + pattern[:500], + context[:500], + outcome[:1000], + confidence, + 1, + datetime.now(UTC).isoformat(), + source_agent, + ), + ) + conn.commit() + self._release_conn(conn) + return hikmah_id + + async def load_hikmah(self, agent_id: str = "", limit: int = 20) -> list[dict]: + """Load learned wisdom patterns from the hikmah table.""" + conn = self._get_conn() + c = conn.cursor() + if agent_id: + c.execute( + """SELECT pattern, context, outcome, confidence, applications + FROM hikmah WHERE source_agent = ? + ORDER BY confidence DESC, applications DESC LIMIT ?""", + (agent_id, limit), + ) + else: + c.execute( + """SELECT pattern, context, outcome, confidence, applications + FROM hikmah ORDER BY confidence DESC, applications DESC LIMIT ?""", + (limit,), + ) + rows = c.fetchall() + self._release_conn(conn) + return [ + { + "pattern": r[0], + "context": r[1], + "outcome": r[2], + "confidence": r[3], + "applications": r[4], + } + for r in rows + ] + + async def get_preference(self, key: str, default: str = "") -> str: + """Read a persisted preference by key.""" + conn = self._get_conn() + c = conn.cursor() + c.execute("SELECT value FROM preferences WHERE key = ?", (key,)) + row = c.fetchone() + self._release_conn(conn) + return row[0] if row else default + + async def set_preference(self, key: str, value: str) -> None: + """Upsert a persisted preference.""" + from datetime import datetime, UTC + + conn = self._get_conn() + c = conn.cursor() + c.execute( + """INSERT INTO preferences (key, value, updated_at) + VALUES (?, ?, ?) + ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at""", + (key, value, datetime.now(UTC).isoformat()), + ) + conn.commit() + self._release_conn(conn) + + async def get_all_preferences(self) -> dict[str, str]: + """Read all persisted preferences as a dict.""" + conn = self._get_conn() + c = conn.cursor() + c.execute("SELECT key, value FROM preferences") + rows = c.fetchall() + self._release_conn(conn) + return {r[0]: r[1] for r in rows} + async def save_task( self, agent_id: str, diff --git a/backend/memory/knowledge_graph.py b/backend/memory/knowledge_graph.py index 899827b..709c3e4 100644 --- a/backend/memory/knowledge_graph.py +++ b/backend/memory/knowledge_graph.py @@ -23,7 +23,7 @@ class KnowledgeGraph: Stores entities and their relationships. """ - def __init__(self, db_path: str = "/tmp/mizan_memory.db"): + def __init__(self, db_path: str = "/data/mizan_memory.db"): self.db_path = db_path self._init_tables() diff --git a/backend/memory/lawh_mahfuz.py b/backend/memory/lawh_mahfuz.py new file mode 100644 index 0000000..9ef6067 --- /dev/null +++ b/backend/memory/lawh_mahfuz.py @@ -0,0 +1,260 @@ +""" +Lawh al-Mahfuz (لَوْح مَحْفُوظ) — The Preserved Tablet +========================================================= + +"Nay, it is a Glorious Quran, inscribed in a Preserved Tablet (Lawh Mahfuz)." +— Quran 85:21-22 + +Immutable core memory with triple-checksum integrity verification. +Once stored, entries CANNOT be modified — only new entries can be added. +Used for: Fitrah axioms, proven theorems, verified facts, immutable truths. + +Integrity mechanism: SHA-256 + CRC-32 + content_length +Any mismatch on read → corruption detected → entry quarantined. +""" + +import binascii +import hashlib +import json +import logging +import os +import sqlite3 +import time +from dataclasses import dataclass, field + +logger = logging.getLogger("mizan.lawh_mahfuz") + + +@dataclass +class LawhEntry: + """An immutable entry in the Preserved Tablet.""" + key: str + content: str + source: str + stored_at: float + sha256: str # SHA-256 of content + crc32: str # CRC-32 hex of content + length: int # len(content) in bytes + certainty: float = 1.0 + category: str = "general" + + def to_dict(self) -> dict: + return { + "key": self.key, + "content": self.content[:500], + "source": self.source, + "certainty": self.certainty, + "category": self.category, + "stored_at": self.stored_at, + } + + +def _sha256(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def _crc32(text: str) -> str: + return format(binascii.crc32(text.encode("utf-8")) & 0xFFFFFFFF, "08x") + + +class LawhMahfuz: + """ + Immutable memory store with triple-checksum integrity. + + All entries are stored with SHA-256 + CRC-32 + length. + Every read re-verifies all three checksums. + Corrupted entries are moved to a quarantine table. + + Usage: + lawh = LawhMahfuz() + key = lawh.store_immutable("TRUTH:1", "Always speak truth", source="Quran 33:70") + lawh.verify_integrity("TRUTH:1") # True if intact + entry = lawh.get("TRUTH:1") # None if corrupted + """ + + def __init__(self, db_path: str = "/data/lawh_mahfuz.db"): + self.db_path = db_path + # In-memory cache for fast reads + self._cache: dict[str, LawhEntry] = {} + self._quarantine: set[str] = set() + self._init_db() + self._load_into_cache() + + def _get_conn(self) -> sqlite3.Connection: + return sqlite3.connect(self.db_path, check_same_thread=False) + + def _init_db(self): + os.makedirs(os.path.dirname(self.db_path) or ".", exist_ok=True) + conn = self._get_conn() + try: + c = conn.cursor() + c.execute(""" + CREATE TABLE IF NOT EXISTS lawh_entries ( + key TEXT PRIMARY KEY, + content TEXT NOT NULL, + source TEXT NOT NULL, + stored_at REAL NOT NULL, + sha256 TEXT NOT NULL, + crc32 TEXT NOT NULL, + length INTEGER NOT NULL, + certainty REAL DEFAULT 1.0, + category TEXT DEFAULT 'general' + ) + """) + c.execute(""" + CREATE TABLE IF NOT EXISTS lawh_quarantine ( + key TEXT PRIMARY KEY, + reason TEXT, + quarantined_at REAL + ) + """) + conn.commit() + finally: + conn.close() + + def _load_into_cache(self): + """Load all entries into memory cache on startup.""" + conn = self._get_conn() + try: + c = conn.cursor() + c.execute("SELECT * FROM lawh_entries") + for row in c.fetchall(): + key, content, source, stored_at, sha256, crc32, length, certainty, category = row + entry = LawhEntry( + key=key, content=content, source=source, stored_at=stored_at, + sha256=sha256, crc32=crc32, length=length, + certainty=certainty, category=category, + ) + self._cache[key] = entry + logger.info("[LAWH] Loaded %d entries from preserved tablet", len(self._cache)) + finally: + conn.close() + + def store_immutable( + self, + key: str, + content: str, + source: str = "system", + certainty: float = 1.0, + category: str = "general", + ) -> str: + """ + Store an immutable entry. Once stored, it cannot be modified. + Returns the key if successful. + Raises ValueError if key already exists (immutability enforcement). + """ + if key in self._cache: + logger.debug("[LAWH] Key '%s' already exists — immutability preserved", key) + return key # Already stored — idempotent + + sha = _sha256(content) + crc = _crc32(content) + length = len(content.encode("utf-8")) + now = time.time() + + entry = LawhEntry( + key=key, content=content, source=source, stored_at=now, + sha256=sha, crc32=crc, length=length, + certainty=certainty, category=category, + ) + + conn = self._get_conn() + try: + c = conn.cursor() + c.execute( + """INSERT OR IGNORE INTO lawh_entries + (key, content, source, stored_at, sha256, crc32, length, certainty, category) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""", + (key, content, source, now, sha, crc, length, certainty, category), + ) + conn.commit() + finally: + conn.close() + + self._cache[key] = entry + logger.debug("[LAWH] Stored '%s' (len=%d sha=%s...)", key, length, sha[:8]) + return key + + def verify_integrity(self, key: str) -> bool: + """ + Verify all three checksums for an entry. + Returns True if intact, False if corrupted. + Quarantines corrupted entries. + """ + if key in self._quarantine: + return False + + entry = self._cache.get(key) + if entry is None: + return False + + # Re-compute all three checksums + actual_sha = _sha256(entry.content) + actual_crc = _crc32(entry.content) + actual_len = len(entry.content.encode("utf-8")) + + if actual_sha != entry.sha256: + self._quarantine_entry(key, f"SHA-256 mismatch: {actual_sha[:8]}≠{entry.sha256[:8]}") + return False + if actual_crc != entry.crc32: + self._quarantine_entry(key, f"CRC-32 mismatch: {actual_crc}≠{entry.crc32}") + return False + if actual_len != entry.length: + self._quarantine_entry(key, f"Length mismatch: {actual_len}≠{entry.length}") + return False + + return True + + def get(self, key: str) -> LawhEntry | None: + """ + Retrieve an entry. Verifies integrity on every read. + Returns None if entry doesn't exist or is corrupted. + """ + if key in self._quarantine: + logger.warning("[LAWH] Attempted read of quarantined key: %s", key) + return None + if not self.verify_integrity(key): + return None + return self._cache.get(key) + + def search(self, query: str, top_k: int = 5) -> list[LawhEntry]: + """Search entries by keyword match in content or key.""" + query_lower = query.lower() + results = [] + for key, entry in self._cache.items(): + if key in self._quarantine: + continue + if query_lower in entry.content.lower() or query_lower in key.lower(): + results.append(entry) + # Sort by certainty desc + results.sort(key=lambda e: -e.certainty) + return results[:top_k] + + def get_by_category(self, category: str) -> list[LawhEntry]: + """Get all entries in a category (e.g., 'ethical', 'epistemic').""" + return [ + e for k, e in self._cache.items() + if e.category == category and k not in self._quarantine + ] + + def _quarantine_entry(self, key: str, reason: str): + """Move corrupted entry to quarantine.""" + self._quarantine.add(key) + conn = self._get_conn() + try: + c = conn.cursor() + c.execute( + "INSERT OR REPLACE INTO lawh_quarantine (key, reason, quarantined_at) VALUES (?,?,?)", + (key, reason, time.time()), + ) + conn.commit() + finally: + conn.close() + logger.error("[LAWH] CORRUPTION DETECTED — quarantined '%s': %s", key, reason) + + def stats(self) -> dict: + return { + "total_entries": len(self._cache), + "quarantined": len(self._quarantine), + "categories": list({e.category for e in self._cache.values()}), + } diff --git a/backend/memory/living_memory.py b/backend/memory/living_memory.py new file mode 100644 index 0000000..5a3a0ff --- /dev/null +++ b/backend/memory/living_memory.py @@ -0,0 +1,687 @@ +""" +Living Memory System — InsÄn-NisyÄn Architecture +================================================== + +"And remember your Lord when you forget" — Quran 18:24 +"Remember Me and I will remember you" — Quran 2:152 + +Memory is NOT a database — it's a living organism that decides what to store, +what to forget, strengthens with use, fades with neglect, transforms over time, +and recalls differently depending on context. + +Implements: +- NOVELTY_GATE: "Do I already know this?" (θ_identical=0.98, θ_similar=0.85, θ_related=0.5) +- IMPORTANCE_SCORER: Multi-factor importance scoring (emotional, goal, surprise, causal, trust, rarity) +- DYNAMIC_RECALL: Context-dependent associative recall with spread activation + emotional modulation +- DHIKR_MAINTENANCE_DAEMON: Spaced repetition, decay, consolidation, Ṣadr→Dhikr→ʿIlm→Lawḥ promotion +- Memory 1+1=2 lifecycle: learn once, activate existing, never re-store + +Principles: + 1. Intelligent forgetting enables intelligent remembering + 2. Memory requires active maintenance (dhikr) — unreviewed decays, reviewed strengthens + 3. Memory has depth: Ḥifẓ (raw) → ʿIlm (structural) → Fahm (applicable) → Tafaqquh (transformative) + 4. Novelty gates storage — only genuinely new information creates new traces +""" + +import hashlib +import logging +import math +import time +from dataclasses import dataclass, field +from enum import Enum +from typing import Any + +logger = logging.getLogger("mizan.living_memory") + +# Novelty Gate thresholds +THETA_IDENTICAL = 0.98 # "I already know this exactly" +THETA_SIMILAR = 0.85 # "I know something like this" +THETA_RELATED = 0.50 # "This is new but related" +THETA_WORTH_STORING = 0.3 # Minimum importance to store unrelated info + +# Importance scoring weights +W_EMOTION = 0.20 +W_GOAL = 0.20 +W_SURPRISE = 0.20 +W_CAUSAL = 0.15 +W_TRUST = 0.10 +W_RARITY = 0.15 + +# Strength / decay parameters +INITIAL_STRENGTH = 0.30 +DELTA_REINFORCE = 0.12 # strength boost on re-encounter +DELTA_RETRIEVAL = 0.08 # testing effect: recall strengthens memory +DELTA_REVIEW = 0.05 # dhikr daemon review boost (diminishing) +DELTA_DECAY = 0.02 # per-cycle decay rate for unreviewed +PHI_SPACING = 1.5 # spacing effect exponent + +# Consolidation thresholds +CONSOLIDATE_RECALL_COUNT = 5 # promote Dhikr→ʿIlm after N retrievals +CONSOLIDATE_CONSISTENCY = 0.7 +PROVE_COUNT = 20 # promote ʿIlm→Lawḥ after N verifications +FORGET_THRESHOLD = 0.05 # archive if strength drops below +MIN_STRENGTH = 0.01 + +# Spread activation parameters +ALPHA_CONTEXT = 0.3 +ALPHA_EMOTION = 0.25 +ALPHA_GOAL = 0.2 +LAMBDA_RECENCY = 0.001 # recency decay constant + +# Working memory capacity (Miller's Law) +SADR_CAPACITY = 7 + +# Review intervals +BASE_REVIEW_INTERVAL = 3600 # 1 hour base + + +class MemoryLevel(Enum): + SADR = "sadr" # Working memory (~4-7 items, seconds) + DHIKR = "dhikr" # Episodic/active (hours-days) + ILM = "ilm" # Semantic/knowledge (weeks-years) + LAWH = "lawh" # Immutable core (forever, write-once-read-many) + + +class GateDecision(Enum): + STORE_NEW = "store_new" + STORE_NEW_WITH_LINKS = "store_new_with_links" + UPDATE_EXISTING = "update_existing" + ACTIVATE_EXISTING = "activate_existing" + IGNORE = "ignore" + + +@dataclass +class MemoryTrace: + """A single memory trace in the living memory system.""" + trace_id: str + content: str + content_hash: str # for fast exact-match + level: MemoryLevel + strength: float + importance: float + emotional_tag: float # -1.0 to +1.0 + context_tags: list[str] = field(default_factory=list) + links: list[str] = field(default_factory=list) # trace_ids of associated memories + recall_count: int = 0 + activation_count: int = 0 + created_at: float = field(default_factory=time.time) + last_accessed: float = field(default_factory=time.time) + optimal_interval: float = BASE_REVIEW_INTERVAL + contradiction_count: int = 0 + proven_count: int = 0 + mutable: bool = True + gist: str = "" # extracted abstract form (for ʿIlm level) + source: str = "" + + def age_hours(self) -> float: + return (time.time() - self.created_at) / 3600 + + def hours_since_access(self) -> float: + return (time.time() - self.last_accessed) / 3600 + + def review_urgency(self) -> float: + """How overdue is this trace for review? >1.0 = overdue.""" + time_since = time.time() - self.last_accessed + return time_since / max(self.optimal_interval, 60) + + +@dataclass +class GateResult: + """Result of the Novelty Gate evaluation.""" + decision: GateDecision + matched_trace: MemoryTrace | None + similarity: float + importance: float + delta_info: str # what's new vs existing + + +@dataclass +class RecallResult: + """A recalled memory with contextual scoring.""" + trace: MemoryTrace + activation: float + context_match: float + emotional_match: float + goal_match: float + recency: float + reconstructed: str # may differ from original (reconstructive recall) + + +@dataclass +class MaintenanceReport: + """Result of one Dhikr Daemon maintenance cycle.""" + reviewed: int + decayed: int + archived: int + promoted_to_ilm: int + promoted_to_lawh: int + cycle_time_ms: float + + +class LivingMemorySystem: + """ + The complete Living Memory architecture. + + 4-level hierarchy: Ṣadr → Dhikr → ʿIlm → Lawḥ + Novelty-gated storage, importance-scored encoding, + context-dependent recall, and Dhikr maintenance daemon. + """ + + def __init__(self, masalik=None, dhikr_db=None, lawh=None): + # Memory stores by level + self.sadr: list[MemoryTrace] = [] # working memory (capacity-limited) + self.traces: dict[str, MemoryTrace] = {} # all traces by ID + self.archive: list[str] = [] # archived (forgotten) trace IDs + + # External system references (for integration) + self._masalik = masalik # MasalikNetwork for spread activation + self._dhikr_db = dhikr_db # DhikrMemorySystem for persistence + self._lawh = lawh # LawhMahfuz for immutable storage + + self._daemon_cycle = 0 + self._content_hashes: dict[str, str] = {} # hash → trace_id for fast lookup + + def process_input( + self, + content: str, + emotional_state: float = 0.0, + goals: list[str] | None = None, + context: str = "", + source_trust: float = 0.7, + ) -> GateResult: + """ + NOVELTY_GATE + IMPORTANCE_SCORER combined entry point. + + 1. Compute similarity to existing memories + 2. Decide: STORE_NEW | UPDATE | ACTIVATE | IGNORE + 3. Score importance + 4. Execute storage or activation + """ + goals = goals or [] + + # Fast exact-match via content hash + content_hash = self._hash_content(content) + if content_hash in self._content_hashes: + existing_id = self._content_hashes[content_hash] + if existing_id in self.traces: + existing = self.traces[existing_id] + return self._activate_existing(existing, "Exact hash match") + + # Similarity search against all traces + best_match, max_similarity = self._find_best_match(content) + + # Importance scoring + importance = self._score_importance( + content, emotional_state, goals, context, source_trust + ) + + # Decision tree (Algorithm: NOVELTY_GATE) + if max_similarity > THETA_IDENTICAL and best_match: + # "I already know this exactly" → like seeing 1+1=2 again + return self._activate_existing(best_match, "Near-identical match") + + elif max_similarity > THETA_SIMILAR and best_match: + # "I know something like this" → enrich existing + delta = self._extract_novel_parts(content, best_match) + return self._update_existing(best_match, delta, importance) + + elif max_similarity > THETA_RELATED and best_match: + # "New but related" → store with links + trace = self._create_trace( + content, content_hash, importance, emotional_state, context + ) + trace.links.append(best_match.trace_id) + self._store_trace(trace) + return GateResult( + decision=GateDecision.STORE_NEW_WITH_LINKS, + matched_trace=trace, + similarity=max_similarity, + importance=importance, + delta_info=f"Linked to '{best_match.content[:40]}'", + ) + + else: + # Completely new + if importance > THETA_WORTH_STORING: + trace = self._create_trace( + content, content_hash, importance, emotional_state, context + ) + self._store_trace(trace) + return GateResult( + decision=GateDecision.STORE_NEW, + matched_trace=trace, + similarity=max_similarity, + importance=importance, + delta_info="Novel information stored", + ) + else: + return GateResult( + decision=GateDecision.IGNORE, + matched_trace=None, + similarity=max_similarity, + importance=importance, + delta_info="Not novel enough and not important enough", + ) + + def recall( + self, + query: str, + context: str = "", + emotional_state: float = 0.0, + goals: list[str] | None = None, + top_k: int = 10, + ) -> list[RecallResult]: + """ + DYNAMIC_RECALL: Context-dependent associative recall. + + 1. Spread activation from query + 2. Context modulation + 3. Emotional modulation (mood-congruent memory) + 4. Goal modulation + 5. Recency weighting + 6. Reconstruction + 7. Post-retrieval update (reconsolidation) + """ + goals = goals or [] + if not self.traces: + return [] + + # Step 1: Spread activation + activations: dict[str, float] = {} + + # Wave 1: Direct matches + query_words = set(query.lower().split()) + for tid, trace in self.traces.items(): + if tid in self.archive: + continue + sim = self._text_similarity(query, trace.content) + if sim > 0.1: + activations[tid] = sim * trace.strength + + # Wave 2: Spread through links (1-hop) + wave2 = {} + for tid, activation in list(activations.items()): + trace = self.traces[tid] + for linked_id in trace.links: + if linked_id in self.traces and linked_id not in self.archive: + link_activation = activation * 0.6 + wave2[linked_id] = max( + wave2.get(linked_id, 0), link_activation + ) + for tid, act in wave2.items(): + activations[tid] = max(activations.get(tid, 0), act) + + # Wave 3: Spread through links (2-hop) + wave3 = {} + for tid, activation in wave2.items(): + trace = self.traces[tid] + for linked_id in trace.links: + if linked_id in self.traces and linked_id not in self.archive: + wave3[linked_id] = max( + wave3.get(linked_id, 0), activation * 0.3 + ) + for tid, act in wave3.items(): + activations[tid] = max(activations.get(tid, 0), act) + + if not activations: + return [] + + # Steps 2-5: Modulation + results = [] + for tid, base_activation in activations.items(): + trace = self.traces[tid] + + # Context modulation + context_match = self._text_similarity(context, " ".join(trace.context_tags)) if context else 0.0 + modulated = base_activation * (1 + ALPHA_CONTEXT * context_match) + + # Emotional modulation (mood-congruent) + emotional_match = 1.0 - abs(emotional_state - trace.emotional_tag) + modulated *= (1 + ALPHA_EMOTION * emotional_match) + + # Goal modulation + goal_match = 0.0 + if goals: + goal_match = max( + self._text_similarity(g, trace.content) for g in goals + ) + modulated *= (1 + ALPHA_GOAL * goal_match) + + # Recency weighting + recency = math.exp(-LAMBDA_RECENCY * trace.hours_since_access()) + modulated *= recency + + results.append(RecallResult( + trace=trace, + activation=modulated, + context_match=context_match, + emotional_match=emotional_match, + goal_match=goal_match, + recency=recency, + reconstructed=trace.content, # simplified: no gap-filling yet + )) + + # Sort by activation, take top-k + results.sort(key=lambda r: r.activation, reverse=True) + results = results[:top_k] + + # Step 7: Post-retrieval update (reconsolidation) + for result in results: + trace = result.trace + trace.recall_count += 1 + trace.strength = min(1.0, trace.strength + DELTA_RETRIEVAL) + trace.last_accessed = time.time() + if context: + if context not in trace.context_tags: + trace.context_tags.append(context) + if len(trace.context_tags) > 20: + trace.context_tags.pop(0) + + return results + + def run_maintenance(self) -> MaintenanceReport: + """ + DHIKR_MAINTENANCE_DAEMON: One maintenance cycle. + + 1. Compute review priority + 2. Review top-priority memories + 3. Decay unreviewed memories + 4. Consolidate: Dhikr→ʿIlm→Lawḥ promotions + 5. Archive forgotten traces + """ + self._daemon_cycle += 1 + start = time.monotonic() + + reviewed = 0 + decayed = 0 + archived = 0 + promoted_ilm = 0 + promoted_lawh = 0 + + # Step 1-2: Review overdue traces + overdue = [ + t for t in self.traces.values() + if t.mutable and t.trace_id not in self.archive and t.review_urgency() > 1.0 + ] + overdue.sort(key=lambda t: t.review_urgency() * t.importance, reverse=True) + + for trace in overdue[:20]: # max 20 reviews per cycle + trace.strength = min(1.0, + trace.strength + DELTA_REVIEW / max(trace.recall_count, 1) + ) + trace.last_accessed = time.time() + # Extend optimal interval (spaced repetition) + trace.optimal_interval *= (1 + trace.strength) ** PHI_SPACING + reviewed += 1 + + # Step 3: Decay unreviewed traces + for trace in self.traces.values(): + if trace.trace_id in self.archive or not trace.mutable: + continue + if trace.review_urgency() < 1.0: + continue # not overdue, no decay + if trace.importance < 0.5: # only decay low-importance + trace.strength *= (1 - DELTA_DECAY) + decayed += 1 + + # Archive if too weak + if trace.strength < FORGET_THRESHOLD: + self.archive.append(trace.trace_id) + archived += 1 + + # Step 4: Consolidation promotions + for trace in list(self.traces.values()): + if trace.trace_id in self.archive: + continue + + # Dhikr → ʿIlm: frequently recalled + consistent + if ( + trace.level == MemoryLevel.DHIKR + and trace.recall_count >= CONSOLIDATE_RECALL_COUNT + and trace.strength >= CONSOLIDATE_CONSISTENCY + ): + trace.level = MemoryLevel.ILM + trace.gist = self._extract_gist(trace) + promoted_ilm += 1 + + # ʿIlm → Lawḥ: proven beyond doubt + elif ( + trace.level == MemoryLevel.ILM + and trace.proven_count >= PROVE_COUNT + and trace.contradiction_count == 0 + ): + trace.level = MemoryLevel.LAWH + trace.mutable = False + trace.strength = 1.0 + # Store in external LawhMahfuz if available + if self._lawh: + try: + self._lawh.store_immutable( + key=f"LM:{trace.trace_id}", + content=trace.gist or trace.content, + source="living_memory_promotion", + ) + except Exception: + pass + promoted_lawh += 1 + + elapsed_ms = (time.monotonic() - start) * 1000 + logger.debug( + "[DHIKR-DAEMON] cycle=%d reviewed=%d decayed=%d archived=%d ilm=%d lawh=%d", + self._daemon_cycle, reviewed, decayed, archived, promoted_ilm, promoted_lawh, + ) + + return MaintenanceReport( + reviewed=reviewed, + decayed=decayed, + archived=archived, + promoted_to_ilm=promoted_ilm, + promoted_to_lawh=promoted_lawh, + cycle_time_ms=round(elapsed_ms, 2), + ) + + def _score_importance( + self, + content: str, + emotional_state: float, + goals: list[str], + context: str, + source_trust: float, + ) -> float: + """ + IMPORTANCE_SCORER: Multi-factor scoring. + + importance = Σ weights · factors: + emotional_weight, goal_relevance, prediction_error (surprise), + causal_impact, source_trust, rarity + """ + content_lower = content.lower() + + # Factor 1: Emotional weight (|affect|) + emotional_weight = abs(emotional_state) + + # Factor 2: Goal relevance + goal_relevance = 0.0 + if goals: + goal_relevance = max( + self._text_similarity(g, content) for g in goals + ) + + # Factor 3: Prediction error (surprise) — novelty proxy + _, max_sim = self._find_best_match(content) + prediction_error = 1.0 - max_sim + + # Factor 4: Causal significance (keyword heuristic) + causal_keywords = {"because", "caused", "leads to", "therefore", "result", "effect"} + causal_impact = min(1.0, + sum(1 for k in causal_keywords if k in content_lower) / 3 + ) + + # Factor 5: Source trust + trust = min(1.0, max(0.0, source_trust)) + + # Factor 6: Rarity (inverse frequency of similar content) + similar_count = sum( + 1 for t in self.traces.values() + if self._text_similarity(content, t.content) > THETA_RELATED + ) + rarity = 1.0 / (1.0 + similar_count) + + # Weighted combination → sigmoid + raw = ( + W_EMOTION * emotional_weight + + W_GOAL * goal_relevance + + W_SURPRISE * prediction_error + + W_CAUSAL * causal_impact + + W_TRUST * trust + + W_RARITY * rarity + ) + return self._sigmoid(raw * 3) # scale into sigmoid range + + def _activate_existing( + self, trace: MemoryTrace, reason: str + ) -> GateResult: + """Activate an existing trace: no new storage, just strengthen.""" + trace.activation_count += 1 + trace.last_accessed = time.time() + trace.strength = min(1.0, trace.strength + DELTA_REINFORCE) + trace.proven_count += 1 + + return GateResult( + decision=GateDecision.ACTIVATE_EXISTING, + matched_trace=trace, + similarity=1.0, + importance=trace.importance, + delta_info=f"{reason} — activated (count={trace.activation_count})", + ) + + def _update_existing( + self, trace: MemoryTrace, delta: str, importance: float + ) -> GateResult: + """Update an existing trace with novel information delta.""" + if delta: + trace.content += f" | UPDATE: {delta[:200]}" + trace.activation_count += 1 + trace.last_accessed = time.time() + trace.strength = min(1.0, trace.strength + DELTA_REINFORCE * 0.7) + trace.importance = max(trace.importance, importance) + + return GateResult( + decision=GateDecision.UPDATE_EXISTING, + matched_trace=trace, + similarity=0.9, + importance=importance, + delta_info=f"Enriched with: {delta[:80]}", + ) + + def _create_trace( + self, + content: str, + content_hash: str, + importance: float, + emotional_tag: float, + context: str, + ) -> MemoryTrace: + trace_id = hashlib.md5( + f"{content[:100]}:{time.time()}".encode() + ).hexdigest()[:12] + + # New traces start in Ṣadr (working memory) + trace = MemoryTrace( + trace_id=trace_id, + content=content, + content_hash=content_hash, + level=MemoryLevel.SADR, + strength=INITIAL_STRENGTH, + importance=importance, + emotional_tag=emotional_tag, + context_tags=[context] if context else [], + source="living_memory", + ) + return trace + + def _store_trace(self, trace: MemoryTrace) -> None: + """Store trace and manage Ṣadr capacity.""" + self.traces[trace.trace_id] = trace + self._content_hashes[trace.content_hash] = trace.trace_id + + # Ṣadr → Dhikr promotion when working memory full + self.sadr.append(trace) + if len(self.sadr) > SADR_CAPACITY: + # Oldest items move to Dhikr level + evicted = self.sadr.pop(0) + if evicted.trace_id in self.traces: + evicted.level = MemoryLevel.DHIKR + + # Persist to external Masalik if available + if self._masalik: + try: + self._masalik.encode(trace.content, importance=trace.importance) + except Exception: + pass + + def _find_best_match(self, content: str) -> tuple[MemoryTrace | None, float]: + """Find the most similar existing trace.""" + best = None + best_sim = 0.0 + for trace in self.traces.values(): + if trace.trace_id in self.archive: + continue + sim = self._text_similarity(content, trace.content) + if sim > best_sim: + best_sim = sim + best = trace + return best, best_sim + + def _extract_novel_parts(self, new_content: str, existing: MemoryTrace) -> str: + """Extract what's genuinely new in content vs existing trace.""" + new_words = set(new_content.lower().split()) + existing_words = set(existing.content.lower().split()) + novel = new_words - existing_words + if novel: + return " ".join(sorted(novel)[:15]) + return "" + + def _extract_gist(self, trace: MemoryTrace) -> str: + """Extract abstract gist from a trace (for ʿIlm promotion).""" + words = trace.content.split() + # Keep first sentence or first 15 words + gist_words = words[:15] + return " ".join(gist_words) + + @staticmethod + def _text_similarity(text_a: str, text_b: str) -> float: + """Jaccard similarity on word sets.""" + if not text_a or not text_b: + return 0.0 + words_a = set(text_a.lower().split()) + words_b = set(text_b.lower().split()) + if not words_a or not words_b: + return 0.0 + intersection = len(words_a & words_b) + union = len(words_a | words_b) + return intersection / union if union > 0 else 0.0 + + @staticmethod + def _hash_content(content: str) -> str: + return hashlib.sha256(content.strip().lower().encode()).hexdigest()[:16] + + @staticmethod + def _sigmoid(x: float) -> float: + return 1.0 / (1.0 + math.exp(-x)) + + def get_level_counts(self) -> dict[str, int]: + counts = {level.value: 0 for level in MemoryLevel} + for trace in self.traces.values(): + if trace.trace_id not in self.archive: + counts[trace.level.value] += 1 + counts["archived"] = len(self.archive) + return counts + + def to_dict(self) -> dict: + return { + "total_traces": len(self.traces), + "levels": self.get_level_counts(), + "sadr_capacity": f"{len(self.sadr)}/{SADR_CAPACITY}", + "daemon_cycles": self._daemon_cycle, + } diff --git a/backend/memory/masalik.py b/backend/memory/masalik.py index d70e779..8242dfd 100644 --- a/backend/memory/masalik.py +++ b/backend/memory/masalik.py @@ -21,8 +21,10 @@ LAWH (لوح): The preserved network — nothing duplicated, everything in place """ +import json import logging import math +import os import re import time from collections import defaultdict @@ -309,17 +311,36 @@ class MasalikNetwork: - Heavily-used pathways become permanent wisdom (Hikmah) - Some pathways are innate/pre-wired (Fitrah) + Persistence: Network state is saved to disk periodically and on shutdown. + On startup, the previously saved state is restored so memory survives restarts. + "And We have certainly made the Quran easy for remembrance (Dhikr). So is there any who will remember?" — 54:17 """ - def __init__(self): + # Default persistence path (inside data/ volume in Docker) + DEFAULT_PERSIST_PATH = "/data/masalik_network.json" + + def __init__(self, persist_path: str | None = None): self.concepts: dict[str, Mafhum] = {} self.pathways: dict[str, Silah] = {} self._activation_history: list[set[str]] = [] self._last_decay: float = time.time() - self._init_fitrah() + # Use environment DB_PATH directory if available, else fallback + if persist_path: + self._persist_path = persist_path + else: + db_dir = os.path.dirname(os.environ.get("DB_PATH", "/data/mizan_memory.db")) + self._persist_path = os.path.join(db_dir, "masalik_network.json") + + self._encode_count_since_save = 0 + self._save_every_n = 10 # Auto-save every N encodes + + # Try to restore from disk first + restored = self._load_from_disk() + if not restored: + self._init_fitrah() def _init_fitrah(self): """ @@ -438,6 +459,9 @@ def encode(self, text: str, importance: float = 0.5) -> dict: if len(self._activation_history) > 100: self._activation_history = self._activation_history[-50:] + # Auto-save periodically to persist across restarts + self._maybe_auto_save() + return { "encoded": len(concepts), "concepts": concepts, @@ -688,4 +712,110 @@ def stats(self) -> dict: "hikmah_pathways": hikmah_pathways, "hikmah_concepts": hikmah_concepts, "avg_pathway_weight": round(avg_weight, 3), + "persisted": self._persist_path, } + + # ─── PERSISTENCE: Save/Load to Disk ────────────────────────────────── + + def save_to_disk(self) -> bool: + """ + Persist the neural network to disk as JSON. + Called periodically and on shutdown to survive restarts. + """ + try: + os.makedirs(os.path.dirname(self._persist_path) or ".", exist_ok=True) + + state = { + "version": 1, + "saved_at": time.time(), + "concepts": { + cid: { + "activation": c.activation, + "resting_level": c.resting_level, + "last_activated": c.last_activated, + "total_activations": c.total_activations, + "is_fitrah": c.is_fitrah, + } + for cid, c in self.concepts.items() + }, + "pathways": { + key: { + "source": p.source, + "target": p.target, + "weight": p.weight, + "pathway_type": p.pathway_type, + "last_activated": p.last_activated, + "co_activations": p.co_activations, + } + for key, p in self.pathways.items() + }, + } + + # Write atomically (write to temp, then rename) + tmp_path = self._persist_path + ".tmp" + with open(tmp_path, "w") as f: + json.dump(state, f) + os.replace(tmp_path, self._persist_path) + + logger.info( + "Masalik saved: %d concepts, %d pathways → %s", + len(self.concepts), + len(self.pathways), + self._persist_path, + ) + return True + except Exception as e: + logger.error("Masalik save failed: %s", e) + return False + + def _load_from_disk(self) -> bool: + """ + Restore neural network state from disk. + Returns True if successfully loaded, False if no saved state. + """ + if not os.path.exists(self._persist_path): + return False + + try: + with open(self._persist_path) as f: + state = json.load(f) + + # Restore concepts + for cid, cdata in state.get("concepts", {}).items(): + self.concepts[cid] = Mafhum( + id=cid, + activation=cdata.get("activation", 0.0), + resting_level=cdata.get("resting_level", 0.0), + last_activated=cdata.get("last_activated", 0.0), + total_activations=cdata.get("total_activations", 0), + is_fitrah=cdata.get("is_fitrah", False), + ) + + # Restore pathways + for key, pdata in state.get("pathways", {}).items(): + self.pathways[key] = Silah( + source=pdata["source"], + target=pdata["target"], + weight=pdata.get("weight", 0.1), + pathway_type=pdata.get("pathway_type", "association"), + last_activated=pdata.get("last_activated", 0.0), + co_activations=pdata.get("co_activations", 0), + ) + + logger.info( + "Masalik restored: %d concepts, %d pathways from %s", + len(self.concepts), + len(self.pathways), + self._persist_path, + ) + return True + except Exception as e: + logger.error("Masalik load failed: %s — starting fresh", e) + return False + + def _maybe_auto_save(self): + """Auto-save after N encode operations.""" + self._encode_count_since_save += 1 + if self._encode_count_since_save >= self._save_every_n: + self._encode_count_since_save = 0 + self.save_to_disk() diff --git a/backend/memory/memory_pyramid.py b/backend/memory/memory_pyramid.py new file mode 100644 index 0000000..f60f660 --- /dev/null +++ b/backend/memory/memory_pyramid.py @@ -0,0 +1,251 @@ +""" +Memory Pyramid (هرم الذاكرة) — Unified 5-Layer Memory Query +============================================================= + +"And We have certainly created man and We know what his soul (nafs) + whispers to him, and We are closer to him than his jugular vein." — Quran 50:16 + +Unifies all five memory systems into a single query interface. +Each layer contributes what it does best: + + Layer 1 — Masalik : Spreading activation → associative concepts + Layer 2 — DhikrMemory : Keyword SQL search → episodic/semantic records + Layer 3 — VectorStore : Semantic embedding search (if available) + Layer 4 — KnowledgeGraph: Entity + relationship lookup + Layer 5 — LawhMahfuz : Immutable fact lookup + +Results are merged, deduplicated, and ranked by (relevance × certainty × recency). +""" + +import logging +import time +from dataclasses import dataclass, field + +logger = logging.getLogger("mizan.memory_pyramid") + + +@dataclass +class MemoryHit: + """A single result from any memory layer.""" + content: str + source_layer: str # "masalik" | "dhikr" | "vector" | "graph" | "lawh" + relevance: float # 0-1 (how well it matches query) + certainty: float # 0-1 (how reliable the memory is) + recency_score: float # 0-1 (1 = very recent) + tags: list[str] = field(default_factory=list) + metadata: dict = field(default_factory=dict) + + @property + def rank_score(self) -> float: + """Combined ranking score: relevance × certainty × recency.""" + return self.relevance * self.certainty * (0.5 + 0.5 * self.recency_score) + + def to_dict(self) -> dict: + return { + "content": self.content[:400], + "source": self.source_layer, + "relevance": round(self.relevance, 3), + "certainty": round(self.certainty, 3), + "rank": round(self.rank_score, 3), + "tags": self.tags[:5], + } + + +def _recency_score(created_at_ts: float, now: float = None) -> float: + """Convert timestamp to 0-1 recency score (1 = just now, 0 = very old).""" + now = now or time.time() + age_hours = (now - created_at_ts) / 3600 + # Half-life of 24 hours → score ~0.5 at 1 day old + return max(0.0, min(1.0, 0.5 ** (age_hours / 24))) + + +class MemoryPyramid: + """ + Unified query across all five memory systems. + + Usage: + pyramid = MemoryPyramid(dhikr, masalik, lawh_mahfuz, vector_store, knowledge_graph) + hits = pyramid.query("zaheer khan", top_k=10) + for hit in hits: + print(f"[{hit.source_layer}] {hit.content[:100]} (rank={hit.rank_score:.2f})") + """ + + def __init__( + self, + dhikr=None, + masalik=None, + lawh_mahfuz=None, + vector_store=None, + knowledge_graph=None, + ): + self.dhikr = dhikr + self.masalik = masalik + self.lawh_mahfuz = lawh_mahfuz + self.vector_store = vector_store + self.knowledge_graph = knowledge_graph + + def query(self, text: str, top_k: int = 10) -> list[MemoryHit]: + """ + Query all available layers and return merged, ranked results. + Layers that are unavailable are silently skipped. + """ + hits: list[MemoryHit] = [] + now = time.time() + + # ── Layer 1: Masalik (neural pathway spreading activation) ── + if self.masalik: + try: + pathway_results = self.masalik.recall(text, top_k=top_k) + for concept, activation in pathway_results: + node = self.masalik.concepts.get(concept) + certainty = node.resting_level if node else 0.3 + hits.append(MemoryHit( + content=f"Associated concept: {concept}", + source_layer="masalik", + relevance=float(activation), + certainty=certainty, + recency_score=_recency_score( + node.last_activated if node else now - 3600, now + ), + tags=["concept", "association"], + )) + except Exception as e: + logger.debug("[PYRAMID] Masalik query failed: %s", e) + + # ── Layer 2: DhikrMemory (SQL keyword search) ── + if self.dhikr: + try: + import asyncio + loop = asyncio.get_event_loop() + if loop.is_running(): + # Use sync fallback if we're inside async context + memories = self._dhikr_sync_recall(text, top_k) + else: + memories = loop.run_until_complete( + self.dhikr.recall(text, limit=top_k) + ) + for mem in memories: + content_str = str(mem.content) + recency = _recency_score(mem.recency.timestamp(), now) + hits.append(MemoryHit( + content=content_str, + source_layer="dhikr", + relevance=float(mem.importance), + certainty=min(0.9, 0.3 + mem.access_count * 0.05), + recency_score=recency, + tags=mem.tags or [], + metadata={"type": mem.memory_type}, + )) + except Exception as e: + logger.debug("[PYRAMID] Dhikr query failed: %s", e) + + # ── Layer 3: VectorStore (semantic embedding search) ── + if self.vector_store and getattr(self.vector_store, "_available", False): + try: + vector_results = self.vector_store.search(text, top_k=top_k) + for result in vector_results: + hits.append(MemoryHit( + content=result.get("content", ""), + source_layer="vector", + relevance=float(result.get("score", 0.5)), + certainty=0.7, # Vector search has good precision + recency_score=0.5, # Unknown recency + tags=result.get("tags", []), + )) + except Exception as e: + logger.debug("[PYRAMID] VectorStore query failed: %s", e) + + # ── Layer 4: KnowledgeGraph (entity + relationship lookup) ── + if self.knowledge_graph: + try: + entities = self.knowledge_graph.search_entities(text, limit=top_k) + for entity in entities: + props = entity.get("properties", {}) + content = f"{entity.get('name', '')} ({entity.get('type', 'entity')})" + if props: + content += f": {list(props.items())[:3]}" + hits.append(MemoryHit( + content=content, + source_layer="graph", + relevance=0.7, # Entity match is high relevance + certainty=0.6, + recency_score=0.4, + tags=["entity", entity.get("type", "concept")], + )) + except Exception as e: + logger.debug("[PYRAMID] KnowledgeGraph query failed: %s", e) + + # ── Layer 5: LawhMahfuz (immutable facts) ── + if self.lawh_mahfuz: + try: + lawh_results = self.lawh_mahfuz.search(text, top_k=top_k) + for entry in lawh_results: + hits.append(MemoryHit( + content=entry.content, + source_layer="lawh", + relevance=0.9, # Immutable facts are highly reliable + certainty=entry.certainty, + recency_score=1.0, # Immutable facts never decay + tags=["immutable", entry.category], + metadata={"source": entry.source}, + )) + except Exception as e: + logger.debug("[PYRAMID] LawhMahfuz query failed: %s", e) + + # ── Merge, deduplicate, rank ── + deduplicated = self._deduplicate(hits) + deduplicated.sort(key=lambda h: -h.rank_score) + return deduplicated[:top_k] + + def _deduplicate(self, hits: list[MemoryHit]) -> list[MemoryHit]: + """Remove near-duplicate hits (same content prefix from different layers).""" + seen_prefixes: set[str] = set() + unique: list[MemoryHit] = [] + for hit in hits: + prefix = hit.content[:80].lower().strip() + if prefix not in seen_prefixes: + seen_prefixes.add(prefix) + unique.append(hit) + return unique + + def _dhikr_sync_recall(self, query: str, limit: int) -> list: + """Synchronous fallback for dhikr recall when inside async context.""" + try: + import asyncio + import concurrent.futures + # Run the coroutine in a separate thread with its own event loop + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit( + asyncio.run, self.dhikr.recall(query, limit=limit) + ) + return future.result(timeout=5) + except Exception: + return [] + + def format_for_prompt(self, text: str, top_k: int = 5) -> str: + """ + Query and format results for injection into an agent system prompt. + Returns empty string if nothing relevant found. + """ + hits = self.query(text, top_k=top_k) + if not hits: + return "" + + lines = ["Unified Memory Recall:"] + for hit in hits: + source = hit.source_layer.upper() + content_preview = hit.content[:150].replace("\n", " ") + lines.append(f" [{source}] {content_preview}") + + return "\n".join(lines) + + def stats(self) -> dict: + """Return availability status of each memory layer.""" + return { + "masalik": self.masalik is not None, + "dhikr": self.dhikr is not None, + "vector_store": self.vector_store is not None + and getattr(self.vector_store, "_available", False), + "knowledge_graph": self.knowledge_graph is not None, + "lawh_mahfuz": self.lawh_mahfuz is not None, + } diff --git a/backend/providers.py b/backend/providers.py index 23e2322..33381f3 100644 --- a/backend/providers.py +++ b/backend/providers.py @@ -17,7 +17,15 @@ import json import logging import os +import re +import uuid from dataclasses import dataclass, field +from pathlib import Path + +from dotenv import load_dotenv + +# Load .env so os.getenv() picks up API keys +load_dotenv(Path(__file__).parent.parent / ".env") logger = logging.getLogger("mizan.providers") @@ -61,6 +69,7 @@ def create( system: str, messages: list[dict], tools: list[dict] = None, + temperature: float = None, ) -> LLMResponse: raise NotImplementedError @@ -88,7 +97,7 @@ def __init__(self, api_key: str): self._client = anthropic.Anthropic(api_key=api_key) - def create(self, model, max_tokens, system, messages, tools=None): + def create(self, model, max_tokens, system, messages, tools=None, temperature=None): kwargs = { "model": model, "max_tokens": max_tokens, @@ -97,6 +106,8 @@ def create(self, model, max_tokens, system, messages, tools=None): } if tools: kwargs["tools"] = tools + if temperature is not None: + kwargs["temperature"] = temperature response = self._client.messages.create(**kwargs) @@ -156,6 +167,61 @@ def __init__( self._client = OpenAI(**kwargs) self.provider_name = provider_name + @staticmethod + def _parse_xml_tool_calls(text: str) -> list["ContentBlock"]: + """Extract tool calls embedded as XML in model text output. + + Handles formats like: + value + ...... + """ + # Match ... blocks + invoke_pattern = re.compile( + r']*>(.*?)', + re.DOTALL, + ) + param_pattern = re.compile( + r']*>(.*?)', + re.DOTALL, + ) + tool_blocks: list[ContentBlock] = [] + for match in invoke_pattern.finditer(text): + tool_name = match.group(1).strip() + body = match.group(2) + params: dict = {} + for param_match in param_pattern.finditer(body): + key = param_match.group(1).strip() + value = param_match.group(2).strip() + # Try to parse JSON values + try: + params[key] = json.loads(value) + except (json.JSONDecodeError, ValueError): + params[key] = value + tool_blocks.append(ContentBlock( + type="tool_use", + id=f"xmltool_{uuid.uuid4().hex[:12]}", + name=tool_name, + input=params, + )) + return tool_blocks + + @staticmethod + def _strip_xml_tool_calls(text: str) -> str: + """Remove XML tool-call markup from text, keeping surrounding prose.""" + # Remove ... wrappers + cleaned = re.sub( + r'.*?', + '', text, flags=re.DOTALL, + ) + # Remove bare ... blocks + cleaned = re.sub( + r']*>.*?', + '', cleaned, flags=re.DOTALL, + ) + # Collapse multiple blank lines + cleaned = re.sub(r'\n{3,}', '\n\n', cleaned) + return cleaned + def _convert_tools_to_openai(self, tools: list[dict]) -> list[dict]: """Convert Anthropic tool schemas to OpenAI function calling format.""" openai_tools = [] @@ -175,7 +241,11 @@ def _convert_tools_to_openai(self, tools: list[dict]) -> list[dict]: return openai_tools def _convert_messages_to_openai(self, system: str, messages: list[dict]) -> list[dict]: - """Convert Anthropic-style messages to OpenAI format.""" + """Convert Anthropic-style messages to OpenAI format. + + Key: OpenAI requires a single assistant message with both content AND + tool_calls (not separate messages). Tool results must have string content. + """ openai_messages = [{"role": "system", "content": system}] for msg in messages: @@ -185,46 +255,71 @@ def _convert_messages_to_openai(self, system: str, messages: list[dict]) -> list if isinstance(content, str): openai_messages.append({"role": role, "content": content}) elif isinstance(content, list): - # Handle Anthropic-style content blocks + # Collect all parts from this Anthropic message into one OpenAI message + text_parts = [] + tool_calls = [] + tool_results = [] + for block in content: if isinstance(block, dict): - if block.get("type") == "tool_result": - openai_messages.append( - { - "role": "tool", - "tool_call_id": block.get("tool_use_id", ""), - "content": block.get("content", ""), - } - ) - elif block.get("type") == "text": - openai_messages.append({"role": role, "content": block.get("text", "")}) - else: - # It's a content block object from Anthropic - if hasattr(block, "type"): - if block.type == "text": - openai_messages.append({"role": role, "content": block.text}) - elif block.type == "tool_use": - openai_messages.append( - { - "role": "assistant", - "tool_calls": [ - { - "id": block.id, - "type": "function", - "function": { - "name": block.name, - "arguments": json.dumps(block.input), - }, - } - ], - } - ) + block_type = block.get("type", "") + if block_type == "tool_result": + raw_content = block.get("content", "") + # Ensure content is a string + if not isinstance(raw_content, str): + raw_content = json.dumps(raw_content) if raw_content else "" + tool_results.append({ + "role": "tool", + "tool_call_id": block.get("tool_use_id", ""), + "content": raw_content, + }) + elif block_type == "tool_use": + tool_calls.append({ + "id": block.get("id", ""), + "type": "function", + "function": { + "name": block.get("name", ""), + "arguments": json.dumps(block.get("input", {})), + }, + }) + elif block_type == "text": + text_parts.append(block.get("text", "")) + elif hasattr(block, "type"): + # Content block object from Anthropic SDK + if block.type == "text": + text_parts.append(block.text) + elif block.type == "tool_use": + tool_calls.append({ + "id": block.id, + "type": "function", + "function": { + "name": block.name, + "arguments": json.dumps(block.input), + }, + }) + + # Emit a single assistant message with text + tool_calls combined + if role == "assistant" and (text_parts or tool_calls): + assistant_msg = {"role": "assistant"} + combined_text = "\n".join(text_parts) + if combined_text: + assistant_msg["content"] = combined_text + elif tool_calls: + assistant_msg["content"] = None + if tool_calls: + assistant_msg["tool_calls"] = tool_calls + openai_messages.append(assistant_msg) + elif text_parts: + openai_messages.append({"role": role, "content": "\n".join(text_parts)}) + + # Tool results are always separate messages with role="tool" + openai_messages.extend(tool_results) else: openai_messages.append({"role": role, "content": str(content)}) return openai_messages - def create(self, model, max_tokens, system, messages, tools=None): + def create(self, model, max_tokens, system, messages, tools=None, temperature=None): openai_messages = self._convert_messages_to_openai(system, messages) kwargs = { @@ -234,6 +329,8 @@ def create(self, model, max_tokens, system, messages, tools=None): } if tools: kwargs["tools"] = self._convert_tools_to_openai(tools) + if temperature is not None: + kwargs["temperature"] = temperature response = self._client.chat.completions.create(**kwargs) choice = response.choices[0] @@ -245,7 +342,7 @@ def create(self, model, max_tokens, system, messages, tools=None): if message.content: blocks.append(ContentBlock(type="text", text=message.content)) - # Tool calls + # Tool calls (structured API) if message.tool_calls: for tc in message.tool_calls: try: @@ -261,9 +358,21 @@ def create(self, model, max_tokens, system, messages, tools=None): ) ) + # Fallback: parse XML-style tool calls from text (MiniMax, etc.) + if not message.tool_calls and message.content and tools: + xml_tools = self._parse_xml_tool_calls(message.content) + if xml_tools: + # Remove XML from text block, replace with cleaned text + cleaned = self._strip_xml_tool_calls(message.content).strip() + blocks = [] + if cleaned: + blocks.append(ContentBlock(type="text", text=cleaned)) + blocks.extend(xml_tools) + # Map finish_reason + has_tool_blocks = any(b.type == "tool_use" for b in blocks) stop_reason = "end_turn" - if choice.finish_reason == "tool_calls": + if choice.finish_reason == "tool_calls" or has_tool_blocks: stop_reason = "tool_use" elif choice.finish_reason == "stop": stop_reason = "end_turn" @@ -331,12 +440,12 @@ def __init__(self, base_url: str = "http://localhost:11434"): ) self._base_url = base_url - def create(self, model, max_tokens, system, messages, tools=None): + def create(self, model, max_tokens, system, messages, tools=None, temperature=None): # Reuse OpenAI-compatible logic provider = OpenAICompatibleProvider.__new__(OpenAICompatibleProvider) provider._client = self._client provider.provider_name = "ollama" - return provider.create(model, max_tokens, system, messages, tools) + return provider.create(model, max_tokens, system, messages, tools, temperature) def stream(self, model, max_tokens, system, messages): openai_messages = [{"role": "system", "content": system}] @@ -363,10 +472,14 @@ def create_provider( Returns None if no provider can be configured. """ - # Auto-detect provider from model name + # Auto-detect provider from model name, verifying keys exist if not provider and model: if model.startswith(("claude-", "anthropic/")): - provider = "anthropic" + # Claude models: prefer Anthropic if key exists, else route via OpenRouter + if os.getenv("ANTHROPIC_API_KEY", "").startswith("sk-ant-"): + provider = "anthropic" + elif os.getenv("OPENROUTER_API_KEY", ""): + provider = "openrouter" elif model.startswith(("gpt-", "o1", "o3", "chatgpt")): provider = "openai" elif "/" in model: @@ -407,6 +520,8 @@ def create_provider( "OPENROUTER_REFERER", "https://github.com/CodeWithJuber/mizan" ), "X-Title": "MIZAN", + # Prevent OpenRouter from auto-routing to incompatible models + "X-Stainless-Retry-Count": "0", }, provider_name="openrouter", ) @@ -427,11 +542,24 @@ def create_provider( return None +def normalize_model_for_provider(model: str, provider_name: str) -> str: + """Translate model IDs between provider formats when needed.""" + # Anthropic → OpenRouter mapping + _anthropic_to_openrouter = { + "claude-opus-4-6": "anthropic/claude-opus-4", + "claude-sonnet-4-20250514": "anthropic/claude-sonnet-4", + "claude-haiku-4-5-20251001": "anthropic/claude-haiku-4.5", + } + if provider_name == "openrouter" and "/" not in model: + return _anthropic_to_openrouter.get(model, model) + return model + + def get_default_model(provider_name: str) -> str: """Get the default model for a provider.""" env_model = os.getenv("DEFAULT_MODEL", "") if env_model: - return env_model + return normalize_model_for_provider(env_model, provider_name) defaults = { "anthropic": "claude-sonnet-4-20250514", @@ -470,6 +598,16 @@ def get_default_model(provider_name: str) -> str: "ollama": [], # Fetched dynamically } +# In-memory active state — updated by switch_provider, read by get_provider_status. +# Falls back to env vars when empty (before any switch). +_active_state: dict[str, str] = {"provider": "", "model": ""} + + +def set_active_state(provider: str, model: str) -> None: + """Update in-memory active provider/model after a switch.""" + _active_state["provider"] = provider + _active_state["model"] = model + def get_provider_status() -> dict: """ @@ -527,25 +665,37 @@ def get_provider_status() -> dict: } ) - # Determine active provider - active = os.getenv("LLM_PROVIDER", "") + # Determine active provider — prefer in-memory state from switch_provider, + # then env var, then auto-detect from configured providers. + active = _active_state["provider"] or os.getenv("LLM_PROVIDER", "") if not active: for p in providers: if p["configured"] and p["name"] != "ollama": active = p["name"] break + default_model = ( + _active_state["model"] + or os.getenv("DEFAULT_MODEL", "") + or get_default_model(active) + ) + return { "active": active, - "default_model": os.getenv("DEFAULT_MODEL", get_default_model(active)), + "default_model": default_model, "providers": providers, } -async def fetch_openrouter_models(limit: int = 50) -> list[dict]: +async def fetch_openrouter_models( + limit: int = 50, + offset: int = 0, + search: str = "", + free_only: bool = False, +) -> dict: """ Fetch available models from OpenRouter's public API. - Returns a curated list of popular models. + Returns paginated results with search/filter support. """ import httpx @@ -553,30 +703,50 @@ async def fetch_openrouter_models(limit: int = 50) -> list[dict]: async with httpx.AsyncClient(timeout=15) as client: resp = await client.get("https://openrouter.ai/api/v1/models") if resp.status_code != 200: - return [] + return {"models": [], "total": 0, "offset": offset, "limit": limit} data = resp.json() models_raw = data.get("data", []) - # Sort by popularity and return curated list models = [] - for m in models_raw[:limit]: + for m in models_raw: + prompt_price = m.get("pricing", {}).get("prompt", "0") + is_free = float(prompt_price or "0") == 0.0 models.append( { "id": m.get("id", ""), "name": m.get("name", m.get("id", "")), "context": m.get("context_length", 0), "pricing": { - "prompt": m.get("pricing", {}).get("prompt", "0"), + "prompt": prompt_price, "completion": m.get("pricing", {}).get("completion", "0"), }, + "free": is_free, } ) - return models + # Apply filters + if search: + search_lower = search.lower() + models = [ + m for m in models + if search_lower in m["id"].lower() or search_lower in m["name"].lower() + ] + if free_only: + models = [m for m in models if m["free"]] + + total = len(models) + paginated = models[offset : offset + limit] + + return { + "models": paginated, + "total": total, + "offset": offset, + "limit": limit, + } except Exception as e: logger.warning(f"Failed to fetch OpenRouter models: {e}") - return [] + return {"models": [], "total": 0, "offset": offset, "limit": limit} async def fetch_ollama_models() -> list[dict]: diff --git a/backend/reasoning/aql_engine.py b/backend/reasoning/aql_engine.py index a6a6a88..5815f39 100644 --- a/backend/reasoning/aql_engine.py +++ b/backend/reasoning/aql_engine.py @@ -27,6 +27,24 @@ logger = logging.getLogger("mizan.aql") +def _lazy_causal_engine(): + """Lazy-load CausalEngine to avoid circular imports.""" + try: + from reasoning.causal_engine import CausalEngine + return CausalEngine() + except ImportError: + return None + + +def _lazy_fuad_engine(): + """Lazy-load FuadEngine to avoid circular imports.""" + try: + from core.fuad import FuadEngine + return FuadEngine() + except ImportError: + return None + + @dataclass class ReasoningStep: """A single step in the reasoning process""" @@ -58,6 +76,8 @@ class AqlEngine: def __init__(self, max_iterations: int = 10): self.max_iterations = max_iterations self._qca = None + self._causal = None + self._fuad = None @property def qca(self): @@ -65,12 +85,25 @@ def qca(self): if self._qca is None: try: from qca.engine import QCAEngine - self._qca = QCAEngine() except ImportError: self._qca = None return self._qca + @property + def causal(self): + """Lazy-load CausalEngine.""" + if self._causal is None: + self._causal = _lazy_causal_engine() + return self._causal + + @property + def fuad(self): + """Lazy-load FuadEngine.""" + if self._fuad is None: + self._fuad = _lazy_fuad_engine() + return self._fuad + async def reason( self, task: str, @@ -287,6 +320,35 @@ async def reason_to_completion( except Exception: pass + # Causal reasoning — analyze task through Pearl's 3-rung ladder + if self.causal and tool_calls: + try: + causal_result = self.causal.analyze_query(task, {}) + result["causal_analysis"] = { + "rung": causal_result.get("rung", 1), + "query_type": causal_result.get("type", "observation"), + "answer": str(causal_result.get("result", ""))[:300], + } + except Exception: + pass + + # Fu'ad conviction formation — evaluate evidence from tool calls + if self.fuad and tool_calls: + try: + sources = [tc["tool"] for tc in tool_calls if tc.get("result")] + if sources: + conviction = self.fuad.evaluate_evidence( + claim=task[:200], + sources=sources, + ) + result["conviction"] = { + "level": conviction.level.value, + "confidence": round(conviction.confidence, 3), + "source_count": conviction.source_count, + } + except Exception: + pass + return result def _build_initial_messages(self, task: str, context: dict = None) -> list[dict]: diff --git a/backend/reasoning/causal_engine.py b/backend/reasoning/causal_engine.py new file mode 100644 index 0000000..ce28a9a --- /dev/null +++ b/backend/reasoning/causal_engine.py @@ -0,0 +1,341 @@ +""" +Causal Engine (عِلَّة) — Pearl's Causal Ladder +================================================ + +"Does the human think that We will not assemble his bones? + Yes. [We are] Able [even] to proportion his fingertips." — Quran 75:3-4 + +Implements Judea Pearl's three rungs of causal reasoning: + Rung 1 — Observation (مشاهدة): P(Y|X) — "What is?" + Rung 2 — Intervention (تدخل): P(Y|do(X)) — "What if I do X?" + Rung 3 — Counterfactual (تفكر): P(Y_x|X', Y') — "What if I had done X?" + +Higher rungs enable deeper understanding of causality — +not just correlation, but actual cause-effect relationships. +""" + +import logging +from dataclasses import dataclass, field +from enum import Enum + +logger = logging.getLogger("mizan.causal") + + +class CausalRung(Enum): + OBSERVATION = 1 # Association — seeing patterns + INTERVENTION = 2 # Doing — what happens when I act + COUNTERFACTUAL = 3 # Imagining — what would have been + + +@dataclass +class CausalNode: + """A variable in the causal graph.""" + name: str + value: float = 0.5 # Normalised probability (0-1) + is_observed: bool = False + + +@dataclass +class CausalEdge: + """A directed causal link: source → target with strength.""" + source: str + target: str + strength: float = 0.5 # Causal strength (0-1) + + +@dataclass +class CausalModel: + """A structural causal model (SCM) built from observations.""" + nodes: dict[str, CausalNode] = field(default_factory=dict) + edges: list[CausalEdge] = field(default_factory=list) + + def parents_of(self, node_name: str) -> list[str]: + return [e.source for e in self.edges if e.target == node_name] + + def children_of(self, node_name: str) -> list[str]: + return [e.target for e in self.edges if e.source == node_name] + + +@dataclass +class ObservationResult: + """Rung 1 result: observed associations.""" + rung: CausalRung = CausalRung.OBSERVATION + associations: dict[str, float] = field(default_factory=dict) + model: CausalModel = field(default_factory=CausalModel) + summary: str = "" + + +@dataclass +class InterventionResult: + """Rung 2 result: effect of doing action.""" + rung: CausalRung = CausalRung.INTERVENTION + action: str = "" + predicted_effects: dict[str, float] = field(default_factory=dict) + confidence: float = 0.5 + summary: str = "" + + def to_dict(self) -> dict: + return { + "rung": self.rung.value, + "action": self.action, + "predicted_effects": self.predicted_effects, + "confidence": round(self.confidence, 3), + "summary": self.summary, + } + + +@dataclass +class CounterfactualResult: + """Rung 3 result: what would have happened.""" + rung: CausalRung = CausalRung.COUNTERFACTUAL + factual: str = "" + alternative: str = "" + counterfactual_outcome: str = "" + probability: float = 0.5 + summary: str = "" + + def to_dict(self) -> dict: + return { + "rung": self.rung.value, + "factual": self.factual, + "alternative": self.alternative, + "counterfactual_outcome": self.counterfactual_outcome, + "probability": round(self.probability, 3), + "summary": self.summary, + } + + +def _detect_causal_rung(text: str) -> CausalRung: + """ + Detect which causal rung a question requires from its phrasing. + Rung 3 indicators take priority, then Rung 2, else Rung 1. + """ + lower = text.lower() + rung3_signals = [ + "what if i had", "what would have", "if only", "had i", + "would have been", "could have", "should have", + ] + rung2_signals = [ + "what if i", "what happens if", "what would happen", + "if i do", "if i delete", "if i change", "if i run", + "what if we", "what if you", + ] + if any(s in lower for s in rung3_signals): + return CausalRung.COUNTERFACTUAL + if any(s in lower for s in rung2_signals): + return CausalRung.INTERVENTION + return CausalRung.OBSERVATION + + +class CausalEngine: + """ + Implements Pearl's three-rung causal ladder for agent reasoning. + + Usage: + engine = CausalEngine() + model = engine.observe({"database_size": 0.8, "response_time": 0.9}) + result = engine.intervene(model, "delete the database") + # → InterventionResult(predicted_effects={"response_time": 0.1, ...}) + """ + + def observe(self, data: dict[str, float]) -> ObservationResult: + """ + Rung 1: Build a causal model from observed co-occurrences. + Data is a dict of {variable_name: probability/value 0-1}. + + Uses correlation as a proxy for causal links (acknowledging the + correlation ≠ causation limitation — this is Rung 1, not Rung 2). + """ + model = CausalModel() + keys = list(data.keys()) + + # Create nodes + for k, v in data.items(): + model.nodes[k] = CausalNode(name=k, value=float(v), is_observed=True) + + # Create edges based on co-occurrence / correlation heuristics + # High-value nodes tend to "cause" downstream effects + for i, a in enumerate(keys): + for b in keys[i + 1:]: + va, vb = data[a], data[b] + # If both are high, assume possible causal relationship + if abs(va - vb) < 0.3: + strength = min(va, vb) + if strength > 0.3: + model.edges.append(CausalEdge(source=a, target=b, strength=strength)) + + associations = {k: round(v, 3) for k, v in data.items()} + summary = f"Observed {len(keys)} variables, identified {len(model.edges)} potential causal links." + + logger.debug("[CAUSAL] Rung 1: %s", summary) + return ObservationResult( + associations=associations, + model=model, + summary=summary, + ) + + def intervene(self, model: CausalModel, action: str) -> InterventionResult: + """ + Rung 2: do-calculus — predict effect of an intervention. + + Simulates "cutting" incoming edges to the action node (Pearl's do-operator) + and propagating the change downstream through the causal graph. + + action: natural language description of the intervention + """ + action_lower = action.lower() + + # Map common actions to their variable effects + # Format: {trigger_word: {variable: delta}} + _action_effects: dict[str, dict[str, float]] = { + "delete": {"data_availability": -0.9, "storage_usage": -0.8, "system_stability": -0.3}, + "restart": {"memory_usage": -0.5, "cpu_load": -0.3, "uptime": -0.7}, + "scale": {"throughput": 0.6, "cost": 0.5, "latency": -0.3}, + "cache": {"response_time": -0.4, "memory_usage": 0.3, "throughput": 0.4}, + "remove": {"resource_availability": -0.6, "dependency_count": -0.3}, + "add": {"capability": 0.5, "complexity": 0.3}, + "disable": {"availability": -0.7, "security_risk": -0.2}, + "enable": {"availability": 0.6, "attack_surface": 0.2}, + } + + predicted_effects: dict[str, float] = {} + matched_action = None + + for trigger, effects in _action_effects.items(): + if trigger in action_lower: + matched_action = trigger + # Apply direct effects + for var, delta in effects.items(): + predicted_effects[var] = max(0.0, min(1.0, 0.5 + delta)) + # Propagate to downstream nodes in model + for node_name in model.nodes: + for trigger_var in effects: + if trigger_var in node_name or node_name in trigger_var: + children = model.children_of(node_name) + for child in children: + if child not in predicted_effects: + edge = next( + (e for e in model.edges + if e.source == node_name and e.target == child), + None + ) + if edge: + propagated = effects[trigger_var] * edge.strength * 0.7 + predicted_effects[child] = max( + 0.0, min(1.0, model.nodes[child].value + propagated) + ) + break + + if not matched_action: + predicted_effects = {"uncertainty": 0.7} + + confidence = 0.6 if matched_action else 0.3 + summary = ( + f"do({action}): predicts effects on {len(predicted_effects)} variables. " + f"Confidence: {'moderate' if confidence >= 0.5 else 'low'} " + f"(Rung 2 do-calculus)." + ) + + logger.debug("[CAUSAL] Rung 2: action='%s' effects=%s", action[:60], predicted_effects) + return InterventionResult( + action=action, + predicted_effects={k: round(v, 3) for k, v in predicted_effects.items()}, + confidence=confidence, + summary=summary, + ) + + def counterfactual( + self, + model: CausalModel, + factual: str, + alternative: str, + ) -> CounterfactualResult: + """ + Rung 3: Counterfactual reasoning — what would have happened + under an alternative course of action. + + Uses abduction (infer hidden state from observed outcome), + then predicts what the outcome would have been under the alternative. + """ + # Step 1: Abduction — infer hidden exogenous factors from factual outcome + # (Simplified: use causal model to estimate prior state) + factual_result = self.intervene(model, factual) + alt_result = self.intervene(model, alternative) + + # Step 2: Prediction — compare outcomes + factual_effects = factual_result.predicted_effects + alt_effects = alt_result.predicted_effects + + # Compute what would have differed + diffs: dict[str, float] = {} + all_vars = set(factual_effects) | set(alt_effects) + for var in all_vars: + fa = factual_effects.get(var, 0.5) + aa = alt_effects.get(var, 0.5) + if abs(fa - aa) > 0.05: + diffs[var] = round(aa - fa, 3) + + if diffs: + changes = ", ".join( + f"{var}: {'+' if d >= 0 else ''}{d:.2f}" + for var, d in sorted(diffs.items(), key=lambda x: -abs(x[1]))[:5] + ) + counterfactual_outcome = f"If '{alternative}' instead of '{factual}': {changes}" + probability = alt_result.confidence * 0.8 # Counterfactuals are less certain + else: + counterfactual_outcome = ( + f"No significant difference between '{factual}' and '{alternative}'." + ) + probability = 0.5 + + summary = ( + f"Counterfactual (Rung 3): {counterfactual_outcome}. " + f"Probability estimate: {probability:.0%}." + ) + + logger.debug("[CAUSAL] Rung 3: factual='%s' alt='%s' prob=%.2f", + factual[:40], alternative[:40], probability) + return CounterfactualResult( + factual=factual, + alternative=alternative, + counterfactual_outcome=counterfactual_outcome, + probability=round(probability, 3), + summary=summary, + ) + + def analyze_query(self, query: str, data: dict[str, float] = None) -> dict: + """ + Auto-detect the required causal rung and run the appropriate analysis. + Entry point for agent tool use. + """ + rung = _detect_causal_rung(query) + data = data or {} + model = self.observe(data) if data else ObservationResult(model=CausalModel()) + + if rung == CausalRung.OBSERVATION: + return { + "rung": 1, + "type": "observation", + "result": model.summary, + "associations": model.associations, + } + elif rung == CausalRung.INTERVENTION: + result = self.intervene(model.model, query) + return { + "rung": 2, + "type": "intervention", + "result": result.summary, + **result.to_dict(), + } + else: + # Counterfactual: parse factual vs alternative from query + parts = query.lower().split("instead of") + alt = parts[0].strip() if parts else query + factual = parts[1].strip() if len(parts) > 1 else "current state" + result = self.counterfactual(model.model, factual, alt) + return { + "rung": 3, + "type": "counterfactual", + "result": result.summary, + **result.to_dict(), + } diff --git a/backend/requirements.txt b/backend/requirements.txt index 10ca75c..f4a9ae3 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -33,6 +33,10 @@ croniter>=2.0.7 click>=8.1.0 rich>=13.7.0 +# Knowledge Ingestion +pymupdf>=1.24.0 +youtube-transcript-api>=0.6.0 + # Optional: Gateway Channels # python-telegram-bot>=21.5 # discord.py>=2.3.2 diff --git a/backend/security/wali.py b/backend/security/wali.py index c223774..33d3670 100644 --- a/backend/security/wali.py +++ b/backend/security/wali.py @@ -30,8 +30,8 @@ class SecurityConfig: ) # Rate limiting - rate_limit_per_minute: int = 60 - rate_limit_burst: int = 10 + rate_limit_per_minute: int = 200 + rate_limit_burst: int = 30 ws_max_connections: int = 50 # File system sandbox diff --git a/backend/skills/builtin/data_analysis.py b/backend/skills/builtin/data_analysis.py index a66cf33..ceb1096 100644 --- a/backend/skills/builtin/data_analysis.py +++ b/backend/skills/builtin/data_analysis.py @@ -38,8 +38,11 @@ async def execute(self, params: dict, context: dict = None) -> dict: return await self.analyze_json(params.get("data", "")) return {"error": f"Unknown action: {action}"} - async def analyze_csv(self, data: str) -> dict: + async def analyze_csv(self, data: str | dict = "") -> dict: """Analyze CSV data""" + # Defensive: handle dict input from invoke system + if isinstance(data, dict): + data = data.get("data", "") try: reader = csv.DictReader(io.StringIO(data)) rows = list(reader) @@ -77,8 +80,11 @@ async def analyze_csv(self, data: str) -> dict: except Exception as e: return {"error": str(e)} - async def analyze_json(self, data: str) -> dict: + async def analyze_json(self, data: str | dict = "") -> dict: """Analyze JSON data""" + # Defensive: handle dict input from invoke system + if isinstance(data, dict) and "data" in data: + data = data.get("data", "") try: parsed = json.loads(data) diff --git a/backend/skills/builtin/linode_cloud.py b/backend/skills/builtin/linode_cloud.py new file mode 100644 index 0000000..8f170d1 --- /dev/null +++ b/backend/skills/builtin/linode_cloud.py @@ -0,0 +1,561 @@ +""" +Linode Cloud Skill (سماء — The Sky/Cloud) +========================================== + +"It is Allah who created the heavens and the earth and sent down rain +from the sky and produced thereby some fruits." — Quran 14:32 + +Purpose-built Linode integration with correct API endpoints, +auto-token loading, and high-level operations: +- Create/list/manage Linode instances +- Reset root passwords via API +- DNS management +- StackScript support +- Full provisioning workflows + +Uses the Linode API v4: https://api.linode.com/v4/ +""" + +import logging +import os +from datetime import UTC, datetime + +import httpx + +from ..base import SkillBase, SkillManifest + +logger = logging.getLogger("mizan.linode") + +# Linode API v4 base URL +LINODE_API_BASE = "https://api.linode.com/v4" + + +class LinodeCloudSkill(SkillBase): + """ + Linode-specific cloud management. + All endpoints use correct Linode API v4 paths. + """ + + manifest = SkillManifest( + name="linode_cloud", + version="1.0.0", + description="Linode cloud management: create/list instances, " + "reset passwords, manage DNS, run StackScripts. " + "Auto-loads LINODE_API_TOKEN from environment.", + permissions=[ + "network:https://api.linode.com/*", + "credentials:read", + ], + tags=["سماء", "Linode", "Cloud"], + ) + + def __init__(self, config: dict = None): + super().__init__(config) + self._token = os.environ.get("LINODE_API_TOKEN", "") + self._tools = { + "linode_list": self.list_instances, + "linode_get": self.get_instance, + "linode_create": self.create_instance, + "linode_delete": self.delete_instance, + "linode_reboot": self.reboot_instance, + "linode_reset_password": self.reset_password, + "linode_list_regions": self.list_regions, + "linode_list_types": self.list_types, + "linode_api": self.raw_api, + "linode_set_token": self.set_token, + } + + def _headers(self, token: str | None = None) -> dict: + """Build auth headers. Uses provided token or env default.""" + t = token or self._token + if not t: + raise ValueError( + "No Linode API token. Set LINODE_API_TOKEN env var " + "or call linode_set_token first." + ) + return { + "Authorization": f"Bearer {t}", + "Content-Type": "application/json", + } + + async def _request( + self, + method: str, + path: str, + body: dict | None = None, + token: str | None = None, + timeout: int = 30, + ) -> dict: + """Make a Linode API request with proper error handling.""" + url = f"{LINODE_API_BASE}{path}" + try: + headers = self._headers(token) + except ValueError as e: + return {"error": str(e)} + + try: + async with httpx.AsyncClient(timeout=timeout) as client: + if method == "GET": + resp = await client.get(url, headers=headers) + elif method == "POST": + resp = await client.post(url, headers=headers, json=body) + elif method == "PUT": + resp = await client.put(url, headers=headers, json=body) + elif method == "DELETE": + resp = await client.delete(url, headers=headers) + else: + return {"error": f"Unsupported method: {method}"} + + try: + data = resp.json() + except Exception: + data = resp.text[:3000] + + if resp.status_code >= 400: + errors = data.get("errors", []) if isinstance(data, dict) else [] + error_msgs = [e.get("reason", str(e)) for e in errors] + return { + "error": "; ".join(error_msgs) if error_msgs else f"HTTP {resp.status_code}", + "status_code": resp.status_code, + "details": data, + } + + return {"status_code": resp.status_code, "data": data} + + except httpx.TimeoutException: + return {"error": f"Request timed out after {timeout}s", "url": url} + except Exception as e: + return {"error": str(e)} + + # ── Token Management ──────────────────────────────────── + + async def set_token(self, params: dict) -> dict: + """Set the Linode API token for this session.""" + token = params.get("token", "") + if not token: + return {"error": "token is required"} + self._token = token + # Verify the token works + result = await self._request("GET", "/profile", token=token) + if result.get("error"): + self._token = "" + return {"error": f"Token validation failed: {result['error']}"} + profile = result.get("data", {}) + logger.info(f"[LINODE] Token set for user: {profile.get('username', 'unknown')}") + return { + "success": True, + "username": profile.get("username"), + "email": profile.get("email"), + "message": "Linode API token configured and verified.", + } + + # ── Instance Management ───────────────────────────────── + + async def list_instances(self, params: dict = None) -> dict: + """List all Linode instances.""" + result = await self._request("GET", "/linode/instances") + if result.get("error"): + return result + instances = result.get("data", {}).get("data", []) + return { + "count": len(instances), + "instances": [ + { + "id": i["id"], + "label": i.get("label", ""), + "status": i.get("status", ""), + "type": i.get("type", ""), + "region": i.get("region", ""), + "ipv4": i.get("ipv4", []), + "created": i.get("created", ""), + } + for i in instances + ], + } + + async def get_instance(self, params: dict) -> dict: + """Get details of a specific Linode instance.""" + linode_id = params.get("linode_id", "") + if not linode_id: + return {"error": "linode_id is required"} + result = await self._request("GET", f"/linode/instances/{linode_id}") + if result.get("error"): + return result + return {"instance": result.get("data", {})} + + async def create_instance(self, params: dict) -> dict: + """Create a new Linode instance. + + Required: region, type, image + Optional: root_pass (auto-generated if omitted), label, + authorized_keys, stackscript_id, stackscript_data + """ + region = params.get("region", "") + instance_type = params.get("type", "") + image = params.get("image", "") + + if not region or not instance_type or not image: + return { + "error": "region, type, and image are required", + "hint": { + "region": "e.g. us-east, us-west, eu-west, ap-south", + "type": "e.g. g6-nanode-1, g6-standard-2, g6-standard-4", + "image": "e.g. linode/ubuntu22.04, linode/ubuntu24.04, linode/debian12", + }, + } + + body = { + "region": region, + "type": instance_type, + "image": image, + } + + # Optional fields + if params.get("root_pass"): + body["root_pass"] = params["root_pass"] + if params.get("label"): + body["label"] = params["label"] + if params.get("authorized_keys"): + body["authorized_keys"] = params["authorized_keys"] + if params.get("stackscript_id"): + body["stackscript_id"] = params["stackscript_id"] + if params.get("stackscript_data"): + body["stackscript_data"] = params["stackscript_data"] + if params.get("tags"): + body["tags"] = params["tags"] + if params.get("booted") is not None: + body["booted"] = params["booted"] + + result = await self._request("POST", "/linode/instances", body=body, timeout=60) + if result.get("error"): + return result + + instance = result.get("data", {}) + return { + "created": True, + "instance": { + "id": instance.get("id"), + "label": instance.get("label"), + "status": instance.get("status"), + "type": instance.get("type"), + "region": instance.get("region"), + "ipv4": instance.get("ipv4", []), + "root_pass": instance.get("root_pass", params.get("root_pass", "(set by you)")), + }, + "message": f"Linode created! ID: {instance.get('id')}. " + f"IPs: {instance.get('ipv4', [])}. " + "Wait ~60s for boot, then SSH in or use ssh_copy_id.", + } + + async def delete_instance(self, params: dict) -> dict: + """Delete a Linode instance.""" + linode_id = params.get("linode_id", "") + if not linode_id: + return {"error": "linode_id is required"} + result = await self._request("DELETE", f"/linode/instances/{linode_id}") + if result.get("error"): + return result + return {"deleted": True, "linode_id": linode_id} + + async def reboot_instance(self, params: dict) -> dict: + """Reboot a Linode instance.""" + linode_id = params.get("linode_id", "") + if not linode_id: + return {"error": "linode_id is required"} + result = await self._request("POST", f"/linode/instances/{linode_id}/reboot") + if result.get("error"): + return result + return {"rebooted": True, "linode_id": linode_id} + + async def reset_password(self, params: dict) -> dict: + """Reset root password on a Linode instance. + + The instance must be powered off first. This tool handles that: + 1. Shuts down the instance + 2. Waits for it to be offline + 3. Resets the password + 4. Boots it back up + """ + linode_id = params.get("linode_id", "") + new_password = params.get("root_pass", "") + + if not linode_id or not new_password: + return {"error": "linode_id and root_pass are required"} + + if len(new_password) < 11: + return {"error": "Password must be at least 11 characters (Linode requirement)"} + + # Step 1: Shut down + shutdown = await self._request( + "POST", f"/linode/instances/{linode_id}/shutdown" + ) + if shutdown.get("error") and "already powered off" not in str(shutdown.get("error", "")).lower(): + return {"error": f"Shutdown failed: {shutdown.get('error')}"} + + # Step 2: Wait for offline status (poll up to 60s) + import asyncio + + for _ in range(12): + await asyncio.sleep(5) + status = await self._request("GET", f"/linode/instances/{linode_id}") + if status.get("data", {}).get("status") == "offline": + break + else: + return {"error": "Instance did not shut down within 60s. Try again."} + + # Step 3: Reset password + reset = await self._request( + "POST", + f"/linode/instances/{linode_id}/password", + body={"root_pass": new_password}, + ) + if reset.get("error"): + # Boot back up even if reset fails + await self._request("POST", f"/linode/instances/{linode_id}/boot") + return {"error": f"Password reset failed: {reset.get('error')}"} + + # Step 4: Boot back up + boot = await self._request("POST", f"/linode/instances/{linode_id}/boot") + + return { + "success": True, + "linode_id": linode_id, + "message": "Root password reset and instance rebooting. " + "Allow ~30s for boot before SSH.", + } + + # ── Reference Data ────────────────────────────────────── + + async def list_regions(self, params: dict = None) -> dict: + """List available Linode regions.""" + result = await self._request("GET", "/regions") + if result.get("error"): + return result + regions = result.get("data", {}).get("data", []) + return { + "regions": [ + { + "id": r["id"], + "label": r.get("label", ""), + "country": r.get("country", ""), + "status": r.get("status", ""), + } + for r in regions + ], + } + + async def list_types(self, params: dict = None) -> dict: + """List available Linode instance types (plans).""" + result = await self._request("GET", "/linode/types") + if result.get("error"): + return result + types = result.get("data", {}).get("data", []) + return { + "types": [ + { + "id": t["id"], + "label": t.get("label", ""), + "vcpus": t.get("vcpus"), + "memory": t.get("memory"), + "disk": t.get("disk"), + "price_monthly": t.get("price", {}).get("monthly"), + } + for t in types[:20] + ], + } + + # ── Raw API (escape hatch) ────────────────────────────── + + async def raw_api(self, params: dict) -> dict: + """Make a raw Linode API call with any path. + + For advanced operations not covered by dedicated tools. + Path is relative to https://api.linode.com/v4/ + """ + method = params.get("method", "GET").upper() + path = params.get("path", "") + body = params.get("body") + + if not path: + return {"error": "path is required (e.g. /linode/instances)"} + + # Normalize path + if not path.startswith("/"): + path = f"/{path}" + + return await self._request(method, path, body=body) + + # ── Execute (generic entry) ───────────────────────────── + + async def execute(self, params: dict, context: dict = None) -> dict: + action = params.get("action", "list") + handler = self._tools.get(f"linode_{action}") + if handler: + return await handler(params) + return {"error": f"Unknown action: {action}. Available: {list(self._tools.keys())}"} + + # ── Tool Schemas ──────────────────────────────────────── + + def get_tool_schemas(self) -> list[dict]: + return [ + { + "name": "linode_set_token", + "description": "Set Linode API token for authentication. Auto-loads from LINODE_API_TOKEN env var if set.", + "input_schema": { + "type": "object", + "properties": { + "token": { + "type": "string", + "description": "Linode Personal Access Token", + }, + }, + "required": ["token"], + }, + }, + { + "name": "linode_list", + "description": "List all Linode instances with ID, label, status, region, and IPs.", + "input_schema": {"type": "object", "properties": {}}, + }, + { + "name": "linode_get", + "description": "Get full details of a specific Linode instance by ID.", + "input_schema": { + "type": "object", + "properties": { + "linode_id": { + "type": "integer", + "description": "Linode instance ID", + }, + }, + "required": ["linode_id"], + }, + }, + { + "name": "linode_create", + "description": "Create a new Linode instance. Requires region, type, and image. " + "Common types: g6-nanode-1, g6-standard-2. " + "Common images: linode/ubuntu22.04, linode/ubuntu24.04. " + "Common regions: us-east, us-west, eu-west.", + "input_schema": { + "type": "object", + "properties": { + "region": { + "type": "string", + "description": "Region ID (e.g. us-east)", + }, + "type": { + "type": "string", + "description": "Instance type/plan (e.g. g6-standard-2)", + }, + "image": { + "type": "string", + "description": "OS image (e.g. linode/ubuntu22.04)", + }, + "root_pass": { + "type": "string", + "description": "Root password (min 11 chars). Auto-generated if omitted.", + }, + "label": { + "type": "string", + "description": "Display label for the instance", + }, + "authorized_keys": { + "type": "array", + "items": {"type": "string"}, + "description": "SSH public keys to install", + }, + "stackscript_id": { + "type": "integer", + "description": "StackScript ID for automated setup", + }, + "tags": { + "type": "array", + "items": {"type": "string"}, + "description": "Instance tags", + }, + }, + "required": ["region", "type", "image"], + }, + }, + { + "name": "linode_delete", + "description": "Delete a Linode instance by ID. This is irreversible.", + "input_schema": { + "type": "object", + "properties": { + "linode_id": { + "type": "integer", + "description": "Linode instance ID to delete", + }, + }, + "required": ["linode_id"], + }, + }, + { + "name": "linode_reboot", + "description": "Reboot a Linode instance.", + "input_schema": { + "type": "object", + "properties": { + "linode_id": { + "type": "integer", + "description": "Linode instance ID to reboot", + }, + }, + "required": ["linode_id"], + }, + }, + { + "name": "linode_reset_password", + "description": "Reset root password on a Linode. Handles shutdown, password reset, and reboot automatically. " + "Password must be at least 11 characters.", + "input_schema": { + "type": "object", + "properties": { + "linode_id": { + "type": "integer", + "description": "Linode instance ID", + }, + "root_pass": { + "type": "string", + "description": "New root password (min 11 chars)", + }, + }, + "required": ["linode_id", "root_pass"], + }, + }, + { + "name": "linode_list_regions", + "description": "List available Linode regions.", + "input_schema": {"type": "object", "properties": {}}, + }, + { + "name": "linode_list_types", + "description": "List available Linode instance types/plans with pricing.", + "input_schema": {"type": "object", "properties": {}}, + }, + { + "name": "linode_api", + "description": "Make a raw Linode API call. Path is relative to https://api.linode.com/v4. " + "For advanced operations not covered by other linode_ tools.", + "input_schema": { + "type": "object", + "properties": { + "method": { + "type": "string", + "enum": ["GET", "POST", "PUT", "DELETE"], + }, + "path": { + "type": "string", + "description": "API path (e.g. /linode/instances, /domains)", + }, + "body": { + "type": "object", + "description": "Request body for POST/PUT", + }, + }, + "required": ["path"], + }, + }, + ] diff --git a/backend/skills/builtin/ssh_remote.py b/backend/skills/builtin/ssh_remote.py new file mode 100644 index 0000000..8e40bb3 --- /dev/null +++ b/backend/skills/builtin/ssh_remote.py @@ -0,0 +1,716 @@ +""" +Jisr Remote Execution Skill (جِسْر — The Bridge) +================================================== + +"And We made from water every living thing" — Quran 21:30 + +Jisr (Bridge) connects MIZAN to remote servers via SSH: +- Execute commands on remote servers +- Transfer files (upload/download) +- Multi-server orchestration +- Server health monitoring +- Automated setup scripts + +Uses subprocess ssh/scp for maximum compatibility (no paramiko needed). +Falls back to paramiko if available for persistent connections. +""" + +import logging +import os +import subprocess +import tempfile +from dataclasses import dataclass, field +from datetime import UTC, datetime + +from ..base import SkillBase, SkillManifest + +logger = logging.getLogger("mizan.jisr") + + +@dataclass +class ServerConnection: + """A registered remote server""" + + id: str = "" + host: str = "" + port: int = 22 + user: str = "root" + key_path: str | None = None + password: str | None = None + label: str = "" + status: str = "unknown" + last_connected: str | None = None + + def to_dict(self) -> dict: + return { + "id": self.id, + "host": self.host, + "port": self.port, + "user": self.user, + "label": self.label, + "status": self.status, + "has_key": bool(self.key_path), + "last_connected": self.last_connected, + } + + +class JisrRemoteSkill(SkillBase): + """ + Jisr — Remote Server Bridge + SSH execution, file transfer, and server management. + """ + + manifest = SkillManifest( + name="jisr_remote", + version="1.0.0", + description="Remote server management via SSH. Execute commands, " + "transfer files, run setup scripts on remote servers.", + permissions=[ + "network:ssh:*", + "shell:ssh", + "shell:scp", + "shell:sshpass", + ], + tags=["جسر", "Remote"], + ) + + # Default key location inside the container + DEFAULT_KEY_DIR = "/data/.ssh" + DEFAULT_KEY_PATH = "/data/.ssh/mizan_id_ed25519" + + def __init__(self, config: dict = None): + super().__init__(config) + self.servers: dict[str, ServerConnection] = {} + self._tools = { + "ssh_register_server": self.register_server, + "ssh_exec": self.ssh_exec, + "ssh_exec_script": self.ssh_exec_script, + "ssh_upload": self.ssh_upload, + "ssh_download": self.ssh_download, + "ssh_check_server": self.check_server, + "ssh_list_servers": self.list_servers, + "ssh_keygen": self.ssh_keygen, + "ssh_copy_id": self.ssh_copy_id, + } + + async def execute(self, params: dict, context: dict = None) -> dict: + action = params.get("action", "list_servers") + handler = self._tools.get(f"ssh_{action}") + if handler: + return await handler(params) + return {"error": f"Unknown SSH action: {action}"} + + def _build_ssh_cmd(self, server: ServerConnection, command: str) -> list[str]: + """Build the SSH command with proper options.""" + cmd = ["ssh"] + cmd.extend(["-o", "StrictHostKeyChecking=no"]) + cmd.extend(["-o", "UserKnownHostsFile=/dev/null"]) + cmd.extend(["-o", "ConnectTimeout=10"]) + cmd.extend(["-o", "LogLevel=ERROR"]) + cmd.extend(["-p", str(server.port)]) + + if server.key_path: + cmd.extend(["-i", server.key_path]) + + cmd.append(f"{server.user}@{server.host}") + cmd.append(command) + return cmd + + def _build_scp_cmd( + self, server: ServerConnection, local: str, remote: str, upload: bool = True + ) -> list[str]: + """Build the SCP command.""" + cmd = ["scp"] + cmd.extend(["-o", "StrictHostKeyChecking=no"]) + cmd.extend(["-o", "UserKnownHostsFile=/dev/null"]) + cmd.extend(["-o", "ConnectTimeout=10"]) + cmd.extend(["-P", str(server.port)]) + + if server.key_path: + cmd.extend(["-i", server.key_path]) + + remote_path = f"{server.user}@{server.host}:{remote}" + if upload: + cmd.extend([local, remote_path]) + else: + cmd.extend([remote_path, local]) + return cmd + + def _wrap_with_sshpass(self, cmd: list[str], password: str) -> list[str]: + """Wrap command with sshpass for password auth.""" + return ["sshpass", "-p", password] + cmd + + def _get_server(self, params: dict) -> ServerConnection | None: + """Resolve server from params (by id or host).""" + server_id = params.get("server_id", "") + host = params.get("host", "") + + if server_id and server_id in self.servers: + return self.servers[server_id] + + # Try by host + for srv in self.servers.values(): + if srv.host == host or srv.label == host: + return srv + + # Auto-create from params if host provided + if host: + srv = ServerConnection( + id=host, + host=host, + port=params.get("port", 22), + user=params.get("user", "root"), + key_path=params.get("key_path"), + password=params.get("password"), + label=params.get("label", host), + ) + self.servers[srv.id] = srv + return srv + + return None + + async def register_server(self, params: dict) -> dict: + """Register a remote server for SSH access.""" + host = params.get("host", "") + if not host: + return {"error": "host is required"} + + import uuid + + server_id = str(uuid.uuid4())[:8] + server = ServerConnection( + id=server_id, + host=host, + port=params.get("port", 22), + user=params.get("user", "root"), + key_path=params.get("key_path"), + password=params.get("password"), + label=params.get("label", host), + ) + self.servers[server_id] = server + logger.info(f"[JISR] Server registered: {server.label} ({server.host})") + return { + "registered": True, + "server": server.to_dict(), + "message": f"Server '{server.label}' registered. Use server_id='{server_id}' for commands.", + } + + async def ssh_exec(self, params: dict) -> dict: + """Execute a command on a remote server via SSH.""" + server = self._get_server(params) + if not server: + return {"error": "Server not found. Register with ssh_register_server first, or provide 'host' in params."} + + command = params.get("command", "") + if not command: + return {"error": "command is required"} + + timeout = min(params.get("timeout", 60), 300) + + ssh_cmd = self._build_ssh_cmd(server, command) + if server.password: + ssh_cmd = self._wrap_with_sshpass(ssh_cmd, server.password) + + try: + result = subprocess.run( + ssh_cmd, + capture_output=True, + text=True, + timeout=timeout, + ) + server.status = "connected" if result.returncode == 0 else "error" + server.last_connected = datetime.now(UTC).isoformat() + return { + "stdout": result.stdout[:20000], + "stderr": result.stderr[:5000], + "returncode": result.returncode, + "server": server.host, + "success": result.returncode == 0, + } + except subprocess.TimeoutExpired: + server.status = "timeout" + return {"error": f"SSH command timed out after {timeout}s", "server": server.host} + except FileNotFoundError: + return { + "error": "SSH client not found. Install openssh-client.", + "hint": "apt-get install -y openssh-client sshpass", + } + except Exception as e: + server.status = "error" + return {"error": str(e), "server": server.host} + + async def ssh_exec_script(self, params: dict) -> dict: + """ + Execute a multi-line script on a remote server. + Writes script to temp file, uploads and executes it. + """ + server = self._get_server(params) + if not server: + return {"error": "Server not found. Register first or provide 'host'."} + + script = params.get("script", "") + if not script: + return {"error": "script content is required"} + + interpreter = params.get("interpreter", "/bin/bash") + timeout = min(params.get("timeout", 300), 600) + + # Write script to temp file + with tempfile.NamedTemporaryFile( + mode="w", suffix=".sh", delete=False, prefix="mizan_" + ) as tmp: + tmp.write(f"#!/bin/bash\nset -e\n{script}") + tmp_path = tmp.name + + try: + # Upload script + remote_script = "/tmp/mizan_remote_script.sh" + scp_cmd = self._build_scp_cmd(server, tmp_path, remote_script, upload=True) + if server.password: + scp_cmd = self._wrap_with_sshpass(scp_cmd, server.password) + + upload_result = subprocess.run( + scp_cmd, capture_output=True, text=True, timeout=30 + ) + if upload_result.returncode != 0: + return { + "error": f"Failed to upload script: {upload_result.stderr[:500]}", + "server": server.host, + } + + # Execute it + exec_cmd = f"chmod +x {remote_script} && {interpreter} {remote_script}" + ssh_cmd = self._build_ssh_cmd(server, exec_cmd) + if server.password: + ssh_cmd = self._wrap_with_sshpass(ssh_cmd, server.password) + + result = subprocess.run( + ssh_cmd, capture_output=True, text=True, timeout=timeout + ) + + # Cleanup remote script + cleanup_cmd = self._build_ssh_cmd(server, f"rm -f {remote_script}") + if server.password: + cleanup_cmd = self._wrap_with_sshpass(cleanup_cmd, server.password) + subprocess.run(cleanup_cmd, capture_output=True, timeout=10) + + server.status = "connected" if result.returncode == 0 else "error" + server.last_connected = datetime.now(UTC).isoformat() + return { + "stdout": result.stdout[:20000], + "stderr": result.stderr[:5000], + "returncode": result.returncode, + "server": server.host, + "success": result.returncode == 0, + } + except subprocess.TimeoutExpired: + return {"error": f"Script execution timed out after {timeout}s"} + except Exception as e: + return {"error": str(e)} + finally: + os.unlink(tmp_path) + + async def ssh_upload(self, params: dict) -> dict: + """Upload a file to a remote server.""" + server = self._get_server(params) + if not server: + return {"error": "Server not found."} + + local_path = params.get("local_path", "") + remote_path = params.get("remote_path", "") + + # Support inline content → write to temp then upload + content = params.get("content", "") + if content and not local_path: + with tempfile.NamedTemporaryFile( + mode="w", delete=False, prefix="mizan_upload_" + ) as tmp: + tmp.write(content) + local_path = tmp.name + + if not local_path or not remote_path: + return {"error": "local_path (or content) and remote_path required"} + + scp_cmd = self._build_scp_cmd(server, local_path, remote_path, upload=True) + if server.password: + scp_cmd = self._wrap_with_sshpass(scp_cmd, server.password) + + try: + result = subprocess.run( + scp_cmd, capture_output=True, text=True, timeout=120 + ) + # Clean up temp file if we created one from content + if content: + os.unlink(local_path) + + return { + "success": result.returncode == 0, + "remote_path": remote_path, + "server": server.host, + "error": result.stderr[:500] if result.returncode != 0 else None, + } + except Exception as e: + return {"error": str(e)} + + async def ssh_download(self, params: dict) -> dict: + """Download a file from a remote server.""" + server = self._get_server(params) + if not server: + return {"error": "Server not found."} + + remote_path = params.get("remote_path", "") + local_path = params.get("local_path", "") + if not remote_path: + return {"error": "remote_path required"} + if not local_path: + local_path = f"/tmp/mizan_download_{os.path.basename(remote_path)}" + + scp_cmd = self._build_scp_cmd(server, local_path, remote_path, upload=False) + if server.password: + scp_cmd = self._wrap_with_sshpass(scp_cmd, server.password) + + try: + result = subprocess.run( + scp_cmd, capture_output=True, text=True, timeout=120 + ) + return { + "success": result.returncode == 0, + "local_path": local_path, + "server": server.host, + "error": result.stderr[:500] if result.returncode != 0 else None, + } + except Exception as e: + return {"error": str(e)} + + async def check_server(self, params: dict) -> dict: + """Check if a server is reachable.""" + server = self._get_server(params) + if not server: + return {"error": "Server not found."} + + # Quick SSH connectivity test + ssh_cmd = self._build_ssh_cmd(server, "echo 'MIZAN_PING_OK' && uname -a && uptime") + if server.password: + ssh_cmd = self._wrap_with_sshpass(ssh_cmd, server.password) + + try: + result = subprocess.run( + ssh_cmd, capture_output=True, text=True, timeout=15 + ) + if result.returncode == 0 and "MIZAN_PING_OK" in result.stdout: + server.status = "connected" + server.last_connected = datetime.now(UTC).isoformat() + lines = result.stdout.strip().split("\n") + return { + "reachable": True, + "server": server.host, + "system": lines[1] if len(lines) > 1 else "", + "uptime": lines[2] if len(lines) > 2 else "", + } + server.status = "error" + return { + "reachable": False, + "server": server.host, + "error": result.stderr[:500], + } + except subprocess.TimeoutExpired: + server.status = "timeout" + return {"reachable": False, "server": server.host, "error": "Connection timed out"} + except Exception as e: + return {"reachable": False, "server": server.host, "error": str(e)} + + async def list_servers(self, params: dict = None) -> dict: + """List all registered servers.""" + return { + "servers": [s.to_dict() for s in self.servers.values()], + "count": len(self.servers), + } + + async def ssh_keygen(self, params: dict = None) -> dict: + """Generate an SSH key pair for MIZAN to use for passwordless auth. + + Creates an ed25519 key at /data/.ssh/mizan_id_ed25519. + If key already exists, returns the existing public key. + """ + params = params or {} + key_path = params.get("key_path", self.DEFAULT_KEY_PATH) + key_dir = os.path.dirname(key_path) + pub_path = f"{key_path}.pub" + + # Return existing key if present + if os.path.exists(pub_path): + with open(pub_path) as f: + pub_key = f.read().strip() + return { + "exists": True, + "key_path": key_path, + "public_key": pub_key, + "message": "Key already exists. Use ssh_copy_id to install on a server.", + } + + # Create directory + os.makedirs(key_dir, mode=0o700, exist_ok=True) + + # Generate key pair + try: + result = subprocess.run( + [ + "ssh-keygen", + "-t", "ed25519", + "-f", key_path, + "-N", "", # No passphrase + "-C", "mizan-agent", + ], + capture_output=True, + text=True, + timeout=15, + ) + if result.returncode != 0: + return {"error": f"ssh-keygen failed: {result.stderr[:500]}"} + + # Lock down permissions + os.chmod(key_path, 0o600) + os.chmod(pub_path, 0o644) + + with open(pub_path) as f: + pub_key = f.read().strip() + + logger.info(f"[JISR] SSH key pair generated at {key_path}") + return { + "generated": True, + "key_path": key_path, + "public_key": pub_key, + "message": "Key pair generated. Use ssh_copy_id to install on a server.", + } + except FileNotFoundError: + return {"error": "ssh-keygen not found. Install openssh-client."} + except Exception as e: + return {"error": str(e)} + + async def ssh_copy_id(self, params: dict) -> dict: + """Install MIZAN's public key on a remote server for passwordless access. + + Requires either: + - A registered server with password auth, OR + - host + password in params + + After success, updates the server entry to use key auth. + """ + server = self._get_server(params) + if not server: + return {"error": "Server not found. Provide 'host' and 'password'."} + + if not server.password: + password = params.get("password", "") + if not password: + return { + "error": "Password required to copy SSH key. " + "Provide 'password' in params or register server with password first." + } + server.password = password + + key_path = params.get("key_path", self.DEFAULT_KEY_PATH) + pub_path = f"{key_path}.pub" + + # Generate key if it doesn't exist + if not os.path.exists(pub_path): + gen_result = await self.ssh_keygen({"key_path": key_path}) + if gen_result.get("error"): + return gen_result + + with open(pub_path) as f: + pub_key = f.read().strip() + + # Copy public key to remote server's authorized_keys + install_cmd = ( + f"mkdir -p ~/.ssh && chmod 700 ~/.ssh && " + f"echo '{pub_key}' >> ~/.ssh/authorized_keys && " + f"chmod 600 ~/.ssh/authorized_keys && " + f"echo 'MIZAN_KEY_INSTALLED'" + ) + + ssh_cmd = self._build_ssh_cmd(server, install_cmd) + ssh_cmd = self._wrap_with_sshpass(ssh_cmd, server.password) + + try: + result = subprocess.run( + ssh_cmd, + capture_output=True, + text=True, + timeout=30, + ) + + if result.returncode == 0 and "MIZAN_KEY_INSTALLED" in result.stdout: + # Update server to use key auth going forward + server.key_path = key_path + old_password = server.password + server.password = None + server.status = "connected" + server.last_connected = datetime.now(UTC).isoformat() + + # Verify key auth works + verify_cmd = self._build_ssh_cmd(server, "echo 'MIZAN_KEY_AUTH_OK'") + verify = subprocess.run( + verify_cmd, capture_output=True, text=True, timeout=15 + ) + + if verify.returncode == 0 and "MIZAN_KEY_AUTH_OK" in verify.stdout: + logger.info( + f"[JISR] SSH key installed and verified on {server.host}" + ) + return { + "success": True, + "server": server.host, + "key_path": key_path, + "message": f"SSH key auth configured for {server.host}. Password no longer needed.", + } + else: + # Key auth failed, restore password + server.password = old_password + server.key_path = None + return { + "success": False, + "server": server.host, + "error": "Key was installed but verification failed. Password auth still active.", + "stderr": verify.stderr[:500], + } + + return { + "success": False, + "server": server.host, + "error": f"Failed to install key: {result.stderr[:500]}", + } + except subprocess.TimeoutExpired: + return {"error": "Connection timed out during key installation"} + except FileNotFoundError: + return {"error": "sshpass not found. Install: apt-get install sshpass"} + except Exception as e: + return {"error": str(e)} + + def get_tool_schemas(self) -> list[dict]: + return [ + { + "name": "ssh_register_server", + "description": "Register a remote server for SSH access. Store connection details for reuse.", + "input_schema": { + "type": "object", + "properties": { + "host": {"type": "string", "description": "Server IP or hostname"}, + "port": {"type": "integer", "description": "SSH port", "default": 22}, + "user": {"type": "string", "description": "SSH username", "default": "root"}, + "key_path": {"type": "string", "description": "Path to SSH private key file"}, + "password": {"type": "string", "description": "SSH password (if no key)"}, + "label": {"type": "string", "description": "Friendly name for this server"}, + }, + "required": ["host"], + }, + }, + { + "name": "ssh_exec", + "description": "Execute a single command on a remote server via SSH.", + "input_schema": { + "type": "object", + "properties": { + "server_id": {"type": "string", "description": "Registered server ID"}, + "host": {"type": "string", "description": "Server host (if not registered)"}, + "user": {"type": "string", "description": "SSH user (default: root)"}, + "password": {"type": "string", "description": "SSH password"}, + "key_path": {"type": "string", "description": "SSH key path"}, + "command": {"type": "string", "description": "Command to execute"}, + "timeout": {"type": "integer", "description": "Timeout in seconds", "default": 60}, + }, + "required": ["command"], + }, + }, + { + "name": "ssh_exec_script", + "description": "Execute a multi-line bash script on a remote server. Uploads and runs the script.", + "input_schema": { + "type": "object", + "properties": { + "server_id": {"type": "string", "description": "Registered server ID"}, + "host": {"type": "string", "description": "Server host"}, + "user": {"type": "string", "description": "SSH user"}, + "password": {"type": "string", "description": "SSH password"}, + "script": {"type": "string", "description": "Multi-line bash script to execute"}, + "interpreter": {"type": "string", "description": "Script interpreter", "default": "/bin/bash"}, + "timeout": {"type": "integer", "description": "Timeout in seconds", "default": 300}, + }, + "required": ["script"], + }, + }, + { + "name": "ssh_upload", + "description": "Upload a file or content to a remote server via SCP.", + "input_schema": { + "type": "object", + "properties": { + "server_id": {"type": "string", "description": "Registered server ID"}, + "host": {"type": "string", "description": "Server host"}, + "local_path": {"type": "string", "description": "Local file path to upload"}, + "content": {"type": "string", "description": "File content to upload (alternative to local_path)"}, + "remote_path": {"type": "string", "description": "Remote destination path"}, + }, + "required": ["remote_path"], + }, + }, + { + "name": "ssh_download", + "description": "Download a file from a remote server via SCP.", + "input_schema": { + "type": "object", + "properties": { + "server_id": {"type": "string", "description": "Registered server ID"}, + "host": {"type": "string", "description": "Server host"}, + "remote_path": {"type": "string", "description": "Remote file path"}, + "local_path": {"type": "string", "description": "Local destination path"}, + }, + "required": ["remote_path"], + }, + }, + { + "name": "ssh_check_server", + "description": "Check if a remote server is reachable via SSH.", + "input_schema": { + "type": "object", + "properties": { + "server_id": {"type": "string", "description": "Registered server ID"}, + "host": {"type": "string", "description": "Server host"}, + }, + }, + }, + { + "name": "ssh_list_servers", + "description": "List all registered remote servers.", + "input_schema": {"type": "object", "properties": {}}, + }, + { + "name": "ssh_keygen", + "description": "Generate an SSH key pair for MIZAN. Creates ed25519 key at /data/.ssh/mizan_id_ed25519. Returns public key.", + "input_schema": { + "type": "object", + "properties": { + "key_path": { + "type": "string", + "description": "Custom key file path (default: /data/.ssh/mizan_id_ed25519)", + }, + }, + }, + }, + { + "name": "ssh_copy_id", + "description": "Install MIZAN's SSH public key on a remote server for passwordless access. Requires password for first-time setup.", + "input_schema": { + "type": "object", + "properties": { + "server_id": {"type": "string", "description": "Registered server ID"}, + "host": {"type": "string", "description": "Server IP or hostname"}, + "port": {"type": "integer", "description": "SSH port", "default": 22}, + "user": {"type": "string", "description": "SSH username", "default": "root"}, + "password": {"type": "string", "description": "Current server password (required for first-time key setup)"}, + "key_path": {"type": "string", "description": "SSH key path (default: /data/.ssh/mizan_id_ed25519)"}, + }, + "required": ["password"], + }, + }, + ] diff --git a/backend/skills/builtin/web_browse.py b/backend/skills/builtin/web_browse.py index 59cd6bf..fb3412f 100644 --- a/backend/skills/builtin/web_browse.py +++ b/backend/skills/builtin/web_browse.py @@ -60,8 +60,11 @@ async def execute(self, params: dict, context: dict = None) -> dict: return await self.search(params.get("query", "")) return {"error": f"Unknown action: {action}"} - async def browse(self, url: str) -> dict: + async def browse(self, url: str | dict = "") -> dict: """Browse a URL and extract text content""" + # Defensive: handle dict input from invoke system + if isinstance(url, dict): + url = url.get("url", "") try: async with httpx.AsyncClient(timeout=30, follow_redirects=True) as client: response = await client.get( @@ -80,11 +83,14 @@ async def browse(self, url: str) -> dict: except Exception as e: return {"error": str(e), "url": url} - async def search(self, query: str) -> dict: + async def search(self, query: str | dict = "") -> dict: """Search the web using DuckDuckGo""" import urllib.parse - encoded = urllib.parse.quote(query) + # Defensive: handle dict input from invoke system + if isinstance(query, dict): + query = query.get("query", "") + encoded = urllib.parse.quote(str(query)) url = f"https://duckduckgo.com/html/?q={encoded}" try: diff --git a/docker-compose.yml b/docker-compose.yml index cc47f9a..782fc66 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,6 +18,8 @@ services: - OLLAMA_URL=${OLLAMA_URL:-http://ollama:11434} - DEFAULT_MODEL=${DEFAULT_MODEL:-claude-opus-4-6} - SECRET_KEY=${SECRET_KEY:?SECRET_KEY environment variable is required} + - DB_PATH=/data/mizan_memory.db + - LINODE_API_TOKEN=${LINODE_API_TOKEN:-} volumes: - mizan-data:/data - mizan-plugins:/app/plugins diff --git a/docker/Dockerfile.backend b/docker/Dockerfile.backend index e5d6f9f..3a9105d 100644 --- a/docker/Dockerfile.backend +++ b/docker/Dockerfile.backend @@ -4,7 +4,7 @@ WORKDIR /app # Install system deps RUN apt-get update && apt-get install -y \ - curl git build-essential \ + curl git build-essential openssh-client sshpass \ && rm -rf /var/lib/apt/lists/* COPY requirements.txt . diff --git a/docs/index.html b/docs/index.html index 37f9025..5ca01cd 100644 --- a/docs/index.html +++ b/docs/index.html @@ -249,7 +249,7 @@

MIZAN

Architecture
System Design - Cognitive Architecture + QALB-7 Architecture Project Structure @@ -1046,24 +1046,131 @@

Docker Deployments

-

Quranic Cognitive Architecture (QCA)

-

MIZAN's reasoning engine is built on a 7-layer cognitive architecture inspired by Quranic epistemology.

+

QALB-7 Cognitive Architecture

+

Every MIZAN agent processes tasks through a 7-layer cognitive pipeline inspired by Quranic psychology. Each layer transforms the task before it reaches the LLM, and post-processes the response.

-

Cognitive Layers

+

QALB-7 Pipeline

+

Each layer runs in sequence on every agent task:

- + - - - - - - - - + + + + + + +
LayerArabicFunction
#ModuleArabicPurposeFile
Sam'سمعSequential temporal input processing
BasarبصرStructural pattern recognition
Fu'adفؤادIntegration engine combining inputs
ISMاسمDeep semantic root-space representation
MizanميزانEpistemic weighting and truth calibration
'AqlعقلTyped relationship binding engine
Lawhلوح4-tier hierarchical memory
FurqanفرقانDiscrimination and articulation output
1FitrahفطرةInnate ethical guardrails (NO_HARM, TRUTH, JUSTICE)core/fitrah.py
2Nafs TriadنفسThree competing inner voices deliberate on approachcore/nafs_triad.py
3Qalb ProcessorقلبCardiac oscillation — modulates LLM temperature & token limitscore/qalb_processor.py
4Fu'adفؤادBayesian conviction engine — evidence accumulationcore/fuad.py
5LubbلبّMetacognition — compress, check coherence, detect biascore/lubb.py
6Developmental GateأطوارProgressive capability gating (7 stages)core/developmental_stages.py
7Causal EngineسببيةPearl's causal ladder (observe/intervene/counterfactual)reasoning/causal_engine.py
+

Layer 1: Fitrah — Innate Ethics

+

Before any tool executes, Fitrah checks three non-negotiable axioms:

+
    +
  • NO_HARM — Blocks destructive commands (rm -rf, DROP TABLE)
    • +
  • TRUTH — Prevents fabricating data or claiming certainty without evidence
    • +
  • JUSTICE — Guards against disproportionate impact actions
    • +
+

On initialization, Fitrah axioms are loaded into QCA Lawh Tier 1 as immutable truths with certainty 1.0.

+ +

Layer 2: Nafs Triad — Inner Deliberation

+

Three competing "inner voices" deliberate on every task. The winner injects behavioral guidance into the system prompt:

+ + + + + + + +
VoiceArabicBiasBehavior
AmmaraأمّارةDrivePush to act quickly, take risks
LawwamaلوّامةCautionQuestion assumptions, double-check
MutmainnaمطمئنةBalanceSeek harmony, measured approach
+

Voice weights shift with the agent's Nafs level: Level 1-2 Ammara dominates (drive), Level 3-4 Lawwama dominates (critical), Level 5-7 Mutmainna dominates (wise).

+ +

Layer 3: Qalb Processor — Cardiac Oscillation

+

The Qalb models a "heartbeat" that alternates between contraction and expansion, directly modulating LLM parameters:

+ + + + + + + +
StateArabicTemperatureMax TokensWhen
Qabdقبض0.32048Focused, analytical tasks
Bastبسط0.84096Creative, exploratory tasks
Khushuخشوع0.26144Deep focus (nafs ≥ 4 + complex task)
+

The oscillation phase advances 0.02 per interaction. Emotional overrides: Frustrated → force Qabd; Positive → allow Bast.

+ +

Layer 4: Fu'ad — Conviction Formation

+

Bayesian evidence accumulation across three conviction levels:

+ + + + + + + +
LevelRequirementConfidence
ImpressionSingle source0.0 – 0.50
Belief2+ independent sources0.50 – 0.78
Conviction3+ sources + temporal consistency0.78 – 1.0
+

Each source closes 35% of the remaining confidence gap: confidence = 1 - (0.65 ^ source_count)

+ +

Layer 5: Lubb — Metacognition

+

After response generation, Lubb performs three evaluations:

+
    +
  • Compress — Information Bottleneck: keeps decisions + tool results + key facts (~20% of input)
    • +
  • Check Coherence — Verify conclusions follow from premises (score 0.0–1.0)
    • +
  • Detect Bias — Flag confirmation bias, anchoring, availability bias
    • +
+

Output: quality label (confident | hedged | uncertain), coherence score, and bias flags. If coherence < 0.5, a caveat is appended to the response.

+ +

Layer 6: Developmental Gate — 7 Embryological Stages

+

Agents grow through seven stages, each unlocking new capabilities:

+ + + + + + + + + + + +
LevelStageArabicMax TurnsKey Unlocks
1Nutfahنطفة5bash, read_file, recall_memory
2Alaqahعلقة8+ write_file, http_get
3Mudghahمضغة10+ python_exec, http_post, delegation
4Izhamعظام12+ create_agent, causal rung 2
5Lahmلحم15All tools, causal rung 3, Lubb metacognition
6Nafkhنفخ20Full metacognition
7Khalq Akharخلق آخر25Full autonomy
+

Promotion criteria: sustained success rate + task count + hikmah threshold (managed by DevelopmentalGate.check_upgrade_readiness()).

+ +

Layer 7: Causal Engine — Pearl's Causal Ladder

+ + + + + + + +
RungTypeQuestionMethod
1Observation"What is?"P(Y|X) — correlational analysis
2Intervention"What if I do X?"P(Y|do(X)) — do-calculus
3Counterfactual"What if I had done X instead?"P(Y_x|X', Y') — alt. history
+ +

Extension Modules

+ + + + + + + + + + +
ModuleArabicPurposeFile
Self-HealingلوّامةImmune memory, health metrics, adaptive checkpointscore/self_healing.py
Parallel Agents—Concurrent task scheduling + skill transfercore/parallel_agents.py
ImaginationتصويرPredictive coding — simulate before actingcore/imagination.py
Creativityإبداع5 creative modes + fitness landscape mathcore/creativity.py
Dream EngineمنامOffline memory consolidation (NREM+REM)core/dream_engine.py
Shura CouncilشورىMulti-agent consultation for complex decisionsagents/shura_council.py
+ +

5-Layer Memory Pyramid

+

All memory layers are queried through a unified MemoryPyramid:

+ + + + + + + + + +
LayerModulePurpose
Dhikrmemory/dhikr.pyThree-tier persistent memory (episodic, semantic, procedural)
Masalikmemory/masalik.pyNeural pathways with spreading activation
Lawh al-Mahfuzmemory/lawh_mahfuz.pyImmutable memory with triple-checksum integrity (SHA-256 + CRC-32 + length)
VectorStorememory/vector_store.pySemantic embedding search (ChromaDB)
KnowledgeGraphmemory/knowledge_graph.pyEntity-relationship graph (SQLite)
+

Unified query: memory/memory_pyramid.py merges, deduplicates, and ranks by relevance × certainty × recency.

+

Yaqin Certainty Engine

Every piece of knowledge is tagged with its certainty level:

@@ -1075,20 +1182,17 @@

Yaqin Certainty Engine

-

Nafs Evolution (Agent Growth)

-

Agents evolve through 7 levels based on their performance:

- - - - - - - - - - - -
LevelNameRequirements
1Ammara (Commanding)Starting level
2Lawwama (Self-Reproaching)60% success, 25+ tasks
3Mulhama (Inspired)75% success, 100+ tasks
4Mutmainna (Tranquil)85% success, 250+ tasks
5Radiya (Pleased)90% success, 500+ tasks
6Mardiyya (Pleasing)95% success, 1000+ tasks
7Kamila (Complete)97% success, 2000+ tasks
+

Cognitive Metadata in the UI

+

Every assistant response includes a CognitiveBar showing real-time cognitive state:

+
    +
  • Qalb state (Qabd/Bast/Khushu) with confidence
    • +
  • Yaqin certainty level with confidence bar
    • +
  • Lubb quality (confident / hedged / uncertain)
    • +
  • Ruh energy percentage
    • +
  • Nafs level and name badge
    • +
  • Lawwama repair indicator (when self-healing is active)
    • +
+

Expandable for detailed signals, bias flags, and evidence lists.

QCA API Endpoints

POST /api/yaqin/tag         Tag knowledge with certainty level
@@ -1134,34 +1238,66 @@ 
Communication Patterns

       
Project Structure

       
mizan/
 +-- backend/
-|   +-- api/main.py              # FastAPI server + all routes
+|   +-- api/main.py                  # FastAPI server + all routes
 |   +-- agents/
-|   |   +-- base.py              # Base agent + agentic loop
-|   |   +-- specialized.py       # Browser, Research, Code agents
-|   |   +-- federation.py        # Agent-to-agent communication
+|   |   +-- base.py                  # Base agent with QALB-7 agentic loop
+|   |   +-- specialized.py           # Browser, Research, Code, SuperAgent (Khalifah)
+|   |   +-- federation.py            # Agent-to-agent communication
+|   |   +-- shura_council.py         # Multi-agent consultation
+|   |   +-- perpetual_rotation.py    # Agent rotation & load balancing
 |   +-- core/
-|   |   +-- events.py            # Event bus (Nida')
-|   |   +-- hooks.py             # Hook system (Ta'liq)
-|   |   +-- plugins.py           # Plugin manager (Wasilah)
-|   |   +-- middleware.py         # Middleware pipeline (Silsilah)
-|   +-- providers.py             # LLM providers (Claude/GPT/Ollama/300+)
-|   +-- memory/dhikr.py          # 3-tier memory
-|   +-- security/                # Auth, permissions, sandbox
-|   +-- skills/                  # Skill registry + builtins
-|   +-- gateway/channels/        # Telegram, Discord, Slack, WhatsApp
-|   +-- automation/              # Cron jobs + webhooks
-|   +-- settings.py              # Configuration
-|   +-- cli.py                   # Terminal interface
+|   |   +-- fitrah.py                # Innate ethical guardrails
+|   |   +-- nafs_triad.py            # 3-voice deliberation (Ammara/Lawwama/Mutmainna)
+|   |   +-- qalb_processor.py        # Cardiac oscillation → LLM param modulation
+|   |   +-- fuad.py                  # Bayesian conviction formation
+|   |   +-- lubb.py                  # Metacognition: compress, cohere, debias
+|   |   +-- developmental_stages.py  # 7-stage capability gating
+|   |   +-- self_healing.py          # Lawwama immune system + health metrics
+|   |   +-- parallel_agents.py       # Concurrent task scheduling
+|   |   +-- imagination.py           # Predictive coding engine
+|   |   +-- creativity.py            # 5 creative modes + landscape math
+|   |   +-- dream_engine.py          # Offline memory consolidation
+|   |   +-- qalb.py                  # Emotional intelligence (sentiment)
+|   |   +-- ruh_engine.py            # Energy/vitality management
+|   |   +-- events.py                # Event bus (Nida')
+|   |   +-- hooks.py                 # Hook system (Ta'liq)
+|   |   +-- plugins.py               # Plugin manager (Wasilah)
+|   |   +-- middleware.py             # Middleware pipeline (Silsilah)
+|   +-- providers.py                 # LLM providers (Claude/GPT/Ollama/300+)
+|   +-- memory/
+|   |   +-- dhikr.py                 # 3-tier persistent memory
+|   |   +-- masalik.py               # Neural pathways (spreading activation)
+|   |   +-- lawh_mahfuz.py           # Immutable memory (triple-checksum)
+|   |   +-- memory_pyramid.py        # Unified 5-layer query engine
+|   |   +-- vector_store.py          # Semantic embeddings (ChromaDB)
+|   |   +-- knowledge_graph.py       # Entity-relationship graph
+|   |   +-- living_memory.py         # Adaptive memory lifecycle
+|   +-- reasoning/
+|   |   +-- aql_engine.py            # Arabic Query Language reasoning
+|   |   +-- causal_engine.py         # Pearl's 3-rung causal ladder
+|   |   +-- planner.py               # Task planning
+|   |   +-- context_manager.py       # Context window management
+|   +-- security/                    # Auth, permissions, sandbox
+|   +-- skills/                      # Skill registry + builtins
+|   +-- knowledge/                   # Knowledge base management
+|   +-- gateway/channels/            # Telegram, Discord, Slack, WhatsApp
+|   +-- automation/                  # Cron jobs + webhooks
+|   +-- settings.py                  # Configuration
+|   +-- cli.py                       # Terminal interface
 +-- frontend/src/
-|   +-- App.tsx                  # Main UI
-|   +-- pages/                   # Feature pages
-|   +-- hooks/                   # React hooks
-+-- plugins/                     # Your plugins go here!
-+-- docs/                        # This documentation
-+-- tests/                       # Test suite
-+-- docker/                      # Docker configs
-+-- pyproject.toml               # Package config
-+-- Makefile                     # Dev commands

+|   +-- App.tsx                      # Main UI + WebSocket handler
+|   +-- components/
+|   |   +-- ChatMessage.tsx          # Chat bubbles + CognitiveBar pills
+|   |   +-- AgentCard.tsx            # Agent card (Nafs + Ruh bars)
+|   +-- pages/                       # Feature pages
+|   +-- hooks/                       # React hooks
+|   +-- types.ts                     # TypeScript types (CognitiveMetadata)
++-- plugins/                         # Your plugins go here!
++-- docs/                            # This documentation
++-- tests/                           # Test suite
++-- docker/                          # Docker configs
++-- pyproject.toml                   # Package config
++-- Makefile                         # Dev commands
diff --git a/frontend/index.html b/frontend/index.html index 20f630b..23b92a6 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -8,7 +8,7 @@ - +
diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 12251db..a983905 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -3,13 +3,40 @@ * Clean, accessible UI with light/dark/system theme support. */ -import { useState, useEffect, useRef, useCallback, Component } from "react"; +import { + useState, + useEffect, + useRef, + useCallback, + Component, + lazy, + Suspense, +} from "react"; import type { ErrorInfo, ReactNode } from "react"; -import type { Agent, ChatMessage, TerminalLine, Memory, Integration, SystemStatus } from "./types"; +import type { + Agent, + ChatMessage, + CognitiveMetadata, + TerminalLine, + Memory, + Integration, + SystemStatus, +} from "./types"; import { config } from "./config"; -import { useTheme } from "./hooks/useTheme"; import { useApi } from "./hooks/useApi"; import { ToastProvider, useToast } from "./components/Toast"; +import { Icons } from "./components/Icons"; +import { ThemeToggle } from "./components/ThemeToggle"; +import { ConnectionBanner } from "./components/ConnectionBanner"; +import { AgentCard, NAFS_LEVELS } from "./components/AgentCard"; +import { + ChatMessageContent, + ChatMessageBubble, +} from "./components/ChatMessage"; +import { Sidebar } from "./components/Sidebar"; +import { MobileNav } from "./components/MobileNav"; +import { AgentModal } from "./components/AgentModal"; +import { SkeletonCard } from "./components/Skeleton"; // ===== ERROR BOUNDARY ===== interface ErrorBoundaryProps { @@ -44,14 +71,27 @@ class ErrorBoundary extends Component {
- - + +
-

Something went wrong

+

+ Something went wrong +

- An unexpected error occurred while rendering this page. You can try again or refresh the browser. + An unexpected error occurred while rendering this page. You can + try again or refresh the browser.

{this.state.error && (

@@ -63,9 +103,23 @@ class ErrorBoundary extends Component { onClick={this.handleRetry} className="inline-flex items-center gap-2 px-5 py-2.5 rounded-lg text-sm font-medium text-white bg-amber-600 hover:bg-amber-700 dark:bg-amber-500 dark:hover:bg-amber-600 transition-colors shadow-sm" > - - - + + + Retry @@ -77,298 +131,18 @@ class ErrorBoundary extends Component { } } -import ChannelsPage from "./pages/ChannelsPage"; -import SkillsPage from "./pages/SkillsPage"; -import SecurityPage from "./pages/SecurityPage"; -import AutomationPage from "./pages/AutomationPage"; -import NotebookPage from "./pages/NotebookPage"; -import ScannerPage from "./pages/ScannerPage"; -import MajlisPage from "./pages/MajlisPage"; -import PluginsPage from "./pages/PluginsPage"; -import ProvidersPage from "./pages/ProvidersPage"; -import DeveloperPage from "./pages/DeveloperPage"; -import WelcomePage from "./pages/WelcomePage"; -import SettingsPage from "./pages/SettingsPage"; - -// ===== ICONS (inline SVG) ===== -const Icons = { - Agent: () => ( - - - - - - ), - Brain: () => ( - - - - - ), - Terminal: () => ( - - - - - ), - Memory: () => ( - - - - - - - - ), - Chat: () => ( - - - - ), - Plus: () => ( - - - - ), - Send: () => ( - - - - - ), - Trash: () => ( - - - - - ), - Globe: () => ( - - - - - - ), - Zap: () => ( - - - - ), - Channel: () => ( - - - - - ), - Skill: () => ( - - - - ), - Shield: () => ( - - - - ), - Clock: () => ( - - - - - ), - Notebook: () => ( - - - - - - ), - Plugin: () => ( - - - - - - ), - Sun: () => ( - - - - - - - - ), - Moon: () => ( - - - - ), - Monitor: () => ( - - - - - ), - Settings: () => ( - - - - ), -}; - -// ===== NAFS LEVELS ===== -const NAFS_LEVELS: Record = { - 1: { latin: "Ammara", color: "#ef4444", desc: "Raw potential" }, - 2: { latin: "Lawwama", color: "#f97316", desc: "Self-correcting" }, - 3: { latin: "Mulhama", color: "#f59e0b", desc: "Inspired" }, - 4: { latin: "Mutmainna", color: "#84cc16", desc: "Tranquil" }, - 5: { latin: "Radiya", color: "#10b981", desc: "Content" }, - 6: { latin: "Mardiyya", color: "#06b6d4", desc: "Pleasing" }, - 7: { latin: "Kamila", color: "#a78bfa", desc: "Perfected" }, -}; - -// ===== AGENT CARD ===== -const AgentCard = ({ agent, selected, onClick }: { agent: Agent; selected: boolean; onClick: () => void }) => { - const nafs = NAFS_LEVELS[agent.nafs_level] || NAFS_LEVELS[1]; - - const stateColors: Record = { - resting: "bg-gray-100 dark:bg-zinc-700/30 text-gray-500 dark:text-gray-400", - thinking: "bg-blue-100 dark:bg-blue-500/15 text-blue-600 dark:text-blue-400", - acting: "bg-amber-100 dark:bg-amber-500/15 text-amber-600 dark:text-amber-400", - learning: "bg-emerald-100 dark:bg-emerald-500/15 text-emerald-600 dark:text-emerald-400", - error: "bg-red-100 dark:bg-red-500/15 text-red-600 dark:text-red-400", - }; - - return ( -

-
-
- {agent.name[0]?.toUpperCase() || "A"} -
-
-
{agent.name}
-
{agent.role}
-
- - {agent.state} - -
- -
- - Level {agent.nafs_level} - -
-
-
- {nafs.desc} -
- -
- {[ - { label: "Tasks", value: agent.total_tasks, color: undefined as boolean | undefined }, - { label: "Success", value: `${(agent.success_rate * 100).toFixed(0)}%`, color: agent.success_rate > 0.7 }, - { label: "Wisdom", value: agent.hikmah_count, color: undefined as boolean | undefined }, - ].map(s => ( -
- - {s.value} - - {s.label} -
- ))} -
- - {(agent.tools || []).length > 0 && ( -
- {(agent.tools || []).slice(0, 4).map(t => ( - {t} - ))} - {(agent.tools || []).length > 4 && ( - +{(agent.tools || []).length - 4} - )} -
- )} -
- ); -}; - -// ===== THEME TOGGLE ===== -function ThemeToggle() { - const { theme, setTheme } = useTheme(); - const modes = ["light", "dark", "system"] as const; - const next = () => { - const idx = modes.indexOf(theme); - setTheme(modes[(idx + 1) % modes.length]); - }; - const icon = theme === "light" ? : theme === "dark" ? : ; - const label = theme === "light" ? "Light" : theme === "dark" ? "Dark" : "System"; - - return ( - - ); -} - -// ===== CONNECTION BANNER ===== -function ConnectionBanner({ status, attempts }: { status: string; attempts: number }) { - if (status === "connected") return null; - if (status === "connecting" || (status === "reconnecting" && attempts < 5)) return null; - - return ( -
-
- - - - Cannot connect to backend. Make sure the server is running: mizan serve or make dev -
- -
- ); -} - -// ===== SIMPLE MARKDOWN TO HTML ===== -function simpleMarkdown(text: string): string { - let html = text - // Escape HTML entities first - .replace(/&/g, "&") - .replace(//g, ">") - // Code blocks (``` ... ```) - .replace(/```(\w*)\n([\s\S]*?)```/g, '
$2
') - // Inline code - .replace(/`([^`]+)`/g, '$1') - // Bold - .replace(/\*\*(.+?)\*\*/g, "$1") - // Italic - .replace(/\*(.+?)\*/g, "$1") - // Headers (only at line start) - .replace(/^### (.+)$/gm, '

$1

') - .replace(/^## (.+)$/gm, '

$1

') - .replace(/^# (.+)$/gm, '

$1

') - // Line breaks - .replace(/\n/g, "
"); - return html; -} +const ChannelsPage = lazy(() => import("./pages/ChannelsPage")); +const SkillsPage = lazy(() => import("./pages/SkillsPage")); +const SecurityPage = lazy(() => import("./pages/SecurityPage")); +const AutomationPage = lazy(() => import("./pages/AutomationPage")); +const NotebookPage = lazy(() => import("./pages/NotebookPage")); +const ScannerPage = lazy(() => import("./pages/ScannerPage")); +const MajlisPage = lazy(() => import("./pages/MajlisPage")); +const PluginsPage = lazy(() => import("./pages/PluginsPage")); +const ProvidersPage = lazy(() => import("./pages/ProvidersPage")); +const DeveloperPage = lazy(() => import("./pages/DeveloperPage")); +const WelcomePage = lazy(() => import("./pages/WelcomePage")); +const SettingsPage = lazy(() => import("./pages/SettingsPage")); // ===== MAIN APP INNER ===== function AppInner() { @@ -378,7 +152,13 @@ function AppInner() { return !localStorage.getItem("mizan_setup_complete"); }); - const [activeTab, setActiveTab] = useState("chat"); + const [activeTab, setActiveTabState] = useState(() => { + return localStorage.getItem("mizan_active_tab") || "chat"; + }); + const setActiveTab = useCallback((tab: string) => { + setActiveTabState(tab); + localStorage.setItem("mizan_active_tab", tab); + }, []); const [agents, setAgents] = useState([]); const [selectedAgent, setSelectedAgent] = useState(null); const [messages, setMessages] = useState([]); @@ -393,31 +173,73 @@ function AppInner() { { text: "Connecting to backend...", type: "" }, ]); const [taskInput, setTaskInput] = useState(""); - const [sessionId] = useState(() => `session_${Date.now()}`); + const [sessionId, setSessionId] = useState(() => { + return localStorage.getItem("mizan_session_id") || `session_${Date.now()}`; + }); const [typingIndicator, setTypingIndicator] = useState(false); const [toolStatus, setToolStatus] = useState(""); const [showCreateAgent, setShowCreateAgent] = useState(false); const [memories, setMemories] = useState([]); const [memoryQuery, setMemoryQuery] = useState(""); + const [memoryTypeFilter, setMemoryTypeFilter] = useState("all"); + const [showAddMemory, setShowAddMemory] = useState(false); + const [newMemory, setNewMemory] = useState({ + content: "", + memory_type: "semantic", + importance: 0.7, + tags: "", + }); + const [knowledgeUrl, setKnowledgeUrl] = useState(""); + const [knowledgeLoading, setKnowledgeLoading] = useState(false); + const [knowledgeSources, setKnowledgeSources] = useState< + { title: string; type: string; chunks: number; last_updated: string }[] + >([]); + const [knowledgeResult, setKnowledgeResult] = useState(null); const [status, setStatus] = useState(null); const [integrations, setIntegrations] = useState([]); - const [newAgent, setNewAgent] = useState({ name: "", type: "general", model: "claude-opus-4-6" }); + const [newAgent, setNewAgent] = useState({ + name: "", + type: "general", + model: "claude-opus-4-6", + system_prompt: "", + }); + const [editingAgent, setEditingAgent] = useState(null); const [appVersion, setAppVersion] = useState("..."); const [showCommandMenu, setShowCommandMenu] = useState(false); const [commandMenuIndex, setCommandMenuIndex] = useState(0); + const [showAgentPicker, setShowAgentPicker] = useState(false); + const [chatModelOverride, setChatModelOverride] = useState(""); + const [chatSessions, setChatSessions] = useState< + { + session_id: string; + started_at: string; + last_message_at: string; + message_count: number; + first_message?: string; + }[] + >([]); + const [showSessionHistory, setShowSessionHistory] = useState(false); const CHAT_COMMANDS = [ { name: "/help", description: "Show available commands" }, { name: "/status", description: "Show system status" }, { name: "/new", description: "Start a new chat session" }, { name: "/reset", description: "Reset agent state" }, - { name: "/model", description: "Switch AI model (e.g. /model claude-sonnet-4-6)" }, + { + name: "/model", + description: "Switch AI model (e.g. /model claude-sonnet-4-6)", + }, { name: "/agents", description: "List available agents" }, - { name: "/compact", description: "Summarize older messages to save context" }, + { + name: "/compact", + description: "Summarize older messages to save context", + }, ]; - const filteredCommands = CHAT_COMMANDS.filter(cmd => - input.startsWith("/") && cmd.name.startsWith(input.split(" ")[0].toLowerCase()) + const filteredCommands = CHAT_COMMANDS.filter( + (cmd) => + input.startsWith("/") && + cmd.name.startsWith(input.split(" ")[0].toLowerCase()), ); const api = useApi(); @@ -426,7 +248,10 @@ function AppInner() { const clientId = useRef(`client_${Date.now()}`); const addTerminalLine = useCallback((text: string, type: string = "") => { - setTerminalLines(prev => [...prev.slice(-100), { text, type: type as TerminalLine["type"], ts: Date.now() }]); + setTerminalLines((prev) => [ + ...prev.slice(-100), + { text, type: type as TerminalLine["type"], ts: Date.now() }, + ]); }, []); // Connect WebSocket @@ -488,88 +313,114 @@ function AppInner() { }; }, []); - const handleWsMessage = useCallback((data: Record) => { - switch (data.type) { - case "connected": - addTerminalLine(`${data.message} — ${data.agents} agents online`, "gold"); - loadAgents(); - loadStatus(); - break; - case "stream": - case "chat_stream": - setTypingIndicator(false); - setStreamingText(prev => prev + (data.chunk as string)); - break; - case "response": - setStreamingText(""); - setStreaming(false); - setTypingIndicator(false); - setToolStatus(""); - setMessages(prev => [...prev, { - id: Date.now(), - role: "assistant" as const, - content: data.content as string, - agent: data.agent as string, - ts: new Date().toLocaleTimeString(), - }]); - addTerminalLine(`Response from ${data.agent}`, "info"); - break; - case "chat_complete": - setStreamingText(""); - setStreaming(false); - setTypingIndicator(false); - setToolStatus(""); - setMessages(prev => [...prev, { - id: Date.now(), - role: "assistant" as const, - content: (data.response as string) || (data.content as string) || "", - agent: data.agent as string, - ts: new Date().toLocaleTimeString(), - }]); - addTerminalLine(`Response from ${data.agent}`, "info"); - break; - case "typing": - setTypingIndicator(true); - break; - case "tool_use": - setToolStatus(`Agent is using ${data.tool_name as string}...`); - addTerminalLine(`Tool: ${data.tool_name as string}`, "info"); - break; - case "command_result": { - setStreaming(false); - setStreamingText(""); - setTypingIndicator(false); - const cmdContent = (data.content as string) || (data.result as string) || "done"; - setMessages(prev => [...prev, { - id: Date.now(), - role: "system" as const, - content: cmdContent, - agent: "system", - ts: new Date().toLocaleTimeString(), - }]); - addTerminalLine(`Command: ${cmdContent.substring(0, 60)}`, "gold"); - break; + const handleWsMessage = useCallback( + (data: Record) => { + switch (data.type) { + case "connected": + addTerminalLine( + `${data.message} — ${data.agents} agents online`, + "gold", + ); + loadAgents(); + loadStatus(); + break; + case "stream": + case "chat_stream": + setTypingIndicator(false); + setStreamingText((prev) => prev + (data.chunk as string)); + break; + case "response": + setStreamingText(""); + setStreaming(false); + setTypingIndicator(false); + setToolStatus(""); + setMessages((prev) => [ + ...prev, + { + id: Date.now(), + role: "assistant" as const, + content: data.content as string, + agent: data.agent as string, + ts: new Date().toLocaleTimeString(), + }, + ]); + addTerminalLine(`Response from ${data.agent}`, "info"); + break; + case "chat_complete": + setStreamingText(""); + setStreaming(false); + setTypingIndicator(false); + setToolStatus(""); + setMessages((prev) => [ + ...prev, + { + id: Date.now(), + role: "assistant" as const, + content: + (data.response as string) || (data.content as string) || "", + agent: data.agent as string, + ts: new Date().toLocaleTimeString(), + cognitive: data.cognitive as CognitiveMetadata | undefined, + }, + ]); + addTerminalLine(`Response from ${data.agent}`, "info"); + break; + case "typing": + setTypingIndicator(true); + break; + case "tool_use": + setToolStatus(`Agent is using ${data.tool_name as string}...`); + addTerminalLine(`Tool: ${data.tool_name as string}`, "info"); + break; + case "command_result": { + setStreaming(false); + setStreamingText(""); + setTypingIndicator(false); + const cmdContent = + (data.content as string) || (data.result as string) || "done"; + // Handle /new — clear messages and start fresh session + if ((data.command as string) === "/new") { + startNewSession(); + } else { + setMessages((prev) => [ + ...prev, + { + id: Date.now(), + role: "system" as const, + content: cmdContent, + agent: "system", + ts: new Date().toLocaleTimeString(), + }, + ]); + } + addTerminalLine(`Command: ${cmdContent.substring(0, 60)}`, "gold"); + break; + } + case "error": + setStreaming(false); + setStreamingText(""); + setTypingIndicator(false); + setToolStatus(""); + addTerminalLine(`Error: ${data.message as string}`, "error"); + break; + case "task_stream": + addTerminalLine(data.chunk as string, ""); + break; + case "task_done": + addTerminalLine("Task completed", "gold"); + loadAgents(); + break; + case "agent_created": + addTerminalLine( + `Agent created: ${(data.agent as Record).name}`, + "gold", + ); + loadAgents(); + break; } - case "error": - setStreaming(false); - setStreamingText(""); - setTypingIndicator(false); - setToolStatus(""); - addTerminalLine(`Error: ${data.message as string}`, "error"); - break; - case "task_stream": - addTerminalLine(data.chunk as string, ""); - break; - case "task_done": - addTerminalLine("Task completed", "gold"); - loadAgents(); - break; - case "agent_created": - addTerminalLine(`Agent created: ${(data.agent as Record).name}`, "gold"); - loadAgents(); - break; - } - }, [addTerminalLine]); + }, + [addTerminalLine], + ); const loadAgents = async () => { try { @@ -577,10 +428,17 @@ function AppInner() { const data = await res.json(); setAgents(data.agents || []); if (!selectedAgent && data.agents?.length > 0) { - setSelectedAgent(data.agents[0]); + const savedAgentId = localStorage.getItem("mizan_selected_agent"); + const restored = savedAgentId + ? data.agents.find((a: Agent) => a.id === savedAgentId) + : null; + setSelectedAgent(restored || data.agents[0]); } } catch (e: unknown) { - addTerminalLine(`Failed to load agents: ${(e as Error).message}`, "error"); + addTerminalLine( + `Failed to load agents: ${(e as Error).message}`, + "error", + ); } }; @@ -589,19 +447,108 @@ function AppInner() { const res = await fetch(`${config.API_URL}/status`); const data = await res.json(); setStatus(data); - } catch { /* ignore */ } + } catch { + /* ignore */ + } }; - const loadMemories = async (query: string = "") => { + const loadMemories = async (query: string = "", typeFilter?: string) => { try { - const res = await fetch(`${config.API_URL}/memory/query`, { + let data; + if (query.trim()) { + const res = await fetch(`${config.API_URL}/memory/query`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + query, + limit: 30, + ...(typeFilter && typeFilter !== "all" + ? { memory_type: typeFilter } + : {}), + }), + }); + data = await res.json(); + } else { + const params = new URLSearchParams({ limit: "30" }); + if (typeFilter && typeFilter !== "all") { + params.set("memory_type", typeFilter); + } + const res = await fetch( + `${config.API_URL}/memory/list?${params.toString()}`, + ); + data = await res.json(); + } + setMemories(data.results || []); + } catch { + /* ignore */ + } + }; + + const loadKnowledgeSources = async () => { + try { + const res = await fetch(`${config.API_URL}/knowledge/sources`); + const data = await res.json(); + setKnowledgeSources(data.sources || []); + } catch { + /* ignore */ + } + }; + + const ingestKnowledge = async () => { + if (!knowledgeUrl.trim()) return; + setKnowledgeLoading(true); + setKnowledgeResult(null); + try { + const res = await fetch(`${config.API_URL}/knowledge/ingest`, { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ query: query || "all", limit: 20 }), + body: JSON.stringify({ source: knowledgeUrl.trim() }), }); const data = await res.json(); - setMemories(data.results || []); - } catch { /* ignore */ } + if (res.ok) { + setKnowledgeResult( + `Ingested "${data.title}" — ${data.chunks_stored} chunks stored (${data.char_count?.toLocaleString()} chars)`, + ); + setKnowledgeUrl(""); + loadKnowledgeSources(); + } else { + setKnowledgeResult(`Error: ${data.detail || "Failed to ingest"}`); + } + } catch (err) { + setKnowledgeResult( + `Error: ${err instanceof Error ? err.message : "Network error"}`, + ); + } finally { + setKnowledgeLoading(false); + } + }; + + const uploadKnowledgePdf = async (file: File) => { + setKnowledgeLoading(true); + setKnowledgeResult(null); + try { + const formData = new FormData(); + formData.append("file", file); + const res = await fetch(`${config.API_URL}/knowledge/upload`, { + method: "POST", + body: formData, + }); + const data = await res.json(); + if (res.ok) { + setKnowledgeResult( + `Uploaded "${data.title}" — ${data.page_count} pages, ${data.chunks_stored} chunks stored`, + ); + loadKnowledgeSources(); + } else { + setKnowledgeResult(`Error: ${data.detail || "Upload failed"}`); + } + } catch (err) { + setKnowledgeResult( + `Error: ${err instanceof Error ? err.message : "Network error"}`, + ); + } finally { + setKnowledgeLoading(false); + } }; const loadIntegrations = async () => { @@ -609,15 +556,74 @@ function AppInner() { const res = await fetch(`${config.API_URL}/integrations`); const data = await res.json(); setIntegrations(data.integrations || []); - } catch { /* ignore */ } + } catch { + /* ignore */ + } }; + const loadChatHistory = useCallback(async (sid: string) => { + try { + const res = await fetch(`${config.API_URL}/chat/${sid}`); + if (!res.ok) return; + const data = await res.json(); + const history = (data.messages || []).map( + ( + m: { + id?: number; + role: string; + content: string; + agent_id?: string; + created_at?: string; + }, + idx: number, + ) => ({ + id: m.id || idx, + role: m.role as "user" | "assistant" | "system", + content: m.content, + agent: m.agent_id, + ts: m.created_at ? new Date(m.created_at).toLocaleTimeString() : "", + }), + ); + if (history.length > 0) { + setMessages(history); + } + } catch { + // No history available — start fresh + } + }, []); + + const loadChatSessions = useCallback(async () => { + try { + const res = await fetch(`${config.API_URL}/chat/sessions/list`); + if (!res.ok) return; + const data = await res.json(); + setChatSessions(data.sessions || []); + } catch { + // ignore + } + }, []); + + const switchSession = useCallback( + async (sid: string) => { + setSessionId(sid); + setMessages([]); + setShowSessionHistory(false); + await loadChatHistory(sid); + }, + [loadChatHistory], + ); + useEffect(() => { loadAgents(); + loadChatHistory(sessionId); + loadChatSessions(); // Fetch version from backend - fetch(`${config.API_URL}/version`).then(r => r.json()).then(data => { - if (data.version) setAppVersion(data.version); - }).catch(() => {}); + fetch(`${config.API_URL}/version`) + .then((r) => r.json()) + .then((data) => { + if (data.version) setAppVersion(data.version); + }) + .catch(() => {}); const interval = setInterval(() => { loadAgents(); loadStatus(); @@ -625,8 +631,21 @@ function AppInner() { return () => clearInterval(interval); }, []); + // Persist sessionId to localStorage useEffect(() => { - if (activeTab === "memory") loadMemories(memoryQuery); + localStorage.setItem("mizan_session_id", sessionId); + }, [sessionId]); + + // Persist selected agent ID + useEffect(() => { + if (selectedAgent) { + localStorage.setItem("mizan_selected_agent", selectedAgent.id); + } + }, [selectedAgent]); + + useEffect(() => { + if (activeTab === "memory") loadMemories(memoryQuery, memoryTypeFilter); + if (activeTab === "knowledge") loadKnowledgeSources(); if (activeTab === "integrations") loadIntegrations(); }, [activeTab]); @@ -634,11 +653,24 @@ function AppInner() { messagesEndRef.current?.scrollIntoView({ behavior: "smooth" }); }, [messages, streamingText]); + const startNewSession = useCallback(() => { + const newId = `session_${Date.now()}`; + setSessionId(newId); + setMessages([]); + setStreamingText(""); + addTerminalLine("New chat session started", "gold"); + }, []); + const sendMessage = async () => { if (!input.trim() || streaming) return; const content = input; - const userMsg: ChatMessage = { id: Date.now(), role: "user", content, ts: new Date().toLocaleTimeString() }; - setMessages(prev => [...prev, userMsg]); + const userMsg: ChatMessage = { + id: Date.now(), + role: "user", + content, + ts: new Date().toLocaleTimeString(), + }; + setMessages((prev) => [...prev, userMsg]); setStreaming(true); setStreamingText(""); setTypingIndicator(true); @@ -656,19 +688,22 @@ function AppInner() { session_id: sessionId, content, agent_id: selectedAgent?.id, + ...(chatModelOverride ? { model_override: chatModelOverride } : {}), }), }); if (!res.ok) throw new Error(`HTTP ${res.status}`); - // Response processing happens via chat_stream/chat_complete WebSocket events + if (chatModelOverride) setChatModelOverride(""); } catch { // Fallback: send via WebSocket directly if (ws) { - ws.send(JSON.stringify({ - type: "chat", - session_id: sessionId, - content, - agent_id: selectedAgent?.id, - })); + ws.send( + JSON.stringify({ + type: "chat", + session_id: sessionId, + content, + agent_id: selectedAgent?.id, + }), + ); } else { setStreaming(false); setTypingIndicator(false); @@ -680,31 +715,86 @@ function AppInner() { const runTask = () => { if (!taskInput.trim() || !ws) return; addTerminalLine(`$ ${taskInput}`, "gold"); - ws.send(JSON.stringify({ - type: "task", - task: taskInput, - agent_id: selectedAgent?.id, - })); + ws.send( + JSON.stringify({ + type: "task", + task: taskInput, + agent_id: selectedAgent?.id, + }), + ); setTaskInput(""); }; const createAgent = async () => { try { - const res = await fetch(`${config.API_URL}/agents`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify(newAgent), + const payload = { + ...newAgent, + system_prompt: newAgent.system_prompt || undefined, + }; + await api.post("/agents", payload); + setShowCreateAgent(false); + setNewAgent({ + name: "", + type: "general", + model: "claude-opus-4-6", + system_prompt: "", }); - await res.json(); + loadAgents(); + addToast({ + type: "success", + title: "Agent created", + description: `${newAgent.name} is ready`, + }); + } catch (e: unknown) { + addToast({ + type: "error", + title: "Failed to create agent", + description: (e as Error).message, + }); + } + }; + + const updateAgent = async () => { + if (!editingAgent) return; + try { + const payload: Record = {}; + if (newAgent.name !== editingAgent.name) payload.name = newAgent.name; + if (newAgent.model !== (editingAgent.model || "")) + payload.model = newAgent.model; + if (newAgent.system_prompt !== (editingAgent.system_prompt || "")) + payload.system_prompt = newAgent.system_prompt; + + await api.put(`/agents/${editingAgent.id}`, payload); setShowCreateAgent(false); - setNewAgent({ name: "", type: "general", model: "claude-opus-4-6" }); + setEditingAgent(null); + setNewAgent({ + name: "", + type: "general", + model: "claude-opus-4-6", + system_prompt: "", + }); loadAgents(); - addToast({ type: "success", title: "Agent created", description: `${newAgent.name} is ready` }); + addToast({ type: "success", title: "Agent updated" }); } catch (e: unknown) { - addToast({ type: "error", title: "Failed to create agent", description: (e as Error).message }); + addToast({ + type: "error", + title: "Failed to update agent", + description: (e as Error).message, + }); } }; + const openEditAgent = (agent: Agent) => { + setEditingAgent(agent); + setNewAgent({ + name: agent.name, + type: agent.role, + model: agent.model || "claude-opus-4-6", + system_prompt: agent.system_prompt || "", + }); + setShowCreateAgent(true); + }; + const deleteAgent = async (agentId: string) => { if (!confirm("Delete this agent?")) return; try { @@ -712,7 +802,9 @@ function AppInner() { if (selectedAgent?.id === agentId) setSelectedAgent(null); loadAgents(); addToast({ type: "success", title: "Agent deleted" }); - } catch { /* ignore */ } + } catch { + /* ignore */ + } }; // ===== WELCOME PAGE ===== @@ -731,79 +823,181 @@ function AppInner() { { label: "Main", items: [ - { id: "chat", label: "Chat", desc: "Talk to your AI", icon: }, - { id: "agents", label: "Agents", desc: "Your AI team", icon: }, - { id: "terminal", label: "Tasks", desc: "Run background jobs", icon: }, + { + id: "chat", + label: "Chat", + desc: "Talk to your AI", + icon: , + }, + { + id: "agents", + label: "Agents", + desc: "Your AI team", + icon: , + }, + { + id: "terminal", + label: "Tasks", + desc: "Run background jobs", + icon: , + }, ], }, { label: "Tools", items: [ - { id: "memory", label: "Memory", desc: "What your AI remembers", icon: }, - { id: "notebooks", label: "Notebooks", desc: "Code scratchpad", icon: }, - { id: "skills", label: "Skills", desc: "AI abilities", icon: }, - { id: "plugins", label: "Plugins", desc: "Extend with add-ons", icon: }, + { + id: "memory", + label: "Memory", + desc: "What your AI remembers", + icon: , + }, + { + id: "knowledge", + label: "Knowledge", + desc: "Feed URLs, PDFs, YouTube", + icon: , + }, + { + id: "notebooks", + label: "Notebooks", + desc: "Code scratchpad", + icon: , + }, + { + id: "skills", + label: "Skills", + desc: "AI abilities", + icon: , + }, + { + id: "plugins", + label: "Plugins", + desc: "Extend with add-ons", + icon: , + }, ], }, { label: "System", items: [ - { id: "providers", label: "Providers", desc: "AI model settings", icon: }, - { id: "channels", label: "Channels", desc: "Telegram, Discord, etc.", icon: }, - { id: "automation", label: "Automation", desc: "Scheduled tasks", icon: }, - { id: "security", label: "Security", desc: "Login & permissions", icon: }, - { id: "settings", label: "Settings", desc: "Configure MIZAN", icon: }, - { id: "developer", label: "Developer", desc: "Build extensions", icon: }, + { + id: "providers", + label: "Providers", + desc: "AI model settings", + icon: , + }, + { + id: "channels", + label: "Channels", + desc: "Telegram, Discord, etc.", + icon: , + }, + { + id: "automation", + label: "Automation", + desc: "Scheduled tasks", + icon: , + }, + { + id: "security", + label: "Security", + desc: "Login & permissions", + icon: , + }, + { + id: "settings", + label: "Settings", + desc: "Configure MIZAN", + icon: , + }, + { + id: "developer", + label: "Developer", + desc: "Build extensions", + icon: , + }, ], }, ]; // ===== STATUS ===== - const statusDot = wsStatus === "connected" - ? "bg-emerald-500" - : wsStatus === "connecting" || wsStatus === "reconnecting" - ? "bg-amber-500 animate-pulse" - : "bg-red-500"; - - const statusLabel = wsStatus === "connected" - ? "Online" - : wsStatus === "connecting" - ? "Connecting..." - : wsStatus === "reconnecting" - ? "Reconnecting..." - : "Offline"; + const statusDot = + wsStatus === "connected" + ? "bg-emerald-500" + : wsStatus === "connecting" || wsStatus === "reconnecting" + ? "bg-amber-500 animate-pulse" + : "bg-red-500"; + + const statusLabel = + wsStatus === "connected" + ? "Online" + : wsStatus === "connecting" + ? "Connecting..." + : wsStatus === "reconnecting" + ? "Reconnecting..." + : "Offline"; // ===== RENDER CONTENT ===== const renderContent = () => { switch (activeTab) { case "agents": return ( -
-
+
+

Agents

-

Your AI team — create and manage intelligent agents

+

+ Your AI team — create and manage intelligent agents +

-
-
-
- {agents.map(agent => ( +
+
+ {agents.map((agent) => (
setSelectedAgent(agent)} /> - +
+ + +
))}
@@ -811,7 +1005,10 @@ function AppInner() {

No agents yet

-

Create your first agent to get started, or make sure the backend is running.

+

+ Create your first agent to get started, or make sure the + backend is running. +

)}
@@ -820,54 +1017,282 @@ function AppInner() { case "chat": return ( -
-
+
+
- Chat - - {selectedAgent && ( - - Level {selectedAgent.nafs_level} — {NAFS_LEVELS[selectedAgent.nafs_level]?.latin} - - )} + + Chat + + + {/* Agent picker pill */} +
+ + + {/* Agent picker dropdown */} + {showAgentPicker && ( +
+ {agents.map((agent) => { + const roleLabels: Record = { + general: "General Purpose", + hafiz: "General Purpose", + wakil: "General Purpose", + browser: "Web Browsing", + mubashir: "Web Browsing", + research: "Deep Research", + mundhir: "Deep Research", + code: "Code Generation", + katib: "Code Generation", + communication: "Communication", + rasul: "Communication", + }; + return ( + + ); + })} +
+ )} +
+ +
+ {selectedAgent && ( + + Level {selectedAgent.nafs_level} —{" "} + {NAFS_LEVELS[selectedAgent.nafs_level]?.latin} + + )} + + {/* Session history dropdown */} +
+ + {showSessionHistory && ( +
+
+ Recent Sessions +
+
+ {chatSessions.length === 0 && ( +
+ No previous sessions +
+ )} + {chatSessions.map((s) => ( + + ))} +
+
+ )} +
+ + {/* New chat button */} + +
{messages.length === 0 && !streaming && ( -
- -

Start a conversation

-

Send a message below. Your AI is ready to help with anything.

+
+
+
+ ميزان +
+

+ How can I help? +

+

+ Start a conversation, ask a question, or try one of these + suggestions. +

+
+
+ {[ + { + label: "Write code", + desc: "Generate, debug, or refactor", + prompt: "Help me write a Python script that ", + }, + { + label: "Analyze data", + desc: "Explore patterns and insights", + prompt: "Analyze this data and help me understand ", + }, + { + label: "Research topic", + desc: "Deep dive into any subject", + prompt: "Research and summarize the key points about ", + }, + { + label: "Brainstorm ideas", + desc: "Creative thinking together", + prompt: "Help me brainstorm ideas for ", + }, + ].map((action) => ( + + ))} +
)} - {messages.map(msg => ( -
+ {messages.map((msg) => ( +
{msg.role === "system" ? (
-
- - + + + System @@ -881,26 +1306,34 @@ function AppInner() {
) : ( <> -
- {msg.role === "user" ? "You" : (msg.agent?.[0] || "AI")} +
+ {msg.role === "user" ? "You" : msg.agent?.[0] || "AI"}
-
+
{msg.role === "assistant" ? ( -
+
+ +
) : ( -
+
{msg.content}
)}
- {msg.role === "assistant" ? msg.agent : "You"} · {msg.ts} + {msg.role === "assistant" ? msg.agent : "You"}{" "} + · {msg.ts}
@@ -917,9 +1350,18 @@ function AppInner() {
- - - + + +
@@ -928,10 +1370,21 @@ function AppInner() { {/* Tool status indicator */} {streaming && toolStatus && ( -
-
-
- {toolStatus} +
+
+
+ + + + + {toolStatus} +
)} @@ -944,17 +1397,17 @@ function AppInner() {
- - + +
)} -
+
-
+
{/* Command autocomplete dropdown */} {showCommandMenu && filteredCommands.length > 0 && (
- {cmd.name} - {cmd.description} + + {cmd.name} + + + {cmd.description} + ))}
@@ -990,7 +1447,7 @@ function AppInner() { className="input flex-1 resize-none min-h-[38px] max-h-[120px]" placeholder="Type your message... (/ for commands, Enter to send)" value={input} - onChange={e => { + onChange={(e) => { const val = e.target.value; setInput(val); if (val.startsWith("/") && !val.includes(" ")) { @@ -1000,19 +1457,28 @@ function AppInner() { setShowCommandMenu(false); } }} - onKeyDown={e => { + onKeyDown={(e) => { if (showCommandMenu && filteredCommands.length > 0) { if (e.key === "ArrowDown") { e.preventDefault(); - setCommandMenuIndex(prev => (prev + 1) % filteredCommands.length); + setCommandMenuIndex( + (prev) => (prev + 1) % filteredCommands.length, + ); return; } if (e.key === "ArrowUp") { e.preventDefault(); - setCommandMenuIndex(prev => (prev - 1 + filteredCommands.length) % filteredCommands.length); + setCommandMenuIndex( + (prev) => + (prev - 1 + filteredCommands.length) % + filteredCommands.length, + ); return; } - if (e.key === "Tab" || (e.key === "Enter" && !e.shiftKey)) { + if ( + e.key === "Tab" || + (e.key === "Enter" && !e.shiftKey) + ) { e.preventDefault(); const selected = filteredCommands[commandMenuIndex]; if (selected) { @@ -1056,17 +1522,21 @@ function AppInner() {

Task Runner

-

Execute tasks through your AI agents

+

+ Execute tasks through your AI agents +

-
-
-
+
+
+
- mizan ~ % + + mizan ~ % + Agent: {selectedAgent?.name || "None"} @@ -1074,26 +1544,40 @@ function AppInner() {
{terminalLines.map((line, i) => ( -
- {line.type === "gold" && {"> "}} +
+ {line.type === "gold" && ( + + {"> "} + + )} {line.text}
))}
- {">"} + + {">"} + setTaskInput(e.target.value)} - onKeyDown={e => e.key === "Enter" && runTask()} + onChange={(e) => setTaskInput(e.target.value)} + onKeyDown={(e) => e.key === "Enter" && runTask()} /> - + + +
+ {/* Add Memory Form */} + {showAddMemory && ( +
+