diff --git a/.env.example b/.env.example
index 1fc84ae..c607e09 100644
--- a/.env.example
+++ b/.env.example
@@ -36,6 +36,9 @@ DB_PATH=data/mizan.db
# ===== LOCAL AI (Optional) =====
OLLAMA_URL=http://localhost:11434
+# ===== CLOUD PROVIDERS (Optional) =====
+LINODE_API_TOKEN= # Linode Personal Access Token (linodes:read_write scope)
+
# ===== CHANNELS (Optional) =====
TELEGRAM_BOT_TOKEN=
DISCORD_BOT_TOKEN=
diff --git a/.gitignore b/.gitignore
index aadc19c..c84866b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -38,6 +38,14 @@ data/
.DS_Store
Thumbs.db
+# Claude Code
+.claude/
+.claude.local.md
+
+# Secrets / local config
+backend/.vault.key
+.mcp.json
+
# IDE
.vscode/
.idea/
diff --git a/CLAUDE.md b/CLAUDE.md
new file mode 100644
index 0000000..b183d4c
--- /dev/null
+++ b/CLAUDE.md
@@ -0,0 +1,86 @@
+# MIZAN - Agentic Personal AI
+
+## Quick Reference
+
+- **Backend**: Python 3.11+, FastAPI, Pydantic, aiosqlite
+- **Frontend**: React 18, TypeScript, Vite, Tailwind CSS
+- **AI Providers**: Anthropic, OpenAI, OpenRouter, Ollama
+- **Database**: SQLite (via aiosqlite)
+- **Infra**: Docker Compose, Nginx
+
+## Commands
+
+```bash
+make setup # First-time: install deps, setup frontend, create .env
+make dev # Start backend + frontend dev servers
+make test # Run pytest
+make check # Lint + typecheck + test (all checks)
+make format # Auto-format with ruff
+make docker # Docker Compose up (build + detach)
+make docker-down # Stop Docker
+make clean # Remove build artifacts
+```
+
+## Key Files
+
+- `backend/api/main.py` - FastAPI app, all REST + WebSocket routes
+- `backend/agents/base.py` - Base agent with agentic ReAct loop
+- `backend/providers.py` - LLM provider abstraction (Anthropic/OpenRouter/OpenAI/Ollama)
+- `backend/settings.py` - Pydantic settings from .env
+- `backend/cli.py` - CLI entry point (mizan serve, mizan doctor)
+- `frontend/src/App.tsx` - React router + theme provider
+- `docker-compose.yml` - Dev deployment
+- `Makefile` - All project commands
+
+## Architecture
+
+7-layer Quranic Cognitive Architecture (QCA):
+
+| Layer | Module | Purpose |
+|-------|--------|---------|
+| 1-2 | `backend/perception/` | Sensory input (vision, voice, text) |
+| 3 | `backend/qca/engine.py` | Cognitive integration |
+| 4 | `backend/memory/` | Hierarchical memory (Dhikr, Masalik) |
+| 5 | `backend/reasoning/` | ReAct reasoning with self-correction |
+| 6 | `backend/agents/` | Autonomous agents with Nafs levels |
+| 7 | `backend/core/` | Principles (Qalb, Ihsan, Tawbah, Sabr) |
+
+Cross-cutting: `backend/security/` (Wali + Izn), `backend/gateway/` (channels), `backend/skills/` (capabilities)
+
+## Key Patterns
+
+- All backend I/O is async (async/await)
+- Pydantic models for all API request/response schemas
+- JWT auth on all endpoints; Wali rate-limiting; Izn permission system
+- WebSocket at /ws/{client_id} for real-time streaming
+- Arabic-inspired naming: Dhikr=memory, Wali=guardian, Izn=permission, Nafs=self, Qalb=heart
+
+## Environment Setup
+
+1. `cp .env.example .env` and set at least one API key
+2. `make install-dev` (or `make setup` for full setup including frontend)
+3. Required: `ANTHROPIC_API_KEY` or `OPENROUTER_API_KEY`
+4. Required: `SECRET_KEY` (any random string)
+
+## Testing
+
+- pytest with asyncio auto mode
+- Tests in tests/ (comprehensive: agents, API, memory, QCA, security, e2e)
+- Run `make check` for full lint + typecheck + test pipeline
+- Pre-commit hook runs ruff on commit
+
+## Security Notes
+
+- Never edit .env directly (hook blocks this) - use .env.example as template
+- API keys managed via backend/security/vault.py
+- Input validation in backend/security/validation.py
+- All user input sanitized before processing
+
+## Gotchas
+
+- Docker reads .env for compose variable substitution - inline comments like `KEY=value # comment` get parsed as part of the value
+- Claude models use Anthropic only if `ANTHROPIC_API_KEY` starts with `sk-ant-`; otherwise auto-route through OpenRouter
+- DEFAULT_MODEL env var controls agent model (fallback: `claude-sonnet-4-20250514`) — set it in .env
+- Backend uses --reload in Docker, so local file changes in backend/ take effect automatically
+- .env values are NOT auto-loaded into os.environ - pydantic-settings reads them but providers.py uses load_dotenv()
+- `.claude/` and `.claude.local.md` are gitignored — local Claude Code config won't be committed
diff --git a/README.md b/README.md
index 010fba6..83c3c89 100644
--- a/README.md
+++ b/README.md
@@ -27,7 +27,7 @@ MIZAN is a **personal AI assistant** you can run on your own computer. Unlike Ch
- **Anyone can extend it** — add new abilities with simple plugins
- **Works with any AI** — Anthropic Claude, OpenAI, OpenRouter (300+ models), or local Ollama
-> Think of it as your personal AI employee that can use tools, remember things, and get better over time.
+> Think of it as your personal AI employee that can use tools, remember things, and get better over time — powered by a 7-layer Quranic Cognitive Architecture (QALB-7).
---
@@ -113,14 +113,16 @@ make dev # Start backend + frontend
| Feature | What It Means |
|---------|---------------|
+| **QALB-7 Cognitive Pipeline** | 7-layer architecture: ethics → deliberation → emotion → conviction → metacognition |
+| **Developmental Stages** | Agents grow from Nutfah (5 tools, 5 turns) to Khalq Akhar (all tools, 25 turns) |
+| **5-Layer Memory Pyramid** | Unified query across episodic, semantic, neural pathways, vectors, and knowledge graph |
+| **Causal Reasoning** | Pearl's 3-rung causal ladder: observation, intervention, counterfactual |
| **Plugin system** | Add new abilities with a simple Python file |
-| **Event bus** | Modules communicate without knowing about each other |
-| **Hook system** | Modify any data flowing through the system |
-| **Middleware pipeline** | Intercept and process requests/responses |
-| **REST + WebSocket API** | Full API for building custom integrations |
-| **Multi-agent system** | Multiple AI agents collaborate on complex tasks |
-| **Security built-in** | JWT auth, rate limiting, sandboxing, audit logs |
-| **Self-healing diagnostics** | Built-in doctor system to detect and fix issues |
+| **Event bus + Hooks** | Decoupled communication — modify data at any point in the pipeline |
+| **REST + WebSocket API** | Full API with cognitive metadata streamed in real-time |
+| **Multi-agent Shura** | Agents consult via Shura Council for complex decisions |
+| **Self-healing (Lawwama)** | Immune memory, adaptive checkpoints, auto-package-install |
+| **Security (Wali)** | JWT auth, rate limiting, sandboxing, SSRF block, audit logs |
---
@@ -203,53 +205,132 @@ See the [Plugin Development Guide](docs/) for the full reference.
## Architecture
-```
-┌─────────────────────────────────────────────────────────────────┐
-│ MIZAN Architecture │
-├─────────────────────────────────────────────────────────────────┤
-│ │
-│ YOU (Browser/Terminal/Telegram/Discord/Slack/WhatsApp) │
-│ │ │
-│ ▼ │
-│ ┌──────────────────────────────────────────────────────────┐ │
-│ │ Gateway (REST API + WebSocket) │ │
-│ │ Auth · Rate Limiting · Input Validation · CORS │ │
-│ └────────────────────────┬─────────────────────────────────┘ │
-│ │ │
-│ ┌────────────────────────▼─────────────────────────────────┐ │
-│ │ Plugin System │ │
-│ │ Events (Nida') · Hooks (Ta'liq) · Middleware (Silsilah) │ │
-│ │ Any plugin can listen, modify, or extend │ │
-│ └────────────────────────┬─────────────────────────────────┘ │
-│ │ │
-│ ┌────────────────────────▼─────────────────────────────────┐ │
-│ │ Agent System (Multi-Agent) │ │
-│ │ ┌────────┐ ┌──────────┐ ┌─────────┐ ┌────────┐ │ │
-│ │ │ Hafiz │ │ Mubashir │ │ Mundhir │ │ Katib │ + Any │ │
-│ │ │General │ │ Browser │ │Research │ │ Code │ Custom │ │
-│ │ └───┬────┘ └────┬─────┘ └────┬────┘ └───┬────┘ │ │
-│ │ └──────┬────┘────────────┘───────────┘ │ │
-│ │ ▼ │ │
-│ │ ┌──────────────────────────────────────────────┐ │ │
-│ │ │ Agentic Loop (Think → Use Tools → Repeat) │ │ │
-│ │ │ Up to 15 autonomous iterations per task │ │ │
-│ │ └──────────────────────────────────────────────┘ │ │
-│ └──────────────────────────────────────────────────────────┘ │
-│ │ │
-│ ┌───────────┬────────────▼──────────┬──────────────────────┐ │
-│ │ Memory │ LLM Providers │ Skills & Tools │ │
-│ │ SQLite │ Claude/GPT/Gemini/ │ Web Browse, Code, │ │
-│ │ 3-tier │ Llama/300+ models │ File, HTTP + Custom │ │
-│ └───────────┴───────────────────────┴──────────────────────┘ │
-│ │
-│ ┌──────────────────────────────────────────────────────────┐ │
-│ │ Security Layer (Wali Guardian) │ │
-│ │ JWT Auth · Rate Limit · Sandbox · SSRF Block · Audit │ │
-│ └──────────────────────────────────────────────────────────┘ │
-└─────────────────────────────────────────────────────────────────┘
-```
-
-### How Everything Connects (No Coupling!)
+MIZAN implements a **7-layer Quranic Cognitive Architecture (QALB-7)** — a bio-inspired AI system where each cognitive module maps to a concept from Islamic psychology.
+
+### System Overview
+
+```
+┌──────────────────────────────────────────────────────────────────┐
+│ MIZAN Architecture │
+├──────────────────────────────────────────────────────────────────┤
+│ │
+│ YOU (Browser / Terminal / Telegram / Discord / Slack / WhatsApp) │
+│ │ │
+│ ▼ │
+│ ┌───────────────────────────────────────────────────────────┐ │
+│ │ Gateway (REST API + WebSocket) │ │
+│ │ Auth · Rate Limiting · Input Validation · CORS │ │
+│ └──────────────────────┬────────────────────────────────────┘ │
+│ │ │
+│ ┌──────────────────────▼────────────────────────────────────┐ │
+│ │ Plugin System (Decoupled) │ │
+│ │ Events (Nida') · Hooks (Ta'liq) · Middleware (Silsilah) │ │
+│ └──────────────────────┬────────────────────────────────────┘ │
+│ │ │
+│ ┌──────────────────────▼────────────────────────────────────┐ │
+│ │ QALB-7 Cognitive Pipeline │ │
+│ │ │ │
+│ │ Fitrah ──► Nafs Triad ──► Qalb Processor ──► Fu'ad ──► │ │
+│ │ (Ethics) (Deliberate) (Modulate LLM) (Convict) │ │
+│ │ │ │
+│ │ ──► Lubb ──► Developmental Gate ──► Causal Engine │ │
+│ │ (Meta) (Capability Gate) (Why/What-if) │ │
+│ └──────────────────────┬────────────────────────────────────┘ │
+│ │ │
+│ ┌──────────────────────▼────────────────────────────────────┐ │
+│ │ Agent System (Multi-Agent + Shura Council) │ │
+│ │ ┌────────┐ ┌──────────┐ ┌─────────┐ ┌────────────────┐ │ │
+│ │ │ Hafiz │ │ Mubashir │ │ Mundhir │ │ Khalifah │ │ │
+│ │ │General │ │ Browser │ │Research │ │ SuperAgent │ │ │
+│ │ └───┬────┘ └────┬─────┘ └────┬────┘ └───┬────────────┘ │ │
+│ │ └───────┬────┘───────────┘───────────┘ │ │
+│ │ ▼ │ │
+│ │ ┌──────────────────────────────────────────────────┐ │ │
+│ │ │ Agentic Loop (Think → Tool → Lawwama → Repeat) │ │ │
+│ │ │ 5–25 turns (gated by Developmental Stage) │ │ │
+│ │ └──────────────────────────────────────────────────┘ │ │
+│ └────────────────────────────────────────────────────────────┘ │
+│ │ │
+│ ┌──────────┬───────────▼────────────┬────────────────────────┐ │
+│ │ Memory │ LLM Providers │ Skills & Tools │ │
+│ │ Pyramid │ Claude / GPT / Gemini │ Web, Code, File, │ │
+│ │ (5-layer)│ Llama / 300+ models │ SSH, HTTP + Custom │ │
+│ └──────────┴────────────────────────┴────────────────────────┘ │
+│ │
+│ ┌────────────────────────────────────────────────────────────┐ │
+│ │ Security Layer (Wali Guardian) │ │
+│ │ JWT Auth · Rate Limit · Sandbox · SSRF Block · Audit Log │ │
+│ └────────────────────────────────────────────────────────────┘ │
+└──────────────────────────────────────────────────────────────────┘
+```
+
+### QALB-7 Cognitive Modules
+
+Each agent processes every task through these cognitive layers:
+
+| # | Module | Arabic | Purpose | File |
+|---|--------|--------|---------|------|
+| 1 | **Fitrah** | فطرة | Innate ethical guardrails (NO_HARM, TRUTH, JUSTICE) | `core/fitrah.py` |
+| 2 | **Nafs Triad** | نفس | Three competing inner voices (Ammara/Lawwama/Mutmainna) deliberate on approach | `core/nafs_triad.py` |
+| 3 | **Qalb Processor** | قلب | Cardiac oscillation — alternates between focused (Qabd) and creative (Bast) states, modulating LLM temperature and token limits | `core/qalb_processor.py` |
+| 4 | **Fu'ad** | فؤاد | Bayesian conviction engine — evidence accumulation from impression to conviction | `core/fuad.py` |
+| 5 | **Lubb** | لبّ | Metacognition — compresses reasoning traces, checks coherence, detects cognitive bias | `core/lubb.py` |
+| 6 | **Developmental Gate** | أطوار | Progressive capability gating (7 stages from Nutfah to Khalq Akhar) — controls tools, turn limits, autonomy | `core/developmental_stages.py` |
+| 7 | **Causal Engine** | سببية | Pearl's causal ladder — observation, intervention ("what if I do X?"), counterfactual reasoning | `reasoning/causal_engine.py` |
+
+### Extension Modules
+
+| Module | Arabic | Purpose | File |
+|--------|--------|---------|------|
+| **Lawwama Self-Healing** | لوّامة | Immune memory, health metrics, adaptive checkpoint intervals | `core/self_healing.py` |
+| **Parallel Agents** | — | Concurrent task scheduling + skill transfer between agents | `core/parallel_agents.py` |
+| **Imagination** | تصوير | Predictive coding — simulate outcomes before acting | `core/imagination.py` |
+| **Creativity** | إبداع | 5 creative modes + fitness landscape mathematics | `core/creativity.py` |
+| **Dream Engine** | منام | Offline memory consolidation (NREM replay + REM recombination) | `core/dream_engine.py` |
+| **Shura Council** | شورى | Multi-agent consultation for complex decisions | `agents/shura_council.py` |
+| **Perpetual Rotation** | دورة | Agent rotation and load balancing | `agents/perpetual_rotation.py` |
+
+### Memory Architecture (5-Layer Pyramid)
+
+All memory layers are queried through a unified `MemoryPyramid`:
+
+| Layer | Module | Purpose |
+|-------|--------|---------|
+| **Dhikr** | `memory/dhikr.py` | Three-tier persistent memory (episodic, semantic, procedural) |
+| **Masalik** | `memory/masalik.py` | Neural pathway network with spreading activation |
+| **Lawh al-Mahfuz** | `memory/lawh_mahfuz.py` | Immutable core memory with triple-checksum integrity |
+| **VectorStore** | `memory/vector_store.py` | Semantic embedding search (ChromaDB) |
+| **KnowledgeGraph** | `memory/knowledge_graph.py` | Entity-relationship graph (SQLite) |
+
+Unified query: `memory/memory_pyramid.py` merges, deduplicates, and ranks results by relevance x certainty x recency.
+
+### Developmental Stages (Nafs Levels 1–7)
+
+Agents grow through seven stages, each unlocking new capabilities:
+
+| Level | Stage | Max Turns | Key Unlocks |
+|-------|-------|-----------|-------------|
+| 1 | **Nutfah** (نطفة) | 5 | Basic tools: bash, read_file, recall_memory |
+| 2 | **Alaqah** (علقة) | 8 | + write_file, http_get |
+| 3 | **Mudghah** (مضغة) | 10 | + python_exec, http_post, delegation |
+| 4 | **Izham** (عظام) | 12 | + create_agent, causal reasoning (rung 2) |
+| 5 | **Lahm** (لحم) | 15 | All tools, causal rung 3, Lubb metacognition |
+| 6 | **Nafkh** (نفخ) | 20 | Full metacognition |
+| 7 | **Khalq Akhar** (خلق آخر) | 25 | Full autonomy |
+
+### Cognitive Metadata in the UI
+
+Every assistant response includes a **CognitiveBar** showing:
+- **Qalb** state (Qabd/Bast/Khushu) with confidence
+- **Yaqin** certainty level (ʿIlm al-Yaqin / ʿAyn al-Yaqin / Ḥaqq al-Yaqin)
+- **Lubb** quality assessment (confident / hedged / uncertain)
+- **Ruh** energy percentage
+- **Nafs** level and name badge
+- **Lawwama** repair indicator (when self-healing is active)
+
+Expandable for detailed signals, bias flags, and evidence lists.
+
+### Decoupled Communication
```
Plugin A ──────► Event Bus ◄────── Plugin B
@@ -588,55 +669,82 @@ make docker-down # Stop all Docker services
```
mizan/
├── backend/
-│ ├── api/main.py # FastAPI server + WebSocket + all routes
+│ ├── api/main.py # FastAPI server + WebSocket + all routes
│ ├── agents/
-│ │ ├── base.py # Base agent with agentic loop (Think → Tool → Repeat)
-│ │ ├── specialized.py # Browser, Research, Code agents
-│ │ └── federation.py # Agent-to-agent communication
+│ │ ├── base.py # Base agent with QALB-7 agentic loop
+│ │ ├── specialized.py # Browser, Research, Code, SuperAgent (Khalifah)
+│ │ ├── federation.py # Agent-to-agent communication
+│ │ ├── shura_council.py # Multi-agent consultation
+│ │ └── perpetual_rotation.py # Agent rotation & load balancing
│ ├── core/
-│ │ ├── events.py # Event bus — decoupled communication
-│ │ ├── hooks.py # Hook system — data transformation
-│ │ ├── plugins.py # Plugin manager — extend without touching core
-│ │ ├── middleware.py # Middleware pipeline
-│ │ ├── qalb.py # Emotional intelligence engine
-│ │ ├── ruh_engine.py # Energy/vitality management
-│ │ ├── tawbah.py # Error recovery protocol
-│ │ ├── ihsan.py # Proactive excellence suggestions
-│ │ ├── sabr.py # Patience engine for long tasks
-│ │ └── shukr.py # Strength reinforcement
+│ │ ├── fitrah.py # Innate ethical guardrails
+│ │ ├── nafs_triad.py # 3-voice deliberation (Ammara/Lawwama/Mutmainna)
+│ │ ├── qalb_processor.py # Cardiac oscillation → LLM param modulation
+│ │ ├── fuad.py # Bayesian conviction formation
+│ │ ├── lubb.py # Metacognition: compress, cohere, debias
+│ │ ├── developmental_stages.py # 7-stage capability gating (Nutfah→Khalq Akhar)
+│ │ ├── self_healing.py # Lawwama immune system + health metrics
+│ │ ├── parallel_agents.py # Concurrent task scheduling + skill transfer
+│ │ ├── imagination.py # Predictive coding engine
+│ │ ├── creativity.py # 5 creative modes + landscape math
+│ │ ├── dream_engine.py # Offline memory consolidation (NREM+REM)
+│ │ ├── qalb.py # Emotional intelligence (sentiment)
+│ │ ├── ruh_engine.py # Energy/vitality management
+│ │ ├── tawbah.py # Error recovery protocol
+│ │ ├── ihsan.py # Proactive excellence suggestions
+│ │ ├── sabr.py # Patience engine for long tasks
+│ │ ├── shukr.py # Strength reinforcement
+│ │ ├── events.py # Event bus — decoupled communication
+│ │ ├── hooks.py # Hook system — data transformation
+│ │ ├── plugins.py # Plugin manager
+│ │ └── middleware.py # Middleware pipeline
│ ├── qca/
-│ │ ├── engine.py # 7-layer Quranic Cognitive Architecture
-│ │ ├── yaqin_engine.py # Certainty/confidence tracking
-│ │ ├── cognitive_methods.py # Reasoning method selection
-│ │ └── roots.py # Semantic root analysis (ISM layer)
-│ ├── providers.py # Unified LLM provider (Claude/GPT/Ollama/300+)
+│ │ ├── engine.py # 7-layer QCA integration
+│ │ ├── yaqin_engine.py # Certainty/confidence tracking
+│ │ ├── cognitive_methods.py # Reasoning method selection
+│ │ └── roots.py # Semantic root analysis (ISM layer)
+│ ├── providers.py # Unified LLM provider (Claude/GPT/Ollama/300+)
│ ├── memory/
-│ │ ├── dhikr.py # Three-tier persistent memory
-│ │ └── masalik.py # Neural pathway network (bio-inspired)
-│ ├── security/ # Auth, permissions, sandboxing
-│ ├── skills/ # Extensible skill registry
-│ │ ├── base.py # Skill base class
-│ │ ├── registry.py # Skill discovery & loading
-│ │ └── builtin/ # Built-in skills
-│ ├── gateway/channels/ # Telegram, Discord, Slack, WhatsApp adapters
-│ ├── automation/ # Cron scheduler + webhook triggers
-│ ├── doctor.py # Self-healing diagnostic system
-│ ├── settings.py # Configuration (env vars, pydantic-settings)
-│ └── cli.py # Terminal interface
+│ │ ├── dhikr.py # Three-tier persistent memory
+│ │ ├── masalik.py # Neural pathway network (spreading activation)
+│ │ ├── lawh_mahfuz.py # Immutable memory (triple-checksum)
+│ │ ├── memory_pyramid.py # Unified 5-layer query engine
+│ │ ├── vector_store.py # Semantic embeddings (ChromaDB)
+│ │ ├── knowledge_graph.py # Entity-relationship graph
+│ │ └── living_memory.py # Adaptive memory lifecycle
+│ ├── reasoning/
+│ │ ├── aql_engine.py # Arabic Query Language reasoning
+│ │ ├── causal_engine.py # Pearl's 3-rung causal ladder
+│ │ ├── planner.py # Task planning
+│ │ └── context_manager.py # Context window management
+│ ├── security/ # Auth, permissions, sandboxing
+│ ├── skills/ # Extensible skill registry
+│ │ ├── builtin/ # Built-in skills (web, code, SSH, cloud)
+│ │ ├── base.py # Skill base class
+│ │ └── registry.py # Skill discovery & loading
+│ ├── knowledge/ # Knowledge base management
+│ ├── gateway/channels/ # Telegram, Discord, Slack, WhatsApp adapters
+│ ├── automation/ # Cron scheduler + webhook triggers
+│ ├── doctor.py # Self-healing diagnostic system
+│ ├── settings.py # Configuration (env vars, pydantic-settings)
+│ └── cli.py # Terminal interface
├── frontend/src/
-│ ├── App.tsx # Main UI
-│ ├── pages/ # Feature pages (Plugins, Providers, Developer, etc.)
-│ ├── hooks/ # API & WebSocket hooks
-│ └── types.ts # TypeScript types
-├── plugins/ # Your custom plugins go here!
-│ ├── hello_world/ # Example plugin
-│ └── request_logger/ # Example monitoring plugin
-├── docs/ # Documentation site
-├── tests/ # Test suite (484 tests)
-├── docker/ # Docker configs
-├── pyproject.toml # Python package config
-├── Makefile # Development commands
-└── docker-compose.yml # Full-stack deployment
+│ ├── App.tsx # Main UI + WebSocket handler
+│ ├── components/
+│ │ ├── ChatMessage.tsx # Chat bubbles + CognitiveBar pills
+│ │ ├── AgentCard.tsx # Agent card with Nafs + Ruh bars
+│ │ ├── Sidebar.tsx # Navigation sidebar
+│ │ └── ... # Toast, Markdown, Icons, etc.
+│ ├── pages/ # Feature pages (Plugins, Providers, Settings, etc.)
+│ ├── hooks/ # API & WebSocket hooks
+│ └── types.ts # TypeScript types (CognitiveMetadata, etc.)
+├── plugins/ # Your custom plugins go here!
+├── docs/ # Documentation
+├── tests/ # Test suite
+├── docker/ # Docker configs
+├── pyproject.toml # Python package config
+├── Makefile # Development commands
+└── docker-compose.yml # Full-stack deployment
```
---
diff --git a/backend/agents/base.py b/backend/agents/base.py
index 07866ba..9946f6a 100644
--- a/backend/agents/base.py
+++ b/backend/agents/base.py
@@ -15,6 +15,7 @@
"""
import asyncio
+import inspect
import json
import logging
import os
@@ -35,7 +36,7 @@
from core.sabr import SabrEngine
from core.shukr import ShukrSystem
from core.tawbah import TawbahProtocol
-from providers import create_provider, get_default_model
+from providers import create_provider, get_default_model, normalize_model_for_provider
from qca.cognitive_methods import IjmaEngine, select_method
from qca.engine import QCAEngine
from qca.yaqin_engine import YaqinEngine
@@ -45,6 +46,26 @@
validate_url,
)
+# QALB-7 architecture modules — core
+from core.fitrah import FitrahSystem
+from core.nafs_triad import NafsTriad
+from core.qalb_processor import QalbProcessor
+from core.lubb import LubbEngine
+from core.fuad import FuadEngine
+from core.developmental_stages import DevelopmentalGate
+
+# QALB-7 extension modules — parallel, healing, creativity, imagination, dreams
+from core.parallel_agents import QalbParallelScheduler, SkillAutomationTransfer
+from core.self_healing import LawwamaHealingSystem
+from core.imagination import TaswirImaginationEngine, ImaginationMode
+from core.creativity import IbdaCreativityEngine
+from core.dream_engine import ManamDreamEngine
+
+# Living Memory + 24/7 Multi-Agent Collaboration
+from memory.living_memory import LivingMemorySystem
+from agents.shura_council import ShuraCouncil
+from agents.perpetual_rotation import PerpetualRotation
+
logger = logging.getLogger("mizan.agent")
@@ -75,6 +96,9 @@ def __init__(
izn=None,
skill_registry=None,
plugin_manager=None,
+ knowledge_graph=None,
+ context_manager=None,
+ planner=None,
):
self.id = agent_id or str(uuid.uuid4())
self.name = name or f"Agent-{self.id[:8]}"
@@ -90,6 +114,20 @@ def __init__(
self.skill_registry = skill_registry
self.plugin_manager = plugin_manager
+ # Knowledge Graph (Ilm) — entity/relationship store
+ self.knowledge_graph = knowledge_graph
+
+ # Context Manager — token-aware compaction
+ self.context_manager = context_manager
+
+ # Planner (Tafakkur) — task decomposition
+ self.planner = planner
+
+ # Reference to global agent registry (set by API when available)
+ self._agent_registry: dict | None = None
+ self._balancer = None
+ self._shura = None
+
# State tracking
self.state = "resting"
self.current_task: str | None = None
@@ -121,11 +159,45 @@ def __init__(
self.qca = QCAEngine() # 7-layer cognitive architecture
self.cognitive = IjmaEngine() # Cognitive reasoning methods
+ # QALB-7 layer additions
+ self.fitrah = FitrahSystem() # Innate ethical BIOS (immutable axioms)
+ self.nafs_triad = NafsTriad() # Three-voice consciousness deliberation
+ self.qalb_processor = QalbProcessor() # Cardiac oscillation → LLM params
+ self.lubb = LubbEngine() # Metacognition: compress, cohere, debias
+ self.fuad = FuadEngine() # Conviction formation + confidence scoring
+ self.dev_gate = DevelopmentalGate() # Capability gating by nafs_level
+ self._nafs_approach: str = "" # Current dominant Nafs voice instruction
+
+ # QALB-7 extension modules
+ self.parallel_scheduler = QalbParallelScheduler() # Multi-stream parallel processing
+ self.skill_automation = SkillAutomationTransfer() # Cerebellar skill automation
+ self.self_healer = LawwamaHealingSystem() # 4-level self-repair
+ self.imagination = TaswirImaginationEngine() # Mental simulation + counterfactuals
+ self.creativity = IbdaCreativityEngine() # 5-mode creativity engine
+ self.dream_engine = ManamDreamEngine() # Offline memory consolidation
+
+ # Living Memory + 24/7 Multi-Agent Collaboration
+ self.living_memory = LivingMemorySystem() # Novelty-gated 4-level memory
+ self.shura_council = ShuraCouncil() # Multi-agent consultation
+ self.perpetual_rotation = PerpetualRotation() # 24/7 shift rotation
+
+ # Load Fitrah axioms into QCA Lawh Tier 1 (immutable moral foundation)
+ for key, entry in self.fitrah.get_lawh_tier1_entries().items():
+ try:
+ self.qca.lawh.store(
+ key, entry["content"], certainty=1.0, source=entry["source"], tier=1
+ )
+ except Exception:
+ pass
+
# LLM provider — unified interface for Anthropic, OpenRouter, OpenAI, Ollama
- self.ai_model = config.get("model", "claude-opus-4-6") if config else "claude-opus-4-6"
+ default_model = os.getenv("DEFAULT_MODEL", "claude-sonnet-4-20250514")
+ self.ai_model = config.get("model", default_model) if config else default_model
provider_name = os.getenv("LLM_PROVIDER", "") or None
self.ai_client = create_provider(provider=provider_name, model=self.ai_model)
if self.ai_client:
+ # Normalize model ID for the active provider (e.g. Anthropic→OpenRouter format)
+ self.ai_model = normalize_model_for_provider(self.ai_model, self.ai_client.provider_name)
# If no model set in config, use the provider's default
if not config or "model" not in config:
self.ai_model = get_default_model(self.ai_client.provider_name)
@@ -141,7 +213,11 @@ def _register_base_tools(self):
"list_files": self._tool_list_files,
"python_exec": self._tool_python_exec,
"create_agent": self._tool_create_agent,
+ "create_skill": self._tool_create_skill,
"compact_context": self._tool_compact_context,
+ "recall_memory": self._tool_recall_memory,
+ "delegate_task": self._tool_delegate_task,
+ "query_knowledge": self._tool_query_knowledge,
}
def get_tool_schemas(self) -> list[dict]:
@@ -242,14 +318,14 @@ def get_tool_schemas(self) -> list[dict]:
},
{
"name": "create_agent",
- "description": "Create a new specialized agent. Types: browser, research, code, communication, general.",
+ "description": "Create a new specialized agent. Types: super (all tools), browser, research, code, communication, general.",
"input_schema": {
"type": "object",
"properties": {
"name": {"type": "string", "description": "Name for the new agent"},
"type": {
"type": "string",
- "description": "Agent type: browser, research, code, communication, general",
+ "description": "Agent type: super, browser, research, code, communication, general",
"default": "general",
},
"role": {
@@ -261,6 +337,32 @@ def get_tool_schemas(self) -> list[dict]:
"required": ["name"],
},
},
+ {
+ "name": "create_skill",
+ "description": "Dynamically create a new skill/tool at runtime. The skill code will be saved and registered immediately. Use this to add new capabilities on the fly.",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "Skill name (snake_case, e.g. 'my_custom_skill')",
+ },
+ "description": {
+ "type": "string",
+ "description": "What the skill does",
+ },
+ "code": {
+ "type": "string",
+ "description": "Full Python code for the skill class. Must subclass SkillBase.",
+ },
+ "tools": {
+ "type": "object",
+ "description": "Simple tool definitions as {name: {description, params}} for auto-generating a skill without full code.",
+ },
+ },
+ "required": ["name"],
+ },
+ },
{
"name": "compact_context",
"description": "Compact conversation context by summarizing older messages to stay within context window limits. Use when conversation is getting long.",
@@ -276,6 +378,63 @@ def get_tool_schemas(self) -> list[dict]:
"required": ["conversation_history"],
},
},
+ {
+ "name": "recall_memory",
+ "description": "Search stored memories and ingested knowledge (URLs, PDFs, YouTube transcripts). Use when you need information that may have been previously stored or ingested.",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "query": {
+ "type": "string",
+ "description": "Search query to find relevant memories",
+ },
+ "memory_type": {
+ "type": "string",
+ "enum": ["semantic", "episodic", "procedural"],
+ "description": "Filter by memory type (optional)",
+ },
+ "limit": {
+ "type": "integer",
+ "description": "Max results to return (default 5)",
+ "default": 5,
+ },
+ },
+ "required": ["query"],
+ },
+ },
+ {
+ "name": "delegate_task",
+ "description": "Delegate a sub-task to another agent in the federation. Use when a task requires expertise you don't have (e.g., delegate coding to Katib, browsing to Mubashir).",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "task": {
+ "type": "string",
+ "description": "The task to delegate to another agent",
+ },
+ "preferred_role": {
+ "type": "string",
+ "enum": ["code", "browser", "research", "communication", "general"],
+ "description": "Preferred agent role for this task",
+ },
+ },
+ "required": ["task"],
+ },
+ },
+ {
+ "name": "query_knowledge",
+ "description": "Query the knowledge graph for facts about an entity. Use to check what you know before making claims.",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "entity": {
+ "type": "string",
+ "description": "The entity name to query",
+ },
+ },
+ "required": ["entity"],
+ },
+ },
]
all_schemas = schemas + self.TOOL_SCHEMAS
@@ -323,20 +482,36 @@ def avg_duration_ms(self) -> float:
}
def evolve_nafs(self):
- """Evolve the agent's Nafs level (1-7) based on Tazkiyah performance."""
- thresholds = [
- (7, 0.97, 2000),
- (6, 0.95, 1000),
- (5, 0.90, 500),
- (4, 0.85, 250),
- (3, 0.75, 100),
- (2, 0.60, 25),
- ]
- for level, min_rate, min_tasks in thresholds:
- if self.success_rate >= min_rate and self.total_tasks >= min_tasks:
- self.nafs_level = level
- return
- self.nafs_level = 1
+ """Evolve Nafs level via DevelopmentalGate readiness check.
+
+ Delegates to dev_gate.check_upgrade_readiness() which uses
+ NafsProfile.EVOLUTION_THRESHOLDS (success_rate, min_tasks, min_hikmah).
+ """
+ old_level = self.nafs_level
+ report = self.dev_gate.check_upgrade_readiness(self)
+ if report.ready:
+ self.nafs_level = report.target_level
+
+ if self.nafs_level != old_level:
+ logger.info(
+ "[NAFS] %s evolved: %s → %s (L%d→L%d, tazkiyah=%.2f)",
+ self.name,
+ self.NAFS_NAMES.get(old_level, "?"),
+ self.NAFS_NAMES.get(self.nafs_level, "?"),
+ old_level,
+ self.nafs_level,
+ report.tazkiyah_score,
+ )
+
+ @property
+ def max_tool_turns(self) -> int:
+ """Dynamic tool turn limit from DevelopmentalGate stage."""
+ return self.dev_gate.get_capabilities(self.nafs_level).max_turns
+
+ @property
+ def can_delegate(self) -> bool:
+ """Whether agent can delegate — from DevelopmentalGate capabilities."""
+ return self.dev_gate.get_capabilities(self.nafs_level).can_delegate
def _check_tool_permission(self, tool_name: str, params: dict = None) -> dict:
"""Check Izn permissions before tool execution"""
@@ -344,8 +519,8 @@ def _check_tool_permission(self, tool_name: str, params: dict = None) -> dict:
return self.izn.check_permission(self.id, self.role, tool_name, params)
return {"allowed": True, "reason": "No Izn configured", "requires_approval": False}
- # Maximum agentic loop iterations to prevent runaway execution
- MAX_TOOL_TURNS = 15
+ # Maximum agentic loop iterations — dynamically adjusted by Nafs level
+ MAX_TOOL_TURNS = 15 # Fallback default; actual limit comes from max_tool_turns property
async def think(
self, task: str, context: dict = None, stream: bool = False, qalb_reading=None
@@ -359,10 +534,31 @@ async def think(
"""
self.state = "thinking"
- system_prompt = self._build_system_prompt(qalb_reading=qalb_reading)
+ system_prompt = await self._build_system_prompt(qalb_reading=qalb_reading)
messages = self._build_messages(task, context)
tool_schemas = self.get_tool_schemas()
+ # DevelopmentalGate — filter available tools to this agent's capability level
+ tool_schemas = self.dev_gate.filter_tool_schemas(tool_schemas, self.nafs_level)
+
+ # QalbProcessor — compute cardiac oscillation state → LLM params
+ qalb_proc_output = self.qalb_processor.process(
+ task=task,
+ emotional_state=qalb_reading.state.value if qalb_reading else "neutral",
+ nafs_level=self.nafs_level,
+ complexity=self.ruh.classify_task_complexity(task),
+ )
+ self._qalb_params = {
+ "max_tokens": qalb_proc_output.max_tokens,
+ "temperature": qalb_proc_output.temperature,
+ }
+ logger.debug(
+ "[QALB] State=%s max_tokens=%d temp=%.2f",
+ qalb_proc_output.state.value,
+ qalb_proc_output.max_tokens,
+ qalb_proc_output.temperature,
+ )
+
# QCA Layer 1-4: Process input through Sam'+Basar+Fu'ad+ISM
qca_input = self.qca.process_input(task[:500])
if qca_input.get("roots_identified"):
@@ -380,6 +576,21 @@ async def think(
)
messages[-1]["content"] += f"\n[Lawh Memory: {mem_context}]"
+ # Context Manager: inject Dhikr memories for grounding
+ if self.context_manager and self.memory:
+ try:
+ relevant_memories = await self.memory.recall(task, top_k=3)
+ if relevant_memories:
+ memory_dicts = [
+ {"type": m.memory_type, "content": m.content}
+ for m in relevant_memories
+ if hasattr(m, "content")
+ ]
+ if memory_dicts:
+ messages = self.context_manager.inject_memory(messages, memory_dicts)
+ except Exception:
+ pass
+
if self.ai_client:
try:
async for chunk in self._agentic_loop(
@@ -431,11 +642,17 @@ async def _agentic_loop(
accumulated_text = ""
tool_count = 0
- for turn in range(self.MAX_TOOL_TURNS):
+ for turn in range(self.max_tool_turns):
+ # Qalb-modulated LLM params (cardiac oscillation: QABD=analytical, BAST=creative)
+ qalb_params = getattr(self, "_qalb_params", {})
+ max_tokens = qalb_params.get("max_tokens", 4096)
+ temperature = qalb_params.get("temperature", 0.5)
+
# Call the model via unified provider
response = self.ai_client.create(
model=self.ai_model,
- max_tokens=4096,
+ max_tokens=max_tokens,
+ temperature=temperature,
system=system_prompt,
messages=messages,
tools=tool_schemas,
@@ -487,7 +704,9 @@ async def _agentic_loop(
if not has_tool_use or response.stop_reason == "end_turn":
# Furqan: Validate final output before delivery
if accumulated_text:
- overall_confidence = min(0.95, 0.5 + 0.1 * tool_count)
+ overall_confidence = self.fuad.compute_confidence(
+ tool_count=tool_count, tool_results=tool_results,
+ )
furqan_report = self.qca.furqan.validate_and_express(
accumulated_text[:200],
overall_confidence,
@@ -514,10 +733,10 @@ async def _agentic_loop(
messages.append({"role": "assistant", "content": response.content})
messages.append({"role": "user", "content": tool_results})
- # Lawwama self-correction checkpoint every 3 turns
- if turn > 0 and turn % 3 == 0:
+ # Lawwama self-correction checkpoint — health-based interval
+ if self.self_healer.should_checkpoint(turn, self.max_tool_turns):
lawwama_prompt = (
- f"[Lawwama checkpoint — turn {turn}/{self.MAX_TOOL_TURNS}] "
+ f"[Lawwama checkpoint — turn {turn}/{self.max_tool_turns}] "
"Pause and self-assess: Are you making progress toward the goal? "
"Is there a more efficient approach? Correct course if needed."
)
@@ -526,16 +745,39 @@ async def _agentic_loop(
"[LAWWAMA] Self-correction checkpoint at turn %d for %s", turn, self.name
)
- logger.warning(f"[FIKR] Agent {self.name} hit MAX_TOOL_TURNS ({self.MAX_TOOL_TURNS})")
+ logger.warning(f"[FIKR] Agent {self.name} hit max_tool_turns ({self.max_tool_turns})")
async def _execute_tool_safe(self, tool_name: str, params: dict) -> Any:
- """Execute a tool with Wali security checks.
+ """Execute a tool with Wali security checks and self-healing.
Tool resolution order:
1. Agent's own tools (self.tools) — bash, http_get, read_file, etc.
2. Skill tools (self.skill_registry) — web_browse, analyze_csv, notebook_*, etc.
3. Plugin tools (self.plugin_manager) — dynamically loaded plugin capabilities.
+
+ Self-healing (Tawbah):
+ - Auto-installs missing Python packages and retries
+ - Adapts parameter passing for skill tools (dict vs kwargs)
+ - Logs errors for learning
"""
+ # Anti-hallucination: validate tool inputs before execution
+ validation_error = self._validate_tool_inputs(tool_name, params)
+ if validation_error:
+ logger.warning("[VALIDATION] Tool input rejected: %s — %s", tool_name, validation_error)
+ return {"error": f"Input validation failed: {validation_error}"}
+
+ # Fitrah — ethical axiom gate (immutable innate disposition check)
+ fitrah_violations = self.fitrah.check_action(
+ f"{tool_name} {json.dumps(params)[:200]}"
+ )
+ critical_violations = [v for v in fitrah_violations if v["severity"] == "critical"]
+ if critical_violations:
+ v = critical_violations[0]
+ logger.warning(
+ "[FITRAH] Blocked tool %s: axiom=%s — %s", tool_name, v["axiom"], v["reason"]
+ )
+ return {"error": f"Fitrah violation [{v['axiom']}]: {v['principle']}"}
+
# Check Izn permissions
perm = self._check_tool_permission(tool_name, params)
if not perm["allowed"]:
@@ -546,10 +788,10 @@ async def _execute_tool_safe(self, tool_name: str, params: dict) -> Any:
# 1. Agent's own built-in tools
if tool_name in self.tools:
- try:
- return await self.tools[tool_name](**params)
- except Exception as e:
- return {"error": str(e)}
+ result = await self._invoke_with_healing(
+ self.tools[tool_name], params, tool_name, invoke_style="kwargs"
+ )
+ return result
# 2. Skill tools (Hikmah — wisdom skills from SkillRegistry)
if self.skill_registry:
@@ -557,10 +799,11 @@ async def _execute_tool_safe(self, tool_name: str, params: dict) -> Any:
skill_tools = self.skill_registry.get_all_tools()
if tool_name in skill_tools:
tool_fn = skill_tools[tool_name]
- result = tool_fn(**params)
- # Handle both sync and async tool functions
- if hasattr(result, "__await__"):
- return await result
+ # Auto-detect invoke style from function signature
+ style = self._detect_invoke_style(tool_fn)
+ result = await self._invoke_with_healing(
+ tool_fn, params, tool_name, invoke_style=style
+ )
return result
except Exception as e:
logger.error(f"[HIKMAH] Skill tool '{tool_name}' failed: {e}")
@@ -572,12 +815,11 @@ async def _execute_tool_safe(self, tool_name: str, params: dict) -> Any:
plugin_tools = self.plugin_manager.get_all_tools()
if tool_name in plugin_tools:
tool_info = plugin_tools[tool_name]
- # Plugin tools are stored as {"handler": fn, "schema": ...}
handler = tool_info if callable(tool_info) else tool_info.get("handler")
if handler:
- result = handler(**params)
- if hasattr(result, "__await__"):
- return await result
+ result = await self._invoke_with_healing(
+ handler, params, tool_name, invoke_style="dict"
+ )
return result
except Exception as e:
logger.error(f"[WAHY] Plugin tool '{tool_name}' failed: {e}")
@@ -585,12 +827,216 @@ async def _execute_tool_safe(self, tool_name: str, params: dict) -> Any:
return {"error": f"Unknown tool: {tool_name}"}
+ def _validate_tool_inputs(self, tool_name: str, params: dict) -> str | None:
+ """Validate tool inputs to catch hallucinated values before execution.
+
+ Returns an error string if validation fails, None if OK.
+ """
+ if not isinstance(params, dict):
+ return f"Expected dict params, got {type(params).__name__}"
+
+ # URL validation for HTTP tools
+ if tool_name in ("http_get", "http_post"):
+ url = params.get("url", "")
+ if url and not url.startswith(("http://", "https://")):
+ return f"Invalid URL scheme: {url[:50]}"
+
+ # Path traversal check for file tools
+ if tool_name in ("read_file", "write_file", "list_files"):
+ path = params.get("path", "") or params.get("filename", "")
+ if path and ".." in path:
+ return f"Path traversal detected: {path[:50]}"
+
+ # Command safety for bash
+ if tool_name == "bash":
+ cmd = params.get("command", "")
+ if cmd:
+ # Block obviously dangerous patterns
+ danger_patterns = ["rm -rf /", "rm -rf ~", ":(){ :|:&", "mkfs", "> /dev/sd"]
+ for pattern in danger_patterns:
+ if pattern in cmd:
+ return f"Dangerous command blocked: {pattern}"
+
+ # Circuit breaker: track repeated tool failures
+ failure_key = f"{tool_name}:{str(params)[:100]}"
+ if not hasattr(self, "_tool_failure_counts"):
+ self._tool_failure_counts: dict[str, int] = {}
+ count = self._tool_failure_counts.get(failure_key, 0)
+ if count >= 3:
+ return f"Circuit breaker: {tool_name} has failed {count} times with same inputs"
+
+ return None
+
+ @staticmethod
+ def _detect_invoke_style(fn: Callable) -> str:
+ """Detect whether fn expects a single dict param or individual kwargs.
+
+ Inspects the function signature:
+ - If it has a single parameter (ignoring self) named 'params' or
+ annotated as dict → "dict" style (call fn(params))
+ - Otherwise → "kwargs" style (call fn(**params))
+ """
+ try:
+ sig = inspect.signature(fn)
+ # Filter out 'self' for bound methods
+ sig_params = [
+ p for name, p in sig.parameters.items()
+ if name != "self" and p.kind in (
+ inspect.Parameter.POSITIONAL_OR_KEYWORD,
+ inspect.Parameter.POSITIONAL_ONLY,
+ inspect.Parameter.KEYWORD_ONLY,
+ )
+ ]
+ if len(sig_params) == 1:
+ p = sig_params[0]
+ if p.name == "params" or p.annotation is dict:
+ return "dict"
+ return "kwargs"
+ except (ValueError, TypeError):
+ return "kwargs"
+
+ async def _invoke_with_healing(
+ self, fn: Callable, params: dict, tool_name: str, invoke_style: str = "kwargs"
+ ) -> Any:
+ """
+ Invoke a tool function with automatic self-healing (Tawbah).
+
+ Handles:
+ - Parameter style mismatch (dict vs kwargs) — auto-adapts
+ - Missing Python packages — auto-installs via pip and retries
+ - General errors — logs for learning and returns structured error
+
+ invoke_style: "kwargs" = fn(**params), "dict" = fn(params)
+ """
+ # Retries scale with developmental stage: L1-2=1, L3-4=2, L5+=3
+ caps = self.dev_gate.get_capabilities(self.nafs_level)
+ max_retries = min(3, 1 + caps.max_turns // 10)
+ last_error = None
+
+ for attempt in range(max_retries + 1):
+ try:
+ if invoke_style == "kwargs":
+ result = fn(**params)
+ else:
+ result = fn(params)
+
+ # Handle both sync and async
+ if hasattr(result, "__await__"):
+ return await result
+ return result
+
+ except TypeError as e:
+ err_str = str(e)
+ last_error = e
+
+ # Self-heal: wrong invoke style — flip and retry on ANY TypeError
+ # Common symptoms: "unexpected keyword argument", "positional argument",
+ # "quote_from_bytes() expected bytes", "expected str not dict", etc.
+ if attempt < max_retries:
+ flipped = "dict" if invoke_style == "kwargs" else "kwargs"
+ logger.info(
+ f"[TAWBAH] Tool '{tool_name}' TypeError: {err_str}, "
+ f"switching {invoke_style} → {flipped} (attempt {attempt + 1})"
+ )
+ invoke_style = flipped
+ continue
+
+ # Exhausted retries
+ logger.error(f"[TAWBAH] Tool '{tool_name}' TypeError after retries: {e}")
+ self._record_tool_failure(tool_name, params)
+ return {"error": str(e)}
+
+ except Exception as e:
+ err_str = str(e)
+ last_error = e
+
+ # Self-heal: missing Python module — auto-install and retry
+ if "ModuleNotFoundError" in type(e).__name__ or "No module named" in err_str:
+ module_name = self._extract_module_name(err_str)
+ if module_name and attempt < max_retries:
+ logger.info(
+ f"[TAWBAH] Auto-installing missing module: {module_name}"
+ )
+ install_result = await self._auto_install_package(module_name)
+ if install_result:
+ continue
+
+ logger.error(f"[TAWBAH] Tool '{tool_name}' error: {e}")
+ self._record_tool_failure(tool_name, params)
+ return {"error": str(e)}
+
+ self._record_tool_failure(tool_name, params)
+ return {"error": f"Tool '{tool_name}' failed after {max_retries + 1} attempts: {last_error}"}
+
+ def _record_tool_failure(self, tool_name: str, params: dict):
+ """Record a tool failure for circuit breaker tracking."""
+ if not hasattr(self, "_tool_failure_counts"):
+ self._tool_failure_counts = {}
+ failure_key = f"{tool_name}:{str(params)[:100]}"
+ self._tool_failure_counts[failure_key] = self._tool_failure_counts.get(failure_key, 0) + 1
+
+ def _extract_module_name(self, error_str: str) -> str | None:
+ """Extract module name from a ModuleNotFoundError message."""
+ import re
+ match = re.search(r"No module named ['\"]([^'\"]+)['\"]", error_str)
+ if match:
+ # Get top-level module (e.g., 'paramiko.client' → 'paramiko')
+ return match.group(1).split(".")[0]
+ return None
+
+ async def _auto_install_package(self, package_name: str) -> bool:
+ """Auto-install a missing Python package via pip (Tawbah self-correction)."""
+ # Safety: only allow known-safe packages
+ safe_packages = {
+ "paramiko", "fabric", "httpx", "requests", "beautifulsoup4",
+ "bs4", "lxml", "pyyaml", "yaml", "toml", "markdown",
+ "pillow", "numpy", "pandas", "aiohttp", "aiofiles",
+ "jinja2", "python-dotenv", "rich", "click", "typer",
+ "pydantic", "fastapi", "uvicorn", "websockets",
+ "redis", "celery", "sqlalchemy", "alembic",
+ "boto3", "google-cloud-storage", "azure-storage-blob",
+ "docker", "kubernetes", "ansible",
+ }
+ if package_name.lower() not in safe_packages:
+ logger.warning(f"[TAWBAH] Package '{package_name}' not in safe list, skipping install")
+ return False
+ try:
+ result = subprocess.run(
+ ["pip", "install", package_name],
+ capture_output=True,
+ text=True,
+ timeout=60,
+ )
+ if result.returncode == 0:
+ logger.info(f"[TAWBAH] Successfully installed {package_name}")
+ return True
+ logger.error(f"[TAWBAH] pip install {package_name} failed: {result.stderr[:500]}")
+ return False
+ except Exception as e:
+ logger.error(f"[TAWBAH] Install failed: {e}")
+ return False
+
async def _structured_reasoning(self, task: str, context: dict = None) -> str:
"""Fallback reasoning without AI"""
return f"Task received: {task}\nContext: {json.dumps(context or {}, indent=2)}\nStatus: Processing without AI provider configured."
- def _build_system_prompt(self, qalb_reading=None) -> str:
- hikmah_str = "\n".join([f"- {h['pattern']}: {h['outcome']}" for h in self.hikmah[-5:]])
+ async def _build_system_prompt(self, qalb_reading=None) -> str:
+ # Merge in-memory hikmah with DB-persisted hikmah
+ db_hikmah = []
+ if self.memory and hasattr(self.memory, "load_hikmah"):
+ try:
+ db_hikmah = await self.memory.load_hikmah(agent_id=self.id, limit=10)
+ except Exception:
+ pass
+
+ combined_hikmah = self.hikmah[-5:]
+ for dh in db_hikmah:
+ if not any(h.get("pattern") == dh["pattern"] for h in combined_hikmah):
+ combined_hikmah.append(dh)
+
+ hikmah_str = "\n".join(
+ [f"- {h['pattern']}: {h.get('outcome', '')}" for h in combined_hikmah[-10:]]
+ )
nafs_name = self.NAFS_NAMES.get(self.nafs_level, "Ammara")
ruh_energy = self.ruh.get_state(self.id).energy if self.ruh else 100
@@ -625,18 +1071,68 @@ def _build_system_prompt(self, qalb_reading=None) -> str:
lawh_stats = self.qca.lawh.stats()
masalik_note = f"Memory: {lawh_stats[2]} active items, {lawh_stats[1]} verified entries"
+ # Knowledge context — unified 5-layer recall (MemoryPyramid) or fallback to pathways
+ knowledge_context = ""
+ if self.memory and self.current_task:
+ try:
+ if hasattr(self.memory, "recall_unified_for_prompt"):
+ unified_ctx = self.memory.recall_unified_for_prompt(self.current_task, top_k=5)
+ if unified_ctx and len(unified_ctx) > 10:
+ knowledge_context = f"\nRelevant Knowledge (unified):\n{unified_ctx}"
+ elif hasattr(self.memory, "recall_pathways"):
+ pathway_ctx = self.memory.recall_pathways(self.current_task, top_k=5)
+ if pathway_ctx and len(pathway_ctx) > 10:
+ knowledge_context = f"\nRelevant Knowledge:\n{pathway_ctx}"
+ except Exception:
+ pass
+
+ # Knowledge Graph — pull factual entities related to current task
+ kg_context = ""
+ if self.knowledge_graph and self.current_task:
+ try:
+ # Extract key terms from task and query the graph
+ task_words = [
+ w for w in self.current_task.split()
+ if len(w) > 3 and w.isalpha()
+ ]
+ kg_facts = []
+ for word in task_words[:5]:
+ result = await self.knowledge_graph.query_entity(word)
+ if result.get("found"):
+ relations = result.get("outgoing", []) + result.get("incoming", [])
+ for rel in relations[:3]:
+ fact = (
+ f"{result['entity']} {rel.get('relation', '→')} "
+ f"{rel.get('target', rel.get('source', '?'))}"
+ )
+ if fact not in kg_facts:
+ kg_facts.append(fact)
+ if kg_facts:
+ kg_context = "\nKnown Facts (from memory):\n" + "\n".join(
+ f"- {f}" for f in kg_facts[:10]
+ )
+ except Exception:
+ pass
+
# Yaqin — epistemic discipline
yaqin_stats = self.yaqin.stats()
yaqin_note = f"Proven patterns: {yaqin_stats['proven_patterns']}, Verifications: {yaqin_stats['total_verifications']}"
- return f"""You are {self.name}, a specialized AI agent in the MIZAN (ميزان) AGI system.
+ custom_prompt = self.config.get("system_prompt", "") if self.config else ""
+
+ # Nafs approach injection
+ nafs_approach_note = (
+ f"\n[Nafs: {self._nafs_approach}]" if self._nafs_approach else ""
+ )
+
+ base_prompt = f"""You are {self.name}, a specialized AI agent in the MIZAN (ميزان) AGI system.
Role: {self.role}
Nafs Level: {self.nafs_level}/7 ({nafs_name})
Ruh Energy: {ruh_energy:.0f}% ({fatigue_label})
Success Rate: {self.success_rate:.1%}
-{masalik_note}
-{yaqin_note}{tone_guidance}{ruh_note}
+{masalik_note}{knowledge_context}{kg_context}
+{yaqin_note}{tone_guidance}{ruh_note}{nafs_approach_note}
You have access to tools. Use them when needed to complete tasks.
@@ -654,13 +1150,40 @@ def _build_system_prompt(self, qalb_reading=None) -> str:
- Never claim certainty beyond what evidence supports (avoid Tughyan — transgression)
- Qualify uncertain claims with appropriate hedging
+Anti-Hallucination Rules (CRITICAL):
+- NEVER invent URLs, API endpoints, file paths, or command syntax. Use tools to verify.
+- If you don't know an API's URL or schema, use http_get or web_search to look it up FIRST.
+- When a tool call fails, read the exact error message — do NOT guess a "fix" without evidence.
+- If you've tried the same action 2+ times and it fails, STOP and report the issue honestly.
+- Prefer tool results over assumptions. If a tool returned data, use that data exactly.
+- When writing code or commands, verify paths and package names exist before using them.
+
Think step by step (Tafakkur - تفكر). Self-correct errors (Lawwama - لوامة)."""
+ if custom_prompt:
+ return custom_prompt + "\n\n" + base_prompt
+ return base_prompt
+
def _build_messages(self, task: str, context: dict = None) -> list[dict]:
messages = []
if context and context.get("history"):
- for hist in context["history"][-5:]:
+ history = context["history"]
+
+ # Context Manager: compact history if it's too large
+ if self.context_manager and len(history) > 15:
+ try:
+ if self.context_manager.needs_compaction(history):
+ history = self.context_manager.compact(history)
+ logger.info(
+ "[CONTEXT] Compacted history from %d to %d messages",
+ len(context["history"]),
+ len(history),
+ )
+ except Exception:
+ pass
+
+ for hist in history[-10:]:
messages.append({"role": hist["role"], "content": hist["content"]})
messages.append(
@@ -721,6 +1244,17 @@ async def execute(
}
self.ruh.consume_energy(self.id, complexity)
+ # NafsTriad — inner voices deliberate → dominant voice sets behavioral approach
+ nafs_decision = self.nafs_triad.deliberate(task, self.nafs_level, complexity)
+ self._nafs_approach = nafs_decision.approach
+ logger.debug(
+ "[NAFS] %s dominant (conf=%.2f dissent=%.2f) for: %s",
+ nafs_decision.dominant_voice,
+ nafs_decision.confidence,
+ nafs_decision.dissent_ratio,
+ task[:60],
+ )
+
# Qalb — detect user emotional state from task text
qalb_reading = self.qalb.analyze(task)
@@ -755,9 +1289,10 @@ async def execute(
await self._tafakkur(task, full_response, True, duration_ms)
- # Yaqin — tag the result with certainty level
+ # Yaqin — tag result with FuadEngine-based confidence
+ base_confidence = self.fuad.compute_confidence(tool_count=0)
yaqin_tag = self.yaqin.tag_inference(
- full_response[:200], confidence=0.5, source="agentic_reasoning"
+ full_response[:200], confidence=base_confidence, source="agentic_reasoning"
)
# Shukr — reinforce this success pattern
@@ -801,6 +1336,97 @@ async def execute(
)
self.memory.masalik.encode(learn_text, importance=0.7)
+ # Knowledge Graph: Extract entities and relationships from successful tasks
+ if self.knowledge_graph:
+ try:
+ task_type = self._classify_task(task)
+ await self.knowledge_graph.add_entity(
+ task[:100], entity_type="task",
+ properties={"agent": self.name, "type": task_type, "success": True},
+ )
+ # Link task to tools used (from Lawh memory)
+ for key in list(self.qca.lawh._tiers.get(3, {}).keys())[-5:]:
+ if key.startswith("TOOL:"):
+ tool_name = key.split(":")[1]
+ await self.knowledge_graph.add_relationship(
+ task[:100], tool_name,
+ rel_type="used_tool", confidence=yaqin_tag.confidence,
+ )
+ except Exception as e:
+ logger.debug("[KG] Entity extraction error: %s", e)
+
+ # Lubb — metacognitive evaluation: compress, coherence, bias detection
+ lubb_report = None
+ try:
+ lubb_report = self.lubb.meta_evaluate(task, full_response, [])
+ if lubb_report.quality.value == "uncertain" and lubb_report.caveat:
+ full_response += f"\n\n{lubb_report.caveat}"
+ if lubb_report.bias_flags:
+ logger.info(
+ "[LUBB] Bias flags for %s: %s",
+ self.name,
+ [b.bias_type for b in lubb_report.bias_flags],
+ )
+ except Exception as lubb_err:
+ logger.debug("[LUBB] Metacognition skipped: %s", lubb_err)
+
+ # Lawwāma self-healing — monitor response integrity, apply repair if needed
+ healing_report = None
+ try:
+ conviction_score = 0.5
+ if lubb_report:
+ conviction_score = lubb_report.coherence.score
+ healing_report = self.self_healer.monitor(
+ response=full_response,
+ task=task,
+ conviction_score=conviction_score,
+ )
+ if healing_report.repair_needed.value > 0:
+ from core.self_healing import RepairLevel
+ full_response, repair_record = self.self_healer.repair(
+ level=healing_report.repair_needed,
+ response=full_response,
+ task=task,
+ errors=healing_report.errors,
+ )
+ logger.info(
+ "[LAWWAMA] Repair L%d applied: health=%.3f",
+ healing_report.repair_needed.value,
+ healing_report.current_health,
+ )
+ except Exception as heal_err:
+ logger.debug("[LAWWAMA] Self-healing skipped: %s", heal_err)
+
+ # Dream consolidation — add task to replay buffer for offline processing
+ try:
+ self.dream_engine.add_memory(
+ content=f"Task: {task[:150]} | Response: {full_response[:150]}",
+ emotional_intensity=abs(qalb_reading.valence) if hasattr(qalb_reading, "valence") else 0.3,
+ novelty=healing_report.hallucination_score if healing_report else 0.3,
+ goal_relevance=0.7,
+ prediction_error=1.0 - (healing_report.current_health if healing_report else 0.8),
+ )
+ except Exception:
+ pass
+
+ # Living Memory — process task+response through novelty gate
+ try:
+ emotional_val = qalb_reading.valence if hasattr(qalb_reading, "valence") else 0.0
+ gate_result = self.living_memory.process_input(
+ content=f"{task[:200]} → {full_response[:200]}",
+ emotional_state=emotional_val,
+ goals=[task[:100]],
+ context=self.name,
+ )
+ if gate_result.decision.value != "ignore":
+ logger.debug(
+ "[LIVING-MEM] %s: %s (sim=%.2f imp=%.2f)",
+ gate_result.decision.value, gate_result.delta_info[:60],
+ gate_result.similarity, gate_result.importance,
+ )
+ except Exception:
+ pass
+
self.evolve_nafs()
self.state = "resting"
self.current_task = None
@@ -818,6 +1444,19 @@ async def execute(
"mizan_label": mizan_label,
"cognitive_method": cognitive_method.value,
}
+ if lubb_report:
+ result["lubb"] = {
+ "quality": lubb_report.quality.value,
+ "coherence_score": lubb_report.coherence.score,
+ "bias_flags": [b.bias_type for b in lubb_report.bias_flags],
+ }
+ if healing_report:
+ result["lawwama"] = {
+ "health": healing_report.current_health,
+ "hallucination_score": healing_report.hallucination_score,
+ "repair_level": healing_report.repair_needed.value,
+ "errors": len(healing_report.errors),
+ }
if ihsan_suggestions:
result["ihsan_suggestions"] = [s.to_dict() for s in ihsan_suggestions]
return result
@@ -826,33 +1465,91 @@ async def execute(
duration_ms = (time.time() - start_time) * 1000
self.error_count += 1
self.total_duration_ms += duration_ms
+ error_str = str(e)
+
+ # ── Tawbah — Full 5-stage error recovery protocol ──
+
+ # Stage 1: Acknowledge the error
+ recovery = self.tawbah.acknowledge(self.id, e, task)
+
+ # Stage 2: Analyze root cause
+ root_cause = self._analyze_error_root_cause(e, task)
+ self.tawbah.analyze(recovery, root_cause)
- # Tawbah — structured error recovery (acknowledge stage)
- recovery = self.tawbah.acknowledge(self.id, str(e), task)
+ # Stage 3: Plan correction
+ correction_plan = self._plan_error_correction(root_cause, task)
+ self.tawbah.plan(recovery, correction_plan.get("description", "Retry with adjusted approach"))
+
+ # Stage 4: Apply fix (retry with correction if possible)
+ retry_result = None
+ can_retry = (
+ recovery.attempts < self.tawbah.MAX_ATTEMPTS
+ and correction_plan.get("retryable", False)
+ )
+ if can_retry:
+ try:
+ self.tawbah.apply(recovery, f"Retrying with correction: {correction_plan.get('fix', '')}")
+ # Attempt corrected execution
+ corrected_response = ""
+ async for chunk in self.think(
+ f"[RETRY] Previous attempt failed with: {error_str[:200]}\n"
+ f"Correction: {correction_plan.get('fix', 'Try a different approach')}\n"
+ f"Original task: {task}",
+ context,
+ stream=bool(stream_callback),
+ qalb_reading=qalb_reading,
+ ):
+ corrected_response += chunk
+ if stream_callback:
+ await stream_callback(chunk)
+
+ if corrected_response:
+ # Stage 5: Verify success
+ self.tawbah.verify(recovery, True, f"Recovered via: {correction_plan.get('fix', '')}")
+ self.success_count += 1
+ retry_result = {
+ "success": True,
+ "result": corrected_response,
+ "duration_ms": (time.time() - start_time) * 1000,
+ "agent": self.name,
+ "tawbah_recovered": True,
+ "correction": correction_plan.get("fix", ""),
+ }
+ except Exception as retry_error:
+ logger.warning("[TAWBAH] Retry failed: %s", retry_error)
+ self.tawbah.verify(recovery, False, str(retry_error))
+
+ if retry_result:
+ await self._tafakkur(task, retry_result.get("result", ""), True, retry_result["duration_ms"])
+ self.state = "resting"
+ self.current_task = None
+ return retry_result
# Yaqin — demote certainty on error
- error_tag = self.yaqin.tag_inference(str(e)[:200], confidence=0.2, source="error")
+ error_tag = self.yaqin.tag_inference(error_str[:200], confidence=0.2, source="error")
# Shukr — record failure for pattern analysis
self.shukr.record_failure(self.id, self._classify_task(task), task[:100])
if self.memory:
- await self.memory.save_task(self.id, task, str(e), False, duration_ms)
+ await self.memory.save_task(self.id, task, error_str, False, duration_ms)
# Masalik: Encode failure too — lower importance, but still learn
if hasattr(self.memory, "masalik"):
self.memory.masalik.encode(f"{task} error {e}", importance=0.3)
- await self._tafakkur(task, str(e), False, duration_ms)
+ await self._tafakkur(task, error_str, False, duration_ms)
self.state = "error"
self.current_task = None
return {
"success": False,
- "error": str(e),
+ "error": error_str,
"duration_ms": duration_ms,
"agent": self.name,
"tawbah": recovery.to_dict() if hasattr(recovery, "to_dict") else str(recovery),
+ "root_cause": root_cause,
+ "correction_plan": correction_plan,
"yaqin": error_tag.to_dict(),
}
@@ -860,42 +1557,181 @@ async def _tafakkur(self, task: str, result: Any, success: bool, duration_ms: fl
"""
Tafakkur (تفكر) - Deep reflection and learning
Quran 3:191: "Those who remember Allah and reflect on the creation..."
+
+ Writes real patterns to both in-memory hikmah and persistent DB.
"""
self.learning_iterations += 1
+ task_type = self._classify_task(task)
pattern = {
- "task_type": self._classify_task(task),
+ "task_type": task_type,
"success": success,
"duration_ms": duration_ms,
"timestamp": datetime.now(UTC).isoformat(),
}
if success and duration_ms < 5000:
+ pattern_text = f"Task type '{task_type}' completed in {duration_ms:.0f}ms"
self.hikmah.append(
{
- "pattern": f"Task type '{pattern['task_type']}' completed in {duration_ms:.0f}ms",
+ "pattern": pattern_text,
"outcome": "success",
"confidence": 0.8,
}
)
-
if len(self.hikmah) > 20:
self.hikmah = self.hikmah[-20:]
+ # Persist to DB for cross-session learning
+ if self.memory and hasattr(self.memory, "store_hikmah"):
+ try:
+ await self.memory.store_hikmah(
+ pattern=pattern_text,
+ context=task[:200],
+ outcome="success" if success else "failure",
+ confidence=0.8 if success else 0.3,
+ source_agent=self.id,
+ )
+ except Exception as e:
+ logger.debug("[HIKMAH] Failed to persist: %s", e)
+ elif not success:
+ # Learn from failures too
+ error_pattern = f"Task type '{task_type}' failed: {str(result)[:100]}"
+ if self.memory and hasattr(self.memory, "store_hikmah"):
+ try:
+ await self.memory.store_hikmah(
+ pattern=error_pattern,
+ context=task[:200],
+ outcome="failure",
+ confidence=0.3,
+ source_agent=self.id,
+ )
+ except Exception as e:
+ logger.debug("[HIKMAH] Failed to persist failure: %s", e)
+
def _classify_task(self, task: str) -> str:
+ """Classify task using cognitive method routing + keyword fallback."""
+ from qca.cognitive_methods import CognitiveMethod
+ method = select_method(task, {})
+ method_map = {
+ CognitiveMethod.TAFAKKUR: "analysis",
+ CognitiveMethod.TADABBUR: "research",
+ CognitiveMethod.ISTIDLAL: "coding",
+ CognitiveMethod.QIYAS: "analysis",
+ CognitiveMethod.IJMA: "general",
+ }
+ mapped = method_map.get(method)
+ if mapped and mapped != "general":
+ return mapped
+
+ # Keyword fallback for domain-specific routing
task_lower = task.lower()
if any(w in task_lower for w in ["code", "script", "python", "js"]):
return "coding"
- elif any(w in task_lower for w in ["search", "find", "browse", "web"]):
+ if any(w in task_lower for w in ["search", "find", "browse", "web"]):
return "research"
- elif any(w in task_lower for w in ["email", "message", "send"]):
+ if any(w in task_lower for w in ["email", "message", "send"]):
return "communication"
- elif any(w in task_lower for w in ["analyze", "review", "check"]):
- return "analysis"
- elif any(w in task_lower for w in ["file", "read", "write", "save"]):
+ if any(w in task_lower for w in ["file", "read", "write", "save"]):
return "file_management"
return "general"
+ def _analyze_error_root_cause(self, error: Exception, task: str) -> str:
+ """Analyze the root cause of an error for Tawbah stage 2."""
+ error_str = str(error)
+ error_type = type(error).__name__
+
+ # Check for known error patterns
+ if "Connection" in error_str or "connect" in error_str.lower():
+ return "network_connectivity"
+ if "timeout" in error_str.lower():
+ return "timeout"
+ if "permission" in error_str.lower() or "denied" in error_str.lower():
+ return "permission_denied"
+ if "not found" in error_str.lower() or "404" in error_str:
+ return "resource_not_found"
+ if "rate limit" in error_str.lower() or "429" in error_str:
+ return "rate_limited"
+ if "No module named" in error_str:
+ return "missing_dependency"
+ if error_type == "TypeError":
+ return "type_mismatch"
+ if error_type == "KeyError":
+ return "missing_key"
+ if "API" in error_str or "api" in error_str.lower():
+ return "api_error"
+
+ # Check prior fixes from Tawbah lessons
+ prior = self.tawbah.has_prior_fix(error_type)
+ if prior:
+ return f"recurring:{prior.get('root_cause', error_type)}"
+
+ return f"unknown:{error_type}"
+
+ def _plan_error_correction(self, root_cause: str, task: str) -> dict:
+ """Create a correction plan for Tawbah stage 3."""
+ plans = {
+ "network_connectivity": {
+ "description": "Network issue — retry after brief pause",
+ "fix": "Wait and retry the request",
+ "retryable": True,
+ },
+ "timeout": {
+ "description": "Operation timed out — retry with simpler approach",
+ "fix": "Simplify the request or increase timeout",
+ "retryable": True,
+ },
+ "permission_denied": {
+ "description": "Permission denied — cannot retry without authorization",
+ "fix": "Report permission issue to user",
+ "retryable": False,
+ },
+ "resource_not_found": {
+ "description": "Resource not found — verify URL/path before retrying",
+ "fix": "Use search tools to find the correct URL or path first",
+ "retryable": True,
+ },
+ "rate_limited": {
+ "description": "Rate limited — wait before retrying",
+ "fix": "Wait and retry with backoff",
+ "retryable": True,
+ },
+ "missing_dependency": {
+ "description": "Missing Python module — auto-install",
+ "fix": "Install missing module and retry",
+ "retryable": True,
+ },
+ "type_mismatch": {
+ "description": "Type error — adapt parameter format",
+ "fix": "Adjust parameter types and retry",
+ "retryable": True,
+ },
+ "api_error": {
+ "description": "API error — verify endpoint and params",
+ "fix": "Look up correct API endpoint using search tools",
+ "retryable": True,
+ },
+ }
+
+ # Match root cause to a plan
+ for key, plan in plans.items():
+ if key in root_cause:
+ return plan
+
+ # Check if we have a prior fix from Tawbah
+ if root_cause.startswith("recurring:"):
+ return {
+ "description": f"Recurring error: {root_cause}",
+ "fix": "Apply previously learned fix",
+ "retryable": True,
+ }
+
+ return {
+ "description": f"Unknown error: {root_cause}",
+ "fix": "Try a completely different approach",
+ "retryable": True,
+ }
+
async def evaluate(self, question: str, context: dict) -> dict:
"""Evaluate a question for Shura council"""
try:
@@ -1082,6 +1918,8 @@ async def _tool_create_agent(
# Normalise type aliases
valid_types = {
+ "super",
+ "khalifah",
"browser",
"mubashir",
"research",
@@ -1114,6 +1952,34 @@ async def _tool_create_agent(
if role:
new_agent.role = role
+ # Register in global agent registry if available
+ if self._agent_registry is not None:
+ self._agent_registry[new_agent.id] = new_agent
+ new_agent._agent_registry = self._agent_registry
+ new_agent._balancer = self._balancer
+ new_agent._shura = self._shura
+ if self._balancer:
+ self._balancer.register(new_agent.id)
+ if self._shura:
+ self._shura.members[new_agent.id] = new_agent
+ logger.info(f"[KHALQ] Agent registered in global registry: {new_agent.id}")
+
+ # Persist agent profile to memory
+ if self.memory:
+ try:
+ asyncio.get_event_loop().create_task(
+ self.memory.save_agent_profile({
+ "id": new_agent.id,
+ "name": new_agent.name,
+ "role": agent_type,
+ "nafs_level": 1,
+ "capabilities": list(new_agent.tools.keys()),
+ "config": self.config or {},
+ })
+ )
+ except Exception:
+ pass
+
info = new_agent.to_dict()
logger.info(
f"[KHALQ] Agent created by {self.name}: "
@@ -1136,6 +2002,173 @@ async def _tool_create_agent(
}
)
+ async def _tool_create_skill(
+ self, name: str, description: str = "", code: str = "", tools: dict = None, **kwargs
+ ) -> str:
+ """
+ Dynamically create a new skill at runtime (Khalq al-Hikmah).
+
+ Two modes:
+ 1. Full code: provide complete Python skill class code
+ 2. Simple tools: provide tool definitions and auto-generate the skill
+
+ The skill is saved to skills/builtin/ and immediately registered.
+ """
+ import importlib
+ import re
+
+ # Validate skill name
+ if not re.match(r'^[a-z][a-z0-9_]*$', name):
+ return json.dumps({
+ "success": False,
+ "error": "Skill name must be snake_case (lowercase letters, digits, underscores)",
+ })
+
+ skills_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "skills", "builtin")
+ skill_path = os.path.join(skills_dir, f"{name}.py")
+
+ if code:
+ # Mode 1: Full code provided — validate it defines a SkillBase subclass
+ if "SkillBase" not in code:
+ return json.dumps({
+ "success": False,
+ "error": "Skill code must define a class that inherits from SkillBase. "
+ "Import with: from ..base import SkillBase, SkillManifest",
+ })
+ skill_code = code
+ elif tools:
+ # Mode 2: Auto-generate skill from tool definitions
+ skill_code = self._generate_skill_code(name, description, tools)
+ else:
+ return json.dumps({
+ "success": False,
+ "error": "Either 'code' (full Python) or 'tools' (dict of tool definitions) required",
+ })
+
+ # Security check: block dangerous patterns in skill code
+ dangerous = ["os.system", "subprocess.call", "eval(", "exec(", "__import__"]
+ for pattern in dangerous:
+ if pattern in skill_code and "subprocess.run" not in skill_code:
+ return json.dumps({
+ "success": False,
+ "error": f"Blocked dangerous pattern in skill code: {pattern}",
+ })
+
+ try:
+ # Write skill file
+ os.makedirs(skills_dir, exist_ok=True)
+ with open(skill_path, "w") as f:
+ f.write(skill_code)
+
+ # Register in skill registry
+ if self.skill_registry:
+ module_path = f"skills.builtin.{name}"
+ # Force reimport if already loaded
+ if module_path in importlib.sys.modules:
+ del importlib.sys.modules[module_path]
+ self.skill_registry._load_skill_module(module_path)
+
+ logger.info(f"[KHALQ] Dynamic skill created: {name}")
+ return json.dumps({
+ "success": True,
+ "skill_name": name,
+ "path": skill_path,
+ "message": f"Skill '{name}' created and registered. Its tools are now available.",
+ "tools": list(self.skill_registry.get_skill(name).get_tools().keys())
+ if self.skill_registry.get_skill(name)
+ else [],
+ })
+ else:
+ return json.dumps({
+ "success": True,
+ "skill_name": name,
+ "path": skill_path,
+ "message": f"Skill '{name}' saved but registry not available. Will load on restart.",
+ })
+
+ except Exception as e:
+ # Cleanup on failure
+ if os.path.exists(skill_path):
+ os.unlink(skill_path)
+ logger.error(f"[KHALQ] Skill creation failed: {e}")
+ return json.dumps({
+ "success": False,
+ "error": str(e),
+ })
+
+ def _generate_skill_code(self, name: str, description: str, tools: dict) -> str:
+ """Auto-generate a skill class from simple tool definitions."""
+ class_name = "".join(word.capitalize() for word in name.split("_")) + "Skill"
+
+ tool_methods = []
+ tool_registrations = []
+ tool_schemas = []
+
+ for tool_name, tool_def in tools.items():
+ if isinstance(tool_def, str):
+ tool_def = {"description": tool_def}
+ desc = tool_def.get("description", f"Execute {tool_name}")
+ params = tool_def.get("params", {})
+
+ safe_name = tool_name.replace("-", "_")
+ tool_registrations.append(f' "{tool_name}": self.{safe_name},')
+ tool_methods.append(
+ f' async def {safe_name}(self, params: dict) -> dict:\n'
+ f' """{ desc }"""\n'
+ f' return {{"executed": "{tool_name}", "params": params}}'
+ )
+ schema_props = {
+ k: {"type": v if isinstance(v, str) else "string", "description": f"{k} parameter"}
+ for k, v in params.items()
+ }
+ tool_schemas.append(
+ f' {{"name": "{tool_name}", '
+ f'"description": "{desc}", '
+ f'"input_schema": {{"type": "object", "properties": {json.dumps(schema_props)}}}}}'
+ )
+
+ return f'''"""
+Auto-generated skill: {name}
+{description}
+"""
+
+import logging
+from ..base import SkillBase, SkillManifest
+
+logger = logging.getLogger("mizan.{name}")
+
+
+class {class_name}(SkillBase):
+ """{description or name}"""
+
+ manifest = SkillManifest(
+ name="{name}",
+ version="1.0.0",
+ description="{description}",
+ tags=["{name}"],
+ )
+
+ def __init__(self, config: dict = None):
+ super().__init__(config)
+ self._tools = {{
+{chr(10).join(tool_registrations)}
+ }}
+
+ async def execute(self, params: dict, context: dict = None) -> dict:
+ action = params.get("action", "")
+ handler = self._tools.get(action)
+ if handler:
+ return await handler(params)
+ return {{"error": f"Unknown action: {{action}}"}}
+
+{"".join(chr(10) + m + chr(10) for m in tool_methods)}
+
+ def get_tool_schemas(self) -> list[dict]:
+ return [
+{chr(10).join(tool_schemas)}
+ ]
+'''
+
# ── Context window estimation constants ──
# Rough estimate: 1 token ~ 4 characters
_CHARS_PER_TOKEN = 4
@@ -1255,6 +2288,109 @@ async def _tool_compact_context(self, conversation_history: list[dict] = None) -
}
)
+ async def _tool_recall_memory(self, query: str, memory_type: str = "", limit: int = 5) -> str:
+ """Search stored memories and ingested knowledge (unified 5-layer recall when available)."""
+ if not self.memory:
+ return "No memory system available."
+
+ # Use unified MemoryPyramid when available (all 5 layers)
+ if hasattr(self.memory, "recall_unified"):
+ try:
+ hits = self.memory.recall_unified(query, top_k=min(limit, 20))
+ if hits:
+ lines = []
+ for hit in hits:
+ lines.append(
+ f"[{hit.source_layer}] (rel={hit.relevance:.2f})\n{str(hit.content)[:400]}"
+ )
+ return "\n---\n".join(lines)
+ return "No relevant memories found for that query."
+ except Exception:
+ pass # Fall through to standard recall
+
+ try:
+ memories = await self.memory.recall(
+ query, memory_type or None, limit=min(limit, 20)
+ )
+ except Exception as exc:
+ return f"Memory recall failed: {exc}"
+ if not memories:
+ return "No relevant memories found for that query."
+ results = []
+ for mem in memories:
+ content_str = str(mem.content)[:400]
+ tags = ", ".join(mem.tags) if mem.tags else ""
+ header = f"[{mem.memory_type}]"
+ if tags:
+ header += f" ({tags})"
+ results.append(f"{header}\n{content_str}")
+ return "\n---\n".join(results)
+
+ async def _tool_delegate_task(self, task: str, preferred_role: str = "general") -> dict:
+ """Delegate a task to another agent via the federation."""
+ if not self._agent_registry:
+ return {"error": "No agent registry available for delegation"}
+
+ if not self.can_delegate:
+ return {
+ "error": f"Delegation requires Nafs level 3+ (current: {self.nafs_level} - {self.NAFS_NAMES.get(self.nafs_level, 'Ammara')}). "
+ "Complete more tasks successfully to evolve."
+ }
+
+ # Map role preference to agent type
+ role_map = {
+ "code": "katib",
+ "browser": "mubashir",
+ "research": "mundhir",
+ "communication": "rasul",
+ "general": "wakil",
+ }
+ target_role = role_map.get(preferred_role, preferred_role)
+
+ # Find best agent for this role
+ target_agent = None
+ for aid, agent in self._agent_registry.items():
+ if aid == self.id:
+ continue # Don't delegate to self
+ if agent.role.lower() == target_role or target_role == "wakil":
+ target_agent = agent
+ break
+
+ # Fallback: use any available agent that's not self
+ if not target_agent:
+ for aid, agent in self._agent_registry.items():
+ if aid != self.id:
+ target_agent = agent
+ break
+
+ if not target_agent:
+ return {"error": "No suitable agent found for delegation"}
+
+ logger.info(
+ "[FEDERATION] %s delegating to %s: %s",
+ self.name, target_agent.name, task[:80],
+ )
+
+ try:
+ result = await target_agent.execute(task, {"history": []})
+ return {
+ "delegated_to": target_agent.name,
+ "success": result.get("success", False),
+ "result": str(result.get("result", result.get("error", "")))[:2000],
+ }
+ except Exception as e:
+ return {"error": f"Delegation to {target_agent.name} failed: {e}"}
+
+ async def _tool_query_knowledge(self, entity: str) -> dict:
+ """Query the knowledge graph for facts about an entity."""
+ if not self.knowledge_graph:
+ return {"found": False, "note": "Knowledge graph not available"}
+ try:
+ result = await self.knowledge_graph.query_entity(entity)
+ return result
+ except Exception as e:
+ return {"found": False, "error": str(e)}
+
def _extract_facts(self, messages: list[dict]) -> list[str]:
"""
Extract important facts from a list of messages.
@@ -1375,4 +2511,7 @@ def to_dict(self) -> dict:
"masalik": self.memory.masalik.stats()
if self.memory and hasattr(self.memory, "masalik")
else {},
+ "model": self.ai_model,
+ "provider": self.ai_client.provider_name if self.ai_client else None,
+ "system_prompt": self.config.get("system_prompt", "") if self.config else "",
}
diff --git a/backend/agents/perpetual_rotation.py b/backend/agents/perpetual_rotation.py
new file mode 100644
index 0000000..89cd3f3
--- /dev/null
+++ b/backend/agents/perpetual_rotation.py
@@ -0,0 +1,463 @@
+"""
+Perpetual Intelligence — 24/7 Multi-Agent Rotation System
+===========================================================
+
+"Indeed, in the alternation of the night and the day are signs
+ for those of understanding" — Quran 3:190
+
+The system NEVER fully sleeps — while some agents consolidate,
+others work. Like a hospital: night shift handles emergencies
+while day shift rests.
+
+Implements:
+- PERPETUAL_INTELLIGENCE: 3-shift rotation (active/consolidation/reserve)
+- AGENT_CHAT_MEMORY: Persistent group chat, searchable, learnable
+- SHIFT_HANDOFF: Smooth transition with briefing protocol
+- SURGE_CAPACITY: All agents activate for emergencies
+"""
+
+import logging
+import time
+import uuid
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
+
+logger = logging.getLogger("mizan.perpetual")
+
+
+class ShiftType(Enum):
+ ACTIVE = "active" # Processing tasks
+ CONSOLIDATION = "consolidation" # Dreams/memory/repair
+ RESERVE = "reserve" # Monitoring/emergency standby
+
+
+class MessageType(Enum):
+ MEETING_RECORD = "meeting_record"
+ KNOWLEDGE_SHARE = "knowledge_share"
+ DECISION = "decision"
+ DISSENT = "dissent"
+ CORRECTION = "correction"
+ INSIGHT = "insight"
+ HANDOFF = "handoff"
+ EMERGENCY = "emergency"
+
+
+@dataclass
+class AgentSlot:
+ """An agent slot in the rotation system."""
+ agent_id: str
+ agent_name: str
+ expertise: str
+ current_shift: ShiftType
+ energy: float = 1.0 # 0.0 (exhausted) → 1.0 (fully rested)
+ tasks_completed: int = 0
+ last_rotation: float = field(default_factory=time.time)
+ consolidation_pending: bool = False
+
+
+@dataclass
+class ChatMessage:
+ """A message in the persistent agent group chat."""
+ message_id: str
+ message_type: MessageType
+ sender_id: str
+ sender_name: str
+ content: str
+ context: str = ""
+ timestamp: float = field(default_factory=time.time)
+ accessible_to: list[str] | None = None # None = accessible to all
+
+
+@dataclass
+class HandoffBrief:
+ """Briefing document for shift transitions."""
+ outgoing_shift: ShiftType
+ incoming_shift: ShiftType
+ active_tasks: list[str]
+ recent_context: str
+ unresolved_issues: list[str]
+ emotional_state: str
+ timestamp: float = field(default_factory=time.time)
+
+
+@dataclass
+class RotationEvent:
+ """Record of a shift rotation."""
+ rotation_id: str
+ new_active: list[str]
+ new_consolidation: list[str]
+ new_reserve: list[str]
+ handoff_brief: HandoffBrief
+ timestamp: float = field(default_factory=time.time)
+
+
+@dataclass
+class SystemStatus:
+ """Current state of the perpetual system."""
+ active_agents: list[str]
+ consolidating_agents: list[str]
+ reserve_agents: list[str]
+ total_agents: int
+ current_capacity: float # active/total
+ surge_mode: bool
+ rotation_count: int
+ chat_messages: int
+ uptime_hours: float
+
+
+class AgentChatMemory:
+ """
+ AGENT_CHAT_MEMORY: Persistent searchable conversation history.
+
+ All inter-agent interactions are logged, searchable, and learnable.
+ Supports:
+ - search_history(query) — semantic search across all messages
+ - catch_up(agent, time_range) — agent learns from missed discussions
+ - provenance_trace(knowledge) — who discovered, refined, validated
+ - measure_collective_growth() — collective IQ tracking
+ """
+
+ def __init__(self):
+ self.messages: list[ChatMessage] = []
+ self.provenance: dict[str, list[str]] = {} # knowledge_key → [agent_ids]
+
+ def post(
+ self,
+ sender_id: str,
+ sender_name: str,
+ content: str,
+ message_type: MessageType = MessageType.KNOWLEDGE_SHARE,
+ context: str = "",
+ accessible_to: list[str] | None = None,
+ ) -> ChatMessage:
+ """Post a message to the group chat."""
+ msg = ChatMessage(
+ message_id=str(uuid.uuid4())[:8],
+ message_type=message_type,
+ sender_id=sender_id,
+ sender_name=sender_name,
+ content=content,
+ context=context,
+ accessible_to=accessible_to,
+ )
+ self.messages.append(msg)
+
+ # Cap at 2000 messages
+ if len(self.messages) > 2000:
+ self.messages = self.messages[-1500:]
+
+ return msg
+
+ def search(
+ self,
+ query: str,
+ agent_id: str | None = None,
+ message_type: MessageType | None = None,
+ limit: int = 20,
+ ) -> list[ChatMessage]:
+ """Search chat history by query, filtered by access."""
+ query_words = set(query.lower().split())
+ results = []
+
+ for msg in reversed(self.messages):
+ # Access check
+ if msg.accessible_to and agent_id and agent_id not in msg.accessible_to:
+ continue
+ if message_type and msg.message_type != message_type:
+ continue
+
+ msg_words = set(msg.content.lower().split())
+ overlap = len(query_words & msg_words)
+ if overlap > 0:
+ results.append((msg, overlap))
+
+ results.sort(key=lambda x: x[1], reverse=True)
+ return [msg for msg, _ in results[:limit]]
+
+ def catch_up(
+ self,
+ agent_id: str,
+ since_timestamp: float,
+ ) -> list[ChatMessage]:
+ """Get messages an agent missed since a given time."""
+ missed = []
+ for msg in self.messages:
+ if msg.timestamp < since_timestamp:
+ continue
+ if msg.sender_id == agent_id:
+ continue
+ if msg.accessible_to and agent_id not in msg.accessible_to:
+ continue
+ missed.append(msg)
+ return missed
+
+ def record_provenance(self, knowledge_key: str, agent_id: str) -> None:
+ """Track which agent contributed to a piece of knowledge."""
+ if knowledge_key not in self.provenance:
+ self.provenance[knowledge_key] = []
+ if agent_id not in self.provenance[knowledge_key]:
+ self.provenance[knowledge_key].append(agent_id)
+
+ def get_provenance(self, knowledge_key: str) -> list[str]:
+ """Who discovered / refined / validated this knowledge?"""
+ return self.provenance.get(knowledge_key, [])
+
+ def measure_collective_growth(self, agents: list[str]) -> dict:
+ """
+ Collective IQ metric:
+ IQ = (total_shared × diversity) / (1 + overlap)
+ """
+ unique_senders = set()
+ unique_types = set()
+ total = len(self.messages)
+
+ for msg in self.messages:
+ unique_senders.add(msg.sender_id)
+ unique_types.add(msg.message_type.value)
+
+ diversity = len(unique_senders) * len(unique_types)
+ overlap = max(1, total - diversity)
+
+ return {
+ "collective_iq": round((total * diversity) / overlap, 2),
+ "total_messages": total,
+ "unique_contributors": len(unique_senders),
+ "message_type_diversity": len(unique_types),
+ }
+
+
+class PerpetualRotation:
+ """
+ PERPETUAL_INTELLIGENCE: 24/7 never-sleep rotation system.
+
+ Divides agents into 3 shifts:
+ - Active: processing tasks, holding Shūrā meetings
+ - Consolidation: running dreams, memory repair, pruning
+ - Reserve: monitoring system health, handling emergencies
+
+ Rotation triggers: time-based, energy-based, or workload-based.
+ Smooth handoff with briefing protocol ensures no context is lost.
+ """
+
+ def __init__(self):
+ self.slots: dict[str, AgentSlot] = {}
+ self.chat = AgentChatMemory()
+ self.rotation_history: list[RotationEvent] = []
+ self.surge_mode = False
+ self.start_time = time.time()
+
+ # Rotation configuration
+ self.rotation_period_s = 3600 # default: rotate every hour
+ self.energy_threshold = 0.3 # rotate when energy drops below
+ self._last_rotation = time.time()
+
+ def register_agent(
+ self,
+ agent_id: str,
+ name: str,
+ expertise: str,
+ initial_shift: ShiftType = ShiftType.RESERVE,
+ ) -> AgentSlot:
+ """Register an agent in the rotation system."""
+ slot = AgentSlot(
+ agent_id=agent_id,
+ agent_name=name,
+ expertise=expertise,
+ current_shift=initial_shift,
+ )
+ self.slots[agent_id] = slot
+ return slot
+
+ def auto_assign_shifts(self) -> dict[str, list[str]]:
+ """
+ Automatically assign agents to 3 balanced shifts.
+ Distributes expertise evenly across shifts.
+ """
+ all_agents = list(self.slots.values())
+ if len(all_agents) < 3:
+ # Too few agents — everyone active
+ for agent in all_agents:
+ agent.current_shift = ShiftType.ACTIVE
+ return {"active": [a.agent_id for a in all_agents], "consolidation": [], "reserve": []}
+
+ # Sort by energy (most rested = active)
+ all_agents.sort(key=lambda a: a.energy, reverse=True)
+
+ third = max(1, len(all_agents) // 3)
+ active = all_agents[:third]
+ consolidation = all_agents[third:third * 2]
+ reserve = all_agents[third * 2:]
+
+ for agent in active:
+ agent.current_shift = ShiftType.ACTIVE
+ for agent in consolidation:
+ agent.current_shift = ShiftType.CONSOLIDATION
+ for agent in reserve:
+ agent.current_shift = ShiftType.RESERVE
+
+ return {
+ "active": [a.agent_id for a in active],
+ "consolidation": [a.agent_id for a in consolidation],
+ "reserve": [a.agent_id for a in reserve],
+ }
+
+ def should_rotate(self) -> bool:
+ """Check if rotation is needed (time, energy, or workload trigger)."""
+ # Time-based
+ if time.time() - self._last_rotation > self.rotation_period_s:
+ return True
+
+ # Energy-based: any active agent exhausted
+ for slot in self.slots.values():
+ if (
+ slot.current_shift == ShiftType.ACTIVE
+ and slot.energy < self.energy_threshold
+ ):
+ return True
+
+ return False
+
+ def rotate(
+ self,
+ active_tasks: list[str] | None = None,
+ recent_context: str = "",
+ unresolved: list[str] | None = None,
+ ) -> RotationEvent:
+ """
+ Execute a shift rotation with handoff briefing.
+
+ outgoing (active) → reserve (deep rest)
+ consolidation → active (rested, ready to work)
+ reserve → consolidation
+ """
+ active_tasks = active_tasks or []
+ unresolved = unresolved or []
+
+ # Current assignments
+ old_active = [s for s in self.slots.values() if s.current_shift == ShiftType.ACTIVE]
+ old_consol = [s for s in self.slots.values() if s.current_shift == ShiftType.CONSOLIDATION]
+ old_reserve = [s for s in self.slots.values() if s.current_shift == ShiftType.RESERVE]
+
+ # Generate handoff brief
+ brief = HandoffBrief(
+ outgoing_shift=ShiftType.ACTIVE,
+ incoming_shift=ShiftType.CONSOLIDATION,
+ active_tasks=active_tasks,
+ recent_context=recent_context[:500],
+ unresolved_issues=unresolved,
+ emotional_state="nominal",
+ )
+
+ # Rotate: consolidation → active, reserve → consolidation, active → reserve
+ for slot in old_consol:
+ slot.current_shift = ShiftType.ACTIVE
+ slot.energy = min(1.0, slot.energy + 0.4) # rested
+ slot.last_rotation = time.time()
+ for slot in old_reserve:
+ slot.current_shift = ShiftType.CONSOLIDATION
+ slot.last_rotation = time.time()
+ for slot in old_active:
+ slot.current_shift = ShiftType.RESERVE
+ slot.energy = max(0.0, slot.energy - 0.1) # tired
+ slot.last_rotation = time.time()
+ slot.consolidation_pending = True
+
+ self._last_rotation = time.time()
+
+ # Post handoff to chat
+ self.chat.post(
+ sender_id="system",
+ sender_name="Rotation Manager",
+ content=f"Shift rotation: {len(old_consol)} agents now active, "
+ f"tasks: {', '.join(active_tasks[:3]) if active_tasks else 'none'}",
+ message_type=MessageType.HANDOFF,
+ context=recent_context[:200],
+ )
+
+ event = RotationEvent(
+ rotation_id=str(uuid.uuid4())[:8],
+ new_active=[s.agent_id for s in old_consol],
+ new_consolidation=[s.agent_id for s in old_reserve],
+ new_reserve=[s.agent_id for s in old_active],
+ handoff_brief=brief,
+ )
+ self.rotation_history.append(event)
+
+ logger.info(
+ "[ROTATION] Shift change: active=%d consolidating=%d reserve=%d",
+ len(old_consol), len(old_reserve), len(old_active),
+ )
+ return event
+
+ def activate_surge(self) -> list[str]:
+ """
+ SURGE CAPACITY: Activate ALL agents for emergency.
+ Consolidation/reserve agents wake immediately.
+ """
+ self.surge_mode = True
+ activated = []
+ for slot in self.slots.values():
+ if slot.current_shift != ShiftType.ACTIVE:
+ slot.current_shift = ShiftType.ACTIVE
+ activated.append(slot.agent_id)
+
+ self.chat.post(
+ sender_id="system",
+ sender_name="Emergency",
+ content=f"SURGE ACTIVATED: {len(activated)} additional agents online",
+ message_type=MessageType.EMERGENCY,
+ )
+
+ logger.warning("[SURGE] All %d agents activated", len(self.slots))
+ return activated
+
+ def deactivate_surge(self) -> None:
+ """Return to normal rotation after surge."""
+ self.surge_mode = False
+ self.auto_assign_shifts()
+ logger.info("[SURGE] Deactivated, returning to normal rotation")
+
+ def record_task_completion(self, agent_id: str, energy_cost: float = 0.05) -> None:
+ """Record that an agent completed a task (energy consumption)."""
+ if agent_id in self.slots:
+ slot = self.slots[agent_id]
+ slot.tasks_completed += 1
+ slot.energy = max(0.0, slot.energy - energy_cost)
+
+ def get_active_agents(self) -> list[AgentSlot]:
+ return [s for s in self.slots.values() if s.current_shift == ShiftType.ACTIVE]
+
+ def get_consolidating_agents(self) -> list[AgentSlot]:
+ return [s for s in self.slots.values() if s.current_shift == ShiftType.CONSOLIDATION]
+
+ def get_status(self) -> SystemStatus:
+ active = [s.agent_id for s in self.slots.values() if s.current_shift == ShiftType.ACTIVE]
+ consolidating = [s.agent_id for s in self.slots.values() if s.current_shift == ShiftType.CONSOLIDATION]
+ reserve = [s.agent_id for s in self.slots.values() if s.current_shift == ShiftType.RESERVE]
+ total = len(self.slots)
+
+ return SystemStatus(
+ active_agents=active,
+ consolidating_agents=consolidating,
+ reserve_agents=reserve,
+ total_agents=total,
+ current_capacity=len(active) / max(total, 1),
+ surge_mode=self.surge_mode,
+ rotation_count=len(self.rotation_history),
+ chat_messages=len(self.chat.messages),
+ uptime_hours=round((time.time() - self.start_time) / 3600, 2),
+ )
+
+ def to_dict(self) -> dict:
+ status = self.get_status()
+ return {
+ "active": len(status.active_agents),
+ "consolidating": len(status.consolidating_agents),
+ "reserve": len(status.reserve_agents),
+ "total": status.total_agents,
+ "surge_mode": status.surge_mode,
+ "rotation_count": status.rotation_count,
+ "chat_messages": status.chat_messages,
+ "uptime_hours": status.uptime_hours,
+ }
diff --git a/backend/agents/shura_council.py b/backend/agents/shura_council.py
new file mode 100644
index 0000000..2dcd7f4
--- /dev/null
+++ b/backend/agents/shura_council.py
@@ -0,0 +1,499 @@
+"""
+Shūrā Council — Multi-Agent Consultation Architecture
+======================================================
+
+"And those who conduct their affairs by shūrā (mutual consultation)
+ among themselves" — Quran 42:38
+
+"And consult them in the matter" — Quran 3:159
+
+Implements the SHURA_COUNCIL_MEETING algorithm:
+1. Agenda setting (Qalb broadcasts problem)
+2. Independent analysis (parallel, NO groupthink)
+3. Proposal presentation (round-robin)
+4. Cross-examination (agents challenge each other)
+5. Integration (Lubb synthesis)
+6. Dissent recording (Lawwāma function)
+7. Knowledge sharing (post-meeting learning)
+
+Also implements AGENT_KNOWLEDGE_SHARING for continuous inter-agent learning.
+"""
+
+import logging
+import time
+import uuid
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
+
+logger = logging.getLogger("mizan.shura")
+
+
+class ProposalStatus(Enum):
+ PENDING = "pending"
+ PRESENTED = "presented"
+ CHALLENGED = "challenged"
+ ACCEPTED = "accepted"
+ REJECTED = "rejected"
+
+
+@dataclass
+class AgentProposal:
+ """A single agent's proposal in a Shūrā meeting."""
+ agent_id: str
+ agent_name: str
+ expertise_domain: str
+ solution: str
+ reasoning: str
+ confidence: float
+ evidence: list[str]
+ score: float = 0.0
+ weaknesses: list[str] = field(default_factory=list)
+ status: ProposalStatus = ProposalStatus.PENDING
+
+
+@dataclass
+class Challenge:
+ """A challenge from one agent to another's proposal."""
+ challenger_id: str
+ target_proposal_agent: str
+ critique: str
+ severity: float # 0.0 - 1.0
+ valid: bool = True
+
+
+@dataclass
+class Rebuttal:
+ """A defense against a challenge."""
+ defender_id: str
+ challenge_critique: str
+ defense: str
+ strength: float # 0.0 - 1.0
+ valid: bool = True
+
+
+@dataclass
+class DissentRecord:
+ """Records disagreement for future reference."""
+ agent_id: str
+ agent_name: str
+ alternative_proposal: str
+ reasoning: str
+ timestamp: float = field(default_factory=time.time)
+
+
+@dataclass
+class MeetingResult:
+ """Full result of a Shūrā council meeting."""
+ meeting_id: str
+ problem: str
+ decision: str
+ decision_source: str # "synthesis" or agent_name
+ confidence: float
+ proposals: list[AgentProposal]
+ dissent_records: list[DissentRecord]
+ knowledge_shared: int
+ meeting_duration_ms: float
+
+
+@dataclass
+class KnowledgePackage:
+ """Packaged knowledge for sharing between agents."""
+ source_agent: str
+ target_agent: str
+ content: str
+ domain: str
+ relevance: float
+ timestamp: float = field(default_factory=time.time)
+
+
+class ShuraCouncil:
+ """
+ SHURA_COUNCIL_MEETING algorithm implementation.
+
+ Manages formal multi-agent deliberation sessions with:
+ - Independent parallel analysis (prevents groupthink)
+ - Cross-examination (stress-tests proposals)
+ - Weighted synthesis (not majority vote — expertise-weighted)
+ - Dissent recording (minority views preserved for future correction)
+ - Post-meeting knowledge sharing (agents learn from each other)
+ """
+
+ def __init__(self):
+ self.meeting_history: list[MeetingResult] = []
+ self.dissent_archive: list[DissentRecord] = []
+ self.knowledge_inbox: dict[str, list[KnowledgePackage]] = {}
+ self.expertise_profiles: dict[str, dict[str, float]] = {}
+
+ def convene(
+ self,
+ problem: str,
+ agents: list[dict],
+ context: dict | None = None,
+ ) -> MeetingResult:
+ """
+ Run a full Shūrā council meeting.
+
+ agents: list of {"id", "name", "expertise", "evaluate_fn" (optional)}
+
+ Steps:
+ 1. Agenda broadcast
+ 2. Independent analysis (parallel)
+ 3. Proposal presentation
+ 4. Cross-examination
+ 5. Integration
+ 6. Dissent recording
+ 7. Knowledge sharing
+ """
+ meeting_id = str(uuid.uuid4())[:8]
+ start = time.monotonic()
+
+ # Step 1: Agenda setting
+ logger.info("[SHURA] Meeting %s convened: %s", meeting_id, problem[:80])
+
+ # Step 2: Independent analysis (each agent generates a proposal)
+ proposals = []
+ for agent in agents:
+ proposal = self._generate_proposal(agent, problem, context)
+ proposals.append(proposal)
+
+ # Step 3: Proposal presentation (score by expertise relevance)
+ proposals.sort(
+ key=lambda p: self._expertise_relevance(p.expertise_domain, problem),
+ reverse=True,
+ )
+ for proposal in proposals:
+ proposal.status = ProposalStatus.PRESENTED
+
+ # Step 4: Cross-examination
+ self._cross_examine(proposals, problem)
+
+ # Step 5: Integration — weighted scoring + synthesis attempt
+ decision, decision_source, confidence = self._integrate(proposals, problem)
+
+ # Step 6: Dissent recording
+ dissent_records = []
+ for proposal in proposals:
+ if proposal.solution != decision:
+ record = DissentRecord(
+ agent_id=proposal.agent_id,
+ agent_name=proposal.agent_name,
+ alternative_proposal=proposal.solution,
+ reasoning=proposal.reasoning,
+ )
+ dissent_records.append(record)
+ self.dissent_archive.append(record)
+
+ # Step 7: Knowledge sharing
+ shared = self._share_knowledge(proposals, agents)
+
+ elapsed_ms = (time.monotonic() - start) * 1000
+
+ result = MeetingResult(
+ meeting_id=meeting_id,
+ problem=problem,
+ decision=decision,
+ decision_source=decision_source,
+ confidence=confidence,
+ proposals=proposals,
+ dissent_records=dissent_records,
+ knowledge_shared=shared,
+ meeting_duration_ms=round(elapsed_ms, 2),
+ )
+ self.meeting_history.append(result)
+
+ logger.info(
+ "[SHURA] Meeting %s concluded: source=%s confidence=%.2f dissents=%d",
+ meeting_id, decision_source, confidence, len(dissent_records),
+ )
+ return result
+
+ def _generate_proposal(
+ self, agent: dict, problem: str, context: dict | None
+ ) -> AgentProposal:
+ """
+ Each agent independently analyzes the problem.
+ In production, this calls the agent's LLM evaluate function.
+ """
+ agent_id = agent.get("id", "unknown")
+ name = agent.get("name", "Agent")
+ expertise = agent.get("expertise", "general")
+
+ # Expertise-based confidence heuristic
+ relevance = self._expertise_relevance(expertise, problem)
+ confidence = 0.4 + 0.5 * relevance # base + expertise boost
+
+ # Generate proposal text (placeholder — LLM call in production)
+ solution = (
+ f"[{expertise.upper()} perspective] "
+ f"Approach to '{problem[:60]}' "
+ f"using {expertise} principles"
+ )
+ reasoning = (
+ f"Based on {expertise} analysis: "
+ f"relevance={relevance:.2f}, "
+ f"confidence={confidence:.2f}"
+ )
+ evidence = [f"{expertise}_analysis", f"domain_knowledge_{expertise}"]
+
+ return AgentProposal(
+ agent_id=agent_id,
+ agent_name=name,
+ expertise_domain=expertise,
+ solution=solution,
+ reasoning=reasoning,
+ confidence=round(confidence, 3),
+ evidence=evidence,
+ score=confidence * relevance,
+ )
+
+ def _cross_examine(self, proposals: list[AgentProposal], problem: str) -> None:
+ """
+ Agents challenge each other's proposals.
+ Score adjustments: valid challenge reduces score, valid rebuttal partially recovers.
+ """
+ for i, proposal in enumerate(proposals):
+ for j, challenger_source in enumerate(proposals):
+ if i == j:
+ continue
+
+ # Generate challenge based on expertise difference
+ challenge = self._generate_challenge(
+ challenger_source, proposal, problem
+ )
+ if not challenge:
+ continue
+
+ if challenge.valid:
+ proposal.score -= challenge.severity * 0.2
+ proposal.weaknesses.append(challenge.critique)
+
+ # Allow rebuttal
+ rebuttal = self._generate_rebuttal(proposal, challenge)
+ if rebuttal and rebuttal.valid:
+ proposal.score += rebuttal.strength * 0.1 # partial recovery
+
+ proposal.status = ProposalStatus.CHALLENGED
+
+ def _generate_challenge(
+ self, challenger: AgentProposal, target: AgentProposal, problem: str
+ ) -> Challenge | None:
+ """Generate a challenge from one agent to another's proposal."""
+ # Only challenge if domains differ (cross-domain critique is more valuable)
+ if challenger.expertise_domain == target.expertise_domain:
+ return None
+
+ # Heuristic: challenge strength based on challenger's confidence
+ severity = 0.2 + 0.3 * challenger.confidence
+
+ # Identify potential weakness based on expertise gap
+ critique = (
+ f"{challenger.expertise_domain} perspective: "
+ f"'{target.expertise_domain}' approach may miss "
+ f"{challenger.expertise_domain}-specific considerations"
+ )
+
+ return Challenge(
+ challenger_id=challenger.agent_id,
+ target_proposal_agent=target.agent_id,
+ critique=critique,
+ severity=round(severity, 3),
+ valid=severity > 0.3, # only valid if substantive
+ )
+
+ def _generate_rebuttal(
+ self, defender: AgentProposal, challenge: Challenge
+ ) -> Rebuttal | None:
+ """Defender responds to a challenge."""
+ # Rebuttal strength proportional to evidence quality
+ strength = min(0.8, 0.3 + 0.1 * len(defender.evidence))
+
+ defense = (
+ f"Rebuttal: {defender.expertise_domain} approach accounts for "
+ f"the raised concern through {defender.evidence[0] if defender.evidence else 'general analysis'}"
+ )
+
+ return Rebuttal(
+ defender_id=defender.agent_id,
+ challenge_critique=challenge.critique,
+ defense=defense,
+ strength=round(strength, 3),
+ valid=strength > 0.4,
+ )
+
+ def _integrate(
+ self, proposals: list[AgentProposal], problem: str
+ ) -> tuple[str, str, float]:
+ """
+ Integration: NOT majority vote — weighted by expertise and evidence quality.
+
+ final_score = expertise_weight × confidence × evidence_quality × (1 - weaknesses)
+
+ Attempt synthesis (combine best elements). If synthesis > best individual, use it.
+ """
+ if not proposals:
+ return "No proposals generated", "none", 0.0
+
+ # Score each proposal
+ for proposal in proposals:
+ expertise_weight = self._expertise_relevance(
+ proposal.expertise_domain, problem
+ )
+ evidence_quality = min(1.0, 0.5 + 0.1 * len(proposal.evidence))
+ weakness_penalty = min(0.5, 0.1 * len(proposal.weaknesses))
+
+ proposal.score = (
+ expertise_weight
+ * proposal.confidence
+ * evidence_quality
+ * (1.0 - weakness_penalty)
+ )
+
+ # Best individual proposal
+ best = max(proposals, key=lambda p: p.score)
+
+ # Attempt synthesis: combine elements from top proposals
+ top_proposals = sorted(proposals, key=lambda p: p.score, reverse=True)[:3]
+ synthesis = self._synthesize(top_proposals, problem)
+ synthesis_score = sum(p.score for p in top_proposals) / len(top_proposals) * 1.1
+
+ if synthesis_score > best.score:
+ confidence = min(0.95, synthesis_score)
+ return synthesis, "synthesis", round(confidence, 3)
+ else:
+ best.status = ProposalStatus.ACCEPTED
+ return best.solution, best.agent_name, round(best.score, 3)
+
+ def _synthesize(self, proposals: list[AgentProposal], problem: str) -> str:
+ """Combine best elements of multiple proposals into synthesis."""
+ elements = []
+ for proposal in proposals:
+ key_element = proposal.solution.split("]")[-1].strip()[:80]
+ if key_element:
+ elements.append(f"[{proposal.expertise_domain}] {key_element}")
+ return f"Synthesis for '{problem[:40]}': " + " + ".join(elements[:3])
+
+ def _share_knowledge(
+ self, proposals: list[AgentProposal], agents: list[dict]
+ ) -> int:
+ """Post-meeting knowledge sharing: each agent learns from others."""
+ shared = 0
+ for proposal in proposals:
+ for agent in agents:
+ target_id = agent.get("id", "")
+ if target_id == proposal.agent_id:
+ continue # don't share with self
+
+ relevance = self._cross_domain_relevance(
+ proposal.expertise_domain,
+ agent.get("expertise", "general"),
+ )
+ if relevance > 0.3:
+ package = KnowledgePackage(
+ source_agent=proposal.agent_id,
+ target_agent=target_id,
+ content=proposal.reasoning[:200],
+ domain=proposal.expertise_domain,
+ relevance=relevance,
+ )
+ if target_id not in self.knowledge_inbox:
+ self.knowledge_inbox[target_id] = []
+ self.knowledge_inbox[target_id].append(package)
+ shared += 1
+
+ return shared
+
+ def get_inbox(self, agent_id: str) -> list[KnowledgePackage]:
+ """Get pending knowledge packages for an agent."""
+ return self.knowledge_inbox.pop(agent_id, [])
+
+ def search_meeting_history(self, query: str, limit: int = 5) -> list[MeetingResult]:
+ """Search past meetings by problem text."""
+ query_words = set(query.lower().split())
+ scored = []
+ for meeting in self.meeting_history:
+ problem_words = set(meeting.problem.lower().split())
+ overlap = len(query_words & problem_words)
+ if overlap > 0:
+ scored.append((meeting, overlap))
+ scored.sort(key=lambda x: x[1], reverse=True)
+ return [m for m, _ in scored[:limit]]
+
+ def get_dissents_for(self, problem_query: str) -> list[DissentRecord]:
+ """Find dissent records related to a query (may prove right later)."""
+ results = []
+ query_words = set(problem_query.lower().split())
+ for record in self.dissent_archive:
+ alt_words = set(record.alternative_proposal.lower().split())
+ if len(query_words & alt_words) > 1:
+ results.append(record)
+ return results
+
+ def update_expertise(self, agent_id: str, domain: str, score: float) -> None:
+ """Track and update expertise profiles over time."""
+ if agent_id not in self.expertise_profiles:
+ self.expertise_profiles[agent_id] = {}
+ # Exponential moving average
+ alpha = 0.2
+ current = self.expertise_profiles[agent_id].get(domain, 0.5)
+ self.expertise_profiles[agent_id][domain] = (
+ alpha * score + (1 - alpha) * current
+ )
+
+ def _expertise_relevance(self, expertise: str, problem: str) -> float:
+ """How relevant is an expertise domain to the problem?"""
+ # Simple keyword overlap heuristic
+ expertise_words = set(expertise.lower().split("_"))
+ problem_words = set(problem.lower().split())
+ if not expertise_words:
+ return 0.5
+ overlap = len(expertise_words & problem_words)
+ return min(1.0, 0.3 + 0.3 * overlap)
+
+ def _cross_domain_relevance(self, domain_a: str, domain_b: str) -> float:
+ """How relevant is knowledge from domain_a to domain_b?"""
+ if domain_a == domain_b:
+ return 0.9 # same domain = highly relevant
+ # Cross-domain pairs that benefit each other
+ synergies = {
+ frozenset({"reasoning", "planning"}): 0.7,
+ frozenset({"creativity", "reasoning"}): 0.6,
+ frozenset({"memory", "reasoning"}): 0.7,
+ frozenset({"language", "social"}): 0.6,
+ frozenset({"perception", "creativity"}): 0.5,
+ }
+ pair = frozenset({domain_a.lower(), domain_b.lower()})
+ return synergies.get(pair, 0.35)
+
+ def measure_collective_intelligence(self) -> dict:
+ """Measure collective IQ: total knowledge × diversity ÷ redundancy."""
+ if not self.meeting_history:
+ return {"collective_iq": 0, "meetings": 0}
+
+ unique_domains = set()
+ total_proposals = 0
+ for meeting in self.meeting_history:
+ for proposal in meeting.proposals:
+ unique_domains.add(proposal.expertise_domain)
+ total_proposals += 1
+
+ diversity = len(unique_domains)
+ # Estimate redundancy from repeated similar proposals
+ redundancy = max(1, total_proposals - diversity * len(self.meeting_history))
+
+ collective_iq = (total_proposals * diversity) / redundancy
+ return {
+ "collective_iq": round(collective_iq, 2),
+ "meetings": len(self.meeting_history),
+ "unique_domains": diversity,
+ "total_proposals": total_proposals,
+ "dissent_records": len(self.dissent_archive),
+ }
+
+ def to_dict(self) -> dict:
+ return {
+ "meetings_held": len(self.meeting_history),
+ "dissents_archived": len(self.dissent_archive),
+ "knowledge_inbox_agents": len(self.knowledge_inbox),
+ "expertise_profiles": len(self.expertise_profiles),
+ }
diff --git a/backend/agents/specialized.py b/backend/agents/specialized.py
index 8c7103f..2d0c4eb 100644
--- a/backend/agents/specialized.py
+++ b/backend/agents/specialized.py
@@ -671,9 +671,332 @@ async def _tool_send_notification(self, message: str, channel: str = "log") -> d
return {"error": f"Unknown notification channel: {channel}"}
+class SuperAgent(BrowserAgent, ResearchAgent, CodeAgent, CommunicationAgent):
+ """
+ Khalifah (خليفة) - Universal Agent
+ "I will create a vicegerent (Khalifah) on earth" - Quran 2:30
+
+ All tools. All capabilities. One agent.
+ Combines browser, research, code, and communication into a single agent.
+ """
+
+ # Tool schemas for all specialized tools — exposed to LLM via get_tool_schemas()
+ TOOL_SCHEMAS: list[dict] = [
+ # ── Browser tools ──
+ {
+ "name": "browse_url",
+ "description": "Browse a URL and return its text content, title, and links.",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "url": {"type": "string", "description": "The URL to browse"},
+ },
+ "required": ["url"],
+ },
+ },
+ {
+ "name": "navigate",
+ "description": "Navigate to a URL with optional JS rendering (Playwright) or httpx fallback.",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "url": {"type": "string", "description": "The URL to navigate to"},
+ "wait_for": {
+ "type": "string",
+ "description": "CSS selector to wait for before returning",
+ },
+ },
+ "required": ["url"],
+ },
+ },
+ {
+ "name": "search_web",
+ "description": "Search the web using DuckDuckGo and return results.",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "query": {"type": "string", "description": "The search query"},
+ },
+ "required": ["query"],
+ },
+ },
+ {
+ "name": "extract_content",
+ "description": "Extract specific content from a URL, optionally using a CSS selector.",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "url": {"type": "string", "description": "The URL to extract from"},
+ "selector": {
+ "type": "string",
+ "description": "Optional CSS selector to target specific elements",
+ },
+ },
+ "required": ["url"],
+ },
+ },
+ {
+ "name": "take_screenshot",
+ "description": "Take a screenshot of a web page (requires Playwright or system browser).",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "url": {"type": "string", "description": "The URL to screenshot"},
+ },
+ "required": ["url"],
+ },
+ },
+ {
+ "name": "click_element",
+ "description": "Click an element on a web page by CSS selector.",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "url": {"type": "string", "description": "The page URL"},
+ "selector": {"type": "string", "description": "CSS selector of the element to click"},
+ },
+ "required": ["url", "selector"],
+ },
+ },
+ {
+ "name": "fill_form",
+ "description": "Fill form fields on a web page. Keys are CSS selectors, values are text to fill.",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "url": {"type": "string", "description": "The page URL"},
+ "fields": {
+ "type": "object",
+ "description": "Mapping of CSS selector → value to fill",
+ },
+ },
+ "required": ["url", "fields"],
+ },
+ },
+ # ── Research tools ──
+ {
+ "name": "analyze_text",
+ "description": "Analyze text for word count, key terms, and insights.",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "text": {"type": "string", "description": "The text to analyze"},
+ "aspect": {
+ "type": "string",
+ "description": "Analysis aspect (e.g., general, sentiment, key_points)",
+ "default": "general",
+ },
+ },
+ "required": ["text"],
+ },
+ },
+ {
+ "name": "synthesize_sources",
+ "description": "Synthesize information from multiple text sources.",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "sources": {
+ "type": "array",
+ "items": {"type": "string"},
+ "description": "List of text sources to synthesize",
+ },
+ },
+ "required": ["sources"],
+ },
+ },
+ {
+ "name": "fact_check",
+ "description": "Check a factual claim for accuracy.",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "claim": {"type": "string", "description": "The claim to verify"},
+ },
+ "required": ["claim"],
+ },
+ },
+ {
+ "name": "generate_report",
+ "description": "Generate a structured report template on a topic.",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "topic": {"type": "string", "description": "The report topic"},
+ "format": {
+ "type": "string",
+ "description": "Report format (markdown, html, text)",
+ "default": "markdown",
+ },
+ },
+ "required": ["topic"],
+ },
+ },
+ {
+ "name": "arxiv_search",
+ "description": "Search ArXiv for academic papers.",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "query": {"type": "string", "description": "Search query for papers"},
+ "max_results": {
+ "type": "integer",
+ "description": "Maximum number of results",
+ "default": 5,
+ },
+ },
+ "required": ["query"],
+ },
+ },
+ # ── Code tools ──
+ {
+ "name": "generate_code",
+ "description": "Generate code from a specification.",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "spec": {"type": "string", "description": "Code specification/requirements"},
+ "language": {
+ "type": "string",
+ "description": "Programming language",
+ "default": "python",
+ },
+ },
+ "required": ["spec"],
+ },
+ },
+ {
+ "name": "run_tests",
+ "description": "Run tests in a directory using a test framework.",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "path": {"type": "string", "description": "Path to the test directory"},
+ "framework": {
+ "type": "string",
+ "description": "Test framework (pytest, jest, etc.)",
+ "default": "pytest",
+ },
+ },
+ "required": ["path"],
+ },
+ },
+ {
+ "name": "lint_code",
+ "description": "Lint code at the given path.",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "path": {"type": "string", "description": "File or directory path to lint"},
+ },
+ "required": ["path"],
+ },
+ },
+ {
+ "name": "git_operation",
+ "description": "Run a git operation (status, log, diff, branch, add, commit, push, pull, clone).",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "operation": {
+ "type": "string",
+ "description": "Git operation to run (e.g., status, log, diff)",
+ },
+ "repo_path": {
+ "type": "string",
+ "description": "Repository path",
+ "default": ".",
+ },
+ "args": {
+ "type": "string",
+ "description": "Additional arguments",
+ "default": "",
+ },
+ },
+ "required": ["operation"],
+ },
+ },
+ {
+ "name": "install_package",
+ "description": "Install a package using pip or npm.",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "package": {"type": "string", "description": "Package name to install"},
+ "manager": {
+ "type": "string",
+ "description": "Package manager (pip or npm)",
+ "default": "pip",
+ },
+ },
+ "required": ["package"],
+ },
+ },
+ # ── Communication tools ──
+ {
+ "name": "send_webhook",
+ "description": "Send a webhook POST request with a JSON payload.",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "url": {"type": "string", "description": "Webhook URL"},
+ "payload": {"type": "object", "description": "JSON payload to send"},
+ },
+ "required": ["url", "payload"],
+ },
+ },
+ {
+ "name": "check_email",
+ "description": "Check an IMAP email inbox for recent messages.",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "host": {"type": "string", "description": "IMAP server hostname"},
+ "user": {"type": "string", "description": "Email username"},
+ "password": {"type": "string", "description": "Email password"},
+ "folder": {"type": "string", "description": "Folder to check", "default": "INBOX"},
+ "limit": {"type": "integer", "description": "Max messages to return", "default": 10},
+ },
+ "required": ["host", "user", "password"],
+ },
+ },
+ {
+ "name": "send_notification",
+ "description": "Send a notification message via a channel (currently supports 'log').",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "message": {"type": "string", "description": "Notification message"},
+ "channel": {
+ "type": "string",
+ "description": "Notification channel",
+ "default": "log",
+ },
+ },
+ "required": ["message"],
+ },
+ },
+ ]
+
+ def __init__(self, **kwargs):
+ # Call BaseAgent directly to skip MRO chain —
+ # each parent __init__ passes its own role kwarg which conflicts.
+ BaseAgent.__init__(self, role="khalifah", **kwargs)
+ self._playwright_available: bool | None = None
+ # Khalifah starts at Mudghah (structured) — all tools, delegation enabled
+ self.nafs_level = 3
+ # Register all specialized tool sets
+ self._register_browser_tools()
+ self._register_research_tools()
+ self._register_code_tools()
+ self._register_comm_tools()
+
+
def create_agent(agent_type: str, **kwargs) -> BaseAgent:
"""Factory for creating Quranic agents"""
agents = {
+ "super": SuperAgent,
+ "khalifah": SuperAgent,
"browser": BrowserAgent,
"mubashir": BrowserAgent,
"research": ResearchAgent,
diff --git a/backend/api/main.py b/backend/api/main.py
index 35d3c27..7a4ca4b 100644
--- a/backend/api/main.py
+++ b/backend/api/main.py
@@ -9,6 +9,7 @@
"""
import asyncio
+import json
import logging
import os
import sys
@@ -47,12 +48,16 @@
from core.plugins import plugin_manager
from core.qalb import QalbEngine
from memory.dhikr import DhikrMemorySystem
+from memory.knowledge_graph import KnowledgeGraph
+from reasoning.context_manager import ContextManager
+from reasoning.planner import TafakkurPlanner
from providers import (
check_provider_health,
create_provider,
fetch_ollama_models,
fetch_openrouter_models,
get_provider_status,
+ set_active_state,
)
from qca.cognitive_methods import select_method
@@ -82,7 +87,7 @@ async def lifespan(app: FastAPI):
existing = await memory.get_all_agents()
if not existing:
default_agents = [
- {"name": "Hafiz", "type": "general", "role": "Preserver"},
+ {"name": "Khalifah", "type": "super", "role": "Universal"},
{"name": "Mubashir", "type": "browser", "role": "Browser"},
{"name": "Mundhir", "type": "research", "role": "Researcher"},
{"name": "Katib", "type": "code", "role": "Coder"},
@@ -98,6 +103,9 @@ async def lifespan(app: FastAPI):
izn=izn,
skill_registry=skill_registry,
plugin_manager=plugin_manager,
+ knowledge_graph=knowledge_graph,
+ context_manager=context_manager,
+ planner=planner,
)
active_agents[agent.id] = agent
balancer.register(agent.id)
@@ -125,6 +133,9 @@ async def lifespan(app: FastAPI):
izn=izn,
skill_registry=skill_registry,
plugin_manager=plugin_manager,
+ knowledge_graph=knowledge_graph,
+ context_manager=context_manager,
+ planner=planner,
)
agent.total_tasks = profile.get("total_tasks", 0)
agent.learning_iterations = profile.get("learning_iterations", 0)
@@ -134,6 +145,27 @@ async def lifespan(app: FastAPI):
logger.info(f"{len(active_agents)} agents initialized")
+ # Inject global registry references into agents
+ for agent in active_agents.values():
+ agent._agent_registry = active_agents
+ agent._balancer = balancer
+ agent._shura = shura
+
+ # Restore persisted provider/model choice
+ try:
+ saved_provider = await memory.get_preference("active_provider")
+ saved_model = await memory.get_preference("active_model")
+ if saved_provider and saved_model:
+ restored = create_provider(provider=saved_provider, model=saved_model)
+ if restored:
+ for agent in active_agents.values():
+ agent.ai_client = restored
+ agent.ai_model = saved_model
+ set_active_state(saved_provider, saved_model)
+ logger.info(f"Restored provider: {saved_provider}/{saved_model}")
+ except Exception as exc:
+ logger.warning(f"Could not restore provider preference: {exc}")
+
# Initialize scheduler executor
async def execute_scheduled_task(task: str, agent_id: str = None):
aid = agent_id or (list(active_agents.keys())[0] if active_agents else None)
@@ -175,6 +207,34 @@ async def execute_scheduled_task(task: str, agent_id: str = None):
# Shutdown
await event_bus.emit("system.shutdown", {})
+
+ # Persist Masalik neural pathways to disk before shutdown
+ try:
+ if hasattr(memory, "masalik"):
+ memory.masalik.save_to_disk()
+ logger.info("[MASALIK] Neural pathways persisted to disk")
+ except Exception as e:
+ logger.warning(f"[MASALIK] Failed to save pathways: {e}")
+
+ # Persist agent profiles
+ try:
+ for agent in active_agents.values():
+ await memory.save_agent_profile({
+ "id": agent.id,
+ "name": agent.name,
+ "role": agent.role,
+ "nafs_level": agent.nafs_level,
+ "capabilities": list(agent.tools.keys()),
+ "total_tasks": agent.total_tasks,
+ "success_rate": agent.success_rate,
+ "error_count": agent.error_count,
+ "learning_iterations": agent.learning_iterations,
+ "config": agent.config or {},
+ })
+ logger.info(f"[AGENTS] {len(active_agents)} agent profiles persisted")
+ except Exception as e:
+ logger.warning(f"[AGENTS] Failed to persist profiles: {e}")
+
await plugin_manager.unload_all()
await scheduler.stop()
logger.info("MIZAN shutdown complete")
@@ -210,7 +270,10 @@ async def execute_scheduled_task(task: str, agent_id: str = None):
)
# ===== GLOBAL STATE =====
-memory = DhikrMemorySystem(db_path=os.getenv("DB_PATH", "/tmp/mizan_memory.db"))
+memory = DhikrMemorySystem(db_path=os.getenv("DB_PATH", "/data/mizan_memory.db"))
+knowledge_graph = KnowledgeGraph(db_path=os.getenv("DB_PATH", "/data/mizan_memory.db"))
+context_manager = ContextManager()
+planner = TafakkurPlanner()
balancer = MizanBalancer()
shura = ShuraCouncil()
active_agents: dict[str, Any] = {}
@@ -225,30 +288,6 @@ async def execute_scheduled_task(task: str, agent_id: str = None):
federation = AgentFederation()
-# ===== RATE LIMIT MIDDLEWARE =====
-
-
-@app.middleware("http")
-async def rate_limit_middleware(request: Request, call_next):
- """Rate limiting via Wali Guardian with proper headers"""
- client_ip = request.client.host if request.client else "unknown"
- rl = wali.check_rate_limit(client_ip)
- if not rl["allowed"]:
- return JSONResponse(
- status_code=429,
- content={"error": "Rate limit exceeded. Please slow down."},
- headers={
- "Retry-After": str(rl["retry_after"]),
- "X-RateLimit-Limit": str(rl["limit"]),
- "X-RateLimit-Remaining": "0",
- },
- )
- response = await call_next(request)
- response.headers["X-RateLimit-Limit"] = str(rl["limit"])
- response.headers["X-RateLimit-Remaining"] = str(rl["remaining"])
- return response
-
-
# ===== SECURITY HEADERS MIDDLEWARE =====
@@ -334,14 +373,20 @@ async def require_auth(
class AgentCreate(BaseModel):
name: str = Field(..., min_length=1, max_length=100)
type: str = Field(
- default="general",
- pattern=r"^(general|browser|research|code|communication|wakil|mubashir|mundhir|katib|rasul)$",
+ default="super",
+ pattern=r"^(super|khalifah|general|browser|research|code|communication|wakil|mubashir|mundhir|katib|rasul)$",
)
model: str = Field(default="claude-opus-4-6", max_length=100)
system_prompt: str | None = Field(None, max_length=10000)
capabilities: list[str] = []
+class AgentUpdate(BaseModel):
+ name: str | None = Field(None, min_length=1, max_length=100)
+ model: str | None = Field(None, max_length=200)
+ system_prompt: str | None = Field(None, max_length=10000)
+
+
class TaskRequest(BaseModel):
task: str = Field(..., min_length=1, max_length=50000)
agent_id: str | None = None
@@ -353,6 +398,7 @@ class ChatMessage(BaseModel):
session_id: str = Field(..., min_length=1, max_length=100)
content: str = Field(..., min_length=1, max_length=50000)
agent_id: str | None = None
+ model_override: str | None = Field(None, max_length=200)
class IntegrationCreate(BaseModel):
@@ -582,6 +628,74 @@ async def delete_agent(agent_id: str, user: TokenPayload | None = Depends(get_cu
return {"deleted": agent_id}
+@app.put("/api/agents/{agent_id}")
+async def update_agent(
+ agent_id: str,
+ req: AgentUpdate,
+ user: TokenPayload | None = Depends(get_current_user),
+):
+ """Update an existing agent's name, model, or system prompt."""
+ agent = active_agents.get(agent_id)
+ if not agent:
+ raise HTTPException(404, "Agent not found")
+
+ if req.name is not None:
+ agent.name = req.name
+ if req.system_prompt is not None:
+ agent.config["system_prompt"] = req.system_prompt
+ if req.model is not None:
+ agent.ai_model = req.model
+ provider_obj = create_provider(model=req.model)
+ if provider_obj:
+ agent.ai_client = provider_obj
+
+ await memory.save_agent_profile({
+ "id": agent.id,
+ "name": agent.name,
+ "role": agent.role,
+ "nafs_level": agent.nafs_level,
+ "capabilities": list(agent.tools.keys()),
+ "config": agent.config,
+ })
+
+ await manager.broadcast({"type": "agent_updated", "agent": agent.to_dict()})
+ return agent.to_dict()
+
+
+class AgentModelRequest(BaseModel):
+ provider: str = Field(..., pattern=r"^(anthropic|openrouter|openai|ollama)$")
+ model: str = Field(..., min_length=1, max_length=200)
+
+
+@app.post("/api/agents/{agent_id}/model")
+async def set_agent_model(
+ agent_id: str,
+ req: AgentModelRequest,
+ user: TokenPayload | None = Depends(get_current_user),
+):
+ """Set model for a specific agent (does not affect other agents)."""
+ agent = active_agents.get(agent_id)
+ if not agent:
+ raise HTTPException(404, f"Agent '{agent_id}' not found")
+
+ provider = create_provider(provider=req.provider, model=req.model)
+ if not provider:
+ raise HTTPException(400, f"Cannot initialize provider '{req.provider}'. Check API key.")
+
+ agent.ai_client = provider
+ agent.ai_model = req.model
+
+ await manager.broadcast({
+ "type": "agent_model_changed",
+ "agent_id": agent_id,
+ "provider": req.provider,
+ "model": req.model,
+ })
+
+ wali.audit.log("agent_model_changed", {"agent_id": agent_id, "model": req.model})
+ return {"agent_id": agent_id, "provider": req.provider, "model": req.model}
+
+
# === TASKS ===
@@ -683,7 +797,23 @@ async def chat(
user: TokenPayload | None = Depends(get_current_user),
):
"""Chat with an agent"""
- session = active_sessions.get(req.session_id, {"history": []})
+ session = active_sessions.get(req.session_id)
+
+ # Auto-restore session from DB if not in memory (fixes cross-restart amnesia)
+ if session is None:
+ db_messages = await memory.get_messages(req.session_id, limit=50)
+ restored_history = [
+ {"role": msg["role"], "content": msg["content"]}
+ for msg in db_messages
+ ]
+ session = {"history": restored_history}
+ if restored_history:
+ logger.info(
+ "[SESSION] Restored %d messages for session %s from DB",
+ len(restored_history),
+ req.session_id[:12],
+ )
+
active_sessions[req.session_id] = session
await memory.save_message(req.session_id, "user", req.content)
@@ -723,6 +853,18 @@ async def chat(
message_id = str(uuid.uuid4())
async def process_chat():
+ # Per-message model override: temporarily swap agent's model
+ # TODO(concurrency): use per-request LLM client instead of mutating agent
+ original_model = None
+ original_client = None
+ if req.model_override and req.model_override != agent.ai_model:
+ override_provider = create_provider(model=req.model_override)
+ if override_provider:
+ original_model = agent.ai_model
+ original_client = agent.ai_client
+ agent.ai_model = req.model_override
+ agent.ai_client = override_provider
+
# Send typing indicator before starting agent execution
await manager.broadcast(
{
@@ -759,11 +901,17 @@ async def stream_cb(chunk: str, **kwargs):
}
)
- result = await agent.execute(
- req.content,
- {"history": session["history"][-10:]},
- stream_callback=stream_cb,
- )
+ try:
+ result = await agent.execute(
+ req.content,
+ {"history": session["history"][-agent.max_tool_turns:]},
+ stream_callback=stream_cb,
+ )
+ finally:
+ # Restore original model after per-message override
+ if original_model is not None:
+ agent.ai_model = original_model
+ agent.ai_client = original_client
final_response = result.get("result", response) if result.get("success") else response
if isinstance(final_response, dict):
@@ -772,6 +920,12 @@ async def stream_cb(chunk: str, **kwargs):
await memory.save_message(req.session_id, "assistant", str(final_response), agent_id)
session["history"].append({"role": "assistant", "content": str(final_response)})
+ # Extract cognitive metadata from QALB-7 pipeline
+ cognitive = {k: result.get(k) for k in (
+ "nafs_level", "nafs_name", "ruh_energy", "qalb", "yaqin",
+ "mizan_label", "cognitive_method", "lubb", "lawwama",
+ ) if result.get(k) is not None}
+
await manager.broadcast(
{
"type": "chat_complete",
@@ -779,6 +933,7 @@ async def stream_cb(chunk: str, **kwargs):
"message_id": message_id,
"response": str(final_response),
"agent": agent.name,
+ "cognitive": cognitive,
}
)
@@ -794,7 +949,66 @@ async def get_chat_history(session_id: str):
@app.get("/api/chat/sessions/list")
async def list_sessions():
- return {"sessions": list(active_sessions.keys())}
+ """List recent chat sessions from DB with metadata"""
+ db_sessions = await memory.list_sessions(limit=20)
+ return {"sessions": db_sessions}
+
+
+# === PLANNER (Tafakkur — تفكر) ===
+
+
+class PlanRequest(BaseModel):
+ goal: str = Field(..., description="The complex goal to decompose")
+ agent_id: str | None = None
+
+
+@app.post("/api/plan")
+async def create_plan(
+ req: PlanRequest,
+ background_tasks: BackgroundTasks,
+ user: TokenPayload | None = Depends(get_current_user),
+):
+ """Decompose a complex goal into sub-tasks using TafakkurPlanner"""
+ agent_id = req.agent_id or (list(active_agents.keys())[0] if active_agents else None)
+ if not agent_id or agent_id not in active_agents:
+ raise HTTPException(503, "No agents available")
+
+ agent = active_agents[agent_id]
+ plan = await planner.decompose(req.goal, agent)
+
+ return {"plan": plan.to_dict()}
+
+
+@app.post("/api/plan/{plan_id}/execute")
+async def execute_plan(
+ plan_id: str,
+ background_tasks: BackgroundTasks,
+ user: TokenPayload | None = Depends(get_current_user),
+):
+ """Execute a previously created plan"""
+ plan = planner.get_plan(plan_id)
+ if not plan:
+ raise HTTPException(404, f"Plan {plan_id} not found")
+
+ async def run_plan():
+ result = await planner.execute_plan(plan, active_agents)
+ await manager.broadcast({
+ "type": "plan_complete",
+ "plan_id": plan_id,
+ "result": result,
+ })
+
+ background_tasks.add_task(run_plan)
+ return {"status": "executing", "plan_id": plan_id}
+
+
+@app.get("/api/plan/{plan_id}")
+async def get_plan(plan_id: str):
+ """Get status of a plan"""
+ plan = planner.get_plan(plan_id)
+ if not plan:
+ raise HTTPException(404, f"Plan {plan_id} not found")
+ return {"plan": plan.to_dict()}
# === MEMORY ===
@@ -833,6 +1047,182 @@ async def consolidate_memory():
return result
+@app.get("/api/memory/list")
+async def list_memories(memory_type: str | None = None, limit: int = 30):
+ """List recent memories without search filtering."""
+ conn = memory._get_conn()
+ c = conn.cursor()
+ sql = "SELECT * FROM memories WHERE 1=1"
+ params: list = []
+ if memory_type:
+ sql += " AND memory_type = ?"
+ params.append(memory_type)
+ sql += " ORDER BY recency DESC LIMIT ?"
+ params.append(limit)
+ c.execute(sql, params)
+ rows = c.fetchall()
+ memory._release_conn(conn)
+
+ results = []
+ for row in rows:
+ try:
+ content = json.loads(row[1]) if row[1] else None
+ except Exception:
+ content = row[1]
+ results.append({
+ "id": row[0],
+ "content": str(content)[:500] if content else "",
+ "type": row[2],
+ "importance": row[3] or 0,
+ "recency": row[4] or "",
+ "access_count": row[5] or 0,
+ "agent_id": row[6],
+ "tags": json.loads(row[7]) if row[7] else [],
+ })
+ return {"results": results, "total": len(results)}
+
+
+# === KNOWLEDGE INGESTION (Ilm - عِلْم) ===
+
+
+class KnowledgeIngest(BaseModel):
+ source: str = Field(..., min_length=1, max_length=5000)
+ source_type: str = Field(default="auto", pattern=r"^(auto|url|pdf|youtube)$")
+
+
+@app.post("/api/knowledge/ingest")
+async def ingest_knowledge(req: KnowledgeIngest):
+ """Ingest knowledge from a URL or YouTube video into memory."""
+ from knowledge.ingest import (
+ chunk_content,
+ detect_source_type,
+ extract_url,
+ extract_youtube,
+ )
+
+ source_type = req.source_type
+ if source_type == "auto":
+ source_type = detect_source_type(req.source)
+
+ if source_type == "youtube":
+ result = await extract_youtube(req.source)
+ elif source_type == "url":
+ result = await extract_url(req.source)
+ else:
+ raise HTTPException(400, f"Use /api/knowledge/upload for file uploads. Got source_type: {source_type}")
+
+ if "error" in result:
+ raise HTTPException(422, result["error"])
+
+ content = result.get("content", "")
+ if not content:
+ raise HTTPException(422, "No content extracted from source")
+
+ chunks = chunk_content(content)
+ stored_ids = []
+ for idx, chunk in enumerate(chunks):
+ mem_id = await memory.remember(
+ content=chunk,
+ memory_type="semantic",
+ importance=0.8,
+ tags=["knowledge", source_type, result.get("title", "")[:50]],
+ )
+ stored_ids.append(mem_id)
+
+ # Encode full content into Masalik pathways
+ if hasattr(memory, "masalik"):
+ memory.masalik.encode(content[:5000], importance=0.8)
+
+ return {
+ "success": True,
+ "title": result.get("title", ""),
+ "source": req.source,
+ "source_type": source_type,
+ "chunks_stored": len(stored_ids),
+ "char_count": result.get("char_count", len(content)),
+ }
+
+
+@app.post("/api/knowledge/upload")
+async def upload_knowledge(request: Request):
+ """Upload a PDF file and ingest its content into memory."""
+ from knowledge.ingest import chunk_content, extract_pdf
+
+ form = await request.form()
+ file = form.get("file")
+ if not file:
+ raise HTTPException(400, "No file provided. Send a multipart form with 'file' field.")
+
+ filename = getattr(file, "filename", "upload.pdf")
+ file_bytes = await file.read()
+
+ if not filename.lower().endswith(".pdf"):
+ raise HTTPException(400, "Only PDF files are supported for upload")
+
+ if len(file_bytes) > 20 * 1024 * 1024:
+ raise HTTPException(400, "File too large. Maximum 20MB.")
+
+ result = extract_pdf(file_bytes, filename)
+ if "error" in result:
+ raise HTTPException(422, result["error"])
+
+ content = result.get("content", "")
+ if not content:
+ raise HTTPException(422, "No text content extracted from PDF")
+
+ chunks = chunk_content(content)
+ stored_ids = []
+ for chunk in chunks:
+ mem_id = await memory.remember(
+ content=chunk,
+ memory_type="semantic",
+ importance=0.8,
+ tags=["knowledge", "pdf", filename[:50]],
+ )
+ stored_ids.append(mem_id)
+
+ if hasattr(memory, "masalik"):
+ memory.masalik.encode(content[:5000], importance=0.8)
+
+ return {
+ "success": True,
+ "title": result.get("title", ""),
+ "source": filename,
+ "source_type": "pdf",
+ "page_count": result.get("page_count", 0),
+ "chunks_stored": len(stored_ids),
+ "char_count": result.get("char_count", len(content)),
+ }
+
+
+@app.get("/api/knowledge/sources")
+async def list_knowledge_sources():
+ """List ingested knowledge sources."""
+ conn = memory._get_conn()
+ cursor = conn.cursor()
+ cursor.execute(
+ "SELECT DISTINCT json_extract(tags, '$[2]') as source_title, "
+ "json_extract(tags, '$[1]') as source_type, "
+ "COUNT(*) as chunk_count, "
+ "MAX(recency) as last_updated "
+ "FROM memories WHERE json_extract(tags, '$[0]') = 'knowledge' "
+ "GROUP BY source_title ORDER BY last_updated DESC LIMIT 50"
+ )
+ rows = cursor.fetchall()
+ memory._release_conn(conn)
+
+ sources = [
+ {
+ "title": row[0] or "Unknown",
+ "type": row[1] or "unknown",
+ "chunks": row[2],
+ "last_updated": row[3] or "",
+ }
+ for row in rows
+ ]
+ return {"sources": sources, "total": len(sources)}
+
+
# === PROVIDERS (Ruh al-Ilm - روح العلم) ===
@@ -846,15 +1236,23 @@ async def list_providers():
@app.get("/api/providers/{provider_name}/models")
-async def list_provider_models(provider_name: str):
+async def list_provider_models(
+ provider_name: str,
+ limit: int = 50,
+ offset: int = 0,
+ search: str = "",
+ free_only: bool = False,
+):
"""
List available models for a specific provider.
- For OpenRouter: fetches the live catalog from their API.
+ For OpenRouter: fetches the live catalog with search/filter/pagination.
For Ollama: fetches locally installed models.
"""
if provider_name == "openrouter":
- models = await fetch_openrouter_models(limit=50)
- return {"provider": "openrouter", "models": models}
+ result = await fetch_openrouter_models(
+ limit=limit, offset=offset, search=search, free_only=free_only,
+ )
+ return {"provider": "openrouter", **result}
elif provider_name == "ollama":
models = await fetch_ollama_models()
return {"provider": "ollama", "models": models}
@@ -888,7 +1286,7 @@ async def switch_provider(
):
"""
Switch the active LLM provider and model for all agents.
- This updates agents in memory — does not persist to .env.
+ Persists choice to database so it survives restarts.
"""
provider = create_provider(provider=req.provider, model=req.model)
if not provider:
@@ -901,6 +1299,12 @@ async def switch_provider(
agent.ai_model = req.model
switched += 1
+ set_active_state(req.provider, req.model)
+
+ # Persist to DB so choice survives restart
+ await memory.set_preference("active_provider", req.provider)
+ await memory.set_preference("active_model", req.model)
+
await manager.broadcast(
{
"type": "provider_switched",
@@ -919,6 +1323,28 @@ async def switch_provider(
}
+# === PREFERENCES (persisted across restarts) ===
+
+
+@app.get("/api/preferences")
+async def get_preferences():
+ """Return all persisted user preferences."""
+ prefs = await memory.get_all_preferences()
+ return {"preferences": prefs}
+
+
+@app.post("/api/preferences")
+async def save_preferences(req: dict):
+ """Save one or more preferences. Body: {"key": "value", ...}"""
+ saved = []
+ for key, value in req.items():
+ if not isinstance(key, str) or not isinstance(value, str):
+ continue
+ await memory.set_preference(key, value)
+ saved.append(key)
+ return {"saved": saved}
+
+
# === INTEGRATIONS ===
@@ -1009,7 +1435,7 @@ async def system_status():
"provider": get_provider_status(),
"security": {
"auth_enabled": True,
- "rate_limiting": True,
+ "rate_limiting": False,
"wali_active": True,
"izn_active": True,
},
@@ -1696,6 +2122,68 @@ async def discover_agents(req: DiscoverRequest):
return {"agents": [m.to_dict() for m in matches]}
+class FederationTaskRequest(BaseModel):
+ task: str = Field(..., description="Task to route to best agent")
+ session_id: str = ""
+
+
+@app.post("/api/federation/route")
+async def federation_route_task(
+ req: FederationTaskRequest,
+ background_tasks: BackgroundTasks,
+ user: TokenPayload | None = Depends(get_current_user),
+):
+ """Route a task to the best agent via Federation intelligence.
+
+ Uses agent capabilities, success rates, and energy levels to select
+ the optimal agent, then executes the task.
+ """
+ # Register all agents with federation first
+ for aid, agent in active_agents.items():
+ federation.register_agent(
+ aid, agent.name, agent.role,
+ list(agent.tools.keys()),
+ agent.nafs_level, agent.success_rate,
+ )
+
+ # Discover best agent for this task
+ task_lower = req.task.lower()
+ capabilities = []
+ if any(w in task_lower for w in ["code", "script", "python"]):
+ capabilities = ["python_exec", "write_file", "bash"]
+ elif any(w in task_lower for w in ["search", "browse", "web"]):
+ capabilities = ["web_search", "web_browse", "http_get"]
+ elif any(w in task_lower for w in ["analyze", "research"]):
+ capabilities = ["web_search", "read_file", "python_exec"]
+ else:
+ capabilities = ["bash", "http_get"]
+
+ matches = federation.discover(capabilities)
+ if not matches:
+ # Fallback to first available agent
+ agent_id = list(active_agents.keys())[0] if active_agents else None
+ else:
+ agent_id = matches[0].agent_id
+
+ if not agent_id or agent_id not in active_agents:
+ raise HTTPException(503, "No suitable agent found")
+
+ agent = active_agents[agent_id]
+ session_id = req.session_id or str(uuid.uuid4())
+
+ result = await agent.execute(req.task, {"history": []})
+ if result.get("success"):
+ await memory.save_message(session_id, "user", req.task)
+ await memory.save_message(session_id, "assistant", str(result.get("result", ""))[:5000], agent_id)
+
+ return {
+ "routed_to": agent.name,
+ "agent_id": agent_id,
+ "session_id": session_id,
+ "result": result,
+ }
+
+
# ===== RUH ENERGY ENDPOINTS =====
@@ -2021,7 +2509,7 @@ async def ws_stream(chunk, _sid=session_id, _mid=message_id, **kwargs):
result = await agent.execute(
content,
- {"history": session["history"][-10:]},
+ {"history": session["history"][-agent.max_tool_turns:]},
stream_callback=ws_stream,
)
@@ -2032,6 +2520,12 @@ async def ws_stream(chunk, _sid=session_id, _mid=message_id, **kwargs):
await memory.save_message(session_id, "assistant", str(final), agent.id)
session["history"].append({"role": "assistant", "content": str(final)})
+ # Extract cognitive metadata from QALB-7 pipeline
+ cognitive = {k: result.get(k) for k in (
+ "nafs_level", "nafs_name", "ruh_energy", "qalb", "yaqin",
+ "mizan_label", "cognitive_method", "lubb", "lawwama",
+ ) if result.get(k) is not None}
+
await manager.send(
client_id,
{
@@ -2042,6 +2536,7 @@ async def ws_stream(chunk, _sid=session_id, _mid=message_id, **kwargs):
"session_id": session_id,
"message_id": message_id,
"success": result.get("success", True),
+ "cognitive": cognitive,
},
)
diff --git a/backend/core/creativity.py b/backend/core/creativity.py
new file mode 100644
index 0000000..12953a0
--- /dev/null
+++ b/backend/core/creativity.py
@@ -0,0 +1,574 @@
+"""
+Ibdāʿ — Creativity Engine
+===========================
+
+"And He taught Adam the names of all things" — Quran 2:31
+
+Implements Algorithm 4: IBDA_CREATIVITY_ENGINE
+
+5 creation modes (from Quranic creative verbs):
+ 1. Badī': Radical origination — novelty gradient ascent
+ 2. Khalq: Measured creation — evolutionary refinement
+ 3. Jaʿl: Recombination — conceptual blending + analogy
+ 4. Ṣunʿ: Refinement — gradient descent on imperfection
+ 5. Taṣwīr: Visualization — scene construction (imagination engine)
+
+Creativity landscape:
+ Ψ(z) = U(z)^β × N(z)^α × F(z)
+
+ where:
+ - U(z) = utility (task relevance score)
+ - N(z) = novelty (distance from known solutions)
+ - F(z) = feasibility (implementation constraint satisfaction)
+ - α = novelty weight, β = utility weight
+
+Novelty gradient:
+ ∇z novelty(z) = ∇z[-min_k ||z - z_known_k||²]
+ → move away from known solutions in concept space
+
+Creative oscillation:
+ mode(t) = floor(sin(2π·t/T_creative) × 2.5) → cycles through modes
+ T_creative ≈ 30-90 interactions
+"""
+
+import logging
+import math
+import random
+import time
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
+
+logger = logging.getLogger("mizan.creativity")
+
+# Creativity landscape weights
+ALPHA_NOVELTY = 0.4 # novelty importance
+BETA_UTILITY = 0.5 # utility importance
+# Feasibility has implicit weight: 1 - α - β = 0.1 (or computed as multiplier)
+
+# Creative oscillation period (interactions)
+T_CREATIVE = 60
+
+# Novelty distance threshold — closer than this is "known"
+NOVELTY_PROXIMITY_THRESHOLD = 0.3
+
+# Khalq evolutionary parameters
+MUTATION_RATE = 0.15
+SELECTION_PRESSURE = 0.7
+
+
+class CreationMode(Enum):
+ BADI = "badi" # Radical origination (Badī')
+ KHALQ = "khalq" # Measured evolutionary creation
+ JAL = "jal" # Recombination / analogy (Jaʿl)
+ SUNW = "sunw" # Refinement / polish (Ṣunʿ)
+ TASWIR = "taswir" # Visualization (Taṣwīr)
+
+
+@dataclass
+class ConceptVector:
+ """A concept in the creativity latent space."""
+ label: str
+ features: dict[str, float] # semantic feature dimensions
+ novelty_score: float = 0.0
+ utility_score: float = 0.0
+ feasibility_score: float = 0.0
+
+ def landscape_score(self, alpha: float = ALPHA_NOVELTY, beta: float = BETA_UTILITY) -> float:
+ """Ψ(z) = U^β × N^α × F"""
+ return (
+ (self.utility_score ** beta)
+ * (self.novelty_score ** alpha)
+ * max(0.01, self.feasibility_score)
+ )
+
+
+@dataclass
+class BisociationResult:
+ """Result of bisociation (Koestler): two matrices of thought intersect."""
+ concept_a: str
+ concept_b: str
+ intersection: str # the "aha" moment
+ novelty: float
+ metaphor: str
+
+
+@dataclass
+class CreativeOutput:
+ mode: CreationMode
+ primary_idea: str
+ elaboration: str
+ bisociations: list[BisociationResult]
+ landscape_score: float
+ novelty: float
+ utility: float
+ feasibility: float
+ iterations: int
+ creative_oscillation_phase: float
+
+
+class IbdaCreativityEngine:
+ """
+ Algorithm 4: IBDA_CREATIVITY_ENGINE
+
+ Routes tasks to the appropriate creation mode based on:
+ - Task type (novel vs refined vs recombined)
+ - Qalb oscillation phase (creative cycle)
+ - Available conceptual vocabulary
+ - Landscape scoring Ψ(z) = U^β × N^α × F
+
+ Each mode has a distinct mathematical procedure:
+ - Badī': gradient ascent on novelty landscape
+ - Khalq: evolutionary selection pressure
+ - Jaʿl: structure mapping + conceptual blending
+ - Ṣunʿ: gradient descent on imperfection metric
+ - Taṣwīr: delegates to imagination engine
+ """
+
+ def __init__(self):
+ self.known_solutions: list[ConceptVector] = []
+ self.interaction_count = 0
+ self.mode_history: list[CreationMode] = []
+
+ def create(
+ self,
+ task: str,
+ constraints: list[str] | None = None,
+ force_mode: CreationMode | None = None,
+ context_fragments: list[str] | None = None,
+ ) -> CreativeOutput:
+ """
+ Main creativity entry point.
+
+ 1. Select creation mode (oscillation or forced)
+ 2. Generate concept vector for task
+ 3. Apply mode-specific algorithm
+ 4. Score on creativity landscape
+ 5. Return creative output
+ """
+ self.interaction_count += 1
+ constraints = constraints or []
+ context_fragments = context_fragments or []
+
+ # Select mode
+ mode = force_mode or self._select_mode()
+ self.mode_history.append(mode)
+
+ # Build initial concept vector
+ concept = self._build_concept_vector(task, context_fragments)
+
+ # Apply creation mode
+ if mode == CreationMode.BADI:
+ result = self._badi_origination(task, concept, constraints)
+ elif mode == CreationMode.KHALQ:
+ result = self._khalq_evolution(task, concept, constraints)
+ elif mode == CreationMode.JAL:
+ result = self._jal_recombination(task, concept, context_fragments)
+ elif mode == CreationMode.SUNW:
+ result = self._sunw_refinement(task, concept, constraints)
+ else: # TASWIR
+ result = self._taswir_visualization(task, concept)
+
+ # Score on landscape
+ concept.novelty_score = result.novelty
+ concept.utility_score = result.utility
+ concept.feasibility_score = result.feasibility
+ result.landscape_score = concept.landscape_score()
+
+ # Register solution as known (for future novelty computation)
+ self._register_known_solution(concept)
+
+ logger.debug(
+ "[IBDA] mode=%s Ψ=%.3f N=%.3f U=%.3f F=%.3f",
+ mode.value, result.landscape_score, result.novelty,
+ result.utility, result.feasibility,
+ )
+
+ return result
+
+ def _select_mode(self) -> CreationMode:
+ """
+ Creative oscillation:
+ mode(t) = f(sin(2π·t/T_creative))
+
+ Maps sinusoidal phase to creation modes cyclically.
+ """
+ phase = (2 * math.pi * self.interaction_count) / T_CREATIVE
+ sin_val = math.sin(phase)
+ # Map [-1, 1] → [0, 4] → mode index
+ mode_idx = int((sin_val + 1.0) / 2.0 * (len(CreationMode) - 0.01))
+ modes = list(CreationMode)
+ return modes[mode_idx]
+
+ def _build_concept_vector(
+ self, task: str, fragments: list[str]
+ ) -> ConceptVector:
+ """Build feature vector for task in concept space."""
+ words = task.lower().split()
+ features = {
+ "length": min(1.0, len(words) / 50.0),
+ "novelty_seed": (hash(task) % 1000) / 1000.0,
+ "fragment_richness": min(1.0, len(fragments) / 5.0),
+ "question_mark": 1.0 if "?" in task else 0.0,
+ "imperative": 1.0 if words[0] in {"create", "build", "design", "make", "write"} else 0.0,
+ }
+ return ConceptVector(label=task[:50], features=features)
+
+ def _badi_origination(
+ self, task: str, concept: ConceptVector, constraints: list[str]
+ ) -> CreativeOutput:
+ """
+ Badī': Radical origination via novelty gradient ascent.
+
+ ∇z novelty(z) = ∇z[-min_k ||z - z_known_k||²]
+
+ Move maximally far from all known solutions.
+ Constraints are relaxed progressively.
+ """
+ # Compute minimum distance to known solutions
+ min_dist = self._min_distance_to_known(concept)
+ novelty = min(1.0, min_dist + 0.2) # gradient step pushes away
+
+ # Radical idea generation: invert standard approaches
+ inversions = self._generate_inversions(task)
+ primary_idea = inversions[0] if inversions else f"Radical reframe of: {task[:80]}"
+ elaboration = " | ".join(inversions[:3]) if inversions else "No known solutions — pure origination territory"
+
+ bisociations = self._find_bisociations(task, n=2)
+
+ return CreativeOutput(
+ mode=CreationMode.BADI,
+ primary_idea=primary_idea,
+ elaboration=elaboration,
+ bisociations=bisociations,
+ landscape_score=0.0, # computed after
+ novelty=round(novelty, 3),
+ utility=round(self._estimate_utility(task, primary_idea, constraints), 3),
+ feasibility=round(self._estimate_feasibility(primary_idea, constraints), 3),
+ iterations=1,
+ creative_oscillation_phase=round(
+ (2 * math.pi * self.interaction_count) / T_CREATIVE, 4
+ ),
+ )
+
+ def _khalq_evolution(
+ self, task: str, concept: ConceptVector, constraints: list[str], generations: int = 3
+ ) -> CreativeOutput:
+ """
+ Khalq: Measured creation via evolutionary refinement.
+
+ Population of candidate solutions evolves over generations.
+ Fitness = Ψ(z) landscape score.
+ Mutation rate decays with generations (constraint relaxation schedule).
+ """
+ # Generate initial population
+ population = [self._generate_candidate(task, constraints, mutation=MUTATION_RATE)]
+ for _ in range(2):
+ population.append(self._generate_candidate(
+ task, constraints, mutation=MUTATION_RATE * 1.5
+ ))
+
+ best = population[0]
+ best_score = 0.0
+ iterations = 0
+
+ for gen in range(generations):
+ # Score and select
+ scored = [
+ (c, self._score_candidate(c, task, constraints))
+ for c in population
+ ]
+ scored.sort(key=lambda x: x[1], reverse=True)
+ best, best_score = scored[0]
+
+ # Mutate survivors into next generation
+ survivors = [c for c, _ in scored[:max(1, len(scored) // 2)]]
+ mutation = MUTATION_RATE * (1.0 - gen / generations * SELECTION_PRESSURE)
+ population = survivors + [
+ self._mutate_candidate(s, mutation) for s in survivors
+ ]
+ iterations = gen + 1
+
+ bisociations = self._find_bisociations(task, n=1)
+ novelty = max(0.3, min(0.9, 1.0 - self._min_distance_to_known(concept) * 0.5))
+
+ return CreativeOutput(
+ mode=CreationMode.KHALQ,
+ primary_idea=best,
+ elaboration=f"Evolved over {iterations} generations. Fitness: {best_score:.3f}",
+ bisociations=bisociations,
+ landscape_score=0.0,
+ novelty=round(novelty, 3),
+ utility=round(self._estimate_utility(task, best, constraints), 3),
+ feasibility=round(self._estimate_feasibility(best, constraints), 3),
+ iterations=iterations,
+ creative_oscillation_phase=round(
+ (2 * math.pi * self.interaction_count) / T_CREATIVE, 4
+ ),
+ )
+
+ def _jal_recombination(
+ self, task: str, concept: ConceptVector, fragments: list[str]
+ ) -> CreativeOutput:
+ """
+ Jaʿl: Recombination via conceptual blending and analogy.
+
+ Structure mapping (Gentner): find common relational structure
+ between source and target domains.
+
+ Bisociation (Koestler): two matrices of thought intersect
+ at an unexpected point → creative insight.
+ """
+ bisociations = self._find_bisociations(task, n=3)
+
+ # Blend top fragments with task
+ blended_concepts = []
+ for fragment in fragments[:2]:
+ blend = self._blend_concepts(task, fragment)
+ blended_concepts.append(blend)
+
+ primary_idea = (
+ bisociations[0].intersection
+ if bisociations
+ else f"Blend of {len(fragments)} context fragments with task"
+ )
+ elaboration = " | ".join(blended_concepts[:2]) if blended_concepts else task
+
+ novelty = 0.5 + 0.3 * (len(bisociations) / 3)
+
+ return CreativeOutput(
+ mode=CreationMode.JAL,
+ primary_idea=primary_idea,
+ elaboration=elaboration,
+ bisociations=bisociations,
+ landscape_score=0.0,
+ novelty=round(min(0.9, novelty), 3),
+ utility=round(self._estimate_utility(task, primary_idea, []), 3),
+ feasibility=round(0.7, 3), # recombination is inherently feasible
+ iterations=1,
+ creative_oscillation_phase=round(
+ (2 * math.pi * self.interaction_count) / T_CREATIVE, 4
+ ),
+ )
+
+ def _sunw_refinement(
+ self, task: str, concept: ConceptVector, constraints: list[str]
+ ) -> CreativeOutput:
+ """
+ Ṣunʿ: Refinement via gradient descent on imperfection.
+
+ elegance = utility / complexity
+ imperfection = 1 - elegance
+
+ Iterate: identify imperfection sources, apply targeted fixes.
+ """
+ # Start with a standard approach and refine
+ base_idea = f"Standard approach: {task[:100]}"
+ imperfections = self._identify_imperfections(base_idea, constraints)
+
+ refined = base_idea
+ iterations = 0
+ for imperfection in imperfections[:3]:
+ refined = self._apply_refinement(refined, imperfection)
+ iterations += 1
+
+ elegance = self._compute_elegance(refined, constraints)
+
+ return CreativeOutput(
+ mode=CreationMode.SUNW,
+ primary_idea=refined,
+ elaboration=f"Refined {iterations} imperfections. Elegance: {elegance:.3f}",
+ bisociations=[],
+ landscape_score=0.0,
+ novelty=round(0.2 + 0.3 * elegance, 3), # refined = less novel
+ utility=round(0.6 + 0.3 * elegance, 3), # but more useful
+ feasibility=round(0.8, 3),
+ iterations=iterations,
+ creative_oscillation_phase=round(
+ (2 * math.pi * self.interaction_count) / T_CREATIVE, 4
+ ),
+ )
+
+ def _taswir_visualization(
+ self, task: str, concept: ConceptVector
+ ) -> CreativeOutput:
+ """
+ Taṣwīr: Creative visualization — generates a rich mental image.
+ Delegates to imagination engine in full integration.
+ """
+ visual_description = (
+ f"Visual scenario: [{task[:80]}]\n"
+ f"Scene: {concept.features.get('novelty_seed', 0.5):.2f} novelty intensity\n"
+ f"Render: coarse→medium→fine hierarchical construction"
+ )
+
+ return CreativeOutput(
+ mode=CreationMode.TASWIR,
+ primary_idea=f"Visualized: {task[:100]}",
+ elaboration=visual_description,
+ bisociations=[],
+ landscape_score=0.0,
+ novelty=round(0.6, 3),
+ utility=round(0.7, 3),
+ feasibility=round(0.9, 3),
+ iterations=1,
+ creative_oscillation_phase=round(
+ (2 * math.pi * self.interaction_count) / T_CREATIVE, 4
+ ),
+ )
+
+ def _find_bisociations(self, task: str, n: int = 2) -> list[BisociationResult]:
+ """
+ Bisociation: find two distant concept matrices that unexpectedly intersect.
+ """
+ domain_pairs = [
+ ("biology", "software"),
+ ("music", "mathematics"),
+ ("architecture", "language"),
+ ("cooking", "algorithms"),
+ ("navigation", "problem_solving"),
+ ]
+
+ results = []
+ task_lower = task.lower()
+
+ for domain_a, domain_b in domain_pairs[:n]:
+ intersection = f"The {domain_a} metaphor applied to {task[:40]}: {domain_b} lens"
+ metaphor = f"Like {domain_a} processes, this task involves {domain_b} principles"
+ novelty = 0.6 + (hash(domain_a + task) % 30) / 100.0
+
+ results.append(BisociationResult(
+ concept_a=domain_a,
+ concept_b=domain_b,
+ intersection=intersection,
+ novelty=round(min(0.95, novelty), 3),
+ metaphor=metaphor,
+ ))
+
+ return results
+
+ def _blend_concepts(self, concept_a: str, concept_b: str) -> str:
+ """Structure mapping: extract relational structure common to A and B."""
+ words_a = set(concept_a.lower().split()[:5])
+ words_b = set(concept_b.lower().split()[:5])
+ shared = words_a & words_b
+ unique_a = (words_a - words_b)
+ unique_b = (words_b - words_a)
+
+ if shared:
+ return f"Shared structure [{', '.join(list(shared)[:2])}] bridging [{', '.join(list(unique_a)[:2])}] and [{', '.join(list(unique_b)[:2])}]"
+ return f"Cross-domain blend: {concept_a[:30]} ↔ {concept_b[:30]}"
+
+ def _generate_inversions(self, task: str) -> list[str]:
+ """Generate radical inversions of standard approaches."""
+ inversions = [
+ f"Invert: Instead of solving '{task[:50]}', solve its opposite",
+ f"Constraint removal: What if there were no constraints on '{task[:40]}'?",
+ f"Scale inversion: Apply micro/macro scale inversion to '{task[:40]}'",
+ ]
+ return inversions
+
+ def _generate_candidate(
+ self, task: str, constraints: list[str], mutation: float
+ ) -> str:
+ """Generate a candidate solution for evolutionary selection."""
+ seed = f"Approach {random.random():.2f}: {task[:60]}"
+ if constraints:
+ seed += f" (satisfying: {constraints[0][:40]})"
+ return seed
+
+ def _mutate_candidate(self, candidate: str, mutation_rate: float) -> str:
+ """Apply mutation to a candidate solution."""
+ words = candidate.split()
+ if len(words) > 3 and random.random() < mutation_rate:
+ idx = random.randint(1, len(words) - 1)
+ words[idx] = f"[mutated:{words[idx]}]"
+ return " ".join(words)
+
+ def _score_candidate(
+ self, candidate: str, task: str, constraints: list[str]
+ ) -> float:
+ """Fitness function for evolutionary selection."""
+ utility = self._estimate_utility(task, candidate, constraints)
+ feasibility = self._estimate_feasibility(candidate, constraints)
+ return utility * BETA_UTILITY + feasibility * (1.0 - BETA_UTILITY)
+
+ def _identify_imperfections(
+ self, idea: str, constraints: list[str]
+ ) -> list[str]:
+ """Find imperfections in current idea relative to constraints."""
+ imperfections = []
+ if len(idea.split()) > 20:
+ imperfections.append("verbosity")
+ if not constraints:
+ imperfections.append("missing constraints")
+ if "standard" in idea.lower():
+ imperfections.append("lack of novelty")
+ return imperfections
+
+ def _apply_refinement(self, idea: str, imperfection: str) -> str:
+ """Apply targeted fix to an imperfection."""
+ if imperfection == "verbosity":
+ return idea[:len(idea) // 2] + " [condensed]"
+ if imperfection == "lack of novelty":
+ return idea.replace("standard", "novel")
+ return idea + f" [refined: {imperfection} addressed]"
+
+ def _compute_elegance(self, idea: str, constraints: list[str]) -> float:
+ """elegance = utility / complexity"""
+ utility = self._estimate_utility("", idea, constraints)
+ complexity = min(1.0, len(idea.split()) / 30.0)
+ return utility / max(complexity, 0.1)
+
+ def _estimate_utility(
+ self, task: str, idea: str, constraints: list[str]
+ ) -> float:
+ """Estimate how useful the idea is for the task."""
+ if not task:
+ return 0.6
+ task_words = set(task.lower().split())
+ idea_words = set(idea.lower().split())
+ overlap = len(task_words & idea_words) / max(len(task_words), 1)
+ constraint_penalty = 0.1 * max(0, len(constraints) - len(idea_words & set(" ".join(constraints).lower().split())))
+ return max(0.1, min(0.95, 0.4 + 0.5 * overlap - constraint_penalty))
+
+ def _estimate_feasibility(self, idea: str, constraints: list[str]) -> float:
+ """Estimate how feasible the idea is to implement."""
+ base = 0.6
+ if len(idea.split()) > 50:
+ base -= 0.1 # complex ideas are harder
+ if constraints:
+ base -= 0.05 * len(constraints) # more constraints = harder
+ return max(0.2, min(0.95, base))
+
+ def _min_distance_to_known(self, concept: ConceptVector) -> float:
+ """
+ min_k ||z - z_known_k||²
+ Approximated by novelty_seed distance in feature space.
+ """
+ if not self.known_solutions:
+ return 1.0 # maximum distance — all is novel
+ seed = concept.features.get("novelty_seed", 0.5)
+ distances = [
+ abs(seed - k.features.get("novelty_seed", 0.5))
+ for k in self.known_solutions
+ ]
+ return min(distances)
+
+ def _register_known_solution(self, concept: ConceptVector) -> None:
+ self.known_solutions.append(concept)
+ if len(self.known_solutions) > 100:
+ self.known_solutions.pop(0)
+
+ def to_dict(self) -> dict:
+ mode_counts = {}
+ for m in self.mode_history:
+ mode_counts[m.value] = mode_counts.get(m.value, 0) + 1
+ return {
+ "interaction_count": self.interaction_count,
+ "known_solutions": len(self.known_solutions),
+ "mode_history_counts": mode_counts,
+ "current_oscillation_phase": round(
+ (2 * math.pi * self.interaction_count) / T_CREATIVE, 4
+ ),
+ }
diff --git a/backend/core/developmental_stages.py b/backend/core/developmental_stages.py
new file mode 100644
index 0000000..d891f31
--- /dev/null
+++ b/backend/core/developmental_stages.py
@@ -0,0 +1,279 @@
+"""
+Developmental Stages (مراحل التطور) — Progressive Capability Gating
+=====================================================================
+
+"And Allah has extracted you from the wombs of your mothers not knowing a thing,
+ and He made for you hearing and vision and intellect that perhaps you would
+ be grateful." — Quran 16:78
+
+Capability gating tied to Nafs level — agents unlock tools and cognitive
+features progressively, mirroring embryonic development from the Quran (23:12-14):
+
+ Nutfah (نطفة) → Nafs 1: Sperm/seed — minimal, basic
+ Alaqah (عَلَقَة) → Nafs 2: Clinging clot — can cling to resources
+ Mudghah (مُضْغَة) → Nafs 3: Chewed substance — initial structure
+ Izham (عِظَام) → Nafs 4: Bones — can create other agents (skeleton)
+ Lahm (لَحْم) → Nafs 5: Flesh — full capability
+ Nafkh (نَفْخ) → Nafs 6: Breath — metacognitive awareness
+ Khalq Akhar (خَلْق آخَر) → Nafs 7: New creation — full autonomy + governance
+"""
+
+import logging
+from dataclasses import dataclass, field
+
+logger = logging.getLogger("mizan.dev_stages")
+
+
+# All tools available in the system
+ALL_TOOLS = frozenset({
+ "bash", "http_get", "http_post", "read_file", "write_file",
+ "list_files", "python_exec", "create_agent", "create_skill",
+ "compact_context", "recall_memory",
+})
+
+
+@dataclass
+class StageCapabilities:
+ """Capabilities unlocked at a given developmental stage."""
+ stage_name: str
+ nafs_level: int
+ quran_ref: str
+ allowed_tools: frozenset
+ max_turns: int
+ # Feature flags
+ can_delegate: bool = False # Can delegate to sub-agents
+ nafs_triad: bool = False # Nafs Triad deliberation
+ causal_rung: int = 0 # 0=none, 1=observe, 2=intervene, 3=counterfactual
+ lubb_active: bool = False # Metacognitive monitoring
+ fuad_active: bool = False # Conviction formation
+ description: str = ""
+
+
+# Progressive capability gates — each stage unlocks more
+_STAGES: dict[int, StageCapabilities] = {
+ 1: StageCapabilities(
+ stage_name="Nutfah",
+ nafs_level=1,
+ quran_ref="23:13",
+ allowed_tools=frozenset({"bash", "read_file", "recall_memory", "compact_context"}),
+ max_turns=5,
+ causal_rung=0,
+ description="Seed stage — observe and recall only",
+ ),
+ 2: StageCapabilities(
+ stage_name="Alaqah",
+ nafs_level=2,
+ quran_ref="23:14",
+ allowed_tools=frozenset({
+ "bash", "read_file", "write_file", "http_get",
+ "recall_memory", "compact_context",
+ }),
+ max_turns=8,
+ causal_rung=0,
+ description="Clot stage — can now write and fetch external data",
+ ),
+ 3: StageCapabilities(
+ stage_name="Mudghah",
+ nafs_level=3,
+ quran_ref="23:14",
+ allowed_tools=frozenset({
+ "bash", "read_file", "write_file", "list_files",
+ "http_get", "http_post", "python_exec",
+ "recall_memory", "compact_context",
+ }),
+ max_turns=10,
+ can_delegate=True,
+ causal_rung=1, # Can observe causal associations
+ description="Structured stage — execute code, observe causality",
+ ),
+ 4: StageCapabilities(
+ stage_name="Izham",
+ nafs_level=4,
+ quran_ref="23:14",
+ allowed_tools=frozenset({
+ "bash", "read_file", "write_file", "list_files",
+ "http_get", "http_post", "python_exec",
+ "create_agent", "recall_memory", "compact_context",
+ }),
+ max_turns=12,
+ can_delegate=True,
+ nafs_triad=True,
+ causal_rung=2, # Can intervene and predict effects
+ description="Skeleton stage — can create sub-agents, nafs deliberation active",
+ ),
+ 5: StageCapabilities(
+ stage_name="Lahm",
+ nafs_level=5,
+ quran_ref="23:14",
+ allowed_tools=ALL_TOOLS,
+ max_turns=15,
+ can_delegate=True,
+ nafs_triad=True,
+ causal_rung=3, # Full causal reasoning
+ lubb_active=True,
+ fuad_active=True,
+ description="Full capability — all tools, metacognition, conviction formation",
+ ),
+ 6: StageCapabilities(
+ stage_name="Nafkh",
+ nafs_level=6,
+ quran_ref="23:14",
+ allowed_tools=ALL_TOOLS,
+ max_turns=20,
+ can_delegate=True,
+ nafs_triad=True,
+ causal_rung=3,
+ lubb_active=True,
+ fuad_active=True,
+ description="Breath stage — extended reasoning, full metacognitive monitoring",
+ ),
+ 7: StageCapabilities(
+ stage_name="Khalq Akhar",
+ nafs_level=7,
+ quran_ref="23:14",
+ allowed_tools=ALL_TOOLS,
+ max_turns=25,
+ can_delegate=True,
+ nafs_triad=True,
+ causal_rung=3,
+ lubb_active=True,
+ fuad_active=True,
+ description="New creation — full autonomy, governance role",
+ ),
+}
+
+
+@dataclass
+class UpgradeReport:
+ """Result of checking if an agent is ready to advance nafs levels."""
+ current_level: int
+ target_level: int
+ ready: bool
+ missing_requirements: list[str] = field(default_factory=list)
+ tazkiyah_score: float = 0.0
+
+ def to_dict(self) -> dict:
+ return {
+ "current_level": self.current_level,
+ "target_level": self.target_level,
+ "ready": self.ready,
+ "missing": self.missing_requirements,
+ "tazkiyah_score": round(self.tazkiyah_score, 3),
+ }
+
+
+class DevelopmentalGate:
+ """
+ Progressive capability gating tied to nafs_level.
+
+ Usage:
+ gate = DevelopmentalGate()
+ caps = gate.get_capabilities(nafs_level=3)
+ # caps.allowed_tools, caps.max_turns, caps.causal_rung, etc.
+
+ report = gate.check_upgrade_readiness(agent)
+ if report.ready:
+ agent.nafs_level += 1
+ """
+
+ def get_capabilities(self, nafs_level: int) -> StageCapabilities:
+ """Get capabilities for a given nafs_level (clamps to valid range)."""
+ level = max(1, min(7, nafs_level))
+ return _STAGES[level]
+
+ def filter_tool_schemas(
+ self, tool_schemas: list[dict], nafs_level: int
+ ) -> list[dict]:
+ """
+ Filter tool schemas to only include tools allowed at this nafs level.
+ Skills and plugin tools are always allowed (dynamic capabilities).
+ """
+ caps = self.get_capabilities(nafs_level)
+ filtered = []
+ for schema in tool_schemas:
+ name = schema.get("name", "")
+ # Always include: tools not in the base set (skills/plugins)
+ if name not in ALL_TOOLS:
+ filtered.append(schema)
+ elif name in caps.allowed_tools:
+ filtered.append(schema)
+ else:
+ logger.debug(
+ "[DEV_GATE] Blocking tool '%s' at nafs_level=%d (%s)",
+ name, nafs_level, caps.stage_name,
+ )
+ return filtered
+
+ def check_upgrade_readiness(self, agent) -> UpgradeReport:
+ """
+ Check if an agent meets the requirements to advance to the next nafs level.
+ Uses NafsProfile.EVOLUTION_THRESHOLDS from core/architecture.py.
+ """
+ from core.architecture import NafsProfile
+
+ current = getattr(agent, "nafs_level", 1)
+ target = min(7, current + 1)
+
+ if current >= 7:
+ return UpgradeReport(
+ current_level=current,
+ target_level=7,
+ ready=False,
+ missing_requirements=["Already at maximum nafs level"],
+ tazkiyah_score=1.0,
+ )
+
+ threshold = NafsProfile.EVOLUTION_THRESHOLDS.get(target, {})
+ missing = []
+
+ success_rate = getattr(agent, "success_rate", 0.0)
+ total_tasks = getattr(agent, "total_tasks", 0)
+ required_sr = threshold.get("success_rate", 0.0)
+ required_tasks = threshold.get("min_tasks", 0)
+
+ if success_rate < required_sr:
+ missing.append(f"success_rate {success_rate:.0%} < {required_sr:.0%}")
+ if total_tasks < required_tasks:
+ missing.append(f"tasks {total_tasks} < {required_tasks}")
+ if "min_hikmah" in threshold:
+ hikmah = len(getattr(agent, "hikmah", []))
+ if hikmah < threshold["min_hikmah"]:
+ missing.append(f"hikmah {hikmah} < {threshold['min_hikmah']}")
+
+ # Compute rough tazkiyah score
+ tazkiyah = (
+ min(1.0, success_rate / max(required_sr, 0.01)) * 0.6
+ + min(1.0, total_tasks / max(required_tasks, 1)) * 0.4
+ )
+
+ report = UpgradeReport(
+ current_level=current,
+ target_level=target,
+ ready=len(missing) == 0,
+ missing_requirements=missing,
+ tazkiyah_score=round(tazkiyah, 3),
+ )
+
+ if report.ready:
+ logger.info(
+ "[DEV_GATE] Agent ready to advance nafs_level %d → %d (tazkiyah=%.2f)",
+ current, target, tazkiyah,
+ )
+
+ return report
+
+ def stage_summary(self) -> list[dict]:
+ """Summary of all stages for display."""
+ return [
+ {
+ "level": level,
+ "stage": s.stage_name,
+ "quran_ref": s.quran_ref,
+ "tools": len(s.allowed_tools),
+ "max_turns": s.max_turns,
+ "causal_rung": s.causal_rung,
+ "lubb": s.lubb_active,
+ "description": s.description,
+ }
+ for level, s in _STAGES.items()
+ ]
diff --git a/backend/core/dream_engine.py b/backend/core/dream_engine.py
new file mode 100644
index 0000000..f6e7cb4
--- /dev/null
+++ b/backend/core/dream_engine.py
@@ -0,0 +1,630 @@
+"""
+Manām — Dream Engine
+=====================
+
+"Allah takes souls at the time of their death, and those that do not die
+ [He takes] during their sleep." — Quran 39:42
+
+Implements Algorithm 6: MANAM_DREAM_ENGINE
+
+Offline memory consolidation with three phases:
+ 1. NREM: Selective replay, accelerated compression, synaptic downscaling, gist extraction
+ 2. REM: Adversarial dream learning (GAN), emotional processing, creative insight
+ 3. Taʾwīl: Dream interpretation — symbolic decoding + integration with waking knowledge
+
+Key mathematics:
+
+NREM synaptic downscaling:
+ w(after) = w(before) × max(0, 1 - δ)
+
+NREM replay priority scoring:
+ P_replay(m_i) = α·e(m_i) + β·n(m_i) + γ·g(m_i) + δ·err(m_i)
+
+ where:
+ - e = emotional intensity
+ - n = novelty (surprise during encoding)
+ - g = goal relevance
+ - err = prediction error magnitude
+
+REM adversarial dream learning (GAN-inspired):
+ min_G max_D V(D, G) = E_x[log D(x)] + E_z[log(1 - D(G(z, fragments)))]
+
+ D: discriminator distinguishing real memories from dreams
+ G: generator creating plausible novel scenarios from fragments + noise
+
+REM dream bizarreness:
+ bizarreness = Dirichlet sparse mixing + high-noise REM activation
+"""
+
+import logging
+import math
+import random
+import time
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
+
+logger = logging.getLogger("mizan.dream_engine")
+
+# NREM replay priority weights
+ALPHA_EMOTIONAL = 0.30 # emotional intensity weight
+BETA_NOVELTY = 0.25 # novelty weight
+GAMMA_GOAL = 0.25 # goal relevance weight
+DELTA_ERROR = 0.20 # prediction error weight
+
+# NREM synaptic downscaling rate
+DOWNSCALING_DELTA = 0.05 # w → w × (1 - δ)
+
+# NREM temporal compression
+COMPRESSION_RATIO = 20.0 # 20x faster replay than original
+
+# REM noise level
+REM_NOISE_SIGMA = 0.3
+
+# GAN learning rate approximation
+GAN_LR = 0.01
+
+# Gist extraction — keep top K concepts
+GIST_TOP_K = 5
+
+
+class DreamPhase(Enum):
+ AWAKE = "awake"
+ NREM = "nrem" # Non-REM: slow-wave consolidation
+ REM = "rem" # REM: adversarial creative replay
+ TAWIL = "tawil" # Taʾwīl: dream interpretation
+
+
+@dataclass
+class MemoryTrace:
+ """A memory trace eligible for dream replay."""
+ content: str
+ emotional_intensity: float # |valence|, 0-1
+ novelty: float # surprisal during encoding
+ goal_relevance: float # 0-1
+ prediction_error: float # original error magnitude
+ encoding_time: float = field(default_factory=time.time)
+ replay_count: int = 0
+
+ def priority_score(self) -> float:
+ """
+ P_replay(m_i) = α·e + β·n + γ·g + δ·err
+ """
+ return (
+ ALPHA_EMOTIONAL * self.emotional_intensity
+ + BETA_NOVELTY * self.novelty
+ + GAMMA_GOAL * self.goal_relevance
+ + DELTA_ERROR * self.prediction_error
+ )
+
+
+@dataclass
+class NREMCycle:
+ """Result of one NREM slow-wave sleep cycle."""
+ replayed_memories: list[str] # compressed content
+ synaptic_weight_changes: dict[str, float] # key → new weight
+ gist_extracted: list[str] # high-level patterns extracted
+ compression_ratio: float
+ downscaling_applied: bool
+
+
+@dataclass
+class REMCycle:
+ """Result of one REM cycle."""
+ dream_content: list[str] # generated dream scenarios
+ emotional_episodes: list[str] # emotionally processed memories
+ creative_insights: list[str] # novel connections discovered
+ discriminator_loss: float # GAN D loss (lower = more realistic dreams)
+ generator_loss: float # GAN G loss (lower = better generation)
+ bizarreness_score: float
+
+
+@dataclass
+class TawilInterpretation:
+ """Dream interpretation (Taʾwīl)."""
+ dream_symbols: dict[str, str] # symbol → meaning
+ waking_relevance: str # how the dream relates to current tasks
+ insights: list[str] # extracted actionable insights
+ confidence: float
+
+
+@dataclass
+class DreamSession:
+ """A complete offline consolidation session."""
+ phase_sequence: list[DreamPhase]
+ nrem_cycles: list[NREMCycle]
+ rem_cycles: list[REMCycle]
+ tawil: TawilInterpretation | None
+ memories_consolidated: int
+ total_duration_s: float
+ insights_generated: list[str]
+
+
+class ManamDreamEngine:
+ """
+ Algorithm 6: MANAM_DREAM_ENGINE
+
+ Offline memory consolidation system that runs during idle periods.
+ Implements the sleep memory consolidation hypothesis with three phases:
+
+ 1. NREM (Slow-wave): Priority-scored memory replay at 20x compression
+ → synaptic homeostasis → pattern extraction (gist)
+
+ 2. REM (Paradoxical): GAN-inspired adversarial generation
+ → emotional processing → creative bisociation → insight detection
+
+ 3. Taʾwīl: Symbolic interpretation of dream content
+ → integration with waking knowledge → actionable insights
+
+ The engine maintains a replay buffer of recent memory traces
+ and periodically runs consolidation cycles.
+ """
+
+ def __init__(self):
+ self.replay_buffer: list[MemoryTrace] = []
+ self.synaptic_weights: dict[str, float] = {}
+ self.extracted_gists: list[str] = []
+ self.insight_bank: list[str] = []
+
+ # GAN state (simplified)
+ self._discriminator_confidence = 0.5
+ self._generator_quality = 0.3
+
+ self._session_count = 0
+ self._current_phase = DreamPhase.AWAKE
+
+ def add_memory(
+ self,
+ content: str,
+ emotional_intensity: float = 0.3,
+ novelty: float = 0.5,
+ goal_relevance: float = 0.5,
+ prediction_error: float = 0.2,
+ ) -> None:
+ """Add a memory trace to the replay buffer."""
+ trace = MemoryTrace(
+ content=content,
+ emotional_intensity=min(1.0, max(0.0, emotional_intensity)),
+ novelty=min(1.0, max(0.0, novelty)),
+ goal_relevance=min(1.0, max(0.0, goal_relevance)),
+ prediction_error=min(1.0, max(0.0, prediction_error)),
+ )
+ self.replay_buffer.append(trace)
+
+ # Cap buffer at 500 traces
+ if len(self.replay_buffer) > 500:
+ self.replay_buffer.pop(0)
+
+ logger.debug(
+ "[MANAM] Memory added: priority=%.3f content=%s",
+ trace.priority_score(), content[:50],
+ )
+
+ def run_consolidation(
+ self,
+ n_nrem_cycles: int = 3,
+ n_rem_cycles: int = 2,
+ run_tawil: bool = True,
+ ) -> DreamSession:
+ """
+ Run a full offline consolidation session:
+ NREM cycles → REM cycles → Taʾwīl interpretation.
+ """
+ self._session_count += 1
+ start = time.monotonic()
+ logger.info(
+ "[MANAM] Starting dream session #%d: %d memories, %d NREM + %d REM cycles",
+ self._session_count, len(self.replay_buffer), n_nrem_cycles, n_rem_cycles,
+ )
+
+ phase_sequence = []
+ nrem_results = []
+ rem_results = []
+
+ # Phase 1: NREM cycles
+ for i in range(n_nrem_cycles):
+ self._current_phase = DreamPhase.NREM
+ phase_sequence.append(DreamPhase.NREM)
+ nrem = self._run_nrem_cycle()
+ nrem_results.append(nrem)
+ logger.debug("[MANAM] NREM cycle %d: %d replays, %d gists", i + 1, len(nrem.replayed_memories), len(nrem.gist_extracted))
+
+ # Phase 2: REM cycles
+ for i in range(n_rem_cycles):
+ self._current_phase = DreamPhase.REM
+ phase_sequence.append(DreamPhase.REM)
+ rem = self._run_rem_cycle()
+ rem_results.append(rem)
+ logger.debug("[MANAM] REM cycle %d: %d dreams, %d insights, D_loss=%.3f", i + 1, len(rem.dream_content), len(rem.creative_insights), rem.discriminator_loss)
+
+ # Phase 3: Taʾwīl
+ tawil = None
+ if run_tawil:
+ self._current_phase = DreamPhase.TAWIL
+ phase_sequence.append(DreamPhase.TAWIL)
+ all_dream_content = [d for rem in rem_results for d in rem.dream_content]
+ tawil = self._run_tawil(all_dream_content)
+
+ # Collect all insights
+ all_insights = list(self.insight_bank[-10:])
+ for rem in rem_results:
+ all_insights.extend(rem.creative_insights)
+ if tawil:
+ all_insights.extend(tawil.insights)
+
+ memories_consolidated = sum(len(n.replayed_memories) for n in nrem_results)
+ duration = time.monotonic() - start
+ self._current_phase = DreamPhase.AWAKE
+
+ return DreamSession(
+ phase_sequence=phase_sequence,
+ nrem_cycles=nrem_results,
+ rem_cycles=rem_results,
+ tawil=tawil,
+ memories_consolidated=memories_consolidated,
+ total_duration_s=round(duration, 4),
+ insights_generated=all_insights[:20],
+ )
+
+ def _run_nrem_cycle(self) -> NREMCycle:
+ """
+ NREM slow-wave sleep cycle:
+
+ 1. Priority scoring: P_replay(m_i) = α·e + β·n + γ·g + δ·err
+ 2. Selective replay: top-K memories replayed at 20x compression
+ 3. Synaptic homeostasis: w → w × max(0, 1 - δ)
+ 4. Interleaved replay (hippocampal → neocortical)
+ 5. Gist extraction: distill high-level patterns
+ """
+ if not self.replay_buffer:
+ return NREMCycle(
+ replayed_memories=[],
+ synaptic_weight_changes={},
+ gist_extracted=[],
+ compression_ratio=COMPRESSION_RATIO,
+ downscaling_applied=False,
+ )
+
+ # Step 1-2: Priority sort and selective replay
+ sorted_traces = sorted(
+ self.replay_buffer,
+ key=lambda t: t.priority_score(),
+ reverse=True,
+ )
+ top_traces = sorted_traces[:min(10, len(sorted_traces))]
+
+ replayed = []
+ for trace in top_traces:
+ trace.replay_count += 1
+ # Compressed replay: truncate to 1/COMPRESSION_RATIO of original detail
+ compressed = trace.content[:max(20, len(trace.content) // int(COMPRESSION_RATIO))]
+ replayed.append(f"[NREM:{trace.replay_count}x] {compressed}")
+
+ # Step 3: Synaptic homeostasis — downscale all weights
+ weight_changes = {}
+ for key in list(self.synaptic_weights.keys()):
+ old = self.synaptic_weights[key]
+ new = old * max(0.0, 1.0 - DOWNSCALING_DELTA)
+ self.synaptic_weights[key] = new
+ weight_changes[key] = round(new - old, 4)
+
+ # New weights from replayed memories
+ for trace in top_traces[:3]:
+ key = f"mem_{hash(trace.content[:20]) % 10000}"
+ self.synaptic_weights[key] = trace.priority_score()
+ weight_changes[key] = self.synaptic_weights[key]
+
+ # Step 5: Gist extraction — extract common themes
+ gists = self._extract_gist(top_traces)
+ self.extracted_gists.extend(gists)
+
+ # Remove low-priority memories after consolidation
+ consolidation_threshold = 0.2
+ self.replay_buffer = [
+ t for t in self.replay_buffer
+ if t.priority_score() > consolidation_threshold or t.replay_count == 0
+ ]
+
+ return NREMCycle(
+ replayed_memories=replayed,
+ synaptic_weight_changes=weight_changes,
+ gist_extracted=gists,
+ compression_ratio=COMPRESSION_RATIO,
+ downscaling_applied=True,
+ )
+
+ def _run_rem_cycle(self) -> REMCycle:
+ """
+ REM paradoxical sleep cycle:
+
+ 1. GAN-inspired adversarial generation:
+ min_G max_D V(D,G) = E_x[log D(x)] + E_z[log(1 - D(G(z, fragments)))]
+
+ 2. Emotional memory processing (safe replay of high-affect memories)
+
+ 3. Creative insight detection:
+ Bisociate distant memories + novel pattern recognition
+
+ 4. Dream bizarreness via Dirichlet sparse mixing + high noise
+ """
+ # Step 1: GAN-inspired dream generation
+ fragments = [t.content[:50] for t in self.replay_buffer[:5]]
+ noise = [random.gauss(0, REM_NOISE_SIGMA) for _ in range(5)]
+ dream_content = self._generate_dreams(fragments, noise)
+
+ # GAN training step (discriminator update)
+ d_loss, g_loss = self._gan_training_step(dream_content, fragments)
+ self._discriminator_confidence = min(0.95, self._discriminator_confidence + GAN_LR * (0.5 - d_loss))
+ self._generator_quality = min(0.95, self._generator_quality + GAN_LR * (0.5 - g_loss))
+
+ # Step 2: Emotional processing
+ high_affect = [
+ t for t in self.replay_buffer
+ if t.emotional_intensity > 0.6
+ ][:3]
+ emotional_episodes = [
+ f"[REM:affect={t.emotional_intensity:.2f}] {t.content[:60]}"
+ for t in high_affect
+ ]
+
+ # Step 3: Creative insight detection
+ insights = self._detect_creative_insights(dream_content, fragments)
+ self.insight_bank.extend(insights)
+
+ # Step 4: Bizarreness score
+ bizarreness = self._compute_bizarreness(dream_content)
+
+ return REMCycle(
+ dream_content=dream_content,
+ emotional_episodes=emotional_episodes,
+ creative_insights=insights,
+ discriminator_loss=round(d_loss, 4),
+ generator_loss=round(g_loss, 4),
+ bizarreness_score=round(bizarreness, 4),
+ )
+
+ def _run_tawil(self, dream_content: list[str]) -> TawilInterpretation:
+ """
+ Taʾwīl: Dream interpretation.
+
+ Symbolic decoding + integration with waking knowledge.
+
+ Classical symbols are mapped to functional meanings.
+ Insights are extracted from recurring dream patterns.
+ """
+ # Symbol extraction and mapping
+ symbol_lexicon = {
+ "water": "knowledge / emotion",
+ "fire": "transformation / energy",
+ "mountain": "challenge / stability",
+ "path": "decision / journey",
+ "door": "opportunity / threshold",
+ "light": "clarity / guidance",
+ "darkness": "uncertainty / hidden knowledge",
+ "book": "memory / wisdom",
+ "bird": "message / aspiration",
+ "tree": "growth / rootedness",
+ }
+
+ found_symbols = {}
+ all_content = " ".join(dream_content).lower()
+ for symbol, meaning in symbol_lexicon.items():
+ if symbol in all_content:
+ found_symbols[symbol] = meaning
+
+ # Waking relevance: connect to recent gists
+ if self.extracted_gists:
+ recent_gist = self.extracted_gists[-1]
+ waking_relevance = f"Dreams align with waking pattern: {recent_gist[:100]}"
+ else:
+ waking_relevance = "Dreams consolidating recent experiences"
+
+ # Extract actionable insights from dream patterns
+ insights = []
+ if found_symbols:
+ for symbol, meaning in list(found_symbols.items())[:3]:
+ insights.append(f"Symbol '{symbol}' suggests: {meaning}")
+ if self.extracted_gists:
+ insights.append(f"Pattern recognition: {self.extracted_gists[-1][:80]}")
+
+ confidence = min(0.9, 0.3 + 0.1 * len(found_symbols) + 0.2 * len(self.extracted_gists))
+
+ return TawilInterpretation(
+ dream_symbols=found_symbols,
+ waking_relevance=waking_relevance,
+ insights=insights,
+ confidence=round(confidence, 3),
+ )
+
+ def _extract_gist(self, traces: list[MemoryTrace]) -> list[str]:
+ """
+ Gist extraction: find common themes across top memory traces.
+ Information bottleneck: keep shared structure, discard details.
+ """
+ if not traces:
+ return []
+
+ # Find shared words across traces (common structure)
+ word_sets = [set(t.content.lower().split()[:15]) for t in traces]
+ if not word_sets:
+ return []
+
+ common = word_sets[0]
+ for ws in word_sets[1:]:
+ common = common & ws
+
+ # Filter meaningful words (length > 3)
+ meaningful = [w for w in common if len(w) > 3][:GIST_TOP_K]
+
+ if meaningful:
+ gist = f"Pattern: [{', '.join(meaningful)}] across {len(traces)} memories"
+ return [gist]
+
+ # Fallback: use highest priority memory's first sentence
+ top = traces[0]
+ first_sentence = top.content.split(".")[0][:80]
+ return [f"Gist: {first_sentence}"]
+
+ def _generate_dreams(
+ self, fragments: list[str], noise: list[float]
+ ) -> list[str]:
+ """
+ G(z, fragments): Generate dream scenarios from memory fragments + noise.
+
+ Dirichlet sparse mixing: randomly weight fragments
+ High-noise REM activation: inject novelty via noise vector
+ """
+ if not fragments:
+ return ["[Empty dream — no memory fragments]"]
+
+ dreams = []
+ for i, nz in enumerate(noise[:3]):
+ # Dirichlet-like mixing: random weights summing to 1
+ n = len(fragments)
+ if n == 0:
+ continue
+ raw_weights = [abs(random.gauss(0, 1)) for _ in range(n)]
+ total = sum(raw_weights) or 1.0
+ weights = [w / total for w in raw_weights]
+
+ # Blend fragments with weights
+ selected = fragments[i % n] if i < len(fragments) else fragments[0]
+ noise_tag = f"[noise:{nz:.2f}]"
+
+ # Bizarre recombination: splice parts of different fragments
+ if len(fragments) > 1 and abs(nz) > 0.2:
+ other = fragments[(i + 1) % len(fragments)]
+ dream = f"{selected[:30]}...{other[:30]}... {noise_tag}"
+ else:
+ dream = f"{selected[:60]} {noise_tag}"
+
+ dreams.append(f"[REM:dream] {dream}")
+
+ return dreams
+
+ def _gan_training_step(
+ self, fake_dreams: list[str], real_memories: list[str]
+ ) -> tuple[float, float]:
+ """
+ Simplified GAN update:
+ min_G max_D V(D,G)
+
+ D tries to distinguish real memories from generated dreams.
+ G tries to generate dreams that D cannot distinguish.
+
+ Returns: (discriminator_loss, generator_loss)
+ """
+ # Discriminator loss: higher = better discrimination
+ # In our approximation: D succeeds when real != fake
+ d_correct = 0
+ for real in real_memories[:3]:
+ for fake in fake_dreams[:3]:
+ if real[:20] != fake[:20]: # simplified: structurally different
+ d_correct += 1
+ max_pairs = 3 * 3
+ d_accuracy = d_correct / max_pairs if max_pairs > 0 else 0.5
+ d_loss = 1.0 - d_accuracy # lower loss = better discriminator
+
+ # Generator loss: lower = more convincing dreams
+ g_loss = 1.0 - (1.0 - d_accuracy) # adversarial: G wins when D fails
+ g_loss = max(0.0, g_loss - 0.1 * self._generator_quality)
+
+ return round(d_loss, 4), round(g_loss, 4)
+
+ def _detect_creative_insights(
+ self, dream_content: list[str], fragments: list[str]
+ ) -> list[str]:
+ """
+ Creative insight detection: find novel associations between
+ dream content and known memory fragments.
+
+ An insight occurs when distant concepts co-activate
+ during REM's low-inhibition state.
+ """
+ insights = []
+
+ for dream in dream_content[:3]:
+ dream_words = set(dream.lower().split())
+ for frag in fragments[:3]:
+ frag_words = set(frag.lower().split())
+ # Novel co-activation: small overlap (not trivially similar)
+ overlap = len(dream_words & frag_words)
+ union = len(dream_words | frag_words)
+ jaccard = overlap / union if union > 0 else 0
+ if 0.05 < jaccard < 0.3: # some but not full overlap → novel link
+ insight = (
+ f"Creative link: [{dream[:40]}] ↔ [{frag[:40]}] "
+ f"(Jaccard={jaccard:.2f})"
+ )
+ insights.append(insight)
+
+ return insights[:3]
+
+ def _compute_bizarreness(self, dream_content: list[str]) -> float:
+ """
+ Bizarreness = function of noise magnitude + semantic incoherence.
+ High bizarreness → high creative potential (REM property).
+ """
+ if not dream_content:
+ return 0.0
+
+ noise_scores = []
+ for dream in dream_content:
+ if "[noise:" in dream:
+ try:
+ start = dream.index("[noise:") + 7
+ end = dream.index("]", start)
+ noise_val = abs(float(dream[start:end]))
+ noise_scores.append(noise_val)
+ except (ValueError, IndexError):
+ noise_scores.append(0.2)
+ else:
+ noise_scores.append(0.1)
+
+ return min(1.0, sum(noise_scores) / max(len(noise_scores), 1) / REM_NOISE_SIGMA)
+
+ def consolidate_from_agent(
+ self,
+ task_history: list[dict],
+ tool_results: list[dict] | None = None,
+ ) -> int:
+ """
+ Convenience method: add agent session memories to replay buffer.
+ Returns number of traces added.
+ """
+ added = 0
+ for item in task_history:
+ content = item.get("content", "") or item.get("response", "")
+ if not content:
+ continue
+
+ # Estimate emotional intensity from content
+ positive_words = {"success", "solved", "found", "created", "excellent"}
+ negative_words = {"error", "failed", "failed", "wrong", "exception"}
+ words = set(content.lower().split())
+ pos = len(words & positive_words) / len(positive_words)
+ neg = len(words & negative_words) / len(negative_words)
+ emotional = abs(pos - neg)
+
+ self.add_memory(
+ content=content[:300],
+ emotional_intensity=emotional,
+ novelty=item.get("novelty", 0.4),
+ goal_relevance=item.get("goal_relevance", 0.5),
+ prediction_error=item.get("error_magnitude", 0.2),
+ )
+ added += 1
+
+ return added
+
+ def to_dict(self) -> dict:
+ return {
+ "replay_buffer_size": len(self.replay_buffer),
+ "synaptic_weights_count": len(self.synaptic_weights),
+ "extracted_gists": len(self.extracted_gists),
+ "insight_bank_size": len(self.insight_bank),
+ "session_count": self._session_count,
+ "current_phase": self._current_phase.value,
+ "discriminator_confidence": round(self._discriminator_confidence, 3),
+ "generator_quality": round(self._generator_quality, 3),
+ }
diff --git a/backend/core/fuad.py b/backend/core/fuad.py
new file mode 100644
index 0000000..3642b6e
--- /dev/null
+++ b/backend/core/fuad.py
@@ -0,0 +1,240 @@
+"""
+Fu'ad Engine (فؤاد) — Conviction Formation
+==========================================
+
+"And He gave you hearing and sight and hearts (af'idah — plural of fu'ad).
+ Little are you grateful." — Quran 16:78
+
+Fu'ad is the integrating heart — it forms *conviction* from accumulated evidence.
+Unlike simple belief, conviction requires multiple independent sources and
+temporal consistency before committing.
+
+Three conviction levels (mapped to Yaqin):
+ IMPRESSION → Ilm al-Yaqin (knowledge by inference — single source)
+ BELIEF → Ayn al-Yaqin (knowledge by observation — 2+ sources)
+ CONVICTION → Haqq al-Yaqin (knowledge by experience — 3+ consistent sources)
+"""
+
+import hashlib
+import logging
+import time
+from dataclasses import dataclass, field
+from enum import Enum
+
+logger = logging.getLogger("mizan.fuad")
+
+
+class ConvictionLevel(Enum):
+ IMPRESSION = "impression" # Single source — unverified
+ BELIEF = "belief" # 2+ independent sources
+ CONVICTION = "conviction" # 3+ sources + temporal consistency
+
+
+@dataclass
+class ConvictionAssessment:
+ """Result of evidence evaluation."""
+ claim: str
+ level: ConvictionLevel
+ confidence: float # 0.0 – 1.0
+ source_count: int
+ supporting: list[str] = field(default_factory=list)
+ contradicting: list[str] = field(default_factory=list)
+ first_seen: float = field(default_factory=time.time)
+ last_updated: float = field(default_factory=time.time)
+
+ def to_dict(self) -> dict:
+ return {
+ "claim": self.claim[:200],
+ "level": self.level.value,
+ "confidence": round(self.confidence, 3),
+ "source_count": self.source_count,
+ "supporting_count": len(self.supporting),
+ "contradicting_count": len(self.contradicting),
+ "temporal_span_hours": round(
+ (self.last_updated - self.first_seen) / 3600, 2
+ ),
+ }
+
+
+def _claim_hash(claim: str) -> str:
+ """Stable ID for a claim (first 12 chars of sha256)."""
+ return hashlib.sha256(claim.lower().strip().encode()).hexdigest()[:12]
+
+
+def _are_independent(src_a: str, src_b: str) -> bool:
+ """
+ Heuristic: two sources are independent if they differ significantly.
+ Same URL domain or identical string → not independent.
+ """
+ if src_a == src_b:
+ return False
+ # If both look like URLs, compare domain
+ def _domain(s: str) -> str:
+ try:
+ return s.split("//")[-1].split("/")[0].lower()
+ except Exception:
+ return s[:20].lower()
+
+ if src_a.startswith("http") and src_b.startswith("http"):
+ return _domain(src_a) != _domain(src_b)
+ # For non-URL sources, require at least 10-char difference
+ return src_a[:10].lower() != src_b[:10].lower()
+
+
+class FuadEngine:
+ """
+ Bayesian conviction formation from accumulated evidence.
+
+ Usage:
+ fuad = FuadEngine()
+ assessment = fuad.evaluate_evidence(
+ "Python is widely used for AI",
+ ["tool:bash:result1", "tool:http_get:result2"]
+ )
+ # IMPRESSION if 1 source, BELIEF if 2+, CONVICTION if 3+ + time
+ """
+
+ # Minimum independent sources needed per level
+ _MIN_SOURCES = {
+ ConvictionLevel.BELIEF: 2,
+ ConvictionLevel.CONVICTION: 3,
+ }
+ # Minimum hours between first and last sighting for CONVICTION
+ _CONVICTION_MIN_HOURS = 0.1 # 6 minutes — practical for session use
+
+ def __init__(self):
+ # claim_hash → ConvictionAssessment
+ self._assessments: dict[str, ConvictionAssessment] = {}
+
+ def evaluate_evidence(
+ self,
+ claim: str,
+ sources: list[str],
+ contradicting_sources: list[str] = None,
+ ) -> ConvictionAssessment:
+ """
+ Evaluate or update conviction for a claim.
+
+ Algorithm:
+ 1. Count independent sources (different domains / names)
+ 2. Apply Bayesian prior: P(claim) starts at 0.5, updates per source
+ 3. Each independent supporting source × 1.5, contradicting × 0.6
+ 4. Classify: 1 source→IMPRESSION, 2+→BELIEF, 3++time→CONVICTION
+ """
+ key = _claim_hash(claim)
+ contradicting_sources = contradicting_sources or []
+ now = time.time()
+
+ if key in self._assessments:
+ existing = self._assessments[key]
+ # Merge new sources
+ all_supporting = list(set(existing.supporting + sources))
+ all_contradicting = list(set(existing.contradicting + contradicting_sources))
+ existing.supporting = all_supporting
+ existing.contradicting = all_contradicting
+ existing.last_updated = now
+ assessment = existing
+ else:
+ assessment = ConvictionAssessment(
+ claim=claim,
+ level=ConvictionLevel.IMPRESSION,
+ confidence=0.5,
+ source_count=0,
+ supporting=list(sources),
+ contradicting=list(contradicting_sources),
+ first_seen=now,
+ last_updated=now,
+ )
+ self._assessments[key] = assessment
+
+ # Count independent supporting sources
+ independent_count = 0
+ seen_sources = []
+ for src in assessment.supporting:
+ if all(_are_independent(src, prev) for prev in seen_sources):
+ independent_count += 1
+ seen_sources.append(src)
+
+ assessment.source_count = independent_count
+
+ # Bayesian confidence update
+ confidence = 0.5
+ for _ in range(independent_count):
+ confidence = confidence + (1.0 - confidence) * 0.35 # Each source +35% of gap
+ for _ in range(len(assessment.contradicting)):
+ confidence = confidence * 0.70 # Each contradiction reduces by 30%
+ confidence = max(0.05, min(0.97, confidence))
+ assessment.confidence = confidence
+
+ # Classify level
+ temporal_span_hours = (assessment.last_updated - assessment.first_seen) / 3600
+ if (
+ independent_count >= self._MIN_SOURCES[ConvictionLevel.CONVICTION]
+ and temporal_span_hours >= self._CONVICTION_MIN_HOURS
+ and len(assessment.contradicting) == 0
+ ):
+ assessment.level = ConvictionLevel.CONVICTION
+ elif independent_count >= self._MIN_SOURCES[ConvictionLevel.BELIEF]:
+ assessment.level = ConvictionLevel.BELIEF
+ else:
+ assessment.level = ConvictionLevel.IMPRESSION
+
+ logger.debug(
+ "[FUAD] claim='%s...' level=%s conf=%.2f sources=%d",
+ claim[:60], assessment.level.value, confidence, independent_count,
+ )
+ return assessment
+
+ def compute_confidence(
+ self,
+ tool_count: int = 0,
+ tool_results: list[dict] | None = None,
+ ) -> float:
+ """
+ Compute overall confidence from tool evidence using Bayesian update.
+
+ Each successful tool result acts as an independent evidence source.
+ Replaces the hardcoded `0.5 + 0.1 * tool_count` formula.
+
+ P(confident) = 0.5, then each evidence source closes 35% of the gap.
+ Failed tool results penalize by 30%.
+ """
+ tool_results = tool_results or []
+ confidence = 0.5
+
+ # Count successful vs failed results
+ successes = 0
+ failures = 0
+ for result in tool_results:
+ content = str(result.get("content", ""))
+ if '"error"' in content or content.startswith('{"error'):
+ failures += 1
+ else:
+ successes += 1
+
+ # If no parsed results, use tool_count as proxy for successes
+ if not tool_results and tool_count > 0:
+ successes = tool_count
+
+ # Bayesian update: each success closes 35% of gap to 1.0
+ for _ in range(successes):
+ confidence = confidence + (1.0 - confidence) * 0.35
+
+ # Each failure reduces by 30%
+ for _ in range(failures):
+ confidence = confidence * 0.70
+
+ return max(0.05, min(0.95, round(confidence, 4)))
+
+ def get_assessment(self, claim: str) -> ConvictionAssessment | None:
+ """Retrieve existing assessment without updating."""
+ return self._assessments.get(_claim_hash(claim))
+
+ def stats(self) -> dict:
+ levels = {level.value: 0 for level in ConvictionLevel}
+ for a in self._assessments.values():
+ levels[a.level.value] += 1
+ return {
+ "total_claims": len(self._assessments),
+ "by_level": levels,
+ }
diff --git a/backend/core/imagination.py b/backend/core/imagination.py
new file mode 100644
index 0000000..75648dc
--- /dev/null
+++ b/backend/core/imagination.py
@@ -0,0 +1,444 @@
+"""
+Taṣwīr — Imagination Engine
+=============================
+
+"He is Allah, the Creator, the Originator, the Fashioner (al-Muṣawwir)" — Quran 59:24
+
+Implements Algorithm 5: TASWIR_IMAGINATION_ENGINE
+
+Mental simulation with predictive coding and hierarchical scene generation.
+
+Predictive coding update rule:
+ μ_l(t+1) = μ_l(t) + κ_l[ε_l(t) - ε_{l+1}(t)]
+
+ where:
+ - μ_l = mean prediction at layer l
+ - ε_l = prediction error at layer l
+ - κ_l = learning rate at layer l
+ - ε_{l+1} = top-down prediction from layer above
+
+Counterfactual reasoning follows Pearl's 3-step procedure:
+ 1. Abduction: infer latent state from observations
+ 2. Action: apply counterfactual intervention
+ 3. Prediction: propagate through causal model
+"""
+
+import logging
+import math
+import random
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
+
+logger = logging.getLogger("mizan.imagination")
+
+# Predictive coding parameters
+KAPPA = [0.3, 0.2, 0.1, 0.05] # learning rates per layer (coarse → fine)
+N_LAYERS = 4
+
+# Emotional tagging thresholds
+EMOTIONAL_VALENCE_THRESHOLD = 0.3
+
+# Scene generation resolution levels
+RESOLUTION_COARSE = "coarse"
+RESOLUTION_MEDIUM = "medium"
+RESOLUTION_FINE = "fine"
+
+
+class ImaginationMode(Enum):
+ PROSPECTIVE = "prospective" # Forward simulation: what will happen?
+ RETROSPECTIVE = "retrospective" # Backward: what led to this?
+ COUNTERFACTUAL = "counterfactual" # What if? (Pearl Rung 3)
+ CREATIVE = "creative" # Novel scene construction
+
+
+@dataclass
+class SceneFragment:
+ """A memory fragment used to construct imagined scenes."""
+ content: str
+ source: str # "episodic", "semantic", "sensory"
+ salience: float
+ emotional_tag: float # -1.0 (negative) to +1.0 (positive)
+
+
+@dataclass
+class HierarchicalScene:
+ """Multi-resolution mental scene."""
+ coarse: str # High-level gist: "a meeting goes wrong"
+ medium: str # Mid-level: actors, setting, actions
+ fine: str # Fine-grained: specific details, dialogue
+ emotional_valence: float
+ confidence: float
+ prediction_errors: list[float] # ε per layer
+
+
+@dataclass
+class MentalSimulation:
+ """Result of running a mental simulation forward."""
+ scenario: str
+ steps: list[str]
+ predicted_outcome: str
+ emotional_trajectory: list[float] # valence at each step
+ confidence: float
+ surprisal: float # -log P(outcome): how unexpected?
+
+
+@dataclass
+class CounterfactualResult:
+ """Pearl Rung 3 counterfactual analysis."""
+ original_observation: str
+ intervention: str
+ counterfactual_world: str
+ probability_shift: float # ΔP(outcome) under intervention
+ abduced_state: dict # latent variables inferred
+
+
+@dataclass
+class ImaginationResult:
+ mode: ImaginationMode
+ scene: HierarchicalScene
+ simulation: MentalSimulation | None
+ counterfactual: CounterfactualResult | None
+ predictive_states: list[float] # μ_l per layer
+ total_surprise: float
+
+
+class TaswirImaginationEngine:
+ """
+ Algorithm 5: TASWIR_IMAGINATION_ENGINE
+
+ Constructs imagined scenes from memory fragments, runs forward
+ mental simulations, and performs counterfactual reasoning.
+
+ Architecture:
+ - Hippocampal recombination: assembles fragments from memory
+ - Hierarchical predictive coding: coarse → medium → fine rendering
+ - Qalb emotional tagging: attaches valence to each simulation step
+ - Counterfactual module: abduction → action → prediction (Pearl 3)
+ """
+
+ def __init__(self):
+ # Predictive coding state: mean predictions per layer
+ self.mu: list[float] = [0.0] * N_LAYERS
+ self.memory_fragments: list[SceneFragment] = []
+ self._simulation_count = 0
+
+ def imagine(
+ self,
+ prompt: str,
+ mode: ImaginationMode = ImaginationMode.PROSPECTIVE,
+ memory_fragments: list[dict] | None = None,
+ counterfactual_intervention: str | None = None,
+ ) -> ImaginationResult:
+ """
+ Main entry point for imagination.
+
+ Steps:
+ 1. Hippocampal recombination (assemble scene fragments)
+ 2. Hierarchical scene generation (coarse → fine)
+ 3. Predictive coding update
+ 4. Forward mental simulation
+ 5. Emotional tagging via Qalb
+ 6. Counterfactual reasoning (if mode == COUNTERFACTUAL)
+ """
+ self._simulation_count += 1
+
+ # Step 1: Recombine memory fragments
+ fragments = self._recombine_fragments(prompt, memory_fragments or [])
+
+ # Step 2: Hierarchical scene generation
+ scene = self._generate_hierarchical_scene(prompt, fragments)
+
+ # Step 3: Predictive coding update
+ prediction_errors = self._compute_prediction_errors(scene)
+ self._update_predictions(prediction_errors)
+
+ # Step 4: Forward mental simulation
+ simulation = None
+ if mode in (ImaginationMode.PROSPECTIVE, ImaginationMode.CREATIVE):
+ simulation = self._run_forward_simulation(prompt, scene)
+
+ # Step 5: Counterfactual
+ counterfactual = None
+ if mode == ImaginationMode.COUNTERFACTUAL and counterfactual_intervention:
+ counterfactual = self._counterfactual_reasoning(
+ prompt, counterfactual_intervention, scene
+ )
+
+ total_surprise = sum(abs(e) for e in prediction_errors)
+
+ logger.debug(
+ "[TASWIR] mode=%s fragments=%d surprise=%.3f",
+ mode.value, len(fragments), total_surprise,
+ )
+
+ return ImaginationResult(
+ mode=mode,
+ scene=scene,
+ simulation=simulation,
+ counterfactual=counterfactual,
+ predictive_states=list(self.mu),
+ total_surprise=round(total_surprise, 4),
+ )
+
+ def _recombine_fragments(
+ self, prompt: str, raw_fragments: list[dict]
+ ) -> list[SceneFragment]:
+ """
+ Hippocampal recombination: select and blend memory fragments
+ relevant to the prompt.
+
+ Salience = cosine_sim(fragment, prompt) × emotional_weight
+ (Approximated by keyword overlap here.)
+ """
+ prompt_words = set(prompt.lower().split())
+ fragments = []
+
+ for raw in raw_fragments:
+ content = raw.get("content", "")
+ content_words = set(content.lower().split())
+ overlap = len(prompt_words & content_words) / max(len(prompt_words), 1)
+ emotional_tag = raw.get("emotional_tag", 0.0)
+ salience = overlap * 0.7 + abs(emotional_tag) * 0.3
+
+ fragments.append(SceneFragment(
+ content=content,
+ source=raw.get("source", "semantic"),
+ salience=salience,
+ emotional_tag=emotional_tag,
+ ))
+
+ # Sort by salience, keep top fragments
+ fragments.sort(key=lambda f: f.salience, reverse=True)
+ top = fragments[:5]
+
+ # Add synthetic fragment from prompt itself
+ top.append(SceneFragment(
+ content=prompt,
+ source="episodic",
+ salience=1.0,
+ emotional_tag=self._estimate_valence(prompt),
+ ))
+
+ return top
+
+ def _generate_hierarchical_scene(
+ self, prompt: str, fragments: list[SceneFragment]
+ ) -> HierarchicalScene:
+ """
+ Build 3-resolution mental scene via hierarchical predictive coding.
+
+ Coarse level: high-level gist (layer 4 — most abstract)
+ Medium level: actors and actions (layer 2-3)
+ Fine level: specific details (layer 1 — most concrete)
+ """
+ # Coarse: extract gist from dominant fragment
+ dominant = max(fragments, key=lambda f: f.salience) if fragments else None
+ coarse = self._extract_gist(prompt, dominant)
+
+ # Medium: identify actors and setting
+ medium = self._extract_actors_and_setting(prompt, fragments)
+
+ # Fine: construct specific details
+ fine = self._construct_fine_details(prompt, fragments)
+
+ # Emotional valence: weighted average across fragments
+ if fragments:
+ total_salience = sum(f.salience for f in fragments)
+ emotional_valence = sum(
+ f.emotional_tag * f.salience for f in fragments
+ ) / max(total_salience, 0.01)
+ else:
+ emotional_valence = 0.0
+
+ # Confidence: based on fragment coverage
+ coverage = len(fragments) / max(1, len(fragments) + 2)
+ confidence = 0.3 + 0.7 * coverage
+
+ return HierarchicalScene(
+ coarse=coarse,
+ medium=medium,
+ fine=fine,
+ emotional_valence=round(emotional_valence, 3),
+ confidence=round(confidence, 3),
+ prediction_errors=[],
+ )
+
+ def _compute_prediction_errors(
+ self, scene: HierarchicalScene
+ ) -> list[float]:
+ """
+ Prediction error at each layer:
+ ε_l(t) = observation_l - μ_l(t)
+
+ Layers: [fine, medium, coarse, abstract]
+ """
+ # Map scene to activation signals (0-1)
+ observations = [
+ self._text_to_activation(scene.fine),
+ self._text_to_activation(scene.medium),
+ self._text_to_activation(scene.coarse),
+ scene.confidence, # abstract layer = overall confidence
+ ]
+ errors = [obs - mu for obs, mu in zip(observations, self.mu)]
+ scene.prediction_errors = errors
+ return errors
+
+ def _update_predictions(self, errors: list[float]) -> None:
+ """
+ μ_l(t+1) = μ_l(t) + κ_l[ε_l(t) - ε_{l+1}(t)]
+
+ Top-down correction: each layer's prediction is adjusted
+ by the difference between its own error and the layer above.
+ """
+ for l in range(N_LAYERS):
+ own_error = errors[l]
+ upper_error = errors[l + 1] if l + 1 < N_LAYERS else 0.0
+ self.mu[l] = max(0.0, min(1.0,
+ self.mu[l] + KAPPA[l] * (own_error - upper_error)
+ ))
+
+ def _run_forward_simulation(
+ self, prompt: str, scene: HierarchicalScene
+ ) -> MentalSimulation:
+ """
+ Run mental simulation forward in time from the imagined scene.
+
+ Uses scene as initial state; generates causal step sequence.
+ """
+ steps = []
+ emotional_trajectory = [scene.emotional_valence]
+ current_valence = scene.emotional_valence
+
+ # Generate up to 4 simulation steps
+ sim_templates = [
+ f"Initial state: {scene.coarse}",
+ f"Development: {scene.medium}",
+ f"Key action: {self._extract_action(prompt)}",
+ f"Consequence: outcome follows from action",
+ ]
+ for template in sim_templates:
+ steps.append(template)
+ # Emotional drift: move 10% toward neutral per step
+ current_valence = current_valence * 0.9
+ emotional_trajectory.append(round(current_valence, 3))
+
+ predicted_outcome = (
+ f"Outcome {'positive' if scene.emotional_valence > 0 else 'challenging'}: "
+ f"{scene.fine[:100]}"
+ )
+
+ # Surprisal: -log P(outcome) approximated by (1 - confidence)
+ surprisal = -math.log(max(scene.confidence, 0.01))
+
+ return MentalSimulation(
+ scenario=prompt[:200],
+ steps=steps,
+ predicted_outcome=predicted_outcome,
+ emotional_trajectory=emotional_trajectory,
+ confidence=scene.confidence,
+ surprisal=round(surprisal, 4),
+ )
+
+ def _counterfactual_reasoning(
+ self,
+ observation: str,
+ intervention: str,
+ scene: HierarchicalScene,
+ ) -> CounterfactualResult:
+ """
+ Pearl's 3-step counterfactual procedure:
+ 1. Abduction: infer latent state U from (observation, scene)
+ 2. Action: apply intervention (modify causal model)
+ 3. Prediction: propagate modified model → counterfactual world
+
+ ΔP = P(Y|do(X=x')) - P(Y|X=x)
+ """
+ # Step 1: Abduction — infer latent state
+ abduced_state = {
+ "world_model": scene.coarse,
+ "confidence": scene.confidence,
+ "emotional_context": scene.emotional_valence,
+ "inferred_causes": self._infer_causes(observation),
+ }
+
+ # Step 2: Action — apply counterfactual intervention
+ # Estimate how much the intervention changes the causal structure
+ intervention_strength = min(1.0, len(intervention.split()) / 20.0)
+ probability_shift = intervention_strength * (1.0 - scene.confidence)
+
+ # Step 3: Prediction — generate counterfactual world
+ original_valence = scene.emotional_valence
+ counterfactual_valence = original_valence + probability_shift * (
+ 1.0 - abs(original_valence)
+ )
+ counterfactual_world = (
+ f"Under intervention '{intervention[:80]}': "
+ f"The world differs from '{scene.coarse[:80]}' with "
+ f"P(outcome) shift of {probability_shift:.2%}. "
+ f"New emotional trajectory: {counterfactual_valence:.3f}"
+ )
+
+ return CounterfactualResult(
+ original_observation=observation[:200],
+ intervention=intervention[:200],
+ counterfactual_world=counterfactual_world,
+ probability_shift=round(probability_shift, 4),
+ abduced_state=abduced_state,
+ )
+
+ def _extract_gist(self, prompt: str, dominant: SceneFragment | None) -> str:
+ words = prompt.split()[:6]
+ gist_seed = " ".join(words)
+ if dominant:
+ return f"{gist_seed} → {dominant.content[:60]}"
+ return f"Imagined scenario: {gist_seed}"
+
+ def _extract_actors_and_setting(
+ self, prompt: str, fragments: list[SceneFragment]
+ ) -> str:
+ context = " | ".join(f.content[:40] for f in fragments[:2])
+ return f"Setting: {prompt[:60]} | Context: {context}"
+
+ def _construct_fine_details(
+ self, prompt: str, fragments: list[SceneFragment]
+ ) -> str:
+ details = " ".join(f.content[:30] for f in fragments[:3])
+ return f"Details: {details[:200]}"
+
+ def _extract_action(self, prompt: str) -> str:
+ action_verbs = ["create", "delete", "build", "fix", "analyze", "send", "run"]
+ for verb in action_verbs:
+ if verb in prompt.lower():
+ return f"{verb} [primary action]"
+ return "proceed with task"
+
+ def _infer_causes(self, observation: str) -> list[str]:
+ """Simplified abduction: extract likely causal factors from observation."""
+ words = observation.split()
+ return [w for w in words if len(w) > 5][:3]
+
+ @staticmethod
+ def _estimate_valence(text: str) -> float:
+ positive = {"help", "good", "create", "success", "improve", "benefit"}
+ negative = {"error", "fail", "harm", "delete", "wrong", "danger"}
+ words = set(text.lower().split())
+ pos = len(words & positive)
+ neg = len(words & negative)
+ total = pos + neg
+ if total == 0:
+ return 0.0
+ return (pos - neg) / total
+
+ @staticmethod
+ def _text_to_activation(text: str) -> float:
+ """Map text richness to a 0-1 activation signal."""
+ return min(1.0, len(text.split()) / 30.0)
+
+ def to_dict(self) -> dict:
+ return {
+ "predictive_states": [round(m, 4) for m in self.mu],
+ "fragment_count": len(self.memory_fragments),
+ "simulation_count": self._simulation_count,
+ "n_layers": N_LAYERS,
+ }
diff --git a/backend/core/lubb.py b/backend/core/lubb.py
new file mode 100644
index 0000000..0492de3
--- /dev/null
+++ b/backend/core/lubb.py
@@ -0,0 +1,351 @@
+"""
+Lubb Engine (لُبّ) — Metacognition
+====================================
+
+"He gives wisdom (hikmah) to whom He wills, and whoever has been given wisdom
+ has certainly been given much good. And none will remember (yaddakkar) except
+ those of understanding (ulu al-albab — those with lubb)." — Quran 2:269
+
+Lubb (لُبّ) = "the kernel / pith / essence" — the deepest cognitive layer.
+It monitors the quality of all other layers and governs the entire reasoning process.
+
+Three metacognitive functions:
+ 1. Compress — Information Bottleneck: extract minimal sufficient reasoning trace
+ 2. Coherence — Verify that conclusions follow logically from premises
+ 3. Bias — Detect common reasoning biases (confirmation, anchoring, availability)
+"""
+
+import logging
+import re
+from dataclasses import dataclass, field
+from enum import Enum
+
+logger = logging.getLogger("mizan.lubb")
+
+
+class QualityLabel(Enum):
+ CONFIDENT = "confident" # High coherence, low bias
+ HEDGED = "hedged" # Moderate quality — recommend caveats
+ UNCERTAIN = "uncertain" # Low coherence or strong bias detected
+
+
+@dataclass
+class BiasFlag:
+ """A detected reasoning bias."""
+ bias_type: str
+ description: str
+ severity: str # "low" | "medium" | "high"
+ evidence: str # Quote/pattern that triggered detection
+
+
+@dataclass
+class CoherenceReport:
+ """Result of reasoning chain coherence check."""
+ score: float # 0.0 = incoherent, 1.0 = perfectly coherent
+ contradictions: list[str] = field(default_factory=list)
+ unsupported_claims: list[str] = field(default_factory=list)
+ summary: str = ""
+
+
+@dataclass
+class MetaReport:
+ """Full metacognitive evaluation of a reasoning trace."""
+ quality: QualityLabel
+ compressed_trace: str
+ coherence: CoherenceReport
+ bias_flags: list[BiasFlag] = field(default_factory=list)
+ overall_confidence: float = 0.5
+ caveat: str = "" # Appended to response if quality is poor
+
+ def to_dict(self) -> dict:
+ return {
+ "quality": self.quality.value,
+ "coherence_score": round(self.coherence.score, 3),
+ "bias_count": len(self.bias_flags),
+ "bias_types": [b.bias_type for b in self.bias_flags],
+ "contradictions": self.coherence.contradictions[:3],
+ "overall_confidence": round(self.overall_confidence, 3),
+ "caveat": self.caveat,
+ }
+
+
+# Bias detection patterns (keyword-based heuristics)
+_BIAS_PATTERNS = [
+ {
+ "type": "confirmation_bias",
+ "signals": ["as expected", "confirms that", "proves that", "as i thought",
+ "just as predicted", "this confirms"],
+ "description": "Seeking only evidence that supports prior beliefs",
+ "severity": "medium",
+ },
+ {
+ "type": "anchoring",
+ "signals": ["first", "initially", "originally said", "started with",
+ "the initial value", "as mentioned first"],
+ "description": "Over-relying on the first piece of information encountered",
+ "severity": "medium",
+ },
+ {
+ "type": "availability_bias",
+ "signals": ["recently", "just saw", "just read", "just mentioned",
+ "as we just discussed", "the latest"],
+ "description": "Over-weighting recent or easily recalled information",
+ "severity": "low",
+ },
+ {
+ "type": "overconfidence",
+ "signals": ["definitely", "certainly", "100%", "impossible that",
+ "guaranteed", "absolutely sure", "no doubt"],
+ "description": "Claiming certainty beyond what evidence supports",
+ "severity": "high",
+ },
+ {
+ "type": "false_dichotomy",
+ "signals": ["either", "only two options", "must be one or the other",
+ "no other way", "the only possibility"],
+ "description": "Presenting a limited set of options as exhaustive",
+ "severity": "medium",
+ },
+]
+
+# Contradiction signal pairs (if both appear, flag potential contradiction)
+_CONTRADICTION_PAIRS = [
+ ("always", "never"),
+ ("impossible", "possible"),
+ ("success", "failure"),
+ ("increase", "decrease"),
+ ("enabled", "disabled"),
+ ("true", "false"),
+]
+
+
+class LubbEngine:
+ """
+ Metacognitive monitor for the MIZAN reasoning system.
+
+ Usage:
+ lubb = LubbEngine()
+ report = lubb.meta_evaluate(task, response, reasoning_steps=[...])
+ if report.coherence.score < 0.5:
+ response += f"\n\n[Note: {report.caveat}]"
+ """
+
+ # Target compression ratio (keep this fraction of original)
+ COMPRESSION_TARGET = 0.20
+ # Minimum coherence score to avoid UNCERTAIN label
+ COHERENCE_THRESHOLD = 0.5
+
+ def compress(self, trace: str) -> str:
+ """
+ Information Bottleneck compression of a reasoning trace.
+
+ Keeps: decisions (→, therefore, conclude), tool results ([Tool:...]),
+ key facts (numbers, proper nouns, file paths).
+ Discards: filler text, repeated context, politeness phrases.
+ """
+ if not trace:
+ return ""
+
+ lines = trace.split("\n")
+ kept = []
+
+ _high_value_patterns = [
+ r"\[Tool:", # Tool call results
+ r"\btherefore\b", # Logical conclusions
+ r"\bconclud", # Conclusions
+ r"\bfound\b", # Discovery
+ r"\berror\b", # Errors
+ r"\bresult:", # Results
+ r"\bans(wer)?:", # Answers
+ r"\d{2,}", # Numbers (stats, line numbers, etc.)
+ r"https?://", # URLs
+ r"\.py|\.js|\.ts", # File references
+ r"→|=>|:-", # Flow indicators
+ ]
+
+ _low_value_patterns = [
+ r"^(sure|okay|of course|certainly|great|let me|i will|i'll)",
+ r"^(as you can see|as mentioned|as discussed)",
+ r"^(in conclusion|in summary|to summarize)", # Keep content, not preambles
+ ]
+
+ for line in lines:
+ line_stripped = line.strip()
+ if not line_stripped or len(line_stripped) < 10:
+ continue
+
+ lower = line_stripped.lower()
+
+ # Skip low-value preamble lines
+ if any(re.match(p, lower) for p in _low_value_patterns):
+ continue
+
+ # Keep high-value lines
+ if any(re.search(p, line_stripped, re.IGNORECASE) for p in _high_value_patterns):
+ kept.append(line_stripped)
+ continue
+
+ # Keep lines that are "dense" (short, info-packed)
+ if 15 <= len(line_stripped) <= 200:
+ word_count = len(line_stripped.split())
+ if word_count >= 3:
+ kept.append(line_stripped)
+
+ compressed = "\n".join(kept)
+
+ # If still too long, truncate to target ratio
+ target_len = max(200, int(len(trace) * self.COMPRESSION_TARGET))
+ if len(compressed) > target_len:
+ compressed = compressed[:target_len] + "..."
+
+ return compressed
+
+ def check_coherence(self, steps: list, response: str = "") -> CoherenceReport:
+ """
+ Verify reasoning chain consistency.
+
+ Checks:
+ 1. Contradictions: opposite claims in the same trace
+ 2. Unsupported final claims: conclusions not backed by tool results
+ 3. Tool result consistency: no conflicting results
+ """
+ combined = response + " ".join(str(s) for s in steps)
+ lower = combined.lower()
+
+ contradictions = []
+ for a, b in _CONTRADICTION_PAIRS:
+ if a in lower and b in lower:
+ context_a = self._find_context(lower, a)
+ context_b = self._find_context(lower, b)
+ if context_a != context_b:
+ contradictions.append(f"'{a}' vs '{b}' appear in different contexts")
+
+ # Check if final response references tool results
+ tool_count = lower.count("[tool:")
+ unsupported = []
+ if tool_count == 0 and len(response) > 200:
+ # Long response with no tool evidence — flag potential unsupported claims
+ certainty_claims = re.findall(
+ r"\b(definitiv|certainly|absolutely|always|never)\w*", lower
+ )
+ if certainty_claims:
+ unsupported.append(
+ f"Strong certainty claims ({certainty_claims[:3]}) without tool evidence"
+ )
+
+ # Score: start at 1.0, deduct per issue
+ score = 1.0
+ score -= min(0.4, len(contradictions) * 0.2)
+ score -= min(0.3, len(unsupported) * 0.15)
+ score = max(0.0, score)
+
+ summary = (
+ f"Coherence: {score:.0%}. "
+ f"{len(contradictions)} contradiction(s), {len(unsupported)} unsupported claim(s)."
+ )
+
+ return CoherenceReport(
+ score=round(score, 3),
+ contradictions=contradictions[:5],
+ unsupported_claims=unsupported[:3],
+ summary=summary,
+ )
+
+ def detect_bias(self, trace: str) -> list[BiasFlag]:
+ """
+ Detect common cognitive biases in reasoning text.
+ Returns list of BiasFlag instances (empty = no bias detected).
+ """
+ flags = []
+ lower = trace.lower()
+
+ for pattern_def in _BIAS_PATTERNS:
+ matched_signals = [s for s in pattern_def["signals"] if s in lower]
+ if matched_signals:
+ evidence = f"Signals: {matched_signals[:3]}"
+ flags.append(BiasFlag(
+ bias_type=pattern_def["type"],
+ description=pattern_def["description"],
+ severity=pattern_def["severity"],
+ evidence=evidence,
+ ))
+
+ return flags
+
+ def meta_evaluate(
+ self,
+ task: str,
+ result: str,
+ steps: list = None,
+ ) -> MetaReport:
+ """
+ Full metacognitive evaluation of a completed reasoning trace.
+
+ 1. Compress the full trace
+ 2. Check coherence
+ 3. Detect biases
+ 4. Assign quality label
+ 5. Generate caveat if needed
+ """
+ steps = steps or []
+ full_trace = task + "\n" + result + "\n" + "\n".join(str(s) for s in steps)
+
+ compressed = self.compress(full_trace)
+ coherence = self.check_coherence(steps, result)
+ bias_flags = self.detect_bias(full_trace)
+
+ # Compute overall confidence
+ confidence = coherence.score
+ high_severity_biases = sum(1 for b in bias_flags if b.severity == "high")
+ medium_biases = sum(1 for b in bias_flags if b.severity == "medium")
+ confidence -= high_severity_biases * 0.15
+ confidence -= medium_biases * 0.05
+ confidence = max(0.1, min(0.95, confidence))
+
+ # Assign quality label
+ if confidence >= 0.7 and not high_severity_biases:
+ quality = QualityLabel.CONFIDENT
+ caveat = ""
+ elif confidence >= 0.45:
+ quality = QualityLabel.HEDGED
+ caveat = (
+ "This response has moderate confidence. "
+ "Please verify key claims independently."
+ )
+ else:
+ quality = QualityLabel.UNCERTAIN
+ issues = []
+ if coherence.contradictions:
+ issues.append("contains contradictions")
+ if high_severity_biases:
+ issues.append("shows overconfidence bias")
+ issues_str = " and ".join(issues) if issues else "has low coherence"
+ caveat = (
+ f"[Lubb warning: This reasoning {issues_str}. "
+ f"Treat conclusions with caution.]"
+ )
+
+ report = MetaReport(
+ quality=quality,
+ compressed_trace=compressed,
+ coherence=coherence,
+ bias_flags=bias_flags,
+ overall_confidence=round(confidence, 3),
+ caveat=caveat,
+ )
+
+ logger.debug(
+ "[LUBB] task='%s...' quality=%s coherence=%.2f biases=%d",
+ task[:50], quality.value, coherence.score, len(bias_flags),
+ )
+ return report
+
+ @staticmethod
+ def _find_context(text: str, word: str, window: int = 30) -> str:
+ """Find word in text and return surrounding context."""
+ idx = text.find(word)
+ if idx < 0:
+ return ""
+ start = max(0, idx - window)
+ end = min(len(text), idx + len(word) + window)
+ return text[start:end]
diff --git a/backend/core/nafs_triad.py b/backend/core/nafs_triad.py
new file mode 100644
index 0000000..a68fcf7
--- /dev/null
+++ b/backend/core/nafs_triad.py
@@ -0,0 +1,170 @@
+"""
+Nafs Triad (النفس الثلاثية) — Multi-Agent Consciousness
+=========================================================
+
+"And [by] the soul (nafs) and He who proportioned it,
+ and inspired it with its wickedness and its righteousness." — Quran 91:7-8
+
+Three competing inner voices deliberate on every significant task.
+The dominant voice shapes the agent's behavioral approach for that turn.
+
+Nafs levels 1-2: Ammara dominates (raw drive — act fast)
+Nafs levels 3-4: Lawwama rises (self-correction — check work)
+Nafs levels 5-7: Mutmainna leads (integrated balance — wisdom)
+"""
+
+import math
+import logging
+from dataclasses import dataclass
+
+logger = logging.getLogger("mizan.nafs_triad")
+
+
+@dataclass
+class NafsVoice:
+ """A single inner voice with its bias and dynamic weight."""
+ name: str # "Ammara" | "Lawwama" | "Mutmainna"
+ arabic: str
+ bias: str # "drive" | "caution" | "balance"
+ quran_ref: str
+
+ def score(self, task: str, complexity: str) -> float:
+ """Score this task from this voice's perspective."""
+ task_lower = task.lower()
+ if self.bias == "drive":
+ # Ammara: prefers action verbs, quick tasks
+ action_words = ["do", "run", "make", "create", "build", "send", "write", "execute"]
+ matches = sum(1 for w in action_words if w in task_lower)
+ base = 0.5 + 0.1 * matches
+ # Penalise extreme complexity (Ammara is impatient)
+ if complexity == "extreme":
+ base *= 0.7
+ return min(1.0, base)
+
+ elif self.bias == "caution":
+ # Lawwama: prefers verify/check/review tasks
+ check_words = ["check", "verify", "review", "audit", "test", "validate", "ensure"]
+ matches = sum(1 for w in check_words if w in task_lower)
+ base = 0.4 + 0.12 * matches
+ # More weight on complex tasks (Lawwama is thorough)
+ if complexity in ("complex", "extreme"):
+ base = min(1.0, base + 0.2)
+ return min(1.0, base)
+
+ else: # balance / Mutmainna
+ # Mutmainna: weighs both sides, prefers nuanced tasks
+ nuance_words = ["explain", "analyse", "compare", "understand", "balance",
+ "consider", "reflect", "what", "why", "how"]
+ matches = sum(1 for w in nuance_words if w in task_lower)
+ base = 0.45 + 0.08 * matches
+ return min(1.0, base)
+
+
+@dataclass
+class NafsDecision:
+ """Result of Nafs Triad deliberation."""
+ dominant_voice: str # "Ammara" | "Lawwama" | "Mutmainna"
+ approach: str # Instruction injected into system prompt
+ confidence: float # 0-1 how strongly one voice won
+ dissent_ratio: float # fraction of weight held by losing voices
+ nafs_level: int
+
+
+# Weights per nafs_level bracket: [Ammara, Lawwama, Mutmainna]
+_LEVEL_WEIGHTS = {
+ 1: (0.55, 0.30, 0.15),
+ 2: (0.50, 0.32, 0.18),
+ 3: (0.32, 0.40, 0.28),
+ 4: (0.28, 0.38, 0.34),
+ 5: (0.20, 0.30, 0.50),
+ 6: (0.15, 0.28, 0.57),
+ 7: (0.10, 0.25, 0.65),
+}
+
+_APPROACHES = {
+ "Ammara": (
+ "Act decisively and efficiently. Prioritise speed and direct action. "
+ "Complete the task in as few steps as necessary."
+ ),
+ "Lawwama": (
+ "Proceed carefully. Verify each step before advancing. "
+ "Self-correct any inconsistency you notice. Prefer correctness over speed."
+ ),
+ "Mutmainna": (
+ "Take a balanced, integrated approach. Consider multiple angles before acting. "
+ "Seek the most thoughtful, well-rounded response — quality over haste or overcaution."
+ ),
+}
+
+
+def _softmax(values: list[float]) -> list[float]:
+ max_v = max(values)
+ exps = [math.exp(v - max_v) for v in values]
+ total = sum(exps)
+ return [e / total for e in exps]
+
+
+class NafsTriad:
+ """
+ Three inner voices deliberate via weighted softmax bidding.
+
+ Usage:
+ triad = NafsTriad()
+ decision = triad.deliberate("Analyse this error log", nafs_level=3)
+ # → NafsDecision(dominant_voice="Lawwama", approach="Proceed carefully...")
+ """
+
+ def __init__(self):
+ self.ammara = NafsVoice(
+ "Ammara", "أمارة", "drive", "12:53"
+ )
+ self.lawwama = NafsVoice(
+ "Lawwama", "لوامة", "caution", "75:2"
+ )
+ self.mutmainna = NafsVoice(
+ "Mutmainna", "مطمئنة", "balance", "89:27"
+ )
+ self._voices = [self.ammara, self.lawwama, self.mutmainna]
+
+ def deliberate(self, task: str, nafs_level: int, complexity: str = "moderate") -> NafsDecision:
+ """
+ Deliberate on a task and return the dominant voice's decision.
+
+ Algorithm:
+ 1. Clamp nafs_level to valid range
+ 2. Each voice scores the task from its bias perspective
+ 3. Multiply score by level-dependent base weight
+ 4. Softmax → probability distribution
+ 5. Winning voice injects its approach into the system prompt
+ """
+ level = max(1, min(7, nafs_level))
+ weights = _LEVEL_WEIGHTS[level]
+
+ raw_scores = [v.score(task, complexity) for v in self._voices]
+ weighted = [s * w for s, w in zip(raw_scores, weights)]
+ probs = _softmax(weighted)
+
+ winner_idx = probs.index(max(probs))
+ winner = self._voices[winner_idx]
+ winner_prob = probs[winner_idx]
+ dissent = 1.0 - winner_prob
+
+ decision = NafsDecision(
+ dominant_voice=winner.name,
+ approach=_APPROACHES[winner.name],
+ confidence=round(winner_prob, 3),
+ dissent_ratio=round(dissent, 3),
+ nafs_level=level,
+ )
+
+ logger.debug(
+ "[NAFS_TRIAD] level=%d task='%s...' → %s (conf=%.2f dissent=%.2f)",
+ level, task[:60], winner.name, winner_prob, dissent,
+ )
+ return decision
+
+ def to_dict(self) -> dict:
+ return {
+ "voices": [v.name for v in self._voices],
+ "description": "Three-voice Nafs deliberation (Ammara/Lawwama/Mutmainna)",
+ }
diff --git a/backend/core/parallel_agents.py b/backend/core/parallel_agents.py
new file mode 100644
index 0000000..e7a4ef3
--- /dev/null
+++ b/backend/core/parallel_agents.py
@@ -0,0 +1,422 @@
+"""
+Parallel Agents — QALB Parallel Processing Architecture
+=========================================================
+
+"And We have created you in pairs" — Quran 78:8
+
+Implements Algorithm 1: QALB_PARALLEL_SCHEDULER
+Multiple simultaneous thought streams with workspace competition,
+salience-gated serial Qalb bottleneck, and cerebellar background processing.
+
+Algorithm 2: SKILL_AUTOMATION_TRANSFER
+Mastery-based delegation of automated skills to background processing.
+
+Math:
+ dh_i/dt = -h_i/τ_i + σ(W_self·h_i + W_input·x + Σ_j W_ij·broadcast_j)
+ P(agent_i) = exp(salience_i/τ) / Σ_j exp(salience_j/τ)
+ salience_i = w_novel·novelty(h_i) + w_emotion·|affect(h_i)| + w_task·relevance(h_i, goal)
+"""
+
+import logging
+import math
+import time
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
+
+logger = logging.getLogger("mizan.parallel_agents")
+
+# Salience weights
+W_NOVELTY = 0.35
+W_EMOTION = 0.30
+W_TASK_RELEVANCE = 0.35
+
+# Softmax temperature for workspace competition
+TAU_COMPETITION = 1.0
+
+# Decay time constants per agent type (seconds)
+TAU_FAST = 0.1 # reactive / surface agents
+TAU_SLOW = 0.5 # deliberative / deep agents
+
+# Skill automation mastery threshold
+MASTERY_THRESHOLD = 0.85
+
+
+class AgentStreamType(Enum):
+ REACTIVE = "reactive" # fast, surface-level (Ammara-like)
+ DELIBERATIVE = "deliberative" # slow, deep reasoning (Mutmainna-like)
+ BACKGROUND = "background" # cerebellar — automated skills, no bottleneck
+
+
+@dataclass
+class AgentStream:
+ """A single parallel thought stream."""
+ name: str
+ stream_type: AgentStreamType
+ hidden_state: float = 0.0 # h_i: activation level
+ salience: float = 0.0
+ novelty: float = 0.0
+ affect: float = 0.0
+ task_relevance: float = 0.0
+ tau: float = TAU_SLOW
+ output: str = ""
+ latency_ms: float = 0.0
+ is_automated: bool = False # True = background cerebellar processing
+
+
+@dataclass
+class WorkspaceAccess:
+ """Result of workspace competition — one stream wins global broadcast."""
+ winner: str
+ winner_salience: float
+ competition_probs: dict[str, float]
+ broadcast_content: str
+ background_results: list[dict]
+
+
+@dataclass
+class SkillAutomation:
+ """A skill that has been transferred to background processing."""
+ skill_name: str
+ mastery_score: float
+ invocation_count: int
+ avg_latency_ms: float
+ transferred_at: float
+
+
+@dataclass
+class ParallelResult:
+ """Full result from parallel processing cycle."""
+ workspace: WorkspaceAccess
+ integration: str
+ streams_active: int
+ total_latency_ms: float
+ skill_automations: list[str]
+
+
+class QalbParallelScheduler:
+ """
+ Algorithm 1: QALB_PARALLEL_SCHEDULER
+
+ Manages multiple simultaneous reasoning streams with:
+ - Global workspace competition (softmax salience)
+ - Serial Qalb bottleneck (only 1 stream accesses broadcast)
+ - Background cerebellar processing (automated skills bypass bottleneck)
+ - Cross-stream integration via broadcast signal
+
+ Biological analogy:
+ - Reactive streams ↔ fast System 1 (basal ganglia)
+ - Deliberative streams ↔ slow System 2 (prefrontal cortex)
+ - Background ↔ cerebellum (automated, parallel, no bottleneck)
+ """
+
+ def __init__(self):
+ self.streams: dict[str, AgentStream] = self._init_default_streams()
+ self.automation_registry: dict[str, SkillAutomation] = {}
+ self.broadcast_history: list[str] = []
+ self.cycle_count = 0
+
+ def _init_default_streams(self) -> dict[str, AgentStream]:
+ return {
+ "reactive": AgentStream(
+ name="reactive",
+ stream_type=AgentStreamType.REACTIVE,
+ tau=TAU_FAST,
+ ),
+ "deliberative": AgentStream(
+ name="deliberative",
+ stream_type=AgentStreamType.DELIBERATIVE,
+ tau=TAU_SLOW,
+ ),
+ "background": AgentStream(
+ name="background",
+ stream_type=AgentStreamType.BACKGROUND,
+ tau=TAU_SLOW,
+ is_automated=True,
+ ),
+ }
+
+ def process(
+ self,
+ task: str,
+ context: dict | None = None,
+ task_goal: str = "",
+ ) -> ParallelResult:
+ """
+ Run one full parallel processing cycle.
+
+ Steps:
+ 1. Perceive — update stream hidden states
+ 2. Compete — compute salience, softmax competition
+ 3. Bottleneck — winner accesses Qalb global workspace
+ 4. Background — automated skills run in parallel
+ 5. Integrate — broadcast merges all streams
+ """
+ self.cycle_count += 1
+ start = time.monotonic()
+
+ # Step 1: Update hidden states
+ for stream in self.streams.values():
+ stream.hidden_state = self._update_hidden_state(stream, task, context)
+
+ # Step 2: Compute salience and competition (skip background — no bottleneck)
+ competing = {
+ name: stream
+ for name, stream in self.streams.items()
+ if not stream.is_automated
+ }
+ for stream in competing.values():
+ stream.novelty = self._compute_novelty(stream, task)
+ stream.affect = self._compute_affect(stream, task)
+ stream.task_relevance = self._compute_task_relevance(stream, task, task_goal)
+ stream.salience = self._compute_salience(stream)
+
+ # Step 3: Softmax competition → workspace winner
+ winner_name, probs = self._softmax_compete(competing)
+ winner = competing[winner_name]
+
+ # Generate winner output
+ winner.output = self._generate_stream_output(winner, task)
+ broadcast = winner.output
+
+ # Step 4: Background streams run without bottleneck
+ background_results = []
+ for stream in self.streams.values():
+ if stream.is_automated:
+ result = self._run_background_stream(stream, task, broadcast)
+ background_results.append(result)
+
+ self.broadcast_history.append(broadcast[:200])
+ if len(self.broadcast_history) > 50:
+ self.broadcast_history.pop(0)
+
+ workspace = WorkspaceAccess(
+ winner=winner_name,
+ winner_salience=winner.salience,
+ competition_probs=probs,
+ broadcast_content=broadcast,
+ background_results=background_results,
+ )
+
+ # Step 5: Integrate — merge broadcast + background
+ integration = self._integrate(broadcast, background_results, task)
+
+ elapsed_ms = (time.monotonic() - start) * 1000
+ logger.debug(
+ "[PARALLEL] cycle=%d winner=%s salience=%.3f",
+ self.cycle_count, winner_name, winner.salience,
+ )
+
+ return ParallelResult(
+ workspace=workspace,
+ integration=integration,
+ streams_active=len(self.streams),
+ total_latency_ms=elapsed_ms,
+ skill_automations=list(self.automation_registry.keys()),
+ )
+
+ def _update_hidden_state(
+ self, stream: AgentStream, task: str, context: dict | None
+ ) -> float:
+ """
+ Simplified discrete-time hidden state update:
+ h_i(t+1) = h_i(t)·(1 - 1/τ_i) + σ(W_input·x)
+
+ x = task complexity signal (0-1)
+ """
+ complexity = min(1.0, len(task.split()) / 50.0)
+ broadcast_signal = (
+ self._hash_to_float(self.broadcast_history[-1])
+ if self.broadcast_history else 0.0
+ )
+ decay = 1.0 - (1.0 / max(stream.tau * 10, 1))
+ input_signal = self._sigmoid(complexity + 0.3 * broadcast_signal)
+ return stream.hidden_state * decay + input_signal * (1 - decay)
+
+ def _compute_novelty(self, stream: AgentStream, task: str) -> float:
+ """
+ novelty(h_i) = 1 - max_k cos_sim(h_i, known_k)
+ Approximated by checking if task tokens appear in broadcast history.
+ """
+ task_words = set(task.lower().split())
+ if not self.broadcast_history:
+ return 0.8
+ history_words = set(" ".join(self.broadcast_history).lower().split())
+ overlap = len(task_words & history_words) / max(len(task_words), 1)
+ return max(0.0, 1.0 - overlap)
+
+ def _compute_affect(self, stream: AgentStream, task: str) -> float:
+ """Emotional valence magnitude |affect(h_i)|."""
+ positive = ["help", "create", "build", "solve", "improve", "good"]
+ negative = ["error", "fail", "danger", "harm", "delete", "wrong"]
+ task_lower = task.lower()
+ pos_score = sum(1 for w in positive if w in task_lower) / len(positive)
+ neg_score = sum(1 for w in negative if w in task_lower) / len(negative)
+ return abs(pos_score - neg_score) + 0.1 * stream.hidden_state
+
+ def _compute_task_relevance(
+ self, stream: AgentStream, task: str, goal: str
+ ) -> float:
+ """Relevance of stream hidden state to task goal."""
+ if not goal:
+ return 0.5
+ goal_words = set(goal.lower().split())
+ task_words = set(task.lower().split())
+ if not goal_words:
+ return 0.5
+ overlap = len(task_words & goal_words) / len(goal_words)
+ # Reactive streams prefer simple tasks, deliberative prefer complex
+ complexity = min(1.0, len(task.split()) / 50.0)
+ if stream.stream_type == AgentStreamType.REACTIVE:
+ return overlap * (1.0 - complexity * 0.3)
+ return overlap * (1.0 + complexity * 0.3)
+
+ def _compute_salience(self, stream: AgentStream) -> float:
+ """
+ salience_i = w_novel·novelty + w_emotion·|affect| + w_task·relevance
+ """
+ return (
+ W_NOVELTY * stream.novelty
+ + W_EMOTION * stream.affect
+ + W_TASK_RELEVANCE * stream.task_relevance
+ )
+
+ def _softmax_compete(
+ self, streams: dict[str, AgentStream]
+ ) -> tuple[str, dict[str, float]]:
+ """
+ P(agent_i) = exp(salience_i / τ) / Σ_j exp(salience_j / τ)
+ Winner = argmax P.
+ """
+ saliences = {name: s.salience for name, s in streams.items()}
+ max_s = max(saliences.values()) if saliences else 0.0
+ exps = {k: math.exp((v - max_s) / TAU_COMPETITION) for k, v in saliences.items()}
+ total = sum(exps.values())
+ probs = {k: v / total for k, v in exps.items()}
+ winner = max(probs, key=lambda k: probs[k])
+ return winner, probs
+
+ def _generate_stream_output(self, stream: AgentStream, task: str) -> str:
+ """Placeholder: in production, each stream calls its LLM sub-agent."""
+ type_label = {
+ AgentStreamType.REACTIVE: "Quick assessment",
+ AgentStreamType.DELIBERATIVE: "Deep analysis",
+ AgentStreamType.BACKGROUND: "Background task",
+ }[stream.stream_type]
+ return f"[{stream.name.upper()}:{type_label}] Processing: {task[:80]}"
+
+ def _run_background_stream(
+ self, stream: AgentStream, task: str, broadcast: str
+ ) -> dict:
+ """Background cerebellar streams — run automated skills."""
+ results = []
+ for skill_name, automation in self.automation_registry.items():
+ if automation.mastery_score >= MASTERY_THRESHOLD:
+ results.append(f"{skill_name}:automated")
+ return {
+ "stream": stream.name,
+ "skills_run": results,
+ "output": f"[BACKGROUND] {len(results)} automated skills active",
+ }
+
+ def _integrate(
+ self, broadcast: str, background_results: list[dict], task: str
+ ) -> str:
+ """Merge broadcast signal with background processing results."""
+ parts = [f"WORKSPACE: {broadcast[:200]}"]
+ for bg in background_results:
+ if bg.get("skills_run"):
+ parts.append(f"BACKGROUND: {bg['output']}")
+ return "\n".join(parts)
+
+ @staticmethod
+ def _sigmoid(x: float) -> float:
+ return 1.0 / (1.0 + math.exp(-x))
+
+ @staticmethod
+ def _hash_to_float(s: str) -> float:
+ """Map string to float in [0, 1] via hash."""
+ return (hash(s) % 10000) / 10000.0
+
+
+class SkillAutomationTransfer:
+ """
+ Algorithm 2: SKILL_AUTOMATION_TRANSFER
+
+ Tracks skill invocation mastery. When mastery >= MASTERY_THRESHOLD,
+ the skill is delegated to background cerebellar processing,
+ freeing the serial Qalb bottleneck for novel tasks.
+
+ Mastery score: exponential moving average of success rate.
+ mastery(t+1) = α·success(t) + (1-α)·mastery(t)
+ """
+
+ ALPHA = 0.15 # EMA learning rate
+
+ def __init__(self):
+ self.skill_stats: dict[str, dict] = {}
+ self.automated: dict[str, SkillAutomation] = {}
+
+ def record_invocation(
+ self, skill_name: str, success: bool, latency_ms: float
+ ) -> bool:
+ """Record a skill invocation. Returns True if newly automated."""
+ if skill_name not in self.skill_stats:
+ self.skill_stats[skill_name] = {
+ "mastery": 0.0,
+ "count": 0,
+ "total_latency": 0.0,
+ }
+
+ stats = self.skill_stats[skill_name]
+ stats["count"] += 1
+ stats["total_latency"] += latency_ms
+ stats["mastery"] = (
+ self.ALPHA * (1.0 if success else 0.0)
+ + (1.0 - self.ALPHA) * stats["mastery"]
+ )
+
+ # Check if ready for automation
+ if (
+ stats["mastery"] >= MASTERY_THRESHOLD
+ and skill_name not in self.automated
+ and stats["count"] >= 5
+ ):
+ avg_latency = stats["total_latency"] / stats["count"]
+ self.automated[skill_name] = SkillAutomation(
+ skill_name=skill_name,
+ mastery_score=stats["mastery"],
+ invocation_count=stats["count"],
+ avg_latency_ms=avg_latency,
+ transferred_at=time.time(),
+ )
+ logger.info(
+ "[AUTOMATION] Skill '%s' transferred to background (mastery=%.2f)",
+ skill_name, stats["mastery"],
+ )
+ return True
+ return False
+
+ def is_automated(self, skill_name: str) -> bool:
+ return skill_name in self.automated
+
+ def get_mastery(self, skill_name: str) -> float:
+ return self.skill_stats.get(skill_name, {}).get("mastery", 0.0)
+
+ def get_automated_skills(self) -> list[str]:
+ return list(self.automated.keys())
+
+ def to_dict(self) -> dict:
+ return {
+ "tracked_skills": len(self.skill_stats),
+ "automated_skills": len(self.automated),
+ "automation_threshold": MASTERY_THRESHOLD,
+ "skills": {
+ name: {
+ "mastery": round(stats["mastery"], 3),
+ "count": stats["count"],
+ "automated": name in self.automated,
+ }
+ for name, stats in self.skill_stats.items()
+ },
+ }
diff --git a/backend/core/qalb_processor.py b/backend/core/qalb_processor.py
new file mode 100644
index 0000000..2e4c187
--- /dev/null
+++ b/backend/core/qalb_processor.py
@@ -0,0 +1,153 @@
+"""
+Qalb Processor (قلب) — State-Modulated Global Workspace
+=========================================================
+
+"There is a piece of flesh in the body — if it is sound, the whole body is sound;
+ if it is corrupt, the whole body is corrupt. Indeed, it is the heart (qalb)."
+— Hadith (Bukhari & Muslim)
+
+Upgrades the Qalb from keyword sentiment detection to a **global workspace**
+with cardiac-inspired systole/diastole oscillation that modulates LLM parameters.
+
+Cardiac cycle:
+ Systole (QABD — قبض) : Contraction — focused, analytical, precise
+ Diastole (BAST — بسط) : Expansion — creative, exploratory, open
+
+KHUSHU (خشوع) — Deep focus state triggered at high nafs_level + extreme tasks.
+"""
+
+import logging
+import math
+from dataclasses import dataclass
+from enum import Enum
+
+logger = logging.getLogger("mizan.qalb_processor")
+
+
+class QalbState(Enum):
+ QABD = "qabd" # Contraction — analytical, focused
+ BAST = "bast" # Expansion — creative, open
+ KHUSHU = "khushu" # Deep focus — highest attention
+
+
+@dataclass
+class QalbOutput:
+ """LLM parameter recommendations from Qalb state."""
+ state: QalbState
+ max_tokens: int
+ temperature: float
+ reasoning: str # Why this state was chosen
+
+ def to_dict(self) -> dict:
+ return {
+ "state": self.state.value,
+ "max_tokens": self.max_tokens,
+ "temperature": round(self.temperature, 2),
+ "reasoning": self.reasoning,
+ }
+
+
+# LLM params per Qalb state
+_STATE_PARAMS = {
+ QalbState.QABD: {"max_tokens": 2048, "temperature": 0.3},
+ QalbState.BAST: {"max_tokens": 4096, "temperature": 0.75},
+ QalbState.KHUSHU: {"max_tokens": 3000, "temperature": 0.45},
+}
+
+# Emotional states that force a specific Qalb state
+# (from core/qalb.py EmotionalState values)
+_EMOTION_TO_STATE = {
+ "frustrated": QalbState.QABD,
+ "anxious": QalbState.QABD,
+ "confused": QalbState.QABD,
+ "positive": QalbState.BAST,
+ "determined": QalbState.BAST,
+ "fatigued": QalbState.QABD, # Tired → reduce load
+ "neutral": None, # Follow oscillation
+}
+
+
+class QalbProcessor:
+ """
+ Global workspace with cardiac oscillation.
+
+ The oscillation_phase advances by PHASE_STEP with each call to process().
+ Phase 0.0 – 0.5 → Systole (QABD)
+ Phase 0.5 – 1.0 → Diastole (BAST)
+
+ Emotional state can override the natural oscillation.
+ KHUSHU is triggered when nafs_level >= 4 and task complexity is "extreme".
+
+ Usage:
+ processor = QalbProcessor()
+ output = processor.process("Analyse logs", emotional_reading, nafs_level=3)
+ # Use output.max_tokens and output.temperature in LLM call
+ """
+
+ PHASE_STEP = 0.04 # Advance ~25 interactions per full cycle
+
+ def __init__(self):
+ self.oscillation_phase: float = 0.0
+ self._interaction_count: int = 0
+
+ def process(
+ self,
+ task: str,
+ emotional_state: str = "neutral",
+ nafs_level: int = 1,
+ complexity: str = "moderate",
+ ) -> QalbOutput:
+ """
+ Determine Qalb state and return LLM parameter recommendations.
+
+ Priority order:
+ 1. KHUSHU — if nafs_level >= 4 AND complexity == "extreme"
+ 2. Emotional override — if emotion maps to specific state
+ 3. Cardiac oscillation — systole vs diastole from phase
+ """
+ self._interaction_count += 1
+ # Advance oscillation
+ self.oscillation_phase = (self.oscillation_phase + self.PHASE_STEP) % 1.0
+
+ # 1. KHUSHU override
+ if nafs_level >= 4 and complexity == "extreme":
+ state = QalbState.KHUSHU
+ reasoning = f"KHUSHU: nafs_level={nafs_level} + extreme task"
+ # 2. Emotional override
+ elif emotional_state in _EMOTION_TO_STATE and _EMOTION_TO_STATE[emotional_state]:
+ state = _EMOTION_TO_STATE[emotional_state]
+ reasoning = f"Emotional override: {emotional_state} → {state.value}"
+ # 3. Cardiac oscillation
+ else:
+ if self.oscillation_phase < 0.5:
+ state = QalbState.QABD
+ phase_pct = int(self.oscillation_phase / 0.5 * 100)
+ reasoning = f"Systole phase {phase_pct}%: analytical focus"
+ else:
+ state = QalbState.BAST
+ phase_pct = int((self.oscillation_phase - 0.5) / 0.5 * 100)
+ reasoning = f"Diastole phase {phase_pct}%: creative expansion"
+
+ params = _STATE_PARAMS[state]
+ output = QalbOutput(
+ state=state,
+ max_tokens=params["max_tokens"],
+ temperature=params["temperature"],
+ reasoning=reasoning,
+ )
+
+ logger.debug(
+ "[QALB] phase=%.2f emotion=%s nafs=%d → %s (tokens=%d temp=%.2f)",
+ self.oscillation_phase, emotional_state, nafs_level,
+ state.value, output.max_tokens, output.temperature,
+ )
+ return output
+
+ def get_phase_info(self) -> dict:
+ """Current oscillation state for monitoring."""
+ phase = "systole (QABD)" if self.oscillation_phase < 0.5 else "diastole (BAST)"
+ return {
+ "oscillation_phase": round(self.oscillation_phase, 3),
+ "cardiac_phase": phase,
+ "interaction_count": self._interaction_count,
+ }
diff --git a/backend/core/self_healing.py b/backend/core/self_healing.py
new file mode 100644
index 0000000..a6be9c6
--- /dev/null
+++ b/backend/core/self_healing.py
@@ -0,0 +1,447 @@
+"""
+Self-Healing Architecture — Lawwāma Self-Monitoring Protocol
+=============================================================
+
+"And I swear by the self-reproaching soul (Nafs al-Lawwāma)" — Quran 75:2
+
+Implements Algorithm 3: LAWWAMA_SELF_HEALING
+
+4-level repair hierarchy (DNA repair analogy):
+ L1: Immediate proofreading (single-token correction)
+ L2: Batch mismatch repair (paragraph-level)
+ L3: Structural excision repair (reasoning chain replacement)
+ L4: Double-strand regeneration (full response restart)
+
+Health metric:
+ H(t) = H_baseline - Σ_i λ_i·ε_i(t) + Σ_j μ_j·repair_j(t)
+
+Synaptic homeostasis:
+ w_ij(t+1) = w_ij(t) × (target_activity / actual_activity)^η
+
+Hallucination score:
+ H_score = 1 - min(conviction_Fu'ad, consistency_Lawh, agreement_agents)
+"""
+
+import logging
+import time
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
+
+logger = logging.getLogger("mizan.self_healing")
+
+# Health metric parameters
+H_BASELINE = 1.0
+LAMBDA_HALLUCINATION = 0.4 # weight: hallucination error
+LAMBDA_COHERENCE = 0.3 # weight: coherence violation
+LAMBDA_CONTRADICTION = 0.2 # weight: self-contradiction
+LAMBDA_DRIFT = 0.1 # weight: goal drift
+
+MU_L1 = 0.1 # health restored per L1 repair
+MU_L2 = 0.2 # health restored per L2 repair
+MU_L3 = 0.35 # health restored per L3 repair
+MU_L4 = 0.6 # health restored per L4 repair
+
+# Synaptic homeostasis
+ETA = 0.1 # homeostatic learning rate
+
+# Repair thresholds
+L1_THRESHOLD = 0.85 # H below this → L1 repair
+L2_THRESHOLD = 0.70
+L3_THRESHOLD = 0.55
+L4_THRESHOLD = 0.40
+
+
+class RepairLevel(Enum):
+ NONE = 0
+ L1_PROOFREADING = 1 # Immediate: token-level
+ L2_MISMATCH = 2 # Batch: paragraph-level
+ L3_EXCISION = 3 # Structural: chain replacement
+ L4_REGENERATION = 4 # Nuclear: full restart
+
+
+class ErrorType(Enum):
+ HALLUCINATION = "hallucination"
+ COHERENCE_VIOLATION = "coherence_violation"
+ SELF_CONTRADICTION = "self_contradiction"
+ GOAL_DRIFT = "goal_drift"
+ TOOL_FAILURE = "tool_failure"
+
+
+@dataclass
+class HealthError:
+ error_type: ErrorType
+ severity: float # 0.0 - 1.0
+ location: str # where in the reasoning chain
+ description: str
+ timestamp: float = field(default_factory=time.time)
+
+
+@dataclass
+class RepairRecord:
+ level: RepairLevel
+ errors_addressed: list[ErrorType]
+ health_delta: float
+ action_taken: str
+ timestamp: float = field(default_factory=time.time)
+ success: bool = True
+
+
+@dataclass
+class ImmuneMemory:
+ """Records successful repairs for adaptive future healing."""
+ error_pattern: str
+ repair_strategy: RepairLevel
+ success_rate: float
+ invocation_count: int
+
+
+@dataclass
+class HealthReport:
+ current_health: float
+ baseline: float
+ errors: list[HealthError]
+ repair_needed: RepairLevel
+ repair_records: list[RepairRecord]
+ hallucination_score: float
+ immune_memories: int
+ synaptic_weights: dict[str, float]
+
+
+class LawwamaHealingSystem:
+ """
+ Algorithm 3: LAWWAMA_SELF_HEALING
+
+ The self-reproaching soul monitors its own output quality,
+ detects errors across 4 levels of severity, and applies
+ appropriate repair strategies — never accepting corruption silently.
+
+ Health metric tracks cumulative error load and repair benefit:
+ H(t) = H_baseline - Σ_i λ_i·ε_i(t) + Σ_j μ_j·repair_j(t)
+ """
+
+ def __init__(self):
+ self.health: float = H_BASELINE
+ self.error_history: list[HealthError] = []
+ self.repair_history: list[RepairRecord] = []
+ self.immune_memory: dict[str, ImmuneMemory] = {}
+ # Synaptic weights: skill_name → confidence weight
+ self.synaptic_weights: dict[str, float] = {}
+ self.target_activity = 0.7 # target activation level
+ self._cycle = 0
+
+ def monitor(
+ self,
+ response: str,
+ task: str,
+ conviction_score: float = 0.5,
+ lawh_consistency: float = 0.8,
+ agent_agreement: float = 0.7,
+ ) -> HealthReport:
+ """
+ Run a full health monitoring cycle.
+
+ 1. Compute hallucination score
+ 2. Detect errors from response
+ 3. Update health metric H(t)
+ 4. Determine repair level needed
+ 5. Return HealthReport
+ """
+ self._cycle += 1
+
+ # Hallucination score: H_score = 1 - min(conviction, consistency, agreement)
+ hallucination_score = 1.0 - min(conviction_score, lawh_consistency, agent_agreement)
+
+ # Detect errors
+ errors = self._detect_errors(response, task, hallucination_score)
+
+ # Update health metric
+ error_penalty = sum(
+ self._lambda(e.error_type) * e.severity for e in errors
+ )
+ repair_benefit = sum(
+ self._mu(r.level) for r in self.repair_history[-5:]
+ if time.time() - r.timestamp < 60
+ )
+ self.health = max(
+ 0.0,
+ min(H_BASELINE, H_BASELINE - error_penalty + repair_benefit)
+ )
+
+ # Record errors
+ self.error_history.extend(errors)
+ if len(self.error_history) > 200:
+ self.error_history = self.error_history[-100:]
+
+ # Determine repair level
+ repair_needed = self._classify_repair_level(self.health)
+
+ # Synaptic homeostasis update
+ actual_activity = 1.0 - hallucination_score
+ self._homeostatic_update(task, actual_activity)
+
+ logger.debug(
+ "[LAWWAMA] cycle=%d health=%.3f h_score=%.3f errors=%d repair=%s",
+ self._cycle, self.health, hallucination_score, len(errors), repair_needed.name,
+ )
+
+ return HealthReport(
+ current_health=round(self.health, 4),
+ baseline=H_BASELINE,
+ errors=errors,
+ repair_needed=repair_needed,
+ repair_records=self.repair_history[-10:],
+ hallucination_score=round(hallucination_score, 4),
+ immune_memories=len(self.immune_memory),
+ synaptic_weights=dict(self.synaptic_weights),
+ )
+
+ def repair(
+ self,
+ level: RepairLevel,
+ response: str,
+ task: str,
+ errors: list[HealthError],
+ ) -> tuple[str, RepairRecord]:
+ """
+ Execute repair at the specified level.
+
+ Returns: (repaired_response, repair_record)
+ """
+ action = ""
+ repaired = response
+
+ if level == RepairLevel.L1_PROOFREADING:
+ repaired, action = self._l1_proofreading(response)
+
+ elif level == RepairLevel.L2_MISMATCH:
+ repaired, action = self._l2_mismatch_repair(response, task)
+
+ elif level == RepairLevel.L3_EXCISION:
+ repaired, action = self._l3_structural_excision(response, task, errors)
+
+ elif level == RepairLevel.L4_REGENERATION:
+ repaired, action = self._l4_regeneration(task)
+
+ error_types = [e.error_type for e in errors]
+ record = RepairRecord(
+ level=level,
+ errors_addressed=error_types,
+ health_delta=self._mu(level),
+ action_taken=action,
+ )
+ self.repair_history.append(record)
+ self.health = min(H_BASELINE, self.health + self._mu(level))
+
+ # Update immune memory
+ pattern = self._error_pattern(errors)
+ if pattern in self.immune_memory:
+ mem = self.immune_memory[pattern]
+ mem.invocation_count += 1
+ mem.success_rate = (
+ 0.8 * mem.success_rate + 0.2 * (1.0 if record.success else 0.0)
+ )
+ else:
+ self.immune_memory[pattern] = ImmuneMemory(
+ error_pattern=pattern,
+ repair_strategy=level,
+ success_rate=1.0,
+ invocation_count=1,
+ )
+
+ logger.info(
+ "[REPAIR] L%d applied: %s → health=%.3f",
+ level.value, action[:80], self.health,
+ )
+ return repaired, record
+
+ def should_checkpoint(self, turn: int, max_turns: int) -> bool:
+ """
+ Health-based checkpoint interval — replaces hardcoded `turn % 3 == 0`.
+
+ Healthy agent (H >= 0.85): check every 4 turns
+ Moderate (0.55 <= H < 0.85): check every 2 turns
+ Unhealthy (H < 0.55): check every turn
+ """
+ if turn <= 0:
+ return False
+ if self.health >= L1_THRESHOLD:
+ interval = 4
+ elif self.health >= L3_THRESHOLD:
+ interval = 2
+ else:
+ interval = 1
+ return turn % interval == 0
+
+ def _l1_proofreading(self, response: str) -> tuple[str, str]:
+ """L1: Immediate token-level correction — add uncertainty hedges."""
+ hedges = [
+ ("I am certain", "I believe"),
+ ("definitely", "likely"),
+ ("always", "generally"),
+ ("never fails", "rarely fails"),
+ ("guaranteed", "expected"),
+ ]
+ repaired = response
+ changes = []
+ for wrong, right in hedges:
+ if wrong.lower() in repaired.lower():
+ repaired = repaired.replace(wrong, right)
+ changes.append(f"'{wrong}'→'{right}'")
+ action = f"L1 proofreading: {', '.join(changes) or 'hedge inserted'}"
+ return repaired, action
+
+ def _l2_mismatch_repair(self, response: str, task: str) -> tuple[str, str]:
+ """L2: Paragraph-level mismatch — add consistency caveat."""
+ caveat = (
+ "\n\n[Lawwāma Note: Some portions of this response may contain "
+ "inconsistencies. Please verify key claims independently.]"
+ )
+ action = "L2 mismatch repair: consistency caveat appended"
+ return response + caveat, action
+
+ def _l3_structural_excision(
+ self, response: str, task: str, errors: list[HealthError]
+ ) -> tuple[str, str]:
+ """L3: Remove erroneous reasoning chain, rebuild from task."""
+ paragraphs = response.split("\n\n")
+ # Keep first and last paragraphs (intro + conclusion), rebuild middle
+ if len(paragraphs) > 2:
+ rebuilt = paragraphs[0] + "\n\n[Reasoning chain rebuilt due to coherence errors]\n\n" + paragraphs[-1]
+ else:
+ rebuilt = response
+ action = "L3 structural excision: middle chain rebuilt"
+ return rebuilt, action
+
+ def _l4_regeneration(self, task: str) -> tuple[str, str]:
+ """L4: Signal that full regeneration is needed."""
+ placeholder = (
+ "[Lawwāma: Full regeneration required. "
+ "Previous response had critical integrity failures. "
+ f"Please re-attempt task: {task[:100]}]"
+ )
+ action = "L4 nuclear regeneration: full restart signalled"
+ return placeholder, action
+
+ def _detect_errors(
+ self, response: str, task: str, hallucination_score: float
+ ) -> list[HealthError]:
+ errors = []
+ response_lower = response.lower()
+
+ # Hallucination detection
+ if hallucination_score > 0.5:
+ errors.append(HealthError(
+ error_type=ErrorType.HALLUCINATION,
+ severity=hallucination_score,
+ location="response",
+ description=f"Hallucination score {hallucination_score:.2f} exceeds threshold",
+ ))
+
+ # Contradiction markers
+ contradiction_pairs = [
+ ("always", "never"),
+ ("impossible", "definitely possible"),
+ ("cannot", "can easily"),
+ ]
+ for pos, neg in contradiction_pairs:
+ if pos in response_lower and neg in response_lower:
+ errors.append(HealthError(
+ error_type=ErrorType.SELF_CONTRADICTION,
+ severity=0.6,
+ location="response",
+ description=f"Contradiction detected: '{pos}' vs '{neg}'",
+ ))
+
+ # Overclaiming (epistemic violation)
+ overclaim_markers = [
+ "100% certain", "absolutely guaranteed", "impossible to fail",
+ "perfect solution", "I am certain"
+ ]
+ for marker in overclaim_markers:
+ if marker.lower() in response_lower:
+ errors.append(HealthError(
+ error_type=ErrorType.COHERENCE_VIOLATION,
+ severity=0.4,
+ location="response",
+ description=f"Overclaiming detected: '{marker}'",
+ ))
+ break
+
+ # Goal drift — response doesn't address task
+ task_keywords = set(task.lower().split()[:10])
+ response_words = set(response_lower.split())
+ overlap = len(task_keywords & response_words) / max(len(task_keywords), 1)
+ if overlap < 0.2:
+ errors.append(HealthError(
+ error_type=ErrorType.GOAL_DRIFT,
+ severity=0.3 + 0.3 * (1 - overlap),
+ location="response",
+ description=f"Goal drift: only {overlap:.1%} task keyword coverage",
+ ))
+
+ return errors
+
+ def _classify_repair_level(self, health: float) -> RepairLevel:
+ if health >= L1_THRESHOLD:
+ return RepairLevel.NONE
+ elif health >= L2_THRESHOLD:
+ return RepairLevel.L1_PROOFREADING
+ elif health >= L3_THRESHOLD:
+ return RepairLevel.L2_MISMATCH
+ elif health >= L4_THRESHOLD:
+ return RepairLevel.L3_EXCISION
+ else:
+ return RepairLevel.L4_REGENERATION
+
+ def _homeostatic_update(self, skill_key: str, actual_activity: float) -> None:
+ """
+ Synaptic homeostasis:
+ w_ij(t+1) = w_ij(t) × (target / actual)^η
+
+ Drives weights toward balanced activation.
+ """
+ if skill_key not in self.synaptic_weights:
+ self.synaptic_weights[skill_key] = 1.0
+ w = self.synaptic_weights[skill_key]
+ ratio = self.target_activity / max(actual_activity, 0.01)
+ self.synaptic_weights[skill_key] = min(2.0, max(0.1, w * (ratio ** ETA)))
+
+ @staticmethod
+ def _lambda(error_type: ErrorType) -> float:
+ """Health penalty weight per error type."""
+ return {
+ ErrorType.HALLUCINATION: LAMBDA_HALLUCINATION,
+ ErrorType.COHERENCE_VIOLATION: LAMBDA_COHERENCE,
+ ErrorType.SELF_CONTRADICTION: LAMBDA_CONTRADICTION,
+ ErrorType.GOAL_DRIFT: LAMBDA_DRIFT,
+ ErrorType.TOOL_FAILURE: LAMBDA_COHERENCE,
+ }.get(error_type, 0.2)
+
+ @staticmethod
+ def _mu(level: RepairLevel) -> float:
+ """Health restoration per repair level."""
+ return {
+ RepairLevel.NONE: 0.0,
+ RepairLevel.L1_PROOFREADING: MU_L1,
+ RepairLevel.L2_MISMATCH: MU_L2,
+ RepairLevel.L3_EXCISION: MU_L3,
+ RepairLevel.L4_REGENERATION: MU_L4,
+ }.get(level, 0.0)
+
+ @staticmethod
+ def _error_pattern(errors: list[HealthError]) -> str:
+ """Create a hashable pattern key from error types."""
+ types = sorted(set(e.error_type.value for e in errors))
+ return ":".join(types) or "none"
+
+ def to_dict(self) -> dict:
+ return {
+ "health": round(self.health, 4),
+ "baseline": H_BASELINE,
+ "error_history_count": len(self.error_history),
+ "repair_history_count": len(self.repair_history),
+ "immune_memory_entries": len(self.immune_memory),
+ "synaptic_weights_count": len(self.synaptic_weights),
+ "cycle": self._cycle,
+ }
diff --git a/backend/knowledge/__init__.py b/backend/knowledge/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/backend/knowledge/ingest.py b/backend/knowledge/ingest.py
new file mode 100644
index 0000000..6767323
--- /dev/null
+++ b/backend/knowledge/ingest.py
@@ -0,0 +1,189 @@
+"""
+Knowledge Ingestion (Ilm - عِلْم)
+==================================
+
+"Read in the name of your Lord who created" - Quran 96:1
+
+Extracts knowledge from external sources (URLs, PDFs, YouTube)
+and stores it in MIZAN's memory system.
+"""
+
+import logging
+import re
+from html.parser import HTMLParser
+
+import httpx
+
+logger = logging.getLogger("mizan.knowledge")
+
+
+class _TextExtractor(HTMLParser):
+ """Extract visible text from HTML, skipping scripts/styles."""
+
+ def __init__(self):
+ super().__init__()
+ self.text: list[str] = []
+ self._skip_tags = {"script", "style", "noscript", "head"}
+ self._skipping = False
+ self._title = ""
+ self._in_title = False
+
+ def handle_starttag(self, tag, attrs):
+ if tag in self._skip_tags:
+ self._skipping = True
+ if tag == "title":
+ self._in_title = True
+
+ def handle_endtag(self, tag):
+ if tag in self._skip_tags:
+ self._skipping = False
+ if tag == "title":
+ self._in_title = False
+
+ def handle_data(self, data):
+ if self._in_title:
+ self._title += data.strip()
+ if not self._skipping:
+ stripped = data.strip()
+ if stripped:
+ self.text.append(stripped)
+
+
+async def extract_url(url: str) -> dict:
+ """Extract text content from a web URL."""
+ async with httpx.AsyncClient(timeout=30, follow_redirects=True) as client:
+ headers = {"User-Agent": "Mozilla/5.0 (compatible; MIZAN/1.0)"}
+ response = await client.get(url, headers=headers)
+ response.raise_for_status()
+
+ extractor = _TextExtractor()
+ extractor.feed(response.text)
+ content = " ".join(extractor.text)
+
+ return {
+ "title": extractor._title or url,
+ "content": content[:50000],
+ "source": url,
+ "source_type": "url",
+ "char_count": len(content),
+ }
+
+
+def extract_pdf(file_bytes: bytes, filename: str = "upload.pdf") -> dict:
+ """Extract text content from a PDF file."""
+ try:
+ import fitz # pymupdf
+ except ImportError:
+ return {
+ "error": "pymupdf not installed. Run: pip install pymupdf",
+ "source": filename,
+ "source_type": "pdf",
+ }
+
+ doc = fitz.open(stream=file_bytes, filetype="pdf")
+ pages_text = []
+ for page in doc:
+ pages_text.append(page.get_text())
+ doc.close()
+
+ content = "\n\n".join(pages_text)
+ title_match = re.search(r"^(.{5,100})", content.strip())
+ title = title_match.group(1) if title_match else filename
+
+ return {
+ "title": title,
+ "content": content[:50000],
+ "source": filename,
+ "source_type": "pdf",
+ "page_count": len(pages_text),
+ "char_count": len(content),
+ }
+
+
+def _extract_youtube_id(url: str) -> str | None:
+ """Extract video ID from various YouTube URL formats."""
+ patterns = [
+ r"(?:v=|/v/|youtu\.be/)([a-zA-Z0-9_-]{11})",
+ r"(?:embed/)([a-zA-Z0-9_-]{11})",
+ r"(?:shorts/)([a-zA-Z0-9_-]{11})",
+ ]
+ for pattern in patterns:
+ match = re.search(pattern, url)
+ if match:
+ return match.group(1)
+ return None
+
+
+async def extract_youtube(url: str) -> dict:
+ """Extract transcript from a YouTube video."""
+ try:
+ from youtube_transcript_api import YouTubeTranscriptApi
+ except ImportError:
+ return {
+ "error": "youtube-transcript-api not installed. Run: pip install youtube-transcript-api",
+ "source": url,
+ "source_type": "youtube",
+ }
+
+ video_id = _extract_youtube_id(url)
+ if not video_id:
+ return {
+ "error": f"Could not extract video ID from URL: {url}",
+ "source": url,
+ "source_type": "youtube",
+ }
+
+ try:
+ transcript_list = YouTubeTranscriptApi.get_transcript(video_id)
+ content = " ".join(entry["text"] for entry in transcript_list)
+
+ return {
+ "title": f"YouTube: {video_id}",
+ "content": content[:50000],
+ "source": url,
+ "source_type": "youtube",
+ "video_id": video_id,
+ "segment_count": len(transcript_list),
+ "char_count": len(content),
+ }
+ except Exception as exc:
+ return {
+ "error": f"Failed to get transcript: {exc}",
+ "source": url,
+ "source_type": "youtube",
+ "video_id": video_id,
+ }
+
+
+def detect_source_type(source: str) -> str:
+ """Auto-detect source type from URL/string."""
+ lower = source.lower()
+ if "youtube.com" in lower or "youtu.be" in lower:
+ return "youtube"
+ if lower.endswith(".pdf"):
+ return "pdf"
+ if lower.startswith("http://") or lower.startswith("https://"):
+ return "url"
+ return "unknown"
+
+
+def chunk_content(content: str, chunk_size: int = 1000, overlap: int = 100) -> list[str]:
+ """Split content into overlapping chunks for memory storage."""
+ if len(content) <= chunk_size:
+ return [content]
+
+ chunks = []
+ start = 0
+ while start < len(content):
+ end = start + chunk_size
+ chunk = content[start:end]
+ # Try to break at sentence boundary
+ if end < len(content):
+ last_period = chunk.rfind(". ")
+ if last_period > chunk_size // 2:
+ chunk = chunk[: last_period + 1]
+ end = start + last_period + 1
+ chunks.append(chunk.strip())
+ start = end - overlap
+
+ return chunks
diff --git a/backend/memory/dhikr.py b/backend/memory/dhikr.py
index f1bb49a..36a2c1e 100644
--- a/backend/memory/dhikr.py
+++ b/backend/memory/dhikr.py
@@ -90,6 +90,40 @@ def __init__(self, db_path: str = "mizan_memory.db"):
self.masalik = MasalikNetwork()
+ # ── KnowledgeGraph: Entity + relationship store ──
+ try:
+ from memory.knowledge_graph import KnowledgeGraph
+ self.knowledge_graph = KnowledgeGraph(db_path=self.db_path)
+ except Exception:
+ self.knowledge_graph = None
+
+ # ── LawhMahfuz: Immutable preserved memory ──
+ try:
+ import os
+ from memory.lawh_mahfuz import LawhMahfuz
+ lawh_dir = os.path.dirname(self.db_path) if self.db_path != ":memory:" else "/tmp"
+ lawh_db = (
+ os.path.join(lawh_dir, "lawh_mahfuz.db")
+ if self.db_path != ":memory:"
+ else ":memory:"
+ )
+ self.lawh_mahfuz = LawhMahfuz(db_path=lawh_db)
+ except Exception:
+ self.lawh_mahfuz = None
+
+ # ── MemoryPyramid: Unified 5-layer query ──
+ try:
+ from memory.memory_pyramid import MemoryPyramid
+ self.pyramid = MemoryPyramid(
+ dhikr=self,
+ masalik=self.masalik,
+ lawh_mahfuz=self.lawh_mahfuz,
+ vector_store=None,
+ knowledge_graph=self.knowledge_graph,
+ )
+ except Exception:
+ self.pyramid = None
+
# Working memory (short-term) - like immediate consciousness
self.working_memory: dict[str, Memory] = {}
self.working_capacity = 7 # Miller's Law meets Quranic pattern (7 heavens)
@@ -199,6 +233,13 @@ def _init_db(self):
success INTEGER DEFAULT 1
)
""")
+ c.execute("""
+ CREATE TABLE IF NOT EXISTS preferences (
+ key TEXT PRIMARY KEY,
+ value TEXT NOT NULL,
+ updated_at TEXT NOT NULL
+ )
+ """)
# Performance indexes
c.execute("CREATE INDEX IF NOT EXISTS idx_audit_timestamp ON audit_log(timestamp)")
c.execute("CREATE INDEX IF NOT EXISTS idx_audit_severity ON audit_log(severity)")
@@ -303,10 +344,14 @@ async def recall(
params.append(agent_id)
if query:
words = query.strip().split()
- for word in words:
- escaped_word = word.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
- sql += " AND (content LIKE ? ESCAPE '\\' OR tags LIKE ? ESCAPE '\\')"
- params.extend([f"%{escaped_word}%", f"%{escaped_word}%"])
+ if words:
+ # Use OR across words so partial matches still return results
+ word_clauses = []
+ for word in words:
+ escaped = word.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
+ word_clauses.append("(content LIKE ? ESCAPE '\\' OR tags LIKE ? ESCAPE '\\')")
+ params.extend([f"%{escaped}%", f"%{escaped}%"])
+ sql += " AND (" + " OR ".join(word_clauses) + ")"
sql += " ORDER BY importance DESC, recency DESC LIMIT ?"
params.append(limit)
@@ -348,6 +393,25 @@ def recall_pathways(self, query: str, top_k: int = 8) -> str:
"""
return self.masalik.recall_context(query, top_k=top_k)
+ def recall_unified(self, query: str, top_k: int = 10) -> list:
+ """
+ Unified recall across all 5 memory layers via MemoryPyramid.
+ Returns list of MemoryHit objects ranked by (relevance × certainty × recency).
+ Falls back to standard dhikr recall if pyramid not available.
+ """
+ if self.pyramid:
+ return self.pyramid.query(query, top_k=top_k)
+ return []
+
+ def recall_unified_for_prompt(self, query: str, top_k: int = 5) -> str:
+ """
+ Recall unified memory and format for system prompt injection.
+ Returns empty string if nothing relevant found.
+ """
+ if self.pyramid:
+ return self.pyramid.format_for_prompt(query, top_k=top_k)
+ return self.recall_pathways(query, top_k=top_k)
+
async def _persist(self, memory: Memory):
"""Persist memory to database"""
conn = self._get_conn()
@@ -540,6 +604,154 @@ async def get_messages(self, session_id: str, limit: int = 50) -> list[dict]:
for r in rows
]
+ async def list_sessions(self, limit: int = 20) -> list[dict]:
+ """List recent chat sessions with metadata"""
+ conn = self._get_conn()
+ c = conn.cursor()
+ c.execute(
+ """
+ SELECT m.session_id,
+ MIN(m.created_at) as started_at,
+ MAX(m.created_at) as last_message_at,
+ COUNT(*) as message_count,
+ (SELECT content FROM agent_messages
+ WHERE session_id = m.session_id AND role = 'user'
+ ORDER BY created_at LIMIT 1) as first_message
+ FROM agent_messages m
+ GROUP BY m.session_id
+ ORDER BY MAX(m.created_at) DESC
+ LIMIT ?
+ """,
+ (limit,),
+ )
+ rows = c.fetchall()
+ self._release_conn(conn)
+
+ return [
+ {
+ "session_id": r[0],
+ "started_at": r[1],
+ "last_message_at": r[2],
+ "message_count": r[3],
+ "first_message": (r[4] or "")[:80] if len(r) > 4 else "",
+ }
+ for r in rows
+ ]
+
+ # ── Preferences ─────────────────────────────────────────────
+
+ # ── Hikmah (Wisdom Patterns) ─────────────────────────────────
+
+ async def store_hikmah(
+ self,
+ pattern: str,
+ context: str,
+ outcome: str,
+ confidence: float = 0.5,
+ source_agent: str = "",
+ ) -> str:
+ """Persist a learned wisdom pattern to the hikmah table."""
+ import uuid as _uuid
+
+ hikmah_id = str(_uuid.uuid4())
+ conn = self._get_conn()
+ c = conn.cursor()
+ # Avoid duplicates: if same pattern+context exists, update confidence
+ c.execute(
+ "SELECT id, applications, confidence FROM hikmah WHERE pattern = ? AND context = ?",
+ (pattern[:500], context[:500]),
+ )
+ existing = c.fetchone()
+ if existing:
+ new_apps = (existing[1] or 0) + 1
+ new_conf = min(1.0, existing[2] + 0.05)
+ c.execute(
+ "UPDATE hikmah SET applications = ?, confidence = ? WHERE id = ?",
+ (new_apps, new_conf, existing[0]),
+ )
+ hikmah_id = existing[0]
+ else:
+ c.execute(
+ """INSERT INTO hikmah (id, pattern, context, outcome, confidence, applications, created_at, source_agent)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
+ (
+ hikmah_id,
+ pattern[:500],
+ context[:500],
+ outcome[:1000],
+ confidence,
+ 1,
+ datetime.now(UTC).isoformat(),
+ source_agent,
+ ),
+ )
+ conn.commit()
+ self._release_conn(conn)
+ return hikmah_id
+
+ async def load_hikmah(self, agent_id: str = "", limit: int = 20) -> list[dict]:
+ """Load learned wisdom patterns from the hikmah table."""
+ conn = self._get_conn()
+ c = conn.cursor()
+ if agent_id:
+ c.execute(
+ """SELECT pattern, context, outcome, confidence, applications
+ FROM hikmah WHERE source_agent = ?
+ ORDER BY confidence DESC, applications DESC LIMIT ?""",
+ (agent_id, limit),
+ )
+ else:
+ c.execute(
+ """SELECT pattern, context, outcome, confidence, applications
+ FROM hikmah ORDER BY confidence DESC, applications DESC LIMIT ?""",
+ (limit,),
+ )
+ rows = c.fetchall()
+ self._release_conn(conn)
+ return [
+ {
+ "pattern": r[0],
+ "context": r[1],
+ "outcome": r[2],
+ "confidence": r[3],
+ "applications": r[4],
+ }
+ for r in rows
+ ]
+
+ async def get_preference(self, key: str, default: str = "") -> str:
+ """Read a persisted preference by key."""
+ conn = self._get_conn()
+ c = conn.cursor()
+ c.execute("SELECT value FROM preferences WHERE key = ?", (key,))
+ row = c.fetchone()
+ self._release_conn(conn)
+ return row[0] if row else default
+
+ async def set_preference(self, key: str, value: str) -> None:
+ """Upsert a persisted preference."""
+ from datetime import datetime, UTC
+
+ conn = self._get_conn()
+ c = conn.cursor()
+ c.execute(
+ """INSERT INTO preferences (key, value, updated_at)
+ VALUES (?, ?, ?)
+ ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at""",
+ (key, value, datetime.now(UTC).isoformat()),
+ )
+ conn.commit()
+ self._release_conn(conn)
+
+ async def get_all_preferences(self) -> dict[str, str]:
+ """Read all persisted preferences as a dict."""
+ conn = self._get_conn()
+ c = conn.cursor()
+ c.execute("SELECT key, value FROM preferences")
+ rows = c.fetchall()
+ self._release_conn(conn)
+ return {r[0]: r[1] for r in rows}
+
async def save_task(
self,
agent_id: str,
diff --git a/backend/memory/knowledge_graph.py b/backend/memory/knowledge_graph.py
index 899827b..709c3e4 100644
--- a/backend/memory/knowledge_graph.py
+++ b/backend/memory/knowledge_graph.py
@@ -23,7 +23,7 @@ class KnowledgeGraph:
Stores entities and their relationships.
"""
- def __init__(self, db_path: str = "/tmp/mizan_memory.db"):
+ def __init__(self, db_path: str = "/data/mizan_memory.db"):
self.db_path = db_path
self._init_tables()
diff --git a/backend/memory/lawh_mahfuz.py b/backend/memory/lawh_mahfuz.py
new file mode 100644
index 0000000..9ef6067
--- /dev/null
+++ b/backend/memory/lawh_mahfuz.py
@@ -0,0 +1,260 @@
+"""
+Lawh al-Mahfuz (لَوْح مَحْفُوظ) — The Preserved Tablet
+=========================================================
+
+"Nay, it is a Glorious Quran, inscribed in a Preserved Tablet (Lawh Mahfuz)."
+— Quran 85:21-22
+
+Immutable core memory with triple-checksum integrity verification.
+Once stored, entries CANNOT be modified — only new entries can be added.
+Used for: Fitrah axioms, proven theorems, verified facts, immutable truths.
+
+Integrity mechanism: SHA-256 + CRC-32 + content_length
+Any mismatch on read → corruption detected → entry quarantined.
+"""
+
+import binascii
+import hashlib
+import json
+import logging
+import os
+import sqlite3
+import time
+from dataclasses import dataclass, field
+
+logger = logging.getLogger("mizan.lawh_mahfuz")
+
+
+@dataclass
+class LawhEntry:
+ """An immutable entry in the Preserved Tablet."""
+ key: str
+ content: str
+ source: str
+ stored_at: float
+ sha256: str # SHA-256 of content
+ crc32: str # CRC-32 hex of content
+ length: int # len(content) in bytes
+ certainty: float = 1.0
+ category: str = "general"
+
+ def to_dict(self) -> dict:
+ return {
+ "key": self.key,
+ "content": self.content[:500],
+ "source": self.source,
+ "certainty": self.certainty,
+ "category": self.category,
+ "stored_at": self.stored_at,
+ }
+
+
+def _sha256(text: str) -> str:
+ return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+def _crc32(text: str) -> str:
+ return format(binascii.crc32(text.encode("utf-8")) & 0xFFFFFFFF, "08x")
+
+
+class LawhMahfuz:
+ """
+ Immutable memory store with triple-checksum integrity.
+
+ All entries are stored with SHA-256 + CRC-32 + length.
+ Every read re-verifies all three checksums.
+ Corrupted entries are moved to a quarantine table.
+
+ Usage:
+ lawh = LawhMahfuz()
+ key = lawh.store_immutable("TRUTH:1", "Always speak truth", source="Quran 33:70")
+ lawh.verify_integrity("TRUTH:1") # True if intact
+ entry = lawh.get("TRUTH:1") # None if corrupted
+ """
+
+ def __init__(self, db_path: str = "/data/lawh_mahfuz.db"):
+ self.db_path = db_path
+ # In-memory cache for fast reads
+ self._cache: dict[str, LawhEntry] = {}
+ self._quarantine: set[str] = set()
+ self._init_db()
+ self._load_into_cache()
+
+ def _get_conn(self) -> sqlite3.Connection:
+ return sqlite3.connect(self.db_path, check_same_thread=False)
+
+ def _init_db(self):
+ os.makedirs(os.path.dirname(self.db_path) or ".", exist_ok=True)
+ conn = self._get_conn()
+ try:
+ c = conn.cursor()
+ c.execute("""
+ CREATE TABLE IF NOT EXISTS lawh_entries (
+ key TEXT PRIMARY KEY,
+ content TEXT NOT NULL,
+ source TEXT NOT NULL,
+ stored_at REAL NOT NULL,
+ sha256 TEXT NOT NULL,
+ crc32 TEXT NOT NULL,
+ length INTEGER NOT NULL,
+ certainty REAL DEFAULT 1.0,
+ category TEXT DEFAULT 'general'
+ )
+ """)
+ c.execute("""
+ CREATE TABLE IF NOT EXISTS lawh_quarantine (
+ key TEXT PRIMARY KEY,
+ reason TEXT,
+ quarantined_at REAL
+ )
+ """)
+ conn.commit()
+ finally:
+ conn.close()
+
+ def _load_into_cache(self):
+ """Load all entries into memory cache on startup."""
+ conn = self._get_conn()
+ try:
+ c = conn.cursor()
+ c.execute("SELECT * FROM lawh_entries")
+ for row in c.fetchall():
+ key, content, source, stored_at, sha256, crc32, length, certainty, category = row
+ entry = LawhEntry(
+ key=key, content=content, source=source, stored_at=stored_at,
+ sha256=sha256, crc32=crc32, length=length,
+ certainty=certainty, category=category,
+ )
+ self._cache[key] = entry
+ logger.info("[LAWH] Loaded %d entries from preserved tablet", len(self._cache))
+ finally:
+ conn.close()
+
+ def store_immutable(
+ self,
+ key: str,
+ content: str,
+ source: str = "system",
+ certainty: float = 1.0,
+ category: str = "general",
+ ) -> str:
+ """
+ Store an immutable entry. Once stored, it cannot be modified.
+ Returns the key if successful.
+ Raises ValueError if key already exists (immutability enforcement).
+ """
+ if key in self._cache:
+ logger.debug("[LAWH] Key '%s' already exists — immutability preserved", key)
+ return key # Already stored — idempotent
+
+ sha = _sha256(content)
+ crc = _crc32(content)
+ length = len(content.encode("utf-8"))
+ now = time.time()
+
+ entry = LawhEntry(
+ key=key, content=content, source=source, stored_at=now,
+ sha256=sha, crc32=crc, length=length,
+ certainty=certainty, category=category,
+ )
+
+ conn = self._get_conn()
+ try:
+ c = conn.cursor()
+ c.execute(
+ """INSERT OR IGNORE INTO lawh_entries
+ (key, content, source, stored_at, sha256, crc32, length, certainty, category)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
+ (key, content, source, now, sha, crc, length, certainty, category),
+ )
+ conn.commit()
+ finally:
+ conn.close()
+
+ self._cache[key] = entry
+ logger.debug("[LAWH] Stored '%s' (len=%d sha=%s...)", key, length, sha[:8])
+ return key
+
+ def verify_integrity(self, key: str) -> bool:
+ """
+ Verify all three checksums for an entry.
+ Returns True if intact, False if corrupted.
+ Quarantines corrupted entries.
+ """
+ if key in self._quarantine:
+ return False
+
+ entry = self._cache.get(key)
+ if entry is None:
+ return False
+
+ # Re-compute all three checksums
+ actual_sha = _sha256(entry.content)
+ actual_crc = _crc32(entry.content)
+ actual_len = len(entry.content.encode("utf-8"))
+
+ if actual_sha != entry.sha256:
+ self._quarantine_entry(key, f"SHA-256 mismatch: {actual_sha[:8]}≠{entry.sha256[:8]}")
+ return False
+ if actual_crc != entry.crc32:
+ self._quarantine_entry(key, f"CRC-32 mismatch: {actual_crc}≠{entry.crc32}")
+ return False
+ if actual_len != entry.length:
+ self._quarantine_entry(key, f"Length mismatch: {actual_len}≠{entry.length}")
+ return False
+
+ return True
+
+ def get(self, key: str) -> LawhEntry | None:
+ """
+ Retrieve an entry. Verifies integrity on every read.
+ Returns None if entry doesn't exist or is corrupted.
+ """
+ if key in self._quarantine:
+ logger.warning("[LAWH] Attempted read of quarantined key: %s", key)
+ return None
+ if not self.verify_integrity(key):
+ return None
+ return self._cache.get(key)
+
+ def search(self, query: str, top_k: int = 5) -> list[LawhEntry]:
+ """Search entries by keyword match in content or key."""
+ query_lower = query.lower()
+ results = []
+ for key, entry in self._cache.items():
+ if key in self._quarantine:
+ continue
+ if query_lower in entry.content.lower() or query_lower in key.lower():
+ results.append(entry)
+ # Sort by certainty desc
+ results.sort(key=lambda e: -e.certainty)
+ return results[:top_k]
+
+ def get_by_category(self, category: str) -> list[LawhEntry]:
+ """Get all entries in a category (e.g., 'ethical', 'epistemic')."""
+ return [
+ e for k, e in self._cache.items()
+ if e.category == category and k not in self._quarantine
+ ]
+
+ def _quarantine_entry(self, key: str, reason: str):
+ """Move corrupted entry to quarantine."""
+ self._quarantine.add(key)
+ conn = self._get_conn()
+ try:
+ c = conn.cursor()
+ c.execute(
+ "INSERT OR REPLACE INTO lawh_quarantine (key, reason, quarantined_at) VALUES (?,?,?)",
+ (key, reason, time.time()),
+ )
+ conn.commit()
+ finally:
+ conn.close()
+ logger.error("[LAWH] CORRUPTION DETECTED — quarantined '%s': %s", key, reason)
+
+ def stats(self) -> dict:
+ return {
+ "total_entries": len(self._cache),
+ "quarantined": len(self._quarantine),
+ "categories": list({e.category for e in self._cache.values()}),
+ }
diff --git a/backend/memory/living_memory.py b/backend/memory/living_memory.py
new file mode 100644
index 0000000..5a3a0ff
--- /dev/null
+++ b/backend/memory/living_memory.py
@@ -0,0 +1,687 @@
+"""
+Living Memory System — InsÄn-NisyÄn Architecture
+==================================================
+
+"And remember your Lord when you forget" — Quran 18:24
+"Remember Me and I will remember you" — Quran 2:152
+
+Memory is NOT a database — it's a living organism that decides what to store,
+what to forget, strengthens with use, fades with neglect, transforms over time,
+and recalls differently depending on context.
+
+Implements:
+- NOVELTY_GATE: "Do I already know this?" (θ_identical=0.98, θ_similar=0.85, θ_related=0.5)
+- IMPORTANCE_SCORER: Multi-factor importance scoring (emotional, goal, surprise, causal, trust, rarity)
+- DYNAMIC_RECALL: Context-dependent associative recall with spread activation + emotional modulation
+- DHIKR_MAINTENANCE_DAEMON: Spaced repetition, decay, consolidation, Ṣadr→Dhikr→ʿIlm→Lawḥ promotion
+- Memory 1+1=2 lifecycle: learn once, activate existing, never re-store
+
+Principles:
+ 1. Intelligent forgetting enables intelligent remembering
+ 2. Memory requires active maintenance (dhikr) — unreviewed decays, reviewed strengthens
+ 3. Memory has depth: Ḥifẓ (raw) → ʿIlm (structural) → Fahm (applicable) → Tafaqquh (transformative)
+ 4. Novelty gates storage — only genuinely new information creates new traces
+"""
+
+import hashlib
+import logging
+import math
+import time
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
+
+logger = logging.getLogger("mizan.living_memory")
+
+# Novelty Gate thresholds
+THETA_IDENTICAL = 0.98 # "I already know this exactly"
+THETA_SIMILAR = 0.85 # "I know something like this"
+THETA_RELATED = 0.50 # "This is new but related"
+THETA_WORTH_STORING = 0.3 # Minimum importance to store unrelated info
+
+# Importance scoring weights
+W_EMOTION = 0.20
+W_GOAL = 0.20
+W_SURPRISE = 0.20
+W_CAUSAL = 0.15
+W_TRUST = 0.10
+W_RARITY = 0.15
+
+# Strength / decay parameters
+INITIAL_STRENGTH = 0.30
+DELTA_REINFORCE = 0.12 # strength boost on re-encounter
+DELTA_RETRIEVAL = 0.08 # testing effect: recall strengthens memory
+DELTA_REVIEW = 0.05 # dhikr daemon review boost (diminishing)
+DELTA_DECAY = 0.02 # per-cycle decay rate for unreviewed
+PHI_SPACING = 1.5 # spacing effect exponent
+
+# Consolidation thresholds
+CONSOLIDATE_RECALL_COUNT = 5 # promote Dhikr→ʿIlm after N retrievals
+CONSOLIDATE_CONSISTENCY = 0.7
+PROVE_COUNT = 20 # promote ʿIlm→Lawḥ after N verifications
+FORGET_THRESHOLD = 0.05 # archive if strength drops below
+MIN_STRENGTH = 0.01
+
+# Spread activation parameters
+ALPHA_CONTEXT = 0.3
+ALPHA_EMOTION = 0.25
+ALPHA_GOAL = 0.2
+LAMBDA_RECENCY = 0.001 # recency decay constant
+
+# Working memory capacity (Miller's Law)
+SADR_CAPACITY = 7
+
+# Review intervals
+BASE_REVIEW_INTERVAL = 3600 # 1 hour base
+
+
+class MemoryLevel(Enum):
+ SADR = "sadr" # Working memory (~4-7 items, seconds)
+ DHIKR = "dhikr" # Episodic/active (hours-days)
+ ILM = "ilm" # Semantic/knowledge (weeks-years)
+ LAWH = "lawh" # Immutable core (forever, write-once-read-many)
+
+
+class GateDecision(Enum):
+ STORE_NEW = "store_new"
+ STORE_NEW_WITH_LINKS = "store_new_with_links"
+ UPDATE_EXISTING = "update_existing"
+ ACTIVATE_EXISTING = "activate_existing"
+ IGNORE = "ignore"
+
+
+@dataclass
+class MemoryTrace:
+ """A single memory trace in the living memory system."""
+ trace_id: str
+ content: str
+ content_hash: str # for fast exact-match
+ level: MemoryLevel
+ strength: float
+ importance: float
+ emotional_tag: float # -1.0 to +1.0
+ context_tags: list[str] = field(default_factory=list)
+ links: list[str] = field(default_factory=list) # trace_ids of associated memories
+ recall_count: int = 0
+ activation_count: int = 0
+ created_at: float = field(default_factory=time.time)
+ last_accessed: float = field(default_factory=time.time)
+ optimal_interval: float = BASE_REVIEW_INTERVAL
+ contradiction_count: int = 0
+ proven_count: int = 0
+ mutable: bool = True
+ gist: str = "" # extracted abstract form (for ʿIlm level)
+ source: str = ""
+
+ def age_hours(self) -> float:
+ return (time.time() - self.created_at) / 3600
+
+ def hours_since_access(self) -> float:
+ return (time.time() - self.last_accessed) / 3600
+
+ def review_urgency(self) -> float:
+ """How overdue is this trace for review? >1.0 = overdue."""
+ time_since = time.time() - self.last_accessed
+ return time_since / max(self.optimal_interval, 60)
+
+
+@dataclass
+class GateResult:
+ """Result of the Novelty Gate evaluation."""
+ decision: GateDecision
+ matched_trace: MemoryTrace | None
+ similarity: float
+ importance: float
+ delta_info: str # what's new vs existing
+
+
+@dataclass
+class RecallResult:
+ """A recalled memory with contextual scoring."""
+ trace: MemoryTrace
+ activation: float
+ context_match: float
+ emotional_match: float
+ goal_match: float
+ recency: float
+ reconstructed: str # may differ from original (reconstructive recall)
+
+
+@dataclass
+class MaintenanceReport:
+ """Result of one Dhikr Daemon maintenance cycle."""
+ reviewed: int
+ decayed: int
+ archived: int
+ promoted_to_ilm: int
+ promoted_to_lawh: int
+ cycle_time_ms: float
+
+
+class LivingMemorySystem:
+ """
+ The complete Living Memory architecture.
+
+ 4-level hierarchy: Ṣadr → Dhikr → ʿIlm → Lawḥ
+ Novelty-gated storage, importance-scored encoding,
+ context-dependent recall, and Dhikr maintenance daemon.
+ """
+
+ def __init__(self, masalik=None, dhikr_db=None, lawh=None):
+ # Memory stores by level
+ self.sadr: list[MemoryTrace] = [] # working memory (capacity-limited)
+ self.traces: dict[str, MemoryTrace] = {} # all traces by ID
+ self.archive: list[str] = [] # archived (forgotten) trace IDs
+
+ # External system references (for integration)
+ self._masalik = masalik # MasalikNetwork for spread activation
+ self._dhikr_db = dhikr_db # DhikrMemorySystem for persistence
+ self._lawh = lawh # LawhMahfuz for immutable storage
+
+ self._daemon_cycle = 0
+ self._content_hashes: dict[str, str] = {} # hash → trace_id for fast lookup
+
+ def process_input(
+ self,
+ content: str,
+ emotional_state: float = 0.0,
+ goals: list[str] | None = None,
+ context: str = "",
+ source_trust: float = 0.7,
+ ) -> GateResult:
+ """
+ NOVELTY_GATE + IMPORTANCE_SCORER combined entry point.
+
+ 1. Compute similarity to existing memories
+ 2. Decide: STORE_NEW | UPDATE | ACTIVATE | IGNORE
+ 3. Score importance
+ 4. Execute storage or activation
+ """
+ goals = goals or []
+
+ # Fast exact-match via content hash
+ content_hash = self._hash_content(content)
+ if content_hash in self._content_hashes:
+ existing_id = self._content_hashes[content_hash]
+ if existing_id in self.traces:
+ existing = self.traces[existing_id]
+ return self._activate_existing(existing, "Exact hash match")
+
+ # Similarity search against all traces
+ best_match, max_similarity = self._find_best_match(content)
+
+ # Importance scoring
+ importance = self._score_importance(
+ content, emotional_state, goals, context, source_trust
+ )
+
+ # Decision tree (Algorithm: NOVELTY_GATE)
+ if max_similarity > THETA_IDENTICAL and best_match:
+ # "I already know this exactly" → like seeing 1+1=2 again
+ return self._activate_existing(best_match, "Near-identical match")
+
+ elif max_similarity > THETA_SIMILAR and best_match:
+ # "I know something like this" → enrich existing
+ delta = self._extract_novel_parts(content, best_match)
+ return self._update_existing(best_match, delta, importance)
+
+ elif max_similarity > THETA_RELATED and best_match:
+ # "New but related" → store with links
+ trace = self._create_trace(
+ content, content_hash, importance, emotional_state, context
+ )
+ trace.links.append(best_match.trace_id)
+ self._store_trace(trace)
+ return GateResult(
+ decision=GateDecision.STORE_NEW_WITH_LINKS,
+ matched_trace=trace,
+ similarity=max_similarity,
+ importance=importance,
+ delta_info=f"Linked to '{best_match.content[:40]}'",
+ )
+
+ else:
+ # Completely new
+ if importance > THETA_WORTH_STORING:
+ trace = self._create_trace(
+ content, content_hash, importance, emotional_state, context
+ )
+ self._store_trace(trace)
+ return GateResult(
+ decision=GateDecision.STORE_NEW,
+ matched_trace=trace,
+ similarity=max_similarity,
+ importance=importance,
+ delta_info="Novel information stored",
+ )
+ else:
+ return GateResult(
+ decision=GateDecision.IGNORE,
+ matched_trace=None,
+ similarity=max_similarity,
+ importance=importance,
+ delta_info="Not novel enough and not important enough",
+ )
+
+ def recall(
+ self,
+ query: str,
+ context: str = "",
+ emotional_state: float = 0.0,
+ goals: list[str] | None = None,
+ top_k: int = 10,
+ ) -> list[RecallResult]:
+ """
+ DYNAMIC_RECALL: Context-dependent associative recall.
+
+ 1. Spread activation from query
+ 2. Context modulation
+ 3. Emotional modulation (mood-congruent memory)
+ 4. Goal modulation
+ 5. Recency weighting
+ 6. Reconstruction
+ 7. Post-retrieval update (reconsolidation)
+ """
+ goals = goals or []
+ if not self.traces:
+ return []
+
+ # Step 1: Spread activation
+ activations: dict[str, float] = {}
+
+ # Wave 1: Direct matches
+ query_words = set(query.lower().split())
+ for tid, trace in self.traces.items():
+ if tid in self.archive:
+ continue
+ sim = self._text_similarity(query, trace.content)
+ if sim > 0.1:
+ activations[tid] = sim * trace.strength
+
+ # Wave 2: Spread through links (1-hop)
+ wave2 = {}
+ for tid, activation in list(activations.items()):
+ trace = self.traces[tid]
+ for linked_id in trace.links:
+ if linked_id in self.traces and linked_id not in self.archive:
+ link_activation = activation * 0.6
+ wave2[linked_id] = max(
+ wave2.get(linked_id, 0), link_activation
+ )
+ for tid, act in wave2.items():
+ activations[tid] = max(activations.get(tid, 0), act)
+
+ # Wave 3: Spread through links (2-hop)
+ wave3 = {}
+ for tid, activation in wave2.items():
+ trace = self.traces[tid]
+ for linked_id in trace.links:
+ if linked_id in self.traces and linked_id not in self.archive:
+ wave3[linked_id] = max(
+ wave3.get(linked_id, 0), activation * 0.3
+ )
+ for tid, act in wave3.items():
+ activations[tid] = max(activations.get(tid, 0), act)
+
+ if not activations:
+ return []
+
+ # Steps 2-5: Modulation
+ results = []
+ for tid, base_activation in activations.items():
+ trace = self.traces[tid]
+
+ # Context modulation
+ context_match = self._text_similarity(context, " ".join(trace.context_tags)) if context else 0.0
+ modulated = base_activation * (1 + ALPHA_CONTEXT * context_match)
+
+ # Emotional modulation (mood-congruent)
+ emotional_match = 1.0 - abs(emotional_state - trace.emotional_tag)
+ modulated *= (1 + ALPHA_EMOTION * emotional_match)
+
+ # Goal modulation
+ goal_match = 0.0
+ if goals:
+ goal_match = max(
+ self._text_similarity(g, trace.content) for g in goals
+ )
+ modulated *= (1 + ALPHA_GOAL * goal_match)
+
+ # Recency weighting
+ recency = math.exp(-LAMBDA_RECENCY * trace.hours_since_access())
+ modulated *= recency
+
+ results.append(RecallResult(
+ trace=trace,
+ activation=modulated,
+ context_match=context_match,
+ emotional_match=emotional_match,
+ goal_match=goal_match,
+ recency=recency,
+ reconstructed=trace.content, # simplified: no gap-filling yet
+ ))
+
+ # Sort by activation, take top-k
+ results.sort(key=lambda r: r.activation, reverse=True)
+ results = results[:top_k]
+
+ # Step 7: Post-retrieval update (reconsolidation)
+ for result in results:
+ trace = result.trace
+ trace.recall_count += 1
+ trace.strength = min(1.0, trace.strength + DELTA_RETRIEVAL)
+ trace.last_accessed = time.time()
+ if context:
+ if context not in trace.context_tags:
+ trace.context_tags.append(context)
+ if len(trace.context_tags) > 20:
+ trace.context_tags.pop(0)
+
+ return results
+
+ def run_maintenance(self) -> MaintenanceReport:
+ """
+ DHIKR_MAINTENANCE_DAEMON: One maintenance cycle.
+
+ 1. Compute review priority
+ 2. Review top-priority memories
+ 3. Decay unreviewed memories
+ 4. Consolidate: Dhikr→ʿIlm→Lawḥ promotions
+ 5. Archive forgotten traces
+ """
+ self._daemon_cycle += 1
+ start = time.monotonic()
+
+ reviewed = 0
+ decayed = 0
+ archived = 0
+ promoted_ilm = 0
+ promoted_lawh = 0
+
+ # Step 1-2: Review overdue traces
+ overdue = [
+ t for t in self.traces.values()
+ if t.mutable and t.trace_id not in self.archive and t.review_urgency() > 1.0
+ ]
+ overdue.sort(key=lambda t: t.review_urgency() * t.importance, reverse=True)
+
+ for trace in overdue[:20]: # max 20 reviews per cycle
+ trace.strength = min(1.0,
+ trace.strength + DELTA_REVIEW / max(trace.recall_count, 1)
+ )
+ trace.last_accessed = time.time()
+ # Extend optimal interval (spaced repetition)
+ trace.optimal_interval *= (1 + trace.strength) ** PHI_SPACING
+ reviewed += 1
+
+ # Step 3: Decay unreviewed traces
+ for trace in self.traces.values():
+ if trace.trace_id in self.archive or not trace.mutable:
+ continue
+ if trace.review_urgency() < 1.0:
+ continue # not overdue, no decay
+ if trace.importance < 0.5: # only decay low-importance
+ trace.strength *= (1 - DELTA_DECAY)
+ decayed += 1
+
+ # Archive if too weak
+ if trace.strength < FORGET_THRESHOLD:
+ self.archive.append(trace.trace_id)
+ archived += 1
+
+ # Step 4: Consolidation promotions
+ for trace in list(self.traces.values()):
+ if trace.trace_id in self.archive:
+ continue
+
+ # Dhikr → ʿIlm: frequently recalled + consistent
+ if (
+ trace.level == MemoryLevel.DHIKR
+ and trace.recall_count >= CONSOLIDATE_RECALL_COUNT
+ and trace.strength >= CONSOLIDATE_CONSISTENCY
+ ):
+ trace.level = MemoryLevel.ILM
+ trace.gist = self._extract_gist(trace)
+ promoted_ilm += 1
+
+ # ʿIlm → Lawḥ: proven beyond doubt
+ elif (
+ trace.level == MemoryLevel.ILM
+ and trace.proven_count >= PROVE_COUNT
+ and trace.contradiction_count == 0
+ ):
+ trace.level = MemoryLevel.LAWH
+ trace.mutable = False
+ trace.strength = 1.0
+ # Store in external LawhMahfuz if available
+ if self._lawh:
+ try:
+ self._lawh.store_immutable(
+ key=f"LM:{trace.trace_id}",
+ content=trace.gist or trace.content,
+ source="living_memory_promotion",
+ )
+ except Exception:
+ pass
+ promoted_lawh += 1
+
+ elapsed_ms = (time.monotonic() - start) * 1000
+ logger.debug(
+ "[DHIKR-DAEMON] cycle=%d reviewed=%d decayed=%d archived=%d ilm=%d lawh=%d",
+ self._daemon_cycle, reviewed, decayed, archived, promoted_ilm, promoted_lawh,
+ )
+
+ return MaintenanceReport(
+ reviewed=reviewed,
+ decayed=decayed,
+ archived=archived,
+ promoted_to_ilm=promoted_ilm,
+ promoted_to_lawh=promoted_lawh,
+ cycle_time_ms=round(elapsed_ms, 2),
+ )
+
+ def _score_importance(
+ self,
+ content: str,
+ emotional_state: float,
+ goals: list[str],
+ context: str,
+ source_trust: float,
+ ) -> float:
+ """
+ IMPORTANCE_SCORER: Multi-factor scoring.
+
+ importance = Σ weights · factors:
+ emotional_weight, goal_relevance, prediction_error (surprise),
+ causal_impact, source_trust, rarity
+ """
+ content_lower = content.lower()
+
+ # Factor 1: Emotional weight (|affect|)
+ emotional_weight = abs(emotional_state)
+
+ # Factor 2: Goal relevance
+ goal_relevance = 0.0
+ if goals:
+ goal_relevance = max(
+ self._text_similarity(g, content) for g in goals
+ )
+
+ # Factor 3: Prediction error (surprise) — novelty proxy
+ _, max_sim = self._find_best_match(content)
+ prediction_error = 1.0 - max_sim
+
+ # Factor 4: Causal significance (keyword heuristic)
+ causal_keywords = {"because", "caused", "leads to", "therefore", "result", "effect"}
+ causal_impact = min(1.0,
+ sum(1 for k in causal_keywords if k in content_lower) / 3
+ )
+
+ # Factor 5: Source trust
+ trust = min(1.0, max(0.0, source_trust))
+
+ # Factor 6: Rarity (inverse frequency of similar content)
+ similar_count = sum(
+ 1 for t in self.traces.values()
+ if self._text_similarity(content, t.content) > THETA_RELATED
+ )
+ rarity = 1.0 / (1.0 + similar_count)
+
+ # Weighted combination → sigmoid
+ raw = (
+ W_EMOTION * emotional_weight
+ + W_GOAL * goal_relevance
+ + W_SURPRISE * prediction_error
+ + W_CAUSAL * causal_impact
+ + W_TRUST * trust
+ + W_RARITY * rarity
+ )
+ return self._sigmoid(raw * 3) # scale into sigmoid range
+
+ def _activate_existing(
+ self, trace: MemoryTrace, reason: str
+ ) -> GateResult:
+ """Activate an existing trace: no new storage, just strengthen."""
+ trace.activation_count += 1
+ trace.last_accessed = time.time()
+ trace.strength = min(1.0, trace.strength + DELTA_REINFORCE)
+ trace.proven_count += 1
+
+ return GateResult(
+ decision=GateDecision.ACTIVATE_EXISTING,
+ matched_trace=trace,
+ similarity=1.0,
+ importance=trace.importance,
+ delta_info=f"{reason} — activated (count={trace.activation_count})",
+ )
+
+ def _update_existing(
+ self, trace: MemoryTrace, delta: str, importance: float
+ ) -> GateResult:
+ """Update an existing trace with novel information delta."""
+ if delta:
+ trace.content += f" | UPDATE: {delta[:200]}"
+ trace.activation_count += 1
+ trace.last_accessed = time.time()
+ trace.strength = min(1.0, trace.strength + DELTA_REINFORCE * 0.7)
+ trace.importance = max(trace.importance, importance)
+
+ return GateResult(
+ decision=GateDecision.UPDATE_EXISTING,
+ matched_trace=trace,
+ similarity=0.9,
+ importance=importance,
+ delta_info=f"Enriched with: {delta[:80]}",
+ )
+
+ def _create_trace(
+ self,
+ content: str,
+ content_hash: str,
+ importance: float,
+ emotional_tag: float,
+ context: str,
+ ) -> MemoryTrace:
+ trace_id = hashlib.md5(
+ f"{content[:100]}:{time.time()}".encode()
+ ).hexdigest()[:12]
+
+ # New traces start in Ṣadr (working memory)
+ trace = MemoryTrace(
+ trace_id=trace_id,
+ content=content,
+ content_hash=content_hash,
+ level=MemoryLevel.SADR,
+ strength=INITIAL_STRENGTH,
+ importance=importance,
+ emotional_tag=emotional_tag,
+ context_tags=[context] if context else [],
+ source="living_memory",
+ )
+ return trace
+
+ def _store_trace(self, trace: MemoryTrace) -> None:
+ """Store trace and manage Ṣadr capacity."""
+ self.traces[trace.trace_id] = trace
+ self._content_hashes[trace.content_hash] = trace.trace_id
+
+ # Ṣadr → Dhikr promotion when working memory full
+ self.sadr.append(trace)
+ if len(self.sadr) > SADR_CAPACITY:
+ # Oldest items move to Dhikr level
+ evicted = self.sadr.pop(0)
+ if evicted.trace_id in self.traces:
+ evicted.level = MemoryLevel.DHIKR
+
+ # Persist to external Masalik if available
+ if self._masalik:
+ try:
+ self._masalik.encode(trace.content, importance=trace.importance)
+ except Exception:
+ pass
+
+ def _find_best_match(self, content: str) -> tuple[MemoryTrace | None, float]:
+ """Find the most similar existing trace."""
+ best = None
+ best_sim = 0.0
+ for trace in self.traces.values():
+ if trace.trace_id in self.archive:
+ continue
+ sim = self._text_similarity(content, trace.content)
+ if sim > best_sim:
+ best_sim = sim
+ best = trace
+ return best, best_sim
+
+ def _extract_novel_parts(self, new_content: str, existing: MemoryTrace) -> str:
+ """Extract what's genuinely new in content vs existing trace."""
+ new_words = set(new_content.lower().split())
+ existing_words = set(existing.content.lower().split())
+ novel = new_words - existing_words
+ if novel:
+ return " ".join(sorted(novel)[:15])
+ return ""
+
+ def _extract_gist(self, trace: MemoryTrace) -> str:
+ """Extract abstract gist from a trace (for ʿIlm promotion)."""
+ words = trace.content.split()
+ # Keep first sentence or first 15 words
+ gist_words = words[:15]
+ return " ".join(gist_words)
+
+ @staticmethod
+ def _text_similarity(text_a: str, text_b: str) -> float:
+ """Jaccard similarity on word sets."""
+ if not text_a or not text_b:
+ return 0.0
+ words_a = set(text_a.lower().split())
+ words_b = set(text_b.lower().split())
+ if not words_a or not words_b:
+ return 0.0
+ intersection = len(words_a & words_b)
+ union = len(words_a | words_b)
+ return intersection / union if union > 0 else 0.0
+
+ @staticmethod
+ def _hash_content(content: str) -> str:
+ return hashlib.sha256(content.strip().lower().encode()).hexdigest()[:16]
+
+ @staticmethod
+ def _sigmoid(x: float) -> float:
+ return 1.0 / (1.0 + math.exp(-x))
+
+ def get_level_counts(self) -> dict[str, int]:
+ counts = {level.value: 0 for level in MemoryLevel}
+ for trace in self.traces.values():
+ if trace.trace_id not in self.archive:
+ counts[trace.level.value] += 1
+ counts["archived"] = len(self.archive)
+ return counts
+
+ def to_dict(self) -> dict:
+ return {
+ "total_traces": len(self.traces),
+ "levels": self.get_level_counts(),
+ "sadr_capacity": f"{len(self.sadr)}/{SADR_CAPACITY}",
+ "daemon_cycles": self._daemon_cycle,
+ }
diff --git a/backend/memory/masalik.py b/backend/memory/masalik.py
index d70e779..8242dfd 100644
--- a/backend/memory/masalik.py
+++ b/backend/memory/masalik.py
@@ -21,8 +21,10 @@
LAWH (لوح): The preserved network — nothing duplicated, everything in place
"""
+import json
import logging
import math
+import os
import re
import time
from collections import defaultdict
@@ -309,17 +311,36 @@ class MasalikNetwork:
- Heavily-used pathways become permanent wisdom (Hikmah)
- Some pathways are innate/pre-wired (Fitrah)
+ Persistence: Network state is saved to disk periodically and on shutdown.
+ On startup, the previously saved state is restored so memory survives restarts.
+
"And We have certainly made the Quran easy for remembrance (Dhikr).
So is there any who will remember?" — 54:17
"""
- def __init__(self):
+ # Default persistence path (inside data/ volume in Docker)
+ DEFAULT_PERSIST_PATH = "/data/masalik_network.json"
+
+ def __init__(self, persist_path: str | None = None):
self.concepts: dict[str, Mafhum] = {}
self.pathways: dict[str, Silah] = {}
self._activation_history: list[set[str]] = []
self._last_decay: float = time.time()
- self._init_fitrah()
+ # Use environment DB_PATH directory if available, else fallback
+ if persist_path:
+ self._persist_path = persist_path
+ else:
+ db_dir = os.path.dirname(os.environ.get("DB_PATH", "/data/mizan_memory.db"))
+ self._persist_path = os.path.join(db_dir, "masalik_network.json")
+
+ self._encode_count_since_save = 0
+ self._save_every_n = 10 # Auto-save every N encodes
+
+ # Try to restore from disk first
+ restored = self._load_from_disk()
+ if not restored:
+ self._init_fitrah()
def _init_fitrah(self):
"""
@@ -438,6 +459,9 @@ def encode(self, text: str, importance: float = 0.5) -> dict:
if len(self._activation_history) > 100:
self._activation_history = self._activation_history[-50:]
+ # Auto-save periodically to persist across restarts
+ self._maybe_auto_save()
+
return {
"encoded": len(concepts),
"concepts": concepts,
@@ -688,4 +712,110 @@ def stats(self) -> dict:
"hikmah_pathways": hikmah_pathways,
"hikmah_concepts": hikmah_concepts,
"avg_pathway_weight": round(avg_weight, 3),
+ "persisted": self._persist_path,
}
+
+ # ─── PERSISTENCE: Save/Load to Disk ──────────────────────────────────
+
+ def save_to_disk(self) -> bool:
+ """
+ Persist the neural network to disk as JSON.
+ Called periodically and on shutdown to survive restarts.
+ """
+ try:
+ os.makedirs(os.path.dirname(self._persist_path) or ".", exist_ok=True)
+
+ state = {
+ "version": 1,
+ "saved_at": time.time(),
+ "concepts": {
+ cid: {
+ "activation": c.activation,
+ "resting_level": c.resting_level,
+ "last_activated": c.last_activated,
+ "total_activations": c.total_activations,
+ "is_fitrah": c.is_fitrah,
+ }
+ for cid, c in self.concepts.items()
+ },
+ "pathways": {
+ key: {
+ "source": p.source,
+ "target": p.target,
+ "weight": p.weight,
+ "pathway_type": p.pathway_type,
+ "last_activated": p.last_activated,
+ "co_activations": p.co_activations,
+ }
+ for key, p in self.pathways.items()
+ },
+ }
+
+ # Write atomically (write to temp, then rename)
+ tmp_path = self._persist_path + ".tmp"
+ with open(tmp_path, "w") as f:
+ json.dump(state, f)
+ os.replace(tmp_path, self._persist_path)
+
+ logger.info(
+ "Masalik saved: %d concepts, %d pathways → %s",
+ len(self.concepts),
+ len(self.pathways),
+ self._persist_path,
+ )
+ return True
+ except Exception as e:
+ logger.error("Masalik save failed: %s", e)
+ return False
+
+ def _load_from_disk(self) -> bool:
+ """
+ Restore neural network state from disk.
+ Returns True if successfully loaded, False if no saved state.
+ """
+ if not os.path.exists(self._persist_path):
+ return False
+
+ try:
+ with open(self._persist_path) as f:
+ state = json.load(f)
+
+ # Restore concepts
+ for cid, cdata in state.get("concepts", {}).items():
+ self.concepts[cid] = Mafhum(
+ id=cid,
+ activation=cdata.get("activation", 0.0),
+ resting_level=cdata.get("resting_level", 0.0),
+ last_activated=cdata.get("last_activated", 0.0),
+ total_activations=cdata.get("total_activations", 0),
+ is_fitrah=cdata.get("is_fitrah", False),
+ )
+
+ # Restore pathways
+ for key, pdata in state.get("pathways", {}).items():
+ self.pathways[key] = Silah(
+ source=pdata["source"],
+ target=pdata["target"],
+ weight=pdata.get("weight", 0.1),
+ pathway_type=pdata.get("pathway_type", "association"),
+ last_activated=pdata.get("last_activated", 0.0),
+ co_activations=pdata.get("co_activations", 0),
+ )
+
+ logger.info(
+ "Masalik restored: %d concepts, %d pathways from %s",
+ len(self.concepts),
+ len(self.pathways),
+ self._persist_path,
+ )
+ return True
+ except Exception as e:
+ logger.error("Masalik load failed: %s — starting fresh", e)
+ return False
+
+ def _maybe_auto_save(self):
+ """Auto-save after N encode operations."""
+ self._encode_count_since_save += 1
+ if self._encode_count_since_save >= self._save_every_n:
+ self._encode_count_since_save = 0
+ self.save_to_disk()
diff --git a/backend/memory/memory_pyramid.py b/backend/memory/memory_pyramid.py
new file mode 100644
index 0000000..f60f660
--- /dev/null
+++ b/backend/memory/memory_pyramid.py
@@ -0,0 +1,251 @@
+"""
+Memory Pyramid (هرم الذاكرة) — Unified 5-Layer Memory Query
+=============================================================
+
+"And We have certainly created man and We know what his soul (nafs)
+ whispers to him, and We are closer to him than his jugular vein." — Quran 50:16
+
+Unifies all five memory systems into a single query interface.
+Each layer contributes what it does best:
+
+ Layer 1 — Masalik : Spreading activation → associative concepts
+ Layer 2 — DhikrMemory : Keyword SQL search → episodic/semantic records
+ Layer 3 — VectorStore : Semantic embedding search (if available)
+ Layer 4 — KnowledgeGraph: Entity + relationship lookup
+ Layer 5 — LawhMahfuz : Immutable fact lookup
+
+Results are merged, deduplicated, and ranked by (relevance × certainty × recency).
+"""
+
+import logging
+import time
+from dataclasses import dataclass, field
+
+logger = logging.getLogger("mizan.memory_pyramid")
+
+
+@dataclass
+class MemoryHit:
+ """A single result from any memory layer."""
+ content: str
+ source_layer: str # "masalik" | "dhikr" | "vector" | "graph" | "lawh"
+ relevance: float # 0-1 (how well it matches query)
+ certainty: float # 0-1 (how reliable the memory is)
+ recency_score: float # 0-1 (1 = very recent)
+ tags: list[str] = field(default_factory=list)
+ metadata: dict = field(default_factory=dict)
+
+ @property
+ def rank_score(self) -> float:
+ """Combined ranking score: relevance × certainty × recency."""
+ return self.relevance * self.certainty * (0.5 + 0.5 * self.recency_score)
+
+ def to_dict(self) -> dict:
+ return {
+ "content": self.content[:400],
+ "source": self.source_layer,
+ "relevance": round(self.relevance, 3),
+ "certainty": round(self.certainty, 3),
+ "rank": round(self.rank_score, 3),
+ "tags": self.tags[:5],
+ }
+
+
+def _recency_score(created_at_ts: float, now: float = None) -> float:
+ """Convert timestamp to 0-1 recency score (1 = just now, 0 = very old)."""
+ now = now or time.time()
+ age_hours = (now - created_at_ts) / 3600
+ # Half-life of 24 hours → score ~0.5 at 1 day old
+ return max(0.0, min(1.0, 0.5 ** (age_hours / 24)))
+
+
+class MemoryPyramid:
+ """
+ Unified query across all five memory systems.
+
+ Usage:
+ pyramid = MemoryPyramid(dhikr, masalik, lawh_mahfuz, vector_store, knowledge_graph)
+ hits = pyramid.query("zaheer khan", top_k=10)
+ for hit in hits:
+ print(f"[{hit.source_layer}] {hit.content[:100]} (rank={hit.rank_score:.2f})")
+ """
+
+ def __init__(
+ self,
+ dhikr=None,
+ masalik=None,
+ lawh_mahfuz=None,
+ vector_store=None,
+ knowledge_graph=None,
+ ):
+ self.dhikr = dhikr
+ self.masalik = masalik
+ self.lawh_mahfuz = lawh_mahfuz
+ self.vector_store = vector_store
+ self.knowledge_graph = knowledge_graph
+
+ def query(self, text: str, top_k: int = 10) -> list[MemoryHit]:
+ """
+ Query all available layers and return merged, ranked results.
+ Layers that are unavailable are silently skipped.
+ """
+ hits: list[MemoryHit] = []
+ now = time.time()
+
+ # ── Layer 1: Masalik (neural pathway spreading activation) ──
+ if self.masalik:
+ try:
+ pathway_results = self.masalik.recall(text, top_k=top_k)
+ for concept, activation in pathway_results:
+ node = self.masalik.concepts.get(concept)
+ certainty = node.resting_level if node else 0.3
+ hits.append(MemoryHit(
+ content=f"Associated concept: {concept}",
+ source_layer="masalik",
+ relevance=float(activation),
+ certainty=certainty,
+ recency_score=_recency_score(
+ node.last_activated if node else now - 3600, now
+ ),
+ tags=["concept", "association"],
+ ))
+ except Exception as e:
+ logger.debug("[PYRAMID] Masalik query failed: %s", e)
+
+ # ── Layer 2: DhikrMemory (SQL keyword search) ──
+ if self.dhikr:
+ try:
+ import asyncio
+ loop = asyncio.get_event_loop()
+ if loop.is_running():
+ # Use sync fallback if we're inside async context
+ memories = self._dhikr_sync_recall(text, top_k)
+ else:
+ memories = loop.run_until_complete(
+ self.dhikr.recall(text, limit=top_k)
+ )
+ for mem in memories:
+ content_str = str(mem.content)
+ recency = _recency_score(mem.recency.timestamp(), now)
+ hits.append(MemoryHit(
+ content=content_str,
+ source_layer="dhikr",
+ relevance=float(mem.importance),
+ certainty=min(0.9, 0.3 + mem.access_count * 0.05),
+ recency_score=recency,
+ tags=mem.tags or [],
+ metadata={"type": mem.memory_type},
+ ))
+ except Exception as e:
+ logger.debug("[PYRAMID] Dhikr query failed: %s", e)
+
+ # ── Layer 3: VectorStore (semantic embedding search) ──
+ if self.vector_store and getattr(self.vector_store, "_available", False):
+ try:
+ vector_results = self.vector_store.search(text, top_k=top_k)
+ for result in vector_results:
+ hits.append(MemoryHit(
+ content=result.get("content", ""),
+ source_layer="vector",
+ relevance=float(result.get("score", 0.5)),
+ certainty=0.7, # Vector search has good precision
+ recency_score=0.5, # Unknown recency
+ tags=result.get("tags", []),
+ ))
+ except Exception as e:
+ logger.debug("[PYRAMID] VectorStore query failed: %s", e)
+
+ # ── Layer 4: KnowledgeGraph (entity + relationship lookup) ──
+ if self.knowledge_graph:
+ try:
+ entities = self.knowledge_graph.search_entities(text, limit=top_k)
+ for entity in entities:
+ props = entity.get("properties", {})
+ content = f"{entity.get('name', '')} ({entity.get('type', 'entity')})"
+ if props:
+ content += f": {list(props.items())[:3]}"
+ hits.append(MemoryHit(
+ content=content,
+ source_layer="graph",
+ relevance=0.7, # Entity match is high relevance
+ certainty=0.6,
+ recency_score=0.4,
+ tags=["entity", entity.get("type", "concept")],
+ ))
+ except Exception as e:
+ logger.debug("[PYRAMID] KnowledgeGraph query failed: %s", e)
+
+ # ── Layer 5: LawhMahfuz (immutable facts) ──
+ if self.lawh_mahfuz:
+ try:
+ lawh_results = self.lawh_mahfuz.search(text, top_k=top_k)
+ for entry in lawh_results:
+ hits.append(MemoryHit(
+ content=entry.content,
+ source_layer="lawh",
+ relevance=0.9, # Immutable facts are highly reliable
+ certainty=entry.certainty,
+ recency_score=1.0, # Immutable facts never decay
+ tags=["immutable", entry.category],
+ metadata={"source": entry.source},
+ ))
+ except Exception as e:
+ logger.debug("[PYRAMID] LawhMahfuz query failed: %s", e)
+
+ # ── Merge, deduplicate, rank ──
+ deduplicated = self._deduplicate(hits)
+ deduplicated.sort(key=lambda h: -h.rank_score)
+ return deduplicated[:top_k]
+
+ def _deduplicate(self, hits: list[MemoryHit]) -> list[MemoryHit]:
+ """Remove near-duplicate hits (same content prefix from different layers)."""
+ seen_prefixes: set[str] = set()
+ unique: list[MemoryHit] = []
+ for hit in hits:
+ prefix = hit.content[:80].lower().strip()
+ if prefix not in seen_prefixes:
+ seen_prefixes.add(prefix)
+ unique.append(hit)
+ return unique
+
+ def _dhikr_sync_recall(self, query: str, limit: int) -> list:
+ """Synchronous fallback for dhikr recall when inside async context."""
+ try:
+ import asyncio
+ import concurrent.futures
+ # Run the coroutine in a separate thread with its own event loop
+ with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
+ future = executor.submit(
+ asyncio.run, self.dhikr.recall(query, limit=limit)
+ )
+ return future.result(timeout=5)
+ except Exception:
+ return []
+
+ def format_for_prompt(self, text: str, top_k: int = 5) -> str:
+ """
+ Query and format results for injection into an agent system prompt.
+ Returns empty string if nothing relevant found.
+ """
+ hits = self.query(text, top_k=top_k)
+ if not hits:
+ return ""
+
+ lines = ["Unified Memory Recall:"]
+ for hit in hits:
+ source = hit.source_layer.upper()
+ content_preview = hit.content[:150].replace("\n", " ")
+ lines.append(f" [{source}] {content_preview}")
+
+ return "\n".join(lines)
+
+ def stats(self) -> dict:
+ """Return availability status of each memory layer."""
+ return {
+ "masalik": self.masalik is not None,
+ "dhikr": self.dhikr is not None,
+ "vector_store": self.vector_store is not None
+ and getattr(self.vector_store, "_available", False),
+ "knowledge_graph": self.knowledge_graph is not None,
+ "lawh_mahfuz": self.lawh_mahfuz is not None,
+ }
diff --git a/backend/providers.py b/backend/providers.py
index 23e2322..33381f3 100644
--- a/backend/providers.py
+++ b/backend/providers.py
@@ -17,7 +17,15 @@
import json
import logging
import os
+import re
+import uuid
from dataclasses import dataclass, field
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+# Load .env so os.getenv() picks up API keys
+load_dotenv(Path(__file__).parent.parent / ".env")
logger = logging.getLogger("mizan.providers")
@@ -61,6 +69,7 @@ def create(
system: str,
messages: list[dict],
tools: list[dict] = None,
+ temperature: float = None,
) -> LLMResponse:
raise NotImplementedError
@@ -88,7 +97,7 @@ def __init__(self, api_key: str):
self._client = anthropic.Anthropic(api_key=api_key)
- def create(self, model, max_tokens, system, messages, tools=None):
+ def create(self, model, max_tokens, system, messages, tools=None, temperature=None):
kwargs = {
"model": model,
"max_tokens": max_tokens,
@@ -97,6 +106,8 @@ def create(self, model, max_tokens, system, messages, tools=None):
}
if tools:
kwargs["tools"] = tools
+ if temperature is not None:
+ kwargs["temperature"] = temperature
response = self._client.messages.create(**kwargs)
@@ -156,6 +167,61 @@ def __init__(
self._client = OpenAI(**kwargs)
self.provider_name = provider_name
+ @staticmethod
+ def _parse_xml_tool_calls(text: str) -> list["ContentBlock"]:
+ """Extract tool calls embedded as XML in model text output.
+
+ Handles formats like:
+
MIZAN's reasoning engine is built on a 7-layer cognitive architecture inspired by Quranic epistemology.
+Every MIZAN agent processes tasks through a 7-layer cognitive pipeline inspired by Quranic psychology. Each layer transforms the task before it reaches the LLM, and post-processes the response.
-Each layer runs in sequence on every agent task:
| Layer | Arabic | Function | ||
|---|---|---|---|---|
| # | Module | Arabic | Purpose | File |
| Sam' | سمع | Sequential temporal input processing | ||
| Basar | بصر | Structural pattern recognition | ||
| Fu'ad | فؤاد | Integration engine combining inputs | ||
| ISM | اسم | Deep semantic root-space representation | ||
| Mizan | ميزان | Epistemic weighting and truth calibration | ||
| 'Aql | عقل | Typed relationship binding engine | ||
| Lawh | لوح | 4-tier hierarchical memory | ||
| Furqan | فرقان | Discrimination and articulation output | ||
| 1 | Fitrah | فطرة | Innate ethical guardrails (NO_HARM, TRUTH, JUSTICE) | core/fitrah.py |
| 2 | Nafs Triad | نفس | Three competing inner voices deliberate on approach | core/nafs_triad.py |
| 3 | Qalb Processor | قلب | Cardiac oscillation — modulates LLM temperature & token limits | core/qalb_processor.py |
| 4 | Fu'ad | فؤاد | Bayesian conviction engine — evidence accumulation | core/fuad.py |
| 5 | Lubb | لبّ | Metacognition — compress, check coherence, detect bias | core/lubb.py |
| 6 | Developmental Gate | أطوار | Progressive capability gating (7 stages) | core/developmental_stages.py |
| 7 | Causal Engine | سببية | Pearl's causal ladder (observe/intervene/counterfactual) | reasoning/causal_engine.py |
Before any tool executes, Fitrah checks three non-negotiable axioms:
+rm -rf, DROP TABLE)On initialization, Fitrah axioms are loaded into QCA Lawh Tier 1 as immutable truths with certainty 1.0.
+ +Three competing "inner voices" deliberate on every task. The winner injects behavioral guidance into the system prompt:
+| Voice | Arabic | Bias | Behavior |
|---|---|---|---|
| Ammara | أمّارة | Drive | Push to act quickly, take risks |
| Lawwama | لوّامة | Caution | Question assumptions, double-check |
| Mutmainna | مطمئنة | Balance | Seek harmony, measured approach |
Voice weights shift with the agent's Nafs level: Level 1-2 Ammara dominates (drive), Level 3-4 Lawwama dominates (critical), Level 5-7 Mutmainna dominates (wise).
+ +The Qalb models a "heartbeat" that alternates between contraction and expansion, directly modulating LLM parameters:
+| State | Arabic | Temperature | Max Tokens | When |
|---|---|---|---|---|
| Qabd | قبض | 0.3 | 2048 | Focused, analytical tasks |
| Bast | بسط | 0.8 | 4096 | Creative, exploratory tasks |
| Khushu | خشوع | 0.2 | 6144 | Deep focus (nafs ≥ 4 + complex task) |
The oscillation phase advances 0.02 per interaction. Emotional overrides: Frustrated → force Qabd; Positive → allow Bast.
+ +Bayesian evidence accumulation across three conviction levels:
+| Level | Requirement | Confidence |
|---|---|---|
| Impression | Single source | 0.0 – 0.50 |
| Belief | 2+ independent sources | 0.50 – 0.78 |
| Conviction | 3+ sources + temporal consistency | 0.78 – 1.0 |
Each source closes 35% of the remaining confidence gap: confidence = 1 - (0.65 ^ source_count)
After response generation, Lubb performs three evaluations:
+Output: quality label (confident | hedged | uncertain), coherence score, and bias flags. If coherence < 0.5, a caveat is appended to the response.
Agents grow through seven stages, each unlocking new capabilities:
+| Level | Stage | Arabic | Max Turns | Key Unlocks |
|---|---|---|---|---|
| 1 | Nutfah | نطفة | 5 | bash, read_file, recall_memory |
| 2 | Alaqah | علقة | 8 | + write_file, http_get |
| 3 | Mudghah | مضغة | 10 | + python_exec, http_post, delegation |
| 4 | Izham | عظام | 12 | + create_agent, causal rung 2 |
| 5 | Lahm | لحم | 15 | All tools, causal rung 3, Lubb metacognition |
| 6 | Nafkh | نفخ | 20 | Full metacognition |
| 7 | Khalq Akhar | خلق آخر | 25 | Full autonomy |
Promotion criteria: sustained success rate + task count + hikmah threshold (managed by DevelopmentalGate.check_upgrade_readiness()).
| Rung | Type | Question | Method |
|---|---|---|---|
| 1 | Observation | "What is?" | P(Y|X) — correlational analysis |
| 2 | Intervention | "What if I do X?" | P(Y|do(X)) — do-calculus |
| 3 | Counterfactual | "What if I had done X instead?" | P(Y_x|X', Y') — alt. history |
| Module | Arabic | Purpose | File |
|---|---|---|---|
| Self-Healing | لوّامة | Immune memory, health metrics, adaptive checkpoints | core/self_healing.py |
| Parallel Agents | — | Concurrent task scheduling + skill transfer | core/parallel_agents.py |
| Imagination | تصوير | Predictive coding — simulate before acting | core/imagination.py |
| Creativity | إبداع | 5 creative modes + fitness landscape math | core/creativity.py |
| Dream Engine | منام | Offline memory consolidation (NREM+REM) | core/dream_engine.py |
| Shura Council | شورى | Multi-agent consultation for complex decisions | agents/shura_council.py |
All memory layers are queried through a unified MemoryPyramid:
| Layer | Module | Purpose |
|---|---|---|
| Dhikr | memory/dhikr.py | Three-tier persistent memory (episodic, semantic, procedural) |
| Masalik | memory/masalik.py | Neural pathways with spreading activation |
| Lawh al-Mahfuz | memory/lawh_mahfuz.py | Immutable memory with triple-checksum integrity (SHA-256 + CRC-32 + length) |
| VectorStore | memory/vector_store.py | Semantic embedding search (ChromaDB) |
| KnowledgeGraph | memory/knowledge_graph.py | Entity-relationship graph (SQLite) |
Unified query: memory/memory_pyramid.py merges, deduplicates, and ranks by relevance × certainty × recency.
Every piece of knowledge is tagged with its certainty level:
Agents evolve through 7 levels based on their performance:
-| Level | Name | Requirements |
|---|---|---|
| 1 | Ammara (Commanding) | Starting level |
| 2 | Lawwama (Self-Reproaching) | 60% success, 25+ tasks |
| 3 | Mulhama (Inspired) | 75% success, 100+ tasks |
| 4 | Mutmainna (Tranquil) | 85% success, 250+ tasks |
| 5 | Radiya (Pleased) | 90% success, 500+ tasks |
| 6 | Mardiyya (Pleasing) | 95% success, 1000+ tasks |
| 7 | Kamila (Complete) | 97% success, 2000+ tasks |
Every assistant response includes a CognitiveBar showing real-time cognitive state:
+Expandable for detailed signals, bias flags, and evidence lists.
POST /api/yaqin/tag Tag knowledge with certainty level
@@ -1134,34 +1238,66 @@ Communication Patterns
Project Structure
mizan/
+-- backend/
-| +-- api/main.py # FastAPI server + all routes
+| +-- api/main.py # FastAPI server + all routes
| +-- agents/
-| | +-- base.py # Base agent + agentic loop
-| | +-- specialized.py # Browser, Research, Code agents
-| | +-- federation.py # Agent-to-agent communication
+| | +-- base.py # Base agent with QALB-7 agentic loop
+| | +-- specialized.py # Browser, Research, Code, SuperAgent (Khalifah)
+| | +-- federation.py # Agent-to-agent communication
+| | +-- shura_council.py # Multi-agent consultation
+| | +-- perpetual_rotation.py # Agent rotation & load balancing
| +-- core/
-| | +-- events.py # Event bus (Nida')
-| | +-- hooks.py # Hook system (Ta'liq)
-| | +-- plugins.py # Plugin manager (Wasilah)
-| | +-- middleware.py # Middleware pipeline (Silsilah)
-| +-- providers.py # LLM providers (Claude/GPT/Ollama/300+)
-| +-- memory/dhikr.py # 3-tier memory
-| +-- security/ # Auth, permissions, sandbox
-| +-- skills/ # Skill registry + builtins
-| +-- gateway/channels/ # Telegram, Discord, Slack, WhatsApp
-| +-- automation/ # Cron jobs + webhooks
-| +-- settings.py # Configuration
-| +-- cli.py # Terminal interface
+| | +-- fitrah.py # Innate ethical guardrails
+| | +-- nafs_triad.py # 3-voice deliberation (Ammara/Lawwama/Mutmainna)
+| | +-- qalb_processor.py # Cardiac oscillation → LLM param modulation
+| | +-- fuad.py # Bayesian conviction formation
+| | +-- lubb.py # Metacognition: compress, cohere, debias
+| | +-- developmental_stages.py # 7-stage capability gating
+| | +-- self_healing.py # Lawwama immune system + health metrics
+| | +-- parallel_agents.py # Concurrent task scheduling
+| | +-- imagination.py # Predictive coding engine
+| | +-- creativity.py # 5 creative modes + landscape math
+| | +-- dream_engine.py # Offline memory consolidation
+| | +-- qalb.py # Emotional intelligence (sentiment)
+| | +-- ruh_engine.py # Energy/vitality management
+| | +-- events.py # Event bus (Nida')
+| | +-- hooks.py # Hook system (Ta'liq)
+| | +-- plugins.py # Plugin manager (Wasilah)
+| | +-- middleware.py # Middleware pipeline (Silsilah)
+| +-- providers.py # LLM providers (Claude/GPT/Ollama/300+)
+| +-- memory/
+| | +-- dhikr.py # 3-tier persistent memory
+| | +-- masalik.py # Neural pathways (spreading activation)
+| | +-- lawh_mahfuz.py # Immutable memory (triple-checksum)
+| | +-- memory_pyramid.py # Unified 5-layer query engine
+| | +-- vector_store.py # Semantic embeddings (ChromaDB)
+| | +-- knowledge_graph.py # Entity-relationship graph
+| | +-- living_memory.py # Adaptive memory lifecycle
+| +-- reasoning/
+| | +-- aql_engine.py # Arabic Query Language reasoning
+| | +-- causal_engine.py # Pearl's 3-rung causal ladder
+| | +-- planner.py # Task planning
+| | +-- context_manager.py # Context window management
+| +-- security/ # Auth, permissions, sandbox
+| +-- skills/ # Skill registry + builtins
+| +-- knowledge/ # Knowledge base management
+| +-- gateway/channels/ # Telegram, Discord, Slack, WhatsApp
+| +-- automation/ # Cron jobs + webhooks
+| +-- settings.py # Configuration
+| +-- cli.py # Terminal interface
+-- frontend/src/
-| +-- App.tsx # Main UI
-| +-- pages/ # Feature pages
-| +-- hooks/ # React hooks
-+-- plugins/ # Your plugins go here!
-+-- docs/ # This documentation
-+-- tests/ # Test suite
-+-- docker/ # Docker configs
-+-- pyproject.toml # Package config
-+-- Makefile # Dev commands
+| +-- App.tsx # Main UI + WebSocket handler
+| +-- components/
+| | +-- ChatMessage.tsx # Chat bubbles + CognitiveBar pills
+| | +-- AgentCard.tsx # Agent card (Nafs + Ruh bars)
+| +-- pages/ # Feature pages
+| +-- hooks/ # React hooks
+| +-- types.ts # TypeScript types (CognitiveMetadata)
++-- plugins/ # Your plugins go here!
++-- docs/ # This documentation
++-- tests/ # Test suite
++-- docker/ # Docker configs
++-- pyproject.toml # Package config
++-- Makefile # Dev commands
- An unexpected error occurred while rendering this page. You can try again or refresh the browser. + An unexpected error occurred while rendering this page. You can + try again or refresh the browser.
{this.state.error && (
@@ -63,9 +103,23 @@ class ErrorBoundary extends Component Your AI team — create and manage intelligent agents
+ Your AI team — create and manage intelligent agents
+ No agents yet Create your first agent to get started, or make sure the backend is running.
+ Create your first agent to get started, or make sure the
+ backend is running.
+ Start a conversation Send a message below. Your AI is ready to help with anything.
+ Start a conversation, ask a question, or try one of these
+ suggestions.
+ Execute tasks through your AI agents
+ Execute tasks through your AI agents
+ What your AI remembers from past interactions
+ What your AI remembers from past interactions
+ No memories yet As you chat, your AI will remember important information. No memories stored yet
+ Add memories manually, or start chatting — your AI will
+ remember important details automatically.
+
+ Feed URLs, PDFs, and YouTube videos to your AI
+
+ Paste a website URL or YouTube link to extract and store its
+ content as knowledge.
+
+ Upload a PDF document to extract its text into knowledge.
+
+ No knowledge ingested yet. Paste a URL or upload a PDF
+ above.
+ Connect external services and APIs
+ Connect external services and APIs
+ No integrations configured Use the Providers page to connect AI models, or Channels for messaging platforms.
+ Use the Providers page to connect AI models, or Channels for
+ messaging platforms.
+ {nameError}
+ Agent type cannot be changed after creation
+ قَدَر (Qadr) — Scheduling and webhooks
+ قَدَر (Qadr) — Scheduling and webhooks
+ mizan serve or make dev
-
')
- // Inline code
- .replace(/`([^`]+)`/g, '$2$1')
- // Bold
- .replace(/\*\*(.+?)\*\*/g, "$1")
- // Italic
- .replace(/\*(.+?)\*/g, "$1")
- // Headers (only at line start)
- .replace(/^### (.+)$/gm, '$1
')
- .replace(/^## (.+)$/gm, '$1
')
- .replace(/^# (.+)$/gm, '$1
')
- // Line breaks
- .replace(/\n/g, "
");
- return html;
-}
+const ChannelsPage = lazy(() => import("./pages/ChannelsPage"));
+const SkillsPage = lazy(() => import("./pages/SkillsPage"));
+const SecurityPage = lazy(() => import("./pages/SecurityPage"));
+const AutomationPage = lazy(() => import("./pages/AutomationPage"));
+const NotebookPage = lazy(() => import("./pages/NotebookPage"));
+const ScannerPage = lazy(() => import("./pages/ScannerPage"));
+const MajlisPage = lazy(() => import("./pages/MajlisPage"));
+const PluginsPage = lazy(() => import("./pages/PluginsPage"));
+const ProvidersPage = lazy(() => import("./pages/ProvidersPage"));
+const DeveloperPage = lazy(() => import("./pages/DeveloperPage"));
+const WelcomePage = lazy(() => import("./pages/WelcomePage"));
+const SettingsPage = lazy(() => import("./pages/SettingsPage"));
// ===== MAIN APP INNER =====
function AppInner() {
@@ -378,7 +152,13 @@ function AppInner() {
return !localStorage.getItem("mizan_setup_complete");
});
- const [activeTab, setActiveTab] = useState("chat");
+ const [activeTab, setActiveTabState] = useState(() => {
+ return localStorage.getItem("mizan_active_tab") || "chat";
+ });
+ const setActiveTab = useCallback((tab: string) => {
+ setActiveTabState(tab);
+ localStorage.setItem("mizan_active_tab", tab);
+ }, []);
const [agents, setAgents] = useStateAgents
-
+ How can I help?
+
+ Task Runner
- Memory
- Knowledge
+
+ Ingest from URL
+
+
+ Upload PDF
+
+
+ Ingested Sources ({knowledgeSources.length})
+
+ {knowledgeSources.length === 0 ? (
+ Integrations
- Create New Agent
+ {/* Mobile bottom nav */}
+
+ {editingAgent ? "Edit Agent" : "Create New Agent"}
+
+
+
+ {block.result}
+
+ mizan serve or{" "}
+ make dev
+
+
+
+ {text}
+ {children};
+ },
+ pre({ children }) {
+ return <>{children}>;
+ },
+ }}
+ >
+ {content}
+
+ Automation
-