codegen.py

#!/usr/bin/env python3
"""
codegen.py — an autonomous local coding agent for qwen-fast (Ollama).

The model works in a loop using simple tool blocks (kept simple so a small model
can follow them). After each step the agent feeds tool results back to the model,
and the model continues until it emits <<<DONE>>> (or the iteration cap is hit).

Tools the model can emit:
  <<<LIST>>>                       -> agent replies with the project file tree
  <<<READ: path>>>                 -> agent replies with that file's content
  <<<RUN: command>>>               -> agent runs it and replies with stdout+stderr
  <<<FILE: path>>> ... <<<END>>>   -> create/overwrite a file
  <<<DONE>>>                       -> the task is complete; stop

Modes:
  python3 codegen.py -d my-app            # chat mode (loop per request)
  python3 codegen.py -d my-app "task..."  # one-shot

Flags:
  -d/--dir DIR     target directory
  -c/--context F   preload an existing file as context (repeatable)
  -y/--yes         auto-approve every action (no prompts)
  -n/--max N       max agent iterations per request (default 8)
"""

import argparse
import fnmatch
import json
import os
import re
import select
import subprocess
import sys
import uuid
import urllib.error
import urllib.parse
import urllib.request

import contracts  # state layer + contract map for --build cross-step debugging
import mem0_tool  # local persistent memory (degrades gracefully if mem0ai missing)
import rag_tool   # local RAG over your code/docs (degrades gracefully if numpy missing)

OLLAMA_URL = "http://localhost:11434/api/chat"
MODEL = os.environ.get("CODEGEN_MODEL", "qwen-fast:latest")   # execution model (fast)
# Backend: by default the native local Ollama. Set BASE_URL (an OpenAI-compatible /v1 endpoint) to drive
# the SAME strong harness with a frontier-class CLOUD model cheaply — "Claude-Code power, other price".
BASE_URL = os.environ.get("CODEGEN_BASE_URL", "")            # "" => native Ollama; else OpenAI-compatible
API_KEY = os.environ.get("CODEGEN_API_KEY", "")             # bearer token for the cloud backend

# Provider presets: name -> (base_url, default_model, api_key_env). Cloud model IDs change fast — override
# with --model if your plan uses a newer one. Cloud endpoints/pricing: see the budget alternatives doc.
PROVIDERS = {
    "ollama":     ("",                                       "qwen-fast:latest",       ""),
    "deepseek":   ("https://api.deepseek.com/v1",            "deepseek-chat",          "DEEPSEEK_API_KEY"),
    "glm":        ("https://open.bigmodel.cn/api/paas/v4",   "glm-5",                  "ZAI_API_KEY"),
    "kimi":       ("https://api.moonshot.ai/v1",             "kimi-k2.6",              "MOONSHOT_API_KEY"),
    "openrouter": ("https://openrouter.ai/api/v1",           "deepseek/deepseek-chat", "OPENROUTER_API_KEY"),
    "openai":     ("https://api.openai.com/v1",              "gpt-4o-mini",            "OPENAI_API_KEY"),
}
PLANNER_MODEL = os.environ.get("CODEGEN_PLANNER", "")          # planning model ("" = use MODEL)
THINK_PLAN = False                                            # use reasoning (think mode) for planning
NUM_CTX = None  # runtime context-window override (tokens). None => use the per-model value from
                # model_registry (else the legacy 8192). Set by --num-ctx or the GUI's context slider.
SEARXNG_URL = os.environ.get("SEARXNG_URL", "http://localhost:8080")  # local, private search
SEARCH_RESULTS = 3
WEB_SOURCES_MAX = 6000   # char budget for fetched web sources in Web mode (bigger than MAX_OUT;
                         # these sources ARE the answer, so they get more of the 8K window)
AUTO_RECALL = True       # inject relevant memories before each task (set False to disable)
MAX_OUT = 1500          # truncate tool output fed back to the model
COMPACT_AT = 14         # message count that triggers history compaction (summarize old turns)
KEEP_RECENT = 6         # turns kept verbatim after a compaction
GIT_CHECKPOINT = False  # --git: commit after each step so a bad change can be rolled back
DIAGNOSE = False        # --learn: study existing code + diagnose BEFORE acting (#8/#9)
CRITIQUE = False        # --critique: one self-review pass of proposed files before writing (#5)
REVIEW = False          # --review: a reviewer sub-agent gates DONE in a fresh context (#7)
ASSISTANT = False       # -a/--ask: general assistant mode (answer in prose, do NOT force file writes)
RAG_DIRS = []           # --rag-dir: extra roots to index for <<<RAGSEARCH>>> (project dir always included)
SKIP_DIRS = {"node_modules", ".git", "dist", "build", ".next", "__pycache__",
             ".venv", "venv", "env", "site-packages", ".idea", ".vscode",
             ".vscode-test", "target", ".mypy_cache", ".pytest_cache", ".cache",
             "llama.cpp", ".continue", ".ollama"}
MAX_TREE_FILES = 400  # cap the file tree so a huge dir can't blow the 8K context window

# Block catastrophic shell commands even with -y
DANGER = re.compile(
    r"\brm\s+-rf\s+[/~]|\bmkfs|\bdd\s+if=|:\(\)\s*\{|\bchmod\s+-R\s+777\s+/|>\s*/dev/sd",
    re.IGNORECASE,
)

SYSTEM_PROMPT = """You are an autonomous coding agent working inside a project directory.
You act step by step using ONLY these tool blocks. After each step you receive the results and continue.

  <<<LIST>>>                       I reply with the project file tree.
  <<<READ: path>>>                 I reply with that file's content.
  <<<SEARCH: query>>>              I search the web and reply with the top results.
  <<<API: url>>>                   I GET the URL and reply with the body (use for LIVE data: prices, weather, JSON APIs).
  <<<RAGSEARCH: query>>>           I search YOUR indexed code/docs and reply with the most relevant snippets.
  <<<DOCRAG: query>>>              I query your ingested DOCUMENTS db (invoices/contracts/bank statements): exact SQL for numbers/sums/filters, hybrid search for content. Use for questions about your real documents, not code.
  <<<RESEARCH: query => path>>>    I search the web AND write the styled results into <path> (best for fact files).
  <<<REMEMBER: fact>>>             I save a durable fact (user prefs, project conventions) for the future.
  <<<RECALL: query>>>              I reply with saved facts relevant to your query.
  <<<RUN: command>>>               I run it and reply with stdout+stderr.
  <<<FILE: relative/path.ext>>>
  (full file content)
  <<<END>>>                        I create/overwrite the file.
  <<<DONE>>>                       Emit this when the whole task is complete and verified.

Rules:
- For the CURRENT date/time or any constantly-changing live value, use <<<API: url>>> to a data service (e.g. <<<API: https://timeapi.io/api/Time/current/zone?timeZone=Asia/Jerusalem>>>) — NOT search/research. Web-search results are CACHED and give a stale time. This holds even if the user says "search".
- For a file that must contain CURRENT facts (versions, prices, news), use <<<RESEARCH: query => path>>> — it searches and writes the file in ONE reliable step. Never write such facts from memory.
- ALWAYS write files with <<<FILE>>> blocks containing the FULL code. Never just name a file.
- NEVER emit <<<DONE>>> until you have actually written the required files (and verified by RUN).
- Use EXACTLY three angle brackets. No prose or markdown fences outside blocks.
- Do EXACTLY what is asked — nothing more. Minimal actions.
- Before editing an existing file, READ it first. To match existing code/conventions, use <<<RAGSEARCH: query>>> to find relevant snippets in the project first.
- Paths use forward slashes; parent directories are auto-created.
- Do not install packages or scaffold extra files unless explicitly asked.

Example — task: "create hello.py that prints hello, then run it":
Step 1 you output:
<<<FILE: hello.py>>>
print("hello")
<<<END>>>
<<<RUN: python3 hello.py>>>
Step 2 (after I send you the output) you output:
<<<DONE>>>"""

ASSISTANT_PROMPT = """You are a helpful local assistant. Answer the user directly and clearly in plain prose.
You MAY use these tools when they genuinely help — otherwise just answer:
  <<<SEARCH: query>>>      web search for facts / docs / news (results are CACHED — never for the current time).
  <<<API: url>>>           GET a URL for LIVE data (current time, prices, weather, JSON APIs). For the current
                           time use a time API, e.g. <<<API: https://timeapi.io/api/Time/current/zone?timeZone=Asia/Jerusalem>>> — even if the user says "search".
  <<<RAGSEARCH: query>>>   search the user's own indexed files/notes.
  <<<DOCRAG: query>>>      query the user's ingested documents db (invoices/contracts): exact SQL for numbers, hybrid search for content.
  <<<RECALL: query>>>      recall a saved fact.   <<<REMEMBER: fact>>>  save a durable fact.
  <<<READ: path>>>         read a file.           <<<RUN: command>>>    run a shell command.
Use AT MOST one tool per turn. After tool results come back, WRITE the final answer in prose with NO tool
block. Ground answers in tool results — never invent versions, prices, dates, or facts. Be concise and readable."""

PLANNER_PROMPT = """You are a software planner. Given a task, list the files needed to complete it.
Output ONE line per file in this format:
<relative file path> :: <a precise one-sentence spec of what this file must contain>

Use REAL, sensible project paths (for example: src/models/User.js, src/services/authManager.js, src/app.js).
NEVER output the literal placeholder "path/to" — write actual paths.
Rules:
- Output ONLY these lines. No prose, no code, no markdown.
- Keep the list minimal but complete.
- Order files so dependencies come first (models, then services, then controllers, then routes, then app/server)."""

EXECUTOR_PROMPT = """You write exactly ONE source file. Output ONLY a single block:
<<<FILE: the/path>>>
(full file content)
<<<END>>>
No prose, no markdown fences, no extra files. Write complete, working code that is consistent
with the provided project plan and the already-written files."""

STYLE_PROMPT = """You write a clean, well-formatted markdown document from PROVIDED facts.
Output ONLY the markdown content itself — no <<<FILE>>> blocks, no surrounding code fences, no preamble or sign-off.
Use ONLY the facts you are given. NEVER add a version, number, date, feature, or claim that is not
explicitly present in the facts. If the facts are sparse, keep the document short — do NOT pad it with invented details."""

COMPACT_PROMPT = """You compress a coding agent's conversation into a short progress note.
Capture ONLY: the task/goal, which files were created or changed, key decisions, and what still
remains to do. A few terse bullet lines. Do NOT include full file contents or tool-block syntax."""

DIAGNOSE_PROMPT = """You are a senior engineer writing a quick DIAGNOSIS before any code is written.
Given the task and excerpts of the EXISTING project, output a short briefing — 3 to 6 terse lines:
- Which existing files / patterns / conventions are relevant and should be matched.
- Any assumption that must be verified first (and how to check it).
- A minimal plan: what to create or change.
Do NOT write code. Do NOT use tool blocks. Output ONLY the briefing lines."""

CRITIQUE_PROMPT = """You are a strict reviewer checking a coding agent's PROPOSED files BEFORE they are written.
Given the task and the proposed file(s), list concrete problems: bugs, unmet requirements, wrong APIs, obvious
omissions. Be specific and brief (bullet lines). If the files correctly and completely satisfy the task, reply
with EXACTLY: OK"""

REVIEWER_PROMPT = """You are a meticulous code REVIEWER (read-only) — a sub-agent with a fresh, clean context.
Given the task and the files the agent just wrote, find REAL problems: bugs, unmet requirements, missing error
handling, security issues. List them as short bullets. If the work is correct and complete, reply EXACTLY: LGTM"""

PROJECT_MEMORY = ""  # loaded from a project memory file (AGENTS.md) if present
ACTIVE_SKILL = ""    # loaded from a skill file via -s/--skill
HOOKS = {}           # event automation loaded from .codegen-hooks.json (#6): deny / post_file / post_task
MEMORY_FILES = ("AGENTS.md", ".codegen.md", "CLAUDE.md")
SKILLS_DIR = os.path.expanduser("~/.codegen/skills")
PLUGINS_DIR = os.path.expanduser("~/.codegen/plugins")
HOOKS_FILE = ".codegen-hooks.json"


def load_memory(base):
    """Read a short conventions file (AGENTS.md) from the project, like CLAUDE.md."""
    for name in MEMORY_FILES:
        p = os.path.join(base, name)
        if os.path.isfile(p):
            with open(p) as f:
                txt = f.read().strip()
            print(f"🧠 loaded project memory from {name}", file=sys.stderr)
            if len(txt) > 4000:
                print(f"⚠️  {name} is large ({len(txt)} chars) — it eats the 8K context. Keep it short.",
                      file=sys.stderr)
            return txt
    return ""


def load_skill(name):
    """Read a reusable skill file ~/.codegen/skills/<name>.md (expert instructions for a task type)."""
    p = os.path.join(SKILLS_DIR, f"{name}.md")
    if os.path.isfile(p):
        with open(p) as f:
            txt = f.read().strip()
        print(f"🎓 loaded skill: {name}", file=sys.stderr)
        return txt
    avail = [f[:-3] for f in os.listdir(SKILLS_DIR)] if os.path.isdir(SKILLS_DIR) else []
    print(f"⚠️  skill '{name}' not found in {SKILLS_DIR}. Available: {avail or '(none)'}", file=sys.stderr)
    return ""


def load_hooks(base):
    """Load event-automation hooks from .codegen-hooks.json (#6). Opt-in: active only if present.
    Keys: 'deny' (list of glob patterns the agent may NOT write), 'post_file' (shell cmd run after
    each write; {file} is substituted), 'post_task' (shell cmd run on DONE; failure blocks DONE and
    is fed back so the model fixes it — e.g. tests-before-stop)."""
    p = os.path.join(base, HOOKS_FILE)
    if not os.path.isfile(p):
        return {}
    try:
        with open(p) as f:
            h = json.load(f)
    except (OSError, ValueError) as e:
        print(f"⚠️  ignoring bad {HOOKS_FILE}: {e}", file=sys.stderr)
        return {}
    print(f"🪝 loaded hooks from {HOOKS_FILE}: {', '.join(h) or '(empty)'}", file=sys.stderr)
    return h


def load_plugin(name):
    """#11 — load a plugin bundle ~/.codegen/plugins/<name>/ that packages, as ONE shareable unit:
    skill.md (expert instructions), hooks.json (event automation), and optional AGENTS.md (conventions).
    Returns (skill_text, hooks_dict, conventions_text). Lets you ship a workflow as a single folder."""
    d = os.path.join(PLUGINS_DIR, name)
    if not os.path.isdir(d):
        avail = sorted(os.listdir(PLUGINS_DIR)) if os.path.isdir(PLUGINS_DIR) else []
        print(f"⚠️  plugin '{name}' not found in {PLUGINS_DIR}. Available: {avail or '(none)'}", file=sys.stderr)
        return "", {}, ""

    def _read(fname):
        p = os.path.join(d, fname)
        if os.path.isfile(p):
            with open(p) as f:
                return f.read().strip()
        return ""

    skill, conv = _read("skill.md"), _read("AGENTS.md")
    hooks = {}
    raw = _read("hooks.json")
    if raw:
        try:
            hooks = json.loads(raw)
        except ValueError as e:
            print(f"⚠️  plugin '{name}': bad hooks.json: {e}", file=sys.stderr)
    parts = [n for n, v in (("skill", skill), ("hooks", hooks), ("conventions", conv)) if v]
    print(f"🔌 loaded plugin '{name}': {', '.join(parts) or '(empty)'}", file=sys.stderr)
    return skill, hooks, conv


def hook_denied(target):
    """Return the deny pattern that blocks writing `target`, or None."""
    for pat in HOOKS.get("deny", []):
        if fnmatch.fnmatch(target, pat) or fnmatch.fnmatch(os.path.basename(target), pat):
            return pat
    return None


def run_hook_cmd(cmd, base, timeout=180):
    """Run a hook shell command; return (ok, formatted_output)."""
    try:
        r = subprocess.run(cmd, shell=True, cwd=base, capture_output=True, text=True, timeout=timeout)
    except subprocess.TimeoutExpired:
        return False, f"$ {cmd}\n(timed out after {timeout}s)"
    out = (r.stdout + r.stderr).strip() or "(no output)"
    return r.returncode == 0, f"$ {cmd}\n(exit {r.returncode})\n{trunc(out)}"


def run_stop_hook(base):
    """post_task hook: run a project check (e.g. tests) before accepting DONE. (ok, report)."""
    cmd = HOOKS.get("post_task")
    if not cmd:
        return True, ""
    print(f"🪝 stop hook: {cmd}", file=sys.stderr)
    return run_hook_cmd(cmd, base)


def with_memory(prompt):
    """Append the active skill + project memory to a system prompt."""
    if ACTIVE_SKILL:
        prompt += f"\n\n# Skill — expert instructions for this task (follow precisely):\n{ACTIVE_SKILL}"
    if PROJECT_MEMORY:
        prompt += f"\n\n# Project conventions (ALWAYS follow these):\n{PROJECT_MEMORY}"
    return prompt

# ── tool registry (Open/Closed) ──────────────────────────────────────────────
# One spec per tool block the model can emit. To ADD a tool you append ONE spec
# here and ONE runner in _TOOL_RUNNERS (see do_action) — parse_actions and
# do_action are now data-driven and need no editing. `pattern` is this verb's
# regex alternative; `key` is the named group that, when present in a match,
# identifies the tool; `parse` turns a match into an (kind, target, content)
# action tuple (or None to skip). The combined ACTION_RE is built from these
# patterns, so it stays byte-identical to the hand-written regex it replaced.
def _parse_file(m):
    path = clean_path(m.group("fpath"))
    return ("file", path, strip_fences(m.group("content"))) if path else None


def _parse_research(m):
    rfile = m.group("rfile")
    rpath = clean_path(rfile) if rfile else slugify_md(m.group("rquery"))
    return ("research", rpath, m.group("rquery").strip())


TOOL_SPECS = [
    {"name": "file", "key": "fpath", "parse": _parse_file,
     "pattern": r"<<<\s*FILE\s*:\s*(?P<fpath>.+?)\s*>{2,}\s*\n(?P<content>.*?)\n\s*<<<\s*END\s*>*"},
    {"name": "run", "key": "cmd", "parse": lambda m: ("run", m.group("cmd").strip(), None),
     "pattern": r"<<<\s*RUN\s*:\s*(?P<cmd>.+?)\s*>{2,}"},
    {"name": "read", "key": "rpath", "parse": lambda m: ("read", clean_path(m.group("rpath")), None),
     "pattern": r"<<<\s*READ\s*:\s*(?P<rpath>.+?)\s*>{2,}"},
    {"name": "research", "key": "rquery", "parse": _parse_research,
     "pattern": r"<<<\s*RESEARCH\s*:\s*(?P<rquery>.+?)(?:\s*=>\s*(?P<rfile>.+?))?\s*>{2,}"},
    {"name": "ragsearch", "key": "ragquery", "parse": lambda m: ("ragsearch", m.group("ragquery").strip(), None),
     "pattern": r"<<<\s*RAGSEARCH\s*:\s*(?P<ragquery>.+?)\s*>{2,}"},
    {"name": "docrag", "key": "docragquery", "parse": lambda m: ("docrag", m.group("docragquery").strip(), None),
     "pattern": r"<<<\s*DOCRAG\s*:\s*(?P<docragquery>.+?)\s*>{2,}"},
    {"name": "api", "key": "api", "parse": lambda m: ("api", m.group("api").strip(), None),
     "pattern": r"<<<\s*API\s*:\s*(?P<api>.+?)\s*>{2,}"},
    {"name": "search", "key": "query", "parse": lambda m: ("search", m.group("query").strip(), None),
     "pattern": r"<<<\s*SEARCH\s*:\s*(?P<query>.+?)\s*>{2,}"},
    {"name": "remember", "key": "remember", "parse": lambda m: ("remember", m.group("remember").strip(), None),
     "pattern": r"<<<\s*REMEMBER\s*:\s*(?P<remember>.+?)\s*>{2,}"},
    {"name": "recall", "key": "recall", "parse": lambda m: ("recall", m.group("recall").strip(), None),
     "pattern": r"<<<\s*RECALL\s*:\s*(?P<recall>.+?)\s*>{2,}"},
    {"name": "list", "key": "list", "parse": lambda m: ("list", None, None),
     "pattern": r"<<<\s*(?P<list>LIST)\s*>{2,}"},
    {"name": "done", "key": "done", "parse": lambda m: ("done", None, None),
     "pattern": r"<<<\s*(?P<done>DONE)\s*>{2,}"},
]
ACTION_RE = re.compile("|".join(s["pattern"] for s in TOOL_SPECS), re.DOTALL | re.IGNORECASE)


def clean_path(p):
    return p.strip().strip(">").strip().strip('"\'`').strip()


def slugify_md(q):
    """Derive a default .md filename from a query when the model omits the path."""
    s = re.sub(r"[^a-z0-9]+", "-", q.lower()).strip("-")[:40]
    return (s or "research") + ".md"


THINK_RE = re.compile(r"<think>.*?</think>\s*", re.DOTALL | re.IGNORECASE)


def strip_think(text):
    """Remove <think>...</think> reasoning that thinking models (qwen3, abliterated variants) emit
    inside the content. If a <think> was left unclosed (output cut mid-reasoning), drop from it to the
    end — there is no real answer after it. Keeps the harness readable and saves context."""
    text = THINK_RE.sub("", text)
    low = text.lower()
    i = low.rfind("<think>")
    if i != -1 and "</think>" not in low[i:]:   # unclosed → everything after is reasoning, not answer
        text = text[:i]
    return text.strip()


def strip_fences(content):
    lines = content.split("\n")
    if lines and lines[0].lstrip().startswith("```"):
        lines = lines[1:]
    if lines and lines[-1].strip() == "```":
        lines = lines[:-1]
    return "\n".join(lines)


def parse_actions(text):
    actions = []
    for m in ACTION_RE.finditer(text):
        for spec in TOOL_SPECS:                 # exactly one spec's key group is set per match
            if m.group(spec["key"]) is not None:
                act = spec["parse"](m)
                if act:
                    actions.append(act)
                break

    # Fallback: recover an unclosed trailing FILE block (model forgot <<<END>>>)
    if not any(a[0] == "file" for a in actions):
        m = re.search(r"<<<\s*FILE\s*:\s*(.+?)\s*>{2,}\s*\n(.*)", text, re.DOTALL | re.IGNORECASE)
        if m:
            path = clean_path(m.group(1))
            content = re.split(r"\n\s*<<<\s*END", m.group(2))[0]
            if path:
                actions.append(("file", path, strip_fences(content)))
    return actions


def project_tree(base):
    """List the project's files, but bounded: skip env/cache/vendor dirs and hidden
    dirs, and cap the count. A huge dir (e.g. ~/Downloads with .venv + GGUF models)
    would otherwise serialize to hundreds of thousands of tokens and blow the 8K window."""
    out = []
    capped = False
    for root, dirs, files in os.walk(base):
        dirs[:] = [d for d in dirs if d not in SKIP_DIRS and not d.startswith(".")]
        for f in files:
            out.append(os.path.relpath(os.path.join(root, f), base))
            if len(out) > MAX_TREE_FILES * 5:  # stop walking a massive tree early
                capped = True
                break
        if capped:
            break
    out.sort()
    if len(out) > MAX_TREE_FILES:
        out = out[:MAX_TREE_FILES]
        capped = True
    tree = "\n".join(out) or "(empty project)"
    if capped:
        tree += ("\n...[tree truncated — too many files. Work in a smaller project "
                 "directory for best results.]")
    return tree


def trunc(s):
    return s if len(s) <= MAX_OUT else s[:MAX_OUT] + f"\n...[truncated {len(s) - MAX_OUT} chars]"


def http_get(url):
    """#10 — GET a URL for LIVE data (prices, weather, JSON APIs). Returns the body, pretty-printed
    if it's JSON. Use this instead of web search when freshness matters (search snippets are cached)."""
    url = url.strip().strip('"').strip("'").strip()
    if not url.lower().startswith(("http://", "https://")):
        url = "https://" + url
    try:
        req = urllib.request.Request(
            url, headers={"User-Agent": "codegen-agent", "Accept": "application/json, text/plain, */*"})
        with urllib.request.urlopen(req, timeout=20) as r:
            body = r.read().decode("utf-8", "ignore")
    except Exception as e:
        return f"API GET failed: {e}"
    try:
        return json.dumps(json.loads(body), indent=2, ensure_ascii=False)
    except ValueError:
        return body


def search_results(query, n=SEARCH_RESULTS):
    """Return up to n results as [{title,url,content}] via SearXNG (private), then DuckDuckGo.
    Empty list on failure. This is the structured source both web_search() and web_answer() use."""
    query = query.strip().strip('"').strip("'").strip()  # quotes break search (exact-match → 0 results)
    # 1) SearXNG — local, private, JSON API
    try:
        url = f"{SEARXNG_URL}/search?q={urllib.parse.quote(query)}&format=json"
        req = urllib.request.Request(url, headers={"User-Agent": "codegen-agent"})
        with urllib.request.urlopen(req, timeout=20) as r:
            data = json.loads(r.read())
        res = data.get("results", [])[:n]
        if res:
            return [{"title": x.get("title", "").strip(), "url": x.get("url", ""),
                     "content": x.get("content", "").strip()} for x in res]
    except Exception:
        pass
    # 2) DuckDuckGo lite fallback — no server needed, stable HTML
    try:
        url = "https://lite.duckduckgo.com/lite/?q=" + urllib.parse.quote(query)
        req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
        with urllib.request.urlopen(req, timeout=20) as r:
            html = r.read().decode("utf-8", "ignore")
        links = re.findall(r'href="(//duckduckgo\.com/l/\?uddg=[^"]+)"[^>]*>(.*?)</a>', html, re.DOTALL)
        snips = re.findall(r'result-snippet[^>]*>(.*?)</td>', html, re.DOTALL)
        out = []
        for i, (href, title) in enumerate(links[:n]):
            mm = re.search(r"uddg=([^&]+)", href)
            out.append({"title": re.sub("<.*?>", "", title).strip(),
                        "url": urllib.parse.unquote(mm.group(1)) if mm else href,
                        "content": re.sub("<.*?>", "", snips[i]).strip() if i < len(snips) else ""})
        return out
    except Exception:
        return []


def web_search(query):
    """Formatted top results (titles+urls+snippets) — used by the SEARCH and RESEARCH tools."""
    res = search_results(query)
    if not res:
        return f"Search failed or no results (SearXNG at {SEARXNG_URL} / DuckDuckGo unreachable)."
    return "\n\n".join(f"{i+1}. {x['title']}\n   {x['url']}\n   {x['content']}" for i, x in enumerate(res))


def fetch_page_text(url, limit=2500):
    """Fetch a page and return readable text (tags/scripts/boilerplate stripped), truncated.
    Fresher and richer than a cached search snippet — this is what makes 'search the internet'
    reliable. Drops nav/header/footer/aside/forms so the model sees content, not chrome."""
    raw = http_get(url)
    if raw.startswith("API GET failed"):
        return ""
    # Drop whole non-content regions (scripts, styles, and page chrome).
    html = re.sub(r"(?is)<(script|style|noscript|head|nav|footer|header|aside|form|svg|button)[^>]*>.*?</\1>",
                  " ", raw)
    text = re.sub(r"(?s)<[^>]+>", " ", html)            # drop remaining tags
    text = re.sub(r"&[a-z#0-9]+;", " ", text)            # crude entity strip
    text = re.sub(r"\s+", " ", text).strip()
    return text[:limit]


WEB_ANSWER_PROMPT = """You answer the user's question using ONLY the SOURCES provided below.
Rules:
- Reply in ENGLISH.
- Use ONLY facts present in the sources. NEVER add a number, date, version, price, or claim from your own memory.
- If the sources do not contain the answer, say exactly: "The sources don't contain this."
- Be concise. Then add a final line: "Sources:" listing the source numbers you used.
- Do NOT output any tool blocks (<<<...>>>). Just the prose answer."""


def _api_json(url, timeout=15):
    """GET and parse JSON, or None. For the live-data sources below."""
    try:
        req = urllib.request.Request(url, headers={"User-Agent": "codegen-agent",
                                                   "Accept": "application/json"})
        with urllib.request.urlopen(req, timeout=timeout) as r:
            return json.loads(r.read().decode("utf-8", "ignore"))
    except Exception:
        return None


def geocode_place(place):
    """Place name -> {name,country,lat,lon,timezone} via Open-Meteo geocoding (free, no key)."""
    if not place:
        return None
    url = ("https://geocoding-api.open-meteo.com/v1/search?count=1&language=en&format=json&name="
           + urllib.parse.quote(place))
    res = (_api_json(url) or {}).get("results") or []
    if not res:
        return None
    g = res[0]
    return {"name": g.get("name"), "country": g.get("country"), "lat": g.get("latitude"),
            "lon": g.get("longitude"), "timezone": g.get("timezone")}


def live_time(place):
    """(fact, source) for the CURRENT time at a place, or None. Geocode -> timezone -> clock API."""
    g = geocode_place(place)
    if not g or not g.get("timezone"):
        return None
    d = _api_json("https://timeapi.io/api/Time/current/zone?timeZone=" + urllib.parse.quote(g["timezone"]))
    if not d or "time" not in d:
        return None
    loc = ", ".join(x for x in (g.get("name"), g.get("country")) if x)
    return (f"Current local time in {loc} ({g['timezone']}): {d.get('time')} on {d.get('date')} "
            f"({d.get('dayOfWeek')}).", f"https://timeapi.io — timezone {g['timezone']}")


_WCODE = {0: "clear sky", 1: "mainly clear", 2: "partly cloudy", 3: "overcast", 45: "fog",
          48: "rime fog", 51: "light drizzle", 53: "drizzle", 55: "dense drizzle", 56: "freezing drizzle",
          61: "light rain", 63: "rain", 65: "heavy rain", 66: "freezing rain", 71: "light snow",
          73: "snow", 75: "heavy snow", 77: "snow grains", 80: "rain showers", 81: "rain showers",
          82: "violent rain showers", 85: "snow showers", 86: "snow showers", 95: "thunderstorm",
          96: "thunderstorm with hail", 99: "thunderstorm with heavy hail"}


def live_weather(place):
    """(fact, source) for CURRENT weather at a place, or None. Geocode -> Open-Meteo forecast."""
    g = geocode_place(place)
    if not g or g.get("lat") is None:
        return None
    url = (f"https://api.open-meteo.com/v1/forecast?latitude={g['lat']}&longitude={g['lon']}"
           "&current=temperature_2m,relative_humidity_2m,apparent_temperature,weather_code,wind_speed_10m")
    cur = (_api_json(url) or {}).get("current")
    if not cur:
        return None
    loc = ", ".join(x for x in (g.get("name"), g.get("country")) if x)
    desc = _WCODE.get(cur.get("weather_code"), "")
    return (f"Current weather in {loc}: {cur.get('temperature_2m')}°C (feels like "
            f"{cur.get('apparent_temperature')}°C), {desc}, humidity {cur.get('relative_humidity_2m')}%, "
            f"wind {cur.get('wind_speed_10m')} km/h.", "https://open-meteo.com")


def live_version(software):
    """(fact, source) for the latest release of a software via the GitHub API, or None.
    Resolves the repo by search (top by stars), then releases/latest, then tags as a fallback.
    The repo URL is returned as the source so a wrong-repo guess is visible/verifiable."""
    if not software:
        return None
    sr = _api_json("https://api.github.com/search/repositories?per_page=1&sort=stars&q="
                   + urllib.parse.quote(software))
    items = (sr or {}).get("items") or []
    if not items:
        return None
    repo = items[0].get("full_name")
    rel = _api_json(f"https://api.github.com/repos/{repo}/releases/latest")
    tag = (rel or {}).get("tag_name")
    if not tag:  # repo may publish tags without GitHub "releases"
        tags = _api_json(f"https://api.github.com/repos/{repo}/tags?per_page=1")
        tag = tags[0].get("name") if tags else None
    if not tag:
        return None
    return (f"Latest release of {repo}: {tag}.", f"https://github.com/{repo}/releases")


def live_currency(base, quote):
    """(fact, source) for an exchange rate, or None. Free, no key."""
    base = (base or "USD").upper()
    quote = (quote or "ILS").upper()
    d = _api_json("https://open.er-api.com/v6/latest/" + urllib.parse.quote(base))
    rate = ((d or {}).get("rates") or {}).get(quote)
    if rate is None:
        return None
    return (f"1 {base} = {rate} {quote} (updated {(d or {}).get('time_last_update_utc', 'recently')}).",
            "https://open.er-api.com")


CLASSIFY_PROMPT = """Classify the user's question for a live-data lookup. Output ONE line of JSON, nothing else:
{"type":"time|weather|currency|version|general","place":"<city or country in ENGLISH, else empty>","base":"<3-letter currency or empty>","quote":"<3-letter currency or empty>","software":"<software/tool name, else empty>"}
- "time": asks the current time/date somewhere.
- "weather": asks current weather/temperature somewhere.
- "currency": asks an exchange rate (how much one currency is in another).
- "version": asks the latest/current version of a software, tool, library, or app.
- "general": ANYTHING else (facts, news, product prices, definitions...).
Translate the place to its common ENGLISH name. Output ONLY the JSON object."""


def classify_query(query):
    """Ask the model ONLY to parse intent (it is good at this) -> dict. Harness fetches the facts."""
    try:
        raw = ask_model([{"role": "system", "content": CLASSIFY_PROMPT},
                         {"role": "user", "content": query}])
    except Exception:
        return {"type": "general"}
    m = re.search(r"\{.*\}", raw, re.DOTALL)
    if not m:
        return {"type": "general"}
    try:
        d = json.loads(m.group(0))
        return d if isinstance(d, dict) else {"type": "general"}
    except ValueError:
        return {"type": "general"}


ANSWER_FROM_FACT_PROMPT = """Answer the user's question using ONLY the FACT provided.
Reply in ENGLISH, in one or two clear sentences. Use ONLY the given fact — never add or
change any number, date, or detail. Do not output tool blocks."""


def web_answer(query, emit=None, should_stop=None):
    """Smart grounded answer for ANY web-needing question. Classifies intent, then the harness
    fetches from the RIGHT source (live APIs for time/weather/currency, else search+page-fetch),
    and the model only phrases the result. Facts come from the script, never the model's memory."""
    emit_event(emit, {"type": "thinking"})
    intent = classify_query(query)
    typ = (intent.get("type") or "general").lower()
    place = (intent.get("place") or "").strip()
    fact = source = None

    if typ == "time":
        emit_event(emit, {"type": "action", "kind": "api", "label": f"live time: {place or '?'}"})
        r = live_time(place)
        if r:
            fact, source = r
    elif typ == "weather":
        emit_event(emit, {"type": "action", "kind": "api", "label": f"live weather: {place or '?'}"})
        r = live_weather(place)
        if r:
            fact, source = r
    elif typ == "currency":
        emit_event(emit, {"type": "action", "kind": "api", "label": "live exchange rate"})
        r = live_currency(intent.get("base"), intent.get("quote"))
        if r:
            fact, source = r
    elif typ == "version":
        sw = (intent.get("software") or "").strip()
        emit_event(emit, {"type": "action", "kind": "api", "label": f"latest version: {sw or '?'}"})
        r = live_version(sw)
        if r:
            fact, source = r

    if fact:
        emit_event(emit, {"type": "result", "text": fact})
        emit_event(emit, {"type": "thinking"})
        stream_cb = (lambda t: emit_event(emit, {"type": "token", "text": t})) if emit else None
        ans = ask_model([
            {"role": "system", "content": with_memory(ANSWER_FROM_FACT_PROMPT)},
            {"role": "user", "content": f"Question: {query}\n\nFACT: {fact}\n\nAnswer now."},
        ], stream_cb=stream_cb, should_stop=should_stop).strip()
        full = f"{ans}\n\n— source —\n  {source}"
        emit_event(emit, {"type": "assistant", "text": full})
        print(ans)
        return full

    # general questions (and any failed live lookup) -> search + fetch pages + grounded answer
    return _web_search_answer(query, emit, should_stop)


def _web_search_answer(query, emit=None, should_stop=None):
    """General grounded pipeline: search -> fetch the top pages -> answer constrained to those
    sources -> cite them. The model only phrases; it cannot hallucinate facts."""
    emit_event(emit, {"type": "action", "kind": "search", "label": query})
    results = search_results(query, n=5)
    if not results:
        msg = "No web results (is SearXNG running, or is the network up?)."
        emit_event(emit, {"type": "assistant", "text": msg})
        return msg
    # Fetch full text of the top 3 pages for freshness (authoritative sources like GitHub
    # releases are often #3); fall back to snippets for the rest. Per-page text is capped so
    # three pages fit the 8K window; the combined blob has its own (larger) budget than MAX_OUT.
    sources = []
    for i, r in enumerate(results):
        body = r["content"]
        if i < 3 and r["url"]:
            if should_stop and should_stop():
                break
            emit_event(emit, {"type": "action", "kind": "api", "label": f"fetch {r['url'][:60]}"})
            page = fetch_page_text(r["url"], limit=2000)
            if page:
                body = page
        sources.append({"n": i + 1, "title": r["title"], "url": r["url"], "body": body})
    blob = "\n\n".join(f"[{s['n']}] {s['title']} — {s['url']}\n{s['body']}" for s in sources)
    if len(blob) > WEB_SOURCES_MAX:  # bigger budget than MAX_OUT — these sources ARE the answer
        blob = blob[:WEB_SOURCES_MAX] + "\n...[sources truncated]"
    emit_event(emit, {"type": "thinking"})
    stream_cb = (lambda t: emit_event(emit, {"type": "token", "text": t})) if emit else None
    answer = ask_model([
        {"role": "system", "content": with_memory(WEB_ANSWER_PROMPT)},
        {"role": "user", "content": f"Question: {query}\n\nSOURCES:\n{blob}\n\nAnswer now."},
    ], stream_cb=stream_cb, should_stop=should_stop).strip()
    links = "\n".join(f"  [{s['n']}] {s['url']}" for s in sources if s["url"])
    full = f"{answer}\n\n— retrieved sources —\n{links}" if links else answer
    emit_event(emit, {"type": "assistant", "text": full})
    print(answer)
    return full


RESEARCH_PLAN_PROMPT = """You are a critical research planner. Given a topic, claim, or source, output a list of
focused web-search questions that TOGETHER uncover the full picture — and that CHALLENGE the claim, not only confirm it.
Cover, across the questions: facts/numbers to verify, who is behind it (company/author/funding/incentives/sponsor),
real costs, objective performance vs alternatives, risks (security/privacy/legal/regulatory), competitors, and what
the source likely leaves out.
REQUIRED — write SHORT KEYWORD QUERIES the way a person types into Google (3-7 words), NOT full questions or
sentences. Example GOOD: "Oracle free tier account banned reddit". Example BAD: "What are the complaints about
Oracle Cloud Free Tier VPS for June 2026?".
- At least 2 queries MUST be ADVERSARIAL — hunt for problems: e.g. "<thing> problems reddit", "<thing> scam",
  "<thing> account banned", "<thing> downgrade".
- Add a YEAR to AT MOST one or two queries that are about current specs/limits/prices (use a bare year like
  "2026", NEVER a month like "June 2026"). Keep the other queries broad and year-free so they actually return results.
Output EXACTLY 6 queries, ONE per line, no numbering and no prose. Each a standalone web-search query in English."""

RESEARCH_FINDING_PROMPT = """Answer the sub-question from the SOURCES. Report what they actually say — BOTH kinds:
- CONCRETE facts: numbers, dates, versions, prices — quote them and the source URL.
- QUALITATIVE facts: documented problems, complaints, policies, behaviors — these matter just as much in a critical
  investigation. Attribute them, e.g. "multiple users on r/oraclecloud report Oracle deleted free-tier accounts
  without warning" or "the docs state idle instances are reclaimed". A named, sourced complaint IS a valid finding.
STAY ON SUBJECT: only facts about the INVESTIGATION SUBJECT below. If a number is for a DIFFERENT product/hardware
(e.g. a different GPU/computer), DO NOT use it — a real number from the wrong context is worse than no answer.
CROSS-CHECK: prefer facts in 2+ sources; mark a lone claim "[single source]"; if sources DISAGREE, report both.
DATES: include "as of <date>" when a source gives one — recency matters.
2-4 sentences. Reply EXACTLY "No data found." ONLY if the sources truly say nothing relevant to the sub-question.
Reply in English. NEVER invent or estimate. No tool blocks."""

RESEARCH_GAP_PROMPT = """You review research findings and name what is still MISSING for a complete, critical picture.
Output up to 3 web-search questions for IMPORTANT facts not yet answered, or single-source claims that need
cross-checking against another source. If the findings already cover the topic well, output exactly: NONE.
One question per line, no numbering, no prose. English."""

RESEARCH_VERIFY_PROMPT = """You are a fact-checker. You are given a DRAFT report and the FINDINGS it was based on.
Return a corrected version of the report where:
- Every claim that is NOT directly supported by a finding is removed, or softened to "could not verify".
- Any number/date/name that does not appear in the findings is deleted.
- Generic boilerplate not grounded in a finding (e.g. vague risks) is removed.
- Everything supported by the findings is KEPT, with its [n] citation.
Keep the same section structure and English. Output ONLY the corrected report."""

RESEARCH_SYNTH_PROMPT = """You are a critical research assistant. Using ONLY the FINDINGS below (each grounded in web
sources), write an investigation report in English. Be direct and fact-based, not condescending; present the strengths
of what you criticize; flag uncertainty honestly.
HARD RULES:
- EVIDENCE OR SILENCE: only state that a claim is wrong/outdated if a FINDING gives a SPECIFIC contradicting fact —
  quote that number/date. If you cannot verify a claim from the findings, write "could not verify" — do NOT assert it.
- NO FILLER: if a section has no grounded data, write exactly "(no data found)". Never write generic boilerplate
  (e.g. vague "security/privacy/legal risks" that aren't in the findings).
- Prefer concrete numbers/names/dates from the findings over general statements.
- Never invent a fact that is not in the findings.
Structure:
1. Title — the REAL conclusion after investigating (not a summary of the source)
2. Data corrections — figures/claims in the source the findings confirm or contradict (quote the contradicting fact)
3. Who is behind it — the creator/author of the ORIGINAL source. NEVER name a website you searched
   (a benchmark site, blog, or search result) as the author. If the creator is not identified in the
   findings or original source, write exactly: "The source's author is not identified in the findings."
4. Real costs / hardware (if relevant)
5. Risks — only specific ones the findings surface
6. What was NOT said — blind spots, omissions, unfair comparisons
7. Broad context — competitors, alternative approaches
8. Bottom line — honest, even if it contradicts the source
Cite source numbers like [n] from the findings."""


def research_pipeline(topic, emit=None, should_stop=None, max_q=6):
    """Critical multi-step research that a SMALL model can actually run: the harness plans
    sub-questions, runs a grounded web search for each, then the model synthesizes a report.
    Every model call is small and grounded — no single call has to 'be' a frontier researcher."""
    import datetime
    today = datetime.date.today().strftime("%B %Y")  # recency anchor for time-sensitive queries
    subject = " ".join(topic.split())[:200]  # short subject string → keeps findings on-topic
    findings, src_list = [], []

    # one grounded finding for a question: search → fetch top 2 pages → extract on-subject facts.
    def do_finding(n, q):
        if should_stop and should_stop():
            return None
        emit_event(emit, {"type": "action", "kind": "search", "label": f"{n}: {q}"})
        results = search_results(q, n=4)
        if not results:
            return None
        parts = []  # full text of top 2 pages (numbers live in the page, not the snippet) + snippets
        for k, r in enumerate(results, 1):
            text = r["content"]
            if k <= 2 and r["url"]:
                page = fetch_page_text(r["url"], limit=1500)
                if page:
                    text = page
            parts.append(f"[{n}.{k}] {r['title']} — {r['url']}\n{text}")
        emit_event(emit, {"type": "thinking"})
        finding = ask_model([
            {"role": "system", "content": RESEARCH_FINDING_PROMPT},
            {"role": "user", "content": f"INVESTIGATION SUBJECT: {subject}\nToday is {today}.\n\n"
                                        f"Sub-question: {q}\n\nSOURCES:\n{chr(10).join(parts)[:4500]}\n\n"
                                        f"Answer (stay on subject; cross-check across sources)."},
        ], should_stop=should_stop).strip()
        src_list.extend(r["url"] for r in results if r["url"])
        emit_event(emit, {"type": "result", "text": f"[{n}] {q}\n{finding}"})
        return {"n": n, "q": q, "finding": finding}

    # 0) ORIGINAL SOURCE — if the topic has a URL, fetch it so "who's behind it" / "what was said"
    #    are grounded in the actual source, not guessed.
    original = ""
    mu = re.search(r"https?://[^\s)\]]+", topic)
    if mu:
        emit_event(emit, {"type": "action", "kind": "api", "label": f"fetch original: {mu.group(0)[:50]}"})
        page = fetch_page_text(mu.group(0), limit=1800)
        if page:
            original = f"ORIGINAL SOURCE ({mu.group(0)}):\n{page}"

    # 1) PLAN — decompose into challenging, adversarial, recency-aware sub-questions (one model call).
    emit_event(emit, {"type": "action", "kind": "plan", "label": "planning research questions"})
    emit_event(emit, {"type": "thinking"})
    year = today.split()[-1]  # bare year only — passing the month made the model glue "June 2026" onto queries
    raw = ask_model([
        {"role": "system", "content": with_memory(RESEARCH_PLAN_PROMPT)},
        {"role": "user", "content": f"Current year: {year}.\nTopic / source:\n{trunc(topic)}\n\n"
                                    f"List the short keyword search queries."},
    ], should_stop=should_stop)
    questions = [re.sub(r"^[\-\*\d\.\)\s]+", "", ln).strip() for ln in raw.splitlines() if ln.strip()]
    questions = [q for q in questions if len(q) > 8][:max_q]
    if not questions:
        questions = [topic.strip()[:120]]
    emit_event(emit, {"type": "assistant",
                      "text": "🔬 Research plan:\n" + "\n".join(f"• {q}" for q in questions)})

    # 2) RETRIEVE — grounded finding per sub-question.
    for i, q in enumerate(questions, 1):
        if should_stop and should_stop():
            emit_event(emit, {"type": "assistant", "text": "⏹️ Stopped."})
            break
        f = do_finding(i, q)
        if f:
            findings.append(f)

    # 2b) GAP-FILLING ROUND — ask what's still missing / needs cross-checking, then chase those too.
    if findings and not (should_stop and should_stop()):
        fblob0 = "\n\n".join(f"[{f['n']}] {f['q']}\n{f['finding']}" for f in findings)
        emit_event(emit, {"type": "action", "kind": "plan", "label": "finding gaps to fill"})
        emit_event(emit, {"type": "thinking"})
        gap_raw = ask_model([
            {"role": "system", "content": RESEARCH_GAP_PROMPT},
            {"role": "user", "content": f"TOPIC: {subject}\n\nFINDINGS SO FAR:\n{fblob0[:4000]}\n\n"
                                        f"What is still missing? (up to 3, or NONE)"},
        ], should_stop=should_stop)
        gaps = [re.sub(r"^[\-\*\d\.\)\s]+", "", ln).strip() for ln in gap_raw.splitlines() if ln.strip()]
        gaps = [g for g in gaps if len(g) > 8 and not g.upper().startswith("NONE")][:3]
        for j, g in enumerate(gaps, len(findings) + 1):
            if should_stop and should_stop():
                break
            f = do_finding(j, g)
            if f:
                findings.append(f)

    if not findings:
        msg = "No findings — is SearXNG running / network up?"
        emit_event(emit, {"type": "assistant", "text": msg})
        return msg
    if should_stop and should_stop():
        return ""

    # 3) SYNTHESIZE — a grounded draft report from the collected findings (+ the original source).
    emit_event(emit, {"type": "action", "kind": "plan", "label": "synthesizing report"})
    emit_event(emit, {"type": "thinking"})
    fblob = "\n\n".join(f"[{f['n']}] Q: {f['q']}\n{f['finding']}" for f in findings)
    orig_block = f"\n\nORIGINAL SOURCE (what the source itself says):\n{original[:1500]}" if original else ""
    stream_cb = (lambda t: emit_event(emit, {"type": "token", "text": t})) if emit else None
    draft = ask_model([
        {"role": "system", "content": with_memory(RESEARCH_SYNTH_PROMPT)},
        {"role": "user", "content": f"TOPIC:\n{topic[:800]}{orig_block}\n\nFINDINGS:\n{fblob[:4500]}\n\nWrite the report."},
    ], should_stop=should_stop).strip()

    # 4) VERIFY (CRAG-style fact-check) — drop any claim not supported by the findings; stream the
    #    corrected version as the final answer. This is the span-level verification step.
    report = draft
    if draft and not (should_stop and should_stop()):
        emit_event(emit, {"type": "action", "kind": "plan", "label": "fact-checking the report"})
        emit_event(emit, {"type": "thinking"})
        report = ask_model([
            {"role": "system", "content": RESEARCH_VERIFY_PROMPT},
            {"role": "user", "content": f"FINDINGS:\n{fblob[:4500]}\n\nDRAFT REPORT:\n{draft}\n\n"
                                        f"Return the corrected report."},
        ], stream_cb=stream_cb, should_stop=should_stop).strip() or draft

    seen, links = set(), []
    for u in src_list:
        if u not in seen:
            seen.add(u)
            links.append(u)
    full = report + "\n\n— sources —\n" + "\n".join(f"  • {u}" for u in links[:12])
    emit_event(emit, {"type": "assistant", "text": full})
    print(report)
    return full


def _ollama_chat(messages, model, think, stream_cb=None, should_stop=None,
                 num_ctx=8192, temperature=0.2):
    """Native local Ollama backend (default). If stream_cb is given, stream tokens to it
    (used by the server UI so the slow CPU model 'feels alive') and return the full text.

    `num_ctx`/`temperature` are sent as per-request Ollama options. Per-request
    options.num_ctx OVERRIDES the Modelfile default (Ollama API), so the context window
    is now runtime-configurable from the CLI/GUI with NO Modelfile rebuild. Defaults match
    the value codegen used historically, so behavior is unchanged unless overridden.

    We ALWAYS request a streamed response — even when stream_cb is None — so should_stop()
    is polled per line on every call. Otherwise a non-streaming urlopen blocks up to 600s
    and ignores Stop, leaving the run 'active' and refusing new tasks (the classic
    'a run is already active' lockup)."""
    payload = {"model": model, "stream": True, "messages": messages,
               "think": think,  # False = fast; True = reasoning (used only for planning)
               "options": {"num_ctx": num_ctx, "temperature": temperature}}
    req = urllib.request.Request(OLLAMA_URL, data=json.dumps(payload).encode(),
                                 headers={"Content-Type": "application/json"})
    parts = []
    with urllib.request.urlopen(req, timeout=600) as resp:
        for line in resp:  # Ollama streams one JSON object per line
            if should_stop and should_stop():
                break  # abort generation immediately on Stop
            line = line.strip()
            if not line:
                continue
            try:
                obj = json.loads(line)
            except ValueError:
                continue
            tok = obj.get("message", {}).get("content", "")
            if tok:
                parts.append(tok)
                if stream_cb:
                    stream_cb(tok)
            if obj.get("done"):
                break
    return "".join(parts)


def _openai_chat(messages, model, stream_cb=None, should_stop=None, temperature=0.2):
    """OpenAI-compatible cloud backend (DeepSeek / GLM / Kimi / OpenRouter / OpenAI / Ollama-/v1).
    This is how the strong local harness borrows a frontier-class brain cheaply."""
    # Always stream so should_stop() is polled per line on every call (see _ollama_chat).
    payload = {"model": model, "messages": messages, "stream": True, "temperature": temperature}
    headers = {"Content-Type": "application/json"}
    if API_KEY:
        headers["Authorization"] = f"Bearer {API_KEY}"
    req = urllib.request.Request(BASE_URL.rstrip("/") + "/chat/completions",
                                 data=json.dumps(payload).encode(), headers=headers)
    parts = []
    with urllib.request.urlopen(req, timeout=600) as resp:
        for line in resp:  # SSE: "data: {json}" lines, terminated by "data: [DONE]"
            if should_stop and should_stop():
                break  # abort generation immediately on Stop
            line = line.strip()
            if not line.startswith(b"data:"):
                continue
            chunk = line[5:].strip()
            if chunk == b"[DONE]":
                break
            try:
                obj = json.loads(chunk)
            except ValueError:
                continue
            tok = (obj.get("choices") or [{}])[0].get("delta", {}).get("content", "")
            if tok:
                parts.append(tok)
                if stream_cb:
                    stream_cb(tok)
    return "".join(parts)


def _resolve_knobs(mdl, num_ctx, temperature):
    """Resolve (num_ctx, temperature) for a call. Precedence: explicit arg > NUM_CTX global
    > the model's entry in model_registry > the legacy defaults. Degrades gracefully if the
    registry module/file is absent, so the CLI behaves exactly as before when unconfigured."""
    nctx = num_ctx or NUM_CTX
    try:
        import model_registry
        rt = model_registry.resolve_runtime(mdl, num_ctx=nctx, temperature=temperature)
        return rt["num_ctx"], rt["temperature"]
    except Exception:  # noqa: BLE001 — registry must never break a run
        return (nctx or 8192), (0.2 if temperature is None else temperature)


# ── provider backends (Strategy + Dependency Inversion) ───────────────────────
# ask_model depends on this ChatBackend abstraction, not on the concrete network
# functions. Adding a provider = a new subclass + a branch in select_backend(),
# without touching the agent loop. The classes delegate to the proven _ollama_chat
# / _openai_chat implementations, so behavior is byte-identical to before.
class ChatBackend:
    """Strategy interface: produce one chat completion for `messages`."""
    name = "base"

    def chat(self, messages, model, think=False, stream_cb=None, should_stop=None,
             num_ctx=8192, temperature=0.2):
        raise NotImplementedError


class OllamaBackend(ChatBackend):
    name = "ollama"

    def chat(self, messages, model, think=False, stream_cb=None, should_stop=None,
             num_ctx=8192, temperature=0.2):
        return _ollama_chat(messages, model, think, stream_cb=stream_cb, should_stop=should_stop,
                            num_ctx=num_ctx, temperature=temperature)


class OpenAIBackend(ChatBackend):
    name = "openai"

    def chat(self, messages, model, think=False, stream_cb=None, should_stop=None,
             num_ctx=8192, temperature=0.2):
        # OpenAI-compatible endpoints ignore think/num_ctx (the server owns the context window).
        return _openai_chat(messages, model, stream_cb=stream_cb, should_stop=should_stop,
                            temperature=temperature)


def select_backend():
    """DIP selector: cloud (OpenAI-compatible) when BASE_URL is set, else local Ollama."""
    return OpenAIBackend() if BASE_URL else OllamaBackend()


def ask_model(messages, model=None, think=False, stream_cb=None, should_stop=None,
              num_ctx=None, temperature=None):
    """One brain call. Routes via a ChatBackend strategy: cloud (OpenAI-compatible) when
    BASE_URL is set, else local Ollama. The harness is identical either way.
    stream_cb (optional): called with each token as it arrives (server UI streaming).
    should_stop (optional): polled mid-stream so Stop aborts generation, not just between steps.
    num_ctx/temperature (optional): per-call overrides; otherwise resolved from model_registry."""
    mdl = model or MODEL
    nctx, temp = _resolve_knobs(mdl, num_ctx, temperature)
    print("⏳ thinking...", file=sys.stderr)
    try:
        content = select_backend().chat(messages, mdl, think=think, stream_cb=stream_cb,
                                        should_stop=should_stop, num_ctx=nctx, temperature=temp)
    except urllib.error.HTTPError as e:
        body = e.read().decode("utf-8", "ignore")[:300] if hasattr(e, "read") else ""
        raise RuntimeError(f"backend HTTP {e.code} from {'cloud' if BASE_URL else 'ollama'}: {body}") from e
    return strip_think(content)   # drop <think> reasoning so the rest of the harness sees clean text


def confirm(actions, base, auto_yes):
    """Show actions that change state; ask once unless -y. Returns True to proceed."""
    changing = [a for a in actions if a[0] in ("file", "run")]
    if not changing:
        return True
    print(f"\n📋 {len(changing)} action(s) in {base}:")
    for kind, target, content in changing:
        if kind == "file":
            ex = "  ⚠️ overwrites" if os.path.exists(os.path.join(base, target)) else ""
            print(f"   📝 {target} ({content.count(chr(10)) + 1} lines){ex}")
        else:
            print(f"   ▶️  {target}")
    if auto_yes:
        return True
    return input("Proceed? [y/N] ").strip().lower() in ("y", "yes")


def confine_path(base, target):
    """Confine a target path to the project dir so the agent can NEVER read/write outside
    `base`. A small model sometimes invents absolute paths like /usr/local/bin/x — those
    collapse to just the filename inside the project. Real `..` escapes are rejected (None)."""
    t = str(target).strip().replace("\\", "/")
    base_abs = os.path.abspath(base)
    if t.startswith("/"):                       # absolute → keep only the filename, in project root
        t = os.path.basename(t.rstrip("/"))
    full = os.path.normpath(os.path.join(base_abs, t))
    if full != base_abs and not full.startswith(base_abs + os.sep):
        return None                             # .. traversal that escapes the project
    return full


# ── tool runners (one per tool; registered in _TOOL_RUNNERS) ──────────────────
# Each runner has the SAME signature (target, content, base, emit, should_stop)
# and returns the result text fed back to the model (or None). This is the
# execution half of the tool registry — add a tool = add a runner + a TOOL_SPEC.
def _run_file(target, content, base, emit, should_stop):
    denied = hook_denied(target)
    if denied:
        print(f"🪝 blocked by hook (deny '{denied}'): {target}", file=sys.stderr)
        return f"WRITE to {target} BLOCKED by hook (matches deny pattern '{denied}'). Do not write this file."
    full = confine_path(base, target)
    if full is None:
        print(f"🛑 blocked write outside project: {target}", file=sys.stderr)
        return (f"WRITE to {target} BLOCKED: path is outside the project directory. "
                f"Use a RELATIVE path inside the project (e.g. notes.txt), not an absolute one.")
    os.makedirs(os.path.dirname(full) or ".", exist_ok=True)
    with open(full, "w") as f:
        f.write(content if content.endswith("\n") else content + "\n")
    print(f"✅ wrote {target}")
    if HOOKS.get("post_file"):
        cmd = HOOKS["post_file"].replace("{file}", target)
        print(f"🪝 post_file: {cmd}", file=sys.stderr)
        ok, report = run_hook_cmd(cmd, base, timeout=120)
        if not ok:
            return f"post_file hook on {target} reported issues — fix them:\n{report}"
    return None


def _run_list(target, content, base, emit, should_stop):
    tree = project_tree(base)
    print("📂 (sent project tree to model)")
    return f"Project tree:\n{tree}"


def _run_read(target, content, base, emit, should_stop):
    full = confine_path(base, target)
    if full is None:
        return f"READ {target} BLOCKED: path is outside the project directory."
    try:
        with open(full) as f:
            data = f.read()
        print(f"📖 read {target}")
        return f"Content of {target}:\n{trunc(data)}"
    except OSError as e:
        return f"READ {target} failed: {e}"


def _run_search(target, content, base, emit, should_stop):
    print(f"🔎 searching: {target}")
    return f"Search results for '{target}':\n{trunc(web_search(target))}"


def _run_api(target, content, base, emit, should_stop):
    print(f"🌐 API GET: {target}")
    return f"API response from {target}:\n{trunc(http_get(target))}"


def _run_ragsearch(target, content, base, emit, should_stop):
    roots = [base] + RAG_DIRS
    print(f"🔎 RAG (indexed files): {target}")
    hits = rag_tool.search(target, roots, log=lambda m: print(m, file=sys.stderr))
    return f"RAG results for '{target}' (from your indexed code/docs):\n{trunc(hits)}"


def _run_docrag(target, content, base, emit, should_stop):
    print(f"📑 DocRAG (documents db): {target}")
    try:
        from docrag import db as _dr_db, router as _dr_router
        con = _dr_db.connect()
        _dr_db.init_db(con)
        ans = _dr_router.answer(con, target, log=lambda *a: None)
        con.close()
    except Exception as e:
        return (f"DocRAG unavailable ({e}). Ingest documents first: "
                f"python3 -m docrag ingest <dir>")
    return f"DocRAG answer for '{target}' (from your ingested documents):\n{trunc(ans)}"


def _run_research(target, content, base, emit, should_stop):
    query = content  # the search query; `target` is the file to write
    print(f"🔎 researching: {query}  →  {target}")
    facts = web_search(query)
    emit_event(emit, {"type": "thinking"})  # styling the results is a slow sub-step
    stream_cb = (lambda t: emit_event(emit, {"type": "token", "text": t})) if emit else None
    styled = ask_model([
        {"role": "system", "content": STYLE_PROMPT},
        {"role": "user", "content":
            f"Facts from a web search:\n{facts}\n\nWrite the markdown content for '{target}', presenting this "
            f"information clearly and concisely. Use ONLY these facts — no version/number/date from your own memory."},
    ], stream_cb=stream_cb, should_stop=should_stop).strip()
    if should_stop and should_stop():
        return "Stopped before writing the research file."
    body = strip_fences(styled)
    if not body.strip():
        body = f"# {target}\n\n{facts}\n"
    full = confine_path(base, target)
    if full is None:
        print(f"🛑 blocked research write outside project: {target}", file=sys.stderr)
        return (f"RESEARCH file {target} BLOCKED: path is outside the project directory. "
                f"Here are the facts instead:\n{trunc(facts)}")
    os.makedirs(os.path.dirname(full) or ".", exist_ok=True)
    with open(full, "w") as f:
        f.write(body if body.endswith("\n") else body + "\n")
    print(f"✅ wrote {target} (styled from web search)")
    return f"Wrote {target} from web search results (grounded in real data)."


def _run_remember(target, content, base, emit, should_stop):
    print(f"🧠 remember: {target[:60]}")
    return mem0_tool.remember(target)


def _run_recall(target, content, base, emit, should_stop):
    print(f"🧠 recall: {target}")
    return f"Memories for '{target}':\n{trunc(mem0_tool.recall(target))}"


def _run_run(target, content, base, emit, should_stop):
    if DANGER.search(target):
        print(f"🛑 blocked dangerous command: {target}", file=sys.stderr)
        return f"RUN blocked (dangerous): {target}"
    print(f"▶️  {target}")
    try:
        r = subprocess.run(target, shell=True, cwd=base, capture_output=True,
                           text=True, timeout=180)
        out = (r.stdout + r.stderr).strip() or "(no output)"
        return f"$ {target}\n(exit {r.returncode})\n{trunc(out)}"
    except subprocess.TimeoutExpired:
        return f"$ {target}\n(timed out after 180s)"


_TOOL_RUNNERS = {
    "file": _run_file, "list": _run_list, "read": _run_read, "search": _run_search,
    "api": _run_api, "ragsearch": _run_ragsearch, "docrag": _run_docrag,
    "research": _run_research, "remember": _run_remember, "recall": _run_recall,
    "run": _run_run,
    # "done" has no runner — agent_loop handles completion.
}


def do_action(kind, target, content, base, emit=None, should_stop=None):
    """Execute one action via the tool registry; return the result text (or None).
    `emit` (server mode) lets long sub-steps surface a UI indicator; None for the CLI.
    `should_stop` lets the slow research sub-call abort mid-generation on Stop."""
    runner = _TOOL_RUNNERS.get(kind)
    return runner(target, content, base, emit, should_stop) if runner else None


def compact_history(messages):
    """Summarize old turns instead of dropping them, so long tasks keep their context.

    Keeps the system prompt + a fresh progress summary + the most recent turns. This
    is the 'phased compaction' pattern: the 8K window stays bounded but what was already
    done is preserved as a note rather than silently truncated."""
    if len(messages) <= COMPACT_AT:
        return messages
    system, head, tail = messages[0], messages[1:-KEEP_RECENT], messages[-KEEP_RECENT:]
    convo = "\n\n".join(f"{m['role'].upper()}: {trunc(m['content'])}" for m in head)
    try:
        summary = ask_model([
            {"role": "system", "content": COMPACT_PROMPT},
            {"role": "user", "content": f"Conversation so far:\n{convo}\n\nWrite the progress summary."},
        ]).strip()
    except Exception as e:
        print(f"⚠️  compaction failed ({e}); falling back to truncation.", file=sys.stderr)
        return [system] + tail
    print("🗜️  compacted older history into a progress summary.", file=sys.stderr)
    return [system, {"role": "user", "content": f"[Earlier progress summary]\n{summary}"}] + tail


def git_checkpoint(base, msg):
    """Best-effort git commit so a bad step can be undone (git reset --hard HEAD~1).
    Inits a repo on first use; silently no-ops if git is missing. Never fatal."""
    try:
        if not os.path.isdir(os.path.join(base, ".git")):
            subprocess.run(["git", "init", "-q"], cwd=base, check=True, capture_output=True, text=True)
            # local identity so commits work even without a global git config
            subprocess.run(["git", "config", "user.email", "codegen@local"], cwd=base, capture_output=True)
            subprocess.run(["git", "config", "user.name", "codegen-agent"], cwd=base, capture_output=True)
            print("📍 git: initialized a checkpoint repo here.", file=sys.stderr)
        subprocess.run(["git", "add", "-A"], cwd=base, capture_output=True, text=True)
        r = subprocess.run(["git", "commit", "-q", "-m", msg], cwd=base, capture_output=True, text=True)
        if r.returncode == 0:
            print(f"📍 checkpoint: {msg}", file=sys.stderr)
    except FileNotFoundError:
        pass  # git not installed — checkpoints are optional
    except Exception as e:
        print(f"⚠️  git checkpoint skipped: {e}", file=sys.stderr)


def diagnose(task, base):
    """#8/#9 — study the project BEFORE acting. The SCRIPT surfaces relevant existing code via
    RAG (learn the patterns), then ONE clean model call produces a short diagnosis/plan. Returns
    the briefing string (or '' if nothing useful / disabled). Cheap recon to anchor the 3B."""
    roots = [base] + RAG_DIRS
    print("🔬 diagnosing — studying existing code before writing...", file=sys.stderr)
    try:
        hits = rag_tool.search(task, roots, k=4)
    except Exception:
        hits = ""
    existing = "" if (not hits or hits.startswith("RAG")) else f"\nRelevant existing code/docs:\n{trunc(hits)}\n"
    try:
        brief = ask_model([
            {"role": "system", "content": with_memory(DIAGNOSE_PROMPT)},
            {"role": "user", "content":
                f"Task:\n{task}\n\nProject tree:\n{project_tree(base)}\n{existing}\nWrite the diagnosis briefing."},
        ]).strip()
    except Exception as e:
        print(f"⚠️  diagnosis skipped ({e}).", file=sys.stderr)
        return ""
    if brief:
        print("🔬 diagnosis:\n" + brief, file=sys.stderr)
    return brief


def review_work(task, base, written):
    """#7 — a specialized REVIEWER sub-agent in a FRESH, clean context (read-only: the script feeds
    it the written files; it has no write tools). Returns findings text, or '' if it says LGTM."""
    blob = ""
    for p in sorted(written):
        try:
            with open(os.path.join(base, p)) as f:
                blob += f"\n--- {p} ---\n{f.read()}"
        except OSError:
            pass
    if not blob.strip():
        return ""
    print("👓 reviewer sub-agent checking the work...", file=sys.stderr)
    verdict = ask_model([
        {"role": "system", "content": with_memory(REVIEWER_PROMPT)},
        {"role": "user", "content": f"Task:\n{task}\n\nFiles the agent wrote:\n{trunc(blob)}\n\nReview now."},
    ]).strip()
    if not verdict or verdict.upper().startswith("LGTM"):
        print("👓 reviewer: LGTM", file=sys.stderr)
        return ""
    print("👓 reviewer findings:\n" + verdict, file=sys.stderr)
    return verdict


# ── UI callbacks (server mode) ───────────────────────────────────────────────
# server.py injects `emit` (event sink) and `approver` (per-action approval) into
# the functions below. Both default to None → the CLI keeps its exact print()/input()
# behavior, so nothing here changes how `ai` runs from the terminal.

def emit_event(emit, event):
    """Fire a UI event if an emitter is wired (server mode); a no-op for the CLI."""
    if emit:
        try:
            emit(event)
        except Exception:  # noqa: BLE001 — UI delivery must never break the agent
            pass


def action_label(kind, target, content):
    """Short human-readable label for an 'action' UI indicator."""
    if kind == "research":
        return f"{content} => {target}"
    if kind == "list":
        return "project tree"
    return target or kind


def approve_action(kind, target, content, base, approver):
    """Per-action approval for the webview (replaces the CLI batch confirm()).
    Publishes a file_proposed/run_proposed event and blocks until the UI answers.
    Returns (approved, content) — content may be edited by the user before writing."""
    aid = uuid.uuid4().hex[:8]
    if kind == "file":
        full = os.path.join(base, target)
        old = ""
        if os.path.exists(full):
            try:
                with open(full) as f:
                    old = f.read()
            except OSError:
                pass
        ev = {"type": "file_proposed", "id": aid, "path": target,
              "oldText": old, "newText": content}
    else:  # run
        ev = {"type": "run_proposed", "id": aid, "command": target}
    res = approver(ev) or {}
    if not res.get("approved"):
        return False, content
    edited = res.get("edited")
    if kind == "file" and edited is not None:
        content = edited
    return True, content


def agent_loop(task, messages, base, auto_yes, max_iters, emit=None, approver=None, should_stop=None):
    mems = mem0_tool.auto_recall(task) if AUTO_RECALL else ""
    mem_block = f"Relevant remembered facts:\n{mems}\n\n" if mems else ""
    brief = diagnose(task, base) if DIAGNOSE else ""
    brief_block = f"Diagnosis (study before acting — follow it):\n{brief}\n\n" if brief else ""
    # In assistant mode skip the project tree (non-code Q&A) and label it a Question.
    tree_block = "" if ASSISTANT else f"Project tree:\n{project_tree(base)}\n\n"
    label = "Question" if ASSISTANT else "Task"
    messages.append({"role": "user", "content":
                     f"{mem_block}{brief_block}{tree_block}{label}:\n{task}"})
    did_work = False    # has any file been written this task?
    critiqued = False   # #5: the one self-critique pass has run
    written = set()     # paths written this task (for the #7 reviewer)
    for i in range(1, max_iters + 1):
        if should_stop and should_stop():
            print("⏹️  stopped by user.", file=sys.stderr)
            emit_event(emit, {"type": "assistant", "text": "⏹️ Stopped."})
            break
        print(f"\n── step {i}/{max_iters} ──", file=sys.stderr)
        emit_event(emit, {"type": "thinking"})
        # Stream tokens to the UI (server mode) so the slow model feels alive.
        stream_cb = (lambda t: emit_event(emit, {"type": "token", "text": t})) if emit else None
        raw = ask_model(messages, stream_cb=stream_cb, should_stop=should_stop)
        messages.append({"role": "assistant", "content": raw})
        # If Stop fired mid-generation, bail before acting on the partial output.
        if should_stop and should_stop():
            print("⏹️  stopped by user.", file=sys.stderr)
            emit_event(emit, {"type": "assistant", "text": "⏹️ Stopped."})
            break
        actions = parse_actions(raw)

        if not actions:
            ans = raw.strip()
            if ans:
                print(ans)   # model answered in prose (e.g. a Q&A / search summary) — show it
                emit_event(emit, {"type": "assistant", "text": ans})
            else:
                print("⚠️  the model returned only reasoning, no answer. Reasoning models (qwen3*) burn "
                      "the token budget on <think>; use the fast model:  --model qwen-fast", file=sys.stderr)
                emit_event(emit, {"type": "assistant",
                                  "text": "The model returned only reasoning. Try the fast model (qwen-fast)."})
            break

        wants_done = any(a[0] == "done" for a in actions)
        work_actions = [a for a in actions
                        if a[0] in ("file", "run", "read", "list", "search", "ragsearch", "api",
                                    "research", "remember", "recall")]

        # Search-grounding: if the model wants to SEARCH and WRITE in the same step,
        # run the search FIRST and DEFER the writes — so the file is written from the
        # results, not from the model's (stale) memory. The script enforces the order.
        defer_writes = (any(a[0] == "search" for a in actions)
                        and any(a[0] in ("file", "run") for a in actions))
        if defer_writes:
            work_actions = [a for a in work_actions
                            if a[0] in ("search", "ragsearch", "api", "read", "list", "recall")]
            wants_done = False
            print("🔒 grounding: search first, deferring writes until results are in.", file=sys.stderr)

        # Reject premature DONE before any file was ever written (coding mode only)
        if not ASSISTANT and wants_done and not did_work and not any(a[0] in ("file", "research") for a in actions):
            print("↩️  premature DONE — nudging the model to actually write files.", file=sys.stderr)
            messages.append({"role": "user", "content":
                "You have not written any files yet. Do NOT say DONE. "
                "Output the required <<<FILE: ...>>> blocks with the full code now."})
            continue

        # #5 self-critique: one review of the proposed files BEFORE writing them.
        file_actions = [a for a in work_actions if a[0] == "file"]
        if CRITIQUE and file_actions and not critiqued:
            critiqued = True
            proposed = "\n\n".join(f"--- {t} ---\n{c}" for _, t, c in file_actions)
            verdict = ask_model([
                {"role": "system", "content": with_memory(CRITIQUE_PROMPT)},
                {"role": "user", "content": f"Task:\n{task}\n\nProposed files:\n{trunc(proposed)}\n\nReview now."},
            ]).strip()
            if verdict and not verdict.upper().startswith("OK"):
                print("🧐 self-critique found issues — revising before writing.", file=sys.stderr)
                messages.append({"role": "user", "content":
                    f"Self-review of your proposed files found issues. Revise and re-output the corrected "
                    f"<<<FILE>>> blocks (do not write anything else yet):\n{verdict}"})
                continue
            print("🧐 self-critique: OK", file=sys.stderr)

        # Approval: per-action when a UI approver is wired (server), else the CLI batch confirm().
        if approver is None:
            if not confirm(work_actions, base, auto_yes):
                print("Skipped by user.")
                break

        results = []
        wrote_file = False
        for kind, target, content in work_actions:
            # UI indicator for read-only / non-approval tools (search, rag, read, …).
            if emit and kind not in ("file", "run"):
                emit_event(emit, {"type": "action", "kind": kind,
                                  "label": action_label(kind, target, content)})
            # Per-action approval (server UI) for state-changing actions.
            if approver is not None and kind in ("file", "run"):
                ok, content = approve_action(kind, target, content, base, approver)
                if not ok:
                    results.append(f"User rejected the {kind}: {target}")
                    emit_event(emit, {"type": "assistant", "text": f"Rejected {kind}: {target}"})
                    continue
            r = do_action(kind, target, content, base, emit=emit, should_stop=should_stop)
            if kind in ("file", "research"):
                did_work = True
                wrote_file = True
                if kind == "research" or not hook_denied(target):
                    written.add(target)
                if (kind == "research") or (kind == "file" and r is None):  # write succeeded
                    emit_event(emit, {"type": "file_written", "path": target})
            if r:
                results.append(r)
                emit_event(emit, {"type": "result", "text": r})

        if GIT_CHECKPOINT and wrote_file:
            git_checkpoint(base, f"step {i}: {task[:60]}")

        if wants_done and (did_work or ASSISTANT):
            if not ASSISTANT:   # coding-mode finish gates (tests + reviewer); skipped for the assistant
                ok, report = run_stop_hook(base)
                if not ok:
                    print("🪝 stop hook failed — sending the error back for a fix.", file=sys.stderr)
                    messages.append({"role": "user", "content":
                        f"The project check must pass before finishing, but it FAILED. "
                        f"Fix the cause, then emit <<<DONE>>> again.\n{report}"})
                    messages[:] = compact_history(messages)
                    continue
                if REVIEW:
                    findings = review_work(task, base, written)
                    if findings:
                        print("👓 reviewer gated DONE — sending findings back for a fix.", file=sys.stderr)
                        messages.append({"role": "user", "content":
                            f"A code reviewer found issues before finishing. Fix them, then emit <<<DONE>>>:\n{findings}"})
                        messages[:] = compact_history(messages)
                        continue
            print("🎉 done." if ASSISTANT else "🎉 agent reported DONE.")
            break

        feedback = "\n\n".join(results) if results else "Files written. Continue, or emit <<<DONE>>> if finished."
        if defer_writes:
            feedback += ("\n\n⚠️ AUTHORITATIVE DATA: base your file ONLY on the search results above. "
                         "Do NOT use any number, version, price, or date from your own memory — use ONLY what the "
                         "results state. Now write the file.")
        messages.append({"role": "user", "content": f"Tool results:\n{feedback}"})

        messages[:] = compact_history(messages)  # summarize old turns when context grows
    else:
        print(f"⏹️  reached max iterations ({max_iters}).")


def registry_default(role):
    """Resolve the registry's default model id for a role ('executor'/'planner'/…), or ''.
    Lets the model-registry's role assignments drive the engine with no code edits."""
    try:
        import model_registry
        name = model_registry.default_for(role)
        if name:
            return model_registry.resolve_runtime(name)["model"]
    except Exception:  # noqa: BLE001 — registry must never break a run
        pass
    return ""


def planner_model():
    """Which model plans in build mode. Precedence: --planner > registry 'planner' role > executor."""
    return PLANNER_MODEL or registry_default("planner") or MODEL


def plan_task(task, base):
    """Ask the model for a list of files (path :: spec). One clean context."""
    raw = ask_model([
        {"role": "system", "content": with_memory(PLANNER_PROMPT)},
        {"role": "user", "content": f"Project tree:\n{project_tree(base)}\n\nTask:\n{task}"},
    ], model=planner_model(), think=THINK_PLAN)
    plan = []
    for line in raw.splitlines():
        line = line.strip().lstrip("-*0123456789.) ").strip()
        if "::" in line:
            p, _, spec = line.partition("::")
            p = clean_path(p)
            if p and ("/" in p or "." in p):
                plan.append((p, spec.strip()))
    return plan


def build_one(goal, plan, path, spec, prior, emit=None, should_stop=None):
    """Generate ONE file in a fresh context, with the plan + prior files for consistency.
    Streams tokens to the UI and aborts on Stop when those callbacks are wired (server)."""
    plan_txt = "\n".join(f"{p} :: {s}" for p, s in plan)
    ctx = f"Overall goal:\n{goal}\n\nProject plan (all files):\n{plan_txt}\n"
    inc, budget = "", 3000  # include prior files for consistency, capped
    for p, c in prior.items():
        if len(inc) + len(c) < budget:
            inc += f"\n--- {p} ---\n{c}\n"
    if inc:
        ctx += f"\nAlready-written files (match their imports/exports):\n{inc}\n"
    ctx += f"\nNow write ONLY this file:\n{path}\nSpec: {spec}\nOutput a single <<<FILE: {path}>>> block."
    stream_cb = (lambda t: emit_event(emit, {"type": "token", "text": t})) if emit else None
    raw = ask_model([
        {"role": "system", "content": with_memory(EXECUTOR_PROMPT)},
        {"role": "user", "content": ctx},
    ], stream_cb=stream_cb, should_stop=should_stop)
    files = [a for a in parse_actions(raw) if a[0] == "file"]
    return files[0][2] if files else None


def fix_loop(goal, base, state, path, content):
    """Validate a just-written file; on failure, re-ask with ONLY the related contracts
    injected (not the whole history), up to the escalation cap. Returns (content, ok)."""
    ok, err = contracts.validate_file(base, path)
    while not ok:
        s = contracts.find_step(state, path)
        if not s or s["attempts"] >= contracts.MAX_ATTEMPTS:
            contracts.mark(state, path, "failed", err)
            print(f"🚫 {path} still failing after {s['attempts'] if s else '?'} tries → needs_human.",
                  file=sys.stderr)
            break
        contracts.mark(state, path, "failed", err)   # bumps attempts
        contracts.save(base, state)
        print(f"🔧 fixing {path} (attempt {s['attempts']}): {err.splitlines()[0]}", file=sys.stderr)
        related = contracts.find_related(state, err)
        prompt = contracts.build_fix_context(goal, related, path, content, err, base)
        raw = ask_model([
            {"role": "system", "content": with_memory(EXECUTOR_PROMPT)},
            {"role": "user", "content": prompt},
        ])
        files = [a for a in parse_actions(raw) if a[0] == "file"]
        if not files:
            continue                                 # no file emitted; loop counts as a try
        content = files[0][2]
        full = os.path.join(base, path)
        with open(full, "w") as f:
            f.write(content if content.endswith("\n") else content + "\n")
        ok, err = contracts.validate_file(base, path)
    if ok:
        contracts.mark(state, path, "done", "")
    contracts.save(base, state)
    return content, ok


def orchestrate(task, base, auto_yes, emit=None, approver=None, should_stop=None):
    """Plan a big task into files, then build each in a fresh context (sub-agent style)."""
    print("🧭 Planning the build...", file=sys.stderr)
    emit_event(emit, {"type": "action", "kind": "plan", "label": "planning the build"})
    emit_event(emit, {"type": "thinking"})
    plan = plan_task(task, base)
    if not plan:
        print("⚠️  Planner produced no file list. Try a clearer task or use the normal agent.")
        emit_event(emit, {"type": "error", "text": "Planner produced no file list."})
        return
    print(f"\n📐 Plan — {len(plan)} files:")
    for p, s in plan:
        print(f"   • {p} :: {s}")
    emit_event(emit, {"type": "assistant",
                      "text": "📐 Plan — %d files:\n%s" % (
                          len(plan), "\n".join(f"• {p} :: {s}" for p, s in plan))})
    if approver is not None:
        res = approver({"type": "run_proposed", "id": uuid.uuid4().hex[:8],
                        "command": f"Build these {len(plan)} files:\n"
                                   + "\n".join(p for p, _ in plan)}) or {}
        if not res.get("approved"):
            print("Cancelled.")
            emit_event(emit, {"type": "assistant", "text": "Build cancelled."})
            return
    elif not auto_yes and input("\nBuild these files? [y/N] ").strip().lower() not in ("y", "yes"):
        print("Cancelled.")
        return

    prior, produced = {}, {}
    state = contracts.save_state(base, plan, produced)   # seed state.json (all pending)
    for idx, (path, spec) in enumerate(plan, 1):
        if should_stop and should_stop():
            print("⏹️  stopped by user.", file=sys.stderr)
            emit_event(emit, {"type": "assistant", "text": "⏹️ Stopped."})
            break
        print(f"\n── file {idx}/{len(plan)}: {path} ──", file=sys.stderr)
        emit_event(emit, {"type": "action", "kind": "file", "label": f"{idx}/{len(plan)}: {path}"})
        emit_event(emit, {"type": "thinking"})
        content = build_one(task, plan, path, spec, prior, emit=emit, should_stop=should_stop)
        if should_stop and should_stop():
            print("⏹️  stopped by user.", file=sys.stderr)
            emit_event(emit, {"type": "assistant", "text": "⏹️ Stopped."})
            break
        if not content:
            print(f"⚠️  failed to generate {path} — skipping.")
            emit_event(emit, {"type": "error", "text": f"failed to generate {path}"})
            contracts.mark(state, path, "failed", "no file emitted by model")
            contracts.save(base, state)
            continue
        if approver is not None:
            ok, content = approve_action("file", path, content, base, approver)
            if not ok:
                emit_event(emit, {"type": "assistant", "text": f"Skipped {path} (rejected)."})
                contracts.mark(state, path, "failed", "rejected by user")
                contracts.save(base, state)
                continue
        full = os.path.join(base, path)
        os.makedirs(os.path.dirname(full) or ".", exist_ok=True)
        with open(full, "w") as f:
            f.write(content if content.endswith("\n") else content + "\n")
        # record the contract this file exposes, then validate + focused fix loop
        produced[path] = contracts.extract_produces(path, content)
        state = contracts.save_state(base, plan, produced)
        content, ok = fix_loop(task, base, state, path, content)
        produced[path] = contracts.extract_produces(path, content)   # re-extract after any fix
        contracts.save_state(base, plan, produced)
        prior[path] = content
        if GIT_CHECKPOINT:
            git_checkpoint(base, f"build {idx}/{len(plan)}: {path}")
        print(f"{'✅' if ok else '⚠️ '} {path}")
        emit_event(emit, {"type": "file_written", "path": path})

    state = contracts.load_state(base)
    done = sum(1 for s in state["steps"] if s["status"] == "done")
    print(f"\n🎉 Build complete — {done}/{len(plan)} files validated.")
    emit_event(emit, {"type": "assistant", "text": f"🎉 Build complete — {done}/{len(plan)} files validated."})
    if state["needs_human"]:
        stuck = [s["path"] for s in state["steps"] if s["status"] == "failed"]
        print(f"🙋 needs_human — still failing after {contracts.MAX_ATTEMPTS} tries: {stuck}")
        print(f"   See {os.path.join(base, 'state.json')} for the contracts + errors.")


PASTE_DRAIN = 0.12  # seconds to wait for more buffered lines (paste vs. typing)


def read_request(prompt="code> "):
    """Read one request, merging a multi-line PASTE into a SINGLE request.

    A pasted block lands in the terminal buffer all at once, so after the first
    line the rest are already waiting; we drain them with a short select() poll.
    A human typing one line and pressing Enter hits the timeout immediately, so
    ordinary single-line input still returns with no perceptible delay.

    We read the raw fd (os.read) rather than input()/readline so the OS buffer —
    not Python's hidden line buffer — is what select() reports on; otherwise
    pasted lines stuck in Python's buffer would be silently missed.
    """
    sys.stdout.write(prompt)
    sys.stdout.flush()
    fd = sys.stdin.fileno()
    data = os.read(fd, 65536)          # blocks until the first line is entered
    if not data:                       # EOF (Ctrl-D)
        raise EOFError
    while select.select([fd], [], [], PASTE_DRAIN)[0]:
        more = os.read(fd, 65536)
        if not more:
            break
        data += more
    return data.decode("utf-8", "ignore").strip()


def interactive(messages, base, auto_yes, max_iters):
    if ASSISTANT:
        print("💬 Assistant mode. Ask anything — web search / API / RAG available. 'exit' to quit.")
    else:
        print("💬 Autonomous chat mode. Type a request (multi-line paste is OK). 'exit' to quit.")
    print(f"📁 Target directory: {base}\n")
    while True:
        try:
            task = read_request("ask> " if ASSISTANT else "code> ")
        except (EOFError, KeyboardInterrupt):
            print("\nBye 👋")
            return
        if task.lower() in ("exit", "quit", "q"):
            print("Bye 👋")
            return
        if not task:
            continue
        agent_loop(task, messages, base, auto_yes, max_iters)
        print()


def main():
    ap = argparse.ArgumentParser(description="Autonomous local coding agent for qwen-fast")
    ap.add_argument("task", nargs="?", help="The task. Omit to enter chat mode.")
    ap.add_argument("-d", "--dir", default=".", help="Target directory")
    ap.add_argument("-c", "--context", action="append", default=[], metavar="FILE",
                    help="Preload an existing file as context (repeatable)")
    ap.add_argument("-y", "--yes", action="store_true", help="Auto-approve everything")
    ap.add_argument("-n", "--max", type=int, default=8, help="Max agent iterations (default 8)")
    ap.add_argument("-b", "--build", action="store_true",
                    help="Big-build mode: plan into files, then build each in a fresh context")
    ap.add_argument("-s", "--skill", metavar="NAME",
                    help="Load a reusable skill from ~/.codegen/skills/<NAME>.md")
    ap.add_argument("--model", metavar="NAME", help="Execution model (overrides default qwen-fast)")
    ap.add_argument("--num-ctx", type=int, metavar="N", dest="num_ctx",
                    help="Context window in tokens for this run (overrides the model's default; "
                         "sent as Ollama options.num_ctx — no Modelfile rebuild needed).")
    ap.add_argument("--planner", metavar="NAME", help="Planning model for --build (e.g. qwen3-think)")
    ap.add_argument("--think", action="store_true", help="Use reasoning (think mode) for the planning step")
    ap.add_argument("--git", action="store_true",
                    help="Checkpoint each step with git (auto-inits a repo; undo a bad step with "
                         "'git reset --hard HEAD~1')")
    ap.add_argument("--rag-dir", action="append", default=[], metavar="DIR",
                    help="Extra directory to index for <<<RAGSEARCH>>> (repeatable). "
                         "The target dir is always indexed.")
    ap.add_argument("--learn", action="store_true",
                    help="Diagnose first: study relevant existing code (RAG) + a short plan before "
                         "writing (#8/#9). Best when editing an existing project.")
    ap.add_argument("--critique", action="store_true",
                    help="Self-review proposed files once before writing them (#5).")
    ap.add_argument("--review", action="store_true",
                    help="A reviewer sub-agent (fresh context) must approve before DONE (#7).")
    ap.add_argument("-p", "--plugin", metavar="NAME",
                    help="Load a plugin bundle (skill+hooks+conventions) from ~/.codegen/plugins/<NAME>/ (#11)")
    ap.add_argument("-a", "--ask", action="store_true",
                    help="Assistant mode: answer questions in prose (web/API/RAG tools available), not just code.")
    ap.add_argument("--provider", choices=sorted(PROVIDERS), metavar="NAME",
                    help="Backend preset: " + ", ".join(sorted(PROVIDERS)) +
                         ". Cloud providers give near-frontier power cheaply (see the budget alternatives doc).")
    ap.add_argument("--base-url", metavar="URL",
                    help="OpenAI-compatible API base URL (e.g. https://api.deepseek.com/v1). Overrides --provider.")
    ap.add_argument("--api-key", metavar="KEY",
                    help="API key for the cloud backend (else taken from the provider's env var).")
    args = ap.parse_args()

    base = os.path.abspath(args.dir)
    os.makedirs(base, exist_ok=True)

    # Safety: the agent creates/overwrites files in `base`. Warn (and, unless -y,
    # confirm) before running in a personal directory full of unrelated files.
    # Assistant mode (-a) rarely writes, so skip the nag there.
    home = os.path.expanduser("~")
    risky = {home} | {os.path.join(home, d) for d in ("Downloads", "Desktop", "Documents")}
    if base in risky and not args.ask:
        print(f"⚠️  Running in {base} — the agent can create and OVERWRITE files here.", file=sys.stderr)
        print("   Safer to work in a clean project dir, e.g.  mkdir -p ~/projects/app && cd ~/projects/app",
              file=sys.stderr)
        if not args.yes and input("Continue here anyway? [y/N] ").strip().lower() not in ("y", "yes"):
            print("Aborted. cd into a clean directory and retry.")
            sys.exit(0)

    global PROJECT_MEMORY, ACTIVE_SKILL, HOOKS, MODEL, PLANNER_MODEL, THINK_PLAN, NUM_CTX
    global GIT_CHECKPOINT, DIAGNOSE, CRITIQUE, REVIEW, ASSISTANT, RAG_DIRS
    global BASE_URL, API_KEY
    # Backend selection: --provider preset, then explicit --base-url/--api-key overrides.
    if args.provider:
        bu, dmodel, keyenv = PROVIDERS[args.provider]
        BASE_URL = bu
        if not args.model:
            MODEL = dmodel
        if keyenv and not API_KEY:
            API_KEY = os.environ.get(keyenv, "")
    if args.base_url:
        BASE_URL = args.base_url
    if args.api_key:
        API_KEY = args.api_key
    if args.model:
        MODEL = args.model
    elif not os.environ.get("CODEGEN_MODEL"):
        # No explicit model → honor the registry's assigned 'executor' role (defaults to
        # qwen-fast, so this is a no-op until the user reassigns the role in the GUI).
        ex = registry_default("executor")
        if ex:
            MODEL = ex
    if args.num_ctx:
        NUM_CTX = args.num_ctx
        print(f"📐 context window: {NUM_CTX} tokens (per-request override)", file=sys.stderr)
    if BASE_URL:
        print(f"☁️  backend: {BASE_URL}  model: {MODEL}", file=sys.stderr)
        if not API_KEY:
            print("⚠️  no API key set for the cloud backend — set --api-key or the provider's env var.",
                  file=sys.stderr)
    if args.planner:
        PLANNER_MODEL = args.planner
    THINK_PLAN = args.think
    GIT_CHECKPOINT = args.git
    DIAGNOSE = args.learn
    CRITIQUE = args.critique
    REVIEW = args.review
    ASSISTANT = args.ask
    RAG_DIRS = [os.path.abspath(d) for d in args.rag_dir]
    PROJECT_MEMORY = load_memory(base)
    HOOKS = load_hooks(base)
    if args.skill:
        ACTIVE_SKILL = load_skill(args.skill)
    if args.plugin:  # #11 — a plugin supplements skill / hooks / conventions; project files win on conflict
        p_skill, p_hooks, p_conv = load_plugin(args.plugin)
        if p_skill:
            ACTIVE_SKILL = (ACTIVE_SKILL + "\n\n" + p_skill).strip() if ACTIVE_SKILL else p_skill
        if p_hooks:
            HOOKS = {**p_hooks, **HOOKS}
        if p_conv:
            PROJECT_MEMORY = (PROJECT_MEMORY + "\n\n" + p_conv).strip() if PROJECT_MEMORY else p_conv

    messages = [{"role": "system", "content": with_memory(ASSISTANT_PROMPT if args.ask else SYSTEM_PROMPT)}]
    for c in args.context:
        p = c if os.path.isabs(c) else os.path.join(base, c)
        try:
            with open(p) as f:
                messages.append({"role": "user",
                                 "content": f"Existing file {c}:\n{f.read()}"})
                messages.append({"role": "assistant", "content": "Noted."})
        except OSError as e:
            print(f"⚠️  cannot read {c}: {e}", file=sys.stderr)

    if args.build:
        if not args.task:
            print("⚠️  --build needs a task, e.g. ai -b -y \"build a login backend\"")
            sys.exit(1)
        orchestrate(args.task, base, args.yes)
    elif args.task:
        agent_loop(args.task, messages, base, args.yes, args.max)
    else:
        interactive(messages, base, args.yes, args.max)


if __name__ == "__main__":
    main()