diff --git a/README.md b/README.md index faa3dea0..482294f8 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,20 @@ The server listens on `127.0.0.1` only. To serve other machines, set `GEMSTONE_H Clients then send `Authorization: Bearer `. `GEMSTONE_ORIGINS` allows more browser origins and `--reload` (or `GEMSTONE_DEV=1`) turns on auto-reload for development. +**Point existing clients at Gemstone** (it listens on `23100`, not Ollama's `11434` or OpenAI's HTTPS host): + +```bash +OLLAMA_HOST=http://127.0.0.1:23100 ollama run qwen3 # the ollama CLI and Ollama clients +OPENAI_BASE_URL=http://127.0.0.1:23100/v1 OPENAI_API_KEY=x python my_script.py # OpenAI SDKs +``` + +`qwen3`, `qwen3:latest` and `qwen3:0.6b` are the built-in model (fetched on first use); any Hugging Face +model is pulled by its id, e.g. `ollama pull Org/Model`. Other Ollama library names (`llama3.2`) are not +available. When `GEMSTONE_API_KEY` is set on the server, send it as the key: `OPENAI_API_KEY=` for +OpenAI SDKs, or an `Authorization: Bearer ` header for other clients (if your Ollama client cannot send +a header, use it on a loopback server without a key). `think: false` (Ollama) and `reasoning_effort: "none"` +(OpenAI) switch Qwen3's reasoning off. + **3. Run a native client** ```bash diff --git a/api/src/main/engine.py b/api/src/main/engine.py index acabc48e..cd362c0d 100644 --- a/api/src/main/engine.py +++ b/api/src/main/engine.py @@ -14,7 +14,7 @@ `model.generate` at a time, with no batched request running. See `Engine.__call__`. """ from contextlib import contextmanager -from typing import Dict, Generator, List, Optional, Union +from typing import Any, Dict, Generator, List, Optional, Union import itertools import logging import os @@ -199,11 +199,15 @@ def __call__( repeat_penalty: float = 1.0, seed: Optional[int] = None, cancel: Optional[threading.Event] = None, + chat_template_kwargs: Optional[Dict[str, Any]] = None, **kwargs ) -> Generator[str, None, None]: """ Stream the reply to `messages` as text chunks. + `chat_template_kwargs` are extra variables for the chat template (`enable_thinking=False` + switches Qwen3's reasoning off). Unlike `kwargs` they do not make the request exclusive. + `stream` is accepted for compatibility with the model layer and ignored: the reply is always produced as a stream. `max_new_tokens <= 0` means "as many as fit": up to the context length, but when the request can join the batch, up to what the paged KV cache holds @@ -223,6 +227,7 @@ def __call__( chat_template=self.chat_template, add_generation_prompt=True, tokenize=False, + **(chat_template_kwargs or {}), ) input_ids = self.tokenizer(prompt, add_special_tokens=False)["input_ids"] prompt_length = len(input_ids) diff --git a/api/src/main/ollama_api.py b/api/src/main/ollama_api.py index fc13b319..838ea5af 100644 --- a/api/src/main/ollama_api.py +++ b/api/src/main/ollama_api.py @@ -12,7 +12,6 @@ `Registry.resolve` (derived models included); residency and keep_alive are handled by `registry.Registry`. """ -from importlib import metadata from typing import Iterator, List, Optional import itertools import json @@ -25,9 +24,10 @@ from starlette.concurrency import run_in_threadpool from . import registry as registry_module +from .version import VERSION from .leases import LeasedStreamingResponse, acquire, once -from .registry import (DERIVED_PARAMETERS, ModelBusy, ModelUnavailable, UnsupportedModel, alias_key, format_parameters, is_builtin_name, now_iso, - parse_keep_alive) +from .registry import (DERIVED_PARAMETERS, ModelBusy, ModelUnavailable, UnsupportedModel, alias_key, format_parameters, + in_catalogue, is_builtin_name, now_iso, parse_keep_alive) router = APIRouter() @@ -246,14 +246,19 @@ def _count_tokens(engine, text: str) -> Optional[int]: return None -def _prompt_tokens(engine, messages, tools) -> Optional[int]: +def _template_kwargs(think: Optional[bool]) -> dict: + """ `think` -> the chat template's `enable_thinking` (Qwen3 and others); unset leaves the template's default. """ + return {} if think is None else {"chat_template_kwargs": {"enable_thinking": think}} + + +def _prompt_tokens(engine, messages, tools, template_kwargs=None) -> Optional[int]: tokenizer = getattr(engine, "tokenizer", None) if tokenizer is None: return None try: prompt = tokenizer.apply_chat_template( messages, tools=tools or None, chat_template=getattr(engine, "chat_template", None), - add_generation_prompt=True, tokenize=False) + add_generation_prompt=True, tokenize=False, **(template_kwargs or {})) except Exception: return None return _count_tokens(engine, prompt) @@ -288,8 +293,9 @@ def _generate(lease, messages, tools, kwargs, stops, think, cancel, started, rel first_token = None chunks = None try: - prompt_eval_count = _prompt_tokens(engine, messages, tools) - chunks = engine(messages, tools=tools or None, cancel=cancel, **kwargs) + template = _template_kwargs(think) + prompt_eval_count = _prompt_tokens(engine, messages, tools, template.get("chat_template_kwargs")) + chunks = engine(messages, tools=tools or None, cancel=cancel, **template, **kwargs) for chunk in chunks: if first_token is None: first_token = time.perf_counter() @@ -381,11 +387,11 @@ async def _run(body: dict, messages: list, tools, render, finish): if keep_alive == 0: await run_in_threadpool(reg.unload, hf_id) return JSONResponse(finish(name, "unload")) - lease = await acquire(reg, hf_id, keep_alive) + lease = await acquire(reg, hf_id, keep_alive, in_catalogue(name)) reg.release(lease) return JSONResponse(finish(name, "load")) - lease = await acquire(reg, hf_id, keep_alive) + lease = await acquire(reg, hf_id, keep_alive, in_catalogue(name)) cancel = threading.Event() release = once(lambda: reg.release(lease)) events = _generate(lease, messages, tools, kwargs, stops, think, cancel, started, release) @@ -785,7 +791,7 @@ async def _embed(body: dict, inputs: List[str]) -> tuple: keep_alive = parse_keep_alive(body.get("keep_alive")) truncate = body.get("truncate", True) is not False reg = _registry() - lease = await acquire(reg, reg.resolve(name), keep_alive) + lease = await acquire(reg, reg.resolve(name), keep_alive, in_catalogue(name)) try: embed = getattr(lease.engine, "embed", None) if embed is None: @@ -828,10 +834,10 @@ async def embeddings(request: Request): return {"embedding": vectors[0] if vectors else []} -@router.get("/api/version") +@router.api_route("/api/version", methods=["GET", "HEAD"]) async def version(): - """ The Gemstone version (clients use it to check that the server is up). """ - try: - return {"version": metadata.version("gemstone")} - except metadata.PackageNotFoundError: - return {"version": "0.0.0"} + """ + The Gemstone version (SPEC S1.15). It is a plain `MAJOR.MINOR.PATCH` string, which is what Ollama + clients compare as semver; it is above every Ollama 0.x, so a "needs Ollama >= 0.x" check passes. + """ + return {"version": VERSION} diff --git a/api/src/main/openai_api.py b/api/src/main/openai_api.py index 35b9e31c..f3677d51 100644 --- a/api/src/main/openai_api.py +++ b/api/src/main/openai_api.py @@ -24,7 +24,7 @@ from . import registry as registry_module from .leases import LeasedStreamingResponse, acquire, once -from .registry import ModelBusy, ModelUnavailable, in_catalogue, parse_keep_alive +from .registry import ModelBusy, ModelUnavailable, in_catalogue, parse_keep_alive, unknown_model_message router = APIRouter(prefix="/v1") @@ -58,6 +58,21 @@ class ChatCompletionRequest(Schema): stream: bool = False stream_options: Optional[Dict[str, Any]] = None n: Optional[int] = None + reasoning_effort: Optional[str] = None # "none" / "minimal" switch Qwen3-style thinking off + chat_template_kwargs: Optional[Dict[str, Any]] = None # extension, as in vLLM + + +def _template_kwargs(req: "ChatCompletionRequest") -> Dict[str, Any]: + """ + Extra variables for the chat template. `reasoning_effort` (OpenAI's field) maps to + `enable_thinking`: `none` / `minimal` is off, any other value on. `chat_template_kwargs` + (vLLM's extension) is passed through and wins. Neither set: the template's own default. + """ + out: Dict[str, Any] = {} + if req.reasoning_effort is not None: + out["enable_thinking"] = req.reasoning_effort.strip().lower() not in ("none", "minimal") + out.update(req.chat_template_kwargs or {}) + return out class APIError(Exception): @@ -264,11 +279,14 @@ class Generation: or None once the reply is complete; `close` stops generation and releases the engine. """ - def __init__(self, engine, messages, tools, parameters: Dict[str, Any], stops: List[str]): + def __init__(self, engine, messages, tools, parameters: Dict[str, Any], stops: List[str], + template_kwargs: Optional[Dict[str, Any]] = None): self.engine, self.messages, self.tools = engine, messages, tools + self.template_kwargs = template_kwargs or {} + extra = dict(chat_template_kwargs=self.template_kwargs) if self.template_kwargs else {} self.max_new_tokens = parameters.get("max_new_tokens", 0) self.cancel = threading.Event() - self.tokens: Iterator[str] = engine(messages, tools=tools, cancel=self.cancel, **parameters) + self.tokens: Iterator[str] = engine(messages, tools=tools, cancel=self.cancel, **extra, **parameters) self.parser = OutputParser(parse_tools=bool(tools)) self.stopper = _StopMatcher(stops) self.raw: List[str] = [] @@ -319,7 +337,7 @@ def _usage(self) -> Dict[str, int]: completion = len(tokenizer.encode("".join(self.raw), add_special_tokens=False)) text = tokenizer.apply_chat_template( self.messages, tools=self.tools, chat_template=getattr(self.engine, "chat_template", None), - add_generation_prompt=True, tokenize=False, + add_generation_prompt=True, tokenize=False, **self.template_kwargs, ) prompt = len(tokenizer.encode(text, add_special_tokens=False)) except Exception: # usage is informative; never fail a reply over it @@ -398,7 +416,8 @@ async def _chat_completions(request: Request): try: hf_id = registry.resolve(req.model) # derived models (`/api/create`, `/api/copy`) too except LookupError: - raise APIError(404, f"The model '{req.model}' does not exist.", param="model", code="model_not_found") + raise APIError(404, f"The {unknown_model_message(req.model)}", + param="model", code="model_not_found") derived = registry.derived(req.model) or {} messages = _engine_messages(req.messages) @@ -446,7 +465,7 @@ async def _complete(req: ChatCompletionRequest, lease, release, messages, tools, parameters["seed"] = req.seed stops = [req.stop] if isinstance(req.stop, str) else list(req.stop or derived.get("stop") or []) - generation = Generation(engine, messages, tools, parameters, stops) + generation = Generation(engine, messages, tools, parameters, stops, _template_kwargs(req)) def finish(): try: diff --git a/api/src/main/registry.py b/api/src/main/registry.py index 6673abd5..02aef348 100644 --- a/api/src/main/registry.py +++ b/api/src/main/registry.py @@ -55,6 +55,7 @@ class CatalogueEntry: **{name: entry.hf_id for name, entry in CATALOGUE.items()}, # qwen3, default "qwen3:0.6b": "Qwen/Qwen3-0.6B", # Ollama library name "qwen3:latest": "Qwen/Qwen3-0.6B", + "qwen3:0.6b-fp16": "Qwen/Qwen3-0.6B", # Ollama's unquantised tag; the weights are served as published "smollm2:135m": "HuggingFaceTB/SmolLM2-135M-Instruct", # Ollama's smollm2:135m is the instruct model } @@ -79,12 +80,27 @@ def resolve_model_name(name: str) -> str: repo, _, tag = name.partition(":") if "/" in repo and tag in ("", "latest") and re.fullmatch(r"[\w.-]+/[\w.-]+", repo): return repo - raise LookupError(f"model '{name}' not found") + raise LookupError(unknown_model_message(name)) + + +def unknown_model_message(name: str) -> str: + """ The 404 text for a name that is neither known nor a Hugging Face id: what exists and how to add more. """ + names = ", ".join(sorted(n for n in MODEL_ALIASES if n != "default")) + return (f"model '{name}' not found. Gemstone serves these names: {names}. " + "Any Hugging Face model can be pulled by its id (`Org/Model`), " + "for example `ollama pull HuggingFaceTB/SmolLM2-360M-Instruct`; Ollama library names such as " + "`llama3.2` are not available.") def in_catalogue(name: str) -> bool: - """ True for a Gemstone model name (`qwen3`, `default`). """ - return isinstance(name, str) and name.strip().lower() in CATALOGUE + """ + True for a Gemstone catalogue model, by its own name (`qwen3`, `default`) or an Ollama name that + maps to it (`qwen3:latest`, `qwen3:0.6b`): such a model is fetched on first use (S1.14). + """ + if not isinstance(name, str): + return False + key = name.strip().lower() + return key in CATALOGUE or MODEL_ALIASES.get(key) in {e.hf_id for e in CATALOGUE.values()} _plain_classes: dict = {} diff --git a/api/src/main/server.py b/api/src/main/server.py index 2727000c..d999eb8c 100644 --- a/api/src/main/server.py +++ b/api/src/main/server.py @@ -1,5 +1,5 @@ from fastapi import FastAPI, WebSocket, WebSocketDisconnect, HTTPException -from fastapi.responses import RedirectResponse, FileResponse +from fastapi.responses import RedirectResponse, FileResponse, Response from fastapi.staticfiles import StaticFiles from starlette.concurrency import run_in_threadpool import anyio @@ -40,6 +40,12 @@ def root(): return FileResponse(os.path.join(WEBPACK_DIR, "gemstone.html")) +@app.head("/") +def root_head(): + """ Health check, as Ollama answers `HEAD /` (SPEC S1.15): 200 with no body and no key. """ + return Response(status_code=200, media_type="text/plain") + + @app.get("/composeResources/{path:path}") def resource(path: str): """ Redirect to the chat page """ diff --git a/api/src/main/version.py b/api/src/main/version.py new file mode 100644 index 00000000..206cc9cc --- /dev/null +++ b/api/src/main/version.py @@ -0,0 +1,6 @@ +""" +The Gemstone version, defined once. The project is not installed as a package (no build system), +so `importlib.metadata` cannot supply it; `pyproject.toml` must carry the same string +(`api/tests/test_client_compat.py` checks that). +""" +VERSION = "1.0.0" diff --git a/api/tests/test_client_compat.py b/api/tests/test_client_compat.py new file mode 100644 index 00000000..c5990f76 --- /dev/null +++ b/api/tests/test_client_compat.py @@ -0,0 +1,202 @@ +""" +Compatibility with stock Ollama and OpenAI clients (SPEC S1.10, S1.15; issue #118): health checks, +the version, Ollama library names, first-use fetch on the Ollama routes, and the thinking switch. + +No model is loaded and nothing is downloaded: a fake engine and a fake model store. +""" +import pathlib +import re +import threading + +import pytest +from fastapi.testclient import TestClient + +from api.src.main import registry as registry_module +from api.src.main import server +from api.src.main.engine import Engine +from api.src.main.registry import Registry, resolve_model_name +from api.src.main.version import VERSION +from api.tests.test_ollama_api import QWEN, SMOL, FakeEngine, FakeStore + +ROOT = pathlib.Path(__file__).resolve().parents[2] +USER = [{"role": "user", "content": "Hi"}] + + +@pytest.fixture(autouse=True) +def no_key(monkeypatch): + monkeypatch.delenv("GEMSTONE_API_KEY", raising=False) + + +def serve(monkeypatch, *cached): + engine = FakeEngine(QWEN, ["Hello", " there", "!"]) + store = FakeStore(*cached) + reg = Registry(loader=lambda hf_id: engine, store=store, close=lambda e: None) + monkeypatch.setattr(registry_module, "registry", reg) + return engine, store, reg + + +@pytest.fixture +def cached(monkeypatch): + engine, store, reg = serve(monkeypatch, QWEN, SMOL) + with TestClient(server.app) as client: + yield dict(client=client, engine=engine, store=store) + reg.unload() + + +@pytest.fixture +def fresh(monkeypatch): + """ A fresh install: nothing in the local store. """ + engine, store, reg = serve(monkeypatch) + with TestClient(server.app) as client: + yield dict(client=client, engine=engine, store=store) + reg.unload() + + +# -- health ---------------------------------------------------------------------------------------- + +def test_head_root_is_200(cached): + r = cached["client"].head("/") + assert r.status_code == 200 and r.content == b"" + + +def test_head_root_and_version_need_no_key(cached, monkeypatch): + monkeypatch.setenv("GEMSTONE_API_KEY", "s3cret") + assert cached["client"].head("/").status_code == 200 + assert cached["client"].head("/api/version").status_code == 200 + assert cached["client"].get("/api/tags").status_code == 401 + + +def test_get_root_still_serves_the_web_client(cached): + assert any(getattr(r, "path", None) == "/" and "GET" in r.methods for r in server.app.routes) + + +# -- version --------------------------------------------------------------------------------------- + +def test_version_is_the_pyproject_version_without_installing(): + declared = re.search(r'^version\s*=\s*"([^"]+)"', (ROOT / "pyproject.toml").read_text(), re.M).group(1) + assert VERSION == declared + + +def test_api_version_is_real_semver(cached): + body = cached["client"].get("/api/version").json() + assert body == {"version": VERSION} and body["version"] != "0.0.0" + assert re.fullmatch(r"\d+\.\d+\.\d+", body["version"]) + + +# -- Ollama names ---------------------------------------------------------------------------------- + +@pytest.mark.parametrize("name", ["qwen3", "qwen3:latest", "qwen3:0.6b", "Qwen3:0.6B", "qwen3:0.6b-fp16", "default"]) +def test_ollama_names_for_the_catalogue_model(name): + assert resolve_model_name(name) == QWEN + + +def test_unknown_ollama_name_says_what_exists(cached): + r = cached["client"].post("/api/chat", json={"model": "llama3.2", "messages": USER}) + assert r.status_code == 404 + error = r.json()["error"] + assert "llama3.2" in error and "qwen3" in error and "Org/Model" in error and "Hugging Face" in error + assert cached["store"].downloads == [] + + +def test_pull_of_unknown_ollama_name_is_404_with_the_same_hint(cached): + r = cached["client"].post("/api/pull", json={"model": "llama3.2", "stream": False}) + assert r.status_code == 404 and "Org/Model" in r.json()["error"] + + +def test_openai_unknown_name_says_what_exists(cached): + r = cached["client"].post("/v1/chat/completions", json={"model": "llama3.2", "messages": USER}) + assert r.status_code == 404 + assert "Org/Model" in r.json()["error"]["message"] and "qwen3" in r.json()["error"]["message"] + + +# -- first use ------------------------------------------------------------------------------------- + +@pytest.mark.parametrize("route, body", [ + ("/api/chat", {"messages": USER}), + ("/api/generate", {"prompt": "Hi"}), + ("/api/embed", {"input": "Hi"}), +]) +@pytest.mark.parametrize("name", ["qwen3", "qwen3:latest", "qwen3:0.6b"]) +def test_ollama_routes_fetch_a_catalogue_model_on_first_use(fresh, route, body, name): + fresh["engine"].embed = lambda texts, truncate=True: [[0.0] for _ in texts] + r = fresh["client"].post(route, json={"model": name, "stream": False, **body}) + assert r.status_code == 200, r.text + assert fresh["store"].downloads == [QWEN] + + +def test_ollama_does_not_fetch_other_models(fresh): + for name in ("someone/not-pulled", "smollm2:135m"): + r = fresh["client"].post("/api/chat", json={"model": name, "messages": USER}) + assert r.status_code == 404 + assert fresh["store"].downloads == [] + + +# -- thinking -------------------------------------------------------------------------------------- + +def ollama_chat(client, **body): + r = client.post("/api/chat", json={"model": "qwen3", "messages": USER, "stream": False, **body}) + assert r.status_code == 200, r.text + + +def test_ollama_think_false_disables_thinking_in_the_template(cached): + ollama_chat(cached["client"], think=False) + assert cached["engine"].calls[-1]["chat_template_kwargs"] == {"enable_thinking": False} + + +def test_ollama_think_true_enables_it_and_unset_changes_nothing(cached): + ollama_chat(cached["client"], think=True) + assert cached["engine"].calls[-1]["chat_template_kwargs"] == {"enable_thinking": True} + ollama_chat(cached["client"]) + assert "chat_template_kwargs" not in cached["engine"].calls[-1] + + +def test_ollama_generate_passes_think_too(cached): + cached["client"].post("/api/generate", json={"model": "qwen3", "prompt": "Hi", "stream": False, "think": False}) + assert cached["engine"].calls[-1]["chat_template_kwargs"] == {"enable_thinking": False} + + +def openai_chat(client, **body): + r = client.post("/v1/chat/completions", json={"model": "qwen3", "messages": USER, **body}) + assert r.status_code == 200, r.text + + +@pytest.mark.parametrize("effort, expected", [("none", False), ("minimal", False), ("low", True), ("high", True)]) +def test_openai_reasoning_effort_sets_enable_thinking(cached, effort, expected): + openai_chat(cached["client"], reasoning_effort=effort) + assert cached["engine"].calls[-1]["chat_template_kwargs"] == {"enable_thinking": expected} + + +def test_openai_chat_template_kwargs_pass_through_and_win(cached): + openai_chat(cached["client"], reasoning_effort="none", chat_template_kwargs={"enable_thinking": True, "x": 1}) + assert cached["engine"].calls[-1]["chat_template_kwargs"] == {"enable_thinking": True, "x": 1} + + +def test_openai_without_a_reasoning_field_changes_nothing(cached): + openai_chat(cached["client"]) + assert "chat_template_kwargs" not in cached["engine"].calls[-1] + + +def test_openai_bad_chat_template_kwargs_is_400(cached): + r = cached["client"].post("/v1/chat/completions", + json={"model": "qwen3", "messages": USER, "chat_template_kwargs": "no"}) + assert r.status_code == 400 + + +# -- the engine ------------------------------------------------------------------------------------ + +def test_engine_gives_chat_template_kwargs_to_the_template_and_stays_batchable(): + seen = {} + + class Tokenizer: + def apply_chat_template(self, messages, **kwargs): + seen.update(kwargs) + return "prompt" + + def __call__(self, text, add_special_tokens=False): + return {"input_ids": [1, 2]} + + engine = Engine.__new__(Engine) + engine.tokenizer, engine.chat_template, engine._batcher, engine.context_length = Tokenizer(), None, None, 100 + engine._generate_exclusive = lambda *args: iter(["ok"]) + assert list(engine(USER, max_new_tokens=5, chat_template_kwargs={"enable_thinking": False})) == ["ok"] + assert seen["enable_thinking"] is False and seen["add_generation_prompt"] is True diff --git a/docs/SPEC.md b/docs/SPEC.md index 4d028505..88c8beda 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -228,6 +228,13 @@ Implemented (#65), `api/src/main/openai_api.py`: recognised. `finish_reason` is `tool_calls` when any call was returned, else `length` when the token limit was reached, else `stop`. With `tool_choice: "none"` the tools are not sent and no output is parsed as a call. A block that is not valid JSON is returned as content. +- `reasoning_effort` switches thinking: `none` or `minimal` passes `enable_thinking=False` to the + chat template (Qwen3 then skips its reasoning), any other value passes `True`. The vLLM extension + `chat_template_kwargs` (an object) is passed to the template as well and wins over + `reasoning_effort`. Neither set: nothing is passed and the template's default applies (Qwen3 + thinks). The engine takes these as `chat_template_kwargs=`; unlike other extra keywords they do + not make a request exclusive (S1.12). An unknown model's 404 message lists the names that exist + and says Hugging Face ids can be pulled (S1.15). - Reasoning between `` and `` is removed from `content` and returned as `reasoning_content` (message field, or delta field when streaming), the extension used by DeepSeek, vLLM and others. Leading and trailing whitespace around content and reasoning is @@ -512,7 +519,7 @@ OpenAI-compatible API (S1.10). Code: `api/src/main/ollama_api.py` (an `APIRouter Implemented (M3 scope, #66; the rest of the commands, #75): - `POST /api/chat`, `POST /api/generate`, `GET /api/tags`, `POST /api/show`, `POST /api/pull`, - `DELETE /api/delete`, `GET /api/ps`, `GET /api/version` (the Gemstone package version). + `DELETE /api/delete`, `GET /api/ps`, `GET /api/version` (below). `GET /api/ps` shows every resident model whichever API loaded it (S1.14). - `POST /api/copy`, `POST /api/create`, `POST /api/embed`, `POST /api/embeddings` and `POST /api/push` (below). @@ -521,12 +528,32 @@ Implemented (M3 scope, #66; the rest of the commands, #75): not offer, 503 when a load timed out waiting for room (S1.14) or a model cannot be downloaded because the machine is offline or the download failed (the message names the model, #114; `/api/pull` too), 500 for a pull of an unknown repository. An error after streaming has started arrives as a final `{"error": ...}` line. A stream whose client is gone before it starts still releases the model (S1.14). +- **Health and version (#118).** `HEAD /` answers 200 with no body and needs no API key, as Ollama's + does; `GET /` keeps serving the bundled web client. `HEAD` and `GET /api/version` return + `{"version": ""}`, the single constant `VERSION` in `api/src/main/version.py` + (equal to `pyproject.toml`, checked by a test; the project is not installed, so + `importlib.metadata` cannot be used). It is a plain `MAJOR.MINOR.PATCH` string, which is what Ollama + clients compare as semver; Gemstone's `1.x` is above every Ollama `0.x`, so a "requires Ollama >= + 0.x" gate passes. +- **Names (#118).** The names that resolve to the catalogue model (`Qwen/Qwen3-0.6B`) are `qwen3`, + `qwen3:latest`, `qwen3:0.6b`, `qwen3:0.6b-fp16` and `default` (case-insensitive); `smollm2:135m` maps + to `HuggingFaceTB/SmolLM2-135M-Instruct`; `Org/Model[:latest]` is a Hugging Face id. Nothing else + is mapped: Ollama library names such as `llama3.2` are not available. Such a name is a 404 + whose message lists the names above and says a Hugging Face id (`Org/Model`) can be pulled. The + same text is used for `/api/pull`, `/api/show` and the OpenAI API. +- **First use (#118).** `/api/chat`, `/api/generate` (also load-only) and `/api/embed`/`/api/embeddings` + download a catalogue model that is not in the cache (`qwen3` and the Ollama names that map to it), + as the app and the OpenAI API do, so `ollama run qwen3` works on a fresh install. Any other model + that is not pulled stays a 404; an offline or failed download is a 503 (#114). - **Chat.** `messages` (`role`, `content`, `tool_calls`, `tool_name`); `images` are dropped (no vision support). `tools` go to the chat template; a `{json}` block the model emits becomes `message.tool_calls: [{function: {name, arguments}}]` and is never executed (pass-through, S1.10). A block that is not valid JSON stays in the content as text. -- **Thinking.** `think: true` returns the `` block as `message.thinking`; `think: false` - drops it from the reply (the model still generates it); unset leaves it in the content as written. +- **Thinking (#118).** `think: false` passes `enable_thinking=False` to the chat template, so a + Qwen3-style model skips its reasoning, and drops any `` block from the reply; `think: true` + (or a level string such as `"high"`) passes `enable_thinking=True` and returns the block as + `message.thinking`; unset passes nothing: the template's default applies (Qwen3 thinks) and the + `` block stays in `content` as the model wrote it. `/api/chat` and `/api/generate` behave alike. - **Options.** `temperature`, `top_p`, `top_k`, `min_p`, `typical_p`, `repeat_penalty` and `seed` go to the engine unchanged; `num_predict` becomes `max_new_tokens` (unset or ≤ 0 means up to the context length); `stop` cuts the output at the first stop string and stops generation (best diff --git a/docs/guide/guide-server.html b/docs/guide/guide-server.html index abca9975..bfa7fe1b 100644 --- a/docs/guide/guide-server.html +++ b/docs/guide/guide-server.html @@ -69,6 +69,14 @@

OpenAI-compatible API

OpenAI 호환 AP -d '{"model": "qwen3", "messages": [{"role": "user", "content": "Hi"}]}'

Tool calls are returned, not run: when the model calls a tool, the reply carries tool_calls and finish_reason: "tool_calls". Your client runs the tool and sends the result back as a tool message. The model's reasoning (<think>) comes back separately as reasoning_content, not in content. The chat app's WebSocket is unchanged and still runs Gemstone's built-in tools on the server.

도구 호출은 실행되지 않고 반환됩니다: 모델이 도구를 부르면 응답에 tool_calls 와 finish_reason: "tool_calls" 가 담깁니다. 클라이언트가 도구를 실행하고 그 결과를 tool 메시지로 다시 보냅니다. 모델의 추론(<think>)은 content 가 아니라 reasoning_content 로 따로 옵니다. 채팅 앱의 WebSocket 은 그대로이며, 여전히 Gemstone 내장 도구를 서버에서 실행합니다.

+

Pointing clients at Gemstone

클라이언트를 Gemstone 에 연결하기

+

Gemstone listens on port 23100, not Ollama's 11434. Set the address the client reads:

Gemstone 은 Ollama 의 11434 가 아니라 23100 포트에서 듣습니다. 클라이언트가 읽는 주소를 설정하세요.

+
OLLAMA_HOST=http://127.0.0.1:23100 ollama run qwen3            # the ollama CLI, Ollama clients
+OPENAI_BASE_URL=http://127.0.0.1:23100/v1 OPENAI_API_KEY=x python app.py  # OpenAI SDKs
+

With GEMSTONE_API_KEY set on the server, give OpenAI SDKs that value as OPENAI_API_KEY, and other clients as an Authorization: Bearer header; an Ollama client that cannot send that header should be used against a loopback server without a key. HEAD / and GET /api/version work without a key, for health checks.

서버에 GEMSTONE_API_KEY 를 설정했다면 OpenAI SDK 에는 그 값을 OPENAI_API_KEY 로, 다른 클라이언트에는 Authorization: Bearer 헤더로 주세요. 그 헤더를 보낼 수 없는 Ollama 클라이언트는 키 없는 loopback 서버에서 쓰세요. 상태 확인용 HEAD / 와 GET /api/version 은 키 없이 동작합니다.

+

Model names: qwen3, qwen3:latest, qwen3:0.6b and qwen3:0.6b-fp16 are the built-in model and are downloaded on first use, by the app, the OpenAI API and the Ollama API alike. smollm2:135m must be pulled first. Any Hugging Face model is pulled by its id (Org/Model); other Ollama library names such as llama3.2 answer 404 with this list. GET /api/version returns Gemstone's own version (1.0.0), a plain MAJOR.MINOR.PATCH that Ollama clients compare as semver.

모델 이름: qwen3, qwen3:latest, qwen3:0.6b, qwen3:0.6b-fp16 는 기본 제공 모델이며 앱, OpenAI API, Ollama API 모두 처음 쓸 때 내려받습니다. smollm2:135m 은 먼저 pull 해야 합니다. Hugging Face 모델은 id(Org/Model)로 pull 합니다. llama3.2 같은 다른 Ollama 라이브러리 이름은 이 목록과 함께 404 를 돌려줍니다. GET /api/version 은 Gemstone 자신의 버전(1.0.0)을 돌려주며, Ollama 클라이언트가 semver 로 비교하는 MAJOR.MINOR.PATCH 형식입니다.

+

Thinking: Qwen3 reasons by default. Ollama think: false and OpenAI reasoning_effort: "none" (or "minimal") pass enable_thinking=False to the chat template, so the model skips the reasoning; vLLM's chat_template_kwargs is passed through. With none of these set the template's default applies and the <think> block stays in the Ollama content as written (OpenAI returns it as reasoning_content).

추론: Qwen3 는 기본적으로 추론합니다. Ollama 의 think: false 와 OpenAI 의 reasoning_effort: "none"(또는 "minimal")은 채팅 템플릿에 enable_thinking=False 를 넘겨 모델이 추론을 건너뜁니다. vLLM 의 chat_template_kwargs 도 그대로 전달됩니다. 아무것도 설정하지 않으면 템플릿 기본값이 적용되고, Ollama 의 content 에는 <think> 블록이 그대로 남습니다(OpenAI 는 reasoning_content 로 돌려줍니다).

+

Keys

키

Web search uses SerpApi when a key is present and falls back to scraping Bing otherwise. Put the key in a .env file at the repository root (never commit it):

웹 검색은 키가 있으면 SerpApi 를, 없으면 Bing 스크래핑을 씁니다. 키는 저장소 루트의 .env 파일에 둡니다 (절대 커밋하지 마세요):

cp .env.example .env
diff --git a/docs/locale/README_ko.md b/docs/locale/README_ko.md
index 7beca81b..173c97ec 100644
--- a/docs/locale/README_ko.md
+++ b/docs/locale/README_ko.md
@@ -69,6 +69,19 @@ python -m api run server
 클라이언트는 `Authorization: Bearer ` 를 보냅니다. `GEMSTONE_ORIGINS` 는 브라우저 origin 을 더 허용하고,
 `--reload` (또는 `GEMSTONE_DEV=1`) 는 개발용 자동 재시작을 켭니다.
 
+**기존 클라이언트를 Gemstone 에 연결하기** (포트는 Ollama 의 `11434` 가 아니라 `23100` 입니다):
+
+```bash
+OLLAMA_HOST=http://127.0.0.1:23100 ollama run qwen3          # ollama CLI 와 Ollama 클라이언트
+OPENAI_BASE_URL=http://127.0.0.1:23100/v1 OPENAI_API_KEY=x python my_script.py   # OpenAI SDK
+```
+
+`qwen3`, `qwen3:latest`, `qwen3:0.6b` 는 기본 제공 모델이며 처음 쓸 때 내려받습니다. Hugging Face 모델은 id 로
+받습니다(`ollama pull Org/Model`). 그 밖의 Ollama 라이브러리 이름(`llama3.2`)은 쓸 수 없습니다. 서버에
+`GEMSTONE_API_KEY` 를 설정했다면 그 값을 키로 보내세요. OpenAI SDK 는 `OPENAI_API_KEY=`, 다른 클라이언트는
+`Authorization: Bearer ` 헤더입니다(헤더를 보낼 수 없는 Ollama 클라이언트는 키 없는 loopback 서버에서 쓰세요).
+`think: false`(Ollama), `reasoning_effort: "none"`(OpenAI)은 Qwen3 의 추론을 끕니다.
+
 **3. 네이티브 클라이언트 실행**
 
 ```bash