diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 602648d..25d2db7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,11 +20,12 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Setup Java 17 + # NetLogo 7.0.4 is compiled for Java 21 + - name: Setup Java 21 uses: actions/setup-java@v4 with: distribution: temurin - java-version: "17" + java-version: "21" cache: sbt - name: Setup sbt @@ -32,3 +33,9 @@ jobs: - name: Run tests run: sbt test + + - name: Build release zip + run: sbt packageZip + + - name: Check release zip contents + run: scripts/check-zip.sh diff --git a/.gitignore b/.gitignore index 2c95842..a9188db 100644 --- a/.gitignore +++ b/.gitignore @@ -54,6 +54,19 @@ config-*.txt !demos/provider-sensitivity/config-multi-provider.txt !demos/provider-sensitivity/config.txt !demos/social-deduction-game/config.txt +!demos/config-reference.txt +!demos/config-placeholder.txt + +# Flattened demo copies staged into demos/ by `packageZip` (build artifacts; +# the canonical sources live in demos// subfolders) +/demos/color-sharing.nlogox +/demos/social-deduction-game.nlogox +/demos/topology-tournament.nlogox +/demos/provider-sensitivity.nlogox +/demos/el-farol-decisions.nlogox +/demos/description-template.yaml +/demos/vote-template.yaml +/llm-*.zip # Live e2e harness run artifact demos/e2e-tests/e2e-results.txt diff --git a/BUILD.md b/BUILD.md index c60fda6..d67294f 100644 --- a/BUILD.md +++ b/BUILD.md @@ -2,8 +2,8 @@ ## Prerequisites -- **Java 17** (required for building) -- **SBT 1.10.6+** (Scala Build Tool) +- **Java 21** (required for building; NetLogo 7.0.4 is compiled for Java 21) +- **SBT 1.10.1** (Scala Build Tool) ## Run Tests @@ -24,8 +24,8 @@ What this does not validate: ### For Development (Fat JAR) ```bash -# Set Java 17 -export JAVA_HOME=$(/usr/libexec/java_home -v 17) +# Set Java 21 (NetLogo 7.0.4 is compiled for Java 21) +export JAVA_HOME=$(/usr/libexec/java_home -v 21) # Build single JAR with all dependencies sbt clean assembly @@ -36,8 +36,8 @@ sbt clean assembly ### For Distribution (ZIP Package) ```bash -# Set Java 17 -export JAVA_HOME=$(/usr/libexec/java_home -v 17) +# Set Java 21 (NetLogo 7.0.4 is compiled for Java 21) +export JAVA_HOME=$(/usr/libexec/java_home -v 21) # Build distribution package sbt clean packageZip @@ -73,15 +73,15 @@ sbt clean packageZip ## Common Issues **Problem**: Build fails with SBT parser errors -**Solution**: Upgrade SBT to 1.10.6+ in `project/build.properties`: +**Solution**: Use the SBT version pinned in `project/build.properties`: ``` -sbt.version=1.10.6 +sbt.version=1.10.1 ``` **Problem**: Build fails with Java version errors -**Solution**: Make sure you're using Java 17: +**Solution**: Make sure you're using Java 21: ```bash -java -version # Should show 17.x.x +java -version # Should show 21.x.x or newer ``` **Problem**: Old JAR still being used diff --git a/INSTALL.md b/INSTALL.md index 8ce9a73..94cec52 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -5,7 +5,7 @@ ### 1. Build the Extension ```bash -export JAVA_HOME=$(/usr/libexec/java_home -v 17) +export JAVA_HOME=$(/usr/libexec/java_home -v 21) sbt clean assembly ``` @@ -13,14 +13,14 @@ sbt clean assembly **macOS:** ```bash -mkdir -p "/Applications/NetLogo 7.0.0/extensions/llm" -cp target/scala-3.7.0/llm.jar "/Applications/NetLogo 7.0.0/extensions/llm/" +mkdir -p "/Applications/NetLogo 7.0.4/extensions/llm" +cp target/scala-3.7.0/llm.jar "/Applications/NetLogo 7.0.4/extensions/llm/" ``` **Windows:** ```bash -mkdir "C:\Program Files\NetLogo 7.0.0\extensions\llm" -copy target\scala-3.7.0\llm.jar "C:\Program Files\NetLogo 7.0.0\extensions\llm\" +mkdir "C:\Program Files\NetLogo 7.0.4\extensions\llm" +copy target\scala-3.7.0\llm.jar "C:\Program Files\NetLogo 7.0.4\extensions\llm\" ``` **Linux:** @@ -31,36 +31,36 @@ cp target/scala-3.7.0/llm.jar "/usr/local/NetLogo-7.0.0/extensions/llm/" ## Install from ZIP Package (Alternative) -If you have a pre-built `llm-0.1.0.zip`: +If you have a pre-built `llm-1.1.0.zip`: ### 1. Extract the ZIP ```bash -unzip llm-0.1.0.zip +unzip llm-1.1.0.zip ``` ### 2. Copy to NetLogo **macOS:** ```bash -cp -r llm-0.1.0 "/Applications/NetLogo 7.0.0/extensions/llm" +cp -r llm-1.1.0 "/Applications/NetLogo 7.0.4/extensions/llm" ``` **Windows:** ```bash -xcopy llm-0.1.0 "C:\Program Files\NetLogo 7.0.0\extensions\llm" /E /I +xcopy llm-1.1.0 "C:\Program Files\NetLogo 7.0.4\extensions\llm" /E /I ``` **Linux:** ```bash -cp -r llm-0.1.0 "/usr/local/NetLogo-7.0.0/extensions/llm" +cp -r llm-1.1.0 "/usr/local/NetLogo-7.0.0/extensions/llm" ``` ## Verify Installation ### 1. Open NetLogo -Launch NetLogo 7.0.0 (or newer) +Launch NetLogo 7.0.4 (or newer) ### 2. Test the Extension @@ -114,24 +114,24 @@ Click Setup, then run `test-llm` from the Command Center. The extension must be in NetLogo's `extensions/llm/` folder: **macOS:** -- `/Applications/NetLogo 7.0.0/extensions/llm/` +- `/Applications/NetLogo 7.0.4/extensions/llm/` **Windows:** -- `C:\Program Files\NetLogo 7.0.0\extensions\llm\` +- `C:\Program Files\NetLogo 7.0.4\extensions\llm\` **Linux:** - `/usr/local/NetLogo-7.0.0/extensions/llm/` **Expected structure:** ``` -NetLogo 7.0.0/ +NetLogo 7.0.4/ └── extensions/ └── llm/ └── llm.jar (for Fat JAR install) OR -NetLogo 7.0.0/ +NetLogo 7.0.4/ └── extensions/ └── llm/ ├── llm.jar diff --git a/RELEASE.md b/RELEASE.md index d0b6e6d..c7574a2 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,133 +1,98 @@ # Release Process -## Creating a New Release +Releases ship to users through the NetLogo Extension Manager, which reads +`libraries.conf` in [NetLogo/NetLogo-Libraries](https://github.com/NetLogo/NetLogo-Libraries). +A release is: a tagged commit here, a GitHub release with the zip, and a PR +to NetLogo-Libraries that adds the zip and bumps the version. -### 1. Update Version +## Requirements -Edit `build.sbt`: -```scala -version := "0.2.0" // Update version number -``` - -### 2. Build Release Package +- **JDK 21 or newer.** NetLogo 7.0.4 is compiled for Java 21; JDK 17 fails with + `UnsupportedClassVersionError` on `org.nlogo.api.APIVersion`. +- **sbt 1.10.1** (pinned in `project/build.properties`; NetLogo notes 1.10.2+ can + hit a ZipException). ```bash -# Set Java 17 -export JAVA_HOME=$(/usr/libexec/java_home -v 17) - -# Build distribution ZIP -sbt clean packageZip +export JAVA_HOME=$(/usr/libexec/java_home -v 21+) ``` -**Output:** `llm-0.2.0.zip` (11MB) +## 1. Set the version -### 3. Test the Release +In `build.sbt`: -```bash -# Automated deterministic tests (no API/network required) -sbt test +```scala +version := "1.1.0" +isSnapshot := false // only for the release build; keep true while developing +``` -# Extract and install release package -unzip llm-0.2.0.zip -cp -r llm-0.2.0 "/Applications/NetLogo 7.0.0/extensions/llm" +With `isSnapshot := false` the plugin refuses to stage the extension from a +dirty working tree, so commit everything first. -# Optional manual integration checks (real providers) -# Open NetLogo and run demos/tests/tests.nlogox -``` +Versioning is semantic: MAJOR for changes that break existing models, MINOR for +new primitives or providers, PATCH for fixes. -### 4. Create Git Tag +## 2. Build and check the zip ```bash -git add build.sbt llm-0.2.0.zip -git commit -m "Release v0.2.0" -git tag v0.2.0 -git push origin main -git push origin v0.2.0 +sbt clean test packageZip +scripts/check-zip.sh # asserts the docs, config files and demos are inside ``` -### 5. Create GitHub Release - -1. Go to: https://github.com/JNK234/Netlogo-LLM-Extension/releases -2. Click "Create a new release" -3. Select tag: `v0.2.0` -4. Title: `LLM Extension v0.2.0` -5. Attach: `llm-0.2.0.zip` -6. Write release notes -7. Publish release - -## Release Checklist +Output: `llm-.zip` in the repo root (gitignored). It contains `llm.jar`, +its dependency jars, `docs/README.md`, `demos/config-reference.txt`, the curated +demos flattened into `demos/`, and `demos/config.txt` copied from the tracked +`demos/config-placeholder.txt` (never the developer's local config, which may +hold real keys). `sbt packageZip` also prints the `libraries.conf` entry to use. -- [ ] Version updated in `build.sbt` -- [ ] `sbt clean packageZip` completed successfully -- [ ] `sbt test` passed -- [ ] ZIP file tested in NetLogo -- [ ] Optional live-provider checks run (`demos/tests/tests.nlogox`) -- [ ] Documentation updated if needed -- [ ] Git tag created -- [ ] GitHub release published -- [ ] ZIP file attached to release +Check the manifest says `NetLogo-Extension-API-Version: 7.0`: -## Build Outputs - -### For Development ```bash -sbt clean assembly +unzip -p llm-.zip llm.jar > /tmp/llm.jar && unzip -p /tmp/llm.jar META-INF/MANIFEST.MF ``` -- Output: `target/scala-3.7.0/llm.jar` (172MB) -- Use: Local testing, quick install - -### For Distribution -```bash -sbt clean packageZip -``` -- Output: `llm-x.x.x.zip` (11MB) -- Use: Official releases, NetLogo Library submission - -## Version Numbering -Follow semantic versioning: `MAJOR.MINOR.PATCH` +## 3. Test the release -- **MAJOR**: Breaking changes to API -- **MINOR**: New features, backward compatible -- **PATCH**: Bug fixes, backward compatible +1. `sbt test` (deterministic, no network) must be green; CI runs the same plus + the zip check. +2. Live harness: open `demos/e2e-tests/e2e-tests.nlogox` with a local + `config-.txt` (see the `.example` files) and run it against at least + one cloud provider, Ollama, and TypeSafe (Jev). +3. Install the zip into a NetLogo 7.0.4 app + (`/extensions/llm/`), open a bundled demo, and run it. -Examples: -- `0.1.0` - Initial release -- `0.2.0` - Added new primitives -- `0.2.1` - Fixed bug in config loading -- `1.0.0` - Stable API, production ready +## 4. Tag and publish on GitHub -## Publishing to NetLogo Extension Library +```bash +git tag v +git push upstream main v +gh release create v llm-.zip -R NetLogo/Netlogo-LLM-Extension \ + --title "LLM Extension v" --notes-file +``` -After creating a GitHub release: +## 5. Publish to the Extension Manager -1. The package metadata is generated by `sbt packageZip`: - ```json - { - name: "LLM-Extension" - codeName: "llm" - version: "0.x.x" - downloadURL: "https://github.com/.../llm-0.x.x.zip" - } - ``` +1. Fork https://github.com/NetLogo/NetLogo-Libraries and branch from **`7.0`** + (the branch NetLogo 7.0.x reads). Maintainers cherry-pick to `7.1`. +2. Add `extensions/llm-.zip`. +3. In `libraries.conf`, update the `llm` entry's `version` (and descriptions if + they changed) using the block `sbt packageZip` printed. +4. Open the PR against `7.0`. For reference, v1.0.0 was NetLogo-Libraries PR #87. -2. Submit to NetLogo Extension Library: - - Fork: https://github.com/NetLogo/NetLogo-Libraries - - Add entry to `libraries.conf` - - Create pull request +## Checklist -## Cleanup After Release +- [ ] `version` set, `isSnapshot := false`, tree committed +- [ ] `sbt clean test packageZip` green, `scripts/check-zip.sh` passes +- [ ] Manifest API version is 7.0 +- [ ] Live e2e harness run (cloud provider, Ollama, Jev) +- [ ] Zip installed and a demo run in NetLogo 7.0.4 +- [ ] Docs updated (`docs/API-REFERENCE.md`, `docs/CONFIGURATION.md`, `docs/PROVIDER-GUIDE.md`) +- [ ] Tag pushed, GitHub release published with the zip +- [ ] NetLogo-Libraries PR opened against `7.0` +- [ ] `isSnapshot := true` restored for development -Remove old build artifacts: +## Development builds ```bash -# Remove old JARs from root (keep only latest ZIP) -rm -f *.jar - -# Remove old version directories -rm -rf llm-0.0.*/ - -# Keep only: -# - llm-0.x.x.zip (latest version) -# - target/ (regenerated on build) +sbt clean assembly # fat jar at target/scala-3.7.0/llm.jar, for local installs +./build.sh # same, plus install when NETLOGO_DIR is set ``` diff --git a/build.sbt b/build.sbt index b59e945..f9f4497 100644 --- a/build.sbt +++ b/build.sbt @@ -3,25 +3,75 @@ import sbt.nio.file.FileTreeView enablePlugins(org.nlogo.build.NetLogoExtension) name := "LLM-Extension" -version := "0.1.0" +version := "1.1.0" isSnapshot := true netLogoExtName := "llm" netLogoClassManager := "org.nlogo.extensions.llm.LLMExtension" -netLogoVersion := "7.0.0-2486d1e" +netLogoVersion := "7.0.4" netLogoShortDescription := "Multi-provider LLM extension for NetLogo agents" -netLogoLongDescription := "A NetLogo extension that provides unified interface for multiple LLM providers including OpenAI, Anthropic, Gemini, and Ollama" -netLogoHomepage := "https://github.com/JNK234/Netlogo-LLM-Extension" +netLogoLongDescription := "A NetLogo extension providing a unified interface to multiple LLM providers — OpenAI, Anthropic/Claude, Google/Gemini, local Ollama, OpenRouter, Together AI, and Groq — plus TypeSafe (Jev) for calibrated typed decisions, with per-agent conversation memory, synchronous, asynchronous and batched chat, prompt templates, constrained choices, typed decisions with probabilities, structured outputs, token and cost tracking, and reasoning/thinking-model support." +netLogoHomepage := "https://github.com/NetLogo/Netlogo-LLM-Extension" -// Package only essential files in the zip distribution +// Package essential files in the zip distribution: docs, the config reference, +// and curated demo models. Demo models + their YAML templates are flattened +// directly into the `demos/` folder so the NetLogo Models Library can +// auto-discover them in the unzipped extension folder. +// +// The NetLogo extension plugin places each extra at its path relative to +// baseDirectory, so to land files flat under `demos/` we stage flat copies in +// `demos/` rather than referencing their subfolders. A missing extra fails the +// build instead of being silently dropped from the zip. netLogoZipExtras := { val base = baseDirectory.value - // Only include core documentation and config reference - Seq( - base / "NetLogo-LLM-Extension-Documentation.md", + // Top-level extras kept at their existing relative paths. + val topLevel = Seq( + base / "docs" / "README.md", base / "demos" / "config-reference.txt" - ).filter(_.exists()) + ) + + // Curated demo files to flatten into `demos/`. Source -> flat target name. + val demoSources = Seq( + base / "demos" / "color-sharing" / "color-sharing.nlogox", + base / "demos" / "social-deduction-game" / "social-deduction-game.nlogox", + base / "demos" / "social-deduction-game" / "description-template.yaml", + base / "demos" / "social-deduction-game" / "vote-template.yaml", + base / "demos" / "topology-tournament" / "topology-tournament.nlogox", + base / "demos" / "provider-sensitivity" / "provider-sensitivity.nlogox", + base / "demos" / "el-farol-decisions" / "el-farol-decisions.nlogox" + ) + + val missing = (topLevel ++ demoSources).filterNot(_.exists()) + if (missing.nonEmpty) + sys.error(s"netLogoZipExtras: missing files: ${missing.mkString(", ")}") + + val demosDir = base / "demos" + val flattened = demoSources.map { src => + val dest = demosDir / src.getName + IO.copyFile(src, dest) + dest + } + + topLevel ++ flattened +} + +// Every bundled demo loads `demos/config.txt`. The developer's local copy may +// hold real API keys, so the zip gets the tracked placeholder instead: it is +// written into the zip after the plugin builds it. +packageZip := { + val zip = packageZip.value + val placeholder = baseDirectory.value / "demos" / "config-placeholder.txt" + if (!placeholder.exists()) + sys.error(s"missing $placeholder") + val uri = java.net.URI.create("jar:" + zip.toURI) + val fs = java.nio.file.FileSystems.newFileSystem(uri, new java.util.HashMap[String, String]()) + try { + val target = fs.getPath("demos", "config.txt") + java.nio.file.Files.createDirectories(target.getParent) + java.nio.file.Files.copy(placeholder.toPath, target, java.nio.file.StandardCopyOption.REPLACE_EXISTING) + } finally fs.close() + zip } scalaVersion := "3.7.0" @@ -44,6 +94,15 @@ libraryDependencies ++= Seq( "com.lihaoyi" %% "ujson" % "4.0.0" ) +// NetLogo 7.0.4's tests jar is built against scalatest 3.2.19; the extension +// plugin injects 3.2.10, so pin the version NetLogo itself uses. +dependencyOverrides += "org.scalatest" %% "scalatest" % "3.2.19" + +// netlogo-prims-json 0.1.1 (added by the extension plugin) depends on NetLogo +// 7.1.0-alpha1, which would otherwise evict 7.0.4 and make the extension +// compile against, and declare the API version of, an unreleased NetLogo. +dependencyOverrides += "org.nlogo" % "netlogo" % netLogoVersion.value + ThisBuild / scalafixDependencies += "com.github.liancheng" %% "organize-imports" % "0.6.0" // Assembly configuration for fat JAR with all dependencies diff --git a/build.sh b/build.sh index 5677827..d4a9da5 100755 --- a/build.sh +++ b/build.sh @@ -4,8 +4,8 @@ set -e -# Set Java 17 -export JAVA_HOME=$(/usr/libexec/java_home -v 17) +# NetLogo 7.0.4 is compiled for Java 21, so the build needs JDK 21 or newer +export JAVA_HOME=${JAVA_HOME:-$(/usr/libexec/java_home -v 21+)} echo "Building LLM Extension..." sbt clean assembly @@ -18,7 +18,7 @@ mkdir -p dist/llm cp target/scala-3.7.0/llm.jar dist/llm/ # Auto-install to NetLogo if NETLOGO_DIR is set -# Usage: NETLOGO_DIR="/Applications/NetLogo 7.0.3" ./build.sh +# Usage: NETLOGO_DIR="/Applications/NetLogo 7.0.4" ./build.sh if [ -n "$NETLOGO_DIR" ]; then NETLOGO_EXT="$NETLOGO_DIR/extensions/llm" mkdir -p "$NETLOGO_EXT" @@ -27,7 +27,7 @@ if [ -n "$NETLOGO_DIR" ]; then else echo "" echo "To auto-install, re-run with your NetLogo path:" - echo " NETLOGO_DIR=\"/Applications/NetLogo 7.0.3\" ./build.sh" + echo " NETLOGO_DIR=\"/Applications/NetLogo 7.0.4\" ./build.sh" echo "" echo "Or manually copy:" echo " mkdir -p /path/to/NetLogo/extensions/llm" diff --git a/demos/baba-is-ai/README.md b/demos/baba-is-ai/README.md index 107da9c..9cf8793 100644 --- a/demos/baba-is-ai/README.md +++ b/demos/baba-is-ai/README.md @@ -83,12 +83,12 @@ api_key=YOUR_KEY_HERE # Anthropic provider=anthropic -model=claude-sonnet-4-20250514 +model=claude-haiku-5-5 api_key=YOUR_KEY_HERE # Gemini provider=gemini -model=gemini-2.0-flash +model=gemini-3.5-flash-lite api_key=YOUR_KEY_HERE ``` diff --git a/demos/color-sharing/config.txt b/demos/color-sharing/config.txt index 9701de3..6726c98 100644 --- a/demos/color-sharing/config.txt +++ b/demos/color-sharing/config.txt @@ -9,9 +9,9 @@ provider=ollama # Choose your model (must match the provider above) -# OpenAI: gpt-4o-mini, gpt-4o, gpt-4-turbo, gpt-3.5-turbo -# Anthropic: claude-3-haiku-20240307, claude-3-sonnet-20240229, claude-3-opus-20240229 -# Gemini: gemini-1.5-flash, gemini-1.5-pro, gemini-pro +# OpenAI: gpt-6-luna (default), gpt-6.1-sol, gpt-6-astra, gpt-4o-mini +# Anthropic: claude-haiku-5-5 (default), claude-sonnet-5-5, claude-opus-5-5 +# Gemini: gemini-3.5-flash-lite (default), gemini-3.8-flash, gemini-4-argon # Ollama: llama3.2, llama3.1, mistral, codellama (must be installed) model=llama3.2:3b diff --git a/demos/config b/demos/config index faa476b..68ff22f 100644 --- a/demos/config +++ b/demos/config @@ -33,7 +33,7 @@ timeout_seconds=30 # Setup: Get API key from https://console.anthropic.com #provider=anthropic #api_key=YOUR_ANTHROPIC_API_KEY_HERE -#model=claude-3-5-sonnet-20241022 +#model=claude-haiku-5-5 #temperature=0.7 #max_tokens=500 #timeout_seconds=30 @@ -45,7 +45,7 @@ timeout_seconds=30 # Setup: Get API key from https://makersuite.google.com #provider=gemini #api_key=YOUR_GEMINI_API_KEY_HERE -#model=gemini-1.5-flash +#model=gemini-3.5-flash-lite #temperature=0.7 #max_tokens=500 #timeout_seconds=30 diff --git a/demos/config-placeholder.txt b/demos/config-placeholder.txt new file mode 100644 index 0000000..b707eb0 --- /dev/null +++ b/demos/config-placeholder.txt @@ -0,0 +1,36 @@ +# NetLogo LLM Extension - configuration used by the bundled demos +# Pick one provider block, set its key, and save. Full option list: config-reference.txt +# Never share this file once it holds a real API key. + +# Local Ollama (free, no key; install from https://ollama.com, then `ollama pull llama3.2`) +provider=ollama +model=llama3.2 + +# Anthropic Claude +# provider=anthropic +# model=claude-haiku-5-5 +# anthropic_api_key=YOUR_ANTHROPIC_API_KEY + +# OpenAI +# provider=openai +# model=gpt-6-luna +# openai_api_key=YOUR_OPENAI_API_KEY + +# Google Gemini (free tier available) +# provider=gemini +# model=gemini-3.5-flash-lite +# gemini_api_key=YOUR_GEMINI_API_KEY + +# Groq (fast, free tier available) +# provider=groq +# model=openai/gpt-oss-20b +# groq_api_key=YOUR_GROQ_API_KEY + +# TypeSafe Jev (decision API for llm:decide / llm:decide-all; used by el-farol-decisions) +# provider=typesafe +# model=jev-latest +# typesafe_api_key=YOUR_TYPESAFE_API_KEY + +temperature=0.7 +max_tokens=500 +timeout_seconds=60 diff --git a/demos/config-reference.txt b/demos/config-reference.txt new file mode 100644 index 0000000..8396702 --- /dev/null +++ b/demos/config-reference.txt @@ -0,0 +1,194 @@ +# NetLogo Multi-LLM Extension - Comprehensive Configuration Reference +# This file contains ALL possible configuration options and values +# Use this as a reference when setting up different providers + +# ============================================================================= +# IMPORTANT NOTES +# ============================================================================= +# +# Configuration Precedence: Runtime commands > Config file > Built-in defaults +# - Use config file for persistent settings (recommended) +# - Runtime commands (llm:set-provider, etc.) override config file values +# - Built-in defaults apply only if no config file or runtime command is used +# - Built-in default models come from the model catalog (models.yaml). To change +# them or add model IDs without rebuilding, put a models.yaml in +# ~/.netlogo-llm/ (see docs/CONFIGURATION.md, "Model catalog") +# +# Immediate Validation: +# - When loading config or setting provider, validation happens immediately +# - Cloud providers (OpenAI, Anthropic, Gemini) require API keys +# - Ollama requires the server to be running and reachable +# - Invalid configurations throw helpful errors with setup instructions +# +# Provider-Specific Keys: +# - Use provider-specific keys: openai_api_key, anthropic_api_key, gemini_api_key +# - This allows configuring multiple providers at once +# - Generic 'api_key' still supported for backward compatibility +# +# Getting Help: +# - Use 'print llm:provider-help "provider-name"' for setup instructions +# - Use 'print llm:provider-status' to see which providers are ready +# - Use 'print llm:active' to verify current provider and model + +# ============================================================================= +# PROVIDER CONFIGURATION +# ============================================================================= + +# Available providers: openai, anthropic, gemini, ollama +# provider=openai + +# ============================================================================= +# OPENAI CONFIGURATION +# ============================================================================= + +# OpenAI Models (choose one): +# model=gpt-6-luna # Default - fast and affordable reasoning model (recommended) +# model=gpt-6.1-sol # Mid-tier GPT-6 +# model=gpt-6-astra # Most capable GPT-6 +# model=gpt-5.6-luna # Previous small GPT-5.x +# model=gpt-5.6-sol # Previous mid-tier GPT-5.x +# model=gpt-4o-mini # Deprecated, served until 2027-01-14 +# Note: GPT-5.x/6.x are reasoning models - temperature is not sent; use +# reasoning_effort (none|low|medium|high|xhigh) to control depth. + +# OpenAI API Configuration: +# openai_api_key=sk-your-openai-api-key-here +# openai_base_url=https://api.openai.com/v1 + +# ============================================================================= +# ANTHROPIC (CLAUDE) CONFIGURATION +# ============================================================================= + +# Claude Models (choose one): +# model=claude-haiku-5-5 # Default - fastest and most affordable (recommended) +# model=claude-sonnet-5-5 # Balanced +# model=claude-opus-5-5 # Most capable Opus +# model=claude-fable-5-1 # Most capable overall +# model=claude-haiku-4-5-20251001 # Previous Haiku (still served) + +# Anthropic API Configuration: +# anthropic_api_key=sk-ant-your-anthropic-api-key-here +# anthropic_base_url=https://api.anthropic.com/v1 + +# ============================================================================= +# GOOGLE GEMINI CONFIGURATION +# ============================================================================= + +# Gemini Models (choose one): +# model=gemini-3.5-flash-lite # Default - fast and affordable (recommended) +# model=gemini-3.8-flash # Stronger Flash tier +# model=gemini-3.1-pro-preview # Pro (preview) +# model=gemini-2.5-flash # Only for projects that already used 2.5 + +# Google AI API Configuration: +# gemini_api_key=your-google-ai-api-key-here +# gemini_base_url=https://generativelanguage.googleapis.com/v1beta + +# ============================================================================= +# OLLAMA (LOCAL) CONFIGURATION +# ============================================================================= + +# Ollama Models (choose one - must be installed locally): +# model=llama3.2 # Latest Llama (default) +# model=llama3.1 # Previous Llama version +# model=llama3 # Llama 3 base +# model=llama2 # Llama 2 base +# model=mistral # Mistral 7B +# model=mixtral # Mixtral 8x7B +# model=codellama # Code-specialized Llama +# model=vicuna # Vicuna chat model +# model=phi3 # Microsoft Phi-3 +# model=gemma # Google Gemma +# model=qwen2 # Alibaba Qwen2 +# model=deepseek-coder # DeepSeek Coder + +# Ollama Server Configuration: +# ollama_base_url=http://localhost:11434 # Default local Ollama server +# ollama_base_url=http://192.168.1.100:11434 # Remote Ollama server example +# No API key needed for Ollama + +# Note: Ollama requires the server to be running. Install from https://ollama.com +# Run 'ollama list' to see installed models, 'ollama pull ' to download new ones + +# ============================================================================= +# TYPESAFE (JEV) CONFIGURATION +# ============================================================================= + +# Jev returns decisions with probabilities, not text. Use llm:decide, +# llm:decide-all or llm:choose; llm:chat and the other text primitives raise +# an error on this provider. temperature and max_tokens do not apply. + +# TypeSafe Models (choose one): +# model=jev-latest # Default - newest stable release (alias) +# model=jev-preview # Newest release, stable or not (alias) +# model=jev-1.14.0 # Pinned version +# model=jev-1.13.0 # Pinned version + +# TypeSafe API Configuration: +# provider=typesafe +# typesafe_api_key=your-typesafe-api-key-here +# typesafe_base_url=https://api.typesafe.ai + +# ============================================================================= +# GENERATION PARAMETERS (ALL PROVIDERS) +# ============================================================================= + +# Temperature: Controls randomness (0.0 = deterministic, 1.0 = very random) +# temperature=0.7 # Default - balanced creativity +# temperature=0.0 # Deterministic output +# temperature=0.3 # Less random, more focused +# temperature=1.0 # More creative/random + +# Max Tokens: Maximum response length +# max_tokens=1000 # Default for OpenAI +# max_tokens=4000 # Higher for longer responses (Claude) +# max_tokens=2048 # Typical for Gemini/Ollama + +# Timeout: How long to wait for response (seconds) +# timeout_seconds=30 # Default timeout +# timeout_seconds=60 # Longer for complex requests +# timeout_seconds=120 # Very long for large responses + +# ============================================================================= +# EXAMPLE CONFIGURATIONS BY PROVIDER +# ============================================================================= + +# Example 1: OpenAI Setup +# provider=openai +# model=gpt-6-luna +# api_key=sk-your-openai-key-here +# temperature=0.7 +# max_tokens=2000 + +# Example 2: Claude Setup +# provider=anthropic +# model=claude-haiku-5-5 +# api_key=sk-ant-your-claude-key-here +# temperature=0.5 +# max_tokens=4000 + +# Example 3: Gemini Setup +# provider=gemini +# model=gemini-3.5-flash-lite +# api_key=your-gemini-key-here +# temperature=0.8 +# max_tokens=2048 + +# Example 4: Local Ollama Setup +# provider=ollama +# model=deepseek-r1:1.5b +# base_url=http://localhost:11434 +# temperature=0.7 +# max_tokens=2048 +# timeout_seconds=60 + +# ============================================================================= +# NOTES +# ============================================================================= + +# 1. Only uncomment the lines you want to use +# 2. API keys are required for OpenAI, Anthropic, and Gemini +# 3. Ollama requires a local server running (ollama serve) +# 4. Base URLs only need to be changed if using custom endpoints +# 5. Models must be available for your API tier/account +# 6. Ollama models must be pulled first: ollama pull llama3.2 \ No newline at end of file diff --git a/demos/e2e-tests/e2e-tests.nlogox b/demos/e2e-tests/e2e-tests.nlogox index 0dd9855..ed06524 100644 --- a/demos/e2e-tests/e2e-tests.nlogox +++ b/demos/e2e-tests/e2e-tests.nlogox @@ -417,7 +417,8 @@ end ;; reuses the active one and the test still proves routing and bookkeeping. to-report alt-model if provider-under-test = "groq" [ report "openai/gpt-oss-120b" ] - if provider-under-test = "anthropic" [ report "claude-3-5-haiku-latest" ] + if provider-under-test = "typesafe" [ report "jev-preview" ] + if provider-under-test = "anthropic" [ report "claude-haiku-4-5-20251001" ] report item 1 llm:active end @@ -435,6 +436,88 @@ to write-config-with-model [src dst model-name] end ;; Run everything in sequence. +;; --------------------------------------------------------------------------- +;; T12 — decisions on a decision provider (TypeSafe Jev) +;; +;; llm:decide must report [answer probability distribution] with the right +;; NetLogo types for each question type, probabilities must form a +;; distribution, llm:decide-all must answer every agent in one batch, and +;; free-text chat must be refused with a pointer to the decision primitives. +;; --------------------------------------------------------------------------- + +to-report probability? [x] + report is-number? x and x >= 0 and x <= 1 +end + +to-report sums-to-one? [dist] + report abs (sum map [ pair -> item 1 pair ] dist - 1) < 0.05 +end + +to test-decisions + log-line "T12 decisions" + let r [] + carefully + [ set r llm:decide "You are a cautious retiree. The bar was packed (90 of 100 people) the last 3 nights. Go out tonight?" ["yes-no"] ] + [ log-line (word " yes-no failed: " error-message) ] + log-line (word " yes-no -> " r) + assert "yes-no answer is a boolean" (length r = 3 and is-boolean? item 0 r) + assert "yes-no probability is in [0, 1]" (length r = 3 and probability? item 1 r) + assert "yes-no distribution is [[true p] [false q]]" + (length r = 3 and map [ pair -> item 0 pair ] item 2 r = [true false] and sums-to-one? item 2 r) + + set r [] + carefully + [ set r llm:decide "A fire alarm rings. Exit A is 5 m away with 40 people queued; exit B is 30 m away and empty. Which exit?" ["choice" ["A" "B"]] ] + [ log-line (word " choice failed: " error-message) ] + log-line (word " choice -> " r) + assert "choice answer is one of the options" (length r = 3 and member? item 0 r ["A" "B"]) + assert "choice distribution covers both options and sums to 1" + (length r = 3 and length item 2 r = 2 and sums-to-one? item 2 r) + + set r [] + carefully + [ set r llm:decide "Smoke is filling the hallway quickly. How fast do you move?" ["score" ["walk" "brisk" "run"]] ] + [ log-line (word " score failed: " error-message) ] + log-line (word " score -> " r) + assert "score answer is a number on the 0-2 scale" + (length r = 3 and is-number? item 0 r and item 0 r >= 0 and item 0 r <= 2) + assert "score distribution is labelled with the level names" + (length r = 3 and map [ pair -> item 0 pair ] item 2 r = ["walk" "brisk" "run"]) + + let picked "" + carefully + [ set picked llm:choose "A fire alarm rings. Exit A is close but crowded; exit B is far but empty. Which exit?" ["A" "B"] ] + [ set picked (word "ERROR: " error-message) ] + log-line (word " choose -> " picked) + assert "llm:choose works on a decision provider" (member? picked ["A" "B"]) + + clear-turtles + create-turtles 20 + let calls-before llm:get llm:usage-total "calls" + reset-timer + let results [] + carefully + [ set results llm:decide-all turtles [ -> (word "You are bar patron " who ". Last 3 nights the bar held " + (40 + 3 * who) " of 100 people; it is only fun below 60. Go out tonight?") ] ["yes-no"] ] + [ log-line (word " decide-all failed: " error-message) ] + let secs timer + log-line (word " decide-all: " length results " replies in " precision secs 2 " s") + assert "decide-all answers every turtle" (length results = 20) + assert "every decide-all reply is ok" (length results = 20 and reduce and map [ x -> item 2 x ] results) + assert "decide-all recorded one call per turtle" (llm:get llm:usage-total "calls" = calls-before + 20) + log-line (word " usage-total -> " llm:usage-total) + assert "a cost was recorded" (is-number? llm:get llm:usage-total "cost" and llm:get llm:usage-total "cost" > 0) + + let refused "" + carefully + [ set refused llm:chat "Say hello" ] + [ set refused error-message ] + log-line (word " chat -> " refused) + assert "free-text chat is refused with a pointer to llm:decide" + (is-number? position "llm:decide" refused) + report-totals +end + to run-headless carefully [ file-delete "e2e-results.txt" ] [ ] setup @@ -448,6 +531,13 @@ to test-all clear-output set pass-count 0 set fail-count 0 + ;; A decision provider answers no free-text prompts, so it runs only T12. + if provider-under-test = "typesafe" [ + test-decisions + output-print "" + log-line (word "TOTAL passed " pass-count " failed " fail-count) + stop + ] test-basic-chat test-think-tag-leakage test-thinking-path @@ -473,6 +563,7 @@ end + diff --git a/demos/el-farol-decisions/README.md b/demos/el-farol-decisions/README.md new file mode 100644 index 0000000..df7a031 --- /dev/null +++ b/demos/el-farol-decisions/README.md @@ -0,0 +1,99 @@ +# El Farol, Calibrated + +The El Farol bar problem (W. Brian Arthur, 1994) played by 150 patrons who ask a +calibrated decision model what to do each night, using `llm:decide-all` and +`llm:choose` with TypeSafe's Jev. + +The lesson is one choice every LLM-driven agent model makes: when the model gives you +probabilities, do you **sample** from them or **always take the top pick**? + +## The model + +One tick is one night. Each patron has a persona (student, cautious retiree, +fear-of-missing-out regular, night-shift nurse, newcomer) and remembers, in turtle +variables, the last three nights' attendance at each bar and how its own last visit to +each bar went. A bar is fun only with at most its capacity (by default 60% of the +patrons, split across 3 bars: 30 each). Jev is stateless, so a reporter builds the +whole memory into each prompt. + +Each night `llm:decide-all` asks every patron three questions, each batch sent at once: + +```netlogo +llm:decide-all patrons [ -> forecast-prompt ] (list "score" crowd-levels) ; how crowded? +llm:decide-all patrons [ -> go-prompt ] ["yes-no"] ; go out? +llm:decide-all goers [ -> bar-prompt ] (list "choice" bar-names) ; which bar? +``` + +The **decision-rule** chooser decides what a patron does with the answer: + +| Rule | Go out | Which bar | +| ---- | ------ | --------- | +| sample from probabilities | `random-float 1 < P(yes)` | drawn from the choice distribution | +| always take the top pick | `yes` if likelier | the likeliest bar | + +The **text LLM** backend asks one `llm:choose` question per patron instead, one patron +at a time ("The Owl", "Blue Note", "Corner Pub" or "stay home"). A text model names only +its top pick, so there is no forecast, nothing to sample, and it is much slower. + +A failed call sends the patron home and adds to **failed calls**; the night goes on. + +## Files + +| File | Purpose | +| ---- | ------- | +| `el-farol-decisions.nlogox` | The model, with an embedded `headless-check` BehaviorSpace experiment | +| `config.txt.example` | Settings for the typesafe (Jev) backend; copy to `config.txt` | +| `config-text.txt.example` | Settings for the text LLM backend (Ollama by default); copy to `config-text.txt` | + +`config.txt` and `config-text.txt` are gitignored, so your keys stay local. + +## Run it + +1. Copy `config.txt.example` to `config.txt` and set `typesafe_api_key` + (key from https://console.typesafe.ai/). +2. Open `el-farol-decisions.nlogox`, press **setup**, then **one night** or **go**. +3. Switch **decision-rule** and press **setup** again to compare. + +In the extension's zip the model sits in `demos/` next to the shared `config.txt`. +Turn on that file's TypeSafe block (`provider=typesafe`, `model=jev-latest`, +`typesafe_api_key`); setup says so if the loaded provider is not `typesafe`. + +Headless, from this folder with the extension installed: + +```bash +java -Djava.awt.headless=true -cp "" org.nlogo.headless.Main \ + --model el-farol-decisions.nlogox --experiment headless-check --table - +``` + +## Measured (Jev, `jev-latest`, 150 patrons, 3 bars of 30, 10 nights per rule) + +| | sample from probabilities | always take the top pick | +| --- | --- | --- | +| patrons out per night (fun limit 90) | 59 to 103 | 0 to 150; 150 on 5 of 10 nights | +| mean P(go) vs fraction out, mean gap | 0.04 | 0.26 | +| ms per night (450 calls) | 2,700 to 4,200, mean 3,400 | 3,000 to 4,700, mean 4,200 | +| cost per night | about $0.008 | about $0.009 | +| failed calls | 0 | 0 | + +What this shows, honestly: + +- **Going out settles with sampling.** Jev's P(go) responds to the remembered + attendance (it drops after a crowded night and rises after a quiet one), and sampling + turns it into a crowd near the fun limit. Top pick sends everyone or no one. +- **The bar choice still herds under both rules.** Everyone remembers the same + attendance, so Jev puts most of the choice probability on the bar that was quiet last + night. Sampling spreads patrons only as much as the probabilities are spread: under + sampling, a typical night had one bar near 55 and another near 2. Private memories + would break this; see EXTENDING THE MODEL in the Info tab. +- **Cost and latency.** Three batches of 150 Jev calls take about 3 to 4 seconds and + under a cent. The text backend with a local 3B model took 5 to 12 seconds for only + 10 patrons, one question each. + +Results vary run to run: the starting history is random and Jev's probabilities are +read live. + +## Credits + +Arthur, W. B. (1994). Inductive reasoning and bounded rationality. *American Economic +Review*, 84(2), 406-411. Rand, W. and Wilensky, U. (2007). NetLogo El Farol model, +NetLogo Models Library. diff --git a/demos/el-farol-decisions/config-text.txt.example b/demos/el-farol-decisions/config-text.txt.example new file mode 100644 index 0000000..1a42702 --- /dev/null +++ b/demos/el-farol-decisions/config-text.txt.example @@ -0,0 +1,17 @@ +# El Farol demo — settings for the "text LLM" backend +# +# Copy this file to config-text.txt. Any chat provider works; local Ollama needs no key. +# config-text.txt is gitignored, so a key you add stays out of version control. + +provider=ollama +model=qwen2.5vl:3b +base_url=http://localhost:11434 + +# --- or a hosted model --- +#provider=groq +#groq_api_key=YOUR_GROQ_API_KEY +#model=openai/gpt-oss-20b + +temperature=0.7 +max_tokens=50 +timeout_seconds=60 diff --git a/demos/el-farol-decisions/config.txt.example b/demos/el-farol-decisions/config.txt.example new file mode 100644 index 0000000..f279319 --- /dev/null +++ b/demos/el-farol-decisions/config.txt.example @@ -0,0 +1,13 @@ +# El Farol demo — settings for the "typesafe (Jev)" backend +# +# Copy this file to config.txt and fill in your key. config.txt is gitignored, +# so your key stays out of version control. Get a key at https://console.typesafe.ai/ + +provider=typesafe +model=jev-latest +typesafe_api_key=YOUR_TYPESAFE_API_KEY +timeout_seconds=30 + +# Each night sends one request per patron, three times, all at once. If your plan's +# rate limit rejects that, cap how many are in flight: +#max_concurrent_requests=50 diff --git a/demos/el-farol-decisions/el-farol-decisions.nlogox b/demos/el-farol-decisions/el-farol-decisions.nlogox new file mode 100644 index 0000000..07acf85 --- /dev/null +++ b/demos/el-farol-decisions/el-farol-decisions.nlogox @@ -0,0 +1,731 @@ + + + = 0 [ + set region floor ((pxcor - min-pxcor) * num-bars / world-width) + ] + ] + ;; a one-patch wall between neighbouring bars keeps the regions readable + let walls patches with [ + region >= 0 and patch-at 1 0 != nobody and [ region ] of patch-at 1 0 != region + ] + ask walls [ set region -2 ] + ;; the top row of each bar is its sign: patrons stand below it, never on it + ask patches with [ pycor = max-pycor ] [ set region -3 set pcolor gray - 4 ] + ;; the street, with a dashed centre line + ask patches with [ region = -2 and pycor < 0 ] [ + set pcolor ifelse-value pycor = -2 and pxcor mod 3 = 0 + [ gray - 1.5 ] [ gray - 3.5 ] + ] + ask patches with [ region = -1 ] [ + set pcolor ifelse-value (pxcor + pycor) mod 2 = 0 [ green - 4 ] [ green - 3.8 ] + ] +end + +;; Every patron starts from the same three random nights of history, so any +;; difference between them comes from persona and from the decision rule. +to setup-patrons + let history n-values num-bars [ n-values 3 [ random (2 * capacity + 1) ] ] + set attendance map [ nights -> last nights ] history + create-patrons num-patrons [ + set shape "person" + set size 0.9 + set color gray + set persona one-of personas + set remembered-attendance history + set last-experience n-values num-bars [ "not visited yet" ] + set forecast-distribution n-values length crowd-levels [ 0 ] + set going? false + set chosen-bar -1 + set failed? false + move-to-spot one-of patches with [ region = -1 ] + ] +end + +;; Jev reads config.txt; a text LLM reads config-text.txt. Both sit beside the model. +to load-backend-config + let config-file ifelse-value jev-backend? [ "config.txt" ] [ "config-text.txt" ] + set config-ok? false + set config-backend backend + ifelse file-exists? config-file [ + carefully [ + llm:load-config config-file + set config-ok? true + ] [ + explain-config-problem (word "Could not load " config-file ": " error-message) + ] + ] [ + explain-config-problem (word "No " config-file " beside this model. " + "Create it with " + ifelse-value jev-backend? + [ "provider=typesafe, model=jev-latest and typesafe_api_key=YOUR_KEY" ] + [ (word "a chat provider, for example provider=groq, " + "model=openai/gpt-oss-20b and groq_api_key=YOUR_KEY") ] + ", one setting per line (see config-reference.txt).") + ] + ;; the extension's zip ships one shared config.txt, which may name a chat provider + if config-ok? and jev-backend? and item 0 llm:active != "typesafe" [ + set config-ok? false + explain-config-problem (word config-file " uses provider " item 0 llm:active + ", but the typesafe (Jev) backend needs provider=typesafe, model=jev-latest" + " and typesafe_api_key=YOUR_KEY in it.") + ] +end + +;; Shows a config problem in the output area, and as a dialog when not in +;; BehaviorSpace (a dialog would block a headless run). +to explain-config-problem [ message ] + output-print message + if behaviorspace-run-number = 0 [ user-message message ] +end + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; GO: one tick is one night +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +to go + ;; the backend chooser changed since setup: load the other backend's config + if backend != config-backend [ load-backend-config ] + if not config-ok? [ + explain-config-problem "Fix the config file, then press setup again." + stop + ] + ask patrons [ + set going? false + set chosen-bar -1 + set failed? false + ] + let cost-before total-cost + reset-timer + let night-ok? true + carefully [ + ifelse jev-backend? [ decide-with-jev ] [ decide-with-text-llm ] + ] [ + ;; an error for the whole batch, such as a bad API key: report it and stop + output-print (word "Night " (ticks + 1) " failed: " error-message) + set night-ok? false + ] + if not night-ok? [ stop ] + set night-ms round (timer * 1000) + let cost-after total-cost + set night-cost ifelse-value is-number? cost-after + [ cost-after - cost-before ] [ "n/a" ] + let answered patrons with [ not failed? ] + set mean-p-go ifelse-value any? answered [ mean [ p-go ] of answered ] [ 0 ] + set attendance n-values num-bars [ i -> count patrons with [ chosen-bar = i ] ] + remember-night + if show-observations? [ show-one-prompt ] + walk-to-destinations + recolor-bars + tick +end + +;; Jev answers three typed questions per patron, each batch sent all at once. +to decide-with-jev + ;; (a) crowd forecast: a score over crowd-levels; it colours the patron and + ;; goes into the go-out prompt + let results llm:decide-all patrons [ -> forecast-prompt ] + (list "score" crowd-levels) + foreach results [ r -> + ask item 0 r [ + ifelse item 2 r [ + set forecast item 0 item 1 r + set forecast-distribution map [ pair -> item 1 pair ] item 2 item 1 r + set color item (round forecast) [ lime yellow orange magenta ] + ] [ note-failure ] + ] + ] + set forecast-shares forecast-averages + + ;; (b) yes-no: go out tonight? + if any? patrons with [ not failed? ] [ + set results llm:decide-all patrons with [ not failed? ] [ -> go-prompt ] + [ "yes-no" ] + foreach results [ r -> + ask item 0 r [ + ifelse item 2 r [ + set p-go probability-of true item 2 item 1 r + ;; THE LESSON: draw from the probability, or always take the top pick + set going? ifelse-value sampling? + [ random-float 1 < p-go ] [ item 0 item 1 r ] + ] [ note-failure ] + ] + ] + ] + + ;; (c) choice: which bar, for those going out + let goers patrons with [ going? ] + if any? goers [ + set results llm:decide-all goers [ -> bar-prompt ] (list "choice" bar-names) + foreach results [ r -> + ask item 0 r [ + ifelse item 2 r [ + let picked ifelse-value sampling? + [ sample-answer item 2 item 1 r ] [ item 0 item 1 r ] + set chosen-bar position picked bar-names + ] [ note-failure ] + ] + ] + ] +end + +;; A text LLM can only name its top pick, so every patron asks one llm:choose +;; question, one after another. There is no forecast and no probability to +;; sample: the decision rule has no effect, and P(go) is just 1 or 0. +to decide-with-text-llm + let first-error "" + ask patrons [ + let picked "stay home" + ;; the prompt already holds the whole memory, so send no chat history + llm:clear-history + carefully [ + set picked llm:choose text-prompt lput "stay home" bar-names + ] [ + if first-error = "" [ set first-error error-message ] + note-failure + ] + if not failed? and picked != "stay home" [ + set going? true + set chosen-bar position picked bar-names + ] + set p-go ifelse-value going? [ 1 ] [ 0 ] + set color gray + ] + ;; one line per night is enough to show why calls failed + if first-error != "" [ + output-print (word "Night " (ticks + 1) ": some calls failed, first error: " + first-error) + ] +end + +;; A failed call sends the patron home and is counted, never aborting the night. +to note-failure ;; patron procedure + set failed? true + set going? false + set chosen-bar -1 + set color gray - 2 + set failed-calls failed-calls + 1 +end + +;; Each patron adds tonight to its memory and remembers how its own bar felt. +to remember-night + ask patrons [ + set remembered-attendance (map [ [nights tonight] -> + lput tonight but-first nights ] remembered-attendance attendance) + if chosen-bar >= 0 [ + let felt ifelse-value item chosen-bar attendance <= capacity + [ "fun" ] [ "too crowded" ] + set last-experience replace-item chosen-bar last-experience felt + ] + ] +end + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; PROMPTS: Jev is stateless, so each prompt carries the patron's whole memory +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +to-report situation ;; patron reporter + report (word "You are a " persona ". There are " num-bars " bars in town. " + "A bar is fun only when at most " capacity " people show up; with more " + "it is too crowded and you would rather have stayed home. " memory-text) +end + +to-report memory-text ;; patron reporter + let text "" + foreach range num-bars [ i -> + let nights item i remembered-attendance + set text (word text item i bar-names ": " item 0 nights ", " item 1 nights + " and " item 2 nights " people on the last three nights (most recent last)" + ", so it was " ifelse-value last nights <= capacity [ "fun" ] [ "too crowded" ] + " last night. The last time you went there it was " + item i last-experience ". ") + ] + report text +end + +to-report forecast-prompt ;; patron reporter + report (word situation "How crowded do you expect the bars to be tonight?") +end + +to-report go-prompt ;; patron reporter + report (word situation "You expect the bars to be " + item (round forecast) crowd-levels " tonight. Do you go out tonight?") +end + +to-report bar-prompt ;; patron reporter + report (word situation + "You have decided to go out tonight. Which bar do you go to?") +end + +to-report text-prompt ;; patron reporter + report (word situation "Do you go out tonight, and if so, to which bar?") +end + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; HELPERS +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +to-report jev-backend? + report backend = "typesafe (Jev)" +end + +to-report sampling? + report decision-rule = "sample from probabilities" +end + +;; Draws one answer from a [[answer probability] ...] distribution. +to-report sample-answer [ distribution ] + let r random-float 1 + foreach distribution [ pair -> + set r r - item 1 pair + if r < 0 [ report item 0 pair ] + ] + report item 0 last distribution +end + +;; The probability a distribution puts on one answer. +to-report probability-of [ answer distribution ] + let matches filter [ pair -> item 0 pair = answer ] distribution + report ifelse-value empty? matches [ 0 ] [ item 1 first matches ] +end + +;; Mean probability on each crowd level, over patrons whose forecast succeeded. +to-report forecast-averages + let answered patrons with [ not failed? ] + if not any? answered [ report n-values length crowd-levels [ 0 ] ] + report n-values length crowd-levels [ i -> + mean [ item i forecast-distribution ] of answered + ] +end + +;; Dollars spent since setup, or "n/a" when the provider reports no cost. +;; Before the first call nothing has been spent, so that is 0, not "n/a". +to-report total-cost + if llm:get llm:usage-total "calls" = 0 [ report 0 ] + let cost llm:get llm:usage-total "cost" + report ifelse-value is-number? cost [ cost ] [ "n/a" ] +end + +to-report fraction-out + report count patrons with [ chosen-bar >= 0 ] / num-patrons +end + +;; Fraction of all patrons the bars can hold while still being fun. +to-report fun-fraction + report capacity * num-bars / num-patrons +end + +to show-one-prompt + let someone one-of patrons + output-print (word "Night " (ticks + 1) ", a " [ persona ] of someone ":") + output-print ifelse-value jev-backend? + [ [ go-prompt ] of someone ] [ [ text-prompt ] of someone ] + output-print "" +end + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; VISUALS +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Patrons walk together, in a few steps, to a spot in their bar or at home. +to walk-to-destinations + let steps 8 + let targets [ ] + ask patrons [ + let spot ifelse-value chosen-bar >= 0 + [ one-of patches with [ region = [ chosen-bar ] of myself ] ] + [ one-of patches with [ region = -1 ] ] + face spot + set targets lput (list self spot (distance spot / steps)) targets + ] + repeat steps [ + foreach targets [ t -> ask item 0 t [ forward item 2 t ] ] + display + ] + foreach targets [ t -> ask item 0 t [ move-to-spot item 1 t ] ] +end + +;; Moves to a random point inside a patch so patrons on one patch stay visible. +to move-to-spot [ spot ] ;; patron procedure + move-to spot + setxy (xcor + random-float 0.8 - 0.4) (ycor + random-float 0.8 - 0.4) +end + +;; A bar turns red when it holds more than capacity tonight. Each sign shows +;; the bar's name and tonight's crowd against its capacity. +to recolor-bars + foreach range num-bars [ i -> + let crowded? item i attendance > capacity + let floor-color ifelse-value crowded? [ red - 3 ] [ item i bar-colors - 3 ] + ask patches with [ region = i ] [ set pcolor floor-color ] + ;; the sign spans the bar's columns; its label sits one patch in from the + ;; right end, because a patch label is drawn to the left of its patch + let sign patches with [ + region = -3 and [ region ] of patch-at 0 -1 = i + ] + ask sign [ + set pcolor ifelse-value crowded? [ red - 1 ] [ item i bar-colors - 1 ] + set plabel "" + ] + ask max-one-of sign with [ pxcor < max [ pxcor ] of sign ] [ pxcor ] [ + set plabel-color white + set plabel (word item i bar-names " " item i attendance " / " capacity + ifelse-value crowded? [ " FULL" ] [ "" ]) + ] + ] + ;; the count of patrons at home sits on the street, just above the lawn + ask patch (max-pxcor - 1) (min-pycor + 9) [ + set plabel-color white + set plabel (word "stayed home: " count patrons with [ chosen-bar < 0 ]) + ] +end +]]> + + + + + + + + + + + + + + + + + + night-ms + night-cost + total-cost + failed-calls + capacity + count patrons with [ chosen-bar >= 0 ] + mean-p-go + fraction-out + + + + create-temporary-plot-pen item i bar-names + set-plot-pen-color item i bar-colors + 1 +] +create-temporary-plot-pen "capacity" +set-plot-pen-color red]]> + + set-current-plot-pen item i bar-names + plot item i attendance +] +set-current-plot-pen "capacity" +plot capacity]]> + + + + + + + if ticks > 0 [ plot mean-p-go ] + + + + if ticks > 0 [ plot fraction-out ] + + + + if ticks > 0 [ plot fun-fraction ] + + + + + + + + plotxy i item i forecast-shares ]]]> + + + Patron colour is its crowd forecast: lime empty, yellow comfortable, orange busy, magenta packed. A bar turns red when more than its capacity show up. Switch decision-rule and compare the attendance plot: top pick herds, sampling spreads. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + setup + go + + item 0 attendance + item 1 attendance + item 2 attendance + capacity + mean-p-go + fraction-out + night-ms + night-cost + total-cost + failed-calls + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/demos/tests/config.txt.example b/demos/tests/config.txt.example index 27c431c..a8cab54 100644 --- a/demos/tests/config.txt.example +++ b/demos/tests/config.txt.example @@ -57,7 +57,7 @@ ollama_base_url=http://localhost:11434 # gemini-2.5-flash-lite fastest, cheapest # gemini-2.5-pro highest quality, slower # gemini-3-flash-preview newest fast model -# gemini-3-pro-preview newest high-end model +# gemini-3.1-pro-preview newest high-end model # ============================================================================= @@ -108,10 +108,10 @@ ollama_base_url=http://localhost:11434 # Models — always vendor-prefixed: # openai/gpt-4o-mini # openai/gpt-4.1-mini -# anthropic/claude-sonnet-4-20250514 -# google/gemini-2.0-flash-001 +# anthropic/claude-haiku-5.5 +# google/gemini-3.5-flash-lite # meta-llama/llama-3.3-70b-instruct -# deepseek/deepseek-r1 reasoning, exposes thinking +# deepseek/deepseek-v4.1-flash # ============================================================================= @@ -120,14 +120,14 @@ ollama_base_url=http://localhost:11434 # Key: https://api.together.xyz/settings/api-keys # provider=together -# model=meta-llama/Llama-3.3-70B-Instruct-Turbo +# model=Qwen/Qwen3.8-Flash # together_api_key=REPLACE_ME # Models: -# meta-llama/Llama-3.3-70B-Instruct-Turbo recommended default -# meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo small and fast -# deepseek-ai/DeepSeek-R1 reasoning model -# Qwen/Qwen2.5-72B-Instruct-Turbo +# Qwen/Qwen3.8-Flash recommended default +# Qwen/Qwen3.5-9B small and fast +# deepseek-ai/DeepSeek-V4.1-Flash DeepSeek model +# openai/gpt-oss-120b # ============================================================================= diff --git a/demos/tests/tests.nlogox b/demos/tests/tests.nlogox index bcd32be..6256724 100644 --- a/demos/tests/tests.nlogox +++ b/demos/tests/tests.nlogox @@ -240,8 +240,8 @@ to test-openrouter-vendor-prefix let original-model item 1 llm:active llm:set-model "openai/gpt-4o-mini" test-pass "set-model openai/gpt-4o-mini accepted" - llm:set-model "anthropic/claude-3.5-haiku" - test-pass "set-model anthropic/claude-3.5-haiku accepted" + llm:set-model "anthropic/claude-haiku-5.5" + test-pass "set-model anthropic/claude-haiku-5.5 accepted" llm:set-model original-model end diff --git a/docs/API-REFERENCE.md b/docs/API-REFERENCE.md index de9f8c6..60805ae 100644 --- a/docs/API-REFERENCE.md +++ b/docs/API-REFERENCE.md @@ -14,6 +14,8 @@ The NetLogo Multi-LLM Extension provides a unified interface for multiple Large | `llm:chat-with-template file vars` | Chat | Send templated prompt with variable substitution | | `llm:chat-with-thinking text` | Chat | Returns `[answer thinking]` for reasoning-capable models | | `llm:choose prompt choices` | Chat | Force selection from provided options | +| `llm:decide prompt spec` | Decision | Typed decision with probabilities: `[answer probability distribution]` (TypeSafe) | +| `llm:decide-all agents [ -> prompt ] spec` | Decision | One decision per agent, sent together; returns `[agent result ok?]` triples | | `llm:chat-with-schema prompt schema` | Structured | Schema-constrained chat, returns parsed nested lists | | `llm:chat-json prompt` | Structured | Force valid JSON output, returns the raw JSON string | | `llm:get parsed key` | Structured | Look up a key in a `[[key value] ...]` list | @@ -31,7 +33,7 @@ The NetLogo Multi-LLM Extension provides a unified interface for multiple Large | `llm:use-profile name` | Profiles | Route the calling agent's calls through that profile | | `llm:profile` | Profiles | The calling agent's profile name (`"default"` if none) | | `llm:profiles` | Profiles | Names of the loaded profiles | -| `llm:set-provider name` | Configuration | Set active provider (openai, anthropic, gemini, ollama, openrouter, together) | +| `llm:set-provider name` | Configuration | Set active provider (openai, anthropic, gemini, ollama, openrouter, together, groq, typesafe) | | `llm:set-api-key key` | Configuration | Set API key for current provider | | `llm:set-model name` | Configuration | Set model to use for current provider | | `llm:providers` | Discovery | List ready providers with configured keys/servers | @@ -92,7 +94,7 @@ llm:load-config "models/gpt4-config.txt" ```netlogo llm:set-provider "openai" -; Sets default model (gpt-4o-mini) and validates API key +; Sets the default model from the model catalog (gpt-6-luna) and validates API key llm:set-provider "ollama" ; Sets default model (llama3.2) and checks if Ollama server is reachable @@ -142,8 +144,8 @@ llm:set-api-key "sk-ant-your-claude-key-here" **Example**: ```netlogo -llm:set-model "gpt-4o-mini" -llm:set-model "claude-3-5-haiku-latest" +llm:set-model "gpt-6-luna" +llm:set-model "claude-haiku-5-5" llm:set-model "llama3.2" ``` @@ -319,6 +321,125 @@ set color read-from-string color-choice more reliable where the constraint is available. - Still raises an error if the reply matches no option. Wrap in `carefully` if a model should tolerate that. +- On a provider with a decision API (`typesafe`), `llm:choose` asks a native choice + question and reports the top option. That call sends only the prompt, with no + history, and writes nothing to history; usage is still recorded. Use + `llm:decide` with `["choice" ...]` to also get the probabilities. The options + follow the same rules as llm:decide's choice spec, checked before anything is + sent: 1 to 255 distinct, non-empty options (for example, `["a" "a"]` raises + `llm:choose: "choice" options must be distinct, but "a" appears more than once`). + If the provider picks something that is not one of the options, the call raises + `llm:choose: provider picked "", which is not one of the options: ...`. + +## Decision Primitives + +Decision providers answer a typed question about a prompt with a probability for +every possible answer, instead of writing text. The first one is TypeSafe's Jev +(`llm:set-provider "typesafe"`, see the Provider Guide). On any other provider these +primitives raise `provider "" has no decision API; use llm:choose, or +llm:set-provider "typesafe"`. On `typesafe`, the free-text primitives (`llm:chat`, +`llm:chat-async`, `llm:chat-all`, `llm:chat-with-template`, `llm:chat-with-thinking`, +`llm:chat-json`, `llm:chat-with-schema`) raise `typesafe (Jev) returns decisions, not +text; use llm:decide, llm:decide-all or llm:choose`. `llm:choose` works on both. + +### llm:decide + +**Syntax**: `llm:decide prompt spec` + +**Description**: Asks one typed question about `prompt` and reports the answer with its +probabilities. + +**Parameters**: + +- `prompt` (string): Everything the model should know. It is the whole context: no + history is sent. +- `spec` (list): The question type and its answer space: + - `["choice" [options]]` — pick one of 1 to 255 distinct, non-empty strings + - `["score" [levels]]` — rate on 2 to 10 ordered levels, lowest first, numbered from 0 + - `["yes-no"]` — answer yes or no + +**Returns**: A list `[answer probability distribution]`. `distribution` lists every +possible answer with its probability, in the order of your spec. + +| Spec | Example result | `answer` | `probability` | +| ---- | -------------- | -------- | ------------- | +| `["choice" ["move" "stay"]]` | `["move" 0.7 [["move" 0.7] ["stay" 0.3]]]` | the most likely option | probability of that option | +| `["score" ["low" "medium" "high"]]` | `[1.43 0.62 [["low" 0.1] ["medium" 0.37] ["high" 0.53]]]` | expected level on the 0-based scale, a number that can fall between levels | the model's confidence in the score (1 = all probability on one level) | +| `["yes-no"]` | `[true 0.82 [[true 0.82] [false 0.18]]]` | `true` or `false`, whichever is likelier | probability of that answer | + +**Example**: El Farol bar. Each agent builds its own prompt from its own variables, then +either takes the top pick or samples from the distribution: + +```netlogo +turtles-own [ last-week-crowd times-went mood ] + +to-report decide-to-go ;; turtle procedure + let prompt (word "You are deciding whether to go to the El Farol bar tonight. " + "Last week " last-week-crowd " of 100 people went. " + "It is fun only if fewer than 60 go. " + "You have gone " times-went " times this month and feel " mood ".") + let result llm:decide prompt ["choice" ["go" "stay home"]] + ;; top pick: always the most likely answer + report item 0 result = "go" +end + +to-report sample-decision [ result ] ;; draw an answer with the model's probabilities + let r random-float 1 + foreach item 2 result [ pair -> + set r r - item 1 pair + if r < 0 [ report item 0 pair ] + ] + report item 0 last item 2 result +end +``` + +Sampling (`sample-decision`) keeps the population heterogeneous: with 0.6 on "go", about +60% of identical agents go, where the top pick would send all of them. +The full model, with a forecast, a bar choice, plots and measured cost, is +`demos/el-farol-decisions/el-farol-decisions.nlogox`. + +**Notes**: + +- The spec is checked before any request: an unknown type, an empty or oversized option + list, duplicate or empty options, fewer than 2 or more than 10 levels, or a list after + `"yes-no"` raises an error and costs nothing. +- Nothing is written to history. Usage is recorded for the calling agent (`llm:usage`), + and a `clear-all` while a call is in flight drops its usage like any other call. +- The request goes through the provider's throttle and retry path, like `llm:chat`. + +### llm:decide-all + +**Syntax**: `llm:decide-all agentset prompt-reporter spec` + +**Description**: `llm:chat-all` for decisions: one request per agent, all sent at once, +one wait. The prompt reporter runs as each agent, so each prompt can read that agent's +variables. + +**Returns**: A list of `[agent result ok?]` triples, one per agent. `result` is the +`[answer probability distribution]` list when `ok?` is `true`, or the error text when that +agent's call failed. One failure never aborts the batch. + +**Example**: + +```netlogo +to go + let results llm:decide-all turtles + [ -> (word "Last week " last-week-crowd " of 100 went. You feel " mood ". Go tonight?") ] + ["yes-no"] + foreach results [ r -> + ask item 0 r [ set going? ifelse-value item 2 r [ item 0 item 1 r ] [ false ] ] + ] + tick +end +``` + +**Notes**: + +- Same batching rules as `llm:chat-all`: every prompt is built before anything is sent, + requests go through each agent's provider and throttle, and the wait gives up only when + no reply has arrived for a whole timeout budget. +- A provider without a decision API raises the error for the whole batch before anything + is sent. Nothing is written to history; usage is recorded per agent. ## Structured Output Primitives @@ -578,7 +699,7 @@ end ## Reasoning / Thinking Primitives -For models that expose intermediate reasoning (Anthropic Claude, Google Gemini, Ollama qwen3/deepseek-r1, OpenRouter, Together AI). OpenAI o-series uses internal reasoning that the API does not return. +For models that expose intermediate reasoning (Anthropic Claude, Google Gemini, Ollama qwen3/deepseek-r1, OpenRouter, Together AI). OpenAI reasoning models (GPT-5.x, GPT-6.x, o-series) use internal reasoning that the API does not return. ### llm:chat-with-thinking @@ -846,7 +967,7 @@ print llm:provider-help "openai" ```netlogo let current llm:active -print current ; ["openai" "gpt-4o-mini"] +print current ; ["openai" "gpt-6-luna"] print (word "Using " item 0 current " with " item 1 current) ``` @@ -867,7 +988,7 @@ print (word "Using " item 0 current " with " item 1 current) ```netlogo print llm:config -; Output: provider=openai, model=gpt-4o-mini, openai_api_key=sk-pr...jYzQ, ... +; Output: provider=openai, model=gpt-6-luna, openai_api_key=sk-pr...jYzQ, ... ``` **Notes**: @@ -895,29 +1016,25 @@ print llm:config ```netlogo print llm:list-models -; Output format: -; === OpenAI Models === -; gpt-4o [ACTIVE] -; gpt-4o-mini -; gpt-4-turbo -; gpt-3.5-turbo -; o1-preview -; o1-mini +; Output format (abridged): +; === Available Models === +; +; --- anthropic --- +; claude-haiku-5-5 +; claude-sonnet-5-5 +; ... ; -; === Anthropic Models === -; claude-3-5-sonnet-20241022 -; claude-3-5-haiku-latest -; claude-3-opus-latest +; --- gemini --- +; gemini-3.5-flash-lite [reasoning] +; gemini-3.8-flash [reasoning] [custom] +; ... ; -; === Gemini Models === [ACTIVE] -; gemini-2.0-flash-exp [custom] [ACTIVE] -; gemini-1.5-pro -; gemini-1.5-flash +; --- openai --- +; gpt-4o-mini +; gpt-6-luna [ACTIVE] [reasoning] +; ... ; -; === Ollama Models === -; llama3.2 -; mistral -; qwen2 +; Currently using: openai / gpt-6-luna ; Use it to see what models are available llm:set-provider "anthropic" @@ -931,6 +1048,9 @@ print llm:list-models ; Shows all providers, with Anthropic marked as ACTIVE - Use this to discover available models across all providers - Custom models added via `models-override.yaml` are marked with `[custom]` - The currently active provider and model are marked with `[ACTIVE]` +- The list and each provider's default model come from the model catalog: the + bundled `models.yaml`, then your `~/.netlogo-llm/models.yaml` if present, then + `models-override.yaml` next to the model. See CONFIGURATION.md, "Model Catalog" ## Profiles: different models for different agents @@ -1357,7 +1477,7 @@ For enterprise or custom deployments: provider=openai base_url=https://your-custom-openai-endpoint.com/v1 api_key=your-key -model=gpt-4o-mini +model=gpt-6-luna ``` ### Timeout Tuning diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 29dd18e..c9db29b 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -49,7 +49,7 @@ Starting from this version, configuration is validated immediately: ``` provider=openai openai_api_key=sk-REPLACE_ME -model=gpt-4o-mini +model=gpt-6-luna temperature=0.7 max_tokens=1000 timeout_seconds=30 @@ -59,7 +59,7 @@ timeout_seconds=30 ``` provider=anthropic anthropic_api_key=sk-ant-REPLACE_ME -model=claude-3-5-sonnet-20241022 +model=claude-haiku-5-5 temperature=0.7 max_tokens=4000 timeout_seconds=30 @@ -69,7 +69,7 @@ timeout_seconds=30 ``` provider=gemini gemini_api_key=REPLACE_ME -model=gemini-1.5-flash +model=gemini-3.5-flash-lite gemini_base_url=https://generativelanguage.googleapis.com/v1beta temperature=0.7 max_tokens=2048 @@ -80,7 +80,7 @@ timeout_seconds=30 ``` provider=openrouter openrouter_api_key=sk-or-REPLACE_ME -model=openai/gpt-4o-mini +model=openai/gpt-oss-20b temperature=0.7 max_tokens=1000 timeout_seconds=30 @@ -90,12 +90,24 @@ timeout_seconds=30 ``` provider=together together_api_key=REPLACE_ME -model=meta-llama/Llama-3.3-70B-Instruct-Turbo +model=Qwen/Qwen3.8-Flash temperature=0.7 max_tokens=1000 timeout_seconds=30 ``` +### TypeSafe (Jev, decisions with probabilities) +Jev answers typed questions instead of writing text: use it with `llm:decide`, +`llm:decide-all` and `llm:choose` (the chat primitives raise an error on it). +`temperature` and `max_tokens` do not apply. +``` +provider=typesafe +typesafe_api_key=REPLACE_ME +model=jev-latest +timeout_seconds=30 +# typesafe_base_url=https://api.typesafe.ai +``` + ### Local (Ollama, no API key) ``` provider=ollama @@ -115,14 +127,15 @@ anthropic_api_key=sk-ant-REPLACE_ME gemini_api_key=REPLACE_ME openrouter_api_key=sk-or-REPLACE_ME together_api_key=REPLACE_ME +typesafe_api_key=REPLACE_ME # Set the active provider provider=openai -model=gpt-4o-mini +model=gpt-6-luna # Switch providers at runtime: # llm:set-provider "anthropic" -# llm:set-model "claude-3-5-sonnet-20241022" +# llm:set-model "claude-sonnet-5-5" ``` ### Profiles: one config file per model, different agents on different models @@ -198,7 +211,7 @@ sudo systemctl enable --now ollama ollama pull llama3.2 # or ollama pull mistral -ollama pull qwen2 +ollama pull qwen3.5:4b ``` 4) Verify installed models @@ -242,7 +255,7 @@ llm:load-config "config.txt" ;; or a path like "configs/openai.txt" ``` You can override values at runtime: ``` -llm:set-model "gpt-4o" +llm:set-model "gpt-6.1-sol" ``` ## Verifying & Troubleshooting @@ -261,11 +274,63 @@ show llm:chat "Say hello in five words." - Do not commit secrets. Keep your config in `.gitignore`. - Use `demos/config-reference.txt` as a safe template and keep your real `config.txt` local. -## Custom Model Registry (models-override.yaml) +## Model Catalog (models.yaml, user catalog, models-override.yaml) + +The list of known models and each provider's default model live in YAML, not in +the extension's code. When a model ID goes stale you edit a YAML file; no +rebuild is needed. + +Three layers are read, each on top of the one before: + +1. **Bundled catalog** — `models.yaml` inside the extension JAR. +2. **User catalog** — `~/.netlogo-llm/models.yaml`, read once when the extension + starts. It applies to every model you open. To use a different path, set the + `llm.models.user.file` Java system property or the `NETLOGO_LLM_MODELS` + environment variable. +3. **Model-directory override** — `models-override.yaml` next to your `.nlogo` + file, read by `llm:load-config`. + +### Format +Each provider is either a plain list of model IDs, or a mapping with a +`default:` and a `models:` list (either may be left out, but not both): + +```yaml +openai: + default: gpt-6-luna + models: + - gpt-6-luna + - gpt-6.1-sol + +ollama: + - llama3.2 + - my-custom-finetune:latest +``` + +`default:` is the model a provider uses when neither your config file +(`model=`) nor `llm:set-model` names one. Those two still take precedence. When +no layer sets `default:` for a provider, the extension's built-in default for +that provider is used. -You can add custom or newly released models without waiting for an extension update by creating a `models-override.yaml` file. +### User catalog (~/.netlogo-llm/models.yaml) +The user catalog **adds** to the bundled catalog rather than replacing it: +- models you list are added to that provider's bundled list +- a `default:` replaces the bundled default, and that model is added to the + provider's list even if you do not list it under `models:`, so it is never + reported as unknown +- a provider the bundled catalog does not have is added -### Location +```yaml +# ~/.netlogo-llm/models.yaml +openai: + default: gpt-6.1-sol # make this the OpenAI default everywhere +gemini: + - gemini-4-argon-preview # a model released after this extension version +``` + +If the file cannot be parsed, the extension prints a warning naming the file and +carries on with the bundled catalog. A missing file is fine. + +### Model-directory override (models-override.yaml) Save the file in the same directory as your `.nlogo` model file: ``` my-model.nlogo @@ -273,49 +338,41 @@ models-override.yaml <-- Place it here config.txt ``` -### Format -The file uses YAML format with provider sections. Each provider section completely replaces the bundled models for that provider: +Each provider section you include **completely replaces** that provider's list +from the layers below (bundled + user catalog): ```yaml openai: - - gpt-4o - - gpt-4o-mini - - gpt-4-turbo - - o1-preview - - o1-mini - - gpt-4o-2024-11-20 # Add new model + - gpt-6-luna + - gpt-6.1-sol + - gpt-6-astra # Add new model anthropic: - - claude-3-5-sonnet-20241022 - - claude-3-5-haiku-latest - - claude-3-opus-latest - - claude-3-7-sonnet-20250219 # Add new model - -gemini: - - gemini-2.0-flash-exp # Override to use only newest model - - gemini-1.5-pro - - gemini-1.5-flash + default: claude-sonnet-5-5 + models: + - claude-haiku-5-5 + - claude-sonnet-5-5 ollama: - llama3.2 - mistral - - qwen2 - deepseek-r1:latest # Add locally installed model ``` ### Behavior -- **Complete replacement**: Each provider section you include completely replaces the bundled models for that provider +- **Complete replacement**: Each provider section you include completely replaces the models for that provider - **Partial override**: You can override just one provider and leave others unchanged +- **Defaults carry over**: a section without `default:` keeps the default from the layers below +- **Default is a known model**: a section's `default:` is added to that section's model list - **Custom marker**: Models from the override file are marked with `[custom]` in `llm:list-models` output -- **Validation**: The extension still validates that the model you select exists in the combined registry +- **Validation**: `llm:set-model` warns (but still proceeds) when a model is not in the combined catalog ### Use Cases 1. **New model releases**: Add newly announced models before the extension is updated ```yaml openai: - - gpt-4o - - gpt-4o-mini - - gpt-5 # Hypothetical future model + - gpt-6-luna + - gpt-7-preview # Hypothetical future model ``` 2. **Local Ollama models**: Include models you've installed locally @@ -329,15 +386,15 @@ ollama: 3. **Simplify model list**: Reduce clutter by listing only the models you use ```yaml anthropic: - - claude-3-5-sonnet-20241022 # Only show the model I use + - claude-haiku-5-5 # Only show the model I use ``` ### Example 1. Create `models-override.yaml` next to your model: ```yaml gemini: - - gemini-2.0-flash-exp - - gemini-1.5-pro + - gemini-3.8-flash + - gemini-3.1-pro-preview ``` 2. Load config and check available models: @@ -347,19 +404,20 @@ ollama: print llm:list-models ; Output shows: - ; === Gemini Models === - ; gemini-2.0-flash-exp [custom] - ; gemini-1.5-pro [custom] + ; --- gemini --- + ; gemini-3.1-pro-preview [reasoning] [custom] + ; gemini-3.8-flash [reasoning] [custom] ``` 3. Set a custom model: ```netlogo llm:set-provider "gemini" - llm:set-model "gemini-2.0-flash-exp" ; Uses custom model + llm:set-model "gemini-3.8-flash" ; Uses custom model ``` ### Notes -- The override file is optional - if not present, bundled models are used -- Invalid YAML syntax will cause an error when loading config +- The override file is optional - if not present, the bundled and user catalogs are used +- Invalid YAML prints a warning and the override is ignored - The file is loaded automatically when you call `llm:load-config` -- You must include ALL models you want for a provider - no merging with bundled list +- You must include ALL models you want for a provider - no merging with the lower layers' list (use the user catalog to add instead) +- If the bundled catalog itself cannot be read, each provider falls back to its built-in default model only diff --git a/docs/EXAMPLES.md b/docs/EXAMPLES.md index 91c4bbd..10a6c72 100644 --- a/docs/EXAMPLES.md +++ b/docs/EXAMPLES.md @@ -11,7 +11,7 @@ extensions [llm] to setup llm:set-provider "openai" - llm:set-model "gpt-4" + llm:set-model "gpt-6.1-sol" end to example-template-usage @@ -481,13 +481,13 @@ end ### Reasoning Models — Show the Model's Work -Use `llm:chat-with-thinking` to get both the final answer and the model's intermediate reasoning. Works with Anthropic Claude, Gemini 2.x, Ollama qwen3 / deepseek-r1, OpenRouter reasoning models, and Together AI's DeepSeek-R1. +Use `llm:chat-with-thinking` to get both the final answer and the model's intermediate reasoning. Works with Anthropic Claude, Gemini 2.5 and 3.x, Ollama qwen3 / deepseek-r1, OpenRouter reasoning models, Groq gpt-oss, and Together AI reasoning models. ```netlogo extensions [llm] to setup-reasoning - llm:load-config "config.txt" ;; e.g. provider=together, model=deepseek-ai/DeepSeek-R1 + llm:load-config "config.txt" ;; e.g. provider=together, model=deepseek-ai/DeepSeek-V4.1-Flash llm:set-thinking true llm:set-reasoning-effort "high" ;; OpenAI o-series + OpenRouter + Together hybrid llm:set-thinking-budget 4096 ;; Anthropic + Gemini @@ -521,7 +521,7 @@ end to compare-vendors-via-openrouter let prompt "Summarize agent-based modeling in one sentence." - foreach ["openai/gpt-4o-mini" "anthropic/claude-3.5-haiku" "meta-llama/llama-3.3-70b-instruct"] [ m -> + foreach ["openai/gpt-oss-20b" "anthropic/claude-haiku-5.5" "meta-llama/llama-3.3-70b-instruct"] [ m -> llm:set-model m print (word m ": " llm:chat prompt) llm:clear-history @@ -991,15 +991,15 @@ end to optimize-for-mode [mode] if mode = "speed" [ - llm:set-model "gpt-4o-mini" ; Fastest model + llm:set-model "gpt-6-luna" ; Fastest model ; Set low temperature and tokens for speed ] if mode = "quality" [ - llm:set-model "gpt-4o" ; Best model + llm:set-model "gpt-6-astra" ; Best model ; Allow higher tokens and temperature ] if mode = "balanced" [ - llm:set-model "gpt-4o-mini" ; Good balance + llm:set-model "gpt-6.1-sol" ; Good balance ; Moderate settings ] end diff --git a/docs/PROVIDER-GUIDE.md b/docs/PROVIDER-GUIDE.md index 8292f2d..1a87290 100644 --- a/docs/PROVIDER-GUIDE.md +++ b/docs/PROVIDER-GUIDE.md @@ -4,6 +4,11 @@ This guide provides comprehensive configuration details for all supported LLM providers, including hyperparameters, model options, and provider-specific settings. +Model lists and each provider's default model come from the extension's model +catalog (`models.yaml`), current as of October 2026. Model IDs retire often; to +add a new ID or change a default without updating the extension, put a +`models.yaml` in `~/.netlogo-llm/` — see [CONFIGURATION.md](CONFIGURATION.md#model-catalog-modelsyaml-user-catalog-models-overrideyaml). + ## OpenAI Configuration ### API Setup @@ -18,7 +23,7 @@ This guide provides comprehensive configuration details for all supported LLM pr # Required Parameters provider=openai api_key=sk-your-openai-key-here -model=gpt-4o-mini +model=gpt-6-luna # Optional Parameters base_url=https://api.openai.com/v1 @@ -31,13 +36,29 @@ timeout_seconds=30 | Model | Description | Context | Cost (per 1K tokens) | |-------|-------------|---------|---------------------| -| `gpt-4o-mini` | Fast, cost-effective GPT-4 | 128K | Input: $0.00015, Output: $0.0006 | -| `gpt-4o` | Most capable GPT-4 variant | 128K | Input: $0.005, Output: $0.015 | -| `gpt-4-turbo` | Fast GPT-4 with large context | 128K | Input: $0.01, Output: $0.03 | -| `gpt-4` | Original GPT-4 | 8K | Input: $0.03, Output: $0.06 | -| `gpt-3.5-turbo` | Legacy but capable | 16K | Input: $0.0005, Output: $0.0015 | - -**Recommended for NetLogo**: `gpt-4o-mini` (best balance of speed, capability, and cost) +| `gpt-6-luna` | Fast, cost-effective reasoning model (default) | 1.05M | Input: $0.0001, Output: $0.0005 | +| `gpt-6.1-sol` | Mid-tier GPT-6 | — | Input: $0.002, Output: $0.01 | +| `gpt-6-astra` | Most capable GPT-6 | — | Input: $0.01, Output: $0.05 | +| `gpt-5.6-luna` / `gpt-5.6-terra` / `gpt-5.6-sol` | Previous GPT-5.x reasoning models | — | See OpenAI pricing | +| `gpt-4o-mini` | Deprecated, served until 2027-01-14 | 128K | Input: $0.00015, Output: $0.0006 | + +**Recommended for NetLogo**: `gpt-6-luna` (best balance of speed, capability, and cost) + +GPT-5.x and GPT-6.x are reasoning models, like the retired o-series. The +extension sends them in reasoning format automatically: system prompts go as +`developer` messages, `max_tokens` is sent as `max_completion_tokens`, and +`temperature` is not sent. Set depth with `reasoning_effort` +(`none|low|medium|high|xhigh`). + +Reasoning tokens count against `max_completion_tokens`. When you have not set +`max_tokens`, reasoning models are sent `max_completion_tokens=4000` instead of +the provider default (1000), so the model has room left to answer. Any +`max_tokens` you set yourself, in a config file or a profile, is sent unchanged, +even one below 4000. If a reply still comes back empty because the model used +its whole budget (`finish_reason=length`), the call fails with an error that +names the tokens used and tells you to set `max_tokens` above that. The tokens +are still counted in `llm:usage` and `llm:usage-total`. The same applies to +reasoning models on OpenRouter, Together AI and Groq. ### Hyperparameters @@ -80,7 +101,7 @@ For Azure OpenAI or custom deployments: provider=openai base_url=https://your-resource.openai.azure.com/openai/deployments/your-deployment/chat/completions?api-version=2024-02-15-preview api_key=your-azure-key -model=gpt-4o-mini +model=gpt-6-luna ``` ## Anthropic (Claude) Configuration @@ -97,7 +118,7 @@ model=gpt-4o-mini # Required Parameters provider=anthropic api_key=sk-ant-your-anthropic-key-here -model=claude-3-haiku-20240307 +model=claude-haiku-5-5 # Optional Parameters base_url=https://api.anthropic.com/v1 @@ -110,12 +131,16 @@ timeout_seconds=30 | Model | Description | Context | Cost (per 1K tokens) | |-------|-------------|---------|---------------------| -| `claude-3-haiku-20240307` | Fast, cost-effective | 200K | Input: $0.00025, Output: $0.00125 | -| `claude-3-sonnet-20240229` | Balanced performance | 200K | Input: $0.003, Output: $0.015 | -| `claude-3-opus-20240229` | Most capable Claude | 200K | Input: $0.015, Output: $0.075 | -| `claude-3-5-sonnet-20241022` | Latest improved Sonnet | 200K | Input: $0.003, Output: $0.015 | +| `claude-haiku-5-5` | Fast, cost-effective (default) | — | Input: $0.0001, Output: $0.0005 (prompts up to 100K) | +| `claude-sonnet-5-5` | Balanced performance | — | Input: $0.002, Output: $0.01 | +| `claude-opus-5-5` | Most capable Opus | — | Input: $0.004, Output: $0.02 | +| `claude-fable-5-1` | Most capable Claude | — | Input: $0.01, Output: $0.05 | +| `claude-haiku-4-5-20251001` | Previous Haiku (still served) | 200K | See Anthropic pricing | + +**Recommended for NetLogo**: `claude-haiku-5-5` (excellent speed and capability for most use cases) -**Recommended for NetLogo**: `claude-3-haiku-20240307` (excellent speed and capability for most use cases) +Claude 4.7 and later (including every 5.x model) reject a non-default +`temperature`, so the extension leaves it out for them. ### Hyperparameters @@ -170,7 +195,7 @@ top_k=20 # Moderately focused # Required Parameters provider=gemini api_key=your-google-ai-api-key-here -model=gemini-1.5-flash +model=gemini-3.5-flash-lite # Optional Parameters base_url=https://generativelanguage.googleapis.com/v1beta @@ -183,12 +208,17 @@ timeout_seconds=30 | Model | Description | Context | Cost (per 1K tokens) | |-------|-------------|---------|---------------------| -| `gemini-1.5-flash` | Fast, free tier available | 1M | Free tier available | -| `gemini-1.5-pro` | Most capable Gemini | 2M | Input: $0.00125, Output: $0.005 | -| `gemini-1.0-pro` | Original Gemini Pro | 32K | Input: $0.0005, Output: $0.0015 | -| `gemini-pro` | Alias for gemini-1.0-pro | 32K | Input: $0.0005, Output: $0.0015 | +| `gemini-3.5-flash-lite` | Fast, free tier available (default) | — | Free tier available | +| `gemini-3.8-flash` | Stronger Flash tier | — | See Google pricing | +| `gemini-3.1-pro-preview` | Most capable Gemini (preview) | — | See Google pricing | +| `gemini-2.5-flash` / `gemini-2.5-pro` | Only for projects that already used 2.5 | 1M | See Google pricing | -**Recommended for NetLogo**: `gemini-1.5-flash` (excellent free tier, very fast) +**Recommended for NetLogo**: `gemini-3.5-flash-lite` (excellent free tier, very fast) + +Thinking depth: Gemini 3 and later take `reasoning_effort`, sent as +`thinkingLevel` (`none` becomes `MINIMAL`, `xhigh` becomes `HIGH`); when an +effort is set, `thinking_budget_tokens` is not sent with it, because the API +rejects the two together. Gemini 2.5 takes only `thinking_budget_tokens`. ### Hyperparameters @@ -250,7 +280,7 @@ candidate_count=1 # Default (recommended) # Required Parameters provider=openrouter openrouter_api_key=sk-or-your-openrouter-key-here -model=openai/gpt-4o-mini +model=openai/gpt-oss-20b # Optional Parameters openrouter_base_url=https://openrouter.ai/api/v1 @@ -265,18 +295,20 @@ Model names use vendor prefixes (`vendor/model-name`): | Model | Description | Context | |-------|-------------|---------| -| `openai/gpt-4o` | OpenAI GPT-4o via OpenRouter | 128K | -| `openai/gpt-4o-mini` | Fast, cost-effective GPT-4 | 128K | -| `openai/o3-mini` | OpenAI reasoning model | 128K | -| `anthropic/claude-3.5-sonnet` | Anthropic Claude 3.5 Sonnet | 200K | -| `anthropic/claude-sonnet-4-20250514` | Latest Claude Sonnet 4 | 200K | -| `google/gemini-2.5-flash` | Google Gemini Flash | 1M | -| `google/gemini-2.5-pro` | Google Gemini Pro | 1M | +| `openai/gpt-oss-20b` | Fast open-weight reasoning model (default) | 131K | +| `openai/gpt-oss-120b` | Larger open-weight reasoning model | 131K | +| `openai/gpt-6-luna` | OpenAI GPT-6 Luna via OpenRouter | 1.05M | +| `anthropic/claude-haiku-5.5` | Anthropic Claude Haiku 5.5 | — | +| `google/gemini-3.5-flash-lite` | Google Gemini Flash-Lite | — | +| `google/gemini-3.8-flash` | Google Gemini Flash | — | | `meta-llama/llama-3.3-70b-instruct` | Meta Llama 3.3 70B | 128K | -| `deepseek/deepseek-r1` | DeepSeek reasoning model | 64K | -| `deepseek/deepseek-chat-v3-0324` | DeepSeek Chat V3 | 64K | +| `deepseek/deepseek-v4.1-flash` | DeepSeek V4.1 Flash | — | +| `qwen/qwen3.8-flash` | Qwen 3.8 Flash | — | +| `openrouter/auto` | OpenRouter picks a model per request | — | + +**Recommended for NetLogo**: `openai/gpt-oss-20b` (fast, cheap, capable) -**Recommended for NetLogo**: `openai/gpt-4o-mini` (fast, cheap, capable) +Avoid `:free` model variants as a default; OpenRouter adds and removes them often. ### Why OpenRouter? @@ -296,7 +328,7 @@ translates automatically: ```netlogo llm:set-provider "openrouter" -llm:set-model "deepseek/deepseek-r1" +llm:set-model "openai/gpt-oss-20b" llm:set-thinking true llm:set-reasoning-effort "high" ``` @@ -315,7 +347,7 @@ llm:set-reasoning-effort "high" # Required Parameters provider=together together_api_key=your-together-key-here -model=meta-llama/Llama-3.3-70B-Instruct-Turbo +model=Qwen/Qwen3.8-Flash # Optional Parameters together_base_url=https://api.together.xyz/v1 @@ -330,31 +362,33 @@ Model names use vendor prefixes (`vendor/model-name`): | Model | Description | Context | |-------|-------------|---------| -| `meta-llama/Llama-3.3-70B-Instruct-Turbo` | Fast Llama 3.3 70B | 128K | -| `meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo` | Largest Llama | 128K | -| `meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo` | Small, fast Llama | 128K | -| `deepseek-ai/DeepSeek-R1` | DeepSeek reasoning model | 64K | -| `deepseek-ai/DeepSeek-V3` | DeepSeek V3 chat | 64K | -| `Qwen/Qwen2.5-72B-Instruct-Turbo` | Qwen 2.5 72B | 128K | -| `google/gemma-3-27b-it` | Google Gemma 3 | 128K | -| `mistralai/Mistral-Small-24B-Instruct-2501` | Mistral Small | 32K | +| `Qwen/Qwen3.8-Flash` | Fast Qwen 3.8 (default) | — | +| `Qwen/Qwen3.5-9B` | Small, cheap Qwen | — | +| `Qwen/Qwen3.7-Plus` | Larger Qwen | — | +| `openai/gpt-oss-120b` | Open-weight reasoning model | 131K | +| `deepseek-ai/DeepSeek-V4.1-Flash` | DeepSeek V4.1 Flash | — | +| `zai-org/GLM-5.3-Flash` | GLM 5.3 Flash | — | +| `meta-models/Muse-Glimmer-30B` | Meta Muse Glimmer 30B | — | +| `moonshotai/Kimi-K3` | Kimi K3 | — | -**Recommended for NetLogo**: `meta-llama/Llama-3.3-70B-Instruct-Turbo` (fast, capable, good value) +**Recommended for NetLogo**: `Qwen/Qwen3.8-Flash` (fast, capable, good value: $0.15 in / $0.47 out per 1M tokens) + +Together IDs are case-sensitive. No Llama model is left on Together serverless. ### Why Together AI? - **Fast inference** on popular open-source models - **Pay-per-token** pricing, often cheaper than proprietary APIs -- **Open-source models** — Llama, DeepSeek, Qwen, Gemma, Mistral -- **Reasoning support** — DeepSeek-R1 with thinking output +- **Open-source models** — Qwen, DeepSeek, GLM, gpt-oss, Kimi +- **Reasoning support** — Qwen 3.8 and gpt-oss with thinking output ### Thinking/Reasoning Models -Together AI supports reasoning models like DeepSeek-R1: +Together AI supports reasoning models like Qwen 3.8: ```netlogo llm:set-provider "together" -llm:set-model "deepseek-ai/DeepSeek-R1" +llm:set-model "Qwen/Qwen3.8-Flash" llm:set-thinking true let result llm:chat-with-thinking "What is 15 * 17?" ; result is [answer thinking-text] @@ -389,11 +423,12 @@ timeout_seconds=30 |-------|-------------|---------| | `openai/gpt-oss-20b` | Fast open-weight reasoning model | 131K | | `openai/gpt-oss-120b` | Larger open-weight reasoning model | 131K | -| `openai/gpt-oss-safeguard-20b` | Safety-tuned gpt-oss variant | 131K | -| `groq/compound` | Agentic system with built-in tool use | 131K | -| `groq/compound-mini` | Smaller agentic system | 131K | -| `qwen/qwen3.6-27b` | Qwen 3.6 reasoning model (preview) | 131K | +| `qwen/qwen3.8-27b` | Qwen 3.8 reasoning model (preview) | 131K | | `minimaxai/minimax-m2.7` | MiniMax M2.7 (preview) | 131K | +| `meta-models/muse-spark-open` | Meta Muse Spark (preview) | 131K | + +Groq shut down `groq/compound*`, the Llama 3.x models (on free and developer +tiers) and `qwen/qwen3.6-27b` in 2026. **Recommended for NetLogo**: `openai/gpt-oss-20b` (fast, cheap, generous free tier) @@ -422,6 +457,71 @@ Groq's model lineup changes frequently — check the [deprecations page](https://console.groq.com/docs/deprecations) before relying on a specific model in a long-lived model file. +## TypeSafe (Jev) Configuration + +Jev, TypeSafe's System One model, does not write text. It answers a typed question +about a prompt (pick an option, rate on levels, or yes/no) with a calibrated +probability for every possible answer. Use it with `llm:decide`, `llm:decide-all` +and `llm:choose`. The free-text primitives (`llm:chat` and friends) raise +`typesafe (Jev) returns decisions, not text; use llm:decide, llm:decide-all or llm:choose`. + +### API Setup + +1. **Get API Key**: Visit [console.typesafe.ai](https://console.typesafe.ai/) +2. **Rate Limits**: View your organization's limits at [console.typesafe.ai/rate-limits](https://console.typesafe.ai/rate-limits) +3. **Docs**: [docs.typesafe.ai](https://docs.typesafe.ai) + +### Configuration Parameters + +```ini +# Required Parameters +provider=typesafe +typesafe_api_key=your-typesafe-key-here +model=jev-latest + +# Optional Parameters +typesafe_base_url=https://api.typesafe.ai +timeout_seconds=30 +``` + +`temperature`, `max_tokens` and the thinking settings do not apply to Jev and are ignored. + +### Available Models + +| Model | Description | Price | +|-------|-------------|-------| +| `jev-latest` | Alias for the newest stable release (the default) | $0.042 per million input tokens; output free | +| `jev-preview` | Alias for the newest release, stable or not | same | +| `jev-1.13.0` | Pinned version | same | +| `jev-1.14.0` | Pinned version | same | + +An alias can move to a new version without notice; pin a version if you calibrated +thresholds against one. `llm:usage` reports `cost` for TypeSafe calls, computed from the +input tokens the API reports at the published price. + +`demos/el-farol-decisions` is a complete Jev model: 150 agents, three `llm:decide-all` +batches per tick, and monitors for cost and latency per tick. + +### How the primitives map to the API + +Each call sends `POST /v1/systemone` with your prompt as `state` and one question: + +| Spec | TypeSafe question | `answer` / `probability` | +|------|-------------------|--------------------------| +| `["choice" [options]]` | `choice` (1-255 options) | top option / its probability | +| `["score" [levels]]` | `score` (2-10 levels, 0-based) | expected level / confidence | +| `["yes-no"]` | `noul` | `true`/`false` / probability of that answer | + +HTTP 429 and 529 (overloaded) are retried with backoff like every other provider; a +401 reports an invalid key and names `typesafe_api_key`. + +```netlogo +llm:set-provider "typesafe" +llm:set-api-key "your-typesafe-key" +show llm:decide "Last week 70 of 100 went to the bar. Go tonight?" ["yes-no"] +; => [false 0.71 [[true 0.29] [false 0.71]]] +``` + ## Ollama (Local) Configuration ### Setup Requirements @@ -582,12 +682,12 @@ Use separate config files for easy switching: # config-openai.txt provider=openai api_key=sk-your-openai-key -model=gpt-4o-mini +model=gpt-6-luna # config-claude.txt provider=anthropic api_key=sk-ant-your-claude-key -model=claude-3-haiku-20240307 +model=claude-haiku-5-5 # config-local.txt provider=ollama @@ -626,8 +726,8 @@ end | Provider | Input Cost/1K | Output Cost/1K | 100 Chat Turns | |----------|---------------|----------------|-----------------| -| OpenAI (gpt-4o-mini) | $0.00015 | $0.0006 | ~$0.08 | -| Claude (Haiku) | $0.00025 | $0.00125 | ~$0.15 | +| OpenAI (gpt-6-luna) | $0.0001 | $0.0005 | ~$0.06 | +| Claude (Haiku 5.5) | $0.0001 | $0.0005 | ~$0.06 | | Gemini (Flash) | Free tier | Free tier | Free | | Ollama | Free | Free | Free | @@ -636,7 +736,7 @@ end For **minimal costs**: ```ini provider=gemini -model=gemini-1.5-flash # Free tier +model=gemini-3.5-flash-lite # Free tier max_tokens=512 # Shorter responses temperature=0.3 # More focused ``` @@ -665,7 +765,7 @@ max_tokens=1024 # Reasonable length **"Model not found"** - Check model name spelling - Verify model access for your tier -- Use `gpt-4o-mini` as fallback +- Use `gpt-6-luna` (the default) as fallback ### Claude Issues diff --git a/docs/README.md b/docs/README.md index ca79e79..458060d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,15 +1,15 @@ # NetLogo Multi-LLM Extension -Unified LLM capabilities for NetLogo with multi-provider support — OpenAI, Anthropic/Claude, Google/Gemini, local Ollama, OpenRouter (200+ models via one key), and Together AI (fast open-source inference) — plus per‑agent memory, async requests, reasoning/thinking models, and simple configuration. +Unified LLM capabilities for NetLogo with multi-provider support — OpenAI, Anthropic/Claude, Google/Gemini, local Ollama, OpenRouter (hundreds of models via one key), Together AI, and Groq — plus TypeSafe's Jev decision model for fast, cheap, calibrated agent decisions. Per‑agent memory, async and batched requests, structured outputs, reasoning/thinking models, token and cost tracking, and simple configuration. ## Quick Links - Usage Guide: `docs/USAGE.md` - Configuration Guide: `docs/CONFIGURATION.md` - API Reference: `docs/API-REFERENCE.md` - Provider Guide: `docs/PROVIDER-GUIDE.md` +- Setup Guide: `docs/SETUP.md` - Testing Guide: `docs/TESTING.md` -- Demos: `demos/` -- Full Reference Doc: `NetLogo-LLM-Extension-Documentation.md` +- Demos: `demos/` (flagship: `demos/el-farol-decisions/`) ## Quick Start 1) Add the extension in your model: @@ -20,33 +20,70 @@ extensions [ llm ] ``` llm:load-config "config.txt" ``` -See `docs/CONFIGURATION.md` for ready-to-copy examples (OpenAI, Anthropic, Gemini, Ollama, OpenRouter, Together AI). For inline setup: +See `docs/CONFIGURATION.md` for ready-to-copy examples for every provider. For inline setup: ``` llm:set-provider "openai" llm:set-api-key "sk-REPLACE_ME" -llm:set-model "gpt-4o-mini" +llm:set-model "gpt-6-luna" ``` -3) Chat: +3) Chat, or decide: ``` show llm:chat "Say hello in five words." +show llm:decide "Go out tonight?" ["yes-no"] ;; with provider "typesafe" ``` ## Core Features - Multi-provider with a single set of primitives. - Per-agent conversation history and `llm:clear-history`. -- Synchronous `llm:chat` and asynchronous `llm:chat-async` (+ `runresult`). +- Synchronous `llm:chat`, asynchronous `llm:chat-async` (+ `runresult`), and batched `llm:chat-all`. - Prompt templates with `llm:chat-with-template` (YAML + variables). -- Constrained decisions with `llm:choose`. +- Constrained decisions with `llm:choose`; structured output with `llm:chat-with-schema`, `llm:chat-json` and `llm:get`. +- Typed decisions with probabilities: `llm:decide` and `llm:decide-all` (choice, score, yes-no) on decision providers such as TypeSafe (Jev). +- Named profiles so different agents use different models (`llm:load-profile`, `llm:use-profile`). +- Request throttling, retries on rate limits, server errors and timeouts, and token/cost/latency tracking (`llm:usage`, `llm:usage-total`). +- Model lists and defaults come from YAML (`models.yaml`, plus an optional `~/.netlogo-llm/models.yaml`), so stale model IDs need no rebuild. ## Install -- Per-model or global installation instructions and build steps are in `docs/USAGE.md`. +- From NetLogo: **Tools → Extensions → LLM** (Extension Manager). +- From source: see `BUILD.md` and `INSTALL.md` (JDK 21+, NetLogo 7.0.4). - The `demos/` folder contains working models you can open immediately. ## Notes -- Keep API keys out of `.nlogo` files; prefer `llm:load-config`. +- Keep API keys out of `.nlogox` files; prefer `llm:load-config` with a gitignored `config.txt`. - For local, no-API usage, see the Ollama quick start in `docs/CONFIGURATION.md`. -If you need deeper details (architecture, patterns, troubleshooting), start with `docs/USAGE.md` and the full reference document. +## For maintainers + +**Layout.** `src/main/LLMExtension.scala` registers every `llm:` primitive and owns +per-agent history and usage (both guarded by a run-generation counter so calls launched +before `clear-all` cannot write into a new run). `src/main/providers/` holds one class per +provider on top of `BaseHttpProvider` (HTTP, retries, throttle, friendly errors) and +`OpenAICompatibleProvider`; each provider is declared once in `ProviderRegistrations.scala`. +`src/main/config/` loads config files, profiles and the model catalog. `src/main/models/` +holds the request/response, structured-output and decision types. + +**Adding a chat provider.** Add a descriptor in `ProviderRegistrations.scala`, a provider +class (usually a small `OpenAICompatibleProvider` subclass, see `GroqProvider.scala`), and a +block in `src/main/resources/config/models.yaml` with `default:` and `models:`. + +**Adding a decision backend.** Implement `DecisionProvider` (`src/main/providers/DecisionProvider.scala`) +next to `BaseHttpProvider`, as `TypeSafeProvider.scala` does. `llm:decide`, `llm:decide-all` +and `llm:choose` pick it up without other changes. + +**Model catalog.** Provider model IDs go stale on the provider's schedule. Check each +provider's deprecation page, edit `models.yaml`, and run `sbt test` +(`ModelCatalogSpec` checks every default is listed). + +**Tests.** `sbt test` is deterministic (no network): `tests.txt` runs NetLogo code through +`DeterministicTestProvider`, and the Scala specs stub HTTP. Live checks live in +`demos/e2e-tests/e2e-tests.nlogox` (T1–T12), run headless as described in `docs/TESTING.md`. + +**Releasing.** See `RELEASE.md`. + +**Decided not to build** (closed issues, with reasons on each): response caching (#54), +record/replay (#55), BehaviorSpace logging (#56, already possible with `llm:usage`), +memory consolidation (#47), observer-level channels and shared blackboards (#57, #58), +streaming (#59), and API keys from environment variables (#50). ## License diff --git a/docs/SETUP.md b/docs/SETUP.md index 488ac68..754c240 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -31,17 +31,19 @@ This extension requires **NetLogo 7.0.0 or later**. Check your version in NetLog 2. **Create config.txt**: ``` provider=openai -model=gpt-4o-mini +model=gpt-6-luna api_key=sk-your-openai-key-here temperature=0.7 max_tokens=1000 ``` **Available Models**: -- `gpt-4o-mini` - Fast, cost-effective (recommended) -- `gpt-4o` - Most capable GPT-4 variant -- `gpt-4-turbo` - Faster GPT-4 with large context -- `gpt-3.5-turbo` - Legacy model, still capable +- `gpt-6-luna` - Fast, cost-effective reasoning model (recommended) +- `gpt-6.1-sol` - Mid-tier GPT-6 +- `gpt-6-astra` - Most capable GPT-6 +- `gpt-4o-mini` - Deprecated, served until 2027-01-14 + +GPT-5.x and GPT-6.x are reasoning models: `temperature` is not sent to them. ### Anthropic (Claude Models) @@ -49,17 +51,19 @@ max_tokens=1000 2. **Create config.txt**: ``` provider=anthropic -model=claude-3-haiku-20240307 +model=claude-haiku-5-5 api_key=sk-ant-your-anthropic-key-here temperature=0.5 max_tokens=4000 ``` **Available Models**: -- `claude-3-haiku-20240307` - Fast, cost-effective (recommended) -- `claude-3-sonnet-20240229` - Balanced performance -- `claude-3-opus-20240229` - Most capable Claude -- `claude-3-5-sonnet-20241022` - Latest improved Sonnet +- `claude-haiku-5-5` - Fast, cost-effective (recommended) +- `claude-sonnet-5-5` - Balanced performance +- `claude-opus-5-5` - Most capable Opus +- `claude-fable-5-1` - Most capable Claude + +Claude 4.7 and later reject a non-default `temperature`, so it is not sent to them. ### Google Gemini @@ -67,16 +71,16 @@ max_tokens=4000 2. **Create config.txt**: ``` provider=gemini -model=gemini-1.5-flash +model=gemini-3.5-flash-lite api_key=your-google-ai-api-key-here temperature=0.8 max_tokens=2048 ``` **Available Models**: -- `gemini-1.5-flash` - Fast, free tier available (recommended) -- `gemini-1.5-pro` - Most capable Gemini -- `gemini-pro` - Original Gemini Pro +- `gemini-3.5-flash-lite` - Fast, free tier available (recommended) +- `gemini-3.8-flash` - Stronger Flash tier +- `gemini-3.1-pro-preview` - Most capable Gemini (preview) ### Ollama (Local Models) @@ -109,47 +113,46 @@ OpenRouter routes requests to many model vendors (OpenAI, Anthropic, Google, Met 2. **Create config.txt**: ``` provider=openrouter -model=openai/gpt-4o-mini +model=openai/gpt-oss-20b openrouter_api_key=sk-or-your-key-here temperature=0.7 max_tokens=1000 ``` **Available Models** (vendor-prefixed): -- `openai/gpt-4o-mini` - Fast, cost-effective (recommended) -- `openai/gpt-4o` - GPT-4o through OpenRouter -- `anthropic/claude-3.5-sonnet` - Claude 3.5 Sonnet -- `anthropic/claude-3.5-haiku` - Fast Claude -- `google/gemini-2.0-flash-exp` - Latest Gemini +- `openai/gpt-oss-20b` - Fast, cost-effective (recommended) +- `openai/gpt-6-luna` - GPT-6 Luna through OpenRouter +- `anthropic/claude-haiku-5.5` - Fast Claude +- `google/gemini-3.5-flash-lite` - Fast Gemini - `meta-llama/llama-3.3-70b-instruct` - Llama 3.3 70B -- `deepseek/deepseek-r1` - Reasoning model +- `deepseek/deepseek-v4.1-flash` - DeepSeek V4.1 Flash Browse the full catalog: [openrouter.ai/models](https://openrouter.ai/models) ### Together AI (open-source models) -Together AI provides fast inference for open-source models (Llama, DeepSeek, Qwen, Mistral, Gemma) via an OpenAI-compatible API. +Together AI provides fast inference for open-source models (Qwen, DeepSeek, GLM, gpt-oss, Kimi) via an OpenAI-compatible API. 1. **Get API Key**: Visit [api.together.ai/settings/api-keys](https://api.together.ai/settings/api-keys) 2. **Create config.txt**: ``` provider=together -model=meta-llama/Llama-3.3-70B-Instruct-Turbo +model=Qwen/Qwen3.8-Flash together_api_key=your-together-key-here temperature=0.7 max_tokens=1000 ``` -**Available Models** (vendor-prefixed): -- `meta-llama/Llama-3.3-70B-Instruct-Turbo` - Fast Llama 3.3 (recommended) -- `meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo` - Llama 3.1 70B -- `deepseek-ai/DeepSeek-R1` - Reasoning model with `` tag output -- `Qwen/Qwen2.5-72B-Instruct-Turbo` - Qwen 2.5 72B -- `mistralai/Mixtral-8x22B-Instruct-v0.1` - Mistral mixture-of-experts +**Available Models** (vendor-prefixed, case-sensitive): +- `Qwen/Qwen3.8-Flash` - Fast Qwen 3.8 (recommended) +- `Qwen/Qwen3.5-9B` - Small, cheap Qwen +- `openai/gpt-oss-120b` - Open-weight reasoning model +- `deepseek-ai/DeepSeek-V4.1-Flash` - DeepSeek V4.1 Flash +- `zai-org/GLM-5.3-Flash` - GLM 5.3 Flash Browse the full catalog: [api.together.ai/models](https://api.together.ai/models) -**Reasoning model note:** DeepSeek-R1 emits its thinking inside `...` tags (sometimes filling the entire response). Use `llm:chat-with-thinking` to get the answer and reasoning split out, and bump `max_tokens` to 2000+ so the model has room to finish its answer after thinking. +**Reasoning model note:** reasoning models may emit their thinking inside `...` tags (sometimes filling the entire response). Use `llm:chat-with-thinking` to get the answer and reasoning split out, and bump `max_tokens` to 2000+ so the model has room to finish its answer after thinking. ### Groq (fast free-tier inference) @@ -168,8 +171,8 @@ max_tokens=1000 **Available Models**: - `openai/gpt-oss-20b` - Fast open-weight reasoning model (recommended) - `openai/gpt-oss-120b` - Larger open-weight reasoning model -- `groq/compound-mini` - Agentic system with built-in tool use -- `qwen/qwen3.6-27b` - Qwen 3.6 reasoning model (preview) +- `qwen/qwen3.8-27b` - Qwen 3.8 reasoning model (preview) +- `minimaxai/minimax-m2.7` - MiniMax M2.7 (preview) Browse the full catalog: [console.groq.com/docs/models](https://console.groq.com/docs/models) @@ -177,20 +180,39 @@ Browse the full catalog: [console.groq.com/docs/models](https://console.groq.com **Model lifecycle note:** Groq retires models often. If a model stops working, check [console.groq.com/docs/deprecations](https://console.groq.com/docs/deprecations). +### TypeSafe (Jev) (typed decisions with probabilities) + +TypeSafe's Jev answers a typed question about a prompt (a choice, a score or a yes/no) with a probability for every possible answer, instead of writing text. It works only with `llm:decide`, `llm:decide-all` and `llm:choose`; `llm:chat` and the other text primitives raise an error on it. + +1. **Get API Key**: Visit [typesafe.ai](https://typesafe.ai) (keys are created in the console at [console.typesafe.ai](https://console.typesafe.ai/)) +2. **Create config.txt**: +``` +provider=typesafe +model=jev-latest +typesafe_api_key=your-typesafe-key-here +``` + +**Available Models**: +- `jev-latest` - Stable release (recommended, the default) +- `jev-preview` - Newest release +- `jev-1.14.0` / `jev-1.13.0` - Pinned versions + +`temperature` and `max_tokens` do not apply. Docs: [docs.typesafe.ai](https://docs.typesafe.ai) + ## Configuration Parameters ### Core Settings | Parameter | Description | Required | Default | |-----------|-------------|----------|---------| -| `provider` | LLM provider (`openai`, `anthropic`, `gemini`, `ollama`, `openrouter`, `together`, `groq`) | Yes | - | +| `provider` | LLM provider (`openai`, `anthropic`, `gemini`, `ollama`, `openrouter`, `together`, `groq`, `typesafe`) | Yes | - | | `model` | Model identifier | Yes | Provider-specific | | `api_key` | API authentication key | Yes* | - | | `temperature` | Response randomness (0.0-1.0) | No | 0.7 | | `max_tokens` | Maximum response length | No | 1000 | | `timeout_seconds` | Request timeout | No | 30 | -| `retry_max_retries` | Retry attempts after a rate-limit (HTTP 429) response | No | 6 | -| `retry_max_elapsed_seconds` | Total time allowed waiting out rate limits | No | 65 | +| `retry_max_retries` | Retry attempts after a rate limit (HTTP 429), server error (500/502/503/504), Anthropic overload (529) or connection timeout | No | 6 | +| `retry_max_elapsed_seconds` | Total time allowed waiting out rate limits and other transient errors | No | 65 | | `max_concurrent_requests` | Maximum requests in flight at once, per provider | No | 0 (off) | | `min_request_interval_ms` | Minimum milliseconds between request starts | No | 0 (off) | @@ -310,6 +332,10 @@ call the LLM every N ticks) rather than only raising `retry_max_elapsed_seconds` - `max_tokens`: 1000 - API key config field: `groq_api_key` +**TypeSafe (Jev)**: +- `base_url`: `https://api.typesafe.ai` +- API key config field: `typesafe_api_key` + ## Testing Your Setup 1. **Create Test Model**: diff --git a/docs/TESTING.md b/docs/TESTING.md index 1023011..06b26d3 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -102,10 +102,10 @@ The suite runs the same 13 test procedures regardless of provider, plus two prov - `test-sync-chat`, `test-async-chat`, `test-choose`, `test-history` — core chat flow - `test-thinking-config`, `test-chat-with-thinking` — reasoning primitives and `[answer thinking]` return shape - `test-openrouter-vendor-prefix` — vendor-prefixed model names (skips unless `provider=openrouter`) -- `test-together-thinking` — DeepSeek-R1 `` tag extraction (skips unless `provider=together` AND model contains "DeepSeek") +- `test-together-thinking` — DeepSeek `` tag extraction (skips unless `provider=together` AND model contains "DeepSeek") - `test-reasoning-marker` — `[reasoning]` marker visible in `llm:list-models` -To exercise everything, run the suite twice — once with `provider=openrouter`, once with `provider=together` (use `model=deepseek-ai/DeepSeek-R1` and `max_tokens=2000+` for the Together reasoning test). +To exercise everything, run the suite twice — once with `provider=openrouter`, once with `provider=together` (use `model=deepseek-ai/DeepSeek-V4.1-Flash` and `max_tokens=2000+` for the Together reasoning test). Suggested usage: - run automated tests first (`sbt test`) @@ -149,3 +149,30 @@ Typical setup later: - separate workflow (manual trigger and/or nightly schedule) - secrets like `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GEMINI_API_KEY`, `OPENROUTER_API_KEY`, `TOGETHER_API_KEY` - not required for normal PR merges (to avoid flaky/costly gating) + +## Live end-to-end harness + +`demos/e2e-tests/e2e-tests.nlogox` makes real API calls through the extension: T1–T11 +cover chat providers (chat, thinking, compile-error, history, structured output, choose, +throttling, usage, profiles, chat-all) and T12 covers decision providers (TypeSafe Jev: +`llm:decide`, `llm:decide-all`, `llm:choose`, cost, and the refusal of free-text chat). + +Put a key config beside the model (`config-groq.txt`, `config-typesafe.txt`, ...; see the +`.example` files; these names are gitignored), pick the provider in the chooser, and press +**run all tests**. Results go to the output area and `e2e-results.txt`. + +To run it headless (no GUI), from the repo root with JDK 21+: + +```bash +sbt package "export Runtime/dependencyClasspath" | tail -1 > target/nl-cp.txt +mkdir -p extensions/llm && cp llm.jar *.jar extensions/llm/ # the extension and its jars +cd demos/e2e-tests +# set the provider-under-test chooser's current="N" in a copy of the model, then: +java -Djava.awt.headless=true -Dnetlogo.extensions.dir=$PWD/../../extensions \ + -cp "$(cat ../../target/nl-cp.txt)" org.nlogo.headless.Main \ + --model e2e-tests.nlogox --experiment e2e --table - +cat e2e-results.txt +``` + +NetLogo 7 headless only runs experiments embedded in the `.nlogox`; an external +`--setup-file` is silently ignored. diff --git a/docs/USAGE.md b/docs/USAGE.md index 48d7511..b2dd873 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -27,7 +27,7 @@ Use inline commands or a config file. A config file keeps secrets out of your mo extensions [ llm ] llm:set-provider "openai" llm:set-api-key "sk-REPLACE_ME" -llm:set-model "gpt-4o-mini" +llm:set-model "gpt-6-luna" ``` - Config file (recommended): - Create a `config.txt` using the key=value format and load it: diff --git a/project/build.properties b/project/build.properties index e88a0d8..ee4c672 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.10.6 +sbt.version=1.10.1 diff --git a/project/plugins.sbt b/project/plugins.sbt index 1360118..dd8a9f9 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -2,6 +2,6 @@ resolvers ++= Seq( "netlogo-extension-plugin" at "https://dl.cloudsmith.io/public/netlogo/netlogo-extension-plugin/maven/" ) -addSbtPlugin("org.nlogo" % "netlogo-extension-plugin" % "6.1.0") +addSbtPlugin("org.nlogo" % "netlogo-extension-plugin" % "7.0.4") addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.10.4") addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.2.0") diff --git a/scripts/check-zip.sh b/scripts/check-zip.sh new file mode 100755 index 0000000..6fd3068 --- /dev/null +++ b/scripts/check-zip.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# ABOUTME: Asserts the release zip built by `sbt packageZip` holds the jar, docs, and demo files. +# ABOUTME: Also checks demos/config.txt in the zip is the tracked placeholder, not a local config. +set -euo pipefail + +# Usage: scripts/check-zip.sh [path/to/llm-.zip] +# With no argument, uses the single llm-*.zip in the repo root. + +root="$(cd "$(dirname "$0")/.." && pwd)" + +if [ $# -ge 1 ]; then + zip="$1" +else + shopt -s nullglob + zips=("$root"/llm-*.zip) + shopt -u nullglob + if [ ${#zips[@]} -ne 1 ]; then + echo "check-zip: expected exactly one llm-*.zip in $root, found ${#zips[@]}" >&2 + exit 1 + fi + zip="${zips[0]}" +fi + +if [ ! -f "$zip" ]; then + echo "check-zip: zip not found: $zip" >&2 + exit 1 +fi + +listing="$(unzip -Z1 "$zip")" + +# Entries may sit under a top-level folder (e.g. llm/), so match by suffix. +required=( + llm.jar + docs/README.md + demos/config-reference.txt + demos/config.txt + demos/color-sharing.nlogox + demos/social-deduction-game.nlogox + demos/topology-tournament.nlogox + demos/provider-sensitivity.nlogox + demos/el-farol-decisions.nlogox +) + +status=0 +for entry in "${required[@]}"; do + if ! grep -Eq "(^|/)${entry//./\\.}\$" <<<"$listing"; then + echo "check-zip: missing $entry in $zip" >&2 + status=1 + fi +done + +if [ $status -ne 0 ]; then + unzip -l "$zip" >&2 + exit $status +fi + +config_entry="$(grep -E '(^|/)demos/config\.txt$' <<<"$listing" | head -1)" +if ! unzip -p "$zip" "$config_entry" | cmp -s - "$root/demos/config-placeholder.txt"; then + echo "check-zip: $config_entry in $zip differs from demos/config-placeholder.txt" >&2 + exit 1 +fi + +unzip -l "$zip" +echo "check-zip: OK ($zip)" diff --git a/src/main/LLMExtension.scala b/src/main/LLMExtension.scala index 445b926..766602f 100644 --- a/src/main/LLMExtension.scala +++ b/src/main/LLMExtension.scala @@ -3,8 +3,8 @@ package org.nlogo.extensions.llm import org.nlogo.api._ import org.nlogo.core.{LogoList, Syntax} import org.nlogo.extensions.llm.config.{ConfigLoader, ConfigStore, ProfileStore} -import org.nlogo.extensions.llm.providers.{LLMProvider, ProviderDescriptor, ProviderFactory, ProviderRegistry, ProviderRegistrations, ModelRegistry, OllamaProvider, ReadinessCheck, RetryPolicy} -import org.nlogo.extensions.llm.models.{ChatMessage, ChatResponse, EnumFormat, JsonObjectFormat, JsonSchemaFormat, ResponseFormat, Usage} +import org.nlogo.extensions.llm.providers.{DecisionProvider, LLMProvider, ProviderDescriptor, ProviderFactory, ProviderRegistry, ProviderRegistrations, ModelRegistry, OllamaProvider, ReadinessCheck, RetryPolicy} +import org.nlogo.extensions.llm.models.{ChatMessage, ChatResponse, DecisionAnswer, DecisionQuestion, DecisionResult, EnumFormat, JsonObjectFormat, JsonSchemaFormat, ResponseFormat, Usage} import org.nlogo.extensions.llm.utils.JsonToNetLogo import scala.collection.mutable.{ArrayBuffer, WeakHashMap} import scala.concurrent.{Await, ExecutionContext, Future} @@ -30,6 +30,38 @@ object LLMExtension { providerFactoryOverride = None } + /** + * Probe an Ollama server for reachability (synchronous, short timeout). + * Takes the provider so tests can hand in one backed by a stub. + */ + private[llm] def probeOllama(config: ConfigStore, provider: OllamaProvider): Boolean = { + try { + val baseUrl = ConfigStore.resolveBaseUrl(config, ConfigStore.OLLAMA_BASE_URL, ConfigStore.DEFAULT_OLLAMA_BASE_URL) + provider.setConfig(ConfigStore.OLLAMA_BASE_URL, baseUrl) + + val checkFuture = provider.checkServerConnection() + Await.result(checkFuture, 1.second) + } catch { + case ex: Exception => + System.err.println(s"WARNING: Ollama reachability check failed: ${ex.getMessage}") + false + } + } + + /** + * The error for a reply that is empty because the model stopped at its token + * limit (finish_reason "length") before writing any answer, or None. + */ + private[llm] def emptyReplyMessage(response: ChatResponse): Option[String] = + response.choices.headOption.collect { + case c if c.finishReason == "length" && c.message.content.trim.isEmpty && c.refusal.isEmpty && + response.thinking.forall(_.trim.isEmpty) => + val used = response.usage.map(u => s" (${u.outputTokens} output tokens)").getOrElse("") + s"${response.model} returned an empty reply: it used its whole token budget$used before " + + "answering, and reasoning tokens count against that budget. Set max_tokens in your config " + + "above that, e.g. max_tokens=8000." + } + private[llm] def createProvider(configStore: ConfigStore)( implicit ec: ExecutionContext ): Try[LLMProvider] = { @@ -40,6 +72,19 @@ object LLMExtension { } } +/** + * Raised when the model declines to answer. Primitives let it through + * unwrapped so the modeler sees exactly "model refused: ". + */ +final class ModelRefusedException(val refusal: String) + extends ExtensionException(s"model refused: $refusal") + +/** Raised by a free-text primitive on a provider that only returns decisions. */ +final class TextUnsupportedException(message: String) extends ExtensionException(message) + +/** Raised by llm:decide / llm:decide-all on a provider without a decision API. */ +final class NoDecisionApiException(message: String) extends ExtensionException(message) + /** * Main extension class for NetLogo Multi-LLM Extension * @@ -75,6 +120,12 @@ class LLMExtension extends DefaultClassManager { private var runUsage: UsageTotals = UsageTotals.empty private val usageLock = new Object + // Run generation, bumped by clear-all. Each primitive captures it at launch + // and its history/usage writes are dropped if a clear-all has happened + // since, so a reply still in flight cannot leak into the next run. Volatile + // so the check inside historyLock/usageLock sees clear-all's bump. + @volatile private var generation: Long = 0L + // Execution context for async operations implicit private val ec: ExecutionContext = ExecutionContext.global @@ -93,6 +144,7 @@ class LLMExtension extends DefaultClassManager { // how long an already-pending call may take. Await.result(future, timeout) } catch { + case e: ModelRefusedException => throw e case e: Exception => throw new ExtensionException(s"Async LLM operation failed: ${e.getMessage}") } @@ -121,6 +173,10 @@ class LLMExtension extends DefaultClassManager { manager.addPrimitive("chat-with-thinking", ChatWithThinkingReporter) manager.addPrimitive("choose", ChooseReporter) + // Decision primitives (providers with a DecisionProvider API) + manager.addPrimitive("decide", DecideReporter) + manager.addPrimitive("decide-all", DecideAllReporter) + // Structured output primitives manager.addPrimitive("chat-with-schema", ChatWithSchemaReporter) manager.addPrimitive("chat-json", ChatJsonReporter) @@ -163,6 +219,8 @@ class LLMExtension extends DefaultClassManager { * Called when NetLogo calls clear-all or when the model is reset */ override def clearAll(): Unit = { + // Bump before clearing so any write that lands after the clear is stale. + historyLock.synchronized { usageLock.synchronized { generation += 1 } } historyLock.synchronized { messageHistory.clear() } usageLock.synchronized { agentUsage.clear() @@ -235,13 +293,16 @@ class LLMExtension extends DefaultClassManager { /** * Append a user/assistant exchange to an agent's history atomically. * Re-resolves the buffer by agent so a commit racing set-history lands in - * the current buffer, not a detached stale one. + * the current buffer, not a detached stale one. Dropped if clear-all ran + * since the call launched in generation `gen`. */ - private def commitExchange(agent: Agent, user: ChatMessage, assistant: ChatMessage): Unit = + private def commitExchange(agent: Agent, gen: Long, user: ChatMessage, assistant: ChatMessage): Unit = historyLock.synchronized { - val h = getAgentHistory(agent) - h += user - h += assistant + if (gen == generation) { + val h = getAgentHistory(agent) + h += user + h += assistant + } } /** @@ -250,16 +311,120 @@ class LLMExtension extends DefaultClassManager { * Called as soon as a response arrives, before the caller decides whether * the reply is acceptable: tokens were billed either way, so a reply the * extension then rejects (schema parse failure, unmatched choice) still - * counts. A call that never produced a response records nothing. + * counts. A call that never produced a response records nothing, and a + * call launched before the latest clear-all (generation `gen`) is dropped. */ - private def recordUsage(agent: Agent, response: ChatResponse): Unit = { - val delta = UsageTotals(response.usage.getOrElse(Usage.empty), 1L) + private def recordUsage(agent: Agent, gen: Long, response: ChatResponse): Unit = + recordUsage(agent, gen, response.usage.getOrElse(Usage.empty)) + + /** As above, for calls that report usage without a ChatResponse (decisions). */ + private def recordUsage(agent: Agent, gen: Long, usage: Usage): Unit = { + val delta = UsageTotals(usage, 1L) usageLock.synchronized { - agentUsage.update(agent, agentUsage.getOrElse(agent, UsageTotals.empty).plus(delta)) - runUsage = runUsage.plus(delta) + if (gen == generation) { + agentUsage.update(agent, agentUsage.getOrElse(agent, UsageTotals.empty).plus(delta)) + runUsage = runUsage.plus(delta) + } } } + /** + * Fail a call the model declined to answer. + * + * Called after recordUsage, so the billed tokens still count, and before any + * commitExchange, so the refused exchange never enters history. + */ + private def rejectRefusal(response: ChatResponse): Unit = + response.refusal.foreach(text => throw new ModelRefusedException(text)) + + /** + * Fail a call whose reply came back empty because the model hit its token + * limit (a reasoning model can spend the whole budget thinking). Called + * after recordUsage like rejectRefusal, so the billed tokens still count. + */ + private def rejectEmptyReply(response: ChatResponse): Unit = + LLMExtension.emptyReplyMessage(response).foreach(m => throw new ExtensionException(m)) + + /** + * Fail a free-text call on a provider that only returns decisions, before + * anything is sent. Raised unwrapped so the modeler sees which primitives + * to use instead. + */ + private def requireText(provider: LLMProvider): Unit = + provider match { + case d: DecisionProvider => d.textUnsupportedMessage.foreach(m => throw new TextUnsupportedException(m)) + case _ => () + } + + /** The agent's provider as a DecisionProvider, or a clear error naming the configured provider. */ + private def decisionProviderFor(agent: Agent): DecisionProvider = + ensureProvider(agent) match { + case d: DecisionProvider => d + case _ => + val name = effectiveConfig(agent).getOrElse(ConfigStore.PROVIDER, ConfigStore.DEFAULT_PROVIDER) + throw new NoDecisionApiException( + s"""provider "$name" has no decision API; use llm:choose, or llm:set-provider "typesafe"""") + } + + /** + * Read a llm:decide spec: ["choice" [options]], ["score" [levels]] or + * ["yes-no"]. Checked before any request so a bad spec costs nothing. + */ + private def parseDecisionSpec(spec: LogoList, primName: String): DecisionQuestion = { + def fail(msg: String): Nothing = throw new ExtensionException(s"$primName: $msg") + spec.toVector match { + case (kind: String) +: rest => + def strings(items: LogoList, what: String): Seq[String] = + items.toVector.map { + case s: String if s.nonEmpty => s + case _ => fail(s""""$kind" $what must be non-empty strings""") + } + kind match { + case "choice" => + rest match { + case Vector(opts: LogoList) if opts.size >= 1 && opts.size <= DecisionQuestion.MaxChoiceOptions => + val options = strings(opts, "options") + options.diff(options.distinct).headOption.foreach { dup => + fail(s""""choice" options must be distinct, but "$dup" appears more than once""") + } + DecisionQuestion.Choice(options) + case _ => fail(s""""choice" needs a list of 1 to ${DecisionQuestion.MaxChoiceOptions} options""") + } + case "score" => + rest match { + case Vector(levels: LogoList) + if levels.size >= DecisionQuestion.MinScoreLevels && levels.size <= DecisionQuestion.MaxScoreLevels => + DecisionQuestion.Score(strings(levels, "levels")) + case _ => + fail(s""""score" needs a list of ${DecisionQuestion.MinScoreLevels} to ${DecisionQuestion.MaxScoreLevels} levels""") + } + case "yes-no" => + if (rest.isEmpty) DecisionQuestion.YesNo else fail(""""yes-no" takes no list""") + case other => + fail(s"""unknown decision type "$other"; use "choice", "score" or "yes-no"""") + } + case _ => + fail("""spec must be ["choice" [options]], ["score" [levels]] or ["yes-no"]""") + } + } + + /** + * A decision as [answer probability distribution]. A yes-no answer and its + * distribution labels are NetLogo booleans; a score's answer is a number. + */ + private def decisionToLogo(result: DecisionResult): LogoList = { + val (answer, yesNo): (AnyRef, scala.Boolean) = result.answer match { + case DecisionAnswer.Picked(option) => (option, false) + case DecisionAnswer.Expected(value) => (Double.box(value), false) + case DecisionAnswer.Verdict(yes) => (Boolean.box(yes), true) + } + val distribution = result.distribution.map { case (label, p) => + val l: AnyRef = if (yesNo) Boolean.box(label == "true") else label + LogoList(l, Double.box(p)) + } + LogoList(answer, Double.box(result.probability), LogoList.fromVector(distribution.toVector)) + } + private def usageFor(agent: Agent): UsageTotals = usageLock.synchronized { agentUsage.getOrElse(agent, UsageTotals.empty) } @@ -335,9 +500,7 @@ class LLMExtension extends DefaultClassManager { desc.readinessCheck match { case ReadinessCheck.ServerReachable => if (!isOllamaReachable(config)) { - val baseUrl = config.get(desc.baseUrlConfigKey) - .orElse(config.get(ConfigStore.BASE_URL)) - .getOrElse(desc.defaultBaseUrl) + val baseUrl = ConfigStore.resolveBaseUrl(config, desc.baseUrlConfigKey, desc.defaultBaseUrl) throw new ExtensionException( s"Config loaded but ${desc.displayName} not reachable at $baseUrl. Please start the server or change ${desc.baseUrlConfigKey} in config. For help: print llm:provider-help \"${desc.name}\"" ) @@ -355,22 +518,8 @@ class LLMExtension extends DefaultClassManager { */ private def isOllamaReachable: Boolean = isOllamaReachable(configStore) - private def isOllamaReachable(config: ConfigStore): Boolean = { - try { - val provider = new OllamaProvider() - val baseUrl = config.get(ConfigStore.OLLAMA_BASE_URL) - .orElse(config.get(ConfigStore.BASE_URL)) - .getOrElse(ConfigStore.DEFAULT_OLLAMA_BASE_URL) - provider.setConfig(ConfigStore.BASE_URL, baseUrl) - - val checkFuture = provider.checkServerConnection() - Await.result(checkFuture, 1.second) - } catch { - case ex: Exception => - System.err.println(s"WARNING: Ollama reachability check failed: ${ex.getMessage}") - false - } - } + private def isOllamaReachable(config: ConfigStore): Boolean = + LLMExtension.probeOllama(config, new OllamaProvider()) /** * Check if a provider is ready to use. @@ -463,25 +612,15 @@ class LLMExtension extends DefaultClassManager { ) } - // Apply provider defaults - val defaults = ProviderFactory.getDefaultConfig(providerName) - defaults.foreach { case (key, value) => - if (!configStore.contains(key) || key == ConfigStore.MODEL) { - configStore.set(key, value) - } - } - - // Set provider - configStore.set(ConfigStore.PROVIDER, providerName) + // Apply provider defaults and set provider + ProviderFactory.applyProviderSwitch(configStore, providerName) // Validate immediately using descriptor ProviderRegistry.get(providerName).foreach { desc => desc.readinessCheck match { case ReadinessCheck.ServerReachable => if (!isOllamaReachable) { - val baseUrl = configStore.get(desc.baseUrlConfigKey) - .orElse(configStore.get(ConfigStore.BASE_URL)) - .getOrElse(desc.defaultBaseUrl) + val baseUrl = ConfigStore.resolveBaseUrl(configStore, desc.baseUrlConfigKey, desc.defaultBaseUrl) throw new ExtensionException( s"${desc.displayName} not reachable at $baseUrl. Please start the server or change ${desc.baseUrlConfigKey}. For help: print llm:provider-help \"$providerName\"" ) @@ -558,13 +697,17 @@ class LLMExtension extends DefaultClassManager { // Snapshot old config so we can roll back on readiness-check failure. val previousConfig = configStore.toMap + val previousDefaults = configStore.defaultKeys configStore.loadFromMap(config) // Load model override file if available - modelDir.foreach { dir => - ModelRegistry.loadOverride(dir).foreach { message => - println(message) - } + modelDir match { + case Some(dir) => + ModelRegistry.loadOverride(dir).foreach { message => + println(message) + } + case None => + ModelRegistry.clearOverride() } // Readiness check uses the now-loaded config (needs apiKeyConfigKey lookup). @@ -573,7 +716,7 @@ class LLMExtension extends DefaultClassManager { checkReadiness(desc, providerName, configStore) } catch { case e: ExtensionException => - configStore.loadFromMap(previousConfig) + configStore.loadFromMap(previousConfig, previousDefaults) throw e } @@ -597,22 +740,28 @@ class LLMExtension extends DefaultClassManager { val agent = context.getAgent try { + val gen = generation val provider = ensureProvider(agent) + requireText(provider) val userMessage = ChatMessage.user(inputText) // Send chat request with user message included, but don't mutate history yet val responseFuture = provider.chatWithFullResponse(snapshotHistory(agent) :+ userMessage) val response = Await.result(responseFuture, awaitTimeoutFor(effectiveConfig(agent))) - recordUsage(agent, response) + recordUsage(agent, gen, response) + rejectRefusal(response) + rejectEmptyReply(response) val responseMessage = replyMessage(response) // Only commit both messages after success - commitExchange(agent, userMessage, responseMessage) + commitExchange(agent, gen, userMessage, responseMessage) responseMessage.content } catch { + case e: ModelRefusedException => throw e + case e: TextUnsupportedException => throw e case e: Exception => throw new ExtensionException(s"LLM chat failed: ${e.getMessage}") } @@ -630,16 +779,20 @@ class LLMExtension extends DefaultClassManager { val agent = context.getAgent try { + val gen = generation val provider = ensureProvider(agent) + requireText(provider) val userMessage = ChatMessage.user(inputText) // Snapshot on the NetLogo thread; commit the pair atomically on success // from the completion thread so overlapping async calls can't interleave. val responseFuture = provider.chatWithFullResponse(snapshotHistory(agent) :+ userMessage).map { response => - recordUsage(agent, response) + recordUsage(agent, gen, response) + rejectRefusal(response) + rejectEmptyReply(response) val responseMessage = replyMessage(response) - commitExchange(agent, userMessage, responseMessage) + commitExchange(agent, gen, userMessage, responseMessage) responseMessage.content } @@ -647,6 +800,7 @@ class LLMExtension extends DefaultClassManager { createAwaitableReporter(responseFuture, awaitTimeoutFor(effectiveConfig(agent))) } catch { + case e: TextUnsupportedException => throw e case e: Exception => throw new ExtensionException(s"Failed to start async LLM chat: ${e.getMessage}") } @@ -719,6 +873,10 @@ class LLMExtension extends DefaultClassManager { // "send all" into "queue all, at most N in flight". Agents killed by a // later agent's prompt reporter are dropped here rather than sent. val live = slots.filter(_.agent.id != -1) + // A decision-only provider cannot serve any of the batch: raise before + // sending. A provider that fails to initialise stays a per-agent failure. + live.foreach(slot => Try(ensureProvider(slot.agent)).foreach(requireText)) + val gen = generation live.foreach { slot => val userMessage = ChatMessage.user(slot.prompt) slot.future = @@ -731,7 +889,9 @@ class LLMExtension extends DefaultClassManager { } sent.onComplete(_ => lastReply.set(System.nanoTime)) sent.map { response => - recordUsage(slot.agent, response) + recordUsage(slot.agent, gen, response) + rejectRefusal(response) + rejectEmptyReply(response) val (value, assistant): (AnyRef, ChatMessage) = format match { case Some(_) => // Parse before committing, as llm:chat-with-schema does. @@ -747,7 +907,7 @@ class LLMExtension extends DefaultClassManager { if (slot.abandoned) { throw new java.util.concurrent.TimeoutException("reply arrived after the wait gave up") } - commitExchange(slot.agent, userMessage, assistant) + commitExchange(slot.agent, gen, userMessage, assistant) slot.result = Some(value) } value @@ -786,6 +946,7 @@ class LLMExtension extends DefaultClassManager { live.map { slot => settle(slot, awaitTimeoutFor(effectiveConfig(slot.agent))) match { case Success(v) => LogoList(slot.agent, v, java.lang.Boolean.TRUE) + case Failure(e: ModelRefusedException) => LogoList(slot.agent, e.getMessage, java.lang.Boolean.FALSE) case Failure(e) => LogoList(slot.agent, s"LLM chat failed: ${e.getMessage}", java.lang.Boolean.FALSE) } } @@ -814,7 +975,9 @@ class LLMExtension extends DefaultClassManager { val agent = context.getAgent try { + val gen = generation val provider = ensureProvider(agent) + requireText(provider) // Extract model directory from workspace val modelDir = Option(context.workspace.getModelPath).flatMap { path => @@ -846,11 +1009,13 @@ class LLMExtension extends DefaultClassManager { // Send chat request val responseFuture = provider.chatWithFullResponse(tempHistory.toSeq) val response = Await.result(responseFuture, awaitTimeoutFor(effectiveConfig(agent))) - recordUsage(agent, response) + recordUsage(agent, gen, response) + rejectRefusal(response) + rejectEmptyReply(response) val responseMessage = replyMessage(response) // Commit both template message and response to permanent history on success - commitExchange(agent, userMessage, responseMessage) + commitExchange(agent, gen, userMessage, responseMessage) responseMessage.content @@ -877,6 +1042,7 @@ class LLMExtension extends DefaultClassManager { val agent = context.getAgent try { + val gen = generation val provider = ensureProvider(agent) val choices = choicesList.map(_.toString).toList @@ -885,6 +1051,28 @@ class LLMExtension extends DefaultClassManager { throw new ExtensionException("Choice list cannot be empty") } + // A decision provider picks natively: ask it a choice question about + // the prompt alone. Like llm:decide, this sends no history and writes + // none; usage is still recorded. + provider match { + case d: DecisionProvider => + // Same option rules as llm:decide's choice spec, checked before sending. + val question = parseDecisionSpec( + LogoList("choice", LogoList.fromVector(choices.toVector)), "llm:choose") + val (result, usage) = Await.result( + d.decide(prompt, question), + awaitTimeoutFor(effectiveConfig(agent))) + recordUsage(agent, gen, usage) + return result.answer match { + case DecisionAnswer.Picked(option) if choices.contains(option) => option + case DecisionAnswer.Picked(option) => + throw new ExtensionException( + s"""llm:choose: provider picked "$option", which is not one of the options: ${choices.mkString(", ")}""") + case other => throw new ExtensionException(s"llm:choose: provider answered $other, not an option") + } + case _ => () + } + // The wording must describe the same shape as the EnumFormat schema // sent with the request. A provider that validates the reply against // the schema server-side (Groq does) rejects the whole response with @@ -908,7 +1096,9 @@ class LLMExtension extends DefaultClassManager { // the constraint behaves exactly as it did before. val responseFuture = provider.chatWithFormat(tempHistory.toSeq, EnumFormat(choices)) val response = Await.result(responseFuture, awaitTimeoutFor(effectiveConfig(agent))) - recordUsage(agent, response) + recordUsage(agent, gen, response) + rejectRefusal(response) + rejectEmptyReply(response) // Extract text: prefer content, fall back to thinking field val text = response.firstContent.filter(_.nonEmpty) @@ -931,7 +1121,7 @@ class LLMExtension extends DefaultClassManager { } // Only on success: store clean messages in permanent history - commitExchange(agent, ChatMessage.user(prompt), ChatMessage.assistant(chosenOption)) + commitExchange(agent, gen, ChatMessage.user(prompt), ChatMessage.assistant(chosenOption)) chosenOption @@ -942,6 +1132,143 @@ class LLMExtension extends DefaultClassManager { } } + /** + * A typed decision about a prompt, with probabilities. + * + * llm:decide prompt ["choice" [options]] + * llm:decide prompt ["score" [levels]] + * llm:decide prompt ["yes-no"] + * + * Reports [answer probability distribution]. Prompt only: no history is + * sent and none is written. Usage is recorded for the calling agent, and + * the request goes through the provider's throttle and retry path. Needs a + * provider with a decision API (DecisionProvider). + */ + object DecideReporter extends Reporter { + override def getSyntax: Syntax = Syntax.reporterSyntax( + right = List(Syntax.StringType, Syntax.ListType), + ret = Syntax.ListType + ) + + override def report(args: Array[Argument], context: Context): AnyRef = { + val prompt = args(0).getString + // Validate before touching the provider so a bad spec costs nothing. + val question = parseDecisionSpec(args(1).getList, "llm:decide") + val agent = context.getAgent + val gen = generation + val provider = decisionProviderFor(agent) + + try { + val (result, usage) = Await.result(provider.decide(prompt, question), awaitTimeoutFor(effectiveConfig(agent))) + recordUsage(agent, gen, usage) + decisionToLogo(result) + } catch { + case e: Exception if !e.isInstanceOf[ExtensionException] => + throw new ExtensionException(s"llm:decide failed: ${e.getMessage}") + } + } + } + + /** + * Batched decisions: one request per agent, sent together, one wait. + * + * llm:decide-all agentset [ -> prompt ] spec + * + * Mirrors llm:chat-all: the prompt reporter runs as each agent, requests go + * out concurrently through the throttle, and one agent's failure is reported + * in its own [agent result ok?] triple. `result` is the [answer probability + * distribution] list, or the error text when ok? is false. Nothing is + * written to history; usage is recorded per agent. + */ + object DecideAllReporter extends Reporter { + override def getSyntax: Syntax = Syntax.reporterSyntax( + right = List(Syntax.AgentsetType, Syntax.ReporterType, Syntax.ListType), + ret = Syntax.ListType + ) + + private final class Slot(val agent: org.nlogo.agent.Agent, val prompt: String) { + var future: Future[AnyRef] = Future.failed(new IllegalStateException("request not sent")) + } + + override def report(args: Array[Argument], context: Context): AnyRef = { + val agents = args(0).getAgentSet + val promptReporter = args(1).getReporter + // Validate before building any prompt so a bad spec costs nothing. + val question = parseDecisionSpec(args(2).getList, "llm:decide-all") + + val ext = context match { + case ec: org.nlogo.nvm.ExtensionContext => ec + case _ => + throw new ExtensionException("llm:decide-all requires a NetLogo workspace and is unavailable in this context") + } + val caller = ext.nvmContext + + // Build every prompt before sending anything, on a copy of the agentset, + // exactly as llm:chat-all does. + val slots = agents.agents.asScala.toList.collect { + case a: org.nlogo.agent.Agent if a.id != -1 => + val agentContext = new org.nlogo.nvm.Context( + caller.job, a, caller.agent, caller.ip, caller.activation, ext.workspace) + promptReporter.report(agentContext, Array.empty[AnyRef]) match { + case prompt: String => new Slot(a, prompt) + case other => + throw new ExtensionException( + s"llm:decide-all: the prompt reporter must report a string, but for " + + s"${Dump.logoObject(a)} it reported ${Dump.logoObject(other, true, false)}") + } + } + + val lastReply = new java.util.concurrent.atomic.AtomicLong(System.nanoTime) + + val live = slots.filter(_.agent.id != -1) + // A provider without a decision API cannot serve any of the batch: raise + // before sending. A provider that fails to initialise stays per agent. + val providers = live.map { slot => + Try(decisionProviderFor(slot.agent)) match { + case Failure(e: NoDecisionApiException) => throw e + case other => other + } + } + val gen = generation + live.zip(providers).foreach { case (slot, provider) => + slot.future = + try { + val sent = provider.get.decide(slot.prompt, question) + sent.onComplete(_ => lastReply.set(System.nanoTime)) + sent.map { case (result, usage) => + recordUsage(slot.agent, gen, usage) + decisionToLogo(result) + } + } catch { + case e: Exception => Future.failed(e) + } + } + + // Same wait as llm:chat-all: give up only when no reply has arrived for + // an agent's whole budget. + def settle(slot: Slot, budget: FiniteDuration): Try[AnyRef] = { + while (!slot.future.isCompleted) { + val remaining = lastReply.get + budget.toNanos - System.nanoTime + if (remaining <= 0) { + return Failure(new java.util.concurrent.TimeoutException(s"no reply within ${budget.toSeconds}s")) + } + try Await.ready(slot.future, remaining.nanos) + catch { case _: java.util.concurrent.TimeoutException => () } + } + slot.future.value.get + } + + val entries = live.map { slot => + settle(slot, awaitTimeoutFor(effectiveConfig(slot.agent))) match { + case Success(v) => LogoList(slot.agent, v, java.lang.Boolean.TRUE) + case Failure(e) => LogoList(slot.agent, s"llm:decide failed: ${e.getMessage}", java.lang.Boolean.FALSE) + } + } + + LogoList.fromVector(entries.toVector) + } + } + /** * Read the selected option out of an enum-constrained reply. * @@ -1011,13 +1338,17 @@ class LLMExtension extends DefaultClassManager { val format = parseObjectSchema(schemaText, "llm:chat-with-schema") try { + val gen = generation val provider = ensureProvider(agent) + requireText(provider) val userMessage = ChatMessage.user(inputText) val responseFuture = provider.chatWithFormat(snapshotHistory(agent) :+ userMessage, format) val response = Await.result(responseFuture, awaitTimeoutFor(effectiveConfig(agent))) - recordUsage(agent, response) + recordUsage(agent, gen, response) + rejectRefusal(response) + rejectEmptyReply(response) val content = response.firstContent.getOrElse("") @@ -1031,7 +1362,7 @@ class LLMExtension extends DefaultClassManager { throw new ExtensionException(s"llm:chat-with-schema: ${e.getMessage}") } - commitExchange(agent, userMessage, ChatMessage.assistant(content)) + commitExchange(agent, gen, userMessage, ChatMessage.assistant(content)) parsed @@ -1062,7 +1393,9 @@ class LLMExtension extends DefaultClassManager { val agent = context.getAgent try { + val gen = generation val provider = ensureProvider(agent) + requireText(provider) // Anthropic has no schemaless JSON mode and Gemini's mime type alone is // only a hint, so the instruction is also stated in the prompt. Providers @@ -1078,7 +1411,9 @@ class LLMExtension extends DefaultClassManager { val responseFuture = provider.chatWithFormat(tempHistory.toSeq, JsonObjectFormat) val response = Await.result(responseFuture, awaitTimeoutFor(effectiveConfig(agent))) - recordUsage(agent, response) + recordUsage(agent, gen, response) + rejectRefusal(response) + rejectEmptyReply(response) val content = response.firstContent.getOrElse("") @@ -1092,13 +1427,13 @@ class LLMExtension extends DefaultClassManager { catch { case e: Exception => throw new ExtensionException( - s"llm:chat-json: Response was not valid JSON: ${e.getMessage}. Response text: $content" + s"llm:chat-json: Response was not valid JSON: ${JsonToNetLogo.parseErrorMessage(e)}. Response text: $content" ) } // Store the clean exchange only — the JSON instruction is prompt // scaffolding, not part of the conversation. - commitExchange(agent, userMessage, ChatMessage.assistant(content)) + commitExchange(agent, gen, userMessage, ChatMessage.assistant(content)) content @@ -1154,25 +1489,31 @@ class LLMExtension extends DefaultClassManager { val agent = context.getAgent try { + val gen = generation val provider = ensureProvider(agent) + requireText(provider) val userMessage = ChatMessage.user(inputText) // Send with user message included, but don't mutate history yet val responseFuture = provider.chatWithFullResponse(snapshotHistory(agent) :+ userMessage) val response = Await.result(responseFuture, awaitTimeoutFor(effectiveConfig(agent))) - recordUsage(agent, response) + recordUsage(agent, gen, response) + rejectRefusal(response) + rejectEmptyReply(response) val answerText = response.firstContent.getOrElse("") val thinkingText = response.thinking.getOrElse("") // Only commit both messages after success (clean answer only, not thinking text) - commitExchange(agent, userMessage, ChatMessage.assistant(answerText)) + commitExchange(agent, gen, userMessage, ChatMessage.assistant(answerText)) // Return [answer thinking] list — always 2 elements LogoList(answerText, thinkingText) } catch { + case e: ModelRefusedException => throw e + case e: TextUnsupportedException => throw e case e: Exception => throw new ExtensionException(s"LLM chat-with-thinking failed: ${e.getMessage}") } @@ -1427,9 +1768,7 @@ class LLMExtension extends DefaultClassManager { val details = provider.toLowerCase.trim match { case "ollama" => - val baseUrl = configStore.get(ConfigStore.OLLAMA_BASE_URL) - .orElse(configStore.get(ConfigStore.BASE_URL)) - .getOrElse(ConfigStore.DEFAULT_OLLAMA_BASE_URL) + val baseUrl = ConfigStore.resolveBaseUrl(configStore, ConfigStore.OLLAMA_BASE_URL, ConfigStore.DEFAULT_OLLAMA_BASE_URL) LogoList( provider, LogoList("ready", Boolean.box(ready)), @@ -1499,7 +1838,7 @@ class LLMExtension extends DefaultClassManager { // Mirror llm:load-config: the file is the whole configuration, with no // OpenAI-flavoured defaults layered underneath. A profile that names no - // model gets its own provider's default, not gpt-4o-mini. + // model gets its own provider's default, not the OpenAI default. val candidate = new ConfigStore() candidate.loadFromMap(config) checkReadiness(desc, providerName, candidate) @@ -1622,8 +1961,9 @@ class LLMExtension extends DefaultClassManager { // Load override from model directory if available Option(context.workspace.getModelPath).flatMap { path => Option(new java.io.File(path).getParent) - }.foreach { modelDir => - ModelRegistry.loadOverride(modelDir) + } match { + case Some(modelDir) => ModelRegistry.loadOverride(modelDir) + case None => ModelRegistry.clearOverride() } val providerName = configStore.getOrElse(ConfigStore.PROVIDER, ConfigStore.DEFAULT_PROVIDER) diff --git a/src/main/config/ConfigStore.scala b/src/main/config/ConfigStore.scala index 2209651..e7e2ae3 100644 --- a/src/main/config/ConfigStore.scala +++ b/src/main/config/ConfigStore.scala @@ -14,6 +14,10 @@ class ConfigStore { private val config: scala.collection.concurrent.Map[String, String] = new ConcurrentHashMap[String, String]().asScala + // Keys whose current value was seeded by setDefault rather than chosen by the + // user. Any later set, load, or remove of a key takes it out of this set. + private val defaulted: java.util.Set[String] = ConcurrentHashMap.newKeySet[String]() + /** * Set a configuration value * @@ -22,8 +26,24 @@ class ConfigStore { */ def set(key: String, value: String): Unit = { config(key) = value + defaulted.remove(key) + } + + /** + * Seed a default value. Unlike set, the key still reads as not chosen by + * the user (isDefault) until something else sets it. + */ + def setDefault(key: String, value: String): Unit = { + config(key) = value + defaulted.add(key) } + /** True when the key holds a value seeded by setDefault, not one the user set. */ + def isDefault(key: String): Boolean = defaulted.contains(key) + + /** The keys currently holding seeded default values. */ + def defaultKeys: Set[String] = defaulted.asScala.toSet + /** * Get a configuration value * @@ -62,6 +82,7 @@ class ConfigStore { * @return Option containing the removed value */ def remove(key: String): Option[String] = { + defaulted.remove(key) config.remove(key) } @@ -70,16 +91,21 @@ class ConfigStore { */ def clear(): Unit = { config.clear() + defaulted.clear() } /** * Load configuration from a map, replacing existing values * * @param newConfig Map of configuration key-value pairs + * @param defaultKeys Keys in newConfig that hold seeded defaults rather than + * user choices (used to restore a snapshot) */ - def loadFromMap(newConfig: Map[String, String]): Unit = { + def loadFromMap(newConfig: Map[String, String], defaultKeys: Set[String] = Set.empty): Unit = { config.clear() + defaulted.clear() config ++= newConfig + defaultKeys.filter(newConfig.contains).foreach(defaulted.add) } /** @@ -89,6 +115,7 @@ class ConfigStore { */ def updateFromMap(newConfig: Map[String, String]): Unit = { config ++= newConfig + newConfig.keys.foreach(defaulted.remove) } /** @@ -178,7 +205,6 @@ object ConfigStore { // Default values val DEFAULT_PROVIDER = "openai" - val DEFAULT_OPENAI_MODEL = "gpt-4o-mini" val DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1" val DEFAULT_ANTHROPIC_BASE_URL = "https://api.anthropic.com/v1" val DEFAULT_GEMINI_BASE_URL = "https://generativelanguage.googleapis.com/v1beta" @@ -192,12 +218,13 @@ object ConfigStore { */ def withDefaults(): ConfigStore = { val store = new ConfigStore() - store.set(PROVIDER, DEFAULT_PROVIDER) - store.set(MODEL, DEFAULT_OPENAI_MODEL) - store.set(BASE_URL, DEFAULT_OPENAI_BASE_URL) - store.set(TEMPERATURE, DEFAULT_TEMPERATURE) - store.set(MAX_TOKENS, DEFAULT_MAX_TOKENS) - store.set(TIMEOUT_SECONDS, DEFAULT_TIMEOUT_SECONDS) + store.setDefault(PROVIDER, DEFAULT_PROVIDER) + // The default provider's default model comes from the model catalog + // (models.yaml `default:`), so it can be updated without a Scala change. + store.setDefault(MODEL, org.nlogo.extensions.llm.providers.ModelRegistry.defaultModel(DEFAULT_PROVIDER)) + store.setDefault(TEMPERATURE, DEFAULT_TEMPERATURE) + store.setDefault(MAX_TOKENS, DEFAULT_MAX_TOKENS) + store.setDefault(TIMEOUT_SECONDS, DEFAULT_TIMEOUT_SECONDS) store } @@ -215,6 +242,18 @@ object ConfigStore { def getProviderBaseUrlName(provider: String): String = org.nlogo.extensions.llm.providers.ProviderRegistry.baseUrlConfigKey(provider) + /** + * Resolve the base URL a provider should send to. + * + * The provider-specific key (e.g. "anthropic_base_url") wins, then the legacy + * generic BASE_URL, then the provider's built-in default. Neither key is ever + * seeded with a default, so a value present here is one the user set. + */ + def resolveBaseUrl(config: ConfigStore, providerBaseUrlKey: String, defaultBaseUrl: String): String = + config.get(providerBaseUrlKey) + .orElse(config.get(BASE_URL)) + .getOrElse(defaultBaseUrl) + /** * Get default base URL for a provider. * Delegates to ProviderRegistry for registered providers. diff --git a/src/main/config/ModelRegistryLoader.scala b/src/main/config/ModelRegistryLoader.scala index 3f44985..06ac789 100644 --- a/src/main/config/ModelRegistryLoader.scala +++ b/src/main/config/ModelRegistryLoader.scala @@ -1,12 +1,12 @@ -// ABOUTME: Loads and merges model registry configurations from bundled YAML and optional override files -// ABOUTME: Provides model lists per provider with custom model counting +// ABOUTME: Loads and merges model registry configurations from bundled YAML, the user catalog, and override files +// ABOUTME: Provides model lists and optional default models per provider, with custom model counting package org.nlogo.extensions.llm.config import scala.util.{Try, Success, Failure} import scala.io.Source import io.circe.yaml.parser -import io.circe.{Json, HCursor} -import java.nio.file.{Files, Paths} +import io.circe.{ACursor, Json, HCursor} +import java.nio.file.{Files, Path, Paths} import java.nio.charset.StandardCharsets /** @@ -14,19 +14,42 @@ import java.nio.charset.StandardCharsets * * @param models Set of model identifiers for this provider * @param isCustom Whether these models come from a custom/override configuration + * @param default The provider's default model, when the YAML names one with `default:` */ -case class ProviderModels(models: Set[String], isCustom: Boolean = false) +case class ProviderModels(models: Set[String], isCustom: Boolean = false, default: Option[String] = None) /** * Utility for loading and merging model registry configurations * * The model registry defines which models are available for each provider. - * Configuration can come from: + * Configuration can come from, lowest layer first: * 1. Bundled config in JAR resources (/config/models.yaml) - * 2. Override config in model directory (models-override.yaml) + * 2. User catalog (~/.netlogo-llm/models.yaml, path overridable), merged additively + * 3. Override config in model directory (models-override.yaml) */ object ModelRegistryLoader { + /** System property naming the user catalog file (takes precedence over the env var). */ + val UserCatalogProperty: String = "llm.models.user.file" + + /** Environment variable naming the user catalog file. */ + val UserCatalogEnv: String = "NETLOGO_LLM_MODELS" + + /** + * Where the user catalog lives: the system property, then the environment + * variable, then ~/.netlogo-llm/models.yaml. + */ + def userCatalogPath: Path = { + val configured = Option(System.getProperty(UserCatalogProperty)) + .orElse(sys.env.get(UserCatalogEnv)) + .map(_.trim) + .filter(_.nonEmpty) + configured match { + case Some(p) => Paths.get(p) + case None => Paths.get(System.getProperty("user.home"), ".netlogo-llm", "models.yaml") + } + } + /** * Load the bundled model configuration from JAR resources * @@ -55,6 +78,20 @@ object ModelRegistryLoader { }.flatten } + /** + * Load the user catalog from a file + * + * The file is optional: a missing file is an empty catalog, not an error. + * Its providers are not marked custom; it is the user's own catalog rather + * than a per-model tweak. + * + * @param path User catalog file + * @return Try containing Map of provider name to ProviderModels + */ + def loadUserConfig(path: Path): Try[Map[String, ProviderModels]] = + if (!Files.exists(path)) Success(Map.empty) + else Try(Files.readString(path, StandardCharsets.UTF_8)).flatMap(parseModelConfig(_, isCustom = false)) + /** * Load override model configuration from a directory * @@ -88,11 +125,45 @@ object ModelRegistryLoader { } } + /** + * Merge the user catalog over a base catalog + * + * Additive, unlike mergeConfigs: a user section adds its models to the + * provider's existing list rather than replacing it, and its `default:` + * (when present) replaces the base default and is added to the list. Providers + * only the user file names are added as-is, plus their own default. + * + * @param base The bundled (or fallback) catalog + * @param user The user catalog + * @return Merged configuration map + */ + def mergeUserConfig( + base: Map[String, ProviderModels], + user: Map[String, ProviderModels] + ): Map[String, ProviderModels] = + user.foldLeft(base) { case (acc, (provider, u)) => + acc.get(provider) match { + case Some(b) => + acc.updated(provider, b.copy(models = b.models ++ withOwnDefault(u).models, default = u.default.orElse(b.default))) + case None => + acc.updated(provider, withOwnDefault(u)) + } + } + + /** + * A section with its own `default:` added to its model list, so a default + * named only under `default:` is still a known model. + */ + private def withOwnDefault(section: ProviderModels): ProviderModels = + section.copy(models = section.models ++ section.default) + /** * Merge bundled and override configurations * * Override sections completely replace bundled sections for each provider. - * If a provider exists in override config, its bundled config is ignored. + * If a provider exists in override config, its bundled model list is ignored. + * An override section without `default:` keeps the lower layer's default; + * one with `default:` gets that model added to its list. * * @param bundled The bundled configuration from JAR resources * @param overrideConfig The override configuration from model directory @@ -104,26 +175,32 @@ object ModelRegistryLoader { ): Map[String, ProviderModels] = { // Start with bundled config, then overlay override config // Override completely replaces bundled for each provider key - bundled ++ overrideConfig + bundled ++ overrideConfig.map { case (provider, o) => + provider -> withOwnDefault(o).copy(default = o.default.orElse(bundled.get(provider).flatMap(_.default))) + } } /** * Parse YAML model configuration content * - * Expected YAML structure: + * Expected YAML structure — each provider is either a plain list or a + * mapping with an optional `default:` and an optional `models:` list (at + * least one of the two): * ```yaml * openai: - * - gpt-4o - * - gpt-4o-mini + * default: gpt-6-luna + * models: + * - gpt-6-luna + * - gpt-4o-mini * anthropic: - * - claude-3-5-sonnet + * - claude-haiku-5-5 * ``` * * @param content YAML content as string * @param isCustom Whether this config comes from a custom/override source * @return Try containing Map of provider name to ProviderModels */ - private def parseModelConfig(content: String, isCustom: Boolean): Try[Map[String, ProviderModels]] = { + private[llm] def parseModelConfig(content: String, isCustom: Boolean): Try[Map[String, ProviderModels]] = { Try { parser.parse(content) match { case Right(json) => @@ -134,9 +211,11 @@ object ModelRegistryLoader { throw new RuntimeException("YAML root must be an object with provider names as keys") ).keys.toSeq - // Parse each provider's model list + // Parse each provider's model list (and default, in the mapping form) val config = providers.map { provider => - val models = cursor.downField(provider).as[Seq[String]] match { + val section = cursor.downField(provider) + + def modelList(c: ACursor): Set[String] = c.as[Seq[String]] match { case Right(modelList) => if (modelList.isEmpty) { throw new RuntimeException(s"Provider '$provider' has empty model list") @@ -149,7 +228,24 @@ object ModelRegistryLoader { ) } - provider -> ProviderModels(models, isCustom) + val parsed = if (section.focus.exists(_.isObject)) { + val defaultField = section.downField("default") + val default = defaultField.focus.map { _ => + defaultField.as[String].toOption.map(_.trim).filter(_.nonEmpty).getOrElse( + throw new RuntimeException(s"Provider '$provider' default must be a model name") + ) + } + val modelsField = section.downField("models") + val models = if (modelsField.focus.isDefined) modelList(modelsField) else Set.empty[String] + if (default.isEmpty && models.isEmpty) { + throw new RuntimeException(s"Provider '$provider' must have a models list or a default") + } + ProviderModels(models, isCustom, default) + } else { + ProviderModels(modelList(section), isCustom) + } + + provider -> parsed }.toMap if (config.isEmpty) { diff --git a/src/main/models/ChatResponse.scala b/src/main/models/ChatResponse.scala index 63c0bb8..0dd690e 100644 --- a/src/main/models/ChatResponse.scala +++ b/src/main/models/ChatResponse.scala @@ -7,12 +7,14 @@ import upickle.default.{ReadWriter => RW, macroRW} * * @param index The index of this choice (usually 0 for single responses) * @param message The message content of the response - * @param finishReason The reason the response ended (e.g., "stop", "length") + * @param finishReason The reason the response ended (e.g., "stop", "length"); empty when not reported + * @param refusal The model's explanation when it declined to answer, if it did */ case class Choice( index: Int, message: ChatMessage, - finishReason: String + finishReason: String, + refusal: Option[String] = None ) object Choice { @@ -32,8 +34,9 @@ object Choice { * not. * - totalTokens is taken from the provider when reported, else computed. * - reasoning / cache counts are absent when the provider does not report them. - * - cost is only present when a provider reports it directly (OpenRouter); - * nothing is ever estimated. + * - cost is only present when a provider reports it directly (OpenRouter), + * or bills one published price per reported token (TypeSafe: input tokens + * only, output free), so it is exact; nothing is ever estimated. * - latencyMs is the wall time of the whole call as the modeler experiences * it, including throttle queueing and rate-limit retries. */ @@ -122,6 +125,13 @@ case class ChatResponse( def firstContent: Option[String] = { firstMessage.map(_.content) } + + /** + * The refusal text of the first choice, when the model declined to answer + */ + def refusal: Option[String] = { + choices.headOption.flatMap(_.refusal) + } } object ChatResponse { diff --git a/src/main/models/Decision.scala b/src/main/models/Decision.scala new file mode 100644 index 0000000..680d11c --- /dev/null +++ b/src/main/models/Decision.scala @@ -0,0 +1,91 @@ +// ABOUTME: Vendor-neutral model of a typed decision question and its probabilistic answer +// ABOUTME: Shared by llm:decide / llm:decide-all and every provider that implements DecisionProvider +package org.nlogo.extensions.llm.models + +/** + * A question with a fixed answer space, asked about a prompt. + * + * Unlike a chat request, the reply is not text: it is one point in the answer + * space plus a probability for every point, so a model can sample from the + * distribution or take the top pick. + */ +enum DecisionQuestion { + /** Pick one of `options`. */ + case Choice(options: Seq[String]) + /** Rate on the ordered `levels`, numbered from 0. */ + case Score(levels: Seq[String]) + /** Answer yes or no. */ + case YesNo +} + +object DecisionQuestion { + val MaxChoiceOptions: Int = 255 + val MinScoreLevels: Int = 2 + val MaxScoreLevels: Int = 10 +} + +/** The answer to a DecisionQuestion; its shape follows the question type. */ +enum DecisionAnswer { + /** The selected option of a Choice. */ + case Picked(option: String) + /** The probability-weighted level of a Score, on the 0-based scale. */ + case Expected(value: Double) + /** The answer to a YesNo. */ + case Verdict(yes: Boolean) +} + +/** + * A decision and how sure the model is of it. + * + * @param answer the decision itself + * @param probability the probability of `answer` for Choice and YesNo; for a + * Score, whose answer can fall between levels, the + * provider's confidence in the score + * @param distribution every possible outcome with its probability, in question + * order: options for Choice, level names for Score, and + * "true" then "false" for YesNo + */ +case class DecisionResult( + answer: DecisionAnswer, + probability: Double, + distribution: Seq[(String, Double)] +) + +object DecisionResult { + + /** + * A Choice result. The distribution follows `options` order rather than the + * provider's, so a modeler's list lines up with what comes back; an option + * the provider gave no probability gets 0. + */ + def choice(options: Seq[String], picked: String, probabilities: Map[String, Double]): DecisionResult = { + val distribution = options.map(o => o -> probabilities.getOrElse(o, 0.0)) + DecisionResult(DecisionAnswer.Picked(picked), probabilities.getOrElse(picked, 0.0), distribution) + } + + /** A Score result; `probabilities(i)` is the probability of `levels(i)`. */ + def score(levels: Seq[String], expected: Double, confidence: Double, probabilities: Seq[Double]): DecisionResult = + DecisionResult( + DecisionAnswer.Expected(expected), + confidence, + levels.zipAll(probabilities, "", 0.0).take(levels.size) + ) + + /** A YesNo result from the probability of yes; the answer is whichever side is likelier. */ + def yesNo(probabilityYes: Double): DecisionResult = { + val yes = probabilityYes >= 0.5 + val probabilityNo = complement(probabilityYes) + DecisionResult( + DecisionAnswer.Verdict(yes), + if (yes) probabilityYes else probabilityNo, + Seq("true" -> probabilityYes, "false" -> probabilityNo) + ) + } + + /** + * 1 - p in decimal rather than binary arithmetic, so a provider's 0.56 gives + * 0.44 instead of 0.43999999999999995 in a modeler's monitors. + */ + private def complement(p: Double): Double = + (BigDecimal(1) - BigDecimal(p.toString)).toDouble +} diff --git a/src/main/models/ResponseFormat.scala b/src/main/models/ResponseFormat.scala index 09e37f3..f78249e 100644 --- a/src/main/models/ResponseFormat.scala +++ b/src/main/models/ResponseFormat.scala @@ -2,6 +2,8 @@ // ABOUTME: Parses and normalizes modeler-supplied JSON Schema strings into a provider-neutral form package org.nlogo.extensions.llm.models +import org.nlogo.extensions.llm.utils.JsonToNetLogo + /** * How a provider should constrain the shape of its reply. * @@ -110,7 +112,7 @@ object ResponseFormat { catch { case e: Exception => throw new IllegalArgumentException( - s"Schema is not valid JSON: ${e.getMessage}" + s"Schema is not valid JSON: ${JsonToNetLogo.parseErrorMessage(e)}" ) } diff --git a/src/main/providers/BaseHttpProvider.scala b/src/main/providers/BaseHttpProvider.scala index 10b2714..800f44a 100644 --- a/src/main/providers/BaseHttpProvider.scala +++ b/src/main/providers/BaseHttpProvider.scala @@ -13,27 +13,100 @@ import scala.concurrent.{Future, ExecutionContext, Promise} import scala.concurrent.duration._ import scala.util.{Try, Success, Failure} +/** A non-success HTTP response, carrying the status code so callers can branch on it. */ +class HttpStatusException(val statusCode: Int, val errorBody: String, message: String) + extends RuntimeException(message) + object BaseHttpProvider { /** Waits at or above this are announced on stderr so a stall isn't mistaken for a hang. */ private[providers] val WaitNoticeThresholdMs = 2000L // Process-wide counters so a modeler can tell how much of a run was spent - // sitting out rate limits, rather than guessing from wall-clock time. - private[providers] val rateLimitWaits = new java.util.concurrent.atomic.LongAdder() - private[providers] val rateLimitWaitMillis = new java.util.concurrent.atomic.LongAdder() + // sitting out retries (rate limits, server errors, overloads, network + // failures), rather than guessing from wall-clock time. + private[providers] val retryWaits = new java.util.concurrent.atomic.LongAdder() + private[providers] val retryWaitMillis = new java.util.concurrent.atomic.LongAdder() + + /** Number of retry waits that have been announced this session. */ + def retryWaitCount: Long = retryWaits.sum() + + /** Total milliseconds spent in announced retry waits this session. */ + def retryWaitTotalMs: Long = retryWaitMillis.sum() - /** Number of rate-limit waits that have been announced this session. */ - def rateLimitWaitCount: Long = rateLimitWaits.sum() + // (provider, model) pairs already told that schema-constrained output is unsupported. + private[providers] val responseFormatFallbackNoticed: java.util.Set[String] = + java.util.concurrent.ConcurrentHashMap.newKeySet[String]() - /** Total milliseconds spent waiting out announced rate limits this session. */ - def rateLimitWaitTotalMs: Long = rateLimitWaitMillis.sum() + /** + * True when a 400 says the model rejected the requested response_format, as + * opposed to any other bad request (bad key, unknown model) that must still fail. + */ + private[providers] def isResponseFormatUnsupported(e: HttpStatusException): Boolean = { + val body = e.errorBody.toLowerCase + e.statusCode == 400 && + Seq("response_format", "json_schema", "json_object", "structured", "not supported", "unsupported") + .exists(body.contains) + } - /** Reset the rate-limit counters. Primarily for tests and fresh runs. */ - def resetRateLimitStats(): Unit = { - rateLimitWaits.reset() - rateLimitWaitMillis.reset() + /** Reset the retry-wait counters. Primarily for tests and fresh runs. */ + def resetRetryWaitStats(): Unit = { + retryWaits.reset() + retryWaitMillis.reset() } + /** + * Server-side statuses worth retrying: the request was fine, the provider was + * briefly unable to serve it. 529 is Anthropic's "overloaded". 429 is handled + * separately because it also carries a requested delay and quota guidance. + */ + val TransientStatuses: Set[Int] = Set(500, 502, 503, 504, 529) + + /** `e` and its chain of causes, outermost first. */ + private[providers] def causes(e: Throwable): List[Throwable] = + Iterator.iterate(e)(_.getCause).takeWhile(_ != null).take(16).toList + + /** + * Whether a failed send is a transport hiccup worth retrying: the connection + * could not be opened, dropped mid-read, or timed out. Checked across the + * cause chain because backends and Future plumbing wrap the original error. + */ + def isTransientFailure(e: Throwable): Boolean = + !isUnknownHost(e) && causes(e).exists { + case _: SttpClientException.ConnectException | _: SttpClientException.ReadException => true + case _: java.net.ConnectException | _: java.net.SocketTimeoutException => true + case _: java.net.http.HttpTimeoutException => true + case _ => false + } + + /** + * Whether a failed send is a host name that does not resolve. java.net.http + * reports this as a ConnectException caused by UnresolvedAddressException, so + * it would otherwise look like a refused connection. Not retried: a mistyped + * base URL will not resolve on the next attempt either. + */ + def isUnknownHost(e: Throwable): Boolean = + causes(e).exists { + case _: java.nio.channels.UnresolvedAddressException | _: java.net.UnknownHostException => true + case _ => false + } + + /** + * Whether an error body says the API key itself was rejected. Gemini reports a + * bad key as HTTP 400 with reason API_KEY_INVALID rather than as a 401. + */ + def isInvalidApiKey(code: Int, errorBody: String): Boolean = + code == 401 || errorBody.contains("API_KEY_INVALID") + + /** + * The stderr line announcing a retry wait, e.g. + * "NOTE: Anthropic returned HTTP 529 (overloaded); retrying in 2.0s (attempt 2 of 4)". + * `attempt` is the 0-based attempt that just failed; attempts are counted 1-based + * out of the initial request plus every allowed retry. + */ + def retryWaitNotice(provider: String, cause: String, delayMs: Long, attempt: Int, maxRetries: Int): String = + f"NOTE: $provider $cause; retrying in ${delayMs / 1000.0}%.1fs " + + s"(attempt ${attempt + 2} of ${maxRetries + 1})" + // Single daemon thread schedules retry attempts. Daemon so it never blocks JVM // shutdown; one idle thread persists across extension reloads, which is fine. private lazy val retryScheduler: ScheduledExecutorService = @@ -73,7 +146,8 @@ abstract class BaseHttpProvider(implicit ec: ExecutionContext) extends LLMProvid // constructing the real HTTP client. protected lazy val backend: Backend[Future] = HttpClientFutureBackend() - // Retry policy for rate-limit (HTTP 429) responses. retryBaseDelayMs is a + // Retry policy for rate-limit (HTTP 429) responses and other transient + // failures (5xx, 529, connect/read timeouts). retryBaseDelayMs is a // protected def so tests can shrink the delays. protected def retryBaseDelayMs: Long = RetryPolicy.DefaultBaseDelay.toMillis @@ -114,27 +188,43 @@ abstract class BaseHttpProvider(implicit ec: ExecutionContext) extends LLMProvid protected def requestThrottle: Option[RequestThrottle] = RequestThrottle.forProvider( providerName, - configStore.get(baseUrlConfigKey).getOrElse(defaultBaseUrl), + resolvedBaseUrl, configStore.get(RequestThrottle.MAX_CONCURRENT_REQUESTS), configStore.get(RequestThrottle.MIN_REQUEST_INTERVAL_MS) ) /** - * Reports a rate-limit wait to the modeler. A silent multi-second stall inside + * Reports a retry wait to the modeler. A silent multi-second stall inside * `go` is indistinguishable from a hang, so long waits are announced on stderr. * Short waits stay quiet to avoid spamming the console on routine backoff. + * + * @param cause what is being waited out, e.g. "returned HTTP 529 (overloaded)" */ - protected def notifyRateLimitWait(delayMs: Long, attempt: Int, maxRetries: Int): Unit = + protected def notifyRetryWait(delayMs: Long, attempt: Int, maxRetries: Int, cause: String): Unit = if (delayMs >= BaseHttpProvider.WaitNoticeThresholdMs) { - System.err.println( - f"NOTE: $providerName rate limited (HTTP 429); waiting ${delayMs / 1000.0}%.1fs " + - s"before retry ${attempt + 1} of $maxRetries. " + - "Set retry_max_elapsed_seconds / retry_max_retries to tune, or reduce request rate." - ) - BaseHttpProvider.rateLimitWaits.increment() - BaseHttpProvider.rateLimitWaitMillis.add(delayMs) + System.err.println(BaseHttpProvider.retryWaitNotice(displayName, cause, delayMs, attempt, maxRetries)) + BaseHttpProvider.retryWaits.increment() + BaseHttpProvider.retryWaitMillis.add(delayMs) } + /** + * Tells the modeler, once per (provider, model), that structured output fell + * back to an unconstrained reply because the model rejected the response_format. + */ + protected def notifyResponseFormatUnsupported(model: String): Unit = + System.err.println( + s"NOTE: $displayName model $model does not support schema-constrained output " + + "(HTTP 400 on response_format); resending without it, so the reply is unconstrained." + ) + + /** Human-readable provider name for messages, from the provider registry. */ + def displayName: String = + ProviderRegistry.get(providerName).map(_.displayName).getOrElse(providerName) + + /** Next step suggested when this provider's endpoint cannot be reached. */ + protected def unreachableHint: String = + s"Check your network connection and $baseUrlConfigKey." + // Initialize with provider-specific defaults initializeDefaults() @@ -148,6 +238,11 @@ abstract class BaseHttpProvider(implicit ec: ExecutionContext) extends LLMProvid protected def requiresApiKey: Boolean protected def buildApiUrl(baseUrl: String): Uri + + /** Base URL for requests: provider-specific key, then legacy base_url, then default. */ + protected def resolvedBaseUrl: String = + ConfigStore.resolveBaseUrl(configStore, baseUrlConfigKey, defaultBaseUrl) + protected def buildHeaders(apiKey: Option[String]): Map[String, String] protected def createProviderRequest(request: ChatRequest): ujson.Value protected def parseProviderResponse(responseBody: String, model: String): ChatResponse @@ -156,11 +251,10 @@ abstract class BaseHttpProvider(implicit ec: ExecutionContext) extends LLMProvid * Initialize provider-specific default configuration */ protected def initializeDefaults(): Unit = { - configStore.set(ConfigStore.PROVIDER, providerName) - configStore.set(ConfigStore.MODEL, defaultModel) - configStore.set(baseUrlConfigKey, defaultBaseUrl) - configStore.set(ConfigStore.TEMPERATURE, ConfigStore.DEFAULT_TEMPERATURE) - configStore.set(ConfigStore.MAX_TOKENS, defaultMaxTokens) + configStore.setDefault(ConfigStore.PROVIDER, providerName) + configStore.setDefault(ConfigStore.MODEL, defaultModel) + configStore.setDefault(ConfigStore.TEMPERATURE, ConfigStore.DEFAULT_TEMPERATURE) + configStore.setDefault(ConfigStore.MAX_TOKENS, defaultMaxTokens) } /** @@ -168,7 +262,16 @@ abstract class BaseHttpProvider(implicit ec: ExecutionContext) extends LLMProvid */ override def chat(request: ChatRequest): Future[ChatResponse] = { validateConfig() match { - case Success(_) => sendChatRequest(request) + case Success(_) => + sendChatRequest(request).recoverWith { + // Some models reject response_format outright; resend once without it. + // Callers already prompt for the expected shape, so the reply still parses. + case e: HttpStatusException + if request.responseFormat.isDefined && BaseHttpProvider.isResponseFormatUnsupported(e) => + if (BaseHttpProvider.responseFormatFallbackNoticed.add(s"$providerName/${request.model}")) + notifyResponseFormatUnsupported(request.model) + sendChatRequest(request.copy(responseFormat = None)) + } case Failure(e) => Future.failed(e) } } @@ -240,7 +343,7 @@ abstract class BaseHttpProvider(implicit ec: ExecutionContext) extends LLMProvid configStore.get(apiKeyConfigKey).orElse(configStore.get(ConfigStore.API_KEY)) } - val baseUrl = configStore.get(baseUrlConfigKey).getOrElse(defaultBaseUrl) + val baseUrl = resolvedBaseUrl val apiUrl = buildApiUrl(baseUrl) val headers = buildHeaders(apiKey) val requestBody = createProviderRequest(request).toString() @@ -269,9 +372,56 @@ abstract class BaseHttpProvider(implicit ec: ExecutionContext) extends LLMProvid errorBody ) + /** Short reason shown next to an HTTP status in messages. */ + protected def statusReason(code: Int): String = code match { + case 400 => "bad request" + case 401 => "invalid API key" + case 403 => "access denied" + case 404 => "not found" + case 408 => "request timeout" + case 413 => "request too large" + case 422 => "unprocessable request" + case 429 => "rate limited" + case 500 => "server error" + case 502 => "bad gateway" + case 503 => "service unavailable" + case 504 => "gateway timeout" + case 529 => "overloaded" + case _ => "error" + } + + /** One next step for a request the provider rejected outright. */ + protected def rejectedHint(code: Int): String = code match { + case 401 | 403 => s"Check $apiKeyConfigKey in your config or call llm:set-api-key." + case 404 => s"Check the model name and $baseUrlConfigKey." + case 413 => "Shorten the prompt or the conversation history." + case _ => "Check the model name and request settings in your config." + } + + /** Short description of a transport failure, e.g. "connection refused". */ + private def transportReason(e: Throwable): String = { + // Checked most-specific first across the whole chain: a TimeoutException is + // a ReadException. Unknown hosts never get here; see isUnknownHost. + val chain = BaseHttpProvider.causes(e) + def has(p: Throwable => Boolean): Boolean = chain.exists(p) + if (has { + case _: SttpClientException.TimeoutException | _: java.net.SocketTimeoutException | + _: java.net.http.HttpTimeoutException => true + case _ => false + }) "timed out" + else if (has { + case _: SttpClientException.ConnectException | _: java.net.ConnectException => true + case _ => false + }) "connection refused" + else if (has(_.isInstanceOf[SttpClientException.ReadException])) "connection dropped" + else "network error" + } + /** - * Send an HTTP request, retrying rate-limit (HTTP 429) responses with jittered - * exponential backoff. Only rate-limit errors retry; all others fail immediately. + * Send an HTTP request, retrying transient failures with jittered exponential + * backoff: rate limits (HTTP 429), server errors (500, 502, 503, 504), Anthropic's + * 529 "overloaded", and connect/read timeouts. Every other error fails at once. + * All transient failures share the same RetryPolicy, backoff and budget. * * The delay the provider asks for wins over our own backoff when it is longer, * because retrying before a quota window reopens is guaranteed to fail. Waiting @@ -292,32 +442,79 @@ abstract class BaseHttpProvider(implicit ec: ExecutionContext) extends LLMProvid val rng = retryRandom val startedAt = System.currentTimeMillis() + val baseUrl = resolvedBaseUrl + def isRateLimited(code: StatusCode, error: String): Boolean = code.code == 429 || error.toLowerCase.contains("rate_limit") + def details(error: String): String = { + val trimmed = error.trim + if (trimmed.isEmpty) "" + else s" Details: ${if (trimmed.length > 300) trimmed.take(300) + "..." else trimmed}" + } + + /** Wait and try again if the policy allows, otherwise fail with `exhausted`. */ + def retryOrFail(n: Int, elapsedMs: Long, requestedMs: Option[Long], cause: String)( + exhausted: => Throwable + ): Future[ChatResponse] = { + val delayMs = policy.delayMsFor(n, requestedMs, rng) + if (policy.canRetry(n, elapsedMs, delayMs)) { + notifyRetryWait(delayMs, n, policy.maxRetries, cause) + BaseHttpProvider.delayedFuture(delayMs)(attempt(n + 1, elapsedMs + delayMs)) + } else Future.failed(exhausted) + } + def attempt(n: Int, elapsedMs: Long): Future[ChatResponse] = - httpRequest.send(backend).flatMap { response => - response.body match { - case Right(responseBody) => - Future.successful(parseProviderResponse(responseBody, model)) - - case Left(error) if isRateLimited(response.code, error) => - val delayMs = policy.delayMsFor(n, parseRetryDelayMs(response, error), rng) - if (policy.canRetry(n, elapsedMs, delayMs)) { - notifyRateLimitWait(delayMs, n, policy.maxRetries) - BaseHttpProvider.delayedFuture(delayMs)(attempt(n + 1, elapsedMs + delayMs)) - } else { - Future.failed(new RuntimeException( - s"HTTP request failed after ${n + 1} attempts " + - s"(rate limited, HTTP ${response.code.code}) after waiting " + - f"${elapsedMs / 1000.0}%.1fs: $error. " + - "This model's quota may be too low for this simulation - reduce the " + - "request rate, or raise retry_max_elapsed_seconds / retry_max_retries.")) - } - - case Left(error) => - Future.failed(new RuntimeException(s"HTTP request failed: $error")) - } + httpRequest.send(backend).transformWith { + case Success(response) => + val code = response.code.code + response.body match { + case Right(responseBody) => + Future.successful(parseProviderResponse(responseBody, model)) + + case Left(error) if isRateLimited(response.code, error) => + retryOrFail(n, elapsedMs, parseRetryDelayMs(response, error), + s"returned HTTP $code (${statusReason(429)})") { + new RuntimeException( + s"$displayName returned HTTP $code (${statusReason(429)}) after ${n + 1} attempts " + + f"and ${elapsedMs / 1000.0}%.1fs of waiting. " + + "This model's quota may be too low for this simulation - reduce the request " + + "rate with the throttling settings, or raise retry_max_elapsed_seconds / " + + "retry_max_retries." + details(error)) + } + + case Left(error) if BaseHttpProvider.TransientStatuses.contains(code) => + val cause = s"returned HTTP $code (${statusReason(code)})" + retryOrFail(n, elapsedMs, parseRetryDelayMs(response, error), cause) { + new RuntimeException( + s"$displayName $cause after ${n + 1} attempts. " + + "Try again shortly, or lower concurrency with the throttling settings." + + details(error)) + } + + case Left(error) => + // Report a rejected key as such whatever status carried it. + val reasonCode = if (BaseHttpProvider.isInvalidApiKey(code, error)) 401 else code + Future.failed(new HttpStatusException(code, error, + s"$displayName rejected the request: HTTP $code (${statusReason(reasonCode)}). " + + rejectedHint(reasonCode) + details(error))) + } + + case Failure(e) if BaseHttpProvider.isUnknownHost(e) => + Future.failed(new RuntimeException( + s"Could not reach $displayName at $baseUrl (unknown host). " + + s"Check $baseUrlConfigKey in your config for a typo.", e)) + + case Failure(e) if BaseHttpProvider.isTransientFailure(e) => + val reason = transportReason(e) + retryOrFail(n, elapsedMs, None, s"could not be reached ($reason)") { + new RuntimeException( + s"Could not reach $displayName at $baseUrl ($reason) after ${n + 1} attempts. " + + unreachableHint, e) + } + + case Failure(e) => + Future.failed(e) } val completed = requestThrottle match { @@ -339,6 +536,10 @@ abstract class BaseHttpProvider(implicit ec: ExecutionContext) extends LLMProvid configStore.set(key, value) } + override def setDefaultConfig(key: String, value: String): Unit = { + configStore.setDefault(key, value) + } + override def getConfig(key: String): Option[String] = { configStore.get(key) } diff --git a/src/main/providers/ClaudeProvider.scala b/src/main/providers/ClaudeProvider.scala index dffd2d3..7d47992 100644 --- a/src/main/providers/ClaudeProvider.scala +++ b/src/main/providers/ClaudeProvider.scala @@ -239,11 +239,20 @@ class ClaudeProvider(implicit ec: ExecutionContext) extends BaseHttpProvider { val thinking = if (thinkingTexts.nonEmpty) Some(thinkingTexts.mkString("\n")) else None + val stopReason = parsed.obj.get("stop_reason").collect { case ujson.Str(s) => s }.getOrElse("") + + // Anthropic signals a refusal by stop_reason alone; any text produced + // before it stopped is the closest thing to an explanation. + val refusal = + if (stopReason == "refusal") Some(if (text.nonEmpty) text else "stop_reason refusal") + else None + val choices = Array( org.nlogo.extensions.llm.models.Choice( index = 0, message = ChatMessage("assistant", text), - finishReason = parsed("stop_reason").str + finishReason = stopReason, + refusal = refusal ) ) diff --git a/src/main/providers/DecisionProvider.scala b/src/main/providers/DecisionProvider.scala new file mode 100644 index 0000000..c8b4386 --- /dev/null +++ b/src/main/providers/DecisionProvider.scala @@ -0,0 +1,30 @@ +// ABOUTME: Capability trait for providers with a typed decision API (choice, score, yes/no) +// ABOUTME: llm:decide, llm:decide-all and llm:choose route through it; any decision-style API implements only this +package org.nlogo.extensions.llm.providers + +import org.nlogo.extensions.llm.models.{DecisionQuestion, DecisionResult, Usage} +import scala.concurrent.Future + +/** + * A provider that answers typed questions with calibrated probabilities + * instead of generating text. + * + * Mixed into an LLMProvider so the provider still has config, a model and a + * name. Implementations must send through their request throttle, as chat does. + */ +trait DecisionProvider { self: LLMProvider => + + /** + * Ask `question` about `prompt`. No history is involved: the prompt is the + * whole context. + * + * @return the decision and the usage the call was billed for + */ + def decide(prompt: String, question: DecisionQuestion): Future[(DecisionResult, Usage)] + + /** + * Why free-text chat is unavailable on this provider, or None when it serves + * text as well as decisions. The text primitives raise this message as is. + */ + def textUnsupportedMessage: Option[String] = None +} diff --git a/src/main/providers/GeminiProvider.scala b/src/main/providers/GeminiProvider.scala index 50100b3..1685bdc 100644 --- a/src/main/providers/GeminiProvider.scala +++ b/src/main/providers/GeminiProvider.scala @@ -47,7 +47,7 @@ class GeminiProvider(implicit ec: ExecutionContext) extends BaseHttpProvider { .orElse(configStore.get(ConfigStore.API_KEY)) .getOrElse(throw new IllegalStateException("API key not configured")) - val baseUrl = configStore.get(baseUrlConfigKey).getOrElse(defaultBaseUrl) + val baseUrl = resolvedBaseUrl val apiUrl = uri"$baseUrl/models/${request.model}:generateContent?key=$apiKey" val headers = Map("Content-Type" -> "application/json") @@ -83,12 +83,17 @@ class GeminiProvider(implicit ec: ExecutionContext) extends BaseHttpProvider { * Gemini expects: * - "contents" array with "role" and "parts" * - Role mapping: "assistant" → "model", "user" → "user" + * - System messages in a top-level "systemInstruction", not in "contents" * - "generationConfig" for optional parameters (temperature, maxOutputTokens) */ override protected def createProviderRequest(request: ChatRequest): ujson.Value = { + // Gemini takes system prompts as a top-level systemInstruction, not as a + // turn in contents; several system messages are joined into one. + val (systemMessages, chatMessages) = request.messages.partition(_.role == "system") + // Convert messages to Gemini's format val contents = ujson.Arr( - request.messages.map { msg => + chatMessages.map { msg => val role = if (msg.role == "assistant") "model" else "user" ujson.Obj( "role" -> role, @@ -103,6 +108,14 @@ class GeminiProvider(implicit ec: ExecutionContext) extends BaseHttpProvider { "contents" -> contents ) + if (systemMessages.nonEmpty) { + baseRequest("systemInstruction") = ujson.Obj( + "parts" -> ujson.Arr( + ujson.Obj("text" -> systemMessages.map(_.content).mkString("\n\n")) + ) + ) + } + // Add generation config if parameters are specified val generationConfig = ujson.Obj() var hasConfig = false @@ -120,12 +133,21 @@ class GeminiProvider(implicit ec: ExecutionContext) extends BaseHttpProvider { if (isThinking) { val thinkingObj = ujson.Obj() - // Map reasoning effort or set budget - request.thinkingConfig.flatMap(_.budgetTokens).foreach { budget => - thinkingObj("thinkingBudget") = budget - } - request.thinkingConfig.flatMap(_.reasoningEffort).foreach { effort => - thinkingObj("thinkingLevel") = effort.toUpperCase + // Map reasoning effort or set budget. + // + // Gemini 3+ steers depth with thinkingLevel and returns a 400 when + // thinkingLevel and thinkingBudget arrive together; it still takes a lone + // thinkingBudget. Gemini 2.5 only knows thinkingBudget. So the effort wins + // on 3+, and is dropped on 2.5. + val effort = request.thinkingConfig.flatMap(_.reasoningEffort) + val budget = request.thinkingConfig.flatMap(_.budgetTokens) + if (GeminiProvider.usesThinkingLevel(request.model)) { + effort match { + case Some(e) => thinkingObj("thinkingLevel") = GeminiProvider.thinkingLevel(e) + case None => budget.foreach(b => thinkingObj("thinkingBudget") = b) + } + } else { + budget.foreach(b => thinkingObj("thinkingBudget") = b) } generationConfig("thinkingConfig") = thinkingObj hasConfig = true @@ -251,3 +273,27 @@ class GeminiProvider(implicit ec: ExecutionContext) extends BaseHttpProvider { } } } + +object GeminiProvider { + + private val Generation = """(?:^|/)gemini-(\d+)""".r + + /** + * Whether a model takes thinkingLevel (Gemini 3 and later) rather than only + * thinkingBudget (2.5 and earlier). Unrecognized names count as newer, the + * direction new models move. + */ + def usesThinkingLevel(model: String): Boolean = + Generation.findFirstMatchIn(model.toLowerCase).forall(_.group(1).toInt >= 3) + + /** + * Map the extension's reasoning_effort onto Gemini's thinkingLevel. Gemini + * has MINIMAL where the extension has "none", and nothing above HIGH. + */ + def thinkingLevel(effort: String): String = + effort.toLowerCase.trim match { + case "none" => "MINIMAL" + case "xhigh" | "max" => "HIGH" + case other => other.toUpperCase + } +} diff --git a/src/main/providers/GroqProvider.scala b/src/main/providers/GroqProvider.scala index 3570658..d856318 100644 --- a/src/main/providers/GroqProvider.scala +++ b/src/main/providers/GroqProvider.scala @@ -24,7 +24,7 @@ class GroqProvider(implicit ec: ExecutionContext) extends OpenAICompatibleProvid override def providerName: String = "groq" - override def defaultModel: String = "openai/gpt-oss-20b" + override def defaultModel: String = ModelRegistry.defaultModel("groq") override protected def defaultBaseUrl: String = "https://api.groq.com/openai/v1" diff --git a/src/main/providers/LLMProvider.scala b/src/main/providers/LLMProvider.scala index 5c7e2c3..642f9fc 100644 --- a/src/main/providers/LLMProvider.scala +++ b/src/main/providers/LLMProvider.scala @@ -58,6 +58,12 @@ trait LLMProvider { */ def setConfig(key: String, value: String): Unit + /** + * Set a configuration parameter that holds a seeded default rather than a + * value the user chose. Providers that treat the two alike need not override. + */ + def setDefaultConfig(key: String, value: String): Unit = setConfig(key, value) + /** * Get a configuration parameter value * diff --git a/src/main/providers/ModelRegistry.scala b/src/main/providers/ModelRegistry.scala index 1696be1..6e68769 100644 --- a/src/main/providers/ModelRegistry.scala +++ b/src/main/providers/ModelRegistry.scala @@ -12,9 +12,13 @@ import scala.util.{Try, Success, Failure} * - Model validation * - Custom model tracking * - * Models are loaded from: + * Models are loaded from, lowest layer first: * 1. Bundled YAML config in JAR resources (/config/models.yaml) - * 2. Optional override config (models-override.yaml) in model directory + * 2. Optional user catalog (~/.netlogo-llm/models.yaml, see ModelRegistryLoader.userCatalogPath) + * 3. Optional override config (models-override.yaml) in model directory + * + * Any layer may name a provider's default model with `default:`; without one + * the ProviderDescriptor default in ProviderRegistrations.scala applies. */ object ModelRegistry { @@ -23,28 +27,58 @@ object ModelRegistry { // Mutable state for dynamic loading (access must be synchronized) private var providers: Map[String, ProviderModels] = Map.empty + // Bundled (or fallback) catalog with the user catalog merged in, kept apart so + // each model-directory override is applied to it fresh + private var bundled: Map[String, ProviderModels] = Map.empty + // The model-directory override currently applied on top of `bundled` + private var overrideLayer: Map[String, ProviderModels] = Map.empty private var isInitialized: Boolean = false private var modelDirLoaded: Option[String] = None private var overrideLoadMessage: Option[String] = None + private var userWarning: Option[String] = None + + /** + * Catalog used when models.yaml cannot be loaded: each registered + * provider's descriptor default and nothing else. + * + * Deliberately no hand-kept model list here — a list in Scala goes stale + * without anyone noticing, which is what #64 removed. + */ + private[llm] def descriptorFallback(): Map[String, ProviderModels] = { + ensureDescriptors() + ProviderRegistry.allNames.toSeq.flatMap(ProviderRegistry.get).map { d => + d.name -> ProviderModels(Set(d.defaultModel), isCustom = false, default = Some(d.defaultModel)) + }.toMap + } - // Fallback config in case YAML loading fails (minimal set for stability). - // - // Every provider's defaultModel must appear here as well as in models.yaml. - // The drift guard in ProviderDefaultsSpec checks descriptors against the - // LOADED registry, so a stale entry here survives it — which is how the - // retired models #62 removed elsewhere lingered in this map. - private[llm] val FALLBACK_CONFIG: Map[String, ProviderModels] = Map( - "openai" -> ProviderModels(Set("gpt-4o", "gpt-4o-mini", "gpt-4", "gpt-3.5-turbo"), isCustom = false), - "anthropic" -> ProviderModels(Set( - "claude-opus-5", "claude-sonnet-5", - "claude-haiku-4-5-20251001", "claude-opus-4-5-20251101" - ), isCustom = false), - "gemini" -> ProviderModels(Set("gemini-2.5-pro", "gemini-2.5-flash", "gemini-3-pro-preview"), isCustom = false), - "ollama" -> ProviderModels(Set("llama3.2:3b", "llama3.2:1b", "mistral", "phi4"), isCustom = false), - "openrouter" -> ProviderModels(Set("openai/gpt-4o", "openai/gpt-4o-mini", "anthropic/claude-3.5-sonnet", "deepseek/deepseek-r1"), isCustom = false), - "together" -> ProviderModels(Set("meta-llama/Llama-3.3-70B-Instruct-Turbo", "deepseek-ai/DeepSeek-R1", "Qwen/Qwen2.5-72B-Instruct-Turbo"), isCustom = false), - "groq" -> ProviderModels(Set("openai/gpt-oss-20b", "openai/gpt-oss-120b", "groq/compound-mini"), isCustom = false) - ) + // Descriptors are normally registered by LLMExtension.load(), but the + // registry can be read before that (ConfigStore.withDefaults runs in the + // extension's constructor). registerAll() is idempotent. + private def ensureDescriptors(): Unit = + if (!ProviderRegistry.isInitialized) ProviderRegistrations.registerAll() + + /** + * Bundled catalog (or the descriptor fallback) with the user catalog merged + * over it. A broken user file is reported and skipped. + */ + private def loadBase(): (Map[String, ProviderModels], Option[String]) = { + val base = ModelRegistryLoader.loadBundledConfig() match { + case Success(config) => config + case Failure(error) => + System.err.println(s"WARNING: Failed to load bundled model config: ${error.getMessage}") + System.err.println("WARNING: Using each provider's default model only") + descriptorFallback() + } + + val userPath = ModelRegistryLoader.userCatalogPath + ModelRegistryLoader.loadUserConfig(userPath) match { + case Success(user) => (ModelRegistryLoader.mergeUserConfig(base, user), None) + case Failure(error) => + val warning = s"WARNING: Ignoring user model catalog $userPath: ${error.getMessage}" + System.err.println(warning) + (base, Some(warning)) + } + } /** * Initialize the registry by loading bundled configuration @@ -55,24 +89,36 @@ object ModelRegistry { def init(): Unit = lock.synchronized { if (isInitialized) return - ModelRegistryLoader.loadBundledConfig() match { - case Success(config) => - providers = config - isInitialized = true + val (base, warning) = loadBase() + bundled = base + providers = base + userWarning = warning + isInitialized = true + } - case Failure(error) => - // Fall back to hardcoded minimal config if YAML loading fails - System.err.println(s"WARNING: Failed to load bundled model config: ${error.getMessage}") - System.err.println("WARNING: Using fallback hardcoded model registry") - providers = FALLBACK_CONFIG - isInitialized = true - } + /** + * Re-read the bundled catalog and the user catalog, keeping any + * model-directory override in effect on top. + * + * Swaps the catalog in one step, so readers never see an empty registry. + */ + private[llm] def reload(): Unit = lock.synchronized { + val (base, warning) = loadBase() + bundled = base + providers = if (overrideLayer.nonEmpty) ModelRegistryLoader.mergeConfigs(base, overrideLayer) else base + userWarning = warning + isInitialized = true } + /** The warning printed for a broken user catalog at the last load, if any. */ + private[llm] def userCatalogWarning: Option[String] = lock.synchronized(userWarning) + /** * Load override configuration from a model directory * * Merges override config with bundled config. Override takes precedence. + * Always rebuilds from the bundled catalog, so an override from a previously + * loaded directory never survives a load from a directory without one. * Returns a notification message if custom models were loaded. * * @param modelDir Directory containing models-override.yaml @@ -84,21 +130,41 @@ object ModelRegistry { ModelRegistryLoader.loadOverrideConfig(modelDir) match { case Success((overrideConfig, customCount)) => if (overrideConfig.nonEmpty) { - providers = ModelRegistryLoader.mergeConfigs(providers, overrideConfig) + providers = ModelRegistryLoader.mergeConfigs(bundled, overrideConfig) + overrideLayer = overrideConfig modelDirLoaded = Some(modelDir) val message = s"Loaded $customCount custom models from override config" overrideLoadMessage = Some(message) Some(message) } else { + clearOverrideState() None } case Failure(error) => System.err.println(s"WARNING: Failed to load override config: ${error.getMessage}") + clearOverrideState() None } } + /** + * Drop any loaded override and restore the bundled catalog + * + * Used when there is no model directory (e.g. an unsaved model). + */ + def clearOverride(): Unit = lock.synchronized { + ensureInitialized() + clearOverrideState() + } + + private def clearOverrideState(): Unit = { + providers = bundled + overrideLayer = Map.empty + modelDirLoaded = None + overrideLoadMessage = None + } + /** * Check if a specific model is from custom/override configuration * @@ -182,9 +248,12 @@ object ModelRegistry { */ def reset(): Unit = lock.synchronized { providers = Map.empty + bundled = Map.empty + overrideLayer = Map.empty isInitialized = false modelDirLoaded = None overrideLoadMessage = None + userWarning = None } /** @@ -211,14 +280,32 @@ object ModelRegistry { /** * Get default model for a provider. * - * Delegates to ProviderRegistry so defaults are defined in one place - * (ProviderRegistrations.scala) rather than duplicated here. + * The loaded catalog's `default:` for the provider (bundled, user catalog, + * or model-directory override — whichever layer set it last), falling back + * to the ProviderDescriptor default in ProviderRegistrations.scala. * * @param providerName Provider name (case-insensitive) * @return Default model name */ - def defaultModel(providerName: String): String = - ProviderRegistry.defaultModel(providerName) + def defaultModel(providerName: String): String = { + ensureInitialized() + defaultFrom(lock.synchronized(providers), providerName) + } + + /** + * Default model for a provider in a given catalog, else its descriptor default. + * + * @param catalog Provider catalog to read `default:` from + * @param providerName Provider name (case-insensitive) + * @return Default model name + */ + private[llm] def defaultFrom(catalog: Map[String, ProviderModels], providerName: String): String = { + val normalizedProvider = providerName.toLowerCase.trim + catalog.get(normalizedProvider).flatMap(_.default).getOrElse { + ensureDescriptors() + ProviderRegistry.defaultModel(normalizedProvider) + } + } /** * Check if a model is valid for a provider diff --git a/src/main/providers/OllamaProvider.scala b/src/main/providers/OllamaProvider.scala index 1a024a4..cc6af0c 100644 --- a/src/main/providers/OllamaProvider.scala +++ b/src/main/providers/OllamaProvider.scala @@ -25,6 +25,7 @@ class OllamaProvider(implicit ec: ExecutionContext) extends BaseHttpProvider { override protected def apiKeyConfigKey: String = ConfigStore.API_KEY override protected def defaultMaxTokens: String = "2048" override protected def requiresApiKey: Boolean = false + override protected def unreachableHint: String = "Is the Ollama server running?" override protected def buildApiUrl(baseUrl: String): Uri = { uri"$baseUrl/api/chat" @@ -131,8 +132,7 @@ class OllamaProvider(implicit ec: ExecutionContext) extends BaseHttpProvider { * Check if Ollama server is accessible */ def checkServerConnection(): Future[Boolean] = { - val baseUrl = configStore.get(ConfigStore.OLLAMA_BASE_URL) - .getOrElse("http://localhost:11434") + val baseUrl = resolvedBaseUrl val apiUrl = uri"$baseUrl/api/tags" val httpRequest = basicRequest.get(apiUrl) @@ -152,8 +152,7 @@ class OllamaProvider(implicit ec: ExecutionContext) extends BaseHttpProvider { * @return Future containing set of installed model names */ def listInstalledModels(): Future[Set[String]] = { - val baseUrl = configStore.get(ConfigStore.OLLAMA_BASE_URL) - .getOrElse("http://localhost:11434") + val baseUrl = resolvedBaseUrl val apiUrl = uri"$baseUrl/api/tags" val httpRequest = basicRequest.get(apiUrl) diff --git a/src/main/providers/OpenAICompatibleProvider.scala b/src/main/providers/OpenAICompatibleProvider.scala index dfe9cb2..a033464 100644 --- a/src/main/providers/OpenAICompatibleProvider.scala +++ b/src/main/providers/OpenAICompatibleProvider.scala @@ -9,6 +9,16 @@ import sttp.model.Uri import ujson._ import scala.concurrent.ExecutionContext +object OpenAICompatibleProvider { + /** + * Smallest completion budget sent to a model in reasoning format when + * max_tokens is still the provider default. Reasoning tokens count against + * max_completion_tokens, so the plain-chat default leaves too little room + * for the visible answer. + */ + val MinReasoningCompletionTokens: Int = 4000 +} + /** * Abstract base for providers that use the OpenAI Chat Completions wire format. * @@ -73,7 +83,7 @@ abstract class OpenAICompatibleProvider(implicit ec: ExecutionContext) extends B if (isReasoning) { // Reasoning models: use max_completion_tokens, no temperature request.maxTokens.foreach { maxTokens => - baseRequest("max_completion_tokens") = maxTokens + baseRequest("max_completion_tokens") = reasoningCompletionBudget(maxTokens) } // Apply provider-specific reasoning fields applyReasoningFields(baseRequest, request) @@ -92,6 +102,19 @@ abstract class OpenAICompatibleProvider(implicit ec: ExecutionContext) extends B baseRequest } + /** + * The max_completion_tokens for a reasoning-format request. A max_tokens + * the user never set (still the seeded default) is raised to + * MinReasoningCompletionTokens; any value the user set, even one equal to + * the default, is the modeler's choice and is sent unchanged. + */ + protected def reasoningCompletionBudget(maxTokens: Int): Int = { + val seededDefault = configStore.isDefault(ConfigStore.MAX_TOKENS) && + configStore.get(ConfigStore.MAX_TOKENS).flatMap(_.trim.toIntOption).contains(maxTokens) + if (seededDefault) math.max(maxTokens, OpenAICompatibleProvider.MinReasoningCompletionTokens) + else maxTokens + } + /** * Add the OpenAI `response_format` field. * @@ -154,13 +177,18 @@ abstract class OpenAICompatibleProvider(implicit ec: ExecutionContext) extends B val choices = parsed("choices").arr.zipWithIndex.map { case (choice, index) => val message = choice("message") val role = message("role").str - val content = message("content").str - val finishReason = choice("finish_reason").str + // A refusal arrives as content: null plus a "refusal" string, and + // finish_reason may be null too. Read both null-safely so the usage + // below is still parsed and the refusal reaches the caller. + val content = message.obj.get("content").collect { case ujson.Str(s) => s }.getOrElse("") + val finishReason = choice.obj.get("finish_reason").collect { case ujson.Str(s) => s }.getOrElse("") + val refusal = message.obj.get("refusal").collect { case ujson.Str(s) => s } Choice( index = index, message = ChatMessage(role, content), - finishReason = finishReason + finishReason = finishReason, + refusal = refusal ) }.toArray diff --git a/src/main/providers/OpenAIProvider.scala b/src/main/providers/OpenAIProvider.scala index 326c0b9..b8b37f7 100644 --- a/src/main/providers/OpenAIProvider.scala +++ b/src/main/providers/OpenAIProvider.scala @@ -14,7 +14,7 @@ class OpenAIProvider(implicit ec: ExecutionContext) extends OpenAICompatibleProv override def providerName: String = "openai" - override def defaultModel: String = "gpt-4o-mini" + override def defaultModel: String = ModelRegistry.defaultModel("openai") override protected def defaultBaseUrl: String = "https://api.openai.com/v1" diff --git a/src/main/providers/OpenRouterProvider.scala b/src/main/providers/OpenRouterProvider.scala index 6f6dd75..d4bcf2a 100644 --- a/src/main/providers/OpenRouterProvider.scala +++ b/src/main/providers/OpenRouterProvider.scala @@ -12,7 +12,7 @@ import scala.concurrent.ExecutionContext * Google, Meta, DeepSeek, etc.) using an OpenAI-compatible API format. * * Key differences from direct OpenAI: - * - Model names are vendor-prefixed (e.g. "openai/gpt-4o", "anthropic/claude-3.5-sonnet") + * - Model names are vendor-prefixed (e.g. "openai/gpt-6-luna", "anthropic/claude-haiku-5.5") * - Extra headers: HTTP-Referer and X-Title for attribution * - Reasoning uses { reasoning: { effort: "..." } } instead of top-level reasoning_effort * - Thinking text is exposed in the response "reasoning" field @@ -21,7 +21,7 @@ class OpenRouterProvider(implicit ec: ExecutionContext) extends OpenAICompatible override def providerName: String = "openrouter" - override def defaultModel: String = "openai/gpt-4o-mini" + override def defaultModel: String = ModelRegistry.defaultModel("openrouter") override protected def defaultBaseUrl: String = "https://openrouter.ai/api/v1" diff --git a/src/main/providers/ProviderFactory.scala b/src/main/providers/ProviderFactory.scala index c722df1..740e68e 100644 --- a/src/main/providers/ProviderFactory.scala +++ b/src/main/providers/ProviderFactory.scala @@ -47,8 +47,22 @@ object ProviderFactory { */ def createProviderFromConfig(configStore: ConfigStore)(implicit ec: ExecutionContext): Try[LLMProvider] = { val providerName = configStore.getOrElse(ConfigStore.PROVIDER, ConfigStore.DEFAULT_PROVIDER) - val config = configStore.toMap - createProvider(providerName, config) + createProvider(providerName).map { provider => + copyConfig(configStore, provider) + provider + } + } + + /** + * Copy every entry of a ConfigStore into a provider, keeping seeded defaults + * marked as defaults so the provider can tell them from values the user set. + */ + def copyConfig(configStore: ConfigStore, provider: LLMProvider): Unit = { + val defaults = configStore.defaultKeys + configStore.toMap.foreach { case (key, value) => + if (defaults.contains(key)) provider.setDefaultConfig(key, value) + else provider.setConfig(key, value) + } } /** @@ -140,14 +154,41 @@ object ProviderFactory { } } + /** + * Apply the config changes llm:set-provider makes when switching to `providerName`. + * + * The legacy generic base_url belongs to the provider that was active when it + * was set. On a switch to a different provider it is moved to the previous + * provider's own key (unless the user already set that key) and cleared, so + * it keeps applying to that provider but never leaks to the new one. A + * base_url set again after the switch applies to the new provider. + */ + def applyProviderSwitch(configStore: ConfigStore, providerName: String): Unit = { + val previous = configStore.getOrElse(ConfigStore.PROVIDER, ConfigStore.DEFAULT_PROVIDER).toLowerCase.trim + if (previous != providerName) { + configStore.remove(ConfigStore.BASE_URL).foreach { url => + val previousKey = ConfigStore.getProviderBaseUrlName(previous) + if (previousKey != ConfigStore.BASE_URL && !configStore.contains(previousKey)) { + configStore.set(previousKey, url) + } + } + } + val defaults = getDefaultConfig(providerName) + defaults.foreach { case (key, value) => + if (!configStore.contains(key) || key == ConfigStore.MODEL) { + configStore.setDefault(key, value) + } + } + configStore.set(ConfigStore.PROVIDER, providerName) + } + /** * Get provider-specific default configuration. */ def getDefaultConfig(providerName: String): Map[String, String] = { ProviderRegistry.get(providerName.toLowerCase.trim) match { case Some(desc) => Map( - ConfigStore.MODEL -> desc.defaultModel, - ConfigStore.BASE_URL -> desc.defaultBaseUrl, + ConfigStore.MODEL -> ModelRegistry.defaultModel(desc.name), ConfigStore.TEMPERATURE -> ConfigStore.DEFAULT_TEMPERATURE, ConfigStore.MAX_TOKENS -> desc.defaultMaxTokens ) diff --git a/src/main/providers/ProviderRegistrations.scala b/src/main/providers/ProviderRegistrations.scala index 680326c..b309121 100644 --- a/src/main/providers/ProviderRegistrations.scala +++ b/src/main/providers/ProviderRegistrations.scala @@ -20,7 +20,7 @@ object ProviderRegistrations { apiKeyConfigKey = "openai_api_key", baseUrlConfigKey = "openai_base_url", defaultBaseUrl = "https://api.openai.com/v1", - defaultModel = "gpt-4o-mini", + defaultModel = "gpt-6-luna", defaultMaxTokens = "1000", requiresApiKey = true, apiKeyPrefix = Some("sk-"), @@ -48,7 +48,7 @@ object ProviderRegistrations { apiKeyConfigKey = "anthropic_api_key", baseUrlConfigKey = "anthropic_base_url", defaultBaseUrl = "https://api.anthropic.com/v1", - defaultModel = "claude-haiku-4-5-20251001", + defaultModel = "claude-haiku-5-5", defaultMaxTokens = "4000", requiresApiKey = true, apiKeyPrefix = None, @@ -76,7 +76,7 @@ object ProviderRegistrations { apiKeyConfigKey = "gemini_api_key", baseUrlConfigKey = "gemini_base_url", defaultBaseUrl = "https://generativelanguage.googleapis.com/v1beta", - defaultModel = "gemini-2.5-flash", + defaultModel = "gemini-3.5-flash-lite", defaultMaxTokens = "2048", requiresApiKey = true, apiKeyPrefix = None, @@ -104,7 +104,7 @@ object ProviderRegistrations { apiKeyConfigKey = "ollama_api_key", baseUrlConfigKey = "ollama_base_url", defaultBaseUrl = "http://localhost:11434", - defaultModel = "llama3.2:3b", + defaultModel = "llama3.2", defaultMaxTokens = "2048", requiresApiKey = false, apiKeyPrefix = None, @@ -122,7 +122,7 @@ object ProviderRegistrations { | - Or start Ollama app (it runs in background) | |3. Pull a model: - | - Run: ollama pull llama3.2:3b + | - Run: ollama pull llama3.2 | - Or try: ollama pull deepseek-r1:1.5b (smaller) | |4. Verify installation: @@ -142,7 +142,7 @@ object ProviderRegistrations { apiKeyConfigKey = "openrouter_api_key", baseUrlConfigKey = "openrouter_base_url", defaultBaseUrl = "https://openrouter.ai/api/v1", - defaultModel = "openai/gpt-4o-mini", + defaultModel = "openai/gpt-oss-20b", defaultMaxTokens = "1000", requiresApiKey = true, apiKeyPrefix = None, @@ -160,9 +160,9 @@ object ProviderRegistrations { | - Or at runtime: llm:set-api-key "sk-or-your-key-here" | |3. Set a model (vendor-prefixed): - | - llm:set-model "openai/gpt-4o" - | - llm:set-model "anthropic/claude-3.5-sonnet" - | - llm:set-model "deepseek/deepseek-r1" + | - llm:set-model "openai/gpt-oss-20b" + | - llm:set-model "anthropic/claude-haiku-5.5" + | - llm:set-model "google/gemini-3.5-flash-lite" | |4. Verify: | - Check llm:provider-status for "has-key: true" @@ -177,7 +177,7 @@ object ProviderRegistrations { apiKeyConfigKey = "together_api_key", baseUrlConfigKey = "together_base_url", defaultBaseUrl = "https://api.together.xyz/v1", - defaultModel = "meta-llama/Llama-3.3-70B-Instruct-Turbo", + defaultModel = "Qwen/Qwen3.8-Flash", defaultMaxTokens = "1000", requiresApiKey = true, apiKeyPrefix = None, @@ -195,9 +195,9 @@ object ProviderRegistrations { | - Or at runtime: llm:set-api-key "your-key-here" | |3. Set a model (vendor-prefixed): - | - llm:set-model "meta-llama/Llama-3.3-70B-Instruct-Turbo" - | - llm:set-model "deepseek-ai/DeepSeek-R1" - | - llm:set-model "Qwen/Qwen2.5-72B-Instruct-Turbo" + | - llm:set-model "Qwen/Qwen3.8-Flash" + | - llm:set-model "openai/gpt-oss-120b" + | - llm:set-model "deepseek-ai/DeepSeek-V4.1-Flash" | |4. Verify: | - Check llm:provider-status for "has-key: true" @@ -243,7 +243,7 @@ object ProviderRegistrations { |3. Set a model: | - llm:set-model "openai/gpt-oss-20b" | - llm:set-model "openai/gpt-oss-120b" - | - llm:set-model "groq/compound-mini" + | - llm:set-model "qwen/qwen3.8-27b" | |4. Verify: | - Check llm:provider-status for "has-key: true" @@ -252,5 +252,45 @@ object ProviderRegistrations { |Browse models: https://console.groq.com/docs/models""".stripMargin, factory = ec => new GroqProvider()(using ec) )) + + ProviderRegistry.register(ProviderDescriptor( + name = "typesafe", + displayName = "TypeSafe (Jev)", + apiKeyConfigKey = "typesafe_api_key", + baseUrlConfigKey = "typesafe_base_url", + defaultBaseUrl = "https://api.typesafe.ai", + defaultModel = "jev-latest", + defaultMaxTokens = "1000", + requiresApiKey = true, + apiKeyPrefix = None, + readinessCheck = ReadinessCheck.ApiKey, + // Jev returns typed decisions with probabilities, never text or reasoning. + exposesThinking = false, + helpText = + """TypeSafe (Jev) Setup Instructions: + | + |Jev answers typed questions with calibrated probabilities instead of + |writing text. Use llm:decide, llm:decide-all or llm:choose with it; + |llm:chat and the other text primitives are not available. + | + |1. Get an API key: + | - Visit https://console.typesafe.ai/ + | - Create a new API key + | + |2. Set the key: + | - In config file: typesafe_api_key=your-key-here + | - Or at runtime: llm:set-api-key "your-key-here" + | + |3. Set a model (optional): + | - llm:set-model "jev-latest" (stable, the default) + | - llm:set-model "jev-preview" (newest release) + | - llm:set-model "jev-1.14.0" (pinned version) + | + |4. Verify: + | - Check llm:provider-status for "has-key: true" + | + |Docs: https://docs.typesafe.ai""".stripMargin, + factory = ec => new TypeSafeProvider()(using ec) + )) } } diff --git a/src/main/providers/ReasoningModelDetector.scala b/src/main/providers/ReasoningModelDetector.scala index 143a397..719606f 100644 --- a/src/main/providers/ReasoningModelDetector.scala +++ b/src/main/providers/ReasoningModelDetector.scala @@ -20,6 +20,9 @@ object ReasoningModelDetector { * Only returns true for models that return HTTP 400 without special handling: * - OpenAI o1, o3, o4 series: must strip temperature, use max_completion_tokens, * convert system->developer role + * - OpenAI GPT-5.x and GPT-6.x: the same reasoning models under a new name + * (no new o-series is coming). The `-chat` variants were non-reasoning and + * are excluded. * * All other providers return false — they don't break, they just won't * activate thinking without explicit config. @@ -27,19 +30,22 @@ object ReasoningModelDetector { def requiresReasoningFormat(provider: String, model: String): Boolean = { provider.toLowerCase.trim match { case "openai" => - val m = model.toLowerCase - m.startsWith("o1") || m.startsWith("o3") || m.startsWith("o4") + isOpenAIReasoningModel(model) case "openrouter" => - // Vendor-prefixed: check if the model after the prefix is an OpenAI o-series + // Vendor-prefixed: check if the model after the prefix is an OpenAI reasoning model val m = model.toLowerCase - if (m.startsWith("openai/")) { - val modelName = m.stripPrefix("openai/") - modelName.startsWith("o1") || modelName.startsWith("o3") || modelName.startsWith("o4") - } else false + if (m.startsWith("openai/")) isOpenAIReasoningModel(m.stripPrefix("openai/")) + else false case _ => false } } + private def isOpenAIReasoningModel(model: String): Boolean = { + val m = model.toLowerCase + m.startsWith("o1") || m.startsWith("o3") || m.startsWith("o4") || + ((m.startsWith("gpt-5") || m.startsWith("gpt-6")) && !m.contains("-chat")) + } + /** * Resolve the effective ThinkingConfig for a request. * @@ -90,8 +96,7 @@ object ReasoningModelDetector { def isReasoningModel(provider: String, model: String): Boolean = { provider.toLowerCase.trim match { case "openai" => - val m = model.toLowerCase - m.startsWith("o1") || m.startsWith("o3") || m.startsWith("o4") + isOpenAIReasoningModel(model) case "anthropic" => model.contains("claude-3-7") || model.contains("claude-4") case "gemini" => @@ -102,7 +107,7 @@ object ReasoningModelDetector { case "openrouter" => // Vendor-prefixed model names: detect reasoning models across vendors val m = model.toLowerCase - m.startsWith("openai/o1") || m.startsWith("openai/o3") || m.startsWith("openai/o4") || + (m.startsWith("openai/") && isOpenAIReasoningModel(m.stripPrefix("openai/"))) || m.contains("claude-3-7") || m.contains("claude-4") || m.contains("deepseek-r1") || m.contains("qwq") case "together" => diff --git a/src/main/providers/TogetherProvider.scala b/src/main/providers/TogetherProvider.scala index d3df53e..9406e18 100644 --- a/src/main/providers/TogetherProvider.scala +++ b/src/main/providers/TogetherProvider.scala @@ -12,7 +12,7 @@ import scala.concurrent.ExecutionContext * Qwen, Gemma, Mistral, etc.) via an OpenAI-compatible API. * * Key differences from direct OpenAI: - * - Model names are vendor-prefixed (e.g. "meta-llama/Llama-3.3-70B-Instruct-Turbo") + * - Model names are vendor-prefixed (e.g. "Qwen/Qwen3.8-Flash", "deepseek-ai/DeepSeek-V4.1-Flash") * - Hybrid reasoning models use { reasoning: { enabled: true } } * - DeepSeek-R1 embeds thinking in ... tags within content */ @@ -20,7 +20,7 @@ class TogetherProvider(implicit ec: ExecutionContext) extends OpenAICompatiblePr override def providerName: String = "together" - override def defaultModel: String = "meta-llama/Llama-3.3-70B-Instruct-Turbo" + override def defaultModel: String = ModelRegistry.defaultModel("together") override protected def defaultBaseUrl: String = "https://api.together.xyz/v1" diff --git a/src/main/providers/TypeSafeProvider.scala b/src/main/providers/TypeSafeProvider.scala new file mode 100644 index 0000000..8294e3a --- /dev/null +++ b/src/main/providers/TypeSafeProvider.scala @@ -0,0 +1,157 @@ +// ABOUTME: TypeSafe provider — the Jev System One API, which answers typed questions with probabilities +// ABOUTME: Implements DecisionProvider (choice, score, noul) over BaseHttpProvider's retry and throttle path +package org.nlogo.extensions.llm.providers + +import org.nlogo.extensions.llm.config.ConfigStore +import org.nlogo.extensions.llm.models.{ChatMessage, ChatRequest, ChatResponse, Choice, DecisionQuestion, DecisionResult, Usage} +import sttp.client4._ +import sttp.model.Uri +import scala.concurrent.{ExecutionContext, Future} +import scala.util.{Failure, Success} + +object TypeSafeProvider { + /** Raised by every free-text primitive while TypeSafe is the active provider. */ + val TextUnsupportedMessage: String = + "typesafe (Jev) returns decisions, not text; use llm:decide, llm:decide-all or llm:choose" + + /** USD per input token for every current Jev model ($0.042 per million); output tokens are free. */ + val InputTokenPriceUsd: Double = 0.042e-6 + + /** Name of the single question each request asks; answers come back under it. */ + private[providers] val QuestionId: String = "decision" +} + +/** + * TypeSafe provider implementation. + * + * Jev, TypeSafe's System One model, does not generate text. A request sends + * `state` (here, the prompt) and named, typed questions; the reply is a typed + * answer per question with calibrated probabilities. Choice, Score and Noul + * questions map onto DecisionQuestion's Choice, Score and YesNo. + * + * Requests go through BaseHttpProvider.executeWithRetry, so the throttle, the + * 429/529 retry, friendly HTTP errors and latency stamping all apply as for + * chat. That path carries a ChatResponse, so parseProviderResponse puts the raw + * reply body in the message content and decide() reads the answer from it. + * + * Docs: https://docs.typesafe.ai/api/system-one.md + */ +class TypeSafeProvider(implicit ec: ExecutionContext) extends BaseHttpProvider with DecisionProvider { + import TypeSafeProvider._ + + override def providerName: String = "typesafe" + + override def defaultModel: String = "jev-latest" + + override protected def defaultBaseUrl: String = "https://api.typesafe.ai" + + override protected def baseUrlConfigKey: String = "typesafe_base_url" + + override protected def apiKeyConfigKey: String = "typesafe_api_key" + + // Unused: Jev takes no max-token setting. BaseHttpProvider stores a default. + override protected def defaultMaxTokens: String = "1000" + + override protected def requiresApiKey: Boolean = true + + override def textUnsupportedMessage: Option[String] = Some(TextUnsupportedMessage) + + override protected def buildApiUrl(baseUrl: String): Uri = uri"$baseUrl/v1/systemone" + + override protected def buildHeaders(apiKey: Option[String]): Map[String, String] = + Map("Content-Type" -> "application/json") ++ + apiKey.map(key => "Authorization" -> s"Bearer $key") + + /** Free-text chat is not offered; fails before any request is sent. */ + override def chat(request: ChatRequest): Future[ChatResponse] = + Future.failed(new UnsupportedOperationException(TextUnsupportedMessage)) + + override protected def createProviderRequest(request: ChatRequest): ujson.Value = + throw new UnsupportedOperationException(TextUnsupportedMessage) + + /** Wrap the raw body so it can travel the shared retry path; decide() parses it. */ + override protected def parseProviderResponse(responseBody: String, model: String): ChatResponse = { + val usage = scala.util.Try(ujson.read(responseBody)).toOption.flatMap(_.obj.get("usage")).map { u => + val in = Usage.longField(u, "input_tokens").getOrElse(0L) + val out = Usage.longField(u, "output_tokens").getOrElse(0L) + Usage(in, out, in + out, cost = Some(in * InputTokenPriceUsd)) + } + ChatResponse( + id = "typesafe-systemone", + created = System.currentTimeMillis() / 1000, + model = model, + choices = Array(Choice(0, ChatMessage.assistant(responseBody), "stop")), + usage = usage + ) + } + + /** The System One request body for one question about `prompt`. */ + private[providers] def decisionRequestBody(prompt: String, question: DecisionQuestion, model: String): ujson.Obj = { + val q = question match { + case DecisionQuestion.Choice(options) => + ujson.Obj( + "type" -> "choice", + "instructions" -> "Choose the option that best fits the state.", + // An option without a description is interpreted by its name alone. + "criteria" -> ujson.Obj.from(options.map(o => o -> ujson.Null)) + ) + case DecisionQuestion.Score(levels) => + ujson.Obj( + "type" -> "score", + "instructions" -> "Rate the state on these levels.", + "criteria" -> ujson.Arr.from(levels.map(ujson.Str(_))) + ) + case DecisionQuestion.YesNo => + ujson.Obj( + "type" -> "noul", + "instructions" -> "Answer the question or judge the statement in the state: yes (true) or no (false)." + ) + } + ujson.Obj("model" -> model, "state" -> prompt, "questions" -> ujson.Obj(QuestionId -> q)) + } + + /** Read the answer to `question` out of a System One reply body. */ + private[providers] def parseDecision(body: String, question: DecisionQuestion): DecisionResult = { + val answer = + try ujson.read(body)("answers")(QuestionId) + catch { + case e: Exception => + throw new RuntimeException(s"$displayName reply had no answer to the question: ${e.getMessage}") + } + def probabilities: Map[String, Double] = + answer.obj.get("probabilities").map(_.obj.view.mapValues(_.num).toMap).getOrElse(Map.empty) + + try question match { + case DecisionQuestion.Choice(options) => + DecisionResult.choice(options, answer("choice").str, probabilities) + case DecisionQuestion.Score(levels) => + val probs = probabilities + DecisionResult.score( + levels, + answer("score").num, + answer.obj.get("confidence").map(_.num).getOrElse(0.0), + levels.indices.map(i => probs.getOrElse(i.toString, 0.0)) + ) + case DecisionQuestion.YesNo => + DecisionResult.yesNo(answer("noul").num) + } catch { + case e: Exception => + throw new RuntimeException(s"$displayName reply could not be read: ${e.getMessage}. Reply: ${body.take(300)}") + } + } + + override def decide(prompt: String, question: DecisionQuestion): Future[(DecisionResult, Usage)] = + validateConfig() match { + case Failure(e) => Future.failed(e) + case Success(_) => + val apiKey = configStore.get(apiKeyConfigKey).orElse(configStore.get(ConfigStore.API_KEY)) + val model = configStore.getOrElse(ConfigStore.MODEL, defaultModel) + val httpRequest = basicRequest + .headers(buildHeaders(apiKey)) + .body(decisionRequestBody(prompt, question, model).toString()) + .post(buildApiUrl(resolvedBaseUrl)) + executeWithRetry(httpRequest, model).map { response => + (parseDecision(response.firstContent.getOrElse(""), question), response.usage.getOrElse(Usage.empty)) + } + } +} diff --git a/src/main/resources/config/models.yaml b/src/main/resources/config/models.yaml index 1603d32..6b68436 100644 --- a/src/main/resources/config/models.yaml +++ b/src/main/resources/config/models.yaml @@ -1,216 +1,242 @@ # NetLogo LLM Extension - Bundled Model Registry # This file defines the supported models for each LLM provider. # Only includes current/recommended models - legacy models removed. +# +# Each provider is either a plain list of model IDs, or a mapping with: +# default: the model used when no model is configured (list it in models +# here; in a user catalog or models-override.yaml a default that +# is not listed is added to the known models automatically) +# models: the list of known model IDs +# When a provider has no default:, the default in ProviderRegistrations.scala applies. +# +# The same format works in a user catalog at ~/.netlogo-llm/models.yaml +# (or the path in the llm.models.user.file system property / NETLOGO_LLM_MODELS +# env var), which adds models and can override default: without rebuilding. +# +# Catalog refreshed October 2026 against each provider's models and +# deprecations pages. Retired and retiring IDs are left out. openai: - # GPT-5 (latest flagship) - - gpt-5 - - gpt-5-mini - - gpt-5-nano - - gpt-5-chat - - # o-series reasoning models (latest) - - o3-deep-research - - o4-mini-deep-research - - o3-pro - - o3 - - o3-mini - - o4-mini - - # GPT-4.1 (current non-reasoning) - - gpt-4.1 - - gpt-4.1-mini - - gpt-4.1-nano - - # o1 series (previous reasoning) - - o1-pro - - o1 - - o1-mini # deprecated - - o1-preview # deprecated - - # GPT-4o (stable multimodal) - - gpt-4o - - gpt-4o-mini - - chatgpt-4o - - gpt-4o-search-preview - - gpt-4o-mini-search-preview - - gpt-3.5-turbo + default: gpt-6-luna + models: + # GPT-6 (reasoning; reasoning_effort none|low|medium|high|xhigh) + - gpt-6-luna + - gpt-6.1-sol + - gpt-6-sol + - gpt-6-astra + + # GPT-5.x (reasoning) + - gpt-5.6-luna + - gpt-5.6-terra + - gpt-5.6-sol + - gpt-5.5 + - gpt-5.4 + - gpt-5.4-mini + + # Deprecated, still served until 2027-01-14 + - gpt-4o-mini anthropic: - # Adaptive thinking (thinking.type "adaptive" + output_config.effort). - # These reject thinking.type "enabled" and any non-default temperature. - # IDs are dateless from the 4.6 generation on, and still pinned snapshots. - - claude-fable-5 - - claude-opus-5 - - claude-sonnet-5 - - claude-opus-4-8 - - claude-opus-4-7 - - # Adaptive thinking, extended thinking deprecated but still accepted - - claude-opus-4-6 - - claude-sonnet-4-6 - - # Extended thinking only (thinking.type "enabled" + budget_tokens). - # These reject "adaptive". - - claude-opus-4-5-20251101 - - claude-sonnet-4-5-20250929 - - claude-haiku-4-5-20251001 - - # Claude 4.1 / 4 (extended thinking) - - claude-opus-4-1-20250805 - - claude-sonnet-4-20250514 - - claude-opus-4-20250514 + default: claude-haiku-5-5 + models: + # Adaptive thinking (thinking.type "adaptive" + output_config.effort). + # These reject thinking.type "enabled" and any non-default temperature. + # IDs are dateless from the 4.6 generation on, and still pinned snapshots. + - claude-fable-5-1 + - claude-opus-5-5 + - claude-sonnet-5-5 + - claude-haiku-5-5 + - claude-fable-5 + - claude-opus-5 + - claude-sonnet-5 + - claude-opus-4-8 + - claude-opus-4-7 + + # Adaptive thinking, extended thinking deprecated but still accepted + - claude-opus-4-6 + - claude-sonnet-4-6 + + # Extended thinking only (thinking.type "enabled" + budget_tokens). + # These reject "adaptive". + - claude-opus-4-5-20251101 + - claude-haiku-4-5-20251001 gemini: - # Gemini 3 (latest preview) - - gemini-3-pro-preview - - gemini-3-pro-image-preview - - gemini-3-flash-preview - - # Gemini 2.5 (current production) - - gemini-2.5-pro - - gemini-2.5-pro-latest - - gemini-2.5-pro-preview-tts - - gemini-2.5-flash - - gemini-2.5-flash-latest - - gemini-2.5-flash-preview-09-2025 - - gemini-2.5-flash-image - - gemini-2.5-flash-native-audio-preview-12-2025 - - gemini-2.5-flash-preview-tts - - gemini-2.5-flash-lite - - gemini-2.5-flash-lite-latest - - gemini-2.5-flash-lite-preview-09-2025 + default: gemini-3.5-flash-lite + models: + # Gemini 3+ (thinking depth set with thinkingLevel) + - gemini-4-argon + - gemini-3.8-flash + - gemini-3.6-flash + - gemini-3.5-flash-lite + - gemini-3.1-pro-preview + - gemini-3-flash-preview + + # Gemini 2.5 (thinking depth set with thinkingBudget). + # Only open to projects that already used them. + - gemini-2.5-pro + - gemini-2.5-flash + - gemini-2.5-flash-lite openrouter: - # OpenAI models via OpenRouter - - openai/gpt-4o - - openai/gpt-4o-mini - - openai/gpt-4.1 - - openai/gpt-4.1-mini - - openai/o3-mini - - openai/o4-mini - - # Anthropic models via OpenRouter - - anthropic/claude-sonnet-4-20250514 - - anthropic/claude-3.5-sonnet - - anthropic/claude-3.5-haiku - - # Google models via OpenRouter - - google/gemini-2.5-flash - - google/gemini-2.5-pro - - # Meta Llama models via OpenRouter - - meta-llama/llama-3.3-70b-instruct - - meta-llama/llama-3.1-405b-instruct - - # DeepSeek models via OpenRouter - - deepseek/deepseek-r1 - - deepseek/deepseek-chat-v3-0324 - - # Qwen models via OpenRouter - - qwen/qwen-max + default: openai/gpt-oss-20b + models: + # OpenAI models via OpenRouter + - openai/gpt-oss-20b + - openai/gpt-oss-120b + - openai/gpt-6-luna + - openai/gpt-4o-mini -together: - # Meta Llama - - meta-llama/Llama-3.3-70B-Instruct-Turbo - - meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo - - meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo - - meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo + # Anthropic models via OpenRouter + - anthropic/claude-haiku-5.5 - # DeepSeek - - deepseek-ai/DeepSeek-R1 - - deepseek-ai/DeepSeek-V3 + # Google models via OpenRouter + - google/gemini-3.5-flash-lite + - google/gemini-3.8-flash + - google/gemini-2.5-flash + - google/gemini-2.5-pro - # Qwen - - Qwen/Qwen2.5-72B-Instruct-Turbo - - Qwen/Qwen2.5-7B-Instruct-Turbo + # Meta Llama models via OpenRouter + - meta-llama/llama-3.3-70b-instruct + - meta-llama/llama-3.1-8b-instruct - # Google Gemma - - google/gemma-3-27b-it + # DeepSeek models via OpenRouter + - deepseek/deepseek-v4.1-flash - # Mistral - - mistralai/Mistral-Small-24B-Instruct-2501 + # Qwen models via OpenRouter + - qwen/qwen3.8-flash + - qwen/qwen3.8-27b -groq: - # OpenAI open-weight models (reasoning, exposed via message.reasoning) - - openai/gpt-oss-120b - - openai/gpt-oss-20b - - openai/gpt-oss-safeguard-20b + # Mistral models via OpenRouter + - mistralai/mistral-small-2603 + + # OpenRouter's automatic router + - openrouter/auto - # Groq agentic systems (built-in tool use) - - groq/compound - - groq/compound-mini +together: + default: Qwen/Qwen3.8-Flash + models: + # Qwen + - Qwen/Qwen3.8-Flash + - Qwen/Qwen3.7-Plus + - Qwen/Qwen3.5-9B + + # OpenAI open-weight + - openai/gpt-oss-120b + + # DeepSeek + - deepseek-ai/DeepSeek-V4.1-Flash + + # Others + - zai-org/GLM-5.3-Flash + - nvidia/NVIDIA-Nemotron-3.5-Super-VL + - meta-models/Muse-Glimmer-30B + - MiniMaxAI/MiniMax-M3 + - moonshotai/Kimi-K3 - # Preview models - - qwen/qwen3.6-27b - - minimaxai/minimax-m2.7 +groq: + default: openai/gpt-oss-20b + models: + # OpenAI open-weight models (reasoning, exposed via message.reasoning) + - openai/gpt-oss-20b + - openai/gpt-oss-120b + + # Preview models + - qwen/qwen3.8-27b + - minimaxai/minimax-m2.7 + - meta-models/muse-spark-open + +typesafe: + # Jev decision models (llm:decide, llm:decide-all, llm:choose only) + default: jev-latest + models: + - jev-latest + - jev-preview + - jev-1.13.0 + - jev-1.14.0 ollama: - # Top reasoning models - - deepseek-r1:70b - - deepseek-r1:32b - - deepseek-r1:14b - - deepseek-r1:8b - - deepseek-r1:1.5b - - # Meta Llama 3 series (best all-round) - - llama3.3:70b - - llama3.3:8b - - llama3.2:3b - - llama3.2:1b - - llama3.1:405b - - llama3.1:70b - - llama3.1:8b - - # Alibaba Qwen series (multilingual) - - qwen2.5:72b - - qwen2.5:32b - - qwen2.5:14b - - qwen2.5:7b - - qwen2.5:3b - - qwen2.5:1.5b - - qwen2.5:0.5b - - # Coding specialists - - qwen2.5-coder:32b - - qwen2.5-coder:7b - - qwen2.5-coder:1.5b - - codellama:34b - - codellama:13b - - codellama:7b - - deepseek-coder-v2:16b - - deepseek-coder-v2:6.7b - - # Google Gemma (efficient) - - gemma2:27b - - gemma2:9b - - gemma2:2b - - # Microsoft Phi (lightweight powerhouse) - - phi4:14b - - phi3:14b - - phi3:3.8b - - # Mistral AI series (fast) - - mistral:7b - - mistral-nemo:12b - - mixtral:8x7b - - mixtral:8x22b - - # Vision and multimodal - - llava:34b - - llava:13b - - llava:7b - - qwen2-vl:7b - - qwen2-vl:2b - - # Ultra-lightweight (edge devices) - - tinyllama:1.1b - - orca-mini:3b - - stablelm2:1.6b - - # Conversational specialists - - vicuna:13b - - vicuna:7b - - solar:10.7b - + default: llama3.2 + models: + # Recommended small default (same as llama3.2:3b) + - llama3.2 + + # Newer small models + - qwen3:4b + - qwen3:8b + - qwen3.5:4b + - qwen3.5:9b + - gemma3:4b + - gemma3:12b + - gemma4:e4b + - gpt-oss:20b + - mistral + - phi4 + + # Top reasoning models + - deepseek-r1:70b + - deepseek-r1:32b + - deepseek-r1:14b + - deepseek-r1:8b + - deepseek-r1:1.5b + + # Meta Llama 3 series (best all-round) + - llama3.3:70b + - llama3.3:8b + - llama3.2:3b + - llama3.2:1b + - llama3.1:405b + - llama3.1:70b + - llama3.1:8b + + # Alibaba Qwen series (multilingual) + - qwen2.5:72b + - qwen2.5:32b + - qwen2.5:14b + - qwen2.5:7b + - qwen2.5:3b + - qwen2.5:1.5b + - qwen2.5:0.5b + + # Coding specialists + - qwen2.5-coder:32b + - qwen2.5-coder:7b + - qwen2.5-coder:1.5b + - codellama:34b + - codellama:13b + - codellama:7b + - deepseek-coder-v2:16b + - deepseek-coder-v2:6.7b + + # Google Gemma (efficient) + - gemma2:27b + - gemma2:9b + - gemma2:2b + + # Microsoft Phi (lightweight powerhouse) + - phi4:14b + - phi3:14b + - phi3:3.8b + + # Mistral AI series (fast) + - mistral:7b + - mistral-nemo:12b + - mixtral:8x7b + - mixtral:8x22b + + # Vision and multimodal + - llava:34b + - llava:13b + - llava:7b + - qwen2-vl:7b + - qwen2-vl:2b + + # Ultra-lightweight (edge devices) + - tinyllama:1.1b + - orca-mini:3b + - stablelm2:1.6b + + # Conversational specialists + - vicuna:13b + - vicuna:7b + - solar:10.7b diff --git a/src/main/utils/JsonToNetLogo.scala b/src/main/utils/JsonToNetLogo.scala index a925f96..75b4016 100644 --- a/src/main/utils/JsonToNetLogo.scala +++ b/src/main/utils/JsonToNetLogo.scala @@ -14,6 +14,20 @@ import org.nlogo.core.LogoList */ object JsonToNetLogo { + /** + * Describe why a `ujson.read` call failed. + * + * The ujson that NetLogo ships (4.4+) wraps the underlying + * `ujson.ParseException` in a trace exception whose message is only the JSON + * path (e.g. `$`), so the useful text lives in the cause chain. + */ + def parseErrorMessage(e: Throwable): String = { + val chain = Iterator.iterate(e)(_.getCause).takeWhile(_ != null).toSeq + chain.collectFirst { case p: ujson.ParseException => s"${p.clue} at index ${p.index}" } + .orElse(chain.collectFirst { case i: ujson.IncompleteParseException => i.getMessage }) + .getOrElse(e.getMessage) + } + /** * Parse a JSON string and convert it to a NetLogo value. * @@ -25,7 +39,7 @@ object JsonToNetLogo { catch { case e: Exception => throw new IllegalArgumentException( - s"Response was not valid JSON: ${e.getMessage}. Response text: $jsonText" + s"Response was not valid JSON: ${parseErrorMessage(e)}. Response text: $jsonText" ) } convert(parsed) @@ -46,7 +60,7 @@ object JsonToNetLogo { catch { case e: Exception => throw new IllegalArgumentException( - s"Response was not valid JSON: ${e.getMessage}. Response text: $jsonText" + s"Response was not valid JSON: ${parseErrorMessage(e)}. Response text: $jsonText" ) } diff --git a/src/test/AnyFunSuiteEx.scala b/src/test/AnyFunSuiteEx.scala new file mode 100644 index 0000000..1c824da --- /dev/null +++ b/src/test/AnyFunSuiteEx.scala @@ -0,0 +1,29 @@ +// ABOUTME: Copy of NetLogo's org.nlogo.util.AnyFunSuiteEx, which netlogo-7.0.4-tests.jar omits. +// ABOUTME: TestLanguage extends it, so without this copy the Tests suite cannot compile (issue #45). + +package org.nlogo.util + +import org.scalatest.{ Failed, Outcome, Succeeded } +import org.scalatest.funsuite.AnyFunSuite + +// Copied verbatim from NetLogo v7.0.4 parser-jvm/src/test/util/AnyFunSuiteEx.scala. +// Remove once NetLogo publishes it in the tests artifact. +// this wrapper trait provides a mechanism for ensuring that exceptions in stray threads +// result in a failed test, when they would otherwise silently print to stdout (Isaac B 11/18/25) +trait AnyFunSuiteEx extends AnyFunSuite { + private var outcome: Outcome = Succeeded + + Thread.setDefaultUncaughtExceptionHandler { (_, t) => + if (outcome == Succeeded) + outcome = Failed(t) + } + + override def withFixture(test: NoArgTest): Outcome = { + outcome = Succeeded + + super.withFixture(test) match { + case Succeeded => outcome + case o => o + } + } +} diff --git a/src/test/BaseUrlResolutionSpec.scala b/src/test/BaseUrlResolutionSpec.scala new file mode 100644 index 0000000..ed5eb6e --- /dev/null +++ b/src/test/BaseUrlResolutionSpec.scala @@ -0,0 +1,142 @@ +// ABOUTME: Tests that the legacy generic base_url is honored, and scoped, consistently across providers +// ABOUTME: Uses sttp BackendStub to capture the URL each provider actually sends to, with no real HTTP +package org.nlogo.extensions.llm.providers + +import org.nlogo.extensions.llm.LLMExtension +import org.nlogo.extensions.llm.config.ConfigStore +import org.nlogo.extensions.llm.models.{ChatMessage, ChatRequest} +import org.scalatest.funsuite.AnyFunSuite +import sttp.client4._ +import sttp.client4.testing.{BackendStub, ResponseStub} +import sttp.model.StatusCode +import java.util.concurrent.ConcurrentLinkedQueue +import scala.concurrent.{Await, Future} +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.duration._ +import scala.jdk.CollectionConverters._ +import scala.util.Try + +/** A stub backend that records every request URL and answers 200 with `body`. */ +class UrlCapture(body: String) { + val urls = new ConcurrentLinkedQueue[String]() + val backend: BackendStub[Future] = BackendStub.asynchronousFuture + .whenRequestMatches { req => urls.add(req.uri.toString); true } + .thenRespond(ResponseStub.adjust(body, StatusCode.Ok)) + def sent: List[String] = urls.asScala.toList +} + +class UrlCapturingOpenAI(capture: UrlCapture) extends OpenAIProvider()(using global) { + override lazy val backend: Backend[Future] = capture.backend +} + +class UrlCapturingClaude(capture: UrlCapture) extends ClaudeProvider()(using global) { + override lazy val backend: Backend[Future] = capture.backend +} + +class UrlCapturingGemini(capture: UrlCapture) extends GeminiProvider()(using global) { + override lazy val backend: Backend[Future] = capture.backend +} + +class UrlCapturingOllama(capture: UrlCapture) extends OllamaProvider()(using global) { + override lazy val backend: Backend[Future] = capture.backend +} + +class BaseUrlResolutionSpec extends AnyFunSuite { + + ProviderRegistrations.registerAll() + + private def request(model: String) = ChatRequest(model = model, messages = Seq(ChatMessage.user("hi"))) + + /** Mirrors ProviderFactory.createProviderFromConfig: every key is copied in. */ + private def configure(provider: BaseHttpProvider, config: ConfigStore): provider.type = { + config.toMap.foreach { case (k, v) => provider.setConfig(k, v) } + provider + } + + private def send(provider: BaseHttpProvider, model: String): Unit = + Try(Await.result(provider.chat(request(model)), 5.seconds)) + + test("OpenAI with only base_url set sends to that base_url") { + val capture = new UrlCapture("{}") + val provider = new UrlCapturingOpenAI(capture) + provider.setConfig(ConfigStore.API_KEY, "sk-test") + provider.setConfig(ConfigStore.BASE_URL, "http://x") + send(provider, "gpt-4o-mini") + assert(capture.sent == List("http://x/chat/completions")) + } + + test("provider-specific base URL wins over the generic base_url") { + val capture = new UrlCapture("{}") + val provider = new UrlCapturingOpenAI(capture) + provider.setConfig(ConfigStore.API_KEY, "sk-test") + provider.setConfig(ConfigStore.BASE_URL, "http://generic") + provider.setConfig(ConfigStore.OPENAI_BASE_URL, "http://specific") + send(provider, "gpt-4o-mini") + assert(capture.sent == List("http://specific/chat/completions")) + } + + test("Gemini with only base_url set sends to that base_url") { + val capture = new UrlCapture("{}") + val provider = new UrlCapturingGemini(capture) + provider.setConfig(ConfigStore.API_KEY, "k") + provider.setConfig(ConfigStore.BASE_URL, "http://g") + send(provider, "gemini-2.5-flash") + assert(capture.sent.headOption.exists(_.startsWith("http://g/models/gemini-2.5-flash:generateContent"))) + } + + test("a fresh extension config does not seed a generic base_url") { + assert(ConfigStore.withDefaults().get(ConfigStore.BASE_URL).isEmpty) + } + + test("Anthropic with no base_url hits the Anthropic default after an OpenAI-only session") { + val config = ConfigStore.withDefaults() + ProviderFactory.applyProviderSwitch(config, "openai") + ProviderFactory.applyProviderSwitch(config, "anthropic") + config.set(ConfigStore.API_KEY, "sk-ant-test") + + val capture = new UrlCapture("{}") + send(configure(new UrlCapturingClaude(capture), config), "claude-sonnet-4-5") + assert(capture.sent == List(s"${ConfigStore.DEFAULT_ANTHROPIC_BASE_URL}/messages")) + } + + test("a base_url set while OpenAI is active does not leak to Anthropic after set-provider") { + val config = ConfigStore.withDefaults() + ProviderFactory.applyProviderSwitch(config, "openai") + config.set(ConfigStore.BASE_URL, "http://openai-proxy") + ProviderFactory.applyProviderSwitch(config, "anthropic") + config.set(ConfigStore.API_KEY, "sk-ant-test") + + val capture = new UrlCapture("{}") + send(configure(new UrlCapturingClaude(capture), config), "claude-sonnet-4-5") + assert(capture.sent == List(s"${ConfigStore.DEFAULT_ANTHROPIC_BASE_URL}/messages")) + + // ...and switching back to OpenAI keeps the URL the user set for it. + ProviderFactory.applyProviderSwitch(config, "openai") + val openAiCapture = new UrlCapture("{}") + send(configure(new UrlCapturingOpenAI(openAiCapture), config), "gpt-4o-mini") + assert(openAiCapture.sent == List("http://openai-proxy/chat/completions")) + } + + test("Ollama with only base_url set probes that host") { + val config = new ConfigStore() + config.set(ConfigStore.BASE_URL, "http://ollama-host:1234") + val capture = new UrlCapture("""{"models":[]}""") + assert(LLMExtension.probeOllama(config, new UrlCapturingOllama(capture))) + assert(capture.sent == List("http://ollama-host:1234/api/tags")) + } + + test("Ollama with no base URL configured probes the default host") { + val capture = new UrlCapture("""{"models":[]}""") + assert(LLMExtension.probeOllama(new ConfigStore(), new UrlCapturingOllama(capture))) + assert(capture.sent == List(s"${ConfigStore.DEFAULT_OLLAMA_BASE_URL}/api/tags")) + } + + test("Ollama chat and model listing honor the generic base_url") { + val capture = new UrlCapture("""{"models":[]}""") + val provider = new UrlCapturingOllama(capture) + provider.setConfig(ConfigStore.BASE_URL, "http://ollama-host:1234") + Await.result(provider.listInstalledModels(), 5.seconds) + send(provider, "llama3.2") + assert(capture.sent == List("http://ollama-host:1234/api/tags", "http://ollama-host:1234/api/chat")) + } +} diff --git a/src/test/DeterministicTestProvider.scala b/src/test/DeterministicTestProvider.scala index 15358ea..360c330 100644 --- a/src/test/DeterministicTestProvider.scala +++ b/src/test/DeterministicTestProvider.scala @@ -1,7 +1,7 @@ package org.nlogo.extensions.llm.providers import org.nlogo.extensions.llm.config.ConfigStore -import org.nlogo.extensions.llm.models.{ChatMessage, ChatRequest, ChatResponse, Choice, EnumFormat, JsonObjectFormat, JsonSchemaFormat, ResponseFormat, Usage} +import org.nlogo.extensions.llm.models.{ChatMessage, ChatRequest, ChatResponse, Choice, DecisionQuestion, DecisionResult, EnumFormat, JsonObjectFormat, JsonSchemaFormat, ResponseFormat, Usage} import scala.concurrent.{ExecutionContext, Future} import scala.util.{Success, Try} @@ -24,6 +24,11 @@ class DeterministicTestProvider(implicit ec: ExecutionContext) extends LLMProvid // response the rest of the stub produces, so accounting can be asserted // independently of which primitive made the call. private val testUsageRegex = """__TEST_USAGE:(\d+),(\d+)(?:,(\d+))?""".r.unanchored + // __TEST_REFUSE: answers with empty content and a refusal, the shape a + // provider returns when the model declines. Captures to end of line. + private val testRefuseRegex = """__TEST_REFUSE:(.+)""".r + // __TEST_TRUNCATE answers with empty content and finish_reason "length", the + // shape a reasoning model returns when it spends the whole budget thinking. private def lastUserContent(messages: Seq[ChatMessage]): String = messages.reverseIterator.find(_.role == "user").map(_.content).getOrElse("") @@ -63,6 +68,21 @@ class DeterministicTestProvider(implicit ec: ExecutionContext) extends LLMProvid // Check for __TEST_THINKING: prefix — return empty content + thinking field lastUserMessage match { + case msg if msg.contains("__TEST_TRUNCATE") => + Future.successful(ChatResponse( + id = "deterministic-test-response", + created = System.currentTimeMillis() / 1000, + model = "deterministic-model", + choices = Array(Choice(0, ChatMessage.assistant(""), "length")) + )) + case msg if msg.contains("__TEST_REFUSE:") => + val refusal = testRefuseRegex.findFirstMatchIn(msg).map(_.group(1).trim).getOrElse("") + Future.successful(ChatResponse( + id = "deterministic-test-response", + created = System.currentTimeMillis() / 1000, + model = "deterministic-model", + choices = Array(Choice(0, ChatMessage.assistant(""), "", refusal = Some(refusal))) + )) case msg if msg.contains("__TEST_THINKING:") => val thinking = testThinkingRegex.findFirstMatchIn(msg).map(_.group(1).trim).getOrElse("") Future.successful(ChatResponse( @@ -101,7 +121,8 @@ class DeterministicTestProvider(implicit ec: ExecutionContext) extends LLMProvid .map(_.content) .getOrElse("") - if (lastUserMessage.contains("__TEST_RESPOND:") || lastUserMessage.contains("__TEST_FAIL")) { + if (lastUserMessage.contains("__TEST_RESPOND:") || lastUserMessage.contains("__TEST_FAIL") || + lastUserMessage.contains("__TEST_REFUSE:")) { return chatWithFullResponse(messages) } @@ -237,3 +258,70 @@ class DeterministicTestProvider(implicit ec: ExecutionContext) extends LLMProvid override def supportsModel(model: String): Boolean = true } + +/** + * Deterministic stand-in for a decision provider (TypeSafe), used by tests + * whenever the configured provider is "typesafe". + * + * Kept separate from DeterministicTestProvider so llm:choose on every other + * provider still exercises the chat path. Answers are prompt-driven: + * - __TEST_DECIDE::

puts probability

on and spreads + * the rest evenly over the other outcomes. is an option for a + * choice, a 0-based level index for a score, and true/false for yes-no. + * - With no marker the first option, level 0, or yes gets probability 1. + * - __TEST_FAIL fails the call; __TEST_DELAY:: delays it; __TEST_USAGE + * attaches usage, as for chat. + * Free-text chat is refused through textUnsupportedMessage, as on TypeSafe. + */ +class DeterministicDecisionTestProvider(implicit ec: ExecutionContext) + extends DeterministicTestProvider with DecisionProvider { + + private val testDecideRegex = """__TEST_DECIDE:([^:\s]+):([0-9.]+)""".r.unanchored + private val testDecisionDelayRegex = """__TEST_DELAY:(\d+):""".r.unanchored + private val testDecisionUsageRegex = """__TEST_USAGE:(\d+),(\d+)""".r.unanchored + + override def providerName: String = "typesafe" + + override def defaultModel: String = "jev-latest" + + override def textUnsupportedMessage: Option[String] = Some(TypeSafeProvider.TextUnsupportedMessage) + + /** `p` on outcome `picked` of `n`, the remainder shared evenly by the others. */ + private def spread(n: Int, picked: Int, p: Double): Seq[Double] = + (0 until n).map(i => if (i == picked) p else if (n > 1) (1.0 - p) / (n - 1) else 0.0) + + private def answer(prompt: String, question: DecisionQuestion): DecisionResult = { + val marker = prompt match { + case testDecideRegex(a, p) => Some((a, p.toDouble)) + case _ => None + } + question match { + case DecisionQuestion.Choice(options) => + val (picked, p) = marker.getOrElse((options.head, 1.0)) + val probs = spread(options.size, options.indexOf(picked), p) + DecisionResult.choice(options, picked, options.zip(probs).toMap) + case DecisionQuestion.Score(levels) => + val (level, p) = marker.map { case (a, q) => (a.toInt, q) }.getOrElse((0, 1.0)) + val probs = spread(levels.size, level, p) + val expected = probs.zipWithIndex.map { case (q, i) => q * i }.sum + DecisionResult.score(levels, expected, p, probs) + case DecisionQuestion.YesNo => + val (yes, p) = marker.map { case (a, q) => (a.toBoolean, q) }.getOrElse((true, 1.0)) + DecisionResult.yesNo(if (yes) p else 1.0 - p) + } + } + + override def decide(prompt: String, question: DecisionQuestion): Future[(DecisionResult, Usage)] = { + val usage = prompt match { + case testDecisionUsageRegex(in, out) => Usage(in.toLong, out.toLong, in.toLong + out.toLong) + case _ => Usage.empty + } + if (prompt.contains("__TEST_FAIL")) Future.failed(new RuntimeException("simulated provider failure")) + else prompt match { + case testDecisionDelayRegex(ms) => + Future { Thread.sleep(ms.toLong) }.map(_ => (answer(prompt, question), usage)) + case _ => + Future.successful((answer(prompt, question), usage)) + } + } +} diff --git a/src/test/ModelCatalogSpec.scala b/src/test/ModelCatalogSpec.scala new file mode 100644 index 0000000..9eb07f0 --- /dev/null +++ b/src/test/ModelCatalogSpec.scala @@ -0,0 +1,279 @@ +// ABOUTME: Deterministic tests for the YAML model catalog: per-provider defaults, the user catalog file, and layering +// ABOUTME: Also pins the refreshed bundled defaults and the request shape their APIs require (#64) +package org.nlogo.extensions.llm.providers + +import org.nlogo.extensions.llm.config.{ModelRegistryLoader, ProviderModels} +import org.nlogo.extensions.llm.models.{ChatMessage, ChatRequest, ThinkingConfig} +import org.scalatest.BeforeAndAfterEach +import org.scalatest.funsuite.AnyFunSuite +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Path} +import java.util.Comparator +import scala.concurrent.ExecutionContext.Implicits.global + +private class CatalogInspectableGemini extends GeminiProvider { + def body(r: ChatRequest): ujson.Value = createProviderRequest(r) +} + +class ModelCatalogSpec extends AnyFunSuite with BeforeAndAfterEach with RegistryLock { + + // registerAll() is idempotent; no reset, which would race other suites. + ProviderRegistrations.registerAll() + + // Provider keys no other suite uses, so the user catalog and overrides loaded + // here never change what a parallel suite reads for a real provider. + private val SpecProvider = "model-catalog-spec-provider" + private val AddedOpenAiModel = "model-catalog-spec-added-model" + + private var dir: Path = scala.compiletime.uninitialized + private var savedProperty: Option[String] = None + + override def beforeEach(): Unit = { + dir = Files.createTempDirectory("model-catalog") + savedProperty = Option(System.getProperty(ModelRegistryLoader.UserCatalogProperty)) + } + + override def afterEach(): Unit = { + savedProperty match { + case Some(v) => System.setProperty(ModelRegistryLoader.UserCatalogProperty, v) + case None => System.clearProperty(ModelRegistryLoader.UserCatalogProperty) + } + ModelRegistry.clearOverride() + ModelRegistry.reload() + Files.walk(dir).sorted(Comparator.reverseOrder[Path]()).forEach(p => Files.deleteIfExists(p)) + } + + private def useUserCatalog(content: String): Path = { + val file = dir.resolve("models.yaml") + Files.writeString(file, content, StandardCharsets.UTF_8) + System.setProperty(ModelRegistryLoader.UserCatalogProperty, file.toString) + ModelRegistry.reload() + file + } + + private lazy val bundled: Map[String, ProviderModels] = ModelRegistryLoader.loadBundledConfig().get + + private def descriptors: Seq[ProviderDescriptor] = + ProviderRegistry.allNames.toSeq.sorted.flatMap(ProviderRegistry.get) + + // --- default: in YAML --- + + test("the bundled catalog declares a default for every registered provider") { + val missing = descriptors.map(_.name).filterNot(n => bundled.get(n).exists(_.default.isDefined)) + assert(missing.isEmpty, s"providers with no default: in models.yaml: ${missing.mkString(", ")}") + } + + test("every bundled provider default is listed in its own catalog list") { + val drifted = bundled.toSeq.sortBy(_._1).collect { + case (name, pm) if pm.default.exists(d => !pm.models.contains(d)) => + s"provider '$name' default '${pm.default.get}' is not in its own list" + } + assert(drifted.isEmpty, s"\n${drifted.mkString("\n")}") + } + + test("the provider default model comes from the catalog's default: key") { + val expected = Map( + "anthropic" -> "claude-haiku-5-5", + "openai" -> "gpt-6-luna", + "gemini" -> "gemini-3.5-flash-lite", + "groq" -> "openai/gpt-oss-20b", + "openrouter" -> "openai/gpt-oss-20b", + "together" -> "Qwen/Qwen3.8-Flash", + "ollama" -> "llama3.2" + ) + expected.foreach { case (provider, model) => + assert(bundled(provider).default.contains(model), s"bundled default for $provider") + assert(ModelRegistry.defaultModel(provider) == model, s"effective default for $provider") + } + } + + test("a provider section without default: falls back to the descriptor default") { + val catalog = Map("openai" -> ProviderModels(Set("some-model"))) + val descriptorDefault = ProviderRegistry.get("openai").get.defaultModel + assert(ModelRegistry.defaultFrom(catalog, "openai") == descriptorDefault) + assert(ModelRegistry.defaultFrom(Map.empty, "anthropic") == ProviderRegistry.get("anthropic").get.defaultModel) + } + + test("a catalog default wins over the descriptor default") { + val catalog = Map("openai" -> ProviderModels(Set("a", "b"), default = Some("b"))) + assert(ModelRegistry.defaultFrom(catalog, "openai") == "b") + } + + test("the parser accepts both the plain-list and the default:/models: provider shapes") { + val parsed = ModelRegistryLoader.parseModelConfig( + "plain:\n - a\n - b\nkeyed:\n default: y\n models:\n - x\n - y\n", + isCustom = false + ).get + assert(parsed("plain") == ProviderModels(Set("a", "b"))) + assert(parsed("keyed") == ProviderModels(Set("x", "y"), default = Some("y"))) + } + + test("a provider section with neither models nor default is rejected") { + assert(ModelRegistryLoader.parseModelConfig("keyed:\n colour: blue\n", isCustom = false).isFailure) + } + + // --- YAML-load fallback --- + + test("the YAML-load fallback is just each descriptor's default model") { + val fallback = ModelRegistry.descriptorFallback() + descriptors.foreach { d => + assert(fallback(d.name).models == Set(d.defaultModel), s"fallback for ${d.name}") + } + } + + test("descriptor defaults match the bundled defaults so the fallback stays current") { + val drifted = descriptors.filterNot(d => bundled.get(d.name).flatMap(_.default).contains(d.defaultModel)) + .map(d => s"${d.name}: descriptor '${d.defaultModel}' vs YAML '${bundled.get(d.name).flatMap(_.default).getOrElse("-")}'") + assert(drifted.isEmpty, s"\n${drifted.mkString("\n")}") + } + + // --- user catalog file --- + + test("user catalog merge adds models, adds providers and overrides a default") { + val user = Map( + "openai" -> ProviderModels(Set("my-model"), default = Some("my-model")), + "brand-new" -> ProviderModels(Set("m")) + ) + val merged = ModelRegistryLoader.mergeUserConfig(bundled, user) + assert(merged("openai").models.contains("my-model")) + assert(merged("openai").models.contains("gpt-6-luna"), "user models are added, not substituted") + assert(merged("openai").default.contains("my-model")) + assert(merged("brand-new").models == Set("m")) + assert(merged("anthropic") == bundled("anthropic")) + } + + test("user catalog merge keeps the bundled default when the user file names none") { + val merged = ModelRegistryLoader.mergeUserConfig(bundled, Map("openai" -> ProviderModels(Set("extra")))) + assert(merged("openai").default == bundled("openai").default) + } + + test("a user catalog file is merged over the bundled catalog at init") { + useUserCatalog( + s"openai:\n - $AddedOpenAiModel\n$SpecProvider:\n default: m1\n models:\n - m1\n - m2\n" + ) + assert(ModelRegistry.userCatalogWarning.isEmpty) + assert(ModelRegistry.isValidModel("openai", AddedOpenAiModel)) + assert(ModelRegistry.isValidModel("openai", "gpt-6-luna")) + assert(ModelRegistry.defaultModel("openai") == "gpt-6-luna") + assert(ModelRegistry.isValidModel(SpecProvider, "m2")) + assert(ModelRegistry.defaultModel(SpecProvider) == "m1") + } + + test("a user catalog default missing from its models list is still a known model") { + useUserCatalog( + s"openai:\n default: $AddedOpenAiModel\n$SpecProvider:\n default: d1\n models:\n - m1\n" + ) + assert(ModelRegistry.defaultModel("openai") == AddedOpenAiModel) + assert(ModelRegistry.isValidModel("openai", AddedOpenAiModel)) + assert(ModelRegistry.isValidModel("openai", "gpt-6-luna")) + assert(ModelRegistry.defaultModel(SpecProvider) == "d1") + assert(ModelRegistry.isValidModel(SpecProvider, "d1")) + assert(ModelRegistry.isValidModel(SpecProvider, "m1")) + } + + test("an override default missing from its models list is still a known model") { + Files.writeString( + dir.resolve("models-override.yaml"), + s"$SpecProvider:\n default: d9\n models:\n - m9\n", + StandardCharsets.UTF_8 + ) + ModelRegistry.loadOverride(dir.toString) + assert(ModelRegistry.defaultModel(SpecProvider) == "d9") + assert(ModelRegistry.isValidModel(SpecProvider, "d9")) + assert(ModelRegistry.isValidModel(SpecProvider, "m9")) + } + + test("a missing user catalog file is not an error") { + System.setProperty(ModelRegistryLoader.UserCatalogProperty, dir.resolve("absent.yaml").toString) + ModelRegistry.reload() + assert(ModelRegistry.userCatalogWarning.isEmpty) + assert(ModelRegistry.defaultModel("openai") == "gpt-6-luna") + } + + test("a broken user catalog file is ignored with a warning") { + val file = useUserCatalog(s"$SpecProvider: [\n") + val warning = ModelRegistry.userCatalogWarning + assert(warning.isDefined) + assert(warning.get.contains(file.toString)) + assert(!ModelRegistry.getAllProviders.contains(SpecProvider)) + assert(ModelRegistry.defaultModel("openai") == "gpt-6-luna") + assert(ModelRegistry.isValidModel("openai", "gpt-6-luna")) + } + + test("models-override.yaml layers on top of the user catalog, and clearing it restores the user layer") { + useUserCatalog(s"$SpecProvider:\n default: m1\n models:\n - m1\n") + Files.writeString( + dir.resolve("models-override.yaml"), + s"$SpecProvider:\n default: m3\n models:\n - m3\n", + StandardCharsets.UTF_8 + ) + assert(ModelRegistry.loadOverride(dir.toString).isDefined) + assert(ModelRegistry.defaultModel(SpecProvider) == "m3") + assert(ModelRegistry.isValidModel(SpecProvider, "m3")) + + ModelRegistry.clearOverride() + assert(ModelRegistry.defaultModel(SpecProvider) == "m1") + assert(!ModelRegistry.isValidModel(SpecProvider, "m3")) + } + + test("an override list without default: keeps the lower layer's default") { + useUserCatalog(s"$SpecProvider:\n default: m1\n models:\n - m1\n") + Files.writeString(dir.resolve("models-override.yaml"), s"$SpecProvider:\n - m9\n", StandardCharsets.UTF_8) + ModelRegistry.loadOverride(dir.toString) + assert(ModelRegistry.defaultModel(SpecProvider) == "m1") + assert(ModelRegistry.isValidModel(SpecProvider, "m9")) + } + + test("the override in effect survives a user catalog reload") { + Files.writeString(dir.resolve("models-override.yaml"), s"$SpecProvider:\n - m9\n", StandardCharsets.UTF_8) + ModelRegistry.loadOverride(dir.toString) + useUserCatalog(s"openai:\n - $AddedOpenAiModel\n") + assert(ModelRegistry.isValidModel(SpecProvider, "m9")) + assert(ModelRegistry.isValidModel("openai", AddedOpenAiModel)) + } + + // --- refreshed defaults get a request shape their API accepts --- + + test("GPT-5 and GPT-6 models get the reasoning request format, alongside the o-series") { + Seq("gpt-6-luna", "gpt-6.1-sol", "gpt-5.6-terra", "gpt-5.4-mini", "o3", "o1").foreach { m => + assert(ReasoningModelDetector.requiresReasoningFormat("openai", m), m) + assert(ReasoningModelDetector.isReasoningModel("openai", m), m) + } + assert(ReasoningModelDetector.requiresReasoningFormat("openrouter", "openai/gpt-6-luna")) + assert(!ReasoningModelDetector.requiresReasoningFormat("openai", "gpt-4o-mini")) + assert(!ReasoningModelDetector.requiresReasoningFormat("openrouter", "openai/gpt-oss-20b")) + } + + private def geminiThinking(model: String, thinking: ThinkingConfig): ujson.Value = + (new CatalogInspectableGemini).body(ChatRequest( + model = model, + messages = Seq(ChatMessage.user("hi")), + maxTokens = Some(1000), + thinkingConfig = Some(thinking) + ))("generationConfig")("thinkingConfig") + + test("Gemini 3 models get thinkingLevel and never thinkingLevel plus thinkingBudget together") { + val tc = geminiThinking("gemini-3.5-flash-lite", + ThinkingConfig(enabled = true, reasoningEffort = Some("low"), budgetTokens = Some(2048))) + assert(tc("thinkingLevel").str == "LOW") + assert(!tc.obj.contains("thinkingBudget")) + } + + test("Gemini 3 maps the extension's none and xhigh efforts onto levels it accepts") { + assert(geminiThinking("gemini-3.8-flash", ThinkingConfig(enabled = true, reasoningEffort = Some("none")))("thinkingLevel").str == "MINIMAL") + assert(geminiThinking("gemini-4-argon", ThinkingConfig(enabled = true, reasoningEffort = Some("xhigh")))("thinkingLevel").str == "HIGH") + } + + test("Gemini 3 still accepts a budget when no effort is set") { + val tc = geminiThinking("gemini-3.5-flash-lite", ThinkingConfig(enabled = true, budgetTokens = Some(2048))) + assert(tc("thinkingBudget").num == 2048) + assert(!tc.obj.contains("thinkingLevel")) + } + + test("Gemini 2.5 models get thinkingBudget and no thinkingLevel") { + val tc = geminiThinking("gemini-2.5-flash", + ThinkingConfig(enabled = true, reasoningEffort = Some("low"), budgetTokens = Some(2048))) + assert(tc("thinkingBudget").num == 2048) + assert(!tc.obj.contains("thinkingLevel")) + } +} diff --git a/src/test/ModelRegistrySpec.scala b/src/test/ModelRegistrySpec.scala new file mode 100644 index 0000000..90bb3e4 --- /dev/null +++ b/src/test/ModelRegistrySpec.scala @@ -0,0 +1,74 @@ +// ABOUTME: Deterministic tests for ModelRegistry override loading across model directories +// ABOUTME: Asserts that a models-override.yaml from one directory does not leak into later loads (#29) +package org.nlogo.extensions.llm.providers + +import org.scalatest.BeforeAndAfterEach +import org.scalatest.funsuite.AnyFunSuite +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Path} +import java.util.Comparator + +class ModelRegistrySpec extends AnyFunSuite with BeforeAndAfterEach with RegistryLock { + + // A provider key no other spec uses, so suites running in parallel against the + // shared registry never see this override touch their providers. + private val CustomProvider = "model-registry-spec-provider" + private val CustomModel = "my-custom-leak-model" + + private var dirA: Path = scala.compiletime.uninitialized + private var dirB: Path = scala.compiletime.uninitialized + + override def beforeEach(): Unit = { + // clearOverride rather than reset: reset briefly empties the shared registry, + // which would race with other suites reading it in parallel. + ModelRegistry.clearOverride() + dirA = Files.createTempDirectory("model-registry-a") + dirB = Files.createTempDirectory("model-registry-b") + Files.writeString( + dirA.resolve("models-override.yaml"), + s"$CustomProvider:\n - $CustomModel\n", + StandardCharsets.UTF_8 + ) + } + + override def afterEach(): Unit = { + ModelRegistry.clearOverride() + Seq(dirA, dirB).foreach { dir => + Files.walk(dir).sorted(Comparator.reverseOrder[Path]()).forEach(p => Files.deleteIfExists(p)) + } + } + + private def assertNoCustomModel(): Unit = { + assert(!ModelRegistry.isValidModel(CustomProvider, CustomModel)) + val listing = ModelRegistry.formatModelList("openai", "gpt-4o-mini") + assert(!listing.contains(CustomProvider)) + assert(!listing.contains(CustomModel)) + assert(!listing.contains("[custom]")) + } + + test("override from directory A is visible after loading A") { + assert(ModelRegistry.loadOverride(dirA.toString).isDefined) + assert(ModelRegistry.isValidModel(CustomProvider, CustomModel)) + assert(ModelRegistry.formatModelList("openai", "gpt-4o-mini").contains("[custom]")) + } + + test("loading a directory without an override drops the previous override") { + ModelRegistry.loadOverride(dirA.toString) + assert(ModelRegistry.loadOverride(dirB.toString).isEmpty) + assertNoCustomModel() + } + + test("a broken override file drops the previous override") { + ModelRegistry.loadOverride(dirA.toString) + Files.writeString(dirB.resolve("models-override.yaml"), s"$CustomProvider: []\n", StandardCharsets.UTF_8) + assert(ModelRegistry.loadOverride(dirB.toString).isEmpty) + assertNoCustomModel() + } + + test("clearOverride restores the bundled catalog") { + ModelRegistry.loadOverride(dirA.toString) + ModelRegistry.clearOverride() + assertNoCustomModel() + assert(ModelRegistry.isValidModel("openai", "gpt-4o-mini")) + } +} diff --git a/src/test/ProviderDefaultsSpec.scala b/src/test/ProviderDefaultsSpec.scala index 73dcd2d..45ba723 100644 --- a/src/test/ProviderDefaultsSpec.scala +++ b/src/test/ProviderDefaultsSpec.scala @@ -48,20 +48,15 @@ class ProviderDefaultsSpec extends AnyFunSuite { assert(empty.isEmpty, s"providers with no models in models.yaml: ${empty.mkString(", ")}") } - test("every provider default model is present in the YAML-load fallback too") { - // The check above reads the LOADED registry, so a stale entry in - // FALLBACK_CONFIG survives it. That map is what the extension falls back on - // when models.yaml cannot be read, and it kept retired models long after - // they were removed from the YAML. + test("every provider's effective default model is present in the loaded registry") { + // The effective default comes from models.yaml's default: key when set, so + // it can drift from the descriptor default checked above. Both must be + // models the extension accepts. (The YAML-load fallback is built from the + // descriptor defaults alone, so it cannot drift; ModelCatalogSpec covers it.) val drifted = descriptors.flatMap { d => - ModelRegistry.FALLBACK_CONFIG.get(d.name) match { - case None => - Some(s"provider '${d.name}' has no FALLBACK_CONFIG entry") - case Some(pm) if !pm.models.contains(d.defaultModel) => - Some(s"provider '${d.name}' default '${d.defaultModel}' missing from FALLBACK_CONFIG " + - s"(has: ${pm.models.toSeq.sorted.mkString(", ")})") - case _ => None - } + val effective = ModelRegistry.defaultModel(d.name) + if (ModelRegistry.isValidModel(d.name, effective)) None + else Some(s"provider '${d.name}' effective default '$effective' is not in the loaded registry") } assert(drifted.isEmpty, s"\n${drifted.mkString("\n")}") } diff --git a/src/test/ReasoningBudgetSpec.scala b/src/test/ReasoningBudgetSpec.scala new file mode 100644 index 0000000..4d5cdf5 --- /dev/null +++ b/src/test/ReasoningBudgetSpec.scala @@ -0,0 +1,93 @@ +// ABOUTME: Deterministic tests for the completion budget sent to OpenAI-format reasoning models +// ABOUTME: Covers lifting the default max_tokens and the clear error for an empty reply cut off at the limit +package org.nlogo.extensions.llm.providers + +import org.nlogo.extensions.llm.LLMExtension +import org.nlogo.extensions.llm.config.ConfigStore +import org.nlogo.extensions.llm.models.{ChatMessage, ChatRequest, ChatResponse} +import org.scalatest.funsuite.AnyFunSuite +import scala.concurrent.ExecutionContext.Implicits.global + +/** Builds requests from the provider's own config, as a real call would. */ +private class BudgetInspectableOpenAI extends OpenAIProvider { + def set(key: String, value: String): Unit = setConfig(key, value) + def body(): ujson.Value = createProviderRequest(buildRequest(Seq(ChatMessage.user("hi")))) + def parse(body: String): ChatResponse = parseProviderResponse(body, "gpt-6-luna") +} + +class ReasoningBudgetSpec extends AnyFunSuite { + + ProviderRegistrations.registerAll() + + private def provider(model: String): BudgetInspectableOpenAI = { + val p = new BudgetInspectableOpenAI + p.set(ConfigStore.MODEL, model) + p + } + + test("a reasoning model left at the default max_tokens gets a larger completion budget") { + val body = provider("gpt-6-luna").body() + assert(body("max_completion_tokens").num.toInt == OpenAICompatibleProvider.MinReasoningCompletionTokens) + assert(OpenAICompatibleProvider.MinReasoningCompletionTokens >= 4000) + assert(!body.obj.contains("max_tokens")) + } + + test("a max_tokens the modeler set is sent unchanged to a reasoning model") { + Seq("1000", "2000", "12000").foreach { n => + val p = provider("gpt-6-luna") + p.set(ConfigStore.MAX_TOKENS, n) + assert(p.body()("max_completion_tokens").num.toInt == n.toInt, n) + } + } + + test("defaults copied from the extension's config still count as unset; user values do not") { + val global = ConfigStore.withDefaults() + global.set(ConfigStore.MODEL, "gpt-6-luna") + val seeded = new BudgetInspectableOpenAI + ProviderFactory.copyConfig(global, seeded) + assert(seeded.body()("max_completion_tokens").num.toInt == OpenAICompatibleProvider.MinReasoningCompletionTokens) + + global.set(ConfigStore.MAX_TOKENS, ConfigStore.DEFAULT_MAX_TOKENS) + val chosen = new BudgetInspectableOpenAI + ProviderFactory.copyConfig(global, chosen) + assert(chosen.body()("max_completion_tokens").num.toInt == ConfigStore.DEFAULT_MAX_TOKENS.toInt) + } + + test("a config file that sets max_tokens to the default value counts as set") { + val global = ConfigStore.withDefaults() + global.loadFromMap(Map(ConfigStore.PROVIDER -> "openai", ConfigStore.MODEL -> "gpt-6-luna", + ConfigStore.MAX_TOKENS -> "1000")) + assert(!global.isDefault(ConfigStore.MAX_TOKENS)) + val restored = ConfigStore.withDefaults() + restored.loadFromMap(ConfigStore.withDefaults().toMap, ConfigStore.withDefaults().defaultKeys) + assert(restored.isDefault(ConfigStore.MAX_TOKENS)) + } + + test("a non-reasoning model keeps the default max_tokens") { + val body = provider("gpt-4o-mini").body() + assert(body("max_tokens").num.toInt == 1000) + assert(!body.obj.contains("max_completion_tokens")) + } + + private def reply(content: String, finish: String): String = + s"""{"id":"c","created":1700000000,"model":"gpt-6-luna", + | "choices":[{"index":0,"message":{"role":"assistant","content":${ujson.write(content)}},"finish_reason":"$finish"}], + | "usage":{"prompt_tokens":5,"completion_tokens":4000,"total_tokens":4005, + | "completion_tokens_details":{"reasoning_tokens":4000}}}""".stripMargin + + test("an empty reply cut off at the token limit parses with its usage, and gets an error naming max_tokens") { + val response = provider("gpt-6-luna").parse(reply("", "length")) + assert(response.usage.map(_.outputTokens).contains(4000L)) + val message = LLMExtension.emptyReplyMessage(response).getOrElse(fail("expected an empty-reply error")) + assert(message.contains("max_tokens"), message) + assert(message.contains("empty"), message) + assert(message.contains("4000 output tokens"), message) + } + + test("a non-empty reply cut off at the limit, or an empty one that stopped normally, is not an error") { + val partial = provider("gpt-6-luna").parse(reply("partial", "length")) + assert(partial.firstContent.contains("partial")) + assert(LLMExtension.emptyReplyMessage(partial).isEmpty) + assert(LLMExtension.emptyReplyMessage(provider("gpt-6-luna").parse(reply("", "stop"))).isEmpty) + } +} diff --git a/src/test/RegistryLock.scala b/src/test/RegistryLock.scala new file mode 100644 index 0000000..2947e94 --- /dev/null +++ b/src/test/RegistryLock.scala @@ -0,0 +1,18 @@ +// ABOUTME: Serializes tests that mutate the global ModelRegistry (overrides, user catalog reloads) +// ABOUTME: Mix into any suite whose tests load or clear catalog layers, so parallel suites never interleave +package org.nlogo.extensions.llm.providers + +import org.scalatest.{Args, Status, Suite, SuiteMixin} + +/** + * ModelRegistry is one global object, and sbt runs suites in parallel. A suite + * that loads a models-override.yaml or a user catalog and then asserts on the + * result can be undone midway by another suite clearing it. Every such suite + * mixes this in, so their tests take turns. + */ +trait RegistryLock extends SuiteMixin { this: Suite => + abstract override protected def runTest(testName: String, args: Args): Status = + RegistryLock.synchronized { super.runTest(testName, args) } +} + +object RegistryLock diff --git a/src/test/RetrySpec.scala b/src/test/RetrySpec.scala index af7e3c3..a9bb206 100644 --- a/src/test/RetrySpec.scala +++ b/src/test/RetrySpec.scala @@ -1,5 +1,5 @@ -// ABOUTME: Unit tests for BaseHttpProvider rate-limit retry with exponential backoff -// ABOUTME: Uses an sttp BackendStub to script 429/200/500 responses without real HTTP +// ABOUTME: Unit tests for BaseHttpProvider retry of rate limits, 5xx/529 and transport failures +// ABOUTME: Uses an sttp BackendStub to script 429/5xx/200 responses and failed sends without real HTTP package org.nlogo.extensions.llm.providers import org.nlogo.extensions.llm.models.{ChatRequest, ChatResponse, ChatMessage, Choice} @@ -27,9 +27,12 @@ class StubbedProvider(stub: BackendStub[Future], counter: AtomicInteger) extends /** Delays actually slept, in order, so tests can assert on backoff behavior. */ val observedDelays = new java.util.concurrent.ConcurrentLinkedQueue[Long]() - override protected def notifyRateLimitWait(delayMs: Long, attempt: Int, maxRetries: Int): Unit = { + /** What each retry was waiting out, e.g. "returned HTTP 529 (overloaded)". */ + val observedCauses = new java.util.concurrent.ConcurrentLinkedQueue[String]() + override protected def notifyRetryWait(delayMs: Long, attempt: Int, maxRetries: Int, cause: String): Unit = { observedDelays.add(delayMs) - super.notifyRateLimitWait(delayMs, attempt, maxRetries) + observedCauses.add(cause) + super.notifyRetryWait(delayMs, attempt, maxRetries, cause) } def delays: List[Long] = observedDelays.toArray(Array.empty[java.lang.Long]).map(_.toLong).toList @@ -59,6 +62,10 @@ class StubbedProvider(stub: BackendStub[Future], counter: AtomicInteger) extends class RetrySpec extends AnyFunSuite { + // Real providers read the registry for their defaults and display names; it is + // normally installed by LLMExtension.load(). registerAll() is idempotent. + ProviderRegistrations.registerAll() + private def request = ChatRequest(model = "stub-model", messages = Seq(ChatMessage.user("hi"))) /** Build a stub that returns the scripted responses in order, counting sends. */ @@ -121,8 +128,7 @@ class RetrySpec extends AnyFunSuite { provider.setConfig(RetryPolicy.MAX_RETRIES, "3") val ex = intercept[RuntimeException](Await.result(provider.chat(request), 10.seconds)) - assert(ex.getMessage.contains("after 4 attempts")) - assert(ex.getMessage.contains("rate limited")) + assert(ex.getMessage.startsWith("stub returned HTTP 429 (rate limited) after 4 attempts"), ex.getMessage) assert(counter.get() == 4) } @@ -131,12 +137,13 @@ class RetrySpec extends AnyFunSuite { assert(RetryPolicy.DefaultMaxRetries > 3) } - test("non-rate-limit errors fail immediately without retry") { - val (stub, counter) = stubReturning(serverError) + test("transient 5xx errors are retried then succeed") { + val (stub, counter) = stubReturning(serverError, serverError, ok("recovered")) val provider = new StubbedProvider(stub, counter) - intercept[RuntimeException](Await.result(provider.chat(request), 5.seconds)) - assert(counter.get() == 1) + val result = Await.result(provider.chat(request), 5.seconds) + assert(result.firstMessage.map(_.content).contains("recovered")) + assert(counter.get() == 3) } test("honors Retry-After header and still succeeds") { @@ -316,10 +323,185 @@ class RetrySpec extends AnyFunSuite { } test("long waits are recorded so a stall is distinguishable from a hang") { - BaseHttpProvider.resetRateLimitStats() + BaseHttpProvider.resetRetryWaitStats() val policy = RetryPolicy() // A 3s wait is above the notice threshold and should be counted. assert(policy.delayMsFor(0, Some(3000L), () => 0.0) >= BaseHttpProvider.WaitNoticeThresholdMs) - assert(BaseHttpProvider.rateLimitWaitCount == 0L, "counters start clean") + assert(BaseHttpProvider.retryWaitCount == 0L, "counters start clean") + } + + // --- Transient server and transport failures (#48, #60) --- + + private def status(code: Int, body: String = "error"): Response[StubBody] = + ResponseStub.adjust(body, StatusCode(code)) + + /** A stub whose first `failures` sends fail with `error`, then answer "ok". */ + private def stubFailingSends(failures: Int)( + error: GenericRequest[?, ?] => Exception + ): (BackendStub[Future], AtomicInteger) = { + val counter = new AtomicInteger(0) + val backend = BackendStub.asynchronousFuture.whenAnyRequest.thenRespondF { req => + if (counter.getAndIncrement() < failures) Future.failed(error(req)) + else Future.successful(ok("ok")) + } + (backend, counter) + } + + private def connectRefused(req: GenericRequest[?, ?]): Exception = + new SttpClientException.ConnectException(req, new java.net.ConnectException("Connection refused")) + + test("Anthropic 529 overloaded is retried then succeeds") { + val (stub, counter) = stubReturning(status(529, """{"type":"error","error":{"type":"overloaded_error"}}"""), ok("fine")) + val provider = new StubbedProvider(stub, counter) + + val result = Await.result(provider.chat(request), 5.seconds) + assert(result.firstMessage.map(_.content).contains("fine")) + assert(counter.get() == 2) + assert(provider.observedCauses.peek() == "returned HTTP 529 (overloaded)") + } + + test("502, 503 and 504 are retried") { + Seq(502, 503, 504).foreach { code => + val (stub, counter) = stubReturning(status(code), ok("fine")) + val provider = new StubbedProvider(stub, counter) + Await.result(provider.chat(request), 5.seconds) + assert(counter.get() == 2, s"HTTP $code must be retried") + } + } + + test("4xx other than 429 is never retried") { + Seq(400, 401, 403, 404, 422).foreach { code => + val (stub, counter) = stubReturning(status(code), ok("fine")) + val provider = new StubbedProvider(stub, counter) + val ex = intercept[RuntimeException](Await.result(provider.chat(request), 5.seconds)) + assert(counter.get() == 1, s"HTTP $code must not be retried") + assert(ex.getMessage.startsWith(s"stub rejected the request: HTTP $code"), ex.getMessage) + } + } + + test("401 names the API key setting to fix") { + val (stub, counter) = stubReturning(status(401, """{"error":{"message":"Incorrect API key"}}""")) + val provider = new StubbedProvider(stub, counter) + + val ex = intercept[RuntimeException](Await.result(provider.chat(request), 5.seconds)) + assert(ex.getMessage.startsWith( + "stub rejected the request: HTTP 401 (invalid API key). " + + "Check stub_api_key in your config or call llm:set-api-key."), ex.getMessage) + assert(ex.getMessage.contains("Incorrect API key"), "the provider's own detail is kept") + } + + test("exhausted 529 retries give a friendly message") { + val (stub, counter) = stubReturning(status(529)) + val provider = new StubbedProvider(stub, counter) + provider.setConfig(RetryPolicy.MAX_RETRIES, "3") + + val ex = intercept[RuntimeException](Await.result(provider.chat(request), 10.seconds)) + assert(ex.getMessage.startsWith( + "stub returned HTTP 529 (overloaded) after 4 attempts. " + + "Try again shortly, or lower concurrency with the throttling settings."), ex.getMessage) + assert(counter.get() == 4) + } + + test("a send that fails to connect is retried then succeeds") { + val (stub, counter) = stubFailingSends(2)(connectRefused) + val provider = new StubbedProvider(stub, counter) + + val result = Await.result(provider.chat(request), 5.seconds) + assert(result.firstMessage.map(_.content).contains("ok")) + assert(counter.get() == 3) + assert(provider.observedCauses.peek() == "could not be reached (connection refused)") + } + + test("read timeouts are retried then succeed") { + val (stub, counter) = stubFailingSends(1) { req => + new SttpClientException.TimeoutException(req, new java.net.http.HttpTimeoutException("request timed out")) + } + val provider = new StubbedProvider(stub, counter) + + Await.result(provider.chat(request), 5.seconds) + assert(counter.get() == 2) + } + + test("timeouts nested as the cause of another exception are retried") { + // The stub backend already maps a bare SocketTimeoutException to sttp's + // TimeoutException, so wrap one to exercise the cause-chain check. + val (stub, counter) = stubFailingSends(1) { _ => + new java.util.concurrent.CompletionException(new java.net.SocketTimeoutException("Read timed out")) + } + val provider = new StubbedProvider(stub, counter) + + Await.result(provider.chat(request), 5.seconds) + assert(counter.get() == 2) + } + + test("non-transport send failures are not retried") { + val (stub, counter) = stubFailingSends(5)(_ => new IllegalArgumentException("bad request object")) + val provider = new StubbedProvider(stub, counter) + + intercept[IllegalArgumentException](Await.result(provider.chat(request), 5.seconds)) + assert(counter.get() == 1) + } + + test("an unreachable Ollama server names the URL and asks if it is running") { + val (stub, counter) = stubFailingSends(100)(connectRefused) + val provider = new OllamaProvider { + override lazy val backend: Backend[Future] = stub + override protected def retryBaseDelayMs: Long = 1L + override protected def retryRandom: () => Double = () => 0.0 + } + provider.setConfig(RetryPolicy.MAX_RETRIES, "1") + provider.setConfig(org.nlogo.extensions.llm.config.ConfigStore.OLLAMA_BASE_URL, "http://localhost:11434") + + val ex = intercept[RuntimeException](Await.result(provider.chat(request), 5.seconds)) + assert(ex.getMessage == "Could not reach Ollama at http://localhost:11434 (connection refused) " + + "after 2 attempts. Is the Ollama server running?", ex.getMessage) + assert(counter.get() == 2) + } + + /** What java.net.http raises for a host that does not resolve, as sttp wraps it. */ + private def unresolvedHost(req: GenericRequest[?, ?]): Exception = { + val connect = new java.net.ConnectException() + connect.initCause(new java.nio.channels.UnresolvedAddressException()) + new SttpClientException.ConnectException(req, connect) + } + + test("an unknown host is reported as such and not retried") { + val (stub, counter) = stubFailingSends(100)(unresolvedHost) + val provider = new OllamaProvider { + override lazy val backend: Backend[Future] = stub + override protected def retryBaseDelayMs: Long = 1L + override protected def retryRandom: () => Double = () => 0.0 + } + provider.setConfig(RetryPolicy.MAX_RETRIES, "2") + provider.setConfig(org.nlogo.extensions.llm.config.ConfigStore.OLLAMA_BASE_URL, "http://no-such-host-zz.invalid") + + val ex = intercept[RuntimeException](Await.result(provider.chat(request), 5.seconds)) + assert(ex.getMessage == "Could not reach Ollama at http://no-such-host-zz.invalid (unknown host). " + + "Check ollama_base_url in your config for a typo.", ex.getMessage) + assert(counter.get() == 1, "a host that does not resolve will not resolve on retry") + } + + test("Gemini's HTTP 400 API_KEY_INVALID points at the API key setting") { + val body = """{"error":{"code":400,"message":"API key not valid. Please pass a valid API key.",""" + + """"status":"INVALID_ARGUMENT","details":[{"reason":"API_KEY_INVALID"}]}}""" + val (stub, counter) = stubReturning(status(400, body)) + val provider = new StubbedProvider(stub, counter) + + val ex = intercept[RuntimeException](Await.result(provider.chat(request), 5.seconds)) + assert(ex.getMessage.startsWith( + "stub rejected the request: HTTP 400 (invalid API key). " + + "Check stub_api_key in your config or call llm:set-api-key."), ex.getMessage) + assert(counter.get() == 1) + } + + test("providers report a readable display name") { + assert(new ClaudeProvider().displayName == "Anthropic (Claude)") + assert(new OpenAIProvider().displayName == "OpenAI") + assert(new OllamaProvider().displayName == "Ollama") + } + + test("the retry wait notice names the provider, status and attempt") { + assert(BaseHttpProvider.retryWaitNotice("Anthropic", "returned HTTP 529 (overloaded)", 2000L, 0, 3) == + "NOTE: Anthropic returned HTTP 529 (overloaded); retrying in 2.0s (attempt 2 of 4)") } } diff --git a/src/test/SchemaFallbackSpec.scala b/src/test/SchemaFallbackSpec.scala new file mode 100644 index 0000000..cf522e9 --- /dev/null +++ b/src/test/SchemaFallbackSpec.scala @@ -0,0 +1,92 @@ +// ABOUTME: Tests that a request whose response_format the model rejects (HTTP 400) is resent once unconstrained +// ABOUTME: Uses an sttp BackendStub behind an OpenAIProvider so the real wire format is exercised +package org.nlogo.extensions.llm.providers + +import org.nlogo.extensions.llm.models.{ChatMessage, EnumFormat} +import org.scalatest.funsuite.AnyFunSuite +import sttp.client4._ +import sttp.client4.testing.{BackendStub, ResponseStub} +import sttp.model.StatusCode +import java.util.concurrent.ConcurrentLinkedQueue +import scala.concurrent.{Await, Future} +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.duration._ +import scala.jdk.CollectionConverters._ + +/** OpenAI provider whose backend is a stub and whose fallback notices are recorded. */ +class SchemaFallbackProvider(stub: BackendStub[Future]) extends OpenAIProvider()(using global) { + override lazy val backend: Backend[Future] = stub + + val notices = new ConcurrentLinkedQueue[String]() + override protected def notifyResponseFormatUnsupported(model: String): Unit = { + notices.add(model) + super.notifyResponseFormatUnsupported(model) + } +} + +class SchemaFallbackSpec extends AnyFunSuite { + + private val okBody = + """{"id":"chatcmpl-1","created":1700000000,"model":"m", + | "choices":[{"index":0,"message":{"role":"assistant","content":"{\"choice\": \"red\"}"},"finish_reason":"stop"}]}""".stripMargin + + private val unsupportedBody = + """{"error":{"message":"response_format json_schema is not supported by this model","type":"invalid_request_error"}}""" + + private def bodyOf(request: GenericRequest[?, ?]): String = request.body match { + case StringBody(s, _, _) => s + case other => other.toString + } + + /** Stub that answers `rejectBody` with a 400 whenever response_format is sent, else 200. */ + private def stub(rejectBody: String, sent: ConcurrentLinkedQueue[String]): BackendStub[Future] = + BackendStub.asynchronousFuture.whenAnyRequest.thenRespondF { (req: GenericRequest[?, ?]) => + val body = bodyOf(req) + sent.add(body) + Future.successful( + if (body.contains("response_format")) ResponseStub.adjust(rejectBody, StatusCode.BadRequest) + else ResponseStub.adjust(okBody, StatusCode.Ok) + ) + } + + private def provider(rejectBody: String, sent: ConcurrentLinkedQueue[String], model: String): SchemaFallbackProvider = { + val p = new SchemaFallbackProvider(stub(rejectBody, sent)) + p.setConfig("api_key", "sk-test") + p.setConfig("model", model) + p + } + + private val messages = Seq(ChatMessage.user("pick a colour")) + + test("a 400 rejecting response_format is resent once without it, with a one-time notice") { + val sent = new ConcurrentLinkedQueue[String]() + val p = provider(unsupportedBody, sent, "fallback-model-a") + + val result = Await.result(p.chatWithFormat(messages, EnumFormat(List("red", "blue"))), 5.seconds) + assert(result.firstMessage.map(_.content).contains("""{"choice": "red"}""")) + val bodies = sent.asScala.toList + assert(bodies.length == 2) + assert(bodies.head.contains("response_format")) + assert(!bodies(1).contains("response_format")) + + Await.result(p.chatWithFormat(messages, EnumFormat(List("red", "blue"))), 5.seconds) + assert(sent.size == 4) + assert(p.notices.asScala.toList == List("fallback-model-a")) + } + + test("a 400 about something else still fails after a single send") { + val sent = new ConcurrentLinkedQueue[String]() + val p = provider( + """{"error":{"message":"The model `nope` does not exist","type":"invalid_request_error"}}""", + sent, + "fallback-model-b" + ) + + val ex = intercept[RuntimeException] { + Await.result(p.chatWithFormat(messages, EnumFormat(List("red", "blue"))), 5.seconds) + } + assert(ex.getMessage.contains("does not exist")) + assert(sent.size == 1) + assert(p.notices.isEmpty) + } +} diff --git a/src/test/StructuredOutputRequestSpec.scala b/src/test/StructuredOutputRequestSpec.scala index d10ccf0..151a31d 100644 --- a/src/test/StructuredOutputRequestSpec.scala +++ b/src/test/StructuredOutputRequestSpec.scala @@ -205,6 +205,36 @@ class StructuredOutputRequestSpec extends AnyFunSuite { assert(!keys(body("generationConfig")).contains("responseJsonSchema")) } + test("Gemini sends system messages as systemInstruction, not user turns (#28)") { + val r = ChatRequest( + model = "gemini-2.5-flash", + messages = Seq( + ChatMessage.system("be terse"), + ChatMessage.user("hi"), + ChatMessage("assistant", "hello"), + ChatMessage.system("answer in French"), + ChatMessage.user("again") + ), + maxTokens = Some(4000), + responseFormat = Some(schemaFormat), + thinkingConfig = Some(ThinkingConfig(enabled = true, budgetTokens = Some(2048))) + ) + val body = (new InspectableGemini).body(r) + assert(body("systemInstruction")("parts")(0)("text").str == "be terse\n\nanswer in French") + val contents = body("contents").arr + assert(contents.map(_("role").str).toSeq == Seq("user", "model", "user")) + assert(contents.map(_("parts")(0)("text").str).toSeq == Seq("hi", "hello", "again")) + val gc = body("generationConfig") + assert(gc("responseMimeType").str == "application/json") + assert(gc("thinkingConfig")("thinkingBudget").num == 2048) + } + + test("Gemini omits systemInstruction when there are no system messages") { + val body = (new InspectableGemini).body(request("gemini-2.5-flash", None)) + assert(!keys(body).contains("systemInstruction")) + assert(body("contents").arr.map(_("role").str).toSeq == Seq("user")) + } + // --- Ollama --- test("Ollama puts the raw schema in format") { diff --git a/src/test/Tests.scala b/src/test/Tests.scala index 46ba2d7..25a2fec 100644 --- a/src/test/Tests.scala +++ b/src/test/Tests.scala @@ -6,7 +6,8 @@ import org.scalatest.BeforeAndAfterAll import java.io.File import org.nlogo.headless.TestLanguage -import org.nlogo.extensions.llm.providers.DeterministicTestProvider +import org.nlogo.extensions.llm.config.ConfigStore +import org.nlogo.extensions.llm.providers.{DeterministicDecisionTestProvider, DeterministicTestProvider} import scala.util.Success object Tests { @@ -21,7 +22,12 @@ class Tests extends TestLanguage(Tests.testFiles) with BeforeAndAfterAll { super.beforeAll() LLMExtension.setProviderFactoryOverride { (configStore, ec) => - val provider = new DeterministicTestProvider()(using ec) + // "typesafe" gets the decision stand-in so llm:decide runs end to end; + // every other provider stays a plain chat provider with no decision API. + val provider = + if (configStore.get(ConfigStore.PROVIDER).contains("typesafe")) + new DeterministicDecisionTestProvider()(using ec) + else new DeterministicTestProvider()(using ec) configStore.toMap.foreach { case (key, value) => provider.setConfig(key, value) } diff --git a/src/test/TypeSafeProviderSpec.scala b/src/test/TypeSafeProviderSpec.scala new file mode 100644 index 0000000..97bc70d --- /dev/null +++ b/src/test/TypeSafeProviderSpec.scala @@ -0,0 +1,191 @@ +// ABOUTME: Request/response mapping tests for TypeSafeProvider's decision API (choice, score, yes/no) +// ABOUTME: A BackendStub captures URL, headers and body and scripts System One replies; no real HTTP +package org.nlogo.extensions.llm.providers + +import org.nlogo.extensions.llm.config.ConfigStore +import org.nlogo.extensions.llm.models.{ChatMessage, ChatRequest, DecisionAnswer, DecisionQuestion, DecisionResult, Usage} +import org.scalatest.funsuite.AnyFunSuite +import sttp.client4._ +import sttp.client4.testing.{BackendStub, ResponseStub} +import sttp.model.StatusCode +import java.util.concurrent.atomic.AtomicReference +import scala.concurrent.{Await, Future} +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.duration._ + +/** Records the last request sent and answers every request with `status` / `body`. */ +class SystemOneStub(status: StatusCode, body: String) { + val last = new AtomicReference[GenericRequest[?, ?]]() + val backend: BackendStub[Future] = BackendStub.asynchronousFuture + .whenRequestMatches { req => last.set(req); true } + .thenRespond(ResponseStub.adjust(body, status)) + + def url: String = last.get.uri.toString + def header(name: String): Option[String] = last.get.header(name) + def json: ujson.Value = last.get.body match { + case StringBody(s, _, _) => ujson.read(s) + case other => throw new AssertionError(s"unexpected body $other") + } +} + +class StubbedTypeSafe(stub: SystemOneStub) extends TypeSafeProvider()(using global) { + override lazy val backend: Backend[Future] = stub.backend + override protected def retryBaseDelayMs: Long = 1L +} + +class TypeSafeProviderSpec extends AnyFunSuite { + + ProviderRegistrations.registerAll() + + private def provider(stub: SystemOneStub, key: Option[String] = Some("ts-key")): TypeSafeProvider = { + val p = new StubbedTypeSafe(stub) + key.foreach(k => p.setConfig("typesafe_api_key", k)) + p + } + + private def decide(p: TypeSafeProvider, prompt: String, q: DecisionQuestion): (DecisionResult, Usage) = + Await.result(p.decide(prompt, q), 5.seconds) + + private def reply(answer: ujson.Value, in: Int = 120, out: Int = 12): String = + ujson.Obj( + "model" -> "jev-1.13.0", + "answers" -> ujson.Obj("decision" -> answer), + "usage" -> ujson.Obj("input_tokens" -> in, "output_tokens" -> out) + ).toString + + private val choiceReply = reply(ujson.Obj( + "type" -> "choice", "choice" -> "stay", "confidence" -> 0.4, + // Deliberately not in request order: the distribution must follow the options. + "probabilities" -> ujson.Obj("stay" -> 0.7, "move" -> 0.3) + )) + + test("choice request goes to /v1/systemone with a bearer key and the documented body") { + val stub = new SystemOneStub(StatusCode.Ok, choiceReply) + decide(provider(stub), "Go to the bar tonight?", DecisionQuestion.Choice(Seq("move", "stay"))) + + assert(stub.url == "https://api.typesafe.ai/v1/systemone") + assert(stub.header("Authorization").contains("Bearer ts-key")) + assert(stub.header("Content-Type").exists(_.startsWith("application/json"))) + + val body = stub.json + assert(body("model").str == "jev-latest") + assert(body("state").str == "Go to the bar tonight?") + val q = body("questions")("decision") + assert(q("type").str == "choice") + assert(q("instructions").str.nonEmpty) + assert(q("criteria").obj.keys.toSeq == Seq("move", "stay")) + assert(q("criteria").obj.values.forall(_ == ujson.Null)) + assert(body.obj.keySet == Set("model", "state", "questions")) + } + + test("choice reply maps to the picked option, its probability, and options-ordered distribution") { + val stub = new SystemOneStub(StatusCode.Ok, choiceReply) + val (result, _) = decide(provider(stub), "p", DecisionQuestion.Choice(Seq("move", "stay"))) + assert(result.answer == DecisionAnswer.Picked("stay")) + assert(result.probability == 0.7) + assert(result.distribution == Seq("move" -> 0.3, "stay" -> 0.7)) + } + + test("score request sends the levels as an ordered list and maps the reply by level index") { + val stub = new SystemOneStub(StatusCode.Ok, reply(ujson.Obj( + "type" -> "score", "score" -> 1.43, "confidence" -> 0.62, + "legend" -> ujson.Obj("0" -> "low", "1" -> "medium", "2" -> "high"), + "probabilities" -> ujson.Obj("2" -> 0.53, "0" -> 0.1, "1" -> 0.37) + ))) + val levels = Seq("low", "medium", "high") + val (result, _) = decide(provider(stub), "How crowded?", DecisionQuestion.Score(levels)) + + val q = stub.json("questions")("decision") + assert(q("type").str == "score") + assert(q("criteria").arr.map(_.str).toSeq == levels) + + assert(result.answer == DecisionAnswer.Expected(1.43)) + assert(result.probability == 0.62) + assert(result.distribution == Seq("low" -> 0.1, "medium" -> 0.37, "high" -> 0.53)) + } + + test("yes-no request is a noul question without criteria, answered by the likelier side") { + val stub = new SystemOneStub(StatusCode.Ok, reply(ujson.Obj("type" -> "noul", "noul" -> 0.82))) + val (yes, _) = decide(provider(stub), "Will it be crowded?", DecisionQuestion.YesNo) + + val q = stub.json("questions")("decision") + assert(q("type").str == "noul") + assert(!q.obj.contains("criteria")) + + assert(yes.answer == DecisionAnswer.Verdict(true)) + assert(yes.probability == 0.82) + assert(yes.distribution.map(_._1) == Seq("true", "false")) + + val stubNo = new SystemOneStub(StatusCode.Ok, reply(ujson.Obj("type" -> "noul", "noul" -> 0.25))) + val (no, _) = decide(provider(stubNo), "p", DecisionQuestion.YesNo) + assert(no.answer == DecisionAnswer.Verdict(false)) + assert(no.probability == 0.75) + assert(no.distribution == Seq("true" -> 0.25, "false" -> 0.75)) + } + + test("usage maps input and output tokens, prices input tokens, and stamps latency") { + val stub = new SystemOneStub(StatusCode.Ok, reply(ujson.Obj("type" -> "noul", "noul" -> 0.9), in = 1000, out = 3)) + val (_, usage) = decide(provider(stub), "p", DecisionQuestion.YesNo) + assert(usage.inputTokens == 1000L) + assert(usage.outputTokens == 3L) + assert(usage.totalTokens == 1003L) + assert(usage.cost.exists(c => math.abs(c - 1000 * 0.042e-6) < 1e-15)) + assert(usage.latencyMs.isDefined) + } + + test("the configured model and typesafe_base_url are honored") { + val stub = new SystemOneStub(StatusCode.Ok, choiceReply) + val p = provider(stub) + p.setConfig(ConfigStore.MODEL, "jev-preview") + p.setConfig("typesafe_base_url", "http://local.test") + decide(p, "p", DecisionQuestion.Choice(Seq("move", "stay"))) + assert(stub.url == "http://local.test/v1/systemone") + assert(stub.json("model").str == "jev-preview") + } + + test("a 401 fails with the friendly invalid-key message") { + val stub = new SystemOneStub(StatusCode.Unauthorized, """{"detail":"Invalid API key"}""") + val e = intercept[HttpStatusException](decide(provider(stub), "p", DecisionQuestion.YesNo)) + assert(e.statusCode == 401) + assert(e.getMessage.contains("TypeSafe (Jev) rejected the request: HTTP 401 (invalid API key)")) + assert(e.getMessage.contains("typesafe_api_key")) + } + + test("a missing key fails before any request is sent") { + val stub = new SystemOneStub(StatusCode.Ok, choiceReply) + val e = intercept[IllegalStateException](decide(provider(stub, key = None), "p", DecisionQuestion.YesNo)) + assert(e.getMessage.contains("typesafe requires an API key")) + assert(stub.last.get == null) + } + + test("a reply missing the answer fails with a clear message") { + val stub = new SystemOneStub(StatusCode.Ok, + """{"model":"jev-latest","answers":{},"usage":{"input_tokens":1,"output_tokens":0}}""") + val e = intercept[RuntimeException](decide(provider(stub), "p", DecisionQuestion.YesNo)) + assert(e.getMessage.contains("TypeSafe (Jev)")) + } + + test("free-text chat is refused with the decisions-not-text message") { + val stub = new SystemOneStub(StatusCode.Ok, choiceReply) + val p = provider(stub) + val e = intercept[UnsupportedOperationException]( + Await.result(p.chat(ChatRequest(model = "jev-latest", messages = Seq(ChatMessage.user("hi")))), 5.seconds)) + assert(e.getMessage == TypeSafeProvider.TextUnsupportedMessage) + assert(p.textUnsupportedMessage.contains(TypeSafeProvider.TextUnsupportedMessage)) + assert(TypeSafeProvider.TextUnsupportedMessage == + "typesafe (Jev) returns decisions, not text; use llm:decide, llm:decide-all or llm:choose") + assert(stub.last.get == null) + } + + test("typesafe is registered with its keys, defaults and models") { + val d = ProviderRegistry.get("typesafe").get + assert(d.displayName == "TypeSafe (Jev)") + assert(d.apiKeyConfigKey == "typesafe_api_key") + assert(d.baseUrlConfigKey == "typesafe_base_url") + assert(d.defaultBaseUrl == "https://api.typesafe.ai") + assert(d.defaultModel == "jev-latest") + assert(d.requiresApiKey) + assert(!d.exposesThinking) + assert(Set("jev-latest", "jev-preview", "jev-1.13.0", "jev-1.14.0").subsetOf(ModelRegistry.getSupportedModels("typesafe"))) + } +} diff --git a/src/test/UsageParsingSpec.scala b/src/test/UsageParsingSpec.scala index d13f229..12fa8b7 100644 --- a/src/test/UsageParsingSpec.scala +++ b/src/test/UsageParsingSpec.scala @@ -64,6 +64,22 @@ class UsageParsingSpec extends AnyFunSuite { assert(usage.contains(Usage(inputTokens = 5, outputTokens = 2, totalTokens = 7))) } + test("OpenAI-compatible parser reads a refusal with null content and still parses usage") { + val body = + """{"id":"chatcmpl-4","created":1700000000,"model":"gpt-4o-mini", + | "choices":[{"index":0,"message":{"role":"assistant","content":null, + | "refusal":"I can't help with that."},"finish_reason":null}], + | "usage":{"prompt_tokens":8,"completion_tokens":6,"total_tokens":14}}""".stripMargin + val response = new InspectableOpenAIProvider().parse(body) + assert(response.usage.contains(Usage(inputTokens = 8, outputTokens = 6, totalTokens = 14))) + assert(response.refusal.contains("I can't help with that.")) + assert(response.firstContent.contains("")) + } + + test("OpenAI-compatible parser reports no refusal for an ordinary reply") { + assert(new InspectableOpenAIProvider().parse(openAiBody).refusal.isEmpty) + } + test("OpenRouter parser exposes the reported dollar cost") { val body = """{"id":"gen-1","created":1700000000,"model":"openai/gpt-4o-mini", @@ -99,6 +115,16 @@ class UsageParsingSpec extends AnyFunSuite { assert(usage.contains(Usage(inputTokens = 3, outputTokens = 1, totalTokens = 4))) } + test("Claude parser maps stop_reason refusal to the refusal field and keeps usage") { + val body = + """{"id":"msg_3","type":"message","role":"assistant","model":"m", + | "content":[{"type":"text","text":"I won't do that."}],"stop_reason":"refusal", + | "usage":{"input_tokens":4,"output_tokens":2}}""".stripMargin + val response = new InspectableClaudeParser().parse(body) + assert(response.refusal.contains("I won't do that.")) + assert(response.usage.contains(Usage(inputTokens = 4, outputTokens = 2, totalTokens = 6))) + } + test("Gemini parser counts thinking tokens as output and reads cached content tokens") { val body = """{"candidates":[{"content":{"parts":[{"text":"hi"}],"role":"model"},"finishReason":"STOP"}], diff --git a/tests.txt b/tests.txt index b03f252..4576d94 100644 --- a/tests.txt +++ b/tests.txt @@ -10,7 +10,7 @@ LLMConfigPrimitives LLMProvidersAll extensions [llm] - length llm:providers-all => 7 + length llm:providers-all => 8 member? "openai" llm:providers-all => true member? "anthropic" llm:providers-all => true member? "gemini" llm:providers-all => true @@ -18,12 +18,13 @@ LLMProvidersAll member? "openrouter" llm:providers-all => true member? "together" llm:providers-all => true member? "groq" llm:providers-all => true + member? "typesafe" llm:providers-all => true LLMProviderStatus extensions [llm] O> llm:set-api-key "test-key" O> llm:set-provider "openai" - length llm:provider-status => 7 + length llm:provider-status => 8 LLMListModels extensions [llm] @@ -268,7 +269,7 @@ LLMOpenRouterProvider O> llm:set-api-key "test-key" O> llm:set-provider "openrouter" item 0 llm:active => "openrouter" - item 1 llm:active => "openai/gpt-4o-mini" + item 1 llm:active => "openai/gpt-oss-20b" empty? (llm:provider-help "openrouter") => false O> llm:set-model "openai/gpt-4o" item 1 llm:active => "openai/gpt-4o" @@ -278,7 +279,7 @@ LLMTogetherProvider O> llm:set-api-key "test-key" O> llm:set-provider "together" item 0 llm:active => "together" - item 1 llm:active => "meta-llama/Llama-3.3-70B-Instruct-Turbo" + item 1 llm:active => "Qwen/Qwen3.8-Flash" empty? (llm:provider-help "together") => false O> llm:set-model "deepseek-ai/DeepSeek-R1" item 1 llm:active => "deepseek-ai/DeepSeek-R1" @@ -1014,7 +1015,7 @@ LLMProfileWithoutModelUsesProviderDefault O> llm:load-profile "claude" "demos/test-profile-anthropic" O> crt 1 O> ask turtle 0 [ llm:use-profile "claude" ] - [llm:active] of turtle 0 => ["anthropic" "claude-haiku-4-5-20251001"] + [llm:active] of turtle 0 => ["anthropic" "claude-haiku-5-5"] O> clear-all LLMProfileUsesItsOwnWaitBudget @@ -1234,3 +1235,257 @@ LLMChatAllRejectsExtraSchemas O> carefully [ set r (llm:chat-all turtles [ -> "hi" ] "{\"type\":\"object\"}" "{\"type\":\"object\"}") ] [ set err error-message ] member? "at most one schema" err => true O> clear-all + +LLMRefusalRaisesAfterRecordingUsage + extensions [llm] + globals [r err] + O> llm:set-api-key "test-key" + O> llm:set-provider "openai" + O> clear-all + O> llm:set-history [["user" "prior"] ["assistant" "context"]] + O> carefully [ set r llm:chat "__TEST_USAGE:5,2 __TEST_REFUSE:cannot help with that" ] [ set err error-message ] + err => "Extension exception: model refused: cannot help with that" + llm:get llm:usage-total "calls" => 1 + llm:get llm:usage "input-tokens" => 5 + llm:history => [["user" "prior"] ["assistant" "context"]] + O> clear-all + +LLMRefusalRaisesFromEveryChatPrimitive + extensions [llm] + globals [p r e2 e3 e4 e5 e6] + O> llm:set-api-key "test-key" + O> llm:set-provider "openai" + O> clear-all + O> set p llm:chat-async "__TEST_REFUSE:no async" + O> wait 0.3 + llm:get llm:usage-total "calls" => 1 + llm:history => [] + O> carefully [ set r llm:chat-with-template "demos/simple-template.yaml" [["text" "__TEST_REFUSE:no template"]] ] [ set e2 error-message ] + member? "model refused: no template" e2 => true + O> carefully [ set r llm:choose "__TEST_REFUSE:no choice" ["north" "south"] ] [ set e3 error-message ] + member? "model refused: no choice" e3 => true + O> carefully [ set r llm:chat-with-schema "__TEST_REFUSE:no schema" "{\"type\":\"object\",\"properties\":{\"a\":{\"type\":\"string\"}}}" ] [ set e4 error-message ] + member? "model refused: no schema" e4 => true + O> carefully [ set r llm:chat-json "__TEST_REFUSE:no json" ] [ set e5 error-message ] + member? "model refused: no json" e5 => true + O> carefully [ set r llm:chat-with-thinking "__TEST_REFUSE:no thinking" ] [ set e6 error-message ] + member? "model refused: no thinking" e6 => true + llm:get llm:usage-total "calls" => 6 + llm:history => [] + O> crt 1 + O> set r llm:chat-all turtles [ -> "__TEST_REFUSE:no batch" ] + item 1 (item 0 r) => "model refused: no batch" + item 2 (item 0 r) => false + [llm:history] of turtle 0 => [] + llm:get llm:usage-total "calls" => 7 + +LLMChatAsyncAfterClearAllWritesNothing + extensions [llm] + globals [r] + O> llm:set-api-key "test-key" + O> llm:set-provider "openai" + O> clear-all + O> let p llm:chat-async "__TEST_USAGE:4,3 __TEST_DELAY:300:slow" clear-all set r (runresult p) + r => "stub:slow" + llm:history => [] + llm:get llm:usage "calls" => 0 + llm:get llm:usage-total "calls" => 0 + O> clear-all + +LLMChatAsyncAfterClearAllSkipsUnawaitedReply + extensions [llm] + globals [p] + O> llm:set-api-key "test-key" + O> llm:set-provider "openai" + O> clear-all + O> let q llm:chat-async "__TEST_USAGE:4,3 __TEST_DELAY:300:slow" clear-all set p q + O> wait 0.6 + llm:history => [] + llm:get llm:usage-total "calls" => 0 + O> clear-all + +LLMDecideChoice + extensions [llm] + globals [r] + O> llm:set-api-key "test-key" + O> llm:set-provider "typesafe" + O> clear-all + item 0 llm:active => "typesafe" + item 1 llm:active => "jev-latest" + O> set r llm:decide "__TEST_DECIDE:stay:0.5 bar tonight?" ["choice" ["move" "stay" "wait"]] + r => ["stay" 0.5 [["move" 0.25] ["stay" 0.5] ["wait" 0.25]]] + llm:decide "bar tonight?" ["choice" ["move" "stay"]] => ["move" 1 [["move" 1] ["stay" 0]]] + llm:history => [] + O> clear-all + +LLMDecideScore + extensions [llm] + globals [r] + O> llm:set-api-key "test-key" + O> llm:set-provider "typesafe" + O> clear-all + O> set r llm:decide "__TEST_DECIDE:2:0.5 how crowded?" ["score" ["low" "medium" "high"]] + r => [1.25 0.5 [["low" 0.25] ["medium" 0.25] ["high" 0.5]]] + is-number? item 0 r => true + llm:history => [] + O> clear-all + +LLMDecideYesNo + extensions [llm] + O> llm:set-api-key "test-key" + O> llm:set-provider "typesafe" + O> clear-all + llm:decide "__TEST_DECIDE:true:0.75 go?" ["yes-no"] => [true 0.75 [[true 0.75] [false 0.25]]] + llm:decide "__TEST_DECIDE:false:0.75 go?" ["yes-no"] => [false 0.75 [[true 0.25] [false 0.75]]] + llm:decide "__TEST_DECIDE:true:0.56 go?" ["yes-no"] => [true 0.56 [[true 0.56] [false 0.44]]] + llm:history => [] + O> clear-all + +LLMDecideRecordsUsageUntilClearAll + extensions [llm] + globals [r] + O> llm:set-api-key "test-key" + O> llm:set-provider "typesafe" + O> clear-all + O> set r llm:decide "__TEST_USAGE:5,1 go?" ["yes-no"] + O> set r llm:decide "__TEST_USAGE:2,0 go?" ["choice" ["a" "b"]] + llm:get llm:usage "input-tokens" => 7 + llm:get llm:usage "calls" => 2 + llm:get llm:usage-total "calls" => 2 + O> clear-all + llm:get llm:usage "calls" => 0 + llm:get llm:usage-total "calls" => 0 + +LLMDecideSpecValidation + extensions [llm] + O> llm:set-api-key "test-key" + O> llm:set-provider "typesafe" + O> clear-all + llm:decide "x" [] => ERROR Extension exception: llm:decide: spec must be ["choice" [options]], ["score" [levels]] or ["yes-no"] + llm:decide "x" ["vote" ["a"]] => ERROR Extension exception: llm:decide: unknown decision type "vote"; use "choice", "score" or "yes-no" + llm:decide "x" ["choice" []] => ERROR Extension exception: llm:decide: "choice" needs a list of 1 to 255 options + llm:decide "x" ["choice" "a"] => ERROR Extension exception: llm:decide: "choice" needs a list of 1 to 255 options + llm:decide "x" ["choice" ["a" "a"]] => ERROR Extension exception: llm:decide: "choice" options must be distinct, but "a" appears more than once + llm:decide "x" ["choice" ["a" ""]] => ERROR Extension exception: llm:decide: "choice" options must be non-empty strings + llm:decide "x" ["choice" ["a" 3]] => ERROR Extension exception: llm:decide: "choice" options must be non-empty strings + llm:decide "x" ["score" ["only"]] => ERROR Extension exception: llm:decide: "score" needs a list of 2 to 10 levels + llm:decide "x" ["score" ["a" "b" "c" "d" "e" "f" "g" "h" "i" "j" "k"]] => ERROR Extension exception: llm:decide: "score" needs a list of 2 to 10 levels + llm:decide "x" ["score" ["a" ""]] => ERROR Extension exception: llm:decide: "score" levels must be non-empty strings + llm:decide "x" ["yes-no" ["a"]] => ERROR Extension exception: llm:decide: "yes-no" takes no list + O> crt 1 + llm:decide-all turtles [ -> "x" ] ["vote"] => ERROR Extension exception: llm:decide-all: unknown decision type "vote"; use "choice", "score" or "yes-no" + llm:get llm:usage-total "calls" => 0 + +LLMDecideNeedsDecisionProvider + extensions [llm] + O> llm:set-api-key "test-key" + O> llm:set-provider "openai" + O> clear-all + llm:decide "x" ["yes-no"] => ERROR Extension exception: provider "openai" has no decision API; use llm:choose, or llm:set-provider "typesafe" + O> crt 2 + llm:decide-all turtles [ -> "x" ] ["yes-no"] => ERROR Extension exception: provider "openai" has no decision API; use llm:choose, or llm:set-provider "typesafe" + llm:get llm:usage-total "calls" => 0 + +LLMDecideAllMixedResults + extensions [llm] + globals [r bad good] + O> llm:set-api-key "test-key" + O> llm:set-provider "typesafe" + O> clear-all + O> crt 3 + O> set r llm:decide-all turtles [ -> ifelse-value who = 1 [ "__TEST_FAIL" ] [ "__TEST_USAGE:2,0 __TEST_DECIDE:stay:0.75 go?" ] ] ["choice" ["move" "stay"]] + length r => 3 + map [ x -> length x ] r => [3 3 3] + sort map [ x -> [who] of item 0 x ] r => [0 1 2] + O> set bad first filter [ x -> [who] of item 0 x = 1 ] r + O> set good filter [ x -> [who] of item 0 x != 1 ] r + item 2 bad => false + member? "simulated provider failure" item 1 bad => true + map [ x -> item 2 x ] good => [true true] + map [ x -> item 1 x ] good => [["stay" 0.75 [["move" 0.25] ["stay" 0.75]]] ["stay" 0.75 [["move" 0.25] ["stay" 0.75]]]] + [llm:get llm:usage "input-tokens"] of turtle 0 => 2 + [llm:get llm:usage "calls"] of turtle 1 => 0 + llm:get llm:usage-total "calls" => 2 + [llm:history] of turtle 0 => [] + O> clear-all + +LLMDecideAllPromptRunsAsEachAgent + extensions [llm] + globals [r] + O> llm:set-api-key "test-key" + O> llm:set-provider "typesafe" + O> clear-all + O> crt 2 + O> set r llm:decide-all turtles [ -> (word "__TEST_DECIDE:" item who ["true" "false"] ":1 go?") ] ["yes-no"] + reduce and map [ x -> item 0 item 1 x = ([who] of item 0 x = 0) ] r => true + llm:decide-all no-turtles [ -> "x" ] ["yes-no"] => [] + O> clear-all + +LLMChooseOnDecisionProvider + extensions [llm] + O> llm:set-api-key "test-key" + O> llm:set-provider "typesafe" + O> clear-all + llm:choose "__TEST_USAGE:3,0 __TEST_DECIDE:south:0.9 where to?" ["north" "south"] => "south" + llm:choose "where to?" ["north" "south"] => "north" + llm:history => [] + llm:get llm:usage "input-tokens" => 3 + llm:get llm:usage "calls" => 2 + O> clear-all + +LLMTextPrimitivesOnDecisionProvider + extensions [llm] + O> llm:set-api-key "test-key" + O> llm:set-provider "typesafe" + O> clear-all + llm:chat "hi" => ERROR Extension exception: typesafe (Jev) returns decisions, not text; use llm:decide, llm:decide-all or llm:choose + llm:chat-async "hi" => ERROR Extension exception: typesafe (Jev) returns decisions, not text; use llm:decide, llm:decide-all or llm:choose + llm:chat-with-thinking "hi" => ERROR Extension exception: typesafe (Jev) returns decisions, not text; use llm:decide, llm:decide-all or llm:choose + llm:chat-json "hi" => ERROR Extension exception: typesafe (Jev) returns decisions, not text; use llm:decide, llm:decide-all or llm:choose + llm:chat-with-schema "hi" "{\"type\":\"object\",\"properties\":{\"a\":{\"type\":\"string\"}}}" => ERROR Extension exception: typesafe (Jev) returns decisions, not text; use llm:decide, llm:decide-all or llm:choose + llm:chat-with-template "demos/simple-template.yaml" [["text" "hi"]] => ERROR Extension exception: typesafe (Jev) returns decisions, not text; use llm:decide, llm:decide-all or llm:choose + O> crt 1 + llm:chat-all turtles [ -> "hi" ] => ERROR Extension exception: typesafe (Jev) returns decisions, not text; use llm:decide, llm:decide-all or llm:choose + llm:history => [] + llm:get llm:usage-total "calls" => 0 + +LLMTypeSafeProviderInfo + extensions [llm] + O> llm:set-api-key "test-key" + O> llm:set-provider "typesafe" + empty? (llm:provider-help "typesafe") => false + member? "typesafe" llm:providers => true + (position "jev-preview" llm:list-models) != false => true + item 1 first filter [ x -> first x = "typesafe" ] llm:provider-status => ["ready" true] + item 2 first filter [ x -> first x = "typesafe" ] llm:provider-status => ["has-key" true] + O> llm:set-model "jev-1.14.0" + item 1 llm:active => "jev-1.14.0" + +LLMChooseOnDecisionProviderValidatesOptions + extensions [llm] + O> llm:set-api-key "test-key" + O> llm:set-provider "typesafe" + O> clear-all + llm:choose "x" ["a" "a"] => ERROR Extension exception: llm:choose: "choice" options must be distinct, but "a" appears more than once + llm:choose "x" ["a" ""] => ERROR Extension exception: llm:choose: "choice" options must be non-empty strings + llm:choose "x" n-values 256 [ i -> (word "o" i) ] => ERROR Extension exception: llm:choose: "choice" needs a list of 1 to 255 options + llm:get llm:usage-total "calls" => 0 + llm:choose "__TEST_DECIDE:west:0.9 where to?" ["north" "south"] => ERROR Extension exception: llm:choose: provider picked "west", which is not one of the options: north, south + llm:history => [] + O> clear-all + +LLMEmptyReplyAtTokenLimitRaisesAfterRecordingUsage + extensions [llm] + globals [r err] + O> llm:set-api-key "test-key" + O> llm:set-provider "openai" + O> clear-all + O> llm:set-history [["user" "prior"] ["assistant" "context"]] + O> carefully [ set r llm:chat "__TEST_USAGE:5,4000,4000 __TEST_TRUNCATE" ] [ set err error-message ] + member? "empty reply" err => true + member? "max_tokens" err => true + member? "4000 output tokens" err => true + llm:get llm:usage-total "calls" => 1 + llm:get llm:usage-total "output-tokens" => 4000 + llm:history => [["user" "prior"] ["assistant" "context"]] + O> clear-all