From 61cfa334f33961c843b108a958b6ddfc5124a1ba Mon Sep 17 00:00:00 2001 From: JNK234 Date: Thu, 24 Sep 2026 15:11:59 -0400 Subject: [PATCH] feat: add llm:chat-all for batched per-agent calls One request per agent, sent together, one wait. A model that runs `ask turtles [ llm:chat ... ]` waits for each turtle in turn; chat-all takes about as long as the slowest single reply. - `llm:chat-all agentset [ -> prompt ]` reports [agent reply ok?] triples; the prompt reporter runs as each agent, like ask - optional schema argument reuses the structured-output path and returns parsed [key value] lists per agent - a failed call is reported in that agent's triple and never aborts the batch; no history is committed for it - every prompt is built before any request is sent, so a prompt error costs no requests - requests go through each agent's provider, so throttling, retry, profiles, history and usage all apply per agent - the batch waits while replies keep arriving and gives up only after a full budget with no reply, so a hung provider costs one budget, not one per agent; late replies and halted waits never write history Closes #53 --- demos/e2e-tests/e2e-tests.nlogox | 68 +++++++++++ docs/API-REFERENCE.md | 81 +++++++++++++ src/main/LLMExtension.scala | 201 +++++++++++++++++++++++++++---- tests.txt | 188 +++++++++++++++++++++++++++++ 4 files changed, 517 insertions(+), 21 deletions(-) diff --git a/demos/e2e-tests/e2e-tests.nlogox b/demos/e2e-tests/e2e-tests.nlogox index e17c639..0dd9855 100644 --- a/demos/e2e-tests/e2e-tests.nlogox +++ b/demos/e2e-tests/e2e-tests.nlogox @@ -345,6 +345,73 @@ to test-profiles report-totals end +;; --------------------------------------------------------------------------- +;; T11 — batched calls (#53) +;; +;; llm:chat-all sends one request per turtle at once and waits once. Run +;; through a throttled copy of the config (cap 3) so the batch also proves it +;; composes with the throttle. Every turtle must get its own reply, its own +;; history and its own usage; a schema batch must come back as parsed lists. +;; --------------------------------------------------------------------------- + +to test-chat-all + log-line "T11 batched calls" + let throttled (word "config-" provider-under-test "-batch.txt") + write-throttled-config active-config throttled 3 0 + llm:load-config throttled + + crt 6 [ set label "batch-turtle" ] + let batch turtles with [label = "batch-turtle"] + let results [] + reset-timer + carefully + [ set results llm:chat-all batch [ -> + (word "Reply with exactly the number " who) ] ] + [ log-line (word " chat-all failed: " error-message) ] + let elapsed timer + log-triples results + log-line (word " 6 calls, cap 3 -> " precision elapsed 2 "s") + + assert "one triple per turtle" (length results = 6) + assert "every call succeeded" (all-ok? results) + assert "each reply names its own turtle" + (all-ok? results and reduce and map [ r -> + member? (word [who] of item 0 r) item 1 r ] results) + assert "each turtle has its own history" (all? batch [ length llm:history = 2 ]) + assert "each turtle was credited one call" + (all? batch [ llm:get llm:usage "calls" = 1 ]) + + let schema (word "{\"type\":\"object\",\"properties\":" + "{\"n\":{\"type\":\"integer\"}}," + "\"required\":[\"n\"],\"additionalProperties\":false}") + let pair (turtle-set min-one-of batch [who] max-one-of batch [who]) + let parsed [] + carefully + [ set parsed (llm:chat-all pair [ -> (word "Return n = " who) ] schema) ] + [ log-line (word " schema chat-all failed: " error-message) ] + log-triples parsed + assert "schema batch succeeded" (length parsed = 2 and all-ok? parsed) + assert "schema replies are parsed per turtle" + (all-ok? parsed and reduce and map [ r -> + llm:get item 1 r "n" = [who] of item 0 r ] parsed) + + ask batch [ die ] + llm:load-config active-config + carefully [ file-delete throttled ] [ ] + report-totals +end + +;; True when a llm:chat-all result is non-empty and every call succeeded. +to-report all-ok? [results] + report not empty? results and reduce and map [ r -> item 2 r ] results +end + +;; Log each [agent reply ok?] triple on its own line. +to log-triples [results] + foreach results [ r -> + log-line (word " " item 0 r " ok? " item 2 r " -> " item 1 r) ] +end + ;; A second model on the provider under test, so a profile can be told apart ;; from the default. Ollama has no guaranteed second model pulled, so it ;; reuses the active one and the test still proves routing and bookkeeping. @@ -391,6 +458,7 @@ to test-all test-throttling test-usage test-profiles + test-chat-all output-print "" log-line (word "TOTAL passed " pass-count " failed " fail-count) end diff --git a/docs/API-REFERENCE.md b/docs/API-REFERENCE.md index cbc7613..de9f8c6 100644 --- a/docs/API-REFERENCE.md +++ b/docs/API-REFERENCE.md @@ -10,6 +10,7 @@ The NetLogo Multi-LLM Extension provides a unified interface for multiple Large | ------------------------------------ | ------------- | ---------------------------------------------------------- | | `llm:chat text` | Chat | Send synchronous chat message, returns response | | `llm:chat-async text` | Chat | Send asynchronous chat message, returns awaitable reporter | +| `llm:chat-all agents [ -> prompt ]` | Chat | One request per agent, sent together; returns `[agent reply ok?]` triples | | `llm:chat-with-template file vars` | Chat | Send templated prompt with variable substitution | | `llm:chat-with-thinking text` | Chat | Returns `[answer thinking]` for reasoning-capable models | | `llm:choose prompt choices` | Chat | Force selection from provided options | @@ -215,6 +216,73 @@ print response - Use `runresult` to retrieve the actual response - Still maintains conversation history per agent +### llm:chat-all + +**Syntax**: `llm:chat-all agentset prompt-reporter` or `(llm:chat-all agentset prompt-reporter schema)` + +**Description**: Sends one chat request per agent, all at once, and waits once for every +reply. A model that does `ask turtles [ llm:chat ... ]` waits for each call in turn — 100 +turtles at 2 seconds each is over 3 minutes per tick. `llm:chat-all` takes about as long as +the slowest single call. + +**Parameters**: + +- `agentset` (agentset): The agents that each send one request +- `prompt-reporter` (anonymous reporter): Run as each agent, like `ask`, to build that + agent's prompt. It can read the agent's own variables, and `myself` is the caller. +- `schema` (string, optional): A JSON Schema, as in `llm:chat-with-schema`. Each reply is then + a parsed `[key value]` list instead of a string. + +**Returns**: A list of `[agent reply ok?]` triples, one per agent. + +- `ok?` is `true` when the call succeeded; `reply` is the response (or the parsed list). +- `ok?` is `false` when that agent's call failed; `reply` is the error message. The other + agents are unaffected — one failure never aborts the batch. + +**Example**: + +```netlogo +turtles-own [ energy decision ] + +to go + let results llm:chat-all turtles [ -> (word "You have " energy " energy. Move or rest?") ] + foreach results [ r -> + let who-asked item 0 r + ifelse item 2 r + [ ask who-asked [ set decision item 1 r ] ] + [ ask who-asked [ set decision "rest" ] ] ; fallback when the call failed + ] + tick +end +``` + +With a schema, each reply arrives as NetLogo values: + +```netlogo +let schema "{\"type\":\"object\",\"properties\":{\"action\":{\"type\":\"string\",\"enum\":[\"move\",\"rest\"]}},\"required\":[\"action\"]}" +let results (llm:chat-all turtles [ -> (word "Energy: " energy) ] schema) +foreach results [ r -> + if item 2 r [ ask item 0 r [ set decision llm:get item 1 r "action" ] ] +] +``` + +**Notes**: + +- Every prompt is built before any request is sent. A prompt reporter that errors or does not + report a string stops the primitive with an error and costs no requests. +- Requests go through each agent's own provider, so everything that applies to `llm:chat` + applies per agent: its profile (`llm:use-profile`), the provider's throttle + (`max_concurrent_requests`, `min_request_interval_ms`), 429 retry, history and usage. +- With throttling on, "send all at once" means "queue all, at most N in flight". Set + `max_concurrent_requests` when running large agentsets on a rate-limited plan. +- Each agent's history gets its own exchange only when its call succeeds. +- The batch keeps waiting as long as replies keep arriving, so a throttled batch whose later + agents queue behind the cap still completes. An agent is reported as failed only when no reply + from any agent has arrived for its whole timeout budget (see Timeout Tuning), so a hung provider + costs one budget, not one per agent. A reply that arrives after that is not added to history. +- The order of the list follows the agentset; read the agent from `item 0` rather than relying + on position. + ### llm:choose **Syntax**: `llm:choose prompt choices` @@ -1195,6 +1263,19 @@ to check-responses end ``` +### Batched Calls + +For one call per agent per tick, prefer `llm:chat-all` over `llm:chat-async` + `runresult`: it +sends every request at once, waits once, and hands back each agent's reply with a success flag. +Use `llm:chat-async` when the model should keep ticking while a reply is pending. + +```netlogo +let results llm:chat-all turtles [ -> (word "Neighbors: " count turtles-on neighbors) ] +foreach filter [ r -> item 2 r ] results [ r -> + ask item 0 r [ set label item 1 r ] +] +``` + ### Decision Making ```netlogo diff --git a/src/main/LLMExtension.scala b/src/main/LLMExtension.scala index 2147ca9..445b926 100644 --- a/src/main/LLMExtension.scala +++ b/src/main/LLMExtension.scala @@ -4,7 +4,7 @@ import org.nlogo.api._ import org.nlogo.core.{LogoList, Syntax} import org.nlogo.extensions.llm.config.{ConfigLoader, ConfigStore, ProfileStore} import org.nlogo.extensions.llm.providers.{LLMProvider, ProviderDescriptor, ProviderFactory, ProviderRegistry, ProviderRegistrations, ModelRegistry, OllamaProvider, ReadinessCheck, RetryPolicy} -import org.nlogo.extensions.llm.models.{ChatMessage, ChatResponse, EnumFormat, JsonObjectFormat, ResponseFormat, Usage} +import org.nlogo.extensions.llm.models.{ChatMessage, ChatResponse, EnumFormat, JsonObjectFormat, JsonSchemaFormat, ResponseFormat, Usage} import org.nlogo.extensions.llm.utils.JsonToNetLogo import scala.collection.mutable.{ArrayBuffer, WeakHashMap} import scala.concurrent.{Await, ExecutionContext, Future} @@ -116,6 +116,7 @@ class LLMExtension extends DefaultClassManager { // Core chat primitives manager.addPrimitive("chat", ChatReporter) manager.addPrimitive("chat-async", ChatAsyncReporter) + manager.addPrimitive("chat-all", ChatAllReporter) manager.addPrimitive("chat-with-template", ChatWithTemplateReporter) manager.addPrimitive("chat-with-thinking", ChatWithThinkingReporter) manager.addPrimitive("choose", ChooseReporter) @@ -652,6 +653,155 @@ class LLMExtension extends DefaultClassManager { } } + /** + * Batched chat: one request per agent, sent together, one wait. + * + * llm:chat-all agentset [ -> prompt ] + * (llm:chat-all agentset [ -> prompt ] schema) + * + * Reports a list of [agent reply ok?] triples. The prompt reporter runs as + * each agent, like `ask`, so it can read that agent's variables. Requests go + * through the same provider path as llm:chat, so throttling, retry, profiles, + * history and usage all apply per agent. One agent's failure is reported in + * its own triple and never aborts the rest of the batch. + */ + object ChatAllReporter extends Reporter { + override def getSyntax: Syntax = Syntax.reporterSyntax( + right = List(Syntax.AgentsetType, Syntax.ReporterType, Syntax.StringType | Syntax.RepeatableType), + ret = Syntax.ListType, + defaultOption = Some(2) + ) + + /** One agent's request; `result` and `abandoned` are only touched under historyLock. */ + private final class Slot(val agent: org.nlogo.agent.Agent, val prompt: String) { + var future: Future[AnyRef] = Future.failed(new IllegalStateException("request not sent")) + var result: Option[AnyRef] = None + var abandoned = false + } + + override def report(args: Array[Argument], context: Context): AnyRef = { + val agents = args(0).getAgentSet + val promptReporter = args(1).getReporter + if (args.length > 3) { + throw new ExtensionException("llm:chat-all takes at most one schema") + } + // Validate before building any prompt so a bad schema costs nothing. + val format = if (args.length > 2) Some(parseObjectSchema(args(2).getString, "llm:chat-all")) else None + + val ext = context match { + case ec: org.nlogo.nvm.ExtensionContext => ec + case _ => + throw new ExtensionException("llm:chat-all requires a NetLogo workspace and is unavailable in this context") + } + val caller = ext.nvmContext + + // Build every prompt before sending anything: an error in the prompt + // reporter is a bug in the model code, and should cost no requests. + // The agentset is copied first, as `ask` does, so a prompt reporter that + // kills agents cannot change the set while it is being walked. + val slots = agents.agents.asScala.toList.collect { + case a: org.nlogo.agent.Agent if a.id != -1 => + val agentContext = new org.nlogo.nvm.Context( + caller.job, a, caller.agent, caller.ip, caller.activation, ext.workspace) + promptReporter.report(agentContext, Array.empty[AnyRef]) match { + case prompt: String => new Slot(a, prompt) + case other => + throw new ExtensionException( + s"llm:chat-all: the prompt reporter must report a string, but for " + + s"${Dump.logoObject(a)} it reported ${Dump.logoObject(other, true, false)}") + } + } + + // Time of the most recent reply from any provider in this batch. + val lastReply = new java.util.concurrent.atomic.AtomicLong(System.nanoTime) + + // Send every request without waiting. The provider's throttle turns + // "send all" into "queue all, at most N in flight". Agents killed by a + // later agent's prompt reporter are dropped here rather than sent. + val live = slots.filter(_.agent.id != -1) + live.foreach { slot => + val userMessage = ChatMessage.user(slot.prompt) + slot.future = + try { + val provider = ensureProvider(slot.agent) + val messages = snapshotHistory(slot.agent) :+ userMessage + val sent = format match { + case Some(f) => provider.chatWithFormat(messages, f) + case None => provider.chatWithFullResponse(messages) + } + sent.onComplete(_ => lastReply.set(System.nanoTime)) + sent.map { response => + recordUsage(slot.agent, response) + val (value, assistant): (AnyRef, ChatMessage) = format match { + case Some(_) => + // Parse before committing, as llm:chat-with-schema does. + val content = response.firstContent.getOrElse("") + (JsonToNetLogo.parseObject(content), ChatMessage.assistant(content)) + case None => + val m = replyMessage(response) + (m.content, m) + } + // A reply that lands after the wait gave up must not write + // history for an agent the batch already reported as failed. + historyLock.synchronized { + if (slot.abandoned) { + throw new java.util.concurrent.TimeoutException("reply arrived after the wait gave up") + } + commitExchange(slot.agent, userMessage, assistant) + slot.result = Some(value) + } + value + } + } catch { + case e: Exception => Future.failed(e) + } + } + + // Wait until no provider has replied for an agent's whole budget. A + // batch that keeps making progress — the normal case behind a throttle, + // where later agents queue for waves — keeps waiting, while a hung + // provider costs one budget in total rather than one per agent. + def settle(slot: Slot, budget: FiniteDuration): Try[AnyRef] = { + while (!slot.future.isCompleted) { + val remaining = lastReply.get + budget.toNanos - System.nanoTime + if (remaining <= 0) { + return historyLock.synchronized { + slot.result match { + case Some(v) => Success(v) + case None => + slot.abandoned = true + Failure(new java.util.concurrent.TimeoutException( + s"no reply within ${budget.toSeconds}s")) + } + } + } + try Await.ready(slot.future, remaining.nanos) + catch { case _: java.util.concurrent.TimeoutException => () } + } + slot.future.value.get + } + + val entries = + try { + live.map { slot => + settle(slot, awaitTimeoutFor(effectiveConfig(slot.agent))) match { + case Success(v) => LogoList(slot.agent, v, java.lang.Boolean.TRUE) + case Failure(e) => LogoList(slot.agent, s"LLM chat failed: ${e.getMessage}", java.lang.Boolean.FALSE) + } + } + } finally { + // If the wait is cut short (halt interrupts the job thread), replies + // still in flight must not write history for calls that never + // returned to the model. + historyLock.synchronized { + live.foreach(slot => if (slot.result.isEmpty) slot.abandoned = true) + } + } + + LogoList.fromVector(entries.toVector) + } + } + object ChatWithTemplateReporter extends Reporter { override def getSyntax: Syntax = Syntax.reporterSyntax( right = List(Syntax.StringType, Syntax.ListType), @@ -804,6 +954,34 @@ class LLMExtension extends DefaultClassManager { ujson.read(text)(EnumFormat.ChoiceKey).str }.toOption + /** + * Parse and check a schema whose replies must convert to [key value] pairs. + * + * The reporters using it promise a list, and only a JSON object converts to + * the [key value] pairs llm:get reads. A top-level scalar or array schema + * would report a bare string or a flat list instead, breaking that promise, + * so it is rejected here rather than at the provider. + */ + private def parseObjectSchema(schemaText: String, primName: String): JsonSchemaFormat = { + val format = + try ResponseFormat.parseSchema(schemaText) + catch { + case e: IllegalArgumentException => + throw new ExtensionException(s"$primName: ${e.getMessage}") + } + + val topLevelType = format.schema.value.get("type").collect { case ujson.Str(s) => s } + if (!topLevelType.contains("object")) { + throw new ExtensionException( + s"$primName: schema must have type 'object' at the top level, but got " + + s"'${topLevelType.getOrElse("none")}'. $primName reports a list of " + + "[key value] pairs, so the reply has to be a JSON object. Wrap it, e.g. " + + """{"type":"object","properties":{"value":{"type":"string"}}}""" + ) + } + format + } + // Structured Output Primitives /** @@ -830,26 +1008,7 @@ class LLMExtension extends DefaultClassManager { val agent = context.getAgent // Validate before touching the provider so a bad schema costs nothing. - val format = - try ResponseFormat.parseSchema(schemaText) - catch { - case e: IllegalArgumentException => - throw new ExtensionException(s"llm:chat-with-schema: ${e.getMessage}") - } - - // This reporter's syntax promises a list, and only a JSON object converts - // to the [key value] pairs llm:get reads. A top-level scalar or array - // schema would report a bare string or a flat list instead, breaking that - // promise, so it is rejected here rather than at the provider. - val topLevelType = format.schema.value.get("type").collect { case ujson.Str(s) => s } - if (!topLevelType.contains("object")) { - throw new ExtensionException( - s"llm:chat-with-schema: schema must have type 'object' at the top level, but got " + - s"'${topLevelType.getOrElse("none")}'. llm:chat-with-schema reports a list of " + - "[key value] pairs, so the reply has to be a JSON object. Wrap it, e.g. " + - """{"type":"object","properties":{"value":{"type":"string"}}}""" - ) - } + val format = parseObjectSchema(schemaText, "llm:chat-with-schema") try { val provider = ensureProvider(agent) diff --git a/tests.txt b/tests.txt index b3545e9..b03f252 100644 --- a/tests.txt +++ b/tests.txt @@ -1046,3 +1046,191 @@ LLMAsyncKeepsTheBudgetItStartedWith O> set r (runresult p) r => "stub:hi" O> clear-all + +LLMChatAllReturnsTriples + extensions [llm] + globals [r] + O> llm:set-api-key "test-key" + O> llm:set-provider "openai" + O> clear-all + O> crt 3 + O> set r llm:chat-all turtles [ -> "hi" ] + length r => 3 + map [ x -> length x ] r => [3 3 3] + sort map [ x -> [who] of item 0 x ] r => [0 1 2] + map [ x -> item 1 x ] r => ["stub:hi" "stub:hi" "stub:hi"] + map [ x -> item 2 x ] r => [true true true] + O> clear-all + +LLMChatAllPromptRunsAsEachAgent + extensions [llm] + globals [r] + O> llm:set-api-key "test-key" + O> llm:set-provider "openai" + O> clear-all + O> crt 3 + O> set r llm:chat-all turtles [ -> (word "id " who) ] + reduce and map [ x -> item 1 x = (word "stub:id " [who] of item 0 x) ] r => true + O> clear-all + +LLMChatAllCommitsHistoryPerAgent + extensions [llm] + globals [r] + O> llm:set-api-key "test-key" + O> llm:set-provider "openai" + O> clear-all + O> crt 2 + O> set r llm:chat-all turtles [ -> (word "id " who) ] + [length llm:history] of turtle 0 => 2 + [item 1 (item 0 llm:history)] of turtle 0 => "id 0" + [item 1 (item 1 llm:history)] of turtle 1 => "stub:id 1" + llm:history => [] + O> clear-all + +LLMChatAllFailureIsolated + extensions [llm] + globals [r bad good] + O> llm:set-api-key "test-key" + O> llm:set-provider "openai" + O> clear-all + O> crt 3 + O> set r llm:chat-all turtles [ -> ifelse-value who = 1 [ "__TEST_FAIL" ] [ "ok" ] ] + O> set bad first filter [ x -> [who] of item 0 x = 1 ] r + O> set good filter [ x -> [who] of item 0 x != 1 ] r + item 2 bad => false + is-string? item 1 bad => true + member? "simulated provider failure" item 1 bad => true + map [ x -> item 2 x ] good => [true true] + [llm:history] of turtle 1 => [] + [length llm:history] of turtle 0 => 2 + O> clear-all + +LLMChatAllRecordsUsage + extensions [llm] + globals [r] + O> llm:set-api-key "test-key" + O> llm:set-provider "openai" + O> clear-all + O> crt 2 + O> set r llm:chat-all turtles [ -> "__TEST_USAGE:4,3 hi" ] + [llm:get llm:usage "total-tokens"] of turtle 0 => 7 + [llm:get llm:usage "calls"] of turtle 1 => 1 + llm:get llm:usage-total "calls" => 2 + llm:get llm:usage-total "total-tokens" => 14 + O> clear-all + +LLMChatAllUsesAgentProfile + extensions [llm] + globals [r] + O> llm:set-api-key "test-key" + O> llm:set-provider "openai" + O> llm:set-model "model-default" + O> clear-all + O> llm:load-profile "a" "demos/test-profile-a" + O> crt 2 + O> ask turtle 0 [ llm:use-profile "a" ] + O> set r llm:chat-all turtles [ -> "__TEST_ECHO_MODEL" ] + item 1 first filter [ x -> [who] of item 0 x = 0 ] r => "model-a" + item 1 first filter [ x -> [who] of item 0 x = 1 ] r => "model-default" + O> clear-all + +LLMChatAllWithSchema + extensions [llm] + globals [r schema] + O> llm:set-api-key "test-key" + O> llm:set-provider "openai" + O> clear-all + O> set schema "{\"type\":\"object\",\"properties\":{\"n\":{\"type\":\"number\"}}}" + O> crt 2 + O> set r (llm:chat-all turtles [ -> ifelse-value who = 0 [ "__TEST_RESPOND:{\"n\": 5}" ] [ "__TEST_RESPOND:not json" ] ] schema) + llm:get item 1 first filter [ x -> [who] of item 0 x = 0 ] r "n" => 5 + item 2 first filter [ x -> [who] of item 0 x = 0 ] r => true + item 2 first filter [ x -> [who] of item 0 x = 1 ] r => false + [llm:history] of turtle 1 => [] + O> clear-all + +LLMChatAllBadSchemaFailsFast + extensions [llm] + globals [r err] + O> llm:set-api-key "test-key" + O> llm:set-provider "openai" + O> clear-all + O> crt 2 + O> carefully [ set r (llm:chat-all turtles [ -> "hi" ] "{\"type\":\"string\"}") ] [ set err error-message ] + member? "type 'object'" err => true + llm:get llm:usage-total "calls" => 0 + O> clear-all + +LLMChatAllNonStringPromptErrors + extensions [llm] + globals [r err] + O> llm:set-api-key "test-key" + O> llm:set-provider "openai" + O> clear-all + O> crt 2 + O> carefully [ set r llm:chat-all turtles [ -> who ] ] [ set err error-message ] + member? "must report a string" err => true + llm:get llm:usage-total "calls" => 0 + O> clear-all + +LLMChatAllEmptyAgentset + extensions [llm] + O> llm:set-api-key "test-key" + O> llm:set-provider "openai" + O> clear-all + llm:chat-all turtles [ -> "hi" ] => [] + +LLMChatAllRunsConcurrently + extensions [llm] + globals [r elapsed] + O> llm:set-api-key "test-key" + O> llm:set-provider "openai" + O> clear-all + O> crt 10 + O> reset-timer + O> set r llm:chat-all turtles [ -> "__TEST_DELAY:400:hi" ] + O> set elapsed timer + map [ x -> item 2 x ] r => [true true true true true true true true true true] + elapsed < 2 => true + O> clear-all + +LLMChatAllHungBatchWaitsOneBudget + extensions [llm] + globals [r elapsed] + O> llm:set-api-key "test-key" + O> llm:set-provider "openai" + O> clear-all + O> llm:load-config "demos/test-config-fast-timeout" + O> crt 3 + O> reset-timer + O> set r llm:chat-all turtles [ -> "__TEST_DELAY:3000:hi" ] + O> set elapsed timer + map [ x -> item 2 x ] r => [false false false] + elapsed < 1.8 => true + O> wait 2.5 + [llm:history] of turtle 0 => [] + [llm:history] of turtle 2 => [] + O> clear-all + +LLMChatAllProgressExtendsTheWait + extensions [llm] + globals [r] + O> llm:set-api-key "test-key" + O> llm:set-provider "openai" + O> clear-all + O> llm:load-config "demos/test-config-fast-timeout" + O> crt 2 + O> set r llm:chat-all turtles [ -> ifelse-value who = 0 [ "__TEST_DELAY:600:a" ] [ "__TEST_DELAY:1400:b" ] ] + map [ x -> item 2 x ] r => [true true] + O> clear-all + +LLMChatAllRejectsExtraSchemas + extensions [llm] + globals [r err] + O> llm:set-api-key "test-key" + O> llm:set-provider "openai" + O> clear-all + O> crt 1 + O> carefully [ set r (llm:chat-all turtles [ -> "hi" ] "{\"type\":\"object\"}" "{\"type\":\"object\"}") ] [ set err error-message ] + member? "at most one schema" err => true + O> clear-all